You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
352 lines
8.8 KiB
352 lines
8.8 KiB
#!/bin/bash
|
|
|
|
# Setup monitoring for Kafka Client Load Test
|
|
# This script sets up Prometheus and Grafana configurations
|
|
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
|
|
MONITORING_DIR="$PROJECT_DIR/monitoring"
|
|
|
|
# Colors
|
|
GREEN='\033[0;32m'
|
|
BLUE='\033[0;34m'
|
|
NC='\033[0m'
|
|
|
|
log_info() {
|
|
echo -e "${BLUE}[INFO]${NC} $1"
|
|
}
|
|
|
|
log_success() {
|
|
echo -e "${GREEN}[SUCCESS]${NC} $1"
|
|
}
|
|
|
|
# Create monitoring directory structure
|
|
setup_directories() {
|
|
log_info "Setting up monitoring directories..."
|
|
|
|
mkdir -p "$MONITORING_DIR/prometheus"
|
|
mkdir -p "$MONITORING_DIR/grafana/dashboards"
|
|
mkdir -p "$MONITORING_DIR/grafana/provisioning/dashboards"
|
|
mkdir -p "$MONITORING_DIR/grafana/provisioning/datasources"
|
|
|
|
log_success "Directories created"
|
|
}
|
|
|
|
# Create Prometheus configuration
|
|
create_prometheus_config() {
|
|
log_info "Creating Prometheus configuration..."
|
|
|
|
cat > "$MONITORING_DIR/prometheus/prometheus.yml" << 'EOF'
|
|
# Prometheus configuration for Kafka Load Test monitoring
|
|
|
|
global:
|
|
scrape_interval: 15s
|
|
evaluation_interval: 15s
|
|
|
|
rule_files:
|
|
# - "first_rules.yml"
|
|
# - "second_rules.yml"
|
|
|
|
scrape_configs:
|
|
# Scrape Prometheus itself
|
|
- job_name: 'prometheus'
|
|
static_configs:
|
|
- targets: ['localhost:9090']
|
|
|
|
# Scrape load test metrics
|
|
- job_name: 'kafka-loadtest'
|
|
static_configs:
|
|
- targets: ['kafka-client-loadtest-runner:8080']
|
|
scrape_interval: 5s
|
|
metrics_path: '/metrics'
|
|
|
|
# Scrape SeaweedFS Master metrics
|
|
- job_name: 'seaweedfs-master'
|
|
static_configs:
|
|
- targets: ['seaweedfs-master:9333']
|
|
metrics_path: '/metrics'
|
|
|
|
# Scrape SeaweedFS Volume metrics
|
|
- job_name: 'seaweedfs-volume'
|
|
static_configs:
|
|
- targets: ['seaweedfs-volume:8080']
|
|
metrics_path: '/metrics'
|
|
|
|
# Scrape SeaweedFS Filer metrics
|
|
- job_name: 'seaweedfs-filer'
|
|
static_configs:
|
|
- targets: ['seaweedfs-filer:8888']
|
|
metrics_path: '/metrics'
|
|
|
|
# Scrape SeaweedFS MQ Broker metrics (if available)
|
|
- job_name: 'seaweedfs-mq-broker'
|
|
static_configs:
|
|
- targets: ['seaweedfs-mq-broker:17777']
|
|
metrics_path: '/metrics'
|
|
scrape_interval: 10s
|
|
|
|
# Scrape Kafka Gateway metrics (if available)
|
|
- job_name: 'kafka-gateway'
|
|
static_configs:
|
|
- targets: ['kafka-gateway:9093']
|
|
metrics_path: '/metrics'
|
|
scrape_interval: 10s
|
|
EOF
|
|
|
|
log_success "Prometheus configuration created"
|
|
}
|
|
|
|
# Create Grafana datasource configuration
|
|
create_grafana_datasource() {
|
|
log_info "Creating Grafana datasource configuration..."
|
|
|
|
cat > "$MONITORING_DIR/grafana/provisioning/datasources/datasource.yml" << 'EOF'
|
|
apiVersion: 1
|
|
|
|
datasources:
|
|
- name: Prometheus
|
|
type: prometheus
|
|
access: proxy
|
|
orgId: 1
|
|
url: http://prometheus:9090
|
|
basicAuth: false
|
|
isDefault: true
|
|
editable: true
|
|
version: 1
|
|
EOF
|
|
|
|
log_success "Grafana datasource configuration created"
|
|
}
|
|
|
|
# Create Grafana dashboard provisioning
|
|
create_grafana_dashboard_provisioning() {
|
|
log_info "Creating Grafana dashboard provisioning..."
|
|
|
|
cat > "$MONITORING_DIR/grafana/provisioning/dashboards/dashboard.yml" << 'EOF'
|
|
apiVersion: 1
|
|
|
|
providers:
|
|
- name: 'default'
|
|
orgId: 1
|
|
folder: ''
|
|
type: file
|
|
disableDeletion: false
|
|
editable: true
|
|
options:
|
|
path: /var/lib/grafana/dashboards
|
|
EOF
|
|
|
|
log_success "Grafana dashboard provisioning created"
|
|
}
|
|
|
|
# Create Kafka Load Test dashboard
|
|
create_loadtest_dashboard() {
|
|
log_info "Creating Kafka Load Test Grafana dashboard..."
|
|
|
|
cat > "$MONITORING_DIR/grafana/dashboards/kafka-loadtest.json" << 'EOF'
|
|
{
|
|
"dashboard": {
|
|
"id": null,
|
|
"title": "Kafka Client Load Test Dashboard",
|
|
"tags": ["kafka", "loadtest", "seaweedfs"],
|
|
"timezone": "browser",
|
|
"panels": [
|
|
{
|
|
"id": 1,
|
|
"title": "Messages Produced/Consumed",
|
|
"type": "stat",
|
|
"targets": [
|
|
{
|
|
"expr": "rate(kafka_loadtest_messages_produced_total[5m])",
|
|
"legendFormat": "Produced/sec"
|
|
},
|
|
{
|
|
"expr": "rate(kafka_loadtest_messages_consumed_total[5m])",
|
|
"legendFormat": "Consumed/sec"
|
|
}
|
|
],
|
|
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 0}
|
|
},
|
|
{
|
|
"id": 2,
|
|
"title": "Message Latency",
|
|
"type": "graph",
|
|
"targets": [
|
|
{
|
|
"expr": "histogram_quantile(0.95, kafka_loadtest_message_latency_seconds)",
|
|
"legendFormat": "95th percentile"
|
|
},
|
|
{
|
|
"expr": "histogram_quantile(0.99, kafka_loadtest_message_latency_seconds)",
|
|
"legendFormat": "99th percentile"
|
|
}
|
|
],
|
|
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 0}
|
|
},
|
|
{
|
|
"id": 3,
|
|
"title": "Error Rates",
|
|
"type": "graph",
|
|
"targets": [
|
|
{
|
|
"expr": "rate(kafka_loadtest_producer_errors_total[5m])",
|
|
"legendFormat": "Producer Errors/sec"
|
|
},
|
|
{
|
|
"expr": "rate(kafka_loadtest_consumer_errors_total[5m])",
|
|
"legendFormat": "Consumer Errors/sec"
|
|
}
|
|
],
|
|
"gridPos": {"h": 8, "w": 24, "x": 0, "y": 8}
|
|
},
|
|
{
|
|
"id": 4,
|
|
"title": "Throughput (MB/s)",
|
|
"type": "graph",
|
|
"targets": [
|
|
{
|
|
"expr": "rate(kafka_loadtest_bytes_produced_total[5m]) / 1024 / 1024",
|
|
"legendFormat": "Produced MB/s"
|
|
},
|
|
{
|
|
"expr": "rate(kafka_loadtest_bytes_consumed_total[5m]) / 1024 / 1024",
|
|
"legendFormat": "Consumed MB/s"
|
|
}
|
|
],
|
|
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 16}
|
|
},
|
|
{
|
|
"id": 5,
|
|
"title": "Active Clients",
|
|
"type": "stat",
|
|
"targets": [
|
|
{
|
|
"expr": "kafka_loadtest_active_producers",
|
|
"legendFormat": "Producers"
|
|
},
|
|
{
|
|
"expr": "kafka_loadtest_active_consumers",
|
|
"legendFormat": "Consumers"
|
|
}
|
|
],
|
|
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 16}
|
|
},
|
|
{
|
|
"id": 6,
|
|
"title": "Consumer Lag",
|
|
"type": "graph",
|
|
"targets": [
|
|
{
|
|
"expr": "kafka_loadtest_consumer_lag_messages",
|
|
"legendFormat": "{{consumer_group}}-{{topic}}-{{partition}}"
|
|
}
|
|
],
|
|
"gridPos": {"h": 8, "w": 24, "x": 0, "y": 24}
|
|
}
|
|
],
|
|
"time": {"from": "now-30m", "to": "now"},
|
|
"refresh": "5s",
|
|
"schemaVersion": 16,
|
|
"version": 0
|
|
}
|
|
}
|
|
EOF
|
|
|
|
log_success "Kafka Load Test dashboard created"
|
|
}
|
|
|
|
# Create SeaweedFS dashboard
|
|
create_seaweedfs_dashboard() {
|
|
log_info "Creating SeaweedFS Grafana dashboard..."
|
|
|
|
cat > "$MONITORING_DIR/grafana/dashboards/seaweedfs.json" << 'EOF'
|
|
{
|
|
"dashboard": {
|
|
"id": null,
|
|
"title": "SeaweedFS Cluster Dashboard",
|
|
"tags": ["seaweedfs", "storage"],
|
|
"timezone": "browser",
|
|
"panels": [
|
|
{
|
|
"id": 1,
|
|
"title": "Master Status",
|
|
"type": "stat",
|
|
"targets": [
|
|
{
|
|
"expr": "up{job=\"seaweedfs-master\"}",
|
|
"legendFormat": "Master Up"
|
|
}
|
|
],
|
|
"gridPos": {"h": 4, "w": 6, "x": 0, "y": 0}
|
|
},
|
|
{
|
|
"id": 2,
|
|
"title": "Volume Status",
|
|
"type": "stat",
|
|
"targets": [
|
|
{
|
|
"expr": "up{job=\"seaweedfs-volume\"}",
|
|
"legendFormat": "Volume Up"
|
|
}
|
|
],
|
|
"gridPos": {"h": 4, "w": 6, "x": 6, "y": 0}
|
|
},
|
|
{
|
|
"id": 3,
|
|
"title": "Filer Status",
|
|
"type": "stat",
|
|
"targets": [
|
|
{
|
|
"expr": "up{job=\"seaweedfs-filer\"}",
|
|
"legendFormat": "Filer Up"
|
|
}
|
|
],
|
|
"gridPos": {"h": 4, "w": 6, "x": 12, "y": 0}
|
|
},
|
|
{
|
|
"id": 4,
|
|
"title": "MQ Broker Status",
|
|
"type": "stat",
|
|
"targets": [
|
|
{
|
|
"expr": "up{job=\"seaweedfs-mq-broker\"}",
|
|
"legendFormat": "MQ Broker Up"
|
|
}
|
|
],
|
|
"gridPos": {"h": 4, "w": 6, "x": 18, "y": 0}
|
|
}
|
|
],
|
|
"time": {"from": "now-30m", "to": "now"},
|
|
"refresh": "10s",
|
|
"schemaVersion": 16,
|
|
"version": 0
|
|
}
|
|
}
|
|
EOF
|
|
|
|
log_success "SeaweedFS dashboard created"
|
|
}
|
|
|
|
# Main setup function
|
|
main() {
|
|
log_info "Setting up monitoring for Kafka Client Load Test..."
|
|
|
|
setup_directories
|
|
create_prometheus_config
|
|
create_grafana_datasource
|
|
create_grafana_dashboard_provisioning
|
|
create_loadtest_dashboard
|
|
create_seaweedfs_dashboard
|
|
|
|
log_success "Monitoring setup completed!"
|
|
log_info "You can now start the monitoring stack with:"
|
|
log_info " ./scripts/run-loadtest.sh monitor"
|
|
log_info ""
|
|
log_info "After starting, access:"
|
|
log_info " Prometheus: http://localhost:9090"
|
|
log_info " Grafana: http://localhost:3000 (admin/admin)"
|
|
}
|
|
|
|
main "$@"
|