#!/bin/bash set -euo pipefail # Colors for output RED='\033[0;31m' GREEN='\033[0;32m' BLUE='\033[0;34m' YELLOW='\033[1;33m' NC='\033[0m' # No Color # Configuration MOUNT_POINT=${MOUNT_POINT:-"/mnt/seaweedfs"} RDMA_SIDECAR_ADDR=${RDMA_SIDECAR_ADDR:-"rdma-sidecar:8081"} PERFORMANCE_RESULTS_DIR=${PERFORMANCE_RESULTS_DIR:-"/performance-results"} # Create results directory mkdir -p "$PERFORMANCE_RESULTS_DIR" # Log file LOG_FILE="$PERFORMANCE_RESULTS_DIR/performance-test.log" exec > >(tee -a "$LOG_FILE") exec 2>&1 echo -e "${BLUE}🏁 SEAWEEDFS RDMA MOUNT PERFORMANCE TESTS${NC}" echo "===========================================" echo "Mount Point: $MOUNT_POINT" echo "RDMA Sidecar: $RDMA_SIDECAR_ADDR" echo "Results Directory: $PERFORMANCE_RESULTS_DIR" echo "Log File: $LOG_FILE" echo "" # Function to wait for mount to be ready wait_for_mount() { local max_attempts=30 local attempt=1 echo -e "${BLUE}âģ Waiting for mount to be ready...${NC}" while [[ $attempt -le $max_attempts ]]; do if mountpoint -q "$MOUNT_POINT" 2>/dev/null && ls "$MOUNT_POINT" >/dev/null 2>&1; then echo -e "${GREEN}✅ Mount is ready${NC}" return 0 fi echo " Attempt $attempt/$max_attempts..." sleep 2 ((attempt++)) done echo -e "${RED}❌ Mount failed to be ready${NC}" return 1 } # Function to get RDMA statistics get_rdma_stats() { curl -s "http://$RDMA_SIDECAR_ADDR/stats" 2>/dev/null || echo "{}" } # Function to run dd performance test run_dd_test() { local test_name=$1 local file_size_mb=$2 local block_size=$3 local operation=$4 # "write" or "read" local test_file="$MOUNT_POINT/perf_test_${test_name}.dat" local result_file="$PERFORMANCE_RESULTS_DIR/dd_${test_name}.json" echo -e "${BLUE}🔎 Running DD test: $test_name${NC}" echo " Size: ${file_size_mb}MB, Block Size: $block_size, Operation: $operation" local start_time end_time duration_ms throughput_mbps if [[ "$operation" == "write" ]]; then start_time=$(date +%s%N) dd if=/dev/zero of="$test_file" bs="$block_size" count=$((file_size_mb * 1024 * 1024 / $(numfmt --from=iec "$block_size"))) 2>/dev/null end_time=$(date +%s%N) else # Create file first if it doesn't exist if [[ ! -f "$test_file" ]]; then dd if=/dev/zero of="$test_file" bs=1M count="$file_size_mb" 2>/dev/null fi start_time=$(date +%s%N) dd if="$test_file" of=/dev/null bs="$block_size" 2>/dev/null end_time=$(date +%s%N) fi duration_ms=$(((end_time - start_time) / 1000000)) throughput_mbps=$(bc <<< "scale=2; $file_size_mb * 1000 / $duration_ms") # Save results cat > "$result_file" << EOF { "test_name": "$test_name", "operation": "$operation", "file_size_mb": $file_size_mb, "block_size": "$block_size", "duration_ms": $duration_ms, "throughput_mbps": $throughput_mbps, "timestamp": "$(date -Iseconds)" } EOF echo " Duration: ${duration_ms}ms" echo " Throughput: ${throughput_mbps} MB/s" echo "" # Cleanup write test files if [[ "$operation" == "write" ]]; then rm -f "$test_file" fi } # Function to run FIO performance test run_fio_test() { local test_name=$1 local rw_type=$2 # "read", "write", "randread", "randwrite" local block_size=$3 local file_size=$4 local iodepth=$5 local test_file="$MOUNT_POINT/fio_test_${test_name}.dat" local result_file="$PERFORMANCE_RESULTS_DIR/fio_${test_name}.json" echo -e "${BLUE}🔎 Running FIO test: $test_name${NC}" echo " Type: $rw_type, Block Size: $block_size, File Size: $file_size, IO Depth: $iodepth" # Run FIO test fio --name="$test_name" \ --filename="$test_file" \ --rw="$rw_type" \ --bs="$block_size" \ --size="$file_size" \ --iodepth="$iodepth" \ --direct=1 \ --runtime=30 \ --time_based \ --group_reporting \ --output-format=json \ --output="$result_file" \ 2>/dev/null # Extract key metrics if [[ -f "$result_file" ]]; then local iops throughput_kbps latency_us iops=$(jq -r '.jobs[0].'"$rw_type"'.iops // 0' "$result_file" 2>/dev/null || echo "0") throughput_kbps=$(jq -r '.jobs[0].'"$rw_type"'.bw // 0' "$result_file" 2>/dev/null || echo "0") latency_us=$(jq -r '.jobs[0].'"$rw_type"'.lat_ns.mean // 0' "$result_file" 2>/dev/null || echo "0") latency_us=$(bc <<< "scale=2; $latency_us / 1000" 2>/dev/null || echo "0") echo " IOPS: $iops" echo " Throughput: $(bc <<< "scale=2; $throughput_kbps / 1024") MB/s" echo " Average Latency: ${latency_us} Ξs" else echo " FIO test failed or no results" fi echo "" # Cleanup rm -f "$test_file" } # Function to run concurrent access test run_concurrent_test() { local num_processes=$1 local file_size_mb=$2 echo -e "${BLUE}🔎 Running concurrent access test${NC}" echo " Processes: $num_processes, File Size per Process: ${file_size_mb}MB" local start_time end_time duration_ms total_throughput local pids=() start_time=$(date +%s%N) # Start concurrent processes for i in $(seq 1 "$num_processes"); do ( local test_file="$MOUNT_POINT/concurrent_test_$i.dat" dd if=/dev/zero of="$test_file" bs=1M count="$file_size_mb" 2>/dev/null dd if="$test_file" of=/dev/null bs=1M 2>/dev/null rm -f "$test_file" ) & pids+=($!) done # Wait for all processes to complete for pid in "${pids[@]}"; do wait "$pid" done end_time=$(date +%s%N) duration_ms=$(((end_time - start_time) / 1000000)) total_throughput=$(bc <<< "scale=2; $num_processes * $file_size_mb * 2 * 1000 / $duration_ms") # Save results cat > "$PERFORMANCE_RESULTS_DIR/concurrent_test.json" << EOF { "test_name": "concurrent_access", "num_processes": $num_processes, "file_size_mb_per_process": $file_size_mb, "total_data_mb": $((num_processes * file_size_mb * 2)), "duration_ms": $duration_ms, "total_throughput_mbps": $total_throughput, "timestamp": "$(date -Iseconds)" } EOF echo " Duration: ${duration_ms}ms" echo " Total Throughput: ${total_throughput} MB/s" echo "" } # Function to generate performance report generate_report() { local report_file="$PERFORMANCE_RESULTS_DIR/performance_report.html" echo -e "${BLUE}📊 Generating performance report...${NC}" cat > "$report_file" << 'EOF' SeaweedFS RDMA Mount Performance Report

🏁 SeaweedFS RDMA Mount Performance Report

Generated: $(date)

Mount Point: $MOUNT_POINT

RDMA Sidecar: $RDMA_SIDECAR_ADDR

EOF # Add DD test results echo '

DD Performance Tests

' >> "$report_file" for result_file in "$PERFORMANCE_RESULTS_DIR"/dd_*.json; do if [[ -f "$result_file" ]]; then local test_name operation file_size_mb block_size throughput_mbps duration_ms test_name=$(jq -r '.test_name' "$result_file" 2>/dev/null || echo "unknown") operation=$(jq -r '.operation' "$result_file" 2>/dev/null || echo "unknown") file_size_mb=$(jq -r '.file_size_mb' "$result_file" 2>/dev/null || echo "0") block_size=$(jq -r '.block_size' "$result_file" 2>/dev/null || echo "unknown") throughput_mbps=$(jq -r '.throughput_mbps' "$result_file" 2>/dev/null || echo "0") duration_ms=$(jq -r '.duration_ms' "$result_file" 2>/dev/null || echo "0") echo "" >> "$report_file" fi done echo '
TestOperationSizeBlock SizeThroughput (MB/s)Duration (ms)
$test_name$operation${file_size_mb}MB$block_size$throughput_mbps$duration_ms
' >> "$report_file" # Add FIO test results echo '

FIO Performance Tests

' >> "$report_file" echo '

Detailed FIO results are available in individual JSON files.

' >> "$report_file" # Add concurrent test results if [[ -f "$PERFORMANCE_RESULTS_DIR/concurrent_test.json" ]]; then echo '

Concurrent Access Test

' >> "$report_file" local num_processes total_throughput duration_ms num_processes=$(jq -r '.num_processes' "$PERFORMANCE_RESULTS_DIR/concurrent_test.json" 2>/dev/null || echo "0") total_throughput=$(jq -r '.total_throughput_mbps' "$PERFORMANCE_RESULTS_DIR/concurrent_test.json" 2>/dev/null || echo "0") duration_ms=$(jq -r '.duration_ms' "$PERFORMANCE_RESULTS_DIR/concurrent_test.json" 2>/dev/null || echo "0") echo "

Processes: $num_processes

" >> "$report_file" echo "

Total Throughput: $total_throughput MB/s

" >> "$report_file" echo "

Duration: $duration_ms ms

" >> "$report_file" echo '
' >> "$report_file" fi echo '' >> "$report_file" echo " Report saved to: $report_file" } # Main test execution main() { echo -e "${BLUE}🚀 Starting performance tests...${NC}" echo "" # Wait for mount to be ready if ! wait_for_mount; then echo -e "${RED}❌ Mount is not ready, aborting tests${NC}" exit 1 fi # Get initial RDMA stats echo -e "${BLUE}📊 Initial RDMA Statistics:${NC}" get_rdma_stats | jq . 2>/dev/null || get_rdma_stats echo "" # Run DD performance tests echo -e "${BLUE}🏃 Running DD Performance Tests...${NC}" run_dd_test "small_write" 10 "4k" "write" run_dd_test "small_read" 10 "4k" "read" run_dd_test "medium_write" 100 "64k" "write" run_dd_test "medium_read" 100 "64k" "read" run_dd_test "large_write" 500 "1M" "write" run_dd_test "large_read" 500 "1M" "read" # Run FIO performance tests echo -e "${BLUE}🏃 Running FIO Performance Tests...${NC}" run_fio_test "seq_read" "read" "64k" "100M" 1 run_fio_test "seq_write" "write" "64k" "100M" 1 run_fio_test "rand_read" "randread" "4k" "100M" 16 run_fio_test "rand_write" "randwrite" "4k" "100M" 16 # Run concurrent access test echo -e "${BLUE}🏃 Running Concurrent Access Test...${NC}" run_concurrent_test 4 50 # Get final RDMA stats echo -e "${BLUE}📊 Final RDMA Statistics:${NC}" get_rdma_stats | jq . 2>/dev/null || get_rdma_stats echo "" # Generate performance report generate_report echo -e "${GREEN}🎉 Performance tests completed!${NC}" echo "Results saved to: $PERFORMANCE_RESULTS_DIR" } # Run main function main "$@"