From ccc0a20dbf7d86baff3b8d88c612ed4f791da1b3 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 21 Dec 2025 21:59:12 -0800 Subject: [PATCH] fix: prevent gRPC port collisions during multi-service fallback allocation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Critical fix for gRPC port allocation safety across multiple services: Problem: When multiple services need gRPC port fallback allocation in sequence (e.g., Master gRPC unavailable → finds alternative, then Filer gRPC unavailable → searches from calculated port), there was no tracking of previously allocated gRPC ports. This could allow two services to claim the same port. Scenario that is now prevented: - Master gRPC: calculated 19333 unavailable → finds 19334 → assigns 19334 - Filer gRPC: calculated 18888 unavailable → searches from 18889, might land on 19334 if consecutive ports in range are unavailable (especially with custom port configurations or in high-port-contention environments) Solution: - Add allocatedGrpcPorts map to track gRPC ports allocated within the function - Check allocatedGrpcPorts before using calculated port for each service - Pass allocatedGrpcPorts to findAvailablePortOnIP when finding fallback ports - Add allocatedGrpcPorts[port] = true after each successful allocation - This ensures no two services can allocate the same gRPC port The fix handles both: 1. Calculated gRPC ports (when grpcPort == 0) 2. Explicitly set gRPC ports (when user provides -service.port.grpc value) While default port spacing makes collision unlikely, this fix is essential for: - Custom port configurations - High-contention environments - Edge cases with many unavailable consecutive ports - Correctness and safety guarantees --- weed/command/mini.go | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/weed/command/mini.go b/weed/command/mini.go index 224dc0c1b..abe699917 100644 --- a/weed/command/mini.go +++ b/weed/command/mini.go @@ -475,6 +475,10 @@ func ensureAllPortsAvailableOnIP(bindIp string) { // If a gRPC port is 0, it will be set to httpPort + GrpcPortOffset // This must be called after HTTP ports are finalized and before services start func initializeGrpcPortsOnIP(bindIp string) { + // Track gRPC ports allocated during this function to prevent collisions between services + // when multiple services need fallback port allocation + allocatedGrpcPorts := make(map[int]bool) + grpcConfigs := []struct { httpPort *int grpcPort *int @@ -496,23 +500,27 @@ func initializeGrpcPortsOnIP(bindIp string) { if *config.grpcPort == 0 { calculatedPort := *config.httpPort + GrpcPortOffset // Check if calculated port is available (on both specific IP and all interfaces) - if !isPortOpenOnIP(bindIp, calculatedPort) || !isPortAvailable(calculatedPort) { + // Also check if it was already allocated to another service in this function + if !isPortOpenOnIP(bindIp, calculatedPort) || !isPortAvailable(calculatedPort) || allocatedGrpcPorts[calculatedPort] { glog.Warningf("Calculated gRPC port %d for %s is not available, finding alternative...", calculatedPort, config.name) - newPort := findAvailablePortOnIP(bindIp, calculatedPort+1, 100, make(map[int]bool)) + newPort := findAvailablePortOnIP(bindIp, calculatedPort+1, 100, allocatedGrpcPorts) if newPort == 0 { glog.Errorf("Could not find available gRPC port for %s starting from %d, will use calculated %d and fail on binding", config.name, calculatedPort+1, calculatedPort) + calculatedPort = calculatedPort } else { calculatedPort = newPort glog.Infof("gRPC port %d for %s is available, using it instead of calculated %d", newPort, config.name, *config.httpPort+GrpcPortOffset) } } *config.grpcPort = calculatedPort + allocatedGrpcPorts[calculatedPort] = true glog.V(1).Infof("%s gRPC port initialized to %d", config.name, calculatedPort) } else { // gRPC port was explicitly set, verify it's still available (check on both specific IP and all interfaces) - if !isPortOpenOnIP(bindIp, *config.grpcPort) || !isPortAvailable(*config.grpcPort) { + // Also check if it was already allocated to another service in this function + if !isPortOpenOnIP(bindIp, *config.grpcPort) || !isPortAvailable(*config.grpcPort) || allocatedGrpcPorts[*config.grpcPort] { glog.Warningf("Explicitly set gRPC port %d for %s is not available, finding alternative...", *config.grpcPort, config.name) - newPort := findAvailablePortOnIP(bindIp, *config.grpcPort+1, 100, make(map[int]bool)) + newPort := findAvailablePortOnIP(bindIp, *config.grpcPort+1, 100, allocatedGrpcPorts) if newPort == 0 { glog.Errorf("Could not find available gRPC port for %s starting from %d, will use original %d and fail on binding", config.name, *config.grpcPort+1, *config.grpcPort) } else { @@ -520,6 +528,7 @@ func initializeGrpcPortsOnIP(bindIp string) { *config.grpcPort = newPort } } + allocatedGrpcPorts[*config.grpcPort] = true } } }