Browse Source

fix: prevent gRPC port collisions during multi-service fallback allocation

Critical fix for gRPC port allocation safety across multiple services:

Problem: When multiple services need gRPC port fallback allocation in sequence
(e.g., Master gRPC unavailable → finds alternative, then Filer gRPC unavailable
→ searches from calculated port), there was no tracking of previously allocated
gRPC ports. This could allow two services to claim the same port.

Scenario that is now prevented:
- Master gRPC: calculated 19333 unavailable → finds 19334 → assigns 19334
- Filer gRPC: calculated 18888 unavailable → searches from 18889, might land on
  19334 if consecutive ports in range are unavailable (especially with custom
  port configurations or in high-port-contention environments)

Solution:
- Add allocatedGrpcPorts map to track gRPC ports allocated within the function
- Check allocatedGrpcPorts before using calculated port for each service
- Pass allocatedGrpcPorts to findAvailablePortOnIP when finding fallback ports
- Add allocatedGrpcPorts[port] = true after each successful allocation
- This ensures no two services can allocate the same gRPC port

The fix handles both:
1. Calculated gRPC ports (when grpcPort == 0)
2. Explicitly set gRPC ports (when user provides -service.port.grpc value)

While default port spacing makes collision unlikely, this fix is essential for:
- Custom port configurations
- High-contention environments
- Edge cases with many unavailable consecutive ports
- Correctness and safety guarantees
pull/7838/head
Chris Lu 2 months ago
parent
commit
ccc0a20dbf
  1. 17
      weed/command/mini.go

17
weed/command/mini.go

@ -475,6 +475,10 @@ func ensureAllPortsAvailableOnIP(bindIp string) {
// If a gRPC port is 0, it will be set to httpPort + GrpcPortOffset // If a gRPC port is 0, it will be set to httpPort + GrpcPortOffset
// This must be called after HTTP ports are finalized and before services start // This must be called after HTTP ports are finalized and before services start
func initializeGrpcPortsOnIP(bindIp string) { func initializeGrpcPortsOnIP(bindIp string) {
// Track gRPC ports allocated during this function to prevent collisions between services
// when multiple services need fallback port allocation
allocatedGrpcPorts := make(map[int]bool)
grpcConfigs := []struct { grpcConfigs := []struct {
httpPort *int httpPort *int
grpcPort *int grpcPort *int
@ -496,23 +500,27 @@ func initializeGrpcPortsOnIP(bindIp string) {
if *config.grpcPort == 0 { if *config.grpcPort == 0 {
calculatedPort := *config.httpPort + GrpcPortOffset calculatedPort := *config.httpPort + GrpcPortOffset
// Check if calculated port is available (on both specific IP and all interfaces) // Check if calculated port is available (on both specific IP and all interfaces)
if !isPortOpenOnIP(bindIp, calculatedPort) || !isPortAvailable(calculatedPort) {
// Also check if it was already allocated to another service in this function
if !isPortOpenOnIP(bindIp, calculatedPort) || !isPortAvailable(calculatedPort) || allocatedGrpcPorts[calculatedPort] {
glog.Warningf("Calculated gRPC port %d for %s is not available, finding alternative...", calculatedPort, config.name) glog.Warningf("Calculated gRPC port %d for %s is not available, finding alternative...", calculatedPort, config.name)
newPort := findAvailablePortOnIP(bindIp, calculatedPort+1, 100, make(map[int]bool))
newPort := findAvailablePortOnIP(bindIp, calculatedPort+1, 100, allocatedGrpcPorts)
if newPort == 0 { if newPort == 0 {
glog.Errorf("Could not find available gRPC port for %s starting from %d, will use calculated %d and fail on binding", config.name, calculatedPort+1, calculatedPort) glog.Errorf("Could not find available gRPC port for %s starting from %d, will use calculated %d and fail on binding", config.name, calculatedPort+1, calculatedPort)
calculatedPort = calculatedPort
} else { } else {
calculatedPort = newPort calculatedPort = newPort
glog.Infof("gRPC port %d for %s is available, using it instead of calculated %d", newPort, config.name, *config.httpPort+GrpcPortOffset) glog.Infof("gRPC port %d for %s is available, using it instead of calculated %d", newPort, config.name, *config.httpPort+GrpcPortOffset)
} }
} }
*config.grpcPort = calculatedPort *config.grpcPort = calculatedPort
allocatedGrpcPorts[calculatedPort] = true
glog.V(1).Infof("%s gRPC port initialized to %d", config.name, calculatedPort) glog.V(1).Infof("%s gRPC port initialized to %d", config.name, calculatedPort)
} else { } else {
// gRPC port was explicitly set, verify it's still available (check on both specific IP and all interfaces) // gRPC port was explicitly set, verify it's still available (check on both specific IP and all interfaces)
if !isPortOpenOnIP(bindIp, *config.grpcPort) || !isPortAvailable(*config.grpcPort) {
// Also check if it was already allocated to another service in this function
if !isPortOpenOnIP(bindIp, *config.grpcPort) || !isPortAvailable(*config.grpcPort) || allocatedGrpcPorts[*config.grpcPort] {
glog.Warningf("Explicitly set gRPC port %d for %s is not available, finding alternative...", *config.grpcPort, config.name) glog.Warningf("Explicitly set gRPC port %d for %s is not available, finding alternative...", *config.grpcPort, config.name)
newPort := findAvailablePortOnIP(bindIp, *config.grpcPort+1, 100, make(map[int]bool))
newPort := findAvailablePortOnIP(bindIp, *config.grpcPort+1, 100, allocatedGrpcPorts)
if newPort == 0 { if newPort == 0 {
glog.Errorf("Could not find available gRPC port for %s starting from %d, will use original %d and fail on binding", config.name, *config.grpcPort+1, *config.grpcPort) glog.Errorf("Could not find available gRPC port for %s starting from %d, will use original %d and fail on binding", config.name, *config.grpcPort+1, *config.grpcPort)
} else { } else {
@ -520,6 +528,7 @@ func initializeGrpcPortsOnIP(bindIp string) {
*config.grpcPort = newPort *config.grpcPort = newPort
} }
} }
allocatedGrpcPorts[*config.grpcPort] = true
} }
} }
} }

Loading…
Cancel
Save