Browse Source

Fix flaky s3tables tests: allocate all ports atomically

The test port allocation had a TOCTOU race where GetFreePort() would
open a listener, grab the port number, then immediately close it.
When called repeatedly, the OS could recycle a just-released port,
causing two services (e.g. Filer and S3) to be assigned the same port.

Replace per-call GetFreePort() with batch AllocatePorts() that holds
all listeners open until every port is obtained, matching the pattern
already used in test/volume_server/framework/cluster.go.
pull/5490/merge
Chris Lu 22 hours ago
parent
commit
e7fc243ee1
  1. 11
      test/s3tables/lakekeeper/lakekeeper_test.go
  2. 13
      test/s3tables/polaris/polaris_env_test.go
  3. 11
      test/s3tables/sts_integration/sts_integration_test.go
  4. 49
      test/s3tables/testutil/docker.go

11
test/s3tables/lakekeeper/lakekeeper_test.go

@ -115,10 +115,13 @@ func NewTestEnvironment(t *testing.T) *TestEnvironment {
bindIP := testutil.FindBindIP()
masterPort, masterGrpcPort := testutil.MustFreePortPair(t, "Master")
volumePort, volumeGrpcPort := testutil.MustFreePortPair(t, "Volume")
filerPort, filerGrpcPort := testutil.MustFreePortPair(t, "Filer")
s3Port, s3GrpcPort := testutil.MustFreePortPair(t, "S3")
// Allocate all ports in a single batch to prevent the OS from recycling
// a released port, which can cause two services to get the same port.
ports := testutil.MustAllocatePorts(t, 8)
masterPort, masterGrpcPort := ports[0], ports[1]
volumePort, volumeGrpcPort := ports[2], ports[3]
filerPort, filerGrpcPort := ports[4], ports[5]
s3Port, s3GrpcPort := ports[6], ports[7]
return &TestEnvironment{
seaweedDir: seaweedDir,

13
test/s3tables/polaris/polaris_env_test.go

@ -89,11 +89,14 @@ func NewTestEnvironment(t *testing.T) *TestEnvironment {
bindIP := testutil.FindBindIP()
masterPort, masterGrpcPort := testutil.MustFreePortPair(t, "Master")
volumePort, volumeGrpcPort := testutil.MustFreePortPair(t, "Volume")
filerPort, filerGrpcPort := testutil.MustFreePortPair(t, "Filer")
s3Port, s3GrpcPort := testutil.MustFreePortPair(t, "S3")
polarisPort, polarisAdminPort := testutil.MustFreePortPair(t, "Polaris")
// Allocate all ports in a single batch to prevent the OS from recycling
// a released port, which can cause two services to get the same port.
ports := testutil.MustAllocatePorts(t, 10)
masterPort, masterGrpcPort := ports[0], ports[1]
volumePort, volumeGrpcPort := ports[2], ports[3]
filerPort, filerGrpcPort := ports[4], ports[5]
s3Port, s3GrpcPort := ports[6], ports[7]
polarisPort, polarisAdminPort := ports[8], ports[9]
return &TestEnvironment{
seaweedDir: seaweedDir,

11
test/s3tables/sts_integration/sts_integration_test.go

@ -93,10 +93,13 @@ func NewTestEnvironment(t *testing.T) *TestEnvironment {
bindIP := testutil.FindBindIP()
masterPort, masterGrpcPort := testutil.MustFreePortPair(t, "Master")
volumePort, volumeGrpcPort := testutil.MustFreePortPair(t, "Volume")
filerPort, filerGrpcPort := testutil.MustFreePortPair(t, "Filer")
s3Port, s3GrpcPort := testutil.MustFreePortPair(t, "S3") // Changed to use testutil.MustFreePortPair
// Allocate all ports in a single batch to prevent the OS from recycling
// a released port, which can cause two services to get the same port.
ports := testutil.MustAllocatePorts(t, 8)
masterPort, masterGrpcPort := ports[0], ports[1]
volumePort, volumeGrpcPort := ports[2], ports[3]
filerPort, filerGrpcPort := ports[4], ports[5]
s3Port, s3GrpcPort := ports[6], ports[7]
return &TestEnvironment{
seaweedDir: seaweedDir,

49
test/s3tables/testutil/docker.go

@ -15,33 +15,44 @@ func HasDocker() bool {
return cmd.Run() == nil
}
// MustFreePortPair is a convenience wrapper for tests that only need a single pair.
// Prefer MustAllocatePorts when allocating multiple pairs to guarantee uniqueness.
func MustFreePortPair(t *testing.T, name string) (int, int) {
httpPort, grpcPort, err := findAvailablePortPair()
if err != nil {
t.Fatalf("Failed to get free port pair for %s: %v", name, err)
}
return httpPort, grpcPort
ports := MustAllocatePorts(t, 2)
return ports[0], ports[1]
}
func findAvailablePortPair() (int, int, error) {
httpPort, err := GetFreePort()
if err != nil {
return 0, 0, err
}
grpcPort, err := GetFreePort()
// MustAllocatePorts allocates count unique free ports atomically.
// All listeners are held open until every port is obtained, preventing
// the OS from recycling a port between successive allocations.
func MustAllocatePorts(t *testing.T, count int) []int {
t.Helper()
ports, err := AllocatePorts(count)
if err != nil {
return 0, 0, err
t.Fatalf("Failed to allocate %d free ports: %v", count, err)
}
return httpPort, grpcPort, nil
return ports
}
func GetFreePort() (int, error) {
listener, err := net.Listen("tcp", "0.0.0.0:0")
if err != nil {
return 0, err
// AllocatePorts allocates count unique free ports atomically.
func AllocatePorts(count int) ([]int, error) {
listeners := make([]net.Listener, 0, count)
ports := make([]int, 0, count)
for i := 0; i < count; i++ {
l, err := net.Listen("tcp", "0.0.0.0:0")
if err != nil {
for _, ll := range listeners {
_ = ll.Close()
}
return nil, err
}
listeners = append(listeners, l)
ports = append(ports, l.Addr().(*net.TCPAddr).Port)
}
for _, l := range listeners {
_ = l.Close()
}
defer listener.Close()
return listener.Addr().(*net.TCPAddr).Port, nil
return ports, nil
}
func WaitForService(url string, timeout time.Duration) bool {

Loading…
Cancel
Save