viaprog/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2026-06-13 23:36:45 +03:00
Code Issues Packages Projects Releases Wiki Activity

fix(test): avoid port collision between master gRPC and volume ports

Browse Source 
AllocateMiniPorts(1) reserved masterPort and masterPort+GrpcPortOffset
by holding listeners open, but closed them on return. The subsequent
AllocatePorts call bound 127.0.0.1:0, so the OS could immediately reuse
the just-released mini gRPC port as a volume port — causing the volume
server to fail at bind time with "address already in use".

Introduce AllocatePortSet(miniCount, regularCount) that holds every
listener open until the full set is chosen, and route the five volume
test cluster builders through it.
This commit is contained in:
Chris Lu
2026-04-21 23:33:57 -07:00
parent 96e5fea08e
commit be9996962d
7 changed files with 107 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/testutil/ports.go
+60
View File
@@ -118,3 +118,63 @@ func MustFreeMiniPort(t *testing.T, name string) int {
t.Helper()
return MustFreeMiniPorts(t, []string{name})[0]
}
// AllocatePortSet atomically allocates miniCount master-style port pairs (each
// with port and port+GrpcPortOffset reserved) plus regularCount additional
// distinct ports. All listeners are held open until the entire batch is
// allocated, so a mini gRPC port cannot be recycled as a regular port mid-batch.
func AllocatePortSet(miniCount, regularCount int) (mini []int, regular []int, err error) {
const (
minPort = 10000
maxPort = 55000
)
reserved := make(map[int]bool)
mini = make([]int, 0, miniCount)
var listeners []net.Listener
defer func() {
for _, l := range listeners {
l.Close()
}
}()
for idx := 0; idx < miniCount; idx++ {
found := false
for i := 0; i < 1000; i++ {
port := minPort + rand.Intn(maxPort-minPort)
grpcPort := port + GrpcPortOffset
if reserved[port] || reserved[grpcPort] {
continue
}
l1, lErr := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port))
if lErr != nil {
continue
}
l2, lErr := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", grpcPort))
if lErr != nil {
l1.Close()
continue
}
listeners = append(listeners, l1, l2)
reserved[port] = true
reserved[grpcPort] = true
mini = append(mini, port)
found = true
break
}
if !found {
return nil, nil, fmt.Errorf("failed to allocate mini port %d of %d", idx+1, miniCount)
}
}
regular = make([]int, 0, regularCount)
for i := 0; i < regularCount; i++ {
l, lErr := net.Listen("tcp", "127.0.0.1:0")
if lErr != nil {
return nil, nil, lErr
}
listeners = append(listeners, l)
regular = append(regular, l.Addr().(*net.TCPAddr).Port)
}
return mini, regular, nil
}
test/testutil/ports_test.go
+29
View File
@@ -0,0 +1,29 @@
package testutil
import "testing"
func TestAllocatePortSetNoGrpcCollision(t *testing.T) {
// Run a few iterations to catch the OS-recycles-just-closed-port race
// that previously hit regular ports when the mini gRPC offset was freed
// between AllocateMiniPorts and AllocatePorts calls.
for iter := 0; iter < 20; iter++ {
mini, regular, err := AllocatePortSet(1, 3)
if err != nil {
t.Fatalf("iter %d: AllocatePortSet: %v", iter, err)
}
if len(mini) != 1 || len(regular) != 3 {
t.Fatalf("iter %d: unexpected counts mini=%d regular=%d", iter, len(mini), len(regular))
}
reserved := map[int]bool{
mini[0]: true,
mini[0] + GrpcPortOffset: true,
}
for _, p := range regular {
if reserved[p] {
t.Fatalf("iter %d: regular port %d collides with mini pair %d/%d",
iter, p, mini[0], mini[0]+GrpcPortOffset)
}
reserved[p] = true
}
}
}
test/volume_server/framework/cluster.go
+3 -8
View File
@@ -85,17 +85,12 @@ func StartSingleVolumeCluster(t testing.TB, profile matrix.Profile) *Cluster {
t.Fatalf("write security config: %v", err)
}
miniPorts, err := testutil.AllocateMiniPorts(1)
if err != nil {
t.Fatalf("allocate master port pair: %v", err)
}
masterPort := miniPorts[0]
masterGrpcPort := masterPort + testutil.GrpcPortOffset
ports, err := testutil.AllocatePorts(3)
miniPorts, ports, err := testutil.AllocatePortSet(1, 3)
if err != nil {
t.Fatalf("allocate ports: %v", err)
}
masterPort := miniPorts[0]
masterGrpcPort := masterPort + testutil.GrpcPortOffset
c := &Cluster{
testingTB: t,
test/volume_server/framework/cluster_mixed.go
+4 -9
View File
@@ -93,22 +93,17 @@ func StartMixedVolumeCluster(t testing.TB, profile matrix.Profile, goCount, rust
t.Fatalf("write security config: %v", err)
}
miniPorts, err := testutil.AllocateMiniPorts(1)
if err != nil {
t.Fatalf("allocate master port pair: %v", err)
}
masterPort := miniPorts[0]
masterGrpcPort := masterPort + testutil.GrpcPortOffset
// 2 ports per server (admin, grpc); add 1 more when public port is split out.
portsPerServer := 2
if profile.SplitPublicPort {
portsPerServer = 3
}
ports, err := testutil.AllocatePorts(total * portsPerServer)
miniPorts, ports, err := testutil.AllocatePortSet(1, total*portsPerServer)
if err != nil {
t.Fatalf("allocate volume ports: %v", err)
t.Fatalf("allocate ports: %v", err)
}
masterPort := miniPorts[0]
masterGrpcPort := masterPort + testutil.GrpcPortOffset
isRust := make([]bool, total)
for i := goCount; i < total; i++ {
test/volume_server/framework/cluster_multi.go
+4 -9
View File
@@ -75,13 +75,6 @@ func StartMultiVolumeCluster(t testing.TB, profile matrix.Profile, serverCount i
t.Fatalf("write security config: %v", err)
}
miniPorts, err := testutil.AllocateMiniPorts(1)
if err != nil {
t.Fatalf("allocate master port pair: %v", err)
}
masterPort := miniPorts[0]
masterGrpcPort := masterPort + testutil.GrpcPortOffset
// Allocate ports for all volume servers (3 ports per server: admin, grpc, public)
// If SplitPublicPort is true, we need an additional port per server
portsPerServer := 3
@@ -89,10 +82,12 @@ func StartMultiVolumeCluster(t testing.TB, profile matrix.Profile, serverCount i
portsPerServer = 4
}
totalPorts := serverCount * portsPerServer
ports, err := testutil.AllocatePorts(totalPorts)
miniPorts, ports, err := testutil.AllocatePortSet(1, totalPorts)
if err != nil {
t.Fatalf("allocate volume ports: %v", err)
t.Fatalf("allocate ports: %v", err)
}
masterPort := miniPorts[0]
masterGrpcPort := masterPort + testutil.GrpcPortOffset
c := &MultiVolumeCluster{
testingTB: t,
test/volume_server/framework/cluster_multi_rust.go
+4 -9
View File
@@ -86,13 +86,6 @@ func StartRustMultiVolumeCluster(t testing.TB, profile matrix.Profile, serverCou
t.Fatalf("write security config: %v", err)
}
miniPorts, err := testutil.AllocateMiniPorts(1)
if err != nil {
t.Fatalf("allocate master port pair: %v", err)
}
masterPort := miniPorts[0]
masterGrpcPort := masterPort + testutil.GrpcPortOffset
// Allocate ports for all volume servers (3 ports per server: admin, grpc, public)
// If SplitPublicPort is true, we need an additional port per server
portsPerServer := 3
@@ -100,10 +93,12 @@ func StartRustMultiVolumeCluster(t testing.TB, profile matrix.Profile, serverCou
portsPerServer = 4
}
totalPorts := serverCount * portsPerServer
ports, err := testutil.AllocatePorts(totalPorts)
miniPorts, ports, err := testutil.AllocatePortSet(1, totalPorts)
if err != nil {
t.Fatalf("allocate volume ports: %v", err)
t.Fatalf("allocate ports: %v", err)
}
masterPort := miniPorts[0]
masterGrpcPort := masterPort + testutil.GrpcPortOffset
c := &RustMultiVolumeCluster{
testingTB: t,
test/volume_server/framework/cluster_rust.go
+3 -8
View File
@@ -80,17 +80,12 @@ func StartRustVolumeCluster(t testing.TB, profile matrix.Profile) *RustCluster {
t.Fatalf("write security config: %v", err)
}
miniPorts, err := testutil.AllocateMiniPorts(1)
if err != nil {
t.Fatalf("allocate master port pair: %v", err)
}
masterPort := miniPorts[0]
masterGrpcPort := masterPort + testutil.GrpcPortOffset
ports, err := testutil.AllocatePorts(3)
miniPorts, ports, err := testutil.AllocatePortSet(1, 3)
if err != nil {
t.Fatalf("allocate ports: %v", err)
}
masterPort := miniPorts[0]
masterGrpcPort := masterPort + testutil.GrpcPortOffset
rc := &RustCluster{
testingTB: t,