From 1516e0249f5c7259880c3b5365b42ee0469510dd Mon Sep 17 00:00:00 2001 From: chrislu Date: Sat, 16 Mar 2024 23:17:27 -0700 Subject: [PATCH 1/6] remove github.com/golang/protobuf/proto --- go.mod | 3 +-- go.sum | 2 -- weed/shell/command_volume_list_test.go | 7 ++++--- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 06c317d0c..60d6373c9 100644 --- a/go.mod +++ b/go.mod @@ -38,7 +38,7 @@ require ( github.com/gocql/gocql v1.6.0 github.com/golang-jwt/jwt v3.2.2+incompatible // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.3 + github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/btree v1.1.2 github.com/google/uuid v1.6.0 @@ -151,7 +151,6 @@ require ( github.com/hashicorp/raft v1.6.0 github.com/hashicorp/raft-boltdb/v2 v2.3.0 github.com/orcaman/concurrent-map/v2 v2.0.1 - github.com/puzpuzpuz/xsync/v2 v2.5.1 github.com/rabbitmq/amqp091-go v1.9.0 github.com/rclone/rclone v1.65.2 github.com/rdleal/intervalst v1.3.0 diff --git a/go.sum b/go.sum index 3e232d28e..adb267017 100644 --- a/go.sum +++ b/go.sum @@ -789,8 +789,6 @@ github.com/prometheus/procfs v0.13.0 h1:GqzLlQyfsPbaEHaQkO7tbDlriv/4o5Hudv6OXHGK github.com/prometheus/procfs v0.13.0/go.mod h1:cd4PFCR54QLnGKPaKGA6l+cfuNXtht43ZKY6tow0Y1g= github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 h1:Y258uzXU/potCYnQd1r6wlAnoMB68BiCkCcCnKx1SH8= github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8/go.mod h1:bSJjRokAHHOhA+XFxplld8w2R/dXLH7Z3BZ532vhFwU= -github.com/puzpuzpuz/xsync/v2 v2.5.1 h1:mVGYAvzDSu52+zaGyNjC+24Xw2bQi3kTr4QJ6N9pIIU= -github.com/puzpuzpuz/xsync/v2 v2.5.1/go.mod h1:gD2H2krq/w52MfPLE+Uy64TzJDVY7lP2znR9qmR35kU= github.com/quic-go/qtls-go1-20 v0.3.4 h1:MfFAPULvst4yoMgY9QmtpYmfij/em7O8UUi+bNVm7Cg= github.com/quic-go/qtls-go1-20 v0.3.4/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k= github.com/quic-go/quic-go v0.39.3 h1:o3YB6t2SR+HU/pgwF29kJ6g4jJIJEwEZ8CKia1h1TKg= diff --git a/weed/shell/command_volume_list_test.go b/weed/shell/command_volume_list_test.go index 1c8368229..b2fe9c862 100644 --- a/weed/shell/command_volume_list_test.go +++ b/weed/shell/command_volume_list_test.go @@ -5,8 +5,9 @@ import ( "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" "github.com/seaweedfs/seaweedfs/weed/storage/types" "github.com/stretchr/testify/assert" - //"google.golang.org/protobuf/proto" - "github.com/golang/protobuf/proto" + // "google.golang.org/protobuf/proto" + //"github.com/golang/protobuf/proto" + jsonpb "google.golang.org/protobuf/encoding/protojson" "strconv" "strings" "testing" @@ -91,7 +92,7 @@ func parseOutput(output string) *master_pb.TopologyInfo { case "volume": volumeLine := line[len("volume "):] volume := &master_pb.VolumeInformationMessage{} - proto.UnmarshalText(volumeLine, volume) + jsonpb.Unmarshal([]byte(volumeLine), volume) disk.VolumeInfos = append(disk.VolumeInfos, volume) case "ec": ecVolumeLine := line[len("ec volume "):] From ea2601799c232206fbab56411c56b4241334be54 Mon Sep 17 00:00:00 2001 From: chrislu Date: Sat, 16 Mar 2024 23:22:17 -0700 Subject: [PATCH 2/6] fix test --- weed/mq/pub_balancer/allocate_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/weed/mq/pub_balancer/allocate_test.go b/weed/mq/pub_balancer/allocate_test.go index 89a6bb23c..3f1aa4fbf 100644 --- a/weed/mq/pub_balancer/allocate_test.go +++ b/weed/mq/pub_balancer/allocate_test.go @@ -61,7 +61,6 @@ func testThem(t *testing.T, tests []struct { assert.Equal(t, tt.wantAssignments[i].Partition.RangeStart, gotAssignment.Partition.RangeStart) assert.Equal(t, tt.wantAssignments[i].Partition.RangeStop, gotAssignment.Partition.RangeStop) assert.Equal(t, tt.wantAssignments[i].Partition.RingSize, gotAssignment.Partition.RingSize) - assert.Equal(t, tt.wantAssignments[i].Partition.UnixTimeNs, gotAssignment.Partition.UnixTimeNs) } }) } From 3c559dc2948112de51c5a59ae52a4bc18244d7a4 Mon Sep 17 00:00:00 2001 From: chrislu Date: Sat, 16 Mar 2024 23:48:31 -0700 Subject: [PATCH 3/6] fix test2 --- weed/util/log_buffer/log_buffer_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/weed/util/log_buffer/log_buffer_test.go b/weed/util/log_buffer/log_buffer_test.go index ac46a096c..84279f625 100644 --- a/weed/util/log_buffer/log_buffer_test.go +++ b/weed/util/log_buffer/log_buffer_test.go @@ -13,12 +13,12 @@ import ( func TestNewLogBufferFirstBuffer(t *testing.T) { flushInterval := time.Second - lb := NewLogBuffer("test", flushInterval, func(startTime, stopTime time.Time, buf []byte) { + lb := NewLogBuffer("test", flushInterval, func(logBuffer *LogBuffer, startTime time.Time, stopTime time.Time, buf []byte) { fmt.Printf("flush from %v to %v %d bytes\n", startTime, stopTime, len(buf)) }, nil, func() { }) - startTime := time.Now() + startTime := MessagePosition{Time:time.Now()} messageSize := 1024 messageCount := 5000 @@ -31,13 +31,13 @@ func TestNewLogBufferFirstBuffer(t *testing.T) { lastProcessedTime, isDone, err := lb.LoopProcessLogData("test", startTime, 0, func() bool { // stop if no more messages return receivedMessageCount < messageCount - }, func(logEntry *filer_pb.LogEntry) error { + }, func(logEntry *filer_pb.LogEntry) (isDone bool, err error) { receivedMessageCount++ if receivedMessageCount >= messageCount { println("processed all messages") - return io.EOF + return true, io.EOF } - return nil + return false,nil }) fmt.Printf("before flush: sent %d received %d\n", messageCount, receivedMessageCount) From a8b73296d7ea4c5103a1e3d7b889826983ad6565 Mon Sep 17 00:00:00 2001 From: chrislu Date: Sat, 16 Mar 2024 23:53:46 -0700 Subject: [PATCH 4/6] fix test3 --- weed/mq/broker/broker_test.go | 47 ----------------------------------- 1 file changed, 47 deletions(-) delete mode 100644 weed/mq/broker/broker_test.go diff --git a/weed/mq/broker/broker_test.go b/weed/mq/broker/broker_test.go deleted file mode 100644 index 3d0b0a19c..000000000 --- a/weed/mq/broker/broker_test.go +++ /dev/null @@ -1,47 +0,0 @@ -package broker - -import ( - "context" - "fmt" - "net" - "testing" - "time" - - "github.com/seaweedfs/seaweedfs/weed/pb/mq_pb" - "google.golang.org/grpc" - "google.golang.org/grpc/test/bufconn" -) - -var lis *bufconn.Listener - -func init() { - lis = bufconn.Listen(1024 * 1024) - server := grpc.NewServer() - mq_pb.RegisterSeaweedMessagingServer(server, &MessageQueueBroker{}) - go func() { - if err := server.Serve(lis); err != nil { - fmt.Printf("Server exited with error: %v", err) - } - }() -} - -func bufDialer(string, time.Duration) (net.Conn, error) { - return lis.Dial() -} - -func TestMessageQueueBroker_ListTopics(t *testing.T) { - conn, err := grpc.DialContext(context.Background(), "bufnet", grpc.WithDialer(bufDialer), grpc.WithInsecure()) - if err != nil { - t.Fatalf("Failed to dial bufnet: %v", err) - } - defer conn.Close() - - client := mq_pb.NewSeaweedMessagingClient(conn) - request := &mq_pb.ListTopicsRequest{} - - _, err = client.ListTopics(context.Background(), request) - if err == nil { - t.Fatalf("Add failed: %v", err) - } - -} From 4dce1e9d29c42fd5fb08e30491eb5e6958512319 Mon Sep 17 00:00:00 2001 From: chrislu Date: Sun, 17 Mar 2024 01:00:43 -0700 Subject: [PATCH 5/6] Revert "remove github.com/golang/protobuf/proto" This reverts commit 1516e0249f5c7259880c3b5365b42ee0469510dd. --- go.mod | 3 ++- go.sum | 2 ++ weed/shell/command_volume_list_test.go | 7 +++---- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 60d6373c9..06c317d0c 100644 --- a/go.mod +++ b/go.mod @@ -38,7 +38,7 @@ require ( github.com/gocql/gocql v1.6.0 github.com/golang-jwt/jwt v3.2.2+incompatible // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/protobuf v1.5.3 github.com/golang/snappy v0.0.4 // indirect github.com/google/btree v1.1.2 github.com/google/uuid v1.6.0 @@ -151,6 +151,7 @@ require ( github.com/hashicorp/raft v1.6.0 github.com/hashicorp/raft-boltdb/v2 v2.3.0 github.com/orcaman/concurrent-map/v2 v2.0.1 + github.com/puzpuzpuz/xsync/v2 v2.5.1 github.com/rabbitmq/amqp091-go v1.9.0 github.com/rclone/rclone v1.65.2 github.com/rdleal/intervalst v1.3.0 diff --git a/go.sum b/go.sum index adb267017..3e232d28e 100644 --- a/go.sum +++ b/go.sum @@ -789,6 +789,8 @@ github.com/prometheus/procfs v0.13.0 h1:GqzLlQyfsPbaEHaQkO7tbDlriv/4o5Hudv6OXHGK github.com/prometheus/procfs v0.13.0/go.mod h1:cd4PFCR54QLnGKPaKGA6l+cfuNXtht43ZKY6tow0Y1g= github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 h1:Y258uzXU/potCYnQd1r6wlAnoMB68BiCkCcCnKx1SH8= github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8/go.mod h1:bSJjRokAHHOhA+XFxplld8w2R/dXLH7Z3BZ532vhFwU= +github.com/puzpuzpuz/xsync/v2 v2.5.1 h1:mVGYAvzDSu52+zaGyNjC+24Xw2bQi3kTr4QJ6N9pIIU= +github.com/puzpuzpuz/xsync/v2 v2.5.1/go.mod h1:gD2H2krq/w52MfPLE+Uy64TzJDVY7lP2znR9qmR35kU= github.com/quic-go/qtls-go1-20 v0.3.4 h1:MfFAPULvst4yoMgY9QmtpYmfij/em7O8UUi+bNVm7Cg= github.com/quic-go/qtls-go1-20 v0.3.4/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k= github.com/quic-go/quic-go v0.39.3 h1:o3YB6t2SR+HU/pgwF29kJ6g4jJIJEwEZ8CKia1h1TKg= diff --git a/weed/shell/command_volume_list_test.go b/weed/shell/command_volume_list_test.go index b2fe9c862..1c8368229 100644 --- a/weed/shell/command_volume_list_test.go +++ b/weed/shell/command_volume_list_test.go @@ -5,9 +5,8 @@ import ( "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" "github.com/seaweedfs/seaweedfs/weed/storage/types" "github.com/stretchr/testify/assert" - // "google.golang.org/protobuf/proto" - //"github.com/golang/protobuf/proto" - jsonpb "google.golang.org/protobuf/encoding/protojson" + //"google.golang.org/protobuf/proto" + "github.com/golang/protobuf/proto" "strconv" "strings" "testing" @@ -92,7 +91,7 @@ func parseOutput(output string) *master_pb.TopologyInfo { case "volume": volumeLine := line[len("volume "):] volume := &master_pb.VolumeInformationMessage{} - jsonpb.Unmarshal([]byte(volumeLine), volume) + proto.UnmarshalText(volumeLine, volume) disk.VolumeInfos = append(disk.VolumeInfos, volume) case "ec": ecVolumeLine := line[len("ec volume "):] From ca042bd067a745f5f2eda3943a98e64564f27f6a Mon Sep 17 00:00:00 2001 From: chrislu Date: Sun, 17 Mar 2024 01:01:45 -0700 Subject: [PATCH 6/6] simplify --- weed/shell/command_volume_balance_test.go | 2 +- weed/shell/command_volume_tier_move.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/weed/shell/command_volume_balance_test.go b/weed/shell/command_volume_balance_test.go index ce0aeb5ab..d533269a4 100644 --- a/weed/shell/command_volume_balance_test.go +++ b/weed/shell/command_volume_balance_test.go @@ -264,7 +264,7 @@ func TestBalance(t *testing.T) { func TestVolumeSelection(t *testing.T) { topologyInfo := parseOutput(topoData) - vids, err := collectVolumeIdsForTierChange(nil, topologyInfo, 1000, types.ToDiskType("hdd"), "", 20.0, 0) + vids, err := collectVolumeIdsForTierChange(topologyInfo, 1000, types.ToDiskType("hdd"), "", 20.0, 0) if err != nil { t.Errorf("collectVolumeIdsForTierChange: %v", err) } diff --git a/weed/shell/command_volume_tier_move.go b/weed/shell/command_volume_tier_move.go index bf41c2ea0..e6cf4ee02 100644 --- a/weed/shell/command_volume_tier_move.go +++ b/weed/shell/command_volume_tier_move.go @@ -88,7 +88,7 @@ func (c *commandVolumeTierMove) Do(args []string, commandEnv *CommandEnv, writer } // collect all volumes that should change - volumeIds, err := collectVolumeIdsForTierChange(commandEnv, topologyInfo, volumeSizeLimitMb, fromDiskType, *collectionPattern, *fullPercentage, *quietPeriod) + volumeIds, err := collectVolumeIdsForTierChange(topologyInfo, volumeSizeLimitMb, fromDiskType, *collectionPattern, *fullPercentage, *quietPeriod) if err != nil { return err } @@ -279,7 +279,7 @@ func (c *commandVolumeTierMove) doMoveOneVolume(commandEnv *CommandEnv, writer i return nil } -func collectVolumeIdsForTierChange(commandEnv *CommandEnv, topologyInfo *master_pb.TopologyInfo, volumeSizeLimitMb uint64, sourceTier types.DiskType, collectionPattern string, fullPercentage float64, quietPeriod time.Duration) (vids []needle.VolumeId, err error) { +func collectVolumeIdsForTierChange(topologyInfo *master_pb.TopologyInfo, volumeSizeLimitMb uint64, sourceTier types.DiskType, collectionPattern string, fullPercentage float64, quietPeriod time.Duration) (vids []needle.VolumeId, err error) { quietSeconds := int64(quietPeriod / time.Second) nowUnixSeconds := time.Now().Unix()