also migrate jsonpb

weed/command/filer_meta_tail.go

@@ -2,8 +2,8 @@ package command
 
 import (
 	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/filer"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
-	"google.golang.org/protobuf/jsonpb"
 	"os"
 	"path/filepath"
 	"strings"
@@ -88,11 +88,8 @@ func runFilerMetaTail(cmd *Command, args []string) bool {
 		return false
 	}
 
-	jsonpbMarshaler := jsonpb.Marshaler{
-		EmitDefaults: false,
-	}
 	eachEntryFunc := func(resp *filer_pb.SubscribeMetadataResponse) error {
-		jsonpbMarshaler.Marshal(os.Stdout, resp)
+		filer.ProtoToText(os.Stdout, resp)
 		fmt.Fprintln(os.Stdout)
 		return nil
 	}

weed/filer/entry_codec.go

@@ -21,7 +21,7 @@ func (entry *Entry) DecodeAttributesAndChunks(blob []byte) error {
 
 	message := &filer_pb.Entry{}
 
-	if err := proto.UnmarshalMerge(blob, message); err != nil {
+	if err := proto.Unmarshal(blob, message); err != nil {
 		return fmt.Errorf("decoding value blob for %s: %v", entry.FullPath, err)
 	}
 

weed/filer/filer_conf.go

@@ -13,7 +13,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/util"
 	"github.com/viant/ptrie"
-	"google.golang.org/protobuf/jsonpb"
+	jsonpb "google.golang.org/protobuf/encoding/protojson"
 )
 
 const (
@@ -93,7 +93,7 @@ func (fc *FilerConf) loadFromChunks(filer *Filer, content []byte, chunks []*file
 func (fc *FilerConf) LoadFromBytes(data []byte) (err error) {
 	conf := &filer_pb.FilerConf{}
 
-	if err := jsonpb.Unmarshal(bytes.NewReader(data), conf); err != nil {
+	if err := jsonpb.Unmarshal(data, conf); err != nil {
 		return err
 	}
 
@@ -181,11 +181,5 @@ func (fc *FilerConf) ToProto() *filer_pb.FilerConf {
 }
 
 func (fc *FilerConf) ToText(writer io.Writer) error {
-
-	m := jsonpb.Marshaler{
-		EmitDefaults: false,
-		Indent:       "  ",
-	}
-
-	return m.Marshal(writer, fc.ToProto())
+	return ProtoToText(writer, fc.ToProto())
 }

weed/filer/s3iam_conf.go

@@ -1,17 +1,16 @@
 package filer
 
 import (
-	"bytes"
 	"fmt"
 	"io"
 
 	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
-	"google.golang.org/protobuf/jsonpb"
+	jsonpb "google.golang.org/protobuf/encoding/protojson"
 	"google.golang.org/protobuf/proto"
 )
 
 func ParseS3ConfigurationFromBytes[T proto.Message](content []byte, config T) error {
-	if err := jsonpb.Unmarshal(bytes.NewBuffer(content), config); err != nil {
+	if err := jsonpb.Unmarshal(content, config); err != nil {
 		return err
 	}
 	return nil
@@ -19,12 +18,22 @@ func ParseS3ConfigurationFromBytes[T proto.Message](content []byte, config T) er
 
 func ProtoToText(writer io.Writer, config proto.Message) error {
 
-	m := jsonpb.Marshaler{
-		EmitDefaults: false,
-		Indent:       "  ",
+	m := jsonpb.MarshalOptions{
+		EmitUnpopulated: true,
+		Indent:          "  ",
 	}
 
-	return m.Marshal(writer, config)
+	text, marshalErr := m.Marshal(config)
+	if marshalErr != nil {
+		return fmt.Errorf("marshal proto message: %v", marshalErr)
+	}
+
+	_, writeErr := writer.Write(text)
+	if writeErr != nil {
+		return fmt.Errorf("fail to write proto message: %v", writeErr)
+	}
+
+	return writeErr
 }
 
 // CheckDuplicateAccessKey returns an error message when s3cfg has duplicate access keys

weed/mq/broker/broker_segment_serde.go

@@ -8,7 +8,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
-	"google.golang.org/protobuf/jsonpb"
+	jsonpb "google.golang.org/protobuf/encoding/protojson"
 	"time"
 )
 
@@ -60,7 +60,7 @@ func (broker *MessageQueueBroker) readSegmentOnFiler(segment *mq.Segment) (info
 
 		// parse into filer conf object
 		info = &mq_pb.SegmentInfo{}
-		if err = jsonpb.Unmarshal(bytes.NewReader(data), info); err != nil {
+		if err = jsonpb.Unmarshal(data, info); err != nil {
 			return err
 		}
 		found = true

weed/notification/aws_sqs/aws_sqs_pub.go

@@ -70,7 +70,10 @@ func (k *AwsSqsPub) initialize(awsAccessKeyId, awsSecretAccessKey, region, queue
 
 func (k *AwsSqsPub) SendMessage(key string, message proto.Message) (err error) {
 
-	text := proto.MarshalTextString(message)
+	text, err := proto.Marshal(message)
+	if err != nil {
+		return fmt.Errorf("send message marshal %+v: %v", message, err)
+	}
 
 	_, err = k.svc.SendMessage(&sqs.SendMessageInput{
 		DelaySeconds: aws.Int64(10),
@@ -80,7 +83,7 @@ func (k *AwsSqsPub) SendMessage(key string, message proto.Message) (err error) {
 				StringValue: aws.String(key),
 			},
 		},
-		MessageBody: aws.String(text),
+		MessageBody: aws.String(string(text)),
 		QueueUrl:    &k.queueUrl,
 	})
 

weed/pb/proto_read_write_test.go

@@ -5,7 +5,7 @@ import (
 	"testing"
 
 	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
-	"google.golang.org/protobuf/jsonpb"
+	jsonpb "google.golang.org/protobuf/encoding/protojson"
 )
 
 func TestJsonpMarshalUnmarshal(t *testing.T) {

weed/replication/sub/notification_aws_sqs.go

@@ -98,7 +98,7 @@ func (k *AwsSqsInput) ReceiveMessage() (key string, message *filer_pb.EventNotif
 	key = *keyValue.StringValue
 	text := *result.Messages[0].Body
 	message = &filer_pb.EventNotification{}
-	err = proto.UnmarshalText(text, message)
+	err = proto.Unmarshal([]byte(text), message)
 
 	// delete the message
 	_, err = k.svc.DeleteMessage(&sqs.DeleteMessageInput{

weed/s3api/auth_credentials_test.go

@@ -5,7 +5,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"testing"
 
-	"google.golang.org/protobuf/jsonpb"
+	jsonpb "google.golang.org/protobuf/encoding/protojson"
 
 	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
 )

weed/shell/command_fs_meta_cat.go

@@ -2,8 +2,7 @@ package shell
 
 import (
 	"fmt"
-	"golang.org/x/exp/slices"
-	"google.golang.org/protobuf/jsonpb"
+	"github.com/seaweedfs/seaweedfs/weed/filer"
 	"google.golang.org/protobuf/proto"
 	"io"
 
@@ -50,22 +49,7 @@ func (c *commandFsMetaCat) Do(args []string, commandEnv *CommandEnv, writer io.W
 			return err
 		}
 
-		m := jsonpb.Marshaler{
-			EmitDefaults: true,
-			Indent:       "  ",
-		}
-		slices.SortFunc(respLookupEntry.Entry.Chunks, func(a, b *filer_pb.FileChunk) bool {
-			if a.Offset == b.Offset {
-				return a.Mtime < b.Mtime
-			}
-			return a.Offset < b.Offset
-		})
-		text, marshalErr := m.MarshalToString(respLookupEntry.Entry)
-		if marshalErr != nil {
-			return fmt.Errorf("marshal meta: %v", marshalErr)
-		}
-
-		fmt.Fprintf(writer, "%s\n", text)
+		filer.ProtoToText(writer, respLookupEntry.Entry)
 
 		bytes, _ := proto.Marshal(respLookupEntry.Entry)
 		gzippedBytes, _ := util.GzipData(bytes)

weed/shell/command_remote_configure.go

@@ -9,7 +9,6 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
 	"github.com/seaweedfs/seaweedfs/weed/remote_storage"
 	"github.com/seaweedfs/seaweedfs/weed/util"
-	"google.golang.org/protobuf/jsonpb"
 	"google.golang.org/protobuf/proto"
 	"io"
 	"regexp"
@@ -159,15 +158,8 @@ func (c *commandRemoteConfigure) listExistingRemoteStorages(commandEnv *CommandE
 		conf.TencentSecretKey = strings.Repeat("*", len(conf.TencentSecretKey))
 		conf.WasabiSecretKey = strings.Repeat("*", len(conf.WasabiSecretKey))
 
-		m := jsonpb.Marshaler{
-			EmitDefaults: false,
-			Indent:       "  ",
-		}
-
-		err := m.Marshal(writer, conf)
-		fmt.Fprintln(writer)
+		return filer.ProtoToText(writer, conf)
 
-		return err
 	})
 
 }

weed/shell/command_remote_mount.go

@@ -9,7 +9,6 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
 	"github.com/seaweedfs/seaweedfs/weed/remote_storage"
 	"github.com/seaweedfs/seaweedfs/weed/util"
-	"google.golang.org/protobuf/jsonpb"
 	"google.golang.org/protobuf/proto"
 	"io"
 	"os"
@@ -101,17 +100,7 @@ func listExistingRemoteStorageMounts(commandEnv *CommandEnv, writer io.Writer) (
 }
 
 func jsonPrintln(writer io.Writer, message proto.Message) error {
-	if message == nil {
-		return nil
-	}
-	m := jsonpb.Marshaler{
-		EmitDefaults: false,
-		Indent:       "  ",
-	}
-
-	err := m.Marshal(writer, message)
-	fmt.Fprintln(writer)
-	return err
+	return filer.ProtoToText(writer, message)
 }
 
 func syncMetadata(commandEnv *CommandEnv, writer io.Writer, dir string, nonEmpty bool, remoteConf *remote_pb.RemoteConf, remote *remote_pb.RemoteStorageLocation) error {

weed/storage/volume_info/volume_info.go

@@ -1,11 +1,10 @@
 package volume_info
 
 import (
-	"bytes"
 	"fmt"
 	"os"
 
-	"google.golang.org/protobuf/jsonpb"
+	jsonpb "google.golang.org/protobuf/encoding/protojson"
 
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
@@ -44,7 +43,7 @@ func MaybeLoadVolumeInfo(fileName string) (volumeInfo *volume_server_pb.VolumeIn
 	}
 
 	glog.V(1).Infof("maybeLoadVolumeInfo Unmarshal volume info %v", fileName)
-	if err = jsonpb.Unmarshal(bytes.NewReader(tierData), volumeInfo); err != nil {
+	if err = jsonpb.Unmarshal(tierData, volumeInfo); err != nil {
 		glog.Warningf("unmarshal error: %v", err)
 		err = fmt.Errorf("unmarshal error: %v", err)
 		return
@@ -65,17 +64,17 @@ func SaveVolumeInfo(fileName string, volumeInfo *volume_server_pb.VolumeInfo) er
 		return fmt.Errorf("%s not writable", fileName)
 	}
 
-	m := jsonpb.Marshaler{
-		EmitDefaults: true,
-		Indent:       "  ",
+	m := jsonpb.MarshalOptions{
+		EmitUnpopulated: true,
+		Indent:          "  ",
 	}
 
-	text, marshalErr := m.MarshalToString(volumeInfo)
+	text, marshalErr := m.Marshal(volumeInfo)
 	if marshalErr != nil {
 		return fmt.Errorf("marshal to %s: %v", fileName, marshalErr)
 	}
 
-	writeErr := util.WriteFile(fileName, []byte(text), 0755)
+	writeErr := util.WriteFile(fileName, text, 0755)
 	if writeErr != nil {
 		return fmt.Errorf("fail to write %s : %v", fileName, writeErr)
 	}

weed/util/skiplist/Makefile

@@ -3,4 +3,4 @@ all: gen
 .PHONY : gen
 
 gen:
-	protoc skiplist.proto --go_out=plugins=grpc:. --go_opt=paths=source_relative
+	protoc skiplist.proto --go_out=. --go-grpc_out=. --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative

weed/util/skiplist/skiplist.pb.go

@@ -1,13 +1,12 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
-// 	protoc-gen-go v1.25.0
-// 	protoc        v3.12.3
+// 	protoc-gen-go v1.26.0
+// 	protoc        v3.17.3
 // source: skiplist.proto
 
 package skiplist
 
 import (
-	proto "google.golang.org/protobuf/proto"
 	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
 	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
 	reflect "reflect"
@@ -21,10 +20,6 @@ const (
 	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
 )
 
-// This is a compile-time assertion that a sufficiently up-to-date version
-// of the legacy proto package is being used.
-const _ = proto.ProtoPackageIsVersion4
-
 type SkipListProto struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
@@ -325,7 +320,7 @@ var file_skiplist_proto_rawDesc = []byte{
 	0x22, 0x25, 0x0a, 0x0d, 0x4e, 0x61, 0x6d, 0x65, 0x42, 0x61, 0x74, 0x63, 0x68, 0x44, 0x61, 0x74,
 	0x61, 0x12, 0x14, 0x0a, 0x05, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c,
 	0x52, 0x05, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x42, 0x33, 0x5a, 0x31, 0x67, 0x69, 0x74, 0x68, 0x75,
-	0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x68, 0x72, 0x69, 0x73, 0x6c, 0x75, 0x73, 0x66, 0x2f,
+	0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f,
 	0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x75,
 	0x74, 0x69, 0x6c, 0x2f, 0x73, 0x6b, 0x69, 0x70, 0x6c, 0x69, 0x73, 0x74, 0x62, 0x06, 0x70, 0x72,
 	0x6f, 0x74, 0x6f, 0x33,