From b881e11a1620619d13647bb53e156cf8e5c3fb28 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 16 Jan 2019 20:44:38 -0800 Subject: [PATCH 1/8] 1.21 --- weed/util/constants.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/util/constants.go b/weed/util/constants.go index dd7314a32..12d09a477 100644 --- a/weed/util/constants.go +++ b/weed/util/constants.go @@ -1,5 +1,5 @@ package util const ( - VERSION = "1.20" + VERSION = "1.21" ) From 1d103e3ed54d0e5cf17dbc489ba412ce96c88a20 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 17 Jan 2019 23:38:33 -0800 Subject: [PATCH 2/8] timeout http connections possible fix for https://github.com/chrislusf/seaweedfs/issues/825 --- weed/server/raft_server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/server/raft_server.go b/weed/server/raft_server.go index 01ca41aac..2cc8252b8 100644 --- a/weed/server/raft_server.go +++ b/weed/server/raft_server.go @@ -42,7 +42,7 @@ func NewRaftServer(r *mux.Router, peers []string, httpAddr string, dataDir strin raft.RegisterCommand(&topology.MaxVolumeIdCommand{}) var err error - transporter := raft.NewHTTPTransporter("/cluster", 0) + transporter := raft.NewHTTPTransporter("/cluster", time.Second) transporter.Transport.MaxIdleConnsPerHost = 1024 glog.V(0).Infof("Starting RaftServer with %v", httpAddr) From e84dcb0bfe2a241f76f66f18b5b85741ee6a805e Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 18 Jan 2019 00:24:40 -0800 Subject: [PATCH 3/8] fix timeout issue, but can not reproduce with local built binaries fix https://github.com/chrislusf/seaweedfs/issues/841 --- weed/util/grpc_client_server.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/weed/util/grpc_client_server.go b/weed/util/grpc_client_server.go index d487e8bb4..f80d71f29 100644 --- a/weed/util/grpc_client_server.go +++ b/weed/util/grpc_client_server.go @@ -25,8 +25,8 @@ func NewGrpcServer() *grpc.Server { } func GrpcDial(address string, opts ...grpc.DialOption) (*grpc.ClientConn, error) { - opts = append(opts, grpc.WithBlock()) - opts = append(opts, grpc.WithTimeout(time.Duration(5*time.Second))) + // opts = append(opts, grpc.WithBlock()) + // opts = append(opts, grpc.WithTimeout(time.Duration(5*time.Second))) opts = append(opts, grpc.WithInsecure()) opts = append(opts, grpc.WithKeepaliveParams(keepalive.ClientParameters{ Time: 30 * time.Second, // client ping server if no activity for this long From dfae0f4e9df1c1b041d6c4f1f13d885bd86f0e20 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 18 Jan 2019 00:25:20 -0800 Subject: [PATCH 4/8] 1.22 --- weed/util/constants.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/util/constants.go b/weed/util/constants.go index 12d09a477..cc5a85c0a 100644 --- a/weed/util/constants.go +++ b/weed/util/constants.go @@ -1,5 +1,5 @@ package util const ( - VERSION = "1.21" + VERSION = "1.22" ) From 67e2ea72be5b0f5bb1f69967165a279cc3067938 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 18 Jan 2019 14:14:47 -0800 Subject: [PATCH 5/8] master add separate grpc port due to https://github.com/soheilhy/cmux/issues/64 fix https://github.com/chrislusf/seaweedfs/issues/820 fix https://github.com/chrislusf/seaweedfs/issues/840 fix https://github.com/chrislusf/seaweedfs/issues/841 --- weed/command/master.go | 39 ++++++++++++--------- weed/command/server.go | 39 ++++++++++++--------- weed/operation/grpc_client.go | 7 +++- weed/operation/upload_content.go | 8 ++--- weed/server/volume_grpc_client_to_master.go | 11 ++++-- weed/util/grpc_client_server.go | 21 +++++++++++ weed/wdclient/masterclient.go | 16 +++++++-- 7 files changed, 95 insertions(+), 46 deletions(-) diff --git a/weed/command/master.go b/weed/command/master.go index 29c8a6833..11f634ed4 100644 --- a/weed/command/master.go +++ b/weed/command/master.go @@ -9,11 +9,10 @@ import ( "time" "github.com/chrislusf/seaweedfs/weed/glog" - "github.com/chrislusf/seaweedfs/weed/pb/master_pb" "github.com/chrislusf/seaweedfs/weed/server" "github.com/chrislusf/seaweedfs/weed/util" + "github.com/chrislusf/seaweedfs/weed/pb/master_pb" "github.com/gorilla/mux" - "github.com/soheilhy/cmux" "google.golang.org/grpc/reflection" ) @@ -32,6 +31,7 @@ var cmdMaster = &Command{ var ( mport = cmdMaster.Flag.Int("port", 9333, "http listen port") + mGrpcPort = cmdMaster.Flag.Int("port.grpc", 0, "grpc server listen port, default to http port + 10000") masterIp = cmdMaster.Flag.String("ip", "localhost", "master | address") masterBindIp = cmdMaster.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to") metaFolder = cmdMaster.Flag.String("mdir", os.TempDir(), "data directory to store meta data") @@ -79,7 +79,7 @@ func runMaster(cmd *Command, args []string) bool { glog.V(0).Infoln("Start Seaweed Master", util.VERSION, "at", listeningAddress) - listener, e := util.NewListener(listeningAddress, 0) + masterListener, e := util.NewListener(listeningAddress, 0) if e != nil { glog.Fatalf("Master startup error: %v", e) } @@ -91,23 +91,28 @@ func runMaster(cmd *Command, args []string) bool { ms.SetRaftServer(raftServer) }() - // start grpc and http server - m := cmux.New(listener) - - grpcL := m.Match(cmux.HTTP2HeaderField("content-type", "application/grpc")) - httpL := m.Match(cmux.Any()) + go func() { + // starting grpc server + grpcPort := *mGrpcPort + if grpcPort == 0 { + grpcPort = *mport + 10000 + } + grpcL, err := util.NewListener(*masterBindIp + ":"+strconv.Itoa(grpcPort), 0) + if err != nil { + glog.Fatalf("master failed to listen on grpc port %d: %v", grpcPort, err) + } + // Create your protocol servers. + grpcS := util.NewGrpcServer() + master_pb.RegisterSeaweedServer(grpcS, ms) + reflection.Register(grpcS) - // Create your protocol servers. - grpcS := util.NewGrpcServer() - master_pb.RegisterSeaweedServer(grpcS, ms) - reflection.Register(grpcS) + glog.V(0).Infof("Start Seaweed Master %s grpc server at %s:%d", util.VERSION, *masterBindIp, grpcPort) + grpcS.Serve(grpcL) + }() + // start http server httpS := &http.Server{Handler: r} - - go grpcS.Serve(grpcL) - go httpS.Serve(httpL) - - if err := m.Serve(); err != nil { + if err := httpS.Serve(masterListener); err != nil { glog.Fatalf("master server failed to serve: %v", err) } diff --git a/weed/command/server.go b/weed/command/server.go index 8f760942c..ba5305a97 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -15,7 +15,6 @@ import ( "github.com/chrislusf/seaweedfs/weed/server" "github.com/chrislusf/seaweedfs/weed/util" "github.com/gorilla/mux" - "github.com/soheilhy/cmux" "google.golang.org/grpc/reflection" ) @@ -62,6 +61,7 @@ var ( serverSecureKey = cmdServer.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)") serverGarbageThreshold = cmdServer.Flag.Float64("garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces") masterPort = cmdServer.Flag.Int("master.port", 9333, "master server http listen port") + masterGrpcPort = cmdServer.Flag.Int("master.port.grpc", 0, "master grpc server listen port, default to http port + 10000") masterMetaFolder = cmdServer.Flag.String("master.dir", "", "data directory to store meta data, default to same as -dir specified") masterVolumeSizeLimitMB = cmdServer.Flag.Uint("master.volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.") masterVolumePreallocate = cmdServer.Flag.Bool("master.volumePreallocate", false, "Preallocate disk space for volumes.") @@ -179,6 +179,25 @@ func runServer(cmd *Command, args []string) bool { glog.Fatalf("Master startup error: %v", e) } + go func() { + // starting grpc server + grpcPort := *masterGrpcPort + if grpcPort == 0 { + grpcPort = *masterPort + 10000 + } + grpcL, err := util.NewListener(*serverIp+":"+strconv.Itoa(grpcPort), 0) + if err != nil { + glog.Fatalf("master failed to listen on grpc port %d: %v", grpcPort, err) + } + // Create your protocol servers. + grpcS := util.NewGrpcServer() + master_pb.RegisterSeaweedServer(grpcS, ms) + reflection.Register(grpcS) + + glog.V(0).Infof("Start Seaweed Master %s grpc server at %s:%d", util.VERSION, *serverIp, grpcPort) + grpcS.Serve(grpcL) + }() + go func() { raftWaitForMaster.Wait() time.Sleep(100 * time.Millisecond) @@ -190,23 +209,9 @@ func runServer(cmd *Command, args []string) bool { raftWaitForMaster.Done() - // start grpc and http server - m := cmux.New(masterListener) - - grpcL := m.Match(cmux.HTTP2HeaderField("content-type", "application/grpc")) - httpL := m.Match(cmux.Any()) - - // Create your protocol servers. - grpcS := util.NewGrpcServer() - master_pb.RegisterSeaweedServer(grpcS, ms) - reflection.Register(grpcS) - + // start http server httpS := &http.Server{Handler: r} - - go grpcS.Serve(grpcL) - go httpS.Serve(httpL) - - if err := m.Serve(); err != nil { + if err := httpS.Serve(masterListener); err != nil { glog.Fatalf("master server failed to serve: %v", err) } diff --git a/weed/operation/grpc_client.go b/weed/operation/grpc_client.go index 300f78b58..d0931a8d3 100644 --- a/weed/operation/grpc_client.go +++ b/weed/operation/grpc_client.go @@ -44,9 +44,14 @@ func toVolumeServerGrpcAddress(volumeServer string) (grpcAddress string, err err func withMasterServerClient(masterServer string, fn func(masterClient master_pb.SeaweedClient) error) error { + masterGrpcAddress, parseErr := util.ParseServerToGrpcAddress(masterServer, 0) + if parseErr != nil { + return fmt.Errorf("failed to parse master grpc %v", masterServer) + } + return util.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error { client := master_pb.NewSeaweedClient(grpcConnection) return fn(client) - }, masterServer) + }, masterGrpcAddress) } diff --git a/weed/operation/upload_content.go b/weed/operation/upload_content.go index 8cf31e382..030bf5889 100644 --- a/weed/operation/upload_content.go +++ b/weed/operation/upload_content.go @@ -13,7 +13,6 @@ import ( "net/textproto" "path/filepath" "strings" - "time" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/security" @@ -31,10 +30,9 @@ var ( ) func init() { - client = &http.Client{ - Transport: &http.Transport{MaxIdleConnsPerHost: 1024}, - Timeout: 5 * time.Second, - } + client = &http.Client{Transport: &http.Transport{ + MaxIdleConnsPerHost: 1024, + }} } var fileNameEscaper = strings.NewReplacer("\\", "\\\\", "\"", "\\\"") diff --git a/weed/server/volume_grpc_client_to_master.go b/weed/server/volume_grpc_client_to_master.go index fcbcfd4d4..bd3ffd7b3 100644 --- a/weed/server/volume_grpc_client_to_master.go +++ b/weed/server/volume_grpc_client_to_master.go @@ -27,7 +27,12 @@ func (vs *VolumeServer) heartbeat() { if newLeader != "" { master = newLeader } - newLeader, err = vs.doHeartbeat(master, time.Duration(vs.pulseSeconds)*time.Second) + masterGrpcAddress, parseErr := util.ParseServerToGrpcAddress(master, 0) + if parseErr != nil { + glog.V(0).Infof("failed to parse master grpc %v", masterGrpcAddress) + continue + } + newLeader, err = vs.doHeartbeat(master, masterGrpcAddress, time.Duration(vs.pulseSeconds)*time.Second) if err != nil { glog.V(0).Infof("heartbeat error: %v", err) time.Sleep(time.Duration(vs.pulseSeconds) * time.Second) @@ -36,9 +41,9 @@ func (vs *VolumeServer) heartbeat() { } } -func (vs *VolumeServer) doHeartbeat(masterNode string, sleepInterval time.Duration) (newLeader string, err error) { +func (vs *VolumeServer) doHeartbeat(masterNode, masterGrpcAddress string, sleepInterval time.Duration) (newLeader string, err error) { - grpcConection, err := util.GrpcDial(masterNode) + grpcConection, err := util.GrpcDial(masterGrpcAddress) if err != nil { return "", fmt.Errorf("fail to dial %s : %v", masterNode, err) } diff --git a/weed/util/grpc_client_server.go b/weed/util/grpc_client_server.go index f80d71f29..d029d21ae 100644 --- a/weed/util/grpc_client_server.go +++ b/weed/util/grpc_client_server.go @@ -2,6 +2,8 @@ package util import ( "fmt" + "strconv" + "strings" "sync" "time" @@ -64,3 +66,22 @@ func WithCachedGrpcClient(fn func(*grpc.ClientConn) error, address string, opts return err } + +func ParseServerToGrpcAddress(server string, optionalGrpcPort int) (serverGrpcAddress string, err error) { + hostnameAndPort := strings.Split(server, ":") + if len(hostnameAndPort) != 2 { + return "", fmt.Errorf("The server should have hostname:port format: %v", hostnameAndPort) + } + + filerPort, parseErr := strconv.ParseUint(hostnameAndPort[1], 10, 64) + if parseErr != nil { + return "", fmt.Errorf("The server port parse error: %v", parseErr) + } + + filerGrpcPort := int(filerPort) + 10000 + if optionalGrpcPort != 0 { + filerGrpcPort = optionalGrpcPort + } + + return fmt.Sprintf("%s:%d", hostnameAndPort[0], filerGrpcPort), nil +} diff --git a/weed/wdclient/masterclient.go b/weed/wdclient/masterclient.go index df0adbd18..f58c28504 100644 --- a/weed/wdclient/masterclient.go +++ b/weed/wdclient/masterclient.go @@ -49,8 +49,9 @@ func (mc *MasterClient) KeepConnectedToMaster() { func (mc *MasterClient) tryAllMasters() { for _, master := range mc.masters { - glog.V(0).Infof("Connecting to %v", master) - withMasterClient(master, func(client master_pb.SeaweedClient) error { + glog.V(0).Infof("Connecting to master %v", master) + gprcErr := withMasterClient(master, func(client master_pb.SeaweedClient) error { + stream, err := client.KeepConnected(context.Background()) if err != nil { glog.V(0).Infof("failed to keep connected to %s: %v", master, err) @@ -88,13 +89,22 @@ func (mc *MasterClient) tryAllMasters() { }) + if gprcErr != nil { + glog.V(0).Infof("%s failed to connect with master %v: %v", mc.name, master, gprcErr) + } + mc.currentMaster = "" } } func withMasterClient(master string, fn func(client master_pb.SeaweedClient) error) error { - grpcConnection, err := util.GrpcDial(master) + masterGrpcAddress, parseErr := util.ParseServerToGrpcAddress(master, 0) + if parseErr != nil { + return fmt.Errorf("failed to parse master grpc %v", master) + } + + grpcConnection, err := util.GrpcDial(masterGrpcAddress) if err != nil { return fmt.Errorf("fail to dial %s: %v", master, err) } From a6bffc58f9517ccb5116b66b1cea77ea0f2abca1 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 18 Jan 2019 14:15:10 -0800 Subject: [PATCH 6/8] go fmt --- weed/command/master.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/weed/command/master.go b/weed/command/master.go index 11f634ed4..bd2267b9e 100644 --- a/weed/command/master.go +++ b/weed/command/master.go @@ -9,9 +9,9 @@ import ( "time" "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb/master_pb" "github.com/chrislusf/seaweedfs/weed/server" "github.com/chrislusf/seaweedfs/weed/util" - "github.com/chrislusf/seaweedfs/weed/pb/master_pb" "github.com/gorilla/mux" "google.golang.org/grpc/reflection" ) @@ -97,7 +97,7 @@ func runMaster(cmd *Command, args []string) bool { if grpcPort == 0 { grpcPort = *mport + 10000 } - grpcL, err := util.NewListener(*masterBindIp + ":"+strconv.Itoa(grpcPort), 0) + grpcL, err := util.NewListener(*masterBindIp+":"+strconv.Itoa(grpcPort), 0) if err != nil { glog.Fatalf("master failed to listen on grpc port %d: %v", grpcPort, err) } From 8ec57e18c736aac1bd5800223c53ef32d547e993 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 18 Jan 2019 14:27:34 -0800 Subject: [PATCH 7/8] 1.23 --- weed/util/constants.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/util/constants.go b/weed/util/constants.go index cc5a85c0a..9ddf07261 100644 --- a/weed/util/constants.go +++ b/weed/util/constants.go @@ -1,5 +1,5 @@ package util const ( - VERSION = "1.22" + VERSION = "1.23" ) From 2b3cef7780a5e91d2072a33411926f9b30c88ee2 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 18 Jan 2019 14:32:02 -0800 Subject: [PATCH 8/8] fix docker to work with master grpc port --- docker/Dockerfile | 4 +++- docker/Dockerfile.go_build | 4 +++- docker/dev-compose.yml | 1 + docker/seaweedfs-compose.yml | 1 + 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 6b6a9d47a..c7a343111 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -15,7 +15,9 @@ EXPOSE 8080 EXPOSE 18888 # filer server http port EXPOSE 8888 -# master server shared gprc+http port +# master server shared gprc port +EXPOSE 19333 +# master server shared http port EXPOSE 9333 # s3 server http port EXPOSE 8333 diff --git a/docker/Dockerfile.go_build b/docker/Dockerfile.go_build index 8a5cb5b1f..d0a214476 100644 --- a/docker/Dockerfile.go_build +++ b/docker/Dockerfile.go_build @@ -9,7 +9,9 @@ EXPOSE 8080 EXPOSE 18888 # filer server http port EXPOSE 8888 -# master server shared gprc+http port +# master server shared gprc port +EXPOSE 19333 +# master server shared http port EXPOSE 9333 # s3 server http port EXPOSE 8333 diff --git a/docker/dev-compose.yml b/docker/dev-compose.yml index e107782e8..0306b3cb0 100644 --- a/docker/dev-compose.yml +++ b/docker/dev-compose.yml @@ -7,6 +7,7 @@ services: dockerfile: Dockerfile.go_build ports: - 9333:9333 + - 19333:19333 command: "master" volume: build: diff --git a/docker/seaweedfs-compose.yml b/docker/seaweedfs-compose.yml index b5105adea..05ed0e69e 100644 --- a/docker/seaweedfs-compose.yml +++ b/docker/seaweedfs-compose.yml @@ -5,6 +5,7 @@ services: image: chrislusf/seaweedfs # use a remote image ports: - 9333:9333 + - 19333:19333 command: "master" volume: image: chrislusf/seaweedfs # use a remote image