From 6608cb5f43d7b26c0f67e18907313dbc52a1eadf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E8=BF=9C=E5=86=9B?= Date: Fri, 10 Jul 2020 10:08:36 +0800 Subject: [PATCH 1/2] Revert "remove fixJpgOrientation" This reverts commit de5ca9b2 --- .../templates/volume-statefulset.yaml | 3 + k8s/seaweedfs/values.yaml | 3 + weed/command/server.go | 7 +- weed/command/volume.go | 4 +- weed/images/orientation.go | 182 ++++++++++++++++++ weed/images/orientation_test.go | 20 ++ weed/images/preprocess.go | 29 +++ weed/pb/shared_values.go | 2 +- weed/server/volume_server.go | 3 + weed/server/volume_server_handlers_write.go | 2 +- weed/storage/needle/needle.go | 10 +- weed/wdclient/masterclient.go | 18 +- 12 files changed, 267 insertions(+), 16 deletions(-) create mode 100644 weed/images/orientation.go create mode 100644 weed/images/orientation_test.go create mode 100644 weed/images/preprocess.go diff --git a/k8s/seaweedfs/templates/volume-statefulset.yaml b/k8s/seaweedfs/templates/volume-statefulset.yaml index 6180bc7df..9c6ddcd9f 100644 --- a/k8s/seaweedfs/templates/volume-statefulset.yaml +++ b/k8s/seaweedfs/templates/volume-statefulset.yaml @@ -88,6 +88,9 @@ spec: {{- if .Values.volume.whiteList }} -whiteList={{ .Values.volume.whiteList }} \ {{- end }} + {{- if .Values.volume.imagesFixOrientation }} + -images.fix.orientation \ + {{- end }} -ip=${POD_NAME}.${SEAWEEDFS_FULLNAME}-volume \ -compactionMBps={{ .Values.volume.compactionMBps }} \ -mserver={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }} diff --git a/k8s/seaweedfs/values.yaml b/k8s/seaweedfs/values.yaml index 0ccc5963b..fa2acfec5 100644 --- a/k8s/seaweedfs/values.yaml +++ b/k8s/seaweedfs/values.yaml @@ -123,6 +123,9 @@ volume: # Comma separated Ip addresses having write permission. No limit if empty. whiteList: null + # Adjust jpg orientation when uploading. + imagesFixOrientation: false + extraVolumes: "" extraVolumeMounts: "" diff --git a/weed/command/server.go b/weed/command/server.go index 443f041c5..8bc98b900 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -58,9 +58,9 @@ var ( volumeMinFreeSpacePercent = cmdServer.Flag.String("volume.minFreeSpacePercent", "1", "minimum free disk space (default to 1%). Low disk space will mark all volumes as ReadOnly.") // pulseSeconds = cmdServer.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats") - isStartingFiler = cmdServer.Flag.Bool("filer", false, "whether to start filer") - isStartingS3 = cmdServer.Flag.Bool("s3", false, "whether to start S3 gateway") - isStartingMsgBroker = cmdServer.Flag.Bool("msgBroker", false, "whether to start message broker") + isStartingFiler = cmdServer.Flag.Bool("filer", false, "whether to start filer") + isStartingS3 = cmdServer.Flag.Bool("s3", false, "whether to start S3 gateway") + isStartingMsgBroker = cmdServer.Flag.Bool("msgBroker", false, "whether to start message broker") serverWhiteList []string @@ -93,6 +93,7 @@ func init() { serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port") serverOptions.v.publicPort = cmdServer.Flag.Int("volume.port.public", 0, "volume server public port") serverOptions.v.indexType = cmdServer.Flag.String("volume.index", "memory", "Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance.") + serverOptions.v.fixJpgOrientation = cmdServer.Flag.Bool("volume.images.fix.orientation", false, "Adjust jpg orientation when uploading.") serverOptions.v.readRedirect = cmdServer.Flag.Bool("volume.read.redirect", true, "Redirect moved or non-local volumes.") serverOptions.v.compactionMBPerSecond = cmdServer.Flag.Int("volume.compactionMBps", 0, "limit compaction speed in mega bytes per second") serverOptions.v.fileSizeLimitMB = cmdServer.Flag.Int("volume.fileSizeLimitMB", 256, "limit file size to avoid out of memory") diff --git a/weed/command/volume.go b/weed/command/volume.go index 0a7d52049..27687af66 100644 --- a/weed/command/volume.go +++ b/weed/command/volume.go @@ -47,6 +47,7 @@ type VolumeServerOptions struct { rack *string whiteList []string indexType *string + fixJpgOrientation *bool readRedirect *bool cpuProfile *string memProfile *string @@ -70,6 +71,7 @@ func init() { v.dataCenter = cmdVolume.Flag.String("dataCenter", "", "current volume server's data center name") v.rack = cmdVolume.Flag.String("rack", "", "current volume server's rack name") v.indexType = cmdVolume.Flag.String("index", "memory", "Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance.") + v.fixJpgOrientation = cmdVolume.Flag.Bool("images.fix.orientation", false, "Adjust jpg orientation when uploading.") v.readRedirect = cmdVolume.Flag.Bool("read.redirect", true, "Redirect moved or non-local volumes.") v.cpuProfile = cmdVolume.Flag.String("cpuprofile", "", "cpu profile output file") v.memProfile = cmdVolume.Flag.String("memprofile", "", "memory profile output file") @@ -200,7 +202,7 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v volumeNeedleMapKind, strings.Split(masters, ","), 5, *v.dataCenter, *v.rack, v.whiteList, - *v.readRedirect, + *v.fixJpgOrientation, *v.readRedirect, *v.compactionMBPerSecond, *v.fileSizeLimitMB, ) diff --git a/weed/images/orientation.go b/weed/images/orientation.go new file mode 100644 index 000000000..a592a7d8b --- /dev/null +++ b/weed/images/orientation.go @@ -0,0 +1,182 @@ +package images + +import ( + "bytes" + "image" + "image/draw" + "image/jpeg" + "log" + + "github.com/seaweedfs/goexif/exif" +) + +//many code is copied from http://camlistore.org/pkg/images/images.go +func FixJpgOrientation(data []byte) (oriented []byte) { + ex, err := exif.Decode(bytes.NewReader(data)) + if err != nil { + return data + } + tag, err := ex.Get(exif.Orientation) + if err != nil { + return data + } + angle := 0 + flipMode := FlipDirection(0) + orient, err := tag.Int(0) + if err != nil { + return data + } + switch orient { + case topLeftSide: + // do nothing + return data + case topRightSide: + flipMode = 2 + case bottomRightSide: + angle = 180 + case bottomLeftSide: + angle = 180 + flipMode = 2 + case leftSideTop: + angle = -90 + flipMode = 2 + case rightSideTop: + angle = -90 + case rightSideBottom: + angle = 90 + flipMode = 2 + case leftSideBottom: + angle = 90 + } + + if srcImage, _, err := image.Decode(bytes.NewReader(data)); err == nil { + dstImage := flip(rotate(srcImage, angle), flipMode) + var buf bytes.Buffer + jpeg.Encode(&buf, dstImage, nil) + return buf.Bytes() + } + + return data +} + +// Exif Orientation Tag values +// http://sylvana.net/jpegcrop/exif_orientation.html +const ( + topLeftSide = 1 + topRightSide = 2 + bottomRightSide = 3 + bottomLeftSide = 4 + leftSideTop = 5 + rightSideTop = 6 + rightSideBottom = 7 + leftSideBottom = 8 +) + +// The FlipDirection type is used by the Flip option in DecodeOpts +// to indicate in which direction to flip an image. +type FlipDirection int + +// FlipVertical and FlipHorizontal are two possible FlipDirections +// values to indicate in which direction an image will be flipped. +const ( + FlipVertical FlipDirection = 1 << iota + FlipHorizontal +) + +type DecodeOpts struct { + // Rotate specifies how to rotate the image. + // If nil, the image is rotated automatically based on EXIF metadata. + // If an int, Rotate is the number of degrees to rotate + // counter clockwise and must be one of 0, 90, -90, 180, or + // -180. + Rotate interface{} + + // Flip specifies how to flip the image. + // If nil, the image is flipped automatically based on EXIF metadata. + // Otherwise, Flip is a FlipDirection bitfield indicating how to flip. + Flip interface{} +} + +func rotate(im image.Image, angle int) image.Image { + var rotated *image.NRGBA + // trigonometric (i.e counter clock-wise) + switch angle { + case 90: + newH, newW := im.Bounds().Dx(), im.Bounds().Dy() + rotated = image.NewNRGBA(image.Rect(0, 0, newW, newH)) + for y := 0; y < newH; y++ { + for x := 0; x < newW; x++ { + rotated.Set(x, y, im.At(newH-1-y, x)) + } + } + case -90: + newH, newW := im.Bounds().Dx(), im.Bounds().Dy() + rotated = image.NewNRGBA(image.Rect(0, 0, newW, newH)) + for y := 0; y < newH; y++ { + for x := 0; x < newW; x++ { + rotated.Set(x, y, im.At(y, newW-1-x)) + } + } + case 180, -180: + newW, newH := im.Bounds().Dx(), im.Bounds().Dy() + rotated = image.NewNRGBA(image.Rect(0, 0, newW, newH)) + for y := 0; y < newH; y++ { + for x := 0; x < newW; x++ { + rotated.Set(x, y, im.At(newW-1-x, newH-1-y)) + } + } + default: + return im + } + return rotated +} + +// flip returns a flipped version of the image im, according to +// the direction(s) in dir. +// It may flip the imput im in place and return it, or it may allocate a +// new NRGBA (if im is an *image.YCbCr). +func flip(im image.Image, dir FlipDirection) image.Image { + if dir == 0 { + return im + } + ycbcr := false + var nrgba image.Image + dx, dy := im.Bounds().Dx(), im.Bounds().Dy() + di, ok := im.(draw.Image) + if !ok { + if _, ok := im.(*image.YCbCr); !ok { + log.Printf("failed to flip image: input does not satisfy draw.Image") + return im + } + // because YCbCr does not implement Set, we replace it with a new NRGBA + ycbcr = true + nrgba = image.NewNRGBA(image.Rect(0, 0, dx, dy)) + di, ok = nrgba.(draw.Image) + if !ok { + log.Print("failed to flip image: could not cast an NRGBA to a draw.Image") + return im + } + } + if dir&FlipHorizontal != 0 { + for y := 0; y < dy; y++ { + for x := 0; x < dx/2; x++ { + old := im.At(x, y) + di.Set(x, y, im.At(dx-1-x, y)) + di.Set(dx-1-x, y, old) + } + } + } + if dir&FlipVertical != 0 { + for y := 0; y < dy/2; y++ { + for x := 0; x < dx; x++ { + old := im.At(x, y) + di.Set(x, y, im.At(x, dy-1-y)) + di.Set(x, dy-1-y, old) + } + } + } + if ycbcr { + return nrgba + } + return im +} diff --git a/weed/images/orientation_test.go b/weed/images/orientation_test.go new file mode 100644 index 000000000..32fa38f76 --- /dev/null +++ b/weed/images/orientation_test.go @@ -0,0 +1,20 @@ +package images + +import ( + "io/ioutil" + "os" + "testing" +) + +func TestXYZ(t *testing.T) { + fname := "sample1.jpg" + + dat, _ := ioutil.ReadFile(fname) + + fixed_data := FixJpgOrientation(dat) + + ioutil.WriteFile("fixed1.jpg", fixed_data, 0644) + + os.Remove("fixed1.jpg") + +} diff --git a/weed/images/preprocess.go b/weed/images/preprocess.go new file mode 100644 index 000000000..f6f3b554d --- /dev/null +++ b/weed/images/preprocess.go @@ -0,0 +1,29 @@ +package images + +import ( + "bytes" + "io" + "path/filepath" + "strings" +) + +/* +* Preprocess image files on client side. +* 1. possibly adjust the orientation +* 2. resize the image to a width or height limit +* 3. remove the exif data +* Call this function on any file uploaded to SeaweedFS +* + */ +func MaybePreprocessImage(filename string, data []byte, width, height int) (resized io.ReadSeeker, w int, h int) { + ext := filepath.Ext(filename) + ext = strings.ToLower(ext) + switch ext { + case ".png", ".gif": + return Resized(ext, bytes.NewReader(data), width, height, "") + case ".jpg", ".jpeg": + data = FixJpgOrientation(data) + return Resized(ext, bytes.NewReader(data), width, height, "") + } + return bytes.NewReader(data), 0, 0 +} diff --git a/weed/pb/shared_values.go b/weed/pb/shared_values.go index 1af19e51a..acc3bb56d 100644 --- a/weed/pb/shared_values.go +++ b/weed/pb/shared_values.go @@ -1,5 +1,5 @@ package pb const ( - AdminShellClient = "adminShell" + AdminShellClient = "shell" ) diff --git a/weed/server/volume_server.go b/weed/server/volume_server.go index b7ed81be0..3af37b491 100644 --- a/weed/server/volume_server.go +++ b/weed/server/volume_server.go @@ -25,6 +25,7 @@ type VolumeServer struct { grpcDialOption grpc.DialOption needleMapKind storage.NeedleMapType + FixJpgOrientation bool ReadRedirect bool compactionBytePerSecond int64 MetricsAddress string @@ -39,6 +40,7 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string, masterNodes []string, pulseSeconds int, dataCenter string, rack string, whiteList []string, + fixJpgOrientation bool, readRedirect bool, compactionMBPerSecond int, fileSizeLimitMB int, @@ -59,6 +61,7 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string, dataCenter: dataCenter, rack: rack, needleMapKind: needleMapKind, + FixJpgOrientation: fixJpgOrientation, ReadRedirect: readRedirect, grpcDialOption: security.LoadClientTLS(util.GetViper(), "grpc.volume"), compactionBytePerSecond: int64(compactionMBPerSecond) * 1024 * 1024, diff --git a/weed/server/volume_server_handlers_write.go b/weed/server/volume_server_handlers_write.go index 5ece46ed0..74dad28de 100644 --- a/weed/server/volume_server_handlers_write.go +++ b/weed/server/volume_server_handlers_write.go @@ -42,7 +42,7 @@ func (vs *VolumeServer) PostHandler(w http.ResponseWriter, r *http.Request) { return } - reqNeedle, originalSize, ne := needle.CreateNeedleFromRequest(r, vs.fileSizeLimitBytes) + reqNeedle, originalSize, ne := needle.CreateNeedleFromRequest(r, vs.FixJpgOrientation, vs.fileSizeLimitBytes) if ne != nil { writeJsonError(w, r, http.StatusBadRequest, ne) return diff --git a/weed/storage/needle/needle.go b/weed/storage/needle/needle.go index 4c62aa00b..7d02758d6 100644 --- a/weed/storage/needle/needle.go +++ b/weed/storage/needle/needle.go @@ -8,6 +8,7 @@ import ( "strings" "time" + "github.com/chrislusf/seaweedfs/weed/images" . "github.com/chrislusf/seaweedfs/weed/storage/types" ) @@ -47,7 +48,7 @@ func (n *Needle) String() (str string) { return } -func CreateNeedleFromRequest(r *http.Request, sizeLimit int64) (n *Needle, originalSize int, e error) { +func CreateNeedleFromRequest(r *http.Request, fixJpgOrientation bool, sizeLimit int64) (n *Needle, originalSize int, e error) { n = new(Needle) pu, e := ParseUpload(r, sizeLimit) if e != nil { @@ -94,6 +95,13 @@ func CreateNeedleFromRequest(r *http.Request, sizeLimit int64) (n *Needle, origi n.SetIsChunkManifest() } + if fixJpgOrientation { + loweredName := strings.ToLower(pu.FileName) + if pu.MimeType == "image/jpeg" || strings.HasSuffix(loweredName, ".jpg") || strings.HasSuffix(loweredName, ".jpeg") { + n.Data = images.FixJpgOrientation(n.Data) + } + } + n.Checksum = NewCRC(n.Data) commaSep := strings.LastIndex(r.URL.Path, ",") diff --git a/weed/wdclient/masterclient.go b/weed/wdclient/masterclient.go index 4c066d535..4f8e0d5ef 100644 --- a/weed/wdclient/masterclient.go +++ b/weed/wdclient/masterclient.go @@ -45,7 +45,7 @@ func (mc *MasterClient) WaitUntilConnected() { } func (mc *MasterClient) KeepConnectedToMaster() { - glog.V(1).Infof("%s masterClient bootstraps with masters %v", mc.clientType, mc.masters) + glog.V(1).Infof("%s bootstraps with masters %v", mc.clientType, mc.masters) for { mc.tryAllMasters() time.Sleep(time.Second) @@ -67,27 +67,27 @@ func (mc *MasterClient) tryAllMasters() { } func (mc *MasterClient) tryConnectToMaster(master string) (nextHintedLeader string) { - glog.V(1).Infof("%s masterClient Connecting to master %v", mc.clientType, master) + glog.V(1).Infof("%s Connecting to master %v", mc.clientType, master) gprcErr := pb.WithMasterClient(master, mc.grpcDialOption, func(client master_pb.SeaweedClient) error { stream, err := client.KeepConnected(context.Background()) if err != nil { - glog.V(0).Infof("%s masterClient failed to keep connected to %s: %v", mc.clientType, master, err) + glog.V(0).Infof("%s failed to keep connected to %s: %v", mc.clientType, master, err) return err } if err = stream.Send(&master_pb.KeepConnectedRequest{Name: mc.clientType, GrpcPort: mc.grpcPort}); err != nil { - glog.V(0).Infof("%s masterClient failed to send to %s: %v", mc.clientType, master, err) + glog.V(0).Infof("%s failed to send to %s: %v", mc.clientType, master, err) return err } - glog.V(1).Infof("%s masterClient Connected to %v", mc.clientType, master) + glog.V(1).Infof("%s Connected to %v", mc.clientType, master) mc.currentMaster = master for { volumeLocation, err := stream.Recv() if err != nil { - glog.V(0).Infof("%s masterClient failed to receive from %s: %v", mc.clientType, master, err) + glog.V(0).Infof("%s failed to receive from %s: %v", mc.clientType, master, err) return err } @@ -104,18 +104,18 @@ func (mc *MasterClient) tryConnectToMaster(master string) (nextHintedLeader stri PublicUrl: volumeLocation.PublicUrl, } for _, newVid := range volumeLocation.NewVids { - glog.V(1).Infof("%s: %s masterClient adds volume %d", mc.clientType, loc.Url, newVid) + glog.V(1).Infof("%s: %s adds volume %d", mc.clientType, loc.Url, newVid) mc.addLocation(newVid, loc) } for _, deletedVid := range volumeLocation.DeletedVids { - glog.V(1).Infof("%s: %s masterClient removes volume %d", mc.clientType, loc.Url, deletedVid) + glog.V(1).Infof("%s: %s removes volume %d", mc.clientType, loc.Url, deletedVid) mc.deleteLocation(deletedVid, loc) } } }) if gprcErr != nil { - glog.V(0).Infof("%s masterClient failed to connect with master %v: %v", mc.clientType, master, gprcErr) + glog.V(0).Infof("%s failed to connect with master %v: %v", mc.clientType, master, gprcErr) } return } From 8c318470dd95b3fc24d39dc3cf253cc17b03ab39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E8=BF=9C=E5=86=9B?= Date: Fri, 10 Jul 2020 10:28:28 +0800 Subject: [PATCH 2/2] Revert "remove fixJpgOrientation" This reverts commit de5ca9b2 minimise the revert changes --- go.mod | 5 ----- go.sum | 29 +++++------------------------ weed/pb/shared_values.go | 2 +- weed/wdclient/masterclient.go | 18 +++++++++--------- 4 files changed, 15 insertions(+), 39 deletions(-) diff --git a/go.mod b/go.mod index 6bd3733c5..3bb1bf4bb 100644 --- a/go.mod +++ b/go.mod @@ -13,10 +13,7 @@ require ( github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72 github.com/cespare/xxhash v1.1.0 github.com/chrislusf/raft v1.0.1 - github.com/coreos/bbolt v1.3.3 // indirect - github.com/coreos/etcd v3.3.15+incompatible // indirect github.com/coreos/go-semver v0.3.0 // indirect - github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect github.com/dgrijalva/jwt-go v3.2.0+incompatible github.com/disintegration/imaging v1.6.2 github.com/dustin/go-humanize v1.0.0 @@ -37,7 +34,6 @@ require ( github.com/google/uuid v1.1.1 github.com/gorilla/mux v1.7.3 github.com/gorilla/websocket v1.4.1 // indirect - github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 // indirect github.com/grpc-ecosystem/grpc-gateway v1.11.0 // indirect github.com/hashicorp/golang-lru v0.5.3 // indirect github.com/jcmturner/gofork v1.0.0 // indirect @@ -92,7 +88,6 @@ require ( gopkg.in/jcmturner/goidentity.v3 v3.0.0 // indirect gopkg.in/jcmturner/gokrb5.v7 v7.3.0 // indirect gopkg.in/karlseguin/expect.v1 v1.0.1 // indirect - sigs.k8s.io/yaml v1.1.0 // indirect ) replace ( diff --git a/go.sum b/go.sum index 098dd0cb6..ec8fb2805 100644 --- a/go.sum +++ b/go.sum @@ -67,31 +67,22 @@ github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/chrislusf/raft v0.0.0-20190225081310-10d6e2182d92 h1:lM9SFsh0EPXkyJyrTJqLZPAIJBtNFP6LNkYXu2MnSZI= -github.com/chrislusf/raft v0.0.0-20190225081310-10d6e2182d92/go.mod h1:4jyiUCD5y548+yKW+oiHtccBiMaLCCbFBpK2t7X4eUo= -github.com/chrislusf/raft v1.0.0 h1:tRGtB3nWOg8VOPSoeZ4NfHMLJcjKgyNkUZQbcyKWf1Y= -github.com/chrislusf/raft v1.0.0/go.mod h1:Ep5DP+mJSosjfKiix1uU7Lc2Df/SX4oGJEpZlXH5l68= github.com/chrislusf/raft v1.0.1 h1:Wa4ffkmkysW7cX3T/gMC/Mk3PhnOXhsqOVwQJcMndhw= github.com/chrislusf/raft v1.0.1/go.mod h1:Ep5DP+mJSosjfKiix1uU7Lc2Df/SX4oGJEpZlXH5l68= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/coreos/bbolt v1.3.2 h1:wZwiHHUieZCquLkDL0B8UhzreNWsPHooDAG3q34zk0s= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/bbolt v1.3.3 h1:n6AiVyVRKQFNb6mJlwESEvvLoDyiTzXX7ORAUlkeBdY= -github.com/coreos/bbolt v1.3.3/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible h1:jFneRYjIvLMLhDLCzuTuU4rSJUjRplcJQ7pD7MnhC04= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/etcd v3.3.15+incompatible h1:+9RjdC18gMxNQVvSiXvObLu29mOFmkgdsB4cRTlV+EE= -github.com/coreos/etcd v3.3.15+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.2.0 h1:3Jm3tLmsgAYcjC+4Up7hJrFBPr+n7rAqYeSw/SZazuY= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f h1:JOrtw2xFKzlg+cbHpyrpLDmnN1HqhBfnX7WDiW7eG2c= -github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.0.0 h1:XJIw/+VlJ+87J+doOxznsAWIdmWuViOVhkQamW5YV28= github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= @@ -509,13 +500,10 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= go.etcd.io/bbolt v1.3.2 h1:Z/90sZLPOeCy2PwprqkFa25PdkusRzaj9P8zm/KNyvk= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk= -go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.4 h1:hi1bXHMVrlQh6WwxAy+qZCV/SYIlqo+Ushwdpa4tAKg= go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/etcd v0.5.0-alpha.5.0.20200425165423-262c93980547 h1:s71VGheLtWmCYsnNjf+s7XE8HsrZnd3EYGrLGWVm7nY= go.etcd.io/etcd v0.5.0-alpha.5.0.20200425165423-262c93980547/go.mod h1:YoUyTScD3Vcv2RBm3eGVOq7i1ULiz3OuXoQFWOirmAM= -go.etcd.io/etcd v3.3.15+incompatible h1:0VpOVCF6EFnJptt8Jh0EWEHO4j2fepyV1fpu9xz/UoQ= -go.etcd.io/etcd v3.3.15+incompatible/go.mod h1:yaeTdrJi5lOmYerz05bd8+V7KubZs8YSFZfzsF9A6aI= go.mongodb.org/mongo-driver v1.3.2 h1:IYppNjEV/C+/3VPbhHVxQ4t04eVW0cLp0/pNdW++6Ug= go.mongodb.org/mongo-driver v1.3.2/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE= go.opencensus.io v0.15.0/go.mod h1:UffZAU+4sDEINUGP/B7UfBBkq4fqLu9zXAX7ke6CHW0= @@ -528,10 +516,9 @@ go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.2.0 h1:6I+W7f5VwC5SV9dNrZ3qXrDB9mD0dyGOi/ZJmYw03T4= -go.uber.org/multierr v1.2.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= @@ -553,8 +540,6 @@ golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5 h1:58fnuSXlxZmFdJyvtTFVmVhcMLU6v5fEb/ok4wyqtNU= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7 h1:0hQKqeLdqlt5iIwVOBErRisrHJAN57yOiPRQItI20fU= -golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc h1:c0o/qxkaO2LF5t6fQrT4b5hzyggAkLLlCUjqfRxd8Q4= golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -569,6 +554,7 @@ golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvx golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -624,8 +610,6 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0 h1:HyfiK1WMnHj5FXFXatD+Qs1A/ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190910064555-bbd175535a8b h1:3S2h5FadpNr0zUUCVZjlKIEYF+KaX/OBplTGo89CYHI= -golang.org/x/sys v0.0.0-20190910064555-bbd175535a8b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -655,8 +639,6 @@ golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190911022129-16c5e0f7d110 h1:6S6bidS7O4yAwA5ORRbRIjvNQ9tGbLd5e+LRIaTeVDQ= -golang.org/x/tools v0.0.0-20190911022129-16c5e0f7d110/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5 h1:hKsoRgsbwY1NafxrwTs+k64bikrLBkAgPir1TNCj3Zs= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -691,8 +673,6 @@ google.golang.org/genproto v0.0.0-20190620144150-6af8c5fc6601/go.mod h1:z3L6/3dT google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190905072037-92dd089d5514 h1:oFSK4421fpCKRrpzIpybyBVWyht05NegY9+L/3TLAZs= -google.golang.org/genproto v0.0.0-20190905072037-92dd089d5514/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -756,6 +736,7 @@ honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= pack.ag/amqp v0.11.2/go.mod h1:4/cbmt4EJXSKlG6LCfWHoqmN0uFdy5i/+YFz+fTfhV4= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/weed/pb/shared_values.go b/weed/pb/shared_values.go index acc3bb56d..1af19e51a 100644 --- a/weed/pb/shared_values.go +++ b/weed/pb/shared_values.go @@ -1,5 +1,5 @@ package pb const ( - AdminShellClient = "shell" + AdminShellClient = "adminShell" ) diff --git a/weed/wdclient/masterclient.go b/weed/wdclient/masterclient.go index 4f8e0d5ef..4c066d535 100644 --- a/weed/wdclient/masterclient.go +++ b/weed/wdclient/masterclient.go @@ -45,7 +45,7 @@ func (mc *MasterClient) WaitUntilConnected() { } func (mc *MasterClient) KeepConnectedToMaster() { - glog.V(1).Infof("%s bootstraps with masters %v", mc.clientType, mc.masters) + glog.V(1).Infof("%s masterClient bootstraps with masters %v", mc.clientType, mc.masters) for { mc.tryAllMasters() time.Sleep(time.Second) @@ -67,27 +67,27 @@ func (mc *MasterClient) tryAllMasters() { } func (mc *MasterClient) tryConnectToMaster(master string) (nextHintedLeader string) { - glog.V(1).Infof("%s Connecting to master %v", mc.clientType, master) + glog.V(1).Infof("%s masterClient Connecting to master %v", mc.clientType, master) gprcErr := pb.WithMasterClient(master, mc.grpcDialOption, func(client master_pb.SeaweedClient) error { stream, err := client.KeepConnected(context.Background()) if err != nil { - glog.V(0).Infof("%s failed to keep connected to %s: %v", mc.clientType, master, err) + glog.V(0).Infof("%s masterClient failed to keep connected to %s: %v", mc.clientType, master, err) return err } if err = stream.Send(&master_pb.KeepConnectedRequest{Name: mc.clientType, GrpcPort: mc.grpcPort}); err != nil { - glog.V(0).Infof("%s failed to send to %s: %v", mc.clientType, master, err) + glog.V(0).Infof("%s masterClient failed to send to %s: %v", mc.clientType, master, err) return err } - glog.V(1).Infof("%s Connected to %v", mc.clientType, master) + glog.V(1).Infof("%s masterClient Connected to %v", mc.clientType, master) mc.currentMaster = master for { volumeLocation, err := stream.Recv() if err != nil { - glog.V(0).Infof("%s failed to receive from %s: %v", mc.clientType, master, err) + glog.V(0).Infof("%s masterClient failed to receive from %s: %v", mc.clientType, master, err) return err } @@ -104,18 +104,18 @@ func (mc *MasterClient) tryConnectToMaster(master string) (nextHintedLeader stri PublicUrl: volumeLocation.PublicUrl, } for _, newVid := range volumeLocation.NewVids { - glog.V(1).Infof("%s: %s adds volume %d", mc.clientType, loc.Url, newVid) + glog.V(1).Infof("%s: %s masterClient adds volume %d", mc.clientType, loc.Url, newVid) mc.addLocation(newVid, loc) } for _, deletedVid := range volumeLocation.DeletedVids { - glog.V(1).Infof("%s: %s removes volume %d", mc.clientType, loc.Url, deletedVid) + glog.V(1).Infof("%s: %s masterClient removes volume %d", mc.clientType, loc.Url, deletedVid) mc.deleteLocation(deletedVid, loc) } } }) if gprcErr != nil { - glog.V(0).Infof("%s failed to connect with master %v: %v", mc.clientType, master, gprcErr) + glog.V(0).Infof("%s masterClient failed to connect with master %v: %v", mc.clientType, master, gprcErr) } return }