From 8e2f3b510e9c33a73f9eaf0f334e5fa1a8578354 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 10 Oct 2020 15:43:04 -0700 Subject: [PATCH 1/8] Hadoop client: 1.4.9 --- other/java/client/pom.xml | 2 +- other/java/client/pom.xml.deploy | 2 +- other/java/client/pom_debug.xml | 2 +- other/java/hdfs2/dependency-reduced-pom.xml | 2 +- other/java/hdfs2/pom.xml | 2 +- other/java/hdfs3/dependency-reduced-pom.xml | 2 +- other/java/hdfs3/pom.xml | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/other/java/client/pom.xml b/other/java/client/pom.xml index ad9145155..01ac3dcfc 100644 --- a/other/java/client/pom.xml +++ b/other/java/client/pom.xml @@ -5,7 +5,7 @@ com.github.chrislusf seaweedfs-client - 1.4.8 + 1.4.9 org.sonatype.oss diff --git a/other/java/client/pom.xml.deploy b/other/java/client/pom.xml.deploy index ad9145155..01ac3dcfc 100644 --- a/other/java/client/pom.xml.deploy +++ b/other/java/client/pom.xml.deploy @@ -5,7 +5,7 @@ com.github.chrislusf seaweedfs-client - 1.4.8 + 1.4.9 org.sonatype.oss diff --git a/other/java/client/pom_debug.xml b/other/java/client/pom_debug.xml index 55aa56e8b..14281f7a3 100644 --- a/other/java/client/pom_debug.xml +++ b/other/java/client/pom_debug.xml @@ -5,7 +5,7 @@ com.github.chrislusf seaweedfs-client - 1.4.8 + 1.4.9 org.sonatype.oss diff --git a/other/java/hdfs2/dependency-reduced-pom.xml b/other/java/hdfs2/dependency-reduced-pom.xml index e3b434b9b..966abd1c7 100644 --- a/other/java/hdfs2/dependency-reduced-pom.xml +++ b/other/java/hdfs2/dependency-reduced-pom.xml @@ -301,7 +301,7 @@ - 1.4.8 + 1.4.9 2.9.2 diff --git a/other/java/hdfs2/pom.xml b/other/java/hdfs2/pom.xml index 595d070e8..ede70eea4 100644 --- a/other/java/hdfs2/pom.xml +++ b/other/java/hdfs2/pom.xml @@ -5,7 +5,7 @@ 4.0.0 - 1.4.8 + 1.4.9 2.9.2 diff --git a/other/java/hdfs3/dependency-reduced-pom.xml b/other/java/hdfs3/dependency-reduced-pom.xml index 46add89fe..9ac40b48b 100644 --- a/other/java/hdfs3/dependency-reduced-pom.xml +++ b/other/java/hdfs3/dependency-reduced-pom.xml @@ -309,7 +309,7 @@ - 1.4.8 + 1.4.9 3.1.1 diff --git a/other/java/hdfs3/pom.xml b/other/java/hdfs3/pom.xml index 240c6eb41..d1a5d8e0f 100644 --- a/other/java/hdfs3/pom.xml +++ b/other/java/hdfs3/pom.xml @@ -5,7 +5,7 @@ 4.0.0 - 1.4.8 + 1.4.9 3.1.1 From cff8bb6554287d50ef3fdc83dd5668ad262d3a79 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 10 Oct 2020 15:43:22 -0700 Subject: [PATCH 2/8] return proper error --- weed/filer/reader_at.go | 4 ++++ weed/filesys/filehandle.go | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/weed/filer/reader_at.go b/weed/filer/reader_at.go index fa51df687..461ef2b2c 100644 --- a/weed/filer/reader_at.go +++ b/weed/filer/reader_at.go @@ -58,6 +58,10 @@ func LookupFn(filerClient filer_pb.FilerClient) LookupFileIdFunctionType { }) } + if err != nil { + return nil, err + } + for _, loc := range locations.Locations { volumeServerAddress := filerClient.AdjustedUrl(loc.Url) targetUrl := fmt.Sprintf("http://%s/%s", volumeServerAddress, fileId) diff --git a/weed/filesys/filehandle.go b/weed/filesys/filehandle.go index 660bbf076..e1524f939 100644 --- a/weed/filesys/filehandle.go +++ b/weed/filesys/filehandle.go @@ -79,7 +79,7 @@ func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fus if err != nil { glog.Warningf("file handle read %s %d: %v", fh.f.fullpath(), totalRead, err) - return nil + return fuse.EIO } if totalRead > int64(len(buff)) { From 70af0ec24c795ef26e9d85813b9b8eecc22d73c5 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 10 Oct 2020 16:02:10 -0700 Subject: [PATCH 3/8] filer: fix hanging on read --- weed/filer/stream.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/filer/stream.go b/weed/filer/stream.go index f6e2a7643..a41aebe22 100644 --- a/weed/filer/stream.go +++ b/weed/filer/stream.go @@ -34,7 +34,7 @@ func StreamContent(masterClient *wdclient.MasterClient, w io.Writer, chunks []*f urlStrings := fileId2Url[chunkView.FileId] data, err := retriedFetchChunkData(urlStrings, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size)) - if err == nil { + if err != nil { return err } w.Write(data) From 8a52379ecb8a11eb7fa096392ab87d222d6892ae Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 10 Oct 2020 16:02:39 -0700 Subject: [PATCH 4/8] add retry if volume can not be found --- weed/filer/reader_at.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/weed/filer/reader_at.go b/weed/filer/reader_at.go index 461ef2b2c..4d0cde850 100644 --- a/weed/filer/reader_at.go +++ b/weed/filer/reader_at.go @@ -11,6 +11,7 @@ import ( "io" "math/rand" "sync" + "time" ) type ChunkReadAt struct { @@ -37,7 +38,8 @@ func LookupFn(filerClient filer_pb.FilerClient) LookupFileIdFunctionType { vid := VolumeId(fileId) locations, found := vidCache[vid] - if !found { + waitTime := time.Second + for !found && waitTime < 6*time.Second { // println("looking up volume", vid) err = filerClient.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { resp, err := client.LookupVolume(context.Background(), &filer_pb.LookupVolumeRequest{ @@ -56,6 +58,12 @@ func LookupFn(filerClient filer_pb.FilerClient) LookupFileIdFunctionType { return nil }) + if err == nil { + break + } + glog.V(1).Infof("wait for volume %s", vid) + time.Sleep(waitTime) + waitTime += waitTime / 2 } if err != nil { From 9b0e8ef026cd74e8c641e922c79a54c5154e5a6b Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 10 Oct 2020 18:00:21 -0700 Subject: [PATCH 5/8] filer: added QR code to transfer files with mobile devices --- go.mod | 1 + go.sum | 2 + weed/server/filer_server_handlers_read_dir.go | 44 ++++++++++++------- weed/server/filer_ui/templates.go | 19 +++++++- 4 files changed, 48 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index 2158cfb65..2a65d47a1 100644 --- a/go.mod +++ b/go.mod @@ -60,6 +60,7 @@ require ( github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 // indirect github.com/seaweedfs/fuse v0.0.0-20190510212405-310228904eff github.com/seaweedfs/goexif v1.0.2 + github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/afero v1.2.2 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect diff --git a/go.sum b/go.sum index 3ac47ec22..ce8ff4a8c 100644 --- a/go.sum +++ b/go.sum @@ -452,6 +452,8 @@ github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0= +github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M= github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= github.com/smartystreets/gunit v1.3.4/go.mod h1:ZjM1ozSIMJlAz/ay4SG8PeKF00ckUp+zMHZXV9/bvak= diff --git a/weed/server/filer_server_handlers_read_dir.go b/weed/server/filer_server_handlers_read_dir.go index 9ca0209f4..99345550c 100644 --- a/weed/server/filer_server_handlers_read_dir.go +++ b/weed/server/filer_server_handlers_read_dir.go @@ -2,6 +2,9 @@ package weed_server import ( "context" + "encoding/base64" + "fmt" + "github.com/skip2/go-qrcode" "net/http" "strconv" "strings" @@ -65,21 +68,30 @@ func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Reque lastFileName, shouldDisplayLoadMore, }) - } else { - ui.StatusTpl.Execute(w, struct { - Path string - Breadcrumbs []ui.Breadcrumb - Entries interface{} - Limit int - LastFileName string - ShouldDisplayLoadMore bool - }{ - path, - ui.ToBreadcrumb(path), - entries, - limit, - lastFileName, - shouldDisplayLoadMore, - }) + return + } + + var qrImageString string + img, err := qrcode.Encode(fmt.Sprintf("http://%s:%d%s", fs.option.Host, fs.option.Port, r.URL.Path), qrcode.Medium, 128) + if err == nil { + qrImageString = base64.StdEncoding.EncodeToString(img) } + + ui.StatusTpl.Execute(w, struct { + Path string + Breadcrumbs []ui.Breadcrumb + Entries interface{} + Limit int + LastFileName string + ShouldDisplayLoadMore bool + QrImage string + }{ + path, + ui.ToBreadcrumb(path), + entries, + limit, + lastFileName, + shouldDisplayLoadMore, + qrImageString, + }) } diff --git a/weed/server/filer_ui/templates.go b/weed/server/filer_ui/templates.go index 04a81433b..f86dde5b1 100644 --- a/weed/server/filer_ui/templates.go +++ b/weed/server/filer_ui/templates.go @@ -21,9 +21,11 @@ var funcMap = template.FuncMap{ var StatusTpl = template.Must(template.New("status").Funcs(funcMap).Parse(` - SeaweedFS Filer - + SeaweedFS Filer + + @@ -116,6 +123,14 @@ var StatusTpl = template.Must(template.New("status").Funcs(funcMap).Parse(` {{end}} + +
+
+ +
+ +
+