From a9990a1dc65dfcfe9e08fdbcc698bab7e9dfca90 Mon Sep 17 00:00:00 2001 From: "ruitao.liu" Date: Fri, 13 Nov 2020 17:13:20 +0800 Subject: [PATCH 01/26] adjust check bucket if exist or has access. --- weed/s3api/s3api_bucket_handlers.go | 32 ++++++++++++++--------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/weed/s3api/s3api_bucket_handlers.go b/weed/s3api/s3api_bucket_handlers.go index fe33cfe78..00b7382cc 100644 --- a/weed/s3api/s3api_bucket_handlers.go +++ b/weed/s3api/s3api_bucket_handlers.go @@ -118,18 +118,12 @@ func (s3a *S3ApiServer) DeleteBucketHandler(w http.ResponseWriter, r *http.Reque bucket, _ := getBucketAndObject(r) - entry, err := s3a.getEntry(s3a.option.BucketsPath, bucket) - if entry == nil || err == filer_pb.ErrNotFound { - writeErrorResponse(w, s3err.ErrNoSuchBucket, r.URL) + if err := s3a.checkBucket(r, bucket); err != s3err.ErrNone { + writeErrorResponse(w, err, r.URL) return } - if !s3a.hasAccess(r, entry) { - writeErrorResponse(w, s3err.ErrAccessDenied, r.URL) - return - } - - err = s3a.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + err := s3a.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { // delete collection deleteCollectionRequest := &filer_pb.DeleteCollectionRequest{ @@ -158,18 +152,24 @@ func (s3a *S3ApiServer) HeadBucketHandler(w http.ResponseWriter, r *http.Request bucket, _ := getBucketAndObject(r) + if err := s3a.checkBucket(r, bucket); err != s3err.ErrNone { + writeErrorResponse(w, err, r.URL) + return + } + + writeSuccessResponseEmpty(w) +} + +func (s3a *S3ApiServer) checkBucket(r *http.Request, bucket string) s3err.ErrorCode { entry, err := s3a.getEntry(s3a.option.BucketsPath, bucket) if entry == nil || err == filer_pb.ErrNotFound { - writeErrorResponse(w, s3err.ErrNoSuchBucket, r.URL) - return + return s3err.ErrNoSuchBucket } if !s3a.hasAccess(r, entry) { - writeErrorResponse(w, s3err.ErrAccessDenied, r.URL) - return + return s3err.ErrAccessDenied } - - writeSuccessResponseEmpty(w) + return s3err.ErrNone } func (s3a *S3ApiServer) hasAccess(r *http.Request, entry *filer_pb.Entry) bool { @@ -188,4 +188,4 @@ func (s3a *S3ApiServer) hasAccess(r *http.Request, entry *filer_pb.Entry) bool { } } return true -} \ No newline at end of file +} From 56d8e5b540caf76f186dc17660e523dc0baa1a54 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 13 Nov 2020 11:33:01 -0800 Subject: [PATCH 02/26] Update release.yml --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c5f58cd72..47b1422f2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -28,7 +28,7 @@ jobs: project_path: weed binary_name: weed-large-disk - name: Go Release Binaries - uses: wangyoucao577/go-release-action@feature/customize-target-release + uses: wangyoucao577/go-release-action@v1.10 with: github_token: ${{ secrets.GITHUB_TOKEN }} goos: linux # default is From 7d249808407588431f00424dbb2ca599cc9441c6 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 13 Nov 2020 11:37:50 -0800 Subject: [PATCH 03/26] HCFS: 1.5.4 --- other/java/client/pom.xml | 2 +- other/java/client/pom.xml.deploy | 2 +- other/java/client/pom_debug.xml | 2 +- other/java/hdfs2/dependency-reduced-pom.xml | 2 +- other/java/hdfs2/pom.xml | 2 +- other/java/hdfs3/dependency-reduced-pom.xml | 2 +- other/java/hdfs3/pom.xml | 2 +- other/java/unzip/pom.xml | 4 ++-- .../unzip/src/main/java/com/example/test/Example.java | 10 ++++++++++ 9 files changed, 19 insertions(+), 9 deletions(-) diff --git a/other/java/client/pom.xml b/other/java/client/pom.xml index 09f891a8a..5a4bbaead 100644 --- a/other/java/client/pom.xml +++ b/other/java/client/pom.xml @@ -5,7 +5,7 @@ com.github.chrislusf seaweedfs-client - 1.5.3 + 1.5.4 org.sonatype.oss diff --git a/other/java/client/pom.xml.deploy b/other/java/client/pom.xml.deploy index 28a62d66f..e24bcca9f 100644 --- a/other/java/client/pom.xml.deploy +++ b/other/java/client/pom.xml.deploy @@ -5,7 +5,7 @@ com.github.chrislusf seaweedfs-client - 1.5.3 + 1.5.4 org.sonatype.oss diff --git a/other/java/client/pom_debug.xml b/other/java/client/pom_debug.xml index 78e4986cf..724c3c3b9 100644 --- a/other/java/client/pom_debug.xml +++ b/other/java/client/pom_debug.xml @@ -5,7 +5,7 @@ com.github.chrislusf seaweedfs-client - 1.5.3 + 1.5.4 org.sonatype.oss diff --git a/other/java/hdfs2/dependency-reduced-pom.xml b/other/java/hdfs2/dependency-reduced-pom.xml index 4be9ff180..b598d8402 100644 --- a/other/java/hdfs2/dependency-reduced-pom.xml +++ b/other/java/hdfs2/dependency-reduced-pom.xml @@ -301,7 +301,7 @@ - 1.5.3 + 1.5.4 2.9.2 diff --git a/other/java/hdfs2/pom.xml b/other/java/hdfs2/pom.xml index 478ae40ad..de518a0dc 100644 --- a/other/java/hdfs2/pom.xml +++ b/other/java/hdfs2/pom.xml @@ -5,7 +5,7 @@ 4.0.0 - 1.5.3 + 1.5.4 2.9.2 diff --git a/other/java/hdfs3/dependency-reduced-pom.xml b/other/java/hdfs3/dependency-reduced-pom.xml index c09ab0040..262c3ca80 100644 --- a/other/java/hdfs3/dependency-reduced-pom.xml +++ b/other/java/hdfs3/dependency-reduced-pom.xml @@ -309,7 +309,7 @@ - 1.5.3 + 1.5.4 3.1.1 diff --git a/other/java/hdfs3/pom.xml b/other/java/hdfs3/pom.xml index d579daa76..1952305e9 100644 --- a/other/java/hdfs3/pom.xml +++ b/other/java/hdfs3/pom.xml @@ -5,7 +5,7 @@ 4.0.0 - 1.5.3 + 1.5.4 3.1.1 diff --git a/other/java/unzip/pom.xml b/other/java/unzip/pom.xml index 1f86bb688..7cbb56ec3 100644 --- a/other/java/unzip/pom.xml +++ b/other/java/unzip/pom.xml @@ -11,13 +11,13 @@ com.github.chrislusf seaweedfs-client - 1.5.3 + 1.5.4 compile com.github.chrislusf seaweedfs-hadoop2-client - 1.5.3 + 1.5.4 compile diff --git a/other/java/unzip/src/main/java/com/example/test/Example.java b/other/java/unzip/src/main/java/com/example/test/Example.java index 393ccb5ab..3d22329a8 100644 --- a/other/java/unzip/src/main/java/com/example/test/Example.java +++ b/other/java/unzip/src/main/java/com/example/test/Example.java @@ -17,9 +17,14 @@ public class Example { public static void main(String[] args) throws IOException { + long startTime = System.currentTimeMillis(); // 本地模式，速度很快 parseZip("/Users/chris/tmp/test.zip"); + long startTime2 = System.currentTimeMillis(); + + long localProcessTime = startTime2 - startTime; + // swfs读取，慢 SeaweedInputStream seaweedInputStream = new SeaweedInputStream( filerGrpcClient, @@ -29,6 +34,11 @@ public class Example { ); parseZip(seaweedInputStream); + long swProcessTime = System.currentTimeMillis() - startTime2; + + System.out.println("Local time: " + localProcessTime); + System.out.println("SeaweedFS time: " + swProcessTime); + } public static void parseZip(String filename) throws IOException { From 0d5355c6141f4d5ff83933f0b7803e88ae1df90b Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 13 Nov 2020 11:43:25 -0800 Subject: [PATCH 04/26] rename --- other/java/{unzip => examples}/pom.xml | 0 .../src/main/java/com/example/test/Example.java | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename other/java/{unzip => examples}/pom.xml (100%) rename other/java/{unzip => examples}/src/main/java/com/example/test/Example.java (100%) diff --git a/other/java/unzip/pom.xml b/other/java/examples/pom.xml similarity index 100% rename from other/java/unzip/pom.xml rename to other/java/examples/pom.xml diff --git a/other/java/unzip/src/main/java/com/example/test/Example.java b/other/java/examples/src/main/java/com/example/test/Example.java similarity index 100% rename from other/java/unzip/src/main/java/com/example/test/Example.java rename to other/java/examples/src/main/java/com/example/test/Example.java From 3e362451d226d9e19b4b652a02926dedc02f6cf9 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 13 Nov 2020 12:10:55 -0800 Subject: [PATCH 05/26] add example of watch files --- .../java/seaweedfs/client/FilerClient.java | 15 +++++-- .../examples/UnzipFile.java} | 12 +++--- .../com/seaweedfs/examples/WatchFiles.java | 42 +++++++++++++++++++ 3 files changed, 59 insertions(+), 10 deletions(-) rename other/java/examples/src/main/java/com/{example/test/Example.java => seaweedfs/examples/UnzipFile.java} (83%) create mode 100644 other/java/examples/src/main/java/com/seaweedfs/examples/WatchFiles.java diff --git a/other/java/client/src/main/java/seaweedfs/client/FilerClient.java b/other/java/client/src/main/java/seaweedfs/client/FilerClient.java index 035b2c852..7338d5bee 100644 --- a/other/java/client/src/main/java/seaweedfs/client/FilerClient.java +++ b/other/java/client/src/main/java/seaweedfs/client/FilerClient.java @@ -275,9 +275,9 @@ public class FilerClient { try { FilerProto.CreateEntryResponse createEntryResponse = filerGrpcClient.getBlockingStub().createEntry(FilerProto.CreateEntryRequest.newBuilder() - .setDirectory(parent) - .setEntry(entry) - .build()); + .setDirectory(parent) + .setEntry(entry) + .build()); if (Strings.isNullOrEmpty(createEntryResponse.getError())) { return true; } @@ -333,4 +333,13 @@ public class FilerClient { return true; } + public Iterator watch(String prefix, String clientName, long sinceNs) { + return filerGrpcClient.getBlockingStub().subscribeMetadata(FilerProto.SubscribeMetadataRequest.newBuilder() + .setPathPrefix(prefix) + .setClientName(clientName) + .setSinceNs(sinceNs) + .build() + ); + } + } diff --git a/other/java/examples/src/main/java/com/example/test/Example.java b/other/java/examples/src/main/java/com/seaweedfs/examples/UnzipFile.java similarity index 83% rename from other/java/examples/src/main/java/com/example/test/Example.java rename to other/java/examples/src/main/java/com/seaweedfs/examples/UnzipFile.java index 3d22329a8..0529a5c73 100644 --- a/other/java/examples/src/main/java/com/example/test/Example.java +++ b/other/java/examples/src/main/java/com/seaweedfs/examples/UnzipFile.java @@ -1,4 +1,4 @@ -package com.example.test; +package com.seaweedfs.examples; import seaweed.hdfs.SeaweedInputStream; import seaweedfs.client.FilerClient; @@ -10,22 +10,20 @@ import java.io.InputStream; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; -public class Example { - - public static FilerClient filerClient = new FilerClient("localhost", 18888); - public static FilerGrpcClient filerGrpcClient = new FilerGrpcClient("localhost", 18888); +public class UnzipFile { public static void main(String[] args) throws IOException { + FilerGrpcClient filerGrpcClient = new FilerGrpcClient("localhost", 18888); + FilerClient filerClient = new FilerClient(filerGrpcClient); + long startTime = System.currentTimeMillis(); - // 本地模式，速度很快 parseZip("/Users/chris/tmp/test.zip"); long startTime2 = System.currentTimeMillis(); long localProcessTime = startTime2 - startTime; - // swfs读取，慢 SeaweedInputStream seaweedInputStream = new SeaweedInputStream( filerGrpcClient, new org.apache.hadoop.fs.FileSystem.Statistics(""), diff --git a/other/java/examples/src/main/java/com/seaweedfs/examples/WatchFiles.java b/other/java/examples/src/main/java/com/seaweedfs/examples/WatchFiles.java new file mode 100644 index 000000000..c4f4c81b0 --- /dev/null +++ b/other/java/examples/src/main/java/com/seaweedfs/examples/WatchFiles.java @@ -0,0 +1,42 @@ +package com.seaweedfs.examples; + +import seaweedfs.client.FilerClient; +import seaweedfs.client.FilerGrpcClient; +import seaweedfs.client.FilerProto; + +import java.io.IOException; +import java.util.Iterator; + +public class WatchFiles { + + public static void main(String[] args) throws IOException { + FilerGrpcClient filerGrpcClient = new FilerGrpcClient("localhost", 18888); + FilerClient filerClient = new FilerClient(filerGrpcClient); + + Iterator watch = filerClient.watch( + "/buckets", + "exampleClient", + System.currentTimeMillis() * 1000000L + ); + + while (watch.hasNext()) { + FilerProto.SubscribeMetadataResponse event = watch.next(); + FilerProto.EventNotification notification = event.getEventNotification(); + if (notification.getNewParentPath() != null) { + // move an entry to a new directory, possibly with a new name + if (notification.hasOldEntry() && notification.hasNewEntry()) { + System.out.println("move " + event.getDirectory() + "/" + notification.getOldEntry().getName() + " to " + notification.getNewParentPath() + "/" + notification.getNewEntry().getName()); + } else { + System.out.println("this should not happen."); + } + } else if (notification.hasNewEntry() && !notification.hasOldEntry()) { + System.out.println("create entry " + event.getDirectory() + "/" + notification.getNewEntry().getName()); + } else if (!notification.hasNewEntry() && notification.hasOldEntry()) { + System.out.println("delete entry " + event.getDirectory() + "/" + notification.getOldEntry().getName()); + } else if (notification.hasNewEntry() && notification.hasOldEntry()) { + System.out.println("updated entry " + event.getDirectory() + "/" + notification.getNewEntry().getName()); + } + } + + } +} From 824e96ffcc3903260479b26aa0f27e07fa23a795 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 13 Nov 2020 12:47:46 -0800 Subject: [PATCH 06/26] updates --- .../com/seaweedfs/examples/WatchFiles.java | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/other/java/examples/src/main/java/com/seaweedfs/examples/WatchFiles.java b/other/java/examples/src/main/java/com/seaweedfs/examples/WatchFiles.java index c4f4c81b0..e489cb3b1 100644 --- a/other/java/examples/src/main/java/com/seaweedfs/examples/WatchFiles.java +++ b/other/java/examples/src/main/java/com/seaweedfs/examples/WatchFiles.java @@ -1,38 +1,42 @@ package com.seaweedfs.examples; import seaweedfs.client.FilerClient; -import seaweedfs.client.FilerGrpcClient; import seaweedfs.client.FilerProto; import java.io.IOException; +import java.util.Date; import java.util.Iterator; public class WatchFiles { public static void main(String[] args) throws IOException { - FilerGrpcClient filerGrpcClient = new FilerGrpcClient("localhost", 18888); - FilerClient filerClient = new FilerClient(filerGrpcClient); + + FilerClient filerClient = new FilerClient("localhost", 18888); + + long sinceNs = (System.currentTimeMillis() - 3600 * 1000) * 1000000L; Iterator watch = filerClient.watch( "/buckets", - "exampleClient", - System.currentTimeMillis() * 1000000L + "exampleClientName", + sinceNs ); + System.out.println("Connected to filer, subscribing from " + new Date()); + while (watch.hasNext()) { FilerProto.SubscribeMetadataResponse event = watch.next(); FilerProto.EventNotification notification = event.getEventNotification(); - if (notification.getNewParentPath() != null) { + if (!event.getDirectory().equals(notification.getNewParentPath())) { // move an entry to a new directory, possibly with a new name if (notification.hasOldEntry() && notification.hasNewEntry()) { - System.out.println("move " + event.getDirectory() + "/" + notification.getOldEntry().getName() + " to " + notification.getNewParentPath() + "/" + notification.getNewEntry().getName()); + System.out.println("moved " + event.getDirectory() + "/" + notification.getOldEntry().getName() + " to " + notification.getNewParentPath() + "/" + notification.getNewEntry().getName()); } else { System.out.println("this should not happen."); } } else if (notification.hasNewEntry() && !notification.hasOldEntry()) { - System.out.println("create entry " + event.getDirectory() + "/" + notification.getNewEntry().getName()); + System.out.println("created entry " + event.getDirectory() + "/" + notification.getNewEntry().getName()); } else if (!notification.hasNewEntry() && notification.hasOldEntry()) { - System.out.println("delete entry " + event.getDirectory() + "/" + notification.getOldEntry().getName()); + System.out.println("deleted entry " + event.getDirectory() + "/" + notification.getOldEntry().getName()); } else if (notification.hasNewEntry() && notification.hasOldEntry()) { System.out.println("updated entry " + event.getDirectory() + "/" + notification.getNewEntry().getName()); } From b2a72630da29b66d056f1666da97db1a7c679425 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 13 Nov 2020 19:47:52 -0800 Subject: [PATCH 07/26] Update release.yml --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 47b1422f2..521418e2a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,7 +15,7 @@ jobs: uses: actions/checkout@v2 - name: Go Release Binaries - uses: wangyoucao577/go-release-action@feature/customize-target-release + uses: wangyoucao577/go-release-action@v1.10 with: github_token: ${{ secrets.GITHUB_TOKEN }} goos: linux # default is From 442e092995011653f2ed2f2b1b83bf4777a35c26 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 14 Nov 2020 14:26:08 -0800 Subject: [PATCH 08/26] filer watch "/etc" folder for configuration changes --- weed/filer/filer_on_meta_event.go | 21 +++++++++++++++++++++ weed/filer/meta_aggregator.go | 3 +++ 2 files changed, 24 insertions(+) create mode 100644 weed/filer/filer_on_meta_event.go diff --git a/weed/filer/filer_on_meta_event.go b/weed/filer/filer_on_meta_event.go new file mode 100644 index 000000000..70b62a974 --- /dev/null +++ b/weed/filer/filer_on_meta_event.go @@ -0,0 +1,21 @@ +package filer + +import ( + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" +) + +const ( + DirectoryEtc = "/etc" +) + +// onMetadataChangeEvent is triggered after filer processed change events from local or remote filers +func (f *Filer) onMetadataChangeEvent(event *filer_pb.SubscribeMetadataResponse) { + if DirectoryEtc != event.Directory { + return + } + + glog.V(0).Infof("procesing %v", event) + + +} diff --git a/weed/filer/meta_aggregator.go b/weed/filer/meta_aggregator.go index b90457339..9437e9992 100644 --- a/weed/filer/meta_aggregator.go +++ b/weed/filer/meta_aggregator.go @@ -141,6 +141,9 @@ func (ma *MetaAggregator) subscribeToOneFiler(f *Filer, self string, peer string return fmt.Errorf("process %v: %v", resp, err) } lastTsNs = resp.TsNs + + f.onMetadataChangeEvent(resp) + } }) if err != nil { From 57c770a7b4d67870b467ddba622a611615ad6f95 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 14 Nov 2020 20:39:12 -0800 Subject: [PATCH 09/26] go get github.com/chrislusf/raft@v1.0.2 --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 08e784510..264b04845 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/aws/aws-sdk-go v1.33.5 github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72 github.com/cespare/xxhash v1.1.0 - github.com/chrislusf/raft v1.0.2-0.20201002174524-b13c3bfdb011 + github.com/chrislusf/raft v1.0.2 github.com/coreos/go-semver v0.3.0 // indirect github.com/dgrijalva/jwt-go v3.2.0+incompatible github.com/disintegration/imaging v1.6.2 diff --git a/go.sum b/go.sum index 11bc7b7b4..a25e560cc 100644 --- a/go.sum +++ b/go.sum @@ -86,6 +86,8 @@ github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chrislusf/raft v1.0.2-0.20201002174524-b13c3bfdb011 h1:vN1GvfLgDg8kIPCdhuVKAjlYpxG1B86jiKejB6MC/Q0= github.com/chrislusf/raft v1.0.2-0.20201002174524-b13c3bfdb011/go.mod h1:Ep5DP+mJSosjfKiix1uU7Lc2Df/SX4oGJEpZlXH5l68= +github.com/chrislusf/raft v1.0.2 h1:3QOYYoZ8HFuJgwmhaIuPIJ1aFep4HoC3fMQYgnf2vms= +github.com/chrislusf/raft v1.0.2/go.mod h1:Ep5DP+mJSosjfKiix1uU7Lc2Df/SX4oGJEpZlXH5l68= github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= From 7cfa616843d88a5557d166cacfc64b9b3fce5e9c Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 14 Nov 2020 20:52:26 -0800 Subject: [PATCH 10/26] raft: grpc wait for ready related to https://github.com/chrislusf/seaweedfs/issues/1618 https://github.com/grpc/grpc/blob/master/doc/wait-for-ready.md https://github.com/chrislusf/raft/commit/7c64624a0692a6058b2482f4f2857a77274187f0 --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 264b04845..d4c1c5a5a 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/aws/aws-sdk-go v1.33.5 github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72 github.com/cespare/xxhash v1.1.0 - github.com/chrislusf/raft v1.0.2 + github.com/chrislusf/raft v1.0.3 github.com/coreos/go-semver v0.3.0 // indirect github.com/dgrijalva/jwt-go v3.2.0+incompatible github.com/disintegration/imaging v1.6.2 diff --git a/go.sum b/go.sum index a25e560cc..ee0b064c2 100644 --- a/go.sum +++ b/go.sum @@ -88,6 +88,8 @@ github.com/chrislusf/raft v1.0.2-0.20201002174524-b13c3bfdb011 h1:vN1GvfLgDg8kIP github.com/chrislusf/raft v1.0.2-0.20201002174524-b13c3bfdb011/go.mod h1:Ep5DP+mJSosjfKiix1uU7Lc2Df/SX4oGJEpZlXH5l68= github.com/chrislusf/raft v1.0.2 h1:3QOYYoZ8HFuJgwmhaIuPIJ1aFep4HoC3fMQYgnf2vms= github.com/chrislusf/raft v1.0.2/go.mod h1:Ep5DP+mJSosjfKiix1uU7Lc2Df/SX4oGJEpZlXH5l68= +github.com/chrislusf/raft v1.0.3 h1:11YrnzJtVa5z7m9lhY2p8VcPHoUlC1UswyoAo+U1m1k= +github.com/chrislusf/raft v1.0.3/go.mod h1:Ep5DP+mJSosjfKiix1uU7Lc2Df/SX4oGJEpZlXH5l68= github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= From 0fc9ffc603cd5ceeb7d268ff6bb559f2967573b2 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 14 Nov 2020 21:21:20 -0800 Subject: [PATCH 11/26] able to read filer.conf if updated --- weed/filer/filer_on_meta_event.go | 33 +++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/weed/filer/filer_on_meta_event.go b/weed/filer/filer_on_meta_event.go index 70b62a974..f0b64f4e2 100644 --- a/weed/filer/filer_on_meta_event.go +++ b/weed/filer/filer_on_meta_event.go @@ -1,6 +1,9 @@ package filer import ( + "bytes" + "math" + "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" ) @@ -12,10 +15,40 @@ const ( // onMetadataChangeEvent is triggered after filer processed change events from local or remote filers func (f *Filer) onMetadataChangeEvent(event *filer_pb.SubscribeMetadataResponse) { if DirectoryEtc != event.Directory { + if DirectoryEtc != event.EventNotification.NewParentPath { + return + } + } + + entry := event.EventNotification.NewEntry + if entry == nil { return } glog.V(0).Infof("procesing %v", event) + if entry.Name == "filer.conf" { + f.reloadFilerConfiguration(entry) + } + +} + +func (f *Filer) readEntry(entry *filer_pb.Entry) ([]byte, error){ + var buf bytes.Buffer + err := StreamContent(f.MasterClient, &buf, entry.Chunks, 0, math.MaxInt64) + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func (f *Filer) reloadFilerConfiguration(entry *filer_pb.Entry) { + data, err := f.readEntry(entry) + if err != nil { + glog.Warningf("read entry %s: %v", entry.Name, err) + return + + } + println(string(data)) } From 68043cfcacd4c35e86d063a08bf0de2cfbf5f7ad Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 14 Nov 2020 21:21:58 -0800 Subject: [PATCH 12/26] add reference implementation to detect create/update/delete/rename events --- weed/pb/filer_pb/filer_pb_helper.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/weed/pb/filer_pb/filer_pb_helper.go b/weed/pb/filer_pb/filer_pb_helper.go index bc0fac36c..75e37ca1b 100644 --- a/weed/pb/filer_pb/filer_pb_helper.go +++ b/weed/pb/filer_pb/filer_pb_helper.go @@ -121,3 +121,20 @@ func LookupEntry(client SeaweedFilerClient, request *LookupDirectoryEntryRequest } var ErrNotFound = errors.New("filer: no entry is found in filer store") + +func IsCreate(event *SubscribeMetadataResponse) bool { + return event.EventNotification.NewEntry != nil && event.EventNotification.OldEntry == nil +} +func IsUpdate(event *SubscribeMetadataResponse) bool { + return event.EventNotification.NewEntry != nil && + event.EventNotification.OldEntry != nil && + event.Directory == event.EventNotification.NewParentPath +} +func IsDelete(event *SubscribeMetadataResponse) bool { + return event.EventNotification.NewEntry == nil && event.EventNotification.OldEntry != nil +} +func IsRename(event *SubscribeMetadataResponse) bool { + return event.EventNotification.NewEntry != nil && + event.EventNotification.OldEntry != nil && + event.Directory != event.EventNotification.NewParentPath +} From 0a406f652ee1104b038e518145e5c3e0008ce4c4 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 15 Nov 2020 00:26:05 -0800 Subject: [PATCH 13/26] load filer conf and match by prefix --- go.mod | 5 +++ go.sum | 14 ++++++--- weed/filer/filer_conf.go | 48 +++++++++++++++++++++++++++++ weed/filer/filer_conf_test.go | 31 +++++++++++++++++++ weed/pb/filer_pb/filer_pb_helper.go | 6 ++++ 5 files changed, 100 insertions(+), 4 deletions(-) create mode 100644 weed/filer/filer_conf.go create mode 100644 weed/filer/filer_conf_test.go diff --git a/go.mod b/go.mod index d4c1c5a5a..51c24bcf0 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect github.com/fclairamb/ftpserverlib v0.8.0 github.com/frankban/quicktest v1.7.2 // indirect + github.com/go-errors/errors v1.1.1 // indirect github.com/go-redis/redis v6.15.7+incompatible github.com/go-sql-driver/mysql v1.5.0 github.com/gocql/gocql v0.0.0-20190829130954-e163eff7a8c6 @@ -45,6 +46,7 @@ require ( github.com/klauspost/reedsolomon v1.9.2 github.com/kurin/blazer v0.5.3 github.com/lib/pq v1.2.0 + github.com/lunixbochs/vtclean v1.0.0 // indirect github.com/magiconair/properties v1.8.1 // indirect github.com/mattn/go-ieproxy v0.0.0-20190805055040-f9202b1cfdeb // indirect github.com/mattn/go-runewidth v0.0.4 // indirect @@ -67,6 +69,9 @@ require ( github.com/tidwall/gjson v1.3.2 github.com/tidwall/match v1.0.1 github.com/valyala/bytebufferpool v1.0.0 + github.com/viant/assertly v0.5.4 // indirect + github.com/viant/ptrie v0.3.0 + github.com/viant/toolbox v0.33.2 // indirect github.com/willf/bitset v1.1.10 // indirect github.com/willf/bloom v2.0.3+incompatible github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 // indirect diff --git a/go.sum b/go.sum index ee0b064c2..51d96d997 100644 --- a/go.sum +++ b/go.sum @@ -84,10 +84,6 @@ github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chrislusf/raft v1.0.2-0.20201002174524-b13c3bfdb011 h1:vN1GvfLgDg8kIPCdhuVKAjlYpxG1B86jiKejB6MC/Q0= -github.com/chrislusf/raft v1.0.2-0.20201002174524-b13c3bfdb011/go.mod h1:Ep5DP+mJSosjfKiix1uU7Lc2Df/SX4oGJEpZlXH5l68= -github.com/chrislusf/raft v1.0.2 h1:3QOYYoZ8HFuJgwmhaIuPIJ1aFep4HoC3fMQYgnf2vms= -github.com/chrislusf/raft v1.0.2/go.mod h1:Ep5DP+mJSosjfKiix1uU7Lc2Df/SX4oGJEpZlXH5l68= github.com/chrislusf/raft v1.0.3 h1:11YrnzJtVa5z7m9lhY2p8VcPHoUlC1UswyoAo+U1m1k= github.com/chrislusf/raft v1.0.3/go.mod h1:Ep5DP+mJSosjfKiix1uU7Lc2Df/SX4oGJEpZlXH5l68= github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= @@ -163,6 +159,8 @@ github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-errors/errors v1.1.1 h1:ljK/pL5ltg3qoN+OtN6yCv9HWSfMwxSx90GJCZQxYNg= +github.com/go-errors/errors v1.1.1/go.mod h1:psDX2osz5VnTOnFWbDeWwS7yejl+uV3FEWEp4lssFEs= github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= @@ -390,6 +388,8 @@ github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= +github.com/lunixbochs/vtclean v1.0.0 h1:xu2sLAri4lGiovBDQKxl5mrXyESr3gUr5m5SM5+LVb8= +github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= @@ -625,6 +625,12 @@ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijb github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/viant/assertly v0.5.4 h1:5Hh4U3pLZa6uhCFAGpYOxck/8l9TZczEzoHNfJAhHEQ= +github.com/viant/assertly v0.5.4/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= +github.com/viant/ptrie v0.3.0 h1:SDaRd7Gqr1+ItCNz0GpTxRdK21nOfqjV6YtBm9jGlMY= +github.com/viant/ptrie v0.3.0/go.mod h1:VguMnbGfz95Zw+V5VarYSqtqslDxJbOv++xLzxkMhec= +github.com/viant/toolbox v0.33.2 h1:Av844IIeGz81gT672qZemyptGfbrcxqGymA5RFnIPjE= +github.com/viant/toolbox v0.33.2/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= github.com/willf/bitset v1.1.10 h1:NotGKqX0KwQ72NUzqrjZq5ipPNDQex9lo3WpaS8L2sc= github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/willf/bloom v2.0.3+incompatible h1:QDacWdqcAUI1MPOwIQZRy9kOR7yxfyEmxX8Wdm2/JPA= diff --git a/weed/filer/filer_conf.go b/weed/filer/filer_conf.go new file mode 100644 index 000000000..079794def --- /dev/null +++ b/weed/filer/filer_conf.go @@ -0,0 +1,48 @@ +package filer + +import ( + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/golang/protobuf/proto" + "github.com/viant/ptrie" +) + +type FilerConf struct { + rules ptrie.Trie +} + +func NewFilerConf(data []byte) (fc *FilerConf) { + fc = &FilerConf{ + rules: ptrie.New(), + } + + conf := &filer_pb.FilerConf{} + err := proto.UnmarshalText(string(data), conf) + if err != nil { + glog.Errorf("unable to parse filer conf: %v", err) + return + } + + fc.doLoadConf(conf) + return fc +} + +func (fc *FilerConf) doLoadConf(conf *filer_pb.FilerConf) { + for _, location := range conf.Locations { + err := fc.rules.Put([]byte(location.LocationPrefix), location) + if err != nil { + glog.Errorf("put location prefix: %v", err) + } + } +} + +func (fc *FilerConf) MatchStorageRule(path string) (pathConf *filer_pb.FilerConf_PathConf){ + fc.rules.MatchPrefix([]byte(path), func(key []byte, value interface{}) bool { + pathConf = value.(*filer_pb.FilerConf_PathConf) + return true + }) + if pathConf == nil { + return &filer_pb.FilerConf_PathConf{} + } + return pathConf +} diff --git a/weed/filer/filer_conf_test.go b/weed/filer/filer_conf_test.go new file mode 100644 index 000000000..6a7b875d3 --- /dev/null +++ b/weed/filer/filer_conf_test.go @@ -0,0 +1,31 @@ +package filer + +import ( + "testing" + + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/stretchr/testify/assert" + "github.com/viant/ptrie" +) + +func TestFilerConf(t *testing.T) { + + fc := &FilerConf{ + rules: ptrie.New(), + } + conf := &filer_pb.FilerConf{Locations: []*filer_pb.FilerConf_PathConf{ + { + LocationPrefix: "/buckets/abc", + Collection: "abc", + }, + { + LocationPrefix: "/buckets/abcd", + Collection: "abcd", + }, + }} + fc.doLoadConf(conf) + + assert.Equal(t, "abc", fc.MatchStorageRule("/buckets/abc/jasdf").Collection) + assert.Equal(t, "abcd", fc.MatchStorageRule("/buckets/abcd/jasdf").Collection) + +} diff --git a/weed/pb/filer_pb/filer_pb_helper.go b/weed/pb/filer_pb/filer_pb_helper.go index 75e37ca1b..fb989a02e 100644 --- a/weed/pb/filer_pb/filer_pb_helper.go +++ b/weed/pb/filer_pb/filer_pb_helper.go @@ -8,6 +8,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/storage/needle" + "github.com/viant/ptrie" ) func ToFileIdObject(fileIdStr string) (*FileId, error) { @@ -138,3 +139,8 @@ func IsRename(event *SubscribeMetadataResponse) bool { event.EventNotification.OldEntry != nil && event.Directory != event.EventNotification.NewParentPath } + +var _ = ptrie.KeyProvider(&FilerConf_PathConf{}) +func (fp *FilerConf_PathConf) Key() interface{} { + return fp.LocationPrefix +} From 590f02179d8837af4e81d36882395d7f728333eb Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 15 Nov 2020 14:06:03 -0800 Subject: [PATCH 14/26] filer: load filer conf when starting --- weed/filer/filer.go | 2 ++ weed/filer/filer_conf.go | 60 ++++++++++++++++++++++++++----- weed/filer/filer_conf_test.go | 6 ++-- weed/filer/filer_on_meta_event.go | 31 +++++++++------- weed/server/filer_server.go | 2 ++ 5 files changed, 77 insertions(+), 24 deletions(-) diff --git a/weed/filer/filer.go b/weed/filer/filer.go index 0c4d610c5..105c8e04f 100644 --- a/weed/filer/filer.go +++ b/weed/filer/filer.go @@ -41,6 +41,7 @@ type Filer struct { metaLogReplication string MetaAggregator *MetaAggregator Signature int32 + FilerConf *FilerConf } func NewFiler(masters []string, grpcDialOption grpc.DialOption, @@ -49,6 +50,7 @@ func NewFiler(masters []string, grpcDialOption grpc.DialOption, MasterClient: wdclient.NewMasterClient(grpcDialOption, "filer", filerHost, filerGrpcPort, dataCenter, masters), fileIdDeletionQueue: util.NewUnboundedQueue(), GrpcDialOption: grpcDialOption, + FilerConf: NewFilerConf(), } f.LocalMetaLogBuffer = log_buffer.NewLogBuffer(LogFlushInterval, f.logFlushFunc, notifyFn) f.metaLogCollection = collection diff --git a/weed/filer/filer_conf.go b/weed/filer/filer_conf.go index 079794def..737b1aeea 100644 --- a/weed/filer/filer_conf.go +++ b/weed/filer/filer_conf.go @@ -1,48 +1,92 @@ package filer import ( + "context" + "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/util" "github.com/golang/protobuf/proto" "github.com/viant/ptrie" ) + +const ( + DirectoryEtc = "/etc" + FilerConfName = "filer.conf" +) + type FilerConf struct { rules ptrie.Trie } -func NewFilerConf(data []byte) (fc *FilerConf) { +func NewFilerConf() (fc *FilerConf) { fc = &FilerConf{ rules: ptrie.New(), } + return fc +} + +func (fc *FilerConf) loadFromFiler(filer *Filer) (err error){ + filerConfPath := util.NewFullPath(DirectoryEtc, FilerConfName) + entry, err := filer.FindEntry(context.Background(), filerConfPath) + if err != nil { + if err == filer_pb.ErrNotFound { + return nil + } + glog.Errorf("read filer conf entry %s: %v", filerConfPath, err) + return + } + + return fc.loadFromChunks(filer, entry.Chunks) +} + +func (fc *FilerConf) loadFromChunks(filer *Filer, chunks []*filer_pb.FileChunk) (err error) { + data, err := filer.readEntry(chunks) + if err != nil { + glog.Errorf("read filer conf content: %v", err) + return + } + + return fc.loadFromBytes(data) +} + +func (fc *FilerConf) loadFromBytes(data []byte) (err error) { conf := &filer_pb.FilerConf{} - err := proto.UnmarshalText(string(data), conf) + err = proto.UnmarshalText(string(data), conf) if err != nil { glog.Errorf("unable to parse filer conf: %v", err) - return + // this is not recoverable + return nil } - fc.doLoadConf(conf) - return fc + return fc.doLoadConf(conf) } -func (fc *FilerConf) doLoadConf(conf *filer_pb.FilerConf) { +func (fc *FilerConf) doLoadConf(conf *filer_pb.FilerConf) (err error) { for _, location := range conf.Locations { - err := fc.rules.Put([]byte(location.LocationPrefix), location) + err = fc.rules.Put([]byte(location.LocationPrefix), location) if err != nil { glog.Errorf("put location prefix: %v", err) + // this is not recoverable + return nil } } + return nil } +var ( + EmptyFilerConfPathConf = &filer_pb.FilerConf_PathConf{} +) + func (fc *FilerConf) MatchStorageRule(path string) (pathConf *filer_pb.FilerConf_PathConf){ fc.rules.MatchPrefix([]byte(path), func(key []byte, value interface{}) bool { pathConf = value.(*filer_pb.FilerConf_PathConf) return true }) if pathConf == nil { - return &filer_pb.FilerConf_PathConf{} + return EmptyFilerConfPathConf } return pathConf } diff --git a/weed/filer/filer_conf_test.go b/weed/filer/filer_conf_test.go index 6a7b875d3..1bfe8bcfe 100644 --- a/weed/filer/filer_conf_test.go +++ b/weed/filer/filer_conf_test.go @@ -5,14 +5,12 @@ import ( "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/stretchr/testify/assert" - "github.com/viant/ptrie" ) func TestFilerConf(t *testing.T) { - fc := &FilerConf{ - rules: ptrie.New(), - } + fc := NewFilerConf() + conf := &filer_pb.FilerConf{Locations: []*filer_pb.FilerConf_PathConf{ { LocationPrefix: "/buckets/abc", diff --git a/weed/filer/filer_on_meta_event.go b/weed/filer/filer_on_meta_event.go index f0b64f4e2..3de27da6e 100644 --- a/weed/filer/filer_on_meta_event.go +++ b/weed/filer/filer_on_meta_event.go @@ -6,10 +6,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" -) - -const ( - DirectoryEtc = "/etc" + "github.com/chrislusf/seaweedfs/weed/util" ) // onMetadataChangeEvent is triggered after filer processed change events from local or remote filers @@ -26,15 +23,15 @@ func (f *Filer) onMetadataChangeEvent(event *filer_pb.SubscribeMetadataResponse) } glog.V(0).Infof("procesing %v", event) - if entry.Name == "filer.conf" { + if entry.Name == FilerConfName { f.reloadFilerConfiguration(entry) } } -func (f *Filer) readEntry(entry *filer_pb.Entry) ([]byte, error){ +func (f *Filer) readEntry(chunks []*filer_pb.FileChunk) ([]byte, error) { var buf bytes.Buffer - err := StreamContent(f.MasterClient, &buf, entry.Chunks, 0, math.MaxInt64) + err := StreamContent(f.MasterClient, &buf, chunks, 0, math.MaxInt64) if err != nil { return nil, err } @@ -42,13 +39,23 @@ func (f *Filer) readEntry(entry *filer_pb.Entry) ([]byte, error){ } func (f *Filer) reloadFilerConfiguration(entry *filer_pb.Entry) { - data, err := f.readEntry(entry) + fc := NewFilerConf() + err := fc.loadFromChunks(f, entry.Chunks) if err != nil { - glog.Warningf("read entry %s: %v", entry.Name, err) + glog.Errorf("read filer conf chunks: %v", err) return - } + f.FilerConf = fc +} - println(string(data)) - +func (f *Filer) LoadFilerConf() { + fc := NewFilerConf() + err := util.Retry("loadFilerConf", func() error { + return fc.loadFromFiler(f) + }) + if err != nil { + glog.Errorf("read filer conf: %v", err) + return + } + f.FilerConf = fc } diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index dc93ae062..ba2d9989c 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -131,6 +131,8 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption) fs.filer.LoadBuckets() + fs.filer.LoadFilerConf() + grace.OnInterrupt(func() { fs.filer.Shutdown() }) From 500bcab9535835404d29b8f68a03bf2ecf5b2991 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 15 Nov 2020 14:41:56 -0800 Subject: [PATCH 15/26] refactoring --- weed/filer/filer_conf.go | 1 - weed/filer/storage_option.go | 16 ++++++++ weed/server/filer_grpc_server.go | 21 ++++++++-- weed/server/filer_server_handlers_write.go | 33 ++++++++++----- .../filer_server_handlers_write_autochunk.go | 40 +++++++++---------- .../filer_server_handlers_write_cipher.go | 12 +++--- 6 files changed, 82 insertions(+), 41 deletions(-) create mode 100644 weed/filer/storage_option.go diff --git a/weed/filer/filer_conf.go b/weed/filer/filer_conf.go index 737b1aeea..0a86192ec 100644 --- a/weed/filer/filer_conf.go +++ b/weed/filer/filer_conf.go @@ -51,7 +51,6 @@ func (fc *FilerConf) loadFromChunks(filer *Filer, chunks []*filer_pb.FileChunk) return fc.loadFromBytes(data) } - func (fc *FilerConf) loadFromBytes(data []byte) (err error) { conf := &filer_pb.FilerConf{} err = proto.UnmarshalText(string(data), conf) diff --git a/weed/filer/storage_option.go b/weed/filer/storage_option.go new file mode 100644 index 000000000..ba9968fcc --- /dev/null +++ b/weed/filer/storage_option.go @@ -0,0 +1,16 @@ +package filer + +import "github.com/chrislusf/seaweedfs/weed/storage/needle" + +type StorageOption struct { + Replication string + Collection string + DataCenter string + Rack string + TtlSeconds int32 + Fsync bool +} + +func (so *StorageOption) TtlString() string { + return needle.SecondsToTTL(so.TtlSeconds) +} \ No newline at end of file diff --git a/weed/server/filer_grpc_server.go b/weed/server/filer_grpc_server.go index 8f326f5c7..e3f4bb161 100644 --- a/weed/server/filer_grpc_server.go +++ b/weed/server/filer_grpc_server.go @@ -257,7 +257,15 @@ func (fs *FilerServer) cleanupChunks(existingEntry *filer.Entry, newEntry *filer garbage = append(garbage, coveredChunks...) if newEntry.Attributes != nil { - chunks, err = filer.MaybeManifestize(fs.saveAsChunk(newEntry.Attributes.Replication, newEntry.Attributes.Collection, "", "", needle.SecondsToTTL(newEntry.Attributes.TtlSec), false), chunks) + so := &filer.StorageOption{ + Replication: newEntry.Attributes.Replication, + Collection: newEntry.Attributes.Collection, + DataCenter: "", + Rack: "", + TtlSeconds: newEntry.Attributes.TtlSec, + Fsync: false, + } + chunks, err = filer.MaybeManifestize(fs.saveAsChunk(so), chunks) if err != nil { // not good, but should be ok glog.V(0).Infof("MaybeManifestize: %v", err) @@ -297,8 +305,15 @@ func (fs *FilerServer) AppendToEntry(ctx context.Context, req *filer_pb.AppendTo } entry.Chunks = append(entry.Chunks, req.Chunks...) - - entry.Chunks, err = filer.MaybeManifestize(fs.saveAsChunk(entry.Replication, entry.Collection, "", "", needle.SecondsToTTL(entry.TtlSec), false), entry.Chunks) + so := &filer.StorageOption{ + Replication: entry.Replication, + Collection: entry.Collection, + DataCenter: "", + Rack: "", + TtlSeconds: entry.TtlSec, + Fsync: false, + } + entry.Chunks, err = filer.MaybeManifestize(fs.saveAsChunk(so), entry.Chunks) if err != nil { // not good, but should be ok glog.V(0).Infof("MaybeManifestize: %v", err) diff --git a/weed/server/filer_server_handlers_write.go b/weed/server/filer_server_handlers_write.go index 267b8752d..e27d9eb26 100644 --- a/weed/server/filer_server_handlers_write.go +++ b/weed/server/filer_server_handlers_write.go @@ -7,6 +7,7 @@ import ( "strings" "time" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/operation" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" @@ -29,7 +30,7 @@ type FilerPostResult struct { Url string `json:"url,omitempty"` } -func (fs *FilerServer) assignNewFileInfo(replication, collection, dataCenter, rack, ttlString string, fsync bool) (fileId, urlLocation string, auth security.EncodedJwt, err error) { +func (fs *FilerServer) assignNewFileInfo(so *filer.StorageOption) (fileId, urlLocation string, auth security.EncodedJwt, err error) { stats.FilerRequestCounter.WithLabelValues("assign").Inc() start := time.Now() @@ -37,18 +38,19 @@ func (fs *FilerServer) assignNewFileInfo(replication, collection, dataCenter, ra ar := &operation.VolumeAssignRequest{ Count: 1, - Replication: replication, - Collection: collection, - Ttl: ttlString, - DataCenter: dataCenter, + Replication: so.Replication, + Collection: so.Collection, + Ttl: so.TtlString(), + DataCenter: so.DataCenter, + Rack: so.Rack, } var altRequest *operation.VolumeAssignRequest - if dataCenter != "" || rack != "" { + if so.DataCenter != "" || so.Rack != "" { altRequest = &operation.VolumeAssignRequest{ Count: 1, - Replication: replication, - Collection: collection, - Ttl: ttlString, + Replication: so.Replication, + Collection: so.Collection, + Ttl: so.TtlString(), DataCenter: "", Rack: "", } @@ -62,7 +64,7 @@ func (fs *FilerServer) assignNewFileInfo(replication, collection, dataCenter, ra } fileId = assignResult.Fid urlLocation = "http://" + assignResult.Url + "/" + assignResult.Fid - if fsync { + if so.Fsync { urlLocation += "?fsync=true" } auth = assignResult.Auth @@ -92,7 +94,16 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) { ttlSeconds = int32(ttl.Minutes()) * 60 } - fs.autoChunk(ctx, w, r, replication, collection, dataCenter, rack, ttlSeconds, ttlString, fsync) + so := &filer.StorageOption{ + Replication: replication, + Collection: collection, + DataCenter: dataCenter, + Rack: rack, + TtlSeconds: ttlSeconds, + Fsync: fsync, + } + + fs.autoChunk(ctx, w, r, so) } diff --git a/weed/server/filer_server_handlers_write_autochunk.go b/weed/server/filer_server_handlers_write_autochunk.go index ee0af9aab..178145e69 100644 --- a/weed/server/filer_server_handlers_write_autochunk.go +++ b/weed/server/filer_server_handlers_write_autochunk.go @@ -25,7 +25,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/util" ) -func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, replication string, collection string, dataCenter string, rack string, ttlSec int32, ttlString string, fsync bool) { +func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, so *filer.StorageOption) { // autoChunking can be set at the command-line level or as a query param. Query param overrides command-line query := r.URL.Query() @@ -51,10 +51,10 @@ func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r * if r.Header.Get("Content-Type") == "" && strings.HasSuffix(r.URL.Path, "/") { reply, err = fs.mkdir(ctx, w, r) } else { - reply, md5bytes, err = fs.doPostAutoChunk(ctx, w, r, chunkSize, replication, collection, dataCenter, rack, ttlSec, ttlString, fsync) + reply, md5bytes, err = fs.doPostAutoChunk(ctx, w, r, chunkSize, so) } } else { - reply, md5bytes, err = fs.doPutAutoChunk(ctx, w, r, chunkSize, replication, collection, dataCenter, rack, ttlSec, ttlString, fsync) + reply, md5bytes, err = fs.doPutAutoChunk(ctx, w, r, chunkSize, so) } if err != nil { writeJsonError(w, r, http.StatusInternalServerError, err) @@ -66,7 +66,7 @@ func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r * } } -func (fs *FilerServer) doPostAutoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, chunkSize int32, replication string, collection string, dataCenter string, rack string, ttlSec int32, ttlString string, fsync bool) (filerResult *FilerPostResult, md5bytes []byte, replyerr error) { +func (fs *FilerServer) doPostAutoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, chunkSize int32, so *filer.StorageOption) (filerResult *FilerPostResult, md5bytes []byte, replyerr error) { multipartReader, multipartReaderErr := r.MultipartReader() if multipartReaderErr != nil { @@ -87,46 +87,46 @@ func (fs *FilerServer) doPostAutoChunk(ctx context.Context, w http.ResponseWrite contentType = "" } - fileChunks, md5Hash, chunkOffset, err := fs.uploadReaderToChunks(w, r, part1, chunkSize, replication, collection, dataCenter, rack, ttlString, fileName, contentType, fsync) + fileChunks, md5Hash, chunkOffset, err := fs.uploadReaderToChunks(w, r, part1, chunkSize, fileName, contentType, so) if err != nil { return nil, nil, err } - fileChunks, replyerr = filer.MaybeManifestize(fs.saveAsChunk(replication, collection, dataCenter, rack, ttlString, fsync), fileChunks) + fileChunks, replyerr = filer.MaybeManifestize(fs.saveAsChunk(so), fileChunks) if replyerr != nil { glog.V(0).Infof("manifestize %s: %v", r.RequestURI, replyerr) return } md5bytes = md5Hash.Sum(nil) - filerResult, replyerr = fs.saveMetaData(ctx, r, fileName, replication, collection, ttlSec, contentType, md5bytes, fileChunks, chunkOffset) + filerResult, replyerr = fs.saveMetaData(ctx, r, fileName, contentType, so, md5bytes, fileChunks, chunkOffset) return } -func (fs *FilerServer) doPutAutoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, chunkSize int32, replication string, collection string, dataCenter string, rack string, ttlSec int32, ttlString string, fsync bool) (filerResult *FilerPostResult, md5bytes []byte, replyerr error) { +func (fs *FilerServer) doPutAutoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, chunkSize int32, so *filer.StorageOption) (filerResult *FilerPostResult, md5bytes []byte, replyerr error) { fileName := "" contentType := "" - fileChunks, md5Hash, chunkOffset, err := fs.uploadReaderToChunks(w, r, r.Body, chunkSize, replication, collection, dataCenter, rack, ttlString, fileName, contentType, fsync) + fileChunks, md5Hash, chunkOffset, err := fs.uploadReaderToChunks(w, r, r.Body, chunkSize, fileName, contentType, so) if err != nil { return nil, nil, err } - fileChunks, replyerr = filer.MaybeManifestize(fs.saveAsChunk(replication, collection, dataCenter, rack, ttlString, fsync), fileChunks) + fileChunks, replyerr = filer.MaybeManifestize(fs.saveAsChunk(so), fileChunks) if replyerr != nil { glog.V(0).Infof("manifestize %s: %v", r.RequestURI, replyerr) return } md5bytes = md5Hash.Sum(nil) - filerResult, replyerr = fs.saveMetaData(ctx, r, fileName, replication, collection, ttlSec, contentType, md5bytes, fileChunks, chunkOffset) + filerResult, replyerr = fs.saveMetaData(ctx, r, fileName, contentType, so, md5bytes, fileChunks, chunkOffset) return } -func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileName string, replication string, collection string, ttlSec int32, contentType string, md5bytes []byte, fileChunks []*filer_pb.FileChunk, chunkOffset int64) (filerResult *FilerPostResult, replyerr error) { +func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileName string, contentType string, so *filer.StorageOption, md5bytes []byte, fileChunks []*filer_pb.FileChunk, chunkOffset int64) (filerResult *FilerPostResult, replyerr error) { // detect file mode modeStr := r.URL.Query().Get("mode") @@ -163,9 +163,9 @@ func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileNa Mode: os.FileMode(mode), Uid: OS_UID, Gid: OS_GID, - Replication: replication, - Collection: collection, - TtlSec: ttlSec, + Replication: so.Replication, + Collection: so.Collection, + TtlSec: so.TtlSeconds, Mime: contentType, Md5: md5bytes, FileSize: uint64(chunkOffset), @@ -199,7 +199,7 @@ func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileNa return filerResult, replyerr } -func (fs *FilerServer) uploadReaderToChunks(w http.ResponseWriter, r *http.Request, reader io.Reader, chunkSize int32, replication string, collection string, dataCenter string, rack string, ttlString string, fileName string, contentType string, fsync bool) ([]*filer_pb.FileChunk, hash.Hash, int64, error) { +func (fs *FilerServer) uploadReaderToChunks(w http.ResponseWriter, r *http.Request, reader io.Reader, chunkSize int32, fileName, contentType string, so *filer.StorageOption) ([]*filer_pb.FileChunk, hash.Hash, int64, error) { var fileChunks []*filer_pb.FileChunk md5Hash := md5.New() @@ -211,7 +211,7 @@ func (fs *FilerServer) uploadReaderToChunks(w http.ResponseWriter, r *http.Reque limitedReader := io.LimitReader(partReader, int64(chunkSize)) // assign one file id for one chunk - fileId, urlLocation, auth, assignErr := fs.assignNewFileInfo(replication, collection, dataCenter, rack, ttlString, fsync) + fileId, urlLocation, auth, assignErr := fs.assignNewFileInfo(so) if assignErr != nil { return nil, nil, 0, assignErr } @@ -255,11 +255,11 @@ func (fs *FilerServer) doUpload(urlLocation string, w http.ResponseWriter, r *ht return uploadResult, err } -func (fs *FilerServer) saveAsChunk(replication string, collection string, dataCenter string, rack string, ttlString string, fsync bool) filer.SaveDataAsChunkFunctionType { +func (fs *FilerServer) saveAsChunk(so *filer.StorageOption) filer.SaveDataAsChunkFunctionType { return func(reader io.Reader, name string, offset int64) (*filer_pb.FileChunk, string, string, error) { // assign one file id for one chunk - fileId, urlLocation, auth, assignErr := fs.assignNewFileInfo(replication, collection, dataCenter, rack, ttlString, fsync) + fileId, urlLocation, auth, assignErr := fs.assignNewFileInfo(so) if assignErr != nil { return nil, "", "", assignErr } @@ -270,7 +270,7 @@ func (fs *FilerServer) saveAsChunk(replication string, collection string, dataCe return nil, "", "", uploadErr } - return uploadResult.ToPbFileChunk(fileId, offset), collection, replication, nil + return uploadResult.ToPbFileChunk(fileId, offset), so.Collection, so.Replication, nil } } diff --git a/weed/server/filer_server_handlers_write_cipher.go b/weed/server/filer_server_handlers_write_cipher.go index 720d97027..fa6b0a44f 100644 --- a/weed/server/filer_server_handlers_write_cipher.go +++ b/weed/server/filer_server_handlers_write_cipher.go @@ -16,12 +16,12 @@ import ( ) // handling single chunk POST or PUT upload -func (fs *FilerServer) encrypt(ctx context.Context, w http.ResponseWriter, r *http.Request, replication string, collection string, dataCenter string, rack string, ttlSeconds int32, ttlString string, fsync bool) (filerResult *FilerPostResult, err error) { +func (fs *FilerServer) encrypt(ctx context.Context, w http.ResponseWriter, r *http.Request, so *filer.StorageOption) (filerResult *FilerPostResult, err error) { - fileId, urlLocation, auth, err := fs.assignNewFileInfo(replication, collection, dataCenter, rack, ttlString, fsync) + fileId, urlLocation, auth, err := fs.assignNewFileInfo(so) if err != nil || fileId == "" || urlLocation == "" { - return nil, fmt.Errorf("fail to allocate volume for %s, collection:%s, datacenter:%s", r.URL.Path, collection, dataCenter) + return nil, fmt.Errorf("fail to allocate volume for %s, collection:%s, datacenter:%s", r.URL.Path, so.Collection, so.DataCenter) } glog.V(4).Infof("write %s to %v", r.URL.Path, urlLocation) @@ -65,9 +65,9 @@ func (fs *FilerServer) encrypt(ctx context.Context, w http.ResponseWriter, r *ht Mode: 0660, Uid: OS_UID, Gid: OS_GID, - Replication: replication, - Collection: collection, - TtlSec: ttlSeconds, + Replication: so.Replication, + Collection: so.Collection, + TtlSec: so.TtlSeconds, Mime: pu.MimeType, Md5: util.Base64Md5ToBytes(pu.ContentMd5), }, From 95c0de285d907cbd826ba6ce97f7c4994c16ffd5 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 15 Nov 2020 16:58:48 -0800 Subject: [PATCH 16/26] refactoring --- other/java/client/src/main/proto/filer.proto | 1 + weed/filer/filer_notify_append.go | 10 +- weed/filer/storage_option.go | 16 -- weed/operation/assign_file_id.go | 37 +++ weed/pb/filer.proto | 1 + weed/pb/filer_pb/filer.pb.go | 235 +++++++++--------- weed/server/common.go | 1 + weed/server/filer_grpc_server.go | 71 ++---- weed/server/filer_server_handlers_write.go | 102 +++----- .../filer_server_handlers_write_autochunk.go | 12 +- .../filer_server_handlers_write_cipher.go | 2 +- weed/util/retry.go | 10 + 12 files changed, 242 insertions(+), 256 deletions(-) delete mode 100644 weed/filer/storage_option.go diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto index f75caec4e..4d9398897 100644 --- a/other/java/client/src/main/proto/filer.proto +++ b/other/java/client/src/main/proto/filer.proto @@ -362,6 +362,7 @@ message FilerConf { SSD = 1; } DiskType disk_type = 5; + bool fsync = 6; } repeated PathConf locations = 2; } diff --git a/weed/filer/filer_notify_append.go b/weed/filer/filer_notify_append.go index b1836b046..19da97f6c 100644 --- a/weed/filer/filer_notify_append.go +++ b/weed/filer/filer_notify_append.go @@ -13,7 +13,7 @@ import ( func (f *Filer) appendToFile(targetFile string, data []byte) error { - assignResult, uploadResult, err2 := f.assignAndUpload(data) + assignResult, uploadResult, err2 := f.assignAndUpload(targetFile, data) if err2 != nil { return err2 } @@ -46,14 +46,16 @@ func (f *Filer) appendToFile(targetFile string, data []byte) error { return err } -func (f *Filer) assignAndUpload(data []byte) (*operation.AssignResult, *operation.UploadResult, error) { +func (f *Filer) assignAndUpload(targetFile string, data []byte) (*operation.AssignResult, *operation.UploadResult, error) { // assign a volume location + rule := f.FilerConf.MatchStorageRule(targetFile) assignRequest := &operation.VolumeAssignRequest{ Count: 1, - Collection: f.metaLogCollection, - Replication: f.metaLogReplication, + Collection: util.Nvl(f.metaLogCollection, rule.Collection), + Replication: util.Nvl(f.metaLogReplication, rule.Replication), WritableVolumeCount: 1, } + assignResult, err := operation.Assign(f.GetMaster(), f.GrpcDialOption, assignRequest) if err != nil { return nil, nil, fmt.Errorf("AssignVolume: %v", err) diff --git a/weed/filer/storage_option.go b/weed/filer/storage_option.go deleted file mode 100644 index ba9968fcc..000000000 --- a/weed/filer/storage_option.go +++ /dev/null @@ -1,16 +0,0 @@ -package filer - -import "github.com/chrislusf/seaweedfs/weed/storage/needle" - -type StorageOption struct { - Replication string - Collection string - DataCenter string - Rack string - TtlSeconds int32 - Fsync bool -} - -func (so *StorageOption) TtlString() string { - return needle.SecondsToTTL(so.TtlSeconds) -} \ No newline at end of file diff --git a/weed/operation/assign_file_id.go b/weed/operation/assign_file_id.go index 893bf516c..c58d3a84d 100644 --- a/weed/operation/assign_file_id.go +++ b/weed/operation/assign_file_id.go @@ -5,6 +5,7 @@ import ( "fmt" "strings" + "github.com/chrislusf/seaweedfs/weed/storage/needle" "google.golang.org/grpc" "github.com/chrislusf/seaweedfs/weed/pb/master_pb" @@ -101,3 +102,39 @@ func LookupJwt(master string, fileId string) security.EncodedJwt { return security.EncodedJwt(tokenStr) } + +type StorageOption struct { + Replication string + Collection string + DataCenter string + Rack string + TtlSeconds int32 + Fsync bool +} + +func (so *StorageOption) TtlString() string { + return needle.SecondsToTTL(so.TtlSeconds) +} + +func (so *StorageOption) ToAssignRequests(count int) (ar *VolumeAssignRequest, altRequest *VolumeAssignRequest) { + ar = &VolumeAssignRequest{ + Count: uint64(count), + Replication: so.Replication, + Collection: so.Collection, + Ttl: so.TtlString(), + DataCenter: so.DataCenter, + Rack: so.Rack, + } + if so.DataCenter != "" || so.Rack != "" { + altRequest = &VolumeAssignRequest{ + Count: uint64(count), + Replication: so.Replication, + Collection: so.Collection, + Ttl: so.TtlString(), + DataCenter: "", + Rack: "", + } + } + return +} + diff --git a/weed/pb/filer.proto b/weed/pb/filer.proto index f75caec4e..4d9398897 100644 --- a/weed/pb/filer.proto +++ b/weed/pb/filer.proto @@ -362,6 +362,7 @@ message FilerConf { SSD = 1; } DiskType disk_type = 5; + bool fsync = 6; } repeated PathConf locations = 2; } diff --git a/weed/pb/filer_pb/filer.pb.go b/weed/pb/filer_pb/filer.pb.go index 42de8b6ef..9ae7e7ef2 100644 --- a/weed/pb/filer_pb/filer.pb.go +++ b/weed/pb/filer_pb/filer.pb.go @@ -3171,6 +3171,7 @@ type FilerConf_PathConf struct { Replication string `protobuf:"bytes,3,opt,name=replication,proto3" json:"replication,omitempty"` Ttl string `protobuf:"bytes,4,opt,name=ttl,proto3" json:"ttl,omitempty"` DiskType FilerConf_PathConf_DiskType `protobuf:"varint,5,opt,name=disk_type,json=diskType,proto3,enum=filer_pb.FilerConf_PathConf_DiskType" json:"disk_type,omitempty"` + Fsync bool `protobuf:"varint,6,opt,name=fsync,proto3" json:"fsync,omitempty"` } func (x *FilerConf_PathConf) Reset() { @@ -3240,6 +3241,13 @@ func (x *FilerConf_PathConf) GetDiskType() FilerConf_PathConf_DiskType { return FilerConf_PathConf_HDD } +func (x *FilerConf_PathConf) GetFsync() bool { + if x != nil { + return x.Fsync + } + return false +} + var File_filer_proto protoreflect.FileDescriptor var file_filer_proto_rawDesc = []byte{ @@ -3614,13 +3622,13 @@ var file_filer_proto_rawDesc = []byte{ 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x25, 0x0a, 0x0d, 0x4b, 0x76, 0x50, 0x75, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0xcd, 0x02, 0x0a, 0x09, 0x46, 0x69, + 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0xe3, 0x02, 0x0a, 0x09, 0x46, 0x69, 0x6c, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x3a, 0x0a, 0x09, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x2e, 0x50, 0x61, 0x74, 0x68, 0x43, 0x6f, - 0x6e, 0x66, 0x52, 0x09, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0xe9, 0x01, + 0x6e, 0x66, 0x52, 0x09, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0xff, 0x01, 0x0a, 0x08, 0x50, 0x61, 0x74, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x12, 0x27, 0x0a, 0x0f, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x65, @@ -3633,117 +3641,118 @@ var file_filer_proto_rawDesc = []byte{ 0x74, 0x79, 0x70, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x25, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x2e, 0x50, 0x61, 0x74, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x2e, 0x44, 0x69, 0x73, 0x6b, 0x54, 0x79, 0x70, - 0x65, 0x52, 0x08, 0x64, 0x69, 0x73, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x22, 0x1c, 0x0a, 0x08, 0x44, - 0x69, 0x73, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x12, 0x07, 0x0a, 0x03, 0x48, 0x44, 0x44, 0x10, 0x00, - 0x12, 0x07, 0x0a, 0x03, 0x53, 0x53, 0x44, 0x10, 0x01, 0x32, 0xdc, 0x0c, 0x0a, 0x0c, 0x53, 0x65, - 0x61, 0x77, 0x65, 0x65, 0x64, 0x46, 0x69, 0x6c, 0x65, 0x72, 0x12, 0x67, 0x0a, 0x14, 0x4c, 0x6f, - 0x6f, 0x6b, 0x75, 0x70, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x12, 0x25, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, - 0x6f, 0x6b, 0x75, 0x70, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x66, 0x69, 0x6c, 0x65, - 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x44, 0x69, 0x72, 0x65, 0x63, - 0x74, 0x6f, 0x72, 0x79, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x22, 0x00, 0x12, 0x4e, 0x0a, 0x0b, 0x4c, 0x69, 0x73, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x69, - 0x65, 0x73, 0x12, 0x1c, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x69, - 0x73, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x1d, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x69, 0x73, 0x74, - 0x45, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, - 0x00, 0x30, 0x01, 0x12, 0x4c, 0x0a, 0x0b, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x12, 0x1c, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x72, - 0x65, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x1d, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x72, 0x65, 0x61, - 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, - 0x00, 0x12, 0x4c, 0x0a, 0x0b, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, - 0x12, 0x1c, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x55, 0x70, 0x64, 0x61, - 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, - 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, - 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, - 0x52, 0x0a, 0x0d, 0x41, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x54, 0x6f, 0x45, 0x6e, 0x74, 0x72, 0x79, - 0x12, 0x1e, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x70, 0x70, 0x65, - 0x6e, 0x64, 0x54, 0x6f, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x1f, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x70, 0x70, 0x65, - 0x6e, 0x64, 0x54, 0x6f, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x22, 0x00, 0x12, 0x4c, 0x0a, 0x0b, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x12, 0x1c, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x44, 0x65, - 0x6c, 0x65, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x1d, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x44, 0x65, 0x6c, 0x65, - 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, - 0x00, 0x12, 0x5e, 0x0a, 0x11, 0x41, 0x74, 0x6f, 0x6d, 0x69, 0x63, 0x52, 0x65, 0x6e, 0x61, 0x6d, - 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x22, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, - 0x62, 0x2e, 0x41, 0x74, 0x6f, 0x6d, 0x69, 0x63, 0x52, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x45, 0x6e, - 0x74, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x66, 0x69, 0x6c, - 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x74, 0x6f, 0x6d, 0x69, 0x63, 0x52, 0x65, 0x6e, 0x61, - 0x6d, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, - 0x00, 0x12, 0x4f, 0x0a, 0x0c, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, - 0x65, 0x12, 0x1d, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x73, 0x73, - 0x69, 0x67, 0x6e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x1e, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x73, 0x73, 0x69, - 0x67, 0x6e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x22, 0x00, 0x12, 0x4f, 0x0a, 0x0c, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75, - 0x6d, 0x65, 0x12, 0x1d, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, - 0x6f, 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x1e, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f, - 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x22, 0x00, 0x12, 0x55, 0x0a, 0x0e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x1f, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, - 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, - 0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5b, 0x0a, 0x10, 0x44, 0x65, - 0x6c, 0x65, 0x74, 0x65, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x21, - 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, - 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x22, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x44, 0x65, 0x6c, - 0x65, 0x74, 0x65, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x49, 0x0a, 0x0a, 0x53, 0x74, 0x61, 0x74, 0x69, - 0x73, 0x74, 0x69, 0x63, 0x73, 0x12, 0x1b, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, - 0x2e, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x74, - 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x22, 0x00, 0x12, 0x6a, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x72, 0x43, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x26, 0x2e, 0x66, 0x69, - 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x72, 0x43, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x47, - 0x65, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x60, - 0x0a, 0x11, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x64, - 0x61, 0x74, 0x61, 0x12, 0x22, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, - 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, - 0x70, 0x62, 0x2e, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x4d, 0x65, 0x74, 0x61, - 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, - 0x12, 0x65, 0x0a, 0x16, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x4c, 0x6f, 0x63, - 0x61, 0x6c, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x22, 0x2e, 0x66, 0x69, 0x6c, - 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x4d, - 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, - 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, - 0x69, 0x62, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x56, 0x0a, 0x0d, 0x4b, 0x65, 0x65, 0x70, 0x43, - 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x12, 0x1e, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, - 0x5f, 0x70, 0x62, 0x2e, 0x4b, 0x65, 0x65, 0x70, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, - 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, - 0x5f, 0x70, 0x62, 0x2e, 0x4b, 0x65, 0x65, 0x70, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, - 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, 0x12, - 0x4f, 0x0a, 0x0c, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x42, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x12, - 0x1d, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x74, - 0x65, 0x42, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, - 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x65, - 0x42, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, - 0x12, 0x3a, 0x0a, 0x05, 0x4b, 0x76, 0x47, 0x65, 0x74, 0x12, 0x16, 0x2e, 0x66, 0x69, 0x6c, 0x65, - 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4b, 0x76, 0x47, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x17, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4b, 0x76, 0x47, - 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x3a, 0x0a, 0x05, - 0x4b, 0x76, 0x50, 0x75, 0x74, 0x12, 0x16, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, - 0x2e, 0x4b, 0x76, 0x50, 0x75, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, - 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4b, 0x76, 0x50, 0x75, 0x74, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x4f, 0x0a, 0x10, 0x73, 0x65, 0x61, 0x77, - 0x65, 0x65, 0x64, 0x66, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x42, 0x0a, 0x46, 0x69, - 0x6c, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, - 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x68, 0x72, 0x69, 0x73, 0x6c, 0x75, 0x73, 0x66, 0x2f, 0x73, - 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x70, 0x62, - 0x2f, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x33, + 0x65, 0x52, 0x08, 0x64, 0x69, 0x73, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x66, + 0x73, 0x79, 0x6e, 0x63, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x66, 0x73, 0x79, 0x6e, + 0x63, 0x22, 0x1c, 0x0a, 0x08, 0x44, 0x69, 0x73, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x12, 0x07, 0x0a, + 0x03, 0x48, 0x44, 0x44, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x53, 0x53, 0x44, 0x10, 0x01, 0x32, + 0xdc, 0x0c, 0x0a, 0x0c, 0x53, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x46, 0x69, 0x6c, 0x65, 0x72, + 0x12, 0x67, 0x0a, 0x14, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, + 0x6f, 0x72, 0x79, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x25, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, + 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, + 0x6f, 0x72, 0x79, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x26, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, + 0x70, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4e, 0x0a, 0x0b, 0x4c, 0x69, 0x73, + 0x74, 0x45, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x12, 0x1c, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, + 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, + 0x62, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x4c, 0x0a, 0x0b, 0x43, 0x72, 0x65, + 0x61, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x1c, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, + 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, + 0x62, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4c, 0x0a, 0x0b, 0x55, 0x70, 0x64, 0x61, 0x74, + 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x1c, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, + 0x62, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, + 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x52, 0x0a, 0x0d, 0x41, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x54, + 0x6f, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x1e, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, + 0x62, 0x2e, 0x41, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x54, 0x6f, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, + 0x62, 0x2e, 0x41, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x54, 0x6f, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4c, 0x0a, 0x0b, 0x44, 0x65, 0x6c, + 0x65, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x1c, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, + 0x5f, 0x70, 0x62, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, + 0x62, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5e, 0x0a, 0x11, 0x41, 0x74, 0x6f, 0x6d, 0x69, + 0x63, 0x52, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x22, 0x2e, 0x66, + 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x74, 0x6f, 0x6d, 0x69, 0x63, 0x52, 0x65, + 0x6e, 0x61, 0x6d, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x23, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x74, 0x6f, 0x6d, + 0x69, 0x63, 0x52, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4f, 0x0a, 0x0c, 0x41, 0x73, 0x73, 0x69, 0x67, + 0x6e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x12, 0x1d, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, + 0x70, 0x62, 0x2e, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, + 0x62, 0x2e, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4f, 0x0a, 0x0c, 0x4c, 0x6f, 0x6f, 0x6b, + 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x12, 0x1d, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, + 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, + 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x55, 0x0a, 0x0e, 0x43, 0x6f, 0x6c, + 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x1f, 0x2e, 0x66, 0x69, + 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x66, + 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, + 0x12, 0x5b, 0x0a, 0x10, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x21, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, + 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, + 0x70, 0x62, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x49, 0x0a, + 0x0a, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x12, 0x1b, 0x2e, 0x66, 0x69, + 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, + 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, + 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x6a, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x46, + 0x69, 0x6c, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x12, 0x26, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, + 0x46, 0x69, 0x6c, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x66, 0x69, 0x6c, 0x65, + 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x72, 0x43, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x22, 0x00, 0x12, 0x60, 0x0a, 0x11, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, + 0x65, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x22, 0x2e, 0x66, 0x69, 0x6c, 0x65, + 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x4d, 0x65, + 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, + 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, + 0x62, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x65, 0x0a, 0x16, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, + 0x69, 0x62, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, + 0x12, 0x22, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x75, 0x62, 0x73, + 0x63, 0x72, 0x69, 0x62, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, + 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, + 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x56, 0x0a, + 0x0d, 0x4b, 0x65, 0x65, 0x70, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x12, 0x1e, + 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4b, 0x65, 0x65, 0x70, 0x43, 0x6f, + 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, + 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4b, 0x65, 0x65, 0x70, 0x43, 0x6f, + 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x00, 0x28, 0x01, 0x30, 0x01, 0x12, 0x4f, 0x0a, 0x0c, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x42, + 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x12, 0x1d, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, + 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x42, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, + 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x42, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x3a, 0x0a, 0x05, 0x4b, 0x76, 0x47, 0x65, 0x74, 0x12, + 0x16, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4b, 0x76, 0x47, 0x65, 0x74, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, + 0x70, 0x62, 0x2e, 0x4b, 0x76, 0x47, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x00, 0x12, 0x3a, 0x0a, 0x05, 0x4b, 0x76, 0x50, 0x75, 0x74, 0x12, 0x16, 0x2e, 0x66, 0x69, + 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4b, 0x76, 0x50, 0x75, 0x74, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4b, + 0x76, 0x50, 0x75, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x4f, + 0x0a, 0x10, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, + 0x6e, 0x74, 0x42, 0x0a, 0x46, 0x69, 0x6c, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x2f, + 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x68, 0x72, 0x69, 0x73, + 0x6c, 0x75, 0x73, 0x66, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, + 0x65, 0x65, 0x64, 0x2f, 0x70, 0x62, 0x2f, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x62, + 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/weed/server/common.go b/weed/server/common.go index 62ddf6e7f..58079032e 100644 --- a/weed/server/common.go +++ b/weed/server/common.go @@ -127,6 +127,7 @@ func submitForClientHandler(w http.ResponseWriter, r *http.Request, masterUrl st ar := &operation.VolumeAssignRequest{ Count: count, DataCenter: r.FormValue("dataCenter"), + Rack: r.FormValue("rack"), Replication: r.FormValue("replication"), Collection: r.FormValue("collection"), Ttl: r.FormValue("ttl"), diff --git a/weed/server/filer_grpc_server.go b/weed/server/filer_grpc_server.go index e3f4bb161..7b04e4fab 100644 --- a/weed/server/filer_grpc_server.go +++ b/weed/server/filer_grpc_server.go @@ -156,7 +156,7 @@ func (fs *FilerServer) CreateEntry(ctx context.Context, req *filer_pb.CreateEntr resp = &filer_pb.CreateEntryResponse{} - chunks, garbage, err2 := fs.cleanupChunks(nil, req.Entry) + chunks, garbage, err2 := fs.cleanupChunks(util.Join(req.Directory, req.Entry.Name), nil, req.Entry) if err2 != nil { return &filer_pb.CreateEntryResponse{}, fmt.Errorf("CreateEntry cleanupChunks %s %s: %v", req.Directory, req.Entry.Name, err2) } @@ -190,7 +190,7 @@ func (fs *FilerServer) UpdateEntry(ctx context.Context, req *filer_pb.UpdateEntr return &filer_pb.UpdateEntryResponse{}, fmt.Errorf("not found %s: %v", fullpath, err) } - chunks, garbage, err2 := fs.cleanupChunks(entry, req.Entry) + chunks, garbage, err2 := fs.cleanupChunks(fullpath, entry, req.Entry) if err2 != nil { return &filer_pb.UpdateEntryResponse{}, fmt.Errorf("UpdateEntry cleanupChunks %s: %v", fullpath, err2) } @@ -240,7 +240,7 @@ func (fs *FilerServer) UpdateEntry(ctx context.Context, req *filer_pb.UpdateEntr return &filer_pb.UpdateEntryResponse{}, err } -func (fs *FilerServer) cleanupChunks(existingEntry *filer.Entry, newEntry *filer_pb.Entry) (chunks, garbage []*filer_pb.FileChunk, err error) { +func (fs *FilerServer) cleanupChunks(fullpath string, existingEntry *filer.Entry, newEntry *filer_pb.Entry) (chunks, garbage []*filer_pb.FileChunk, err error) { // remove old chunks if not included in the new ones if existingEntry != nil { @@ -257,14 +257,13 @@ func (fs *FilerServer) cleanupChunks(existingEntry *filer.Entry, newEntry *filer garbage = append(garbage, coveredChunks...) if newEntry.Attributes != nil { - so := &filer.StorageOption{ - Replication: newEntry.Attributes.Replication, - Collection: newEntry.Attributes.Collection, - DataCenter: "", - Rack: "", - TtlSeconds: newEntry.Attributes.TtlSec, - Fsync: false, - } + so := fs.detectStorageOption(fullpath, + newEntry.Attributes.Collection, + newEntry.Attributes.Replication, + newEntry.Attributes.TtlSec, + "", + "", + ) chunks, err = filer.MaybeManifestize(fs.saveAsChunk(so), chunks) if err != nil { // not good, but should be ok @@ -283,7 +282,7 @@ func (fs *FilerServer) AppendToEntry(ctx context.Context, req *filer_pb.AppendTo fullpath := util.NewFullPath(req.Directory, req.EntryName) var offset int64 = 0 - entry, err := fs.filer.FindEntry(ctx, util.FullPath(fullpath)) + entry, err := fs.filer.FindEntry(ctx, fullpath) if err == filer_pb.ErrNotFound { entry = &filer.Entry{ FullPath: fullpath, @@ -305,14 +304,7 @@ func (fs *FilerServer) AppendToEntry(ctx context.Context, req *filer_pb.AppendTo } entry.Chunks = append(entry.Chunks, req.Chunks...) - so := &filer.StorageOption{ - Replication: entry.Replication, - Collection: entry.Collection, - DataCenter: "", - Rack: "", - TtlSeconds: entry.TtlSec, - Fsync: false, - } + so := fs.detectStorageOption(string(fullpath), entry.Collection, entry.Replication, entry.TtlSec, "", "") entry.Chunks, err = filer.MaybeManifestize(fs.saveAsChunk(so), entry.Chunks) if err != nil { // not good, but should be ok @@ -338,41 +330,10 @@ func (fs *FilerServer) DeleteEntry(ctx context.Context, req *filer_pb.DeleteEntr func (fs *FilerServer) AssignVolume(ctx context.Context, req *filer_pb.AssignVolumeRequest) (resp *filer_pb.AssignVolumeResponse, err error) { - ttlStr := "" - if req.TtlSec > 0 { - ttlStr = strconv.Itoa(int(req.TtlSec)) - } - collection, replication, _ := fs.detectCollection(req.Path, req.Collection, req.Replication) - - var altRequest *operation.VolumeAssignRequest + so := fs.detectStorageOption(req.Path, req.Collection, req.Replication, req.TtlSec, req.DataCenter, req.Rack) - dataCenter := req.DataCenter - if dataCenter == "" { - dataCenter = fs.option.DataCenter - } - rack := req.Rack - if rack == "" { - rack = fs.option.Rack - } + assignRequest, altRequest := so.ToAssignRequests(int(req.Count)) - assignRequest := &operation.VolumeAssignRequest{ - Count: uint64(req.Count), - Replication: replication, - Collection: collection, - Ttl: ttlStr, - DataCenter: dataCenter, - Rack: rack, - } - if dataCenter != "" || rack != "" { - altRequest = &operation.VolumeAssignRequest{ - Count: uint64(req.Count), - Replication: replication, - Collection: collection, - Ttl: ttlStr, - DataCenter: "", - Rack: "", - } - } assignResult, err := operation.Assign(fs.filer.GetMaster(), fs.grpcDialOption, assignRequest, altRequest) if err != nil { glog.V(3).Infof("AssignVolume: %v", err) @@ -389,8 +350,8 @@ func (fs *FilerServer) AssignVolume(ctx context.Context, req *filer_pb.AssignVol Url: assignResult.Url, PublicUrl: assignResult.PublicUrl, Auth: string(assignResult.Auth), - Collection: collection, - Replication: replication, + Collection: so.Collection, + Replication: so.Replication, }, nil } diff --git a/weed/server/filer_server_handlers_write.go b/weed/server/filer_server_handlers_write.go index e27d9eb26..0344d84dc 100644 --- a/weed/server/filer_server_handlers_write.go +++ b/weed/server/filer_server_handlers_write.go @@ -7,7 +7,6 @@ import ( "strings" "time" - "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/operation" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" @@ -30,31 +29,13 @@ type FilerPostResult struct { Url string `json:"url,omitempty"` } -func (fs *FilerServer) assignNewFileInfo(so *filer.StorageOption) (fileId, urlLocation string, auth security.EncodedJwt, err error) { +func (fs *FilerServer) assignNewFileInfo(so *operation.StorageOption) (fileId, urlLocation string, auth security.EncodedJwt, err error) { stats.FilerRequestCounter.WithLabelValues("assign").Inc() start := time.Now() defer func() { stats.FilerRequestHistogram.WithLabelValues("assign").Observe(time.Since(start).Seconds()) }() - ar := &operation.VolumeAssignRequest{ - Count: 1, - Replication: so.Replication, - Collection: so.Collection, - Ttl: so.TtlString(), - DataCenter: so.DataCenter, - Rack: so.Rack, - } - var altRequest *operation.VolumeAssignRequest - if so.DataCenter != "" || so.Rack != "" { - altRequest = &operation.VolumeAssignRequest{ - Count: 1, - Replication: so.Replication, - Collection: so.Collection, - Ttl: so.TtlString(), - DataCenter: "", - Rack: "", - } - } + ar, altRequest := so.ToAssignRequests(1) assignResult, ae := operation.Assign(fs.filer.GetMaster(), fs.grpcDialOption, ar, altRequest) if ae != nil { @@ -76,32 +57,13 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) { ctx := context.Background() query := r.URL.Query() - collection, replication, fsync := fs.detectCollection(r.RequestURI, query.Get("collection"), query.Get("replication")) - dataCenter := query.Get("dataCenter") - if dataCenter == "" { - dataCenter = fs.option.DataCenter - } - rack := query.Get("rack") - if dataCenter == "" { - rack = fs.option.Rack - } - ttlString := r.URL.Query().Get("ttl") - - // read ttl in seconds - ttl, err := needle.ReadTTL(ttlString) - ttlSeconds := int32(0) - if err == nil { - ttlSeconds = int32(ttl.Minutes()) * 60 - } - - so := &filer.StorageOption{ - Replication: replication, - Collection: collection, - DataCenter: dataCenter, - Rack: rack, - TtlSeconds: ttlSeconds, - Fsync: fsync, - } + so := fs.detectStorageOption0(r.RequestURI, + query.Get("collection"), + query.Get("replication"), + query.Get("ttl"), + query.Get("dataCenter"), + query.Get("rack"), + ) fs.autoChunk(ctx, w, r, so) @@ -141,21 +103,12 @@ func (fs *FilerServer) DeleteHandler(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNoContent) } -func (fs *FilerServer) detectCollection(requestURI, qCollection, qReplication string) (collection, replication string, fsync bool) { - // default - collection = fs.option.Collection - replication = fs.option.DefaultReplication - - // get default collection settings - if qCollection != "" { - collection = qCollection - } - if qReplication != "" { - replication = qReplication - } +func (fs *FilerServer) detectStorageOption(requestURI, qCollection, qReplication string, ttlSeconds int32, dataCenter, rack string) (*operation.StorageOption) { + collection := util.Nvl(qCollection, fs.option.Collection) + replication := util.Nvl(qReplication, fs.option.DefaultReplication) // required by buckets folder - bucketDefaultReplication := "" + bucketDefaultReplication, fsync := "", false if strings.HasPrefix(requestURI, fs.filer.DirBucketsPath+"/") { bucketAndObjectKey := requestURI[len(fs.filer.DirBucketsPath)+1:] t := strings.Index(bucketAndObjectKey, "/") @@ -171,5 +124,32 @@ func (fs *FilerServer) detectCollection(requestURI, qCollection, qReplication st replication = bucketDefaultReplication } - return + rule := fs.filer.FilerConf.MatchStorageRule(requestURI) + + if ttlSeconds == 0 { + ttl, err := needle.ReadTTL(rule.GetTtl()) + if err != nil { + glog.Errorf("fail to parse %s ttl setting %s: %v", rule.LocationPrefix, rule.Ttl, err) + } + ttlSeconds = int32(ttl.Minutes())*60 + } + + return &operation.StorageOption{ + Replication: util.Nvl(replication, rule.Replication), + Collection: util.Nvl(collection, rule.Collection), + DataCenter: util.Nvl(dataCenter, fs.option.DataCenter), + Rack: util.Nvl(rack, fs.option.Rack), + TtlSeconds: ttlSeconds, + Fsync: fsync || rule.Fsync, + } +} + +func (fs *FilerServer) detectStorageOption0(requestURI, qCollection, qReplication string, qTtl string, dataCenter, rack string) (*operation.StorageOption) { + + ttl, err := needle.ReadTTL(qTtl) + if err != nil { + glog.Errorf("fail to parse ttl %s: %v", qTtl, err) + } + + return fs.detectStorageOption(requestURI, qCollection, qReplication, int32(ttl.Minutes())*60, dataCenter, rack) } diff --git a/weed/server/filer_server_handlers_write_autochunk.go b/weed/server/filer_server_handlers_write_autochunk.go index 178145e69..fd2db884f 100644 --- a/weed/server/filer_server_handlers_write_autochunk.go +++ b/weed/server/filer_server_handlers_write_autochunk.go @@ -25,7 +25,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/util" ) -func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, so *filer.StorageOption) { +func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, so *operation.StorageOption) { // autoChunking can be set at the command-line level or as a query param. Query param overrides command-line query := r.URL.Query() @@ -66,7 +66,7 @@ func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r * } } -func (fs *FilerServer) doPostAutoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, chunkSize int32, so *filer.StorageOption) (filerResult *FilerPostResult, md5bytes []byte, replyerr error) { +func (fs *FilerServer) doPostAutoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, chunkSize int32, so *operation.StorageOption) (filerResult *FilerPostResult, md5bytes []byte, replyerr error) { multipartReader, multipartReaderErr := r.MultipartReader() if multipartReaderErr != nil { @@ -104,7 +104,7 @@ func (fs *FilerServer) doPostAutoChunk(ctx context.Context, w http.ResponseWrite return } -func (fs *FilerServer) doPutAutoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, chunkSize int32, so *filer.StorageOption) (filerResult *FilerPostResult, md5bytes []byte, replyerr error) { +func (fs *FilerServer) doPutAutoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, chunkSize int32, so *operation.StorageOption) (filerResult *FilerPostResult, md5bytes []byte, replyerr error) { fileName := "" contentType := "" @@ -126,7 +126,7 @@ func (fs *FilerServer) doPutAutoChunk(ctx context.Context, w http.ResponseWriter return } -func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileName string, contentType string, so *filer.StorageOption, md5bytes []byte, fileChunks []*filer_pb.FileChunk, chunkOffset int64) (filerResult *FilerPostResult, replyerr error) { +func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileName string, contentType string, so *operation.StorageOption, md5bytes []byte, fileChunks []*filer_pb.FileChunk, chunkOffset int64) (filerResult *FilerPostResult, replyerr error) { // detect file mode modeStr := r.URL.Query().Get("mode") @@ -199,7 +199,7 @@ func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileNa return filerResult, replyerr } -func (fs *FilerServer) uploadReaderToChunks(w http.ResponseWriter, r *http.Request, reader io.Reader, chunkSize int32, fileName, contentType string, so *filer.StorageOption) ([]*filer_pb.FileChunk, hash.Hash, int64, error) { +func (fs *FilerServer) uploadReaderToChunks(w http.ResponseWriter, r *http.Request, reader io.Reader, chunkSize int32, fileName, contentType string, so *operation.StorageOption) ([]*filer_pb.FileChunk, hash.Hash, int64, error) { var fileChunks []*filer_pb.FileChunk md5Hash := md5.New() @@ -255,7 +255,7 @@ func (fs *FilerServer) doUpload(urlLocation string, w http.ResponseWriter, r *ht return uploadResult, err } -func (fs *FilerServer) saveAsChunk(so *filer.StorageOption) filer.SaveDataAsChunkFunctionType { +func (fs *FilerServer) saveAsChunk(so *operation.StorageOption) filer.SaveDataAsChunkFunctionType { return func(reader io.Reader, name string, offset int64) (*filer_pb.FileChunk, string, string, error) { // assign one file id for one chunk diff --git a/weed/server/filer_server_handlers_write_cipher.go b/weed/server/filer_server_handlers_write_cipher.go index fa6b0a44f..3cc0d0c41 100644 --- a/weed/server/filer_server_handlers_write_cipher.go +++ b/weed/server/filer_server_handlers_write_cipher.go @@ -16,7 +16,7 @@ import ( ) // handling single chunk POST or PUT upload -func (fs *FilerServer) encrypt(ctx context.Context, w http.ResponseWriter, r *http.Request, so *filer.StorageOption) (filerResult *FilerPostResult, err error) { +func (fs *FilerServer) encrypt(ctx context.Context, w http.ResponseWriter, r *http.Request, so *operation.StorageOption) (filerResult *FilerPostResult, err error) { fileId, urlLocation, auth, err := fs.assignNewFileInfo(so) diff --git a/weed/util/retry.go b/weed/util/retry.go index faaab0351..85c4d150d 100644 --- a/weed/util/retry.go +++ b/weed/util/retry.go @@ -29,3 +29,13 @@ func Retry(name string, job func() error) (err error) { } return err } + +// return the first non empty string +func Nvl(values ...string) string { + for _, s := range values { + if s != "" { + return s + } + } + return "" +} From 0ea5c087ce8414e2fc6a0d470d316bac7fd1167b Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 15 Nov 2020 16:59:28 -0800 Subject: [PATCH 17/26] go fmt --- weed/filer/filer_conf.go | 7 +++---- weed/filesys/file.go | 2 +- weed/operation/assign_file_id.go | 1 - weed/pb/filer_pb/filer_pb_helper.go | 1 + weed/s3api/http/header.go | 2 +- weed/server/filer_server_handlers.go | 4 ++-- weed/server/filer_server_handlers_write.go | 6 +++--- 7 files changed, 11 insertions(+), 12 deletions(-) diff --git a/weed/filer/filer_conf.go b/weed/filer/filer_conf.go index 0a86192ec..b75fb5084 100644 --- a/weed/filer/filer_conf.go +++ b/weed/filer/filer_conf.go @@ -10,9 +10,8 @@ import ( "github.com/viant/ptrie" ) - const ( - DirectoryEtc = "/etc" + DirectoryEtc = "/etc" FilerConfName = "filer.conf" ) @@ -27,7 +26,7 @@ func NewFilerConf() (fc *FilerConf) { return fc } -func (fc *FilerConf) loadFromFiler(filer *Filer) (err error){ +func (fc *FilerConf) loadFromFiler(filer *Filer) (err error) { filerConfPath := util.NewFullPath(DirectoryEtc, FilerConfName) entry, err := filer.FindEntry(context.Background(), filerConfPath) if err != nil { @@ -79,7 +78,7 @@ var ( EmptyFilerConfPathConf = &filer_pb.FilerConf_PathConf{} ) -func (fc *FilerConf) MatchStorageRule(path string) (pathConf *filer_pb.FilerConf_PathConf){ +func (fc *FilerConf) MatchStorageRule(path string) (pathConf *filer_pb.FilerConf_PathConf) { fc.rules.MatchPrefix([]byte(path), func(key []byte, value interface{}) bool { pathConf = value.(*filer_pb.FilerConf_PathConf) return true diff --git a/weed/filesys/file.go b/weed/filesys/file.go index 9e1342370..3bffa156e 100644 --- a/weed/filesys/file.go +++ b/weed/filesys/file.go @@ -294,7 +294,7 @@ func (file *File) addChunks(chunks []*filer_pb.FileChunk) { // find the earliest incoming chunk newChunks := chunks earliestChunk := newChunks[0] - for i:=1;i Date: Sun, 15 Nov 2020 18:09:35 -0800 Subject: [PATCH 18/26] fs.configure: read and local add filer configuration --- weed/filer/filer_conf.go | 31 +++++++++-- weed/shell/command_fs_configure.go | 87 ++++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+), 4 deletions(-) create mode 100644 weed/shell/command_fs_configure.go diff --git a/weed/filer/filer_conf.go b/weed/filer/filer_conf.go index b75fb5084..3e3e9f8a9 100644 --- a/weed/filer/filer_conf.go +++ b/weed/filer/filer_conf.go @@ -2,6 +2,7 @@ package filer import ( "context" + "io" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" @@ -47,10 +48,10 @@ func (fc *FilerConf) loadFromChunks(filer *Filer, chunks []*filer_pb.FileChunk) return } - return fc.loadFromBytes(data) + return fc.LoadFromBytes(data) } -func (fc *FilerConf) loadFromBytes(data []byte) (err error) { +func (fc *FilerConf) LoadFromBytes(data []byte) (err error) { conf := &filer_pb.FilerConf{} err = proto.UnmarshalText(string(data), conf) if err != nil { @@ -64,9 +65,8 @@ func (fc *FilerConf) loadFromBytes(data []byte) (err error) { func (fc *FilerConf) doLoadConf(conf *filer_pb.FilerConf) (err error) { for _, location := range conf.Locations { - err = fc.rules.Put([]byte(location.LocationPrefix), location) + err = fc.AddLocationConf(location) if err != nil { - glog.Errorf("put location prefix: %v", err) // this is not recoverable return nil } @@ -74,6 +74,15 @@ func (fc *FilerConf) doLoadConf(conf *filer_pb.FilerConf) (err error) { return nil } +func (fc *FilerConf) AddLocationConf(locConf *filer_pb.FilerConf_PathConf) (err error) { + err = fc.rules.Put([]byte(locConf.LocationPrefix), locConf) + if err != nil { + glog.Errorf("put location prefix: %v", err) + } + return +} + + var ( EmptyFilerConfPathConf = &filer_pb.FilerConf_PathConf{} ) @@ -88,3 +97,17 @@ func (fc *FilerConf) MatchStorageRule(path string) (pathConf *filer_pb.FilerConf } return pathConf } + +func (fc *FilerConf) ToProto() *filer_pb.FilerConf { + m := &filer_pb.FilerConf{} + fc.rules.Walk(func(key []byte, value interface{}) bool { + pathConf := value.(*filer_pb.FilerConf_PathConf) + m.Locations = append(m.Locations, pathConf) + return true + }) + return m +} + +func (fc *FilerConf) ToText(writer io.Writer) error { + return proto.MarshalText(writer, fc.ToProto()) +} \ No newline at end of file diff --git a/weed/shell/command_fs_configure.go b/weed/shell/command_fs_configure.go new file mode 100644 index 000000000..d8fcda83b --- /dev/null +++ b/weed/shell/command_fs_configure.go @@ -0,0 +1,87 @@ +package shell + +import ( + "bytes" + "flag" + "io" + "math" + + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" +) + +func init() { + Commands = append(Commands, &commandFsConfigure{}) +} + +type commandFsConfigure struct { +} + +func (c *commandFsConfigure) Name() string { + return "fs.configure" +} + +func (c *commandFsConfigure) Help() string { + return `configure and apply storage options for each location + + fs.configure -locationPrfix=/my/folder - + +` +} + +func (c *commandFsConfigure) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { + + fsConfigureCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) + locationPrefix := fsConfigureCommand.String("locationPrefix", "", "path prefix") + collection := fsConfigureCommand.String("collection", "", "assign writes to this colletion") + replication := fsConfigureCommand.String("replication", "", "assign writes with this replication") + ttl := fsConfigureCommand.String("ttl", "", "assign writes with this ttl") + fsync := fsConfigureCommand.Bool("fsync", false, "fsync for the writes") + apply := fsConfigureCommand.Bool("apply", false, "update and apply filer configuration") + if err = fsConfigureCommand.Parse(args); err != nil { + return nil + } + + var buf bytes.Buffer + if err = commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + + request := &filer_pb.LookupDirectoryEntryRequest{ + Directory: filer.DirectoryEtc, + Name: filer.FilerConfName, + } + respLookupEntry, err := filer_pb.LookupEntry(client, request) + if err != nil { + return err + } + + return filer.StreamContent(commandEnv.MasterClient, &buf, respLookupEntry.Entry.Chunks, 0, math.MaxInt64) + + }); err != nil { + return err + } + + fc := filer.NewFilerConf() + if err = fc.LoadFromBytes(buf.Bytes()); err != nil { + return err + } + + if *locationPrefix != "" { + locConf := &filer_pb.FilerConf_PathConf{ + LocationPrefix: *locationPrefix, + Collection: *collection, + Replication: *replication, + Ttl: *ttl, + Fsync: *fsync, + } + fc.AddLocationConf(locConf) + } + + fc.ToText(writer) + + if *apply { + + } + + return nil + +} From 2bd6fd3bbe4489e040abc3a8b6bd957eeab3910c Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 15 Nov 2020 20:15:07 -0800 Subject: [PATCH 19/26] remove unused function --- weed/util/http_util.go | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/weed/util/http_util.go b/weed/util/http_util.go index 48823c1c3..7851d8293 100644 --- a/weed/util/http_util.go +++ b/weed/util/http_util.go @@ -1,7 +1,6 @@ package util import ( - "bytes" "compress/gzip" "encoding/json" "errors" @@ -29,22 +28,6 @@ func init() { } } -func PostBytes(url string, body []byte) ([]byte, error) { - r, err := client.Post(url, "", bytes.NewReader(body)) - if err != nil { - return nil, fmt.Errorf("Post to %s: %v", url, err) - } - defer r.Body.Close() - b, err := ioutil.ReadAll(r.Body) - if err != nil { - return nil, fmt.Errorf("Read response body: %v", err) - } - if r.StatusCode >= 400 { - return nil, fmt.Errorf("%s: %s", url, r.Status) - } - return b, nil -} - func Post(url string, values url.Values) ([]byte, error) { r, err := client.PostForm(url, values) if err != nil { From ee2fa14dbeed5496043422e87f44226cdda59830 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 15 Nov 2020 20:15:47 -0800 Subject: [PATCH 20/26] filer conf: delete location specific configuration --- weed/filer/filer_conf.go | 14 +++++++++++- weed/pb/filer_pb/filer_pb_helper.go | 4 +++- weed/shell/command_fs_configure.go | 33 ++++++++++++++++++++++++++--- 3 files changed, 46 insertions(+), 5 deletions(-) diff --git a/weed/filer/filer_conf.go b/weed/filer/filer_conf.go index 3e3e9f8a9..4f317371b 100644 --- a/weed/filer/filer_conf.go +++ b/weed/filer/filer_conf.go @@ -82,6 +82,18 @@ func (fc *FilerConf) AddLocationConf(locConf *filer_pb.FilerConf_PathConf) (err return } +func (fc *FilerConf) DeleteLocationConf(locationPrefix string) { + rules := ptrie.New() + fc.rules.Walk(func(key []byte, value interface{}) bool { + if string(key) == locationPrefix { + return true + } + rules.Put(key, value) + return true + }) + fc.rules = rules + return +} var ( EmptyFilerConfPathConf = &filer_pb.FilerConf_PathConf{} @@ -110,4 +122,4 @@ func (fc *FilerConf) ToProto() *filer_pb.FilerConf { func (fc *FilerConf) ToText(writer io.Writer) error { return proto.MarshalText(writer, fc.ToProto()) -} \ No newline at end of file +} diff --git a/weed/pb/filer_pb/filer_pb_helper.go b/weed/pb/filer_pb/filer_pb_helper.go index 8dde9c80c..b46385c8f 100644 --- a/weed/pb/filer_pb/filer_pb_helper.go +++ b/weed/pb/filer_pb/filer_pb_helper.go @@ -8,6 +8,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/storage/needle" + "github.com/golang/protobuf/proto" "github.com/viant/ptrie" ) @@ -143,5 +144,6 @@ func IsRename(event *SubscribeMetadataResponse) bool { var _ = ptrie.KeyProvider(&FilerConf_PathConf{}) func (fp *FilerConf_PathConf) Key() interface{} { - return fp.LocationPrefix + key, _ := proto.Marshal(fp) + return string(key) } diff --git a/weed/shell/command_fs_configure.go b/weed/shell/command_fs_configure.go index d8fcda83b..71082ddd4 100644 --- a/weed/shell/command_fs_configure.go +++ b/weed/shell/command_fs_configure.go @@ -3,11 +3,14 @@ package shell import ( "bytes" "flag" + "fmt" "io" "math" + "net/http" "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/util" ) func init() { @@ -32,11 +35,12 @@ func (c *commandFsConfigure) Help() string { func (c *commandFsConfigure) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { fsConfigureCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) - locationPrefix := fsConfigureCommand.String("locationPrefix", "", "path prefix") + locationPrefix := fsConfigureCommand.String("locationPrefix", "", "path prefix, required to update the path-specific configuration") collection := fsConfigureCommand.String("collection", "", "assign writes to this colletion") replication := fsConfigureCommand.String("replication", "", "assign writes with this replication") ttl := fsConfigureCommand.String("ttl", "", "assign writes with this ttl") fsync := fsConfigureCommand.Bool("fsync", false, "fsync for the writes") + isDelete := fsConfigureCommand.Bool("delete", false, "delete the configuration by locationPrefix") apply := fsConfigureCommand.Bool("apply", false, "update and apply filer configuration") if err = fsConfigureCommand.Parse(args); err != nil { return nil @@ -73,13 +77,36 @@ func (c *commandFsConfigure) Do(args []string, commandEnv *CommandEnv, writer io Ttl: *ttl, Fsync: *fsync, } - fc.AddLocationConf(locConf) + if *isDelete { + fc.DeleteLocationConf(*locationPrefix) + } else { + fc.AddLocationConf(locConf) + } } - fc.ToText(writer) + buf.Reset() + fc.ToText(&buf) + + fmt.Fprintf(writer, string(buf.Bytes())) if *apply { + target := fmt.Sprintf("http://%s:%d%s/%s", commandEnv.option.FilerHost, commandEnv.option.FilerPort, filer.DirectoryEtc, filer.FilerConfName) + + // set the HTTP method, url, and request body + req, err := http.NewRequest(http.MethodPut, target, &buf) + if err != nil { + return err + } + + // set the request header Content-Type for json + req.Header.Set("Content-Type", "text/plain; charset=utf-8") + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + util.CloseResponse(resp) + } return nil From 98827d68d9f6dc17ff635a5ad6aef7eac0a5db89 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 15 Nov 2020 20:25:56 -0800 Subject: [PATCH 21/26] 2.11 --- k8s/seaweedfs/Chart.yaml | 2 +- k8s/seaweedfs/values.yaml | 2 +- weed/util/constants.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/k8s/seaweedfs/Chart.yaml b/k8s/seaweedfs/Chart.yaml index 99bb96dce..ee08dbec5 100644 --- a/k8s/seaweedfs/Chart.yaml +++ b/k8s/seaweedfs/Chart.yaml @@ -1,4 +1,4 @@ apiVersion: v1 description: SeaweedFS name: seaweedfs -version: 2.10 \ No newline at end of file +version: 2.11 \ No newline at end of file diff --git a/k8s/seaweedfs/values.yaml b/k8s/seaweedfs/values.yaml index 414330dd2..1bc619cb6 100644 --- a/k8s/seaweedfs/values.yaml +++ b/k8s/seaweedfs/values.yaml @@ -4,7 +4,7 @@ global: registry: "" repository: "" imageName: chrislusf/seaweedfs - imageTag: "2.10" + imageTag: "2.11" imagePullPolicy: IfNotPresent imagePullSecrets: imagepullsecret restartPolicy: Always diff --git a/weed/util/constants.go b/weed/util/constants.go index f1864cf95..0aeee111e 100644 --- a/weed/util/constants.go +++ b/weed/util/constants.go @@ -5,7 +5,7 @@ import ( ) var ( - VERSION = fmt.Sprintf("%s %d.%02d", sizeLimit, 2, 10) + VERSION = fmt.Sprintf("%s %d.%02d", sizeLimit, 2, 11) COMMIT = "" ) From af658ea97016c95c09938bed7d728ef06740a389 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 15 Nov 2020 21:16:28 -0800 Subject: [PATCH 22/26] fix typo --- weed/shell/command_fs_configure.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/shell/command_fs_configure.go b/weed/shell/command_fs_configure.go index 71082ddd4..a616b6bd2 100644 --- a/weed/shell/command_fs_configure.go +++ b/weed/shell/command_fs_configure.go @@ -36,7 +36,7 @@ func (c *commandFsConfigure) Do(args []string, commandEnv *CommandEnv, writer io fsConfigureCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) locationPrefix := fsConfigureCommand.String("locationPrefix", "", "path prefix, required to update the path-specific configuration") - collection := fsConfigureCommand.String("collection", "", "assign writes to this colletion") + collection := fsConfigureCommand.String("collection", "", "assign writes to this collection") replication := fsConfigureCommand.String("replication", "", "assign writes with this replication") ttl := fsConfigureCommand.String("ttl", "", "assign writes with this ttl") fsync := fsConfigureCommand.Bool("fsync", false, "fsync for the writes") From 61d96fde011f81fb1addfd1a6599a4081c14b5e2 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 15 Nov 2020 21:26:04 -0800 Subject: [PATCH 23/26] protect against edge cases when locations expires --- .../main/java/seaweedfs/client/SeaweedRead.java | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java b/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java index a9ddd51db..2b530d2dd 100644 --- a/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java +++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java @@ -28,20 +28,26 @@ public class SeaweedRead { List chunkViews = viewFromVisibles(visibleIntervals, position, bufferLength); + Map knownLocations = new HashMap<>(); + FilerProto.LookupVolumeRequest.Builder lookupRequest = FilerProto.LookupVolumeRequest.newBuilder(); for (ChunkView chunkView : chunkViews) { String vid = parseVolumeId(chunkView.fileId); - if (volumeIdCache.getLocations(vid)==null){ + FilerProto.Locations locations = volumeIdCache.getLocations(vid); + if (locations == null) { lookupRequest.addVolumeIds(vid); + } else { + knownLocations.put(vid, locations); } } - if (lookupRequest.getVolumeIdsCount()>0){ + if (lookupRequest.getVolumeIdsCount() > 0) { FilerProto.LookupVolumeResponse lookupResponse = filerGrpcClient .getBlockingStub().lookupVolume(lookupRequest.build()); Map vid2Locations = lookupResponse.getLocationsMapMap(); - for (Map.Entry entry : vid2Locations.entrySet()) { + for (Map.Entry entry : vid2Locations.entrySet()) { volumeIdCache.setLocations(entry.getKey(), entry.getValue()); + knownLocations.put(entry.getKey(), entry.getValue()); } } @@ -57,7 +63,7 @@ public class SeaweedRead { startOffset += gap; } - FilerProto.Locations locations = volumeIdCache.getLocations(parseVolumeId(chunkView.fileId)); + FilerProto.Locations locations = knownLocations.get(parseVolumeId(chunkView.fileId)); if (locations == null || locations.getLocationsCount() == 0) { LOG.error("failed to locate {}", chunkView.fileId); // log here! From fe0105967520ea9072fc19b0a94216fddb28f4f5 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 15 Nov 2020 21:27:21 -0800 Subject: [PATCH 24/26] volume id cache: reduce cache time --- .../client/src/main/java/seaweedfs/client/VolumeIdCache.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/other/java/client/src/main/java/seaweedfs/client/VolumeIdCache.java b/other/java/client/src/main/java/seaweedfs/client/VolumeIdCache.java index 38daa14ac..fd2649cc2 100644 --- a/other/java/client/src/main/java/seaweedfs/client/VolumeIdCache.java +++ b/other/java/client/src/main/java/seaweedfs/client/VolumeIdCache.java @@ -15,7 +15,7 @@ public class VolumeIdCache { } this.cache = CacheBuilder.newBuilder() .maximumSize(maxEntries) - .expireAfterAccess(1, TimeUnit.HOURS) + .expireAfterAccess(5, TimeUnit.MINUTES) .build(); } From 83527a8f55784cf28ba6fd2bef4365be8da2df25 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 15 Nov 2020 21:31:14 -0800 Subject: [PATCH 25/26] adds more help message --- weed/shell/command_fs_configure.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/weed/shell/command_fs_configure.go b/weed/shell/command_fs_configure.go index a616b6bd2..9f6b14439 100644 --- a/weed/shell/command_fs_configure.go +++ b/weed/shell/command_fs_configure.go @@ -27,7 +27,18 @@ func (c *commandFsConfigure) Name() string { func (c *commandFsConfigure) Help() string { return `configure and apply storage options for each location - fs.configure -locationPrfix=/my/folder - + # see the possible configuration file content + fs.configure + + # trying the changes and see the possible configuration file content + fs.configure -locationPrfix=/my/folder -collection=abc + fs.configure -locationPrfix=/my/folder -collection=abc -ttl=7d + + # apply the changes + fs.configure -locationPrfix=/my/folder -collection=abc -apply + + # delete the changes + fs.configure -locationPrfix=/my/folder -delete -apply ` } From c0d279c54e56882bc0ecdf496ecfcbcfa1c6d6e3 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 15 Nov 2020 21:48:17 -0800 Subject: [PATCH 26/26] filere.conf: prefer to use json format --- weed/filer/filer_conf.go | 24 +++++++++++++++++++----- weed/shell/command_fs_configure.go | 1 + 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/weed/filer/filer_conf.go b/weed/filer/filer_conf.go index 4f317371b..182449d49 100644 --- a/weed/filer/filer_conf.go +++ b/weed/filer/filer_conf.go @@ -1,12 +1,14 @@ package filer import ( + "bytes" "context" "io" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/util" + "github.com/golang/protobuf/jsonpb" "github.com/golang/protobuf/proto" "github.com/viant/ptrie" ) @@ -53,10 +55,16 @@ func (fc *FilerConf) loadFromChunks(filer *Filer, chunks []*filer_pb.FileChunk) func (fc *FilerConf) LoadFromBytes(data []byte) (err error) { conf := &filer_pb.FilerConf{} - err = proto.UnmarshalText(string(data), conf) - if err != nil { - glog.Errorf("unable to parse filer conf: %v", err) - // this is not recoverable + + if err := jsonpb.Unmarshal(bytes.NewReader(data), conf); err != nil { + + err = proto.UnmarshalText(string(data), conf) + if err != nil { + glog.Errorf("unable to parse filer conf: %v", err) + // this is not recoverable + return nil + } + return nil } @@ -121,5 +129,11 @@ func (fc *FilerConf) ToProto() *filer_pb.FilerConf { } func (fc *FilerConf) ToText(writer io.Writer) error { - return proto.MarshalText(writer, fc.ToProto()) + + m := jsonpb.Marshaler{ + EmitDefaults: false, + Indent: " ", + } + + return m.Marshal(writer, fc.ToProto()) } diff --git a/weed/shell/command_fs_configure.go b/weed/shell/command_fs_configure.go index 9f6b14439..d4d70048d 100644 --- a/weed/shell/command_fs_configure.go +++ b/weed/shell/command_fs_configure.go @@ -99,6 +99,7 @@ func (c *commandFsConfigure) Do(args []string, commandEnv *CommandEnv, writer io fc.ToText(&buf) fmt.Fprintf(writer, string(buf.Bytes())) + fmt.Fprintln(writer) if *apply {