From 64c48c9724c77faa0f7c97049eda6845b61a1649 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 23 Dec 2020 21:44:53 -0800 Subject: [PATCH 001/260] seems compiling --- go.mod | 1 + go.sum | 15 ++ weed/filer/hbase/hbase_store.go | 223 +++++++++++++++++++++++++++++ weed/filer/hbase/hbase_store_kv.go | 67 +++++++++ 4 files changed, 306 insertions(+) create mode 100644 weed/filer/hbase/hbase_store.go create mode 100644 weed/filer/hbase/hbase_store_kv.go diff --git a/go.mod b/go.mod index 1ecbfd2a9..87bc93284 100644 --- a/go.mod +++ b/go.mod @@ -68,6 +68,7 @@ require ( github.com/syndtr/goleveldb v1.0.0 github.com/tidwall/gjson v1.3.2 github.com/tidwall/match v1.0.1 + github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365 // indirect github.com/valyala/bytebufferpool v1.0.0 github.com/viant/assertly v0.5.4 // indirect github.com/viant/ptrie v0.3.0 diff --git a/go.sum b/go.sum index 387d2a8ac..66fb72a6b 100644 --- a/go.sum +++ b/go.sum @@ -179,6 +179,8 @@ github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gG github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-zookeeper/zk v1.0.2 h1:4mx0EYENAdX/B/rbunjlt5+4RTA/a9SMHBRuSKdGxPM= +github.com/go-zookeeper/zk v1.0.2/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw= github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= @@ -223,6 +225,7 @@ github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4er github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= @@ -539,6 +542,7 @@ github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a h1:9ZKAASQSHhD github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 h1:dY6ETXrvDG7Sa4vE8ZQG4yqWg6UnOcbqTAahkV813vQ= github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -562,6 +566,8 @@ github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.5.0 h1:1N5EYkVAPEywqZRJd7cwnRtCb6xJx7NH3T3WUTF980Q= +github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= @@ -621,6 +627,8 @@ github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhV github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365 h1:6iRwZdrFUzbcVYZwa8dXTIILGIxmmhjyUPJEcwzPGaU= +github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365/go.mod h1:zj0GJHGvyf1ed3Jm/Tb4830c/ZKDq+YoLsCt2rGQuT0= github.com/ugorji/go v1.1.4 h1:j4s+tAvLfL3bZyefP2SEWmhBzmuIlH/eqNuPdFPgngw= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= @@ -785,6 +793,7 @@ golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfru golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201022201747-fb209a7c41cd h1:WgqgiQvkiZWz7XLhphjt2GI2GcGCTIZs9jqXMWmH+oc= golang.org/x/sys v0.0.0-20201022201747-fb209a7c41cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= @@ -930,8 +939,14 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +modernc.org/b v1.0.0 h1:vpvqeyp17ddcQWF29Czawql4lDdABCDRbXRAS4+aF2o= +modernc.org/b v1.0.0/go.mod h1:uZWcZfRj1BpYzfN9JTerzlNUnnPsV9O2ZA8JsRcubNg= +modernc.org/mathutil v1.1.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/strutil v1.1.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= pack.ag/amqp v0.11.2/go.mod h1:4/cbmt4EJXSKlG6LCfWHoqmN0uFdy5i/+YFz+fTfhV4= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= diff --git a/weed/filer/hbase/hbase_store.go b/weed/filer/hbase/hbase_store.go new file mode 100644 index 000000000..6e1ccea35 --- /dev/null +++ b/weed/filer/hbase/hbase_store.go @@ -0,0 +1,223 @@ +package hbase + +import ( + "bytes" + "context" + "fmt" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/util" + "github.com/tsuna/gohbase" + "github.com/tsuna/gohbase/hrpc" + "io" +) + +func init() { + filer.Stores = append(filer.Stores, &HbaseStore{}) +} + +type HbaseStore struct { + Client gohbase.Client + table []byte + cfKv string + cfMetaDir string + cfFlatDir string + column string +} + +func (store *HbaseStore) GetName() string { + return "hbase" +} + +func (store *HbaseStore) Initialize(configuration util.Configuration, prefix string) (err error) { + return store.initialize( + configuration.GetString(prefix+"zkquorum"), + configuration.GetString(prefix+"table"), + ) +} + +func (store *HbaseStore) initialize(zkquorum, table string) (err error) { + store.Client = gohbase.NewClient(zkquorum) + store.table = []byte(table) + store.cfKv = "kv" + store.cfMetaDir = "meta" + store.cfFlatDir = "flat" + store.column = "a" + + // check table exists + key := "whatever" + headers := map[string][]string{store.cfMetaDir: nil} + get, err := hrpc.NewGet(context.Background(), store.table, []byte(key), hrpc.Families(headers)) + if err != nil { + return fmt.Errorf("NewGet returned an error: %v", err) + } + _, err = store.Client.Get(get) + if err != gohbase.TableNotFound { + return nil + } + + // create table + adminClient := gohbase.NewAdminClient(zkquorum) + cFamilies := []string{store.cfKv, store.cfMetaDir, store.cfFlatDir} + cf := make(map[string]map[string]string, len(cFamilies)) + for _, f := range cFamilies { + cf[f] = nil + } + ct := hrpc.NewCreateTable(context.Background(), []byte(table), cf) + if err := adminClient.CreateTable(ct); err != nil { + return err + } + + return nil +} + +func (store *HbaseStore) InsertEntry(ctx context.Context, entry *filer.Entry) error { + value, err := entry.EncodeAttributesAndChunks() + if err != nil { + return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err) + } + if len(entry.Chunks) > 50 { + value = util.MaybeGzipData(value) + } + + return store.doPut(ctx, store.cfMetaDir, []byte(entry.FullPath), value) +} + +func (store *HbaseStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) { + return store.InsertEntry(ctx, entry) +} + +func (store *HbaseStore) FindEntry(ctx context.Context, path util.FullPath) (entry *filer.Entry, err error) { + value, err := store.doGet(ctx, store.cfMetaDir, []byte(path)) + if err != nil { + if err == filer.ErrKvNotFound { + return nil, filer_pb.ErrNotFound + } + return nil, err + } + + entry = &filer.Entry{ + FullPath: path, + } + err = entry.DecodeAttributesAndChunks(util.MaybeDecompressData(value)) + if err != nil { + return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err) + } + return entry, nil +} + +func (store *HbaseStore) DeleteEntry(ctx context.Context, path util.FullPath) (err error) { + return store.doDelete(ctx, store.cfMetaDir, []byte(path)) +} + +func (store *HbaseStore) DeleteFolderChildren(ctx context.Context, path util.FullPath) (err error) { + + family := map[string][]string{store.cfMetaDir: {COLUMN_NAME}} + expectedPrefix := []byte(dirPath+"/") + scan, err := hrpc.NewScanRange(ctx, store.table, expectedPrefix, nil, hrpc.Families(family)) + if err != nil { + return err + } + + scanner := store.Client.Scan(scan) + defer scanner.Close() + for { + res, err := scanner.Next() + if err != nil { + break + } + if len(res.Cells) == 0 { + continue + } + cell := res.Cells[0] + + if !bytes.HasPrefix(cell.Row, expectedPrefix) { + break + } + + err = store.doDelete(ctx, store.cfMetaDir, cell.Row) + if err != nil { + break + } + + } + return +} + +func (store *HbaseStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int) ([]*filer.Entry, error) { + return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "") +} + +func (store *HbaseStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) ([]*filer.Entry, error) { + family := map[string][]string{store.cfMetaDir: {COLUMN_NAME}} + expectedPrefix := []byte(dirPath.Child(prefix)) + scan, err := hrpc.NewScanRange(ctx, store.table, expectedPrefix, nil, hrpc.Families(family)) + if err != nil { + return nil, err + } + + var entries []*filer.Entry + scanner := store.Client.Scan(scan) + defer scanner.Close() + for { + res, err := scanner.Next() + if err == io.EOF { + break + } + if err != nil { + return entries, err + } + if len(res.Cells) == 0 { + continue + } + cell := res.Cells[0] + + if !bytes.HasPrefix(cell.Row, expectedPrefix) { + break + } + + fullpath := util.FullPath(cell.Row) + + + value := cell.Value + + _, fileName := fullpath.DirAndName() + + if fileName == startFileName && !includeStartFile { + continue + } + + limit-- + if limit < 0 { + break + } + entry := &filer.Entry{ + FullPath: fullpath, + } + if decodeErr := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(value)); decodeErr != nil { + err = decodeErr + glog.V(0).Infof("list %s : %v", entry.FullPath, err) + break + } + entries = append(entries, entry) + } + + return entries, nil +} + +func (store *HbaseStore) BeginTransaction(ctx context.Context) (context.Context, error) { + return ctx, nil +} + +func (store *HbaseStore) CommitTransaction(ctx context.Context) error { + return nil +} + +func (store *HbaseStore) RollbackTransaction(ctx context.Context) error { + return nil +} + +func (store *HbaseStore) Shutdown() { + store.Client.Close() +} diff --git a/weed/filer/hbase/hbase_store_kv.go b/weed/filer/hbase/hbase_store_kv.go new file mode 100644 index 000000000..b98eb3ed6 --- /dev/null +++ b/weed/filer/hbase/hbase_store_kv.go @@ -0,0 +1,67 @@ +package hbase + +import ( + "context" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/tsuna/gohbase/hrpc" +) + +const( + COLUMN_NAME = "a" +) +func (store *HbaseStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) { + return store.doPut(ctx, store.cfKv, key, value) +} + +func (store *HbaseStore) KvGet(ctx context.Context, key []byte) (value []byte, err error) { + return store.doGet(ctx, store.cfKv, key) +} + +func (store *HbaseStore) KvDelete(ctx context.Context, key []byte) (err error) { + return store.doDelete(ctx, store.cfKv, key) +} + +func (store *HbaseStore) doPut(ctx context.Context, cf string, key, value []byte) (err error) { + values := map[string]map[string][]byte{store.cfKv: map[string][]byte{}} + values[cf][COLUMN_NAME] = value + putRequest, err := hrpc.NewPut(ctx, store.table, key, values) + if err != nil { + return err + } + _, err = store.Client.Put(putRequest) + if err != nil { + return err + } + return nil +} + +func (store *HbaseStore) doGet(ctx context.Context, cf string, key []byte) (value []byte, err error) { + family := map[string][]string{cf: {COLUMN_NAME}} + getRequest, err := hrpc.NewGet(context.Background(), store.table, key, hrpc.Families(family)) + if err != nil { + return nil, err + } + getResp, err := store.Client.Get(getRequest) + if err != nil { + return nil, err + } + if len(getResp.Cells) == 0 { + return nil, filer.ErrKvNotFound + } + + return getResp.Cells[0].Value, nil +} + +func (store *HbaseStore) doDelete(ctx context.Context, cf string, key []byte) (err error) { + values := map[string]map[string][]byte{store.cfKv: map[string][]byte{}} + values[cf][COLUMN_NAME] = nil + deleteRequest, err := hrpc.NewDel(ctx, store.table, key, values) + if err != nil { + return err + } + _, err = store.Client.Delete(deleteRequest) + if err != nil { + return err + } + return nil +} From 94e3757c08909c6796c2bf02b579f8ee36128c4d Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 23 Dec 2020 21:45:16 -0800 Subject: [PATCH 002/260] fix --- weed/filer/hbase/hbase_store.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/filer/hbase/hbase_store.go b/weed/filer/hbase/hbase_store.go index 6e1ccea35..fea6004be 100644 --- a/weed/filer/hbase/hbase_store.go +++ b/weed/filer/hbase/hbase_store.go @@ -114,7 +114,7 @@ func (store *HbaseStore) DeleteEntry(ctx context.Context, path util.FullPath) (e func (store *HbaseStore) DeleteFolderChildren(ctx context.Context, path util.FullPath) (err error) { family := map[string][]string{store.cfMetaDir: {COLUMN_NAME}} - expectedPrefix := []byte(dirPath+"/") + expectedPrefix := []byte(path+"/") scan, err := hrpc.NewScanRange(ctx, store.table, expectedPrefix, nil, hrpc.Families(family)) if err != nil { return err From c3d1b3b5aa5aef8d69cd1254818c296238e0fe64 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 23 Dec 2020 21:49:01 -0800 Subject: [PATCH 003/260] hook up --- weed/command/scaffold.go | 4 ++++ weed/server/filer_server.go | 1 + 2 files changed, 5 insertions(+) diff --git a/weed/command/scaffold.go b/weed/command/scaffold.go index 04a988027..f3a094d5a 100644 --- a/weed/command/scaffold.go +++ b/weed/command/scaffold.go @@ -141,6 +141,10 @@ password="" # This changes the data layout. Only add new directories. Removing/Updating will cause data loss. superLargeDirectories = [] +[hbase] +zkquorum = "" +table = "seaweedfs" + [redis2] enabled = false address = "localhost:6379" diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index 2991d14ab..2da129ab2 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -23,6 +23,7 @@ import ( _ "github.com/chrislusf/seaweedfs/weed/filer/cassandra" _ "github.com/chrislusf/seaweedfs/weed/filer/elastic/v7" _ "github.com/chrislusf/seaweedfs/weed/filer/etcd" + _ "github.com/chrislusf/seaweedfs/weed/filer/hbase" _ "github.com/chrislusf/seaweedfs/weed/filer/leveldb" _ "github.com/chrislusf/seaweedfs/weed/filer/leveldb2" _ "github.com/chrislusf/seaweedfs/weed/filer/mongodb" From 32955b04b83a8132624ee332427e05fd59a32e59 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 23 Dec 2020 22:08:41 -0800 Subject: [PATCH 004/260] Update go.mod --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 87bc93284..d93a44379 100644 --- a/go.mod +++ b/go.mod @@ -68,7 +68,7 @@ require ( github.com/syndtr/goleveldb v1.0.0 github.com/tidwall/gjson v1.3.2 github.com/tidwall/match v1.0.1 - github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365 // indirect + github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365 github.com/valyala/bytebufferpool v1.0.0 github.com/viant/assertly v0.5.4 // indirect github.com/viant/ptrie v0.3.0 From c4a202ec417554f170a21d7331b44f860ea72274 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 23 Dec 2020 23:23:05 -0800 Subject: [PATCH 005/260] fix wrong column family --- weed/filer/hbase/hbase_store.go | 4 +--- weed/filer/hbase/hbase_store_kv.go | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/weed/filer/hbase/hbase_store.go b/weed/filer/hbase/hbase_store.go index fea6004be..8a9ea7fc1 100644 --- a/weed/filer/hbase/hbase_store.go +++ b/weed/filer/hbase/hbase_store.go @@ -22,7 +22,6 @@ type HbaseStore struct { table []byte cfKv string cfMetaDir string - cfFlatDir string column string } @@ -42,7 +41,6 @@ func (store *HbaseStore) initialize(zkquorum, table string) (err error) { store.table = []byte(table) store.cfKv = "kv" store.cfMetaDir = "meta" - store.cfFlatDir = "flat" store.column = "a" // check table exists @@ -59,7 +57,7 @@ func (store *HbaseStore) initialize(zkquorum, table string) (err error) { // create table adminClient := gohbase.NewAdminClient(zkquorum) - cFamilies := []string{store.cfKv, store.cfMetaDir, store.cfFlatDir} + cFamilies := []string{store.cfKv, store.cfMetaDir} cf := make(map[string]map[string]string, len(cFamilies)) for _, f := range cFamilies { cf[f] = nil diff --git a/weed/filer/hbase/hbase_store_kv.go b/weed/filer/hbase/hbase_store_kv.go index b98eb3ed6..8f5794ac1 100644 --- a/weed/filer/hbase/hbase_store_kv.go +++ b/weed/filer/hbase/hbase_store_kv.go @@ -22,7 +22,7 @@ func (store *HbaseStore) KvDelete(ctx context.Context, key []byte) (err error) { } func (store *HbaseStore) doPut(ctx context.Context, cf string, key, value []byte) (err error) { - values := map[string]map[string][]byte{store.cfKv: map[string][]byte{}} + values := map[string]map[string][]byte{cf: map[string][]byte{}} values[cf][COLUMN_NAME] = value putRequest, err := hrpc.NewPut(ctx, store.table, key, values) if err != nil { @@ -53,7 +53,7 @@ func (store *HbaseStore) doGet(ctx context.Context, cf string, key []byte) (valu } func (store *HbaseStore) doDelete(ctx context.Context, cf string, key []byte) (err error) { - values := map[string]map[string][]byte{store.cfKv: map[string][]byte{}} + values := map[string]map[string][]byte{cf: map[string][]byte{}} values[cf][COLUMN_NAME] = nil deleteRequest, err := hrpc.NewDel(ctx, store.table, key, values) if err != nil { From b5e2be635adb51808282358a1eab0b61080982d6 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 23 Dec 2020 23:49:22 -0800 Subject: [PATCH 006/260] adjust for directory listing --- weed/filer/hbase/hbase_store.go | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/weed/filer/hbase/hbase_store.go b/weed/filer/hbase/hbase_store.go index 8a9ea7fc1..cc690c48a 100644 --- a/weed/filer/hbase/hbase_store.go +++ b/weed/filer/hbase/hbase_store.go @@ -112,7 +112,7 @@ func (store *HbaseStore) DeleteEntry(ctx context.Context, path util.FullPath) (e func (store *HbaseStore) DeleteFolderChildren(ctx context.Context, path util.FullPath) (err error) { family := map[string][]string{store.cfMetaDir: {COLUMN_NAME}} - expectedPrefix := []byte(path+"/") + expectedPrefix := []byte(path + "/") scan, err := hrpc.NewScanRange(ctx, store.table, expectedPrefix, nil, hrpc.Families(family)) if err != nil { return err @@ -133,6 +133,11 @@ func (store *HbaseStore) DeleteFolderChildren(ctx context.Context, path util.Ful if !bytes.HasPrefix(cell.Row, expectedPrefix) { break } + fullpath := util.FullPath(cell.Row) + dir, _ := fullpath.DirAndName() + if dir != string(dirPath) { + continue + } err = store.doDelete(ctx, store.cfMetaDir, cell.Row) if err != nil { @@ -149,7 +154,7 @@ func (store *HbaseStore) ListDirectoryEntries(ctx context.Context, dirPath util. func (store *HbaseStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) ([]*filer.Entry, error) { family := map[string][]string{store.cfMetaDir: {COLUMN_NAME}} - expectedPrefix := []byte(dirPath.Child(prefix)) + expectedPrefix := []byte(string(dirPath) + "/" + prefix) scan, err := hrpc.NewScanRange(ctx, store.table, expectedPrefix, nil, hrpc.Families(family)) if err != nil { return nil, err @@ -176,12 +181,13 @@ func (store *HbaseStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPa } fullpath := util.FullPath(cell.Row) - + dir, fileName := fullpath.DirAndName() + if dir != string(dirPath) { + continue + } value := cell.Value - _, fileName := fullpath.DirAndName() - if fileName == startFileName && !includeStartFile { continue } From 53bc1ea25bf707afd828141255f3fb7841c23686 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 23 Dec 2020 23:53:46 -0800 Subject: [PATCH 007/260] fix compilation --- weed/filer/hbase/hbase_store.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/filer/hbase/hbase_store.go b/weed/filer/hbase/hbase_store.go index cc690c48a..68170ec18 100644 --- a/weed/filer/hbase/hbase_store.go +++ b/weed/filer/hbase/hbase_store.go @@ -135,7 +135,7 @@ func (store *HbaseStore) DeleteFolderChildren(ctx context.Context, path util.Ful } fullpath := util.FullPath(cell.Row) dir, _ := fullpath.DirAndName() - if dir != string(dirPath) { + if dir != string(dir) { continue } From 2fea8cfa0fd56d7f2f459681b727d9625b62bc16 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 23 Dec 2020 23:54:15 -0800 Subject: [PATCH 008/260] fix compilation --- weed/filer/hbase/hbase_store.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/filer/hbase/hbase_store.go b/weed/filer/hbase/hbase_store.go index 68170ec18..65cdfd847 100644 --- a/weed/filer/hbase/hbase_store.go +++ b/weed/filer/hbase/hbase_store.go @@ -135,7 +135,7 @@ func (store *HbaseStore) DeleteFolderChildren(ctx context.Context, path util.Ful } fullpath := util.FullPath(cell.Row) dir, _ := fullpath.DirAndName() - if dir != string(dir) { + if dir != string(path) { continue } From 9bf6c10505bef2e9f679b203ab2380672d5afb4e Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 24 Dec 2020 00:18:59 -0800 Subject: [PATCH 009/260] fix prefix --- weed/filer/hbase/hbase_store.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/weed/filer/hbase/hbase_store.go b/weed/filer/hbase/hbase_store.go index 65cdfd847..03aecfd2b 100644 --- a/weed/filer/hbase/hbase_store.go +++ b/weed/filer/hbase/hbase_store.go @@ -112,7 +112,7 @@ func (store *HbaseStore) DeleteEntry(ctx context.Context, path util.FullPath) (e func (store *HbaseStore) DeleteFolderChildren(ctx context.Context, path util.FullPath) (err error) { family := map[string][]string{store.cfMetaDir: {COLUMN_NAME}} - expectedPrefix := []byte(path + "/") + expectedPrefix := []byte(path.Child("")) scan, err := hrpc.NewScanRange(ctx, store.table, expectedPrefix, nil, hrpc.Families(family)) if err != nil { return err @@ -154,7 +154,7 @@ func (store *HbaseStore) ListDirectoryEntries(ctx context.Context, dirPath util. func (store *HbaseStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) ([]*filer.Entry, error) { family := map[string][]string{store.cfMetaDir: {COLUMN_NAME}} - expectedPrefix := []byte(string(dirPath) + "/" + prefix) + expectedPrefix := []byte(dirPath.Child(prefix)) scan, err := hrpc.NewScanRange(ctx, store.table, expectedPrefix, nil, hrpc.Families(family)) if err != nil { return nil, err From 75c6edba9e9f09ff102a059c46a55c16a8e9063e Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 24 Dec 2020 00:19:16 -0800 Subject: [PATCH 010/260] filer: hbase add enabled flag --- weed/command/scaffold.go | 1 + 1 file changed, 1 insertion(+) diff --git a/weed/command/scaffold.go b/weed/command/scaffold.go index f3a094d5a..6cfd46427 100644 --- a/weed/command/scaffold.go +++ b/weed/command/scaffold.go @@ -142,6 +142,7 @@ password="" superLargeDirectories = [] [hbase] +enabled = false zkquorum = "" table = "seaweedfs" From a09cd28986077dc3fe9252475a4da16d8acaad98 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 24 Dec 2020 01:42:15 -0800 Subject: [PATCH 011/260] add debug messages --- weed/s3api/s3api_objects_list_handlers.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/weed/s3api/s3api_objects_list_handlers.go b/weed/s3api/s3api_objects_list_handlers.go index 0af099967..a4c9bf0f1 100644 --- a/weed/s3api/s3api_objects_list_handlers.go +++ b/weed/s3api/s3api_objects_list_handlers.go @@ -310,6 +310,8 @@ func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string, func (s3a *S3ApiServer) isDirectoryAllEmpty(filerClient filer_pb.SeaweedFilerClient, parentDir, name string) (isEmpty bool, err error) { // println("+ isDirectoryAllEmpty", dir, name) + glog.V(4).Infof("+ start parentDir: %s name: %s", parentDir, name) + defer glog.V(4).Infof("- start parentDir: %s name: %s isEmpty: %v", parentDir, name, isEmpty) var fileCounter int var subDirs []string currentDir := parentDir + "/" + name @@ -324,6 +326,7 @@ func (s3a *S3ApiServer) isDirectoryAllEmpty(filerClient filer_pb.SeaweedFilerCli } startFrom = entry.Name isExhausted = isExhausted || isLast + glog.V(4).Infof("list: %s/%s isExhausted: %t", currentDir, startFrom, isExhausted) return nil }, startFrom, false, 8) } From 1620de08aea14366a81d55b9db26fc7cf9a7fd67 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 24 Dec 2020 01:52:06 -0800 Subject: [PATCH 012/260] added more logs --- weed/s3api/s3api_objects_list_handlers.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/weed/s3api/s3api_objects_list_handlers.go b/weed/s3api/s3api_objects_list_handlers.go index a4c9bf0f1..5c3b898aa 100644 --- a/weed/s3api/s3api_objects_list_handlers.go +++ b/weed/s3api/s3api_objects_list_handlers.go @@ -310,8 +310,8 @@ func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string, func (s3a *S3ApiServer) isDirectoryAllEmpty(filerClient filer_pb.SeaweedFilerClient, parentDir, name string) (isEmpty bool, err error) { // println("+ isDirectoryAllEmpty", dir, name) - glog.V(4).Infof("+ start parentDir: %s name: %s", parentDir, name) - defer glog.V(4).Infof("- start parentDir: %s name: %s isEmpty: %v", parentDir, name, isEmpty) + glog.V(4).Infof("+ isEmpty %s/%s", parentDir, name) + defer glog.V(4).Infof("- isEmpty %s/%s %v", parentDir, name, isEmpty) var fileCounter int var subDirs []string currentDir := parentDir + "/" + name @@ -326,7 +326,7 @@ func (s3a *S3ApiServer) isDirectoryAllEmpty(filerClient filer_pb.SeaweedFilerCli } startFrom = entry.Name isExhausted = isExhausted || isLast - glog.V(4).Infof("list: %s/%s isExhausted: %t", currentDir, startFrom, isExhausted) + glog.V(4).Infof(" * %s/%s isLast: %t", currentDir, startFrom, isLast) return nil }, startFrom, false, 8) } From 8e48a235e2e0eb3edaeb30aa82b3c5624d564e01 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 24 Dec 2020 11:34:52 -0800 Subject: [PATCH 013/260] s3: avoid looping if the directory is empty fix https://github.com/chrislusf/seaweedfs/issues/1701 --- weed/s3api/s3api_objects_list_handlers.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/weed/s3api/s3api_objects_list_handlers.go b/weed/s3api/s3api_objects_list_handlers.go index 5c3b898aa..c1c6e2f89 100644 --- a/weed/s3api/s3api_objects_list_handlers.go +++ b/weed/s3api/s3api_objects_list_handlers.go @@ -317,8 +317,10 @@ func (s3a *S3ApiServer) isDirectoryAllEmpty(filerClient filer_pb.SeaweedFilerCli currentDir := parentDir + "/" + name var startFrom string var isExhausted bool + var foundEntry bool for fileCounter == 0 && !isExhausted && err == nil { err = filer_pb.SeaweedList(filerClient, currentDir, "", func(entry *filer_pb.Entry, isLast bool) error { + foundEntry = true if entry.IsDirectory { subDirs = append(subDirs, entry.Name) } else { @@ -329,6 +331,9 @@ func (s3a *S3ApiServer) isDirectoryAllEmpty(filerClient filer_pb.SeaweedFilerCli glog.V(4).Infof(" * %s/%s isLast: %t", currentDir, startFrom, isLast) return nil }, startFrom, false, 8) + if !foundEntry { + break + } } if err != nil { From 0e016bc7bd3d08d4c843388d6af55802ebec0dad Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 24 Dec 2020 12:10:35 -0800 Subject: [PATCH 014/260] hbase add ttl --- weed/filer/hbase/hbase_store.go | 2 +- weed/filer/hbase/hbase_store_kv.go | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/weed/filer/hbase/hbase_store.go b/weed/filer/hbase/hbase_store.go index 03aecfd2b..6b0ad58b9 100644 --- a/weed/filer/hbase/hbase_store.go +++ b/weed/filer/hbase/hbase_store.go @@ -79,7 +79,7 @@ func (store *HbaseStore) InsertEntry(ctx context.Context, entry *filer.Entry) er value = util.MaybeGzipData(value) } - return store.doPut(ctx, store.cfMetaDir, []byte(entry.FullPath), value) + return store.doPut(ctx, store.cfMetaDir, []byte(entry.FullPath), value, entry.TtlSec) } func (store *HbaseStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) { diff --git a/weed/filer/hbase/hbase_store_kv.go b/weed/filer/hbase/hbase_store_kv.go index 8f5794ac1..26bf763e2 100644 --- a/weed/filer/hbase/hbase_store_kv.go +++ b/weed/filer/hbase/hbase_store_kv.go @@ -4,13 +4,14 @@ import ( "context" "github.com/chrislusf/seaweedfs/weed/filer" "github.com/tsuna/gohbase/hrpc" + "time" ) const( COLUMN_NAME = "a" ) func (store *HbaseStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) { - return store.doPut(ctx, store.cfKv, key, value) + return store.doPut(ctx, store.cfKv, key, value, 0) } func (store *HbaseStore) KvGet(ctx context.Context, key []byte) (value []byte, err error) { @@ -21,10 +22,17 @@ func (store *HbaseStore) KvDelete(ctx context.Context, key []byte) (err error) { return store.doDelete(ctx, store.cfKv, key) } -func (store *HbaseStore) doPut(ctx context.Context, cf string, key, value []byte) (err error) { +func (store *HbaseStore) doPut(ctx context.Context, cf string, key, value []byte, ttlSecond int32) (err error) { + if ttlSecond > 0 { + return store.doPutWithOptions(ctx, cf, key, value, hrpc.Durability(hrpc.AsyncWal), hrpc.TTL(time.Duration(ttlSecond)*time.Second)) + } + return store.doPutWithOptions(ctx, cf, key, value, hrpc.Durability(hrpc.AsyncWal)) +} + +func (store *HbaseStore) doPutWithOptions(ctx context.Context, cf string, key, value []byte, options ...func(hrpc.Call) error) (err error) { values := map[string]map[string][]byte{cf: map[string][]byte{}} values[cf][COLUMN_NAME] = value - putRequest, err := hrpc.NewPut(ctx, store.table, key, values) + putRequest, err := hrpc.NewPut(ctx, store.table, key, values, options...) if err != nil { return err } @@ -55,7 +63,7 @@ func (store *HbaseStore) doGet(ctx context.Context, cf string, key []byte) (valu func (store *HbaseStore) doDelete(ctx context.Context, cf string, key []byte) (err error) { values := map[string]map[string][]byte{cf: map[string][]byte{}} values[cf][COLUMN_NAME] = nil - deleteRequest, err := hrpc.NewDel(ctx, store.table, key, values) + deleteRequest, err := hrpc.NewDel(ctx, store.table, key, values, hrpc.Durability(hrpc.AsyncWal)) if err != nil { return err } From 090f85be4b4c02dfd57942e2816dd609b002995d Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 25 Dec 2020 00:38:56 -0800 Subject: [PATCH 015/260] s3: support config action Admin:bucket --- weed/s3api/auth_credentials.go | 33 ++++++++++++++++++++--------- weed/s3api/s3api_bucket_handlers.go | 13 +++++++++++- weed/s3api/s3api_server.go | 2 +- 3 files changed, 36 insertions(+), 12 deletions(-) diff --git a/weed/s3api/auth_credentials.go b/weed/s3api/auth_credentials.go index 93544b75e..c305fee6f 100644 --- a/weed/s3api/auth_credentials.go +++ b/weed/s3api/auth_credentials.go @@ -3,6 +3,7 @@ package s3api import ( "fmt" "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/s3api/s3_constants" "io/ioutil" "net/http" @@ -155,6 +156,24 @@ func (iam *IdentityAccessManagement) Auth(f http.HandlerFunc, action Action) htt // check whether the request has valid access keys func (iam *IdentityAccessManagement) authRequest(r *http.Request, action Action) (*Identity, s3err.ErrorCode) { + identity, s3Err := iam.authUser(r) + if s3Err != s3err.ErrNone { + return identity, s3Err + } + + glog.V(3).Infof("user name: %v actions: %v", identity.Name, identity.Actions) + + bucket, _ := getBucketAndObject(r) + + if !identity.canDo(action, bucket) { + return identity, s3err.ErrAccessDenied + } + + return identity, s3err.ErrNone + +} + +func (iam *IdentityAccessManagement) authUser(r *http.Request) (*Identity, s3err.ErrorCode) { var identity *Identity var s3Err s3err.ErrorCode var found bool @@ -189,17 +208,7 @@ func (iam *IdentityAccessManagement) authRequest(r *http.Request, action Action) if s3Err != s3err.ErrNone { return identity, s3Err } - - glog.V(3).Infof("user name: %v actions: %v", identity.Name, identity.Actions) - - bucket, _ := getBucketAndObject(r) - - if !identity.canDo(action, bucket) { - return identity, s3err.ErrAccessDenied - } - return identity, s3err.ErrNone - } func (identity *Identity) canDo(action Action, bucket string) bool { @@ -215,10 +224,14 @@ func (identity *Identity) canDo(action Action, bucket string) bool { return false } limitedByBucket := string(action) + ":" + bucket + adminLimitedByBucket := s3_constants.ACTION_ADMIN + ":" + bucket for _, a := range identity.Actions { if string(a) == limitedByBucket { return true } + if string(a) == adminLimitedByBucket { + return true + } } return false } diff --git a/weed/s3api/s3api_bucket_handlers.go b/weed/s3api/s3api_bucket_handlers.go index 00b7382cc..f750f6e53 100644 --- a/weed/s3api/s3api_bucket_handlers.go +++ b/weed/s3api/s3api_bucket_handlers.go @@ -4,6 +4,7 @@ import ( "context" "encoding/xml" "fmt" + "github.com/chrislusf/seaweedfs/weed/s3api/s3_constants" "math" "net/http" "time" @@ -26,6 +27,16 @@ type ListAllMyBucketsResult struct { func (s3a *S3ApiServer) ListBucketsHandler(w http.ResponseWriter, r *http.Request) { + var identity *Identity + var s3Err s3err.ErrorCode + if s3a.iam.isEnabled() { + identity, s3Err = s3a.iam.authUser(r) + if s3Err != s3err.ErrNone { + writeErrorResponse(w, s3Err, r.URL) + return + } + } + var response ListAllMyBucketsResult entries, _, err := s3a.list(s3a.option.BucketsPath, "", "", false, math.MaxInt32) @@ -40,7 +51,7 @@ func (s3a *S3ApiServer) ListBucketsHandler(w http.ResponseWriter, r *http.Reques var buckets []*s3.Bucket for _, entry := range entries { if entry.IsDirectory { - if !s3a.hasAccess(r, entry) { + if identity!=nil && !identity.canDo(s3_constants.ACTION_ADMIN, entry.Name) { continue } buckets = append(buckets, &s3.Bucket{ diff --git a/weed/s3api/s3api_server.go b/weed/s3api/s3api_server.go index 93e2bb575..e4a07a443 100644 --- a/weed/s3api/s3api_server.go +++ b/weed/s3api/s3api_server.go @@ -128,7 +128,7 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) { } // ListBuckets - apiRouter.Methods("GET").Path("/").HandlerFunc(track(s3a.iam.Auth(s3a.ListBucketsHandler, ACTION_ADMIN), "LIST")) + apiRouter.Methods("GET").Path("/").HandlerFunc(track(s3a.ListBucketsHandler, "LIST")) // NotFound apiRouter.NotFoundHandler = http.HandlerFunc(notFoundHandler) From 1b1d18246962b89b9d2339e0611a8789d7e4423b Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 25 Dec 2020 02:07:30 -0800 Subject: [PATCH 016/260] Update release.yml --- .github/workflows/release.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2e1e9c162..39108c30a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,13 +11,13 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - goos: [linux, windows, darwin, freebsd, netbsd, openbsd ] - goarch: ["386", amd64, arm] + goos: [linux, windows, darwin ] + goarch: [amd64, arm] exclude: - goarch: arm goos: darwin - - goarch: 386 - goos: freebsd + - goarch: arm + goos: windows steps: From 0a067944cc5fb022943aaf9b7afcffb1ce6dba51 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 25 Dec 2020 02:32:52 -0800 Subject: [PATCH 017/260] filer: add retries during volume moving fix https://github.com/chrislusf/seaweedfs/issues/1704 --- .../filer_server_handlers_write_autochunk.go | 31 +++++++++++++++---- 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/weed/server/filer_server_handlers_write_autochunk.go b/weed/server/filer_server_handlers_write_autochunk.go index 82c6c11b4..eee39152b 100644 --- a/weed/server/filer_server_handlers_write_autochunk.go +++ b/weed/server/filer_server_handlers_write_autochunk.go @@ -209,17 +209,36 @@ func (fs *FilerServer) uploadReaderToChunks(w http.ResponseWriter, r *http.Reque for { limitedReader := io.LimitReader(partReader, int64(chunkSize)) - // assign one file id for one chunk - fileId, urlLocation, auth, assignErr := fs.assignNewFileInfo(so) - if assignErr != nil { - return nil, nil, 0, assignErr, nil + data, err := ioutil.ReadAll(limitedReader) + if err != nil { + return nil, nil, 0, err, nil } + dataReader := util.NewBytesReader(data) + + // retry to assign a different file id + var fileId, urlLocation string + var auth security.EncodedJwt + var assignErr, uploadErr error + var uploadResult *operation.UploadResult + for i := 0; i < 3; i++ { + // assign one file id for one chunk + fileId, urlLocation, auth, assignErr = fs.assignNewFileInfo(so) + if assignErr != nil { + return nil, nil, 0, assignErr, nil + } - // upload the chunk to the volume server - uploadResult, uploadErr, data := fs.doUpload(urlLocation, w, r, limitedReader, fileName, contentType, nil, auth) + // upload the chunk to the volume server + uploadResult, uploadErr, _ = fs.doUpload(urlLocation, w, r, dataReader, fileName, contentType, nil, auth) + if uploadErr != nil { + time.Sleep(251 * time.Millisecond) + continue + } + break + } if uploadErr != nil { return nil, nil, 0, uploadErr, nil } + content = data // if last chunk exhausted the reader exactly at the border From 515b7632d7531727fceca340e42d2aff126306b0 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 25 Dec 2020 23:58:22 -0800 Subject: [PATCH 018/260] 2.17 --- k8s/seaweedfs/values.yaml | 2 +- weed/util/constants.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/k8s/seaweedfs/values.yaml b/k8s/seaweedfs/values.yaml index 7667d2815..0e8ce3a35 100644 --- a/k8s/seaweedfs/values.yaml +++ b/k8s/seaweedfs/values.yaml @@ -4,7 +4,7 @@ global: registry: "" repository: "" imageName: chrislusf/seaweedfs - imageTag: "2.16" + imageTag: "2.17" imagePullPolicy: IfNotPresent imagePullSecrets: imagepullsecret restartPolicy: Always diff --git a/weed/util/constants.go b/weed/util/constants.go index 89155e9a2..95370746b 100644 --- a/weed/util/constants.go +++ b/weed/util/constants.go @@ -5,7 +5,7 @@ import ( ) var ( - VERSION = fmt.Sprintf("%s %d.%02d", sizeLimit, 2, 16) + VERSION = fmt.Sprintf("%s %d.%02d", sizeLimit, 2, 17) COMMIT = "" ) From 0a7c5f85a966460a034de3ef782f54323f6f8238 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 26 Dec 2020 15:05:31 -0800 Subject: [PATCH 019/260] filer: add namePattern to search in current folder --- weed/filer/filer.go | 17 ---- weed/filer/filer_search.go | 79 +++++++++++++++++++ weed/server/filer_server_handlers_read_dir.go | 3 +- 3 files changed, 81 insertions(+), 18 deletions(-) create mode 100644 weed/filer/filer_search.go diff --git a/weed/filer/filer.go b/weed/filer/filer.go index 800dd35dc..13dedea1e 100644 --- a/weed/filer/filer.go +++ b/weed/filer/filer.go @@ -280,23 +280,6 @@ func (f *Filer) FindEntry(ctx context.Context, p util.FullPath) (entry *Entry, e } -func (f *Filer) ListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int, prefix string) ([]*Entry, error) { - if strings.HasSuffix(string(p), "/") && len(p) > 1 { - p = p[0 : len(p)-1] - } - - var makeupEntries []*Entry - entries, expiredCount, lastFileName, err := f.doListDirectoryEntries(ctx, p, startFileName, inclusive, limit, prefix) - for expiredCount > 0 && err == nil { - makeupEntries, expiredCount, lastFileName, err = f.doListDirectoryEntries(ctx, p, lastFileName, false, expiredCount, prefix) - if err == nil { - entries = append(entries, makeupEntries...) - } - } - - return entries, err -} - func (f *Filer) doListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*Entry, expiredCount int, lastFileName string, err error) { listedEntries, listErr := f.Store.ListDirectoryPrefixedEntries(ctx, p, startFileName, inclusive, limit, prefix) if listErr != nil { diff --git a/weed/filer/filer_search.go b/weed/filer/filer_search.go new file mode 100644 index 000000000..9622b51eb --- /dev/null +++ b/weed/filer/filer_search.go @@ -0,0 +1,79 @@ +package filer + +import ( + "context" + "github.com/chrislusf/seaweedfs/weed/util" + "path/filepath" + "strings" +) + +func splitPattern(pattern string) (prefix string, restPattern string, hasUpper bool) { + for i := 0; i < len(pattern); i++ { + hasUpper = hasUpper || ('A' <= pattern[i] && pattern[i] <= 'Z') + } + position := strings.Index(pattern, "*") + if position >= 0 { + return pattern[:position], pattern[position:], hasUpper + } + position = strings.Index(pattern, "?") + if position >= 0 { + return pattern[:position], pattern[position:], hasUpper + } + return "", restPattern, hasUpper +} + +func (f *Filer) ListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int, namePattern string) (entries []*Entry, err error) { + if strings.HasSuffix(string(p), "/") && len(p) > 1 { + p = p[0 : len(p)-1] + } + + prefix, restNamePattern, hasUpper := splitPattern(namePattern) + var missedCount int + var lastFileName string + + entries, missedCount, lastFileName, err = f.doListPatternMatchedEntries(ctx, p, startFileName, inclusive, limit, prefix, restNamePattern, hasUpper) + + for missedCount > 0 && err == nil { + var makeupEntries []*Entry + makeupEntries, missedCount, lastFileName, err = f.doListPatternMatchedEntries(ctx, p, lastFileName, false, missedCount, prefix, restNamePattern, hasUpper) + for _, entry := range makeupEntries { + entries = append(entries, entry) + } + } + + return entries, err +} + +func (f *Filer) doListPatternMatchedEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int, prefix, restNamePattern string, hasUpper bool) (matchedEntries []*Entry, missedCount int, lastFileName string, err error) { + var foundEntries []*Entry + + foundEntries, lastFileName, err = f.doListValidEntries(ctx, p, startFileName, inclusive, limit, prefix) + if err != nil { + return + } + if len(restNamePattern) == 0 { + return foundEntries, 0, lastFileName, nil + } + for _, entry := range foundEntries { + nameToTest := strings.ToLower(entry.Name()) + if matched, matchErr := filepath.Match(restNamePattern, nameToTest[len(prefix):]); matchErr == nil && matched { + matchedEntries = append(matchedEntries, entry) + } else { + missedCount++ + } + } + return +} + +func (f *Filer) doListValidEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*Entry, lastFileName string, err error) { + var makeupEntries []*Entry + var expiredCount int + entries, expiredCount, lastFileName, err = f.doListDirectoryEntries(ctx, p, startFileName, inclusive, limit, prefix) + for expiredCount > 0 && err == nil { + makeupEntries, expiredCount, lastFileName, err = f.doListDirectoryEntries(ctx, p, lastFileName, false, expiredCount, prefix) + if err == nil { + entries = append(entries, makeupEntries...) + } + } + return +} diff --git a/weed/server/filer_server_handlers_read_dir.go b/weed/server/filer_server_handlers_read_dir.go index 99345550c..f303ba1d4 100644 --- a/weed/server/filer_server_handlers_read_dir.go +++ b/weed/server/filer_server_handlers_read_dir.go @@ -34,8 +34,9 @@ func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Reque } lastFileName := r.FormValue("lastFileName") + namePattern := r.FormValue("namePattern") - entries, err := fs.filer.ListDirectoryEntries(context.Background(), util.FullPath(path), lastFileName, false, limit, "") + entries, err := fs.filer.ListDirectoryEntries(context.Background(), util.FullPath(path), lastFileName, false, limit, namePattern) if err != nil { glog.V(0).Infof("listDirectory %s %s %d: %s", path, lastFileName, limit, err) From da87f6b2653fbfff88a1c345d703a063155d76ef Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 26 Dec 2020 15:21:12 -0800 Subject: [PATCH 020/260] remove unused code --- weed/filer/filer_search.go | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/weed/filer/filer_search.go b/weed/filer/filer_search.go index 9622b51eb..b26959cb0 100644 --- a/weed/filer/filer_search.go +++ b/weed/filer/filer_search.go @@ -7,19 +7,16 @@ import ( "strings" ) -func splitPattern(pattern string) (prefix string, restPattern string, hasUpper bool) { - for i := 0; i < len(pattern); i++ { - hasUpper = hasUpper || ('A' <= pattern[i] && pattern[i] <= 'Z') - } +func splitPattern(pattern string) (prefix string, restPattern string) { position := strings.Index(pattern, "*") if position >= 0 { - return pattern[:position], pattern[position:], hasUpper + return pattern[:position], pattern[position:] } position = strings.Index(pattern, "?") if position >= 0 { - return pattern[:position], pattern[position:], hasUpper + return pattern[:position], pattern[position:] } - return "", restPattern, hasUpper + return "", restPattern } func (f *Filer) ListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int, namePattern string) (entries []*Entry, err error) { @@ -27,15 +24,15 @@ func (f *Filer) ListDirectoryEntries(ctx context.Context, p util.FullPath, start p = p[0 : len(p)-1] } - prefix, restNamePattern, hasUpper := splitPattern(namePattern) + prefix, restNamePattern := splitPattern(namePattern) var missedCount int var lastFileName string - entries, missedCount, lastFileName, err = f.doListPatternMatchedEntries(ctx, p, startFileName, inclusive, limit, prefix, restNamePattern, hasUpper) + entries, missedCount, lastFileName, err = f.doListPatternMatchedEntries(ctx, p, startFileName, inclusive, limit, prefix, restNamePattern) for missedCount > 0 && err == nil { var makeupEntries []*Entry - makeupEntries, missedCount, lastFileName, err = f.doListPatternMatchedEntries(ctx, p, lastFileName, false, missedCount, prefix, restNamePattern, hasUpper) + makeupEntries, missedCount, lastFileName, err = f.doListPatternMatchedEntries(ctx, p, lastFileName, false, missedCount, prefix, restNamePattern) for _, entry := range makeupEntries { entries = append(entries, entry) } @@ -44,7 +41,7 @@ func (f *Filer) ListDirectoryEntries(ctx context.Context, p util.FullPath, start return entries, err } -func (f *Filer) doListPatternMatchedEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int, prefix, restNamePattern string, hasUpper bool) (matchedEntries []*Entry, missedCount int, lastFileName string, err error) { +func (f *Filer) doListPatternMatchedEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int, prefix, restNamePattern string) (matchedEntries []*Entry, missedCount int, lastFileName string, err error) { var foundEntries []*Entry foundEntries, lastFileName, err = f.doListValidEntries(ctx, p, startFileName, inclusive, limit, prefix) From aa020ee7e765f200976cd513d296530bbe3704a8 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 26 Dec 2020 22:01:16 -0800 Subject: [PATCH 021/260] s3: restore V2 auth type The added headers were included in calculating the signatures, failing the comparison --- weed/s3api/http/header.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/weed/s3api/http/header.go b/weed/s3api/http/header.go index a960d2370..6614b0af0 100644 --- a/weed/s3api/http/header.go +++ b/weed/s3api/http/header.go @@ -31,6 +31,6 @@ const ( // Non-Standard S3 HTTP request constants const ( - AmzIdentityId = "x-amz-identity-id" - AmzIsAdmin = "x-amz-is-admin" // only set to http request header as a context + AmzIdentityId = "s3-identity-id" + AmzIsAdmin = "s3-is-admin" // only set to http request header as a context ) From 561a2ff0bcdb72f320b1d3a81da63dd6beb41548 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 27 Dec 2020 10:50:27 -0800 Subject: [PATCH 022/260] s3: add option for "alllowEmptyFolder" --- weed/command/s3.go | 17 ++++++++++------- weed/s3api/s3api_objects_list_handlers.go | 6 ++++-- weed/s3api/s3api_server.go | 1 + 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/weed/command/s3.go b/weed/command/s3.go index ed5bb0b80..d8e3e306b 100644 --- a/weed/command/s3.go +++ b/weed/command/s3.go @@ -23,13 +23,14 @@ var ( ) type S3Options struct { - filer *string - port *int - config *string - domainName *string - tlsPrivateKey *string - tlsCertificate *string - metricsHttpPort *int + filer *string + port *int + config *string + domainName *string + tlsPrivateKey *string + tlsCertificate *string + metricsHttpPort *int + allowEmptyFolder *bool } func init() { @@ -41,6 +42,7 @@ func init() { s3StandaloneOptions.tlsPrivateKey = cmdS3.Flag.String("key.file", "", "path to the TLS private key file") s3StandaloneOptions.tlsCertificate = cmdS3.Flag.String("cert.file", "", "path to the TLS certificate file") s3StandaloneOptions.metricsHttpPort = cmdS3.Flag.Int("metricsPort", 0, "Prometheus metrics listen port") + s3StandaloneOptions.allowEmptyFolder = cmdS3.Flag.Bool("allowEmptyFolder", false, "allow empty folders") } var cmdS3 = &Command{ @@ -181,6 +183,7 @@ func (s3opt *S3Options) startS3Server() bool { DomainName: *s3opt.domainName, BucketsPath: filerBucketsPath, GrpcDialOption: grpcDialOption, + AllowEmptyFolder: *s3opt.allowEmptyFolder, }) if s3ApiServer_err != nil { glog.Fatalf("S3 API Server startup error: %v", s3ApiServer_err) diff --git a/weed/s3api/s3api_objects_list_handlers.go b/weed/s3api/s3api_objects_list_handlers.go index c1c6e2f89..fb1497b78 100644 --- a/weed/s3api/s3api_objects_list_handlers.go +++ b/weed/s3api/s3api_objects_list_handlers.go @@ -264,8 +264,10 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d } } else { var isEmpty bool - if isEmpty, err = s3a.isDirectoryAllEmpty(client, dir, entry.Name); err != nil { - glog.Errorf("check empty folder %s: %v", dir, err) + if !s3a.option.AllowEmptyFolder { + if isEmpty, err = s3a.isDirectoryAllEmpty(client, dir, entry.Name); err != nil { + glog.Errorf("check empty folder %s: %v", dir, err) + } } if !isEmpty { eachEntryFn(dir, entry) diff --git a/weed/s3api/s3api_server.go b/weed/s3api/s3api_server.go index e4a07a443..4993104ae 100644 --- a/weed/s3api/s3api_server.go +++ b/weed/s3api/s3api_server.go @@ -20,6 +20,7 @@ type S3ApiServerOption struct { DomainName string BucketsPath string GrpcDialOption grpc.DialOption + AllowEmptyFolder bool } type S3ApiServer struct { From 7a70caac3cffdc049790df805ae101588fbffc0f Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 27 Dec 2020 11:03:41 -0800 Subject: [PATCH 023/260] Update README.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 923331853..70e2ba0b2 100644 --- a/README.md +++ b/README.md @@ -84,7 +84,7 @@ There is only 40 bytes of disk storage overhead for each file's metadata. It is SeaweedFS started by implementing [Facebook's Haystack design paper](http://www.usenix.org/event/osdi10/tech/full_papers/Beaver.pdf). Also, SeaweedFS implements erasure coding with ideas from [f4: Facebook’s Warm BLOB Storage System](https://www.usenix.org/system/files/conference/osdi14/osdi14-paper-muralidhar.pdf) -On top of the object store, optional [Filer] can support directories and POSIX attributes. Filer is a separate linearly-scalable stateless server with customizable metadata stores, e.g., MySql, Postgres, Mongodb, Redis, Cassandra, Elastic Search, LevelDB, MemSql, TiDB, Etcd, CockroachDB, etc. +On top of the object store, optional [Filer] can support directories and POSIX attributes. Filer is a separate linearly-scalable stateless server with customizable metadata stores, e.g., MySql, Postgres, Redis, Cassandra, HBase, Mongodb, Elastic Search, LevelDB, MemSql, TiDB, Etcd, CockroachDB, etc. [Back to TOC](#table-of-contents) @@ -365,7 +365,7 @@ The architectures are mostly the same. SeaweedFS aims to store and read files fa * SeaweedFS optimizes for small files, ensuring O(1) disk seek operation, and can also handle large files. * SeaweedFS statically assigns a volume id for a file. Locating file content becomes just a lookup of the volume id, which can be easily cached. -* SeaweedFS Filer metadata store can be any well-known and proven data stores, e.g., Cassandra, Mongodb, Redis, Elastic Search, MySql, Postgres, MemSql, TiDB, CockroachDB, Etcd etc, and is easy to customized. +* SeaweedFS Filer metadata store can be any well-known and proven data stores, e.g., Redis, Cassandra, HBase, Mongodb, Elastic Search, MySql, Postgres, MemSql, TiDB, CockroachDB, Etcd etc, and is easy to customized. * SeaweedFS Volume server also communicates directly with clients via HTTP, supporting range queries, direct uploads, etc. | System | File Metadata | File Content Read| POSIX | REST API | Optimized for large number of small files | @@ -407,7 +407,7 @@ Ceph uses CRUSH hashing to automatically manage the data placement. SeaweedFS pl SeaweedFS is optimized for small files. Small files are stored as one continuous block of content, with at most 8 unused bytes between files. Small file access is O(1) disk read. -SeaweedFS Filer uses off-the-shelf stores, such as MySql, Postgres, Mongodb, Redis, Elastic Search, Cassandra, MemSql, TiDB, CockroachCB, Etcd, to manage file directories. These stores are proven, scalable, and easier to manage. +SeaweedFS Filer uses off-the-shelf stores, such as MySql, Postgres, Mongodb, Redis, Elastic Search, Cassandra, HBase, MemSql, TiDB, CockroachCB, Etcd, to manage file directories. These stores are proven, scalable, and easier to manage. | SeaweedFS | comparable to Ceph | advantage | | ------------- | ------------- | ---------------- | From 7458ff952311deff8aa3ca1a8ca13f788e311dce Mon Sep 17 00:00:00 2001 From: henry Date: Mon, 28 Dec 2020 11:28:33 +0800 Subject: [PATCH 024/260] Solve the Chinese name disorder of download file --- weed/server/filer_server_handlers_read.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.go index d55bf7cbb..4d61193ec 100644 --- a/weed/server/filer_server_handlers_read.go +++ b/weed/server/filer_server_handlers_read.go @@ -6,6 +6,7 @@ import ( "io" "mime" "net/http" + "net/url" "path/filepath" "strconv" "strings" @@ -121,6 +122,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, setEtag(w, etag) filename := entry.Name() + filename = url.QueryEscape(filename) adjustHeaderContentDisposition(w, r, filename) totalSize := int64(entry.Size()) From da7e5aaa65fc6e54bfdb8829d95d3e5f46bf2b30 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 27 Dec 2020 20:46:15 -0800 Subject: [PATCH 025/260] filer: optimize for less number of directory lookup bottom up directory lookup --- weed/filer/filer.go | 131 ++++++++++++++++++++++---------------------- 1 file changed, 66 insertions(+), 65 deletions(-) diff --git a/weed/filer/filer.go b/weed/filer/filer.go index 13dedea1e..920d79da5 100644 --- a/weed/filer/filer.go +++ b/weed/filer/filer.go @@ -134,69 +134,7 @@ func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, o_excl bool, isFr return nil } - dirParts := strings.Split(string(entry.FullPath), "/") - - // fmt.Printf("directory parts: %+v\n", dirParts) - - var lastDirectoryEntry *Entry - - for i := 1; i < len(dirParts); i++ { - dirPath := "/" + util.Join(dirParts[:i]...) - // fmt.Printf("%d directory: %+v\n", i, dirPath) - - // check the store directly - glog.V(4).Infof("find uncached directory: %s", dirPath) - dirEntry, _ := f.FindEntry(ctx, util.FullPath(dirPath)) - - // no such existing directory - if dirEntry == nil { - - // create the directory - now := time.Now() - - dirEntry = &Entry{ - FullPath: util.FullPath(dirPath), - Attr: Attr{ - Mtime: now, - Crtime: now, - Mode: os.ModeDir | entry.Mode | 0110, - Uid: entry.Uid, - Gid: entry.Gid, - Collection: entry.Collection, - Replication: entry.Replication, - UserName: entry.UserName, - GroupNames: entry.GroupNames, - }, - } - - glog.V(2).Infof("create directory: %s %v", dirPath, dirEntry.Mode) - mkdirErr := f.Store.InsertEntry(ctx, dirEntry) - if mkdirErr != nil { - if _, err := f.FindEntry(ctx, util.FullPath(dirPath)); err == filer_pb.ErrNotFound { - glog.V(3).Infof("mkdir %s: %v", dirPath, mkdirErr) - return fmt.Errorf("mkdir %s: %v", dirPath, mkdirErr) - } - } else { - f.maybeAddBucket(dirEntry) - f.NotifyUpdateEvent(ctx, nil, dirEntry, false, isFromOtherCluster, nil) - } - - } else if !dirEntry.IsDirectory() { - glog.Errorf("CreateEntry %s: %s should be a directory", entry.FullPath, dirPath) - return fmt.Errorf("%s is a file", dirPath) - } - - // remember the direct parent directory entry - if i == len(dirParts)-1 { - lastDirectoryEntry = dirEntry - } - - } - - if lastDirectoryEntry == nil { - glog.Errorf("CreateEntry %s: lastDirectoryEntry is nil", entry.FullPath) - return fmt.Errorf("parent folder not found: %v", entry.FullPath) - } + oldEntry, _ := f.FindEntry(ctx, entry.FullPath) /* if !hasWritePermission(lastDirectoryEntry, entry) { @@ -206,9 +144,13 @@ func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, o_excl bool, isFr } */ - oldEntry, _ := f.FindEntry(ctx, entry.FullPath) - if oldEntry == nil { + + dirParts := strings.Split(string(entry.FullPath), "/") + if err := f.ensureParentDirecotryEntry(ctx, entry, dirParts, len(dirParts)-1, isFromOtherCluster); err != nil { + return err + } + glog.V(4).Infof("InsertEntry %s: new entry: %v", entry.FullPath, entry.Name()) if err := f.Store.InsertEntry(ctx, entry); err != nil { glog.Errorf("insert entry %s: %v", entry.FullPath, err) @@ -236,6 +178,65 @@ func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, o_excl bool, isFr return nil } +func (f *Filer) ensureParentDirecotryEntry(ctx context.Context, entry *Entry, dirParts []string, level int, isFromOtherCluster bool) (err error) { + + if level == 0 { + return nil + } + + dirPath := "/" + util.Join(dirParts[:level]...) + // fmt.Printf("%d directory: %+v\n", i, dirPath) + + // check the store directly + glog.V(4).Infof("find uncached directory: %s", dirPath) + dirEntry, _ := f.FindEntry(ctx, util.FullPath(dirPath)) + + // no such existing directory + if dirEntry == nil { + + // ensure parent directory + if err = f.ensureParentDirecotryEntry(ctx, entry, dirParts, level-1, isFromOtherCluster); err != nil { + return err + } + + // create the directory + now := time.Now() + + dirEntry = &Entry{ + FullPath: util.FullPath(dirPath), + Attr: Attr{ + Mtime: now, + Crtime: now, + Mode: os.ModeDir | entry.Mode | 0110, + Uid: entry.Uid, + Gid: entry.Gid, + Collection: entry.Collection, + Replication: entry.Replication, + UserName: entry.UserName, + GroupNames: entry.GroupNames, + }, + } + + glog.V(2).Infof("create directory: %s %v", dirPath, dirEntry.Mode) + mkdirErr := f.Store.InsertEntry(ctx, dirEntry) + if mkdirErr != nil { + if _, err := f.FindEntry(ctx, util.FullPath(dirPath)); err == filer_pb.ErrNotFound { + glog.V(3).Infof("mkdir %s: %v", dirPath, mkdirErr) + return fmt.Errorf("mkdir %s: %v", dirPath, mkdirErr) + } + } else { + f.maybeAddBucket(dirEntry) + f.NotifyUpdateEvent(ctx, nil, dirEntry, false, isFromOtherCluster, nil) + } + + } else if !dirEntry.IsDirectory() { + glog.Errorf("CreateEntry %s: %s should be a directory", entry.FullPath, dirPath) + return fmt.Errorf("%s is a file", dirPath) + } + + return nil +} + func (f *Filer) UpdateEntry(ctx context.Context, oldEntry, entry *Entry) (err error) { if oldEntry != nil { entry.Attr.Crtime = oldEntry.Attr.Crtime From 90df52d001ba116a622d14fb3ed7c78c7b4f2225 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 27 Dec 2020 21:09:06 -0800 Subject: [PATCH 026/260] s3: add command option "allowEmptyFolder" to filer and server --- weed/command/filer.go | 1 + weed/command/server.go | 1 + 2 files changed, 2 insertions(+) diff --git a/weed/command/filer.go b/weed/command/filer.go index a3008eb29..146840e00 100644 --- a/weed/command/filer.go +++ b/weed/command/filer.go @@ -74,6 +74,7 @@ func init() { filerS3Options.tlsPrivateKey = cmdFiler.Flag.String("s3.key.file", "", "path to the TLS private key file") filerS3Options.tlsCertificate = cmdFiler.Flag.String("s3.cert.file", "", "path to the TLS certificate file") filerS3Options.config = cmdFiler.Flag.String("s3.config", "", "path to the config file") + filerS3Options.allowEmptyFolder = cmdFiler.Flag.Bool("s3.allowEmptyFolder", false, "allow empty folders") } var cmdFiler = &Command{ diff --git a/weed/command/server.go b/weed/command/server.go index 7e63f8e8a..bd25f94b1 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -113,6 +113,7 @@ func init() { s3Options.tlsPrivateKey = cmdServer.Flag.String("s3.key.file", "", "path to the TLS private key file") s3Options.tlsCertificate = cmdServer.Flag.String("s3.cert.file", "", "path to the TLS certificate file") s3Options.config = cmdServer.Flag.String("s3.config", "", "path to the config file") + s3Options.allowEmptyFolder = cmdServer.Flag.Bool("s3.allowEmptyFolder", false, "allow empty folders") msgBrokerOptions.port = cmdServer.Flag.Int("msgBroker.port", 17777, "broker gRPC listen port") From 6d2c979999363763090a686ae995e3582e006453 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 27 Dec 2020 21:09:45 -0800 Subject: [PATCH 027/260] s3: fix regression fix https://github.com/chrislusf/seaweedfs/issues/1707 --- weed/s3api/auth_credentials.go | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/weed/s3api/auth_credentials.go b/weed/s3api/auth_credentials.go index c305fee6f..b8af6381a 100644 --- a/weed/s3api/auth_credentials.go +++ b/weed/s3api/auth_credentials.go @@ -156,7 +156,36 @@ func (iam *IdentityAccessManagement) Auth(f http.HandlerFunc, action Action) htt // check whether the request has valid access keys func (iam *IdentityAccessManagement) authRequest(r *http.Request, action Action) (*Identity, s3err.ErrorCode) { - identity, s3Err := iam.authUser(r) + var identity *Identity + var s3Err s3err.ErrorCode + var found bool + switch getRequestAuthType(r) { + case authTypeStreamingSigned: + return identity, s3err.ErrNone + case authTypeUnknown: + glog.V(3).Infof("unknown auth type") + return identity, s3err.ErrAccessDenied + case authTypePresignedV2, authTypeSignedV2: + glog.V(3).Infof("v2 auth type") + identity, s3Err = iam.isReqAuthenticatedV2(r) + case authTypeSigned, authTypePresigned: + glog.V(3).Infof("v4 auth type") + identity, s3Err = iam.reqSignatureV4Verify(r) + case authTypePostPolicy: + glog.V(3).Infof("post policy auth type") + return identity, s3err.ErrNone + case authTypeJWT: + glog.V(3).Infof("jwt auth type") + return identity, s3err.ErrNotImplemented + case authTypeAnonymous: + identity, found = iam.lookupAnonymous() + if !found { + return identity, s3err.ErrAccessDenied + } + default: + return identity, s3err.ErrNotImplemented + } + if s3Err != s3err.ErrNone { return identity, s3Err } From 2e6cf541e7bac8f21224857449720b6b836bc4e3 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 28 Dec 2020 00:18:37 -0800 Subject: [PATCH 028/260] filer: avoid deletion error when item is not found --- weed/server/filer_grpc_server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/server/filer_grpc_server.go b/weed/server/filer_grpc_server.go index 5f1b2d819..38472df2a 100644 --- a/weed/server/filer_grpc_server.go +++ b/weed/server/filer_grpc_server.go @@ -326,7 +326,7 @@ func (fs *FilerServer) DeleteEntry(ctx context.Context, req *filer_pb.DeleteEntr err = fs.filer.DeleteEntryMetaAndData(ctx, util.JoinPath(req.Directory, req.Name), req.IsRecursive, req.IgnoreRecursiveError, req.IsDeleteData, req.IsFromOtherCluster, req.Signatures) resp = &filer_pb.DeleteEntryResponse{} - if err != nil { + if err != nil && err != filer_pb.ErrNotFound { resp.Error = err.Error() } return resp, nil From d2ead72918726eeac01e2ba9cc63765a24d99d5c Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 28 Dec 2020 12:37:10 -0800 Subject: [PATCH 029/260] 2.18 --- k8s/seaweedfs/values.yaml | 2 +- weed/util/constants.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/k8s/seaweedfs/values.yaml b/k8s/seaweedfs/values.yaml index 0e8ce3a35..7144b0016 100644 --- a/k8s/seaweedfs/values.yaml +++ b/k8s/seaweedfs/values.yaml @@ -4,7 +4,7 @@ global: registry: "" repository: "" imageName: chrislusf/seaweedfs - imageTag: "2.17" + imageTag: "2.18" imagePullPolicy: IfNotPresent imagePullSecrets: imagepullsecret restartPolicy: Always diff --git a/weed/util/constants.go b/weed/util/constants.go index 95370746b..08d169545 100644 --- a/weed/util/constants.go +++ b/weed/util/constants.go @@ -5,7 +5,7 @@ import ( ) var ( - VERSION = fmt.Sprintf("%s %d.%02d", sizeLimit, 2, 17) + VERSION = fmt.Sprintf("%s %d.%02d", sizeLimit, 2, 18) COMMIT = "" ) From efd4253ab664d10def4b4721a8385ca33a025ad4 Mon Sep 17 00:00:00 2001 From: nivekuil Date: Mon, 28 Dec 2020 13:54:02 -0800 Subject: [PATCH 030/260] Add fuse to all dockerfiles --- docker/Dockerfile.go_build | 1 + docker/Dockerfile.go_build_large | 1 + docker/Dockerfile.local | 1 + 3 files changed, 3 insertions(+) diff --git a/docker/Dockerfile.go_build b/docker/Dockerfile.go_build index 726046b56..d9ebeff48 100644 --- a/docker/Dockerfile.go_build +++ b/docker/Dockerfile.go_build @@ -14,6 +14,7 @@ COPY --from=builder /root/go/bin/weed /usr/bin/ RUN mkdir -p /etc/seaweedfs COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/entrypoint.sh /entrypoint.sh +RUN apk add fuse # for weed mount # volume server gprc port EXPOSE 18080 diff --git a/docker/Dockerfile.go_build_large b/docker/Dockerfile.go_build_large index 8fc85e868..d5e9bb4e1 100644 --- a/docker/Dockerfile.go_build_large +++ b/docker/Dockerfile.go_build_large @@ -14,6 +14,7 @@ COPY --from=builder /root/go/bin/weed /usr/bin/ RUN mkdir -p /etc/seaweedfs COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/entrypoint.sh /entrypoint.sh +RUN apk add fuse # for weed mount # volume server gprc port EXPOSE 18080 diff --git a/docker/Dockerfile.local b/docker/Dockerfile.local index 693d8a952..199c21e81 100644 --- a/docker/Dockerfile.local +++ b/docker/Dockerfile.local @@ -4,6 +4,7 @@ COPY ./weed /usr/bin/ RUN mkdir -p /etc/seaweedfs COPY ./filer.toml /etc/seaweedfs/filer.toml COPY ./entrypoint.sh /entrypoint.sh +RUN apk add fuse # for weed mount # volume server grpc port EXPOSE 18080 From 89977123b6da298e07c429f78e68c0529fe611c5 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 28 Dec 2020 15:07:16 -0800 Subject: [PATCH 031/260] shell: add shell.toml support fix https://github.com/chrislusf/seaweedfs/issues/1664 --- weed/command/scaffold.go | 16 ++++++++++++++++ weed/command/shell.go | 25 +++++++++++++++++++++++-- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/weed/command/scaffold.go b/weed/command/scaffold.go index 6cfd46427..d1b4b27ac 100644 --- a/weed/command/scaffold.go +++ b/weed/command/scaffold.go @@ -44,6 +44,8 @@ func runScaffold(cmd *Command, args []string) bool { content = SECURITY_TOML_EXAMPLE case "master": content = MASTER_TOML_EXAMPLE + case "shell": + content = SHELL_TOML_EXAMPLE } if content == "" { println("need a valid -config option") @@ -459,5 +461,19 @@ copy_other = 1 # create n x 1 = n actual volumes # if you are doing your own replication or periodic sync of volumes. treat_replication_as_minimums = false +` + SHELL_TOML_EXAMPLE = ` + +[cluster] +default = "c1" + +[cluster.c1] +master = "localhost:9333" # comma-separated master servers +filer = "localhost:8888" # filer host and port + +[cluster.c2] +master = "" +filer = "" + ` ) diff --git a/weed/command/shell.go b/weed/command/shell.go index 6dd768f47..c9976e809 100644 --- a/weed/command/shell.go +++ b/weed/command/shell.go @@ -11,12 +11,14 @@ import ( var ( shellOptions shell.ShellOptions shellInitialFiler *string + shellCluster *string ) func init() { cmdShell.Run = runShell // break init cycle - shellOptions.Masters = cmdShell.Flag.String("master", "localhost:9333", "comma-separated master servers") - shellInitialFiler = cmdShell.Flag.String("filer", "localhost:8888", "filer host and port") + shellOptions.Masters = cmdShell.Flag.String("master", "", "comma-separated master servers, e.g. localhost:9333") + shellInitialFiler = cmdShell.Flag.String("filer", "", "filer host and port, e.g. localhost:8888") + shellCluster = cmdShell.Flag.String("cluster", "", "cluster defined in shell.toml") } var cmdShell = &Command{ @@ -24,6 +26,8 @@ var cmdShell = &Command{ Short: "run interactive administrative commands", Long: `run interactive administrative commands. + Generate shell.toml via "weed scaffold -config=shell" + `, } @@ -32,6 +36,23 @@ func runShell(command *Command, args []string) bool { util.LoadConfiguration("security", false) shellOptions.GrpcDialOption = security.LoadClientTLS(util.GetViper(), "grpc.client") + if *shellOptions.Masters == "" && *shellInitialFiler == "" { + util.LoadConfiguration("shell", false) + v := util.GetViper() + cluster := v.GetString("cluster.default") + if *shellCluster != "" { + cluster = *shellCluster + } + if cluster == "" { + *shellOptions.Masters, *shellInitialFiler = "localhost:9333", "localhost:8888" + } else { + *shellOptions.Masters = v.GetString("cluster." + cluster + ".master") + *shellInitialFiler = v.GetString("cluster." + cluster + ".filer") + } + } + + fmt.Printf("master: %s filer: %s\n", *shellOptions.Masters, *shellInitialFiler) + var err error shellOptions.FilerHost, shellOptions.FilerPort, err = util.ParseHostPort(*shellInitialFiler) if err != nil { From adef4ddc879dc01f2cffe86d0901f14adb36fd72 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 29 Dec 2020 11:51:38 -0800 Subject: [PATCH 032/260] mount: load security keys fix https://github.com/chrislusf/seaweedfs/issues/1714 --- weed/command/mount_std.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/weed/command/mount_std.go b/weed/command/mount_std.go index 83cb352ff..7e75b082d 100644 --- a/weed/command/mount_std.go +++ b/weed/command/mount_std.go @@ -58,6 +58,7 @@ func RunMount(option *MountOptions, umask os.FileMode) bool { return true } + util.LoadConfiguration("security", false) // try to connect to filer, filerBucketsPath may be useful later grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client") var cipher bool @@ -78,8 +79,6 @@ func RunMount(option *MountOptions, umask os.FileMode) bool { dir := util.ResolvePath(*option.dir) chunkSizeLimitMB := *mountOptions.chunkSizeLimitMB - util.LoadConfiguration("security", false) - fmt.Printf("This is SeaweedFS version %s %s %s\n", util.Version(), runtime.GOOS, runtime.GOARCH) if dir == "" { fmt.Printf("Please specify the mount directory via \"-dir\"") From 66ebc7c98fefd6803e9fc23bb7ad6beb18d9a088 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 29 Dec 2020 12:12:07 -0800 Subject: [PATCH 033/260] adjust actions --- .github/workflows/cleanup.yml | 22 ++++++++++++++++++++++ .github/workflows/release.yml | 10 +++------- 2 files changed, 25 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/cleanup.yml diff --git a/.github/workflows/cleanup.yml b/.github/workflows/cleanup.yml new file mode 100644 index 000000000..6c27689af --- /dev/null +++ b/.github/workflows/cleanup.yml @@ -0,0 +1,22 @@ +name: Release + +on: + push: + branches: [ master ] + +jobs: + + build: + name: Build + runs-on: ubuntu-latest + + steps: + + - name: Delete old release assets + uses: mknejp/delete-release-assets@v1 + with: + token: ${{ github.token }} + tag: dev + fail-if-no-assets: false + assets: | + weed-* diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 39108c30a..c4aa4e6b7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,14 +24,10 @@ jobs: - name: Check out code into the Go module directory uses: actions/checkout@v2 - - name: Delete old release assets - uses: mknejp/delete-release-assets@v1 + - name: Wait for the deletion + uses: jakejarvis/wait-action@master with: - token: ${{ github.token }} - tag: dev - fail-if-no-assets: false - assets: | - weed-* + time: '30s' - name: Set BUILD_TIME env run: echo BUILD_TIME=$(date -u +%Y-%m-%d-%H-%M) >> ${GITHUB_ENV} From c1cd6dd8d69c14d8048a409cc352e071b5821a3c Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 29 Dec 2020 12:13:39 -0800 Subject: [PATCH 034/260] rename --- .github/workflows/cleanup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cleanup.yml b/.github/workflows/cleanup.yml index 6c27689af..d8719c36f 100644 --- a/.github/workflows/cleanup.yml +++ b/.github/workflows/cleanup.yml @@ -1,4 +1,4 @@ -name: Release +name: Cleanup on: push: From adda3207c4d98532b6666d7da7a4dccdc26090be Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 29 Dec 2020 21:15:09 -0800 Subject: [PATCH 035/260] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 70e2ba0b2..f705e4ad7 100644 --- a/README.md +++ b/README.md @@ -335,7 +335,7 @@ Usually hot data are fresh and warm data are old. SeaweedFS puts the newly creat With the O(1) access time, the network latency cost is kept at minimum. -If the hot~warm data is split as 20~80, with 20 servers, you can achieve storage capacity of 100 servers. That's a cost saving of 80%! Or you can repurpose the 80 servers to store new data also, and get 5X storage throughput. +If the hot/warm data is split as 20/80, with 20 servers, you can achieve storage capacity of 100 servers. That's a cost saving of 80%! Or you can repurpose the 80 servers to store new data also, and get 5X storage throughput. [Back to TOC](#table-of-contents) From 1fc37a68c412718ef59997b855fcce9af7d77d46 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 29 Dec 2020 21:23:59 -0800 Subject: [PATCH 036/260] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f705e4ad7..53147b9c0 100644 --- a/README.md +++ b/README.md @@ -403,7 +403,7 @@ SeaweedFS has a centralized master group to look up free volumes, while Ceph use Same as SeaweedFS, Ceph is also based on the object store RADOS. Ceph is rather complicated with mixed reviews. -Ceph uses CRUSH hashing to automatically manage the data placement. SeaweedFS places data by assigned volumes. +Ceph uses CRUSH hashing to automatically manage the data placement, which is efficient to locate the data. But the data has to be placed according to the CRUSH algorithm. Any wrong configuration would cause data loss. SeaweedFS places data by assigning them to any writable volumes. If writes to one volume failed, just pick another volume to write. Adding more volumes are also as simple as it can be. SeaweedFS is optimized for small files. Small files are stored as one continuous block of content, with at most 8 unused bytes between files. Small file access is O(1) disk read. From 0d8c1f976bf1b95e927bbe6d6605eebdf3e41820 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 29 Dec 2020 21:30:17 -0800 Subject: [PATCH 037/260] Update README.md --- README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 53147b9c0..d4377b6ec 100644 --- a/README.md +++ b/README.md @@ -117,7 +117,10 @@ On top of the object store, optional [Filer] can support directories and POSIX a * [WebDAV] accesses as a mapped drive on Mac and Windows, or from mobile devices. * [AES256-GCM Encrypted Storage][FilerDataEncryption] safely stores the encrypted data. * [Super Large Files][SuperLargeFiles] stores large or super large files in tens of TB. + +## Kubernetes ## * [Kubernetes CSI Driver][SeaweedFsCsiDriver] A Container Storage Interface (CSI) Driver. [![Docker Pulls](https://img.shields.io/docker/pulls/chrislusf/seaweedfs-csi-driver.svg?maxAge=4800)](https://hub.docker.com/r/chrislusf/seaweedfs-csi-driver/) +* [SeaweedFS Operator](https://github.com/seaweedfs/seaweedfs-operator) [Filer]: https://github.com/chrislusf/seaweedfs/wiki/Directories-and-Files [SuperLargeFiles]: https://github.com/chrislusf/seaweedfs/wiki/Data-Structure-for-Large-Files @@ -135,6 +138,7 @@ On top of the object store, optional [Filer] can support directories and POSIX a [ActiveActiveAsyncReplication]: https://github.com/chrislusf/seaweedfs/wiki/Filer-Active-Active-cross-cluster-continuous-synchronization [FilerStoreReplication]: https://github.com/chrislusf/seaweedfs/wiki/Filer-Store-Replication + [Back to TOC](#table-of-contents) ## Example: Using Seaweed Object Store ## @@ -436,9 +440,7 @@ MinIO has specific requirements on storage layout. It is not flexible to adjust ## Dev Plan ## -* More tools and documentation, on how to manage and scale the system. For example, how to move volumes, automatically balancing data, how to grow volumes, how to check system status, etc. -* Integrate with Kubernetes. build [SeaweedFS Operator](https://github.com/seaweedfs/seaweedfs-operator). -* Add ftp server. +* More tools and documentation, on how to manage and scale the system. * Read and write stream data. * Support structured data. From d1a05d261af33b2dd6eb3e1f3813dd054fb40943 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 29 Dec 2020 21:35:40 -0800 Subject: [PATCH 038/260] Update README.md --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d4377b6ec..01be6d791 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,8 @@ SeaweedFS started by implementing [Facebook's Haystack design paper](http://www. On top of the object store, optional [Filer] can support directories and POSIX attributes. Filer is a separate linearly-scalable stateless server with customizable metadata stores, e.g., MySql, Postgres, Redis, Cassandra, HBase, Mongodb, Elastic Search, LevelDB, MemSql, TiDB, Etcd, CockroachDB, etc. +For any distributed key value stores, the large values can be offloaded to SeaweedFS. With the fast access speed and linearly scalable capacity, SeaweedFS can work as a distributed [Key-Large-Value store][KeyLargeValueStore]. + [Back to TOC](#table-of-contents) ## Additional Features ## @@ -137,7 +139,7 @@ On top of the object store, optional [Filer] can support directories and POSIX a [SeaweedFsCsiDriver]: https://github.com/seaweedfs/seaweedfs-csi-driver [ActiveActiveAsyncReplication]: https://github.com/chrislusf/seaweedfs/wiki/Filer-Active-Active-cross-cluster-continuous-synchronization [FilerStoreReplication]: https://github.com/chrislusf/seaweedfs/wiki/Filer-Store-Replication - +[KeyLargeValueStore]: https://github.com/chrislusf/seaweedfs/wiki/Filer-as-a-Key-Large-Value-Store [Back to TOC](#table-of-contents) From 14ddd155082a455542a992220d6c61e658420c6b Mon Sep 17 00:00:00 2001 From: henry Date: Thu, 31 Dec 2020 00:03:22 +0800 Subject: [PATCH 039/260] 1 Distinguish between folders and common files, folders are listed first 2 Add some custom headers that need to be exposed --- weed/filer/filer_search.go | 12 +++++++++++- weed/server/filer_server_handlers_read.go | 10 ++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/weed/filer/filer_search.go b/weed/filer/filer_search.go index b26959cb0..2fe83b49d 100644 --- a/weed/filer/filer_search.go +++ b/weed/filer/filer_search.go @@ -38,7 +38,17 @@ func (f *Filer) ListDirectoryEntries(ctx context.Context, p util.FullPath, start } } - return entries, err + //Distinguish between folders and common files, folders are listed first + var folderEntries, commonEntries []*Entry + for i, _ := range entries { + if entries[i].IsDirectory() { + folderEntries = append(folderEntries, entries[i]) + } else { + commonEntries = append(commonEntries, entries[i]) + } + } + folderEntries = append(folderEntries, commonEntries...) + return folderEntries, err } func (f *Filer) doListPatternMatchedEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int, prefix, restNamePattern string) (matchedEntries []*Entry, missedCount int, lastFileName string, err error) { diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.go index 4d61193ec..28027a009 100644 --- a/weed/server/filer_server_handlers_read.go +++ b/weed/server/filer_server_handlers_read.go @@ -100,6 +100,16 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, w.Header().Set(k, string(v)) } + //Add some custom headers that need to be exposed + seaweedHeaders := []string{} + for i, _ := range w.Header() { + if strings.Contains(i, "Seaweed") { + seaweedHeaders = append(seaweedHeaders, i) + } + } + seaweedHeaders = append(seaweedHeaders, "Content-Disposition") + w.Header().Set("Access-Control-Expose-Headers", strings.Join(seaweedHeaders, ",")) + //set tag count if r.Method == "GET" { tagCount := 0 From 3433accb1bdb1f311ecba4b87740e853cdfcf93d Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 30 Dec 2020 15:46:07 -0800 Subject: [PATCH 040/260] weed upload: add TTL --- weed/operation/submit.go | 1 + 1 file changed, 1 insertion(+) diff --git a/weed/operation/submit.go b/weed/operation/submit.go index 25843c892..c8f979d83 100644 --- a/weed/operation/submit.go +++ b/weed/operation/submit.go @@ -69,6 +69,7 @@ func SubmitFiles(master string, grpcDialOption grpc.DialOption, files []FilePart file.Replication = replication file.Collection = collection file.DataCenter = dataCenter + file.Ttl = ttl results[index].Size, err = file.Upload(maxMB, master, usePublicUrl, ret.Auth, grpcDialOption) if err != nil { results[index].Error = err.Error() From cd2a9865edfe8c0c362d460312d363848fbef7f1 Mon Sep 17 00:00:00 2001 From: Peter Cai Date: Thu, 31 Dec 2020 11:32:45 +0800 Subject: [PATCH 041/260] filer: respect Content-Type on PUT This is needed for the S3 gateway to conform to the S3 protocol -- the server should always respect the Content-Type set by client. --- weed/server/filer_server_handlers_write_autochunk.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/weed/server/filer_server_handlers_write_autochunk.go b/weed/server/filer_server_handlers_write_autochunk.go index eee39152b..fe4e68140 100644 --- a/weed/server/filer_server_handlers_write_autochunk.go +++ b/weed/server/filer_server_handlers_write_autochunk.go @@ -111,7 +111,10 @@ func (fs *FilerServer) doPostAutoChunk(ctx context.Context, w http.ResponseWrite func (fs *FilerServer) doPutAutoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, chunkSize int32, so *operation.StorageOption) (filerResult *FilerPostResult, md5bytes []byte, replyerr error) { fileName := "" - contentType := "" + contentType := r.Header.Get("Content-Type") + if contentType == "application/octet-stream" { + contentType = "" + } fileChunks, md5Hash, chunkOffset, err, smallContent := fs.uploadReaderToChunks(w, r, r.Body, chunkSize, fileName, contentType, so) if err != nil { From 2a5fc7aa4def384b31ee83c60a27e15b180c7fbf Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 31 Dec 2020 02:13:19 -0800 Subject: [PATCH 042/260] Update release.yml --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c4aa4e6b7..55457ba6a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - goos: [linux, windows, darwin ] + goos: [linux, windows, darwin, freebsd ] goarch: [amd64, arm] exclude: - goarch: arm From 712563fbddc6b1d04f5019c4d406668c7808b912 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 31 Dec 2020 19:54:31 -0800 Subject: [PATCH 043/260] dev cluster adds fuse mount --- docker/Dockerfile.go_build | 2 +- docker/Dockerfile.go_build_large | 2 +- docker/local-dev-compose.yml | 12 +++++++++++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/docker/Dockerfile.go_build b/docker/Dockerfile.go_build index d9ebeff48..36d9fd07a 100644 --- a/docker/Dockerfile.go_build +++ b/docker/Dockerfile.go_build @@ -1,5 +1,5 @@ FROM frolvlad/alpine-glibc as builder -RUN apk add git go g++ +RUN apk add git go g++ fuse RUN mkdir -p /go/src/github.com/chrislusf/ RUN git clone https://github.com/chrislusf/seaweedfs /go/src/github.com/chrislusf/seaweedfs ARG BRANCH=${BRANCH:-master} diff --git a/docker/Dockerfile.go_build_large b/docker/Dockerfile.go_build_large index d5e9bb4e1..08cfbd547 100644 --- a/docker/Dockerfile.go_build_large +++ b/docker/Dockerfile.go_build_large @@ -1,5 +1,5 @@ FROM frolvlad/alpine-glibc as builder -RUN apk add git go g++ +RUN apk add git go g++ fuse RUN mkdir -p /go/src/github.com/chrislusf/ RUN git clone https://github.com/chrislusf/seaweedfs /go/src/github.com/chrislusf/seaweedfs ARG BRANCH=${BRANCH:-master} diff --git a/docker/local-dev-compose.yml b/docker/local-dev-compose.yml index f6fd0f4ce..91aee63f6 100644 --- a/docker/local-dev-compose.yml +++ b/docker/local-dev-compose.yml @@ -12,7 +12,7 @@ services: ports: - 8080:8080 - 18080:18080 - command: "volume -mserver=master:9333 -port=8080 -ip=volume" + command: "volume -mserver=master:9333 -port=8080 -ip=volume -preStopSeconds=1" depends_on: - master filer: @@ -33,3 +33,13 @@ services: - master - volume - filer + mount: + image: chrislusf/seaweedfs:local + privileged: true + cap_add: + - SYS_ADMIN + command: 'mount -filer="filer:8888" -dirAutoCreate -dir=/mnt/seaweedfs' + depends_on: + - master + - volume + - filer From 3fdf73e51449605180e7e062416c85ed2b4411e6 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 1 Jan 2021 20:20:35 -0800 Subject: [PATCH 044/260] add mem_limit for mount --- docker/local-dev-compose.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/local-dev-compose.yml b/docker/local-dev-compose.yml index 91aee63f6..752086acd 100644 --- a/docker/local-dev-compose.yml +++ b/docker/local-dev-compose.yml @@ -38,7 +38,8 @@ services: privileged: true cap_add: - SYS_ADMIN - command: 'mount -filer="filer:8888" -dirAutoCreate -dir=/mnt/seaweedfs' + mem_limit: 4096m + command: '-v=4 mount -filer="filer:8888" -dirAutoCreate -dir=/mnt/seaweedfs -cacheCapacityMB=0 ' depends_on: - master - volume From 2c3c2c27d7a900838734fac5c7cb3f4cc7340ea1 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 1 Jan 2021 20:23:23 -0800 Subject: [PATCH 045/260] separate prefix from namePattern fix https://github.com/chrislusf/seaweedfs/issues/1722 --- weed/filer/filer_buckets.go | 2 +- weed/filer/filer_delete_entry.go | 2 +- weed/filer/filer_notify.go | 4 ++-- weed/filer/filer_search.go | 8 ++++++-- weed/filer/leveldb/leveldb_store.go | 6 +++++- weed/filer/leveldb/leveldb_store_test.go | 6 +++--- weed/filer/leveldb2/leveldb2_store.go | 5 ++++- weed/filer/leveldb2/leveldb2_store_test.go | 6 +++--- weed/server/filer_grpc_server.go | 2 +- weed/server/filer_grpc_server_rename.go | 2 +- weed/server/filer_server_handlers_read_dir.go | 2 +- 11 files changed, 28 insertions(+), 17 deletions(-) diff --git a/weed/filer/filer_buckets.go b/weed/filer/filer_buckets.go index 4d4f4abc3..87ad9e452 100644 --- a/weed/filer/filer_buckets.go +++ b/weed/filer/filer_buckets.go @@ -29,7 +29,7 @@ func (f *Filer) LoadBuckets() { limit := math.MaxInt32 - entries, err := f.ListDirectoryEntries(context.Background(), util.FullPath(f.DirBucketsPath), "", false, limit, "") + entries, err := f.ListDirectoryEntries(context.Background(), util.FullPath(f.DirBucketsPath), "", false, limit, "", "") if err != nil { glog.V(1).Infof("no buckets found: %v", err) diff --git a/weed/filer/filer_delete_entry.go b/weed/filer/filer_delete_entry.go index da92c4f4b..0d1fd7e47 100644 --- a/weed/filer/filer_delete_entry.go +++ b/weed/filer/filer_delete_entry.go @@ -68,7 +68,7 @@ func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry lastFileName := "" includeLastFile := false for { - entries, err := f.ListDirectoryEntries(ctx, entry.FullPath, lastFileName, includeLastFile, PaginationSize, "") + entries, err := f.ListDirectoryEntries(ctx, entry.FullPath, lastFileName, includeLastFile, PaginationSize, "", "") if err != nil { glog.Errorf("list folder %s: %v", entry.FullPath, err) return nil, nil, fmt.Errorf("list folder %s: %v", entry.FullPath, err) diff --git a/weed/filer/filer_notify.go b/weed/filer/filer_notify.go index 40755e6a7..563e0eb51 100644 --- a/weed/filer/filer_notify.go +++ b/weed/filer/filer_notify.go @@ -113,13 +113,13 @@ func (f *Filer) ReadPersistedLogBuffer(startTime time.Time, eachLogEntryFn func( sizeBuf := make([]byte, 4) startTsNs := startTime.UnixNano() - dayEntries, listDayErr := f.ListDirectoryEntries(context.Background(), SystemLogDir, startDate, true, 366, "") + dayEntries, listDayErr := f.ListDirectoryEntries(context.Background(), SystemLogDir, startDate, true, 366, "", "") if listDayErr != nil { return lastTsNs, fmt.Errorf("fail to list log by day: %v", listDayErr) } for _, dayEntry := range dayEntries { // println("checking day", dayEntry.FullPath) - hourMinuteEntries, listHourMinuteErr := f.ListDirectoryEntries(context.Background(), util.NewFullPath(SystemLogDir, dayEntry.Name()), "", false, 24*60, "") + hourMinuteEntries, listHourMinuteErr := f.ListDirectoryEntries(context.Background(), util.NewFullPath(SystemLogDir, dayEntry.Name()), "", false, 24*60, "", "") if listHourMinuteErr != nil { return lastTsNs, fmt.Errorf("fail to list log %s by day: %v", dayEntry.Name(), listHourMinuteErr) } diff --git a/weed/filer/filer_search.go b/weed/filer/filer_search.go index b26959cb0..34358d849 100644 --- a/weed/filer/filer_search.go +++ b/weed/filer/filer_search.go @@ -19,12 +19,16 @@ func splitPattern(pattern string) (prefix string, restPattern string) { return "", restPattern } -func (f *Filer) ListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int, namePattern string) (entries []*Entry, err error) { +// For now, prefix and namePattern are mutually exclusive +func (f *Filer) ListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int, prefix string, namePattern string) (entries []*Entry, err error) { if strings.HasSuffix(string(p), "/") && len(p) > 1 { p = p[0 : len(p)-1] } - prefix, restNamePattern := splitPattern(namePattern) + prefixInNamePattern, restNamePattern := splitPattern(namePattern) + if prefixInNamePattern != "" { + prefix = prefixInNamePattern + } var missedCount int var lastFileName string diff --git a/weed/filer/leveldb/leveldb_store.go b/weed/filer/leveldb/leveldb_store.go index b879f3a6e..1aefbf5d4 100644 --- a/weed/filer/leveldb/leveldb_store.go +++ b/weed/filer/leveldb/leveldb_store.go @@ -170,8 +170,12 @@ func (store *LevelDBStore) ListDirectoryEntries(ctx context.Context, fullpath we func (store *LevelDBStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer.Entry, err error) { directoryPrefix := genDirectoryKeyPrefix(fullpath, prefix) + lastFileStart := directoryPrefix + if startFileName != "" { + lastFileStart = genDirectoryKeyPrefix(fullpath, startFileName) + } - iter := store.db.NewIterator(&leveldb_util.Range{Start: genDirectoryKeyPrefix(fullpath, startFileName)}, nil) + iter := store.db.NewIterator(&leveldb_util.Range{Start: lastFileStart}, nil) for iter.Next() { key := iter.Key() if !bytes.HasPrefix(key, directoryPrefix) { diff --git a/weed/filer/leveldb/leveldb_store_test.go b/weed/filer/leveldb/leveldb_store_test.go index c5bfb8474..c85a37332 100644 --- a/weed/filer/leveldb/leveldb_store_test.go +++ b/weed/filer/leveldb/leveldb_store_test.go @@ -49,14 +49,14 @@ func TestCreateAndFind(t *testing.T) { } // checking one upper directory - entries, _ := testFiler.ListDirectoryEntries(ctx, util.FullPath("/home/chris/this/is/one"), "", false, 100, "") + entries, _ := testFiler.ListDirectoryEntries(ctx, util.FullPath("/home/chris/this/is/one"), "", false, 100, "", "") if len(entries) != 1 { t.Errorf("list entries count: %v", len(entries)) return } // checking one upper directory - entries, _ = testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "") + entries, _ = testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "") if len(entries) != 1 { t.Errorf("list entries count: %v", len(entries)) return @@ -75,7 +75,7 @@ func TestEmptyRoot(t *testing.T) { ctx := context.Background() // checking one upper directory - entries, err := testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "") + entries, err := testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "") if err != nil { t.Errorf("list entries: %v", err) return diff --git a/weed/filer/leveldb2/leveldb2_store.go b/weed/filer/leveldb2/leveldb2_store.go index 4b41554b9..ceda64153 100644 --- a/weed/filer/leveldb2/leveldb2_store.go +++ b/weed/filer/leveldb2/leveldb2_store.go @@ -179,7 +179,10 @@ func (store *LevelDB2Store) ListDirectoryEntries(ctx context.Context, fullpath w func (store *LevelDB2Store) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer.Entry, err error) { directoryPrefix, partitionId := genDirectoryKeyPrefix(fullpath, prefix, store.dbCount) - lastFileStart, _ := genDirectoryKeyPrefix(fullpath, startFileName, store.dbCount) + lastFileStart := directoryPrefix + if startFileName != "" { + lastFileStart, _ = genDirectoryKeyPrefix(fullpath, startFileName, store.dbCount) + } iter := store.dbs[partitionId].NewIterator(&leveldb_util.Range{Start: lastFileStart}, nil) for iter.Next() { diff --git a/weed/filer/leveldb2/leveldb2_store_test.go b/weed/filer/leveldb2/leveldb2_store_test.go index 22c0d6052..a7f5ae95b 100644 --- a/weed/filer/leveldb2/leveldb2_store_test.go +++ b/weed/filer/leveldb2/leveldb2_store_test.go @@ -49,14 +49,14 @@ func TestCreateAndFind(t *testing.T) { } // checking one upper directory - entries, _ := testFiler.ListDirectoryEntries(ctx, util.FullPath("/home/chris/this/is/one"), "", false, 100, "") + entries, _ := testFiler.ListDirectoryEntries(ctx, util.FullPath("/home/chris/this/is/one"), "", false, 100, "", "") if len(entries) != 1 { t.Errorf("list entries count: %v", len(entries)) return } // checking one upper directory - entries, _ = testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "") + entries, _ = testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "") if len(entries) != 1 { t.Errorf("list entries count: %v", len(entries)) return @@ -75,7 +75,7 @@ func TestEmptyRoot(t *testing.T) { ctx := context.Background() // checking one upper directory - entries, err := testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "") + entries, err := testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "") if err != nil { t.Errorf("list entries: %v", err) return diff --git a/weed/server/filer_grpc_server.go b/weed/server/filer_grpc_server.go index 38472df2a..efc06f0d6 100644 --- a/weed/server/filer_grpc_server.go +++ b/weed/server/filer_grpc_server.go @@ -61,7 +61,7 @@ func (fs *FilerServer) ListEntries(req *filer_pb.ListEntriesRequest, stream file lastFileName := req.StartFromFileName includeLastFile := req.InclusiveStartFrom for limit > 0 { - entries, err := fs.filer.ListDirectoryEntries(stream.Context(), util.FullPath(req.Directory), lastFileName, includeLastFile, paginationLimit, req.Prefix) + entries, err := fs.filer.ListDirectoryEntries(stream.Context(), util.FullPath(req.Directory), lastFileName, includeLastFile, paginationLimit, req.Prefix, "") if err != nil { return err diff --git a/weed/server/filer_grpc_server_rename.go b/weed/server/filer_grpc_server_rename.go index fa86737ac..a0c9aedb9 100644 --- a/weed/server/filer_grpc_server_rename.go +++ b/weed/server/filer_grpc_server_rename.go @@ -75,7 +75,7 @@ func (fs *FilerServer) moveFolderSubEntries(ctx context.Context, oldParent util. includeLastFile := false for { - entries, err := fs.filer.ListDirectoryEntries(ctx, currentDirPath, lastFileName, includeLastFile, 1024, "") + entries, err := fs.filer.ListDirectoryEntries(ctx, currentDirPath, lastFileName, includeLastFile, 1024, "", "") if err != nil { return err } diff --git a/weed/server/filer_server_handlers_read_dir.go b/weed/server/filer_server_handlers_read_dir.go index f303ba1d4..ba01ce4f7 100644 --- a/weed/server/filer_server_handlers_read_dir.go +++ b/weed/server/filer_server_handlers_read_dir.go @@ -36,7 +36,7 @@ func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Reque lastFileName := r.FormValue("lastFileName") namePattern := r.FormValue("namePattern") - entries, err := fs.filer.ListDirectoryEntries(context.Background(), util.FullPath(path), lastFileName, false, limit, namePattern) + entries, err := fs.filer.ListDirectoryEntries(context.Background(), util.FullPath(path), lastFileName, false, limit, "", namePattern) if err != nil { glog.V(0).Infof("listDirectory %s %s %d: %s", path, lastFileName, limit, err) From f4d87f8b9dcaab0e52c5c0aaf8b006e00af623ae Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 2 Jan 2021 23:32:58 -0800 Subject: [PATCH 046/260] works performance seems about the same as leveldb for insert entries --- go.mod | 1 + go.sum | 8 +- weed/filer/leveldb/leveldb_store_test.go | 27 +++ weed/filer/rocksdb/README.md | 38 +++ weed/filer/rocksdb/rocksdb_store.go | 293 +++++++++++++++++++++++ weed/filer/rocksdb/rocksdb_store_kv.go | 51 ++++ weed/filer/rocksdb/rocksdb_store_test.go | 117 +++++++++ weed/server/filer_server_rocksdb.go | 7 + 8 files changed, 540 insertions(+), 2 deletions(-) create mode 100644 weed/filer/rocksdb/README.md create mode 100644 weed/filer/rocksdb/rocksdb_store.go create mode 100644 weed/filer/rocksdb/rocksdb_store_kv.go create mode 100644 weed/filer/rocksdb/rocksdb_store_test.go create mode 100644 weed/server/filer_server_rocksdb.go diff --git a/go.mod b/go.mod index d93a44379..e49838059 100644 --- a/go.mod +++ b/go.mod @@ -66,6 +66,7 @@ require ( github.com/spf13/viper v1.4.0 github.com/stretchr/testify v1.6.1 github.com/syndtr/goleveldb v1.0.0 + github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c github.com/tidwall/gjson v1.3.2 github.com/tidwall/match v1.0.1 github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365 diff --git a/go.sum b/go.sum index 66fb72a6b..0ba1d5f31 100644 --- a/go.sum +++ b/go.sum @@ -225,6 +225,7 @@ github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4er github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.3 h1:GV+pQPG/EUUbkh47niozDcADz6go/dUwhVzdUQHIVRw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -542,6 +543,7 @@ github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a h1:9ZKAASQSHhD github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 h1:dY6ETXrvDG7Sa4vE8ZQG4yqWg6UnOcbqTAahkV813vQ= github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -551,8 +553,6 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/seaweedfs/fuse v1.0.7 h1:tESMXhI3gXzN+dlWsCUrkIZDiWA4dZX18rQMoqmvazw= -github.com/seaweedfs/fuse v1.0.7/go.mod h1:W7ubwr1l7KQsMeUpxFFOFOSxUL/ucTRMAlVYs4xdfQ8= github.com/seaweedfs/fuse v1.0.8 h1:HBPJTC77OlxwSd2JiTwvLPn8bWTElqQp3xs9vf3C15s= github.com/seaweedfs/fuse v1.0.8/go.mod h1:W7ubwr1l7KQsMeUpxFFOFOSxUL/ucTRMAlVYs4xdfQ8= github.com/seaweedfs/goexif v1.0.2 h1:p+rTXYdQ2mgxd+1JaTrQ9N8DvYuw9UH9xgYmJ+Bb29E= @@ -618,6 +618,8 @@ github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= +github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c h1:g+WoO5jjkqGAzHWCjJB1zZfXPIAaDpzXIEJ0eS6B5Ok= +github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c/go.mod h1:ahpPrc7HpcfEWDQRZEmnXMzHY03mLDYMCxeDzy46i+8= github.com/tidwall/gjson v1.3.2 h1:+7p3qQFaH3fOMXAJSrdZwGKcOO/lYdGS0HqGhPqDdTI= github.com/tidwall/gjson v1.3.2/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls= github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc= @@ -941,7 +943,9 @@ honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXe honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= modernc.org/b v1.0.0 h1:vpvqeyp17ddcQWF29Czawql4lDdABCDRbXRAS4+aF2o= modernc.org/b v1.0.0/go.mod h1:uZWcZfRj1BpYzfN9JTerzlNUnnPsV9O2ZA8JsRcubNg= +modernc.org/mathutil v1.1.1 h1:FeylZSVX8S+58VsyJlkEj2bcpdytmp9MmDKZkKx8OIE= modernc.org/mathutil v1.1.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/strutil v1.1.0 h1:+1/yCzZxY2pZwwrsbH+4T7BQMoLQ9QiBshRC9eicYsc= modernc.org/strutil v1.1.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= pack.ag/amqp v0.11.2/go.mod h1:4/cbmt4EJXSKlG6LCfWHoqmN0uFdy5i/+YFz+fTfhV4= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/weed/filer/leveldb/leveldb_store_test.go b/weed/filer/leveldb/leveldb_store_test.go index c85a37332..edf629814 100644 --- a/weed/filer/leveldb/leveldb_store_test.go +++ b/weed/filer/leveldb/leveldb_store_test.go @@ -2,9 +2,11 @@ package leveldb import ( "context" + "fmt" "io/ioutil" "os" "testing" + "time" "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/util" @@ -86,3 +88,28 @@ func TestEmptyRoot(t *testing.T) { } } + +func BenchmarkInsertEntry(b *testing.B) { + testFiler := filer.NewFiler(nil, nil, "", 0, "", "", "", nil) + dir, _ := ioutil.TempDir("", "seaweedfs_filer_bench") + defer os.RemoveAll(dir) + store := &LevelDBStore{} + store.initialize(dir) + testFiler.SetStore(store) + + ctx := context.Background() + + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + entry := &filer.Entry{ + FullPath: util.FullPath(fmt.Sprintf("/file%d.txt", i)), + Attr: filer.Attr{ + Crtime: time.Now(), + Mtime: time.Now(), + Mode: os.FileMode(0644), + }, + } + store.InsertEntry(ctx, entry) + } +} diff --git a/weed/filer/rocksdb/README.md b/weed/filer/rocksdb/README.md new file mode 100644 index 000000000..de4f41b0a --- /dev/null +++ b/weed/filer/rocksdb/README.md @@ -0,0 +1,38 @@ +# Prepare the compilation environment on linux +- sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test +- sudo apt-get update -qq +- sudo apt-get install gcc-6 g++-6 libsnappy-dev zlib1g-dev libbz2-dev -qq +- export CXX="g++-6" CC="gcc-6" + +- wget https://launchpad.net/ubuntu/+archive/primary/+files/libgflags2_2.0-1.1ubuntu1_amd64.deb +- sudo dpkg -i libgflags2_2.0-1.1ubuntu1_amd64.deb +- wget https://launchpad.net/ubuntu/+archive/primary/+files/libgflags-dev_2.0-1.1ubuntu1_amd64.deb +- sudo dpkg -i libgflags-dev_2.0-1.1ubuntu1_amd64.deb + +# Prepare the compilation environment on mac os +``` +brew install snappy +``` + +# install rocksdb: +``` + + export ROCKSDB_HOME=/Users/chris/dev/rocksdb + + git clone https://github.com/facebook/rocksdb.git $ROCKSDB_HOME + pushd $ROCKSDB_HOME + make clean + make install-static + popd + +``` + +# install gorocksdb + +``` +export CGO_CFLAGS="-I$ROCKSDB_HOME/include" +export CGO_LDFLAGS="-L$ROCKSDB_HOME -lrocksdb -lstdc++ -lm -lz -lbz2 -lsnappy -llz4 -lzstd" + +go get github.com/tecbot/gorocksdb + +``` diff --git a/weed/filer/rocksdb/rocksdb_store.go b/weed/filer/rocksdb/rocksdb_store.go new file mode 100644 index 000000000..a8992cf03 --- /dev/null +++ b/weed/filer/rocksdb/rocksdb_store.go @@ -0,0 +1,293 @@ +// +build rocksdb + +package rocksdb + +import ( + "bytes" + "context" + "crypto/md5" + "fmt" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + weed_util "github.com/chrislusf/seaweedfs/weed/util" + "github.com/tecbot/gorocksdb" + "io" +) + +func init() { + filer.Stores = append(filer.Stores, &RocksDBStore{}) +} + +type RocksDBStore struct { + path string + db *gorocksdb.DB +} + +func (store *RocksDBStore) GetName() string { + return "rocksdb" +} + +func (store *RocksDBStore) Initialize(configuration weed_util.Configuration, prefix string) (err error) { + dir := configuration.GetString(prefix + "dir") + return store.initialize(dir) +} + +func (store *RocksDBStore) initialize(dir string) (err error) { + glog.Infof("filer store rocksdb dir: %s", dir) + if err := weed_util.TestFolderWritable(dir); err != nil { + return fmt.Errorf("Check Level Folder %s Writable: %s", dir, err) + } + + options := gorocksdb.NewDefaultOptions() + options.SetCreateIfMissing(true) + store.db, err = gorocksdb.OpenDb(options, dir) + + return +} + +func (store *RocksDBStore) BeginTransaction(ctx context.Context) (context.Context, error) { + return ctx, nil +} +func (store *RocksDBStore) CommitTransaction(ctx context.Context) error { + return nil +} +func (store *RocksDBStore) RollbackTransaction(ctx context.Context) error { + return nil +} + +func (store *RocksDBStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) { + dir, name := entry.DirAndName() + key := genKey(dir, name) + + value, err := entry.EncodeAttributesAndChunks() + if err != nil { + return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err) + } + + wo := gorocksdb.NewDefaultWriteOptions() + err = store.db.Put(wo, key, value) + + if err != nil { + return fmt.Errorf("persisting %s : %v", entry.FullPath, err) + } + + // println("saved", entry.FullPath, "chunks", len(entry.Chunks)) + + return nil +} + +func (store *RocksDBStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) { + + return store.InsertEntry(ctx, entry) +} + +func (store *RocksDBStore) FindEntry(ctx context.Context, fullpath weed_util.FullPath) (entry *filer.Entry, err error) { + dir, name := fullpath.DirAndName() + key := genKey(dir, name) + + ro := gorocksdb.NewDefaultReadOptions() + data, err := store.db.GetBytes(ro, key) + + if data == nil { + return nil, filer_pb.ErrNotFound + } + if err != nil { + return nil, fmt.Errorf("get %s : %v", entry.FullPath, err) + } + + entry = &filer.Entry{ + FullPath: fullpath, + } + err = entry.DecodeAttributesAndChunks(weed_util.MaybeDecompressData(data)) + if err != nil { + return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err) + } + + // println("read", entry.FullPath, "chunks", len(entry.Chunks), "data", len(data), string(data)) + + return entry, nil +} + +func (store *RocksDBStore) DeleteEntry(ctx context.Context, fullpath weed_util.FullPath) (err error) { + dir, name := fullpath.DirAndName() + key := genKey(dir, name) + + wo := gorocksdb.NewDefaultWriteOptions() + err = store.db.Delete(wo, key) + if err != nil { + return fmt.Errorf("delete %s : %v", fullpath, err) + } + + return nil +} + +func (store *RocksDBStore) DeleteFolderChildren(ctx context.Context, fullpath weed_util.FullPath) (err error) { + directoryPrefix := genDirectoryKeyPrefix(fullpath, "") + + batch := new(gorocksdb.WriteBatch) + + ro := gorocksdb.NewDefaultReadOptions() + ro.SetFillCache(false) + iter := store.db.NewIterator(ro) + defer iter.Close() + err = enumerate(iter, directoryPrefix, nil, false, -1, func(key, value []byte) bool { + batch.Delete(key) + return true + }) + if err != nil { + return fmt.Errorf("delete list %s : %v", fullpath, err) + } + + wo := gorocksdb.NewDefaultWriteOptions() + err = store.db.Write(wo, batch) + + if err != nil { + return fmt.Errorf("delete %s : %v", fullpath, err) + } + + return nil +} + +func enumerate(iter *gorocksdb.Iterator, prefix, lastKey []byte, includeLastKey bool, limit int, fn func(key, value []byte) bool) error { + + if len(lastKey) == 0 { + iter.Seek(prefix) + } else { + iter.Seek(lastKey) + + if !includeLastKey { + k := iter.Key() + v := iter.Value() + key := k.Data() + defer k.Free() + defer v.Free() + + if !bytes.HasPrefix(key, prefix) { + return nil + } + + if bytes.Equal(key, lastKey) { + iter.Next() + } + + } + } + + i := 0 + for ; iter.Valid(); iter.Next() { + + if limit > 0 { + i++ + if i > limit { + break + } + } + + k := iter.Key() + v := iter.Value() + key := k.Data() + value := v.Data() + + if !bytes.HasPrefix(key, prefix) { + k.Free() + v.Free() + break + } + + ret := fn(key, value) + + k.Free() + v.Free() + + if !ret { + break + } + + } + + if err := iter.Err(); err != nil { + return fmt.Errorf("prefix scan iterator: %v", err) + } + return nil +} + +func (store *RocksDBStore) ListDirectoryEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, + limit int) (entries []*filer.Entry, err error) { + return store.ListDirectoryPrefixedEntries(ctx, fullpath, startFileName, inclusive, limit, "") +} + +func (store *RocksDBStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer.Entry, err error) { + + directoryPrefix := genDirectoryKeyPrefix(fullpath, prefix) + lastFileStart := directoryPrefix + if startFileName != "" { + lastFileStart = genDirectoryKeyPrefix(fullpath, startFileName) + } + + ro := gorocksdb.NewDefaultReadOptions() + ro.SetFillCache(false) + iter := store.db.NewIterator(ro) + defer iter.Close() + err = enumerate(iter, directoryPrefix, lastFileStart, inclusive, limit, func(key, value []byte) bool { + fileName := getNameFromKey(key) + if fileName == "" { + return true + } + limit-- + if limit < 0 { + return false + } + entry := &filer.Entry{ + FullPath: weed_util.NewFullPath(string(fullpath), fileName), + } + + // println("list", entry.FullPath, "chunks", len(entry.Chunks)) + if decodeErr := entry.DecodeAttributesAndChunks(weed_util.MaybeDecompressData(value)); decodeErr != nil { + err = decodeErr + glog.V(0).Infof("list %s : %v", entry.FullPath, err) + return false + } + entries = append(entries, entry) + return true + }) + if err != nil { + return entries, fmt.Errorf("prefix list %s : %v", fullpath, err) + } + + return entries, err +} + +func genKey(dirPath, fileName string) (key []byte) { + key = hashToBytes(dirPath) + key = append(key, []byte(fileName)...) + return key +} + +func genDirectoryKeyPrefix(fullpath weed_util.FullPath, startFileName string) (keyPrefix []byte) { + keyPrefix = hashToBytes(string(fullpath)) + if len(startFileName) > 0 { + keyPrefix = append(keyPrefix, []byte(startFileName)...) + } + return keyPrefix +} + +func getNameFromKey(key []byte) string { + + return string(key[md5.Size:]) + +} + +// hash directory, and use last byte for partitioning +func hashToBytes(dir string) []byte { + h := md5.New() + io.WriteString(h, dir) + + b := h.Sum(nil) + + return b +} + +func (store *RocksDBStore) Shutdown() { + store.db.Close() +} diff --git a/weed/filer/rocksdb/rocksdb_store_kv.go b/weed/filer/rocksdb/rocksdb_store_kv.go new file mode 100644 index 000000000..093a905e8 --- /dev/null +++ b/weed/filer/rocksdb/rocksdb_store_kv.go @@ -0,0 +1,51 @@ +// +build rocksdb + +package rocksdb + +import ( + "context" + "fmt" + "github.com/tecbot/gorocksdb" + + "github.com/chrislusf/seaweedfs/weed/filer" +) + +func (store *RocksDBStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) { + + wo := gorocksdb.NewDefaultWriteOptions() + err = store.db.Put(wo, key, value) + + if err != nil { + return fmt.Errorf("kv put: %v", err) + } + + return nil +} + +func (store *RocksDBStore) KvGet(ctx context.Context, key []byte) (value []byte, err error) { + + ro := gorocksdb.NewDefaultReadOptions() + value, err = store.db.GetBytes(ro, key) + + if value == nil { + return nil, filer.ErrKvNotFound + } + + if err != nil { + return nil, fmt.Errorf("kv get: %v", err) + } + + return +} + +func (store *RocksDBStore) KvDelete(ctx context.Context, key []byte) (err error) { + + wo := gorocksdb.NewDefaultWriteOptions() + err = store.db.Delete(wo, key) + + if err != nil { + return fmt.Errorf("kv delete: %v", err) + } + + return nil +} diff --git a/weed/filer/rocksdb/rocksdb_store_test.go b/weed/filer/rocksdb/rocksdb_store_test.go new file mode 100644 index 000000000..5c48e9bd9 --- /dev/null +++ b/weed/filer/rocksdb/rocksdb_store_test.go @@ -0,0 +1,117 @@ +// +build rocksdb + +package rocksdb + +import ( + "context" + "fmt" + "io/ioutil" + "os" + "testing" + "time" + + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/util" +) + +func TestCreateAndFind(t *testing.T) { + testFiler := filer.NewFiler(nil, nil, "", 0, "", "", "", nil) + dir, _ := ioutil.TempDir("", "seaweedfs_filer_test") + defer os.RemoveAll(dir) + store := &RocksDBStore{} + store.initialize(dir) + testFiler.SetStore(store) + + fullpath := util.FullPath("/home/chris/this/is/one/file1.jpg") + + ctx := context.Background() + + entry1 := &filer.Entry{ + FullPath: fullpath, + Attr: filer.Attr{ + Mode: 0440, + Uid: 1234, + Gid: 5678, + }, + } + + if err := testFiler.CreateEntry(ctx, entry1, false, false, nil); err != nil { + t.Errorf("create entry %v: %v", entry1.FullPath, err) + return + } + + entry, err := testFiler.FindEntry(ctx, fullpath) + + if err != nil { + t.Errorf("find entry: %v", err) + return + } + + if entry.FullPath != entry1.FullPath { + t.Errorf("find wrong entry: %v", entry.FullPath) + return + } + + // checking one upper directory + entries, _ := testFiler.ListDirectoryEntries(ctx, util.FullPath("/home/chris/this/is/one"), "", false, 100, "", "") + if len(entries) != 1 { + t.Errorf("list entries count: %v", len(entries)) + return + } + + // checking one upper directory + entries, _ = testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "") + if len(entries) != 1 { + t.Errorf("list entries count: %v", len(entries)) + return + } + +} + +func TestEmptyRoot(t *testing.T) { + testFiler := filer.NewFiler(nil, nil, "", 0, "", "", "", nil) + dir, _ := ioutil.TempDir("", "seaweedfs_filer_test2") + defer os.RemoveAll(dir) + store := &RocksDBStore{} + store.initialize(dir) + testFiler.SetStore(store) + + ctx := context.Background() + + // checking one upper directory + entries, err := testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "") + if err != nil { + t.Errorf("list entries: %v", err) + return + } + if len(entries) != 0 { + t.Errorf("list entries count: %v", len(entries)) + return + } + +} + +func BenchmarkInsertEntry(b *testing.B) { + testFiler := filer.NewFiler(nil, nil, "", 0, "", "", "", nil) + dir, _ := ioutil.TempDir("", "seaweedfs_filer_bench") + defer os.RemoveAll(dir) + store := &RocksDBStore{} + store.initialize(dir) + testFiler.SetStore(store) + + ctx := context.Background() + + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + entry := &filer.Entry{ + FullPath: util.FullPath(fmt.Sprintf("/file%d.txt", i)), + Attr: filer.Attr{ + Crtime: time.Now(), + Mtime: time.Now(), + Mode: os.FileMode(0644), + }, + } + store.InsertEntry(ctx, entry) + } +} diff --git a/weed/server/filer_server_rocksdb.go b/weed/server/filer_server_rocksdb.go new file mode 100644 index 000000000..5fcc7e88f --- /dev/null +++ b/weed/server/filer_server_rocksdb.go @@ -0,0 +1,7 @@ +// +build rocksdb + +package weed_server + +import ( + _ "github.com/chrislusf/seaweedfs/weed/filer/rocksdb" +) From 4250a3a67510dda9fe9d6e38d4ea617c97c4c214 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 2 Jan 2021 23:43:01 -0800 Subject: [PATCH 047/260] upload: add read default collection from master fix https://github.com/chrislusf/seaweedfs/issues/1724#issuecomment-753576071 --- weed/command/upload.go | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/weed/command/upload.go b/weed/command/upload.go index 45b15535b..7115da587 100644 --- a/weed/command/upload.go +++ b/weed/command/upload.go @@ -1,8 +1,12 @@ package command import ( + "context" "encoding/json" "fmt" + "github.com/chrislusf/seaweedfs/weed/pb" + "github.com/chrislusf/seaweedfs/weed/pb/master_pb" + "google.golang.org/grpc" "os" "path/filepath" @@ -65,6 +69,15 @@ func runUpload(cmd *Command, args []string) bool { util.LoadConfiguration("security", false) grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client") + defaultCollection, err := readMasterConfiguration(grpcDialOption, *upload.master) + if err != nil { + fmt.Printf("upload: %v", err) + return false + } + if *upload.replication == "" { + *upload.replication = defaultCollection + } + if len(args) == 0 { if *upload.dir == "" { return false @@ -104,3 +117,15 @@ func runUpload(cmd *Command, args []string) bool { } return true } + +func readMasterConfiguration(grpcDialOption grpc.DialOption, masterAddress string) (replication string, err error) { + err = pb.WithMasterClient(masterAddress, grpcDialOption, func(client master_pb.SeaweedClient) error { + resp, err := client.GetMasterConfiguration(context.Background(), &master_pb.GetMasterConfigurationRequest{}) + if err != nil { + return fmt.Errorf("get master %s configuration: %v", masterAddress, err) + } + replication = resp.DefaultReplication + return nil + }) + return +} From a04d8927a8af9bf4aa128c93370729f416dceb4d Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 3 Jan 2021 00:44:52 -0800 Subject: [PATCH 048/260] add scaffold --- weed/command/scaffold.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/weed/command/scaffold.go b/weed/command/scaffold.go index d1b4b27ac..600a4a8d4 100644 --- a/weed/command/scaffold.go +++ b/weed/command/scaffold.go @@ -87,7 +87,13 @@ buckets_folder = "/buckets" # local on disk, mostly for simple single-machine setup, fairly scalable # faster than previous leveldb, recommended. enabled = true -dir = "." # directory to store level db files +dir = "./filerldb2" # directory to store level db files + +[rocksdb] +# local on disk, similar to leveldb +# since it is using a C wrapper, you need to install rocksdb and build it by yourself +enabled = false +dir = "./filerrdb" # directory to store rocksdb files [mysql] # or tidb # CREATE TABLE IF NOT EXISTS filemeta ( From bdd166c03136d55abae1769c73806cfd20a5a226 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 3 Jan 2021 00:47:58 -0800 Subject: [PATCH 049/260] adjust readme --- weed/filer/rocksdb/README.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/weed/filer/rocksdb/README.md b/weed/filer/rocksdb/README.md index de4f41b0a..6bae6d34e 100644 --- a/weed/filer/rocksdb/README.md +++ b/weed/filer/rocksdb/README.md @@ -16,7 +16,6 @@ brew install snappy # install rocksdb: ``` - export ROCKSDB_HOME=/Users/chris/dev/rocksdb git clone https://github.com/facebook/rocksdb.git $ROCKSDB_HOME @@ -24,7 +23,6 @@ brew install snappy make clean make install-static popd - ``` # install gorocksdb @@ -34,5 +32,10 @@ export CGO_CFLAGS="-I$ROCKSDB_HOME/include" export CGO_LDFLAGS="-L$ROCKSDB_HOME -lrocksdb -lstdc++ -lm -lz -lbz2 -lsnappy -llz4 -lzstd" go get github.com/tecbot/gorocksdb +``` +# compile with rocksdb ``` +cd ~/go/src/github.com/chrislusf/seaweedfs/weed +go install -tags rocksdb +``` From 039ad101b18e71fd8f0fb0006702678d11fc54fd Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 3 Jan 2021 01:42:21 -0800 Subject: [PATCH 050/260] manifest file also need to detect compression fix https://github.com/chrislusf/seaweedfs/issues/1724#issuecomment-753585046 --- weed/storage/needle/needle_parse_upload.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/weed/storage/needle/needle_parse_upload.go b/weed/storage/needle/needle_parse_upload.go index 8f457be1d..7201503f1 100644 --- a/weed/storage/needle/needle_parse_upload.go +++ b/weed/storage/needle/needle_parse_upload.go @@ -193,9 +193,9 @@ func parseMultipart(r *http.Request, sizeLimit int64, pu *ParsedUpload) (e error mtype = contentType } - pu.IsGzipped = part.Header.Get("Content-Encoding") == "gzip" - // pu.IsZstd = part.Header.Get("Content-Encoding") == "zstd" } + pu.IsGzipped = part.Header.Get("Content-Encoding") == "gzip" + // pu.IsZstd = part.Header.Get("Content-Encoding") == "zstd" return } From 4b5adc60b4869c00e1f3256ac79f8f9e0846b7aa Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 3 Jan 2021 01:44:22 -0800 Subject: [PATCH 051/260] upload result changes to the right output variable --- weed/operation/submit.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/operation/submit.go b/weed/operation/submit.go index c8f979d83..e785b68a9 100644 --- a/weed/operation/submit.go +++ b/weed/operation/submit.go @@ -32,7 +32,7 @@ type FilePart struct { type SubmitResult struct { FileName string `json:"fileName,omitempty"` - FileUrl string `json:"fileUrl,omitempty"` + FileUrl string `json:"url,omitempty"` Fid string `json:"fid,omitempty"` Size uint32 `json:"size,omitempty"` Error string `json:"error,omitempty"` From 9c9ba3c209a4e833727b5e4a797bfdd332b2cb4f Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 3 Jan 2021 12:25:58 -0800 Subject: [PATCH 052/260] nil related related to https://github.com/chrislusf/seaweedfs/issues/1676 --- weed/topology/data_node.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/weed/topology/data_node.go b/weed/topology/data_node.go index 0a4df63d0..eaed51654 100644 --- a/weed/topology/data_node.go +++ b/weed/topology/data_node.go @@ -151,7 +151,10 @@ func (dn *DataNode) GetVolumesById(id needle.VolumeId) (storage.VolumeInfo, erro } func (dn *DataNode) GetDataCenter() *DataCenter { - return dn.Parent().Parent().(*NodeImpl).value.(*DataCenter) + rack := dn.Parent() + dcNode := rack.Parent() + dcValue := dcNode.GetValue() + return dcValue.(*DataCenter) } func (dn *DataNode) GetRack() *Rack { From c3f674ac1d693634a2630341b06cbcf85691e62d Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 3 Jan 2021 13:38:27 -0800 Subject: [PATCH 053/260] address raft nil snapshot during flush fix https://github.com/chrislusf/seaweedfs/issues/1718 --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index e49838059..467b3b1fc 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/aws/aws-sdk-go v1.33.5 github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72 github.com/cespare/xxhash v1.1.0 - github.com/chrislusf/raft v1.0.3 + github.com/chrislusf/raft v1.0.4 github.com/coreos/go-semver v0.3.0 // indirect github.com/dgrijalva/jwt-go v3.2.0+incompatible github.com/disintegration/imaging v1.6.2 diff --git a/go.sum b/go.sum index 0ba1d5f31..6f4519aef 100644 --- a/go.sum +++ b/go.sum @@ -86,6 +86,8 @@ github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chrislusf/raft v1.0.3 h1:11YrnzJtVa5z7m9lhY2p8VcPHoUlC1UswyoAo+U1m1k= github.com/chrislusf/raft v1.0.3/go.mod h1:Ep5DP+mJSosjfKiix1uU7Lc2Df/SX4oGJEpZlXH5l68= +github.com/chrislusf/raft v1.0.4 h1:THhbsVik2hxdE0/VXX834f64Wn9RzgVPp+E+XCWZdKM= +github.com/chrislusf/raft v1.0.4/go.mod h1:Ep5DP+mJSosjfKiix1uU7Lc2Df/SX4oGJEpZlXH5l68= github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= From 2ce86f308ea4836cf534e50dc1388932253b5cd5 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 3 Jan 2021 17:24:02 -0800 Subject: [PATCH 054/260] 2.19 --- k8s/seaweedfs/values.yaml | 2 +- weed/util/constants.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/k8s/seaweedfs/values.yaml b/k8s/seaweedfs/values.yaml index 7144b0016..9d621a496 100644 --- a/k8s/seaweedfs/values.yaml +++ b/k8s/seaweedfs/values.yaml @@ -4,7 +4,7 @@ global: registry: "" repository: "" imageName: chrislusf/seaweedfs - imageTag: "2.18" + imageTag: "2.19" imagePullPolicy: IfNotPresent imagePullSecrets: imagepullsecret restartPolicy: Always diff --git a/weed/util/constants.go b/weed/util/constants.go index 08d169545..cfb33516b 100644 --- a/weed/util/constants.go +++ b/weed/util/constants.go @@ -5,7 +5,7 @@ import ( ) var ( - VERSION = fmt.Sprintf("%s %d.%02d", sizeLimit, 2, 18) + VERSION = fmt.Sprintf("%s %d.%02d", sizeLimit, 2, 19) COMMIT = "" ) From 40b3207f18d7b1a5f04c0333c970a42527d16df0 Mon Sep 17 00:00:00 2001 From: chenqieqie Date: Mon, 4 Jan 2021 18:48:55 +0800 Subject: [PATCH 055/260] fix #1726 --- weed/filer/rocksdb/rocksdb_store.go | 81 ++++++++++++++++------------- 1 file changed, 45 insertions(+), 36 deletions(-) diff --git a/weed/filer/rocksdb/rocksdb_store.go b/weed/filer/rocksdb/rocksdb_store.go index a8992cf03..ca6391386 100644 --- a/weed/filer/rocksdb/rocksdb_store.go +++ b/weed/filer/rocksdb/rocksdb_store.go @@ -11,7 +11,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" weed_util "github.com/chrislusf/seaweedfs/weed/util" - "github.com/tecbot/gorocksdb" + rocksdb "github.com/tecbot/gorocksdb" "io" ) @@ -19,9 +19,28 @@ func init() { filer.Stores = append(filer.Stores, &RocksDBStore{}) } +type options struct { + opt *rocksdb.Options + ro *rocksdb.ReadOptions + wo *rocksdb.WriteOptions +} + +func (opt *options) init() { + opt.opt = rocksdb.NewDefaultOptions() + opt.ro = rocksdb.NewDefaultReadOptions() + opt.wo = rocksdb.NewDefaultWriteOptions() +} + +func (opt *options) close() { + opt.opt.Destroy() + opt.ro.Destroy() + opt.wo.Destroy() +} + type RocksDBStore struct { path string - db *gorocksdb.DB + db *rocksdb.DB + options } func (store *RocksDBStore) GetName() string { @@ -38,10 +57,9 @@ func (store *RocksDBStore) initialize(dir string) (err error) { if err := weed_util.TestFolderWritable(dir); err != nil { return fmt.Errorf("Check Level Folder %s Writable: %s", dir, err) } - - options := gorocksdb.NewDefaultOptions() - options.SetCreateIfMissing(true) - store.db, err = gorocksdb.OpenDb(options, dir) + store.options.init() + store.opt.SetCreateIfMissing(true) + store.db, err = rocksdb.OpenDb(store.opt, dir) return } @@ -65,8 +83,7 @@ func (store *RocksDBStore) InsertEntry(ctx context.Context, entry *filer.Entry) return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err) } - wo := gorocksdb.NewDefaultWriteOptions() - err = store.db.Put(wo, key, value) + err = store.db.Put(store.wo, key, value) if err != nil { return fmt.Errorf("persisting %s : %v", entry.FullPath, err) @@ -85,21 +102,21 @@ func (store *RocksDBStore) UpdateEntry(ctx context.Context, entry *filer.Entry) func (store *RocksDBStore) FindEntry(ctx context.Context, fullpath weed_util.FullPath) (entry *filer.Entry, err error) { dir, name := fullpath.DirAndName() key := genKey(dir, name) - - ro := gorocksdb.NewDefaultReadOptions() - data, err := store.db.GetBytes(ro, key) + data, err := store.db.Get(store.ro, key) if data == nil { return nil, filer_pb.ErrNotFound } + defer data.Free() + if err != nil { - return nil, fmt.Errorf("get %s : %v", entry.FullPath, err) + return nil, fmt.Errorf("get %s : %v", fullpath, err) } entry = &filer.Entry{ FullPath: fullpath, } - err = entry.DecodeAttributesAndChunks(weed_util.MaybeDecompressData(data)) + err = entry.DecodeAttributesAndChunks(weed_util.MaybeDecompressData(data.Data())) if err != nil { return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err) } @@ -113,8 +130,7 @@ func (store *RocksDBStore) DeleteEntry(ctx context.Context, fullpath weed_util.F dir, name := fullpath.DirAndName() key := genKey(dir, name) - wo := gorocksdb.NewDefaultWriteOptions() - err = store.db.Delete(wo, key) + err = store.db.Delete(store.wo, key) if err != nil { return fmt.Errorf("delete %s : %v", fullpath, err) } @@ -125,10 +141,13 @@ func (store *RocksDBStore) DeleteEntry(ctx context.Context, fullpath weed_util.F func (store *RocksDBStore) DeleteFolderChildren(ctx context.Context, fullpath weed_util.FullPath) (err error) { directoryPrefix := genDirectoryKeyPrefix(fullpath, "") - batch := new(gorocksdb.WriteBatch) + batch := rocksdb.NewWriteBatch() + defer batch.Destroy() - ro := gorocksdb.NewDefaultReadOptions() + ro := rocksdb.NewDefaultReadOptions() + defer ro.Destroy() ro.SetFillCache(false) + iter := store.db.NewIterator(ro) defer iter.Close() err = enumerate(iter, directoryPrefix, nil, false, -1, func(key, value []byte) bool { @@ -139,8 +158,7 @@ func (store *RocksDBStore) DeleteFolderChildren(ctx context.Context, fullpath we return fmt.Errorf("delete list %s : %v", fullpath, err) } - wo := gorocksdb.NewDefaultWriteOptions() - err = store.db.Write(wo, batch) + err = store.db.Write(store.wo, batch) if err != nil { return fmt.Errorf("delete %s : %v", fullpath, err) @@ -149,7 +167,7 @@ func (store *RocksDBStore) DeleteFolderChildren(ctx context.Context, fullpath we return nil } -func enumerate(iter *gorocksdb.Iterator, prefix, lastKey []byte, includeLastKey bool, limit int, fn func(key, value []byte) bool) error { +func enumerate(iter *rocksdb.Iterator, prefix, lastKey []byte, includeLastKey bool, limit int, fn func(key, value []byte) bool) error { if len(lastKey) == 0 { iter.Seek(prefix) @@ -157,11 +175,7 @@ func enumerate(iter *gorocksdb.Iterator, prefix, lastKey []byte, includeLastKey iter.Seek(lastKey) if !includeLastKey { - k := iter.Key() - v := iter.Value() - key := k.Data() - defer k.Free() - defer v.Free() + key := iter.Key().Data() if !bytes.HasPrefix(key, prefix) { return nil @@ -184,21 +198,13 @@ func enumerate(iter *gorocksdb.Iterator, prefix, lastKey []byte, includeLastKey } } - k := iter.Key() - v := iter.Value() - key := k.Data() - value := v.Data() + key := iter.Key().Data() if !bytes.HasPrefix(key, prefix) { - k.Free() - v.Free() break } - ret := fn(key, value) - - k.Free() - v.Free() + ret := fn(key, iter.Value().Data()) if !ret { break @@ -225,8 +231,10 @@ func (store *RocksDBStore) ListDirectoryPrefixedEntries(ctx context.Context, ful lastFileStart = genDirectoryKeyPrefix(fullpath, startFileName) } - ro := gorocksdb.NewDefaultReadOptions() + ro := rocksdb.NewDefaultReadOptions() + defer ro.Destroy() ro.SetFillCache(false) + iter := store.db.NewIterator(ro) defer iter.Close() err = enumerate(iter, directoryPrefix, lastFileStart, inclusive, limit, func(key, value []byte) bool { @@ -290,4 +298,5 @@ func hashToBytes(dir string) []byte { func (store *RocksDBStore) Shutdown() { store.db.Close() + store.options.close() } From ebb223c190385a92c72b4d8096e71558c9f03354 Mon Sep 17 00:00:00 2001 From: qieqieplus Date: Mon, 4 Jan 2021 21:21:46 +0800 Subject: [PATCH 056/260] fix for rocksdb kv --- weed/filer/rocksdb/rocksdb_store_kv.go | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/weed/filer/rocksdb/rocksdb_store_kv.go b/weed/filer/rocksdb/rocksdb_store_kv.go index 093a905e8..cf1214d5b 100644 --- a/weed/filer/rocksdb/rocksdb_store_kv.go +++ b/weed/filer/rocksdb/rocksdb_store_kv.go @@ -5,15 +5,13 @@ package rocksdb import ( "context" "fmt" - "github.com/tecbot/gorocksdb" "github.com/chrislusf/seaweedfs/weed/filer" ) func (store *RocksDBStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) { - wo := gorocksdb.NewDefaultWriteOptions() - err = store.db.Put(wo, key, value) + err = store.db.Put(store.wo, key, value) if err != nil { return fmt.Errorf("kv put: %v", err) @@ -24,8 +22,7 @@ func (store *RocksDBStore) KvPut(ctx context.Context, key []byte, value []byte) func (store *RocksDBStore) KvGet(ctx context.Context, key []byte) (value []byte, err error) { - ro := gorocksdb.NewDefaultReadOptions() - value, err = store.db.GetBytes(ro, key) + value, err = store.db.GetBytes(store.ro, key) if value == nil { return nil, filer.ErrKvNotFound @@ -40,8 +37,7 @@ func (store *RocksDBStore) KvGet(ctx context.Context, key []byte) (value []byte, func (store *RocksDBStore) KvDelete(ctx context.Context, key []byte) (err error) { - wo := gorocksdb.NewDefaultWriteOptions() - err = store.db.Delete(wo, key) + err = store.db.Delete(store.wo, key) if err != nil { return fmt.Errorf("kv delete: %v", err) From e5a3ad372fff51f882a03498d523990b5bca2a4c Mon Sep 17 00:00:00 2001 From: henry Date: Tue, 5 Jan 2021 11:17:04 +0800 Subject: [PATCH 057/260] gitignore --- .gitignore | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.gitignore b/.gitignore index 671b01051..7a45c2ea0 100644 --- a/.gitignore +++ b/.gitignore @@ -83,3 +83,21 @@ other/java/hdfs/dependency-reduced-pom.xml # binary file weed/weed + + + +weed-ui/node_modules/ +weed-ui/dist/ +weed-ui/npm-debug.log* +weed-ui/yarn-debug.log* +weed-ui/yarn-error.log* +weed-ui//test/e2e/reports/ +weed-ui/selenium-debug.log + +# Editor directories and files + +weed-ui/.vscode +*.suo +*.ntvs* +*.njsproj +*.sln \ No newline at end of file From 942a30f04c74d7a73dce4169d558358ee7fd6689 Mon Sep 17 00:00:00 2001 From: henry Date: Tue, 5 Jan 2021 12:24:32 +0800 Subject: [PATCH 058/260] remove sort --- weed/filer/filer_search.go | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/weed/filer/filer_search.go b/weed/filer/filer_search.go index 7489d8e34..34358d849 100644 --- a/weed/filer/filer_search.go +++ b/weed/filer/filer_search.go @@ -42,17 +42,7 @@ func (f *Filer) ListDirectoryEntries(ctx context.Context, p util.FullPath, start } } - //Distinguish between folders and common files, folders are listed first - var folderEntries, commonEntries []*Entry - for i, _ := range entries { - if entries[i].IsDirectory() { - folderEntries = append(folderEntries, entries[i]) - } else { - commonEntries = append(commonEntries, entries[i]) - } - } - folderEntries = append(folderEntries, commonEntries...) - return folderEntries, err + return entries, err } func (f *Filer) doListPatternMatchedEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int, prefix, restNamePattern string) (matchedEntries []*Entry, missedCount int, lastFileName string, err error) { From e92ce1d51756cd3b1a311bf241f89df000c510a6 Mon Sep 17 00:00:00 2001 From: henry Date: Tue, 5 Jan 2021 12:26:51 +0800 Subject: [PATCH 059/260] remove sort --- .gitignore | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/.gitignore b/.gitignore index 7a45c2ea0..ca451260a 100644 --- a/.gitignore +++ b/.gitignore @@ -85,19 +85,3 @@ other/java/hdfs/dependency-reduced-pom.xml weed/weed - -weed-ui/node_modules/ -weed-ui/dist/ -weed-ui/npm-debug.log* -weed-ui/yarn-debug.log* -weed-ui/yarn-error.log* -weed-ui//test/e2e/reports/ -weed-ui/selenium-debug.log - -# Editor directories and files - -weed-ui/.vscode -*.suo -*.ntvs* -*.njsproj -*.sln \ No newline at end of file From 247d5eed937f624228e629fa5e89406616e26dd5 Mon Sep 17 00:00:00 2001 From: henry Date: Tue, 5 Jan 2021 12:27:26 +0800 Subject: [PATCH 060/260] remove sort --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index ca451260a..473918513 100644 --- a/.gitignore +++ b/.gitignore @@ -84,4 +84,3 @@ other/java/hdfs/dependency-reduced-pom.xml # binary file weed/weed - From e4ee25e6f0cd2b10b2f7745f5847845185bf2858 Mon Sep 17 00:00:00 2001 From: henry Date: Tue, 5 Jan 2021 12:29:46 +0800 Subject: [PATCH 061/260] remove sort --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index 473918513..671b01051 100644 --- a/.gitignore +++ b/.gitignore @@ -83,4 +83,3 @@ other/java/hdfs/dependency-reduced-pom.xml # binary file weed/weed - From 1309dfe612ae818134d2241c38fc3858516ea729 Mon Sep 17 00:00:00 2001 From: henry Date: Tue, 5 Jan 2021 12:58:46 +0800 Subject: [PATCH 062/260] Seaweed custom header are not visible to Vue or javascript --- weed/server/filer_server_handlers_read.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.go index 28027a009..46a7ca9d9 100644 --- a/weed/server/filer_server_handlers_read.go +++ b/weed/server/filer_server_handlers_read.go @@ -102,9 +102,9 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, //Add some custom headers that need to be exposed seaweedHeaders := []string{} - for i, _ := range w.Header() { - if strings.Contains(i, "Seaweed") { - seaweedHeaders = append(seaweedHeaders, i) + for header, _ := range w.Header() { + if strings.HasPrefix(header, "Seaweed-") { + seaweedHeaders = append(seaweedHeaders, header) } } seaweedHeaders = append(seaweedHeaders, "Content-Disposition") From b42c5b71c1ec97bd096785ee1428fe257f10c88b Mon Sep 17 00:00:00 2001 From: henry Date: Tue, 5 Jan 2021 13:01:29 +0800 Subject: [PATCH 063/260] Seaweed custom header are not visible to Vue or javascript --- weed/server/filer_server_handlers_read.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.go index 46a7ca9d9..5cfbfe86c 100644 --- a/weed/server/filer_server_handlers_read.go +++ b/weed/server/filer_server_handlers_read.go @@ -100,7 +100,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, w.Header().Set(k, string(v)) } - //Add some custom headers that need to be exposed + //Seaweed custom header are not visible to Vue or javascript seaweedHeaders := []string{} for header, _ := range w.Header() { if strings.HasPrefix(header, "Seaweed-") { From bbae13e41581d8fa9dca2e5df7823d15ce94afe9 Mon Sep 17 00:00:00 2001 From: qieqieplus Date: Tue, 5 Jan 2021 17:47:59 +0800 Subject: [PATCH 064/260] impl: TTL per entry for rocksdb; fix package name --- weed/filer/rocksdb/rocksdb_store.go | 40 +++++++++++++++++----------- weed/filer/rocksdb/rocksdb_ttl.go | 41 +++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 16 deletions(-) create mode 100644 weed/filer/rocksdb/rocksdb_ttl.go diff --git a/weed/filer/rocksdb/rocksdb_store.go b/weed/filer/rocksdb/rocksdb_store.go index ca6391386..9e058a96d 100644 --- a/weed/filer/rocksdb/rocksdb_store.go +++ b/weed/filer/rocksdb/rocksdb_store.go @@ -7,12 +7,14 @@ import ( "context" "crypto/md5" "fmt" + "io" + + "github.com/tecbot/gorocksdb" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" weed_util "github.com/chrislusf/seaweedfs/weed/util" - rocksdb "github.com/tecbot/gorocksdb" - "io" ) func init() { @@ -20,15 +22,15 @@ func init() { } type options struct { - opt *rocksdb.Options - ro *rocksdb.ReadOptions - wo *rocksdb.WriteOptions + opt *gorocksdb.Options + ro *gorocksdb.ReadOptions + wo *gorocksdb.WriteOptions } func (opt *options) init() { - opt.opt = rocksdb.NewDefaultOptions() - opt.ro = rocksdb.NewDefaultReadOptions() - opt.wo = rocksdb.NewDefaultWriteOptions() + opt.opt = gorocksdb.NewDefaultOptions() + opt.ro = gorocksdb.NewDefaultReadOptions() + opt.wo = gorocksdb.NewDefaultWriteOptions() } func (opt *options) close() { @@ -39,7 +41,7 @@ func (opt *options) close() { type RocksDBStore struct { path string - db *rocksdb.DB + db *gorocksdb.DB options } @@ -59,7 +61,13 @@ func (store *RocksDBStore) initialize(dir string) (err error) { } store.options.init() store.opt.SetCreateIfMissing(true) - store.db, err = rocksdb.OpenDb(store.opt, dir) + // reduce write amplification + // also avoid expired data stored in highest level never get compacted + store.opt.SetLevelCompactionDynamicLevelBytes(true) + store.opt.SetCompactionFilter(NewTTLFilter()) + // store.opt.SetMaxBackgroundCompactions(2) + + store.db, err = gorocksdb.OpenDb(store.opt, dir) return } @@ -116,7 +124,7 @@ func (store *RocksDBStore) FindEntry(ctx context.Context, fullpath weed_util.Ful entry = &filer.Entry{ FullPath: fullpath, } - err = entry.DecodeAttributesAndChunks(weed_util.MaybeDecompressData(data.Data())) + err = entry.DecodeAttributesAndChunks(data.Data()) if err != nil { return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err) } @@ -141,10 +149,10 @@ func (store *RocksDBStore) DeleteEntry(ctx context.Context, fullpath weed_util.F func (store *RocksDBStore) DeleteFolderChildren(ctx context.Context, fullpath weed_util.FullPath) (err error) { directoryPrefix := genDirectoryKeyPrefix(fullpath, "") - batch := rocksdb.NewWriteBatch() + batch := gorocksdb.NewWriteBatch() defer batch.Destroy() - ro := rocksdb.NewDefaultReadOptions() + ro := gorocksdb.NewDefaultReadOptions() defer ro.Destroy() ro.SetFillCache(false) @@ -167,7 +175,7 @@ func (store *RocksDBStore) DeleteFolderChildren(ctx context.Context, fullpath we return nil } -func enumerate(iter *rocksdb.Iterator, prefix, lastKey []byte, includeLastKey bool, limit int, fn func(key, value []byte) bool) error { +func enumerate(iter *gorocksdb.Iterator, prefix, lastKey []byte, includeLastKey bool, limit int, fn func(key, value []byte) bool) error { if len(lastKey) == 0 { iter.Seek(prefix) @@ -231,7 +239,7 @@ func (store *RocksDBStore) ListDirectoryPrefixedEntries(ctx context.Context, ful lastFileStart = genDirectoryKeyPrefix(fullpath, startFileName) } - ro := rocksdb.NewDefaultReadOptions() + ro := gorocksdb.NewDefaultReadOptions() defer ro.Destroy() ro.SetFillCache(false) @@ -251,7 +259,7 @@ func (store *RocksDBStore) ListDirectoryPrefixedEntries(ctx context.Context, ful } // println("list", entry.FullPath, "chunks", len(entry.Chunks)) - if decodeErr := entry.DecodeAttributesAndChunks(weed_util.MaybeDecompressData(value)); decodeErr != nil { + if decodeErr := entry.DecodeAttributesAndChunks(value); decodeErr != nil { err = decodeErr glog.V(0).Infof("list %s : %v", entry.FullPath, err) return false diff --git a/weed/filer/rocksdb/rocksdb_ttl.go b/weed/filer/rocksdb/rocksdb_ttl.go new file mode 100644 index 000000000..1fc239c0c --- /dev/null +++ b/weed/filer/rocksdb/rocksdb_ttl.go @@ -0,0 +1,41 @@ +//+build rocksdb + +package rocksdb + +import ( + "time" + + "github.com/tecbot/gorocksdb" + + "github.com/chrislusf/seaweedfs/weed/filer" +) + +type TTLFilter struct { + skipLevel0 bool +} + +func NewTTLFilter() gorocksdb.CompactionFilter { + return &TTLFilter{ + skipLevel0: true, + } +} + +func (t *TTLFilter) Filter(level int, key, val []byte) (remove bool, newVal []byte) { + // decode could be slow, causing write stall + // level >0 sst can run compaction in parallel + if t.skipLevel0 && level == 0 { + return false, val + } + entry := filer.Entry{} + if err := entry.DecodeAttributesAndChunks(val); err == nil { + if entry.TtlSec == 0 || + entry.Crtime.Add(time.Duration(entry.TtlSec)*time.Second).Before(time.Now()) { + return false, val + } + } + return true, nil +} + +func (t *TTLFilter) Name() string { + return "TTLFilter" +} From b2e50f602f23f919b6837b1207056cf1ebc7fc94 Mon Sep 17 00:00:00 2001 From: qieqieplus Date: Tue, 5 Jan 2021 19:36:40 +0800 Subject: [PATCH 065/260] fix time comparison --- weed/filer/rocksdb/rocksdb_ttl.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/filer/rocksdb/rocksdb_ttl.go b/weed/filer/rocksdb/rocksdb_ttl.go index 1fc239c0c..98918b5d7 100644 --- a/weed/filer/rocksdb/rocksdb_ttl.go +++ b/weed/filer/rocksdb/rocksdb_ttl.go @@ -29,7 +29,7 @@ func (t *TTLFilter) Filter(level int, key, val []byte) (remove bool, newVal []by entry := filer.Entry{} if err := entry.DecodeAttributesAndChunks(val); err == nil { if entry.TtlSec == 0 || - entry.Crtime.Add(time.Duration(entry.TtlSec)*time.Second).Before(time.Now()) { + entry.Crtime.Add(time.Duration(entry.TtlSec)*time.Second).After(time.Now()) { return false, val } } From a08e9d59ecaaa563b993478359aab6794b2d7269 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 5 Jan 2021 12:41:11 -0800 Subject: [PATCH 066/260] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 01be6d791..8edaa6075 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ Table of Contents * Download the latest binary from https://github.com/chrislusf/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe` * Run `weed server -dir=/some/data/dir -s3` to start one master, one volume server, one filer, and one S3 gateway. -Also, to increase capacity, just add more volume servers by `weed volume -dir="/some/data/dir2" -mserver=":9333" -port=8081` locally or a different machine. That is it! +Also, to increase capacity, just add more volume servers by `weed volume -dir="/some/data/dir2" -mserver=":9333" -port=8081` locally, or a different machine, or thoudsands of machines. That is it! ## Introduction ## From 9e670c2e2302df31322d8afda7ef8512ca498eba Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 5 Jan 2021 13:20:58 -0800 Subject: [PATCH 067/260] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8edaa6075..92979a174 100644 --- a/README.md +++ b/README.md @@ -84,7 +84,7 @@ There is only 40 bytes of disk storage overhead for each file's metadata. It is SeaweedFS started by implementing [Facebook's Haystack design paper](http://www.usenix.org/event/osdi10/tech/full_papers/Beaver.pdf). Also, SeaweedFS implements erasure coding with ideas from [f4: Facebook’s Warm BLOB Storage System](https://www.usenix.org/system/files/conference/osdi14/osdi14-paper-muralidhar.pdf) -On top of the object store, optional [Filer] can support directories and POSIX attributes. Filer is a separate linearly-scalable stateless server with customizable metadata stores, e.g., MySql, Postgres, Redis, Cassandra, HBase, Mongodb, Elastic Search, LevelDB, MemSql, TiDB, Etcd, CockroachDB, etc. +On top of the object store, optional [Filer] can support directories and POSIX attributes. Filer is a separate linearly-scalable stateless server with customizable metadata stores, e.g., MySql, Postgres, Redis, Cassandra, HBase, Mongodb, Elastic Search, LevelDB, RocksDB, MemSql, TiDB, Etcd, CockroachDB, etc. For any distributed key value stores, the large values can be offloaded to SeaweedFS. With the fast access speed and linearly scalable capacity, SeaweedFS can work as a distributed [Key-Large-Value store][KeyLargeValueStore]. From d253e0ea6e1136bf849bb46d3459f78af8082918 Mon Sep 17 00:00:00 2001 From: yuwang Date: Wed, 6 Jan 2021 15:11:54 +0800 Subject: [PATCH 068/260] fix s3 upload "use of closed network connection" --- go.mod | 2 +- go.sum | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 467b3b1fc..c46456f25 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/Azure/azure-storage-blob-go v0.8.0 github.com/OneOfOne/xxhash v1.2.2 github.com/Shopify/sarama v1.23.1 - github.com/aws/aws-sdk-go v1.33.5 + github.com/aws/aws-sdk-go v1.34.30 github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72 github.com/cespare/xxhash v1.1.0 github.com/chrislusf/raft v1.0.4 diff --git a/go.sum b/go.sum index 6f4519aef..f542c3a14 100644 --- a/go.sum +++ b/go.sum @@ -61,6 +61,8 @@ github.com/aws/aws-sdk-go v1.19.45/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpi github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.33.5 h1:p2fr1ryvNTU6avUWLI+/H7FGv0TBIjzVM5WDgXBBv4U= github.com/aws/aws-sdk-go v1.33.5/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= +github.com/aws/aws-sdk-go v1.34.30 h1:izATc/E0+HcT5YHmaQVjn7GHCoqaBxn0PGo6Zq5UNFA= +github.com/aws/aws-sdk-go v1.34.30/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= @@ -84,8 +86,6 @@ github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chrislusf/raft v1.0.3 h1:11YrnzJtVa5z7m9lhY2p8VcPHoUlC1UswyoAo+U1m1k= -github.com/chrislusf/raft v1.0.3/go.mod h1:Ep5DP+mJSosjfKiix1uU7Lc2Df/SX4oGJEpZlXH5l68= github.com/chrislusf/raft v1.0.4 h1:THhbsVik2hxdE0/VXX834f64Wn9RzgVPp+E+XCWZdKM= github.com/chrislusf/raft v1.0.4/go.mod h1:Ep5DP+mJSosjfKiix1uU7Lc2Df/SX4oGJEpZlXH5l68= github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= @@ -345,6 +345,10 @@ github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5i github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= @@ -934,6 +938,8 @@ gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bl gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From 0764fccde7f88fb0d3b2b5824f03ed2cdc2ca42b Mon Sep 17 00:00:00 2001 From: qieqieplus Date: Wed, 6 Jan 2021 17:42:43 +0800 Subject: [PATCH 069/260] ignore decode error for non-entry data --- weed/filer/rocksdb/rocksdb_ttl.go | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/weed/filer/rocksdb/rocksdb_ttl.go b/weed/filer/rocksdb/rocksdb_ttl.go index 98918b5d7..faed22310 100644 --- a/weed/filer/rocksdb/rocksdb_ttl.go +++ b/weed/filer/rocksdb/rocksdb_ttl.go @@ -23,17 +23,16 @@ func NewTTLFilter() gorocksdb.CompactionFilter { func (t *TTLFilter) Filter(level int, key, val []byte) (remove bool, newVal []byte) { // decode could be slow, causing write stall // level >0 sst can run compaction in parallel - if t.skipLevel0 && level == 0 { - return false, val - } - entry := filer.Entry{} - if err := entry.DecodeAttributesAndChunks(val); err == nil { - if entry.TtlSec == 0 || - entry.Crtime.Add(time.Duration(entry.TtlSec)*time.Second).After(time.Now()) { - return false, val + if !t.skipLevel0 || level > 0 { + entry := filer.Entry{} + if err := entry.DecodeAttributesAndChunks(val); err == nil { + if entry.TtlSec > 0 && + entry.Crtime.Add(time.Duration(entry.TtlSec)*time.Second).Before(time.Now()) { + return true, nil + } } } - return true, nil + return false, val } func (t *TTLFilter) Name() string { From 2b7685464139e8fbfedbb357f72c4ff70bd869e1 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 6 Jan 2021 04:21:34 -0800 Subject: [PATCH 070/260] add "weed filer.cat" to read files directly from volume servers --- weed/command/command.go | 1 + weed/command/filer_cat.go | 118 ++++++++++++++++++ weed/command/filer_copy.go | 2 +- weed/filer/filechunk_manifest.go | 7 +- weed/filer/filechunks.go | 9 +- weed/filer/reader_at.go | 6 +- weed/filer/stream.go | 8 +- weed/replication/sink/filersink/filer_sink.go | 3 +- weed/wdclient/vid_map.go | 10 ++ 9 files changed, 147 insertions(+), 17 deletions(-) create mode 100644 weed/command/filer_cat.go diff --git a/weed/command/command.go b/weed/command/command.go index 0df22b575..8f9cec087 100644 --- a/weed/command/command.go +++ b/weed/command/command.go @@ -15,6 +15,7 @@ var Commands = []*Command{ cmdDownload, cmdExport, cmdFiler, + cmdFilerCat, cmdFilerReplicate, cmdFilerSynchronize, cmdFix, diff --git a/weed/command/filer_cat.go b/weed/command/filer_cat.go new file mode 100644 index 000000000..a46098b04 --- /dev/null +++ b/weed/command/filer_cat.go @@ -0,0 +1,118 @@ +package command + +import ( + "context" + "fmt" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/pb" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/wdclient" + "google.golang.org/grpc" + "math" + "net/url" + "os" + "strings" + + "github.com/chrislusf/seaweedfs/weed/security" + "github.com/chrislusf/seaweedfs/weed/util" +) + +var ( + filerCat FilerCatOptions +) + +type FilerCatOptions struct { + grpcDialOption grpc.DialOption + filerAddress string + filerClient filer_pb.SeaweedFilerClient + output *string +} + +func (fco *FilerCatOptions) GetLookupFileIdFunction() wdclient.LookupFileIdFunctionType { + return func(fileId string) (targetUrls []string, err error) { + vid := filer.VolumeId(fileId) + resp, err := fco.filerClient.LookupVolume(context.Background(), &filer_pb.LookupVolumeRequest{ + VolumeIds: []string{vid}, + }) + if err != nil { + return nil, err + } + locations := resp.LocationsMap[vid] + for _, loc := range locations.Locations { + targetUrls = append(targetUrls, fmt.Sprintf("http://%s/%s", loc.Url, fileId)) + } + return + } +} + +func init() { + cmdFilerCat.Run = runFilerCat // break init cycle + filerCat.output = cmdFilerCat.Flag.String("o", "", "write to file instead of stdout") +} + +var cmdFilerCat = &Command{ + UsageLine: "filer.cat [-o ] http://localhost:8888/path/to/file", + Short: "copy one file to local", + Long: `read one file to stdout or write to a file + +`, +} + +func runFilerCat(cmd *Command, args []string) bool { + + util.LoadConfiguration("security", false) + + if len(args) == 0 { + return false + } + filerSource := args[len(args)-1] + + filerUrl, err := url.Parse(filerSource) + if err != nil { + fmt.Printf("The last argument should be a URL on filer: %v\n", err) + return false + } + urlPath := filerUrl.Path + if strings.HasSuffix(urlPath, "/") { + fmt.Printf("The last argument should be a file: %v\n", err) + return false + } + + filerCat.filerAddress = filerUrl.Host + filerCat.grpcDialOption = security.LoadClientTLS(util.GetViper(), "grpc.client") + + dir, name := util.FullPath(urlPath).DirAndName() + + writer := os.Stdout + if *filerCat.output != "" { + + fmt.Printf("saving %s to %s\n", filerSource, *filerCat.output) + + f, err := os.OpenFile(*filerCat.output, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0755) + if err != nil { + fmt.Printf("open file %s: %v\n", *filerCat.output, err) + return false + } + defer f.Close() + writer = f + } + + pb.WithFilerClient(filerCat.filerAddress, filerCat.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { + + request := &filer_pb.LookupDirectoryEntryRequest{ + Name: name, + Directory: dir, + } + respLookupEntry, err := filer_pb.LookupEntry(client, request) + if err != nil { + return err + } + + filerCat.filerClient = client + + return filer.StreamContent(&filerCat, writer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64) + + }) + + return true +} diff --git a/weed/command/filer_copy.go b/weed/command/filer_copy.go index 93248f357..b95df696c 100644 --- a/weed/command/filer_copy.go +++ b/weed/command/filer_copy.go @@ -92,7 +92,7 @@ func runCopy(cmd *Command, args []string) bool { } urlPath := filerUrl.Path if !strings.HasSuffix(urlPath, "/") { - fmt.Printf("The last argument should be a folder and end with \"/\": %v\n", err) + fmt.Printf("The last argument should be a folder and end with \"/\"\n") return false } diff --git a/weed/filer/filechunk_manifest.go b/weed/filer/filechunk_manifest.go index f5ab36d37..845bfaec1 100644 --- a/weed/filer/filechunk_manifest.go +++ b/weed/filer/filechunk_manifest.go @@ -3,6 +3,7 @@ package filer import ( "bytes" "fmt" + "github.com/chrislusf/seaweedfs/weed/wdclient" "io" "math" "time" @@ -38,7 +39,7 @@ func SeparateManifestChunks(chunks []*filer_pb.FileChunk) (manifestChunks, nonMa return } -func ResolveChunkManifest(lookupFileIdFn LookupFileIdFunctionType, chunks []*filer_pb.FileChunk) (dataChunks, manifestChunks []*filer_pb.FileChunk, manifestResolveErr error) { +func ResolveChunkManifest(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk) (dataChunks, manifestChunks []*filer_pb.FileChunk, manifestResolveErr error) { // TODO maybe parallel this for _, chunk := range chunks { if !chunk.IsChunkManifest { @@ -63,7 +64,7 @@ func ResolveChunkManifest(lookupFileIdFn LookupFileIdFunctionType, chunks []*fil return } -func ResolveOneChunkManifest(lookupFileIdFn LookupFileIdFunctionType, chunk *filer_pb.FileChunk) (dataChunks []*filer_pb.FileChunk, manifestResolveErr error) { +func ResolveOneChunkManifest(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunk *filer_pb.FileChunk) (dataChunks []*filer_pb.FileChunk, manifestResolveErr error) { if !chunk.IsChunkManifest { return } @@ -84,7 +85,7 @@ func ResolveOneChunkManifest(lookupFileIdFn LookupFileIdFunctionType, chunk *fil } // TODO fetch from cache for weed mount? -func fetchChunk(lookupFileIdFn LookupFileIdFunctionType, fileId string, cipherKey []byte, isGzipped bool) ([]byte, error) { +func fetchChunk(lookupFileIdFn wdclient.LookupFileIdFunctionType, fileId string, cipherKey []byte, isGzipped bool) ([]byte, error) { urlStrings, err := lookupFileIdFn(fileId) if err != nil { glog.Errorf("operation LookupFileId %s failed, err: %v", fileId, err) diff --git a/weed/filer/filechunks.go b/weed/filer/filechunks.go index c75a35f79..68f308a51 100644 --- a/weed/filer/filechunks.go +++ b/weed/filer/filechunks.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/hex" "fmt" + "github.com/chrislusf/seaweedfs/weed/wdclient" "math" "sort" "sync" @@ -52,7 +53,7 @@ func ETagChunks(chunks []*filer_pb.FileChunk) (etag string) { return fmt.Sprintf("%x-%d", util.Md5(bytes.Join(md5_digests, nil)), len(chunks)) } -func CompactFileChunks(lookupFileIdFn LookupFileIdFunctionType, chunks []*filer_pb.FileChunk) (compacted, garbage []*filer_pb.FileChunk) { +func CompactFileChunks(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk) (compacted, garbage []*filer_pb.FileChunk) { visibles, _ := NonOverlappingVisibleIntervals(lookupFileIdFn, chunks) @@ -71,7 +72,7 @@ func CompactFileChunks(lookupFileIdFn LookupFileIdFunctionType, chunks []*filer_ return } -func MinusChunks(lookupFileIdFn LookupFileIdFunctionType, as, bs []*filer_pb.FileChunk) (delta []*filer_pb.FileChunk, err error) { +func MinusChunks(lookupFileIdFn wdclient.LookupFileIdFunctionType, as, bs []*filer_pb.FileChunk) (delta []*filer_pb.FileChunk, err error) { aData, aMeta, aErr := ResolveChunkManifest(lookupFileIdFn, as) if aErr != nil { @@ -116,7 +117,7 @@ func (cv *ChunkView) IsFullChunk() bool { return cv.Size == cv.ChunkSize } -func ViewFromChunks(lookupFileIdFn LookupFileIdFunctionType, chunks []*filer_pb.FileChunk, offset int64, size int64) (views []*ChunkView) { +func ViewFromChunks(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk, offset int64, size int64) (views []*ChunkView) { visibles, _ := NonOverlappingVisibleIntervals(lookupFileIdFn, chunks) @@ -222,7 +223,7 @@ func MergeIntoVisibles(visibles []VisibleInterval, chunk *filer_pb.FileChunk) (n // NonOverlappingVisibleIntervals translates the file chunk into VisibleInterval in memory // If the file chunk content is a chunk manifest -func NonOverlappingVisibleIntervals(lookupFileIdFn LookupFileIdFunctionType, chunks []*filer_pb.FileChunk) (visibles []VisibleInterval, err error) { +func NonOverlappingVisibleIntervals(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk) (visibles []VisibleInterval, err error) { chunks, _, err = ResolveChunkManifest(lookupFileIdFn, chunks) diff --git a/weed/filer/reader_at.go b/weed/filer/reader_at.go index 6193dbd45..1f594a1bf 100644 --- a/weed/filer/reader_at.go +++ b/weed/filer/reader_at.go @@ -18,7 +18,7 @@ import ( type ChunkReadAt struct { masterClient *wdclient.MasterClient chunkViews []*ChunkView - lookupFileId LookupFileIdFunctionType + lookupFileId wdclient.LookupFileIdFunctionType readerLock sync.Mutex fileSize int64 @@ -31,9 +31,7 @@ type ChunkReadAt struct { var _ = io.ReaderAt(&ChunkReadAt{}) var _ = io.Closer(&ChunkReadAt{}) -type LookupFileIdFunctionType func(fileId string) (targetUrls []string, err error) - -func LookupFn(filerClient filer_pb.FilerClient) LookupFileIdFunctionType { +func LookupFn(filerClient filer_pb.FilerClient) wdclient.LookupFileIdFunctionType { vidCache := make(map[string]*filer_pb.Locations) var vicCacheLock sync.RWMutex diff --git a/weed/filer/stream.go b/weed/filer/stream.go index cffdc8303..f0042a0ff 100644 --- a/weed/filer/stream.go +++ b/weed/filer/stream.go @@ -13,16 +13,16 @@ import ( "github.com/chrislusf/seaweedfs/weed/wdclient" ) -func StreamContent(masterClient *wdclient.MasterClient, w io.Writer, chunks []*filer_pb.FileChunk, offset int64, size int64) error { +func StreamContent(masterClient wdclient.HasLookupFileIdFunction, w io.Writer, chunks []*filer_pb.FileChunk, offset int64, size int64) error { // fmt.Printf("start to stream content for chunks: %+v\n", chunks) - chunkViews := ViewFromChunks(masterClient.LookupFileId, chunks, offset, size) + chunkViews := ViewFromChunks(masterClient.GetLookupFileIdFunction(), chunks, offset, size) fileId2Url := make(map[string][]string) for _, chunkView := range chunkViews { - urlStrings, err := masterClient.LookupFileId(chunkView.FileId) + urlStrings, err := masterClient.GetLookupFileIdFunction()(chunkView.FileId) if err != nil { glog.V(1).Infof("operation LookupFileId %s failed, err: %v", chunkView.FileId, err) return err @@ -86,7 +86,7 @@ type ChunkStreamReader struct { bufferOffset int64 bufferPos int chunkIndex int - lookupFileId LookupFileIdFunctionType + lookupFileId wdclient.LookupFileIdFunctionType } var _ = io.ReadSeeker(&ChunkStreamReader{}) diff --git a/weed/replication/sink/filersink/filer_sink.go b/weed/replication/sink/filersink/filer_sink.go index a58c8f296..9c0e4176f 100644 --- a/weed/replication/sink/filersink/filer_sink.go +++ b/weed/replication/sink/filersink/filer_sink.go @@ -3,6 +3,7 @@ package filersink import ( "context" "fmt" + "github.com/chrislusf/seaweedfs/weed/wdclient" "google.golang.org/grpc" @@ -206,7 +207,7 @@ func (fs *FilerSink) UpdateEntry(key string, oldEntry *filer_pb.Entry, newParent }) } -func compareChunks(lookupFileIdFn filer.LookupFileIdFunctionType, oldEntry, newEntry *filer_pb.Entry) (deletedChunks, newChunks []*filer_pb.FileChunk, err error) { +func compareChunks(lookupFileIdFn wdclient.LookupFileIdFunctionType, oldEntry, newEntry *filer_pb.Entry) (deletedChunks, newChunks []*filer_pb.FileChunk, err error) { aData, aMeta, aErr := filer.ResolveChunkManifest(lookupFileIdFn, oldEntry.Chunks) if aErr != nil { return nil, nil, aErr diff --git a/weed/wdclient/vid_map.go b/weed/wdclient/vid_map.go index 773da0191..271baa132 100644 --- a/weed/wdclient/vid_map.go +++ b/weed/wdclient/vid_map.go @@ -15,6 +15,12 @@ const ( maxCursorIndex = 4096 ) +type HasLookupFileIdFunction interface { + GetLookupFileIdFunction() LookupFileIdFunctionType +} + +type LookupFileIdFunctionType func(fileId string) (targetUrls []string, err error) + type Location struct { Url string `json:"url,omitempty"` PublicUrl string `json:"publicUrl,omitempty"` @@ -67,6 +73,10 @@ func (vc *vidMap) LookupVolumeServerUrl(vid string) (serverUrls []string, err er return } +func (vc *vidMap) GetLookupFileIdFunction() LookupFileIdFunctionType { + return vc.LookupFileId +} + func (vc *vidMap) LookupFileId(fileId string) (fullUrls []string, err error) { parts := strings.Split(fileId, ",") if len(parts) != 2 { From e327385644324776d0a2a06935bfc89dc58546a3 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 6 Jan 2021 04:21:57 -0800 Subject: [PATCH 071/260] go fmt --- weed/filer/hbase/hbase_store_kv.go | 3 ++- weed/s3api/s3api_bucket_handlers.go | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/weed/filer/hbase/hbase_store_kv.go b/weed/filer/hbase/hbase_store_kv.go index 26bf763e2..990e55a24 100644 --- a/weed/filer/hbase/hbase_store_kv.go +++ b/weed/filer/hbase/hbase_store_kv.go @@ -7,9 +7,10 @@ import ( "time" ) -const( +const ( COLUMN_NAME = "a" ) + func (store *HbaseStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) { return store.doPut(ctx, store.cfKv, key, value, 0) } diff --git a/weed/s3api/s3api_bucket_handlers.go b/weed/s3api/s3api_bucket_handlers.go index f750f6e53..338f82668 100644 --- a/weed/s3api/s3api_bucket_handlers.go +++ b/weed/s3api/s3api_bucket_handlers.go @@ -51,7 +51,7 @@ func (s3a *S3ApiServer) ListBucketsHandler(w http.ResponseWriter, r *http.Reques var buckets []*s3.Bucket for _, entry := range entries { if entry.IsDirectory { - if identity!=nil && !identity.canDo(s3_constants.ACTION_ADMIN, entry.Name) { + if identity != nil && !identity.canDo(s3_constants.ACTION_ADMIN, entry.Name) { continue } buckets = append(buckets, &s3.Bucket{ From c7e7431fd367117590930cc1948aa9a25da60d7a Mon Sep 17 00:00:00 2001 From: qieqieplus Date: Thu, 7 Jan 2021 17:07:56 +0800 Subject: [PATCH 072/260] fix seek lastkey may reach EOF --- weed/filer/rocksdb/rocksdb_store.go | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/weed/filer/rocksdb/rocksdb_store.go b/weed/filer/rocksdb/rocksdb_store.go index 9e058a96d..2d5501637 100644 --- a/weed/filer/rocksdb/rocksdb_store.go +++ b/weed/filer/rocksdb/rocksdb_store.go @@ -182,17 +182,9 @@ func enumerate(iter *gorocksdb.Iterator, prefix, lastKey []byte, includeLastKey } else { iter.Seek(lastKey) - if !includeLastKey { - key := iter.Key().Data() - - if !bytes.HasPrefix(key, prefix) { - return nil - } - - if bytes.Equal(key, lastKey) { - iter.Next() - } - + if iter.Valid() && !includeLastKey && + bytes.Equal(iter.Key().Data(), lastKey) { + iter.Next() } } @@ -250,10 +242,6 @@ func (store *RocksDBStore) ListDirectoryPrefixedEntries(ctx context.Context, ful if fileName == "" { return true } - limit-- - if limit < 0 { - return false - } entry := &filer.Entry{ FullPath: weed_util.NewFullPath(string(fullpath), fileName), } From 0f18592315e8a184f1733d9eb9ddc77e682e0c23 Mon Sep 17 00:00:00 2001 From: qieqieplus Date: Thu, 7 Jan 2021 18:21:48 +0800 Subject: [PATCH 073/260] alter style --- weed/filer/rocksdb/rocksdb_store.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/weed/filer/rocksdb/rocksdb_store.go b/weed/filer/rocksdb/rocksdb_store.go index 2d5501637..0793f547c 100644 --- a/weed/filer/rocksdb/rocksdb_store.go +++ b/weed/filer/rocksdb/rocksdb_store.go @@ -181,10 +181,12 @@ func enumerate(iter *gorocksdb.Iterator, prefix, lastKey []byte, includeLastKey iter.Seek(prefix) } else { iter.Seek(lastKey) - - if iter.Valid() && !includeLastKey && - bytes.Equal(iter.Key().Data(), lastKey) { - iter.Next() + if !includeLastKey { + if iter.Valid() { + if bytes.Equal(iter.Key().Data(), lastKey) { + iter.Next() + } + } } } From 30efaa66c69a20df91244d060af1067edb263bf7 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 7 Jan 2021 02:35:47 -0800 Subject: [PATCH 074/260] server: add option to disable master --- weed/command/server.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/weed/command/server.go b/weed/command/server.go index bd25f94b1..187563db1 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -61,6 +61,7 @@ var ( serverMetricsHttpPort = cmdServer.Flag.Int("metricsPort", 0, "Prometheus metrics listen port") // pulseSeconds = cmdServer.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats") + isStartingMasterServer = cmdServer.Flag.Bool("master", true, "whether to start master server") isStartingVolumeServer = cmdServer.Flag.Bool("volume", true, "whether to start volume server") isStartingFiler = cmdServer.Flag.Bool("filer", false, "whether to start filer") isStartingS3 = cmdServer.Flag.Bool("s3", false, "whether to start S3 gateway") @@ -224,7 +225,11 @@ func runServer(cmd *Command, args []string) bool { } - startMaster(masterOptions, serverWhiteList) + if *isStartingMasterServer { + go startMaster(masterOptions, serverWhiteList) + } + + select {} return true } From 40161c9cec3b8a0ed16bb355c6d6f3e7f23a094c Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 8 Jan 2021 01:08:44 -0800 Subject: [PATCH 075/260] filer store: redis module upgrade --- go.mod | 6 +-- go.sum | 41 ++++++++++++++++--- weed/filer/redis/redis_cluster_store.go | 2 +- weed/filer/redis/redis_store.go | 2 +- weed/filer/redis/universal_redis_store.go | 22 +++++----- weed/filer/redis/universal_redis_store_kv.go | 8 ++-- weed/filer/redis2/redis_cluster_store.go | 2 +- weed/filer/redis2/redis_store.go | 2 +- weed/filer/redis2/universal_redis_store.go | 26 ++++++------ weed/filer/redis2/universal_redis_store_kv.go | 8 ++-- 10 files changed, 73 insertions(+), 46 deletions(-) diff --git a/go.mod b/go.mod index c46456f25..40708b9f9 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/fclairamb/ftpserverlib v0.8.0 github.com/frankban/quicktest v1.7.2 // indirect github.com/go-errors/errors v1.1.1 // indirect - github.com/go-redis/redis v6.15.7+incompatible + github.com/go-redis/redis/v8 v8.4.4 github.com/go-sql-driver/mysql v1.5.0 github.com/gocql/gocql v0.0.0-20190829130954-e163eff7a8c6 github.com/gogo/protobuf v1.2.2-0.20190730201129-28a6bbf47e48 // indirect @@ -51,8 +51,6 @@ require ( github.com/mattn/go-ieproxy v0.0.0-20190805055040-f9202b1cfdeb // indirect github.com/mattn/go-runewidth v0.0.4 // indirect github.com/olivere/elastic/v7 v7.0.19 - github.com/onsi/ginkgo v1.10.1 // indirect - github.com/onsi/gomega v1.7.0 // indirect github.com/peterh/liner v1.1.0 github.com/pierrec/lz4 v2.2.7+incompatible // indirect github.com/prometheus/client_golang v1.3.0 @@ -83,7 +81,7 @@ require ( gocloud.dev/pubsub/natspubsub v0.16.0 gocloud.dev/pubsub/rabbitpubsub v0.16.0 golang.org/x/image v0.0.0-20200119044424-58c23975cae1 // indirect - golang.org/x/net v0.0.0-20200202094626-16171245cfb2 + golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb golang.org/x/sync v0.0.0-20200930132711-30421366ff76 // indirect golang.org/x/sys v0.0.0-20201022201747-fb209a7c41cd golang.org/x/tools v0.0.0-20200103221440-774c71fcf114 diff --git a/go.sum b/go.sum index f542c3a14..3c0396f85 100644 --- a/go.sum +++ b/go.sum @@ -116,6 +116,8 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/devigned/tab v0.1.1/go.mod h1:XG9mPq0dFghrYvoBF3xdRrJzSTX1b7IQrvaL9mzjeJY= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c= @@ -159,6 +161,8 @@ github.com/frankban/quicktest v1.7.2 h1:2QxQoC1TS09S7fhCPsrvqYdvP1H5M1P1ih5ABm3B github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-errors/errors v1.1.1 h1:ljK/pL5ltg3qoN+OtN6yCv9HWSfMwxSx90GJCZQxYNg= @@ -172,8 +176,8 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih4= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-redis/redis v6.15.7+incompatible h1:3skhDh95XQMpnqeqNftPkQD9jL9e5e36z/1SUm6dy1U= -github.com/go-redis/redis v6.15.7+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= +github.com/go-redis/redis/v8 v8.4.4 h1:fGqgxCTR1sydaKI00oQf3OmkU/DIe/I/fYXvGklCIuc= +github.com/go-redis/redis/v8 v8.4.4/go.mod h1:nA0bQuF0i5JFx4Ta9RZxGKXFrQ8cRWntra97f0196iY= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= @@ -259,6 +263,8 @@ github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-replayers/grpcreplay v0.1.0 h1:eNb1y9rZFmY4ax45uEEECSa8fsxGRU+8Bil52ASAwic= github.com/google/go-replayers/grpcreplay v0.1.0/go.mod h1:8Ig2Idjpr6gifRd6pNVggX6TC1Zw6Jx74AKp7QNH2QE= github.com/google/go-replayers/httpreplay v0.1.0 h1:AX7FUb4BjrrzNvblr/OlgwrmFiep6soj5K2QSDW7BGk= @@ -463,6 +469,8 @@ github.com/nats-io/nkeys v0.1.3 h1:6JrEfig+HzTH85yxzhSVbjHRJv9cn0p6n3IngIcM5/k= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= @@ -472,12 +480,15 @@ github.com/olivere/elastic/v7 v7.0.19/go.mod h1:4Jqt5xvjqpjCqgnTcHwl3j8TLs8mvoOK github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo= -github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.2 h1:8mVmC9kjFFmA8H4pKMUhcblgifdkOIXPvbhN1T36q1M= +github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= -github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.10.4 h1:NiTx7EEvBzu9sFOD1zORteLSt3o8gnlvZZwSE9TnY9U= +github.com/onsi/gomega v1.10.4/go.mod h1:g/HbgYopi++010VEqkFgJHKC09uJiW9UkXvMUuKHUCQ= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= @@ -680,6 +691,8 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4 h1:LYy1Hy3MJdrCdMwwzxA/dRok4ejH+RwNGbuoD9fCjto= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opentelemetry.io/otel v0.15.0 h1:CZFy2lPhxd4HlhZnYK8gRyDotksO3Ip9rBweY1vVYJw= +go.opentelemetry.io/otel v0.15.0/go.mod h1:e4GKElweB8W2gWUqbghw0B8t5MCTccc9212eNHnOHwA= go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -719,6 +732,8 @@ golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc h1:c0o/qxkaO2LF5t6fQrT4b5 golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067 h1:KYGJGHOQy8oSi1fDlSpcZF0+juKwk/hEMv5SiwHogR0= @@ -758,6 +773,9 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb h1:eBmm0M9fYhWpKZLjQUUKka/LtIxf46G4fxeEz5KJr9U= +golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -796,9 +814,15 @@ golang.org/x/sys v0.0.0-20190620070143-6f217b454f45/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0 h1:HyfiK1WMnHj5FXFXatD+Qs1A/xC2Run6RzeW1SyHxpc= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201022201747-fb209a7c41cd h1:WgqgiQvkiZWz7XLhphjt2GI2GcGCTIZs9jqXMWmH+oc= golang.org/x/sys v0.0.0-20201022201747-fb209a7c41cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -806,6 +830,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= @@ -938,8 +964,11 @@ gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bl gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/weed/filer/redis/redis_cluster_store.go b/weed/filer/redis/redis_cluster_store.go index 8af94ee55..3afa1cd43 100644 --- a/weed/filer/redis/redis_cluster_store.go +++ b/weed/filer/redis/redis_cluster_store.go @@ -3,7 +3,7 @@ package redis import ( "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/util" - "github.com/go-redis/redis" + "github.com/go-redis/redis/v8" ) func init() { diff --git a/weed/filer/redis/redis_store.go b/weed/filer/redis/redis_store.go index e152457ed..665352a63 100644 --- a/weed/filer/redis/redis_store.go +++ b/weed/filer/redis/redis_store.go @@ -3,7 +3,7 @@ package redis import ( "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/util" - "github.com/go-redis/redis" + "github.com/go-redis/redis/v8" ) func init() { diff --git a/weed/filer/redis/universal_redis_store.go b/weed/filer/redis/universal_redis_store.go index 0de9924a3..a3e09c3a1 100644 --- a/weed/filer/redis/universal_redis_store.go +++ b/weed/filer/redis/universal_redis_store.go @@ -7,7 +7,7 @@ import ( "strings" "time" - "github.com/go-redis/redis" + "github.com/go-redis/redis/v8" "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" @@ -44,7 +44,7 @@ func (store *UniversalRedisStore) InsertEntry(ctx context.Context, entry *filer. value = util.MaybeGzipData(value) } - _, err = store.Client.Set(string(entry.FullPath), value, time.Duration(entry.TtlSec)*time.Second).Result() + _, err = store.Client.Set(ctx, string(entry.FullPath), value, time.Duration(entry.TtlSec)*time.Second).Result() if err != nil { return fmt.Errorf("persisting %s : %v", entry.FullPath, err) @@ -52,7 +52,7 @@ func (store *UniversalRedisStore) InsertEntry(ctx context.Context, entry *filer. dir, name := entry.FullPath.DirAndName() if name != "" { - _, err = store.Client.SAdd(genDirectoryListKey(dir), name).Result() + _, err = store.Client.SAdd(ctx, genDirectoryListKey(dir), name).Result() if err != nil { return fmt.Errorf("persisting %s in parent dir: %v", entry.FullPath, err) } @@ -68,7 +68,7 @@ func (store *UniversalRedisStore) UpdateEntry(ctx context.Context, entry *filer. func (store *UniversalRedisStore) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) { - data, err := store.Client.Get(string(fullpath)).Result() + data, err := store.Client.Get(ctx, string(fullpath)).Result() if err == redis.Nil { return nil, filer_pb.ErrNotFound } @@ -90,7 +90,7 @@ func (store *UniversalRedisStore) FindEntry(ctx context.Context, fullpath util.F func (store *UniversalRedisStore) DeleteEntry(ctx context.Context, fullpath util.FullPath) (err error) { - _, err = store.Client.Del(string(fullpath)).Result() + _, err = store.Client.Del(ctx, string(fullpath)).Result() if err != nil { return fmt.Errorf("delete %s : %v", fullpath, err) @@ -98,7 +98,7 @@ func (store *UniversalRedisStore) DeleteEntry(ctx context.Context, fullpath util dir, name := fullpath.DirAndName() if name != "" { - _, err = store.Client.SRem(genDirectoryListKey(dir), name).Result() + _, err = store.Client.SRem(ctx, genDirectoryListKey(dir), name).Result() if err != nil { return fmt.Errorf("delete %s in parent dir: %v", fullpath, err) } @@ -109,14 +109,14 @@ func (store *UniversalRedisStore) DeleteEntry(ctx context.Context, fullpath util func (store *UniversalRedisStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) (err error) { - members, err := store.Client.SMembers(genDirectoryListKey(string(fullpath))).Result() + members, err := store.Client.SMembers(ctx, genDirectoryListKey(string(fullpath))).Result() if err != nil { return fmt.Errorf("delete folder %s : %v", fullpath, err) } for _, fileName := range members { path := util.NewFullPath(string(fullpath), fileName) - _, err = store.Client.Del(string(path)).Result() + _, err = store.Client.Del(ctx, string(path)).Result() if err != nil { return fmt.Errorf("delete %s in parent dir: %v", fullpath, err) } @@ -133,7 +133,7 @@ func (store *UniversalRedisStore) ListDirectoryEntries(ctx context.Context, full limit int) (entries []*filer.Entry, err error) { dirListKey := genDirectoryListKey(string(fullpath)) - members, err := store.Client.SMembers(dirListKey).Result() + members, err := store.Client.SMembers(ctx, dirListKey).Result() if err != nil { return nil, fmt.Errorf("list %s : %v", fullpath, err) } @@ -174,8 +174,8 @@ func (store *UniversalRedisStore) ListDirectoryEntries(ctx context.Context, full } else { if entry.TtlSec > 0 { if entry.Attr.Crtime.Add(time.Duration(entry.TtlSec) * time.Second).Before(time.Now()) { - store.Client.Del(string(path)).Result() - store.Client.SRem(dirListKey, fileName).Result() + store.Client.Del(ctx, string(path)).Result() + store.Client.SRem(ctx, dirListKey, fileName).Result() continue } } diff --git a/weed/filer/redis/universal_redis_store_kv.go b/weed/filer/redis/universal_redis_store_kv.go index 0fc12c631..ad6e389ed 100644 --- a/weed/filer/redis/universal_redis_store_kv.go +++ b/weed/filer/redis/universal_redis_store_kv.go @@ -5,12 +5,12 @@ import ( "fmt" "github.com/chrislusf/seaweedfs/weed/filer" - "github.com/go-redis/redis" + "github.com/go-redis/redis/v8" ) func (store *UniversalRedisStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) { - _, err = store.Client.Set(string(key), value, 0).Result() + _, err = store.Client.Set(ctx, string(key), value, 0).Result() if err != nil { return fmt.Errorf("kv put: %v", err) @@ -21,7 +21,7 @@ func (store *UniversalRedisStore) KvPut(ctx context.Context, key []byte, value [ func (store *UniversalRedisStore) KvGet(ctx context.Context, key []byte) (value []byte, err error) { - data, err := store.Client.Get(string(key)).Result() + data, err := store.Client.Get(ctx, string(key)).Result() if err == redis.Nil { return nil, filer.ErrKvNotFound @@ -32,7 +32,7 @@ func (store *UniversalRedisStore) KvGet(ctx context.Context, key []byte) (value func (store *UniversalRedisStore) KvDelete(ctx context.Context, key []byte) (err error) { - _, err = store.Client.Del(string(key)).Result() + _, err = store.Client.Del(ctx, string(key)).Result() if err != nil { return fmt.Errorf("kv delete: %v", err) diff --git a/weed/filer/redis2/redis_cluster_store.go b/weed/filer/redis2/redis_cluster_store.go index c7742bb19..4dbcca913 100644 --- a/weed/filer/redis2/redis_cluster_store.go +++ b/weed/filer/redis2/redis_cluster_store.go @@ -3,7 +3,7 @@ package redis2 import ( "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/util" - "github.com/go-redis/redis" + "github.com/go-redis/redis/v8" ) func init() { diff --git a/weed/filer/redis2/redis_store.go b/weed/filer/redis2/redis_store.go index da404ed4c..8eb97e374 100644 --- a/weed/filer/redis2/redis_store.go +++ b/weed/filer/redis2/redis_store.go @@ -3,7 +3,7 @@ package redis2 import ( "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/util" - "github.com/go-redis/redis" + "github.com/go-redis/redis/v8" ) func init() { diff --git a/weed/filer/redis2/universal_redis_store.go b/weed/filer/redis2/universal_redis_store.go index 00d02ea14..6a8d90768 100644 --- a/weed/filer/redis2/universal_redis_store.go +++ b/weed/filer/redis2/universal_redis_store.go @@ -5,7 +5,7 @@ import ( "fmt" "time" - "github.com/go-redis/redis" + "github.com/go-redis/redis/v8" "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" @@ -56,7 +56,7 @@ func (store *UniversalRedis2Store) InsertEntry(ctx context.Context, entry *filer value = util.MaybeGzipData(value) } - if err = store.Client.Set(string(entry.FullPath), value, time.Duration(entry.TtlSec)*time.Second).Err(); err != nil { + if err = store.Client.Set(ctx, string(entry.FullPath), value, time.Duration(entry.TtlSec)*time.Second).Err(); err != nil { return fmt.Errorf("persisting %s : %v", entry.FullPath, err) } @@ -66,7 +66,7 @@ func (store *UniversalRedis2Store) InsertEntry(ctx context.Context, entry *filer } if name != "" { - if err = store.Client.ZAddNX(genDirectoryListKey(dir), redis.Z{Score: 0, Member: name}).Err(); err != nil { + if err = store.Client.ZAddNX(ctx, genDirectoryListKey(dir), &redis.Z{Score: 0, Member: name}).Err(); err != nil { return fmt.Errorf("persisting %s in parent dir: %v", entry.FullPath, err) } } @@ -81,7 +81,7 @@ func (store *UniversalRedis2Store) UpdateEntry(ctx context.Context, entry *filer func (store *UniversalRedis2Store) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) { - data, err := store.Client.Get(string(fullpath)).Result() + data, err := store.Client.Get(ctx, string(fullpath)).Result() if err == redis.Nil { return nil, filer_pb.ErrNotFound } @@ -103,12 +103,12 @@ func (store *UniversalRedis2Store) FindEntry(ctx context.Context, fullpath util. func (store *UniversalRedis2Store) DeleteEntry(ctx context.Context, fullpath util.FullPath) (err error) { - _, err = store.Client.Del(genDirectoryListKey(string(fullpath))).Result() + _, err = store.Client.Del(ctx, genDirectoryListKey(string(fullpath))).Result() if err != nil { return fmt.Errorf("delete dir list %s : %v", fullpath, err) } - _, err = store.Client.Del(string(fullpath)).Result() + _, err = store.Client.Del(ctx, string(fullpath)).Result() if err != nil { return fmt.Errorf("delete %s : %v", fullpath, err) } @@ -118,7 +118,7 @@ func (store *UniversalRedis2Store) DeleteEntry(ctx context.Context, fullpath uti return nil } if name != "" { - _, err = store.Client.ZRem(genDirectoryListKey(dir), name).Result() + _, err = store.Client.ZRem(ctx, genDirectoryListKey(dir), name).Result() if err != nil { return fmt.Errorf("DeleteEntry %s in parent dir: %v", fullpath, err) } @@ -133,14 +133,14 @@ func (store *UniversalRedis2Store) DeleteFolderChildren(ctx context.Context, ful return nil } - members, err := store.Client.ZRange(genDirectoryListKey(string(fullpath)), 0, -1).Result() + members, err := store.Client.ZRange(ctx, genDirectoryListKey(string(fullpath)), 0, -1).Result() if err != nil { return fmt.Errorf("DeleteFolderChildren %s : %v", fullpath, err) } for _, fileName := range members { path := util.NewFullPath(string(fullpath), fileName) - _, err = store.Client.Del(string(path)).Result() + _, err = store.Client.Del(ctx, string(path)).Result() if err != nil { return fmt.Errorf("DeleteFolderChildren %s in parent dir: %v", fullpath, err) } @@ -159,12 +159,12 @@ func (store *UniversalRedis2Store) ListDirectoryEntries(ctx context.Context, ful dirListKey := genDirectoryListKey(string(fullpath)) start := int64(0) if startFileName != "" { - start, _ = store.Client.ZRank(dirListKey, startFileName).Result() + start, _ = store.Client.ZRank(ctx, dirListKey, startFileName).Result() if !inclusive { start++ } } - members, err := store.Client.ZRange(dirListKey, start, start+int64(limit)-1).Result() + members, err := store.Client.ZRange(ctx, dirListKey, start, start+int64(limit)-1).Result() if err != nil { return nil, fmt.Errorf("list %s : %v", fullpath, err) } @@ -178,8 +178,8 @@ func (store *UniversalRedis2Store) ListDirectoryEntries(ctx context.Context, ful } else { if entry.TtlSec > 0 { if entry.Attr.Crtime.Add(time.Duration(entry.TtlSec) * time.Second).Before(time.Now()) { - store.Client.Del(string(path)).Result() - store.Client.ZRem(dirListKey, fileName).Result() + store.Client.Del(ctx, string(path)).Result() + store.Client.ZRem(ctx, dirListKey, fileName).Result() continue } } diff --git a/weed/filer/redis2/universal_redis_store_kv.go b/weed/filer/redis2/universal_redis_store_kv.go index 658491ddf..bde994dc9 100644 --- a/weed/filer/redis2/universal_redis_store_kv.go +++ b/weed/filer/redis2/universal_redis_store_kv.go @@ -5,12 +5,12 @@ import ( "fmt" "github.com/chrislusf/seaweedfs/weed/filer" - "github.com/go-redis/redis" + "github.com/go-redis/redis/v8" ) func (store *UniversalRedis2Store) KvPut(ctx context.Context, key []byte, value []byte) (err error) { - _, err = store.Client.Set(string(key), value, 0).Result() + _, err = store.Client.Set(ctx, string(key), value, 0).Result() if err != nil { return fmt.Errorf("kv put: %v", err) @@ -21,7 +21,7 @@ func (store *UniversalRedis2Store) KvPut(ctx context.Context, key []byte, value func (store *UniversalRedis2Store) KvGet(ctx context.Context, key []byte) (value []byte, err error) { - data, err := store.Client.Get(string(key)).Result() + data, err := store.Client.Get(ctx, string(key)).Result() if err == redis.Nil { return nil, filer.ErrKvNotFound @@ -32,7 +32,7 @@ func (store *UniversalRedis2Store) KvGet(ctx context.Context, key []byte) (value func (store *UniversalRedis2Store) KvDelete(ctx context.Context, key []byte) (err error) { - _, err = store.Client.Del(string(key)).Result() + _, err = store.Client.Del(ctx, string(key)).Result() if err != nil { return fmt.Errorf("kv delete: %v", err) From e2e0e30a518d8ab4b89e0235b3986924d769256e Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 8 Jan 2021 01:12:44 -0800 Subject: [PATCH 076/260] filer: redis default to false for useReadOnly and routeByLatency --- weed/filer/redis/redis_cluster_store.go | 4 ++-- weed/filer/redis2/redis_cluster_store.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/weed/filer/redis/redis_cluster_store.go b/weed/filer/redis/redis_cluster_store.go index 3afa1cd43..9572058a8 100644 --- a/weed/filer/redis/redis_cluster_store.go +++ b/weed/filer/redis/redis_cluster_store.go @@ -20,8 +20,8 @@ func (store *RedisClusterStore) GetName() string { func (store *RedisClusterStore) Initialize(configuration util.Configuration, prefix string) (err error) { - configuration.SetDefault(prefix+"useReadOnly", true) - configuration.SetDefault(prefix+"routeByLatency", true) + configuration.SetDefault(prefix+"useReadOnly", false) + configuration.SetDefault(prefix+"routeByLatency", false) return store.initialize( configuration.GetStringSlice(prefix+"addresses"), diff --git a/weed/filer/redis2/redis_cluster_store.go b/weed/filer/redis2/redis_cluster_store.go index 4dbcca913..22d09da25 100644 --- a/weed/filer/redis2/redis_cluster_store.go +++ b/weed/filer/redis2/redis_cluster_store.go @@ -20,8 +20,8 @@ func (store *RedisCluster2Store) GetName() string { func (store *RedisCluster2Store) Initialize(configuration util.Configuration, prefix string) (err error) { - configuration.SetDefault(prefix+"useReadOnly", true) - configuration.SetDefault(prefix+"routeByLatency", true) + configuration.SetDefault(prefix+"useReadOnly", false) + configuration.SetDefault(prefix+"routeByLatency", false) return store.initialize( configuration.GetStringSlice(prefix+"addresses"), From 90b117acf1bdf617ffa549a3f229530d905f7f5d Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 8 Jan 2021 02:17:43 -0800 Subject: [PATCH 077/260] update ccache version --- go.mod | 2 +- go.sum | 3 +++ weed/util/chunk_cache/chunk_cache_in_memory.go | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 40708b9f9..c0bcdc8cd 100644 --- a/go.mod +++ b/go.mod @@ -39,7 +39,7 @@ require ( github.com/jcmturner/gofork v1.0.0 // indirect github.com/json-iterator/go v1.1.10 github.com/karlseguin/ccache v2.0.3+incompatible - github.com/karlseguin/expect v1.0.1 // indirect + github.com/karlseguin/ccache/v2 v2.0.7 github.com/klauspost/compress v1.10.9 // indirect github.com/klauspost/cpuid v1.2.1 // indirect github.com/klauspost/crc32 v1.2.0 diff --git a/go.sum b/go.sum index 3c0396f85..d91a570f8 100644 --- a/go.sum +++ b/go.sum @@ -369,8 +369,11 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/karlseguin/ccache v2.0.3+incompatible h1:j68C9tWOROiOLWTS/kCGg9IcJG+ACqn5+0+t8Oh83UU= github.com/karlseguin/ccache v2.0.3+incompatible/go.mod h1:CM9tNPzT6EdRh14+jiW8mEF9mkNZuuE51qmgGYUB93w= +github.com/karlseguin/ccache/v2 v2.0.7 h1:y5Pfi4eiyYCOD6LS/Kj+o6Nb4M5Ngpw9qFQs+v44ZYM= +github.com/karlseguin/ccache/v2 v2.0.7/go.mod h1:2BDThcfQMf/c0jnZowt16eW405XIqZPavt+HoYEtcxQ= github.com/karlseguin/expect v1.0.1 h1:z4wy4npwwHSWKjGWH85WNJO42VQhovxTCZDSzhjo8hY= github.com/karlseguin/expect v1.0.1/go.mod h1:zNBxMY8P21owkeogJELCLeHIt+voOSduHYTFUbwRAV8= +github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003/go.mod h1:zNBxMY8P21owkeogJELCLeHIt+voOSduHYTFUbwRAV8= github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= diff --git a/weed/util/chunk_cache/chunk_cache_in_memory.go b/weed/util/chunk_cache/chunk_cache_in_memory.go index 931e45e9a..1eb00e1fa 100644 --- a/weed/util/chunk_cache/chunk_cache_in_memory.go +++ b/weed/util/chunk_cache/chunk_cache_in_memory.go @@ -3,7 +3,7 @@ package chunk_cache import ( "time" - "github.com/karlseguin/ccache" + "github.com/karlseguin/ccache/v2" ) // a global cache for recently accessed file chunks From e9ccc0a6c5428128ce5b128ba43543205dbbd12f Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 9 Jan 2021 22:53:09 -0800 Subject: [PATCH 078/260] adjust --- docker/local-dev-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/local-dev-compose.yml b/docker/local-dev-compose.yml index 752086acd..18cccab3e 100644 --- a/docker/local-dev-compose.yml +++ b/docker/local-dev-compose.yml @@ -39,7 +39,7 @@ services: cap_add: - SYS_ADMIN mem_limit: 4096m - command: '-v=4 mount -filer="filer:8888" -dirAutoCreate -dir=/mnt/seaweedfs -cacheCapacityMB=0 ' + command: '-v=4 mount -filer="filer:8888" -dirAutoCreate -dir=/mnt/seaweedfs -cacheCapacityMB=100 -concurrentWriters=128' depends_on: - master - volume From d3d3f2fb9b2be9393fd84c89d1b25cb5cbbd2396 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 9 Jan 2021 22:53:37 -0800 Subject: [PATCH 079/260] mount: default to 128 concurrent writers --- weed/command/mount.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/command/mount.go b/weed/command/mount.go index f325cb0a5..fa75919aa 100644 --- a/weed/command/mount.go +++ b/weed/command/mount.go @@ -43,7 +43,7 @@ func init() { mountOptions.replication = cmdMount.Flag.String("replication", "", "replication(e.g. 000, 001) to create to files. If empty, let filer decide.") mountOptions.ttlSec = cmdMount.Flag.Int("ttl", 0, "file ttl in seconds") mountOptions.chunkSizeLimitMB = cmdMount.Flag.Int("chunkSizeLimitMB", 2, "local write buffer size, also chunk large files") - mountOptions.concurrentWriters = cmdMount.Flag.Int("concurrentWriters", 0, "limit concurrent goroutine writers if not 0") + mountOptions.concurrentWriters = cmdMount.Flag.Int("concurrentWriters", 128, "limit concurrent goroutine writers if not 0") mountOptions.cacheDir = cmdMount.Flag.String("cacheDir", os.TempDir(), "local cache directory for file chunks and meta data") mountOptions.cacheSizeMB = cmdMount.Flag.Int64("cacheCapacityMB", 1000, "local file chunk cache capacity in MB (0 will disable cache)") mountOptions.dataCenter = cmdMount.Flag.String("dataCenter", "", "prefer to write to the data center") From f0d3b3bf9397fe5bfb64fd0d46c40055a950d9cf Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 9 Jan 2021 22:55:22 -0800 Subject: [PATCH 080/260] 2.20 --- k8s/seaweedfs/values.yaml | 2 +- weed/util/constants.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/k8s/seaweedfs/values.yaml b/k8s/seaweedfs/values.yaml index 9d621a496..964521693 100644 --- a/k8s/seaweedfs/values.yaml +++ b/k8s/seaweedfs/values.yaml @@ -4,7 +4,7 @@ global: registry: "" repository: "" imageName: chrislusf/seaweedfs - imageTag: "2.19" + imageTag: "2.20" imagePullPolicy: IfNotPresent imagePullSecrets: imagepullsecret restartPolicy: Always diff --git a/weed/util/constants.go b/weed/util/constants.go index cfb33516b..ac653c4d7 100644 --- a/weed/util/constants.go +++ b/weed/util/constants.go @@ -5,7 +5,7 @@ import ( ) var ( - VERSION = fmt.Sprintf("%s %d.%02d", sizeLimit, 2, 19) + VERSION = fmt.Sprintf("%s %d.%02d", sizeLimit, 2, 20) COMMIT = "" ) From 1efb51ba843d809df5617cf9c1c288b7af5d10d4 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 10 Jan 2021 23:14:46 -0800 Subject: [PATCH 081/260] filer: change to saveToFilerLimit from cacheToFilerLimit short circuit saving small files to volume server --- weed/command/filer.go | 28 +++++++++---------- weed/command/server.go | 2 +- weed/server/filer_server.go | 24 ++++++++-------- .../filer_server_handlers_write_autochunk.go | 14 ++++++---- 4 files changed, 35 insertions(+), 33 deletions(-) diff --git a/weed/command/filer.go b/weed/command/filer.go index 146840e00..aa09c3c98 100644 --- a/weed/command/filer.go +++ b/weed/command/filer.go @@ -42,7 +42,7 @@ type FilerOptions struct { cipher *bool peers *string metricsHttpPort *int - cacheToFilerLimit *int + saveToFilerLimit *int defaultLevelDbDirectory *string } @@ -64,7 +64,7 @@ func init() { f.cipher = cmdFiler.Flag.Bool("encryptVolumeData", false, "encrypt data on volume servers") f.peers = cmdFiler.Flag.String("peers", "", "all filers sharing the same filer store in comma separated ip:port list") f.metricsHttpPort = cmdFiler.Flag.Int("metricsPort", 0, "Prometheus metrics listen port") - f.cacheToFilerLimit = cmdFiler.Flag.Int("cacheToFilerLimit", 0, "Small files smaller than this limit can be cached in filer store.") + f.saveToFilerLimit = cmdFiler.Flag.Int("saveToFilerLimit", 0, "files smaller than this limit will be saved in filer store") f.defaultLevelDbDirectory = cmdFiler.Flag.String("defaultStoreDir", ".", "if filer.toml is empty, use an embedded filer store in the directory") // start s3 on filer @@ -139,18 +139,18 @@ func (fo *FilerOptions) startFiler() { Masters: strings.Split(*fo.masters, ","), Collection: *fo.collection, DefaultReplication: *fo.defaultReplicaPlacement, - DisableDirListing: *fo.disableDirListing, - MaxMB: *fo.maxMB, - DirListingLimit: *fo.dirListingLimit, - DataCenter: *fo.dataCenter, - Rack: *fo.rack, - DefaultLevelDbDir: defaultLevelDbDirectory, - DisableHttp: *fo.disableHttp, - Host: *fo.ip, - Port: uint32(*fo.port), - Cipher: *fo.cipher, - CacheToFilerLimit: int64(*fo.cacheToFilerLimit), - Filers: peers, + DisableDirListing: *fo.disableDirListing, + MaxMB: *fo.maxMB, + DirListingLimit: *fo.dirListingLimit, + DataCenter: *fo.dataCenter, + Rack: *fo.rack, + DefaultLevelDbDir: defaultLevelDbDirectory, + DisableHttp: *fo.disableHttp, + Host: *fo.ip, + Port: uint32(*fo.port), + Cipher: *fo.cipher, + SaveToFilerLimit: *fo.saveToFilerLimit, + Filers: peers, }) if nfs_err != nil { glog.Fatalf("Filer startup error: %v", nfs_err) diff --git a/weed/command/server.go b/weed/command/server.go index 187563db1..9976db2ea 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -95,7 +95,7 @@ func init() { filerOptions.dirListingLimit = cmdServer.Flag.Int("filer.dirListLimit", 1000, "limit sub dir listing size") filerOptions.cipher = cmdServer.Flag.Bool("filer.encryptVolumeData", false, "encrypt data on volume servers") filerOptions.peers = cmdServer.Flag.String("filer.peers", "", "all filers sharing the same filer store in comma separated ip:port list") - filerOptions.cacheToFilerLimit = cmdServer.Flag.Int("filer.cacheToFilerLimit", 0, "Small files smaller than this limit can be cached in filer store.") + filerOptions.saveToFilerLimit = cmdServer.Flag.Int("filer.saveToFilerLimit", 0, "Small files smaller than this limit can be cached in filer store.") serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port") serverOptions.v.publicPort = cmdServer.Flag.Int("volume.port.public", 0, "volume server public port") diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index 2da129ab2..efd601717 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -46,18 +46,18 @@ type FilerOption struct { Collection string DefaultReplication string DisableDirListing bool - MaxMB int - DirListingLimit int - DataCenter string - Rack string - DefaultLevelDbDir string - DisableHttp bool - Host string - Port uint32 - recursiveDelete bool - Cipher bool - CacheToFilerLimit int64 - Filers []string + MaxMB int + DirListingLimit int + DataCenter string + Rack string + DefaultLevelDbDir string + DisableHttp bool + Host string + Port uint32 + recursiveDelete bool + Cipher bool + SaveToFilerLimit int + Filers []string } type FilerServer struct { diff --git a/weed/server/filer_server_handlers_write_autochunk.go b/weed/server/filer_server_handlers_write_autochunk.go index fe4e68140..43884bfde 100644 --- a/weed/server/filer_server_handlers_write_autochunk.go +++ b/weed/server/filer_server_handlers_write_autochunk.go @@ -207,7 +207,7 @@ func (fs *FilerServer) uploadReaderToChunks(w http.ResponseWriter, r *http.Reque var partReader = ioutil.NopCloser(io.TeeReader(reader, md5Hash)) chunkOffset := int64(0) - var smallContent, content []byte + var smallContent []byte for { limitedReader := io.LimitReader(partReader, int64(chunkSize)) @@ -216,6 +216,13 @@ func (fs *FilerServer) uploadReaderToChunks(w http.ResponseWriter, r *http.Reque if err != nil { return nil, nil, 0, err, nil } + if chunkOffset == 0 { + if len(data) < fs.option.SaveToFilerLimit || strings.HasPrefix(r.URL.Path, filer.DirectoryEtcRoot) && len(data) < 4*1024 { + smallContent = data + chunkOffset += int64(len(data)) + break + } + } dataReader := util.NewBytesReader(data) // retry to assign a different file id @@ -242,8 +249,6 @@ func (fs *FilerServer) uploadReaderToChunks(w http.ResponseWriter, r *http.Reque return nil, nil, 0, uploadErr, nil } - content = data - // if last chunk exhausted the reader exactly at the border if uploadResult.Size == 0 { break @@ -263,9 +268,6 @@ func (fs *FilerServer) uploadReaderToChunks(w http.ResponseWriter, r *http.Reque } } - if chunkOffset < fs.option.CacheToFilerLimit || strings.HasPrefix(r.URL.Path, filer.DirectoryEtcRoot) && chunkOffset < 4*1024 { - smallContent = content - } return fileChunks, md5Hash, chunkOffset, nil, smallContent } From 62ee484d12c2753402e67a4a8ba8490324e4731b Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 11 Jan 2021 00:03:13 -0800 Subject: [PATCH 082/260] refactoring --- weed/command/watch.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/weed/command/watch.go b/weed/command/watch.go index fd7dd6fb2..431a326d7 100644 --- a/weed/command/watch.go +++ b/weed/command/watch.go @@ -76,6 +76,10 @@ func runWatch(cmd *Command, args []string) bool { return false } + eachEntryFunc := func(resp *filer_pb.SubscribeMetadataResponse) { + fmt.Printf("dir:%s %+v\n", resp.Directory, resp.EventNotification) + } + watchErr := pb.WithFilerClient(*watchFiler, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { ctx, cancel := context.WithCancel(context.Background()) @@ -101,7 +105,7 @@ func runWatch(cmd *Command, args []string) bool { if !shouldPrint(resp) { continue } - fmt.Printf("dir:%s %+v\n", resp.Directory, resp.EventNotification) + eachEntryFunc(resp) } }) From 6e12a3a490fe5bc4f57fccf51a9fbdc9d1675ca5 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 11 Jan 2021 02:08:26 -0800 Subject: [PATCH 083/260] refactoring --- weed/command/watch.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/weed/command/watch.go b/weed/command/watch.go index 431a326d7..8aad22019 100644 --- a/weed/command/watch.go +++ b/weed/command/watch.go @@ -76,8 +76,9 @@ func runWatch(cmd *Command, args []string) bool { return false } - eachEntryFunc := func(resp *filer_pb.SubscribeMetadataResponse) { + eachEntryFunc := func(resp *filer_pb.SubscribeMetadataResponse) error { fmt.Printf("dir:%s %+v\n", resp.Directory, resp.EventNotification) + return nil } watchErr := pb.WithFilerClient(*watchFiler, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { @@ -105,7 +106,9 @@ func runWatch(cmd *Command, args []string) bool { if !shouldPrint(resp) { continue } - eachEntryFunc(resp) + if err = eachEntryFunc(resp); err != nil { + return err + } } }) From 394513f598f5fac7889086ab6420d6ccf1b8d53a Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 11 Jan 2021 02:08:55 -0800 Subject: [PATCH 084/260] filer: ensure seamless meta data updates --- weed/filer/filer_notify.go | 2 +- .../broker/broker_grpc_server_subscribe.go | 26 ++++++---- weed/server/filer_grpc_server_sub_meta.go | 52 ++++++++++++------- weed/util/log_buffer/log_buffer.go | 24 +++++---- weed/util/log_buffer/log_read.go | 6 ++- 5 files changed, 69 insertions(+), 41 deletions(-) diff --git a/weed/filer/filer_notify.go b/weed/filer/filer_notify.go index 563e0eb51..b690a4cd1 100644 --- a/weed/filer/filer_notify.go +++ b/weed/filer/filer_notify.go @@ -170,7 +170,7 @@ func ReadEachLogEntry(r io.Reader, sizeBuf []byte, ns int64, eachLogEntryFn func return lastTsNs, err } if logEntry.TsNs <= ns { - return lastTsNs, nil + continue } // println("each log: ", logEntry.TsNs) if err := eachLogEntryFn(logEntry); err != nil { diff --git a/weed/messaging/broker/broker_grpc_server_subscribe.go b/weed/messaging/broker/broker_grpc_server_subscribe.go index df4052096..3021473e5 100644 --- a/weed/messaging/broker/broker_grpc_server_subscribe.go +++ b/weed/messaging/broker/broker_grpc_server_subscribe.go @@ -101,20 +101,21 @@ func (broker *MessageBroker) Subscribe(stream messaging_pb.SeaweedMessaging_Subs return nil } - if err = broker.readPersistedLogBuffer(&tp, lastReadTime, eachLogEntryFn); err != nil { - if err != io.EOF { - // println("stopping from persisted logs", err.Error()) - return err - } - } - - if processedTsNs != 0 { - lastReadTime = time.Unix(0, processedTsNs) - } - // fmt.Printf("subscriber %s read %d on disk log %v\n", subscriberId, messageCount, lastReadTime) for { + + if err = broker.readPersistedLogBuffer(&tp, lastReadTime, eachLogEntryFn); err != nil { + if err != io.EOF { + // println("stopping from persisted logs", err.Error()) + return err + } + } + + if processedTsNs != 0 { + lastReadTime = time.Unix(0, processedTsNs) + } + lastReadTime, err = lock.logBuffer.LoopProcessLogData(lastReadTime, func() bool { lock.Mutex.Lock() lock.cond.Wait() @@ -122,6 +123,9 @@ func (broker *MessageBroker) Subscribe(stream messaging_pb.SeaweedMessaging_Subs return isConnected }, eachLogEntryFn) if err != nil { + if err == log_buffer.ResumeFromDiskError { + continue + } glog.Errorf("processed to %v: %v", lastReadTime, err) time.Sleep(3127 * time.Millisecond) if err != log_buffer.ResumeError { diff --git a/weed/server/filer_grpc_server_sub_meta.go b/weed/server/filer_grpc_server_sub_meta.go index 3b8ced675..d9f91b125 100644 --- a/weed/server/filer_grpc_server_sub_meta.go +++ b/weed/server/filer_grpc_server_sub_meta.go @@ -29,16 +29,20 @@ func (fs *FilerServer) SubscribeMetadata(req *filer_pb.SubscribeMetadataRequest, eachLogEntryFn := eachLogEntryFn(eachEventNotificationFn) - processedTsNs, err := fs.filer.ReadPersistedLogBuffer(lastReadTime, eachLogEntryFn) - if err != nil { - return fmt.Errorf("reading from persisted logs: %v", err) - } - - if processedTsNs != 0 { - lastReadTime = time.Unix(0, processedTsNs) - } + var processedTsNs int64 + var err error for { + + processedTsNs, err = fs.filer.ReadPersistedLogBuffer(lastReadTime, eachLogEntryFn) + if err != nil { + return fmt.Errorf("reading from persisted logs: %v", err) + } + + if processedTsNs != 0 { + lastReadTime = time.Unix(0, processedTsNs) + } + lastReadTime, err = fs.filer.MetaAggregator.MetaLogBuffer.LoopProcessLogData(lastReadTime, func() bool { fs.filer.MetaAggregator.ListenersLock.Lock() fs.filer.MetaAggregator.ListenersCond.Wait() @@ -46,6 +50,9 @@ func (fs *FilerServer) SubscribeMetadata(req *filer_pb.SubscribeMetadataRequest, return true }, eachLogEntryFn) if err != nil { + if err == log_buffer.ResumeFromDiskError { + continue + } glog.Errorf("processed to %v: %v", lastReadTime, err) time.Sleep(3127 * time.Millisecond) if err != log_buffer.ResumeError { @@ -73,19 +80,23 @@ func (fs *FilerServer) SubscribeLocalMetadata(req *filer_pb.SubscribeMetadataReq eachLogEntryFn := eachLogEntryFn(eachEventNotificationFn) - // println("reading from persisted logs ...") - processedTsNs, err := fs.filer.ReadPersistedLogBuffer(lastReadTime, eachLogEntryFn) - if err != nil { - return fmt.Errorf("reading from persisted logs: %v", err) - } - - if processedTsNs != 0 { - lastReadTime = time.Unix(0, processedTsNs) - } - glog.V(0).Infof("after local log reads, %v local subscribe %s from %+v", clientName, req.PathPrefix, lastReadTime) + var processedTsNs int64 + var err error - // println("reading from in memory logs ...") for { + // println("reading from persisted logs ...") + processedTsNs, err = fs.filer.ReadPersistedLogBuffer(lastReadTime, eachLogEntryFn) + if err != nil { + return fmt.Errorf("reading from persisted logs: %v", err) + } + + if processedTsNs != 0 { + lastReadTime = time.Unix(0, processedTsNs) + } + // glog.V(0).Infof("after local log reads, %v local subscribe %s from %+v", clientName, req.PathPrefix, lastReadTime) + + // println("reading from in memory logs ...") + lastReadTime, err = fs.filer.LocalMetaLogBuffer.LoopProcessLogData(lastReadTime, func() bool { fs.listenersLock.Lock() fs.listenersCond.Wait() @@ -93,6 +104,9 @@ func (fs *FilerServer) SubscribeLocalMetadata(req *filer_pb.SubscribeMetadataReq return true }, eachLogEntryFn) if err != nil { + if err == log_buffer.ResumeFromDiskError { + continue + } glog.Errorf("processed to %v: %v", lastReadTime, err) time.Sleep(3127 * time.Millisecond) if err != log_buffer.ResumeError { diff --git a/weed/util/log_buffer/log_buffer.go b/weed/util/log_buffer/log_buffer.go index e4310b5c5..f84c674ff 100644 --- a/weed/util/log_buffer/log_buffer.go +++ b/weed/util/log_buffer/log_buffer.go @@ -28,6 +28,7 @@ type LogBuffer struct { pos int startTime time.Time stopTime time.Time + lastFlushTime time.Time sizeBuf []byte flushInterval time.Duration flushFn func(startTime, stopTime time.Time, buf []byte) @@ -129,6 +130,7 @@ func (m *LogBuffer) loopFlush() { // fmt.Printf("flush [%v, %v] size %d\n", d.startTime, d.stopTime, len(d.data.Bytes())) m.flushFn(d.startTime, d.stopTime, d.data.Bytes()) d.releaseMemory() + m.lastFlushTime = d.stopTime } } } @@ -174,10 +176,14 @@ func (d *dataToFlush) releaseMemory() { bufferPool.Put(d.data) } -func (m *LogBuffer) ReadFromBuffer(lastReadTime time.Time) (bufferCopy *bytes.Buffer) { +func (m *LogBuffer) ReadFromBuffer(lastReadTime time.Time) (bufferCopy *bytes.Buffer, err error) { m.RLock() defer m.RUnlock() + if !m.lastFlushTime.IsZero() && m.lastFlushTime.After(lastReadTime) { + return nil, ResumeFromDiskError + } + /* fmt.Printf("read buffer %p: %v last stop time: [%v,%v], pos %d, entries:%d, prevBufs:%d\n", m, lastReadTime, m.startTime, m.stopTime, m.pos, len(m.idx), len(m.prevBuffers.buffers)) for i, prevBuf := range m.prevBuffers.buffers { @@ -186,11 +192,11 @@ func (m *LogBuffer) ReadFromBuffer(lastReadTime time.Time) (bufferCopy *bytes.Bu */ if lastReadTime.Equal(m.stopTime) { - return nil + return nil, nil } if lastReadTime.After(m.stopTime) { // glog.Fatalf("unexpected last read time %v, older than latest %v", lastReadTime, m.stopTime) - return nil + return nil, nil } if lastReadTime.Before(m.startTime) { // println("checking ", lastReadTime.UnixNano()) @@ -198,19 +204,19 @@ func (m *LogBuffer) ReadFromBuffer(lastReadTime time.Time) (bufferCopy *bytes.Bu if buf.startTime.After(lastReadTime) { if i == 0 { // println("return the earliest in memory", buf.startTime.UnixNano()) - return copiedBytes(buf.buf[:buf.size]) + return copiedBytes(buf.buf[:buf.size]), nil } // println("return the", i, "th in memory", buf.startTime.UnixNano()) - return copiedBytes(buf.buf[:buf.size]) + return copiedBytes(buf.buf[:buf.size]), nil } if !buf.startTime.After(lastReadTime) && buf.stopTime.After(lastReadTime) { pos := buf.locateByTs(lastReadTime) // fmt.Printf("locate buffer[%d] pos %d\n", i, pos) - return copiedBytes(buf.buf[pos:buf.size]) + return copiedBytes(buf.buf[pos:buf.size]), nil } } // println("return the current buf", lastReadTime.UnixNano()) - return copiedBytes(m.buf[:m.pos]) + return copiedBytes(m.buf[:m.pos]), nil } lastTs := lastReadTime.UnixNano() @@ -243,7 +249,7 @@ func (m *LogBuffer) ReadFromBuffer(lastReadTime time.Time) (bufferCopy *bytes.Bu } if prevT <= lastTs { // fmt.Printf("found l=%d, m-1=%d(ts=%d), m=%d(ts=%d), h=%d [%d, %d) \n", l, mid-1, prevT, mid, t, h, pos, m.pos) - return copiedBytes(m.buf[pos:m.pos]) + return copiedBytes(m.buf[pos:m.pos]), nil } h = mid } @@ -251,7 +257,7 @@ func (m *LogBuffer) ReadFromBuffer(lastReadTime time.Time) (bufferCopy *bytes.Bu } // FIXME: this could be that the buffer has been flushed already - return nil + return nil, nil } func (m *LogBuffer) ReleaseMemory(b *bytes.Buffer) { diff --git a/weed/util/log_buffer/log_read.go b/weed/util/log_buffer/log_read.go index 57f4b0115..3c314a135 100644 --- a/weed/util/log_buffer/log_read.go +++ b/weed/util/log_buffer/log_read.go @@ -14,6 +14,7 @@ import ( var ( ResumeError = fmt.Errorf("resume") + ResumeFromDiskError = fmt.Errorf("resumeFromDisk") ) func (logBuffer *LogBuffer) LoopProcessLogData( @@ -34,7 +35,10 @@ func (logBuffer *LogBuffer) LoopProcessLogData( if bytesBuf != nil { logBuffer.ReleaseMemory(bytesBuf) } - bytesBuf = logBuffer.ReadFromBuffer(lastReadTime) + bytesBuf, err = logBuffer.ReadFromBuffer(lastReadTime) + if err == ResumeFromDiskError { + return lastReadTime, ResumeFromDiskError + } // fmt.Printf("ReadFromBuffer by %v\n", lastReadTime) if bytesBuf == nil { if waitForDataFn() { From dd0171e35cb032781b703bc8ae95d8a21f59ea76 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 11 Jan 2021 02:09:18 -0800 Subject: [PATCH 085/260] Revert "filer store: redis module upgrade" This reverts commit 40161c9cec3b8a0ed16bb355c6d6f3e7f23a094c. --- go.mod | 6 ++- go.sum | 41 +++---------------- weed/filer/redis/redis_cluster_store.go | 2 +- weed/filer/redis/redis_store.go | 2 +- weed/filer/redis/universal_redis_store.go | 22 +++++----- weed/filer/redis/universal_redis_store_kv.go | 8 ++-- weed/filer/redis2/redis_cluster_store.go | 2 +- weed/filer/redis2/redis_store.go | 2 +- weed/filer/redis2/universal_redis_store.go | 26 ++++++------ weed/filer/redis2/universal_redis_store_kv.go | 8 ++-- 10 files changed, 46 insertions(+), 73 deletions(-) diff --git a/go.mod b/go.mod index c0bcdc8cd..18b1097f8 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/fclairamb/ftpserverlib v0.8.0 github.com/frankban/quicktest v1.7.2 // indirect github.com/go-errors/errors v1.1.1 // indirect - github.com/go-redis/redis/v8 v8.4.4 + github.com/go-redis/redis v6.15.7+incompatible github.com/go-sql-driver/mysql v1.5.0 github.com/gocql/gocql v0.0.0-20190829130954-e163eff7a8c6 github.com/gogo/protobuf v1.2.2-0.20190730201129-28a6bbf47e48 // indirect @@ -51,6 +51,8 @@ require ( github.com/mattn/go-ieproxy v0.0.0-20190805055040-f9202b1cfdeb // indirect github.com/mattn/go-runewidth v0.0.4 // indirect github.com/olivere/elastic/v7 v7.0.19 + github.com/onsi/ginkgo v1.10.1 // indirect + github.com/onsi/gomega v1.7.0 // indirect github.com/peterh/liner v1.1.0 github.com/pierrec/lz4 v2.2.7+incompatible // indirect github.com/prometheus/client_golang v1.3.0 @@ -81,7 +83,7 @@ require ( gocloud.dev/pubsub/natspubsub v0.16.0 gocloud.dev/pubsub/rabbitpubsub v0.16.0 golang.org/x/image v0.0.0-20200119044424-58c23975cae1 // indirect - golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb + golang.org/x/net v0.0.0-20200202094626-16171245cfb2 golang.org/x/sync v0.0.0-20200930132711-30421366ff76 // indirect golang.org/x/sys v0.0.0-20201022201747-fb209a7c41cd golang.org/x/tools v0.0.0-20200103221440-774c71fcf114 diff --git a/go.sum b/go.sum index d91a570f8..e019b01f7 100644 --- a/go.sum +++ b/go.sum @@ -116,8 +116,6 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/devigned/tab v0.1.1/go.mod h1:XG9mPq0dFghrYvoBF3xdRrJzSTX1b7IQrvaL9mzjeJY= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= -github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c= @@ -161,8 +159,6 @@ github.com/frankban/quicktest v1.7.2 h1:2QxQoC1TS09S7fhCPsrvqYdvP1H5M1P1ih5ABm3B github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-errors/errors v1.1.1 h1:ljK/pL5ltg3qoN+OtN6yCv9HWSfMwxSx90GJCZQxYNg= @@ -176,8 +172,8 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih4= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-redis/redis/v8 v8.4.4 h1:fGqgxCTR1sydaKI00oQf3OmkU/DIe/I/fYXvGklCIuc= -github.com/go-redis/redis/v8 v8.4.4/go.mod h1:nA0bQuF0i5JFx4Ta9RZxGKXFrQ8cRWntra97f0196iY= +github.com/go-redis/redis v6.15.7+incompatible h1:3skhDh95XQMpnqeqNftPkQD9jL9e5e36z/1SUm6dy1U= +github.com/go-redis/redis v6.15.7+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= @@ -263,8 +259,6 @@ github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-replayers/grpcreplay v0.1.0 h1:eNb1y9rZFmY4ax45uEEECSa8fsxGRU+8Bil52ASAwic= github.com/google/go-replayers/grpcreplay v0.1.0/go.mod h1:8Ig2Idjpr6gifRd6pNVggX6TC1Zw6Jx74AKp7QNH2QE= github.com/google/go-replayers/httpreplay v0.1.0 h1:AX7FUb4BjrrzNvblr/OlgwrmFiep6soj5K2QSDW7BGk= @@ -472,8 +466,6 @@ github.com/nats-io/nkeys v0.1.3 h1:6JrEfig+HzTH85yxzhSVbjHRJv9cn0p6n3IngIcM5/k= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= -github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= @@ -483,15 +475,12 @@ github.com/olivere/elastic/v7 v7.0.19/go.mod h1:4Jqt5xvjqpjCqgnTcHwl3j8TLs8mvoOK github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.14.2 h1:8mVmC9kjFFmA8H4pKMUhcblgifdkOIXPvbhN1T36q1M= -github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo= +github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.10.4 h1:NiTx7EEvBzu9sFOD1zORteLSt3o8gnlvZZwSE9TnY9U= -github.com/onsi/gomega v1.10.4/go.mod h1:g/HbgYopi++010VEqkFgJHKC09uJiW9UkXvMUuKHUCQ= +github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= @@ -694,8 +683,6 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4 h1:LYy1Hy3MJdrCdMwwzxA/dRok4ejH+RwNGbuoD9fCjto= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opentelemetry.io/otel v0.15.0 h1:CZFy2lPhxd4HlhZnYK8gRyDotksO3Ip9rBweY1vVYJw= -go.opentelemetry.io/otel v0.15.0/go.mod h1:e4GKElweB8W2gWUqbghw0B8t5MCTccc9212eNHnOHwA= go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -735,8 +722,6 @@ golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc h1:c0o/qxkaO2LF5t6fQrT4b5 golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067 h1:KYGJGHOQy8oSi1fDlSpcZF0+juKwk/hEMv5SiwHogR0= @@ -776,9 +761,6 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb h1:eBmm0M9fYhWpKZLjQUUKka/LtIxf46G4fxeEz5KJr9U= -golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -817,15 +799,9 @@ golang.org/x/sys v0.0.0-20190620070143-6f217b454f45/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0 h1:HyfiK1WMnHj5FXFXatD+Qs1A/xC2Run6RzeW1SyHxpc= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201022201747-fb209a7c41cd h1:WgqgiQvkiZWz7XLhphjt2GI2GcGCTIZs9jqXMWmH+oc= golang.org/x/sys v0.0.0-20201022201747-fb209a7c41cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -833,8 +809,6 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= @@ -967,11 +941,8 @@ gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bl gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/weed/filer/redis/redis_cluster_store.go b/weed/filer/redis/redis_cluster_store.go index 9572058a8..a2ab5b5f9 100644 --- a/weed/filer/redis/redis_cluster_store.go +++ b/weed/filer/redis/redis_cluster_store.go @@ -3,7 +3,7 @@ package redis import ( "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/util" - "github.com/go-redis/redis/v8" + "github.com/go-redis/redis" ) func init() { diff --git a/weed/filer/redis/redis_store.go b/weed/filer/redis/redis_store.go index 665352a63..e152457ed 100644 --- a/weed/filer/redis/redis_store.go +++ b/weed/filer/redis/redis_store.go @@ -3,7 +3,7 @@ package redis import ( "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/util" - "github.com/go-redis/redis/v8" + "github.com/go-redis/redis" ) func init() { diff --git a/weed/filer/redis/universal_redis_store.go b/weed/filer/redis/universal_redis_store.go index a3e09c3a1..0de9924a3 100644 --- a/weed/filer/redis/universal_redis_store.go +++ b/weed/filer/redis/universal_redis_store.go @@ -7,7 +7,7 @@ import ( "strings" "time" - "github.com/go-redis/redis/v8" + "github.com/go-redis/redis" "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" @@ -44,7 +44,7 @@ func (store *UniversalRedisStore) InsertEntry(ctx context.Context, entry *filer. value = util.MaybeGzipData(value) } - _, err = store.Client.Set(ctx, string(entry.FullPath), value, time.Duration(entry.TtlSec)*time.Second).Result() + _, err = store.Client.Set(string(entry.FullPath), value, time.Duration(entry.TtlSec)*time.Second).Result() if err != nil { return fmt.Errorf("persisting %s : %v", entry.FullPath, err) @@ -52,7 +52,7 @@ func (store *UniversalRedisStore) InsertEntry(ctx context.Context, entry *filer. dir, name := entry.FullPath.DirAndName() if name != "" { - _, err = store.Client.SAdd(ctx, genDirectoryListKey(dir), name).Result() + _, err = store.Client.SAdd(genDirectoryListKey(dir), name).Result() if err != nil { return fmt.Errorf("persisting %s in parent dir: %v", entry.FullPath, err) } @@ -68,7 +68,7 @@ func (store *UniversalRedisStore) UpdateEntry(ctx context.Context, entry *filer. func (store *UniversalRedisStore) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) { - data, err := store.Client.Get(ctx, string(fullpath)).Result() + data, err := store.Client.Get(string(fullpath)).Result() if err == redis.Nil { return nil, filer_pb.ErrNotFound } @@ -90,7 +90,7 @@ func (store *UniversalRedisStore) FindEntry(ctx context.Context, fullpath util.F func (store *UniversalRedisStore) DeleteEntry(ctx context.Context, fullpath util.FullPath) (err error) { - _, err = store.Client.Del(ctx, string(fullpath)).Result() + _, err = store.Client.Del(string(fullpath)).Result() if err != nil { return fmt.Errorf("delete %s : %v", fullpath, err) @@ -98,7 +98,7 @@ func (store *UniversalRedisStore) DeleteEntry(ctx context.Context, fullpath util dir, name := fullpath.DirAndName() if name != "" { - _, err = store.Client.SRem(ctx, genDirectoryListKey(dir), name).Result() + _, err = store.Client.SRem(genDirectoryListKey(dir), name).Result() if err != nil { return fmt.Errorf("delete %s in parent dir: %v", fullpath, err) } @@ -109,14 +109,14 @@ func (store *UniversalRedisStore) DeleteEntry(ctx context.Context, fullpath util func (store *UniversalRedisStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) (err error) { - members, err := store.Client.SMembers(ctx, genDirectoryListKey(string(fullpath))).Result() + members, err := store.Client.SMembers(genDirectoryListKey(string(fullpath))).Result() if err != nil { return fmt.Errorf("delete folder %s : %v", fullpath, err) } for _, fileName := range members { path := util.NewFullPath(string(fullpath), fileName) - _, err = store.Client.Del(ctx, string(path)).Result() + _, err = store.Client.Del(string(path)).Result() if err != nil { return fmt.Errorf("delete %s in parent dir: %v", fullpath, err) } @@ -133,7 +133,7 @@ func (store *UniversalRedisStore) ListDirectoryEntries(ctx context.Context, full limit int) (entries []*filer.Entry, err error) { dirListKey := genDirectoryListKey(string(fullpath)) - members, err := store.Client.SMembers(ctx, dirListKey).Result() + members, err := store.Client.SMembers(dirListKey).Result() if err != nil { return nil, fmt.Errorf("list %s : %v", fullpath, err) } @@ -174,8 +174,8 @@ func (store *UniversalRedisStore) ListDirectoryEntries(ctx context.Context, full } else { if entry.TtlSec > 0 { if entry.Attr.Crtime.Add(time.Duration(entry.TtlSec) * time.Second).Before(time.Now()) { - store.Client.Del(ctx, string(path)).Result() - store.Client.SRem(ctx, dirListKey, fileName).Result() + store.Client.Del(string(path)).Result() + store.Client.SRem(dirListKey, fileName).Result() continue } } diff --git a/weed/filer/redis/universal_redis_store_kv.go b/weed/filer/redis/universal_redis_store_kv.go index ad6e389ed..0fc12c631 100644 --- a/weed/filer/redis/universal_redis_store_kv.go +++ b/weed/filer/redis/universal_redis_store_kv.go @@ -5,12 +5,12 @@ import ( "fmt" "github.com/chrislusf/seaweedfs/weed/filer" - "github.com/go-redis/redis/v8" + "github.com/go-redis/redis" ) func (store *UniversalRedisStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) { - _, err = store.Client.Set(ctx, string(key), value, 0).Result() + _, err = store.Client.Set(string(key), value, 0).Result() if err != nil { return fmt.Errorf("kv put: %v", err) @@ -21,7 +21,7 @@ func (store *UniversalRedisStore) KvPut(ctx context.Context, key []byte, value [ func (store *UniversalRedisStore) KvGet(ctx context.Context, key []byte) (value []byte, err error) { - data, err := store.Client.Get(ctx, string(key)).Result() + data, err := store.Client.Get(string(key)).Result() if err == redis.Nil { return nil, filer.ErrKvNotFound @@ -32,7 +32,7 @@ func (store *UniversalRedisStore) KvGet(ctx context.Context, key []byte) (value func (store *UniversalRedisStore) KvDelete(ctx context.Context, key []byte) (err error) { - _, err = store.Client.Del(ctx, string(key)).Result() + _, err = store.Client.Del(string(key)).Result() if err != nil { return fmt.Errorf("kv delete: %v", err) diff --git a/weed/filer/redis2/redis_cluster_store.go b/weed/filer/redis2/redis_cluster_store.go index 22d09da25..38a930d5d 100644 --- a/weed/filer/redis2/redis_cluster_store.go +++ b/weed/filer/redis2/redis_cluster_store.go @@ -3,7 +3,7 @@ package redis2 import ( "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/util" - "github.com/go-redis/redis/v8" + "github.com/go-redis/redis" ) func init() { diff --git a/weed/filer/redis2/redis_store.go b/weed/filer/redis2/redis_store.go index 8eb97e374..da404ed4c 100644 --- a/weed/filer/redis2/redis_store.go +++ b/weed/filer/redis2/redis_store.go @@ -3,7 +3,7 @@ package redis2 import ( "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/util" - "github.com/go-redis/redis/v8" + "github.com/go-redis/redis" ) func init() { diff --git a/weed/filer/redis2/universal_redis_store.go b/weed/filer/redis2/universal_redis_store.go index 6a8d90768..00d02ea14 100644 --- a/weed/filer/redis2/universal_redis_store.go +++ b/weed/filer/redis2/universal_redis_store.go @@ -5,7 +5,7 @@ import ( "fmt" "time" - "github.com/go-redis/redis/v8" + "github.com/go-redis/redis" "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" @@ -56,7 +56,7 @@ func (store *UniversalRedis2Store) InsertEntry(ctx context.Context, entry *filer value = util.MaybeGzipData(value) } - if err = store.Client.Set(ctx, string(entry.FullPath), value, time.Duration(entry.TtlSec)*time.Second).Err(); err != nil { + if err = store.Client.Set(string(entry.FullPath), value, time.Duration(entry.TtlSec)*time.Second).Err(); err != nil { return fmt.Errorf("persisting %s : %v", entry.FullPath, err) } @@ -66,7 +66,7 @@ func (store *UniversalRedis2Store) InsertEntry(ctx context.Context, entry *filer } if name != "" { - if err = store.Client.ZAddNX(ctx, genDirectoryListKey(dir), &redis.Z{Score: 0, Member: name}).Err(); err != nil { + if err = store.Client.ZAddNX(genDirectoryListKey(dir), redis.Z{Score: 0, Member: name}).Err(); err != nil { return fmt.Errorf("persisting %s in parent dir: %v", entry.FullPath, err) } } @@ -81,7 +81,7 @@ func (store *UniversalRedis2Store) UpdateEntry(ctx context.Context, entry *filer func (store *UniversalRedis2Store) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) { - data, err := store.Client.Get(ctx, string(fullpath)).Result() + data, err := store.Client.Get(string(fullpath)).Result() if err == redis.Nil { return nil, filer_pb.ErrNotFound } @@ -103,12 +103,12 @@ func (store *UniversalRedis2Store) FindEntry(ctx context.Context, fullpath util. func (store *UniversalRedis2Store) DeleteEntry(ctx context.Context, fullpath util.FullPath) (err error) { - _, err = store.Client.Del(ctx, genDirectoryListKey(string(fullpath))).Result() + _, err = store.Client.Del(genDirectoryListKey(string(fullpath))).Result() if err != nil { return fmt.Errorf("delete dir list %s : %v", fullpath, err) } - _, err = store.Client.Del(ctx, string(fullpath)).Result() + _, err = store.Client.Del(string(fullpath)).Result() if err != nil { return fmt.Errorf("delete %s : %v", fullpath, err) } @@ -118,7 +118,7 @@ func (store *UniversalRedis2Store) DeleteEntry(ctx context.Context, fullpath uti return nil } if name != "" { - _, err = store.Client.ZRem(ctx, genDirectoryListKey(dir), name).Result() + _, err = store.Client.ZRem(genDirectoryListKey(dir), name).Result() if err != nil { return fmt.Errorf("DeleteEntry %s in parent dir: %v", fullpath, err) } @@ -133,14 +133,14 @@ func (store *UniversalRedis2Store) DeleteFolderChildren(ctx context.Context, ful return nil } - members, err := store.Client.ZRange(ctx, genDirectoryListKey(string(fullpath)), 0, -1).Result() + members, err := store.Client.ZRange(genDirectoryListKey(string(fullpath)), 0, -1).Result() if err != nil { return fmt.Errorf("DeleteFolderChildren %s : %v", fullpath, err) } for _, fileName := range members { path := util.NewFullPath(string(fullpath), fileName) - _, err = store.Client.Del(ctx, string(path)).Result() + _, err = store.Client.Del(string(path)).Result() if err != nil { return fmt.Errorf("DeleteFolderChildren %s in parent dir: %v", fullpath, err) } @@ -159,12 +159,12 @@ func (store *UniversalRedis2Store) ListDirectoryEntries(ctx context.Context, ful dirListKey := genDirectoryListKey(string(fullpath)) start := int64(0) if startFileName != "" { - start, _ = store.Client.ZRank(ctx, dirListKey, startFileName).Result() + start, _ = store.Client.ZRank(dirListKey, startFileName).Result() if !inclusive { start++ } } - members, err := store.Client.ZRange(ctx, dirListKey, start, start+int64(limit)-1).Result() + members, err := store.Client.ZRange(dirListKey, start, start+int64(limit)-1).Result() if err != nil { return nil, fmt.Errorf("list %s : %v", fullpath, err) } @@ -178,8 +178,8 @@ func (store *UniversalRedis2Store) ListDirectoryEntries(ctx context.Context, ful } else { if entry.TtlSec > 0 { if entry.Attr.Crtime.Add(time.Duration(entry.TtlSec) * time.Second).Before(time.Now()) { - store.Client.Del(ctx, string(path)).Result() - store.Client.ZRem(ctx, dirListKey, fileName).Result() + store.Client.Del(string(path)).Result() + store.Client.ZRem(dirListKey, fileName).Result() continue } } diff --git a/weed/filer/redis2/universal_redis_store_kv.go b/weed/filer/redis2/universal_redis_store_kv.go index bde994dc9..658491ddf 100644 --- a/weed/filer/redis2/universal_redis_store_kv.go +++ b/weed/filer/redis2/universal_redis_store_kv.go @@ -5,12 +5,12 @@ import ( "fmt" "github.com/chrislusf/seaweedfs/weed/filer" - "github.com/go-redis/redis/v8" + "github.com/go-redis/redis" ) func (store *UniversalRedis2Store) KvPut(ctx context.Context, key []byte, value []byte) (err error) { - _, err = store.Client.Set(ctx, string(key), value, 0).Result() + _, err = store.Client.Set(string(key), value, 0).Result() if err != nil { return fmt.Errorf("kv put: %v", err) @@ -21,7 +21,7 @@ func (store *UniversalRedis2Store) KvPut(ctx context.Context, key []byte, value func (store *UniversalRedis2Store) KvGet(ctx context.Context, key []byte) (value []byte, err error) { - data, err := store.Client.Get(ctx, string(key)).Result() + data, err := store.Client.Get(string(key)).Result() if err == redis.Nil { return nil, filer.ErrKvNotFound @@ -32,7 +32,7 @@ func (store *UniversalRedis2Store) KvGet(ctx context.Context, key []byte) (value func (store *UniversalRedis2Store) KvDelete(ctx context.Context, key []byte) (err error) { - _, err = store.Client.Del(ctx, string(key)).Result() + _, err = store.Client.Del(string(key)).Result() if err != nil { return fmt.Errorf("kv delete: %v", err) From acc0d5cd2c0294631642fe9b4db64bead721e260 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 11 Jan 2021 02:30:19 -0800 Subject: [PATCH 086/260] Revert "Revert "filer store: redis module upgrade"" This reverts commit dd0171e35cb032781b703bc8ae95d8a21f59ea76. --- go.mod | 6 +-- go.sum | 41 ++++++++++++++++--- weed/filer/redis/redis_cluster_store.go | 2 +- weed/filer/redis/redis_store.go | 2 +- weed/filer/redis/universal_redis_store.go | 22 +++++----- weed/filer/redis/universal_redis_store_kv.go | 8 ++-- weed/filer/redis2/redis_cluster_store.go | 2 +- weed/filer/redis2/redis_store.go | 2 +- weed/filer/redis2/universal_redis_store.go | 26 ++++++------ weed/filer/redis2/universal_redis_store_kv.go | 8 ++-- 10 files changed, 73 insertions(+), 46 deletions(-) diff --git a/go.mod b/go.mod index 18b1097f8..c0bcdc8cd 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/fclairamb/ftpserverlib v0.8.0 github.com/frankban/quicktest v1.7.2 // indirect github.com/go-errors/errors v1.1.1 // indirect - github.com/go-redis/redis v6.15.7+incompatible + github.com/go-redis/redis/v8 v8.4.4 github.com/go-sql-driver/mysql v1.5.0 github.com/gocql/gocql v0.0.0-20190829130954-e163eff7a8c6 github.com/gogo/protobuf v1.2.2-0.20190730201129-28a6bbf47e48 // indirect @@ -51,8 +51,6 @@ require ( github.com/mattn/go-ieproxy v0.0.0-20190805055040-f9202b1cfdeb // indirect github.com/mattn/go-runewidth v0.0.4 // indirect github.com/olivere/elastic/v7 v7.0.19 - github.com/onsi/ginkgo v1.10.1 // indirect - github.com/onsi/gomega v1.7.0 // indirect github.com/peterh/liner v1.1.0 github.com/pierrec/lz4 v2.2.7+incompatible // indirect github.com/prometheus/client_golang v1.3.0 @@ -83,7 +81,7 @@ require ( gocloud.dev/pubsub/natspubsub v0.16.0 gocloud.dev/pubsub/rabbitpubsub v0.16.0 golang.org/x/image v0.0.0-20200119044424-58c23975cae1 // indirect - golang.org/x/net v0.0.0-20200202094626-16171245cfb2 + golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb golang.org/x/sync v0.0.0-20200930132711-30421366ff76 // indirect golang.org/x/sys v0.0.0-20201022201747-fb209a7c41cd golang.org/x/tools v0.0.0-20200103221440-774c71fcf114 diff --git a/go.sum b/go.sum index e019b01f7..d91a570f8 100644 --- a/go.sum +++ b/go.sum @@ -116,6 +116,8 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/devigned/tab v0.1.1/go.mod h1:XG9mPq0dFghrYvoBF3xdRrJzSTX1b7IQrvaL9mzjeJY= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c= @@ -159,6 +161,8 @@ github.com/frankban/quicktest v1.7.2 h1:2QxQoC1TS09S7fhCPsrvqYdvP1H5M1P1ih5ABm3B github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-errors/errors v1.1.1 h1:ljK/pL5ltg3qoN+OtN6yCv9HWSfMwxSx90GJCZQxYNg= @@ -172,8 +176,8 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih4= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-redis/redis v6.15.7+incompatible h1:3skhDh95XQMpnqeqNftPkQD9jL9e5e36z/1SUm6dy1U= -github.com/go-redis/redis v6.15.7+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= +github.com/go-redis/redis/v8 v8.4.4 h1:fGqgxCTR1sydaKI00oQf3OmkU/DIe/I/fYXvGklCIuc= +github.com/go-redis/redis/v8 v8.4.4/go.mod h1:nA0bQuF0i5JFx4Ta9RZxGKXFrQ8cRWntra97f0196iY= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= @@ -259,6 +263,8 @@ github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-replayers/grpcreplay v0.1.0 h1:eNb1y9rZFmY4ax45uEEECSa8fsxGRU+8Bil52ASAwic= github.com/google/go-replayers/grpcreplay v0.1.0/go.mod h1:8Ig2Idjpr6gifRd6pNVggX6TC1Zw6Jx74AKp7QNH2QE= github.com/google/go-replayers/httpreplay v0.1.0 h1:AX7FUb4BjrrzNvblr/OlgwrmFiep6soj5K2QSDW7BGk= @@ -466,6 +472,8 @@ github.com/nats-io/nkeys v0.1.3 h1:6JrEfig+HzTH85yxzhSVbjHRJv9cn0p6n3IngIcM5/k= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= @@ -475,12 +483,15 @@ github.com/olivere/elastic/v7 v7.0.19/go.mod h1:4Jqt5xvjqpjCqgnTcHwl3j8TLs8mvoOK github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo= -github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.2 h1:8mVmC9kjFFmA8H4pKMUhcblgifdkOIXPvbhN1T36q1M= +github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= -github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.10.4 h1:NiTx7EEvBzu9sFOD1zORteLSt3o8gnlvZZwSE9TnY9U= +github.com/onsi/gomega v1.10.4/go.mod h1:g/HbgYopi++010VEqkFgJHKC09uJiW9UkXvMUuKHUCQ= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= @@ -683,6 +694,8 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4 h1:LYy1Hy3MJdrCdMwwzxA/dRok4ejH+RwNGbuoD9fCjto= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opentelemetry.io/otel v0.15.0 h1:CZFy2lPhxd4HlhZnYK8gRyDotksO3Ip9rBweY1vVYJw= +go.opentelemetry.io/otel v0.15.0/go.mod h1:e4GKElweB8W2gWUqbghw0B8t5MCTccc9212eNHnOHwA= go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -722,6 +735,8 @@ golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc h1:c0o/qxkaO2LF5t6fQrT4b5 golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067 h1:KYGJGHOQy8oSi1fDlSpcZF0+juKwk/hEMv5SiwHogR0= @@ -761,6 +776,9 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb h1:eBmm0M9fYhWpKZLjQUUKka/LtIxf46G4fxeEz5KJr9U= +golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -799,9 +817,15 @@ golang.org/x/sys v0.0.0-20190620070143-6f217b454f45/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0 h1:HyfiK1WMnHj5FXFXatD+Qs1A/xC2Run6RzeW1SyHxpc= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201022201747-fb209a7c41cd h1:WgqgiQvkiZWz7XLhphjt2GI2GcGCTIZs9jqXMWmH+oc= golang.org/x/sys v0.0.0-20201022201747-fb209a7c41cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -809,6 +833,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= @@ -941,8 +967,11 @@ gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bl gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/weed/filer/redis/redis_cluster_store.go b/weed/filer/redis/redis_cluster_store.go index a2ab5b5f9..9572058a8 100644 --- a/weed/filer/redis/redis_cluster_store.go +++ b/weed/filer/redis/redis_cluster_store.go @@ -3,7 +3,7 @@ package redis import ( "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/util" - "github.com/go-redis/redis" + "github.com/go-redis/redis/v8" ) func init() { diff --git a/weed/filer/redis/redis_store.go b/weed/filer/redis/redis_store.go index e152457ed..665352a63 100644 --- a/weed/filer/redis/redis_store.go +++ b/weed/filer/redis/redis_store.go @@ -3,7 +3,7 @@ package redis import ( "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/util" - "github.com/go-redis/redis" + "github.com/go-redis/redis/v8" ) func init() { diff --git a/weed/filer/redis/universal_redis_store.go b/weed/filer/redis/universal_redis_store.go index 0de9924a3..a3e09c3a1 100644 --- a/weed/filer/redis/universal_redis_store.go +++ b/weed/filer/redis/universal_redis_store.go @@ -7,7 +7,7 @@ import ( "strings" "time" - "github.com/go-redis/redis" + "github.com/go-redis/redis/v8" "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" @@ -44,7 +44,7 @@ func (store *UniversalRedisStore) InsertEntry(ctx context.Context, entry *filer. value = util.MaybeGzipData(value) } - _, err = store.Client.Set(string(entry.FullPath), value, time.Duration(entry.TtlSec)*time.Second).Result() + _, err = store.Client.Set(ctx, string(entry.FullPath), value, time.Duration(entry.TtlSec)*time.Second).Result() if err != nil { return fmt.Errorf("persisting %s : %v", entry.FullPath, err) @@ -52,7 +52,7 @@ func (store *UniversalRedisStore) InsertEntry(ctx context.Context, entry *filer. dir, name := entry.FullPath.DirAndName() if name != "" { - _, err = store.Client.SAdd(genDirectoryListKey(dir), name).Result() + _, err = store.Client.SAdd(ctx, genDirectoryListKey(dir), name).Result() if err != nil { return fmt.Errorf("persisting %s in parent dir: %v", entry.FullPath, err) } @@ -68,7 +68,7 @@ func (store *UniversalRedisStore) UpdateEntry(ctx context.Context, entry *filer. func (store *UniversalRedisStore) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) { - data, err := store.Client.Get(string(fullpath)).Result() + data, err := store.Client.Get(ctx, string(fullpath)).Result() if err == redis.Nil { return nil, filer_pb.ErrNotFound } @@ -90,7 +90,7 @@ func (store *UniversalRedisStore) FindEntry(ctx context.Context, fullpath util.F func (store *UniversalRedisStore) DeleteEntry(ctx context.Context, fullpath util.FullPath) (err error) { - _, err = store.Client.Del(string(fullpath)).Result() + _, err = store.Client.Del(ctx, string(fullpath)).Result() if err != nil { return fmt.Errorf("delete %s : %v", fullpath, err) @@ -98,7 +98,7 @@ func (store *UniversalRedisStore) DeleteEntry(ctx context.Context, fullpath util dir, name := fullpath.DirAndName() if name != "" { - _, err = store.Client.SRem(genDirectoryListKey(dir), name).Result() + _, err = store.Client.SRem(ctx, genDirectoryListKey(dir), name).Result() if err != nil { return fmt.Errorf("delete %s in parent dir: %v", fullpath, err) } @@ -109,14 +109,14 @@ func (store *UniversalRedisStore) DeleteEntry(ctx context.Context, fullpath util func (store *UniversalRedisStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) (err error) { - members, err := store.Client.SMembers(genDirectoryListKey(string(fullpath))).Result() + members, err := store.Client.SMembers(ctx, genDirectoryListKey(string(fullpath))).Result() if err != nil { return fmt.Errorf("delete folder %s : %v", fullpath, err) } for _, fileName := range members { path := util.NewFullPath(string(fullpath), fileName) - _, err = store.Client.Del(string(path)).Result() + _, err = store.Client.Del(ctx, string(path)).Result() if err != nil { return fmt.Errorf("delete %s in parent dir: %v", fullpath, err) } @@ -133,7 +133,7 @@ func (store *UniversalRedisStore) ListDirectoryEntries(ctx context.Context, full limit int) (entries []*filer.Entry, err error) { dirListKey := genDirectoryListKey(string(fullpath)) - members, err := store.Client.SMembers(dirListKey).Result() + members, err := store.Client.SMembers(ctx, dirListKey).Result() if err != nil { return nil, fmt.Errorf("list %s : %v", fullpath, err) } @@ -174,8 +174,8 @@ func (store *UniversalRedisStore) ListDirectoryEntries(ctx context.Context, full } else { if entry.TtlSec > 0 { if entry.Attr.Crtime.Add(time.Duration(entry.TtlSec) * time.Second).Before(time.Now()) { - store.Client.Del(string(path)).Result() - store.Client.SRem(dirListKey, fileName).Result() + store.Client.Del(ctx, string(path)).Result() + store.Client.SRem(ctx, dirListKey, fileName).Result() continue } } diff --git a/weed/filer/redis/universal_redis_store_kv.go b/weed/filer/redis/universal_redis_store_kv.go index 0fc12c631..ad6e389ed 100644 --- a/weed/filer/redis/universal_redis_store_kv.go +++ b/weed/filer/redis/universal_redis_store_kv.go @@ -5,12 +5,12 @@ import ( "fmt" "github.com/chrislusf/seaweedfs/weed/filer" - "github.com/go-redis/redis" + "github.com/go-redis/redis/v8" ) func (store *UniversalRedisStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) { - _, err = store.Client.Set(string(key), value, 0).Result() + _, err = store.Client.Set(ctx, string(key), value, 0).Result() if err != nil { return fmt.Errorf("kv put: %v", err) @@ -21,7 +21,7 @@ func (store *UniversalRedisStore) KvPut(ctx context.Context, key []byte, value [ func (store *UniversalRedisStore) KvGet(ctx context.Context, key []byte) (value []byte, err error) { - data, err := store.Client.Get(string(key)).Result() + data, err := store.Client.Get(ctx, string(key)).Result() if err == redis.Nil { return nil, filer.ErrKvNotFound @@ -32,7 +32,7 @@ func (store *UniversalRedisStore) KvGet(ctx context.Context, key []byte) (value func (store *UniversalRedisStore) KvDelete(ctx context.Context, key []byte) (err error) { - _, err = store.Client.Del(string(key)).Result() + _, err = store.Client.Del(ctx, string(key)).Result() if err != nil { return fmt.Errorf("kv delete: %v", err) diff --git a/weed/filer/redis2/redis_cluster_store.go b/weed/filer/redis2/redis_cluster_store.go index 38a930d5d..22d09da25 100644 --- a/weed/filer/redis2/redis_cluster_store.go +++ b/weed/filer/redis2/redis_cluster_store.go @@ -3,7 +3,7 @@ package redis2 import ( "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/util" - "github.com/go-redis/redis" + "github.com/go-redis/redis/v8" ) func init() { diff --git a/weed/filer/redis2/redis_store.go b/weed/filer/redis2/redis_store.go index da404ed4c..8eb97e374 100644 --- a/weed/filer/redis2/redis_store.go +++ b/weed/filer/redis2/redis_store.go @@ -3,7 +3,7 @@ package redis2 import ( "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/util" - "github.com/go-redis/redis" + "github.com/go-redis/redis/v8" ) func init() { diff --git a/weed/filer/redis2/universal_redis_store.go b/weed/filer/redis2/universal_redis_store.go index 00d02ea14..6a8d90768 100644 --- a/weed/filer/redis2/universal_redis_store.go +++ b/weed/filer/redis2/universal_redis_store.go @@ -5,7 +5,7 @@ import ( "fmt" "time" - "github.com/go-redis/redis" + "github.com/go-redis/redis/v8" "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" @@ -56,7 +56,7 @@ func (store *UniversalRedis2Store) InsertEntry(ctx context.Context, entry *filer value = util.MaybeGzipData(value) } - if err = store.Client.Set(string(entry.FullPath), value, time.Duration(entry.TtlSec)*time.Second).Err(); err != nil { + if err = store.Client.Set(ctx, string(entry.FullPath), value, time.Duration(entry.TtlSec)*time.Second).Err(); err != nil { return fmt.Errorf("persisting %s : %v", entry.FullPath, err) } @@ -66,7 +66,7 @@ func (store *UniversalRedis2Store) InsertEntry(ctx context.Context, entry *filer } if name != "" { - if err = store.Client.ZAddNX(genDirectoryListKey(dir), redis.Z{Score: 0, Member: name}).Err(); err != nil { + if err = store.Client.ZAddNX(ctx, genDirectoryListKey(dir), &redis.Z{Score: 0, Member: name}).Err(); err != nil { return fmt.Errorf("persisting %s in parent dir: %v", entry.FullPath, err) } } @@ -81,7 +81,7 @@ func (store *UniversalRedis2Store) UpdateEntry(ctx context.Context, entry *filer func (store *UniversalRedis2Store) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) { - data, err := store.Client.Get(string(fullpath)).Result() + data, err := store.Client.Get(ctx, string(fullpath)).Result() if err == redis.Nil { return nil, filer_pb.ErrNotFound } @@ -103,12 +103,12 @@ func (store *UniversalRedis2Store) FindEntry(ctx context.Context, fullpath util. func (store *UniversalRedis2Store) DeleteEntry(ctx context.Context, fullpath util.FullPath) (err error) { - _, err = store.Client.Del(genDirectoryListKey(string(fullpath))).Result() + _, err = store.Client.Del(ctx, genDirectoryListKey(string(fullpath))).Result() if err != nil { return fmt.Errorf("delete dir list %s : %v", fullpath, err) } - _, err = store.Client.Del(string(fullpath)).Result() + _, err = store.Client.Del(ctx, string(fullpath)).Result() if err != nil { return fmt.Errorf("delete %s : %v", fullpath, err) } @@ -118,7 +118,7 @@ func (store *UniversalRedis2Store) DeleteEntry(ctx context.Context, fullpath uti return nil } if name != "" { - _, err = store.Client.ZRem(genDirectoryListKey(dir), name).Result() + _, err = store.Client.ZRem(ctx, genDirectoryListKey(dir), name).Result() if err != nil { return fmt.Errorf("DeleteEntry %s in parent dir: %v", fullpath, err) } @@ -133,14 +133,14 @@ func (store *UniversalRedis2Store) DeleteFolderChildren(ctx context.Context, ful return nil } - members, err := store.Client.ZRange(genDirectoryListKey(string(fullpath)), 0, -1).Result() + members, err := store.Client.ZRange(ctx, genDirectoryListKey(string(fullpath)), 0, -1).Result() if err != nil { return fmt.Errorf("DeleteFolderChildren %s : %v", fullpath, err) } for _, fileName := range members { path := util.NewFullPath(string(fullpath), fileName) - _, err = store.Client.Del(string(path)).Result() + _, err = store.Client.Del(ctx, string(path)).Result() if err != nil { return fmt.Errorf("DeleteFolderChildren %s in parent dir: %v", fullpath, err) } @@ -159,12 +159,12 @@ func (store *UniversalRedis2Store) ListDirectoryEntries(ctx context.Context, ful dirListKey := genDirectoryListKey(string(fullpath)) start := int64(0) if startFileName != "" { - start, _ = store.Client.ZRank(dirListKey, startFileName).Result() + start, _ = store.Client.ZRank(ctx, dirListKey, startFileName).Result() if !inclusive { start++ } } - members, err := store.Client.ZRange(dirListKey, start, start+int64(limit)-1).Result() + members, err := store.Client.ZRange(ctx, dirListKey, start, start+int64(limit)-1).Result() if err != nil { return nil, fmt.Errorf("list %s : %v", fullpath, err) } @@ -178,8 +178,8 @@ func (store *UniversalRedis2Store) ListDirectoryEntries(ctx context.Context, ful } else { if entry.TtlSec > 0 { if entry.Attr.Crtime.Add(time.Duration(entry.TtlSec) * time.Second).Before(time.Now()) { - store.Client.Del(string(path)).Result() - store.Client.ZRem(dirListKey, fileName).Result() + store.Client.Del(ctx, string(path)).Result() + store.Client.ZRem(ctx, dirListKey, fileName).Result() continue } } diff --git a/weed/filer/redis2/universal_redis_store_kv.go b/weed/filer/redis2/universal_redis_store_kv.go index 658491ddf..bde994dc9 100644 --- a/weed/filer/redis2/universal_redis_store_kv.go +++ b/weed/filer/redis2/universal_redis_store_kv.go @@ -5,12 +5,12 @@ import ( "fmt" "github.com/chrislusf/seaweedfs/weed/filer" - "github.com/go-redis/redis" + "github.com/go-redis/redis/v8" ) func (store *UniversalRedis2Store) KvPut(ctx context.Context, key []byte, value []byte) (err error) { - _, err = store.Client.Set(string(key), value, 0).Result() + _, err = store.Client.Set(ctx, string(key), value, 0).Result() if err != nil { return fmt.Errorf("kv put: %v", err) @@ -21,7 +21,7 @@ func (store *UniversalRedis2Store) KvPut(ctx context.Context, key []byte, value func (store *UniversalRedis2Store) KvGet(ctx context.Context, key []byte) (value []byte, err error) { - data, err := store.Client.Get(string(key)).Result() + data, err := store.Client.Get(ctx, string(key)).Result() if err == redis.Nil { return nil, filer.ErrKvNotFound @@ -32,7 +32,7 @@ func (store *UniversalRedis2Store) KvGet(ctx context.Context, key []byte) (value func (store *UniversalRedis2Store) KvDelete(ctx context.Context, key []byte) (err error) { - _, err = store.Client.Del(string(key)).Result() + _, err = store.Client.Del(ctx, string(key)).Result() if err != nil { return fmt.Errorf("kv delete: %v", err) From 2c7148cd550ebd6cd6b25223fc8cbba575758a47 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 11 Jan 2021 02:43:53 -0800 Subject: [PATCH 087/260] redis cluster: do not enable route by latency by default --- weed/command/scaffold.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/weed/command/scaffold.go b/weed/command/scaffold.go index 600a4a8d4..2991fda69 100644 --- a/weed/command/scaffold.go +++ b/weed/command/scaffold.go @@ -174,9 +174,9 @@ addresses = [ ] password = "" # allows reads from slave servers or the master, but all writes still go to the master -readOnly = true +readOnly = false # automatically use the closest Redis server for reads -routeByLatency = true +routeByLatency = false # This changes the data layout. Only add new directories. Removing/Updating will cause data loss. superLargeDirectories = [] From 8a7302ce377dd24a789b11b83a921dc974050947 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 11 Jan 2021 03:50:45 -0800 Subject: [PATCH 088/260] s3: KeyCount is zero fix https://github.com/chrislusf/seaweedfs/issues/1741 --- weed/s3api/s3api_objects_list_handlers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/s3api/s3api_objects_list_handlers.go b/weed/s3api/s3api_objects_list_handlers.go index fb1497b78..2d36c6ec9 100644 --- a/weed/s3api/s3api_objects_list_handlers.go +++ b/weed/s3api/s3api_objects_list_handlers.go @@ -71,7 +71,7 @@ func (s3a *S3ApiServer) ListObjectsV2Handler(w http.ResponseWriter, r *http.Requ ContinuationToken: continuationToken, Delimiter: response.Delimiter, IsTruncated: response.IsTruncated, - KeyCount: len(response.Contents), + KeyCount: len(response.Contents) + len(response.CommonPrefixes), MaxKeys: response.MaxKeys, NextContinuationToken: response.NextMarker, Prefix: response.Prefix, From a4f0bac9b73aa0f2a24eccc1558910c8135e2927 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 11 Jan 2021 13:26:59 -0800 Subject: [PATCH 089/260] mount: avoid fatal error --- weed/filesys/meta_cache/meta_cache_subscribe.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/filesys/meta_cache/meta_cache_subscribe.go b/weed/filesys/meta_cache/meta_cache_subscribe.go index f9973f436..5b6503a55 100644 --- a/weed/filesys/meta_cache/meta_cache_subscribe.go +++ b/weed/filesys/meta_cache/meta_cache_subscribe.go @@ -73,7 +73,7 @@ func SubscribeMetaEvents(mc *MetaCache, selfSignature int32, client filer_pb.Fil } if err := processEventFn(resp); err != nil { - glog.Fatalf("process %v: %v", resp, err) + return err } lastTsNs = resp.TsNs } From 38d516251eb080e56ee16747684e808a06cb6702 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 11 Jan 2021 13:50:15 -0800 Subject: [PATCH 090/260] Revert "mount: avoid fatal error" This reverts commit a4f0bac9b73aa0f2a24eccc1558910c8135e2927. --- weed/filesys/meta_cache/meta_cache_subscribe.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/filesys/meta_cache/meta_cache_subscribe.go b/weed/filesys/meta_cache/meta_cache_subscribe.go index 5b6503a55..f9973f436 100644 --- a/weed/filesys/meta_cache/meta_cache_subscribe.go +++ b/weed/filesys/meta_cache/meta_cache_subscribe.go @@ -73,7 +73,7 @@ func SubscribeMetaEvents(mc *MetaCache, selfSignature int32, client filer_pb.Fil } if err := processEventFn(resp); err != nil { - return err + glog.Fatalf("process %v: %v", resp, err) } lastTsNs = resp.TsNs } From cfb9342a15ce73c1e04c0e690cf87f65133bcf1b Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 12 Jan 2021 02:28:13 -0800 Subject: [PATCH 091/260] avoid concurrent map updates to viper --- weed/command/filer_replication.go | 3 +-- weed/filer/configuration.go | 6 +++--- weed/notification/configuration.go | 5 ++--- weed/security/tls.go | 7 +++---- weed/storage/backend/backend.go | 4 ++-- weed/util/config.go | 18 +++++++++++++++--- 6 files changed, 26 insertions(+), 17 deletions(-) diff --git a/weed/command/filer_replication.go b/weed/command/filer_replication.go index 40f2b570b..4f698e375 100644 --- a/weed/command/filer_replication.go +++ b/weed/command/filer_replication.go @@ -14,7 +14,6 @@ import ( _ "github.com/chrislusf/seaweedfs/weed/replication/sink/s3sink" "github.com/chrislusf/seaweedfs/weed/replication/sub" "github.com/chrislusf/seaweedfs/weed/util" - "github.com/spf13/viper" ) func init() { @@ -123,7 +122,7 @@ func runFilerReplicate(cmd *Command, args []string) bool { } -func validateOneEnabledInput(config *viper.Viper) { +func validateOneEnabledInput(config *util.ViperProxy) { enabledInput := "" for _, input := range sub.NotificationInputs { if config.GetBool("notification." + input.GetName() + ".enabled") { diff --git a/weed/filer/configuration.go b/weed/filer/configuration.go index a6f18709e..9ef2f3e0f 100644 --- a/weed/filer/configuration.go +++ b/weed/filer/configuration.go @@ -2,7 +2,7 @@ package filer import ( "github.com/chrislusf/seaweedfs/weed/glog" - "github.com/spf13/viper" + "github.com/chrislusf/seaweedfs/weed/util" "os" "reflect" "strings" @@ -12,7 +12,7 @@ var ( Stores []FilerStore ) -func (f *Filer) LoadConfiguration(config *viper.Viper) { +func (f *Filer) LoadConfiguration(config *util.ViperProxy) { validateOneEnabledStore(config) @@ -79,7 +79,7 @@ func (f *Filer) LoadConfiguration(config *viper.Viper) { } -func validateOneEnabledStore(config *viper.Viper) { +func validateOneEnabledStore(config *util.ViperProxy) { enabledStore := "" for _, store := range Stores { if config.GetBool(store.GetName() + ".enabled") { diff --git a/weed/notification/configuration.go b/weed/notification/configuration.go index 36211692c..541a453e9 100644 --- a/weed/notification/configuration.go +++ b/weed/notification/configuration.go @@ -4,7 +4,6 @@ import ( "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/util" "github.com/golang/protobuf/proto" - "github.com/spf13/viper" ) type MessageQueue interface { @@ -21,7 +20,7 @@ var ( Queue MessageQueue ) -func LoadConfiguration(config *viper.Viper, prefix string) { +func LoadConfiguration(config *util.ViperProxy, prefix string) { if config == nil { return @@ -43,7 +42,7 @@ func LoadConfiguration(config *viper.Viper, prefix string) { } -func validateOneEnabledQueue(config *viper.Viper) { +func validateOneEnabledQueue(config *util.ViperProxy) { enabledQueue := "" for _, queue := range MessageQueues { if config.GetBool(queue.GetName() + ".enabled") { diff --git a/weed/security/tls.go b/weed/security/tls.go index 72363768f..b4bf84e2d 100644 --- a/weed/security/tls.go +++ b/weed/security/tls.go @@ -3,17 +3,16 @@ package security import ( "crypto/tls" "crypto/x509" + "github.com/chrislusf/seaweedfs/weed/util" "io/ioutil" - "github.com/spf13/viper" - "google.golang.org/grpc" "google.golang.org/grpc/credentials" "github.com/chrislusf/seaweedfs/weed/glog" ) -func LoadServerTLS(config *viper.Viper, component string) grpc.ServerOption { +func LoadServerTLS(config *util.ViperProxy, component string) grpc.ServerOption { if config == nil { return nil } @@ -40,7 +39,7 @@ func LoadServerTLS(config *viper.Viper, component string) grpc.ServerOption { return grpc.Creds(ta) } -func LoadClientTLS(config *viper.Viper, component string) grpc.DialOption { +func LoadClientTLS(config *util.ViperProxy, component string) grpc.DialOption { if config == nil { return grpc.WithInsecure() } diff --git a/weed/storage/backend/backend.go b/weed/storage/backend/backend.go index daab29621..b8b883be6 100644 --- a/weed/storage/backend/backend.go +++ b/weed/storage/backend/backend.go @@ -1,6 +1,7 @@ package backend import ( + "github.com/chrislusf/seaweedfs/weed/util" "io" "os" "strings" @@ -9,7 +10,6 @@ import ( "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/master_pb" "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb" - "github.com/spf13/viper" ) type BackendStorageFile interface { @@ -45,7 +45,7 @@ var ( ) // used by master to load remote storage configurations -func LoadConfiguration(config *viper.Viper) { +func LoadConfiguration(config *util.ViperProxy) { StorageBackendPrefix := "storage.backend" diff --git a/weed/util/config.go b/weed/util/config.go index 94e621e34..54edf5a3c 100644 --- a/weed/util/config.go +++ b/weed/util/config.go @@ -2,6 +2,7 @@ package util import ( "strings" + "sync" "github.com/spf13/viper" @@ -46,9 +47,20 @@ func LoadConfiguration(configFileName string, required bool) (loaded bool) { return true } -func GetViper() *viper.Viper { - v := &viper.Viper{} - *v = *viper.GetViper() +type ViperProxy struct { + *viper.Viper + sync.Mutex +} + +func (vp *ViperProxy) SetDefault(key string, value interface{}) { + vp.Lock() + defer vp.Unlock() + vp.Viper.SetDefault(key, value) +} + +func GetViper() *ViperProxy { + v := &ViperProxy{} + v.Viper = viper.GetViper() v.AutomaticEnv() v.SetEnvPrefix("weed") v.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) From 361043e6c17771c0c57f29630c6e5feb617d2eab Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 12 Jan 2021 02:28:57 -0800 Subject: [PATCH 092/260] filer store: leveldb2 fix nil entry error if not found --- weed/filer/etcd/etcd_store.go | 2 +- weed/filer/leveldb/leveldb_store.go | 2 +- weed/filer/leveldb2/leveldb2_store.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/weed/filer/etcd/etcd_store.go b/weed/filer/etcd/etcd_store.go index 634fba1eb..01c453158 100644 --- a/weed/filer/etcd/etcd_store.go +++ b/weed/filer/etcd/etcd_store.go @@ -101,7 +101,7 @@ func (store *EtcdStore) FindEntry(ctx context.Context, fullpath weed_util.FullPa resp, err := store.client.Get(ctx, string(key)) if err != nil { - return nil, fmt.Errorf("get %s : %v", entry.FullPath, err) + return nil, fmt.Errorf("get %s : %v", fullpath, err) } if len(resp.Kvs) == 0 { diff --git a/weed/filer/leveldb/leveldb_store.go b/weed/filer/leveldb/leveldb_store.go index 1aefbf5d4..e596a180e 100644 --- a/weed/filer/leveldb/leveldb_store.go +++ b/weed/filer/leveldb/leveldb_store.go @@ -107,7 +107,7 @@ func (store *LevelDBStore) FindEntry(ctx context.Context, fullpath weed_util.Ful return nil, filer_pb.ErrNotFound } if err != nil { - return nil, fmt.Errorf("get %s : %v", entry.FullPath, err) + return nil, fmt.Errorf("get %s : %v", fullpath, err) } entry = &filer.Entry{ diff --git a/weed/filer/leveldb2/leveldb2_store.go b/weed/filer/leveldb2/leveldb2_store.go index ceda64153..66ad61b8b 100644 --- a/weed/filer/leveldb2/leveldb2_store.go +++ b/weed/filer/leveldb2/leveldb2_store.go @@ -115,7 +115,7 @@ func (store *LevelDB2Store) FindEntry(ctx context.Context, fullpath weed_util.Fu return nil, filer_pb.ErrNotFound } if err != nil { - return nil, fmt.Errorf("get %s : %v", entry.FullPath, err) + return nil, fmt.Errorf("get %s : %v", fullpath, err) } entry = &filer.Entry{ From 629c9962e7d268752cf8f836783f34dc009ceda7 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 12 Jan 2021 02:29:44 -0800 Subject: [PATCH 093/260] filer: add leveldb3 support dedicated leveldb instance for each bucket --- weed/command/scaffold.go | 6 + weed/filer/leveldb3/leveldb3_store.go | 363 +++++++++++++++++++++ weed/filer/leveldb3/leveldb3_store_kv.go | 46 +++ weed/filer/leveldb3/leveldb3_store_test.go | 88 +++++ weed/server/filer_server.go | 1 + 5 files changed, 504 insertions(+) create mode 100644 weed/filer/leveldb3/leveldb3_store.go create mode 100644 weed/filer/leveldb3/leveldb3_store_kv.go create mode 100644 weed/filer/leveldb3/leveldb3_store_test.go diff --git a/weed/command/scaffold.go b/weed/command/scaffold.go index 2991fda69..f43a9d143 100644 --- a/weed/command/scaffold.go +++ b/weed/command/scaffold.go @@ -89,6 +89,12 @@ buckets_folder = "/buckets" enabled = true dir = "./filerldb2" # directory to store level db files +[leveldb3] +# similar to leveldb2. +# each bucket has its own meta store. +enabled = false +dir = "./filerldb3" # directory to store level db files + [rocksdb] # local on disk, similar to leveldb # since it is using a C wrapper, you need to install rocksdb and build it by yourself diff --git a/weed/filer/leveldb3/leveldb3_store.go b/weed/filer/leveldb3/leveldb3_store.go new file mode 100644 index 000000000..31956daf9 --- /dev/null +++ b/weed/filer/leveldb3/leveldb3_store.go @@ -0,0 +1,363 @@ +package leveldb + +import ( + "bytes" + "context" + "crypto/md5" + "fmt" + "github.com/syndtr/goleveldb/leveldb" + leveldb_errors "github.com/syndtr/goleveldb/leveldb/errors" + "github.com/syndtr/goleveldb/leveldb/opt" + leveldb_util "github.com/syndtr/goleveldb/leveldb/util" + "io" + "os" + "strings" + "sync" + + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + weed_util "github.com/chrislusf/seaweedfs/weed/util" +) + +const ( + DEFAULT = "_main" +) + +func init() { + filer.Stores = append(filer.Stores, &LevelDB3Store{}) +} + +type LevelDB3Store struct { + dir string + dbs map[string]*leveldb.DB + dbsLock sync.RWMutex +} + +func (store *LevelDB3Store) GetName() string { + return "leveldb3" +} + +func (store *LevelDB3Store) Initialize(configuration weed_util.Configuration, prefix string) (err error) { + dir := configuration.GetString(prefix + "dir") + return store.initialize(dir) +} + +func (store *LevelDB3Store) initialize(dir string) (err error) { + glog.Infof("filer store leveldb3 dir: %s", dir) + if err := weed_util.TestFolderWritable(dir); err != nil { + return fmt.Errorf("Check Level Folder %s Writable: %s", dir, err) + } + store.dir = dir + + db, loadDbErr := store.loadDB(DEFAULT) + if loadDbErr != nil { + return loadDbErr + } + store.dbs = make(map[string]*leveldb.DB) + store.dbs[DEFAULT] = db + + return +} + +func (store *LevelDB3Store) loadDB(name string) (*leveldb.DB, error) { + + opts := &opt.Options{ + BlockCacheCapacity: 32 * 1024 * 1024, // default value is 8MiB + WriteBuffer: 16 * 1024 * 1024, // default value is 4MiB + CompactionTableSizeMultiplier: 4, + } + if name != DEFAULT { + opts = &opt.Options{ + BlockCacheCapacity: 4 * 1024 * 1024, // default value is 8MiB + WriteBuffer: 2 * 1024 * 1024, // default value is 4MiB + CompactionTableSizeMultiplier: 4, + } + } + + dbFolder := fmt.Sprintf("%s/%s", store.dir, name) + os.MkdirAll(dbFolder, 0755) + db, dbErr := leveldb.OpenFile(dbFolder, opts) + if leveldb_errors.IsCorrupted(dbErr) { + db, dbErr = leveldb.RecoverFile(dbFolder, opts) + } + if dbErr != nil { + glog.Errorf("filer store open dir %s: %v", dbFolder, dbErr) + return nil, dbErr + } + return db, nil +} + +func (store *LevelDB3Store) findDB(fullpath weed_util.FullPath, isForChildren bool) (*leveldb.DB, string, weed_util.FullPath, error) { + + store.dbsLock.RLock() + + defaultDB := store.dbs[DEFAULT] + if !strings.HasPrefix(string(fullpath), "/buckets/") { + store.dbsLock.RUnlock() + return defaultDB, DEFAULT, fullpath, nil + } + + // detect bucket + bucketAndObjectKey := string(fullpath)[len("/buckets/"):] + t := strings.Index(bucketAndObjectKey, "/") + if t < 0 && !isForChildren { + store.dbsLock.RUnlock() + return defaultDB, DEFAULT, fullpath, nil + } + bucket := bucketAndObjectKey + shortPath := weed_util.FullPath("/") + if t > 0 { + bucket = bucketAndObjectKey[:t] + shortPath = weed_util.FullPath(bucketAndObjectKey[t:]) + } + + println("bucket:", bucket, "shortPath", shortPath) + + if db, found := store.dbs[bucket]; found { + store.dbsLock.RUnlock() + return db, bucket, shortPath, nil + } + + store.dbsLock.RUnlock() + // upgrade to write lock + store.dbsLock.Lock() + defer store.dbsLock.Unlock() + + // double check after getting the write lock + if db, found := store.dbs[bucket]; found { + return db, bucket, shortPath, nil + } + + // create db + db, err := store.loadDB(bucket) + if err != nil { + return nil, bucket, shortPath, err + } + store.dbs[bucket] = db + + return db, bucket, shortPath, nil +} + +func (store *LevelDB3Store) BeginTransaction(ctx context.Context) (context.Context, error) { + return ctx, nil +} +func (store *LevelDB3Store) CommitTransaction(ctx context.Context) error { + return nil +} +func (store *LevelDB3Store) RollbackTransaction(ctx context.Context) error { + return nil +} + +func (store *LevelDB3Store) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) { + + db, _, shortPath, err := store.findDB(entry.FullPath, false) + if err != nil { + return fmt.Errorf("findDB %s : %v", entry.FullPath, err) + } + + dir, name := shortPath.DirAndName() + key := genKey(dir, name) + + value, err := entry.EncodeAttributesAndChunks() + if err != nil { + return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err) + } + + if len(entry.Chunks) > 50 { + value = weed_util.MaybeGzipData(value) + } + + err = db.Put(key, value, nil) + + if err != nil { + return fmt.Errorf("persisting %s : %v", entry.FullPath, err) + } + + // println("saved", entry.FullPath, "chunks", len(entry.Chunks)) + + return nil +} + +func (store *LevelDB3Store) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) { + + return store.InsertEntry(ctx, entry) +} + +func (store *LevelDB3Store) FindEntry(ctx context.Context, fullpath weed_util.FullPath) (entry *filer.Entry, err error) { + + db, _, shortPath, err := store.findDB(fullpath, false) + if err != nil { + return nil, fmt.Errorf("findDB %s : %v", fullpath, err) + } + + dir, name := shortPath.DirAndName() + key := genKey(dir, name) + + data, err := db.Get(key, nil) + + if err == leveldb.ErrNotFound { + return nil, filer_pb.ErrNotFound + } + if err != nil { + return nil, fmt.Errorf("get %s : %v", fullpath, err) + } + + entry = &filer.Entry{ + FullPath: fullpath, + } + err = entry.DecodeAttributesAndChunks(weed_util.MaybeDecompressData(data)) + if err != nil { + return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err) + } + + // println("read", entry.FullPath, "chunks", len(entry.Chunks), "data", len(data), string(data)) + + return entry, nil +} + +func (store *LevelDB3Store) DeleteEntry(ctx context.Context, fullpath weed_util.FullPath) (err error) { + + db, _, shortPath, err := store.findDB(fullpath, false) + if err != nil { + return fmt.Errorf("findDB %s : %v", fullpath, err) + } + + dir, name := shortPath.DirAndName() + key := genKey(dir, name) + + err = db.Delete(key, nil) + if err != nil { + return fmt.Errorf("delete %s : %v", fullpath, err) + } + + return nil +} + +func (store *LevelDB3Store) DeleteFolderChildren(ctx context.Context, fullpath weed_util.FullPath) (err error) { + + db, bucket, shortPath, err := store.findDB(fullpath, true) + if err != nil { + return fmt.Errorf("findDB %s : %v", fullpath, err) + } + + if bucket != DEFAULT && shortPath == "/" { + db.Close() + if bucket != "" { // just to make sure + os.RemoveAll(store.dir + "/" + bucket) + } + return nil + } + + directoryPrefix := genDirectoryKeyPrefix(shortPath, "") + + batch := new(leveldb.Batch) + + iter := db.NewIterator(&leveldb_util.Range{Start: directoryPrefix}, nil) + for iter.Next() { + key := iter.Key() + if !bytes.HasPrefix(key, directoryPrefix) { + break + } + fileName := getNameFromKey(key) + if fileName == "" { + continue + } + batch.Delete(append(directoryPrefix, []byte(fileName)...)) + } + iter.Release() + + err = db.Write(batch, nil) + + if err != nil { + return fmt.Errorf("delete %s : %v", fullpath, err) + } + + return nil +} + +func (store *LevelDB3Store) ListDirectoryEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, + limit int) (entries []*filer.Entry, err error) { + return store.ListDirectoryPrefixedEntries(ctx, fullpath, startFileName, inclusive, limit, "") +} + +func (store *LevelDB3Store) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer.Entry, err error) { + + db, _, shortPath, err := store.findDB(fullpath, true) + if err != nil { + return nil, fmt.Errorf("findDB %s : %v", fullpath, err) + } + + directoryPrefix := genDirectoryKeyPrefix(shortPath, prefix) + lastFileStart := directoryPrefix + if startFileName != "" { + lastFileStart = genDirectoryKeyPrefix(shortPath, startFileName) + } + + iter := db.NewIterator(&leveldb_util.Range{Start: lastFileStart}, nil) + for iter.Next() { + key := iter.Key() + if !bytes.HasPrefix(key, directoryPrefix) { + break + } + fileName := getNameFromKey(key) + if fileName == "" { + continue + } + if fileName == startFileName && !inclusive { + continue + } + limit-- + if limit < 0 { + break + } + entry := &filer.Entry{ + FullPath: weed_util.NewFullPath(string(fullpath), fileName), + } + + // println("list", entry.FullPath, "chunks", len(entry.Chunks)) + if decodeErr := entry.DecodeAttributesAndChunks(weed_util.MaybeDecompressData(iter.Value())); decodeErr != nil { + err = decodeErr + glog.V(0).Infof("list %s : %v", entry.FullPath, err) + break + } + entries = append(entries, entry) + } + iter.Release() + + return entries, err +} + +func genKey(dirPath, fileName string) (key []byte) { + key = hashToBytes(dirPath) + key = append(key, []byte(fileName)...) + return key +} + +func genDirectoryKeyPrefix(fullpath weed_util.FullPath, startFileName string) (keyPrefix []byte) { + keyPrefix = hashToBytes(string(fullpath)) + if len(startFileName) > 0 { + keyPrefix = append(keyPrefix, []byte(startFileName)...) + } + return keyPrefix +} + +func getNameFromKey(key []byte) string { + + return string(key[md5.Size:]) + +} + +// hash directory +func hashToBytes(dir string) []byte { + h := md5.New() + io.WriteString(h, dir) + b := h.Sum(nil) + return b +} + +func (store *LevelDB3Store) Shutdown() { + for _, db := range store.dbs { + db.Close() + } +} diff --git a/weed/filer/leveldb3/leveldb3_store_kv.go b/weed/filer/leveldb3/leveldb3_store_kv.go new file mode 100644 index 000000000..18d782b80 --- /dev/null +++ b/weed/filer/leveldb3/leveldb3_store_kv.go @@ -0,0 +1,46 @@ +package leveldb + +import ( + "context" + "fmt" + + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/syndtr/goleveldb/leveldb" +) + +func (store *LevelDB3Store) KvPut(ctx context.Context, key []byte, value []byte) (err error) { + + err = store.dbs[DEFAULT].Put(key, value, nil) + + if err != nil { + return fmt.Errorf("kv put: %v", err) + } + + return nil +} + +func (store *LevelDB3Store) KvGet(ctx context.Context, key []byte) (value []byte, err error) { + + value, err = store.dbs[DEFAULT].Get(key, nil) + + if err == leveldb.ErrNotFound { + return nil, filer.ErrKvNotFound + } + + if err != nil { + return nil, fmt.Errorf("kv get: %v", err) + } + + return +} + +func (store *LevelDB3Store) KvDelete(ctx context.Context, key []byte) (err error) { + + err = store.dbs[DEFAULT].Delete(key, nil) + + if err != nil { + return fmt.Errorf("kv delete: %v", err) + } + + return nil +} diff --git a/weed/filer/leveldb3/leveldb3_store_test.go b/weed/filer/leveldb3/leveldb3_store_test.go new file mode 100644 index 000000000..81f759e78 --- /dev/null +++ b/weed/filer/leveldb3/leveldb3_store_test.go @@ -0,0 +1,88 @@ +package leveldb + +import ( + "context" + "io/ioutil" + "os" + "testing" + + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/util" +) + +func TestCreateAndFind(t *testing.T) { + testFiler := filer.NewFiler(nil, nil, "", 0, "", "", "", nil) + dir, _ := ioutil.TempDir("", "seaweedfs_filer_test") + defer os.RemoveAll(dir) + store := &LevelDB3Store{} + store.initialize(dir, 2) + testFiler.SetStore(store) + + fullpath := util.FullPath("/home/chris/this/is/one/file1.jpg") + + ctx := context.Background() + + entry1 := &filer.Entry{ + FullPath: fullpath, + Attr: filer.Attr{ + Mode: 0440, + Uid: 1234, + Gid: 5678, + }, + } + + if err := testFiler.CreateEntry(ctx, entry1, false, false, nil); err != nil { + t.Errorf("create entry %v: %v", entry1.FullPath, err) + return + } + + entry, err := testFiler.FindEntry(ctx, fullpath) + + if err != nil { + t.Errorf("find entry: %v", err) + return + } + + if entry.FullPath != entry1.FullPath { + t.Errorf("find wrong entry: %v", entry.FullPath) + return + } + + // checking one upper directory + entries, _ := testFiler.ListDirectoryEntries(ctx, util.FullPath("/home/chris/this/is/one"), "", false, 100, "", "") + if len(entries) != 1 { + t.Errorf("list entries count: %v", len(entries)) + return + } + + // checking one upper directory + entries, _ = testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "") + if len(entries) != 1 { + t.Errorf("list entries count: %v", len(entries)) + return + } + +} + +func TestEmptyRoot(t *testing.T) { + testFiler := filer.NewFiler(nil, nil, "", 0, "", "", "", nil) + dir, _ := ioutil.TempDir("", "seaweedfs_filer_test2") + defer os.RemoveAll(dir) + store := &LevelDB3Store{} + store.initialize(dir, 2) + testFiler.SetStore(store) + + ctx := context.Background() + + // checking one upper directory + entries, err := testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "") + if err != nil { + t.Errorf("list entries: %v", err) + return + } + if len(entries) != 0 { + t.Errorf("list entries count: %v", len(entries)) + return + } + +} diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index efd601717..9256fd676 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -26,6 +26,7 @@ import ( _ "github.com/chrislusf/seaweedfs/weed/filer/hbase" _ "github.com/chrislusf/seaweedfs/weed/filer/leveldb" _ "github.com/chrislusf/seaweedfs/weed/filer/leveldb2" + _ "github.com/chrislusf/seaweedfs/weed/filer/leveldb3" _ "github.com/chrislusf/seaweedfs/weed/filer/mongodb" _ "github.com/chrislusf/seaweedfs/weed/filer/mysql" _ "github.com/chrislusf/seaweedfs/weed/filer/postgres" From 0ef43a23a70d1c4fd25f373d1ae83b6949552f33 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 12 Jan 2021 02:30:12 -0800 Subject: [PATCH 094/260] go fmt --- weed/command/filer.go | 24 ++++++++++++------------ weed/server/filer_server.go | 24 ++++++++++++------------ weed/util/log_buffer/log_read.go | 2 +- 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/weed/command/filer.go b/weed/command/filer.go index aa09c3c98..633c25cac 100644 --- a/weed/command/filer.go +++ b/weed/command/filer.go @@ -139,18 +139,18 @@ func (fo *FilerOptions) startFiler() { Masters: strings.Split(*fo.masters, ","), Collection: *fo.collection, DefaultReplication: *fo.defaultReplicaPlacement, - DisableDirListing: *fo.disableDirListing, - MaxMB: *fo.maxMB, - DirListingLimit: *fo.dirListingLimit, - DataCenter: *fo.dataCenter, - Rack: *fo.rack, - DefaultLevelDbDir: defaultLevelDbDirectory, - DisableHttp: *fo.disableHttp, - Host: *fo.ip, - Port: uint32(*fo.port), - Cipher: *fo.cipher, - SaveToFilerLimit: *fo.saveToFilerLimit, - Filers: peers, + DisableDirListing: *fo.disableDirListing, + MaxMB: *fo.maxMB, + DirListingLimit: *fo.dirListingLimit, + DataCenter: *fo.dataCenter, + Rack: *fo.rack, + DefaultLevelDbDir: defaultLevelDbDirectory, + DisableHttp: *fo.disableHttp, + Host: *fo.ip, + Port: uint32(*fo.port), + Cipher: *fo.cipher, + SaveToFilerLimit: *fo.saveToFilerLimit, + Filers: peers, }) if nfs_err != nil { glog.Fatalf("Filer startup error: %v", nfs_err) diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index 9256fd676..2f03dc289 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -47,18 +47,18 @@ type FilerOption struct { Collection string DefaultReplication string DisableDirListing bool - MaxMB int - DirListingLimit int - DataCenter string - Rack string - DefaultLevelDbDir string - DisableHttp bool - Host string - Port uint32 - recursiveDelete bool - Cipher bool - SaveToFilerLimit int - Filers []string + MaxMB int + DirListingLimit int + DataCenter string + Rack string + DefaultLevelDbDir string + DisableHttp bool + Host string + Port uint32 + recursiveDelete bool + Cipher bool + SaveToFilerLimit int + Filers []string } type FilerServer struct { diff --git a/weed/util/log_buffer/log_read.go b/weed/util/log_buffer/log_read.go index 3c314a135..d6917abfe 100644 --- a/weed/util/log_buffer/log_read.go +++ b/weed/util/log_buffer/log_read.go @@ -13,7 +13,7 @@ import ( ) var ( - ResumeError = fmt.Errorf("resume") + ResumeError = fmt.Errorf("resume") ResumeFromDiskError = fmt.Errorf("resumeFromDisk") ) From 2880160113535549d3ee1a1eb3a1920705907f22 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 12 Jan 2021 11:03:08 -0800 Subject: [PATCH 095/260] close opened file --- weed/sequence/etcd_sequencer.go | 1 + 1 file changed, 1 insertion(+) diff --git a/weed/sequence/etcd_sequencer.go b/weed/sequence/etcd_sequencer.go index 1fc378640..c0d7fbdb6 100644 --- a/weed/sequence/etcd_sequencer.go +++ b/weed/sequence/etcd_sequencer.go @@ -47,6 +47,7 @@ func NewEtcdSequencer(etcdUrls string, metaFolder string) (*EtcdSequencer, error if nil != err { return nil, fmt.Errorf("open sequence file fialed, %v", err) } + defer file.Close() cli, err := client.New(client.Config{ Endpoints: strings.Split(etcdUrls, ","), From 1c7faf1100736a15cac958f96a0f7a433fc17684 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 12 Jan 2021 11:07:14 -0800 Subject: [PATCH 096/260] Revert "close opened file" This reverts commit 2880160113535549d3ee1a1eb3a1920705907f22. --- weed/sequence/etcd_sequencer.go | 1 - 1 file changed, 1 deletion(-) diff --git a/weed/sequence/etcd_sequencer.go b/weed/sequence/etcd_sequencer.go index c0d7fbdb6..1fc378640 100644 --- a/weed/sequence/etcd_sequencer.go +++ b/weed/sequence/etcd_sequencer.go @@ -47,7 +47,6 @@ func NewEtcdSequencer(etcdUrls string, metaFolder string) (*EtcdSequencer, error if nil != err { return nil, fmt.Errorf("open sequence file fialed, %v", err) } - defer file.Close() cli, err := client.New(client.Config{ Endpoints: strings.Split(etcdUrls, ","), From def6d8e5650ff6a7d0c575bcc3779b282febc462 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 12 Jan 2021 12:32:54 -0800 Subject: [PATCH 097/260] fix test --- weed/filer/leveldb3/leveldb3_store_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/weed/filer/leveldb3/leveldb3_store_test.go b/weed/filer/leveldb3/leveldb3_store_test.go index 81f759e78..20d86053f 100644 --- a/weed/filer/leveldb3/leveldb3_store_test.go +++ b/weed/filer/leveldb3/leveldb3_store_test.go @@ -15,7 +15,7 @@ func TestCreateAndFind(t *testing.T) { dir, _ := ioutil.TempDir("", "seaweedfs_filer_test") defer os.RemoveAll(dir) store := &LevelDB3Store{} - store.initialize(dir, 2) + store.initialize(dir) testFiler.SetStore(store) fullpath := util.FullPath("/home/chris/this/is/one/file1.jpg") @@ -69,7 +69,7 @@ func TestEmptyRoot(t *testing.T) { dir, _ := ioutil.TempDir("", "seaweedfs_filer_test2") defer os.RemoveAll(dir) store := &LevelDB3Store{} - store.initialize(dir, 2) + store.initialize(dir) testFiler.SetStore(store) ctx := context.Background() From 5cd9c9e6421215f64cc84090f4384cdec6190b85 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 12 Jan 2021 18:47:38 -0800 Subject: [PATCH 098/260] remove println --- weed/filer/leveldb3/leveldb3_store.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/weed/filer/leveldb3/leveldb3_store.go b/weed/filer/leveldb3/leveldb3_store.go index 31956daf9..a582235f4 100644 --- a/weed/filer/leveldb3/leveldb3_store.go +++ b/weed/filer/leveldb3/leveldb3_store.go @@ -112,8 +112,6 @@ func (store *LevelDB3Store) findDB(fullpath weed_util.FullPath, isForChildren bo shortPath = weed_util.FullPath(bucketAndObjectKey[t:]) } - println("bucket:", bucket, "shortPath", shortPath) - if db, found := store.dbs[bucket]; found { store.dbsLock.RUnlock() return db, bucket, shortPath, nil From 90c507761084821692fcbd25a0a729ce4bbc4806 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 12 Jan 2021 18:48:01 -0800 Subject: [PATCH 099/260] rename "weed watch" to "weed filer.meta.tail" --- weed/command/command.go | 2 +- weed/command/{watch.go => filer_meta_tail.go} | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) rename weed/command/{watch.go => filer_meta_tail.go} (75%) diff --git a/weed/command/command.go b/weed/command/command.go index 8f9cec087..3fa52c922 100644 --- a/weed/command/command.go +++ b/weed/command/command.go @@ -16,6 +16,7 @@ var Commands = []*Command{ cmdExport, cmdFiler, cmdFilerCat, + cmdFilerMetaTail, cmdFilerReplicate, cmdFilerSynchronize, cmdFix, @@ -26,7 +27,6 @@ var Commands = []*Command{ cmdScaffold, cmdServer, cmdShell, - cmdWatch, cmdUpload, cmdVersion, cmdVolume, diff --git a/weed/command/watch.go b/weed/command/filer_meta_tail.go similarity index 75% rename from weed/command/watch.go rename to weed/command/filer_meta_tail.go index 8aad22019..af56c5394 100644 --- a/weed/command/watch.go +++ b/weed/command/filer_meta_tail.go @@ -15,11 +15,11 @@ import ( ) func init() { - cmdWatch.Run = runWatch // break init cycle + cmdFilerMetaTail.Run = runFilerMetaTail // break init cycle } -var cmdWatch = &Command{ - UsageLine: "watch [-filer=localhost:8888] [-target=/]", +var cmdFilerMetaTail = &Command{ + UsageLine: "filer.meta.tail [-filer=localhost:8888] [-target=/]", Short: "see recent changes on a filer", Long: `See recent changes on a filer. @@ -27,13 +27,13 @@ var cmdWatch = &Command{ } var ( - watchFiler = cmdWatch.Flag.String("filer", "localhost:8888", "filer hostname:port") - watchTarget = cmdWatch.Flag.String("pathPrefix", "/", "path to a folder or file, or common prefix for the folders or files on filer") - watchStart = cmdWatch.Flag.Duration("timeAgo", 0, "start time before now. \"300ms\", \"1.5h\" or \"2h45m\". Valid time units are \"ns\", \"us\" (or \"µs\"), \"ms\", \"s\", \"m\", \"h\"") - watchPattern = cmdWatch.Flag.String("pattern", "", "full path or just filename pattern, ex: \"/home/?opher\", \"*.pdf\", see https://golang.org/pkg/path/filepath/#Match ") + watchFiler = cmdFilerMetaTail.Flag.String("filer", "localhost:8888", "filer hostname:port") + watchTarget = cmdFilerMetaTail.Flag.String("pathPrefix", "/", "path to a folder or file, or common prefix for the folders or files on filer") + watchStart = cmdFilerMetaTail.Flag.Duration("timeAgo", 0, "start time before now. \"300ms\", \"1.5h\" or \"2h45m\". Valid time units are \"ns\", \"us\" (or \"µs\"), \"ms\", \"s\", \"m\", \"h\"") + watchPattern = cmdFilerMetaTail.Flag.String("pattern", "", "full path or just filename pattern, ex: \"/home/?opher\", \"*.pdf\", see https://golang.org/pkg/path/filepath/#Match ") ) -func runWatch(cmd *Command, args []string) bool { +func runFilerMetaTail(cmd *Command, args []string) bool { grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client") From c5df2577f5a1922d4e93bbaa1ad6b7a84abe8a37 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 13 Jan 2021 00:31:19 -0800 Subject: [PATCH 100/260] rename --- weed/command/filer_meta_tail.go | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/weed/command/filer_meta_tail.go b/weed/command/filer_meta_tail.go index af56c5394..ce53fae98 100644 --- a/weed/command/filer_meta_tail.go +++ b/weed/command/filer_meta_tail.go @@ -27,10 +27,10 @@ var cmdFilerMetaTail = &Command{ } var ( - watchFiler = cmdFilerMetaTail.Flag.String("filer", "localhost:8888", "filer hostname:port") - watchTarget = cmdFilerMetaTail.Flag.String("pathPrefix", "/", "path to a folder or file, or common prefix for the folders or files on filer") - watchStart = cmdFilerMetaTail.Flag.Duration("timeAgo", 0, "start time before now. \"300ms\", \"1.5h\" or \"2h45m\". Valid time units are \"ns\", \"us\" (or \"µs\"), \"ms\", \"s\", \"m\", \"h\"") - watchPattern = cmdFilerMetaTail.Flag.String("pattern", "", "full path or just filename pattern, ex: \"/home/?opher\", \"*.pdf\", see https://golang.org/pkg/path/filepath/#Match ") + tailFiler = cmdFilerMetaTail.Flag.String("filer", "localhost:8888", "filer hostname:port") + tailTarget = cmdFilerMetaTail.Flag.String("pathPrefix", "/", "path to a folder or file, or common prefix for the folders or files on filer") + tailStart = cmdFilerMetaTail.Flag.Duration("timeAgo", 0, "start time before now. \"300ms\", \"1.5h\" or \"2h45m\". Valid time units are \"ns\", \"us\" (or \"µs\"), \"ms\", \"s\", \"m\", \"h\"") + tailPattern = cmdFilerMetaTail.Flag.String("pattern", "", "full path or just filename pattern, ex: \"/home/?opher\", \"*.pdf\", see https://golang.org/pkg/path/filepath/#Match ") ) func runFilerMetaTail(cmd *Command, args []string) bool { @@ -38,20 +38,20 @@ func runFilerMetaTail(cmd *Command, args []string) bool { grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client") var filterFunc func(dir, fname string) bool - if *watchPattern != "" { - if strings.Contains(*watchPattern, "/") { - println("watch path pattern", *watchPattern) + if *tailPattern != "" { + if strings.Contains(*tailPattern, "/") { + println("watch path pattern", *tailPattern) filterFunc = func(dir, fname string) bool { - matched, err := filepath.Match(*watchPattern, dir+"/"+fname) + matched, err := filepath.Match(*tailPattern, dir+"/"+fname) if err != nil { fmt.Printf("error: %v", err) } return matched } } else { - println("watch file pattern", *watchPattern) + println("watch file pattern", *tailPattern) filterFunc = func(dir, fname string) bool { - matched, err := filepath.Match(*watchPattern, fname) + matched, err := filepath.Match(*tailPattern, fname) if err != nil { fmt.Printf("error: %v", err) } @@ -81,15 +81,15 @@ func runFilerMetaTail(cmd *Command, args []string) bool { return nil } - watchErr := pb.WithFilerClient(*watchFiler, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { + tailErr := pb.WithFilerClient(*tailFiler, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() stream, err := client.SubscribeMetadata(ctx, &filer_pb.SubscribeMetadataRequest{ - ClientName: "watch", - PathPrefix: *watchTarget, - SinceNs: time.Now().Add(-*watchStart).UnixNano(), + ClientName: "tail", + PathPrefix: *tailTarget, + SinceNs: time.Now().Add(-*tailStart).UnixNano(), }) if err != nil { return fmt.Errorf("listen: %v", err) @@ -112,8 +112,8 @@ func runFilerMetaTail(cmd *Command, args []string) bool { } }) - if watchErr != nil { - fmt.Printf("watch %s: %v\n", *watchFiler, watchErr) + if tailErr != nil { + fmt.Printf("tail %s: %v\n", *tailFiler, tailErr) } return true From 3fb2ed90932f608b3fd8e9c29d32663b7402e8eb Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 13 Jan 2021 03:31:37 -0800 Subject: [PATCH 101/260] filer.meta.tail: optionally submit metadata changes to ElasticSearch --- weed/command/filer_meta_tail.go | 81 +++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/weed/command/filer_meta_tail.go b/weed/command/filer_meta_tail.go index ce53fae98..fa0262160 100644 --- a/weed/command/filer_meta_tail.go +++ b/weed/command/filer_meta_tail.go @@ -3,6 +3,8 @@ package command import ( "context" "fmt" + jsoniter "github.com/json-iterator/go" + "github.com/olivere/elastic/v7" "io" "path/filepath" "strings" @@ -31,6 +33,8 @@ var ( tailTarget = cmdFilerMetaTail.Flag.String("pathPrefix", "/", "path to a folder or file, or common prefix for the folders or files on filer") tailStart = cmdFilerMetaTail.Flag.Duration("timeAgo", 0, "start time before now. \"300ms\", \"1.5h\" or \"2h45m\". Valid time units are \"ns\", \"us\" (or \"µs\"), \"ms\", \"s\", \"m\", \"h\"") tailPattern = cmdFilerMetaTail.Flag.String("pattern", "", "full path or just filename pattern, ex: \"/home/?opher\", \"*.pdf\", see https://golang.org/pkg/path/filepath/#Match ") + esServers = cmdFilerMetaTail.Flag.String("es", "", "comma-separated elastic servers http://") + esIndex = cmdFilerMetaTail.Flag.String("es.index", "seaweedfs", "ES index name") ) func runFilerMetaTail(cmd *Command, args []string) bool { @@ -80,6 +84,14 @@ func runFilerMetaTail(cmd *Command, args []string) bool { fmt.Printf("dir:%s %+v\n", resp.Directory, resp.EventNotification) return nil } + if *esServers != "" { + var err error + eachEntryFunc, err = sendToElasticSearchFunc(*esServers, *esIndex) + if err != nil { + fmt.Printf("create elastic search client to %s: %+v\n", *esServers, err) + return false + } + } tailErr := pb.WithFilerClient(*tailFiler, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { @@ -118,3 +130,72 @@ func runFilerMetaTail(cmd *Command, args []string) bool { return true } + +type EsDocument struct { + Dir string `json:"dir,omitempty"` + Name string `json:"name,omitempty"` + IsDirectory bool `json:"isDir,omitempty"` + Size uint64 `json:"size,omitempty"` + Uid uint32 `json:"uid,omitempty"` + Gid uint32 `json:"gid,omitempty"` + UserName string `json:"userName,omitempty"` + Collection string `json:"collection,omitempty"` + Crtime int64 `json:"crtime,omitempty"` + Mtime int64 `json:"mtime,omitempty"` + Mime string `json:"mime,omitempty"` +} + +func toEsEntry(event *filer_pb.EventNotification) (*EsDocument, string) { + entry := event.NewEntry + dir, name := event.NewParentPath, entry.Name + id := util.Md5String([]byte(util.NewFullPath(dir, name))) + esEntry := &EsDocument{ + Dir: dir, + Name: name, + IsDirectory: entry.IsDirectory, + Size: entry.Attributes.FileSize, + Uid: entry.Attributes.Uid, + Gid: entry.Attributes.Gid, + UserName: entry.Attributes.UserName, + Collection: entry.Attributes.Collection, + Crtime: entry.Attributes.Crtime, + Mtime: entry.Attributes.Mtime, + Mime: entry.Attributes.Mime, + } + return esEntry, id +} + +func sendToElasticSearchFunc(servers string, esIndex string) (func(resp *filer_pb.SubscribeMetadataResponse) error, error) { + options := []elastic.ClientOptionFunc{} + options = append(options, elastic.SetURL(strings.Split(servers, ",")...)) + options = append(options, elastic.SetSniff(false)) + options = append(options, elastic.SetHealthcheck(false)) + client, err := elastic.NewClient(options...) + if err != nil { + return nil, err + } + return func(resp *filer_pb.SubscribeMetadataResponse) error { + event := resp.EventNotification + if event.OldEntry != nil && + (event.NewEntry == nil || resp.Directory != event.NewParentPath || event.OldEntry.Name != event.NewEntry.Name) { + // delete or not update the same file + dir, name := resp.Directory, event.OldEntry.Name + id := util.Md5String([]byte(util.NewFullPath(dir, name))) + println("delete", id) + _, err := client.Delete().Index(esIndex).Id(id).Do(context.Background()) + return err + } + if event.NewEntry != nil { + // add a new file or update the same file + esEntry, id := toEsEntry(event) + value, err := jsoniter.Marshal(esEntry) + if err != nil { + return err + } + println(string(value)) + _, err = client.Index().Index(esIndex).Id(id).BodyJson(string(value)).Do(context.Background()) + return err + } + return nil + }, nil +} From 8b553312478a7730218846c7fc9d6600be371fc0 Mon Sep 17 00:00:00 2001 From: LazyDBA247-Anyvision Date: Wed, 13 Jan 2021 22:11:50 +0200 Subject: [PATCH 102/260] appVersion start publishing and using appVersion as the image tag, so its clear that the appVersion of the chart and containers are the same. --- k8s/seaweedfs/Chart.yaml | 3 ++- k8s/seaweedfs/templates/_helpers.tpl | 14 +++++++------- k8s/seaweedfs/values.yaml | 2 +- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/k8s/seaweedfs/Chart.yaml b/k8s/seaweedfs/Chart.yaml index ea3317e61..a734d3f9e 100644 --- a/k8s/seaweedfs/Chart.yaml +++ b/k8s/seaweedfs/Chart.yaml @@ -1,4 +1,5 @@ apiVersion: v1 description: SeaweedFS name: seaweedfs -version: 2.14 \ No newline at end of file +appVersion: "2.20" +version: 2.20 diff --git a/k8s/seaweedfs/templates/_helpers.tpl b/k8s/seaweedfs/templates/_helpers.tpl index 44d480e66..f6c4fa570 100644 --- a/k8s/seaweedfs/templates/_helpers.tpl +++ b/k8s/seaweedfs/templates/_helpers.tpl @@ -52,12 +52,12 @@ Inject extra environment vars in the format key:value, if populated {{- $registryName := default .Values.image.registry .Values.global.localRegistry | toString -}} {{- $repositoryName := .Values.image.repository | toString -}} {{- $name := .Values.global.imageName | toString -}} -{{- $tag := .Values.global.imageTag | toString -}} +{{- $tag := .Chart.AppVersion | toString -}} {{- printf "%s%s%s:%s" $registryName $repositoryName $name $tag -}} {{- end -}} {{- end -}} -{{/* Return the proper postgresqlSchema image */}} +{{/* Return the proper dbSchema image */}} {{- define "filer.dbSchema.image" -}} {{- if .Values.filer.dbSchema.imageOverride -}} {{- $imageOverride := .Values.filer.dbSchema.imageOverride -}} @@ -80,7 +80,7 @@ Inject extra environment vars in the format key:value, if populated {{- $registryName := default .Values.image.registry .Values.global.localRegistry | toString -}} {{- $repositoryName := .Values.image.repository | toString -}} {{- $name := .Values.global.imageName | toString -}} -{{- $tag := .Values.global.imageTag | toString -}} +{{- $tag := .Chart.AppVersion | toString -}} {{- printf "%s%s%s:%s" $registryName $repositoryName $name $tag -}} {{- end -}} {{- end -}} @@ -94,7 +94,7 @@ Inject extra environment vars in the format key:value, if populated {{- $registryName := default .Values.image.registry .Values.global.localRegistry | toString -}} {{- $repositoryName := .Values.image.repository | toString -}} {{- $name := .Values.global.imageName | toString -}} -{{- $tag := .Values.global.imageTag | toString -}} +{{- $tag := .Chart.AppVersion | toString -}} {{- printf "%s%s%s:%s" $registryName $repositoryName $name $tag -}} {{- end -}} {{- end -}} @@ -108,7 +108,7 @@ Inject extra environment vars in the format key:value, if populated {{- $registryName := default .Values.image.registry .Values.global.localRegistry | toString -}} {{- $repositoryName := .Values.image.repository | toString -}} {{- $name := .Values.global.imageName | toString -}} -{{- $tag := .Values.global.imageTag | toString -}} +{{- $tag := .Chart.AppVersion | toString -}} {{- printf "%s%s%s:%s" $registryName $repositoryName $name $tag -}} {{- end -}} {{- end -}} @@ -122,7 +122,7 @@ Inject extra environment vars in the format key:value, if populated {{- $registryName := default .Values.image.registry .Values.global.localRegistry | toString -}} {{- $repositoryName := .Values.image.repository | toString -}} {{- $name := .Values.global.imageName | toString -}} -{{- $tag := .Values.global.imageTag | toString -}} +{{- $tag := .Chart.AppVersion | toString -}} {{- printf "%s%s%s:%s" $registryName $repositoryName $name $tag -}} {{- end -}} -{{- end -}} \ No newline at end of file +{{- end -}} diff --git a/k8s/seaweedfs/values.yaml b/k8s/seaweedfs/values.yaml index 9d621a496..1ae7be731 100644 --- a/k8s/seaweedfs/values.yaml +++ b/k8s/seaweedfs/values.yaml @@ -4,7 +4,7 @@ global: registry: "" repository: "" imageName: chrislusf/seaweedfs - imageTag: "2.19" + # imageTag: "2.20" - started using {.Chart.appVersion} imagePullPolicy: IfNotPresent imagePullSecrets: imagepullsecret restartPolicy: Always From 8491fbf1c7e435016cc8eadf8079763c690c3fbd Mon Sep 17 00:00:00 2001 From: LazyDBA247-Anyvision Date: Wed, 13 Jan 2021 22:13:07 +0200 Subject: [PATCH 103/260] fix cronjob reschedule avoid errors of "more than 100 jobs skipped/failed" and stopping of job by k8s --- k8s/seaweedfs/templates/cronjob.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/k8s/seaweedfs/templates/cronjob.yaml b/k8s/seaweedfs/templates/cronjob.yaml index ad4406d4f..6f4ed8c70 100644 --- a/k8s/seaweedfs/templates/cronjob.yaml +++ b/k8s/seaweedfs/templates/cronjob.yaml @@ -6,6 +6,7 @@ metadata: name: {{ include "seaweedfs.fullname" . }}-cronjob spec: schedule: "{{ .Values.cronjob.schedule }}" + startingDeadlineSeconds: 200 concurrencyPolicy: Forbid failedJobsHistoryLimit: 2 successfulJobsHistoryLimit: 2 @@ -51,4 +52,4 @@ spec: -filer {{ template "seaweedfs.name" . }}-filer.{{ .Release.Namespace }}.svc:{{ .Values.filer.port }} {{- end }} {{- end }} -{{- end }} \ No newline at end of file +{{- end }} From cfc2033bf84dbff131c4e7afceae8b51a5cc88e9 Mon Sep 17 00:00:00 2001 From: LazyDBA247-Anyvision Date: Wed, 13 Jan 2021 22:15:47 +0200 Subject: [PATCH 104/260] shell.toml enable running shell from each pod with default filer & master config --- k8s/seaweedfs/templates/filer-statefulset.yaml | 6 ++++++ k8s/seaweedfs/templates/master-statefulset.yaml | 6 ++++++ k8s/seaweedfs/templates/s3-deployment.yaml | 6 ++++++ k8s/seaweedfs/templates/volume-statefulset.yaml | 6 ++++++ k8s/seaweedfs/values.yaml | 4 ++++ 5 files changed, 28 insertions(+) diff --git a/k8s/seaweedfs/templates/filer-statefulset.yaml b/k8s/seaweedfs/templates/filer-statefulset.yaml index a4f38830f..f54d7d455 100644 --- a/k8s/seaweedfs/templates/filer-statefulset.yaml +++ b/k8s/seaweedfs/templates/filer-statefulset.yaml @@ -87,6 +87,12 @@ spec: value: {{ $value | quote }} {{- end }} {{- end }} + {{- if .Values.global.extraEnvironmentVars }} + {{- range $key, $value := .Values.global.extraEnvironmentVars }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} + {{- end }} command: - "/bin/sh" - "-ec" diff --git a/k8s/seaweedfs/templates/master-statefulset.yaml b/k8s/seaweedfs/templates/master-statefulset.yaml index fe90f3d81..48a42f155 100644 --- a/k8s/seaweedfs/templates/master-statefulset.yaml +++ b/k8s/seaweedfs/templates/master-statefulset.yaml @@ -76,6 +76,12 @@ spec: value: {{ $value | quote }} {{- end }} {{- end }} + {{- if .Values.global.extraEnvironmentVars }} + {{- range $key, $value := .Values.global.extraEnvironmentVars }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} + {{- end }} command: - "/bin/sh" - "-ec" diff --git a/k8s/seaweedfs/templates/s3-deployment.yaml b/k8s/seaweedfs/templates/s3-deployment.yaml index f7244ad76..82644bde7 100644 --- a/k8s/seaweedfs/templates/s3-deployment.yaml +++ b/k8s/seaweedfs/templates/s3-deployment.yaml @@ -59,6 +59,12 @@ spec: fieldPath: metadata.namespace - name: SEAWEEDFS_FULLNAME value: "{{ template "seaweedfs.name" . }}" + {{- if .Values.global.extraEnvironmentVars }} + {{- range $key, $value := .Values.global.extraEnvironmentVars }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} + {{- end }} command: - "/bin/sh" - "-ec" diff --git a/k8s/seaweedfs/templates/volume-statefulset.yaml b/k8s/seaweedfs/templates/volume-statefulset.yaml index 6c519f374..4dcbf634d 100644 --- a/k8s/seaweedfs/templates/volume-statefulset.yaml +++ b/k8s/seaweedfs/templates/volume-statefulset.yaml @@ -64,6 +64,12 @@ spec: fieldPath: status.hostIP - name: SEAWEEDFS_FULLNAME value: "{{ template "seaweedfs.name" . }}" + {{- if .Values.global.extraEnvironmentVars }} + {{- range $key, $value := .Values.global.extraEnvironmentVars }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} + {{- end }} command: - "/bin/sh" - "-ec" diff --git a/k8s/seaweedfs/values.yaml b/k8s/seaweedfs/values.yaml index 1ae7be731..3737a7bd9 100644 --- a/k8s/seaweedfs/values.yaml +++ b/k8s/seaweedfs/values.yaml @@ -21,6 +21,10 @@ global: # Y number of replica in other racks in the same data center # Z number of replica in other servers in the same rack replicationPlacment: "001" + extraEnvironmentVars: + WEED_CLUSTER_DEFAULT: "sw" + WEED_CLUSTER_SW_MASTER: "seaweedfs-master:9333" + WEED_CLUSTER_SW_FILER: "seaweedfs-filer-client:8888" image: registry: "" From 5e62f77681c77169111051157230181ef19fba95 Mon Sep 17 00:00:00 2001 From: LazyDBA247-Anyvision Date: Wed, 13 Jan 2021 22:16:39 +0200 Subject: [PATCH 105/260] dir.idx volume config enable setting dir.idx for volume server --- k8s/seaweedfs/templates/volume-statefulset.yaml | 3 +++ k8s/seaweedfs/values.yaml | 2 ++ 2 files changed, 5 insertions(+) diff --git a/k8s/seaweedfs/templates/volume-statefulset.yaml b/k8s/seaweedfs/templates/volume-statefulset.yaml index 4dcbf634d..fff48095b 100644 --- a/k8s/seaweedfs/templates/volume-statefulset.yaml +++ b/k8s/seaweedfs/templates/volume-statefulset.yaml @@ -86,6 +86,9 @@ spec: -metricsPort {{ .Values.volume.metricsPort }} \ {{- end }} -dir={{ .Values.volume.dir }} \ + {{- if .Values.volume.dir_idx }} + -dir.idx={{ .Values.volume.dir_idx }} \ + {{- end }} -max={{ .Values.volume.maxVolumes }} \ {{- if .Values.volume.rack }} -rack={{ .Values.volume.rack }} \ diff --git a/k8s/seaweedfs/values.yaml b/k8s/seaweedfs/values.yaml index 3737a7bd9..d2b4b1956 100644 --- a/k8s/seaweedfs/values.yaml +++ b/k8s/seaweedfs/values.yaml @@ -144,6 +144,8 @@ volume: # Directories to store data files. dir[,dir]... (default "/tmp") dir: "/data" + # Directories to store index files. dir[,dir]... (default "/tmp") + dir_idx: null # Maximum numbers of volumes, count[,count]... # If set to zero on non-windows OS, the limit will be auto configured. (default "7") From 9ca1787e7ec19d29d9bea518e365f2bb226e1873 Mon Sep 17 00:00:00 2001 From: LazyDBA247-Anyvision Date: Wed, 13 Jan 2021 22:17:34 +0200 Subject: [PATCH 106/260] s3 allowEmptyFolder enable setting allowEmptyFolder for s3 server --- k8s/seaweedfs/templates/s3-deployment.yaml | 6 ++++-- k8s/seaweedfs/values.yaml | 2 ++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/k8s/seaweedfs/templates/s3-deployment.yaml b/k8s/seaweedfs/templates/s3-deployment.yaml index 82644bde7..794d4da30 100644 --- a/k8s/seaweedfs/templates/s3-deployment.yaml +++ b/k8s/seaweedfs/templates/s3-deployment.yaml @@ -87,8 +87,10 @@ spec: {{- if .Values.s3.domainName }} -domainName={{ .Values.s3.domainName }} \ {{- end }} - -filer={{ template "seaweedfs.name" . }}-filer:{{ .Values.filer.port }} - {{- if or (.Values.global.enableSecurity) (.Values.s3.extraVolumeMounts) }} + {{- if .Values.s3.allowEmptyFolder }} + -allowEmptyFolder={{ .Values.s3.allowEmptyFolder }} \ + {{- end }} + -filer={{ template "seaweedfs.name" . }}-filer-client:{{ .Values.filer.port }} volumeMounts: {{- if .Values.global.enableSecurity }} - name: security-config diff --git a/k8s/seaweedfs/values.yaml b/k8s/seaweedfs/values.yaml index d2b4b1956..632bbd440 100644 --- a/k8s/seaweedfs/values.yaml +++ b/k8s/seaweedfs/values.yaml @@ -319,6 +319,8 @@ s3: port: 8333 metricsPort: 9327 loggingOverrideLevel: null + #allow empty folders + allowEmptyFolder: true # Suffix of the host name, {bucket}.{domainName} domainName: "" From e57e6d667702680d19eba6c5a282aafdfbb7d91d Mon Sep 17 00:00:00 2001 From: LazyDBA247-Anyvision Date: Wed, 13 Jan 2021 22:18:24 +0200 Subject: [PATCH 107/260] s3 logs set logs to hostpath same as other pods --- k8s/seaweedfs/templates/s3-deployment.yaml | 10 +++++++++- k8s/seaweedfs/values.yaml | 5 +++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/k8s/seaweedfs/templates/s3-deployment.yaml b/k8s/seaweedfs/templates/s3-deployment.yaml index 794d4da30..46bf91d85 100644 --- a/k8s/seaweedfs/templates/s3-deployment.yaml +++ b/k8s/seaweedfs/templates/s3-deployment.yaml @@ -69,7 +69,7 @@ spec: - "/bin/sh" - "-ec" - | - exec /usr/bin/weed \ + exec /usr/bin/weed -logdir=/logs \ {{- if .Values.s3.loggingOverrideLevel }} -v={{ .Values.s3.loggingOverrideLevel }} \ {{- else }} @@ -92,6 +92,8 @@ spec: {{- end }} -filer={{ template "seaweedfs.name" . }}-filer-client:{{ .Values.filer.port }} volumeMounts: + - name: logs + mountPath: "/logs/" {{- if .Values.global.enableSecurity }} - name: security-config readOnly: true @@ -143,6 +145,12 @@ spec: {{ tpl .Values.s3.resources . | nindent 12 | trim }} {{- end }} volumes: + {{- if eq .Values.s3.logs.type "hostPath" }} + - name: logs + hostPath: + path: /storage/logs/seaweedfs/s3 + type: DirectoryOrCreate + {{- end }} {{- if .Values.global.enableSecurity }} - name: security-config configMap: diff --git a/k8s/seaweedfs/values.yaml b/k8s/seaweedfs/values.yaml index 632bbd440..cdb9437ee 100644 --- a/k8s/seaweedfs/values.yaml +++ b/k8s/seaweedfs/values.yaml @@ -351,6 +351,11 @@ s3: # ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/ priorityClassName: "" + logs: + type: "hostPath" + size: "" + storageClass: "" + cronjob: enabled: false schedule: "*/7 * * * *" From c1ed647dafae94b1e784c09f9b595105ed69a1f1 Mon Sep 17 00:00:00 2001 From: LazyDBA247-Anyvision Date: Wed, 13 Jan 2021 22:23:44 +0200 Subject: [PATCH 108/260] increase timeout reduce unwanted k8s pod restarts on slow response --- k8s/seaweedfs/templates/filer-statefulset.yaml | 4 ++-- k8s/seaweedfs/templates/master-statefulset.yaml | 4 ++-- k8s/seaweedfs/templates/s3-deployment.yaml | 5 ++--- k8s/seaweedfs/templates/volume-statefulset.yaml | 4 ++-- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/k8s/seaweedfs/templates/filer-statefulset.yaml b/k8s/seaweedfs/templates/filer-statefulset.yaml index f54d7d455..023b0d5b9 100644 --- a/k8s/seaweedfs/templates/filer-statefulset.yaml +++ b/k8s/seaweedfs/templates/filer-statefulset.yaml @@ -178,7 +178,7 @@ spec: periodSeconds: 15 successThreshold: 1 failureThreshold: 100 - timeoutSeconds: 3 + timeoutSeconds: 10 livenessProbe: httpGet: path: / @@ -188,7 +188,7 @@ spec: periodSeconds: 30 successThreshold: 1 failureThreshold: 5 - timeoutSeconds: 3 + timeoutSeconds: 10 {{- if .Values.filer.resources }} resources: {{ tpl .Values.filer.resources . | nindent 12 | trim }} diff --git a/k8s/seaweedfs/templates/master-statefulset.yaml b/k8s/seaweedfs/templates/master-statefulset.yaml index 48a42f155..e5a7a537a 100644 --- a/k8s/seaweedfs/templates/master-statefulset.yaml +++ b/k8s/seaweedfs/templates/master-statefulset.yaml @@ -163,7 +163,7 @@ spec: periodSeconds: 45 successThreshold: 2 failureThreshold: 100 - timeoutSeconds: 5 + timeoutSeconds: 10 livenessProbe: httpGet: path: /cluster/status @@ -173,7 +173,7 @@ spec: periodSeconds: 30 successThreshold: 1 failureThreshold: 4 - timeoutSeconds: 5 + timeoutSeconds: 10 {{- if .Values.master.resources }} resources: {{ tpl .Values.master.resources . | nindent 12 | trim }} diff --git a/k8s/seaweedfs/templates/s3-deployment.yaml b/k8s/seaweedfs/templates/s3-deployment.yaml index 46bf91d85..ed01758b8 100644 --- a/k8s/seaweedfs/templates/s3-deployment.yaml +++ b/k8s/seaweedfs/templates/s3-deployment.yaml @@ -116,7 +116,6 @@ spec: mountPath: /usr/local/share/ca-certificates/client/ {{- end }} {{ tpl .Values.s3.extraVolumeMounts . | nindent 12 | trim }} - {{- end }} ports: - containerPort: {{ .Values.s3.port }} name: swfs-s3 @@ -129,7 +128,7 @@ spec: periodSeconds: 15 successThreshold: 1 failureThreshold: 100 - timeoutSeconds: 3 + timeoutSeconds: 10 livenessProbe: httpGet: path: / @@ -139,7 +138,7 @@ spec: periodSeconds: 60 successThreshold: 1 failureThreshold: 20 - timeoutSeconds: 3 + timeoutSeconds: 10 {{- if .Values.s3.resources }} resources: {{ tpl .Values.s3.resources . | nindent 12 | trim }} diff --git a/k8s/seaweedfs/templates/volume-statefulset.yaml b/k8s/seaweedfs/templates/volume-statefulset.yaml index fff48095b..f9e55e0d3 100644 --- a/k8s/seaweedfs/templates/volume-statefulset.yaml +++ b/k8s/seaweedfs/templates/volume-statefulset.yaml @@ -158,7 +158,7 @@ spec: periodSeconds: 90 successThreshold: 1 failureThreshold: 100 - timeoutSeconds: 5 + timeoutSeconds: 30 livenessProbe: httpGet: path: /status @@ -168,7 +168,7 @@ spec: periodSeconds: 90 successThreshold: 1 failureThreshold: 4 - timeoutSeconds: 5 + timeoutSeconds: 30 {{- if .Values.volume.resources }} resources: {{ tpl .Values.volume.resources . | nindent 12 | trim }} From d595b35ca7cf16be7024a7d3c971ac775b0ee64c Mon Sep 17 00:00:00 2001 From: LazyDBA247-Anyvision Date: Wed, 13 Jan 2021 22:26:50 +0200 Subject: [PATCH 109/260] filer peers & client service in order to start several filers with peers setting, they need to be discoverable by dns name, we need a service with - tolerate-unready-endpoints: "true" - publishNotReadyAddresses: true so the filer will start and "See" each other... added "client" service which expose only ready and line filer changed s3 config to use the new service --- .../templates/filer-service-client.yaml | 31 +++++++++++++++++++ k8s/seaweedfs/templates/filer-service.yaml | 11 +++---- .../templates/filer-statefulset.yaml | 2 +- 3 files changed, 36 insertions(+), 8 deletions(-) create mode 100644 k8s/seaweedfs/templates/filer-service-client.yaml diff --git a/k8s/seaweedfs/templates/filer-service-client.yaml b/k8s/seaweedfs/templates/filer-service-client.yaml new file mode 100644 index 000000000..65568418f --- /dev/null +++ b/k8s/seaweedfs/templates/filer-service-client.yaml @@ -0,0 +1,31 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "seaweedfs.name" . }}-filer-client + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "seaweedfs.name" . }} + component: filer +{{- if .Values.filer.metricsPort }} + monitoring: "true" +{{- end }} +spec: + clusterIP: None + ports: + - name: "swfs-filer" + port: {{ .Values.filer.port }} + targetPort: {{ .Values.filer.port }} + protocol: TCP + - name: "swfs-filer-grpc" + port: {{ .Values.filer.grpcPort }} + targetPort: {{ .Values.filer.grpcPort }} + protocol: TCP +{{- if .Values.filer.metricsPort }} + - name: "metrics" + port: {{ .Values.filer.metricsPort }} + targetPort: {{ .Values.filer.metricsPort }} + protocol: TCP +{{- end }} + selector: + app: {{ template "seaweedfs.name" . }} + component: filer \ No newline at end of file diff --git a/k8s/seaweedfs/templates/filer-service.yaml b/k8s/seaweedfs/templates/filer-service.yaml index debc31f6c..45035fc27 100644 --- a/k8s/seaweedfs/templates/filer-service.yaml +++ b/k8s/seaweedfs/templates/filer-service.yaml @@ -1,6 +1,8 @@ apiVersion: v1 kind: Service metadata: + annotations: + service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" name: {{ template "seaweedfs.name" . }}-filer namespace: {{ .Release.Namespace }} labels: @@ -8,6 +10,7 @@ metadata: component: filer spec: clusterIP: None + publishNotReadyAddresses: true ports: - name: "swfs-filer" port: {{ .Values.filer.port }} @@ -17,12 +20,6 @@ spec: port: {{ .Values.filer.grpcPort }} targetPort: {{ .Values.filer.grpcPort }} protocol: TCP -{{- if .Values.filer.metricsPort }} - - name: "swfs-filer-metrics" - port: {{ .Values.filer.metricsPort }} - targetPort: {{ .Values.filer.metricsPort }} - protocol: TCP -{{- end }} selector: app: {{ template "seaweedfs.name" . }} - component: filer \ No newline at end of file + component: filer diff --git a/k8s/seaweedfs/templates/filer-statefulset.yaml b/k8s/seaweedfs/templates/filer-statefulset.yaml index 023b0d5b9..d284e9992 100644 --- a/k8s/seaweedfs/templates/filer-statefulset.yaml +++ b/k8s/seaweedfs/templates/filer-statefulset.yaml @@ -134,7 +134,7 @@ spec: {{- end }} -ip=${POD_IP} \ {{- if gt (.Values.filer.replicas | int) 1 }} - -peers={{ range $index := until (.Values.filer.replicas | int) }}${SEAWEEDFS_FULLNAME}-filer-{{ $index }}.${SEAWEEDFS_FULLNAME}-filer:{{ $.Values.filer.port }}{{ if lt $index (sub ($.Values.filer.replicas | int) 1) }},{{ end }}{{ end }} + -peers=$(echo -n "{{ range $index := until (.Values.filer.replicas | int) }}${SEAWEEDFS_FULLNAME}-filer-{{ $index }}.${SEAWEEDFS_FULLNAME}-filer:{{ $.Values.filer.port }}{{ if lt $index (sub ($.Values.filer.replicas | int) 1) }},{{ end }}{{ end }}" | sed "s/$HOSTNAME.${SEAWEEDFS_FULLNAME}-filer:{{ $.Values.filer.port }}//" | sed 's/,$//; 's/^,//'; s/,,/,/;' ) \ {{- end }} -master={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }} {{- if or (.Values.global.enableSecurity) (.Values.filer.extraVolumeMounts) }} From 95982e0856dcde5cd7542532367c436503d39a7a Mon Sep 17 00:00:00 2001 From: LazyDBA247-Anyvision Date: Wed, 13 Jan 2021 22:27:27 +0200 Subject: [PATCH 110/260] correct master service should have also: publishNotReadyAddresses: true --- k8s/seaweedfs/templates/master-service.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/k8s/seaweedfs/templates/master-service.yaml b/k8s/seaweedfs/templates/master-service.yaml index f7603bd91..0ce467538 100644 --- a/k8s/seaweedfs/templates/master-service.yaml +++ b/k8s/seaweedfs/templates/master-service.yaml @@ -10,6 +10,7 @@ metadata: service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" spec: clusterIP: None + publishNotReadyAddresses: true ports: - name: "swfs-master" port: {{ .Values.master.port }} @@ -21,4 +22,4 @@ spec: protocol: TCP selector: app: {{ template "seaweedfs.name" . }} - component: master \ No newline at end of file + component: master From 9d1d6f7adff74665ecd4ca5a500a024a1af6e10a Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 13 Jan 2021 13:03:21 -0800 Subject: [PATCH 111/260] Update README.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 92979a174..d23df4224 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ Table of Contents * Download the latest binary from https://github.com/chrislusf/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe` * Run `weed server -dir=/some/data/dir -s3` to start one master, one volume server, one filer, and one S3 gateway. -Also, to increase capacity, just add more volume servers by `weed volume -dir="/some/data/dir2" -mserver=":9333" -port=8081` locally, or a different machine, or thoudsands of machines. That is it! +Also, to increase capacity, just add more volume servers by running `weed volume -dir="/some/data/dir2" -mserver=":9333" -port=8081` locally, or on a different machine, or on thoudsands of machines. That is it! ## Introduction ## @@ -76,9 +76,9 @@ SeaweedFS is a simple and highly scalable distributed file system. There are two 1. to store billions of files! 2. to serve the files fast! -SeaweedFS started as an Object Store to handle small files efficiently. Instead of managing all file metadata in a central master, the central master only manages file volumes, and it lets these volume servers manage files and their metadata. This relieves concurrency pressure from the central master and spreads file metadata into volume servers, allowing faster file access (O(1), usually just one disk read operation). +SeaweedFS started as an Object Store to handle small files efficiently. Instead of managing all file metadata in a central master, the central master only manages volumes on volume servers, and these volume servers manage files and their metadata. This relieves concurrency pressure from the central master and spreads file metadata into volume servers, allowing faster file access (O(1), usually just one disk read operation). -SeaweedFS can transparently integrate with the cloud. With hot data on local cluster, and warm data on the cloud with O(1) access time, SeaweedFS can achieve both fast local access time and elastic cloud storage capacity, without any client side changes. +SeaweedFS can transparently integrate with the cloud. With hot data on local cluster, and warm data on the cloud with O(1) access time, SeaweedFS can achieve both fast local access time and elastic cloud storage capacity. What's more, the cloud storage access API cost is minimized. Faster and cheaper than direct cloud storage. Signup for future managed Seaweed cloud storage offering at "seaweedfilesystem at gmail dot com". There is only 40 bytes of disk storage overhead for each file's metadata. It is so simple with O(1) disk reads that you are welcome to challenge the performance with your actual use cases. From 3c5b0ed8e37ae503f355cfad92d0059a44007cc7 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 13 Jan 2021 13:04:47 -0800 Subject: [PATCH 112/260] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d23df4224..f55f35826 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,7 @@ SeaweedFS is a simple and highly scalable distributed file system. There are two SeaweedFS started as an Object Store to handle small files efficiently. Instead of managing all file metadata in a central master, the central master only manages volumes on volume servers, and these volume servers manage files and their metadata. This relieves concurrency pressure from the central master and spreads file metadata into volume servers, allowing faster file access (O(1), usually just one disk read operation). -SeaweedFS can transparently integrate with the cloud. With hot data on local cluster, and warm data on the cloud with O(1) access time, SeaweedFS can achieve both fast local access time and elastic cloud storage capacity. What's more, the cloud storage access API cost is minimized. Faster and cheaper than direct cloud storage. Signup for future managed Seaweed cloud storage offering at "seaweedfilesystem at gmail dot com". +SeaweedFS can transparently integrate with the cloud. With hot data on local cluster, and warm data on the cloud with O(1) access time, SeaweedFS can achieve both fast local access time and elastic cloud storage capacity. What's more, the cloud storage access API cost is minimized. Faster and Cheaper than direct cloud storage! There is only 40 bytes of disk storage overhead for each file's metadata. It is so simple with O(1) disk reads that you are welcome to challenge the performance with your actual use cases. From 819a85f59aad5f8e1e8a46f9bba705cb31db7af5 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 13 Jan 2021 13:14:48 -0800 Subject: [PATCH 113/260] avoid viper concurrent access --- weed/util/config.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/weed/util/config.go b/weed/util/config.go index 54edf5a3c..e746b0bd1 100644 --- a/weed/util/config.go +++ b/weed/util/config.go @@ -58,6 +58,30 @@ func (vp *ViperProxy) SetDefault(key string, value interface{}) { vp.Viper.SetDefault(key, value) } +func (vp *ViperProxy) GetString(key string) string { + vp.Lock() + defer vp.Unlock() + return vp.Viper.GetString(key) +} + +func (vp *ViperProxy) GetBool(key string) bool { + vp.Lock() + defer vp.Unlock() + return vp.Viper.GetBool(key) +} + +func (vp *ViperProxy) GetInt(key string) int { + vp.Lock() + defer vp.Unlock() + return vp.Viper.GetInt(key) +} + +func (vp *ViperProxy) GetStringSlice(key string) []string { + vp.Lock() + defer vp.Unlock() + return vp.Viper.GetStringSlice(key) +} + func GetViper() *ViperProxy { v := &ViperProxy{} v.Viper = viper.GetViper() From e2c7e3fe6d44c183731bf67eb5eb27f4acfe802c Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 13 Jan 2021 13:20:33 -0800 Subject: [PATCH 114/260] better locks --- weed/util/config.go | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/weed/util/config.go b/weed/util/config.go index e746b0bd1..2af1ad09e 100644 --- a/weed/util/config.go +++ b/weed/util/config.go @@ -52,6 +52,10 @@ type ViperProxy struct { sync.Mutex } +var ( + vp = &ViperProxy{} +) + func (vp *ViperProxy) SetDefault(key string, value interface{}) { vp.Lock() defer vp.Unlock() @@ -83,10 +87,15 @@ func (vp *ViperProxy) GetStringSlice(key string) []string { } func GetViper() *ViperProxy { - v := &ViperProxy{} - v.Viper = viper.GetViper() - v.AutomaticEnv() - v.SetEnvPrefix("weed") - v.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) - return v + vp.Lock() + defer vp.Unlock() + + if vp.Viper == nil { + vp.Viper = viper.GetViper() + vp.AutomaticEnv() + vp.SetEnvPrefix("weed") + vp.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) + } + + return vp } From ca730134538e3afe4d9806acac95ba0a808f518f Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 13 Jan 2021 13:48:48 -0800 Subject: [PATCH 115/260] leveldb3: ensure repeatable adding removing buckets --- weed/filer/leveldb3/leveldb3_store.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/weed/filer/leveldb3/leveldb3_store.go b/weed/filer/leveldb3/leveldb3_store.go index a582235f4..332f1569b 100644 --- a/weed/filer/leveldb3/leveldb3_store.go +++ b/weed/filer/leveldb3/leveldb3_store.go @@ -137,6 +137,18 @@ func (store *LevelDB3Store) findDB(fullpath weed_util.FullPath, isForChildren bo return db, bucket, shortPath, nil } +func (store *LevelDB3Store) closeDB(bucket string) { + + store.dbsLock.Lock() + defer store.dbsLock.Unlock() + + if db, found := store.dbs[bucket]; found { + db.Close() + delete(store.dbs, bucket) + } + +} + func (store *LevelDB3Store) BeginTransaction(ctx context.Context) (context.Context, error) { return ctx, nil } @@ -240,7 +252,7 @@ func (store *LevelDB3Store) DeleteFolderChildren(ctx context.Context, fullpath w } if bucket != DEFAULT && shortPath == "/" { - db.Close() + store.closeDB(bucket) if bucket != "" { // just to make sure os.RemoveAll(store.dir + "/" + bucket) } From f17aa1d06c10c59124b08ca92480e4bba9b324b8 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 13 Jan 2021 13:49:04 -0800 Subject: [PATCH 116/260] skip directory loop if dropping the bucket --- weed/filer/filer_delete_entry.go | 69 +++++++++++++++++--------------- 1 file changed, 36 insertions(+), 33 deletions(-) diff --git a/weed/filer/filer_delete_entry.go b/weed/filer/filer_delete_entry.go index 0d1fd7e47..8dd3cfed0 100644 --- a/weed/filer/filer_delete_entry.go +++ b/weed/filer/filer_delete_entry.go @@ -30,7 +30,7 @@ func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isR // delete the folder children, not including the folder itself var dirChunks []*filer_pb.FileChunk var dirHardLinkIds []HardLinkId - dirChunks, dirHardLinkIds, err = f.doBatchDeleteFolderMetaAndData(ctx, entry, isRecursive, ignoreRecursiveError, shouldDeleteChunks && !isDeleteCollection, isFromOtherCluster, signatures) + dirChunks, dirHardLinkIds, err = f.doBatchDeleteFolderMetaAndData(ctx, entry, isRecursive, ignoreRecursiveError, shouldDeleteChunks && !isDeleteCollection, isDeleteCollection, isFromOtherCluster, signatures) if err != nil { glog.V(0).Infof("delete directory %s: %v", p, err) return fmt.Errorf("delete directory %s: %v", p, err) @@ -63,46 +63,49 @@ func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isR return nil } -func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry, isRecursive, ignoreRecursiveError, shouldDeleteChunks, isFromOtherCluster bool, signatures []int32) (chunks []*filer_pb.FileChunk, hardlinkIds []HardLinkId, err error) { +func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry, isRecursive, ignoreRecursiveError, shouldDeleteChunks, isDeletingBucket, isFromOtherCluster bool, signatures []int32) (chunks []*filer_pb.FileChunk, hardlinkIds []HardLinkId, err error) { lastFileName := "" includeLastFile := false - for { - entries, err := f.ListDirectoryEntries(ctx, entry.FullPath, lastFileName, includeLastFile, PaginationSize, "", "") - if err != nil { - glog.Errorf("list folder %s: %v", entry.FullPath, err) - return nil, nil, fmt.Errorf("list folder %s: %v", entry.FullPath, err) - } - if lastFileName == "" && !isRecursive && len(entries) > 0 { - // only for first iteration in the loop - glog.Errorf("deleting a folder %s has children: %+v ...", entry.FullPath, entries[0].Name()) - return nil, nil, fmt.Errorf("fail to delete non-empty folder: %s", entry.FullPath) - } + if !isDeletingBucket { + for { + entries, err := f.ListDirectoryEntries(ctx, entry.FullPath, lastFileName, includeLastFile, PaginationSize, "", "") + if err != nil { + glog.Errorf("list folder %s: %v", entry.FullPath, err) + return nil, nil, fmt.Errorf("list folder %s: %v", entry.FullPath, err) + } + if lastFileName == "" && !isRecursive && len(entries) > 0 { + // only for first iteration in the loop + glog.Errorf("deleting a folder %s has children: %+v ...", entry.FullPath, entries[0].Name()) + return nil, nil, fmt.Errorf("fail to delete non-empty folder: %s", entry.FullPath) + } - for _, sub := range entries { - lastFileName = sub.Name() - var dirChunks []*filer_pb.FileChunk - var dirHardLinkIds []HardLinkId - if sub.IsDirectory() { - dirChunks, dirHardLinkIds, err = f.doBatchDeleteFolderMetaAndData(ctx, sub, isRecursive, ignoreRecursiveError, shouldDeleteChunks, false, nil) - chunks = append(chunks, dirChunks...) - hardlinkIds = append(hardlinkIds, dirHardLinkIds...) - } else { - f.NotifyUpdateEvent(ctx, sub, nil, shouldDeleteChunks, isFromOtherCluster, nil) - if len(sub.HardLinkId) != 0 { - // hard link chunk data are deleted separately - hardlinkIds = append(hardlinkIds, sub.HardLinkId) + for _, sub := range entries { + lastFileName = sub.Name() + var dirChunks []*filer_pb.FileChunk + var dirHardLinkIds []HardLinkId + if sub.IsDirectory() { + subIsDeletingBucket := f.isBucket(sub) + dirChunks, dirHardLinkIds, err = f.doBatchDeleteFolderMetaAndData(ctx, sub, isRecursive, ignoreRecursiveError, shouldDeleteChunks, subIsDeletingBucket, false, nil) + chunks = append(chunks, dirChunks...) + hardlinkIds = append(hardlinkIds, dirHardLinkIds...) } else { - chunks = append(chunks, sub.Chunks...) + f.NotifyUpdateEvent(ctx, sub, nil, shouldDeleteChunks, isFromOtherCluster, nil) + if len(sub.HardLinkId) != 0 { + // hard link chunk data are deleted separately + hardlinkIds = append(hardlinkIds, sub.HardLinkId) + } else { + chunks = append(chunks, sub.Chunks...) + } + } + if err != nil && !ignoreRecursiveError { + return nil, nil, err } } - if err != nil && !ignoreRecursiveError { - return nil, nil, err - } - } - if len(entries) < PaginationSize { - break + if len(entries) < PaginationSize { + break + } } } From 8eed763b97acc3af6420f7e370c3e61ecc7befe2 Mon Sep 17 00:00:00 2001 From: LazyDBA247-Anyvision Date: Thu, 14 Jan 2021 08:14:21 +0200 Subject: [PATCH 117/260] better mysql connection pool management adding SetConnMaxLifetime configuration (https://golang.org/pkg/database/sql/#DB.SetConnMaxLifetime) to enable refresh of connections. --- weed/command/scaffold.go | 1 + weed/filer/mysql/mysql_store.go | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/weed/command/scaffold.go b/weed/command/scaffold.go index f43a9d143..887c9883f 100644 --- a/weed/command/scaffold.go +++ b/weed/command/scaffold.go @@ -118,6 +118,7 @@ password = "" database = "" # create or use an existing database connection_max_idle = 2 connection_max_open = 100 +connection_max_lifetime_seconds = 0 interpolateParams = false [postgres] # or cockroachdb diff --git a/weed/filer/mysql/mysql_store.go b/weed/filer/mysql/mysql_store.go index 5bc132980..62b9b287a 100644 --- a/weed/filer/mysql/mysql_store.go +++ b/weed/filer/mysql/mysql_store.go @@ -35,12 +35,13 @@ func (store *MysqlStore) Initialize(configuration util.Configuration, prefix str configuration.GetString(prefix+"database"), configuration.GetInt(prefix+"connection_max_idle"), configuration.GetInt(prefix+"connection_max_open"), + configuration.GetInt(prefix+"connection_max_lifetime_seconds"), configuration.GetBool(prefix+"interpolateParams"), ) } -func (store *MysqlStore) initialize(user, password, hostname string, port int, database string, maxIdle, maxOpen int, - interpolateParams bool) (err error) { +func (store *MysqlStore) initialize(user, password, hostname string, port int, database string, maxIdle, maxOpen, + maxLifetimeSeconds int, interpolateParams bool) (err error) { // store.SqlInsert = "INSERT INTO filemeta (dirhash,name,directory,meta) VALUES(?,?,?,?)" store.SqlUpdate = "UPDATE filemeta SET meta=? WHERE dirhash=? AND name=? AND directory=?" @@ -65,6 +66,7 @@ func (store *MysqlStore) initialize(user, password, hostname string, port int, d store.DB.SetMaxIdleConns(maxIdle) store.DB.SetMaxOpenConns(maxOpen) + store.DB.SetConnMaxLifetime(maxLifetimeSeconds*time.Second) if err = store.DB.Ping(); err != nil { return fmt.Errorf("connect to %s error:%v", sqlUrl, err) From f20743bb383782b0071c0cf0bc8088096321adc6 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 13 Jan 2021 22:21:34 -0800 Subject: [PATCH 118/260] remove WIP status --- weed/shell/command_s3_configure.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/shell/command_s3_configure.go b/weed/shell/command_s3_configure.go index 869949a25..ca51ef72f 100644 --- a/weed/shell/command_s3_configure.go +++ b/weed/shell/command_s3_configure.go @@ -25,7 +25,7 @@ func (c *commandS3Configure) Name() string { } func (c *commandS3Configure) Help() string { - return ` configure and apply s3 options for each bucket + return `configure and apply s3 options for each bucket # see the current configuration file content s3.configure From 5581d087021560a4b83cf6434726c00c668abb75 Mon Sep 17 00:00:00 2001 From: LazyDBA247-Anyvision Date: Thu, 14 Jan 2021 09:48:12 +0200 Subject: [PATCH 119/260] K8S mysql connection pool set max lifetime to 10 minutes, keeping connection "fresh" and eliminating mysql/memsql closing old connection (causing filer errors) --- k8s/seaweedfs/values.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/k8s/seaweedfs/values.yaml b/k8s/seaweedfs/values.yaml index cdb9437ee..e08731354 100644 --- a/k8s/seaweedfs/values.yaml +++ b/k8s/seaweedfs/values.yaml @@ -298,8 +298,10 @@ filer: WEED_MYSQL_HOSTNAME: "mysql-db-host" WEED_MYSQL_PORT: "3306" WEED_MYSQL_DATABASE: "sw_database" - WEED_MYSQL_CONNECTION_MAX_IDLE: "10" - WEED_MYSQL_CONNECTION_MAX_OPEN: "150" + WEED_MYSQL_CONNECTION_MAX_IDLE: "5" + WEED_MYSQL_CONNECTION_MAX_OPEN: "75" + # "refresh" connection every 10 minutes, eliminating mysql closing "old" connections + WEED_MYSQL_CONNECTION_MAX_LIFETIME_SECONDS: "600" # enable usage of memsql as filer backend WEED_MYSQL_INTERPOLATEPARAMS: "true" WEED_LEVELDB2_ENABLED: "false" From fd1d8a2a0958e4256ac0bbd77a982c5fc2274546 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 14 Jan 2021 00:06:15 -0800 Subject: [PATCH 120/260] fix compilation --- weed/filer/mysql/mysql_store.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/weed/filer/mysql/mysql_store.go b/weed/filer/mysql/mysql_store.go index 62b9b287a..b9f7f06dd 100644 --- a/weed/filer/mysql/mysql_store.go +++ b/weed/filer/mysql/mysql_store.go @@ -3,6 +3,7 @@ package mysql import ( "database/sql" "fmt" + "time" "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/filer/abstract_sql" @@ -66,7 +67,7 @@ func (store *MysqlStore) initialize(user, password, hostname string, port int, d store.DB.SetMaxIdleConns(maxIdle) store.DB.SetMaxOpenConns(maxOpen) - store.DB.SetConnMaxLifetime(maxLifetimeSeconds*time.Second) + store.DB.SetConnMaxLifetime(time.Duration(maxLifetimeSeconds)*time.Second) if err = store.DB.Ping(); err != nil { return fmt.Errorf("connect to %s error:%v", sqlUrl, err) From 698f58f7c400eae6b8ca416025c8ae44b5b6a134 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 14 Jan 2021 02:59:40 -0800 Subject: [PATCH 121/260] filer, s3: add http status 206 as late as possible fix https://github.com/chrislusf/seaweedfs/issues/1746 --- weed/server/common.go | 9 ++++----- weed/server/filer_server_handlers_read.go | 5 ++++- weed/server/volume_server_handlers_read.go | 5 ++++- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/weed/server/common.go b/weed/server/common.go index 58079032e..cf9547950 100644 --- a/weed/server/common.go +++ b/weed/server/common.go @@ -233,12 +233,12 @@ func adjustHeaderContentDisposition(w http.ResponseWriter, r *http.Request, file } } -func processRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64, mimeType string, writeFn func(writer io.Writer, offset int64, size int64) error) { +func processRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64, mimeType string, writeFn func(writer io.Writer, offset int64, size int64, httpStatusCode int) error) { rangeReq := r.Header.Get("Range") if rangeReq == "" { w.Header().Set("Content-Length", strconv.FormatInt(totalSize, 10)) - if err := writeFn(w, 0, totalSize); err != nil { + if err := writeFn(w, 0, totalSize, 0); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } @@ -277,9 +277,8 @@ func processRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64 ra := ranges[0] w.Header().Set("Content-Length", strconv.FormatInt(ra.length, 10)) w.Header().Set("Content-Range", ra.contentRange(totalSize)) - w.WriteHeader(http.StatusPartialContent) - err = writeFn(w, ra.start, ra.length) + err = writeFn(w, ra.start, ra.length, http.StatusPartialContent) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return @@ -307,7 +306,7 @@ func processRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64 pw.CloseWithError(e) return } - if e = writeFn(part, ra.start, ra.length); e != nil { + if e = writeFn(part, ra.start, ra.length, 0); e != nil { pw.CloseWithError(e) return } diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.go index 5cfbfe86c..9fdc03dea 100644 --- a/weed/server/filer_server_handlers_read.go +++ b/weed/server/filer_server_handlers_read.go @@ -158,8 +158,11 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, } } - processRangeRequest(r, w, totalSize, mimeType, func(writer io.Writer, offset int64, size int64) error { + processRangeRequest(r, w, totalSize, mimeType, func(writer io.Writer, offset int64, size int64, httpStatusCode int) error { if offset+size <= int64(len(entry.Content)) { + if httpStatusCode != 0 { + w.WriteHeader(httpStatusCode) + } _, err := writer.Write(entry.Content[offset : offset+size]) return err } diff --git a/weed/server/volume_server_handlers_read.go b/weed/server/volume_server_handlers_read.go index 1c963b39c..1cd4ee21d 100644 --- a/weed/server/volume_server_handlers_read.go +++ b/weed/server/volume_server_handlers_read.go @@ -261,10 +261,13 @@ func writeResponseContent(filename, mimeType string, rs io.ReadSeeker, w http.Re return nil } - processRangeRequest(r, w, totalSize, mimeType, func(writer io.Writer, offset int64, size int64) error { + processRangeRequest(r, w, totalSize, mimeType, func(writer io.Writer, offset int64, size int64, httpStatusCode int) error { if _, e = rs.Seek(offset, 0); e != nil { return e } + if httpStatusCode != 0 { + w.WriteHeader(httpStatusCode) + } _, e = io.CopyN(writer, rs, size) return e }) From d0826c2f62d45331bb72cb2781a015f92c467cd2 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 14 Jan 2021 19:03:15 -0800 Subject: [PATCH 122/260] mount: avoid showing "topics" folder fix https://github.com/chrislusf/seaweedfs/issues/1325 --- weed/filesys/meta_cache/meta_cache_init.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/weed/filesys/meta_cache/meta_cache_init.go b/weed/filesys/meta_cache/meta_cache_init.go index 4089cea28..1ca3b16d5 100644 --- a/weed/filesys/meta_cache/meta_cache_init.go +++ b/weed/filesys/meta_cache/meta_cache_init.go @@ -19,6 +19,9 @@ func EnsureVisited(mc *MetaCache, client filer_pb.FilerClient, dirPath util.Full util.Retry("ReadDirAllEntries", func() error { err = filer_pb.ReadDirAllEntries(client, dirPath, "", func(pbEntry *filer_pb.Entry, isLast bool) error { entry := filer.FromPbEntry(string(dirPath), pbEntry) + if IsHiddenSystemEntry(string(dirPath), entry.Name()) { + return nil + } if err := mc.doInsertEntry(context.Background(), entry); err != nil { glog.V(0).Infof("read %s: %v", entry.FullPath, err) return err @@ -38,3 +41,7 @@ func EnsureVisited(mc *MetaCache, client filer_pb.FilerClient, dirPath util.Full return }) } + +func IsHiddenSystemEntry(dir, name string) bool { + return dir == "/" && name == "topics" +} From 4aaa97c78995b2fb9df1d22f8bfa7340f20105e6 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 14 Jan 2021 19:56:14 -0800 Subject: [PATCH 123/260] skip not found error in case of race condition --- weed/filer/redis/universal_redis_store.go | 3 +++ weed/filer/redis2/universal_redis_store.go | 3 +++ 2 files changed, 6 insertions(+) diff --git a/weed/filer/redis/universal_redis_store.go b/weed/filer/redis/universal_redis_store.go index a3e09c3a1..fc7ee0399 100644 --- a/weed/filer/redis/universal_redis_store.go +++ b/weed/filer/redis/universal_redis_store.go @@ -171,6 +171,9 @@ func (store *UniversalRedisStore) ListDirectoryEntries(ctx context.Context, full entry, err := store.FindEntry(ctx, path) if err != nil { glog.V(0).Infof("list %s : %v", path, err) + if err == filer_pb.ErrNotFound { + continue + } } else { if entry.TtlSec > 0 { if entry.Attr.Crtime.Add(time.Duration(entry.TtlSec) * time.Second).Before(time.Now()) { diff --git a/weed/filer/redis2/universal_redis_store.go b/weed/filer/redis2/universal_redis_store.go index 6a8d90768..b1cf228e8 100644 --- a/weed/filer/redis2/universal_redis_store.go +++ b/weed/filer/redis2/universal_redis_store.go @@ -175,6 +175,9 @@ func (store *UniversalRedis2Store) ListDirectoryEntries(ctx context.Context, ful entry, err := store.FindEntry(ctx, path) if err != nil { glog.V(0).Infof("list %s : %v", path, err) + if err == filer_pb.ErrNotFound { + continue + } } else { if entry.TtlSec > 0 { if entry.Attr.Crtime.Add(time.Duration(entry.TtlSec) * time.Second).Before(time.Now()) { From e1c7bc66f9a2f613df5eef55d6143f9b2a551cf3 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 14 Jan 2021 22:17:49 -0800 Subject: [PATCH 124/260] change interface --- weed/filer/filerstore.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/weed/filer/filerstore.go b/weed/filer/filerstore.go index f1e6c6c35..a44d9b6d6 100644 --- a/weed/filer/filerstore.go +++ b/weed/filer/filerstore.go @@ -24,8 +24,8 @@ type FilerStore interface { FindEntry(context.Context, util.FullPath) (entry *Entry, err error) DeleteEntry(context.Context, util.FullPath) (err error) DeleteFolderChildren(context.Context, util.FullPath) (err error) - ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int) ([]*Entry, error) - ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) ([]*Entry, error) + ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int) ([]*Entry, bool, error) + ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) ([]*Entry, bool, error) BeginTransaction(ctx context.Context) (context.Context, error) CommitTransaction(ctx context.Context) error From 9a50dbcda08fd49ada647dccb9b755793f3207a5 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 14 Jan 2021 22:18:38 -0800 Subject: [PATCH 125/260] chagned api --- weed/filer/filer.go | 2 +- weed/filesys/meta_cache/meta_cache.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/weed/filer/filer.go b/weed/filer/filer.go index 920d79da5..9a1b7be3a 100644 --- a/weed/filer/filer.go +++ b/weed/filer/filer.go @@ -282,7 +282,7 @@ func (f *Filer) FindEntry(ctx context.Context, p util.FullPath) (entry *Entry, e } func (f *Filer) doListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*Entry, expiredCount int, lastFileName string, err error) { - listedEntries, listErr := f.Store.ListDirectoryPrefixedEntries(ctx, p, startFileName, inclusive, limit, prefix) + listedEntries, _, listErr := f.Store.ListDirectoryPrefixedEntries(ctx, p, startFileName, inclusive, limit, prefix) if listErr != nil { return listedEntries, expiredCount, "", listErr } diff --git a/weed/filesys/meta_cache/meta_cache.go b/weed/filesys/meta_cache/meta_cache.go index 4b282253d..4cb3a4930 100644 --- a/weed/filesys/meta_cache/meta_cache.go +++ b/weed/filesys/meta_cache/meta_cache.go @@ -125,7 +125,7 @@ func (mc *MetaCache) ListDirectoryEntries(ctx context.Context, dirPath util.Full return nil, fmt.Errorf("unsynchronized dir: %v", dirPath) } - entries, err := mc.localStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit) + entries, _, err := mc.localStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit) if err != nil { return nil, err } From d2ece29fff3853dc05f96ac2c9b002877561a051 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 14 Jan 2021 22:21:31 -0800 Subject: [PATCH 126/260] implement redis changes --- weed/filer/redis/universal_redis_store.go | 14 ++++++------- weed/filer/redis2/universal_redis_store.go | 24 +++++++++++++--------- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/weed/filer/redis/universal_redis_store.go b/weed/filer/redis/universal_redis_store.go index fc7ee0399..bc2644cb0 100644 --- a/weed/filer/redis/universal_redis_store.go +++ b/weed/filer/redis/universal_redis_store.go @@ -125,17 +125,16 @@ func (store *UniversalRedisStore) DeleteFolderChildren(ctx context.Context, full return nil } -func (store *UniversalRedisStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer.Entry, err error) { - return nil, filer.ErrUnsupportedListDirectoryPrefixed +func (store *UniversalRedisStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) (entries []*filer.Entry, hasMore bool, err error) { + return nil, false, filer.ErrUnsupportedListDirectoryPrefixed } -func (store *UniversalRedisStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, - limit int) (entries []*filer.Entry, err error) { +func (store *UniversalRedisStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, includeStartFile bool, limit int) (entries []*filer.Entry, hasMore bool, err error) { dirListKey := genDirectoryListKey(string(fullpath)) members, err := store.Client.SMembers(ctx, dirListKey).Result() if err != nil { - return nil, fmt.Errorf("list %s : %v", fullpath, err) + return nil, false, fmt.Errorf("list %s : %v", fullpath, err) } // skip @@ -144,7 +143,7 @@ func (store *UniversalRedisStore) ListDirectoryEntries(ctx context.Context, full for _, m := range members { if strings.Compare(m, startFileName) >= 0 { if m == startFileName { - if inclusive { + if includeStartFile { t = append(t, m) } } else { @@ -163,6 +162,7 @@ func (store *UniversalRedisStore) ListDirectoryEntries(ctx context.Context, full // limit if limit < len(members) { members = members[:limit] + hasMore = true } // fetch entry meta @@ -186,7 +186,7 @@ func (store *UniversalRedisStore) ListDirectoryEntries(ctx context.Context, full } } - return entries, err + return entries, hasMore, err } func genDirectoryListKey(dir string) (dirList string) { diff --git a/weed/filer/redis2/universal_redis_store.go b/weed/filer/redis2/universal_redis_store.go index b1cf228e8..8009b6a45 100644 --- a/weed/filer/redis2/universal_redis_store.go +++ b/weed/filer/redis2/universal_redis_store.go @@ -149,29 +149,33 @@ func (store *UniversalRedis2Store) DeleteFolderChildren(ctx context.Context, ful return nil } -func (store *UniversalRedis2Store) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer.Entry, err error) { - return nil, filer.ErrUnsupportedListDirectoryPrefixed +func (store *UniversalRedis2Store) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) (entries []*filer.Entry, hasMore bool, err error) { + return nil, false, filer.ErrUnsupportedListDirectoryPrefixed } -func (store *UniversalRedis2Store) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, - limit int) (entries []*filer.Entry, err error) { +func (store *UniversalRedis2Store) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int) (entries []*filer.Entry, hasMore bool, err error) { - dirListKey := genDirectoryListKey(string(fullpath)) + dirListKey := genDirectoryListKey(string(dirPath)) start := int64(0) if startFileName != "" { start, _ = store.Client.ZRank(ctx, dirListKey, startFileName).Result() - if !inclusive { + if !includeStartFile { start++ } } - members, err := store.Client.ZRange(ctx, dirListKey, start, start+int64(limit)-1).Result() + members, err := store.Client.ZRange(ctx, dirListKey, start, start+int64(limit)-1+1).Result() if err != nil { - return nil, fmt.Errorf("list %s : %v", fullpath, err) + return nil, false, fmt.Errorf("list %s : %v", dirPath, err) + } + + hasMore = len(members) == limit+1 + if hasMore { + members = members[:len(members)-1] } // fetch entry meta for _, fileName := range members { - path := util.NewFullPath(string(fullpath), fileName) + path := util.NewFullPath(string(dirPath), fileName) entry, err := store.FindEntry(ctx, path) if err != nil { glog.V(0).Infof("list %s : %v", path, err) @@ -190,7 +194,7 @@ func (store *UniversalRedis2Store) ListDirectoryEntries(ctx context.Context, ful } } - return entries, err + return entries, hasMore, err } func genDirectoryListKey(dir string) (dirList string) { From 34a846009db264393a6f4ecbf251a2262e176dcc Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 14 Jan 2021 22:24:26 -0800 Subject: [PATCH 127/260] implement rocksdb changes --- weed/filer/rocksdb/rocksdb_store.go | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/weed/filer/rocksdb/rocksdb_store.go b/weed/filer/rocksdb/rocksdb_store.go index 0793f547c..fd0c96663 100644 --- a/weed/filer/rocksdb/rocksdb_store.go +++ b/weed/filer/rocksdb/rocksdb_store.go @@ -158,7 +158,7 @@ func (store *RocksDBStore) DeleteFolderChildren(ctx context.Context, fullpath we iter := store.db.NewIterator(ro) defer iter.Close() - err = enumerate(iter, directoryPrefix, nil, false, -1, func(key, value []byte) bool { + _, err = enumerate(iter, directoryPrefix, nil, false, -1, func(key, value []byte) bool { batch.Delete(key) return true }) @@ -175,7 +175,7 @@ func (store *RocksDBStore) DeleteFolderChildren(ctx context.Context, fullpath we return nil } -func enumerate(iter *gorocksdb.Iterator, prefix, lastKey []byte, includeLastKey bool, limit int, fn func(key, value []byte) bool) error { +func enumerate(iter *gorocksdb.Iterator, prefix, lastKey []byte, includeLastKey bool, limit int, fn func(key, value []byte) bool) (hasMore bool, err error) { if len(lastKey) == 0 { iter.Seek(prefix) @@ -196,6 +196,7 @@ func enumerate(iter *gorocksdb.Iterator, prefix, lastKey []byte, includeLastKey if limit > 0 { i++ if i > limit { + hasMore = true break } } @@ -215,22 +216,21 @@ func enumerate(iter *gorocksdb.Iterator, prefix, lastKey []byte, includeLastKey } if err := iter.Err(); err != nil { - return fmt.Errorf("prefix scan iterator: %v", err) + return hasMore, fmt.Errorf("prefix scan iterator: %v", err) } - return nil + return hasMore, nil } -func (store *RocksDBStore) ListDirectoryEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, - limit int) (entries []*filer.Entry, err error) { - return store.ListDirectoryPrefixedEntries(ctx, fullpath, startFileName, inclusive, limit, "") +func (store *RocksDBStore) ListDirectoryEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int) (entries []*filer.Entry, hasMore bool, err error) { + return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "") } -func (store *RocksDBStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer.Entry, err error) { +func (store *RocksDBStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) (entries []*filer.Entry, hasMore bool, err error) { - directoryPrefix := genDirectoryKeyPrefix(fullpath, prefix) + directoryPrefix := genDirectoryKeyPrefix(dirPath, prefix) lastFileStart := directoryPrefix if startFileName != "" { - lastFileStart = genDirectoryKeyPrefix(fullpath, startFileName) + lastFileStart = genDirectoryKeyPrefix(dirPath, startFileName) } ro := gorocksdb.NewDefaultReadOptions() @@ -239,13 +239,13 @@ func (store *RocksDBStore) ListDirectoryPrefixedEntries(ctx context.Context, ful iter := store.db.NewIterator(ro) defer iter.Close() - err = enumerate(iter, directoryPrefix, lastFileStart, inclusive, limit, func(key, value []byte) bool { + hasMore, err = enumerate(iter, directoryPrefix, lastFileStart, includeStartFile, limit, func(key, value []byte) bool { fileName := getNameFromKey(key) if fileName == "" { return true } entry := &filer.Entry{ - FullPath: weed_util.NewFullPath(string(fullpath), fileName), + FullPath: weed_util.NewFullPath(string(dirPath), fileName), } // println("list", entry.FullPath, "chunks", len(entry.Chunks)) @@ -258,10 +258,10 @@ func (store *RocksDBStore) ListDirectoryPrefixedEntries(ctx context.Context, ful return true }) if err != nil { - return entries, fmt.Errorf("prefix list %s : %v", fullpath, err) + return entries, false, fmt.Errorf("prefix list %s : %v", dirPath, err) } - return entries, err + return entries, false, err } func genKey(dirPath, fileName string) (key []byte) { From c64bfb0e2ebf0ca547edd445ad7d7dd30d517280 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 14 Jan 2021 22:28:51 -0800 Subject: [PATCH 128/260] implement mongodb changes --- weed/filer/mongodb/mongodb_store.go | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/weed/filer/mongodb/mongodb_store.go b/weed/filer/mongodb/mongodb_store.go index d20c6477a..5f601aae3 100644 --- a/weed/filer/mongodb/mongodb_store.go +++ b/weed/filer/mongodb/mongodb_store.go @@ -178,30 +178,30 @@ func (store *MongodbStore) DeleteFolderChildren(ctx context.Context, fullpath ut return nil } -func (store *MongodbStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer.Entry, err error) { - return nil, filer.ErrUnsupportedListDirectoryPrefixed +func (store *MongodbStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) (entries []*filer.Entry, hasMore bool, err error) { + return nil, false, filer.ErrUnsupportedListDirectoryPrefixed } -func (store *MongodbStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer.Entry, err error) { +func (store *MongodbStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int) (entries []*filer.Entry, hasMore bool, err error) { - var where = bson.M{"directory": string(fullpath), "name": bson.M{"$gt": startFileName}} - if inclusive { + var where = bson.M{"directory": string(dirPath), "name": bson.M{"$gt": startFileName}} + if includeStartFile { where["name"] = bson.M{ "$gte": startFileName, } } - optLimit := int64(limit) + optLimit := int64(limit+1) opts := &options.FindOptions{Limit: &optLimit, Sort: bson.M{"name": 1}} cur, err := store.connect.Database(store.database).Collection(store.collectionName).Find(ctx, where, opts) for cur.Next(ctx) { var data Model err := cur.Decode(&data) if err != nil && err != mongo.ErrNoDocuments { - return nil, err + return nil, false, err } entry := &filer.Entry{ - FullPath: util.NewFullPath(string(fullpath), data.Name), + FullPath: util.NewFullPath(string(dirPath), data.Name), } if decodeErr := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(data.Meta)); decodeErr != nil { err = decodeErr @@ -212,11 +212,16 @@ func (store *MongodbStore) ListDirectoryEntries(ctx context.Context, fullpath ut entries = append(entries, entry) } + hasMore = len(entries) == limit + 1 + if hasMore { + entries = entries[:limit] + } + if err := cur.Close(ctx); err != nil { glog.V(0).Infof("list iterator close: %v", err) } - return entries, err + return entries, hasMore, err } func (store *MongodbStore) Shutdown() { From b5ceffe18845a08d3f709b41b39b69ba8dff2f35 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 14 Jan 2021 22:33:05 -0800 Subject: [PATCH 129/260] implement leveldb changes --- weed/filer/leveldb/leveldb_store.go | 10 +++++----- weed/filer/leveldb2/leveldb2_store.go | 10 +++++----- weed/filer/leveldb3/leveldb3_store.go | 18 +++++++++--------- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/weed/filer/leveldb/leveldb_store.go b/weed/filer/leveldb/leveldb_store.go index e596a180e..32814af14 100644 --- a/weed/filer/leveldb/leveldb_store.go +++ b/weed/filer/leveldb/leveldb_store.go @@ -162,12 +162,11 @@ func (store *LevelDBStore) DeleteFolderChildren(ctx context.Context, fullpath we return nil } -func (store *LevelDBStore) ListDirectoryEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, - limit int) (entries []*filer.Entry, err error) { - return store.ListDirectoryPrefixedEntries(ctx, fullpath, startFileName, inclusive, limit, "") +func (store *LevelDBStore) ListDirectoryEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int) (entries []*filer.Entry, hasMore bool, err error) { + return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "") } -func (store *LevelDBStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer.Entry, err error) { +func (store *LevelDBStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer.Entry, hasMore bool, err error) { directoryPrefix := genDirectoryKeyPrefix(fullpath, prefix) lastFileStart := directoryPrefix @@ -190,6 +189,7 @@ func (store *LevelDBStore) ListDirectoryPrefixedEntries(ctx context.Context, ful } limit-- if limit < 0 { + hasMore = true break } entry := &filer.Entry{ @@ -204,7 +204,7 @@ func (store *LevelDBStore) ListDirectoryPrefixedEntries(ctx context.Context, ful } iter.Release() - return entries, err + return entries, hasMore, err } func genKey(dirPath, fileName string) (key []byte) { diff --git a/weed/filer/leveldb2/leveldb2_store.go b/weed/filer/leveldb2/leveldb2_store.go index 66ad61b8b..32be56243 100644 --- a/weed/filer/leveldb2/leveldb2_store.go +++ b/weed/filer/leveldb2/leveldb2_store.go @@ -171,12 +171,11 @@ func (store *LevelDB2Store) DeleteFolderChildren(ctx context.Context, fullpath w return nil } -func (store *LevelDB2Store) ListDirectoryEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, - limit int) (entries []*filer.Entry, err error) { - return store.ListDirectoryPrefixedEntries(ctx, fullpath, startFileName, inclusive, limit, "") +func (store *LevelDB2Store) ListDirectoryEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int) (entries []*filer.Entry, hasMore bool, err error) { + return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "") } -func (store *LevelDB2Store) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer.Entry, err error) { +func (store *LevelDB2Store) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer.Entry, hasMore bool, err error) { directoryPrefix, partitionId := genDirectoryKeyPrefix(fullpath, prefix, store.dbCount) lastFileStart := directoryPrefix @@ -199,6 +198,7 @@ func (store *LevelDB2Store) ListDirectoryPrefixedEntries(ctx context.Context, fu } limit-- if limit < 0 { + hasMore = true break } entry := &filer.Entry{ @@ -215,7 +215,7 @@ func (store *LevelDB2Store) ListDirectoryPrefixedEntries(ctx context.Context, fu } iter.Release() - return entries, err + return entries, hasMore, err } func genKey(dirPath, fileName string, dbCount int) (key []byte, partitionId int) { diff --git a/weed/filer/leveldb3/leveldb3_store.go b/weed/filer/leveldb3/leveldb3_store.go index 332f1569b..8cf2ca7c6 100644 --- a/weed/filer/leveldb3/leveldb3_store.go +++ b/weed/filer/leveldb3/leveldb3_store.go @@ -286,16 +286,15 @@ func (store *LevelDB3Store) DeleteFolderChildren(ctx context.Context, fullpath w return nil } -func (store *LevelDB3Store) ListDirectoryEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, - limit int) (entries []*filer.Entry, err error) { - return store.ListDirectoryPrefixedEntries(ctx, fullpath, startFileName, inclusive, limit, "") +func (store *LevelDB3Store) ListDirectoryEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int) (entries []*filer.Entry, hasMore bool, err error) { + return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "") } -func (store *LevelDB3Store) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer.Entry, err error) { +func (store *LevelDB3Store) ListDirectoryPrefixedEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) (entries []*filer.Entry, hasMore bool, err error) { - db, _, shortPath, err := store.findDB(fullpath, true) + db, _, shortPath, err := store.findDB(dirPath, true) if err != nil { - return nil, fmt.Errorf("findDB %s : %v", fullpath, err) + return nil, false, fmt.Errorf("findDB %s : %v", dirPath, err) } directoryPrefix := genDirectoryKeyPrefix(shortPath, prefix) @@ -314,15 +313,16 @@ func (store *LevelDB3Store) ListDirectoryPrefixedEntries(ctx context.Context, fu if fileName == "" { continue } - if fileName == startFileName && !inclusive { + if fileName == startFileName && !includeStartFile { continue } limit-- if limit < 0 { + hasMore = true break } entry := &filer.Entry{ - FullPath: weed_util.NewFullPath(string(fullpath), fileName), + FullPath: weed_util.NewFullPath(string(dirPath), fileName), } // println("list", entry.FullPath, "chunks", len(entry.Chunks)) @@ -335,7 +335,7 @@ func (store *LevelDB3Store) ListDirectoryPrefixedEntries(ctx context.Context, fu } iter.Release() - return entries, err + return entries, hasMore, err } func genKey(dirPath, fileName string) (key []byte) { From 54527f03266629844a178eddf7aa9a8d01026b00 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 14 Jan 2021 22:34:35 -0800 Subject: [PATCH 130/260] implement hbase changes --- weed/filer/hbase/hbase_store.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/weed/filer/hbase/hbase_store.go b/weed/filer/hbase/hbase_store.go index 6b0ad58b9..5a64ffe7e 100644 --- a/weed/filer/hbase/hbase_store.go +++ b/weed/filer/hbase/hbase_store.go @@ -148,18 +148,19 @@ func (store *HbaseStore) DeleteFolderChildren(ctx context.Context, path util.Ful return } -func (store *HbaseStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int) ([]*filer.Entry, error) { +func (store *HbaseStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int) ([]*filer.Entry, bool, error) { return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "") } -func (store *HbaseStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) ([]*filer.Entry, error) { +func (store *HbaseStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) ([]*filer.Entry, bool, error) { family := map[string][]string{store.cfMetaDir: {COLUMN_NAME}} expectedPrefix := []byte(dirPath.Child(prefix)) scan, err := hrpc.NewScanRange(ctx, store.table, expectedPrefix, nil, hrpc.Families(family)) if err != nil { - return nil, err + return nil, false, err } + var hasMore bool var entries []*filer.Entry scanner := store.Client.Scan(scan) defer scanner.Close() @@ -169,7 +170,7 @@ func (store *HbaseStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPa break } if err != nil { - return entries, err + return entries, hasMore, err } if len(res.Cells) == 0 { continue @@ -194,6 +195,7 @@ func (store *HbaseStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPa limit-- if limit < 0 { + hasMore = true break } entry := &filer.Entry{ @@ -207,7 +209,7 @@ func (store *HbaseStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPa entries = append(entries, entry) } - return entries, nil + return entries, hasMore, nil } func (store *HbaseStore) BeginTransaction(ctx context.Context) (context.Context, error) { From 5ef43b9b09d7d7465bd5f1d6ee0da998c3f42e9b Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 14 Jan 2021 22:35:56 -0800 Subject: [PATCH 131/260] implement etcd changes --- weed/filer/etcd/etcd_store.go | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/weed/filer/etcd/etcd_store.go b/weed/filer/etcd/etcd_store.go index 01c453158..6139d9fa0 100644 --- a/weed/filer/etcd/etcd_store.go +++ b/weed/filer/etcd/etcd_store.go @@ -139,17 +139,17 @@ func (store *EtcdStore) DeleteFolderChildren(ctx context.Context, fullpath weed_ return nil } -func (store *EtcdStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer.Entry, err error) { - return nil, filer.ErrUnsupportedListDirectoryPrefixed +func (store *EtcdStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) (entries []*filer.Entry, hasMore bool, err error) { + return nil, false, filer.ErrUnsupportedListDirectoryPrefixed } -func (store *EtcdStore) ListDirectoryEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer.Entry, err error) { - directoryPrefix := genDirectoryKeyPrefix(fullpath, "") +func (store *EtcdStore) ListDirectoryEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int) (entries []*filer.Entry, hasMore bool, err error) { + directoryPrefix := genDirectoryKeyPrefix(dirPath, "") resp, err := store.client.Get(ctx, string(directoryPrefix), clientv3.WithPrefix(), clientv3.WithSort(clientv3.SortByKey, clientv3.SortDescend)) if err != nil { - return nil, fmt.Errorf("list %s : %v", fullpath, err) + return nil, false, fmt.Errorf("list %s : %v", dirPath, err) } for _, kv := range resp.Kvs { @@ -157,15 +157,16 @@ func (store *EtcdStore) ListDirectoryEntries(ctx context.Context, fullpath weed_ if fileName == "" { continue } - if fileName == startFileName && !inclusive { + if fileName == startFileName && !includeStartFile { continue } limit-- if limit < 0 { + hasMore = true break } entry := &filer.Entry{ - FullPath: weed_util.NewFullPath(string(fullpath), fileName), + FullPath: weed_util.NewFullPath(string(dirPath), fileName), } if decodeErr := entry.DecodeAttributesAndChunks(weed_util.MaybeDecompressData(kv.Value)); decodeErr != nil { err = decodeErr @@ -175,7 +176,7 @@ func (store *EtcdStore) ListDirectoryEntries(ctx context.Context, fullpath weed_ entries = append(entries, entry) } - return entries, err + return entries, hasMore, err } func genKey(dirPath, fileName string) (key []byte) { From 893cbc8482bc88807e01b62718c18dfba4a234c2 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 14 Jan 2021 22:38:34 -0800 Subject: [PATCH 132/260] implement c* changes --- weed/filer/cassandra/cassandra_store.go | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/weed/filer/cassandra/cassandra_store.go b/weed/filer/cassandra/cassandra_store.go index 49f5625d9..578ff3172 100644 --- a/weed/filer/cassandra/cassandra_store.go +++ b/weed/filer/cassandra/cassandra_store.go @@ -168,28 +168,27 @@ func (store *CassandraStore) DeleteFolderChildren(ctx context.Context, fullpath return nil } -func (store *CassandraStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer.Entry, err error) { - return nil, filer.ErrUnsupportedListDirectoryPrefixed +func (store *CassandraStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) (entries []*filer.Entry, hasMore bool, err error) { + return nil, false, filer.ErrUnsupportedListDirectoryPrefixed } -func (store *CassandraStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, - limit int) (entries []*filer.Entry, err error) { +func (store *CassandraStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int) (entries []*filer.Entry, hasMore bool, err error) { - if _, ok := store.isSuperLargeDirectory(string(fullpath)); ok { + if _, ok := store.isSuperLargeDirectory(string(dirPath)); ok { return // nil, filer.ErrUnsupportedSuperLargeDirectoryListing } cqlStr := "SELECT NAME, meta FROM filemeta WHERE directory=? AND name>? ORDER BY NAME ASC LIMIT ?" - if inclusive { + if includeStartFile { cqlStr = "SELECT NAME, meta FROM filemeta WHERE directory=? AND name>=? ORDER BY NAME ASC LIMIT ?" } var data []byte var name string - iter := store.session.Query(cqlStr, string(fullpath), startFileName, limit).Iter() + iter := store.session.Query(cqlStr, string(dirPath), startFileName, limit+1).Iter() for iter.Scan(&name, &data) { entry := &filer.Entry{ - FullPath: util.NewFullPath(string(fullpath), name), + FullPath: util.NewFullPath(string(dirPath), name), } if decodeErr := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(data)); decodeErr != nil { err = decodeErr @@ -202,7 +201,12 @@ func (store *CassandraStore) ListDirectoryEntries(ctx context.Context, fullpath glog.V(0).Infof("list iterator close: %v", err) } - return entries, err + hasMore = len(entries) == limit + 1 + if hasMore { + entries = entries[:limit] + } + + return entries, hasMore, err } func (store *CassandraStore) Shutdown() { From 5d4568b91fae7fbc843367ff597427200af17e9a Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 14 Jan 2021 22:42:25 -0800 Subject: [PATCH 133/260] implement elastic changes --- weed/filer/elastic/v7/elastic_store.go | 36 +++++++++++++------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/weed/filer/elastic/v7/elastic_store.go b/weed/filer/elastic/v7/elastic_store.go index ec88e10a5..d08fc8ba8 100644 --- a/weed/filer/elastic/v7/elastic_store.go +++ b/weed/filer/elastic/v7/elastic_store.go @@ -96,8 +96,8 @@ func (store *ElasticStore) RollbackTransaction(ctx context.Context) error { return nil } -func (store *ElasticStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer.Entry, err error) { - return nil, filer.ErrUnsupportedListDirectoryPrefixed +func (store *ElasticStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) (entries []*filer.Entry, hasMore bool, err error) { + return nil, false, filer.ErrUnsupportedListDirectoryPrefixed } func (store *ElasticStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) { @@ -187,7 +187,7 @@ func (store *ElasticStore) deleteEntry(ctx context.Context, index, id string) (e } func (store *ElasticStore) DeleteFolderChildren(ctx context.Context, fullpath weed_util.FullPath) (err error) { - if entries, err := store.ListDirectoryEntries(ctx, fullpath, "", false, math.MaxInt32); err == nil { + if entries, _, err := store.ListDirectoryEntries(ctx, fullpath, "", false, math.MaxInt32); err == nil { for _, entry := range entries { store.DeleteEntry(ctx, entry.FullPath) } @@ -195,20 +195,18 @@ func (store *ElasticStore) DeleteFolderChildren(ctx context.Context, fullpath we return nil } -func (store *ElasticStore) ListDirectoryEntries( - ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, -) (entries []*filer.Entry, err error) { - if string(fullpath) == "/" { - return store.listRootDirectoryEntries(ctx, startFileName, inclusive, limit) +func (store *ElasticStore) ListDirectoryEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int) (entries []*filer.Entry, hasMore bool, err error) { + if string(dirPath) == "/" { + return store.listRootDirectoryEntries(ctx, startFileName, includeStartFile, limit) } - return store.listDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) + return store.listDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit) } -func (store *ElasticStore) listRootDirectoryEntries(ctx context.Context, startFileName string, inclusive bool, limit int) (entries []*filer.Entry, err error) { +func (store *ElasticStore) listRootDirectoryEntries(ctx context.Context, startFileName string, inclusive bool, limit int) (entries []*filer.Entry, hasMore bool, err error) { indexResult, err := store.client.CatIndices().Do(ctx) if err != nil { glog.Errorf("list indices %v.", err) - return entries, err + return entries, false, err } for _, index := range indexResult { if index.Index == indexKV { @@ -223,18 +221,19 @@ func (store *ElasticStore) listRootDirectoryEntries(ctx context.Context, startFi } limit-- if limit < 0 { + hasMore = true break } entries = append(entries, entry) } } } - return entries, nil + return entries, hasMore, nil } func (store *ElasticStore) listDirectoryEntries( ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, -) (entries []*filer.Entry, err error) { +) (entries []*filer.Entry, hasMore bool, err error) { first := true index := getIndex(fullpath) nextStart := "" @@ -242,7 +241,7 @@ func (store *ElasticStore) listDirectoryEntries( if _, err := store.client.Refresh(index).Do(ctx); err != nil { if elastic.IsNotFound(err) { store.client.CreateIndex(index).Do(ctx) - return entries, nil + return entries, hasMore, nil } } for { @@ -250,7 +249,7 @@ func (store *ElasticStore) listDirectoryEntries( if (startFileName == "" && first) || inclusive { if result, err = store.search(ctx, index, parentId); err != nil { glog.Errorf("search (%s,%s,%t,%d) %v.", string(fullpath), startFileName, inclusive, limit, err) - return entries, err + return entries, hasMore, err } } else { fullPath := string(fullpath) + "/" + startFileName @@ -260,7 +259,7 @@ func (store *ElasticStore) listDirectoryEntries( after := weed_util.Md5String([]byte(fullPath)) if result, err = store.searchAfter(ctx, index, parentId, after); err != nil { glog.Errorf("searchAfter (%s,%s,%t,%d) %v.", string(fullpath), startFileName, inclusive, limit, err) - return entries, err + return entries, hasMore, err } } first = false @@ -272,7 +271,8 @@ func (store *ElasticStore) listDirectoryEntries( if err := jsoniter.Unmarshal(hit.Source, esEntry); err == nil { limit-- if limit < 0 { - return entries, nil + hasMore = true + return entries, hasMore, nil } nextStart = string(esEntry.Entry.FullPath) fileName := getFileName(esEntry.Entry.FullPath) @@ -286,7 +286,7 @@ func (store *ElasticStore) listDirectoryEntries( break } } - return entries, nil + return entries, hasMore, nil } func (store *ElasticStore) search(ctx context.Context, index, parentId string) (result *elastic.SearchResult, err error) { From c0bcf6f6e16126f5d3a1a158d57d9c0c3f7d98ab Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 14 Jan 2021 22:44:22 -0800 Subject: [PATCH 134/260] implement sql changes --- weed/filer/abstract_sql/abstract_sql_store.go | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/weed/filer/abstract_sql/abstract_sql_store.go b/weed/filer/abstract_sql/abstract_sql_store.go index da104358b..475dec2be 100644 --- a/weed/filer/abstract_sql/abstract_sql_store.go +++ b/weed/filer/abstract_sql/abstract_sql_store.go @@ -172,15 +172,15 @@ func (store *AbstractSqlStore) DeleteFolderChildren(ctx context.Context, fullpat return nil } -func (store *AbstractSqlStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer.Entry, err error) { +func (store *AbstractSqlStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) (entries []*filer.Entry, hasMore bool, err error) { sqlText := store.SqlListExclusive - if inclusive { + if includeStartFile { sqlText = store.SqlListInclusive } - rows, err := store.getTxOrDB(ctx).QueryContext(ctx, sqlText, util.HashStringToLong(string(fullpath)), startFileName, string(fullpath), prefix+"%", limit) + rows, err := store.getTxOrDB(ctx).QueryContext(ctx, sqlText, util.HashStringToLong(string(dirPath)), startFileName, string(dirPath), prefix+"%", limit+1) if err != nil { - return nil, fmt.Errorf("list %s : %v", fullpath, err) + return nil, false, fmt.Errorf("list %s : %v", dirPath, err) } defer rows.Close() @@ -188,26 +188,31 @@ func (store *AbstractSqlStore) ListDirectoryPrefixedEntries(ctx context.Context, var name string var data []byte if err = rows.Scan(&name, &data); err != nil { - glog.V(0).Infof("scan %s : %v", fullpath, err) - return nil, fmt.Errorf("scan %s: %v", fullpath, err) + glog.V(0).Infof("scan %s : %v", dirPath, err) + return nil, false, fmt.Errorf("scan %s: %v", dirPath, err) } entry := &filer.Entry{ - FullPath: util.NewFullPath(string(fullpath), name), + FullPath: util.NewFullPath(string(dirPath), name), } if err = entry.DecodeAttributesAndChunks(util.MaybeDecompressData(data)); err != nil { glog.V(0).Infof("scan decode %s : %v", entry.FullPath, err) - return nil, fmt.Errorf("scan decode %s : %v", entry.FullPath, err) + return nil, false, fmt.Errorf("scan decode %s : %v", entry.FullPath, err) } entries = append(entries, entry) } - return entries, nil + hasMore = len(entries) == limit + 1 + if hasMore { + entries = entries[:limit] + } + + return entries, hasMore, nil } -func (store *AbstractSqlStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer.Entry, err error) { - return store.ListDirectoryPrefixedEntries(ctx, fullpath, startFileName, inclusive, limit, "") +func (store *AbstractSqlStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int) (entries []*filer.Entry, hasMore bool, err error) { + return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "") } func (store *AbstractSqlStore) Shutdown() { From 19e52fd4146d32b4da48a3369385442e836a7647 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 14 Jan 2021 22:49:29 -0800 Subject: [PATCH 135/260] pass along hasMore --- weed/filer/filerstore_translate_path.go | 16 +++++++-------- weed/filer/filerstore_wrapper.go | 26 ++++++++++++------------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/weed/filer/filerstore_translate_path.go b/weed/filer/filerstore_translate_path.go index ea0f9db77..eb060ea89 100644 --- a/weed/filer/filerstore_translate_path.go +++ b/weed/filer/filerstore_translate_path.go @@ -106,32 +106,32 @@ func (t *FilerStorePathTranlator) DeleteFolderChildren(ctx context.Context, fp u return t.actualStore.DeleteFolderChildren(ctx, newFullPath) } -func (t *FilerStorePathTranlator) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int) ([]*Entry, error) { +func (t *FilerStorePathTranlator) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int) ([]*Entry, bool, error) { newFullPath := t.translatePath(dirPath) - entries, err := t.actualStore.ListDirectoryEntries(ctx, newFullPath, startFileName, includeStartFile, limit) + entries, hasMore, err := t.actualStore.ListDirectoryEntries(ctx, newFullPath, startFileName, includeStartFile, limit) if err != nil { - return nil, err + return nil, hasMore, err } for _, entry := range entries { entry.FullPath = dirPath[:len(t.storeRoot)-1] + entry.FullPath } - return entries, err + return entries, hasMore, err } -func (t *FilerStorePathTranlator) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) ([]*Entry, error) { +func (t *FilerStorePathTranlator) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) ([]*Entry, bool, error) { newFullPath := t.translatePath(dirPath) - entries, err := t.actualStore.ListDirectoryPrefixedEntries(ctx, newFullPath, startFileName, includeStartFile, limit, prefix) + entries, hasMore, err := t.actualStore.ListDirectoryPrefixedEntries(ctx, newFullPath, startFileName, includeStartFile, limit, prefix) if err != nil { - return nil, err + return nil, hasMore, err } for _, entry := range entries { entry.FullPath = dirPath[:len(t.storeRoot)-1] + entry.FullPath } - return entries, nil + return entries, hasMore, nil } func (t *FilerStorePathTranlator) BeginTransaction(ctx context.Context) (context.Context, error) { diff --git a/weed/filer/filerstore_wrapper.go b/weed/filer/filerstore_wrapper.go index 3206d5ba4..92f113991 100644 --- a/weed/filer/filerstore_wrapper.go +++ b/weed/filer/filerstore_wrapper.go @@ -194,7 +194,7 @@ func (fsw *FilerStoreWrapper) DeleteFolderChildren(ctx context.Context, fp util. return actualStore.DeleteFolderChildren(ctx, fp) } -func (fsw *FilerStoreWrapper) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int) ([]*Entry, error) { +func (fsw *FilerStoreWrapper) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int) ([]*Entry, bool, error) { actualStore := fsw.getActualStore(dirPath + "/") stats.FilerStoreCounter.WithLabelValues(actualStore.GetName(), "list").Inc() start := time.Now() @@ -203,18 +203,18 @@ func (fsw *FilerStoreWrapper) ListDirectoryEntries(ctx context.Context, dirPath }() glog.V(4).Infof("ListDirectoryEntries %s from %s limit %d", dirPath, startFileName, limit) - entries, err := actualStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit) + entries, hasMore, err := actualStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit) if err != nil { - return nil, err + return nil, hasMore, err } for _, entry := range entries { fsw.maybeReadHardLink(ctx, entry) filer_pb.AfterEntryDeserialization(entry.Chunks) } - return entries, err + return entries, hasMore, err } -func (fsw *FilerStoreWrapper) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) ([]*Entry, error) { +func (fsw *FilerStoreWrapper) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) ([]*Entry, bool, error) { actualStore := fsw.getActualStore(dirPath + "/") stats.FilerStoreCounter.WithLabelValues(actualStore.GetName(), "prefixList").Inc() start := time.Now() @@ -222,25 +222,25 @@ func (fsw *FilerStoreWrapper) ListDirectoryPrefixedEntries(ctx context.Context, stats.FilerStoreHistogram.WithLabelValues(actualStore.GetName(), "prefixList").Observe(time.Since(start).Seconds()) }() glog.V(4).Infof("ListDirectoryPrefixedEntries %s from %s prefix %s limit %d", dirPath, startFileName, prefix, limit) - entries, err := actualStore.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, prefix) + entries, hasMore, err := actualStore.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, prefix) if err == ErrUnsupportedListDirectoryPrefixed { - entries, err = fsw.prefixFilterEntries(ctx, dirPath, startFileName, includeStartFile, limit, prefix) + entries, hasMore, err = fsw.prefixFilterEntries(ctx, dirPath, startFileName, includeStartFile, limit, prefix) } if err != nil { - return nil, err + return nil, hasMore, err } for _, entry := range entries { fsw.maybeReadHardLink(ctx, entry) filer_pb.AfterEntryDeserialization(entry.Chunks) } - return entries, nil + return entries, hasMore, nil } -func (fsw *FilerStoreWrapper) prefixFilterEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) (entries []*Entry, err error) { +func (fsw *FilerStoreWrapper) prefixFilterEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) (entries []*Entry, hasMore bool, err error) { actualStore := fsw.getActualStore(dirPath + "/") - entries, err = actualStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit) + entries, hasMore, err = actualStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit) if err != nil { - return nil, err + return nil, hasMore, err } if prefix == "" { @@ -263,7 +263,7 @@ func (fsw *FilerStoreWrapper) prefixFilterEntries(ctx context.Context, dirPath u } } if count < limit { - notPrefixed, err = actualStore.ListDirectoryEntries(ctx, dirPath, lastFileName, false, limit) + notPrefixed, hasMore, err = actualStore.ListDirectoryEntries(ctx, dirPath, lastFileName, false, limit) if err != nil { return } From f002e668def4abdfda5fe8a76599ae6d0fb76319 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 14 Jan 2021 23:10:37 -0800 Subject: [PATCH 136/260] change limit to int64 in case of overflow --- weed/filer/abstract_sql/abstract_sql_store.go | 6 ++--- weed/filer/cassandra/cassandra_store.go | 6 ++--- weed/filer/elastic/v7/elastic_store.go | 8 +++---- weed/filer/etcd/etcd_store.go | 4 ++-- weed/filer/filer.go | 7 +++--- weed/filer/filer_buckets.go | 4 ++-- weed/filer/filer_delete_entry.go | 2 +- weed/filer/filer_notify.go | 4 ++-- weed/filer/filer_search.go | 24 +++++++++---------- weed/filer/filerstore.go | 4 ++-- weed/filer/filerstore_translate_path.go | 4 ++-- weed/filer/filerstore_wrapper.go | 8 +++---- weed/filer/hbase/hbase_store.go | 4 ++-- weed/filer/leveldb/leveldb_store.go | 12 +++++----- weed/filer/leveldb/leveldb_store_test.go | 6 ++--- weed/filer/leveldb2/leveldb2_store.go | 12 +++++----- weed/filer/leveldb2/leveldb2_store_test.go | 6 ++--- weed/filer/leveldb3/leveldb3_store.go | 4 ++-- weed/filer/leveldb3/leveldb3_store_test.go | 6 ++--- weed/filer/mongodb/mongodb_store.go | 6 ++--- weed/filer/redis/universal_redis_store.go | 12 +++++----- weed/filer/redis2/universal_redis_store.go | 6 ++--- weed/filer/rocksdb/rocksdb_store.go | 4 ++-- weed/filer/rocksdb/rocksdb_store_test.go | 6 ++--- weed/filesys/dir.go | 2 +- weed/filesys/meta_cache/meta_cache.go | 2 +- weed/server/filer_grpc_server.go | 4 ++-- weed/server/filer_grpc_server_rename.go | 4 ++-- weed/server/filer_server_handlers_read_dir.go | 3 +-- 29 files changed, 90 insertions(+), 90 deletions(-) diff --git a/weed/filer/abstract_sql/abstract_sql_store.go b/weed/filer/abstract_sql/abstract_sql_store.go index 475dec2be..33a237e87 100644 --- a/weed/filer/abstract_sql/abstract_sql_store.go +++ b/weed/filer/abstract_sql/abstract_sql_store.go @@ -172,7 +172,7 @@ func (store *AbstractSqlStore) DeleteFolderChildren(ctx context.Context, fullpat return nil } -func (store *AbstractSqlStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) (entries []*filer.Entry, hasMore bool, err error) { +func (store *AbstractSqlStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string) (entries []*filer.Entry, hasMore bool, err error) { sqlText := store.SqlListExclusive if includeStartFile { sqlText = store.SqlListInclusive @@ -203,7 +203,7 @@ func (store *AbstractSqlStore) ListDirectoryPrefixedEntries(ctx context.Context, entries = append(entries, entry) } - hasMore = len(entries) == limit + 1 + hasMore = int64(len(entries)) == limit + 1 if hasMore { entries = entries[:limit] } @@ -211,7 +211,7 @@ func (store *AbstractSqlStore) ListDirectoryPrefixedEntries(ctx context.Context, return entries, hasMore, nil } -func (store *AbstractSqlStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int) (entries []*filer.Entry, hasMore bool, err error) { +func (store *AbstractSqlStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64) (entries []*filer.Entry, hasMore bool, err error) { return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "") } diff --git a/weed/filer/cassandra/cassandra_store.go b/weed/filer/cassandra/cassandra_store.go index 578ff3172..2c815aa1f 100644 --- a/weed/filer/cassandra/cassandra_store.go +++ b/weed/filer/cassandra/cassandra_store.go @@ -168,11 +168,11 @@ func (store *CassandraStore) DeleteFolderChildren(ctx context.Context, fullpath return nil } -func (store *CassandraStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) (entries []*filer.Entry, hasMore bool, err error) { +func (store *CassandraStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string) (entries []*filer.Entry, hasMore bool, err error) { return nil, false, filer.ErrUnsupportedListDirectoryPrefixed } -func (store *CassandraStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int) (entries []*filer.Entry, hasMore bool, err error) { +func (store *CassandraStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64) (entries []*filer.Entry, hasMore bool, err error) { if _, ok := store.isSuperLargeDirectory(string(dirPath)); ok { return // nil, filer.ErrUnsupportedSuperLargeDirectoryListing @@ -201,7 +201,7 @@ func (store *CassandraStore) ListDirectoryEntries(ctx context.Context, dirPath u glog.V(0).Infof("list iterator close: %v", err) } - hasMore = len(entries) == limit + 1 + hasMore = int64(len(entries)) == limit + 1 if hasMore { entries = entries[:limit] } diff --git a/weed/filer/elastic/v7/elastic_store.go b/weed/filer/elastic/v7/elastic_store.go index d08fc8ba8..0e055e1fc 100644 --- a/weed/filer/elastic/v7/elastic_store.go +++ b/weed/filer/elastic/v7/elastic_store.go @@ -96,7 +96,7 @@ func (store *ElasticStore) RollbackTransaction(ctx context.Context) error { return nil } -func (store *ElasticStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) (entries []*filer.Entry, hasMore bool, err error) { +func (store *ElasticStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string) (entries []*filer.Entry, hasMore bool, err error) { return nil, false, filer.ErrUnsupportedListDirectoryPrefixed } @@ -195,14 +195,14 @@ func (store *ElasticStore) DeleteFolderChildren(ctx context.Context, fullpath we return nil } -func (store *ElasticStore) ListDirectoryEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int) (entries []*filer.Entry, hasMore bool, err error) { +func (store *ElasticStore) ListDirectoryEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64) (entries []*filer.Entry, hasMore bool, err error) { if string(dirPath) == "/" { return store.listRootDirectoryEntries(ctx, startFileName, includeStartFile, limit) } return store.listDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit) } -func (store *ElasticStore) listRootDirectoryEntries(ctx context.Context, startFileName string, inclusive bool, limit int) (entries []*filer.Entry, hasMore bool, err error) { +func (store *ElasticStore) listRootDirectoryEntries(ctx context.Context, startFileName string, inclusive bool, limit int64) (entries []*filer.Entry, hasMore bool, err error) { indexResult, err := store.client.CatIndices().Do(ctx) if err != nil { glog.Errorf("list indices %v.", err) @@ -232,7 +232,7 @@ func (store *ElasticStore) listRootDirectoryEntries(ctx context.Context, startFi } func (store *ElasticStore) listDirectoryEntries( - ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, + ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int64, ) (entries []*filer.Entry, hasMore bool, err error) { first := true index := getIndex(fullpath) diff --git a/weed/filer/etcd/etcd_store.go b/weed/filer/etcd/etcd_store.go index 6139d9fa0..b7acc2049 100644 --- a/weed/filer/etcd/etcd_store.go +++ b/weed/filer/etcd/etcd_store.go @@ -139,11 +139,11 @@ func (store *EtcdStore) DeleteFolderChildren(ctx context.Context, fullpath weed_ return nil } -func (store *EtcdStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) (entries []*filer.Entry, hasMore bool, err error) { +func (store *EtcdStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string) (entries []*filer.Entry, hasMore bool, err error) { return nil, false, filer.ErrUnsupportedListDirectoryPrefixed } -func (store *EtcdStore) ListDirectoryEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int) (entries []*filer.Entry, hasMore bool, err error) { +func (store *EtcdStore) ListDirectoryEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64) (entries []*filer.Entry, hasMore bool, err error) { directoryPrefix := genDirectoryKeyPrefix(dirPath, "") resp, err := store.client.Get(ctx, string(directoryPrefix), diff --git a/weed/filer/filer.go b/weed/filer/filer.go index 9a1b7be3a..dd0fcf2cf 100644 --- a/weed/filer/filer.go +++ b/weed/filer/filer.go @@ -281,10 +281,11 @@ func (f *Filer) FindEntry(ctx context.Context, p util.FullPath) (entry *Entry, e } -func (f *Filer) doListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*Entry, expiredCount int, lastFileName string, err error) { - listedEntries, _, listErr := f.Store.ListDirectoryPrefixedEntries(ctx, p, startFileName, inclusive, limit, prefix) +func (f *Filer) doListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int64, prefix string) (entries []*Entry, hasMore bool, expiredCount int64, lastFileName string, err error) { + listedEntries, listHasMore, listErr := f.Store.ListDirectoryPrefixedEntries(ctx, p, startFileName, inclusive, limit, prefix) + hasMore = listHasMore if listErr != nil { - return listedEntries, expiredCount, "", listErr + return listedEntries, hasMore, expiredCount, "", listErr } for _, entry := range listedEntries { lastFileName = entry.Name() diff --git a/weed/filer/filer_buckets.go b/weed/filer/filer_buckets.go index 87ad9e452..ba170f02e 100644 --- a/weed/filer/filer_buckets.go +++ b/weed/filer/filer_buckets.go @@ -27,9 +27,9 @@ func (f *Filer) LoadBuckets() { buckets: make(map[BucketName]*BucketOption), } - limit := math.MaxInt32 + limit := int64(math.MaxInt32) - entries, err := f.ListDirectoryEntries(context.Background(), util.FullPath(f.DirBucketsPath), "", false, limit, "", "") + entries, _, err := f.ListDirectoryEntries(context.Background(), util.FullPath(f.DirBucketsPath), "", false, limit, "", "") if err != nil { glog.V(1).Infof("no buckets found: %v", err) diff --git a/weed/filer/filer_delete_entry.go b/weed/filer/filer_delete_entry.go index 8dd3cfed0..50a669f40 100644 --- a/weed/filer/filer_delete_entry.go +++ b/weed/filer/filer_delete_entry.go @@ -69,7 +69,7 @@ func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry includeLastFile := false if !isDeletingBucket { for { - entries, err := f.ListDirectoryEntries(ctx, entry.FullPath, lastFileName, includeLastFile, PaginationSize, "", "") + entries, _, err := f.ListDirectoryEntries(ctx, entry.FullPath, lastFileName, includeLastFile, PaginationSize, "", "") if err != nil { glog.Errorf("list folder %s: %v", entry.FullPath, err) return nil, nil, fmt.Errorf("list folder %s: %v", entry.FullPath, err) diff --git a/weed/filer/filer_notify.go b/weed/filer/filer_notify.go index b690a4cd1..f3a795ad0 100644 --- a/weed/filer/filer_notify.go +++ b/weed/filer/filer_notify.go @@ -113,13 +113,13 @@ func (f *Filer) ReadPersistedLogBuffer(startTime time.Time, eachLogEntryFn func( sizeBuf := make([]byte, 4) startTsNs := startTime.UnixNano() - dayEntries, listDayErr := f.ListDirectoryEntries(context.Background(), SystemLogDir, startDate, true, 366, "", "") + dayEntries, _, listDayErr := f.ListDirectoryEntries(context.Background(), SystemLogDir, startDate, true, 366, "", "") if listDayErr != nil { return lastTsNs, fmt.Errorf("fail to list log by day: %v", listDayErr) } for _, dayEntry := range dayEntries { // println("checking day", dayEntry.FullPath) - hourMinuteEntries, listHourMinuteErr := f.ListDirectoryEntries(context.Background(), util.NewFullPath(SystemLogDir, dayEntry.Name()), "", false, 24*60, "", "") + hourMinuteEntries, _, listHourMinuteErr := f.ListDirectoryEntries(context.Background(), util.NewFullPath(SystemLogDir, dayEntry.Name()), "", false, 24*60, "", "") if listHourMinuteErr != nil { return lastTsNs, fmt.Errorf("fail to list log %s by day: %v", dayEntry.Name(), listHourMinuteErr) } diff --git a/weed/filer/filer_search.go b/weed/filer/filer_search.go index 34358d849..d37699a5a 100644 --- a/weed/filer/filer_search.go +++ b/weed/filer/filer_search.go @@ -20,7 +20,7 @@ func splitPattern(pattern string) (prefix string, restPattern string) { } // For now, prefix and namePattern are mutually exclusive -func (f *Filer) ListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int, prefix string, namePattern string) (entries []*Entry, err error) { +func (f *Filer) ListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int64, prefix string, namePattern string) (entries []*Entry, hasMore bool, err error) { if strings.HasSuffix(string(p), "/") && len(p) > 1 { p = p[0 : len(p)-1] } @@ -29,31 +29,31 @@ func (f *Filer) ListDirectoryEntries(ctx context.Context, p util.FullPath, start if prefixInNamePattern != "" { prefix = prefixInNamePattern } - var missedCount int + var missedCount int64 var lastFileName string - entries, missedCount, lastFileName, err = f.doListPatternMatchedEntries(ctx, p, startFileName, inclusive, limit, prefix, restNamePattern) + entries, hasMore, missedCount, lastFileName, err = f.doListPatternMatchedEntries(ctx, p, startFileName, inclusive, limit, prefix, restNamePattern) for missedCount > 0 && err == nil { var makeupEntries []*Entry - makeupEntries, missedCount, lastFileName, err = f.doListPatternMatchedEntries(ctx, p, lastFileName, false, missedCount, prefix, restNamePattern) + makeupEntries, hasMore, missedCount, lastFileName, err = f.doListPatternMatchedEntries(ctx, p, lastFileName, false, missedCount, prefix, restNamePattern) for _, entry := range makeupEntries { entries = append(entries, entry) } } - return entries, err + return entries, hasMore, err } -func (f *Filer) doListPatternMatchedEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int, prefix, restNamePattern string) (matchedEntries []*Entry, missedCount int, lastFileName string, err error) { +func (f *Filer) doListPatternMatchedEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int64, prefix, restNamePattern string) (matchedEntries []*Entry, hasMore bool, missedCount int64, lastFileName string, err error) { var foundEntries []*Entry - foundEntries, lastFileName, err = f.doListValidEntries(ctx, p, startFileName, inclusive, limit, prefix) + foundEntries, hasMore, lastFileName, err = f.doListValidEntries(ctx, p, startFileName, inclusive, limit, prefix) if err != nil { return } if len(restNamePattern) == 0 { - return foundEntries, 0, lastFileName, nil + return foundEntries, false,0, lastFileName, nil } for _, entry := range foundEntries { nameToTest := strings.ToLower(entry.Name()) @@ -66,12 +66,12 @@ func (f *Filer) doListPatternMatchedEntries(ctx context.Context, p util.FullPath return } -func (f *Filer) doListValidEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*Entry, lastFileName string, err error) { +func (f *Filer) doListValidEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int64, prefix string) (entries []*Entry, hasMore bool, lastFileName string, err error) { var makeupEntries []*Entry - var expiredCount int - entries, expiredCount, lastFileName, err = f.doListDirectoryEntries(ctx, p, startFileName, inclusive, limit, prefix) + var expiredCount int64 + entries, hasMore, expiredCount, lastFileName, err = f.doListDirectoryEntries(ctx, p, startFileName, inclusive, limit, prefix) for expiredCount > 0 && err == nil { - makeupEntries, expiredCount, lastFileName, err = f.doListDirectoryEntries(ctx, p, lastFileName, false, expiredCount, prefix) + makeupEntries, hasMore, expiredCount, lastFileName, err = f.doListDirectoryEntries(ctx, p, lastFileName, false, expiredCount, prefix) if err == nil { entries = append(entries, makeupEntries...) } diff --git a/weed/filer/filerstore.go b/weed/filer/filerstore.go index a44d9b6d6..a28c4d01b 100644 --- a/weed/filer/filerstore.go +++ b/weed/filer/filerstore.go @@ -24,8 +24,8 @@ type FilerStore interface { FindEntry(context.Context, util.FullPath) (entry *Entry, err error) DeleteEntry(context.Context, util.FullPath) (err error) DeleteFolderChildren(context.Context, util.FullPath) (err error) - ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int) ([]*Entry, bool, error) - ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) ([]*Entry, bool, error) + ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64) ([]*Entry, bool, error) + ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string) ([]*Entry, bool, error) BeginTransaction(ctx context.Context) (context.Context, error) CommitTransaction(ctx context.Context) error diff --git a/weed/filer/filerstore_translate_path.go b/weed/filer/filerstore_translate_path.go index eb060ea89..0e2ddd914 100644 --- a/weed/filer/filerstore_translate_path.go +++ b/weed/filer/filerstore_translate_path.go @@ -106,7 +106,7 @@ func (t *FilerStorePathTranlator) DeleteFolderChildren(ctx context.Context, fp u return t.actualStore.DeleteFolderChildren(ctx, newFullPath) } -func (t *FilerStorePathTranlator) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int) ([]*Entry, bool, error) { +func (t *FilerStorePathTranlator) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64) ([]*Entry, bool, error) { newFullPath := t.translatePath(dirPath) @@ -120,7 +120,7 @@ func (t *FilerStorePathTranlator) ListDirectoryEntries(ctx context.Context, dirP return entries, hasMore, err } -func (t *FilerStorePathTranlator) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) ([]*Entry, bool, error) { +func (t *FilerStorePathTranlator) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string) ([]*Entry, bool, error) { newFullPath := t.translatePath(dirPath) diff --git a/weed/filer/filerstore_wrapper.go b/weed/filer/filerstore_wrapper.go index 92f113991..2bb7793f0 100644 --- a/weed/filer/filerstore_wrapper.go +++ b/weed/filer/filerstore_wrapper.go @@ -194,7 +194,7 @@ func (fsw *FilerStoreWrapper) DeleteFolderChildren(ctx context.Context, fp util. return actualStore.DeleteFolderChildren(ctx, fp) } -func (fsw *FilerStoreWrapper) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int) ([]*Entry, bool, error) { +func (fsw *FilerStoreWrapper) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64) ([]*Entry, bool, error) { actualStore := fsw.getActualStore(dirPath + "/") stats.FilerStoreCounter.WithLabelValues(actualStore.GetName(), "list").Inc() start := time.Now() @@ -214,7 +214,7 @@ func (fsw *FilerStoreWrapper) ListDirectoryEntries(ctx context.Context, dirPath return entries, hasMore, err } -func (fsw *FilerStoreWrapper) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) ([]*Entry, bool, error) { +func (fsw *FilerStoreWrapper) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string) ([]*Entry, bool, error) { actualStore := fsw.getActualStore(dirPath + "/") stats.FilerStoreCounter.WithLabelValues(actualStore.GetName(), "prefixList").Inc() start := time.Now() @@ -236,7 +236,7 @@ func (fsw *FilerStoreWrapper) ListDirectoryPrefixedEntries(ctx context.Context, return entries, hasMore, nil } -func (fsw *FilerStoreWrapper) prefixFilterEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) (entries []*Entry, hasMore bool, err error) { +func (fsw *FilerStoreWrapper) prefixFilterEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string) (entries []*Entry, hasMore bool, err error) { actualStore := fsw.getActualStore(dirPath + "/") entries, hasMore, err = actualStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit) if err != nil { @@ -247,7 +247,7 @@ func (fsw *FilerStoreWrapper) prefixFilterEntries(ctx context.Context, dirPath u return } - count := 0 + count := int64(0) var lastFileName string notPrefixed := entries entries = nil diff --git a/weed/filer/hbase/hbase_store.go b/weed/filer/hbase/hbase_store.go index 5a64ffe7e..c93374b15 100644 --- a/weed/filer/hbase/hbase_store.go +++ b/weed/filer/hbase/hbase_store.go @@ -148,11 +148,11 @@ func (store *HbaseStore) DeleteFolderChildren(ctx context.Context, path util.Ful return } -func (store *HbaseStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int) ([]*filer.Entry, bool, error) { +func (store *HbaseStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64) ([]*filer.Entry, bool, error) { return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "") } -func (store *HbaseStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) ([]*filer.Entry, bool, error) { +func (store *HbaseStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string) ([]*filer.Entry, bool, error) { family := map[string][]string{store.cfMetaDir: {COLUMN_NAME}} expectedPrefix := []byte(dirPath.Child(prefix)) scan, err := hrpc.NewScanRange(ctx, store.table, expectedPrefix, nil, hrpc.Families(family)) diff --git a/weed/filer/leveldb/leveldb_store.go b/weed/filer/leveldb/leveldb_store.go index 32814af14..c968ca2da 100644 --- a/weed/filer/leveldb/leveldb_store.go +++ b/weed/filer/leveldb/leveldb_store.go @@ -162,16 +162,16 @@ func (store *LevelDBStore) DeleteFolderChildren(ctx context.Context, fullpath we return nil } -func (store *LevelDBStore) ListDirectoryEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int) (entries []*filer.Entry, hasMore bool, err error) { +func (store *LevelDBStore) ListDirectoryEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64) (entries []*filer.Entry, hasMore bool, err error) { return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "") } -func (store *LevelDBStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer.Entry, hasMore bool, err error) { +func (store *LevelDBStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string) (entries []*filer.Entry, hasMore bool, err error) { - directoryPrefix := genDirectoryKeyPrefix(fullpath, prefix) + directoryPrefix := genDirectoryKeyPrefix(dirPath, prefix) lastFileStart := directoryPrefix if startFileName != "" { - lastFileStart = genDirectoryKeyPrefix(fullpath, startFileName) + lastFileStart = genDirectoryKeyPrefix(dirPath, startFileName) } iter := store.db.NewIterator(&leveldb_util.Range{Start: lastFileStart}, nil) @@ -184,7 +184,7 @@ func (store *LevelDBStore) ListDirectoryPrefixedEntries(ctx context.Context, ful if fileName == "" { continue } - if fileName == startFileName && !inclusive { + if fileName == startFileName && !includeStartFile { continue } limit-- @@ -193,7 +193,7 @@ func (store *LevelDBStore) ListDirectoryPrefixedEntries(ctx context.Context, ful break } entry := &filer.Entry{ - FullPath: weed_util.NewFullPath(string(fullpath), fileName), + FullPath: weed_util.NewFullPath(string(dirPath), fileName), } if decodeErr := entry.DecodeAttributesAndChunks(weed_util.MaybeDecompressData(iter.Value())); decodeErr != nil { err = decodeErr diff --git a/weed/filer/leveldb/leveldb_store_test.go b/weed/filer/leveldb/leveldb_store_test.go index edf629814..9c342605e 100644 --- a/weed/filer/leveldb/leveldb_store_test.go +++ b/weed/filer/leveldb/leveldb_store_test.go @@ -51,14 +51,14 @@ func TestCreateAndFind(t *testing.T) { } // checking one upper directory - entries, _ := testFiler.ListDirectoryEntries(ctx, util.FullPath("/home/chris/this/is/one"), "", false, 100, "", "") + entries, _, _ := testFiler.ListDirectoryEntries(ctx, util.FullPath("/home/chris/this/is/one"), "", false, 100, "", "") if len(entries) != 1 { t.Errorf("list entries count: %v", len(entries)) return } // checking one upper directory - entries, _ = testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "") + entries, _, _ = testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "") if len(entries) != 1 { t.Errorf("list entries count: %v", len(entries)) return @@ -77,7 +77,7 @@ func TestEmptyRoot(t *testing.T) { ctx := context.Background() // checking one upper directory - entries, err := testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "") + entries, _, err := testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "") if err != nil { t.Errorf("list entries: %v", err) return diff --git a/weed/filer/leveldb2/leveldb2_store.go b/weed/filer/leveldb2/leveldb2_store.go index 32be56243..cc0d67eac 100644 --- a/weed/filer/leveldb2/leveldb2_store.go +++ b/weed/filer/leveldb2/leveldb2_store.go @@ -171,16 +171,16 @@ func (store *LevelDB2Store) DeleteFolderChildren(ctx context.Context, fullpath w return nil } -func (store *LevelDB2Store) ListDirectoryEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int) (entries []*filer.Entry, hasMore bool, err error) { +func (store *LevelDB2Store) ListDirectoryEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64) (entries []*filer.Entry, hasMore bool, err error) { return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "") } -func (store *LevelDB2Store) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer.Entry, hasMore bool, err error) { +func (store *LevelDB2Store) ListDirectoryPrefixedEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string) (entries []*filer.Entry, hasMore bool, err error) { - directoryPrefix, partitionId := genDirectoryKeyPrefix(fullpath, prefix, store.dbCount) + directoryPrefix, partitionId := genDirectoryKeyPrefix(dirPath, prefix, store.dbCount) lastFileStart := directoryPrefix if startFileName != "" { - lastFileStart, _ = genDirectoryKeyPrefix(fullpath, startFileName, store.dbCount) + lastFileStart, _ = genDirectoryKeyPrefix(dirPath, startFileName, store.dbCount) } iter := store.dbs[partitionId].NewIterator(&leveldb_util.Range{Start: lastFileStart}, nil) @@ -193,7 +193,7 @@ func (store *LevelDB2Store) ListDirectoryPrefixedEntries(ctx context.Context, fu if fileName == "" { continue } - if fileName == startFileName && !inclusive { + if fileName == startFileName && !includeStartFile { continue } limit-- @@ -202,7 +202,7 @@ func (store *LevelDB2Store) ListDirectoryPrefixedEntries(ctx context.Context, fu break } entry := &filer.Entry{ - FullPath: weed_util.NewFullPath(string(fullpath), fileName), + FullPath: weed_util.NewFullPath(string(dirPath), fileName), } // println("list", entry.FullPath, "chunks", len(entry.Chunks)) diff --git a/weed/filer/leveldb2/leveldb2_store_test.go b/weed/filer/leveldb2/leveldb2_store_test.go index a7f5ae95b..495c73fdd 100644 --- a/weed/filer/leveldb2/leveldb2_store_test.go +++ b/weed/filer/leveldb2/leveldb2_store_test.go @@ -49,14 +49,14 @@ func TestCreateAndFind(t *testing.T) { } // checking one upper directory - entries, _ := testFiler.ListDirectoryEntries(ctx, util.FullPath("/home/chris/this/is/one"), "", false, 100, "", "") + entries, _, _ := testFiler.ListDirectoryEntries(ctx, util.FullPath("/home/chris/this/is/one"), "", false, 100, "", "") if len(entries) != 1 { t.Errorf("list entries count: %v", len(entries)) return } // checking one upper directory - entries, _ = testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "") + entries, _, _ = testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "") if len(entries) != 1 { t.Errorf("list entries count: %v", len(entries)) return @@ -75,7 +75,7 @@ func TestEmptyRoot(t *testing.T) { ctx := context.Background() // checking one upper directory - entries, err := testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "") + entries, _, err := testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "") if err != nil { t.Errorf("list entries: %v", err) return diff --git a/weed/filer/leveldb3/leveldb3_store.go b/weed/filer/leveldb3/leveldb3_store.go index 8cf2ca7c6..afbcabaa6 100644 --- a/weed/filer/leveldb3/leveldb3_store.go +++ b/weed/filer/leveldb3/leveldb3_store.go @@ -286,11 +286,11 @@ func (store *LevelDB3Store) DeleteFolderChildren(ctx context.Context, fullpath w return nil } -func (store *LevelDB3Store) ListDirectoryEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int) (entries []*filer.Entry, hasMore bool, err error) { +func (store *LevelDB3Store) ListDirectoryEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64) (entries []*filer.Entry, hasMore bool, err error) { return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "") } -func (store *LevelDB3Store) ListDirectoryPrefixedEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) (entries []*filer.Entry, hasMore bool, err error) { +func (store *LevelDB3Store) ListDirectoryPrefixedEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string) (entries []*filer.Entry, hasMore bool, err error) { db, _, shortPath, err := store.findDB(dirPath, true) if err != nil { diff --git a/weed/filer/leveldb3/leveldb3_store_test.go b/weed/filer/leveldb3/leveldb3_store_test.go index 20d86053f..53b0e927f 100644 --- a/weed/filer/leveldb3/leveldb3_store_test.go +++ b/weed/filer/leveldb3/leveldb3_store_test.go @@ -49,14 +49,14 @@ func TestCreateAndFind(t *testing.T) { } // checking one upper directory - entries, _ := testFiler.ListDirectoryEntries(ctx, util.FullPath("/home/chris/this/is/one"), "", false, 100, "", "") + entries, _, _ := testFiler.ListDirectoryEntries(ctx, util.FullPath("/home/chris/this/is/one"), "", false, 100, "", "") if len(entries) != 1 { t.Errorf("list entries count: %v", len(entries)) return } // checking one upper directory - entries, _ = testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "") + entries, _, _ = testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "") if len(entries) != 1 { t.Errorf("list entries count: %v", len(entries)) return @@ -75,7 +75,7 @@ func TestEmptyRoot(t *testing.T) { ctx := context.Background() // checking one upper directory - entries, err := testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "") + entries, _, err := testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "") if err != nil { t.Errorf("list entries: %v", err) return diff --git a/weed/filer/mongodb/mongodb_store.go b/weed/filer/mongodb/mongodb_store.go index 5f601aae3..2549b6029 100644 --- a/weed/filer/mongodb/mongodb_store.go +++ b/weed/filer/mongodb/mongodb_store.go @@ -178,11 +178,11 @@ func (store *MongodbStore) DeleteFolderChildren(ctx context.Context, fullpath ut return nil } -func (store *MongodbStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) (entries []*filer.Entry, hasMore bool, err error) { +func (store *MongodbStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string) (entries []*filer.Entry, hasMore bool, err error) { return nil, false, filer.ErrUnsupportedListDirectoryPrefixed } -func (store *MongodbStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int) (entries []*filer.Entry, hasMore bool, err error) { +func (store *MongodbStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64) (entries []*filer.Entry, hasMore bool, err error) { var where = bson.M{"directory": string(dirPath), "name": bson.M{"$gt": startFileName}} if includeStartFile { @@ -212,7 +212,7 @@ func (store *MongodbStore) ListDirectoryEntries(ctx context.Context, dirPath uti entries = append(entries, entry) } - hasMore = len(entries) == limit + 1 + hasMore = int64(len(entries)) == limit + 1 if hasMore { entries = entries[:limit] } diff --git a/weed/filer/redis/universal_redis_store.go b/weed/filer/redis/universal_redis_store.go index bc2644cb0..8399b4e99 100644 --- a/weed/filer/redis/universal_redis_store.go +++ b/weed/filer/redis/universal_redis_store.go @@ -125,16 +125,16 @@ func (store *UniversalRedisStore) DeleteFolderChildren(ctx context.Context, full return nil } -func (store *UniversalRedisStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) (entries []*filer.Entry, hasMore bool, err error) { +func (store *UniversalRedisStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string) (entries []*filer.Entry, hasMore bool, err error) { return nil, false, filer.ErrUnsupportedListDirectoryPrefixed } -func (store *UniversalRedisStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, includeStartFile bool, limit int) (entries []*filer.Entry, hasMore bool, err error) { +func (store *UniversalRedisStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64) (entries []*filer.Entry, hasMore bool, err error) { - dirListKey := genDirectoryListKey(string(fullpath)) + dirListKey := genDirectoryListKey(string(dirPath)) members, err := store.Client.SMembers(ctx, dirListKey).Result() if err != nil { - return nil, false, fmt.Errorf("list %s : %v", fullpath, err) + return nil, false, fmt.Errorf("list %s : %v", dirPath, err) } // skip @@ -160,14 +160,14 @@ func (store *UniversalRedisStore) ListDirectoryEntries(ctx context.Context, full }) // limit - if limit < len(members) { + if limit < int64(len(entries)) { members = members[:limit] hasMore = true } // fetch entry meta for _, fileName := range members { - path := util.NewFullPath(string(fullpath), fileName) + path := util.NewFullPath(string(dirPath), fileName) entry, err := store.FindEntry(ctx, path) if err != nil { glog.V(0).Infof("list %s : %v", path, err) diff --git a/weed/filer/redis2/universal_redis_store.go b/weed/filer/redis2/universal_redis_store.go index 8009b6a45..7b4e9d325 100644 --- a/weed/filer/redis2/universal_redis_store.go +++ b/weed/filer/redis2/universal_redis_store.go @@ -149,11 +149,11 @@ func (store *UniversalRedis2Store) DeleteFolderChildren(ctx context.Context, ful return nil } -func (store *UniversalRedis2Store) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) (entries []*filer.Entry, hasMore bool, err error) { +func (store *UniversalRedis2Store) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string) (entries []*filer.Entry, hasMore bool, err error) { return nil, false, filer.ErrUnsupportedListDirectoryPrefixed } -func (store *UniversalRedis2Store) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int) (entries []*filer.Entry, hasMore bool, err error) { +func (store *UniversalRedis2Store) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64) (entries []*filer.Entry, hasMore bool, err error) { dirListKey := genDirectoryListKey(string(dirPath)) start := int64(0) @@ -168,7 +168,7 @@ func (store *UniversalRedis2Store) ListDirectoryEntries(ctx context.Context, dir return nil, false, fmt.Errorf("list %s : %v", dirPath, err) } - hasMore = len(members) == limit+1 + hasMore = int64(len(entries)) == limit+1 if hasMore { members = members[:len(members)-1] } diff --git a/weed/filer/rocksdb/rocksdb_store.go b/weed/filer/rocksdb/rocksdb_store.go index fd0c96663..abeadb019 100644 --- a/weed/filer/rocksdb/rocksdb_store.go +++ b/weed/filer/rocksdb/rocksdb_store.go @@ -221,11 +221,11 @@ func enumerate(iter *gorocksdb.Iterator, prefix, lastKey []byte, includeLastKey return hasMore, nil } -func (store *RocksDBStore) ListDirectoryEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int) (entries []*filer.Entry, hasMore bool, err error) { +func (store *RocksDBStore) ListDirectoryEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64) (entries []*filer.Entry, hasMore bool, err error) { return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "") } -func (store *RocksDBStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) (entries []*filer.Entry, hasMore bool, err error) { +func (store *RocksDBStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string) (entries []*filer.Entry, hasMore bool, err error) { directoryPrefix := genDirectoryKeyPrefix(dirPath, prefix) lastFileStart := directoryPrefix diff --git a/weed/filer/rocksdb/rocksdb_store_test.go b/weed/filer/rocksdb/rocksdb_store_test.go index 5c48e9bd9..439663524 100644 --- a/weed/filer/rocksdb/rocksdb_store_test.go +++ b/weed/filer/rocksdb/rocksdb_store_test.go @@ -53,14 +53,14 @@ func TestCreateAndFind(t *testing.T) { } // checking one upper directory - entries, _ := testFiler.ListDirectoryEntries(ctx, util.FullPath("/home/chris/this/is/one"), "", false, 100, "", "") + entries, _, _ := testFiler.ListDirectoryEntries(ctx, util.FullPath("/home/chris/this/is/one"), "", false, 100, "", "") if len(entries) != 1 { t.Errorf("list entries count: %v", len(entries)) return } // checking one upper directory - entries, _ = testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "") + entries, _, _ = testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "") if len(entries) != 1 { t.Errorf("list entries count: %v", len(entries)) return @@ -79,7 +79,7 @@ func TestEmptyRoot(t *testing.T) { ctx := context.Background() // checking one upper directory - entries, err := testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "") + entries, _, err := testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "") if err != nil { t.Errorf("list entries: %v", err) return diff --git a/weed/filesys/dir.go b/weed/filesys/dir.go index a8481a435..f2fbd99d3 100644 --- a/weed/filesys/dir.go +++ b/weed/filesys/dir.go @@ -319,7 +319,7 @@ func (dir *Dir) ReadDirAll(ctx context.Context) (ret []fuse.Dirent, err error) { glog.Errorf("dir ReadDirAll %s: %v", dirPath, err) return nil, fuse.EIO } - listedEntries, listErr := dir.wfs.metaCache.ListDirectoryEntries(context.Background(), util.FullPath(dir.FullPath()), "", false, int(math.MaxInt32)) + listedEntries, listErr := dir.wfs.metaCache.ListDirectoryEntries(context.Background(), util.FullPath(dir.FullPath()), "", false, int64(math.MaxInt32)) if listErr != nil { glog.Errorf("list meta cache: %v", listErr) return nil, fuse.EIO diff --git a/weed/filesys/meta_cache/meta_cache.go b/weed/filesys/meta_cache/meta_cache.go index 4cb3a4930..c16c4a938 100644 --- a/weed/filesys/meta_cache/meta_cache.go +++ b/weed/filesys/meta_cache/meta_cache.go @@ -117,7 +117,7 @@ func (mc *MetaCache) DeleteEntry(ctx context.Context, fp util.FullPath) (err err return mc.localStore.DeleteEntry(ctx, fp) } -func (mc *MetaCache) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int) ([]*filer.Entry, error) { +func (mc *MetaCache) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64) ([]*filer.Entry, error) { mc.RLock() defer mc.RUnlock() diff --git a/weed/server/filer_grpc_server.go b/weed/server/filer_grpc_server.go index efc06f0d6..5cdf44e96 100644 --- a/weed/server/filer_grpc_server.go +++ b/weed/server/filer_grpc_server.go @@ -61,7 +61,7 @@ func (fs *FilerServer) ListEntries(req *filer_pb.ListEntriesRequest, stream file lastFileName := req.StartFromFileName includeLastFile := req.InclusiveStartFrom for limit > 0 { - entries, err := fs.filer.ListDirectoryEntries(stream.Context(), util.FullPath(req.Directory), lastFileName, includeLastFile, paginationLimit, req.Prefix, "") + entries, hasMore, err := fs.filer.ListDirectoryEntries(stream.Context(), util.FullPath(req.Directory), lastFileName, includeLastFile, int64(paginationLimit), req.Prefix, "") if err != nil { return err @@ -97,7 +97,7 @@ func (fs *FilerServer) ListEntries(req *filer_pb.ListEntriesRequest, stream file } } - if len(entries) < paginationLimit { + if !hasMore { break } diff --git a/weed/server/filer_grpc_server_rename.go b/weed/server/filer_grpc_server_rename.go index a0c9aedb9..5b68b64de 100644 --- a/weed/server/filer_grpc_server_rename.go +++ b/weed/server/filer_grpc_server_rename.go @@ -75,7 +75,7 @@ func (fs *FilerServer) moveFolderSubEntries(ctx context.Context, oldParent util. includeLastFile := false for { - entries, err := fs.filer.ListDirectoryEntries(ctx, currentDirPath, lastFileName, includeLastFile, 1024, "", "") + entries, hasMore, err := fs.filer.ListDirectoryEntries(ctx, currentDirPath, lastFileName, includeLastFile, 1024, "", "") if err != nil { return err } @@ -90,7 +90,7 @@ func (fs *FilerServer) moveFolderSubEntries(ctx context.Context, oldParent util. return err } } - if len(entries) < 1024 { + if !hasMore { break } } diff --git a/weed/server/filer_server_handlers_read_dir.go b/weed/server/filer_server_handlers_read_dir.go index ba01ce4f7..9cf79ab41 100644 --- a/weed/server/filer_server_handlers_read_dir.go +++ b/weed/server/filer_server_handlers_read_dir.go @@ -36,7 +36,7 @@ func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Reque lastFileName := r.FormValue("lastFileName") namePattern := r.FormValue("namePattern") - entries, err := fs.filer.ListDirectoryEntries(context.Background(), util.FullPath(path), lastFileName, false, limit, "", namePattern) + entries, shouldDisplayLoadMore, err := fs.filer.ListDirectoryEntries(context.Background(), util.FullPath(path), lastFileName, false, int64(limit), "", namePattern) if err != nil { glog.V(0).Infof("listDirectory %s %s %d: %s", path, lastFileName, limit, err) @@ -44,7 +44,6 @@ func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Reque return } - shouldDisplayLoadMore := len(entries) == limit if path == "/" { path = "" } From 16ad74f4779bd059017366843ada396d513d4fcc Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 14 Jan 2021 23:11:27 -0800 Subject: [PATCH 137/260] go fmt --- weed/filer/abstract_sql/abstract_sql_store.go | 2 +- weed/filer/cassandra/cassandra_store.go | 2 +- weed/filer/filer_search.go | 2 +- weed/filer/mongodb/mongodb_store.go | 4 ++-- weed/filer/mysql/mysql_store.go | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/weed/filer/abstract_sql/abstract_sql_store.go b/weed/filer/abstract_sql/abstract_sql_store.go index 33a237e87..bd1ac9734 100644 --- a/weed/filer/abstract_sql/abstract_sql_store.go +++ b/weed/filer/abstract_sql/abstract_sql_store.go @@ -203,7 +203,7 @@ func (store *AbstractSqlStore) ListDirectoryPrefixedEntries(ctx context.Context, entries = append(entries, entry) } - hasMore = int64(len(entries)) == limit + 1 + hasMore = int64(len(entries)) == limit+1 if hasMore { entries = entries[:limit] } diff --git a/weed/filer/cassandra/cassandra_store.go b/weed/filer/cassandra/cassandra_store.go index 2c815aa1f..06fb3af46 100644 --- a/weed/filer/cassandra/cassandra_store.go +++ b/weed/filer/cassandra/cassandra_store.go @@ -201,7 +201,7 @@ func (store *CassandraStore) ListDirectoryEntries(ctx context.Context, dirPath u glog.V(0).Infof("list iterator close: %v", err) } - hasMore = int64(len(entries)) == limit + 1 + hasMore = int64(len(entries)) == limit+1 if hasMore { entries = entries[:limit] } diff --git a/weed/filer/filer_search.go b/weed/filer/filer_search.go index d37699a5a..0fa57dd06 100644 --- a/weed/filer/filer_search.go +++ b/weed/filer/filer_search.go @@ -53,7 +53,7 @@ func (f *Filer) doListPatternMatchedEntries(ctx context.Context, p util.FullPath return } if len(restNamePattern) == 0 { - return foundEntries, false,0, lastFileName, nil + return foundEntries, false, 0, lastFileName, nil } for _, entry := range foundEntries { nameToTest := strings.ToLower(entry.Name()) diff --git a/weed/filer/mongodb/mongodb_store.go b/weed/filer/mongodb/mongodb_store.go index 2549b6029..2e9556f1f 100644 --- a/weed/filer/mongodb/mongodb_store.go +++ b/weed/filer/mongodb/mongodb_store.go @@ -190,7 +190,7 @@ func (store *MongodbStore) ListDirectoryEntries(ctx context.Context, dirPath uti "$gte": startFileName, } } - optLimit := int64(limit+1) + optLimit := int64(limit + 1) opts := &options.FindOptions{Limit: &optLimit, Sort: bson.M{"name": 1}} cur, err := store.connect.Database(store.database).Collection(store.collectionName).Find(ctx, where, opts) for cur.Next(ctx) { @@ -212,7 +212,7 @@ func (store *MongodbStore) ListDirectoryEntries(ctx context.Context, dirPath uti entries = append(entries, entry) } - hasMore = int64(len(entries)) == limit + 1 + hasMore = int64(len(entries)) == limit+1 if hasMore { entries = entries[:limit] } diff --git a/weed/filer/mysql/mysql_store.go b/weed/filer/mysql/mysql_store.go index b9f7f06dd..70f729fc9 100644 --- a/weed/filer/mysql/mysql_store.go +++ b/weed/filer/mysql/mysql_store.go @@ -42,7 +42,7 @@ func (store *MysqlStore) Initialize(configuration util.Configuration, prefix str } func (store *MysqlStore) initialize(user, password, hostname string, port int, database string, maxIdle, maxOpen, - maxLifetimeSeconds int, interpolateParams bool) (err error) { + maxLifetimeSeconds int, interpolateParams bool) (err error) { // store.SqlInsert = "INSERT INTO filemeta (dirhash,name,directory,meta) VALUES(?,?,?,?)" store.SqlUpdate = "UPDATE filemeta SET meta=? WHERE dirhash=? AND name=? AND directory=?" @@ -67,7 +67,7 @@ func (store *MysqlStore) initialize(user, password, hostname string, port int, d store.DB.SetMaxIdleConns(maxIdle) store.DB.SetMaxOpenConns(maxOpen) - store.DB.SetConnMaxLifetime(time.Duration(maxLifetimeSeconds)*time.Second) + store.DB.SetConnMaxLifetime(time.Duration(maxLifetimeSeconds) * time.Second) if err = store.DB.Ping(); err != nil { return fmt.Errorf("connect to %s error:%v", sqlUrl, err) From 01dc8a43ba007d21f97c40271b366e538f752227 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 15 Jan 2021 18:31:29 -0800 Subject: [PATCH 138/260] fix rocksdb --- weed/filer/rocksdb/rocksdb_store.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/weed/filer/rocksdb/rocksdb_store.go b/weed/filer/rocksdb/rocksdb_store.go index abeadb019..98023e82e 100644 --- a/weed/filer/rocksdb/rocksdb_store.go +++ b/weed/filer/rocksdb/rocksdb_store.go @@ -175,7 +175,7 @@ func (store *RocksDBStore) DeleteFolderChildren(ctx context.Context, fullpath we return nil } -func enumerate(iter *gorocksdb.Iterator, prefix, lastKey []byte, includeLastKey bool, limit int, fn func(key, value []byte) bool) (hasMore bool, err error) { +func enumerate(iter *gorocksdb.Iterator, prefix, lastKey []byte, includeLastKey bool, limit int64, fn func(key, value []byte) bool) (hasMore bool, err error) { if len(lastKey) == 0 { iter.Seek(prefix) @@ -190,7 +190,7 @@ func enumerate(iter *gorocksdb.Iterator, prefix, lastKey []byte, includeLastKey } } - i := 0 + i := int64(0) for ; iter.Valid(); iter.Next() { if limit > 0 { From a4063a5437f0554962bcdadefc96c9131f8c0395 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 15 Jan 2021 23:56:24 -0800 Subject: [PATCH 139/260] add stream list directory entries --- weed/filer/abstract_sql/abstract_sql_store.go | 23 +++---- weed/filer/cassandra/cassandra_store.go | 18 +++-- weed/filer/elastic/v7/elastic_store.go | 58 ++++++++-------- weed/filer/etcd/etcd_store.go | 16 +++-- weed/filer/filer.go | 17 ++--- weed/filer/filer_search.go | 67 +++++++++++++------ weed/filer/filerstore.go | 6 +- weed/filer/filerstore_translate_path.go | 24 +++---- weed/filer/filerstore_wrapper.go | 60 ++++++++--------- weed/filer/hbase/hbase_store.go | 21 +++--- weed/filer/leveldb/leveldb_store.go | 14 ++-- weed/filer/leveldb2/leveldb2_store.go | 14 ++-- weed/filer/leveldb3/leveldb3_store.go | 16 +++-- weed/filer/mongodb/mongodb_store.go | 21 +++--- weed/filer/redis/universal_redis_store.go | 18 ++--- weed/filer/redis2/universal_redis_store.go | 22 +++--- weed/filer/rocksdb/rocksdb_store.go | 26 +++---- weed/filesys/dir.go | 8 +-- weed/filesys/meta_cache/meta_cache.go | 16 ++--- weed/server/filer_grpc_server.go | 42 ++++++------ 20 files changed, 266 insertions(+), 241 deletions(-) diff --git a/weed/filer/abstract_sql/abstract_sql_store.go b/weed/filer/abstract_sql/abstract_sql_store.go index bd1ac9734..f5dcf6e03 100644 --- a/weed/filer/abstract_sql/abstract_sql_store.go +++ b/weed/filer/abstract_sql/abstract_sql_store.go @@ -172,7 +172,7 @@ func (store *AbstractSqlStore) DeleteFolderChildren(ctx context.Context, fullpat return nil } -func (store *AbstractSqlStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string) (entries []*filer.Entry, hasMore bool, err error) { +func (store *AbstractSqlStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { sqlText := store.SqlListExclusive if includeStartFile { sqlText = store.SqlListInclusive @@ -180,7 +180,7 @@ func (store *AbstractSqlStore) ListDirectoryPrefixedEntries(ctx context.Context, rows, err := store.getTxOrDB(ctx).QueryContext(ctx, sqlText, util.HashStringToLong(string(dirPath)), startFileName, string(dirPath), prefix+"%", limit+1) if err != nil { - return nil, false, fmt.Errorf("list %s : %v", dirPath, err) + return lastFileName, fmt.Errorf("list %s : %v", dirPath, err) } defer rows.Close() @@ -189,30 +189,29 @@ func (store *AbstractSqlStore) ListDirectoryPrefixedEntries(ctx context.Context, var data []byte if err = rows.Scan(&name, &data); err != nil { glog.V(0).Infof("scan %s : %v", dirPath, err) - return nil, false, fmt.Errorf("scan %s: %v", dirPath, err) + return lastFileName, fmt.Errorf("scan %s: %v", dirPath, err) } + lastFileName = name entry := &filer.Entry{ FullPath: util.NewFullPath(string(dirPath), name), } if err = entry.DecodeAttributesAndChunks(util.MaybeDecompressData(data)); err != nil { glog.V(0).Infof("scan decode %s : %v", entry.FullPath, err) - return nil, false, fmt.Errorf("scan decode %s : %v", entry.FullPath, err) + return lastFileName, fmt.Errorf("scan decode %s : %v", entry.FullPath, err) } - entries = append(entries, entry) - } + if !eachEntryFunc(entry) { + break + } - hasMore = int64(len(entries)) == limit+1 - if hasMore { - entries = entries[:limit] } - return entries, hasMore, nil + return lastFileName, nil } -func (store *AbstractSqlStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64) (entries []*filer.Entry, hasMore bool, err error) { - return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "") +func (store *AbstractSqlStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { + return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "", nil) } func (store *AbstractSqlStore) Shutdown() { diff --git a/weed/filer/cassandra/cassandra_store.go b/weed/filer/cassandra/cassandra_store.go index 06fb3af46..fd2ce91a6 100644 --- a/weed/filer/cassandra/cassandra_store.go +++ b/weed/filer/cassandra/cassandra_store.go @@ -168,11 +168,11 @@ func (store *CassandraStore) DeleteFolderChildren(ctx context.Context, fullpath return nil } -func (store *CassandraStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string) (entries []*filer.Entry, hasMore bool, err error) { - return nil, false, filer.ErrUnsupportedListDirectoryPrefixed +func (store *CassandraStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { + return lastFileName, filer.ErrUnsupportedListDirectoryPrefixed } -func (store *CassandraStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64) (entries []*filer.Entry, hasMore bool, err error) { +func (store *CassandraStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { if _, ok := store.isSuperLargeDirectory(string(dirPath)); ok { return // nil, filer.ErrUnsupportedSuperLargeDirectoryListing @@ -190,23 +190,21 @@ func (store *CassandraStore) ListDirectoryEntries(ctx context.Context, dirPath u entry := &filer.Entry{ FullPath: util.NewFullPath(string(dirPath), name), } + lastFileName = name if decodeErr := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(data)); decodeErr != nil { err = decodeErr glog.V(0).Infof("list %s : %v", entry.FullPath, err) break } - entries = append(entries, entry) + if !eachEntryFunc(entry) { + break + } } if err := iter.Close(); err != nil { glog.V(0).Infof("list iterator close: %v", err) } - hasMore = int64(len(entries)) == limit+1 - if hasMore { - entries = entries[:limit] - } - - return entries, hasMore, err + return lastFileName, err } func (store *CassandraStore) Shutdown() { diff --git a/weed/filer/elastic/v7/elastic_store.go b/weed/filer/elastic/v7/elastic_store.go index 0e055e1fc..1e7f55599 100644 --- a/weed/filer/elastic/v7/elastic_store.go +++ b/weed/filer/elastic/v7/elastic_store.go @@ -96,8 +96,8 @@ func (store *ElasticStore) RollbackTransaction(ctx context.Context) error { return nil } -func (store *ElasticStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string) (entries []*filer.Entry, hasMore bool, err error) { - return nil, false, filer.ErrUnsupportedListDirectoryPrefixed +func (store *ElasticStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { + return lastFileName, filer.ErrUnsupportedListDirectoryPrefixed } func (store *ElasticStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) { @@ -187,26 +187,28 @@ func (store *ElasticStore) deleteEntry(ctx context.Context, index, id string) (e } func (store *ElasticStore) DeleteFolderChildren(ctx context.Context, fullpath weed_util.FullPath) (err error) { - if entries, _, err := store.ListDirectoryEntries(ctx, fullpath, "", false, math.MaxInt32); err == nil { - for _, entry := range entries { - store.DeleteEntry(ctx, entry.FullPath) + _, err = store.ListDirectoryEntries(ctx, fullpath, "", false, math.MaxInt32, func(entry *filer.Entry) bool { + if err := store.DeleteEntry(ctx, entry.FullPath); err != nil { + glog.Errorf("elastic delete %s: %v.", entry.FullPath, err) + return false } - } - return nil + return true + }) + return } -func (store *ElasticStore) ListDirectoryEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64) (entries []*filer.Entry, hasMore bool, err error) { +func (store *ElasticStore) ListDirectoryEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { if string(dirPath) == "/" { - return store.listRootDirectoryEntries(ctx, startFileName, includeStartFile, limit) + return store.listRootDirectoryEntries(ctx, startFileName, includeStartFile, limit, eachEntryFunc) } - return store.listDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit) + return store.listDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit, eachEntryFunc) } -func (store *ElasticStore) listRootDirectoryEntries(ctx context.Context, startFileName string, inclusive bool, limit int64) (entries []*filer.Entry, hasMore bool, err error) { +func (store *ElasticStore) listRootDirectoryEntries(ctx context.Context, startFileName string, inclusive bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { indexResult, err := store.client.CatIndices().Do(ctx) if err != nil { glog.Errorf("list indices %v.", err) - return entries, false, err + return } for _, index := range indexResult { if index.Index == indexKV { @@ -216,32 +218,33 @@ func (store *ElasticStore) listRootDirectoryEntries(ctx context.Context, startFi if entry, err := store.FindEntry(ctx, weed_util.FullPath("/"+strings.Replace(index.Index, indexPrefix, "", 1))); err == nil { fileName := getFileName(entry.FullPath) + lastFileName = fileName if fileName == startFileName && !inclusive { continue } limit-- if limit < 0 { - hasMore = true break } - entries = append(entries, entry) + if !eachEntryFunc(entry) { + break + } } } } - return entries, hasMore, nil + return } func (store *ElasticStore) listDirectoryEntries( - ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int64, -) (entries []*filer.Entry, hasMore bool, err error) { + ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { first := true index := getIndex(fullpath) nextStart := "" parentId := weed_util.Md5String([]byte(fullpath)) - if _, err := store.client.Refresh(index).Do(ctx); err != nil { + if _, err = store.client.Refresh(index).Do(ctx); err != nil { if elastic.IsNotFound(err) { store.client.CreateIndex(index).Do(ctx) - return entries, hasMore, nil + return } } for { @@ -249,7 +252,7 @@ func (store *ElasticStore) listDirectoryEntries( if (startFileName == "" && first) || inclusive { if result, err = store.search(ctx, index, parentId); err != nil { glog.Errorf("search (%s,%s,%t,%d) %v.", string(fullpath), startFileName, inclusive, limit, err) - return entries, hasMore, err + return } } else { fullPath := string(fullpath) + "/" + startFileName @@ -259,7 +262,7 @@ func (store *ElasticStore) listDirectoryEntries( after := weed_util.Md5String([]byte(fullPath)) if result, err = store.searchAfter(ctx, index, parentId, after); err != nil { glog.Errorf("searchAfter (%s,%s,%t,%d) %v.", string(fullpath), startFileName, inclusive, limit, err) - return entries, hasMore, err + return } } first = false @@ -271,22 +274,21 @@ func (store *ElasticStore) listDirectoryEntries( if err := jsoniter.Unmarshal(hit.Source, esEntry); err == nil { limit-- if limit < 0 { - hasMore = true - return entries, hasMore, nil + return lastFileName, nil } nextStart = string(esEntry.Entry.FullPath) fileName := getFileName(esEntry.Entry.FullPath) + lastFileName = fileName if fileName == startFileName && !inclusive { continue } - entries = append(entries, esEntry.Entry) + if !eachEntryFunc(esEntry.Entry) { + break + } } } - if len(result.Hits.Hits) < store.maxPageSize { - break - } } - return entries, hasMore, nil + return } func (store *ElasticStore) search(ctx context.Context, index, parentId string) (result *elastic.SearchResult, err error) { diff --git a/weed/filer/etcd/etcd_store.go b/weed/filer/etcd/etcd_store.go index b7acc2049..8159c634d 100644 --- a/weed/filer/etcd/etcd_store.go +++ b/weed/filer/etcd/etcd_store.go @@ -139,17 +139,17 @@ func (store *EtcdStore) DeleteFolderChildren(ctx context.Context, fullpath weed_ return nil } -func (store *EtcdStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string) (entries []*filer.Entry, hasMore bool, err error) { - return nil, false, filer.ErrUnsupportedListDirectoryPrefixed +func (store *EtcdStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { + return lastFileName, filer.ErrUnsupportedListDirectoryPrefixed } -func (store *EtcdStore) ListDirectoryEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64) (entries []*filer.Entry, hasMore bool, err error) { +func (store *EtcdStore) ListDirectoryEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { directoryPrefix := genDirectoryKeyPrefix(dirPath, "") resp, err := store.client.Get(ctx, string(directoryPrefix), clientv3.WithPrefix(), clientv3.WithSort(clientv3.SortByKey, clientv3.SortDescend)) if err != nil { - return nil, false, fmt.Errorf("list %s : %v", dirPath, err) + return lastFileName, fmt.Errorf("list %s : %v", dirPath, err) } for _, kv := range resp.Kvs { @@ -160,9 +160,9 @@ func (store *EtcdStore) ListDirectoryEntries(ctx context.Context, dirPath weed_u if fileName == startFileName && !includeStartFile { continue } + lastFileName = fileName limit-- if limit < 0 { - hasMore = true break } entry := &filer.Entry{ @@ -173,10 +173,12 @@ func (store *EtcdStore) ListDirectoryEntries(ctx context.Context, dirPath weed_u glog.V(0).Infof("list %s : %v", entry.FullPath, err) break } - entries = append(entries, entry) + if !eachEntryFunc(entry) { + break + } } - return entries, hasMore, err + return lastFileName, err } func genKey(dirPath, fileName string) (key []byte) { diff --git a/weed/filer/filer.go b/weed/filer/filer.go index dd0fcf2cf..e59887763 100644 --- a/weed/filer/filer.go +++ b/weed/filer/filer.go @@ -281,22 +281,19 @@ func (f *Filer) FindEntry(ctx context.Context, p util.FullPath) (entry *Entry, e } -func (f *Filer) doListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int64, prefix string) (entries []*Entry, hasMore bool, expiredCount int64, lastFileName string, err error) { - listedEntries, listHasMore, listErr := f.Store.ListDirectoryPrefixedEntries(ctx, p, startFileName, inclusive, limit, prefix) - hasMore = listHasMore - if listErr != nil { - return listedEntries, hasMore, expiredCount, "", listErr - } - for _, entry := range listedEntries { - lastFileName = entry.Name() +func (f *Filer) doListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int64, prefix string, eachEntryFunc ListEachEntryFunc) (expiredCount int64, lastFileName string, err error) { + lastFileName, err = f.Store.ListDirectoryPrefixedEntries(ctx, p, startFileName, inclusive, limit, prefix, func(entry *Entry) bool { if entry.TtlSec > 0 { if entry.Crtime.Add(time.Duration(entry.TtlSec) * time.Second).Before(time.Now()) { f.Store.DeleteOneEntry(ctx, entry) expiredCount++ - continue + return true } } - entries = append(entries, entry) + return eachEntryFunc(entry) + }) + if err != nil { + return expiredCount, lastFileName, err } return } diff --git a/weed/filer/filer_search.go b/weed/filer/filer_search.go index 0fa57dd06..8c9688ceb 100644 --- a/weed/filer/filer_search.go +++ b/weed/filer/filer_search.go @@ -32,49 +32,76 @@ func (f *Filer) ListDirectoryEntries(ctx context.Context, p util.FullPath, start var missedCount int64 var lastFileName string - entries, hasMore, missedCount, lastFileName, err = f.doListPatternMatchedEntries(ctx, p, startFileName, inclusive, limit, prefix, restNamePattern) + missedCount, lastFileName, err = f.doListPatternMatchedEntries(ctx, p, startFileName, inclusive, limit+1, prefix, restNamePattern, func(entry *Entry) bool { + entries = append(entries, entry) + return true + }) for missedCount > 0 && err == nil { - var makeupEntries []*Entry - makeupEntries, hasMore, missedCount, lastFileName, err = f.doListPatternMatchedEntries(ctx, p, lastFileName, false, missedCount, prefix, restNamePattern) - for _, entry := range makeupEntries { + missedCount, lastFileName, err = f.doListPatternMatchedEntries(ctx, p, lastFileName, false, missedCount+1, prefix, restNamePattern, func(entry *Entry) bool { entries = append(entries, entry) - } + return true + }) + } + + hasMore = int64(len(entries)) >= limit+1 + if hasMore { + entries = entries[:limit] } return entries, hasMore, err } -func (f *Filer) doListPatternMatchedEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int64, prefix, restNamePattern string) (matchedEntries []*Entry, hasMore bool, missedCount int64, lastFileName string, err error) { - var foundEntries []*Entry +// For now, prefix and namePattern are mutually exclusive +func (f *Filer) StreamListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int64, prefix string, namePattern string, eachEntryFunc ListEachEntryFunc) (lastFileName string, err error) { + if strings.HasSuffix(string(p), "/") && len(p) > 1 { + p = p[0 : len(p)-1] + } - foundEntries, hasMore, lastFileName, err = f.doListValidEntries(ctx, p, startFileName, inclusive, limit, prefix) - if err != nil { - return + prefixInNamePattern, restNamePattern := splitPattern(namePattern) + if prefixInNamePattern != "" { + prefix = prefixInNamePattern + } + var missedCount int64 + + missedCount, lastFileName, err = f.doListPatternMatchedEntries(ctx, p, startFileName, inclusive, limit, prefix, restNamePattern, eachEntryFunc) + + for missedCount > 0 && err == nil { + missedCount, lastFileName, err = f.doListPatternMatchedEntries(ctx, p, lastFileName, false, missedCount, prefix, restNamePattern, eachEntryFunc) } + + return +} + +func (f *Filer) doListPatternMatchedEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int64, prefix, restNamePattern string, eachEntryFunc ListEachEntryFunc) (missedCount int64, lastFileName string, err error) { + if len(restNamePattern) == 0 { - return foundEntries, false, 0, lastFileName, nil + lastFileName, err = f.doListValidEntries(ctx, p, startFileName, inclusive, limit, prefix, eachEntryFunc) + return 0, lastFileName, err } - for _, entry := range foundEntries { + + lastFileName, err = f.doListValidEntries(ctx, p, startFileName, inclusive, limit, prefix, func(entry *Entry) bool { nameToTest := strings.ToLower(entry.Name()) if matched, matchErr := filepath.Match(restNamePattern, nameToTest[len(prefix):]); matchErr == nil && matched { - matchedEntries = append(matchedEntries, entry) + if !eachEntryFunc(entry) { + return false + } } else { missedCount++ } + return true + }) + if err != nil { + return } return } -func (f *Filer) doListValidEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int64, prefix string) (entries []*Entry, hasMore bool, lastFileName string, err error) { - var makeupEntries []*Entry +func (f *Filer) doListValidEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int64, prefix string, eachEntryFunc ListEachEntryFunc) (lastFileName string, err error) { var expiredCount int64 - entries, hasMore, expiredCount, lastFileName, err = f.doListDirectoryEntries(ctx, p, startFileName, inclusive, limit, prefix) + expiredCount, lastFileName, err = f.doListDirectoryEntries(ctx, p, startFileName, inclusive, limit, prefix, eachEntryFunc) for expiredCount > 0 && err == nil { - makeupEntries, hasMore, expiredCount, lastFileName, err = f.doListDirectoryEntries(ctx, p, lastFileName, false, expiredCount, prefix) - if err == nil { - entries = append(entries, makeupEntries...) - } + expiredCount, lastFileName, err = f.doListDirectoryEntries(ctx, p, lastFileName, false, expiredCount, prefix, eachEntryFunc) } return } diff --git a/weed/filer/filerstore.go b/weed/filer/filerstore.go index a28c4d01b..8955a25c7 100644 --- a/weed/filer/filerstore.go +++ b/weed/filer/filerstore.go @@ -13,6 +13,8 @@ var ( ErrKvNotFound = errors.New("kv: not found") ) +type ListEachEntryFunc func(entry *Entry) bool + type FilerStore interface { // GetName gets the name to locate the configuration in filer.toml file GetName() string @@ -24,8 +26,8 @@ type FilerStore interface { FindEntry(context.Context, util.FullPath) (entry *Entry, err error) DeleteEntry(context.Context, util.FullPath) (err error) DeleteFolderChildren(context.Context, util.FullPath) (err error) - ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64) ([]*Entry, bool, error) - ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string) ([]*Entry, bool, error) + ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc ListEachEntryFunc) (lastFileName string, err error) + ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc ListEachEntryFunc) (lastFileName string, err error) BeginTransaction(ctx context.Context) (context.Context, error) CommitTransaction(ctx context.Context) error diff --git a/weed/filer/filerstore_translate_path.go b/weed/filer/filerstore_translate_path.go index 0e2ddd914..00bf82ed4 100644 --- a/weed/filer/filerstore_translate_path.go +++ b/weed/filer/filerstore_translate_path.go @@ -106,32 +106,24 @@ func (t *FilerStorePathTranlator) DeleteFolderChildren(ctx context.Context, fp u return t.actualStore.DeleteFolderChildren(ctx, newFullPath) } -func (t *FilerStorePathTranlator) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64) ([]*Entry, bool, error) { +func (t *FilerStorePathTranlator) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc ListEachEntryFunc) (string, error) { newFullPath := t.translatePath(dirPath) - entries, hasMore, err := t.actualStore.ListDirectoryEntries(ctx, newFullPath, startFileName, includeStartFile, limit) - if err != nil { - return nil, hasMore, err - } - for _, entry := range entries { + return t.actualStore.ListDirectoryEntries(ctx, newFullPath, startFileName, includeStartFile, limit, func(entry *Entry) bool { entry.FullPath = dirPath[:len(t.storeRoot)-1] + entry.FullPath - } - return entries, hasMore, err + return eachEntryFunc(entry) + }) } -func (t *FilerStorePathTranlator) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string) ([]*Entry, bool, error) { +func (t *FilerStorePathTranlator) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc ListEachEntryFunc) (string, error) { newFullPath := t.translatePath(dirPath) - entries, hasMore, err := t.actualStore.ListDirectoryPrefixedEntries(ctx, newFullPath, startFileName, includeStartFile, limit, prefix) - if err != nil { - return nil, hasMore, err - } - for _, entry := range entries { + return t.actualStore.ListDirectoryPrefixedEntries(ctx, newFullPath, startFileName, includeStartFile, limit, prefix, func(entry *Entry) bool { entry.FullPath = dirPath[:len(t.storeRoot)-1] + entry.FullPath - } - return entries, hasMore, nil + return eachEntryFunc(entry) + }) } func (t *FilerStorePathTranlator) BeginTransaction(ctx context.Context) (context.Context, error) { diff --git a/weed/filer/filerstore_wrapper.go b/weed/filer/filerstore_wrapper.go index 2bb7793f0..64baac371 100644 --- a/weed/filer/filerstore_wrapper.go +++ b/weed/filer/filerstore_wrapper.go @@ -194,7 +194,7 @@ func (fsw *FilerStoreWrapper) DeleteFolderChildren(ctx context.Context, fp util. return actualStore.DeleteFolderChildren(ctx, fp) } -func (fsw *FilerStoreWrapper) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64) ([]*Entry, bool, error) { +func (fsw *FilerStoreWrapper) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc ListEachEntryFunc) (string, error) { actualStore := fsw.getActualStore(dirPath + "/") stats.FilerStoreCounter.WithLabelValues(actualStore.GetName(), "list").Inc() start := time.Now() @@ -203,18 +203,14 @@ func (fsw *FilerStoreWrapper) ListDirectoryEntries(ctx context.Context, dirPath }() glog.V(4).Infof("ListDirectoryEntries %s from %s limit %d", dirPath, startFileName, limit) - entries, hasMore, err := actualStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit) - if err != nil { - return nil, hasMore, err - } - for _, entry := range entries { + return actualStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit, func(entry *Entry) bool { fsw.maybeReadHardLink(ctx, entry) filer_pb.AfterEntryDeserialization(entry.Chunks) - } - return entries, hasMore, err + return eachEntryFunc(entry) + }) } -func (fsw *FilerStoreWrapper) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string) ([]*Entry, bool, error) { +func (fsw *FilerStoreWrapper) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc ListEachEntryFunc) (lastFileName string, err error) { actualStore := fsw.getActualStore(dirPath + "/") stats.FilerStoreCounter.WithLabelValues(actualStore.GetName(), "prefixList").Inc() start := time.Now() @@ -222,48 +218,52 @@ func (fsw *FilerStoreWrapper) ListDirectoryPrefixedEntries(ctx context.Context, stats.FilerStoreHistogram.WithLabelValues(actualStore.GetName(), "prefixList").Observe(time.Since(start).Seconds()) }() glog.V(4).Infof("ListDirectoryPrefixedEntries %s from %s prefix %s limit %d", dirPath, startFileName, prefix, limit) - entries, hasMore, err := actualStore.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, prefix) + lastFileName, err = actualStore.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, prefix, eachEntryFunc) if err == ErrUnsupportedListDirectoryPrefixed { - entries, hasMore, err = fsw.prefixFilterEntries(ctx, dirPath, startFileName, includeStartFile, limit, prefix) - } - if err != nil { - return nil, hasMore, err - } - for _, entry := range entries { - fsw.maybeReadHardLink(ctx, entry) - filer_pb.AfterEntryDeserialization(entry.Chunks) + lastFileName, err = fsw.prefixFilterEntries(ctx, dirPath, startFileName, includeStartFile, limit, prefix, func(entry *Entry) bool { + fsw.maybeReadHardLink(ctx, entry) + filer_pb.AfterEntryDeserialization(entry.Chunks) + return eachEntryFunc(entry) + }) } - return entries, hasMore, nil + return lastFileName, err } -func (fsw *FilerStoreWrapper) prefixFilterEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string) (entries []*Entry, hasMore bool, err error) { +func (fsw *FilerStoreWrapper) prefixFilterEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc ListEachEntryFunc) (lastFileName string, err error) { actualStore := fsw.getActualStore(dirPath + "/") - entries, hasMore, err = actualStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit) - if err != nil { - return nil, hasMore, err - } if prefix == "" { + return actualStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit, eachEntryFunc) + } + + var notPrefixed []*Entry + lastFileName, err = actualStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit, func(entry *Entry) bool { + notPrefixed = append(notPrefixed, entry) + return true + }) + if err != nil { return } count := int64(0) - var lastFileName string - notPrefixed := entries - entries = nil for count < limit && len(notPrefixed) > 0 { for _, entry := range notPrefixed { - lastFileName = entry.Name() if strings.HasPrefix(entry.Name(), prefix) { count++ - entries = append(entries, entry) + if !eachEntryFunc(entry) { + return + } if count >= limit { break } } } if count < limit { - notPrefixed, hasMore, err = actualStore.ListDirectoryEntries(ctx, dirPath, lastFileName, false, limit) + notPrefixed = notPrefixed[:0] + _, err = actualStore.ListDirectoryEntries(ctx, dirPath, lastFileName, false, limit, func(entry *Entry) bool { + notPrefixed = append(notPrefixed, entry) + return true + }) if err != nil { return } diff --git a/weed/filer/hbase/hbase_store.go b/weed/filer/hbase/hbase_store.go index c93374b15..2e4491515 100644 --- a/weed/filer/hbase/hbase_store.go +++ b/weed/filer/hbase/hbase_store.go @@ -148,20 +148,18 @@ func (store *HbaseStore) DeleteFolderChildren(ctx context.Context, path util.Ful return } -func (store *HbaseStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64) ([]*filer.Entry, bool, error) { - return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "") +func (store *HbaseStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (string, error) { + return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "", eachEntryFunc) } -func (store *HbaseStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string) ([]*filer.Entry, bool, error) { +func (store *HbaseStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { family := map[string][]string{store.cfMetaDir: {COLUMN_NAME}} expectedPrefix := []byte(dirPath.Child(prefix)) scan, err := hrpc.NewScanRange(ctx, store.table, expectedPrefix, nil, hrpc.Families(family)) if err != nil { - return nil, false, err + return lastFileName, err } - var hasMore bool - var entries []*filer.Entry scanner := store.Client.Scan(scan) defer scanner.Close() for { @@ -170,7 +168,7 @@ func (store *HbaseStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPa break } if err != nil { - return entries, hasMore, err + return lastFileName, err } if len(res.Cells) == 0 { continue @@ -187,6 +185,8 @@ func (store *HbaseStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPa continue } + lastFileName = fileName + value := cell.Value if fileName == startFileName && !includeStartFile { @@ -195,7 +195,6 @@ func (store *HbaseStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPa limit-- if limit < 0 { - hasMore = true break } entry := &filer.Entry{ @@ -206,10 +205,12 @@ func (store *HbaseStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPa glog.V(0).Infof("list %s : %v", entry.FullPath, err) break } - entries = append(entries, entry) + if !eachEntryFunc(entry) { + break + } } - return entries, hasMore, nil + return lastFileName, nil } func (store *HbaseStore) BeginTransaction(ctx context.Context) (context.Context, error) { diff --git a/weed/filer/leveldb/leveldb_store.go b/weed/filer/leveldb/leveldb_store.go index c968ca2da..f0ae64769 100644 --- a/weed/filer/leveldb/leveldb_store.go +++ b/weed/filer/leveldb/leveldb_store.go @@ -162,11 +162,11 @@ func (store *LevelDBStore) DeleteFolderChildren(ctx context.Context, fullpath we return nil } -func (store *LevelDBStore) ListDirectoryEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64) (entries []*filer.Entry, hasMore bool, err error) { - return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "") +func (store *LevelDBStore) ListDirectoryEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { + return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "", eachEntryFunc) } -func (store *LevelDBStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string) (entries []*filer.Entry, hasMore bool, err error) { +func (store *LevelDBStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { directoryPrefix := genDirectoryKeyPrefix(dirPath, prefix) lastFileStart := directoryPrefix @@ -187,9 +187,9 @@ func (store *LevelDBStore) ListDirectoryPrefixedEntries(ctx context.Context, dir if fileName == startFileName && !includeStartFile { continue } + lastFileName = fileName limit-- if limit < 0 { - hasMore = true break } entry := &filer.Entry{ @@ -200,11 +200,13 @@ func (store *LevelDBStore) ListDirectoryPrefixedEntries(ctx context.Context, dir glog.V(0).Infof("list %s : %v", entry.FullPath, err) break } - entries = append(entries, entry) + if !eachEntryFunc(entry) { + break + } } iter.Release() - return entries, hasMore, err + return lastFileName, err } func genKey(dirPath, fileName string) (key []byte) { diff --git a/weed/filer/leveldb2/leveldb2_store.go b/weed/filer/leveldb2/leveldb2_store.go index cc0d67eac..965721460 100644 --- a/weed/filer/leveldb2/leveldb2_store.go +++ b/weed/filer/leveldb2/leveldb2_store.go @@ -171,11 +171,11 @@ func (store *LevelDB2Store) DeleteFolderChildren(ctx context.Context, fullpath w return nil } -func (store *LevelDB2Store) ListDirectoryEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64) (entries []*filer.Entry, hasMore bool, err error) { - return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "") +func (store *LevelDB2Store) ListDirectoryEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { + return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "", eachEntryFunc) } -func (store *LevelDB2Store) ListDirectoryPrefixedEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string) (entries []*filer.Entry, hasMore bool, err error) { +func (store *LevelDB2Store) ListDirectoryPrefixedEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { directoryPrefix, partitionId := genDirectoryKeyPrefix(dirPath, prefix, store.dbCount) lastFileStart := directoryPrefix @@ -196,9 +196,9 @@ func (store *LevelDB2Store) ListDirectoryPrefixedEntries(ctx context.Context, di if fileName == startFileName && !includeStartFile { continue } + lastFileName = fileName limit-- if limit < 0 { - hasMore = true break } entry := &filer.Entry{ @@ -211,11 +211,13 @@ func (store *LevelDB2Store) ListDirectoryPrefixedEntries(ctx context.Context, di glog.V(0).Infof("list %s : %v", entry.FullPath, err) break } - entries = append(entries, entry) + if !eachEntryFunc(entry) { + break + } } iter.Release() - return entries, hasMore, err + return lastFileName, err } func genKey(dirPath, fileName string, dbCount int) (key []byte, partitionId int) { diff --git a/weed/filer/leveldb3/leveldb3_store.go b/weed/filer/leveldb3/leveldb3_store.go index afbcabaa6..24e00edc7 100644 --- a/weed/filer/leveldb3/leveldb3_store.go +++ b/weed/filer/leveldb3/leveldb3_store.go @@ -286,15 +286,15 @@ func (store *LevelDB3Store) DeleteFolderChildren(ctx context.Context, fullpath w return nil } -func (store *LevelDB3Store) ListDirectoryEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64) (entries []*filer.Entry, hasMore bool, err error) { - return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "") +func (store *LevelDB3Store) ListDirectoryEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { + return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "", eachEntryFunc) } -func (store *LevelDB3Store) ListDirectoryPrefixedEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string) (entries []*filer.Entry, hasMore bool, err error) { +func (store *LevelDB3Store) ListDirectoryPrefixedEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { db, _, shortPath, err := store.findDB(dirPath, true) if err != nil { - return nil, false, fmt.Errorf("findDB %s : %v", dirPath, err) + return lastFileName, fmt.Errorf("findDB %s : %v", dirPath, err) } directoryPrefix := genDirectoryKeyPrefix(shortPath, prefix) @@ -316,9 +316,9 @@ func (store *LevelDB3Store) ListDirectoryPrefixedEntries(ctx context.Context, di if fileName == startFileName && !includeStartFile { continue } + lastFileName = fileName limit-- if limit < 0 { - hasMore = true break } entry := &filer.Entry{ @@ -331,11 +331,13 @@ func (store *LevelDB3Store) ListDirectoryPrefixedEntries(ctx context.Context, di glog.V(0).Infof("list %s : %v", entry.FullPath, err) break } - entries = append(entries, entry) + if !eachEntryFunc(entry) { + break + } } iter.Release() - return entries, hasMore, err + return lastFileName, err } func genKey(dirPath, fileName string) (key []byte) { diff --git a/weed/filer/mongodb/mongodb_store.go b/weed/filer/mongodb/mongodb_store.go index 2e9556f1f..1ef5056f4 100644 --- a/weed/filer/mongodb/mongodb_store.go +++ b/weed/filer/mongodb/mongodb_store.go @@ -178,11 +178,11 @@ func (store *MongodbStore) DeleteFolderChildren(ctx context.Context, fullpath ut return nil } -func (store *MongodbStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string) (entries []*filer.Entry, hasMore bool, err error) { - return nil, false, filer.ErrUnsupportedListDirectoryPrefixed +func (store *MongodbStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { + return lastFileName, filer.ErrUnsupportedListDirectoryPrefixed } -func (store *MongodbStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64) (entries []*filer.Entry, hasMore bool, err error) { +func (store *MongodbStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { var where = bson.M{"directory": string(dirPath), "name": bson.M{"$gt": startFileName}} if includeStartFile { @@ -190,38 +190,37 @@ func (store *MongodbStore) ListDirectoryEntries(ctx context.Context, dirPath uti "$gte": startFileName, } } - optLimit := int64(limit + 1) + optLimit := int64(limit) opts := &options.FindOptions{Limit: &optLimit, Sort: bson.M{"name": 1}} cur, err := store.connect.Database(store.database).Collection(store.collectionName).Find(ctx, where, opts) for cur.Next(ctx) { var data Model err := cur.Decode(&data) if err != nil && err != mongo.ErrNoDocuments { - return nil, false, err + return lastFileName, err } entry := &filer.Entry{ FullPath: util.NewFullPath(string(dirPath), data.Name), } + lastFileName = data.Name if decodeErr := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(data.Meta)); decodeErr != nil { err = decodeErr glog.V(0).Infof("list %s : %v", entry.FullPath, err) break } - entries = append(entries, entry) - } + if !eachEntryFunc(entry) { + break + } - hasMore = int64(len(entries)) == limit+1 - if hasMore { - entries = entries[:limit] } if err := cur.Close(ctx); err != nil { glog.V(0).Infof("list iterator close: %v", err) } - return entries, hasMore, err + return lastFileName, err } func (store *MongodbStore) Shutdown() { diff --git a/weed/filer/redis/universal_redis_store.go b/weed/filer/redis/universal_redis_store.go index 8399b4e99..30d11a7f4 100644 --- a/weed/filer/redis/universal_redis_store.go +++ b/weed/filer/redis/universal_redis_store.go @@ -125,16 +125,16 @@ func (store *UniversalRedisStore) DeleteFolderChildren(ctx context.Context, full return nil } -func (store *UniversalRedisStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string) (entries []*filer.Entry, hasMore bool, err error) { - return nil, false, filer.ErrUnsupportedListDirectoryPrefixed +func (store *UniversalRedisStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { + return lastFileName, filer.ErrUnsupportedListDirectoryPrefixed } -func (store *UniversalRedisStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64) (entries []*filer.Entry, hasMore bool, err error) { +func (store *UniversalRedisStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { dirListKey := genDirectoryListKey(string(dirPath)) members, err := store.Client.SMembers(ctx, dirListKey).Result() if err != nil { - return nil, false, fmt.Errorf("list %s : %v", dirPath, err) + return lastFileName, fmt.Errorf("list %s : %v", dirPath, err) } // skip @@ -160,15 +160,15 @@ func (store *UniversalRedisStore) ListDirectoryEntries(ctx context.Context, dirP }) // limit - if limit < int64(len(entries)) { + if limit < int64(len(members)) { members = members[:limit] - hasMore = true } // fetch entry meta for _, fileName := range members { path := util.NewFullPath(string(dirPath), fileName) entry, err := store.FindEntry(ctx, path) + lastFileName = fileName if err != nil { glog.V(0).Infof("list %s : %v", path, err) if err == filer_pb.ErrNotFound { @@ -182,11 +182,13 @@ func (store *UniversalRedisStore) ListDirectoryEntries(ctx context.Context, dirP continue } } - entries = append(entries, entry) + if !eachEntryFunc(entry) { + break + } } } - return entries, hasMore, err + return lastFileName, err } func genDirectoryListKey(dir string) (dirList string) { diff --git a/weed/filer/redis2/universal_redis_store.go b/weed/filer/redis2/universal_redis_store.go index 7b4e9d325..aab3d1f4a 100644 --- a/weed/filer/redis2/universal_redis_store.go +++ b/weed/filer/redis2/universal_redis_store.go @@ -149,11 +149,11 @@ func (store *UniversalRedis2Store) DeleteFolderChildren(ctx context.Context, ful return nil } -func (store *UniversalRedis2Store) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string) (entries []*filer.Entry, hasMore bool, err error) { - return nil, false, filer.ErrUnsupportedListDirectoryPrefixed +func (store *UniversalRedis2Store) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { + return lastFileName, filer.ErrUnsupportedListDirectoryPrefixed } -func (store *UniversalRedis2Store) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64) (entries []*filer.Entry, hasMore bool, err error) { +func (store *UniversalRedis2Store) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { dirListKey := genDirectoryListKey(string(dirPath)) start := int64(0) @@ -163,20 +163,16 @@ func (store *UniversalRedis2Store) ListDirectoryEntries(ctx context.Context, dir start++ } } - members, err := store.Client.ZRange(ctx, dirListKey, start, start+int64(limit)-1+1).Result() + members, err := store.Client.ZRange(ctx, dirListKey, start, start+int64(limit)-1).Result() if err != nil { - return nil, false, fmt.Errorf("list %s : %v", dirPath, err) - } - - hasMore = int64(len(entries)) == limit+1 - if hasMore { - members = members[:len(members)-1] + return lastFileName, fmt.Errorf("list %s : %v", dirPath, err) } // fetch entry meta for _, fileName := range members { path := util.NewFullPath(string(dirPath), fileName) entry, err := store.FindEntry(ctx, path) + lastFileName = fileName if err != nil { glog.V(0).Infof("list %s : %v", path, err) if err == filer_pb.ErrNotFound { @@ -190,11 +186,13 @@ func (store *UniversalRedis2Store) ListDirectoryEntries(ctx context.Context, dir continue } } - entries = append(entries, entry) + if !eachEntryFunc(entry) { + break + } } } - return entries, hasMore, err + return lastFileName, err } func genDirectoryListKey(dir string) (dirList string) { diff --git a/weed/filer/rocksdb/rocksdb_store.go b/weed/filer/rocksdb/rocksdb_store.go index 98023e82e..70c301725 100644 --- a/weed/filer/rocksdb/rocksdb_store.go +++ b/weed/filer/rocksdb/rocksdb_store.go @@ -158,7 +158,7 @@ func (store *RocksDBStore) DeleteFolderChildren(ctx context.Context, fullpath we iter := store.db.NewIterator(ro) defer iter.Close() - _, err = enumerate(iter, directoryPrefix, nil, false, -1, func(key, value []byte) bool { + err = enumerate(iter, directoryPrefix, nil, false, -1, func(key, value []byte) bool { batch.Delete(key) return true }) @@ -175,7 +175,7 @@ func (store *RocksDBStore) DeleteFolderChildren(ctx context.Context, fullpath we return nil } -func enumerate(iter *gorocksdb.Iterator, prefix, lastKey []byte, includeLastKey bool, limit int64, fn func(key, value []byte) bool) (hasMore bool, err error) { +func enumerate(iter *gorocksdb.Iterator, prefix, lastKey []byte, includeLastKey bool, limit int64, fn func(key, value []byte) bool) (err error) { if len(lastKey) == 0 { iter.Seek(prefix) @@ -196,7 +196,6 @@ func enumerate(iter *gorocksdb.Iterator, prefix, lastKey []byte, includeLastKey if limit > 0 { i++ if i > limit { - hasMore = true break } } @@ -216,16 +215,16 @@ func enumerate(iter *gorocksdb.Iterator, prefix, lastKey []byte, includeLastKey } if err := iter.Err(); err != nil { - return hasMore, fmt.Errorf("prefix scan iterator: %v", err) + return fmt.Errorf("prefix scan iterator: %v", err) } - return hasMore, nil + return nil } -func (store *RocksDBStore) ListDirectoryEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64) (entries []*filer.Entry, hasMore bool, err error) { - return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "") +func (store *RocksDBStore) ListDirectoryEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { + return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "", eachEntryFunc) } -func (store *RocksDBStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string) (entries []*filer.Entry, hasMore bool, err error) { +func (store *RocksDBStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { directoryPrefix := genDirectoryKeyPrefix(dirPath, prefix) lastFileStart := directoryPrefix @@ -239,7 +238,7 @@ func (store *RocksDBStore) ListDirectoryPrefixedEntries(ctx context.Context, dir iter := store.db.NewIterator(ro) defer iter.Close() - hasMore, err = enumerate(iter, directoryPrefix, lastFileStart, includeStartFile, limit, func(key, value []byte) bool { + err = enumerate(iter, directoryPrefix, lastFileStart, includeStartFile, limit, func(key, value []byte) bool { fileName := getNameFromKey(key) if fileName == "" { return true @@ -247,6 +246,7 @@ func (store *RocksDBStore) ListDirectoryPrefixedEntries(ctx context.Context, dir entry := &filer.Entry{ FullPath: weed_util.NewFullPath(string(dirPath), fileName), } + lastFileName = fileName // println("list", entry.FullPath, "chunks", len(entry.Chunks)) if decodeErr := entry.DecodeAttributesAndChunks(value); decodeErr != nil { @@ -254,14 +254,16 @@ func (store *RocksDBStore) ListDirectoryPrefixedEntries(ctx context.Context, dir glog.V(0).Infof("list %s : %v", entry.FullPath, err) return false } - entries = append(entries, entry) + if !eachEntryFunc(entry) { + return false + } return true }) if err != nil { - return entries, false, fmt.Errorf("prefix list %s : %v", dirPath, err) + return lastFileName, fmt.Errorf("prefix list %s : %v", dirPath, err) } - return entries, false, err + return lastFileName, err } func genKey(dirPath, fileName string) (key []byte) { diff --git a/weed/filesys/dir.go b/weed/filesys/dir.go index f2fbd99d3..2cc332635 100644 --- a/weed/filesys/dir.go +++ b/weed/filesys/dir.go @@ -319,14 +319,14 @@ func (dir *Dir) ReadDirAll(ctx context.Context) (ret []fuse.Dirent, err error) { glog.Errorf("dir ReadDirAll %s: %v", dirPath, err) return nil, fuse.EIO } - listedEntries, listErr := dir.wfs.metaCache.ListDirectoryEntries(context.Background(), util.FullPath(dir.FullPath()), "", false, int64(math.MaxInt32)) + listErr := dir.wfs.metaCache.ListDirectoryEntries(context.Background(), util.FullPath(dir.FullPath()), "", false, int64(math.MaxInt32), func(entry *filer.Entry) bool { + processEachEntryFn(entry.ToProtoEntry(), false) + return true + }) if listErr != nil { glog.Errorf("list meta cache: %v", listErr) return nil, fuse.EIO } - for _, cachedEntry := range listedEntries { - processEachEntryFn(cachedEntry.ToProtoEntry(), false) - } return } diff --git a/weed/filesys/meta_cache/meta_cache.go b/weed/filesys/meta_cache/meta_cache.go index c16c4a938..f4e4d7d6e 100644 --- a/weed/filesys/meta_cache/meta_cache.go +++ b/weed/filesys/meta_cache/meta_cache.go @@ -117,22 +117,22 @@ func (mc *MetaCache) DeleteEntry(ctx context.Context, fp util.FullPath) (err err return mc.localStore.DeleteEntry(ctx, fp) } -func (mc *MetaCache) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64) ([]*filer.Entry, error) { +func (mc *MetaCache) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) error { mc.RLock() defer mc.RUnlock() if !mc.visitedBoundary.HasVisited(dirPath) { - return nil, fmt.Errorf("unsynchronized dir: %v", dirPath) + return fmt.Errorf("unsynchronized dir: %v", dirPath) } - entries, _, err := mc.localStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit) - if err != nil { - return nil, err - } - for _, entry := range entries { + _, err := mc.localStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit, func(entry *filer.Entry) bool { mc.mapIdFromFilerToLocal(entry) + return eachEntryFunc(entry) + }) + if err != nil { + return err } - return entries, err + return err } func (mc *MetaCache) Shutdown() { diff --git a/weed/server/filer_grpc_server.go b/weed/server/filer_grpc_server.go index 5cdf44e96..b0563d8bd 100644 --- a/weed/server/filer_grpc_server.go +++ b/weed/server/filer_grpc_server.go @@ -44,7 +44,7 @@ func (fs *FilerServer) LookupDirectoryEntry(ctx context.Context, req *filer_pb.L }, nil } -func (fs *FilerServer) ListEntries(req *filer_pb.ListEntriesRequest, stream filer_pb.SeaweedFiler_ListEntriesServer) error { +func (fs *FilerServer) ListEntries(req *filer_pb.ListEntriesRequest, stream filer_pb.SeaweedFiler_ListEntriesServer) (err error) { glog.V(4).Infof("ListEntries %v", req) @@ -60,23 +60,12 @@ func (fs *FilerServer) ListEntries(req *filer_pb.ListEntriesRequest, stream file lastFileName := req.StartFromFileName includeLastFile := req.InclusiveStartFrom + var listErr error for limit > 0 { - entries, hasMore, err := fs.filer.ListDirectoryEntries(stream.Context(), util.FullPath(req.Directory), lastFileName, includeLastFile, int64(paginationLimit), req.Prefix, "") - - if err != nil { - return err - } - if len(entries) == 0 { - return nil - } - - includeLastFile = false - - for _, entry := range entries { - - lastFileName = entry.Name() - - if err := stream.Send(&filer_pb.ListEntriesResponse{ + var hasEntries bool + lastFileName, listErr = fs.filer.StreamListDirectoryEntries(stream.Context(), util.FullPath(req.Directory), lastFileName, includeLastFile, int64(paginationLimit), req.Prefix, "", func(entry *filer.Entry) bool { + hasEntries = true + if err = stream.Send(&filer_pb.ListEntriesResponse{ Entry: &filer_pb.Entry{ Name: entry.Name(), IsDirectory: entry.IsDirectory(), @@ -88,19 +77,28 @@ func (fs *FilerServer) ListEntries(req *filer_pb.ListEntriesRequest, stream file Content: entry.Content, }, }); err != nil { - return err + return false } limit-- if limit == 0 { - return nil + return false } - } + return true + }) - if !hasMore { - break + if listErr != nil { + return listErr + } + if err != nil { + return err + } + if !hasEntries { + return nil } + includeLastFile = false + } return nil From 09f49d1c0440a55041c60643480d4159a9af3f0e Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 16 Jan 2021 19:52:15 -0800 Subject: [PATCH 140/260] refactoring --- weed/filer/filer_search.go | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/weed/filer/filer_search.go b/weed/filer/filer_search.go index 8c9688ceb..0a14d3756 100644 --- a/weed/filer/filer_search.go +++ b/weed/filer/filer_search.go @@ -21,29 +21,12 @@ func splitPattern(pattern string) (prefix string, restPattern string) { // For now, prefix and namePattern are mutually exclusive func (f *Filer) ListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int64, prefix string, namePattern string) (entries []*Entry, hasMore bool, err error) { - if strings.HasSuffix(string(p), "/") && len(p) > 1 { - p = p[0 : len(p)-1] - } - - prefixInNamePattern, restNamePattern := splitPattern(namePattern) - if prefixInNamePattern != "" { - prefix = prefixInNamePattern - } - var missedCount int64 - var lastFileName string - missedCount, lastFileName, err = f.doListPatternMatchedEntries(ctx, p, startFileName, inclusive, limit+1, prefix, restNamePattern, func(entry *Entry) bool { + _, err = f.StreamListDirectoryEntries(ctx, p, startFileName, inclusive, limit+1, prefix, namePattern, func(entry *Entry) bool { entries = append(entries, entry) return true }) - for missedCount > 0 && err == nil { - missedCount, lastFileName, err = f.doListPatternMatchedEntries(ctx, p, lastFileName, false, missedCount+1, prefix, restNamePattern, func(entry *Entry) bool { - entries = append(entries, entry) - return true - }) - } - hasMore = int64(len(entries)) >= limit+1 if hasMore { entries = entries[:limit] From 0f1c08d8ec6151c4e23bf3e62ebbc5ac8e8931d4 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev Date: Sun, 17 Jan 2021 18:33:14 +0500 Subject: [PATCH 141/260] docker-compose - prometheus - nginx --- docker/local-k8s-compose.yml | 41 +++++++++++++++++++++++++++----- docker/nginx/proxy.conf | 30 +++++++++++++++++++++++ docker/prometheus/prometheus.yml | 13 ++++++++++ docker/seaweedfs-compose.yml | 18 +++++++++++--- 4 files changed, 93 insertions(+), 9 deletions(-) create mode 100644 docker/nginx/proxy.conf create mode 100644 docker/prometheus/prometheus.yml diff --git a/docker/local-k8s-compose.yml b/docker/local-k8s-compose.yml index 0dda89ca4..9a25465c4 100644 --- a/docker/local-k8s-compose.yml +++ b/docker/local-k8s-compose.yml @@ -38,28 +38,57 @@ services: - WEED_MYSQL_USERNAME=seaweedfs - WEED_MYSQL_PASSWORD=secret - WEED_MYSQL_ENABLED=true + - WEED_MYSQL_CONNECTION_MAX_IDLE=5 + - WEED_MYSQL_CONNECTION_MAX_OPEN=75 + # "refresh" connection every 10 minutes, eliminating mysql closing "old" connections + - WEED_MYSQL_CONNECTION_MAX_LIFETIME_SECONDS=600 + # enable usage of memsql as filer backend + - WEED_MYSQL_INTERPOLATEPARAMS=true - WEED_LEVELDB2_ENABLED=false - command: 'filer -master="master:9333"' + command: '-v 9 filer -master="master:9333"' depends_on: - master - volume - mysql ingress: - image: jwilder/nginx-proxy + image: jwilder/nginx-proxy:alpine ports: - "80:80" volumes: - /var/run/docker.sock:/tmp/docker.sock:ro - - /tmp/nginx:/etc/nginx/conf.d + - ./nginx/proxy.conf:/etc/nginx/proxy.conf s3: image: chrislusf/seaweedfs:local ports: - 8333:8333 - command: 's3 -filer="filer:8888"' + command: '-v 9 s3 -filer="filer:8888"' depends_on: - master - volume - filer environment: - - VIRTUAL_HOST=s3 - - VIRTUAL_PORT=8333 \ No newline at end of file + - VIRTUAL_HOST=ingress + - VIRTUAL_PORT=8333 + registry: + image: registry:2 + environment: + REGISTRY_HTTP_ADDR: "0.0.0.0:5001" # seaweedfs s3 + REGISTRY_LOG_LEVEL: "debug" + REGISTRY_STORAGE: "s3" + REGISTRY_STORAGE_S3_REGION: "us-east-1" + REGISTRY_STORAGE_S3_REGIONENDPOINT: "http://ingress" + REGISTRY_STORAGE_S3_BUCKET: "registry" + REGISTRY_STORAGE_S3_ACCESSKEY: "some_access_key1" + REGISTRY_STORAGE_S3_SECRETKEY: "some_secret_key1" + REGISTRY_STORAGE_S3_V4AUTH: "true" + REGISTRY_STORAGE_S3_SECURE: "false" + REGISTRY_STORAGE_S3_SKIPVERIFY: "true" + REGISTRY_STORAGE_S3_ROOTDIRECTORY: "/" + REGISTRY_STORAGE_DELETE_ENABLED: "true" + REGISTRY_STORAGE_REDIRECT_DISABLE: "true" + REGISTRY_VALIDATION_DISABLED: "true" + ports: + - 5001:5001 + depends_on: + - s3 + - ingress \ No newline at end of file diff --git a/docker/nginx/proxy.conf b/docker/nginx/proxy.conf new file mode 100644 index 000000000..59ff30ce2 --- /dev/null +++ b/docker/nginx/proxy.conf @@ -0,0 +1,30 @@ +# HTTP 1.1 support +proxy_http_version 1.1; +#proxy_buffering off; +proxy_set_header Host $http_host; +proxy_set_header Upgrade $http_upgrade; +proxy_set_header Connection $proxy_connection; +proxy_set_header X-Real-IP $remote_addr; +proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; +proxy_set_header X-Forwarded-Proto $proxy_x_forwarded_proto; +proxy_set_header X-Forwarded-Ssl $proxy_x_forwarded_ssl; +proxy_set_header X-Forwarded-Port $proxy_x_forwarded_port; + +# Mitigate httpoxy attack (see README for details) +proxy_set_header Proxy ""; + +# aws default max_concurrent_requests 10 +# aws default multipart_threshold 8MB +proxy_buffering on; # GET buffering or “X-Accel-Buffering” enables or disables buffering of a response; +proxy_buffers 64 1m; # buffers used for reading a response from the proxied server, for a single connection +proxy_buffer_size 8k; # maximum size of the data that nginx can receive from the server at a time is set +proxy_busy_buffers_size 2m; + +proxy_request_buffering on; # PUT buffering +client_body_buffer_size 64m; # buffer size for reading client request body +client_max_body_size 64m; + +proxy_next_upstream error timeout non_idempotent http_500; # PUT request should be passed to the next server: +proxy_connect_timeout 200ms; +proxy_read_timeout 3s; #timeout is set only between two successive read operations +proxy_send_timeout 3s; #timeout is set only between two successive write operations diff --git a/docker/prometheus/prometheus.yml b/docker/prometheus/prometheus.yml new file mode 100644 index 000000000..34f669d56 --- /dev/null +++ b/docker/prometheus/prometheus.yml @@ -0,0 +1,13 @@ +global: + scrape_interval: 30s + scrape_timeout: 10s + +scrape_configs: + - job_name: services + metrics_path: /metrics + static_configs: + - targets: + - 'prometheus:9090' + - 'volume:9325' + - 'filer:9326' + - 's3:9327' \ No newline at end of file diff --git a/docker/seaweedfs-compose.yml b/docker/seaweedfs-compose.yml index 70d005017..3c8a8e3e3 100644 --- a/docker/seaweedfs-compose.yml +++ b/docker/seaweedfs-compose.yml @@ -12,7 +12,8 @@ services: ports: - 8080:8080 - 18080:18080 - command: 'volume -mserver="master:9333" -port=8080' + - 9325:9325 + command: 'volume -mserver="master:9333" -port=8080 -metricsPort=9325' depends_on: - master filer: @@ -20,7 +21,8 @@ services: ports: - 8888:8888 - 18888:18888 - command: 'filer -master="master:9333"' + - 9326:9326 + command: 'filer -master="master:9333" -metricsPort=9326' tty: true stdin_open: true depends_on: @@ -40,8 +42,18 @@ services: image: chrislusf/seaweedfs # use a remote image ports: - 8333:8333 - command: 's3 -filer="filer:8888"' + - 9327:9327 + command: 's3 -filer="filer:8888" -metricsPort=9327' depends_on: - master - volume - filer + prometheus: + image: prom/prometheus:v2.21.0 + ports: + - 9000:9090 + volumes: + - ./prometheus:/etc/prometheus + command: --web.enable-lifecycle --config.file=/etc/prometheus/prometheus.yml + depends_on: + - s3 From 389426bbb7ad54b4967fd26c454fa5b6de1276e2 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 17 Jan 2021 22:59:15 -0800 Subject: [PATCH 142/260] s3: listing may repeat on the edge --- weed/s3api/s3api_objects_list_handlers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/s3api/s3api_objects_list_handlers.go b/weed/s3api/s3api_objects_list_handlers.go index 2d36c6ec9..a2407fced 100644 --- a/weed/s3api/s3api_objects_list_handlers.go +++ b/weed/s3api/s3api_objects_list_handlers.go @@ -238,7 +238,7 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d return } } - if counter >= maxKeys { + if counter >= maxKeys + 1 { isTruncated = true return } From d7d907be27e2441f54fac70b6723d6a41ae62af8 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 17 Jan 2021 23:46:39 -0800 Subject: [PATCH 143/260] mount: add more entry type --- weed/filesys/dir.go | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/weed/filesys/dir.go b/weed/filesys/dir.go index 2cc332635..35527546b 100644 --- a/weed/filesys/dir.go +++ b/weed/filesys/dir.go @@ -6,6 +6,7 @@ import ( "math" "os" "strings" + "syscall" "time" "github.com/seaweedfs/fuse" @@ -308,7 +309,7 @@ func (dir *Dir) ReadDirAll(ctx context.Context) (ret []fuse.Dirent, err error) { dirent := fuse.Dirent{Inode: inode, Name: entry.Name, Type: fuse.DT_Dir} ret = append(ret, dirent) } else { - dirent := fuse.Dirent{Inode: inode, Name: entry.Name, Type: fuse.DT_File} + dirent := fuse.Dirent{Inode: inode, Name: entry.Name, Type: findFileType(uint16(entry.Attributes.FileMode))} ret = append(ret, dirent) } return nil @@ -330,6 +331,26 @@ func (dir *Dir) ReadDirAll(ctx context.Context) (ret []fuse.Dirent, err error) { return } +func findFileType(mode uint16) fuse.DirentType { + switch mode & (syscall.S_IFMT & 0xffff) { + case syscall.S_IFSOCK: + return fuse.DT_Socket + case syscall.S_IFLNK: + return fuse.DT_Link + case syscall.S_IFREG: + return fuse.DT_File + case syscall.S_IFBLK: + return fuse.DT_Block + case syscall.S_IFDIR: + return fuse.DT_Dir + case syscall.S_IFCHR: + return fuse.DT_Char + case syscall.S_IFIFO: + return fuse.DT_FIFO + } + return fuse.DT_File +} + func (dir *Dir) Remove(ctx context.Context, req *fuse.RemoveRequest) error { if !req.Dir { From 61ef2d865824cf26871dd60877fa712010db1420 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 17 Jan 2021 23:51:10 -0800 Subject: [PATCH 144/260] mount: fake support for socket/block/character/fifo devices to pass pjdfstest. A distributed file system can not really support these things anyway. --- weed/filesys/dir.go | 79 ++++++++++++++++++++++++--------------------- 1 file changed, 42 insertions(+), 37 deletions(-) diff --git a/weed/filesys/dir.go b/weed/filesys/dir.go index 35527546b..afa361561 100644 --- a/weed/filesys/dir.go +++ b/weed/filesys/dir.go @@ -128,28 +128,59 @@ func (dir *Dir) newDirectory(fullpath util.FullPath, entry *filer_pb.Entry) fs.N func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest, resp *fuse.CreateResponse) (fs.Node, fs.Handle, error) { + request, err := dir.doCreateEntry(req.Name, req.Mode, req.Uid, req.Gid, req.Flags&fuse.OpenExclusive != 0) + + if err != nil { + return nil, nil, err + } + var node fs.Node + if request.Entry.IsDirectory { + node = dir.newDirectory(util.NewFullPath(dir.FullPath(), req.Name), request.Entry) + return node, nil, nil + } + + node = dir.newFile(req.Name, request.Entry) + file := node.(*File) + fh := dir.wfs.AcquireHandle(file, req.Uid, req.Gid) + return file, fh, nil + +} + +func (dir *Dir) Mknod(ctx context.Context, req *fuse.MknodRequest) (fs.Node, error) { + + request, err := dir.doCreateEntry(req.Name, req.Mode, req.Uid, req.Gid, false) + + if err != nil { + return nil, err + } + var node fs.Node + node = dir.newFile(req.Name, request.Entry) + return node, nil +} + +func (dir *Dir) doCreateEntry(name string, mode os.FileMode, uid, gid uint32, exlusive bool) (*filer_pb.CreateEntryRequest, error) { request := &filer_pb.CreateEntryRequest{ Directory: dir.FullPath(), Entry: &filer_pb.Entry{ - Name: req.Name, - IsDirectory: req.Mode&os.ModeDir > 0, + Name: name, + IsDirectory: mode&os.ModeDir > 0, Attributes: &filer_pb.FuseAttributes{ Mtime: time.Now().Unix(), Crtime: time.Now().Unix(), - FileMode: uint32(req.Mode &^ dir.wfs.option.Umask), - Uid: req.Uid, - Gid: req.Gid, + FileMode: uint32(mode &^ dir.wfs.option.Umask), + Uid: uid, + Gid: gid, Collection: dir.wfs.option.Collection, Replication: dir.wfs.option.Replication, TtlSec: dir.wfs.option.TtlSec, }, }, - OExcl: req.Flags&fuse.OpenExclusive != 0, + OExcl: exlusive, Signatures: []int32{dir.wfs.signature}, } - glog.V(1).Infof("create %s/%s: %v", dir.FullPath(), req.Name, req.Flags) + glog.V(1).Infof("create %s/%s: %v", dir.FullPath(), name) - if err := dir.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + err := dir.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { dir.wfs.mapPbIdFromLocalToFiler(request.Entry) defer dir.wfs.mapPbIdFromFilerToLocal(request.Entry) @@ -158,41 +189,15 @@ func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest, if strings.Contains(err.Error(), "EEXIST") { return fuse.EEXIST } - glog.V(0).Infof("create %s/%s: %v", dir.FullPath(), req.Name, err) + glog.V(0).Infof("create %s/%s: %v", dir.FullPath(), name, err) return fuse.EIO } dir.wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry)) return nil - }); err != nil { - return nil, nil, err - } - var node fs.Node - if request.Entry.IsDirectory { - node = dir.newDirectory(util.NewFullPath(dir.FullPath(), req.Name), request.Entry) - return node, nil, nil - } - - node = dir.newFile(req.Name, request.Entry) - file := node.(*File) - fh := dir.wfs.AcquireHandle(file, req.Uid, req.Gid) - return file, fh, nil - -} - -func (dir *Dir) Mknod(ctx context.Context, req *fuse.MknodRequest) (fs.Node, error) { - if req.Mode&os.ModeNamedPipe != 0 { - glog.V(1).Infof("mknod named pipe %s", req.String()) - return nil, fuse.ENOSYS - } - if req.Mode&req.Mode&os.ModeSocket != 0 { - glog.V(1).Infof("mknod socket %s", req.String()) - return nil, fuse.ENOSYS - } - // not going to support mknod for normal files either - glog.V(1).Infof("mknod %s", req.String()) - return nil, fuse.ENOSYS + }) + return request, err } func (dir *Dir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fs.Node, error) { From 3d5cb7eb86cc1eb8971caaa27fd8cc27b7cc2d5d Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 18 Jan 2021 00:18:57 -0800 Subject: [PATCH 145/260] Revert "mount: fake support for socket/block/character/fifo devices" This reverts commit 61ef2d865824cf26871dd60877fa712010db1420. --- weed/filesys/dir.go | 79 +++++++++++++++++++++------------------------ 1 file changed, 37 insertions(+), 42 deletions(-) diff --git a/weed/filesys/dir.go b/weed/filesys/dir.go index afa361561..35527546b 100644 --- a/weed/filesys/dir.go +++ b/weed/filesys/dir.go @@ -128,59 +128,28 @@ func (dir *Dir) newDirectory(fullpath util.FullPath, entry *filer_pb.Entry) fs.N func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest, resp *fuse.CreateResponse) (fs.Node, fs.Handle, error) { - request, err := dir.doCreateEntry(req.Name, req.Mode, req.Uid, req.Gid, req.Flags&fuse.OpenExclusive != 0) - - if err != nil { - return nil, nil, err - } - var node fs.Node - if request.Entry.IsDirectory { - node = dir.newDirectory(util.NewFullPath(dir.FullPath(), req.Name), request.Entry) - return node, nil, nil - } - - node = dir.newFile(req.Name, request.Entry) - file := node.(*File) - fh := dir.wfs.AcquireHandle(file, req.Uid, req.Gid) - return file, fh, nil - -} - -func (dir *Dir) Mknod(ctx context.Context, req *fuse.MknodRequest) (fs.Node, error) { - - request, err := dir.doCreateEntry(req.Name, req.Mode, req.Uid, req.Gid, false) - - if err != nil { - return nil, err - } - var node fs.Node - node = dir.newFile(req.Name, request.Entry) - return node, nil -} - -func (dir *Dir) doCreateEntry(name string, mode os.FileMode, uid, gid uint32, exlusive bool) (*filer_pb.CreateEntryRequest, error) { request := &filer_pb.CreateEntryRequest{ Directory: dir.FullPath(), Entry: &filer_pb.Entry{ - Name: name, - IsDirectory: mode&os.ModeDir > 0, + Name: req.Name, + IsDirectory: req.Mode&os.ModeDir > 0, Attributes: &filer_pb.FuseAttributes{ Mtime: time.Now().Unix(), Crtime: time.Now().Unix(), - FileMode: uint32(mode &^ dir.wfs.option.Umask), - Uid: uid, - Gid: gid, + FileMode: uint32(req.Mode &^ dir.wfs.option.Umask), + Uid: req.Uid, + Gid: req.Gid, Collection: dir.wfs.option.Collection, Replication: dir.wfs.option.Replication, TtlSec: dir.wfs.option.TtlSec, }, }, - OExcl: exlusive, + OExcl: req.Flags&fuse.OpenExclusive != 0, Signatures: []int32{dir.wfs.signature}, } - glog.V(1).Infof("create %s/%s: %v", dir.FullPath(), name) + glog.V(1).Infof("create %s/%s: %v", dir.FullPath(), req.Name, req.Flags) - err := dir.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + if err := dir.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { dir.wfs.mapPbIdFromLocalToFiler(request.Entry) defer dir.wfs.mapPbIdFromFilerToLocal(request.Entry) @@ -189,15 +158,41 @@ func (dir *Dir) doCreateEntry(name string, mode os.FileMode, uid, gid uint32, ex if strings.Contains(err.Error(), "EEXIST") { return fuse.EEXIST } - glog.V(0).Infof("create %s/%s: %v", dir.FullPath(), name, err) + glog.V(0).Infof("create %s/%s: %v", dir.FullPath(), req.Name, err) return fuse.EIO } dir.wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry)) return nil - }) - return request, err + }); err != nil { + return nil, nil, err + } + var node fs.Node + if request.Entry.IsDirectory { + node = dir.newDirectory(util.NewFullPath(dir.FullPath(), req.Name), request.Entry) + return node, nil, nil + } + + node = dir.newFile(req.Name, request.Entry) + file := node.(*File) + fh := dir.wfs.AcquireHandle(file, req.Uid, req.Gid) + return file, fh, nil + +} + +func (dir *Dir) Mknod(ctx context.Context, req *fuse.MknodRequest) (fs.Node, error) { + if req.Mode&os.ModeNamedPipe != 0 { + glog.V(1).Infof("mknod named pipe %s", req.String()) + return nil, fuse.ENOSYS + } + if req.Mode&req.Mode&os.ModeSocket != 0 { + glog.V(1).Infof("mknod socket %s", req.String()) + return nil, fuse.ENOSYS + } + // not going to support mknod for normal files either + glog.V(1).Infof("mknod %s", req.String()) + return nil, fuse.ENOSYS } func (dir *Dir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fs.Node, error) { From 4085b79d389ba2566b4a6b7f3dce7f1fca8d481b Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 18 Jan 2021 00:19:19 -0800 Subject: [PATCH 146/260] Revert "mount: add more entry type" This reverts commit d7d907be27e2441f54fac70b6723d6a41ae62af8. --- weed/filesys/dir.go | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/weed/filesys/dir.go b/weed/filesys/dir.go index 35527546b..2cc332635 100644 --- a/weed/filesys/dir.go +++ b/weed/filesys/dir.go @@ -6,7 +6,6 @@ import ( "math" "os" "strings" - "syscall" "time" "github.com/seaweedfs/fuse" @@ -309,7 +308,7 @@ func (dir *Dir) ReadDirAll(ctx context.Context) (ret []fuse.Dirent, err error) { dirent := fuse.Dirent{Inode: inode, Name: entry.Name, Type: fuse.DT_Dir} ret = append(ret, dirent) } else { - dirent := fuse.Dirent{Inode: inode, Name: entry.Name, Type: findFileType(uint16(entry.Attributes.FileMode))} + dirent := fuse.Dirent{Inode: inode, Name: entry.Name, Type: fuse.DT_File} ret = append(ret, dirent) } return nil @@ -331,26 +330,6 @@ func (dir *Dir) ReadDirAll(ctx context.Context) (ret []fuse.Dirent, err error) { return } -func findFileType(mode uint16) fuse.DirentType { - switch mode & (syscall.S_IFMT & 0xffff) { - case syscall.S_IFSOCK: - return fuse.DT_Socket - case syscall.S_IFLNK: - return fuse.DT_Link - case syscall.S_IFREG: - return fuse.DT_File - case syscall.S_IFBLK: - return fuse.DT_Block - case syscall.S_IFDIR: - return fuse.DT_Dir - case syscall.S_IFCHR: - return fuse.DT_Char - case syscall.S_IFIFO: - return fuse.DT_FIFO - } - return fuse.DT_File -} - func (dir *Dir) Remove(ctx context.Context, req *fuse.RemoveRequest) error { if !req.Dir { From de876c795d034a7e9fc10b74f31ad600c33dd2e4 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 18 Jan 2021 01:14:27 -0800 Subject: [PATCH 147/260] minor fix --- weed/filer/reader_at.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/filer/reader_at.go b/weed/filer/reader_at.go index 1f594a1bf..307224f35 100644 --- a/weed/filer/reader_at.go +++ b/weed/filer/reader_at.go @@ -107,7 +107,7 @@ func (c *ChunkReadAt) ReadAt(p []byte, offset int64) (n int, err error) { defer c.readerLock.Unlock() glog.V(4).Infof("ReadAt [%d,%d) of total file size %d bytes %d chunk views", offset, offset+int64(len(p)), c.fileSize, len(c.chunkViews)) - return c.doReadAt(p[n:], offset+int64(n)) + return c.doReadAt(p, offset) } func (c *ChunkReadAt) doReadAt(p []byte, offset int64) (n int, err error) { From 7119e7b4912008f50c78639d393c95ee4eaea9e8 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 18 Jan 2021 01:14:42 -0800 Subject: [PATCH 148/260] mount: ensure reading from dirty pages --- weed/filesys/filehandle.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/filesys/filehandle.go b/weed/filesys/filehandle.go index c273eec8a..d5f6cc38e 100644 --- a/weed/filesys/filehandle.go +++ b/weed/filesys/filehandle.go @@ -72,7 +72,7 @@ func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fus } totalRead, err := fh.readFromChunks(buff, req.Offset) - if err == nil { + if err == nil || err == io.EOF { maxStop := fh.readFromDirtyPages(buff, req.Offset) totalRead = max(maxStop-req.Offset, totalRead) } From 67faa56920e9a534c1fc531230e7d90b68e53d44 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 18 Jan 2021 01:14:58 -0800 Subject: [PATCH 149/260] Revert "Revert "mount: add more entry type"" This reverts commit 4085b79d389ba2566b4a6b7f3dce7f1fca8d481b. --- weed/filesys/dir.go | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/weed/filesys/dir.go b/weed/filesys/dir.go index 2cc332635..35527546b 100644 --- a/weed/filesys/dir.go +++ b/weed/filesys/dir.go @@ -6,6 +6,7 @@ import ( "math" "os" "strings" + "syscall" "time" "github.com/seaweedfs/fuse" @@ -308,7 +309,7 @@ func (dir *Dir) ReadDirAll(ctx context.Context) (ret []fuse.Dirent, err error) { dirent := fuse.Dirent{Inode: inode, Name: entry.Name, Type: fuse.DT_Dir} ret = append(ret, dirent) } else { - dirent := fuse.Dirent{Inode: inode, Name: entry.Name, Type: fuse.DT_File} + dirent := fuse.Dirent{Inode: inode, Name: entry.Name, Type: findFileType(uint16(entry.Attributes.FileMode))} ret = append(ret, dirent) } return nil @@ -330,6 +331,26 @@ func (dir *Dir) ReadDirAll(ctx context.Context) (ret []fuse.Dirent, err error) { return } +func findFileType(mode uint16) fuse.DirentType { + switch mode & (syscall.S_IFMT & 0xffff) { + case syscall.S_IFSOCK: + return fuse.DT_Socket + case syscall.S_IFLNK: + return fuse.DT_Link + case syscall.S_IFREG: + return fuse.DT_File + case syscall.S_IFBLK: + return fuse.DT_Block + case syscall.S_IFDIR: + return fuse.DT_Dir + case syscall.S_IFCHR: + return fuse.DT_Char + case syscall.S_IFIFO: + return fuse.DT_FIFO + } + return fuse.DT_File +} + func (dir *Dir) Remove(ctx context.Context, req *fuse.RemoveRequest) error { if !req.Dir { From 5a3386e39bb2ed1e0b1c4a8aaae5c9cbc12b98bd Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 18 Jan 2021 01:15:07 -0800 Subject: [PATCH 150/260] Revert "Revert "mount: fake support for socket/block/character/fifo devices"" This reverts commit 3d5cb7eb86cc1eb8971caaa27fd8cc27b7cc2d5d. --- weed/filesys/dir.go | 79 ++++++++++++++++++++++++--------------------- 1 file changed, 42 insertions(+), 37 deletions(-) diff --git a/weed/filesys/dir.go b/weed/filesys/dir.go index 35527546b..afa361561 100644 --- a/weed/filesys/dir.go +++ b/weed/filesys/dir.go @@ -128,28 +128,59 @@ func (dir *Dir) newDirectory(fullpath util.FullPath, entry *filer_pb.Entry) fs.N func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest, resp *fuse.CreateResponse) (fs.Node, fs.Handle, error) { + request, err := dir.doCreateEntry(req.Name, req.Mode, req.Uid, req.Gid, req.Flags&fuse.OpenExclusive != 0) + + if err != nil { + return nil, nil, err + } + var node fs.Node + if request.Entry.IsDirectory { + node = dir.newDirectory(util.NewFullPath(dir.FullPath(), req.Name), request.Entry) + return node, nil, nil + } + + node = dir.newFile(req.Name, request.Entry) + file := node.(*File) + fh := dir.wfs.AcquireHandle(file, req.Uid, req.Gid) + return file, fh, nil + +} + +func (dir *Dir) Mknod(ctx context.Context, req *fuse.MknodRequest) (fs.Node, error) { + + request, err := dir.doCreateEntry(req.Name, req.Mode, req.Uid, req.Gid, false) + + if err != nil { + return nil, err + } + var node fs.Node + node = dir.newFile(req.Name, request.Entry) + return node, nil +} + +func (dir *Dir) doCreateEntry(name string, mode os.FileMode, uid, gid uint32, exlusive bool) (*filer_pb.CreateEntryRequest, error) { request := &filer_pb.CreateEntryRequest{ Directory: dir.FullPath(), Entry: &filer_pb.Entry{ - Name: req.Name, - IsDirectory: req.Mode&os.ModeDir > 0, + Name: name, + IsDirectory: mode&os.ModeDir > 0, Attributes: &filer_pb.FuseAttributes{ Mtime: time.Now().Unix(), Crtime: time.Now().Unix(), - FileMode: uint32(req.Mode &^ dir.wfs.option.Umask), - Uid: req.Uid, - Gid: req.Gid, + FileMode: uint32(mode &^ dir.wfs.option.Umask), + Uid: uid, + Gid: gid, Collection: dir.wfs.option.Collection, Replication: dir.wfs.option.Replication, TtlSec: dir.wfs.option.TtlSec, }, }, - OExcl: req.Flags&fuse.OpenExclusive != 0, + OExcl: exlusive, Signatures: []int32{dir.wfs.signature}, } - glog.V(1).Infof("create %s/%s: %v", dir.FullPath(), req.Name, req.Flags) + glog.V(1).Infof("create %s/%s: %v", dir.FullPath(), name) - if err := dir.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + err := dir.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { dir.wfs.mapPbIdFromLocalToFiler(request.Entry) defer dir.wfs.mapPbIdFromFilerToLocal(request.Entry) @@ -158,41 +189,15 @@ func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest, if strings.Contains(err.Error(), "EEXIST") { return fuse.EEXIST } - glog.V(0).Infof("create %s/%s: %v", dir.FullPath(), req.Name, err) + glog.V(0).Infof("create %s/%s: %v", dir.FullPath(), name, err) return fuse.EIO } dir.wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry)) return nil - }); err != nil { - return nil, nil, err - } - var node fs.Node - if request.Entry.IsDirectory { - node = dir.newDirectory(util.NewFullPath(dir.FullPath(), req.Name), request.Entry) - return node, nil, nil - } - - node = dir.newFile(req.Name, request.Entry) - file := node.(*File) - fh := dir.wfs.AcquireHandle(file, req.Uid, req.Gid) - return file, fh, nil - -} - -func (dir *Dir) Mknod(ctx context.Context, req *fuse.MknodRequest) (fs.Node, error) { - if req.Mode&os.ModeNamedPipe != 0 { - glog.V(1).Infof("mknod named pipe %s", req.String()) - return nil, fuse.ENOSYS - } - if req.Mode&req.Mode&os.ModeSocket != 0 { - glog.V(1).Infof("mknod socket %s", req.String()) - return nil, fuse.ENOSYS - } - // not going to support mknod for normal files either - glog.V(1).Infof("mknod %s", req.String()) - return nil, fuse.ENOSYS + }) + return request, err } func (dir *Dir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fs.Node, error) { From c7197470fdf947323fb31574ee328ea084f23ac8 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 18 Jan 2021 01:25:04 -0800 Subject: [PATCH 151/260] 2.21 --- k8s/seaweedfs/Chart.yaml | 4 ++-- k8s/seaweedfs/values.yaml | 2 +- weed/util/constants.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/k8s/seaweedfs/Chart.yaml b/k8s/seaweedfs/Chart.yaml index a734d3f9e..7cf6b62a7 100644 --- a/k8s/seaweedfs/Chart.yaml +++ b/k8s/seaweedfs/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v1 description: SeaweedFS name: seaweedfs -appVersion: "2.20" -version: 2.20 +appVersion: "2.21" +version: 2.21 diff --git a/k8s/seaweedfs/values.yaml b/k8s/seaweedfs/values.yaml index e08731354..5e40bae2b 100644 --- a/k8s/seaweedfs/values.yaml +++ b/k8s/seaweedfs/values.yaml @@ -4,7 +4,7 @@ global: registry: "" repository: "" imageName: chrislusf/seaweedfs - # imageTag: "2.20" - started using {.Chart.appVersion} + # imageTag: "2.21" - started using {.Chart.appVersion} imagePullPolicy: IfNotPresent imagePullSecrets: imagepullsecret restartPolicy: Always diff --git a/weed/util/constants.go b/weed/util/constants.go index ac653c4d7..4e6a28334 100644 --- a/weed/util/constants.go +++ b/weed/util/constants.go @@ -5,7 +5,7 @@ import ( ) var ( - VERSION = fmt.Sprintf("%s %d.%02d", sizeLimit, 2, 20) + VERSION = fmt.Sprintf("%s %d.%02d", sizeLimit, 2, 21) COMMIT = "" ) From e439b65e387609b2a761d9a976c0efc44060dff5 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 18 Jan 2021 01:48:00 -0800 Subject: [PATCH 152/260] fix test --- weed/filesys/dir.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/filesys/dir.go b/weed/filesys/dir.go index afa361561..d86d92ac9 100644 --- a/weed/filesys/dir.go +++ b/weed/filesys/dir.go @@ -178,7 +178,7 @@ func (dir *Dir) doCreateEntry(name string, mode os.FileMode, uid, gid uint32, ex OExcl: exlusive, Signatures: []int32{dir.wfs.signature}, } - glog.V(1).Infof("create %s/%s: %v", dir.FullPath(), name) + glog.V(1).Infof("create %s/%s", dir.FullPath(), name) err := dir.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { From f13c082583e78a8ab85c457dca93c6eb800961fc Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 19 Jan 2021 11:31:55 -0800 Subject: [PATCH 153/260] add memsql to comments --- weed/command/scaffold.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/command/scaffold.go b/weed/command/scaffold.go index 887c9883f..f55123573 100644 --- a/weed/command/scaffold.go +++ b/weed/command/scaffold.go @@ -101,7 +101,7 @@ dir = "./filerldb3" # directory to store level db files enabled = false dir = "./filerrdb" # directory to store rocksdb files -[mysql] # or tidb +[mysql] # or memsql, tidb # CREATE TABLE IF NOT EXISTS filemeta ( # dirhash BIGINT COMMENT 'first 64 bits of MD5 hash value of directory field', # name VARCHAR(1000) COMMENT 'directory or file name', From 90ce1eec19f31b762f08b2558fe63e9118911498 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 19 Jan 2021 11:32:55 -0800 Subject: [PATCH 154/260] add more input params --- weed/filer/abstract_sql/abstract_sql_store.go | 16 ++++++++-------- weed/filer/abstract_sql/abstract_sql_store_kv.go | 8 ++++---- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/weed/filer/abstract_sql/abstract_sql_store.go b/weed/filer/abstract_sql/abstract_sql_store.go index f5dcf6e03..78f408dcc 100644 --- a/weed/filer/abstract_sql/abstract_sql_store.go +++ b/weed/filer/abstract_sql/abstract_sql_store.go @@ -52,7 +52,7 @@ func (store *AbstractSqlStore) RollbackTransaction(ctx context.Context) error { return nil } -func (store *AbstractSqlStore) getTxOrDB(ctx context.Context) TxOrDB { +func (store *AbstractSqlStore) getTxOrDB(ctx context.Context, fullpath util.FullPath, isForChildren bool) TxOrDB { if tx, ok := ctx.Value("tx").(*sql.Tx); ok { return tx } @@ -71,7 +71,7 @@ func (store *AbstractSqlStore) InsertEntry(ctx context.Context, entry *filer.Ent meta = util.MaybeGzipData(meta) } - res, err := store.getTxOrDB(ctx).ExecContext(ctx, store.SqlInsert, util.HashStringToLong(dir), name, dir, meta) + res, err := store.getTxOrDB(ctx, entry.FullPath, false).ExecContext(ctx, store.SqlInsert, util.HashStringToLong(dir), name, dir, meta) if err == nil { return } @@ -84,7 +84,7 @@ func (store *AbstractSqlStore) InsertEntry(ctx context.Context, entry *filer.Ent // now the insert failed possibly due to duplication constraints glog.V(1).Infof("insert %s falls back to update: %v", entry.FullPath, err) - res, err = store.getTxOrDB(ctx).ExecContext(ctx, store.SqlUpdate, meta, util.HashStringToLong(dir), name, dir) + res, err = store.getTxOrDB(ctx, entry.FullPath, false).ExecContext(ctx, store.SqlUpdate, meta, util.HashStringToLong(dir), name, dir) if err != nil { return fmt.Errorf("upsert %s: %s", entry.FullPath, err) } @@ -105,7 +105,7 @@ func (store *AbstractSqlStore) UpdateEntry(ctx context.Context, entry *filer.Ent return fmt.Errorf("encode %s: %s", entry.FullPath, err) } - res, err := store.getTxOrDB(ctx).ExecContext(ctx, store.SqlUpdate, meta, util.HashStringToLong(dir), name, dir) + res, err := store.getTxOrDB(ctx, entry.FullPath, false).ExecContext(ctx, store.SqlUpdate, meta, util.HashStringToLong(dir), name, dir) if err != nil { return fmt.Errorf("update %s: %s", entry.FullPath, err) } @@ -120,7 +120,7 @@ func (store *AbstractSqlStore) UpdateEntry(ctx context.Context, entry *filer.Ent func (store *AbstractSqlStore) FindEntry(ctx context.Context, fullpath util.FullPath) (*filer.Entry, error) { dir, name := fullpath.DirAndName() - row := store.getTxOrDB(ctx).QueryRowContext(ctx, store.SqlFind, util.HashStringToLong(dir), name, dir) + row := store.getTxOrDB(ctx, fullpath, false).QueryRowContext(ctx, store.SqlFind, util.HashStringToLong(dir), name, dir) var data []byte if err := row.Scan(&data); err != nil { @@ -144,7 +144,7 @@ func (store *AbstractSqlStore) DeleteEntry(ctx context.Context, fullpath util.Fu dir, name := fullpath.DirAndName() - res, err := store.getTxOrDB(ctx).ExecContext(ctx, store.SqlDelete, util.HashStringToLong(dir), name, dir) + res, err := store.getTxOrDB(ctx, fullpath, false).ExecContext(ctx, store.SqlDelete, util.HashStringToLong(dir), name, dir) if err != nil { return fmt.Errorf("delete %s: %s", fullpath, err) } @@ -159,7 +159,7 @@ func (store *AbstractSqlStore) DeleteEntry(ctx context.Context, fullpath util.Fu func (store *AbstractSqlStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) error { - res, err := store.getTxOrDB(ctx).ExecContext(ctx, store.SqlDeleteFolderChildren, util.HashStringToLong(string(fullpath)), fullpath) + res, err := store.getTxOrDB(ctx, fullpath, true).ExecContext(ctx, store.SqlDeleteFolderChildren, util.HashStringToLong(string(fullpath)), fullpath) if err != nil { return fmt.Errorf("deleteFolderChildren %s: %s", fullpath, err) } @@ -178,7 +178,7 @@ func (store *AbstractSqlStore) ListDirectoryPrefixedEntries(ctx context.Context, sqlText = store.SqlListInclusive } - rows, err := store.getTxOrDB(ctx).QueryContext(ctx, sqlText, util.HashStringToLong(string(dirPath)), startFileName, string(dirPath), prefix+"%", limit+1) + rows, err := store.getTxOrDB(ctx, dirPath, true).QueryContext(ctx, sqlText, util.HashStringToLong(string(dirPath)), startFileName, string(dirPath), prefix+"%", limit+1) if err != nil { return lastFileName, fmt.Errorf("list %s : %v", dirPath, err) } diff --git a/weed/filer/abstract_sql/abstract_sql_store_kv.go b/weed/filer/abstract_sql/abstract_sql_store_kv.go index 792a45ff4..e85a85f8c 100644 --- a/weed/filer/abstract_sql/abstract_sql_store_kv.go +++ b/weed/filer/abstract_sql/abstract_sql_store_kv.go @@ -15,7 +15,7 @@ func (store *AbstractSqlStore) KvPut(ctx context.Context, key []byte, value []by dirStr, dirHash, name := genDirAndName(key) - res, err := store.getTxOrDB(ctx).ExecContext(ctx, store.SqlInsert, dirHash, name, dirStr, value) + res, err := store.getTxOrDB(ctx, "", false).ExecContext(ctx, store.SqlInsert, dirHash, name, dirStr, value) if err == nil { return } @@ -28,7 +28,7 @@ func (store *AbstractSqlStore) KvPut(ctx context.Context, key []byte, value []by // now the insert failed possibly due to duplication constraints glog.V(1).Infof("kv insert falls back to update: %s", err) - res, err = store.getTxOrDB(ctx).ExecContext(ctx, store.SqlUpdate, value, dirHash, name, dirStr) + res, err = store.getTxOrDB(ctx, "", false).ExecContext(ctx, store.SqlUpdate, value, dirHash, name, dirStr) if err != nil { return fmt.Errorf("kv upsert: %s", err) } @@ -44,7 +44,7 @@ func (store *AbstractSqlStore) KvPut(ctx context.Context, key []byte, value []by func (store *AbstractSqlStore) KvGet(ctx context.Context, key []byte) (value []byte, err error) { dirStr, dirHash, name := genDirAndName(key) - row := store.getTxOrDB(ctx).QueryRowContext(ctx, store.SqlFind, dirHash, name, dirStr) + row := store.getTxOrDB(ctx, "", false).QueryRowContext(ctx, store.SqlFind, dirHash, name, dirStr) err = row.Scan(&value) @@ -63,7 +63,7 @@ func (store *AbstractSqlStore) KvDelete(ctx context.Context, key []byte) (err er dirStr, dirHash, name := genDirAndName(key) - res, err := store.getTxOrDB(ctx).ExecContext(ctx, store.SqlDelete, dirHash, name, dirStr) + res, err := store.getTxOrDB(ctx, "", false).ExecContext(ctx, store.SqlDelete, dirHash, name, dirStr) if err != nil { return fmt.Errorf("kv delete: %s", err) } From ca8f793978622f566c04abf543142fcda3e2c50a Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 19 Jan 2021 12:34:58 -0800 Subject: [PATCH 155/260] refactor: prepare for bucket specific tables --- weed/filer/abstract_sql/abstract_sql_store.go | 72 +++++++++++++++---- .../abstract_sql/abstract_sql_store_kv.go | 23 ++++-- 2 files changed, 77 insertions(+), 18 deletions(-) diff --git a/weed/filer/abstract_sql/abstract_sql_store.go b/weed/filer/abstract_sql/abstract_sql_store.go index 78f408dcc..9aafd448e 100644 --- a/weed/filer/abstract_sql/abstract_sql_store.go +++ b/weed/filer/abstract_sql/abstract_sql_store.go @@ -22,6 +22,10 @@ type AbstractSqlStore struct { SqlListInclusive string } +const ( + DEFAULT = "_main" +) + type TxOrDB interface { ExecContext(ctx context.Context, query string, args ...interface{}) (sql.Result, error) QueryRowContext(ctx context.Context, query string, args ...interface{}) *sql.Row @@ -52,16 +56,22 @@ func (store *AbstractSqlStore) RollbackTransaction(ctx context.Context) error { return nil } -func (store *AbstractSqlStore) getTxOrDB(ctx context.Context, fullpath util.FullPath, isForChildren bool) TxOrDB { +func (store *AbstractSqlStore) getTxOrDB(ctx context.Context, fullpath util.FullPath, isForChildren bool) (txOrDB TxOrDB, bucket string, shortPath util.FullPath, err error) { + shortPath = fullpath if tx, ok := ctx.Value("tx").(*sql.Tx); ok { - return tx + return tx, bucket, shortPath, err } - return store.DB + return store.DB, bucket, shortPath, err } func (store *AbstractSqlStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) { - dir, name := entry.FullPath.DirAndName() + db, _, shortPath, err := store.getTxOrDB(ctx, entry.FullPath, false) + if err != nil { + return fmt.Errorf("findDB %s : %v", entry.FullPath, err) + } + + dir, name := shortPath.DirAndName() meta, err := entry.EncodeAttributesAndChunks() if err != nil { return fmt.Errorf("encode %s: %s", entry.FullPath, err) @@ -71,7 +81,7 @@ func (store *AbstractSqlStore) InsertEntry(ctx context.Context, entry *filer.Ent meta = util.MaybeGzipData(meta) } - res, err := store.getTxOrDB(ctx, entry.FullPath, false).ExecContext(ctx, store.SqlInsert, util.HashStringToLong(dir), name, dir, meta) + res, err := db.ExecContext(ctx, store.SqlInsert, util.HashStringToLong(dir), name, dir, meta) if err == nil { return } @@ -84,7 +94,7 @@ func (store *AbstractSqlStore) InsertEntry(ctx context.Context, entry *filer.Ent // now the insert failed possibly due to duplication constraints glog.V(1).Infof("insert %s falls back to update: %v", entry.FullPath, err) - res, err = store.getTxOrDB(ctx, entry.FullPath, false).ExecContext(ctx, store.SqlUpdate, meta, util.HashStringToLong(dir), name, dir) + res, err = db.ExecContext(ctx, store.SqlUpdate, meta, util.HashStringToLong(dir), name, dir) if err != nil { return fmt.Errorf("upsert %s: %s", entry.FullPath, err) } @@ -99,13 +109,18 @@ func (store *AbstractSqlStore) InsertEntry(ctx context.Context, entry *filer.Ent func (store *AbstractSqlStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) { - dir, name := entry.FullPath.DirAndName() + db, _, shortPath, err := store.getTxOrDB(ctx, entry.FullPath, false) + if err != nil { + return fmt.Errorf("findDB %s : %v", entry.FullPath, err) + } + + dir, name := shortPath.DirAndName() meta, err := entry.EncodeAttributesAndChunks() if err != nil { return fmt.Errorf("encode %s: %s", entry.FullPath, err) } - res, err := store.getTxOrDB(ctx, entry.FullPath, false).ExecContext(ctx, store.SqlUpdate, meta, util.HashStringToLong(dir), name, dir) + res, err := db.ExecContext(ctx, store.SqlUpdate, meta, util.HashStringToLong(dir), name, dir) if err != nil { return fmt.Errorf("update %s: %s", entry.FullPath, err) } @@ -119,8 +134,13 @@ func (store *AbstractSqlStore) UpdateEntry(ctx context.Context, entry *filer.Ent func (store *AbstractSqlStore) FindEntry(ctx context.Context, fullpath util.FullPath) (*filer.Entry, error) { - dir, name := fullpath.DirAndName() - row := store.getTxOrDB(ctx, fullpath, false).QueryRowContext(ctx, store.SqlFind, util.HashStringToLong(dir), name, dir) + db, _, shortPath, err := store.getTxOrDB(ctx, fullpath, false) + if err != nil { + return nil, fmt.Errorf("findDB %s : %v", fullpath, err) + } + + dir, name := shortPath.DirAndName() + row := db.QueryRowContext(ctx, store.SqlFind, util.HashStringToLong(dir), name, dir) var data []byte if err := row.Scan(&data); err != nil { @@ -142,9 +162,14 @@ func (store *AbstractSqlStore) FindEntry(ctx context.Context, fullpath util.Full func (store *AbstractSqlStore) DeleteEntry(ctx context.Context, fullpath util.FullPath) error { - dir, name := fullpath.DirAndName() + db, _, shortPath, err := store.getTxOrDB(ctx, fullpath, false) + if err != nil { + return fmt.Errorf("findDB %s : %v", fullpath, err) + } - res, err := store.getTxOrDB(ctx, fullpath, false).ExecContext(ctx, store.SqlDelete, util.HashStringToLong(dir), name, dir) + dir, name := shortPath.DirAndName() + + res, err := db.ExecContext(ctx, store.SqlDelete, util.HashStringToLong(dir), name, dir) if err != nil { return fmt.Errorf("delete %s: %s", fullpath, err) } @@ -159,7 +184,17 @@ func (store *AbstractSqlStore) DeleteEntry(ctx context.Context, fullpath util.Fu func (store *AbstractSqlStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) error { - res, err := store.getTxOrDB(ctx, fullpath, true).ExecContext(ctx, store.SqlDeleteFolderChildren, util.HashStringToLong(string(fullpath)), fullpath) + db, bucket, shortPath, err := store.getTxOrDB(ctx, fullpath, true) + if err != nil { + return fmt.Errorf("findDB %s : %v", fullpath, err) + } + + if bucket != DEFAULT && shortPath == "/" { + store.deleteTable(bucket) + return nil + } + + res, err := db.ExecContext(ctx, store.SqlDeleteFolderChildren, util.HashStringToLong(string(shortPath)), fullpath) if err != nil { return fmt.Errorf("deleteFolderChildren %s: %s", fullpath, err) } @@ -173,12 +208,18 @@ func (store *AbstractSqlStore) DeleteFolderChildren(ctx context.Context, fullpat } func (store *AbstractSqlStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { + + db, _, shortPath, err := store.getTxOrDB(ctx, dirPath, true) + if err != nil { + return lastFileName, fmt.Errorf("findDB %s : %v", dirPath, err) + } + sqlText := store.SqlListExclusive if includeStartFile { sqlText = store.SqlListInclusive } - rows, err := store.getTxOrDB(ctx, dirPath, true).QueryContext(ctx, sqlText, util.HashStringToLong(string(dirPath)), startFileName, string(dirPath), prefix+"%", limit+1) + rows, err := db.QueryContext(ctx, sqlText, util.HashStringToLong(string(shortPath)), startFileName, string(shortPath), prefix+"%", limit+1) if err != nil { return lastFileName, fmt.Errorf("list %s : %v", dirPath, err) } @@ -217,3 +258,6 @@ func (store *AbstractSqlStore) ListDirectoryEntries(ctx context.Context, dirPath func (store *AbstractSqlStore) Shutdown() { store.DB.Close() } + +func (store *AbstractSqlStore) deleteTable(bucket string) { +} diff --git a/weed/filer/abstract_sql/abstract_sql_store_kv.go b/weed/filer/abstract_sql/abstract_sql_store_kv.go index e85a85f8c..4e56c5db2 100644 --- a/weed/filer/abstract_sql/abstract_sql_store_kv.go +++ b/weed/filer/abstract_sql/abstract_sql_store_kv.go @@ -13,9 +13,14 @@ import ( func (store *AbstractSqlStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) { + db, _, _, err := store.getTxOrDB(ctx, "", false) + if err != nil { + return fmt.Errorf("findDB: %v", err) + } + dirStr, dirHash, name := genDirAndName(key) - res, err := store.getTxOrDB(ctx, "", false).ExecContext(ctx, store.SqlInsert, dirHash, name, dirStr, value) + res, err := db.ExecContext(ctx, store.SqlInsert, dirHash, name, dirStr, value) if err == nil { return } @@ -28,7 +33,7 @@ func (store *AbstractSqlStore) KvPut(ctx context.Context, key []byte, value []by // now the insert failed possibly due to duplication constraints glog.V(1).Infof("kv insert falls back to update: %s", err) - res, err = store.getTxOrDB(ctx, "", false).ExecContext(ctx, store.SqlUpdate, value, dirHash, name, dirStr) + res, err = db.ExecContext(ctx, store.SqlUpdate, value, dirHash, name, dirStr) if err != nil { return fmt.Errorf("kv upsert: %s", err) } @@ -43,8 +48,13 @@ func (store *AbstractSqlStore) KvPut(ctx context.Context, key []byte, value []by func (store *AbstractSqlStore) KvGet(ctx context.Context, key []byte) (value []byte, err error) { + db, _, _, err := store.getTxOrDB(ctx, "", false) + if err != nil { + return nil, fmt.Errorf("findDB: %v", err) + } + dirStr, dirHash, name := genDirAndName(key) - row := store.getTxOrDB(ctx, "", false).QueryRowContext(ctx, store.SqlFind, dirHash, name, dirStr) + row := db.QueryRowContext(ctx, store.SqlFind, dirHash, name, dirStr) err = row.Scan(&value) @@ -61,9 +71,14 @@ func (store *AbstractSqlStore) KvGet(ctx context.Context, key []byte) (value []b func (store *AbstractSqlStore) KvDelete(ctx context.Context, key []byte) (err error) { + db, _, _, err := store.getTxOrDB(ctx, "", false) + if err != nil { + return fmt.Errorf("findDB: %v", err) + } + dirStr, dirHash, name := genDirAndName(key) - res, err := store.getTxOrDB(ctx, "", false).ExecContext(ctx, store.SqlDelete, dirHash, name, dirStr) + res, err := db.ExecContext(ctx, store.SqlDelete, dirHash, name, dirStr) if err != nil { return fmt.Errorf("kv delete: %s", err) } From 96354208c57c69b05a9c94fc26ae13860f1f5008 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 19 Jan 2021 12:35:14 -0800 Subject: [PATCH 156/260] scaffold: add YugabyteDB --- weed/command/scaffold.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/command/scaffold.go b/weed/command/scaffold.go index f55123573..949f22588 100644 --- a/weed/command/scaffold.go +++ b/weed/command/scaffold.go @@ -121,7 +121,7 @@ connection_max_open = 100 connection_max_lifetime_seconds = 0 interpolateParams = false -[postgres] # or cockroachdb +[postgres] # or cockroachdb, YugabyteDB # CREATE TABLE IF NOT EXISTS filemeta ( # dirhash BIGINT, # name VARCHAR(65535), From 4c5b752b040bbbee34fdc1db61fe6e210fb11961 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 19 Jan 2021 13:53:16 -0800 Subject: [PATCH 157/260] restructuring sql stores --- weed/filer/abstract_sql/abstract_sql_store.go | 113 +++++++++++++----- .../abstract_sql/abstract_sql_store_kv.go | 8 +- weed/filer/mysql/mysql_store.go | 45 +++++-- weed/filer/postgres/postgres_store.go | 43 +++++-- 4 files changed, 162 insertions(+), 47 deletions(-) diff --git a/weed/filer/abstract_sql/abstract_sql_store.go b/weed/filer/abstract_sql/abstract_sql_store.go index 9aafd448e..dd35112a2 100644 --- a/weed/filer/abstract_sql/abstract_sql_store.go +++ b/weed/filer/abstract_sql/abstract_sql_store.go @@ -9,21 +9,28 @@ import ( "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/util" "strings" + "sync" ) +type SqlGenerator interface { + GetSqlInsert(bucket string) string + GetSqlUpdate(bucket string) string + GetSqlFind(bucket string) string + GetSqlDelete(bucket string) string + GetSqlDeleteFolderChildren(bucket string) string + GetSqlListExclusive(bucket string) string + GetSqlListInclusive(bucket string) string +} + type AbstractSqlStore struct { - DB *sql.DB - SqlInsert string - SqlUpdate string - SqlFind string - SqlDelete string - SqlDeleteFolderChildren string - SqlListExclusive string - SqlListInclusive string + DB *sql.DB + SqlGenerator + dbs map[string]bool + dbsLock sync.Mutex } const ( - DEFAULT = "_main" + DEFAULT_TABLE = "filemeta" ) type TxOrDB interface { @@ -57,16 +64,53 @@ func (store *AbstractSqlStore) RollbackTransaction(ctx context.Context) error { } func (store *AbstractSqlStore) getTxOrDB(ctx context.Context, fullpath util.FullPath, isForChildren bool) (txOrDB TxOrDB, bucket string, shortPath util.FullPath, err error) { + shortPath = fullpath + bucket = DEFAULT_TABLE + if tx, ok := ctx.Value("tx").(*sql.Tx); ok { - return tx, bucket, shortPath, err + txOrDB = tx + } else { + txOrDB = store.DB + } + + if strings.HasPrefix(string(fullpath), "/buckets/") { + return } - return store.DB, bucket, shortPath, err + + // detect bucket + bucketAndObjectKey := string(fullpath)[len("/buckets/"):] + t := strings.Index(bucketAndObjectKey, "/") + if t < 0 && !isForChildren { + return + } + if t > 0 { + bucket = bucketAndObjectKey[:t] + shortPath = util.FullPath(bucketAndObjectKey[t:]) + } + + if isValidBucket(bucket) { + store.dbsLock.Lock() + defer store.dbsLock.Unlock() + + if store.dbs == nil { + store.dbs = make(map[string]bool) + } + + if _, found := store.dbs[bucket]; !found { + if err = store.createTable(bucket); err != nil { + store.dbs[bucket] = true + } + } + + } + + return } func (store *AbstractSqlStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) { - db, _, shortPath, err := store.getTxOrDB(ctx, entry.FullPath, false) + db, bucket, shortPath, err := store.getTxOrDB(ctx, entry.FullPath, false) if err != nil { return fmt.Errorf("findDB %s : %v", entry.FullPath, err) } @@ -81,7 +125,7 @@ func (store *AbstractSqlStore) InsertEntry(ctx context.Context, entry *filer.Ent meta = util.MaybeGzipData(meta) } - res, err := db.ExecContext(ctx, store.SqlInsert, util.HashStringToLong(dir), name, dir, meta) + res, err := db.ExecContext(ctx, store.GetSqlInsert(bucket), util.HashStringToLong(dir), name, dir, meta) if err == nil { return } @@ -94,7 +138,7 @@ func (store *AbstractSqlStore) InsertEntry(ctx context.Context, entry *filer.Ent // now the insert failed possibly due to duplication constraints glog.V(1).Infof("insert %s falls back to update: %v", entry.FullPath, err) - res, err = db.ExecContext(ctx, store.SqlUpdate, meta, util.HashStringToLong(dir), name, dir) + res, err = db.ExecContext(ctx, store.GetSqlUpdate(bucket), meta, util.HashStringToLong(dir), name, dir) if err != nil { return fmt.Errorf("upsert %s: %s", entry.FullPath, err) } @@ -109,7 +153,7 @@ func (store *AbstractSqlStore) InsertEntry(ctx context.Context, entry *filer.Ent func (store *AbstractSqlStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) { - db, _, shortPath, err := store.getTxOrDB(ctx, entry.FullPath, false) + db, bucket, shortPath, err := store.getTxOrDB(ctx, entry.FullPath, false) if err != nil { return fmt.Errorf("findDB %s : %v", entry.FullPath, err) } @@ -120,7 +164,7 @@ func (store *AbstractSqlStore) UpdateEntry(ctx context.Context, entry *filer.Ent return fmt.Errorf("encode %s: %s", entry.FullPath, err) } - res, err := db.ExecContext(ctx, store.SqlUpdate, meta, util.HashStringToLong(dir), name, dir) + res, err := db.ExecContext(ctx, store.GetSqlUpdate(bucket), meta, util.HashStringToLong(dir), name, dir) if err != nil { return fmt.Errorf("update %s: %s", entry.FullPath, err) } @@ -134,13 +178,13 @@ func (store *AbstractSqlStore) UpdateEntry(ctx context.Context, entry *filer.Ent func (store *AbstractSqlStore) FindEntry(ctx context.Context, fullpath util.FullPath) (*filer.Entry, error) { - db, _, shortPath, err := store.getTxOrDB(ctx, fullpath, false) + db, bucket, shortPath, err := store.getTxOrDB(ctx, fullpath, false) if err != nil { return nil, fmt.Errorf("findDB %s : %v", fullpath, err) } dir, name := shortPath.DirAndName() - row := db.QueryRowContext(ctx, store.SqlFind, util.HashStringToLong(dir), name, dir) + row := db.QueryRowContext(ctx, store.GetSqlFind(bucket), util.HashStringToLong(dir), name, dir) var data []byte if err := row.Scan(&data); err != nil { @@ -162,14 +206,14 @@ func (store *AbstractSqlStore) FindEntry(ctx context.Context, fullpath util.Full func (store *AbstractSqlStore) DeleteEntry(ctx context.Context, fullpath util.FullPath) error { - db, _, shortPath, err := store.getTxOrDB(ctx, fullpath, false) + db, bucket, shortPath, err := store.getTxOrDB(ctx, fullpath, false) if err != nil { return fmt.Errorf("findDB %s : %v", fullpath, err) } dir, name := shortPath.DirAndName() - res, err := db.ExecContext(ctx, store.SqlDelete, util.HashStringToLong(dir), name, dir) + res, err := db.ExecContext(ctx, store.GetSqlDelete(bucket), util.HashStringToLong(dir), name, dir) if err != nil { return fmt.Errorf("delete %s: %s", fullpath, err) } @@ -189,12 +233,16 @@ func (store *AbstractSqlStore) DeleteFolderChildren(ctx context.Context, fullpat return fmt.Errorf("findDB %s : %v", fullpath, err) } - if bucket != DEFAULT && shortPath == "/" { - store.deleteTable(bucket) - return nil + if isValidBucket(bucket) && shortPath == "/" { + if store.deleteTable(bucket) { + store.dbsLock.Lock() + delete(store.dbs, bucket) + store.dbsLock.Unlock() + return nil + } } - res, err := db.ExecContext(ctx, store.SqlDeleteFolderChildren, util.HashStringToLong(string(shortPath)), fullpath) + res, err := db.ExecContext(ctx, store.GetSqlDeleteFolderChildren(bucket), util.HashStringToLong(string(shortPath)), fullpath) if err != nil { return fmt.Errorf("deleteFolderChildren %s: %s", fullpath, err) } @@ -209,14 +257,14 @@ func (store *AbstractSqlStore) DeleteFolderChildren(ctx context.Context, fullpat func (store *AbstractSqlStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { - db, _, shortPath, err := store.getTxOrDB(ctx, dirPath, true) + db, bucket, shortPath, err := store.getTxOrDB(ctx, dirPath, true) if err != nil { return lastFileName, fmt.Errorf("findDB %s : %v", dirPath, err) } - sqlText := store.SqlListExclusive + sqlText := store.GetSqlListExclusive(bucket) if includeStartFile { - sqlText = store.SqlListInclusive + sqlText = store.GetSqlListInclusive(bucket) } rows, err := db.QueryContext(ctx, sqlText, util.HashStringToLong(string(shortPath)), startFileName, string(shortPath), prefix+"%", limit+1) @@ -259,5 +307,14 @@ func (store *AbstractSqlStore) Shutdown() { store.DB.Close() } -func (store *AbstractSqlStore) deleteTable(bucket string) { +func isValidBucket(bucket string) bool { + return bucket != DEFAULT_TABLE && bucket != "" +} + +func (store *AbstractSqlStore) createTable(bucket string) error { + return nil +} + +func (store *AbstractSqlStore) deleteTable(bucket string) bool { + return false } diff --git a/weed/filer/abstract_sql/abstract_sql_store_kv.go b/weed/filer/abstract_sql/abstract_sql_store_kv.go index 4e56c5db2..03b016c76 100644 --- a/weed/filer/abstract_sql/abstract_sql_store_kv.go +++ b/weed/filer/abstract_sql/abstract_sql_store_kv.go @@ -20,7 +20,7 @@ func (store *AbstractSqlStore) KvPut(ctx context.Context, key []byte, value []by dirStr, dirHash, name := genDirAndName(key) - res, err := db.ExecContext(ctx, store.SqlInsert, dirHash, name, dirStr, value) + res, err := db.ExecContext(ctx, store.GetSqlInsert(DEFAULT_TABLE), dirHash, name, dirStr, value) if err == nil { return } @@ -33,7 +33,7 @@ func (store *AbstractSqlStore) KvPut(ctx context.Context, key []byte, value []by // now the insert failed possibly due to duplication constraints glog.V(1).Infof("kv insert falls back to update: %s", err) - res, err = db.ExecContext(ctx, store.SqlUpdate, value, dirHash, name, dirStr) + res, err = db.ExecContext(ctx, store.GetSqlUpdate(DEFAULT_TABLE), value, dirHash, name, dirStr) if err != nil { return fmt.Errorf("kv upsert: %s", err) } @@ -54,7 +54,7 @@ func (store *AbstractSqlStore) KvGet(ctx context.Context, key []byte) (value []b } dirStr, dirHash, name := genDirAndName(key) - row := db.QueryRowContext(ctx, store.SqlFind, dirHash, name, dirStr) + row := db.QueryRowContext(ctx, store.GetSqlFind(DEFAULT_TABLE), dirHash, name, dirStr) err = row.Scan(&value) @@ -78,7 +78,7 @@ func (store *AbstractSqlStore) KvDelete(ctx context.Context, key []byte) (err er dirStr, dirHash, name := genDirAndName(key) - res, err := db.ExecContext(ctx, store.SqlDelete, dirHash, name, dirStr) + res, err := db.ExecContext(ctx, store.GetSqlDelete(DEFAULT_TABLE), dirHash, name, dirStr) if err != nil { return fmt.Errorf("kv delete: %s", err) } diff --git a/weed/filer/mysql/mysql_store.go b/weed/filer/mysql/mysql_store.go index 70f729fc9..479d2eed1 100644 --- a/weed/filer/mysql/mysql_store.go +++ b/weed/filer/mysql/mysql_store.go @@ -15,6 +15,41 @@ const ( CONNECTION_URL_PATTERN = "%s:%s@tcp(%s:%d)/%s?charset=utf8" ) +type SqlGenMysql struct { +} + +var ( + _ = abstract_sql.SqlGenerator(&SqlGenMysql{}) +) + +func (gen *SqlGenMysql) GetSqlInsert(bucket string) string { + return "INSERT INTO filemeta (dirhash,name,directory,meta) VALUES(?,?,?,?)" +} + +func (gen *SqlGenMysql) GetSqlUpdate(bucket string) string { + return "UPDATE filemeta SET meta=? WHERE dirhash=? AND name=? AND directory=?" +} + +func (gen *SqlGenMysql) GetSqlFind(bucket string) string { + return "SELECT meta FROM filemeta WHERE dirhash=? AND name=? AND directory=?" +} + +func (gen *SqlGenMysql) GetSqlDelete(bucket string) string { + return "DELETE FROM filemeta WHERE dirhash=? AND name=? AND directory=?" +} + +func (gen *SqlGenMysql) GetSqlDeleteFolderChildren(bucket string) string { + return "DELETE FROM filemeta WHERE dirhash=? AND directory=?" +} + +func (gen *SqlGenMysql) GetSqlListExclusive(bucket string) string { + return "SELECT NAME, meta FROM filemeta WHERE dirhash=? AND name>? AND directory=? AND name like ? ORDER BY NAME ASC LIMIT ?" +} + +func (gen *SqlGenMysql) GetSqlListInclusive(bucket string) string { + return "SELECT NAME, meta FROM filemeta WHERE dirhash=? AND name>=? AND directory=? AND name like ? ORDER BY NAME ASC LIMIT ?" +} + func init() { filer.Stores = append(filer.Stores, &MysqlStore{}) } @@ -43,14 +78,8 @@ func (store *MysqlStore) Initialize(configuration util.Configuration, prefix str func (store *MysqlStore) initialize(user, password, hostname string, port int, database string, maxIdle, maxOpen, maxLifetimeSeconds int, interpolateParams bool) (err error) { - // - store.SqlInsert = "INSERT INTO filemeta (dirhash,name,directory,meta) VALUES(?,?,?,?)" - store.SqlUpdate = "UPDATE filemeta SET meta=? WHERE dirhash=? AND name=? AND directory=?" - store.SqlFind = "SELECT meta FROM filemeta WHERE dirhash=? AND name=? AND directory=?" - store.SqlDelete = "DELETE FROM filemeta WHERE dirhash=? AND name=? AND directory=?" - store.SqlDeleteFolderChildren = "DELETE FROM filemeta WHERE dirhash=? AND directory=?" - store.SqlListExclusive = "SELECT NAME, meta FROM filemeta WHERE dirhash=? AND name>? AND directory=? AND name like ? ORDER BY NAME ASC LIMIT ?" - store.SqlListInclusive = "SELECT NAME, meta FROM filemeta WHERE dirhash=? AND name>=? AND directory=? AND name like ? ORDER BY NAME ASC LIMIT ?" + + store.SqlGenerator = &SqlGenMysql{} sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, user, password, hostname, port, database) if interpolateParams { diff --git a/weed/filer/postgres/postgres_store.go b/weed/filer/postgres/postgres_store.go index 2325568fe..783f27c10 100644 --- a/weed/filer/postgres/postgres_store.go +++ b/weed/filer/postgres/postgres_store.go @@ -14,6 +14,41 @@ const ( CONNECTION_URL_PATTERN = "host=%s port=%d sslmode=%s connect_timeout=30" ) +type SqlGenPostgres struct { +} + +var ( + _ = abstract_sql.SqlGenerator(&SqlGenPostgres{}) +) + +func (gen *SqlGenPostgres) GetSqlInsert(bucket string) string { + return "INSERT INTO filemeta (dirhash,name,directory,meta) VALUES($1,$2,$3,$4)" +} + +func (gen *SqlGenPostgres) GetSqlUpdate(bucket string) string { + return "UPDATE filemeta SET meta=$1 WHERE dirhash=$2 AND name=$3 AND directory=$4" +} + +func (gen *SqlGenPostgres) GetSqlFind(bucket string) string { + return "SELECT meta FROM filemeta WHERE dirhash=$1 AND name=$2 AND directory=$3" +} + +func (gen *SqlGenPostgres) GetSqlDelete(bucket string) string { + return "DELETE FROM filemeta WHERE dirhash=$1 AND name=$2 AND directory=$3" +} + +func (gen *SqlGenPostgres) GetSqlDeleteFolderChildren(bucket string) string { + return "DELETE FROM filemeta WHERE dirhash=$1 AND directory=$2" +} + +func (gen *SqlGenPostgres) GetSqlListExclusive(bucket string) string { + return "SELECT NAME, meta FROM filemeta WHERE dirhash=$1 AND name>$2 AND directory=$3 AND name like $4 ORDER BY NAME ASC LIMIT $5" +} + +func (gen *SqlGenPostgres) GetSqlListInclusive(bucket string) string { + return "SELECT NAME, meta FROM filemeta WHERE dirhash=$1 AND name>=$2 AND directory=$3 AND name like $4 ORDER BY NAME ASC LIMIT $5" +} + func init() { filer.Stores = append(filer.Stores, &PostgresStore{}) } @@ -41,13 +76,7 @@ func (store *PostgresStore) Initialize(configuration util.Configuration, prefix func (store *PostgresStore) initialize(user, password, hostname string, port int, database, sslmode string, maxIdle, maxOpen int) (err error) { - store.SqlInsert = "INSERT INTO filemeta (dirhash,name,directory,meta) VALUES($1,$2,$3,$4)" - store.SqlUpdate = "UPDATE filemeta SET meta=$1 WHERE dirhash=$2 AND name=$3 AND directory=$4" - store.SqlFind = "SELECT meta FROM filemeta WHERE dirhash=$1 AND name=$2 AND directory=$3" - store.SqlDelete = "DELETE FROM filemeta WHERE dirhash=$1 AND name=$2 AND directory=$3" - store.SqlDeleteFolderChildren = "DELETE FROM filemeta WHERE dirhash=$1 AND directory=$2" - store.SqlListExclusive = "SELECT NAME, meta FROM filemeta WHERE dirhash=$1 AND name>$2 AND directory=$3 AND name like $4 ORDER BY NAME ASC LIMIT $5" - store.SqlListInclusive = "SELECT NAME, meta FROM filemeta WHERE dirhash=$1 AND name>=$2 AND directory=$3 AND name like $4 ORDER BY NAME ASC LIMIT $5" + store.SqlGenerator = &SqlGenPostgres{} sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, hostname, port, sslmode) if user != "" { From fa0c8d528370ee419cf4ed64213b1b4cdf014fac Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 19 Jan 2021 14:05:48 -0800 Subject: [PATCH 158/260] fix error --- weed/filer/abstract_sql/abstract_sql_store.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/filer/abstract_sql/abstract_sql_store.go b/weed/filer/abstract_sql/abstract_sql_store.go index dd35112a2..878386cf7 100644 --- a/weed/filer/abstract_sql/abstract_sql_store.go +++ b/weed/filer/abstract_sql/abstract_sql_store.go @@ -74,7 +74,7 @@ func (store *AbstractSqlStore) getTxOrDB(ctx context.Context, fullpath util.Full txOrDB = store.DB } - if strings.HasPrefix(string(fullpath), "/buckets/") { + if !strings.HasPrefix(string(fullpath), "/buckets/") { return } From 93b3adba9864dd278a778ccfe2165bcadb010574 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 19 Jan 2021 15:55:51 -0800 Subject: [PATCH 159/260] fix bucket creation --- weed/filer/abstract_sql/abstract_sql_store.go | 33 ++++++++++++++----- weed/filer/mysql/mysql_store.go | 16 ++++++++- weed/filer/postgres/postgres_store.go | 16 ++++++++- 3 files changed, 54 insertions(+), 11 deletions(-) diff --git a/weed/filer/abstract_sql/abstract_sql_store.go b/weed/filer/abstract_sql/abstract_sql_store.go index 878386cf7..aada9cd4a 100644 --- a/weed/filer/abstract_sql/abstract_sql_store.go +++ b/weed/filer/abstract_sql/abstract_sql_store.go @@ -20,13 +20,16 @@ type SqlGenerator interface { GetSqlDeleteFolderChildren(bucket string) string GetSqlListExclusive(bucket string) string GetSqlListInclusive(bucket string) string + GetSqlCreateTable(bucket string) string + GetSqlDropTable(bucket string) string } type AbstractSqlStore struct { - DB *sql.DB SqlGenerator - dbs map[string]bool - dbsLock sync.Mutex + DB *sql.DB + SupportBucketTable bool + dbs map[string]bool + dbsLock sync.Mutex } const ( @@ -74,6 +77,10 @@ func (store *AbstractSqlStore) getTxOrDB(ctx context.Context, fullpath util.Full txOrDB = store.DB } + if !store.SupportBucketTable { + return + } + if !strings.HasPrefix(string(fullpath), "/buckets/") { return } @@ -98,7 +105,7 @@ func (store *AbstractSqlStore) getTxOrDB(ctx context.Context, fullpath util.Full } if _, found := store.dbs[bucket]; !found { - if err = store.createTable(bucket); err != nil { + if err = store.createTable(ctx, bucket); err != nil { store.dbs[bucket] = true } } @@ -234,7 +241,7 @@ func (store *AbstractSqlStore) DeleteFolderChildren(ctx context.Context, fullpat } if isValidBucket(bucket) && shortPath == "/" { - if store.deleteTable(bucket) { + if err = store.deleteTable(ctx, bucket); err != nil { store.dbsLock.Lock() delete(store.dbs, bucket) store.dbsLock.Unlock() @@ -311,10 +318,18 @@ func isValidBucket(bucket string) bool { return bucket != DEFAULT_TABLE && bucket != "" } -func (store *AbstractSqlStore) createTable(bucket string) error { - return nil +func (store *AbstractSqlStore) createTable(ctx context.Context, bucket string) error { + if !store.SupportBucketTable { + return nil + } + _, err := store.DB.ExecContext(ctx, store.SqlGenerator.GetSqlCreateTable(bucket)) + return err } -func (store *AbstractSqlStore) deleteTable(bucket string) bool { - return false +func (store *AbstractSqlStore) deleteTable(ctx context.Context, bucket string) error { + if !store.SupportBucketTable { + return nil + } + _, err := store.DB.ExecContext(ctx, store.SqlGenerator.GetSqlDropTable(bucket)) + return err } diff --git a/weed/filer/mysql/mysql_store.go b/weed/filer/mysql/mysql_store.go index 479d2eed1..0e29852f5 100644 --- a/weed/filer/mysql/mysql_store.go +++ b/weed/filer/mysql/mysql_store.go @@ -16,6 +16,8 @@ const ( ) type SqlGenMysql struct { + createTableSqlTemplate string + dropTableSqlTemplate string } var ( @@ -50,6 +52,14 @@ func (gen *SqlGenMysql) GetSqlListInclusive(bucket string) string { return "SELECT NAME, meta FROM filemeta WHERE dirhash=? AND name>=? AND directory=? AND name like ? ORDER BY NAME ASC LIMIT ?" } +func (gen *SqlGenMysql) GetSqlCreateTable(bucket string) string { + return fmt.Sprintf(gen.createTableSqlTemplate, bucket) +} + +func (gen *SqlGenMysql) GetSqlDropTable(bucket string) string { + return fmt.Sprintf(gen.dropTableSqlTemplate, bucket) +} + func init() { filer.Stores = append(filer.Stores, &MysqlStore{}) } @@ -79,7 +89,11 @@ func (store *MysqlStore) Initialize(configuration util.Configuration, prefix str func (store *MysqlStore) initialize(user, password, hostname string, port int, database string, maxIdle, maxOpen, maxLifetimeSeconds int, interpolateParams bool) (err error) { - store.SqlGenerator = &SqlGenMysql{} + store.SupportBucketTable = false + store.SqlGenerator = &SqlGenMysql{ + createTableSqlTemplate: "", + dropTableSqlTemplate: "drop table %s", + } sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, user, password, hostname, port, database) if interpolateParams { diff --git a/weed/filer/postgres/postgres_store.go b/weed/filer/postgres/postgres_store.go index 783f27c10..8c36b8672 100644 --- a/weed/filer/postgres/postgres_store.go +++ b/weed/filer/postgres/postgres_store.go @@ -15,6 +15,8 @@ const ( ) type SqlGenPostgres struct { + createTableSqlTemplate string + dropTableSqlTemplate string } var ( @@ -49,6 +51,14 @@ func (gen *SqlGenPostgres) GetSqlListInclusive(bucket string) string { return "SELECT NAME, meta FROM filemeta WHERE dirhash=$1 AND name>=$2 AND directory=$3 AND name like $4 ORDER BY NAME ASC LIMIT $5" } +func (gen *SqlGenPostgres) GetSqlCreateTable(bucket string) string { + return fmt.Sprintf(gen.createTableSqlTemplate, bucket) +} + +func (gen *SqlGenPostgres) GetSqlDropTable(bucket string) string { + return fmt.Sprintf(gen.dropTableSqlTemplate, bucket) +} + func init() { filer.Stores = append(filer.Stores, &PostgresStore{}) } @@ -76,7 +86,11 @@ func (store *PostgresStore) Initialize(configuration util.Configuration, prefix func (store *PostgresStore) initialize(user, password, hostname string, port int, database, sslmode string, maxIdle, maxOpen int) (err error) { - store.SqlGenerator = &SqlGenPostgres{} + store.SupportBucketTable = false + store.SqlGenerator = &SqlGenPostgres{ + createTableSqlTemplate: "", + dropTableSqlTemplate: "drop table %s", + } sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, hostname, port, sslmode) if user != "" { From 52a8f1470ef92b9d0075c3dccc76a4170ab30c07 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 19 Jan 2021 17:21:50 -0800 Subject: [PATCH 160/260] filer store: add mysql2 --- weed/command/scaffold.go | 21 +++++ weed/filer/abstract_sql/abstract_sql_store.go | 6 +- weed/filer/mysql/mysql_sql_gen.go | 52 +++++++++++++ weed/filer/mysql/mysql_store.go | 51 +----------- weed/filer/mysql2/mysql_store.go | 77 +++++++++++++++++++ weed/server/filer_server.go | 1 + weed/util/config.go | 6 +- 7 files changed, 162 insertions(+), 52 deletions(-) create mode 100644 weed/filer/mysql/mysql_sql_gen.go create mode 100644 weed/filer/mysql2/mysql_store.go diff --git a/weed/command/scaffold.go b/weed/command/scaffold.go index 949f22588..b44bc41c1 100644 --- a/weed/command/scaffold.go +++ b/weed/command/scaffold.go @@ -121,6 +121,27 @@ connection_max_open = 100 connection_max_lifetime_seconds = 0 interpolateParams = false +[mysql2] # or memsql, tidb +enabled = false +createTable = """ + CREATE TABLE IF NOT EXISTS %s ( + dirhash BIGINT, + name VARCHAR(1000), + directory TEXT, + meta LONGBLOB, + PRIMARY KEY (dirhash, name) + ) DEFAULT CHARSET=utf8; +""" +hostname = "localhost" +port = 3306 +username = "root" +password = "" +database = "" # create or use an existing database +connection_max_idle = 2 +connection_max_open = 100 +connection_max_lifetime_seconds = 0 +interpolateParams = false + [postgres] # or cockroachdb, YugabyteDB # CREATE TABLE IF NOT EXISTS filemeta ( # dirhash BIGINT, diff --git a/weed/filer/abstract_sql/abstract_sql_store.go b/weed/filer/abstract_sql/abstract_sql_store.go index aada9cd4a..8345d7a7b 100644 --- a/weed/filer/abstract_sql/abstract_sql_store.go +++ b/weed/filer/abstract_sql/abstract_sql_store.go @@ -91,6 +91,8 @@ func (store *AbstractSqlStore) getTxOrDB(ctx context.Context, fullpath util.Full if t < 0 && !isForChildren { return } + bucket = bucketAndObjectKey + shortPath = "/" if t > 0 { bucket = bucketAndObjectKey[:t] shortPath = util.FullPath(bucketAndObjectKey[t:]) @@ -241,11 +243,13 @@ func (store *AbstractSqlStore) DeleteFolderChildren(ctx context.Context, fullpat } if isValidBucket(bucket) && shortPath == "/" { - if err = store.deleteTable(ctx, bucket); err != nil { + if err = store.deleteTable(ctx, bucket); err == nil { store.dbsLock.Lock() delete(store.dbs, bucket) store.dbsLock.Unlock() return nil + } else { + return err } } diff --git a/weed/filer/mysql/mysql_sql_gen.go b/weed/filer/mysql/mysql_sql_gen.go new file mode 100644 index 000000000..057484c37 --- /dev/null +++ b/weed/filer/mysql/mysql_sql_gen.go @@ -0,0 +1,52 @@ +package mysql + +import ( + "fmt" + "github.com/chrislusf/seaweedfs/weed/filer/abstract_sql" + _ "github.com/go-sql-driver/mysql" +) + +type SqlGenMysql struct { + CreateTableSqlTemplate string + DropTableSqlTemplate string +} + +var ( + _ = abstract_sql.SqlGenerator(&SqlGenMysql{}) +) + +func (gen *SqlGenMysql) GetSqlInsert(bucket string) string { + return fmt.Sprintf("INSERT INTO %s (dirhash,name,directory,meta) VALUES(?,?,?,?)", bucket) +} + +func (gen *SqlGenMysql) GetSqlUpdate(bucket string) string { + return fmt.Sprintf("UPDATE %s SET meta=? WHERE dirhash=? AND name=? AND directory=?", bucket) +} + +func (gen *SqlGenMysql) GetSqlFind(bucket string) string { + return fmt.Sprintf("SELECT meta FROM %s WHERE dirhash=? AND name=? AND directory=?", bucket) +} + +func (gen *SqlGenMysql) GetSqlDelete(bucket string) string { + return fmt.Sprintf("DELETE FROM %s WHERE dirhash=? AND name=? AND directory=?", bucket) +} + +func (gen *SqlGenMysql) GetSqlDeleteFolderChildren(bucket string) string { + return fmt.Sprintf("DELETE FROM %s WHERE dirhash=? AND directory=?", bucket) +} + +func (gen *SqlGenMysql) GetSqlListExclusive(bucket string) string { + return fmt.Sprintf("SELECT NAME, meta FROM %s WHERE dirhash=? AND name>? AND directory=? AND name like ? ORDER BY NAME ASC LIMIT ?", bucket) +} + +func (gen *SqlGenMysql) GetSqlListInclusive(bucket string) string { + return fmt.Sprintf("SELECT NAME, meta FROM %s WHERE dirhash=? AND name>=? AND directory=? AND name like ? ORDER BY NAME ASC LIMIT ?", bucket) +} + +func (gen *SqlGenMysql) GetSqlCreateTable(bucket string) string { + return fmt.Sprintf(gen.CreateTableSqlTemplate, bucket) +} + +func (gen *SqlGenMysql) GetSqlDropTable(bucket string) string { + return fmt.Sprintf(gen.DropTableSqlTemplate, bucket) +} diff --git a/weed/filer/mysql/mysql_store.go b/weed/filer/mysql/mysql_store.go index 0e29852f5..686628740 100644 --- a/weed/filer/mysql/mysql_store.go +++ b/weed/filer/mysql/mysql_store.go @@ -3,9 +3,9 @@ package mysql import ( "database/sql" "fmt" + "github.com/chrislusf/seaweedfs/weed/filer" "time" - "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/filer/abstract_sql" "github.com/chrislusf/seaweedfs/weed/util" _ "github.com/go-sql-driver/mysql" @@ -15,51 +15,6 @@ const ( CONNECTION_URL_PATTERN = "%s:%s@tcp(%s:%d)/%s?charset=utf8" ) -type SqlGenMysql struct { - createTableSqlTemplate string - dropTableSqlTemplate string -} - -var ( - _ = abstract_sql.SqlGenerator(&SqlGenMysql{}) -) - -func (gen *SqlGenMysql) GetSqlInsert(bucket string) string { - return "INSERT INTO filemeta (dirhash,name,directory,meta) VALUES(?,?,?,?)" -} - -func (gen *SqlGenMysql) GetSqlUpdate(bucket string) string { - return "UPDATE filemeta SET meta=? WHERE dirhash=? AND name=? AND directory=?" -} - -func (gen *SqlGenMysql) GetSqlFind(bucket string) string { - return "SELECT meta FROM filemeta WHERE dirhash=? AND name=? AND directory=?" -} - -func (gen *SqlGenMysql) GetSqlDelete(bucket string) string { - return "DELETE FROM filemeta WHERE dirhash=? AND name=? AND directory=?" -} - -func (gen *SqlGenMysql) GetSqlDeleteFolderChildren(bucket string) string { - return "DELETE FROM filemeta WHERE dirhash=? AND directory=?" -} - -func (gen *SqlGenMysql) GetSqlListExclusive(bucket string) string { - return "SELECT NAME, meta FROM filemeta WHERE dirhash=? AND name>? AND directory=? AND name like ? ORDER BY NAME ASC LIMIT ?" -} - -func (gen *SqlGenMysql) GetSqlListInclusive(bucket string) string { - return "SELECT NAME, meta FROM filemeta WHERE dirhash=? AND name>=? AND directory=? AND name like ? ORDER BY NAME ASC LIMIT ?" -} - -func (gen *SqlGenMysql) GetSqlCreateTable(bucket string) string { - return fmt.Sprintf(gen.createTableSqlTemplate, bucket) -} - -func (gen *SqlGenMysql) GetSqlDropTable(bucket string) string { - return fmt.Sprintf(gen.dropTableSqlTemplate, bucket) -} - func init() { filer.Stores = append(filer.Stores, &MysqlStore{}) } @@ -91,8 +46,8 @@ func (store *MysqlStore) initialize(user, password, hostname string, port int, d store.SupportBucketTable = false store.SqlGenerator = &SqlGenMysql{ - createTableSqlTemplate: "", - dropTableSqlTemplate: "drop table %s", + CreateTableSqlTemplate: "", + DropTableSqlTemplate: "drop table %s", } sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, user, password, hostname, port, database) diff --git a/weed/filer/mysql2/mysql_store.go b/weed/filer/mysql2/mysql_store.go new file mode 100644 index 000000000..fb74b6f7f --- /dev/null +++ b/weed/filer/mysql2/mysql_store.go @@ -0,0 +1,77 @@ +package mysql2 + +import ( + "database/sql" + "fmt" + "time" + + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/filer/abstract_sql" + "github.com/chrislusf/seaweedfs/weed/filer/mysql" + "github.com/chrislusf/seaweedfs/weed/util" + _ "github.com/go-sql-driver/mysql" +) + +const ( + CONNECTION_URL_PATTERN = "%s:%s@tcp(%s:%d)/%s?charset=utf8" +) + +func init() { + filer.Stores = append(filer.Stores, &MysqlStore2{}) +} + +type MysqlStore2 struct { + abstract_sql.AbstractSqlStore +} + +func (store *MysqlStore2) GetName() string { + return "mysql2" +} + +func (store *MysqlStore2) Initialize(configuration util.Configuration, prefix string) (err error) { + return store.initialize( + configuration.GetString(prefix+"createTable"), + configuration.GetString(prefix+"username"), + configuration.GetString(prefix+"password"), + configuration.GetString(prefix+"hostname"), + configuration.GetInt(prefix+"port"), + configuration.GetString(prefix+"database"), + configuration.GetInt(prefix+"connection_max_idle"), + configuration.GetInt(prefix+"connection_max_open"), + configuration.GetInt(prefix+"connection_max_lifetime_seconds"), + configuration.GetBool(prefix+"interpolateParams"), + ) +} + +func (store *MysqlStore2) initialize(createTable, user, password, hostname string, port int, database string, maxIdle, maxOpen, + maxLifetimeSeconds int, interpolateParams bool) (err error) { + + store.SupportBucketTable = true + store.SqlGenerator = &mysql.SqlGenMysql{ + CreateTableSqlTemplate: createTable, + DropTableSqlTemplate: "drop table %s", + } + + sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, user, password, hostname, port, database) + if interpolateParams { + sqlUrl += "&interpolateParams=true" + } + + var dbErr error + store.DB, dbErr = sql.Open("mysql", sqlUrl) + if dbErr != nil { + store.DB.Close() + store.DB = nil + return fmt.Errorf("can not connect to %s error:%v", sqlUrl, err) + } + + store.DB.SetMaxIdleConns(maxIdle) + store.DB.SetMaxOpenConns(maxOpen) + store.DB.SetConnMaxLifetime(time.Duration(maxLifetimeSeconds) * time.Second) + + if err = store.DB.Ping(); err != nil { + return fmt.Errorf("connect to %s error:%v", sqlUrl, err) + } + + return nil +} diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index 2f03dc289..8ea2b8b82 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -29,6 +29,7 @@ import ( _ "github.com/chrislusf/seaweedfs/weed/filer/leveldb3" _ "github.com/chrislusf/seaweedfs/weed/filer/mongodb" _ "github.com/chrislusf/seaweedfs/weed/filer/mysql" + _ "github.com/chrislusf/seaweedfs/weed/filer/mysql2" _ "github.com/chrislusf/seaweedfs/weed/filer/postgres" _ "github.com/chrislusf/seaweedfs/weed/filer/redis" _ "github.com/chrislusf/seaweedfs/weed/filer/redis2" diff --git a/weed/util/config.go b/weed/util/config.go index 2af1ad09e..ee805f26a 100644 --- a/weed/util/config.go +++ b/weed/util/config.go @@ -29,11 +29,11 @@ func LoadConfiguration(configFileName string, required bool) (loaded bool) { glog.V(1).Infof("Reading %s.toml from %s", configFileName, viper.ConfigFileUsed()) if err := viper.MergeInConfig(); err != nil { // Handle errors reading the config file - logLevel := glog.Level(0) if strings.Contains(err.Error(), "Not Found") { - logLevel = 1 + glog.V(1).Infof("Reading %s: %v", viper.ConfigFileUsed(), err) + } else { + glog.Fatalf("Reading %s: %v", viper.ConfigFileUsed(), err) } - glog.V(logLevel).Infof("Reading %s: %v", viper.ConfigFileUsed(), err) if required { glog.Fatalf("Failed to load %s.toml file from current directory, or $HOME/.seaweedfs/, or /etc/seaweedfs/"+ "\n\nPlease use this command to generate the default %s.toml file\n"+ From d5add83e85da0c61fe107842e7dd82b52af2bcdb Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 19 Jan 2021 18:07:29 -0800 Subject: [PATCH 161/260] filer store: add postgres2 --- weed/command/scaffold.go | 20 +++++ weed/filer/abstract_sql/abstract_sql_store.go | 4 +- .../{mysql_store.go => mysql2_store.go} | 5 ++ weed/filer/postgres/postgres_sql_gen.go | 53 +++++++++++ weed/filer/postgres/postgres_store.go | 55 ++---------- weed/filer/postgres2/postgres2_store.go | 87 +++++++++++++++++++ weed/server/filer_server.go | 1 + 7 files changed, 175 insertions(+), 50 deletions(-) rename weed/filer/mysql2/{mysql_store.go => mysql2_store.go} (92%) create mode 100644 weed/filer/postgres/postgres_sql_gen.go create mode 100644 weed/filer/postgres2/postgres2_store.go diff --git a/weed/command/scaffold.go b/weed/command/scaffold.go index b44bc41c1..c42dce7be 100644 --- a/weed/command/scaffold.go +++ b/weed/command/scaffold.go @@ -160,6 +160,26 @@ sslmode = "disable" connection_max_idle = 100 connection_max_open = 100 +[postgres2] +enabled = false +createTable = """ + CREATE TABLE IF NOT EXISTS %s ( + dirhash BIGINT, + name VARCHAR(65535), + directory VARCHAR(65535), + meta bytea, + PRIMARY KEY (dirhash, name) + ); +""" +hostname = "localhost" +port = 5432 +username = "postgres" +password = "" +database = "" # create or use an existing database +sslmode = "disable" +connection_max_idle = 100 +connection_max_open = 100 + [cassandra] # CREATE TABLE filemeta ( # directory varchar, diff --git a/weed/filer/abstract_sql/abstract_sql_store.go b/weed/filer/abstract_sql/abstract_sql_store.go index 8345d7a7b..91b0bc98f 100644 --- a/weed/filer/abstract_sql/abstract_sql_store.go +++ b/weed/filer/abstract_sql/abstract_sql_store.go @@ -107,7 +107,7 @@ func (store *AbstractSqlStore) getTxOrDB(ctx context.Context, fullpath util.Full } if _, found := store.dbs[bucket]; !found { - if err = store.createTable(ctx, bucket); err != nil { + if err = store.CreateTable(ctx, bucket); err != nil { store.dbs[bucket] = true } } @@ -322,7 +322,7 @@ func isValidBucket(bucket string) bool { return bucket != DEFAULT_TABLE && bucket != "" } -func (store *AbstractSqlStore) createTable(ctx context.Context, bucket string) error { +func (store *AbstractSqlStore) CreateTable(ctx context.Context, bucket string) error { if !store.SupportBucketTable { return nil } diff --git a/weed/filer/mysql2/mysql_store.go b/weed/filer/mysql2/mysql2_store.go similarity index 92% rename from weed/filer/mysql2/mysql_store.go rename to weed/filer/mysql2/mysql2_store.go index fb74b6f7f..15216b651 100644 --- a/weed/filer/mysql2/mysql_store.go +++ b/weed/filer/mysql2/mysql2_store.go @@ -1,6 +1,7 @@ package mysql2 import ( + "context" "database/sql" "fmt" "time" @@ -73,5 +74,9 @@ func (store *MysqlStore2) initialize(createTable, user, password, hostname strin return fmt.Errorf("connect to %s error:%v", sqlUrl, err) } + if err = store.CreateTable(context.Background(), abstract_sql.DEFAULT_TABLE); err != nil { + return fmt.Errorf("init table %s: %v", abstract_sql.DEFAULT_TABLE, err) + } + return nil } diff --git a/weed/filer/postgres/postgres_sql_gen.go b/weed/filer/postgres/postgres_sql_gen.go new file mode 100644 index 000000000..284cf254b --- /dev/null +++ b/weed/filer/postgres/postgres_sql_gen.go @@ -0,0 +1,53 @@ +package postgres + +import ( + "fmt" + + "github.com/chrislusf/seaweedfs/weed/filer/abstract_sql" + _ "github.com/lib/pq" +) + +type SqlGenPostgres struct { + CreateTableSqlTemplate string + DropTableSqlTemplate string +} + +var ( + _ = abstract_sql.SqlGenerator(&SqlGenPostgres{}) +) + +func (gen *SqlGenPostgres) GetSqlInsert(bucket string) string { + return fmt.Sprintf("INSERT INTO %s (dirhash,name,directory,meta) VALUES($1,$2,$3,$4)", bucket) +} + +func (gen *SqlGenPostgres) GetSqlUpdate(bucket string) string { + return fmt.Sprintf("UPDATE %s SET meta=$1 WHERE dirhash=$2 AND name=$3 AND directory=$4", bucket) +} + +func (gen *SqlGenPostgres) GetSqlFind(bucket string) string { + return fmt.Sprintf("SELECT meta FROM %s WHERE dirhash=$1 AND name=$2 AND directory=$3", bucket) +} + +func (gen *SqlGenPostgres) GetSqlDelete(bucket string) string { + return fmt.Sprintf("DELETE FROM %s WHERE dirhash=$1 AND name=$2 AND directory=$3", bucket) +} + +func (gen *SqlGenPostgres) GetSqlDeleteFolderChildren(bucket string) string { + return fmt.Sprintf("DELETE FROM %s WHERE dirhash=$1 AND directory=$2", bucket) +} + +func (gen *SqlGenPostgres) GetSqlListExclusive(bucket string) string { + return fmt.Sprintf("SELECT NAME, meta FROM %s WHERE dirhash=$1 AND name>$2 AND directory=$3 AND name like $4 ORDER BY NAME ASC LIMIT $5", bucket) +} + +func (gen *SqlGenPostgres) GetSqlListInclusive(bucket string) string { + return fmt.Sprintf("SELECT NAME, meta FROM %s WHERE dirhash=$1 AND name>=$2 AND directory=$3 AND name like $4 ORDER BY NAME ASC LIMIT $5", bucket) +} + +func (gen *SqlGenPostgres) GetSqlCreateTable(bucket string) string { + return fmt.Sprintf(gen.CreateTableSqlTemplate, bucket) +} + +func (gen *SqlGenPostgres) GetSqlDropTable(bucket string) string { + return fmt.Sprintf(gen.DropTableSqlTemplate, bucket) +} diff --git a/weed/filer/postgres/postgres_store.go b/weed/filer/postgres/postgres_store.go index 8c36b8672..27c6278c7 100644 --- a/weed/filer/postgres/postgres_store.go +++ b/weed/filer/postgres/postgres_store.go @@ -14,51 +14,6 @@ const ( CONNECTION_URL_PATTERN = "host=%s port=%d sslmode=%s connect_timeout=30" ) -type SqlGenPostgres struct { - createTableSqlTemplate string - dropTableSqlTemplate string -} - -var ( - _ = abstract_sql.SqlGenerator(&SqlGenPostgres{}) -) - -func (gen *SqlGenPostgres) GetSqlInsert(bucket string) string { - return "INSERT INTO filemeta (dirhash,name,directory,meta) VALUES($1,$2,$3,$4)" -} - -func (gen *SqlGenPostgres) GetSqlUpdate(bucket string) string { - return "UPDATE filemeta SET meta=$1 WHERE dirhash=$2 AND name=$3 AND directory=$4" -} - -func (gen *SqlGenPostgres) GetSqlFind(bucket string) string { - return "SELECT meta FROM filemeta WHERE dirhash=$1 AND name=$2 AND directory=$3" -} - -func (gen *SqlGenPostgres) GetSqlDelete(bucket string) string { - return "DELETE FROM filemeta WHERE dirhash=$1 AND name=$2 AND directory=$3" -} - -func (gen *SqlGenPostgres) GetSqlDeleteFolderChildren(bucket string) string { - return "DELETE FROM filemeta WHERE dirhash=$1 AND directory=$2" -} - -func (gen *SqlGenPostgres) GetSqlListExclusive(bucket string) string { - return "SELECT NAME, meta FROM filemeta WHERE dirhash=$1 AND name>$2 AND directory=$3 AND name like $4 ORDER BY NAME ASC LIMIT $5" -} - -func (gen *SqlGenPostgres) GetSqlListInclusive(bucket string) string { - return "SELECT NAME, meta FROM filemeta WHERE dirhash=$1 AND name>=$2 AND directory=$3 AND name like $4 ORDER BY NAME ASC LIMIT $5" -} - -func (gen *SqlGenPostgres) GetSqlCreateTable(bucket string) string { - return fmt.Sprintf(gen.createTableSqlTemplate, bucket) -} - -func (gen *SqlGenPostgres) GetSqlDropTable(bucket string) string { - return fmt.Sprintf(gen.dropTableSqlTemplate, bucket) -} - func init() { filer.Stores = append(filer.Stores, &PostgresStore{}) } @@ -78,18 +33,19 @@ func (store *PostgresStore) Initialize(configuration util.Configuration, prefix configuration.GetString(prefix+"hostname"), configuration.GetInt(prefix+"port"), configuration.GetString(prefix+"database"), + configuration.GetString(prefix+"schema"), configuration.GetString(prefix+"sslmode"), configuration.GetInt(prefix+"connection_max_idle"), configuration.GetInt(prefix+"connection_max_open"), ) } -func (store *PostgresStore) initialize(user, password, hostname string, port int, database, sslmode string, maxIdle, maxOpen int) (err error) { +func (store *PostgresStore) initialize(user, password, hostname string, port int, database, schema, sslmode string, maxIdle, maxOpen int) (err error) { store.SupportBucketTable = false store.SqlGenerator = &SqlGenPostgres{ - createTableSqlTemplate: "", - dropTableSqlTemplate: "drop table %s", + CreateTableSqlTemplate: "", + DropTableSqlTemplate: "drop table %s", } sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, hostname, port, sslmode) @@ -102,6 +58,9 @@ func (store *PostgresStore) initialize(user, password, hostname string, port int if database != "" { sqlUrl += " dbname=" + database } + if schema != "" { + sqlUrl += " search_path=" + schema + } var dbErr error store.DB, dbErr = sql.Open("postgres", sqlUrl) if dbErr != nil { diff --git a/weed/filer/postgres2/postgres2_store.go b/weed/filer/postgres2/postgres2_store.go new file mode 100644 index 000000000..82552376f --- /dev/null +++ b/weed/filer/postgres2/postgres2_store.go @@ -0,0 +1,87 @@ +package postgres2 + +import ( + "context" + "database/sql" + "fmt" + + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/filer/abstract_sql" + "github.com/chrislusf/seaweedfs/weed/filer/postgres" + "github.com/chrislusf/seaweedfs/weed/util" + _ "github.com/lib/pq" +) + +const ( + CONNECTION_URL_PATTERN = "host=%s port=%d sslmode=%s connect_timeout=30" +) + +func init() { + filer.Stores = append(filer.Stores, &PostgresStore2{}) +} + +type PostgresStore2 struct { + abstract_sql.AbstractSqlStore +} + +func (store *PostgresStore2) GetName() string { + return "postgres2" +} + +func (store *PostgresStore2) Initialize(configuration util.Configuration, prefix string) (err error) { + return store.initialize( + configuration.GetString(prefix+"createTable"), + configuration.GetString(prefix+"username"), + configuration.GetString(prefix+"password"), + configuration.GetString(prefix+"hostname"), + configuration.GetInt(prefix+"port"), + configuration.GetString(prefix+"database"), + configuration.GetString(prefix+"schema"), + configuration.GetString(prefix+"sslmode"), + configuration.GetInt(prefix+"connection_max_idle"), + configuration.GetInt(prefix+"connection_max_open"), + ) +} + +func (store *PostgresStore2) initialize(createTable, user, password, hostname string, port int, database, schema, sslmode string, maxIdle, maxOpen int) (err error) { + + store.SupportBucketTable = true + store.SqlGenerator = &postgres.SqlGenPostgres{ + CreateTableSqlTemplate: createTable, + DropTableSqlTemplate: "drop table %s", + } + + sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, hostname, port, sslmode) + if user != "" { + sqlUrl += " user=" + user + } + if password != "" { + sqlUrl += " password=" + password + } + if database != "" { + sqlUrl += " dbname=" + database + } + if schema != "" { + sqlUrl += " search_path=" + schema + } + var dbErr error + store.DB, dbErr = sql.Open("postgres", sqlUrl) + if dbErr != nil { + store.DB.Close() + store.DB = nil + return fmt.Errorf("can not connect to %s error:%v", sqlUrl, err) + } + + store.DB.SetMaxIdleConns(maxIdle) + store.DB.SetMaxOpenConns(maxOpen) + + if err = store.DB.Ping(); err != nil { + return fmt.Errorf("connect to %s error:%v", sqlUrl, err) + } + + if err = store.CreateTable(context.Background(), abstract_sql.DEFAULT_TABLE); err != nil { + return fmt.Errorf("init table %s: %v", abstract_sql.DEFAULT_TABLE, err) + } + + return nil +} diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index 8ea2b8b82..22474a5e2 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -31,6 +31,7 @@ import ( _ "github.com/chrislusf/seaweedfs/weed/filer/mysql" _ "github.com/chrislusf/seaweedfs/weed/filer/mysql2" _ "github.com/chrislusf/seaweedfs/weed/filer/postgres" + _ "github.com/chrislusf/seaweedfs/weed/filer/postgres2" _ "github.com/chrislusf/seaweedfs/weed/filer/redis" _ "github.com/chrislusf/seaweedfs/weed/filer/redis2" "github.com/chrislusf/seaweedfs/weed/glog" From b434f7e4e00263aa57d6ee349986ba59c539a997 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 19 Jan 2021 18:10:36 -0800 Subject: [PATCH 162/260] filer store: postgres adjust default config, add optioanl schema --- weed/command/scaffold.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/weed/command/scaffold.go b/weed/command/scaffold.go index c42dce7be..c640d9544 100644 --- a/weed/command/scaffold.go +++ b/weed/command/scaffold.go @@ -155,7 +155,8 @@ hostname = "localhost" port = 5432 username = "postgres" password = "" -database = "" # create or use an existing database +database = "postgres" # create or use an existing database +schema = "public" # create or use an existing schema sslmode = "disable" connection_max_idle = 100 connection_max_open = 100 @@ -175,7 +176,8 @@ hostname = "localhost" port = 5432 username = "postgres" password = "" -database = "" # create or use an existing database +database = "postgres" # create or use an existing database +schema = "public" # create or use an existing schema sslmode = "disable" connection_max_idle = 100 connection_max_open = 100 From 1a3f859c58fd60e803a917a6b65814b7611b31fe Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 19 Jan 2021 18:12:07 -0800 Subject: [PATCH 163/260] filer store: postgres configuration adjust text --- weed/command/scaffold.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/weed/command/scaffold.go b/weed/command/scaffold.go index c640d9544..8b74274e5 100644 --- a/weed/command/scaffold.go +++ b/weed/command/scaffold.go @@ -156,7 +156,7 @@ port = 5432 username = "postgres" password = "" database = "postgres" # create or use an existing database -schema = "public" # create or use an existing schema +schema = "" sslmode = "disable" connection_max_idle = 100 connection_max_open = 100 @@ -177,7 +177,7 @@ port = 5432 username = "postgres" password = "" database = "postgres" # create or use an existing database -schema = "public" # create or use an existing schema +schema = "" sslmode = "disable" connection_max_idle = 100 connection_max_open = 100 From c78f3dddcbc28d22bacad8eb52c1fa6ef8e72215 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 20 Jan 2021 03:03:49 -0800 Subject: [PATCH 164/260] update to wangyoucao577/go-release-action@v1.14 --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 55457ba6a..13714825d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -33,7 +33,7 @@ jobs: run: echo BUILD_TIME=$(date -u +%Y-%m-%d-%H-%M) >> ${GITHUB_ENV} - name: Go Release Binaries - uses: wangyoucao577/go-release-action@feature/asset-name + uses: wangyoucao577/go-release-action@v1.14 with: github_token: ${{ secrets.GITHUB_TOKEN }} goos: ${{ matrix.goos }} @@ -48,7 +48,7 @@ jobs: asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}" - name: Go Release Binaries - uses: wangyoucao577/go-release-action@feature/asset-name + uses: wangyoucao577/go-release-action@v1.14 with: github_token: ${{ secrets.GITHUB_TOKEN }} goos: ${{ matrix.goos }} From 1b8e3da295d1a5e8a09c0a65af3003583bca33ab Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 20 Jan 2021 12:59:51 -0800 Subject: [PATCH 165/260] refactoring --- .../filer_server_handlers_write_autochunk.go | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/weed/server/filer_server_handlers_write_autochunk.go b/weed/server/filer_server_handlers_write_autochunk.go index 43884bfde..2bd1104d3 100644 --- a/weed/server/filer_server_handlers_write_autochunk.go +++ b/weed/server/filer_server_handlers_write_autochunk.go @@ -96,12 +96,6 @@ func (fs *FilerServer) doPostAutoChunk(ctx context.Context, w http.ResponseWrite return nil, nil, err } - fileChunks, replyerr = filer.MaybeManifestize(fs.saveAsChunk(so), fileChunks) - if replyerr != nil { - glog.V(0).Infof("manifestize %s: %v", r.RequestURI, replyerr) - return - } - md5bytes = md5Hash.Sum(nil) filerResult, replyerr = fs.saveMetaData(ctx, r, fileName, contentType, so, md5bytes, fileChunks, chunkOffset, smallContent) @@ -121,12 +115,6 @@ func (fs *FilerServer) doPutAutoChunk(ctx context.Context, w http.ResponseWriter return nil, nil, err } - fileChunks, replyerr = filer.MaybeManifestize(fs.saveAsChunk(so), fileChunks) - if replyerr != nil { - glog.V(0).Infof("manifestize %s: %v", r.RequestURI, replyerr) - return - } - md5bytes = md5Hash.Sum(nil) filerResult, replyerr = fs.saveMetaData(ctx, r, fileName, contentType, so, md5bytes, fileChunks, chunkOffset, smallContent) @@ -135,6 +123,13 @@ func (fs *FilerServer) doPutAutoChunk(ctx context.Context, w http.ResponseWriter func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileName string, contentType string, so *operation.StorageOption, md5bytes []byte, fileChunks []*filer_pb.FileChunk, chunkOffset int64, content []byte) (filerResult *FilerPostResult, replyerr error) { + // maybe compact chunks + fileChunks, replyerr = filer.MaybeManifestize(fs.saveAsChunk(so), fileChunks) + if replyerr != nil { + glog.V(0).Infof("manifestize %s: %v", r.RequestURI, replyerr) + return + } + // detect file mode modeStr := r.URL.Query().Get("mode") if modeStr == "" { From f98817cfe655d0285ed35f6d30068aa15938bd3d Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 20 Jan 2021 13:40:32 -0800 Subject: [PATCH 166/260] filer: support appending to a file --- .../filer_server_handlers_write_autochunk.go | 87 +++++++++++++------ 1 file changed, 60 insertions(+), 27 deletions(-) diff --git a/weed/server/filer_server_handlers_write_autochunk.go b/weed/server/filer_server_handlers_write_autochunk.go index 2bd1104d3..237f121fe 100644 --- a/weed/server/filer_server_handlers_write_autochunk.go +++ b/weed/server/filer_server_handlers_write_autochunk.go @@ -121,14 +121,11 @@ func (fs *FilerServer) doPutAutoChunk(ctx context.Context, w http.ResponseWriter return } -func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileName string, contentType string, so *operation.StorageOption, md5bytes []byte, fileChunks []*filer_pb.FileChunk, chunkOffset int64, content []byte) (filerResult *FilerPostResult, replyerr error) { +func isAppend(r *http.Request) bool { + return r.URL.Query().Get("op") == "append" +} - // maybe compact chunks - fileChunks, replyerr = filer.MaybeManifestize(fs.saveAsChunk(so), fileChunks) - if replyerr != nil { - glog.V(0).Infof("manifestize %s: %v", r.RequestURI, replyerr) - return - } +func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileName string, contentType string, so *operation.StorageOption, md5bytes []byte, fileChunks []*filer_pb.FileChunk, chunkOffset int64, content []byte) (filerResult *FilerPostResult, replyerr error) { // detect file mode modeStr := r.URL.Query().Get("mode") @@ -149,25 +146,61 @@ func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileNa } } - glog.V(4).Infoln("saving", path) - entry := &filer.Entry{ - FullPath: util.FullPath(path), - Attr: filer.Attr{ - Mtime: time.Now(), - Crtime: time.Now(), - Mode: os.FileMode(mode), - Uid: OS_UID, - Gid: OS_GID, - Replication: so.Replication, - Collection: so.Collection, - TtlSec: so.TtlSeconds, - Mime: contentType, - Md5: md5bytes, - FileSize: uint64(chunkOffset), - }, - Chunks: fileChunks, - Content: content, + var entry *filer.Entry + var mergedChunks []*filer_pb.FileChunk + // when it is an append + if isAppend(r) { + existingEntry, findErr := fs.filer.FindEntry(ctx, util.FullPath(path)) + if findErr != nil && findErr != filer_pb.ErrNotFound { + glog.V(0).Infof("failing to find %s: %v", path, findErr) + } + entry = existingEntry + } + if entry != nil { + entry.Mtime = time.Now() + entry.Md5 = nil + // adjust chunk offsets + for _, chunk := range fileChunks { + chunk.Offset += int64(entry.FileSize) + } + mergedChunks = append(entry.Chunks, fileChunks...) + entry.FileSize += uint64(chunkOffset) + + // TODO + if len(entry.Content) > 0 { + replyerr = fmt.Errorf("append to small file is not supported yet") + return + } + + } else { + glog.V(4).Infoln("saving", path) + mergedChunks = fileChunks + entry = &filer.Entry{ + FullPath: util.FullPath(path), + Attr: filer.Attr{ + Mtime: time.Now(), + Crtime: time.Now(), + Mode: os.FileMode(mode), + Uid: OS_UID, + Gid: OS_GID, + Replication: so.Replication, + Collection: so.Collection, + TtlSec: so.TtlSeconds, + Mime: contentType, + Md5: md5bytes, + FileSize: uint64(chunkOffset), + }, + Content: content, + } + } + + // maybe compact entry chunks + mergedChunks, replyerr = filer.MaybeManifestize(fs.saveAsChunk(so), mergedChunks) + if replyerr != nil { + glog.V(0).Infof("manifestize %s: %v", r.RequestURI, replyerr) + return } + entry.Chunks = mergedChunks filerResult = &FilerPostResult{ Name: fileName, @@ -187,7 +220,7 @@ func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileNa } if dbErr := fs.filer.CreateEntry(ctx, entry, false, false, nil); dbErr != nil { - fs.filer.DeleteChunks(entry.Chunks) + fs.filer.DeleteChunks(fileChunks) replyerr = dbErr filerResult.Error = dbErr.Error() glog.V(0).Infof("failing to write %s to filer server : %v", path, dbErr) @@ -211,7 +244,7 @@ func (fs *FilerServer) uploadReaderToChunks(w http.ResponseWriter, r *http.Reque if err != nil { return nil, nil, 0, err, nil } - if chunkOffset == 0 { + if chunkOffset == 0 && !isAppend(r) { if len(data) < fs.option.SaveToFilerLimit || strings.HasPrefix(r.URL.Path, filer.DirectoryEtcRoot) && len(data) < 4*1024 { smallContent = data chunkOffset += int64(len(data)) From 84f05787f8eecfcb61e49882346ad5855b6bb784 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 20 Jan 2021 23:30:14 -0800 Subject: [PATCH 167/260] mount: limit background requests and congestion threshold --- go.mod | 2 +- go.sum | 3 +++ weed/command/mount_std.go | 2 ++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index c0bcdc8cd..44950d25c 100644 --- a/go.mod +++ b/go.mod @@ -56,7 +56,7 @@ require ( github.com/prometheus/client_golang v1.3.0 github.com/rakyll/statik v0.1.7 github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 // indirect - github.com/seaweedfs/fuse v1.0.8 + github.com/seaweedfs/fuse v1.0.9 github.com/seaweedfs/goexif v1.0.2 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e github.com/spaolacci/murmur3 v1.1.0 // indirect diff --git a/go.sum b/go.sum index d91a570f8..f0b86cb62 100644 --- a/go.sum +++ b/go.sum @@ -575,6 +575,8 @@ github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/seaweedfs/fuse v1.0.8 h1:HBPJTC77OlxwSd2JiTwvLPn8bWTElqQp3xs9vf3C15s= github.com/seaweedfs/fuse v1.0.8/go.mod h1:W7ubwr1l7KQsMeUpxFFOFOSxUL/ucTRMAlVYs4xdfQ8= +github.com/seaweedfs/fuse v1.0.9 h1:3JZoGsW7cmmrd/U5KYcIGR2+EqyBvCYCoCpEdZAz/Dc= +github.com/seaweedfs/fuse v1.0.9/go.mod h1:+PP6WlkrRUG6KPE+Th2EX5To/PjHaFsvqg/UgQ39aj8= github.com/seaweedfs/goexif v1.0.2 h1:p+rTXYdQ2mgxd+1JaTrQ9N8DvYuw9UH9xgYmJ+Bb29E= github.com/seaweedfs/goexif v1.0.2/go.mod h1:MrKs5LK0HXdffrdCZrW3OIMegL2xXpC6ThLyXMyjdrk= github.com/secsy/goftp v0.0.0-20190720192957-f31499d7c79a h1:C6IhVTxNkhlb0tlCB6JfHOUv1f0xHPK7V8X4HlJZEJw= @@ -824,6 +826,7 @@ golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201022201747-fb209a7c41cd h1:WgqgiQvkiZWz7XLhphjt2GI2GcGCTIZs9jqXMWmH+oc= diff --git a/weed/command/mount_std.go b/weed/command/mount_std.go index 7e75b082d..9e955e344 100644 --- a/weed/command/mount_std.go +++ b/weed/command/mount_std.go @@ -150,6 +150,8 @@ func RunMount(option *MountOptions, umask os.FileMode) bool { fuse.MaxReadahead(1024 * 128), fuse.AsyncRead(), fuse.WritebackCache(), + fuse.MaxBackground(128), + fuse.CongestionThreshold(128), } options = append(options, osSpecificMountOptions()...) From 024c186a487a36031dd47bdd8bc3aa026444c27e Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 21 Jan 2021 22:39:25 -0800 Subject: [PATCH 168/260] filer.meta.tail: output json format for easier parsing --- weed/command/filer_meta_tail.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/weed/command/filer_meta_tail.go b/weed/command/filer_meta_tail.go index fa0262160..f055b19a8 100644 --- a/weed/command/filer_meta_tail.go +++ b/weed/command/filer_meta_tail.go @@ -3,9 +3,11 @@ package command import ( "context" "fmt" + "github.com/golang/protobuf/jsonpb" jsoniter "github.com/json-iterator/go" "github.com/olivere/elastic/v7" "io" + "os" "path/filepath" "strings" "time" @@ -25,6 +27,10 @@ var cmdFilerMetaTail = &Command{ Short: "see recent changes on a filer", Long: `See recent changes on a filer. + weed filer.meta.tail -timeAgo=30h | grep truncate + weed filer.meta.tail -timeAgo=30h | jq . + weed filer.meta.tail -timeAgo=30h | jq .eventNotification.newEntry.name + `, } @@ -80,8 +86,12 @@ func runFilerMetaTail(cmd *Command, args []string) bool { return false } + jsonpbMarshaler := jsonpb.Marshaler{ + EmitDefaults: false, + } eachEntryFunc := func(resp *filer_pb.SubscribeMetadataResponse) error { - fmt.Printf("dir:%s %+v\n", resp.Directory, resp.EventNotification) + jsonpbMarshaler.Marshal(os.Stdout, resp) + fmt.Fprintln(os.Stdout) return nil } if *esServers != "" { From 711c3f393980f96971904d52d078e8f272451374 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 22 Jan 2021 00:26:30 -0800 Subject: [PATCH 169/260] filer: fix http status 206 setting fix https://github.com/chrislusf/seaweedfs/issues/1753 --- weed/server/filer_server_handlers_read.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.go index 9fdc03dea..f77462adb 100644 --- a/weed/server/filer_server_handlers_read.go +++ b/weed/server/filer_server_handlers_read.go @@ -159,10 +159,10 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, } processRangeRequest(r, w, totalSize, mimeType, func(writer io.Writer, offset int64, size int64, httpStatusCode int) error { + if httpStatusCode != 0 { + w.WriteHeader(httpStatusCode) + } if offset+size <= int64(len(entry.Content)) { - if httpStatusCode != 0 { - w.WriteHeader(httpStatusCode) - } _, err := writer.Write(entry.Content[offset : offset+size]) return err } From 20ef3bb8d4e228cdcd2c81c46c98ffb95a6398aa Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 22 Jan 2021 22:39:46 -0800 Subject: [PATCH 170/260] mount: use direct_io to avoid OS page cache fix https://github.com/chrislusf/seaweedfs/issues/1752 --- weed/filesys/file.go | 3 +++ weed/filesys/file_darwin.go | 8 ++++++++ weed/filesys/file_other.go | 7 +++++++ weed/filesys/filehandle.go | 5 +++-- 4 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 weed/filesys/file_darwin.go create mode 100644 weed/filesys/file_other.go diff --git a/weed/filesys/file.go b/weed/filesys/file.go index 3bffa156e..a2b6660d8 100644 --- a/weed/filesys/file.go +++ b/weed/filesys/file.go @@ -91,6 +91,9 @@ func (file *File) Getxattr(ctx context.Context, req *fuse.GetxattrRequest, resp func (file *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenResponse) (fs.Handle, error) { glog.V(4).Infof("file %v open %+v", file.fullpath(), req) + if USE_DIRECT_IO { + resp.Flags |= fuse.OpenDirectIO + } handle := file.wfs.AcquireHandle(file, req.Uid, req.Gid) diff --git a/weed/filesys/file_darwin.go b/weed/filesys/file_darwin.go new file mode 100644 index 000000000..b8806cff7 --- /dev/null +++ b/weed/filesys/file_darwin.go @@ -0,0 +1,8 @@ +//+build darwin + +package filesys + +const ( + USE_DIRECT_IO = false +) + diff --git a/weed/filesys/file_other.go b/weed/filesys/file_other.go new file mode 100644 index 000000000..8a8693511 --- /dev/null +++ b/weed/filesys/file_other.go @@ -0,0 +1,7 @@ +//+build !darwin + +package filesys + +const ( + USE_DIRECT_IO = true +) diff --git a/weed/filesys/filehandle.go b/weed/filesys/filehandle.go index d5f6cc38e..6225ab968 100644 --- a/weed/filesys/filehandle.go +++ b/weed/filesys/filehandle.go @@ -90,8 +90,9 @@ func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fus glog.Warningf("%s FileHandle Read %d: [%d,%d) size %d totalRead %d", fh.f.fullpath(), fh.handle, req.Offset, req.Offset+int64(req.Size), req.Size, totalRead) totalRead = min(int64(len(buff)), totalRead) } - // resp.Data = buff[:totalRead] - resp.Data = buff + if err == nil { + resp.Data = buff[:totalRead] + } return err } From 3ecd989500c9ee23460b3707e5713f944346d070 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 22 Jan 2021 22:47:23 -0800 Subject: [PATCH 171/260] add local mount for testing --- docker/Makefile | 3 +++ docker/local-mount-compose.yml | 47 ++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 docker/local-mount-compose.yml diff --git a/docker/Makefile b/docker/Makefile index c2e9a12e7..3712db4be 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -12,6 +12,9 @@ build: dev: build docker-compose -f local-dev-compose.yml -p seaweedfs up +dev_mount: build + docker-compose -f local-mount-compose.yml -p seaweedfs up + k8s: build docker-compose -f local-k8s-compose.yml -p seaweedfs up diff --git a/docker/local-mount-compose.yml b/docker/local-mount-compose.yml new file mode 100644 index 000000000..97fc652d4 --- /dev/null +++ b/docker/local-mount-compose.yml @@ -0,0 +1,47 @@ +version: '2' + +services: + master: + image: chrislusf/seaweedfs:local + ports: + - 9333:9333 + - 19333:19333 + command: "master -ip=master" + volume: + image: chrislusf/seaweedfs:local + ports: + - 8080:8080 + - 18080:18080 + - 9325:9325 + command: 'volume -mserver="master:9333" -port=8080 -metricsPort=9325 -preStopSeconds=1' + depends_on: + - master + filer: + image: chrislusf/seaweedfs:local + ports: + - 8888:8888 + - 18888:18888 + - 9326:9326 + command: 'filer -master="master:9333" -metricsPort=9326' + tty: true + stdin_open: true + depends_on: + - master + - volume + mount_1: + image: chrislusf/seaweedfs:local + privileged: true + entrypoint: '/bin/sh -c "mkdir -p t1 && mkdir -p cache/t1 && weed -v=4 mount -filer=filer:8888 -cacheDir=./cache/t1 -dir=./t1 -filer.path=/"' + depends_on: + - master + - volume + - filer + mount_2: + image: chrislusf/seaweedfs:local + privileged: true + entrypoint: '/bin/sh -c "mkdir -p t2 && mkdir -p cache/t2 && weed -v=4 mount -filer=filer:8888 -cacheDir=./cache/t2 -dir=./t2 -filer.path=/"' + depends_on: + - master + - volume + - filer + - mount_1 From 937cfacc01c0acd27f352f1f57b8abd6f1e8bc56 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 23 Jan 2021 03:43:48 -0800 Subject: [PATCH 172/260] filer: add "proxyToFileId" to reverse proxy to a volume server --- weed/server/filer_server_handlers.go | 13 ++++- weed/server/filer_server_handlers_proxy.go | 66 ++++++++++++++++++++++ 2 files changed, 76 insertions(+), 3 deletions(-) create mode 100644 weed/server/filer_server_handlers_proxy.go diff --git a/weed/server/filer_server_handlers.go b/weed/server/filer_server_handlers.go index 451e2a2de..5b93c6d08 100644 --- a/weed/server/filer_server_handlers.go +++ b/weed/server/filer_server_handlers.go @@ -17,9 +17,16 @@ func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) { start := time.Now() switch r.Method { case "GET": - stats.FilerRequestCounter.WithLabelValues("get").Inc() - fs.GetOrHeadHandler(w, r, true) - stats.FilerRequestHistogram.WithLabelValues("get").Observe(time.Since(start).Seconds()) + fileId := r.FormValue("proxyToFileId") + if fileId != "" { + stats.FilerRequestCounter.WithLabelValues("proxy").Inc() + fs.proxyToVolumeServer(w,r,fileId) + stats.FilerRequestHistogram.WithLabelValues("proxy").Observe(time.Since(start).Seconds()) + } else { + stats.FilerRequestCounter.WithLabelValues("get").Inc() + fs.GetOrHeadHandler(w, r, true) + stats.FilerRequestHistogram.WithLabelValues("get").Observe(time.Since(start).Seconds()) + } case "HEAD": stats.FilerRequestCounter.WithLabelValues("head").Inc() fs.GetOrHeadHandler(w, r, false) diff --git a/weed/server/filer_server_handlers_proxy.go b/weed/server/filer_server_handlers_proxy.go new file mode 100644 index 000000000..5ba42e71d --- /dev/null +++ b/weed/server/filer_server_handlers_proxy.go @@ -0,0 +1,66 @@ +package weed_server + +import ( + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/util" + "io" + "math/rand" + "net/http" +) + +var ( + client *http.Client +) + +func init() { + client = &http.Client{Transport: &http.Transport{ + MaxIdleConnsPerHost: 1024, + }} +} + +func (fs *FilerServer) proxyToVolumeServer(w http.ResponseWriter, r *http.Request, fileId string) { + + urlStrings, err := fs.filer.MasterClient.GetLookupFileIdFunction()(fileId) + if err != nil { + glog.Errorf("locate %s: %v", fileId, err) + w.WriteHeader(http.StatusInternalServerError) + return + } + + if len(urlStrings) == 0 { + w.WriteHeader(http.StatusNotFound) + return + } + + proxyReq, err := http.NewRequest("GET", urlStrings[rand.Intn(len(urlStrings))], r.Body) + if err != nil { + glog.Errorf("NewRequest %s: %v", urlStrings[0], err) + w.WriteHeader(http.StatusInternalServerError) + return + } + + proxyReq.Header.Set("Host", r.Host) + proxyReq.Header.Set("X-Forwarded-For", r.RemoteAddr) + + for header, values := range r.Header { + for _, value := range values { + proxyReq.Header.Add(header, value) + } + } + + proxyResponse, postErr := client.Do(proxyReq) + + if postErr != nil { + glog.Errorf("post to filer: %v", postErr) + w.WriteHeader(http.StatusInternalServerError) + return + } + defer util.CloseResponse(proxyResponse) + + for k, v := range proxyResponse.Header { + w.Header()[k] = v + } + w.WriteHeader(proxyResponse.StatusCode) + io.Copy(w, proxyResponse.Body) + +} From 759482e2aca798523b8d361c9e7d1303c0e2b579 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 23 Jan 2021 03:52:38 -0800 Subject: [PATCH 173/260] filer: proxy all http methods to volume servers --- weed/server/filer_server_handlers_proxy.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/server/filer_server_handlers_proxy.go b/weed/server/filer_server_handlers_proxy.go index 5ba42e71d..8d73bc960 100644 --- a/weed/server/filer_server_handlers_proxy.go +++ b/weed/server/filer_server_handlers_proxy.go @@ -32,7 +32,7 @@ func (fs *FilerServer) proxyToVolumeServer(w http.ResponseWriter, r *http.Reques return } - proxyReq, err := http.NewRequest("GET", urlStrings[rand.Intn(len(urlStrings))], r.Body) + proxyReq, err := http.NewRequest(r.Method, urlStrings[rand.Intn(len(urlStrings))], r.Body) if err != nil { glog.Errorf("NewRequest %s: %v", urlStrings[0], err) w.WriteHeader(http.StatusInternalServerError) From f8dbb03bddc64d04b70a2e052e8204d09fc9d397 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 23 Jan 2021 04:19:49 -0800 Subject: [PATCH 174/260] filer: append operation returns final file size --- weed/server/filer_server_handlers_write_autochunk.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/server/filer_server_handlers_write_autochunk.go b/weed/server/filer_server_handlers_write_autochunk.go index 237f121fe..ec1ecbe0d 100644 --- a/weed/server/filer_server_handlers_write_autochunk.go +++ b/weed/server/filer_server_handlers_write_autochunk.go @@ -204,7 +204,7 @@ func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileNa filerResult = &FilerPostResult{ Name: fileName, - Size: chunkOffset, + Size: int64(entry.FileSize), } if entry.Extended == nil { From 6897f1bfbc0731b5e1581313a3e248ad7991cef8 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 24 Jan 2021 00:00:42 -0800 Subject: [PATCH 175/260] makefile: add env for 2 clusters --- docker/Makefile | 3 +++ docker/local-clusters-compose.yml | 23 +++++++++++++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 docker/local-clusters-compose.yml diff --git a/docker/Makefile b/docker/Makefile index 3712db4be..fe278f9b4 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -24,5 +24,8 @@ dev_registry: build cluster: build docker-compose -f local-cluster-compose.yml -p seaweedfs up +2clusters: build + docker-compose -f local-clusters-compose.yml -p seaweedfs up + clean: rm ./weed diff --git a/docker/local-clusters-compose.yml b/docker/local-clusters-compose.yml new file mode 100644 index 000000000..e813ca35f --- /dev/null +++ b/docker/local-clusters-compose.yml @@ -0,0 +1,23 @@ +version: '2' + +services: + server1: + image: chrislusf/seaweedfs:local + ports: + - 9333:9333 + - 19333:19333 + - 8084:8080 + - 18084:18080 + - 8888:8888 + - 18888:18888 + command: "server -ip=server1 -filer -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1" + server2: + image: chrislusf/seaweedfs:local + ports: + - 9334:9333 + - 19334:19333 + - 8085:8080 + - 18085:18080 + - 8889:8888 + - 18889:18888 + command: "server -ip=server2 -filer -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1" From 80b869268898798c662618b11933af82ee374f1c Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 24 Jan 2021 00:01:44 -0800 Subject: [PATCH 176/260] filer.sync: replicate outside of either cluster, only need to see filers --- weed/command/filer_sync.go | 54 ++++++++++--------- weed/pb/grpc_client_server.go | 16 ++++++ .../replication/sink/filersink/fetch_write.go | 4 ++ weed/replication/sink/filersink/filer_sink.go | 32 +++++++---- weed/replication/source/filer_source.go | 19 +++++-- weed/server/filer_server_handlers.go | 30 +++++++---- 6 files changed, 106 insertions(+), 49 deletions(-) diff --git a/weed/command/filer_sync.go b/weed/command/filer_sync.go index af0a624b1..587174059 100644 --- a/weed/command/filer_sync.go +++ b/weed/command/filer_sync.go @@ -20,19 +20,21 @@ import ( ) type SyncOptions struct { - isActivePassive *bool - filerA *string - filerB *string - aPath *string - bPath *string - aReplication *string - bReplication *string - aCollection *string - bCollection *string - aTtlSec *int - bTtlSec *int - aDebug *bool - bDebug *bool + isActivePassive *bool + filerA *string + filerB *string + aPath *string + bPath *string + aReplication *string + bReplication *string + aCollection *string + bCollection *string + aTtlSec *int + bTtlSec *int + aDebug *bool + bDebug *bool + aProxyByFiler *bool + bProxyByFiler *bool } var ( @@ -43,7 +45,7 @@ var ( func init() { cmdFilerSynchronize.Run = runFilerSynchronize // break init cycle - syncOptions.isActivePassive = cmdFilerSynchronize.Flag.Bool("isActivePassive", false, "one directional follow if true") + syncOptions.isActivePassive = cmdFilerSynchronize.Flag.Bool("isActivePassive", false, "one directional follow from A to B if true") syncOptions.filerA = cmdFilerSynchronize.Flag.String("a", "", "filer A in one SeaweedFS cluster") syncOptions.filerB = cmdFilerSynchronize.Flag.String("b", "", "filer B in the other SeaweedFS cluster") syncOptions.aPath = cmdFilerSynchronize.Flag.String("a.path", "/", "directory to sync on filer A") @@ -54,6 +56,8 @@ func init() { syncOptions.bCollection = cmdFilerSynchronize.Flag.String("b.collection", "", "collection on filer B") syncOptions.aTtlSec = cmdFilerSynchronize.Flag.Int("a.ttlSec", 0, "ttl in seconds on filer A") syncOptions.bTtlSec = cmdFilerSynchronize.Flag.Int("b.ttlSec", 0, "ttl in seconds on filer B") + syncOptions.aProxyByFiler = cmdFilerSynchronize.Flag.Bool("a.filerProxy", true, "read and write file chunks by filer A instead of volume servers") + syncOptions.bProxyByFiler = cmdFilerSynchronize.Flag.Bool("b.filerProxy", true, "read and write file chunks by filer B instead of volume servers") syncOptions.aDebug = cmdFilerSynchronize.Flag.Bool("a.debug", false, "debug mode to print out filer A received files") syncOptions.bDebug = cmdFilerSynchronize.Flag.Bool("b.debug", false, "debug mode to print out filer B received files") syncCpuProfile = cmdFilerSynchronize.Flag.String("cpuprofile", "", "cpu profile output file") @@ -62,8 +66,8 @@ func init() { var cmdFilerSynchronize = &Command{ UsageLine: "filer.sync -a=: -b=:", - Short: "continuously synchronize between two active-active or active-passive SeaweedFS clusters", - Long: `continuously synchronize file changes between two active-active or active-passive filers + Short: "resumeable continuous synchronization between two active-active or active-passive SeaweedFS clusters", + Long: `resumeable continuous synchronization for file changes between two active-active or active-passive filers filer.sync listens on filer notifications. If any file is updated, it will fetch the updated content, and write to the other destination. Different from filer.replicate: @@ -86,8 +90,9 @@ func runFilerSynchronize(cmd *Command, args []string) bool { go func() { for { - err := doSubscribeFilerMetaChanges(grpcDialOption, *syncOptions.filerA, *syncOptions.aPath, *syncOptions.filerB, - *syncOptions.bPath, *syncOptions.bReplication, *syncOptions.bCollection, *syncOptions.bTtlSec, *syncOptions.bDebug) + err := doSubscribeFilerMetaChanges(grpcDialOption, *syncOptions.filerA, *syncOptions.aPath, *syncOptions.aProxyByFiler, + *syncOptions.filerB, *syncOptions.bPath, *syncOptions.bReplication, *syncOptions.bCollection, *syncOptions.bTtlSec, *syncOptions.bProxyByFiler, + *syncOptions.bDebug) if err != nil { glog.Errorf("sync from %s to %s: %v", *syncOptions.filerA, *syncOptions.filerB, err) time.Sleep(1747 * time.Millisecond) @@ -98,8 +103,9 @@ func runFilerSynchronize(cmd *Command, args []string) bool { if !*syncOptions.isActivePassive { go func() { for { - err := doSubscribeFilerMetaChanges(grpcDialOption, *syncOptions.filerB, *syncOptions.bPath, *syncOptions.filerA, - *syncOptions.aPath, *syncOptions.aReplication, *syncOptions.aCollection, *syncOptions.aTtlSec, *syncOptions.aDebug) + err := doSubscribeFilerMetaChanges(grpcDialOption, *syncOptions.filerB, *syncOptions.bPath, *syncOptions.bProxyByFiler, + *syncOptions.filerA, *syncOptions.aPath, *syncOptions.aReplication, *syncOptions.aCollection, *syncOptions.aTtlSec, *syncOptions.aProxyByFiler, + *syncOptions.aDebug) if err != nil { glog.Errorf("sync from %s to %s: %v", *syncOptions.filerB, *syncOptions.filerA, err) time.Sleep(2147 * time.Millisecond) @@ -113,8 +119,8 @@ func runFilerSynchronize(cmd *Command, args []string) bool { return true } -func doSubscribeFilerMetaChanges(grpcDialOption grpc.DialOption, sourceFiler, sourcePath, targetFiler, targetPath string, - replicationStr, collection string, ttlSec int, debug bool) error { +func doSubscribeFilerMetaChanges(grpcDialOption grpc.DialOption, sourceFiler, sourcePath string, sourceReadChunkFromFiler bool, targetFiler, targetPath string, + replicationStr, collection string, ttlSec int, sinkWriteChunkByFiler, debug bool) error { // read source filer signature sourceFilerSignature, sourceErr := replication.ReadFilerSignature(grpcDialOption, sourceFiler) @@ -138,9 +144,9 @@ func doSubscribeFilerMetaChanges(grpcDialOption grpc.DialOption, sourceFiler, so // create filer sink filerSource := &source.FilerSource{} - filerSource.DoInitialize(pb.ServerToGrpcAddress(sourceFiler), sourcePath) + filerSource.DoInitialize(sourceFiler, pb.ServerToGrpcAddress(sourceFiler), sourcePath, sourceReadChunkFromFiler) filerSink := &filersink.FilerSink{} - filerSink.DoInitialize(pb.ServerToGrpcAddress(targetFiler), targetPath, replicationStr, collection, ttlSec, grpcDialOption) + filerSink.DoInitialize(targetFiler, pb.ServerToGrpcAddress(targetFiler), targetPath, replicationStr, collection, ttlSec, grpcDialOption, sinkWriteChunkByFiler) filerSink.SetSourceFiler(filerSource) processEventFn := func(resp *filer_pb.SubscribeMetadataResponse) error { diff --git a/weed/pb/grpc_client_server.go b/weed/pb/grpc_client_server.go index f19af43b2..4d78d769f 100644 --- a/weed/pb/grpc_client_server.go +++ b/weed/pb/grpc_client_server.go @@ -138,6 +138,22 @@ func ServerToGrpcAddress(server string) (serverGrpcAddress string) { return fmt.Sprintf("%s:%d", hostnameAndPort[0], grpcPort) } +func GrpcAddressToServerAddress(grpcAddress string) (serverAddress string) { + hostnameAndPort := strings.Split(grpcAddress, ":") + if len(hostnameAndPort) != 2 { + return fmt.Sprintf("unexpected grpcAddress: %s", grpcAddress) + } + + grpcPort, parseErr := strconv.ParseUint(hostnameAndPort[1], 10, 64) + if parseErr != nil { + return fmt.Sprintf("failed to parse port for %s:%s", hostnameAndPort[0], hostnameAndPort[1]) + } + + port := int(grpcPort) - 10000 + + return fmt.Sprintf("%s:%d", hostnameAndPort[0], port) +} + func WithMasterClient(master string, grpcDialOption grpc.DialOption, fn func(client master_pb.SeaweedClient) error) error { masterGrpcAddress, parseErr := ParseServerToGrpcAddress(master) diff --git a/weed/replication/sink/filersink/fetch_write.go b/weed/replication/sink/filersink/fetch_write.go index d193ff81c..b062adcfe 100644 --- a/weed/replication/sink/filersink/fetch_write.go +++ b/weed/replication/sink/filersink/fetch_write.go @@ -30,6 +30,7 @@ func (fs *FilerSink) replicateChunks(sourceChunks []*filer_pb.FileChunk, path st replicatedChunk, e := fs.replicateOneChunk(chunk, path) if e != nil { err = e + return } replicatedChunks[index] = replicatedChunk }(sourceChunk, chunkIndex) @@ -97,6 +98,9 @@ func (fs *FilerSink) fetchAndWrite(sourceChunk *filer_pb.FileChunk, path string) } fileUrl := fmt.Sprintf("http://%s/%s", host, fileId) + if fs.writeChunkByFiler { + fileUrl = fmt.Sprintf("http://%s/?proxyChunkId=%s", fs.address, fileId) + } glog.V(4).Infof("replicating %s to %s header:%+v", filename, fileUrl, header) diff --git a/weed/replication/sink/filersink/filer_sink.go b/weed/replication/sink/filersink/filer_sink.go index 9c0e4176f..600ff51f0 100644 --- a/weed/replication/sink/filersink/filer_sink.go +++ b/weed/replication/sink/filersink/filer_sink.go @@ -3,6 +3,7 @@ package filersink import ( "context" "fmt" + "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/wdclient" "google.golang.org/grpc" @@ -18,14 +19,16 @@ import ( ) type FilerSink struct { - filerSource *source.FilerSource - grpcAddress string - dir string - replication string - collection string - ttlSec int32 - dataCenter string - grpcDialOption grpc.DialOption + filerSource *source.FilerSource + grpcAddress string + dir string + replication string + collection string + ttlSec int32 + dataCenter string + grpcDialOption grpc.DialOption + address string + writeChunkByFiler bool } func init() { @@ -42,26 +45,33 @@ func (fs *FilerSink) GetSinkToDirectory() string { func (fs *FilerSink) Initialize(configuration util.Configuration, prefix string) error { return fs.DoInitialize( + "", configuration.GetString(prefix+"grpcAddress"), configuration.GetString(prefix+"directory"), configuration.GetString(prefix+"replication"), configuration.GetString(prefix+"collection"), configuration.GetInt(prefix+"ttlSec"), - security.LoadClientTLS(util.GetViper(), "grpc.client")) + security.LoadClientTLS(util.GetViper(), "grpc.client"), + false) } func (fs *FilerSink) SetSourceFiler(s *source.FilerSource) { fs.filerSource = s } -func (fs *FilerSink) DoInitialize(grpcAddress string, dir string, - replication string, collection string, ttlSec int, grpcDialOption grpc.DialOption) (err error) { +func (fs *FilerSink) DoInitialize(address, grpcAddress string, dir string, + replication string, collection string, ttlSec int, grpcDialOption grpc.DialOption, writeChunkByFiler bool) (err error) { + fs.address = address + if fs.address == "" { + fs.address = pb.GrpcAddressToServerAddress(grpcAddress) + } fs.grpcAddress = grpcAddress fs.dir = dir fs.replication = replication fs.collection = collection fs.ttlSec = int32(ttlSec) fs.grpcDialOption = grpcDialOption + fs.writeChunkByFiler = writeChunkByFiler return nil } diff --git a/weed/replication/source/filer_source.go b/weed/replication/source/filer_source.go index ff4f2eb26..3982360b0 100644 --- a/weed/replication/source/filer_source.go +++ b/weed/replication/source/filer_source.go @@ -25,19 +25,28 @@ type FilerSource struct { grpcAddress string grpcDialOption grpc.DialOption Dir string + address string + proxyByFiler bool } func (fs *FilerSource) Initialize(configuration util.Configuration, prefix string) error { return fs.DoInitialize( + "", configuration.GetString(prefix+"grpcAddress"), configuration.GetString(prefix+"directory"), + false, ) } -func (fs *FilerSource) DoInitialize(grpcAddress string, dir string) (err error) { +func (fs *FilerSource) DoInitialize(address, grpcAddress string, dir string, readChunkFromFiler bool) (err error) { + fs.address = address + if fs.address == "" { + fs.address = pb.GrpcAddressToServerAddress(grpcAddress) + } fs.grpcAddress = grpcAddress fs.Dir = dir fs.grpcDialOption = security.LoadClientTLS(util.GetViper(), "grpc.client") + fs.proxyByFiler = readChunkFromFiler return nil } @@ -81,9 +90,13 @@ func (fs *FilerSource) LookupFileId(part string) (fileUrls []string, err error) return } -func (fs *FilerSource) ReadPart(part string) (filename string, header http.Header, resp *http.Response, err error) { +func (fs *FilerSource) ReadPart(fileId string) (filename string, header http.Header, resp *http.Response, err error) { + + if fs.proxyByFiler { + return util.DownloadFile("http://" + fs.address + "/?proxyChunkId=" + fileId) + } - fileUrls, err := fs.LookupFileId(part) + fileUrls, err := fs.LookupFileId(fileId) if err != nil { return "", nil, nil, err } diff --git a/weed/server/filer_server_handlers.go b/weed/server/filer_server_handlers.go index 5b93c6d08..fcafe3893 100644 --- a/weed/server/filer_server_handlers.go +++ b/weed/server/filer_server_handlers.go @@ -3,30 +3,38 @@ package weed_server import ( "github.com/chrislusf/seaweedfs/weed/util" "net/http" + "strings" "time" "github.com/chrislusf/seaweedfs/weed/stats" ) func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) { + + start := time.Now() + + // proxy to volume servers + var fileId string + if strings.HasPrefix(r.RequestURI, "/?proxyChunkId=") { + fileId = r.RequestURI[len("/?proxyChunkId="):] + } + if fileId != "" { + stats.FilerRequestCounter.WithLabelValues("proxy").Inc() + fs.proxyToVolumeServer(w,r,fileId) + stats.FilerRequestHistogram.WithLabelValues("proxy").Observe(time.Since(start).Seconds()) + return + } + w.Header().Set("Server", "SeaweedFS Filer "+util.VERSION) if r.Header.Get("Origin") != "" { w.Header().Set("Access-Control-Allow-Origin", "*") w.Header().Set("Access-Control-Allow-Credentials", "true") } - start := time.Now() switch r.Method { case "GET": - fileId := r.FormValue("proxyToFileId") - if fileId != "" { - stats.FilerRequestCounter.WithLabelValues("proxy").Inc() - fs.proxyToVolumeServer(w,r,fileId) - stats.FilerRequestHistogram.WithLabelValues("proxy").Observe(time.Since(start).Seconds()) - } else { - stats.FilerRequestCounter.WithLabelValues("get").Inc() - fs.GetOrHeadHandler(w, r, true) - stats.FilerRequestHistogram.WithLabelValues("get").Observe(time.Since(start).Seconds()) - } + stats.FilerRequestCounter.WithLabelValues("get").Inc() + fs.GetOrHeadHandler(w, r, true) + stats.FilerRequestHistogram.WithLabelValues("get").Observe(time.Since(start).Seconds()) case "HEAD": stats.FilerRequestCounter.WithLabelValues("head").Inc() fs.GetOrHeadHandler(w, r, false) From ccbdb38c89a79a482c46faaa9b0dd53e3dacb822 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 24 Jan 2021 00:27:52 -0800 Subject: [PATCH 177/260] s3: fix for listing objects if more than 1000 in the folder many existing tools have max key set to 1000 --- weed/s3api/s3api_objects_list_handlers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/s3api/s3api_objects_list_handlers.go b/weed/s3api/s3api_objects_list_handlers.go index a2407fced..2d36c6ec9 100644 --- a/weed/s3api/s3api_objects_list_handlers.go +++ b/weed/s3api/s3api_objects_list_handlers.go @@ -238,7 +238,7 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d return } } - if counter >= maxKeys + 1 { + if counter >= maxKeys { isTruncated = true return } From 096e088d7bb2a5dce7573b24c2d3006d1cb6f9ec Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 24 Jan 2021 01:41:38 -0800 Subject: [PATCH 178/260] mount: when outside cluster network, use filer as proxy to access volume servers --- weed/command/mount_std.go | 1 + weed/filer/reader_at.go | 6 +++--- weed/filesys/file.go | 4 ++-- weed/filesys/filehandle.go | 6 +++--- weed/filesys/wfs.go | 13 +++++++++++++ weed/filesys/wfs_deletion.go | 4 ++-- weed/filesys/wfs_filer_client.go | 7 ------- weed/filesys/wfs_write.go | 5 ++++- weed/messaging/broker/broker_append.go | 4 ---- weed/pb/filer_pb/filer_client.go | 1 - weed/replication/sink/filersink/fetch_write.go | 3 --- weed/replication/source/filer_source.go | 4 ---- weed/s3api/s3api_handlers.go | 3 --- weed/server/webdav_server.go | 5 +---- weed/shell/commands.go | 4 ---- 15 files changed, 29 insertions(+), 41 deletions(-) diff --git a/weed/command/mount_std.go b/weed/command/mount_std.go index 9e955e344..a95ecd567 100644 --- a/weed/command/mount_std.go +++ b/weed/command/mount_std.go @@ -169,6 +169,7 @@ func RunMount(option *MountOptions, umask os.FileMode) bool { } seaweedFileSystem := filesys.NewSeaweedFileSystem(&filesys.Option{ + FilerAddress: filer, FilerGrpcAddress: filerGrpcAddress, GrpcDialOption: grpcDialOption, FilerMountRootPath: mountRoot, diff --git a/weed/filer/reader_at.go b/weed/filer/reader_at.go index 307224f35..41d177210 100644 --- a/weed/filer/reader_at.go +++ b/weed/filer/reader_at.go @@ -71,7 +71,7 @@ func LookupFn(filerClient filer_pb.FilerClient) wdclient.LookupFileIdFunctionTyp } for _, loc := range locations.Locations { - volumeServerAddress := filerClient.AdjustedUrl(loc) + volumeServerAddress := loc.Url targetUrl := fmt.Sprintf("http://%s/%s", volumeServerAddress, fileId) targetUrls = append(targetUrls, targetUrl) } @@ -85,11 +85,11 @@ func LookupFn(filerClient filer_pb.FilerClient) wdclient.LookupFileIdFunctionTyp } } -func NewChunkReaderAtFromClient(filerClient filer_pb.FilerClient, chunkViews []*ChunkView, chunkCache chunk_cache.ChunkCache, fileSize int64) *ChunkReadAt { +func NewChunkReaderAtFromClient(lookupFn wdclient.LookupFileIdFunctionType, chunkViews []*ChunkView, chunkCache chunk_cache.ChunkCache, fileSize int64) *ChunkReadAt { return &ChunkReadAt{ chunkViews: chunkViews, - lookupFileId: LookupFn(filerClient), + lookupFileId: lookupFn, chunkCache: chunkCache, fileSize: fileSize, } diff --git a/weed/filesys/file.go b/weed/filesys/file.go index a2b6660d8..8054435b1 100644 --- a/weed/filesys/file.go +++ b/weed/filesys/file.go @@ -147,7 +147,7 @@ func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *f } } file.entry.Chunks = chunks - file.entryViewCache, _ = filer.NonOverlappingVisibleIntervals(filer.LookupFn(file.wfs), chunks) + file.entryViewCache, _ = filer.NonOverlappingVisibleIntervals(file.wfs.LookupFn(), chunks) file.reader = nil file.wfs.deleteFileChunks(truncatedChunks) } @@ -329,7 +329,7 @@ func (file *File) addChunks(chunks []*filer_pb.FileChunk) { func (file *File) setEntry(entry *filer_pb.Entry) { file.entry = entry - file.entryViewCache, _ = filer.NonOverlappingVisibleIntervals(filer.LookupFn(file.wfs), entry.Chunks) + file.entryViewCache, _ = filer.NonOverlappingVisibleIntervals(file.wfs.LookupFn(), entry.Chunks) file.reader = nil } diff --git a/weed/filesys/filehandle.go b/weed/filesys/filehandle.go index 6225ab968..da42ae562 100644 --- a/weed/filesys/filehandle.go +++ b/weed/filesys/filehandle.go @@ -119,7 +119,7 @@ func (fh *FileHandle) readFromChunks(buff []byte, offset int64) (int64, error) { var chunkResolveErr error if fh.f.entryViewCache == nil { - fh.f.entryViewCache, chunkResolveErr = filer.NonOverlappingVisibleIntervals(filer.LookupFn(fh.f.wfs), fh.f.entry.Chunks) + fh.f.entryViewCache, chunkResolveErr = filer.NonOverlappingVisibleIntervals(fh.f.wfs.LookupFn(), fh.f.entry.Chunks) if chunkResolveErr != nil { return 0, fmt.Errorf("fail to resolve chunk manifest: %v", chunkResolveErr) } @@ -128,7 +128,7 @@ func (fh *FileHandle) readFromChunks(buff []byte, offset int64) (int64, error) { if fh.f.reader == nil { chunkViews := filer.ViewFromVisibleIntervals(fh.f.entryViewCache, 0, math.MaxInt64) - fh.f.reader = filer.NewChunkReaderAtFromClient(fh.f.wfs, chunkViews, fh.f.wfs.chunkCache, fileSize) + fh.f.reader = filer.NewChunkReaderAtFromClient(fh.f.wfs.LookupFn(), chunkViews, fh.f.wfs.chunkCache, fileSize) } totalRead, err := fh.f.reader.ReadAt(buff, offset) @@ -269,7 +269,7 @@ func (fh *FileHandle) doFlush(ctx context.Context, header fuse.Header) error { manifestChunks, nonManifestChunks := filer.SeparateManifestChunks(fh.f.entry.Chunks) - chunks, _ := filer.CompactFileChunks(filer.LookupFn(fh.f.wfs), nonManifestChunks) + chunks, _ := filer.CompactFileChunks(fh.f.wfs.LookupFn(), nonManifestChunks) chunks, manifestErr := filer.MaybeManifestize(fh.f.wfs.saveDataAsChunk(fh.f.fullpath()), chunks) if manifestErr != nil { // not good, but should be ok diff --git a/weed/filesys/wfs.go b/weed/filesys/wfs.go index cd14e8032..236ecdacb 100644 --- a/weed/filesys/wfs.go +++ b/weed/filesys/wfs.go @@ -3,6 +3,8 @@ package filesys import ( "context" "fmt" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/wdclient" "math" "os" "path" @@ -24,6 +26,7 @@ import ( ) type Option struct { + FilerAddress string FilerGrpcAddress string GrpcDialOption grpc.DialOption FilerMountRootPath string @@ -237,3 +240,13 @@ func (wfs *WFS) mapPbIdFromLocalToFiler(entry *filer_pb.Entry) { } entry.Attributes.Uid, entry.Attributes.Gid = wfs.option.UidGidMapper.LocalToFiler(entry.Attributes.Uid, entry.Attributes.Gid) } + +func (wfs *WFS) LookupFn() wdclient.LookupFileIdFunctionType { + if wfs.option.OutsideContainerClusterMode { + return func(fileId string) (targetUrls []string, err error) { + return []string{"http://" + wfs.option.FilerAddress + "/?proxyChunkId=" + fileId}, nil + } + } + return filer.LookupFn(wfs) + +} diff --git a/weed/filesys/wfs_deletion.go b/weed/filesys/wfs_deletion.go index a245b6795..7ed936a62 100644 --- a/weed/filesys/wfs_deletion.go +++ b/weed/filesys/wfs_deletion.go @@ -22,7 +22,7 @@ func (wfs *WFS) deleteFileChunks(chunks []*filer_pb.FileChunk) { fileIds = append(fileIds, chunk.GetFileIdString()) continue } - dataChunks, manifestResolveErr := filer.ResolveOneChunkManifest(filer.LookupFn(wfs), chunk) + dataChunks, manifestResolveErr := filer.ResolveOneChunkManifest(wfs.LookupFn(), chunk) if manifestResolveErr != nil { glog.V(0).Infof("failed to resolve manifest %s: %v", chunk.FileId, manifestResolveErr) } @@ -68,7 +68,7 @@ func (wfs *WFS) deleteFileIds(grpcDialOption grpc.DialOption, client filer_pb.Se } for _, loc := range locations.Locations { lr.Locations = append(lr.Locations, operation.Location{ - Url: wfs.AdjustedUrl(loc), + Url: loc.Url, PublicUrl: loc.PublicUrl, }) } diff --git a/weed/filesys/wfs_filer_client.go b/weed/filesys/wfs_filer_client.go index ef4213af1..e0d352a7b 100644 --- a/weed/filesys/wfs_filer_client.go +++ b/weed/filesys/wfs_filer_client.go @@ -25,10 +25,3 @@ func (wfs *WFS) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) error) erro return err } - -func (wfs *WFS) AdjustedUrl(location *filer_pb.Location) string { - if wfs.option.OutsideContainerClusterMode { - return location.PublicUrl - } - return location.Url -} diff --git a/weed/filesys/wfs_write.go b/weed/filesys/wfs_write.go index 83e40e7f5..dfe6e57a6 100644 --- a/weed/filesys/wfs_write.go +++ b/weed/filesys/wfs_write.go @@ -44,7 +44,7 @@ func (wfs *WFS) saveDataAsChunk(fullPath util.FullPath) filer.SaveDataAsChunkFun Url: resp.Url, PublicUrl: resp.PublicUrl, } - host = wfs.AdjustedUrl(loc) + host = loc.Url collection, replication = resp.Collection, resp.Replication return nil @@ -53,6 +53,9 @@ func (wfs *WFS) saveDataAsChunk(fullPath util.FullPath) filer.SaveDataAsChunkFun } fileUrl := fmt.Sprintf("http://%s/%s", host, fileId) + if wfs.option.OutsideContainerClusterMode { + fileUrl = fmt.Sprintf("http://%s/?proxyChunkId=%s", wfs.option.FilerAddress, fileId) + } uploadResult, err, data := operation.Upload(fileUrl, filename, wfs.option.Cipher, reader, false, "", nil, auth) if err != nil { glog.V(0).Infof("upload data %v to %s: %v", filename, fileUrl, err) diff --git a/weed/messaging/broker/broker_append.go b/weed/messaging/broker/broker_append.go index 8e5b56fd0..67c9bcb79 100644 --- a/weed/messaging/broker/broker_append.go +++ b/weed/messaging/broker/broker_append.go @@ -107,7 +107,3 @@ func (broker *MessageBroker) WithFilerClient(fn func(filer_pb.SeaweedFilerClient return } - -func (broker *MessageBroker) AdjustedUrl(location *filer_pb.Location) string { - return location.Url -} diff --git a/weed/pb/filer_pb/filer_client.go b/weed/pb/filer_pb/filer_client.go index 079fbd671..7198de95c 100644 --- a/weed/pb/filer_pb/filer_client.go +++ b/weed/pb/filer_pb/filer_client.go @@ -20,7 +20,6 @@ var ( type FilerClient interface { WithFilerClient(fn func(SeaweedFilerClient) error) error - AdjustedUrl(location *Location) string } func GetEntry(filerClient FilerClient, fullFilePath util.FullPath) (entry *Entry, err error) { diff --git a/weed/replication/sink/filersink/fetch_write.go b/weed/replication/sink/filersink/fetch_write.go index b062adcfe..544b84995 100644 --- a/weed/replication/sink/filersink/fetch_write.go +++ b/weed/replication/sink/filersink/fetch_write.go @@ -128,6 +128,3 @@ func (fs *FilerSink) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) error) }, fs.grpcAddress, fs.grpcDialOption) } -func (fs *FilerSink) AdjustedUrl(location *filer_pb.Location) string { - return location.Url -} diff --git a/weed/replication/source/filer_source.go b/weed/replication/source/filer_source.go index 3982360b0..eff1da8dc 100644 --- a/weed/replication/source/filer_source.go +++ b/weed/replication/source/filer_source.go @@ -124,10 +124,6 @@ func (fs *FilerSource) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) erro } -func (fs *FilerSource) AdjustedUrl(location *filer_pb.Location) string { - return location.Url -} - func volumeId(fileId string) string { lastCommaIndex := strings.LastIndex(fileId, ",") if lastCommaIndex > 0 { diff --git a/weed/s3api/s3api_handlers.go b/weed/s3api/s3api_handlers.go index 6935c75bd..57b26f3dd 100644 --- a/weed/s3api/s3api_handlers.go +++ b/weed/s3api/s3api_handlers.go @@ -50,9 +50,6 @@ func (s3a *S3ApiServer) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) err }, s3a.option.FilerGrpcAddress, s3a.option.GrpcDialOption) } -func (s3a *S3ApiServer) AdjustedUrl(location *filer_pb.Location) string { - return location.Url -} // If none of the http routes match respond with MethodNotAllowed func notFoundHandler(w http.ResponseWriter, r *http.Request) { diff --git a/weed/server/webdav_server.go b/weed/server/webdav_server.go index 2b238e534..5bd92a136 100644 --- a/weed/server/webdav_server.go +++ b/weed/server/webdav_server.go @@ -123,9 +123,6 @@ func (fs *WebDavFileSystem) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) }, fs.option.FilerGrpcAddress, fs.option.GrpcDialOption) } -func (fs *WebDavFileSystem) AdjustedUrl(location *filer_pb.Location) string { - return location.Url -} func clearName(name string) (string, error) { slashed := strings.HasSuffix(name, "/") @@ -523,7 +520,7 @@ func (f *WebDavFile) Read(p []byte) (readSize int, err error) { } if f.reader == nil { chunkViews := filer.ViewFromVisibleIntervals(f.entryViewCache, 0, math.MaxInt64) - f.reader = filer.NewChunkReaderAtFromClient(f.fs, chunkViews, f.fs.chunkCache, fileSize) + f.reader = filer.NewChunkReaderAtFromClient(filer.LookupFn(f.fs), chunkViews, f.fs.chunkCache, fileSize) } readSize, err = f.reader.ReadAt(p, f.off) diff --git a/weed/shell/commands.go b/weed/shell/commands.go index 0e285214b..6e1348ca5 100644 --- a/weed/shell/commands.go +++ b/weed/shell/commands.go @@ -102,10 +102,6 @@ func (ce *CommandEnv) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) error } -func (ce *CommandEnv) AdjustedUrl(location *filer_pb.Location) string { - return location.Url -} - func parseFilerUrl(entryPath string) (filerServer string, filerPort int64, path string, err error) { if strings.HasPrefix(entryPath, "http") { var u *url.URL From 6ca10725b81fd20d8bce2f582f1b93b047d0a4c3 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 24 Jan 2021 03:15:19 -0800 Subject: [PATCH 179/260] Revert "mount: when outside cluster network, use filer as proxy to access volume servers" This reverts commit 096e088d7bb2a5dce7573b24c2d3006d1cb6f9ec. --- weed/command/mount_std.go | 1 - weed/filer/reader_at.go | 6 +++--- weed/filesys/file.go | 4 ++-- weed/filesys/filehandle.go | 6 +++--- weed/filesys/wfs.go | 13 ------------- weed/filesys/wfs_deletion.go | 4 ++-- weed/filesys/wfs_filer_client.go | 7 +++++++ weed/filesys/wfs_write.go | 5 +---- weed/messaging/broker/broker_append.go | 4 ++++ weed/pb/filer_pb/filer_client.go | 1 + weed/replication/sink/filersink/fetch_write.go | 3 +++ weed/replication/source/filer_source.go | 4 ++++ weed/s3api/s3api_handlers.go | 3 +++ weed/server/webdav_server.go | 5 ++++- weed/shell/commands.go | 4 ++++ 15 files changed, 41 insertions(+), 29 deletions(-) diff --git a/weed/command/mount_std.go b/weed/command/mount_std.go index a95ecd567..9e955e344 100644 --- a/weed/command/mount_std.go +++ b/weed/command/mount_std.go @@ -169,7 +169,6 @@ func RunMount(option *MountOptions, umask os.FileMode) bool { } seaweedFileSystem := filesys.NewSeaweedFileSystem(&filesys.Option{ - FilerAddress: filer, FilerGrpcAddress: filerGrpcAddress, GrpcDialOption: grpcDialOption, FilerMountRootPath: mountRoot, diff --git a/weed/filer/reader_at.go b/weed/filer/reader_at.go index 41d177210..307224f35 100644 --- a/weed/filer/reader_at.go +++ b/weed/filer/reader_at.go @@ -71,7 +71,7 @@ func LookupFn(filerClient filer_pb.FilerClient) wdclient.LookupFileIdFunctionTyp } for _, loc := range locations.Locations { - volumeServerAddress := loc.Url + volumeServerAddress := filerClient.AdjustedUrl(loc) targetUrl := fmt.Sprintf("http://%s/%s", volumeServerAddress, fileId) targetUrls = append(targetUrls, targetUrl) } @@ -85,11 +85,11 @@ func LookupFn(filerClient filer_pb.FilerClient) wdclient.LookupFileIdFunctionTyp } } -func NewChunkReaderAtFromClient(lookupFn wdclient.LookupFileIdFunctionType, chunkViews []*ChunkView, chunkCache chunk_cache.ChunkCache, fileSize int64) *ChunkReadAt { +func NewChunkReaderAtFromClient(filerClient filer_pb.FilerClient, chunkViews []*ChunkView, chunkCache chunk_cache.ChunkCache, fileSize int64) *ChunkReadAt { return &ChunkReadAt{ chunkViews: chunkViews, - lookupFileId: lookupFn, + lookupFileId: LookupFn(filerClient), chunkCache: chunkCache, fileSize: fileSize, } diff --git a/weed/filesys/file.go b/weed/filesys/file.go index 8054435b1..a2b6660d8 100644 --- a/weed/filesys/file.go +++ b/weed/filesys/file.go @@ -147,7 +147,7 @@ func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *f } } file.entry.Chunks = chunks - file.entryViewCache, _ = filer.NonOverlappingVisibleIntervals(file.wfs.LookupFn(), chunks) + file.entryViewCache, _ = filer.NonOverlappingVisibleIntervals(filer.LookupFn(file.wfs), chunks) file.reader = nil file.wfs.deleteFileChunks(truncatedChunks) } @@ -329,7 +329,7 @@ func (file *File) addChunks(chunks []*filer_pb.FileChunk) { func (file *File) setEntry(entry *filer_pb.Entry) { file.entry = entry - file.entryViewCache, _ = filer.NonOverlappingVisibleIntervals(file.wfs.LookupFn(), entry.Chunks) + file.entryViewCache, _ = filer.NonOverlappingVisibleIntervals(filer.LookupFn(file.wfs), entry.Chunks) file.reader = nil } diff --git a/weed/filesys/filehandle.go b/weed/filesys/filehandle.go index da42ae562..6225ab968 100644 --- a/weed/filesys/filehandle.go +++ b/weed/filesys/filehandle.go @@ -119,7 +119,7 @@ func (fh *FileHandle) readFromChunks(buff []byte, offset int64) (int64, error) { var chunkResolveErr error if fh.f.entryViewCache == nil { - fh.f.entryViewCache, chunkResolveErr = filer.NonOverlappingVisibleIntervals(fh.f.wfs.LookupFn(), fh.f.entry.Chunks) + fh.f.entryViewCache, chunkResolveErr = filer.NonOverlappingVisibleIntervals(filer.LookupFn(fh.f.wfs), fh.f.entry.Chunks) if chunkResolveErr != nil { return 0, fmt.Errorf("fail to resolve chunk manifest: %v", chunkResolveErr) } @@ -128,7 +128,7 @@ func (fh *FileHandle) readFromChunks(buff []byte, offset int64) (int64, error) { if fh.f.reader == nil { chunkViews := filer.ViewFromVisibleIntervals(fh.f.entryViewCache, 0, math.MaxInt64) - fh.f.reader = filer.NewChunkReaderAtFromClient(fh.f.wfs.LookupFn(), chunkViews, fh.f.wfs.chunkCache, fileSize) + fh.f.reader = filer.NewChunkReaderAtFromClient(fh.f.wfs, chunkViews, fh.f.wfs.chunkCache, fileSize) } totalRead, err := fh.f.reader.ReadAt(buff, offset) @@ -269,7 +269,7 @@ func (fh *FileHandle) doFlush(ctx context.Context, header fuse.Header) error { manifestChunks, nonManifestChunks := filer.SeparateManifestChunks(fh.f.entry.Chunks) - chunks, _ := filer.CompactFileChunks(fh.f.wfs.LookupFn(), nonManifestChunks) + chunks, _ := filer.CompactFileChunks(filer.LookupFn(fh.f.wfs), nonManifestChunks) chunks, manifestErr := filer.MaybeManifestize(fh.f.wfs.saveDataAsChunk(fh.f.fullpath()), chunks) if manifestErr != nil { // not good, but should be ok diff --git a/weed/filesys/wfs.go b/weed/filesys/wfs.go index 236ecdacb..cd14e8032 100644 --- a/weed/filesys/wfs.go +++ b/weed/filesys/wfs.go @@ -3,8 +3,6 @@ package filesys import ( "context" "fmt" - "github.com/chrislusf/seaweedfs/weed/filer" - "github.com/chrislusf/seaweedfs/weed/wdclient" "math" "os" "path" @@ -26,7 +24,6 @@ import ( ) type Option struct { - FilerAddress string FilerGrpcAddress string GrpcDialOption grpc.DialOption FilerMountRootPath string @@ -240,13 +237,3 @@ func (wfs *WFS) mapPbIdFromLocalToFiler(entry *filer_pb.Entry) { } entry.Attributes.Uid, entry.Attributes.Gid = wfs.option.UidGidMapper.LocalToFiler(entry.Attributes.Uid, entry.Attributes.Gid) } - -func (wfs *WFS) LookupFn() wdclient.LookupFileIdFunctionType { - if wfs.option.OutsideContainerClusterMode { - return func(fileId string) (targetUrls []string, err error) { - return []string{"http://" + wfs.option.FilerAddress + "/?proxyChunkId=" + fileId}, nil - } - } - return filer.LookupFn(wfs) - -} diff --git a/weed/filesys/wfs_deletion.go b/weed/filesys/wfs_deletion.go index 7ed936a62..a245b6795 100644 --- a/weed/filesys/wfs_deletion.go +++ b/weed/filesys/wfs_deletion.go @@ -22,7 +22,7 @@ func (wfs *WFS) deleteFileChunks(chunks []*filer_pb.FileChunk) { fileIds = append(fileIds, chunk.GetFileIdString()) continue } - dataChunks, manifestResolveErr := filer.ResolveOneChunkManifest(wfs.LookupFn(), chunk) + dataChunks, manifestResolveErr := filer.ResolveOneChunkManifest(filer.LookupFn(wfs), chunk) if manifestResolveErr != nil { glog.V(0).Infof("failed to resolve manifest %s: %v", chunk.FileId, manifestResolveErr) } @@ -68,7 +68,7 @@ func (wfs *WFS) deleteFileIds(grpcDialOption grpc.DialOption, client filer_pb.Se } for _, loc := range locations.Locations { lr.Locations = append(lr.Locations, operation.Location{ - Url: loc.Url, + Url: wfs.AdjustedUrl(loc), PublicUrl: loc.PublicUrl, }) } diff --git a/weed/filesys/wfs_filer_client.go b/weed/filesys/wfs_filer_client.go index e0d352a7b..ef4213af1 100644 --- a/weed/filesys/wfs_filer_client.go +++ b/weed/filesys/wfs_filer_client.go @@ -25,3 +25,10 @@ func (wfs *WFS) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) error) erro return err } + +func (wfs *WFS) AdjustedUrl(location *filer_pb.Location) string { + if wfs.option.OutsideContainerClusterMode { + return location.PublicUrl + } + return location.Url +} diff --git a/weed/filesys/wfs_write.go b/weed/filesys/wfs_write.go index dfe6e57a6..83e40e7f5 100644 --- a/weed/filesys/wfs_write.go +++ b/weed/filesys/wfs_write.go @@ -44,7 +44,7 @@ func (wfs *WFS) saveDataAsChunk(fullPath util.FullPath) filer.SaveDataAsChunkFun Url: resp.Url, PublicUrl: resp.PublicUrl, } - host = loc.Url + host = wfs.AdjustedUrl(loc) collection, replication = resp.Collection, resp.Replication return nil @@ -53,9 +53,6 @@ func (wfs *WFS) saveDataAsChunk(fullPath util.FullPath) filer.SaveDataAsChunkFun } fileUrl := fmt.Sprintf("http://%s/%s", host, fileId) - if wfs.option.OutsideContainerClusterMode { - fileUrl = fmt.Sprintf("http://%s/?proxyChunkId=%s", wfs.option.FilerAddress, fileId) - } uploadResult, err, data := operation.Upload(fileUrl, filename, wfs.option.Cipher, reader, false, "", nil, auth) if err != nil { glog.V(0).Infof("upload data %v to %s: %v", filename, fileUrl, err) diff --git a/weed/messaging/broker/broker_append.go b/weed/messaging/broker/broker_append.go index 67c9bcb79..8e5b56fd0 100644 --- a/weed/messaging/broker/broker_append.go +++ b/weed/messaging/broker/broker_append.go @@ -107,3 +107,7 @@ func (broker *MessageBroker) WithFilerClient(fn func(filer_pb.SeaweedFilerClient return } + +func (broker *MessageBroker) AdjustedUrl(location *filer_pb.Location) string { + return location.Url +} diff --git a/weed/pb/filer_pb/filer_client.go b/weed/pb/filer_pb/filer_client.go index 7198de95c..079fbd671 100644 --- a/weed/pb/filer_pb/filer_client.go +++ b/weed/pb/filer_pb/filer_client.go @@ -20,6 +20,7 @@ var ( type FilerClient interface { WithFilerClient(fn func(SeaweedFilerClient) error) error + AdjustedUrl(location *Location) string } func GetEntry(filerClient FilerClient, fullFilePath util.FullPath) (entry *Entry, err error) { diff --git a/weed/replication/sink/filersink/fetch_write.go b/weed/replication/sink/filersink/fetch_write.go index 544b84995..b062adcfe 100644 --- a/weed/replication/sink/filersink/fetch_write.go +++ b/weed/replication/sink/filersink/fetch_write.go @@ -128,3 +128,6 @@ func (fs *FilerSink) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) error) }, fs.grpcAddress, fs.grpcDialOption) } +func (fs *FilerSink) AdjustedUrl(location *filer_pb.Location) string { + return location.Url +} diff --git a/weed/replication/source/filer_source.go b/weed/replication/source/filer_source.go index eff1da8dc..3982360b0 100644 --- a/weed/replication/source/filer_source.go +++ b/weed/replication/source/filer_source.go @@ -124,6 +124,10 @@ func (fs *FilerSource) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) erro } +func (fs *FilerSource) AdjustedUrl(location *filer_pb.Location) string { + return location.Url +} + func volumeId(fileId string) string { lastCommaIndex := strings.LastIndex(fileId, ",") if lastCommaIndex > 0 { diff --git a/weed/s3api/s3api_handlers.go b/weed/s3api/s3api_handlers.go index 57b26f3dd..6935c75bd 100644 --- a/weed/s3api/s3api_handlers.go +++ b/weed/s3api/s3api_handlers.go @@ -50,6 +50,9 @@ func (s3a *S3ApiServer) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) err }, s3a.option.FilerGrpcAddress, s3a.option.GrpcDialOption) } +func (s3a *S3ApiServer) AdjustedUrl(location *filer_pb.Location) string { + return location.Url +} // If none of the http routes match respond with MethodNotAllowed func notFoundHandler(w http.ResponseWriter, r *http.Request) { diff --git a/weed/server/webdav_server.go b/weed/server/webdav_server.go index 5bd92a136..2b238e534 100644 --- a/weed/server/webdav_server.go +++ b/weed/server/webdav_server.go @@ -123,6 +123,9 @@ func (fs *WebDavFileSystem) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) }, fs.option.FilerGrpcAddress, fs.option.GrpcDialOption) } +func (fs *WebDavFileSystem) AdjustedUrl(location *filer_pb.Location) string { + return location.Url +} func clearName(name string) (string, error) { slashed := strings.HasSuffix(name, "/") @@ -520,7 +523,7 @@ func (f *WebDavFile) Read(p []byte) (readSize int, err error) { } if f.reader == nil { chunkViews := filer.ViewFromVisibleIntervals(f.entryViewCache, 0, math.MaxInt64) - f.reader = filer.NewChunkReaderAtFromClient(filer.LookupFn(f.fs), chunkViews, f.fs.chunkCache, fileSize) + f.reader = filer.NewChunkReaderAtFromClient(f.fs, chunkViews, f.fs.chunkCache, fileSize) } readSize, err = f.reader.ReadAt(p, f.off) diff --git a/weed/shell/commands.go b/weed/shell/commands.go index 6e1348ca5..0e285214b 100644 --- a/weed/shell/commands.go +++ b/weed/shell/commands.go @@ -102,6 +102,10 @@ func (ce *CommandEnv) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) error } +func (ce *CommandEnv) AdjustedUrl(location *filer_pb.Location) string { + return location.Url +} + func parseFilerUrl(entryPath string) (filerServer string, filerPort int64, path string, err error) { if strings.HasPrefix(entryPath, "http") { var u *url.URL From 2c5eac5705c12b4dc0930d0a27478a73924b9e16 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 24 Jan 2021 14:06:09 -0800 Subject: [PATCH 180/260] 2.22 --- k8s/seaweedfs/Chart.yaml | 4 ++-- k8s/seaweedfs/values.yaml | 2 +- weed/util/constants.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/k8s/seaweedfs/Chart.yaml b/k8s/seaweedfs/Chart.yaml index 7cf6b62a7..112648a60 100644 --- a/k8s/seaweedfs/Chart.yaml +++ b/k8s/seaweedfs/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v1 description: SeaweedFS name: seaweedfs -appVersion: "2.21" -version: 2.21 +appVersion: "2.22" +version: 2.22 diff --git a/k8s/seaweedfs/values.yaml b/k8s/seaweedfs/values.yaml index 5e40bae2b..4b4833da3 100644 --- a/k8s/seaweedfs/values.yaml +++ b/k8s/seaweedfs/values.yaml @@ -4,7 +4,7 @@ global: registry: "" repository: "" imageName: chrislusf/seaweedfs - # imageTag: "2.21" - started using {.Chart.appVersion} + # imageTag: "2.22" - started using {.Chart.appVersion} imagePullPolicy: IfNotPresent imagePullSecrets: imagepullsecret restartPolicy: Always diff --git a/weed/util/constants.go b/weed/util/constants.go index 4e6a28334..b9a9bd492 100644 --- a/weed/util/constants.go +++ b/weed/util/constants.go @@ -5,7 +5,7 @@ import ( ) var ( - VERSION = fmt.Sprintf("%s %d.%02d", sizeLimit, 2, 21) + VERSION = fmt.Sprintf("%s %d.%02d", sizeLimit, 2, 22) COMMIT = "" ) From 9bcb28a3eabcb2c869afc52da6b0341e8e2d8427 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 24 Jan 2021 17:19:35 -0800 Subject: [PATCH 181/260] avoid possible nil reader fix https://github.com/chrislusf/seaweedfs/issues/1754 --- weed/filesys/filehandle.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/weed/filesys/filehandle.go b/weed/filesys/filehandle.go index 6225ab968..5701b92b7 100644 --- a/weed/filesys/filehandle.go +++ b/weed/filesys/filehandle.go @@ -126,12 +126,14 @@ func (fh *FileHandle) readFromChunks(buff []byte, offset int64) (int64, error) { fh.f.reader = nil } - if fh.f.reader == nil { + reader := fh.f.reader + if reader == nil { chunkViews := filer.ViewFromVisibleIntervals(fh.f.entryViewCache, 0, math.MaxInt64) - fh.f.reader = filer.NewChunkReaderAtFromClient(fh.f.wfs, chunkViews, fh.f.wfs.chunkCache, fileSize) + reader = filer.NewChunkReaderAtFromClient(fh.f.wfs, chunkViews, fh.f.wfs.chunkCache, fileSize) } + fh.f.reader = reader - totalRead, err := fh.f.reader.ReadAt(buff, offset) + totalRead, err := reader.ReadAt(buff, offset) if err != nil && err != io.EOF { glog.Errorf("file handle read %s: %v", fh.f.fullpath(), err) From 00707ec00fb8016ac9ef8858a01a9784a6aee1a0 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 24 Jan 2021 19:01:58 -0800 Subject: [PATCH 182/260] mount: outsideContainerClusterMode proxy through filer Running mount outside of the cluster would not need to expose all the volume servers to outside of the cluster. The chunk read and write will go through the filer. --- weed/command/mount_std.go | 1 + weed/filer/reader_at.go | 6 +- weed/filesys/dir.go | 5 -- weed/filesys/file.go | 5 +- weed/filesys/filehandle.go | 6 +- weed/filesys/wfs.go | 13 +++ weed/filesys/wfs_deletion.go | 84 ------------------- weed/filesys/wfs_filer_client.go | 7 -- weed/filesys/wfs_write.go | 5 +- weed/messaging/broker/broker_append.go | 4 - weed/operation/delete_content.go | 3 +- weed/pb/filer_pb/filer_client.go | 1 - .../replication/sink/filersink/fetch_write.go | 3 - weed/replication/source/filer_source.go | 4 - weed/s3api/s3api_handlers.go | 3 - weed/server/webdav_server.go | 5 +- weed/shell/commands.go | 4 - 17 files changed, 28 insertions(+), 131 deletions(-) delete mode 100644 weed/filesys/wfs_deletion.go diff --git a/weed/command/mount_std.go b/weed/command/mount_std.go index 9e955e344..a95ecd567 100644 --- a/weed/command/mount_std.go +++ b/weed/command/mount_std.go @@ -169,6 +169,7 @@ func RunMount(option *MountOptions, umask os.FileMode) bool { } seaweedFileSystem := filesys.NewSeaweedFileSystem(&filesys.Option{ + FilerAddress: filer, FilerGrpcAddress: filerGrpcAddress, GrpcDialOption: grpcDialOption, FilerMountRootPath: mountRoot, diff --git a/weed/filer/reader_at.go b/weed/filer/reader_at.go index 307224f35..41d177210 100644 --- a/weed/filer/reader_at.go +++ b/weed/filer/reader_at.go @@ -71,7 +71,7 @@ func LookupFn(filerClient filer_pb.FilerClient) wdclient.LookupFileIdFunctionTyp } for _, loc := range locations.Locations { - volumeServerAddress := filerClient.AdjustedUrl(loc) + volumeServerAddress := loc.Url targetUrl := fmt.Sprintf("http://%s/%s", volumeServerAddress, fileId) targetUrls = append(targetUrls, targetUrl) } @@ -85,11 +85,11 @@ func LookupFn(filerClient filer_pb.FilerClient) wdclient.LookupFileIdFunctionTyp } } -func NewChunkReaderAtFromClient(filerClient filer_pb.FilerClient, chunkViews []*ChunkView, chunkCache chunk_cache.ChunkCache, fileSize int64) *ChunkReadAt { +func NewChunkReaderAtFromClient(lookupFn wdclient.LookupFileIdFunctionType, chunkViews []*ChunkView, chunkCache chunk_cache.ChunkCache, fileSize int64) *ChunkReadAt { return &ChunkReadAt{ chunkViews: chunkViews, - lookupFileId: LookupFn(filerClient), + lookupFileId: lookupFn, chunkCache: chunkCache, fileSize: fileSize, } diff --git a/weed/filesys/dir.go b/weed/filesys/dir.go index d86d92ac9..3d0a00a8b 100644 --- a/weed/filesys/dir.go +++ b/weed/filesys/dir.go @@ -404,11 +404,6 @@ func (dir *Dir) removeOneFile(req *fuse.RemoveRequest) error { inodeId := util.NewFullPath(dir.FullPath(), req.Name).AsInode() delete(dir.wfs.handles, inodeId) - // delete the chunks last - if isDeleteData { - dir.wfs.deleteFileChunks(entry.Chunks) - } - return nil } diff --git a/weed/filesys/file.go b/weed/filesys/file.go index a2b6660d8..a8d6dac29 100644 --- a/weed/filesys/file.go +++ b/weed/filesys/file.go @@ -147,9 +147,8 @@ func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *f } } file.entry.Chunks = chunks - file.entryViewCache, _ = filer.NonOverlappingVisibleIntervals(filer.LookupFn(file.wfs), chunks) + file.entryViewCache, _ = filer.NonOverlappingVisibleIntervals(file.wfs.LookupFn(), chunks) file.reader = nil - file.wfs.deleteFileChunks(truncatedChunks) } file.entry.Attributes.FileSize = req.Size file.dirtyMetadata = true @@ -329,7 +328,7 @@ func (file *File) addChunks(chunks []*filer_pb.FileChunk) { func (file *File) setEntry(entry *filer_pb.Entry) { file.entry = entry - file.entryViewCache, _ = filer.NonOverlappingVisibleIntervals(filer.LookupFn(file.wfs), entry.Chunks) + file.entryViewCache, _ = filer.NonOverlappingVisibleIntervals(file.wfs.LookupFn(), entry.Chunks) file.reader = nil } diff --git a/weed/filesys/filehandle.go b/weed/filesys/filehandle.go index 6225ab968..da42ae562 100644 --- a/weed/filesys/filehandle.go +++ b/weed/filesys/filehandle.go @@ -119,7 +119,7 @@ func (fh *FileHandle) readFromChunks(buff []byte, offset int64) (int64, error) { var chunkResolveErr error if fh.f.entryViewCache == nil { - fh.f.entryViewCache, chunkResolveErr = filer.NonOverlappingVisibleIntervals(filer.LookupFn(fh.f.wfs), fh.f.entry.Chunks) + fh.f.entryViewCache, chunkResolveErr = filer.NonOverlappingVisibleIntervals(fh.f.wfs.LookupFn(), fh.f.entry.Chunks) if chunkResolveErr != nil { return 0, fmt.Errorf("fail to resolve chunk manifest: %v", chunkResolveErr) } @@ -128,7 +128,7 @@ func (fh *FileHandle) readFromChunks(buff []byte, offset int64) (int64, error) { if fh.f.reader == nil { chunkViews := filer.ViewFromVisibleIntervals(fh.f.entryViewCache, 0, math.MaxInt64) - fh.f.reader = filer.NewChunkReaderAtFromClient(fh.f.wfs, chunkViews, fh.f.wfs.chunkCache, fileSize) + fh.f.reader = filer.NewChunkReaderAtFromClient(fh.f.wfs.LookupFn(), chunkViews, fh.f.wfs.chunkCache, fileSize) } totalRead, err := fh.f.reader.ReadAt(buff, offset) @@ -269,7 +269,7 @@ func (fh *FileHandle) doFlush(ctx context.Context, header fuse.Header) error { manifestChunks, nonManifestChunks := filer.SeparateManifestChunks(fh.f.entry.Chunks) - chunks, _ := filer.CompactFileChunks(filer.LookupFn(fh.f.wfs), nonManifestChunks) + chunks, _ := filer.CompactFileChunks(fh.f.wfs.LookupFn(), nonManifestChunks) chunks, manifestErr := filer.MaybeManifestize(fh.f.wfs.saveDataAsChunk(fh.f.fullpath()), chunks) if manifestErr != nil { // not good, but should be ok diff --git a/weed/filesys/wfs.go b/weed/filesys/wfs.go index cd14e8032..236ecdacb 100644 --- a/weed/filesys/wfs.go +++ b/weed/filesys/wfs.go @@ -3,6 +3,8 @@ package filesys import ( "context" "fmt" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/wdclient" "math" "os" "path" @@ -24,6 +26,7 @@ import ( ) type Option struct { + FilerAddress string FilerGrpcAddress string GrpcDialOption grpc.DialOption FilerMountRootPath string @@ -237,3 +240,13 @@ func (wfs *WFS) mapPbIdFromLocalToFiler(entry *filer_pb.Entry) { } entry.Attributes.Uid, entry.Attributes.Gid = wfs.option.UidGidMapper.LocalToFiler(entry.Attributes.Uid, entry.Attributes.Gid) } + +func (wfs *WFS) LookupFn() wdclient.LookupFileIdFunctionType { + if wfs.option.OutsideContainerClusterMode { + return func(fileId string) (targetUrls []string, err error) { + return []string{"http://" + wfs.option.FilerAddress + "/?proxyChunkId=" + fileId}, nil + } + } + return filer.LookupFn(wfs) + +} diff --git a/weed/filesys/wfs_deletion.go b/weed/filesys/wfs_deletion.go deleted file mode 100644 index a245b6795..000000000 --- a/weed/filesys/wfs_deletion.go +++ /dev/null @@ -1,84 +0,0 @@ -package filesys - -import ( - "context" - - "google.golang.org/grpc" - - "github.com/chrislusf/seaweedfs/weed/filer" - "github.com/chrislusf/seaweedfs/weed/glog" - "github.com/chrislusf/seaweedfs/weed/operation" - "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" -) - -func (wfs *WFS) deleteFileChunks(chunks []*filer_pb.FileChunk) { - if len(chunks) == 0 { - return - } - - var fileIds []string - for _, chunk := range chunks { - if !chunk.IsChunkManifest { - fileIds = append(fileIds, chunk.GetFileIdString()) - continue - } - dataChunks, manifestResolveErr := filer.ResolveOneChunkManifest(filer.LookupFn(wfs), chunk) - if manifestResolveErr != nil { - glog.V(0).Infof("failed to resolve manifest %s: %v", chunk.FileId, manifestResolveErr) - } - for _, dChunk := range dataChunks { - fileIds = append(fileIds, dChunk.GetFileIdString()) - } - fileIds = append(fileIds, chunk.GetFileIdString()) - } - - wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { - wfs.deleteFileIds(wfs.option.GrpcDialOption, client, fileIds) - return nil - }) -} - -func (wfs *WFS) deleteFileIds(grpcDialOption grpc.DialOption, client filer_pb.SeaweedFilerClient, fileIds []string) error { - - var vids []string - for _, fileId := range fileIds { - vids = append(vids, filer.VolumeId(fileId)) - } - - lookupFunc := func(vids []string) (map[string]operation.LookupResult, error) { - - m := make(map[string]operation.LookupResult) - - glog.V(4).Infof("deleteFileIds lookup volume id locations: %v", vids) - resp, err := client.LookupVolume(context.Background(), &filer_pb.LookupVolumeRequest{ - VolumeIds: vids, - }) - if err != nil { - return m, err - } - - for _, vid := range vids { - lr := operation.LookupResult{ - VolumeId: vid, - Locations: nil, - } - locations, found := resp.LocationsMap[vid] - if !found { - continue - } - for _, loc := range locations.Locations { - lr.Locations = append(lr.Locations, operation.Location{ - Url: wfs.AdjustedUrl(loc), - PublicUrl: loc.PublicUrl, - }) - } - m[vid] = lr - } - - return m, err - } - - _, err := operation.DeleteFilesWithLookupVolumeId(grpcDialOption, fileIds, lookupFunc) - - return err -} diff --git a/weed/filesys/wfs_filer_client.go b/weed/filesys/wfs_filer_client.go index ef4213af1..e0d352a7b 100644 --- a/weed/filesys/wfs_filer_client.go +++ b/weed/filesys/wfs_filer_client.go @@ -25,10 +25,3 @@ func (wfs *WFS) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) error) erro return err } - -func (wfs *WFS) AdjustedUrl(location *filer_pb.Location) string { - if wfs.option.OutsideContainerClusterMode { - return location.PublicUrl - } - return location.Url -} diff --git a/weed/filesys/wfs_write.go b/weed/filesys/wfs_write.go index 83e40e7f5..dfe6e57a6 100644 --- a/weed/filesys/wfs_write.go +++ b/weed/filesys/wfs_write.go @@ -44,7 +44,7 @@ func (wfs *WFS) saveDataAsChunk(fullPath util.FullPath) filer.SaveDataAsChunkFun Url: resp.Url, PublicUrl: resp.PublicUrl, } - host = wfs.AdjustedUrl(loc) + host = loc.Url collection, replication = resp.Collection, resp.Replication return nil @@ -53,6 +53,9 @@ func (wfs *WFS) saveDataAsChunk(fullPath util.FullPath) filer.SaveDataAsChunkFun } fileUrl := fmt.Sprintf("http://%s/%s", host, fileId) + if wfs.option.OutsideContainerClusterMode { + fileUrl = fmt.Sprintf("http://%s/?proxyChunkId=%s", wfs.option.FilerAddress, fileId) + } uploadResult, err, data := operation.Upload(fileUrl, filename, wfs.option.Cipher, reader, false, "", nil, auth) if err != nil { glog.V(0).Infof("upload data %v to %s: %v", filename, fileUrl, err) diff --git a/weed/messaging/broker/broker_append.go b/weed/messaging/broker/broker_append.go index 8e5b56fd0..67c9bcb79 100644 --- a/weed/messaging/broker/broker_append.go +++ b/weed/messaging/broker/broker_append.go @@ -107,7 +107,3 @@ func (broker *MessageBroker) WithFilerClient(fn func(filer_pb.SeaweedFilerClient return } - -func (broker *MessageBroker) AdjustedUrl(location *filer_pb.Location) string { - return location.Url -} diff --git a/weed/operation/delete_content.go b/weed/operation/delete_content.go index 9868a411d..65baaddf2 100644 --- a/weed/operation/delete_content.go +++ b/weed/operation/delete_content.go @@ -4,12 +4,11 @@ import ( "context" "errors" "fmt" + "google.golang.org/grpc" "net/http" "strings" "sync" - "google.golang.org/grpc" - "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb" ) diff --git a/weed/pb/filer_pb/filer_client.go b/weed/pb/filer_pb/filer_client.go index 079fbd671..7198de95c 100644 --- a/weed/pb/filer_pb/filer_client.go +++ b/weed/pb/filer_pb/filer_client.go @@ -20,7 +20,6 @@ var ( type FilerClient interface { WithFilerClient(fn func(SeaweedFilerClient) error) error - AdjustedUrl(location *Location) string } func GetEntry(filerClient FilerClient, fullFilePath util.FullPath) (entry *Entry, err error) { diff --git a/weed/replication/sink/filersink/fetch_write.go b/weed/replication/sink/filersink/fetch_write.go index b062adcfe..544b84995 100644 --- a/weed/replication/sink/filersink/fetch_write.go +++ b/weed/replication/sink/filersink/fetch_write.go @@ -128,6 +128,3 @@ func (fs *FilerSink) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) error) }, fs.grpcAddress, fs.grpcDialOption) } -func (fs *FilerSink) AdjustedUrl(location *filer_pb.Location) string { - return location.Url -} diff --git a/weed/replication/source/filer_source.go b/weed/replication/source/filer_source.go index 3982360b0..eff1da8dc 100644 --- a/weed/replication/source/filer_source.go +++ b/weed/replication/source/filer_source.go @@ -124,10 +124,6 @@ func (fs *FilerSource) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) erro } -func (fs *FilerSource) AdjustedUrl(location *filer_pb.Location) string { - return location.Url -} - func volumeId(fileId string) string { lastCommaIndex := strings.LastIndex(fileId, ",") if lastCommaIndex > 0 { diff --git a/weed/s3api/s3api_handlers.go b/weed/s3api/s3api_handlers.go index 6935c75bd..57b26f3dd 100644 --- a/weed/s3api/s3api_handlers.go +++ b/weed/s3api/s3api_handlers.go @@ -50,9 +50,6 @@ func (s3a *S3ApiServer) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) err }, s3a.option.FilerGrpcAddress, s3a.option.GrpcDialOption) } -func (s3a *S3ApiServer) AdjustedUrl(location *filer_pb.Location) string { - return location.Url -} // If none of the http routes match respond with MethodNotAllowed func notFoundHandler(w http.ResponseWriter, r *http.Request) { diff --git a/weed/server/webdav_server.go b/weed/server/webdav_server.go index 2b238e534..5bd92a136 100644 --- a/weed/server/webdav_server.go +++ b/weed/server/webdav_server.go @@ -123,9 +123,6 @@ func (fs *WebDavFileSystem) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) }, fs.option.FilerGrpcAddress, fs.option.GrpcDialOption) } -func (fs *WebDavFileSystem) AdjustedUrl(location *filer_pb.Location) string { - return location.Url -} func clearName(name string) (string, error) { slashed := strings.HasSuffix(name, "/") @@ -523,7 +520,7 @@ func (f *WebDavFile) Read(p []byte) (readSize int, err error) { } if f.reader == nil { chunkViews := filer.ViewFromVisibleIntervals(f.entryViewCache, 0, math.MaxInt64) - f.reader = filer.NewChunkReaderAtFromClient(f.fs, chunkViews, f.fs.chunkCache, fileSize) + f.reader = filer.NewChunkReaderAtFromClient(filer.LookupFn(f.fs), chunkViews, f.fs.chunkCache, fileSize) } readSize, err = f.reader.ReadAt(p, f.off) diff --git a/weed/shell/commands.go b/weed/shell/commands.go index 0e285214b..6e1348ca5 100644 --- a/weed/shell/commands.go +++ b/weed/shell/commands.go @@ -102,10 +102,6 @@ func (ce *CommandEnv) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) error } -func (ce *CommandEnv) AdjustedUrl(location *filer_pb.Location) string { - return location.Url -} - func parseFilerUrl(entryPath string) (filerServer string, filerPort int64, path string, err error) { if strings.HasPrefix(entryPath, "http") { var u *url.URL From 62ebb917e016b0e0ad87f79540328fe209dd5707 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 25 Jan 2021 09:10:20 -0800 Subject: [PATCH 183/260] mount: avoid the need to specify different cache directory for different mount fix https://github.com/seaweedfs/seaweedfs-csi-driver/issues/11 --- weed/command/mount_std.go | 1 + weed/filesys/wfs.go | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/weed/command/mount_std.go b/weed/command/mount_std.go index a95ecd567..23f224cc2 100644 --- a/weed/command/mount_std.go +++ b/weed/command/mount_std.go @@ -169,6 +169,7 @@ func RunMount(option *MountOptions, umask os.FileMode) bool { } seaweedFileSystem := filesys.NewSeaweedFileSystem(&filesys.Option{ + MountDirectory: dir, FilerAddress: filer, FilerGrpcAddress: filerGrpcAddress, GrpcDialOption: grpcDialOption, diff --git a/weed/filesys/wfs.go b/weed/filesys/wfs.go index 236ecdacb..6cfadcc18 100644 --- a/weed/filesys/wfs.go +++ b/weed/filesys/wfs.go @@ -26,6 +26,7 @@ import ( ) type Option struct { + MountDirectory string FilerAddress string FilerGrpcAddress string GrpcDialOption grpc.DialOption @@ -92,7 +93,7 @@ func NewSeaweedFileSystem(option *Option) *WFS { }, signature: util.RandomInt32(), } - cacheUniqueId := util.Md5String([]byte(option.FilerGrpcAddress + option.FilerMountRootPath + util.Version()))[0:4] + cacheUniqueId := util.Md5String([]byte(option.MountDirectory + option.FilerGrpcAddress + option.FilerMountRootPath + util.Version()))[0:8] cacheDir := path.Join(option.CacheDir, cacheUniqueId) if option.CacheSizeMB > 0 { os.MkdirAll(cacheDir, os.FileMode(0777)&^option.Umask) From cc1f3907ffec17965ce69c3ab7ac700913e939d0 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 25 Jan 2021 10:14:42 -0800 Subject: [PATCH 184/260] filer.sync: default filerProxy to true --- weed/command/filer_sync.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/weed/command/filer_sync.go b/weed/command/filer_sync.go index 587174059..f4411b756 100644 --- a/weed/command/filer_sync.go +++ b/weed/command/filer_sync.go @@ -56,8 +56,8 @@ func init() { syncOptions.bCollection = cmdFilerSynchronize.Flag.String("b.collection", "", "collection on filer B") syncOptions.aTtlSec = cmdFilerSynchronize.Flag.Int("a.ttlSec", 0, "ttl in seconds on filer A") syncOptions.bTtlSec = cmdFilerSynchronize.Flag.Int("b.ttlSec", 0, "ttl in seconds on filer B") - syncOptions.aProxyByFiler = cmdFilerSynchronize.Flag.Bool("a.filerProxy", true, "read and write file chunks by filer A instead of volume servers") - syncOptions.bProxyByFiler = cmdFilerSynchronize.Flag.Bool("b.filerProxy", true, "read and write file chunks by filer B instead of volume servers") + syncOptions.aProxyByFiler = cmdFilerSynchronize.Flag.Bool("a.filerProxy", false, "read and write file chunks by filer A instead of volume servers") + syncOptions.bProxyByFiler = cmdFilerSynchronize.Flag.Bool("b.filerProxy", false, "read and write file chunks by filer B instead of volume servers") syncOptions.aDebug = cmdFilerSynchronize.Flag.Bool("a.debug", false, "debug mode to print out filer A received files") syncOptions.bDebug = cmdFilerSynchronize.Flag.Bool("b.debug", false, "debug mode to print out filer B received files") syncCpuProfile = cmdFilerSynchronize.Flag.String("cpuprofile", "", "cpu profile output file") From ff35520a1a3a51476b5f864ee3f3c20d3cd7a5a8 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 25 Jan 2021 10:34:51 -0800 Subject: [PATCH 185/260] Update README.md --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index f55f35826..b4e8dc628 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,9 @@ [![GoDoc](https://godoc.org/github.com/chrislusf/seaweedfs/weed?status.svg)](https://godoc.org/github.com/chrislusf/seaweedfs/weed) [![Wiki](https://img.shields.io/badge/docs-wiki-blue.svg)](https://github.com/chrislusf/seaweedfs/wiki) [![Docker Pulls](https://img.shields.io/docker/pulls/chrislusf/seaweedfs.svg?maxAge=4800)](https://hub.docker.com/r/chrislusf/seaweedfs/) +[![Slack](https://img.shields.io/badge/slack-purple)](https://join.slack.com/t/seaweedfs/shared_invite/enQtMzI4MTMwMjU2MzA3LTEyYzZmZWYzOGQ3MDJlZWMzYmI0OTE4OTJiZjJjODBmMzUxNmYwODg0YjY3MTNlMjBmZDQ1NzQ5NDJhZWI2ZmY) +[![Twitter](https://img.shields.io/twitter/follow/seaweedfs.svg?style=social&label=Follow)](https://twitter.com/intent/follow?screen_name=seaweedfs) + ![SeaweedFS Logo](https://raw.githubusercontent.com/chrislusf/seaweedfs/master/note/seaweedfs.png) From c41961d5cccf4a4b49e5968e0da283c6fb1f47c3 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 25 Jan 2021 10:44:16 -0800 Subject: [PATCH 186/260] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b4e8dc628..58885eb61 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,12 @@ # SeaweedFS +[![Slack](https://img.shields.io/badge/slack-purple)](https://join.slack.com/t/seaweedfs/shared_invite/enQtMzI4MTMwMjU2MzA3LTEyYzZmZWYzOGQ3MDJlZWMzYmI0OTE4OTJiZjJjODBmMzUxNmYwODg0YjY3MTNlMjBmZDQ1NzQ5NDJhZWI2ZmY) +[![Twitter](https://img.shields.io/twitter/follow/seaweedfs.svg?style=social&label=Follow)](https://twitter.com/intent/follow?screen_name=seaweedfs) [![Build Status](https://travis-ci.org/chrislusf/seaweedfs.svg?branch=master)](https://travis-ci.org/chrislusf/seaweedfs) [![GoDoc](https://godoc.org/github.com/chrislusf/seaweedfs/weed?status.svg)](https://godoc.org/github.com/chrislusf/seaweedfs/weed) [![Wiki](https://img.shields.io/badge/docs-wiki-blue.svg)](https://github.com/chrislusf/seaweedfs/wiki) [![Docker Pulls](https://img.shields.io/docker/pulls/chrislusf/seaweedfs.svg?maxAge=4800)](https://hub.docker.com/r/chrislusf/seaweedfs/) -[![Slack](https://img.shields.io/badge/slack-purple)](https://join.slack.com/t/seaweedfs/shared_invite/enQtMzI4MTMwMjU2MzA3LTEyYzZmZWYzOGQ3MDJlZWMzYmI0OTE4OTJiZjJjODBmMzUxNmYwODg0YjY3MTNlMjBmZDQ1NzQ5NDJhZWI2ZmY) -[![Twitter](https://img.shields.io/twitter/follow/seaweedfs.svg?style=social&label=Follow)](https://twitter.com/intent/follow?screen_name=seaweedfs) ![SeaweedFS Logo](https://raw.githubusercontent.com/chrislusf/seaweedfs/master/note/seaweedfs.png) From 3a1d3d3413c5fc6a49c138ed5024f2b907f63d37 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 26 Jan 2021 02:50:50 -0800 Subject: [PATCH 187/260] mount: properly invalidate kernel node cache entry fix https://github.com/chrislusf/seaweedfs/issues/1752 --- go.mod | 2 +- go.sum | 2 ++ weed/command/mount_std.go | 4 +++- weed/filesys/dir.go | 16 +++++++--------- weed/filesys/file.go | 7 ++----- weed/filesys/file_darwin.go | 8 -------- weed/filesys/file_other.go | 7 ------- weed/filesys/wfs.go | 14 ++++++++++++++ 8 files changed, 29 insertions(+), 31 deletions(-) delete mode 100644 weed/filesys/file_darwin.go delete mode 100644 weed/filesys/file_other.go diff --git a/go.mod b/go.mod index 44950d25c..39acd0e9b 100644 --- a/go.mod +++ b/go.mod @@ -56,7 +56,7 @@ require ( github.com/prometheus/client_golang v1.3.0 github.com/rakyll/statik v0.1.7 github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 // indirect - github.com/seaweedfs/fuse v1.0.9 + github.com/seaweedfs/fuse v1.1.0 github.com/seaweedfs/goexif v1.0.2 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e github.com/spaolacci/murmur3 v1.1.0 // indirect diff --git a/go.sum b/go.sum index f0b86cb62..eb16fac4c 100644 --- a/go.sum +++ b/go.sum @@ -577,6 +577,8 @@ github.com/seaweedfs/fuse v1.0.8 h1:HBPJTC77OlxwSd2JiTwvLPn8bWTElqQp3xs9vf3C15s= github.com/seaweedfs/fuse v1.0.8/go.mod h1:W7ubwr1l7KQsMeUpxFFOFOSxUL/ucTRMAlVYs4xdfQ8= github.com/seaweedfs/fuse v1.0.9 h1:3JZoGsW7cmmrd/U5KYcIGR2+EqyBvCYCoCpEdZAz/Dc= github.com/seaweedfs/fuse v1.0.9/go.mod h1:+PP6WlkrRUG6KPE+Th2EX5To/PjHaFsvqg/UgQ39aj8= +github.com/seaweedfs/fuse v1.1.0 h1:cL1qPHFNtFv0UuJTLjKKgWDzfJ4iZzTa4Y7ipc2acGw= +github.com/seaweedfs/fuse v1.1.0/go.mod h1:+PP6WlkrRUG6KPE+Th2EX5To/PjHaFsvqg/UgQ39aj8= github.com/seaweedfs/goexif v1.0.2 h1:p+rTXYdQ2mgxd+1JaTrQ9N8DvYuw9UH9xgYmJ+Bb29E= github.com/seaweedfs/goexif v1.0.2/go.mod h1:MrKs5LK0HXdffrdCZrW3OIMegL2xXpC6ThLyXMyjdrk= github.com/secsy/goftp v0.0.0-20190720192957-f31499d7c79a h1:C6IhVTxNkhlb0tlCB6JfHOUv1f0xHPK7V8X4HlJZEJw= diff --git a/weed/command/mount_std.go b/weed/command/mount_std.go index 23f224cc2..eaa66519e 100644 --- a/weed/command/mount_std.go +++ b/weed/command/mount_std.go @@ -208,7 +208,9 @@ func RunMount(option *MountOptions, umask os.FileMode) bool { }) glog.V(0).Infof("mounted %s%s to %s", filer, mountRoot, dir) - err = fs.Serve(c, seaweedFileSystem) + server := fs.New(c, nil) + seaweedFileSystem.Server = server + err = server.Serve(seaweedFileSystem) // check if the mount process has an error to report <-c.Ready diff --git a/weed/filesys/dir.go b/weed/filesys/dir.go index 3d0a00a8b..e3d0057db 100644 --- a/weed/filesys/dir.go +++ b/weed/filesys/dir.go @@ -58,7 +58,7 @@ func (dir *Dir) Attr(ctx context.Context, attr *fuse.Attr) error { return err } - attr.Inode = util.FullPath(dir.FullPath()).AsInode() + // attr.Inode = util.FullPath(dir.FullPath()).AsInode() attr.Mode = os.FileMode(dir.entry.Attributes.FileMode) | os.ModeDir attr.Mtime = time.Unix(dir.entry.Attributes.Mtime, 0) attr.Crtime = time.Unix(dir.entry.Attributes.Crtime, 0) @@ -82,8 +82,8 @@ func (dir *Dir) Getxattr(ctx context.Context, req *fuse.GetxattrRequest, resp *f } func (dir *Dir) setRootDirAttributes(attr *fuse.Attr) { - attr.Inode = 1 // filer2.FullPath(dir.Path).AsInode() - attr.Valid = time.Hour + // attr.Inode = 1 // filer2.FullPath(dir.Path).AsInode() + attr.Valid = time.Second attr.Uid = dir.wfs.option.MountUid attr.Gid = dir.wfs.option.MountGid attr.Mode = dir.wfs.option.MountMode @@ -91,7 +91,7 @@ func (dir *Dir) setRootDirAttributes(attr *fuse.Attr) { attr.Ctime = dir.wfs.option.MountCtime attr.Mtime = dir.wfs.option.MountMtime attr.Atime = dir.wfs.option.MountMtime - attr.BlockSize = 1024 * 1024 + attr.BlockSize = blockSize } func (dir *Dir) Fsync(ctx context.Context, req *fuse.FsyncRequest) error { @@ -285,7 +285,7 @@ func (dir *Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse. } // resp.EntryValid = time.Second - resp.Attr.Inode = fullFilePath.AsInode() + // resp.Attr.Inode = fullFilePath.AsInode() resp.Attr.Valid = time.Second resp.Attr.Mtime = time.Unix(entry.Attributes.Mtime, 0) resp.Attr.Crtime = time.Unix(entry.Attributes.Crtime, 0) @@ -308,13 +308,11 @@ func (dir *Dir) ReadDirAll(ctx context.Context) (ret []fuse.Dirent, err error) { glog.V(4).Infof("dir ReadDirAll %s", dir.FullPath()) processEachEntryFn := func(entry *filer_pb.Entry, isLast bool) error { - fullpath := util.NewFullPath(dir.FullPath(), entry.Name) - inode := fullpath.AsInode() if entry.IsDirectory { - dirent := fuse.Dirent{Inode: inode, Name: entry.Name, Type: fuse.DT_Dir} + dirent := fuse.Dirent{Name: entry.Name, Type: fuse.DT_Dir} ret = append(ret, dirent) } else { - dirent := fuse.Dirent{Inode: inode, Name: entry.Name, Type: findFileType(uint16(entry.Attributes.FileMode))} + dirent := fuse.Dirent{Name: entry.Name, Type: findFileType(uint16(entry.Attributes.FileMode))} ret = append(ret, dirent) } return nil diff --git a/weed/filesys/file.go b/weed/filesys/file.go index a8d6dac29..a210c5152 100644 --- a/weed/filesys/file.go +++ b/weed/filesys/file.go @@ -45,7 +45,7 @@ func (file *File) fullpath() util.FullPath { func (file *File) Attr(ctx context.Context, attr *fuse.Attr) (err error) { - glog.V(4).Infof("file Attr %s, open:%v, existing attr: %+v", file.fullpath(), file.isOpen, attr) + glog.V(4).Infof("file Attr %s, open:%v existing:%v", file.fullpath(), file.isOpen, attr) entry := file.entry if file.isOpen <= 0 || entry == nil { @@ -54,7 +54,7 @@ func (file *File) Attr(ctx context.Context, attr *fuse.Attr) (err error) { } } - attr.Inode = file.fullpath().AsInode() + // attr.Inode = file.fullpath().AsInode() attr.Valid = time.Second attr.Mode = os.FileMode(entry.Attributes.FileMode) attr.Size = filer.FileSize(entry) @@ -91,9 +91,6 @@ func (file *File) Getxattr(ctx context.Context, req *fuse.GetxattrRequest, resp func (file *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenResponse) (fs.Handle, error) { glog.V(4).Infof("file %v open %+v", file.fullpath(), req) - if USE_DIRECT_IO { - resp.Flags |= fuse.OpenDirectIO - } handle := file.wfs.AcquireHandle(file, req.Uid, req.Gid) diff --git a/weed/filesys/file_darwin.go b/weed/filesys/file_darwin.go deleted file mode 100644 index b8806cff7..000000000 --- a/weed/filesys/file_darwin.go +++ /dev/null @@ -1,8 +0,0 @@ -//+build darwin - -package filesys - -const ( - USE_DIRECT_IO = false -) - diff --git a/weed/filesys/file_other.go b/weed/filesys/file_other.go deleted file mode 100644 index 8a8693511..000000000 --- a/weed/filesys/file_other.go +++ /dev/null @@ -1,7 +0,0 @@ -//+build !darwin - -package filesys - -const ( - USE_DIRECT_IO = true -) diff --git a/weed/filesys/wfs.go b/weed/filesys/wfs.go index 6cfadcc18..108e23c85 100644 --- a/weed/filesys/wfs.go +++ b/weed/filesys/wfs.go @@ -76,6 +76,7 @@ type WFS struct { // throttle writers concurrentWriters *util.LimitedConcurrentExecutor + Server *fs.Server } type statsCache struct { filer_pb.StatisticsResponse @@ -104,9 +105,22 @@ func NewSeaweedFileSystem(option *Option) *WFS { fsNode := wfs.fsNodeCache.GetFsNode(filePath) if fsNode != nil { if file, ok := fsNode.(*File); ok { + if err := wfs.Server.InvalidateNodeData(file); err != nil { + glog.V(4).Infof("InvalidateNodeData %s : %v", filePath, err) + } file.clearEntry() } } + dir, name := filePath.DirAndName() + parent := wfs.root + if dir != "/" { + parent = wfs.fsNodeCache.GetFsNode(util.FullPath(dir)) + } + if parent != nil { + if err := wfs.Server.InvalidateEntry(parent, name); err != nil { + glog.V(4).Infof("InvalidateEntry %s : %v", filePath, err) + } + } }) startTime := time.Now() go meta_cache.SubscribeMetaEvents(wfs.metaCache, wfs.signature, wfs, wfs.option.FilerMountRootPath, startTime.UnixNano()) From 612b7975a12da0ff7d84781b1bea338a5f4814c6 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev Date: Tue, 26 Jan 2021 22:50:25 +0500 Subject: [PATCH 188/260] replication to local disk storage --- docker/Makefile | 3 + docker/local-replicate-compose.yml | 59 ++++++++++ docker/notification.toml | 16 +++ docker/replication.toml | 11 ++ weed/command/filer_replication.go | 1 + weed/command/scaffold.go | 4 + weed/replication/sink/localsink/local_sink.go | 101 ++++++++++++++++++ 7 files changed, 195 insertions(+) create mode 100644 docker/local-replicate-compose.yml create mode 100644 docker/notification.toml create mode 100644 docker/replication.toml create mode 100644 weed/replication/sink/localsink/local_sink.go diff --git a/docker/Makefile b/docker/Makefile index fe278f9b4..5949842f1 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -21,6 +21,9 @@ k8s: build dev_registry: build docker-compose -f local-registry-compose.yml -p seaweedfs up +dev_replicate: build + docker-compose -f local-replicate-compose.yml -p seaweedfs up + cluster: build docker-compose -f local-cluster-compose.yml -p seaweedfs up diff --git a/docker/local-replicate-compose.yml b/docker/local-replicate-compose.yml new file mode 100644 index 000000000..a8e0f808e --- /dev/null +++ b/docker/local-replicate-compose.yml @@ -0,0 +1,59 @@ +version: '2' + +services: + master: + image: chrislusf/seaweedfs:local + ports: + - 9333:9333 + - 19333:19333 + command: "master -ip=master" + volume: + image: chrislusf/seaweedfs:local + ports: + - 8080:8080 + - 18080:18080 + command: "volume -mserver=master:9333 -port=8080 -ip=volume -preStopSeconds=1" + depends_on: + - master + filer: + image: chrislusf/seaweedfs:local + ports: + - 8888:8888 + - 18888:18888 + command: '-v=9 filer -master="master:9333"' + restart: on-failure + volumes: + - ./notification.toml:/etc/seaweedfs/notification.toml + depends_on: + - master + - volume + - rabbitmq + - replicate + environment: + RABBIT_SERVER_URL: "amqp://guest:guest@rabbitmq:5672/" + replicate: + image: chrislusf/seaweedfs:local + command: '-v=9 filer.replicate' + restart: on-failure + volumes: + - ./notification.toml:/etc/seaweedfs/notification.toml + - ./replication.toml:/etc/seaweedfs/replication.toml + depends_on: + - rabbitmq + environment: + RABBIT_SERVER_URL: "amqp://guest:guest@rabbitmq:5672/" + s3: + image: chrislusf/seaweedfs:local + ports: + - 8333:8333 + command: 's3 -filer="filer:8888"' + depends_on: + - master + - volume + - filer + rabbitmq: + image: rabbitmq:3.8.10-management-alpine + ports: + - 5672:5672 + - 15671:15671 + - 15672:15672 \ No newline at end of file diff --git a/docker/notification.toml b/docker/notification.toml new file mode 100644 index 000000000..4ed76825e --- /dev/null +++ b/docker/notification.toml @@ -0,0 +1,16 @@ +[notification.log] +# this is only for debugging perpose and does not work with "weed filer.replicate" +enabled = false + + +[notification.gocdk_pub_sub] +# The Go Cloud Development Kit (https://gocloud.dev). +# PubSub API (https://godoc.org/gocloud.dev/pubsub). +# Supports AWS SNS/SQS, Azure Service Bus, Google PubSub, NATS and RabbitMQ. +enabled = true +# This URL will Dial the RabbitMQ server at the URL in the environment +# variable RABBIT_SERVER_URL and open the exchange "myexchange". +# The exchange must have already been created by some other means, like +# the RabbitMQ management plugin. +topic_url = "rabbit://swexchange" +sub_url = "rabbit://swqueue" \ No newline at end of file diff --git a/docker/replication.toml b/docker/replication.toml new file mode 100644 index 000000000..84c0bec28 --- /dev/null +++ b/docker/replication.toml @@ -0,0 +1,11 @@ +[source.filer] +enabled = true +grpcAddress = "filer:18888" +# all files under this directory tree are replicated. +# this is not a directory on your hard drive, but on your filer. +# i.e., all files with this "prefix" are sent to notification message queue. +directory = "/buckets" + +[sink.local] +enabled = true +directory = "/data" \ No newline at end of file diff --git a/weed/command/filer_replication.go b/weed/command/filer_replication.go index 4f698e375..b6515e505 100644 --- a/weed/command/filer_replication.go +++ b/weed/command/filer_replication.go @@ -11,6 +11,7 @@ import ( _ "github.com/chrislusf/seaweedfs/weed/replication/sink/b2sink" _ "github.com/chrislusf/seaweedfs/weed/replication/sink/filersink" _ "github.com/chrislusf/seaweedfs/weed/replication/sink/gcssink" + _ "github.com/chrislusf/seaweedfs/weed/replication/sink/localsink" _ "github.com/chrislusf/seaweedfs/weed/replication/sink/s3sink" "github.com/chrislusf/seaweedfs/weed/replication/sub" "github.com/chrislusf/seaweedfs/weed/util" diff --git a/weed/command/scaffold.go b/weed/command/scaffold.go index 8b74274e5..415e0cba3 100644 --- a/weed/command/scaffold.go +++ b/weed/command/scaffold.go @@ -350,6 +350,10 @@ grpcAddress = "localhost:18888" # i.e., all files with this "prefix" are sent to notification message queue. directory = "/buckets" +[sink.local] +enabled = false +directory = "/backup" + [sink.filer] enabled = false grpcAddress = "localhost:18888" diff --git a/weed/replication/sink/localsink/local_sink.go b/weed/replication/sink/localsink/local_sink.go new file mode 100644 index 000000000..76e0384e5 --- /dev/null +++ b/weed/replication/sink/localsink/local_sink.go @@ -0,0 +1,101 @@ +package localsink + +import ( + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/replication/repl_util" + "github.com/chrislusf/seaweedfs/weed/replication/sink" + "github.com/chrislusf/seaweedfs/weed/replication/source" + "github.com/chrislusf/seaweedfs/weed/util" + "io/ioutil" + "os" + "path/filepath" + "strings" +) + +type LocalSink struct { + dir string + filerSource *source.FilerSource +} + +func init() { + sink.Sinks = append(sink.Sinks, &LocalSink{}) +} + +func (localsink *LocalSink) SetSourceFiler(s *source.FilerSource) { + localsink.filerSource = s +} + +func (localsink *LocalSink) GetName() string { + return "local" +} + +func (localsink *LocalSink) isMultiPartEntry(key string) bool { + return strings.HasSuffix(key, ".part") && strings.Contains(key, "/.uploads/") +} + +func (localsink *LocalSink) initialize(dir string) error { + localsink.dir = dir + return nil +} + +func (localsink *LocalSink) Initialize(configuration util.Configuration, prefix string) error { + dir := configuration.GetString(prefix + "directory") + glog.V(4).Infof("sink.local.directory: %v", dir) + return localsink.initialize(dir) +} + +func (localsink *LocalSink) GetSinkToDirectory() string { + return localsink.dir +} + +func (localsink *LocalSink) DeleteEntry(key string, isDirectory, deleteIncludeChunks bool, signatures []int32) error { + if localsink.isMultiPartEntry(key) { + return nil + } + glog.V(4).Infof("Delete Entry key: %s", key) + if err := os.Remove(key); err != nil { + return err + } + return nil +} + +func (localsink *LocalSink) CreateEntry(key string, entry *filer_pb.Entry, signatures []int32) error { + if entry.IsDirectory || localsink.isMultiPartEntry(key) { + return nil + } + glog.V(4).Infof("Create Entry key: %s", key) + + totalSize := filer.FileSize(entry) + chunkViews := filer.ViewFromChunks(localsink.filerSource.LookupFileId, entry.Chunks, 0, int64(totalSize)) + + dir := filepath.Dir(key) + + if _, err := os.Stat(dir); os.IsNotExist(err) { + glog.V(4).Infof("Create Direcotry key: %s", dir) + if err = os.MkdirAll(dir, 0); err != nil { + return err + } + } + + writeFunc := func(data []byte) error { + writeErr := ioutil.WriteFile(key, data, 0) + return writeErr + } + + if err := repl_util.CopyFromChunkViews(chunkViews, localsink.filerSource, writeFunc); err != nil { + return err + } + + return nil +} + +func (localsink *LocalSink) UpdateEntry(key string, oldEntry *filer_pb.Entry, newParentPath string, newEntry *filer_pb.Entry, deleteIncludeChunks bool, signatures []int32) (foundExistingEntry bool, err error) { + if localsink.isMultiPartEntry(key) { + return true, nil + } + glog.V(4).Infof("Update Entry key: %s", key) + // do delete and create + return false, nil +} From ad2a20c8a5827461d07ab2a46932d0c1c89d1b37 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 26 Jan 2021 11:08:44 -0800 Subject: [PATCH 189/260] notification add ack and nack --- weed/command/filer_replication.go | 14 +++++++++++++- weed/replication/sub/notification_aws_sqs.go | 2 +- .../sub/notification_gocdk_pub_sub.go | 17 ++++++++++++++--- .../sub/notification_google_pub_sub.go | 10 ++++++++-- weed/replication/sub/notification_kafka.go | 2 +- weed/replication/sub/notifications.go | 2 +- 6 files changed, 38 insertions(+), 9 deletions(-) diff --git a/weed/command/filer_replication.go b/weed/command/filer_replication.go index 4f698e375..0b6eaf94e 100644 --- a/weed/command/filer_replication.go +++ b/weed/command/filer_replication.go @@ -97,13 +97,19 @@ func runFilerReplicate(cmd *Command, args []string) bool { replicator := replication.NewReplicator(config, "source.filer.", dataSink) for { - key, m, err := notificationInput.ReceiveMessage() + key, m, onSuccessFn, onFailureFn, err := notificationInput.ReceiveMessage() if err != nil { glog.Errorf("receive %s: %+v", key, err) + if onFailureFn != nil { + onFailureFn() + } continue } if key == "" { // long poll received no messages + if onSuccessFn != nil { + onSuccessFn() + } continue } if m.OldEntry != nil && m.NewEntry == nil { @@ -115,8 +121,14 @@ func runFilerReplicate(cmd *Command, args []string) bool { } if err = replicator.Replicate(context.Background(), key, m); err != nil { glog.Errorf("replicate %s: %+v", key, err) + if onFailureFn != nil { + onFailureFn() + } } else { glog.V(1).Infof("replicated %s", key) + if onSuccessFn != nil { + onSuccessFn() + } } } diff --git a/weed/replication/sub/notification_aws_sqs.go b/weed/replication/sub/notification_aws_sqs.go index 1dd386ba7..642834c72 100644 --- a/weed/replication/sub/notification_aws_sqs.go +++ b/weed/replication/sub/notification_aws_sqs.go @@ -68,7 +68,7 @@ func (k *AwsSqsInput) initialize(awsAccessKeyId, awsSecretAccessKey, region, que return nil } -func (k *AwsSqsInput) ReceiveMessage() (key string, message *filer_pb.EventNotification, err error) { +func (k *AwsSqsInput) ReceiveMessage() (key string, message *filer_pb.EventNotification, onSuccessFn func(), onFailureFn func(), err error) { // receive message result, err := k.svc.ReceiveMessage(&sqs.ReceiveMessageInput{ diff --git a/weed/replication/sub/notification_gocdk_pub_sub.go b/weed/replication/sub/notification_gocdk_pub_sub.go index 9726096e5..09a96d5d5 100644 --- a/weed/replication/sub/notification_gocdk_pub_sub.go +++ b/weed/replication/sub/notification_gocdk_pub_sub.go @@ -38,13 +38,24 @@ func (k *GoCDKPubSubInput) Initialize(configuration util.Configuration, prefix s return nil } -func (k *GoCDKPubSubInput) ReceiveMessage() (key string, message *filer_pb.EventNotification, err error) { +func (k *GoCDKPubSubInput) ReceiveMessage() (key string, message *filer_pb.EventNotification, onSuccessFn func(), onFailureFn func(), err error) { msg, err := k.sub.Receive(context.Background()) + if err != nil { + return + } + onSuccessFn = func() { + msg.Ack() + } + onFailureFn = func() { + if msg.Nackable() { + msg.Nack() + } + } key = msg.Metadata["key"] message = &filer_pb.EventNotification{} err = proto.Unmarshal(msg.Body, message) if err != nil { - return "", nil, err + return "", nil, onSuccessFn, onFailureFn, err } - return key, message, nil + return key, message, onSuccessFn, onFailureFn, nil } diff --git a/weed/replication/sub/notification_google_pub_sub.go b/weed/replication/sub/notification_google_pub_sub.go index a950bb42b..f7c767d4a 100644 --- a/weed/replication/sub/notification_google_pub_sub.go +++ b/weed/replication/sub/notification_google_pub_sub.go @@ -85,16 +85,22 @@ func (k *GooglePubSubInput) initialize(google_application_credentials, projectId go k.sub.Receive(ctx, func(ctx context.Context, m *pubsub.Message) { k.messageChan <- m - m.Ack() }) return err } -func (k *GooglePubSubInput) ReceiveMessage() (key string, message *filer_pb.EventNotification, err error) { +func (k *GooglePubSubInput) ReceiveMessage() (key string, message *filer_pb.EventNotification, onSuccessFn func(), onFailureFn func(), err error) { m := <-k.messageChan + onSuccessFn = func() { + m.Ack() + } + onFailureFn = func() { + m.Nack() + } + // process the message key = m.Attributes["key"] message = &filer_pb.EventNotification{} diff --git a/weed/replication/sub/notification_kafka.go b/weed/replication/sub/notification_kafka.go index fa9cfad9b..622a759ea 100644 --- a/weed/replication/sub/notification_kafka.go +++ b/weed/replication/sub/notification_kafka.go @@ -97,7 +97,7 @@ func (k *KafkaInput) initialize(hosts []string, topic string, offsetFile string, return nil } -func (k *KafkaInput) ReceiveMessage() (key string, message *filer_pb.EventNotification, err error) { +func (k *KafkaInput) ReceiveMessage() (key string, message *filer_pb.EventNotification, onSuccessFn func(), onFailureFn func(), err error) { msg := <-k.messageChan diff --git a/weed/replication/sub/notifications.go b/weed/replication/sub/notifications.go index 8a2668f98..d5a910db9 100644 --- a/weed/replication/sub/notifications.go +++ b/weed/replication/sub/notifications.go @@ -10,7 +10,7 @@ type NotificationInput interface { GetName() string // Initialize initializes the file store Initialize(configuration util.Configuration, prefix string) error - ReceiveMessage() (key string, message *filer_pb.EventNotification, err error) + ReceiveMessage() (key string, message *filer_pb.EventNotification, onSuccessFn func(), onFailureFn func(), err error) } var ( From 5a761bb7a4a2d9a9e41a089b5091d674b4ff3297 Mon Sep 17 00:00:00 2001 From: Daniel Santos Date: Tue, 26 Jan 2021 20:42:10 -0700 Subject: [PATCH 190/260] Enforce umask correctly --- weed/command/mount_std.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/weed/command/mount_std.go b/weed/command/mount_std.go index eaa66519e..fe9ba28c4 100644 --- a/weed/command/mount_std.go +++ b/weed/command/mount_std.go @@ -100,7 +100,7 @@ func RunMount(option *MountOptions, umask os.FileMode) bool { uid, gid := uint32(0), uint32(0) mountMode := os.ModeDir | 0777 if err == nil { - mountMode = os.ModeDir | fileInfo.Mode() + mountMode := os.ModeDir | os.FileMode(0777)&^umask uid, gid = util.GetFileUidGid(fileInfo) fmt.Printf("mount point owner uid=%d gid=%d mode=%s\n", uid, gid, fileInfo.Mode()) } else { @@ -208,9 +208,7 @@ func RunMount(option *MountOptions, umask os.FileMode) bool { }) glog.V(0).Infof("mounted %s%s to %s", filer, mountRoot, dir) - server := fs.New(c, nil) - seaweedFileSystem.Server = server - err = server.Serve(seaweedFileSystem) + err = fs.Serve(c, seaweedFileSystem) // check if the mount process has an error to report <-c.Ready From 0aaaa9b470155401be55e8cb5c26d8d9af0cd85d Mon Sep 17 00:00:00 2001 From: Daniel Santos Date: Tue, 26 Jan 2021 21:06:08 -0700 Subject: [PATCH 191/260] Fix var assignment --- weed/command/mount_std.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/command/mount_std.go b/weed/command/mount_std.go index fe9ba28c4..8119edd01 100644 --- a/weed/command/mount_std.go +++ b/weed/command/mount_std.go @@ -100,7 +100,7 @@ func RunMount(option *MountOptions, umask os.FileMode) bool { uid, gid := uint32(0), uint32(0) mountMode := os.ModeDir | 0777 if err == nil { - mountMode := os.ModeDir | os.FileMode(0777)&^umask + mountMode = os.ModeDir | os.FileMode(0777)&^umask uid, gid = util.GetFileUidGid(fileInfo) fmt.Printf("mount point owner uid=%d gid=%d mode=%s\n", uid, gid, fileInfo.Mode()) } else { From 0c64d60fdb29d48e2ab2805a33904191c78349b5 Mon Sep 17 00:00:00 2001 From: Daniel Santos Date: Tue, 26 Jan 2021 21:45:52 -0700 Subject: [PATCH 192/260] Fix log message with correct mode --- weed/command/mount_std.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/command/mount_std.go b/weed/command/mount_std.go index 8119edd01..5df005202 100644 --- a/weed/command/mount_std.go +++ b/weed/command/mount_std.go @@ -102,7 +102,7 @@ func RunMount(option *MountOptions, umask os.FileMode) bool { if err == nil { mountMode = os.ModeDir | os.FileMode(0777)&^umask uid, gid = util.GetFileUidGid(fileInfo) - fmt.Printf("mount point owner uid=%d gid=%d mode=%s\n", uid, gid, fileInfo.Mode()) + fmt.Printf("mount point owner uid=%d gid=%d mode=%s\n", uid, gid, mountMode) } else { fmt.Printf("can not stat %s\n", dir) return false From 2945a64fd694d53b901f23b850d717494e3d4a80 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 26 Jan 2021 22:03:22 -0800 Subject: [PATCH 193/260] Update issue templates --- .github/ISSUE_TEMPLATE/bug_report.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 53423f4b7..edb95fac1 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -9,6 +9,7 @@ assignees: '' Sponsors SeaweedFS via Patreon https://www.patreon.com/seaweedfs Report issues here. Ask questions here https://stackoverflow.com/questions/tagged/seaweedfs +Please ask questions in https://github.com/chrislusf/seaweedfs/discussions example of a good issue report: https://github.com/chrislusf/seaweedfs/issues/1005 From b3f66199db4b6a9da48f091a33e87470877d6d73 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 26 Jan 2021 22:30:37 -0800 Subject: [PATCH 194/260] shell: volume.fix.replication adds collectionPattern with wildcard characters fix https://github.com/chrislusf/seaweedfs/issues/1758 --- weed/shell/command_volume_fix_replication.go | 30 ++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/weed/shell/command_volume_fix_replication.go b/weed/shell/command_volume_fix_replication.go index 9b9abd8eb..c737133f5 100644 --- a/weed/shell/command_volume_fix_replication.go +++ b/weed/shell/command_volume_fix_replication.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/chrislusf/seaweedfs/weed/storage/needle" "io" + "path/filepath" "sort" "github.com/chrislusf/seaweedfs/weed/operation" @@ -19,6 +20,7 @@ func init() { } type commandVolumeFixReplication struct { + collectionPattern *string } func (c *commandVolumeFixReplication) Name() string { @@ -33,8 +35,9 @@ func (c *commandVolumeFixReplication) Help() string { This command also finds all under-replicated volumes, and finds volume servers with free slots. If the free slots satisfy the replication requirement, the volume content is copied over and mounted. - volume.fix.replication -n # do not take action - volume.fix.replication # actually deleting or copying the volume files and mount the volume + volume.fix.replication -n # do not take action + volume.fix.replication # actually deleting or copying the volume files and mount the volume + volume.fix.replication -collectionPattern=important* # fix any collections with prefix "important" Note: * each time this will only add back one replica for one volume id. If there are multiple replicas @@ -52,6 +55,7 @@ func (c *commandVolumeFixReplication) Do(args []string, commandEnv *CommandEnv, } volFixReplicationCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) + c.collectionPattern = volFixReplicationCommand.String("collectionPattern", "", "match with wildcard characters '*' and '?'") skipChange := volFixReplicationCommand.Bool("n", false, "skip the changes") if err = volFixReplicationCommand.Parse(args); err != nil { return nil @@ -127,6 +131,17 @@ func (c *commandVolumeFixReplication) fixOverReplicatedVolumes(commandEnv *Comma replica := pickOneReplicaToDelete(replicas, replicaPlacement) + // check collection name pattern + if *c.collectionPattern != "" { + matched, err := filepath.Match(*c.collectionPattern, replica.info.Collection) + if err != nil { + return fmt.Errorf("match pattern %s with collection %s: %v", *c.collectionPattern, replica.info.Collection, err) + } + if !matched { + break + } + } + fmt.Fprintf(writer, "deleting volume %d from %s ...\n", replica.info.Id, replica.location.dataNode.Id) if !takeAction { @@ -150,6 +165,17 @@ func (c *commandVolumeFixReplication) fixUnderReplicatedVolumes(commandEnv *Comm for _, dst := range allLocations { // check whether data nodes satisfy the constraints if dst.dataNode.FreeVolumeCount > 0 && satisfyReplicaPlacement(replicaPlacement, replicas, dst) { + // check collection name pattern + if *c.collectionPattern != "" { + matched, err := filepath.Match(*c.collectionPattern, replica.info.Collection) + if err != nil { + return fmt.Errorf("match pattern %s with collection %s: %v", *c.collectionPattern, replica.info.Collection, err) + } + if !matched { + break + } + } + // ask the volume server to replicate the volume foundNewLocation = true fmt.Fprintf(writer, "replicating volume %d %s from %s to dataNode %s ...\n", replica.info.Id, replicaPlacement, replica.location.dataNode.Id, dst.dataNode.Id) From 36348114084e7ff583f52ef93a73e1cd8c8f58b5 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev Date: Wed, 27 Jan 2021 12:45:58 +0500 Subject: [PATCH 195/260] replication to todays date directory --- docker/notification.toml | 3 ++- docker/replication.toml | 3 ++- weed/command/scaffold.go | 5 ++++- weed/replication/sink/localsink/local_sink.go | 17 ++++++++++++----- 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/docker/notification.toml b/docker/notification.toml index 4ed76825e..dcd5f2c6f 100644 --- a/docker/notification.toml +++ b/docker/notification.toml @@ -11,6 +11,7 @@ enabled = true # This URL will Dial the RabbitMQ server at the URL in the environment # variable RABBIT_SERVER_URL and open the exchange "myexchange". # The exchange must have already been created by some other means, like -# the RabbitMQ management plugin. +# the RabbitMQ management plugin. Сreate myexchange of type fanout and myqueue then +# create binding myexchange => myqueue topic_url = "rabbit://swexchange" sub_url = "rabbit://swqueue" \ No newline at end of file diff --git a/docker/replication.toml b/docker/replication.toml index 84c0bec28..2cee755a5 100644 --- a/docker/replication.toml +++ b/docker/replication.toml @@ -8,4 +8,5 @@ directory = "/buckets" [sink.local] enabled = true -directory = "/data" \ No newline at end of file +directory = "/data" +todays_date_format = "2006-02-01" \ No newline at end of file diff --git a/weed/command/scaffold.go b/weed/command/scaffold.go index 415e0cba3..3b88e107d 100644 --- a/weed/command/scaffold.go +++ b/weed/command/scaffold.go @@ -329,7 +329,8 @@ enabled = false # This URL will Dial the RabbitMQ server at the URL in the environment # variable RABBIT_SERVER_URL and open the exchange "myexchange". # The exchange must have already been created by some other means, like -# the RabbitMQ management plugin. +# the RabbitMQ management plugin. Сreate myexchange of type fanout and myqueue then +# create binding myexchange => myqueue topic_url = "rabbit://myexchange" sub_url = "rabbit://myqueue" ` @@ -353,6 +354,8 @@ directory = "/buckets" [sink.local] enabled = false directory = "/backup" +# all replicated files are under todays date directory tree +todays_date_format = "" [sink.filer] enabled = false diff --git a/weed/replication/sink/localsink/local_sink.go b/weed/replication/sink/localsink/local_sink.go index 76e0384e5..5ca562ec8 100644 --- a/weed/replication/sink/localsink/local_sink.go +++ b/weed/replication/sink/localsink/local_sink.go @@ -12,11 +12,13 @@ import ( "os" "path/filepath" "strings" + "time" ) type LocalSink struct { - dir string - filerSource *source.FilerSource + dir string + todaysDateFormat string + filerSource *source.FilerSource } func init() { @@ -35,18 +37,23 @@ func (localsink *LocalSink) isMultiPartEntry(key string) bool { return strings.HasSuffix(key, ".part") && strings.Contains(key, "/.uploads/") } -func (localsink *LocalSink) initialize(dir string) error { +func (localsink *LocalSink) initialize(dir string, todaysDateFormat string) error { localsink.dir = dir + localsink.todaysDateFormat = todaysDateFormat return nil } func (localsink *LocalSink) Initialize(configuration util.Configuration, prefix string) error { dir := configuration.GetString(prefix + "directory") + todaysDateFormat := configuration.GetString(prefix + "todays_date_format") glog.V(4).Infof("sink.local.directory: %v", dir) - return localsink.initialize(dir) + return localsink.initialize(dir, todaysDateFormat) } func (localsink *LocalSink) GetSinkToDirectory() string { + if localsink.todaysDateFormat != "" { + return filepath.Join(localsink.dir, time.Now().Format(localsink.todaysDateFormat)) + } return localsink.dir } @@ -56,7 +63,7 @@ func (localsink *LocalSink) DeleteEntry(key string, isDirectory, deleteIncludeCh } glog.V(4).Infof("Delete Entry key: %s", key) if err := os.Remove(key); err != nil { - return err + glog.V(0).Infof("remove entry key %s: %s", key, err) } return nil } From 0488c84beac91a57d9383252b8c58fb6b0ebd84d Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 27 Jan 2021 00:17:52 -0800 Subject: [PATCH 196/260] adjust help message --- weed/command/scaffold.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/weed/command/scaffold.go b/weed/command/scaffold.go index 3b88e107d..1705c6ae4 100644 --- a/weed/command/scaffold.go +++ b/weed/command/scaffold.go @@ -354,8 +354,7 @@ directory = "/buckets" [sink.local] enabled = false directory = "/backup" -# all replicated files are under todays date directory tree -todays_date_format = "" +todays_date_format = "" # set this to 2006-02-01 for incremental backup [sink.filer] enabled = false From 1045ab8b074e1b2d49de566dcfd384fc57b58d2c Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 27 Jan 2021 00:21:59 -0800 Subject: [PATCH 197/260] avoid error message --- weed/shell/command_volume_fix_replication.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/weed/shell/command_volume_fix_replication.go b/weed/shell/command_volume_fix_replication.go index c737133f5..d3fa7d4ac 100644 --- a/weed/shell/command_volume_fix_replication.go +++ b/weed/shell/command_volume_fix_replication.go @@ -162,6 +162,7 @@ func (c *commandVolumeFixReplication) fixUnderReplicatedVolumes(commandEnv *Comm replica := pickOneReplicaToCopyFrom(replicas) replicaPlacement, _ := super_block.NewReplicaPlacementFromByte(byte(replica.info.ReplicaPlacement)) foundNewLocation := false + hasSkippedCollection := false for _, dst := range allLocations { // check whether data nodes satisfy the constraints if dst.dataNode.FreeVolumeCount > 0 && satisfyReplicaPlacement(replicaPlacement, replicas, dst) { @@ -172,6 +173,7 @@ func (c *commandVolumeFixReplication) fixUnderReplicatedVolumes(commandEnv *Comm return fmt.Errorf("match pattern %s with collection %s: %v", *c.collectionPattern, replica.info.Collection, err) } if !matched { + hasSkippedCollection = true break } } @@ -205,7 +207,7 @@ func (c *commandVolumeFixReplication) fixUnderReplicatedVolumes(commandEnv *Comm break } } - if !foundNewLocation { + if !foundNewLocation && !hasSkippedCollection { fmt.Fprintf(writer, "failed to place volume %d replica as %s, existing:%+v\n", replica.info.Id, replicaPlacement, len(replicas)) } From f20ec82a286973a7e3658830637e2e1675729fc5 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 27 Jan 2021 00:48:31 -0800 Subject: [PATCH 198/260] update help message see https://github.com/chrislusf/seaweedfs/issues/1758#issuecomment-768123224 --- weed/shell/command_volume_fix_replication.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/weed/shell/command_volume_fix_replication.go b/weed/shell/command_volume_fix_replication.go index d3fa7d4ac..8ae8850f3 100644 --- a/weed/shell/command_volume_fix_replication.go +++ b/weed/shell/command_volume_fix_replication.go @@ -40,8 +40,8 @@ func (c *commandVolumeFixReplication) Help() string { volume.fix.replication -collectionPattern=important* # fix any collections with prefix "important" Note: - * each time this will only add back one replica for one volume id. If there are multiple replicas - are missing, e.g. multiple volume servers are new, you may need to run this multiple times. + * each time this will only add back one replica for each volume id that is under replicated. + If there are multiple replicas are missing, e.g. replica count is > 2, you may need to run this multiple times. * do not run this too quickly within seconds, since the new volume replica may take a few seconds to register itself to the master. From 6b54ff991249d2ce5dfbe28ffb503fc770d5db6c Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev Date: Wed, 27 Jan 2021 15:01:33 +0500 Subject: [PATCH 199/260] replication to create time date directory --- docker/replication.toml | 5 ++--- weed/command/filer_replication.go | 1 + weed/command/scaffold.go | 7 ++++++- weed/replication/replicator.go | 13 ++++++++++++- .../sink/backupsink/backup_sink.go | 18 ++++++++++++++++++ weed/replication/sink/localsink/local_sink.go | 19 ++++++------------- 6 files changed, 45 insertions(+), 18 deletions(-) create mode 100644 weed/replication/sink/backupsink/backup_sink.go diff --git a/docker/replication.toml b/docker/replication.toml index 2cee755a5..75e34a899 100644 --- a/docker/replication.toml +++ b/docker/replication.toml @@ -6,7 +6,6 @@ grpcAddress = "filer:18888" # i.e., all files with this "prefix" are sent to notification message queue. directory = "/buckets" -[sink.local] +[sink.backup] enabled = true -directory = "/data" -todays_date_format = "2006-02-01" \ No newline at end of file +directory = "/data" \ No newline at end of file diff --git a/weed/command/filer_replication.go b/weed/command/filer_replication.go index e8c06b208..f2754139b 100644 --- a/weed/command/filer_replication.go +++ b/weed/command/filer_replication.go @@ -9,6 +9,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/replication/sink" _ "github.com/chrislusf/seaweedfs/weed/replication/sink/azuresink" _ "github.com/chrislusf/seaweedfs/weed/replication/sink/b2sink" + _ "github.com/chrislusf/seaweedfs/weed/replication/sink/backupsink" _ "github.com/chrislusf/seaweedfs/weed/replication/sink/filersink" _ "github.com/chrislusf/seaweedfs/weed/replication/sink/gcssink" _ "github.com/chrislusf/seaweedfs/weed/replication/sink/localsink" diff --git a/weed/command/scaffold.go b/weed/command/scaffold.go index 1705c6ae4..9cbb1fa1b 100644 --- a/weed/command/scaffold.go +++ b/weed/command/scaffold.go @@ -353,9 +353,14 @@ directory = "/buckets" [sink.local] enabled = false -directory = "/backup" +directory = "/data" todays_date_format = "" # set this to 2006-02-01 for incremental backup +[sink.backup] +enabled = false +# all replicated files are under create time date directory tree +directory = "/backup" + [sink.filer] enabled = false grpcAddress = "localhost:18888" diff --git a/weed/replication/replicator.go b/weed/replication/replicator.go index c4228434f..81d546c3c 100644 --- a/weed/replication/replicator.go +++ b/weed/replication/replicator.go @@ -6,6 +6,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/pb" "google.golang.org/grpc" "strings" + "time" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" @@ -40,7 +41,17 @@ func (r *Replicator) Replicate(ctx context.Context, key string, message *filer_p glog.V(4).Infof("skipping %v outside of %v", key, r.source.Dir) return nil } - newKey := util.Join(r.sink.GetSinkToDirectory(), key[len(r.source.Dir):]) + var dateKey string + if r.sink.GetName() == "backup" { + var crTime int64 + if message.NewEntry != nil { + crTime = message.NewEntry.Attributes.Crtime + } else if message.OldEntry != nil { + crTime = message.OldEntry.Attributes.Crtime + } + dateKey = time.Unix(crTime, 0).Format("2006-01-02") + } + newKey := util.Join(r.sink.GetSinkToDirectory(), dateKey, key[len(r.source.Dir):]) glog.V(3).Infof("replicate %s => %s", key, newKey) key = newKey if message.OldEntry != nil && message.NewEntry == nil { diff --git a/weed/replication/sink/backupsink/backup_sink.go b/weed/replication/sink/backupsink/backup_sink.go new file mode 100644 index 000000000..df0a778d1 --- /dev/null +++ b/weed/replication/sink/backupsink/backup_sink.go @@ -0,0 +1,18 @@ +package backupsink + +import ( + "github.com/chrislusf/seaweedfs/weed/replication/sink" + "github.com/chrislusf/seaweedfs/weed/replication/sink/localsink" +) + +type BackupSink struct { + localsink.LocalSink +} + +func (backupsink *BackupSink) GetName() string { + return "backup" +} + +func init() { + sink.Sinks = append(sink.Sinks, &BackupSink{}) +} diff --git a/weed/replication/sink/localsink/local_sink.go b/weed/replication/sink/localsink/local_sink.go index 5ca562ec8..21c625c3f 100644 --- a/weed/replication/sink/localsink/local_sink.go +++ b/weed/replication/sink/localsink/local_sink.go @@ -12,13 +12,11 @@ import ( "os" "path/filepath" "strings" - "time" ) type LocalSink struct { - dir string - todaysDateFormat string - filerSource *source.FilerSource + Dir string + filerSource *source.FilerSource } func init() { @@ -37,24 +35,19 @@ func (localsink *LocalSink) isMultiPartEntry(key string) bool { return strings.HasSuffix(key, ".part") && strings.Contains(key, "/.uploads/") } -func (localsink *LocalSink) initialize(dir string, todaysDateFormat string) error { - localsink.dir = dir - localsink.todaysDateFormat = todaysDateFormat +func (localsink *LocalSink) initialize(dir string) error { + localsink.Dir = dir return nil } func (localsink *LocalSink) Initialize(configuration util.Configuration, prefix string) error { dir := configuration.GetString(prefix + "directory") - todaysDateFormat := configuration.GetString(prefix + "todays_date_format") glog.V(4).Infof("sink.local.directory: %v", dir) - return localsink.initialize(dir, todaysDateFormat) + return localsink.initialize(dir) } func (localsink *LocalSink) GetSinkToDirectory() string { - if localsink.todaysDateFormat != "" { - return filepath.Join(localsink.dir, time.Now().Format(localsink.todaysDateFormat)) - } - return localsink.dir + return localsink.Dir } func (localsink *LocalSink) DeleteEntry(key string, isDirectory, deleteIncludeChunks bool, signatures []int32) error { From b81956bcb573166a505d87c2f085e298639fe227 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 27 Jan 2021 10:28:37 -0800 Subject: [PATCH 200/260] mount: invalidate kernel cache when mounted to a filer path fix https://github.com/chrislusf/seaweedfs/issues/1752#issuecomment-768178422 --- weed/filesys/meta_cache/meta_cache.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/weed/filesys/meta_cache/meta_cache.go b/weed/filesys/meta_cache/meta_cache.go index f4e4d7d6e..b9d4724c9 100644 --- a/weed/filesys/meta_cache/meta_cache.go +++ b/weed/filesys/meta_cache/meta_cache.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "os" + "strings" "sync" "github.com/chrislusf/seaweedfs/weed/filer" @@ -29,7 +30,12 @@ func NewMetaCache(dbFolder string, baseDir util.FullPath, uidGidMapper *UidGidMa localStore: openMetaStore(dbFolder), visitedBoundary: bounded_tree.NewBoundedTree(baseDir), uidGidMapper: uidGidMapper, - invalidateFunc: invalidateFunc, + invalidateFunc: func(fullpath util.FullPath) { + if baseDir != "/" && strings.HasPrefix(string(fullpath), string(baseDir)) { + fullpath = fullpath[len(baseDir):] + } + invalidateFunc(fullpath) + }, } } From 7a96276f3416bcb8e4368d032bc7347a4dfabf78 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 27 Jan 2021 10:29:09 -0800 Subject: [PATCH 201/260] adjust dev local docker compose yaml --- docker/local-mount-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/local-mount-compose.yml b/docker/local-mount-compose.yml index 97fc652d4..62a6691d6 100644 --- a/docker/local-mount-compose.yml +++ b/docker/local-mount-compose.yml @@ -31,7 +31,7 @@ services: mount_1: image: chrislusf/seaweedfs:local privileged: true - entrypoint: '/bin/sh -c "mkdir -p t1 && mkdir -p cache/t1 && weed -v=4 mount -filer=filer:8888 -cacheDir=./cache/t1 -dir=./t1 -filer.path=/"' + entrypoint: '/bin/sh -c "mkdir -p t1 && mkdir -p cache/t1 && weed -v=4 mount -filer=filer:8888 -cacheDir=./cache/t1 -dir=./t1 -filer.path=/c1"' depends_on: - master - volume @@ -39,7 +39,7 @@ services: mount_2: image: chrislusf/seaweedfs:local privileged: true - entrypoint: '/bin/sh -c "mkdir -p t2 && mkdir -p cache/t2 && weed -v=4 mount -filer=filer:8888 -cacheDir=./cache/t2 -dir=./t2 -filer.path=/"' + entrypoint: '/bin/sh -c "mkdir -p t2 && mkdir -p cache/t2 && weed -v=4 mount -filer=filer:8888 -cacheDir=./cache/t2 -dir=./t2 -filer.path=/c1"' depends_on: - master - volume From 22e167bc8a30c1bc750ba5ffeae65700b07e85ab Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 27 Jan 2021 15:34:43 -0800 Subject: [PATCH 202/260] mount: skip daemon timeout on FreeBSD --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 39acd0e9b..cbd806799 100644 --- a/go.mod +++ b/go.mod @@ -56,7 +56,7 @@ require ( github.com/prometheus/client_golang v1.3.0 github.com/rakyll/statik v0.1.7 github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 // indirect - github.com/seaweedfs/fuse v1.1.0 + github.com/seaweedfs/fuse v1.1.1 github.com/seaweedfs/goexif v1.0.2 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e github.com/spaolacci/murmur3 v1.1.0 // indirect diff --git a/go.sum b/go.sum index eb16fac4c..db7d2748a 100644 --- a/go.sum +++ b/go.sum @@ -579,6 +579,8 @@ github.com/seaweedfs/fuse v1.0.9 h1:3JZoGsW7cmmrd/U5KYcIGR2+EqyBvCYCoCpEdZAz/Dc= github.com/seaweedfs/fuse v1.0.9/go.mod h1:+PP6WlkrRUG6KPE+Th2EX5To/PjHaFsvqg/UgQ39aj8= github.com/seaweedfs/fuse v1.1.0 h1:cL1qPHFNtFv0UuJTLjKKgWDzfJ4iZzTa4Y7ipc2acGw= github.com/seaweedfs/fuse v1.1.0/go.mod h1:+PP6WlkrRUG6KPE+Th2EX5To/PjHaFsvqg/UgQ39aj8= +github.com/seaweedfs/fuse v1.1.1 h1:WD51YFJcBViOx8I89jeqPD+vAKl4EowzBy9GUw0plb0= +github.com/seaweedfs/fuse v1.1.1/go.mod h1:+PP6WlkrRUG6KPE+Th2EX5To/PjHaFsvqg/UgQ39aj8= github.com/seaweedfs/goexif v1.0.2 h1:p+rTXYdQ2mgxd+1JaTrQ9N8DvYuw9UH9xgYmJ+Bb29E= github.com/seaweedfs/goexif v1.0.2/go.mod h1:MrKs5LK0HXdffrdCZrW3OIMegL2xXpC6ThLyXMyjdrk= github.com/secsy/goftp v0.0.0-20190720192957-f31499d7c79a h1:C6IhVTxNkhlb0tlCB6JfHOUv1f0xHPK7V8X4HlJZEJw= From d38613c49acbca55937770ebf3db79817a313a05 Mon Sep 17 00:00:00 2001 From: Daniel Santos Date: Wed, 27 Jan 2021 20:56:41 -0700 Subject: [PATCH 203/260] match upstream --- weed/command/mount_std.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/weed/command/mount_std.go b/weed/command/mount_std.go index 5df005202..8c6072255 100644 --- a/weed/command/mount_std.go +++ b/weed/command/mount_std.go @@ -208,7 +208,9 @@ func RunMount(option *MountOptions, umask os.FileMode) bool { }) glog.V(0).Infof("mounted %s%s to %s", filer, mountRoot, dir) - err = fs.Serve(c, seaweedFileSystem) + server := fs.New(c, nil) + seaweedFileSystem.Server = server + err = server.Serve(seaweedFileSystem) // check if the mount process has an error to report <-c.Ready From 02fdc0a333bedac882c5f9c103d073c3180ec6bb Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev Date: Thu, 28 Jan 2021 14:56:13 +0500 Subject: [PATCH 204/260] rename backup to local_incremental and use mtime --- docker/replication.toml | 2 +- weed/command/filer_replication.go | 2 +- weed/replication/replicator.go | 10 +++++----- .../replication/sink/backupsink/backup_sink.go | 18 ------------------ .../local_incremental_sink.go | 18 ++++++++++++++++++ 5 files changed, 25 insertions(+), 25 deletions(-) delete mode 100644 weed/replication/sink/backupsink/backup_sink.go create mode 100644 weed/replication/sink/localincrementalsink/local_incremental_sink.go diff --git a/docker/replication.toml b/docker/replication.toml index 75e34a899..833bb1692 100644 --- a/docker/replication.toml +++ b/docker/replication.toml @@ -6,6 +6,6 @@ grpcAddress = "filer:18888" # i.e., all files with this "prefix" are sent to notification message queue. directory = "/buckets" -[sink.backup] +[sink.local_incremental] enabled = true directory = "/data" \ No newline at end of file diff --git a/weed/command/filer_replication.go b/weed/command/filer_replication.go index f2754139b..47ef5d1b6 100644 --- a/weed/command/filer_replication.go +++ b/weed/command/filer_replication.go @@ -9,9 +9,9 @@ import ( "github.com/chrislusf/seaweedfs/weed/replication/sink" _ "github.com/chrislusf/seaweedfs/weed/replication/sink/azuresink" _ "github.com/chrislusf/seaweedfs/weed/replication/sink/b2sink" - _ "github.com/chrislusf/seaweedfs/weed/replication/sink/backupsink" _ "github.com/chrislusf/seaweedfs/weed/replication/sink/filersink" _ "github.com/chrislusf/seaweedfs/weed/replication/sink/gcssink" + _ "github.com/chrislusf/seaweedfs/weed/replication/sink/localincrementalsink" _ "github.com/chrislusf/seaweedfs/weed/replication/sink/localsink" _ "github.com/chrislusf/seaweedfs/weed/replication/sink/s3sink" "github.com/chrislusf/seaweedfs/weed/replication/sub" diff --git a/weed/replication/replicator.go b/weed/replication/replicator.go index 81d546c3c..7688029e6 100644 --- a/weed/replication/replicator.go +++ b/weed/replication/replicator.go @@ -42,14 +42,14 @@ func (r *Replicator) Replicate(ctx context.Context, key string, message *filer_p return nil } var dateKey string - if r.sink.GetName() == "backup" { - var crTime int64 + if r.sink.GetName() == "local_incremental" { + var mTime int64 if message.NewEntry != nil { - crTime = message.NewEntry.Attributes.Crtime + mTime = message.NewEntry.Attributes.Mtime } else if message.OldEntry != nil { - crTime = message.OldEntry.Attributes.Crtime + mTime = message.OldEntry.Attributes.Mtime } - dateKey = time.Unix(crTime, 0).Format("2006-01-02") + dateKey = time.Unix(mTime, 0).Format("2006-01-02") } newKey := util.Join(r.sink.GetSinkToDirectory(), dateKey, key[len(r.source.Dir):]) glog.V(3).Infof("replicate %s => %s", key, newKey) diff --git a/weed/replication/sink/backupsink/backup_sink.go b/weed/replication/sink/backupsink/backup_sink.go deleted file mode 100644 index df0a778d1..000000000 --- a/weed/replication/sink/backupsink/backup_sink.go +++ /dev/null @@ -1,18 +0,0 @@ -package backupsink - -import ( - "github.com/chrislusf/seaweedfs/weed/replication/sink" - "github.com/chrislusf/seaweedfs/weed/replication/sink/localsink" -) - -type BackupSink struct { - localsink.LocalSink -} - -func (backupsink *BackupSink) GetName() string { - return "backup" -} - -func init() { - sink.Sinks = append(sink.Sinks, &BackupSink{}) -} diff --git a/weed/replication/sink/localincrementalsink/local_incremental_sink.go b/weed/replication/sink/localincrementalsink/local_incremental_sink.go new file mode 100644 index 000000000..97da3a5f7 --- /dev/null +++ b/weed/replication/sink/localincrementalsink/local_incremental_sink.go @@ -0,0 +1,18 @@ +package localincrementalsink + +import ( + "github.com/chrislusf/seaweedfs/weed/replication/sink" + "github.com/chrislusf/seaweedfs/weed/replication/sink/localsink" +) + +type LocalIncSink struct { + localsink.LocalSink +} + +func (localincsink *LocalIncSink) GetName() string { + return "local_incremental" +} + +func init() { + sink.Sinks = append(sink.Sinks, &LocalIncSink{}) +} From be1062b7fcabf793ea3cbfd5a67b81c07610ce83 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev Date: Thu, 28 Jan 2021 14:59:20 +0500 Subject: [PATCH 205/260] rename in scaffold --- weed/command/scaffold.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/weed/command/scaffold.go b/weed/command/scaffold.go index 9cbb1fa1b..a95c41054 100644 --- a/weed/command/scaffold.go +++ b/weed/command/scaffold.go @@ -356,9 +356,9 @@ enabled = false directory = "/data" todays_date_format = "" # set this to 2006-02-01 for incremental backup -[sink.backup] +[sink.local_incremental] enabled = false -# all replicated files are under create time date directory tree +# all replicated files are under modification time date directory tree directory = "/backup" [sink.filer] From da08402ba253fe5391f0152905f51f82c51fd527 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 28 Jan 2021 02:17:41 -0800 Subject: [PATCH 206/260] replicate: use creation time for local incremental file sink related to https://github.com/chrislusf/seaweedfs/pull/1762 --- weed/command/filer_replication.go | 1 - weed/command/scaffold.go | 3 +-- weed/replication/replicator.go | 8 ++++---- .../local_incremental_sink.go | 5 ++--- 4 files changed, 7 insertions(+), 10 deletions(-) rename weed/replication/sink/{localincrementalsink => localsink}/local_incremental_sink.go (68%) diff --git a/weed/command/filer_replication.go b/weed/command/filer_replication.go index 47ef5d1b6..e8c06b208 100644 --- a/weed/command/filer_replication.go +++ b/weed/command/filer_replication.go @@ -11,7 +11,6 @@ import ( _ "github.com/chrislusf/seaweedfs/weed/replication/sink/b2sink" _ "github.com/chrislusf/seaweedfs/weed/replication/sink/filersink" _ "github.com/chrislusf/seaweedfs/weed/replication/sink/gcssink" - _ "github.com/chrislusf/seaweedfs/weed/replication/sink/localincrementalsink" _ "github.com/chrislusf/seaweedfs/weed/replication/sink/localsink" _ "github.com/chrislusf/seaweedfs/weed/replication/sink/s3sink" "github.com/chrislusf/seaweedfs/weed/replication/sub" diff --git a/weed/command/scaffold.go b/weed/command/scaffold.go index a95c41054..3ac7c021d 100644 --- a/weed/command/scaffold.go +++ b/weed/command/scaffold.go @@ -354,11 +354,10 @@ directory = "/buckets" [sink.local] enabled = false directory = "/data" -todays_date_format = "" # set this to 2006-02-01 for incremental backup [sink.local_incremental] +# all replicated files are under creation time as yyyy-mm-dd directories enabled = false -# all replicated files are under modification time date directory tree directory = "/backup" [sink.filer] diff --git a/weed/replication/replicator.go b/weed/replication/replicator.go index 7688029e6..90e1f7cdd 100644 --- a/weed/replication/replicator.go +++ b/weed/replication/replicator.go @@ -43,13 +43,13 @@ func (r *Replicator) Replicate(ctx context.Context, key string, message *filer_p } var dateKey string if r.sink.GetName() == "local_incremental" { - var mTime int64 + var cTime int64 if message.NewEntry != nil { - mTime = message.NewEntry.Attributes.Mtime + cTime = message.NewEntry.Attributes.Crtime } else if message.OldEntry != nil { - mTime = message.OldEntry.Attributes.Mtime + cTime = message.OldEntry.Attributes.Crtime } - dateKey = time.Unix(mTime, 0).Format("2006-01-02") + dateKey = time.Unix(cTime, 0).Format("2006-01-02") } newKey := util.Join(r.sink.GetSinkToDirectory(), dateKey, key[len(r.source.Dir):]) glog.V(3).Infof("replicate %s => %s", key, newKey) diff --git a/weed/replication/sink/localincrementalsink/local_incremental_sink.go b/weed/replication/sink/localsink/local_incremental_sink.go similarity index 68% rename from weed/replication/sink/localincrementalsink/local_incremental_sink.go rename to weed/replication/sink/localsink/local_incremental_sink.go index 97da3a5f7..a1d49e28a 100644 --- a/weed/replication/sink/localincrementalsink/local_incremental_sink.go +++ b/weed/replication/sink/localsink/local_incremental_sink.go @@ -1,12 +1,11 @@ -package localincrementalsink +package localsink import ( "github.com/chrislusf/seaweedfs/weed/replication/sink" - "github.com/chrislusf/seaweedfs/weed/replication/sink/localsink" ) type LocalIncSink struct { - localsink.LocalSink + LocalSink } func (localincsink *LocalIncSink) GetName() string { From 9a06c35da40b9d4b5b687b7300fed4196020f375 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 28 Jan 2021 02:39:22 -0800 Subject: [PATCH 207/260] replicate: incremental sink only contains new and updated files address https://github.com/chrislusf/seaweedfs/commit/da08402ba253fe5391f0152905f51f82c51fd527 --- weed/command/scaffold.go | 3 ++- weed/replication/replicator.go | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/weed/command/scaffold.go b/weed/command/scaffold.go index 3ac7c021d..58143a0ad 100644 --- a/weed/command/scaffold.go +++ b/weed/command/scaffold.go @@ -356,7 +356,8 @@ enabled = false directory = "/data" [sink.local_incremental] -# all replicated files are under creation time as yyyy-mm-dd directories +# all replicated files are under modified time as yyyy-mm-dd directories +# so each date directory contains all new and updated files. enabled = false directory = "/backup" diff --git a/weed/replication/replicator.go b/weed/replication/replicator.go index 90e1f7cdd..7688029e6 100644 --- a/weed/replication/replicator.go +++ b/weed/replication/replicator.go @@ -43,13 +43,13 @@ func (r *Replicator) Replicate(ctx context.Context, key string, message *filer_p } var dateKey string if r.sink.GetName() == "local_incremental" { - var cTime int64 + var mTime int64 if message.NewEntry != nil { - cTime = message.NewEntry.Attributes.Crtime + mTime = message.NewEntry.Attributes.Mtime } else if message.OldEntry != nil { - cTime = message.OldEntry.Attributes.Crtime + mTime = message.OldEntry.Attributes.Mtime } - dateKey = time.Unix(cTime, 0).Format("2006-01-02") + dateKey = time.Unix(mTime, 0).Format("2006-01-02") } newKey := util.Join(r.sink.GetSinkToDirectory(), dateKey, key[len(r.source.Dir):]) glog.V(3).Infof("replicate %s => %s", key, newKey) From cf252fc0cdbe2e65a1fcb4853ba2d125c47ab1a0 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 28 Jan 2021 04:46:37 -0800 Subject: [PATCH 208/260] mount: report error when Flush() The error in Release() is not working. See https://github.com/jaderhs/libfuse/blob/master/FAQ related to https://github.com/chrislusf/seaweedfs/issues/1765 --- weed/filesys/dirty_page.go | 22 +--------------------- weed/filesys/filehandle.go | 26 +++++++++++++++----------- 2 files changed, 16 insertions(+), 32 deletions(-) diff --git a/weed/filesys/dirty_page.go b/weed/filesys/dirty_page.go index 11089186f..c4c206a05 100644 --- a/weed/filesys/dirty_page.go +++ b/weed/filesys/dirty_page.go @@ -3,7 +3,6 @@ package filesys import ( "bytes" "io" - "runtime" "sync" "time" @@ -16,8 +15,6 @@ type ContinuousDirtyPages struct { f *File writeWaitGroup sync.WaitGroup chunkAddLock sync.Mutex - chunkSaveErrChan chan error - chunkSaveErrChanClosed bool lastErr error collection string replication string @@ -27,15 +24,7 @@ func newDirtyPages(file *File) *ContinuousDirtyPages { dirtyPages := &ContinuousDirtyPages{ intervals: &ContinuousIntervals{}, f: file, - chunkSaveErrChan: make(chan error, runtime.NumCPU()), } - go func() { - for t := range dirtyPages.chunkSaveErrChan { - if t != nil { - dirtyPages.lastErr = t - } - } - }() return dirtyPages } @@ -94,15 +83,6 @@ func (pages *ContinuousDirtyPages) saveExistingLargestPageToStorage() (hasSavedD func (pages *ContinuousDirtyPages) saveToStorage(reader io.Reader, offset int64, size int64) { - errChanSize := pages.f.wfs.option.ConcurrentWriters - if errChanSize == 0 { - errChanSize = runtime.NumCPU() - } - if pages.chunkSaveErrChanClosed { - pages.chunkSaveErrChan = make(chan error, errChanSize) - pages.chunkSaveErrChanClosed = false - } - mtime := time.Now().UnixNano() pages.writeWaitGroup.Add(1) writer := func() { @@ -112,7 +92,7 @@ func (pages *ContinuousDirtyPages) saveToStorage(reader io.Reader, offset int64, chunk, collection, replication, err := pages.f.wfs.saveDataAsChunk(pages.f.fullpath())(reader, pages.f.Name, offset) if err != nil { glog.V(0).Infof("%s saveToStorage [%d,%d): %v", pages.f.fullpath(), offset, offset+size, err) - pages.chunkSaveErrChan <- err + pages.lastErr = err return } chunk.Mtime = mtime diff --git a/weed/filesys/filehandle.go b/weed/filesys/filehandle.go index 168bf792d..85f64d292 100644 --- a/weed/filesys/filehandle.go +++ b/weed/filesys/filehandle.go @@ -184,25 +184,20 @@ func (fh *FileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) err fh.Lock() defer fh.Unlock() - fh.f.isOpen-- - - if fh.f.isOpen < 0 { + if fh.f.isOpen <= 0 { glog.V(0).Infof("Release reset %s open count %d => %d", fh.f.Name, fh.f.isOpen, 0) fh.f.isOpen = 0 return nil } - if fh.f.isOpen == 0 { + if fh.f.isOpen == 1 { if err := fh.doFlush(ctx, req.Header); err != nil { glog.Errorf("Release doFlush %s: %v", fh.f.Name, err) + return err } - // stop the goroutine - if !fh.dirtyPages.chunkSaveErrChanClosed { - fh.dirtyPages.chunkSaveErrChanClosed = true - close(fh.dirtyPages.chunkSaveErrChan) - } + fh.f.isOpen-- fh.f.wfs.ReleaseHandle(fh.f.fullpath(), fuse.HandleID(fh.handle)) if closer, ok := fh.f.reader.(io.Closer); ok { @@ -216,10 +211,18 @@ func (fh *FileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) err func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error { + glog.V(4).Infof("Flush %v fh %d", fh.f.fullpath(), fh.handle) + fh.Lock() defer fh.Unlock() - return fh.doFlush(ctx, req.Header) + if err := fh.doFlush(ctx, req.Header); err != nil { + glog.Errorf("Flush doFlush %s: %v", fh.f.Name, err) + return err + } + + glog.V(4).Infof("Flush %v fh %d success", fh.f.fullpath(), fh.handle) + return nil } func (fh *FileHandle) doFlush(ctx context.Context, header fuse.Header) error { @@ -232,7 +235,8 @@ func (fh *FileHandle) doFlush(ctx context.Context, header fuse.Header) error { fh.dirtyPages.writeWaitGroup.Wait() if fh.dirtyPages.lastErr != nil { - return fh.dirtyPages.lastErr + glog.Errorf("%v doFlush last err: %v", fh.f.fullpath(), fh.dirtyPages.lastErr) + return fuse.EIO } if !fh.f.dirtyMetadata { From e9d820192587e6ae01d7183f99b898953f192cf7 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 28 Jan 2021 13:20:06 -0800 Subject: [PATCH 209/260] s3: batch purge empty folders --- weed/s3api/s3api_object_handlers.go | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/weed/s3api/s3api_object_handlers.go b/weed/s3api/s3api_object_handlers.go index 401e2f96c..e8c1b59d7 100644 --- a/weed/s3api/s3api_object_handlers.go +++ b/weed/s3api/s3api_object_handlers.go @@ -175,16 +175,15 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h var deletedObjects []ObjectIdentifier var deleteErrors []DeleteError + directoriesWithDeletion := make(map[string]int) + s3a.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + // delete file entries for _, object := range deleteObjects.Objects { - response, _ := s3a.listFilerEntries(bucket, object.ObjectName, 1, "", "/") - if len(response.Contents) != 0 && strings.HasSuffix(object.ObjectName, "/") { - continue - } lastSeparator := strings.LastIndex(object.ObjectName, "/") - parentDirectoryPath, entryName, isDeleteData, isRecursive := "/", object.ObjectName, true, true + parentDirectoryPath, entryName, isDeleteData, isRecursive := "/", object.ObjectName, true, false if lastSeparator > 0 && lastSeparator+1 < len(object.ObjectName) { entryName = object.ObjectName[lastSeparator+1:] parentDirectoryPath = "/" + object.ObjectName[:lastSeparator] @@ -193,8 +192,10 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h err := doDeleteEntry(client, parentDirectoryPath, entryName, isDeleteData, isRecursive) if err == nil { + directoriesWithDeletion[parentDirectoryPath]++ deletedObjects = append(deletedObjects, object) } else { + delete(directoriesWithDeletion, parentDirectoryPath) deleteErrors = append(deleteErrors, DeleteError{ Code: "", Message: err.Error(), @@ -202,6 +203,15 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h }) } } + + // purge empty folders, only checking folders with deletions + for dir, deletionCount := range directoriesWithDeletion { + parentDir, dirName := util.FullPath(dir).DirAndName() + if err := doDeleteEntry(client, parentDir, dirName, false, false); err != nil { + glog.V(4).Infof("directory %s has %d deletion but still not empty: %v", dir, deletionCount, err) + } + } + return nil }) From c2bf1a88ac1af1cc573f21c6d98c191be7f59281 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 28 Jan 2021 14:28:40 -0800 Subject: [PATCH 210/260] delete from the deepest directory first when checking empty folders --- weed/s3api/s3api_object_handlers.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/weed/s3api/s3api_object_handlers.go b/weed/s3api/s3api_object_handlers.go index e8c1b59d7..ab79d1976 100644 --- a/weed/s3api/s3api_object_handlers.go +++ b/weed/s3api/s3api_object_handlers.go @@ -8,6 +8,7 @@ import ( "io" "io/ioutil" "net/http" + "sort" "strings" "github.com/chrislusf/seaweedfs/weed/s3api/s3err" @@ -204,11 +205,20 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h } } + + // purge empty folders, only checking folders with deletions - for dir, deletionCount := range directoriesWithDeletion { + var allDirs []string + for dir, _ := range directoriesWithDeletion { + allDirs = append(allDirs, dir) + } + sort.Slice(allDirs, func(i, j int) bool { + return len(allDirs[i]) > len(allDirs[j]) + }) + for _, dir := range allDirs { parentDir, dirName := util.FullPath(dir).DirAndName() if err := doDeleteEntry(client, parentDir, dirName, false, false); err != nil { - glog.V(4).Infof("directory %s has %d deletion but still not empty: %v", dir, deletionCount, err) + glog.V(4).Infof("directory %s has %d deletion but still not empty: %v", dir, directoriesWithDeletion[dir], err) } } From 9292796ec2d8f796f3e9162fcd1362cba801e7e4 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 28 Jan 2021 14:29:14 -0800 Subject: [PATCH 211/260] fmt --- weed/s3api/s3api_object_handlers.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/weed/s3api/s3api_object_handlers.go b/weed/s3api/s3api_object_handlers.go index ab79d1976..1c0b290c9 100644 --- a/weed/s3api/s3api_object_handlers.go +++ b/weed/s3api/s3api_object_handlers.go @@ -205,8 +205,6 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h } } - - // purge empty folders, only checking folders with deletions var allDirs []string for dir, _ := range directoriesWithDeletion { From 990fa69bfeac65a72bec1539993e32633858c8f5 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 28 Jan 2021 14:36:29 -0800 Subject: [PATCH 212/260] add back AdjustedUrl() related code --- weed/filer/reader_at.go | 2 +- weed/filesys/wfs_filer_client.go | 7 +++++++ weed/filesys/wfs_write.go | 2 +- weed/messaging/broker/broker_append.go | 4 ++++ weed/pb/filer_pb/filer_client.go | 1 + weed/replication/sink/filersink/fetch_write.go | 3 +++ weed/replication/source/filer_source.go | 4 ++++ weed/s3api/s3api_handlers.go | 3 +++ weed/server/webdav_server.go | 3 +++ weed/shell/commands.go | 4 ++++ 10 files changed, 31 insertions(+), 2 deletions(-) diff --git a/weed/filer/reader_at.go b/weed/filer/reader_at.go index 41d177210..a1e989684 100644 --- a/weed/filer/reader_at.go +++ b/weed/filer/reader_at.go @@ -71,7 +71,7 @@ func LookupFn(filerClient filer_pb.FilerClient) wdclient.LookupFileIdFunctionTyp } for _, loc := range locations.Locations { - volumeServerAddress := loc.Url + volumeServerAddress := filerClient.AdjustedUrl(loc) targetUrl := fmt.Sprintf("http://%s/%s", volumeServerAddress, fileId) targetUrls = append(targetUrls, targetUrl) } diff --git a/weed/filesys/wfs_filer_client.go b/weed/filesys/wfs_filer_client.go index e0d352a7b..ef4213af1 100644 --- a/weed/filesys/wfs_filer_client.go +++ b/weed/filesys/wfs_filer_client.go @@ -25,3 +25,10 @@ func (wfs *WFS) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) error) erro return err } + +func (wfs *WFS) AdjustedUrl(location *filer_pb.Location) string { + if wfs.option.OutsideContainerClusterMode { + return location.PublicUrl + } + return location.Url +} diff --git a/weed/filesys/wfs_write.go b/weed/filesys/wfs_write.go index dfe6e57a6..54e114013 100644 --- a/weed/filesys/wfs_write.go +++ b/weed/filesys/wfs_write.go @@ -44,7 +44,7 @@ func (wfs *WFS) saveDataAsChunk(fullPath util.FullPath) filer.SaveDataAsChunkFun Url: resp.Url, PublicUrl: resp.PublicUrl, } - host = loc.Url + host = wfs.AdjustedUrl(loc) collection, replication = resp.Collection, resp.Replication return nil diff --git a/weed/messaging/broker/broker_append.go b/weed/messaging/broker/broker_append.go index 67c9bcb79..8e5b56fd0 100644 --- a/weed/messaging/broker/broker_append.go +++ b/weed/messaging/broker/broker_append.go @@ -107,3 +107,7 @@ func (broker *MessageBroker) WithFilerClient(fn func(filer_pb.SeaweedFilerClient return } + +func (broker *MessageBroker) AdjustedUrl(location *filer_pb.Location) string { + return location.Url +} diff --git a/weed/pb/filer_pb/filer_client.go b/weed/pb/filer_pb/filer_client.go index 7198de95c..079fbd671 100644 --- a/weed/pb/filer_pb/filer_client.go +++ b/weed/pb/filer_pb/filer_client.go @@ -20,6 +20,7 @@ var ( type FilerClient interface { WithFilerClient(fn func(SeaweedFilerClient) error) error + AdjustedUrl(location *Location) string } func GetEntry(filerClient FilerClient, fullFilePath util.FullPath) (entry *Entry, err error) { diff --git a/weed/replication/sink/filersink/fetch_write.go b/weed/replication/sink/filersink/fetch_write.go index 544b84995..b062adcfe 100644 --- a/weed/replication/sink/filersink/fetch_write.go +++ b/weed/replication/sink/filersink/fetch_write.go @@ -128,3 +128,6 @@ func (fs *FilerSink) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) error) }, fs.grpcAddress, fs.grpcDialOption) } +func (fs *FilerSink) AdjustedUrl(location *filer_pb.Location) string { + return location.Url +} diff --git a/weed/replication/source/filer_source.go b/weed/replication/source/filer_source.go index eff1da8dc..3982360b0 100644 --- a/weed/replication/source/filer_source.go +++ b/weed/replication/source/filer_source.go @@ -124,6 +124,10 @@ func (fs *FilerSource) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) erro } +func (fs *FilerSource) AdjustedUrl(location *filer_pb.Location) string { + return location.Url +} + func volumeId(fileId string) string { lastCommaIndex := strings.LastIndex(fileId, ",") if lastCommaIndex > 0 { diff --git a/weed/s3api/s3api_handlers.go b/weed/s3api/s3api_handlers.go index 57b26f3dd..6935c75bd 100644 --- a/weed/s3api/s3api_handlers.go +++ b/weed/s3api/s3api_handlers.go @@ -50,6 +50,9 @@ func (s3a *S3ApiServer) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) err }, s3a.option.FilerGrpcAddress, s3a.option.GrpcDialOption) } +func (s3a *S3ApiServer) AdjustedUrl(location *filer_pb.Location) string { + return location.Url +} // If none of the http routes match respond with MethodNotAllowed func notFoundHandler(w http.ResponseWriter, r *http.Request) { diff --git a/weed/server/webdav_server.go b/weed/server/webdav_server.go index 5bd92a136..feb193230 100644 --- a/weed/server/webdav_server.go +++ b/weed/server/webdav_server.go @@ -123,6 +123,9 @@ func (fs *WebDavFileSystem) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) }, fs.option.FilerGrpcAddress, fs.option.GrpcDialOption) } +func (fs *WebDavFileSystem) AdjustedUrl(location *filer_pb.Location) string { + return location.Url +} func clearName(name string) (string, error) { slashed := strings.HasSuffix(name, "/") diff --git a/weed/shell/commands.go b/weed/shell/commands.go index 6e1348ca5..0e285214b 100644 --- a/weed/shell/commands.go +++ b/weed/shell/commands.go @@ -102,6 +102,10 @@ func (ce *CommandEnv) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) error } +func (ce *CommandEnv) AdjustedUrl(location *filer_pb.Location) string { + return location.Url +} + func parseFilerUrl(entryPath string) (filerServer string, filerPort int64, path string, err error) { if strings.HasPrefix(entryPath, "http") { var u *url.URL From 19295600f94fd18d09323cb17e89c115c811fb52 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 28 Jan 2021 15:23:16 -0800 Subject: [PATCH 213/260] mount: change option name to volumeServerAccess, with publicUrl and filerProxy modes --- weed/command/mount.go | 28 ++++++++++++++-------------- weed/command/mount_std.go | 26 +++++++++++++------------- weed/filesys/wfs.go | 8 ++++---- weed/filesys/wfs_filer_client.go | 2 +- weed/filesys/wfs_write.go | 2 +- 5 files changed, 33 insertions(+), 33 deletions(-) diff --git a/weed/command/mount.go b/weed/command/mount.go index fa75919aa..febc80683 100644 --- a/weed/command/mount.go +++ b/weed/command/mount.go @@ -11,19 +11,19 @@ type MountOptions struct { dir *string dirAutoCreate *bool collection *string - replication *string - ttlSec *int - chunkSizeLimitMB *int - concurrentWriters *int - cacheDir *string - cacheSizeMB *int64 - dataCenter *string - allowOthers *bool - umaskString *string - nonempty *bool - outsideContainerClusterMode *bool - uidMap *string - gidMap *string + replication *string + ttlSec *int + chunkSizeLimitMB *int + concurrentWriters *int + cacheDir *string + cacheSizeMB *int64 + dataCenter *string + allowOthers *bool + umaskString *string + nonempty *bool + volumeServerAccess *string + uidMap *string + gidMap *string } var ( @@ -50,7 +50,7 @@ func init() { mountOptions.allowOthers = cmdMount.Flag.Bool("allowOthers", true, "allows other users to access the file system") mountOptions.umaskString = cmdMount.Flag.String("umask", "022", "octal umask, e.g., 022, 0111") mountOptions.nonempty = cmdMount.Flag.Bool("nonempty", false, "allows the mounting over a non-empty directory") - mountOptions.outsideContainerClusterMode = cmdMount.Flag.Bool("outsideContainerClusterMode", false, "allows other users to access volume servers with publicUrl") + mountOptions.volumeServerAccess = cmdMount.Flag.String("volumeServerAccess", "direct", "access volume servers by [direct|publicUrl|filerProxy]") mountOptions.uidMap = cmdMount.Flag.String("map.uid", "", "map local uid to uid on filer, comma-separated :") mountOptions.gidMap = cmdMount.Flag.String("map.gid", "", "map local gid to gid on filer, comma-separated :") diff --git a/weed/command/mount_std.go b/weed/command/mount_std.go index 8c6072255..26c59a1c7 100644 --- a/weed/command/mount_std.go +++ b/weed/command/mount_std.go @@ -179,19 +179,19 @@ func RunMount(option *MountOptions, umask os.FileMode) bool { TtlSec: int32(*option.ttlSec), ChunkSizeLimit: int64(chunkSizeLimitMB) * 1024 * 1024, ConcurrentWriters: *option.concurrentWriters, - CacheDir: *option.cacheDir, - CacheSizeMB: *option.cacheSizeMB, - DataCenter: *option.dataCenter, - EntryCacheTtl: 3 * time.Second, - MountUid: uid, - MountGid: gid, - MountMode: mountMode, - MountCtime: fileInfo.ModTime(), - MountMtime: time.Now(), - Umask: umask, - OutsideContainerClusterMode: *mountOptions.outsideContainerClusterMode, - Cipher: cipher, - UidGidMapper: uidGidMapper, + CacheDir: *option.cacheDir, + CacheSizeMB: *option.cacheSizeMB, + DataCenter: *option.dataCenter, + EntryCacheTtl: 3 * time.Second, + MountUid: uid, + MountGid: gid, + MountMode: mountMode, + MountCtime: fileInfo.ModTime(), + MountMtime: time.Now(), + Umask: umask, + VolumeServerAccess: *mountOptions.volumeServerAccess, + Cipher: cipher, + UidGidMapper: uidGidMapper, }) // mount diff --git a/weed/filesys/wfs.go b/weed/filesys/wfs.go index 108e23c85..f4e5ac38a 100644 --- a/weed/filesys/wfs.go +++ b/weed/filesys/wfs.go @@ -48,9 +48,9 @@ type Option struct { MountCtime time.Time MountMtime time.Time - OutsideContainerClusterMode bool // whether the mount runs outside SeaweedFS containers - Cipher bool // whether encrypt data on volume server - UidGidMapper *meta_cache.UidGidMapper + VolumeServerAccess string // how to access volume servers + Cipher bool // whether encrypt data on volume server + UidGidMapper *meta_cache.UidGidMapper } var _ = fs.FS(&WFS{}) @@ -257,7 +257,7 @@ func (wfs *WFS) mapPbIdFromLocalToFiler(entry *filer_pb.Entry) { } func (wfs *WFS) LookupFn() wdclient.LookupFileIdFunctionType { - if wfs.option.OutsideContainerClusterMode { + if wfs.option.VolumeServerAccess == "filerProxy" { return func(fileId string) (targetUrls []string, err error) { return []string{"http://" + wfs.option.FilerAddress + "/?proxyChunkId=" + fileId}, nil } diff --git a/weed/filesys/wfs_filer_client.go b/weed/filesys/wfs_filer_client.go index ef4213af1..671d20ba2 100644 --- a/weed/filesys/wfs_filer_client.go +++ b/weed/filesys/wfs_filer_client.go @@ -27,7 +27,7 @@ func (wfs *WFS) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) error) erro } func (wfs *WFS) AdjustedUrl(location *filer_pb.Location) string { - if wfs.option.OutsideContainerClusterMode { + if wfs.option.VolumeServerAccess == "publicUrl" { return location.PublicUrl } return location.Url diff --git a/weed/filesys/wfs_write.go b/weed/filesys/wfs_write.go index 54e114013..4e1e71672 100644 --- a/weed/filesys/wfs_write.go +++ b/weed/filesys/wfs_write.go @@ -53,7 +53,7 @@ func (wfs *WFS) saveDataAsChunk(fullPath util.FullPath) filer.SaveDataAsChunkFun } fileUrl := fmt.Sprintf("http://%s/%s", host, fileId) - if wfs.option.OutsideContainerClusterMode { + if wfs.option.VolumeServerAccess == "filerProxy" { fileUrl = fmt.Sprintf("http://%s/?proxyChunkId=%s", wfs.option.FilerAddress, fileId) } uploadResult, err, data := operation.Upload(fileUrl, filename, wfs.option.Cipher, reader, false, "", nil, auth) From d475c89fccf1f020e6f552a5923a688a56e0f1ed Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 28 Jan 2021 15:23:46 -0800 Subject: [PATCH 214/260] go fmt --- weed/command/filer_sync.go | 30 ++++++++++++++-------------- weed/command/mount.go | 10 +++++----- weed/command/mount_std.go | 20 +++++++++---------- weed/filesys/dir.go | 2 +- weed/filesys/dirty_page.go | 18 ++++++++--------- weed/server/filer_server_handlers.go | 2 +- 6 files changed, 41 insertions(+), 41 deletions(-) diff --git a/weed/command/filer_sync.go b/weed/command/filer_sync.go index f4411b756..9a23fd731 100644 --- a/weed/command/filer_sync.go +++ b/weed/command/filer_sync.go @@ -20,21 +20,21 @@ import ( ) type SyncOptions struct { - isActivePassive *bool - filerA *string - filerB *string - aPath *string - bPath *string - aReplication *string - bReplication *string - aCollection *string - bCollection *string - aTtlSec *int - bTtlSec *int - aDebug *bool - bDebug *bool - aProxyByFiler *bool - bProxyByFiler *bool + isActivePassive *bool + filerA *string + filerB *string + aPath *string + bPath *string + aReplication *string + bReplication *string + aCollection *string + bCollection *string + aTtlSec *int + bTtlSec *int + aDebug *bool + bDebug *bool + aProxyByFiler *bool + bProxyByFiler *bool } var ( diff --git a/weed/command/mount.go b/weed/command/mount.go index febc80683..fce3eba22 100644 --- a/weed/command/mount.go +++ b/weed/command/mount.go @@ -6,11 +6,11 @@ import ( ) type MountOptions struct { - filer *string - filerMountRootPath *string - dir *string - dirAutoCreate *bool - collection *string + filer *string + filerMountRootPath *string + dir *string + dirAutoCreate *bool + collection *string replication *string ttlSec *int chunkSizeLimitMB *int diff --git a/weed/command/mount_std.go b/weed/command/mount_std.go index 26c59a1c7..83ae870e6 100644 --- a/weed/command/mount_std.go +++ b/weed/command/mount_std.go @@ -169,16 +169,16 @@ func RunMount(option *MountOptions, umask os.FileMode) bool { } seaweedFileSystem := filesys.NewSeaweedFileSystem(&filesys.Option{ - MountDirectory: dir, - FilerAddress: filer, - FilerGrpcAddress: filerGrpcAddress, - GrpcDialOption: grpcDialOption, - FilerMountRootPath: mountRoot, - Collection: *option.collection, - Replication: *option.replication, - TtlSec: int32(*option.ttlSec), - ChunkSizeLimit: int64(chunkSizeLimitMB) * 1024 * 1024, - ConcurrentWriters: *option.concurrentWriters, + MountDirectory: dir, + FilerAddress: filer, + FilerGrpcAddress: filerGrpcAddress, + GrpcDialOption: grpcDialOption, + FilerMountRootPath: mountRoot, + Collection: *option.collection, + Replication: *option.replication, + TtlSec: int32(*option.ttlSec), + ChunkSizeLimit: int64(chunkSizeLimitMB) * 1024 * 1024, + ConcurrentWriters: *option.concurrentWriters, CacheDir: *option.cacheDir, CacheSizeMB: *option.cacheSizeMB, DataCenter: *option.dataCenter, diff --git a/weed/filesys/dir.go b/weed/filesys/dir.go index e3d0057db..10a0a2b44 100644 --- a/weed/filesys/dir.go +++ b/weed/filesys/dir.go @@ -175,7 +175,7 @@ func (dir *Dir) doCreateEntry(name string, mode os.FileMode, uid, gid uint32, ex TtlSec: dir.wfs.option.TtlSec, }, }, - OExcl: exlusive, + OExcl: exlusive, Signatures: []int32{dir.wfs.signature}, } glog.V(1).Infof("create %s/%s", dir.FullPath(), name) diff --git a/weed/filesys/dirty_page.go b/weed/filesys/dirty_page.go index c4c206a05..f05a3a56a 100644 --- a/weed/filesys/dirty_page.go +++ b/weed/filesys/dirty_page.go @@ -11,19 +11,19 @@ import ( ) type ContinuousDirtyPages struct { - intervals *ContinuousIntervals - f *File - writeWaitGroup sync.WaitGroup - chunkAddLock sync.Mutex - lastErr error - collection string - replication string + intervals *ContinuousIntervals + f *File + writeWaitGroup sync.WaitGroup + chunkAddLock sync.Mutex + lastErr error + collection string + replication string } func newDirtyPages(file *File) *ContinuousDirtyPages { dirtyPages := &ContinuousDirtyPages{ - intervals: &ContinuousIntervals{}, - f: file, + intervals: &ContinuousIntervals{}, + f: file, } return dirtyPages } diff --git a/weed/server/filer_server_handlers.go b/weed/server/filer_server_handlers.go index fcafe3893..3bc0c5d0d 100644 --- a/weed/server/filer_server_handlers.go +++ b/weed/server/filer_server_handlers.go @@ -20,7 +20,7 @@ func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) { } if fileId != "" { stats.FilerRequestCounter.WithLabelValues("proxy").Inc() - fs.proxyToVolumeServer(w,r,fileId) + fs.proxyToVolumeServer(w, r, fileId) stats.FilerRequestHistogram.WithLabelValues("proxy").Observe(time.Since(start).Seconds()) return } From 678b9a60be690c6d4c8fb03c0a1bdd7516eaed8a Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 30 Jan 2021 05:38:31 -0800 Subject: [PATCH 215/260] test different volumeServerAcccss --- docker/local-mount-compose.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/docker/local-mount-compose.yml b/docker/local-mount-compose.yml index 62a6691d6..4361d6d99 100644 --- a/docker/local-mount-compose.yml +++ b/docker/local-mount-compose.yml @@ -10,10 +10,9 @@ services: volume: image: chrislusf/seaweedfs:local ports: - - 8080:8080 - - 18080:18080 + - 7455:8080 - 9325:9325 - command: 'volume -mserver="master:9333" -port=8080 -metricsPort=9325 -preStopSeconds=1' + command: 'volume -mserver="master:9333" -port=8080 -metricsPort=9325 -preStopSeconds=1 -publicUrl=localhost:7455' depends_on: - master filer: @@ -31,7 +30,7 @@ services: mount_1: image: chrislusf/seaweedfs:local privileged: true - entrypoint: '/bin/sh -c "mkdir -p t1 && mkdir -p cache/t1 && weed -v=4 mount -filer=filer:8888 -cacheDir=./cache/t1 -dir=./t1 -filer.path=/c1"' + entrypoint: '/bin/sh -c "mkdir -p t1 && mkdir -p cache/t1 && weed -v=4 mount -filer=filer:8888 -cacheDir=./cache/t1 -dir=./t1 -filer.path=/c1" -volumeServerAccess=filerProxy' depends_on: - master - volume @@ -39,7 +38,7 @@ services: mount_2: image: chrislusf/seaweedfs:local privileged: true - entrypoint: '/bin/sh -c "mkdir -p t2 && mkdir -p cache/t2 && weed -v=4 mount -filer=filer:8888 -cacheDir=./cache/t2 -dir=./t2 -filer.path=/c1"' + entrypoint: '/bin/sh -c "mkdir -p t2 && mkdir -p cache/t2 && weed -v=4 mount -filer=filer:8888 -cacheDir=./cache/t2 -dir=./t2 -filer.path=/c1" -volumeServerAcess=publicUrl' depends_on: - master - volume From 043c2d796098e1f88da1442eefdb9bd5df075708 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 30 Jan 2021 05:39:09 -0800 Subject: [PATCH 216/260] refactoring SeaweedOutputStream --- .../client}/SeaweedOutputStream.java | 41 +-- .../seaweed/hdfs/SeaweedFileSystemStore.java | 2 +- .../hdfs/SeaweedHadoopOutputStream.java | 16 + .../seaweed/hdfs/SeaweedFileSystemStore.java | 2 +- .../hdfs/SeaweedHadoopOutputStream.java | 64 ++++ .../seaweed/hdfs/SeaweedOutputStream.java | 337 ------------------ 6 files changed, 101 insertions(+), 361 deletions(-) rename other/java/{hdfs2/src/main/java/seaweed/hdfs => client/src/main/java/seaweedfs/client}/SeaweedOutputStream.java (89%) create mode 100644 other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedHadoopOutputStream.java create mode 100644 other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedHadoopOutputStream.java delete mode 100644 other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedOutputStream.java diff --git a/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedOutputStream.java b/other/java/client/src/main/java/seaweedfs/client/SeaweedOutputStream.java similarity index 89% rename from other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedOutputStream.java rename to other/java/client/src/main/java/seaweedfs/client/SeaweedOutputStream.java index 26290c46c..b09a15a5c 100644 --- a/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedOutputStream.java +++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedOutputStream.java @@ -1,16 +1,9 @@ -package seaweed.hdfs; +package seaweedfs.client; // adapted from org.apache.hadoop.fs.azurebfs.services.AbfsOutputStream -import com.google.common.base.Preconditions; -import org.apache.hadoop.fs.FSExceptionMessages; -import org.apache.hadoop.fs.Path; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import seaweedfs.client.ByteBufferPool; -import seaweedfs.client.FilerGrpcClient; -import seaweedfs.client.FilerProto; -import seaweedfs.client.SeaweedWrite; import java.io.IOException; import java.io.InterruptedIOException; @@ -18,20 +11,18 @@ import java.io.OutputStream; import java.nio.ByteBuffer; import java.util.concurrent.*; -import static seaweed.hdfs.SeaweedFileSystemStore.getParentDirectory; - public class SeaweedOutputStream extends OutputStream { private static final Logger LOG = LoggerFactory.getLogger(SeaweedOutputStream.class); private final FilerGrpcClient filerGrpcClient; - private final Path path; + private final String path; private final int bufferSize; private final int maxConcurrentRequestCount; private final ThreadPoolExecutor threadExecutor; private final ExecutorCompletionService completionService; private final FilerProto.Entry.Builder entry; - private final boolean supportFlush = false; // true; + protected final boolean supportFlush = false; // true; private final ConcurrentLinkedDeque writeOperations; private long position; private boolean closed; @@ -42,7 +33,7 @@ public class SeaweedOutputStream extends OutputStream { private long outputIndex; private String replication = "000"; - public SeaweedOutputStream(FilerGrpcClient filerGrpcClient, final Path path, FilerProto.Entry.Builder entry, + public SeaweedOutputStream(FilerGrpcClient filerGrpcClient, final String path, FilerProto.Entry.Builder entry, final long position, final int bufferSize, final String replication) { this.filerGrpcClient = filerGrpcClient; this.replication = replication; @@ -70,6 +61,14 @@ public class SeaweedOutputStream extends OutputStream { } + public static String getParentDirectory(String path) { + if (path.equals("/")) { + return path; + } + int lastSlashIndex = path.lastIndexOf("/"); + return path.substring(0, lastSlashIndex); + } + private synchronized void flushWrittenBytesToServiceInternal(final long offset) throws IOException { try { SeaweedWrite.writeMeta(filerGrpcClient, getParentDirectory(path), entry); @@ -89,7 +88,9 @@ public class SeaweedOutputStream extends OutputStream { throws IOException { maybeThrowLastError(); - Preconditions.checkArgument(data != null, "null data"); + if (data == null) { + return; + } if (off < 0 || length < 0 || length > data.length - off) { throw new IndexOutOfBoundsException(); @@ -152,7 +153,7 @@ public class SeaweedOutputStream extends OutputStream { flushInternal(); threadExecutor.shutdown(); } finally { - lastError = new IOException(FSExceptionMessages.STREAM_IS_CLOSED); + lastError = new IOException("Stream is closed!"); ByteBufferPool.release(buffer); buffer = null; outputIndex = 0; @@ -185,7 +186,7 @@ public class SeaweedOutputStream extends OutputStream { } final Future job = completionService.submit(() -> { // System.out.println(path + " is going to save [" + (writePosition) + "," + ((writePosition) + bytesLength) + ")"); - SeaweedWrite.writeData(entry, replication, filerGrpcClient, writePosition, bufferToWrite.array(), bufferToWrite.position(), bufferToWrite.limit(), path.toUri().getPath()); + SeaweedWrite.writeData(entry, replication, filerGrpcClient, writePosition, bufferToWrite.array(), bufferToWrite.position(), bufferToWrite.limit(), path); // System.out.println(path + " saved [" + (writePosition) + "," + ((writePosition) + bytesLength) + ")"); ByteBufferPool.release(bufferToWrite); return null; @@ -239,13 +240,13 @@ public class SeaweedOutputStream extends OutputStream { } } - private synchronized void flushInternal() throws IOException { + protected synchronized void flushInternal() throws IOException { maybeThrowLastError(); writeCurrentBufferToService(); flushWrittenBytesToService(); } - private synchronized void flushInternalAsync() throws IOException { + protected synchronized void flushInternalAsync() throws IOException { maybeThrowLastError(); writeCurrentBufferToService(); flushWrittenBytesToServiceAsync(); @@ -278,10 +279,6 @@ public class SeaweedOutputStream extends OutputStream { private final long length; WriteOperation(final Future task, final long startOffset, final long length) { - Preconditions.checkNotNull(task, "task"); - Preconditions.checkArgument(startOffset >= 0, "startOffset"); - Preconditions.checkArgument(length >= 0, "length"); - this.task = task; this.startOffset = startOffset; this.length = length; diff --git a/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java b/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java index 14b32528e..719e52579 100644 --- a/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java +++ b/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java @@ -216,7 +216,7 @@ public class SeaweedFileSystemStore { SeaweedWrite.writeMeta(filerGrpcClient, getParentDirectory(path), entry); } - return new SeaweedOutputStream(filerGrpcClient, path, entry, writePosition, bufferSize, replication); + return new SeaweedHadoopOutputStream(filerGrpcClient, path.toString(), entry, writePosition, bufferSize, replication); } diff --git a/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedHadoopOutputStream.java b/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedHadoopOutputStream.java new file mode 100644 index 000000000..f7a6225d8 --- /dev/null +++ b/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedHadoopOutputStream.java @@ -0,0 +1,16 @@ +package seaweed.hdfs; + +// adapted from org.apache.hadoop.fs.azurebfs.services.AbfsOutputStream + +import seaweedfs.client.FilerGrpcClient; +import seaweedfs.client.FilerProto; +import seaweedfs.client.SeaweedOutputStream; + +public class SeaweedHadoopOutputStream extends SeaweedOutputStream { + + public SeaweedHadoopOutputStream(FilerGrpcClient filerGrpcClient, final String path, FilerProto.Entry.Builder entry, + final long position, final int bufferSize, final String replication) { + super(filerGrpcClient, path.toString(), entry, position, bufferSize, replication); + } + +} diff --git a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java index 14b32528e..719e52579 100644 --- a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java +++ b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java @@ -216,7 +216,7 @@ public class SeaweedFileSystemStore { SeaweedWrite.writeMeta(filerGrpcClient, getParentDirectory(path), entry); } - return new SeaweedOutputStream(filerGrpcClient, path, entry, writePosition, bufferSize, replication); + return new SeaweedHadoopOutputStream(filerGrpcClient, path.toString(), entry, writePosition, bufferSize, replication); } diff --git a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedHadoopOutputStream.java b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedHadoopOutputStream.java new file mode 100644 index 000000000..f65aef619 --- /dev/null +++ b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedHadoopOutputStream.java @@ -0,0 +1,64 @@ +package seaweed.hdfs; + +// adapted from org.apache.hadoop.fs.azurebfs.services.AbfsOutputStream + +import org.apache.hadoop.fs.StreamCapabilities; +import org.apache.hadoop.fs.Syncable; +import seaweedfs.client.FilerGrpcClient; +import seaweedfs.client.FilerProto; +import seaweedfs.client.SeaweedOutputStream; + +import java.io.IOException; +import java.util.Locale; + +public class SeaweedHadoopOutputStream extends SeaweedOutputStream implements Syncable, StreamCapabilities { + + public SeaweedHadoopOutputStream(FilerGrpcClient filerGrpcClient, final String path, FilerProto.Entry.Builder entry, + final long position, final int bufferSize, final String replication) { + super(filerGrpcClient, path, entry, position, bufferSize, replication); + } + + /** + * Similar to posix fsync, flush out the data in client's user buffer + * all the way to the disk device (but the disk may have it in its cache). + * + * @throws IOException if error occurs + */ + @Override + public void hsync() throws IOException { + if (supportFlush) { + flushInternal(); + } + } + + /** + * Flush out the data in client's user buffer. After the return of + * this call, new readers will see the data. + * + * @throws IOException if any error occurs + */ + @Override + public void hflush() throws IOException { + if (supportFlush) { + flushInternal(); + } + } + + /** + * Query the stream for a specific capability. + * + * @param capability string to query the stream support for. + * @return true for hsync and hflush. + */ + @Override + public boolean hasCapability(String capability) { + switch (capability.toLowerCase(Locale.ENGLISH)) { + case StreamCapabilities.HSYNC: + case StreamCapabilities.HFLUSH: + return supportFlush; + default: + return false; + } + } + +} diff --git a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedOutputStream.java b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedOutputStream.java deleted file mode 100644 index d4c967a06..000000000 --- a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedOutputStream.java +++ /dev/null @@ -1,337 +0,0 @@ -package seaweed.hdfs; - -// adapted from org.apache.hadoop.fs.azurebfs.services.AbfsOutputStream - -import com.google.common.base.Preconditions; -import org.apache.hadoop.fs.FSExceptionMessages; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.StreamCapabilities; -import org.apache.hadoop.fs.Syncable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import seaweedfs.client.ByteBufferPool; -import seaweedfs.client.FilerGrpcClient; -import seaweedfs.client.FilerProto; -import seaweedfs.client.SeaweedWrite; - -import java.io.IOException; -import java.io.InterruptedIOException; -import java.io.OutputStream; -import java.nio.ByteBuffer; -import java.util.Locale; -import java.util.concurrent.*; - -import static seaweed.hdfs.SeaweedFileSystemStore.getParentDirectory; - -public class SeaweedOutputStream extends OutputStream implements Syncable, StreamCapabilities { - - private static final Logger LOG = LoggerFactory.getLogger(SeaweedOutputStream.class); - - private final FilerGrpcClient filerGrpcClient; - private final Path path; - private final int bufferSize; - private final int maxConcurrentRequestCount; - private final ThreadPoolExecutor threadExecutor; - private final ExecutorCompletionService completionService; - private final FilerProto.Entry.Builder entry; - private final boolean supportFlush = false; // true; - private final ConcurrentLinkedDeque writeOperations; - private long position; - private boolean closed; - private volatile IOException lastError; - private long lastFlushOffset; - private long lastTotalAppendOffset = 0; - private ByteBuffer buffer; - private long outputIndex; - private String replication = "000"; - - public SeaweedOutputStream(FilerGrpcClient filerGrpcClient, final Path path, FilerProto.Entry.Builder entry, - final long position, final int bufferSize, final String replication) { - this.filerGrpcClient = filerGrpcClient; - this.replication = replication; - this.path = path; - this.position = position; - this.closed = false; - this.lastError = null; - this.lastFlushOffset = 0; - this.bufferSize = bufferSize; - this.buffer = ByteBufferPool.request(bufferSize); - this.outputIndex = 0; - this.writeOperations = new ConcurrentLinkedDeque<>(); - - this.maxConcurrentRequestCount = Runtime.getRuntime().availableProcessors(); - - this.threadExecutor - = new ThreadPoolExecutor(maxConcurrentRequestCount, - maxConcurrentRequestCount, - 120L, - TimeUnit.SECONDS, - new LinkedBlockingQueue()); - this.completionService = new ExecutorCompletionService<>(this.threadExecutor); - - this.entry = entry; - - } - - private synchronized void flushWrittenBytesToServiceInternal(final long offset) throws IOException { - try { - SeaweedWrite.writeMeta(filerGrpcClient, getParentDirectory(path), entry); - } catch (Exception ex) { - throw new IOException(ex); - } - this.lastFlushOffset = offset; - } - - @Override - public void write(final int byteVal) throws IOException { - write(new byte[]{(byte) (byteVal & 0xFF)}); - } - - @Override - public synchronized void write(final byte[] data, final int off, final int length) - throws IOException { - maybeThrowLastError(); - - Preconditions.checkArgument(data != null, "null data"); - - if (off < 0 || length < 0 || length > data.length - off) { - throw new IndexOutOfBoundsException(); - } - - // System.out.println(path + " write [" + (outputIndex + off) + "," + ((outputIndex + off) + length) + ")"); - - int currentOffset = off; - int writableBytes = bufferSize - buffer.position(); - int numberOfBytesToWrite = length; - - while (numberOfBytesToWrite > 0) { - - if (numberOfBytesToWrite < writableBytes) { - buffer.put(data, currentOffset, numberOfBytesToWrite); - outputIndex += numberOfBytesToWrite; - break; - } - - // System.out.println(path + " [" + (outputIndex + currentOffset) + "," + ((outputIndex + currentOffset) + writableBytes) + ") " + buffer.capacity()); - buffer.put(data, currentOffset, writableBytes); - outputIndex += writableBytes; - currentOffset += writableBytes; - writeCurrentBufferToService(); - numberOfBytesToWrite = numberOfBytesToWrite - writableBytes; - writableBytes = bufferSize - buffer.position(); - } - - } - - /** - * Flushes this output stream and forces any buffered output bytes to be - * written out. If any data remains in the payload it is committed to the - * service. Data is queued for writing and forced out to the service - * before the call returns. - */ - @Override - public void flush() throws IOException { - if (supportFlush) { - flushInternalAsync(); - } - } - - /** - * Similar to posix fsync, flush out the data in client's user buffer - * all the way to the disk device (but the disk may have it in its cache). - * - * @throws IOException if error occurs - */ - @Override - public void hsync() throws IOException { - if (supportFlush) { - flushInternal(); - } - } - - /** - * Flush out the data in client's user buffer. After the return of - * this call, new readers will see the data. - * - * @throws IOException if any error occurs - */ - @Override - public void hflush() throws IOException { - if (supportFlush) { - flushInternal(); - } - } - - /** - * Query the stream for a specific capability. - * - * @param capability string to query the stream support for. - * @return true for hsync and hflush. - */ - @Override - public boolean hasCapability(String capability) { - switch (capability.toLowerCase(Locale.ENGLISH)) { - case StreamCapabilities.HSYNC: - case StreamCapabilities.HFLUSH: - return supportFlush; - default: - return false; - } - } - - /** - * Force all data in the output stream to be written to Azure storage. - * Wait to return until this is complete. Close the access to the stream and - * shutdown the upload thread pool. - * If the blob was created, its lease will be released. - * Any error encountered caught in threads and stored will be rethrown here - * after cleanup. - */ - @Override - public synchronized void close() throws IOException { - if (closed) { - return; - } - - LOG.debug("close path: {}", path); - try { - flushInternal(); - threadExecutor.shutdown(); - } finally { - lastError = new IOException(FSExceptionMessages.STREAM_IS_CLOSED); - ByteBufferPool.release(buffer); - buffer = null; - outputIndex = 0; - closed = true; - writeOperations.clear(); - if (!threadExecutor.isShutdown()) { - threadExecutor.shutdownNow(); - } - } - } - - private synchronized void writeCurrentBufferToService() throws IOException { - if (buffer.position() == 0) { - return; - } - - position += submitWriteBufferToService(buffer, position); - - buffer = ByteBufferPool.request(bufferSize); - - } - - private synchronized int submitWriteBufferToService(final ByteBuffer bufferToWrite, final long writePosition) throws IOException { - - bufferToWrite.flip(); - int bytesLength = bufferToWrite.limit() - bufferToWrite.position(); - - if (threadExecutor.getQueue().size() >= maxConcurrentRequestCount) { - waitForTaskToComplete(); - } - final Future job = completionService.submit(() -> { - // System.out.println(path + " is going to save [" + (writePosition) + "," + ((writePosition) + bytesLength) + ")"); - SeaweedWrite.writeData(entry, replication, filerGrpcClient, writePosition, bufferToWrite.array(), bufferToWrite.position(), bufferToWrite.limit(), path.toUri().getPath()); - // System.out.println(path + " saved [" + (writePosition) + "," + ((writePosition) + bytesLength) + ")"); - ByteBufferPool.release(bufferToWrite); - return null; - }); - - writeOperations.add(new WriteOperation(job, writePosition, bytesLength)); - - // Try to shrink the queue - shrinkWriteOperationQueue(); - - return bytesLength; - - } - - private void waitForTaskToComplete() throws IOException { - boolean completed; - for (completed = false; completionService.poll() != null; completed = true) { - // keep polling until there is no data - } - - if (!completed) { - try { - completionService.take(); - } catch (InterruptedException e) { - lastError = (IOException) new InterruptedIOException(e.toString()).initCause(e); - throw lastError; - } - } - } - - private void maybeThrowLastError() throws IOException { - if (lastError != null) { - throw lastError; - } - } - - /** - * Try to remove the completed write operations from the beginning of write - * operation FIFO queue. - */ - private synchronized void shrinkWriteOperationQueue() throws IOException { - try { - while (writeOperations.peek() != null && writeOperations.peek().task.isDone()) { - writeOperations.peek().task.get(); - lastTotalAppendOffset += writeOperations.peek().length; - writeOperations.remove(); - } - } catch (Exception e) { - lastError = new IOException(e); - throw lastError; - } - } - - private synchronized void flushInternal() throws IOException { - maybeThrowLastError(); - writeCurrentBufferToService(); - flushWrittenBytesToService(); - } - - private synchronized void flushInternalAsync() throws IOException { - maybeThrowLastError(); - writeCurrentBufferToService(); - flushWrittenBytesToServiceAsync(); - } - - private synchronized void flushWrittenBytesToService() throws IOException { - for (WriteOperation writeOperation : writeOperations) { - try { - writeOperation.task.get(); - } catch (Exception ex) { - lastError = new IOException(ex); - throw lastError; - } - } - LOG.debug("flushWrittenBytesToService: {} position:{}", path, position); - flushWrittenBytesToServiceInternal(position); - } - - private synchronized void flushWrittenBytesToServiceAsync() throws IOException { - shrinkWriteOperationQueue(); - - if (this.lastTotalAppendOffset > this.lastFlushOffset) { - this.flushWrittenBytesToServiceInternal(this.lastTotalAppendOffset); - } - } - - private static class WriteOperation { - private final Future task; - private final long startOffset; - private final long length; - - WriteOperation(final Future task, final long startOffset, final long length) { - Preconditions.checkNotNull(task, "task"); - Preconditions.checkArgument(startOffset >= 0, "startOffset"); - Preconditions.checkArgument(length >= 0, "length"); - - this.task = task; - this.startOffset = startOffset; - this.length = length; - } - } - -} From 6f4aab51f9d7e8d9721c3c4fd7b57da7ab8b6d8e Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 30 Jan 2021 06:16:02 -0800 Subject: [PATCH 217/260] refactoring SeaweedInputStream --- .../seaweedfs/client}/SeaweedInputStream.java | 93 +------ .../com/seaweedfs/examples/UnzipFile.java | 4 +- .../seaweed/hdfs/SeaweedFileSystemStore.java | 2 +- .../hdfs/SeaweedHadoopInputStream.java | 150 ++++++++++ .../seaweed/hdfs/SeaweedFileSystemStore.java | 2 +- .../hdfs/SeaweedHadoopInputStream.java | 150 ++++++++++ .../java/seaweed/hdfs/SeaweedInputStream.java | 259 ------------------ 7 files changed, 317 insertions(+), 343 deletions(-) rename other/java/{hdfs2/src/main/java/seaweed/hdfs => client/src/main/java/seaweedfs/client}/SeaweedInputStream.java (64%) create mode 100644 other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedHadoopInputStream.java create mode 100644 other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedHadoopInputStream.java delete mode 100644 other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedInputStream.java diff --git a/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedInputStream.java b/other/java/client/src/main/java/seaweedfs/client/SeaweedInputStream.java similarity index 64% rename from other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedInputStream.java rename to other/java/client/src/main/java/seaweedfs/client/SeaweedInputStream.java index 690366849..312e77aa2 100644 --- a/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedInputStream.java +++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedInputStream.java @@ -1,28 +1,22 @@ -package seaweed.hdfs; +package seaweedfs.client; // based on org.apache.hadoop.fs.azurebfs.services.AbfsInputStream -import org.apache.hadoop.fs.ByteBufferReadable; -import org.apache.hadoop.fs.FSExceptionMessages; -import org.apache.hadoop.fs.FSInputStream; -import org.apache.hadoop.fs.FileSystem.Statistics; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import seaweedfs.client.FilerGrpcClient; -import seaweedfs.client.FilerProto; -import seaweedfs.client.SeaweedRead; import java.io.EOFException; import java.io.IOException; +import java.io.InputStream; import java.nio.ByteBuffer; import java.util.List; -public class SeaweedInputStream extends FSInputStream implements ByteBufferReadable { +public class SeaweedInputStream extends InputStream { private static final Logger LOG = LoggerFactory.getLogger(SeaweedInputStream.class); + private static final IOException EXCEPTION_STREAM_IS_CLOSED = new IOException("Stream is closed!"); private final FilerGrpcClient filerGrpcClient; - private final Statistics statistics; private final String path; private final FilerProto.Entry entry; private final List visibleIntervalList; @@ -34,11 +28,9 @@ public class SeaweedInputStream extends FSInputStream implements ByteBufferReada public SeaweedInputStream( final FilerGrpcClient filerGrpcClient, - final Statistics statistics, final String path, final FilerProto.Entry entry) throws IOException { this.filerGrpcClient = filerGrpcClient; - this.statistics = statistics; this.path = path; this.entry = entry; this.contentLength = SeaweedRead.fileSize(entry); @@ -86,7 +78,6 @@ public class SeaweedInputStream extends FSInputStream implements ByteBufferReada } // implement ByteBufferReadable - @Override public synchronized int read(ByteBuffer buf) throws IOException { if (position < 0) { @@ -111,45 +102,32 @@ public class SeaweedInputStream extends FSInputStream implements ByteBufferReada if (bytesRead > 0) { this.position += bytesRead; - if (statistics != null) { - statistics.incrementBytesRead(bytesRead); - } } return (int) bytesRead; } - /** - * Seek to given position in stream. - * - * @param n position to seek to - * @throws IOException if there is an error - * @throws EOFException if attempting to seek past end of file - */ - @Override public synchronized void seek(long n) throws IOException { if (closed) { - throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED); + throw EXCEPTION_STREAM_IS_CLOSED; } if (n < 0) { - throw new EOFException(FSExceptionMessages.NEGATIVE_SEEK); + throw new EOFException("Cannot seek to a negative offset"); } if (n > contentLength) { - throw new EOFException(FSExceptionMessages.CANNOT_SEEK_PAST_EOF); + throw new EOFException("Attempted to seek or read past the end of the file"); } - this.position = n; - } @Override public synchronized long skip(long n) throws IOException { if (closed) { - throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED); + throw EXCEPTION_STREAM_IS_CLOSED; } if (this.position == contentLength) { if (n > 0) { - throw new EOFException(FSExceptionMessages.CANNOT_SEEK_PAST_EOF); + throw new EOFException("Attempted to seek or read past the end of the file"); } } long newPos = this.position + n; @@ -177,10 +155,9 @@ public class SeaweedInputStream extends FSInputStream implements ByteBufferReada @Override public synchronized int available() throws IOException { if (closed) { - throw new IOException( - FSExceptionMessages.STREAM_IS_CLOSED); + throw EXCEPTION_STREAM_IS_CLOSED; } - final long remaining = this.contentLength - this.getPos(); + final long remaining = this.contentLength - this.position; return remaining <= Integer.MAX_VALUE ? (int) remaining : Integer.MAX_VALUE; } @@ -195,65 +172,21 @@ public class SeaweedInputStream extends FSInputStream implements ByteBufferReada */ public long length() throws IOException { if (closed) { - throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED); + throw EXCEPTION_STREAM_IS_CLOSED; } return contentLength; } - /** - * Return the current offset from the start of the file - * - * @throws IOException throws {@link IOException} if there is an error - */ - @Override public synchronized long getPos() throws IOException { if (closed) { - throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED); + throw EXCEPTION_STREAM_IS_CLOSED; } return position; } - /** - * Seeks a different copy of the data. Returns true if - * found a new source, false otherwise. - * - * @throws IOException throws {@link IOException} if there is an error - */ - @Override - public boolean seekToNewSource(long l) throws IOException { - return false; - } - @Override public synchronized void close() throws IOException { closed = true; } - /** - * Not supported by this stream. Throws {@link UnsupportedOperationException} - * - * @param readlimit ignored - */ - @Override - public synchronized void mark(int readlimit) { - throw new UnsupportedOperationException("mark()/reset() not supported on this stream"); - } - - /** - * Not supported by this stream. Throws {@link UnsupportedOperationException} - */ - @Override - public synchronized void reset() throws IOException { - throw new UnsupportedOperationException("mark()/reset() not supported on this stream"); - } - - /** - * gets whether mark and reset are supported by {@code ADLFileInputStream}. Always returns false. - * - * @return always {@code false} - */ - @Override - public boolean markSupported() { - return false; - } } diff --git a/other/java/examples/src/main/java/com/seaweedfs/examples/UnzipFile.java b/other/java/examples/src/main/java/com/seaweedfs/examples/UnzipFile.java index 0529a5c73..317327be4 100644 --- a/other/java/examples/src/main/java/com/seaweedfs/examples/UnzipFile.java +++ b/other/java/examples/src/main/java/com/seaweedfs/examples/UnzipFile.java @@ -1,6 +1,6 @@ package com.seaweedfs.examples; -import seaweed.hdfs.SeaweedInputStream; +import seaweed.hdfs.SeaweedHadoopInputStream; import seaweedfs.client.FilerClient; import seaweedfs.client.FilerGrpcClient; @@ -24,7 +24,7 @@ public class UnzipFile { long localProcessTime = startTime2 - startTime; - SeaweedInputStream seaweedInputStream = new SeaweedInputStream( + SeaweedHadoopInputStream seaweedInputStream = new SeaweedHadoopInputStream( filerGrpcClient, new org.apache.hadoop.fs.FileSystem.Statistics(""), "/", diff --git a/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java b/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java index 719e52579..223036c13 100644 --- a/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java +++ b/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java @@ -230,7 +230,7 @@ public class SeaweedFileSystemStore { throw new FileNotFoundException("read non-exist file " + path); } - return new SeaweedInputStream(filerGrpcClient, + return new SeaweedHadoopInputStream(filerGrpcClient, statistics, path.toUri().getPath(), entry); diff --git a/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedHadoopInputStream.java b/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedHadoopInputStream.java new file mode 100644 index 000000000..dd9bf4032 --- /dev/null +++ b/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedHadoopInputStream.java @@ -0,0 +1,150 @@ +package seaweed.hdfs; + +// based on org.apache.hadoop.fs.azurebfs.services.AbfsInputStream + +import org.apache.hadoop.fs.ByteBufferReadable; +import org.apache.hadoop.fs.FSInputStream; +import org.apache.hadoop.fs.FileSystem.Statistics; +import seaweedfs.client.FilerGrpcClient; +import seaweedfs.client.FilerProto; +import seaweedfs.client.SeaweedInputStream; + +import java.io.EOFException; +import java.io.IOException; +import java.nio.ByteBuffer; + +public class SeaweedHadoopInputStream extends FSInputStream implements ByteBufferReadable { + + private final SeaweedInputStream seaweedInputStream; + private final Statistics statistics; + + public SeaweedHadoopInputStream( + final FilerGrpcClient filerGrpcClient, + final Statistics statistics, + final String path, + final FilerProto.Entry entry) throws IOException { + this.seaweedInputStream = new SeaweedInputStream(filerGrpcClient, path, entry); + this.statistics = statistics; + } + + @Override + public int read() throws IOException { + return seaweedInputStream.read(); + } + + @Override + public int read(final byte[] b, final int off, final int len) throws IOException { + return seaweedInputStream.read(b, off, len); + } + + // implement ByteBufferReadable + @Override + public synchronized int read(ByteBuffer buf) throws IOException { + int bytesRead = seaweedInputStream.read(buf); + + if (bytesRead > 0) { + if (statistics != null) { + statistics.incrementBytesRead(bytesRead); + } + } + + return bytesRead; + } + + /** + * Seek to given position in stream. + * + * @param n position to seek to + * @throws IOException if there is an error + * @throws EOFException if attempting to seek past end of file + */ + @Override + public synchronized void seek(long n) throws IOException { + seaweedInputStream.seek(n); + } + + @Override + public synchronized long skip(long n) throws IOException { + return seaweedInputStream.skip(n); + } + + /** + * Return the size of the remaining available bytes + * if the size is less than or equal to {@link Integer#MAX_VALUE}, + * otherwise, return {@link Integer#MAX_VALUE}. + *

+ * This is to match the behavior of DFSInputStream.available(), + * which some clients may rely on (HBase write-ahead log reading in + * particular). + */ + @Override + public synchronized int available() throws IOException { + return seaweedInputStream.available(); + } + + /** + * Returns the length of the file that this stream refers to. Note that the length returned is the length + * as of the time the Stream was opened. Specifically, if there have been subsequent appends to the file, + * they wont be reflected in the returned length. + * + * @return length of the file. + * @throws IOException if the stream is closed + */ + public long length() throws IOException { + return seaweedInputStream.length(); + } + + /** + * Return the current offset from the start of the file + * + * @throws IOException throws {@link IOException} if there is an error + */ + @Override + public synchronized long getPos() throws IOException { + return seaweedInputStream.getPos(); + } + + /** + * Seeks a different copy of the data. Returns true if + * found a new source, false otherwise. + * + * @throws IOException throws {@link IOException} if there is an error + */ + @Override + public boolean seekToNewSource(long l) throws IOException { + return false; + } + + @Override + public synchronized void close() throws IOException { + seaweedInputStream.close(); + } + + /** + * Not supported by this stream. Throws {@link UnsupportedOperationException} + * + * @param readlimit ignored + */ + @Override + public synchronized void mark(int readlimit) { + throw new UnsupportedOperationException("mark()/reset() not supported on this stream"); + } + + /** + * Not supported by this stream. Throws {@link UnsupportedOperationException} + */ + @Override + public synchronized void reset() throws IOException { + throw new UnsupportedOperationException("mark()/reset() not supported on this stream"); + } + + /** + * gets whether mark and reset are supported by {@code ADLFileInputStream}. Always returns false. + * + * @return always {@code false} + */ + @Override + public boolean markSupported() { + return false; + } +} diff --git a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java index 719e52579..223036c13 100644 --- a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java +++ b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java @@ -230,7 +230,7 @@ public class SeaweedFileSystemStore { throw new FileNotFoundException("read non-exist file " + path); } - return new SeaweedInputStream(filerGrpcClient, + return new SeaweedHadoopInputStream(filerGrpcClient, statistics, path.toUri().getPath(), entry); diff --git a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedHadoopInputStream.java b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedHadoopInputStream.java new file mode 100644 index 000000000..dd9bf4032 --- /dev/null +++ b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedHadoopInputStream.java @@ -0,0 +1,150 @@ +package seaweed.hdfs; + +// based on org.apache.hadoop.fs.azurebfs.services.AbfsInputStream + +import org.apache.hadoop.fs.ByteBufferReadable; +import org.apache.hadoop.fs.FSInputStream; +import org.apache.hadoop.fs.FileSystem.Statistics; +import seaweedfs.client.FilerGrpcClient; +import seaweedfs.client.FilerProto; +import seaweedfs.client.SeaweedInputStream; + +import java.io.EOFException; +import java.io.IOException; +import java.nio.ByteBuffer; + +public class SeaweedHadoopInputStream extends FSInputStream implements ByteBufferReadable { + + private final SeaweedInputStream seaweedInputStream; + private final Statistics statistics; + + public SeaweedHadoopInputStream( + final FilerGrpcClient filerGrpcClient, + final Statistics statistics, + final String path, + final FilerProto.Entry entry) throws IOException { + this.seaweedInputStream = new SeaweedInputStream(filerGrpcClient, path, entry); + this.statistics = statistics; + } + + @Override + public int read() throws IOException { + return seaweedInputStream.read(); + } + + @Override + public int read(final byte[] b, final int off, final int len) throws IOException { + return seaweedInputStream.read(b, off, len); + } + + // implement ByteBufferReadable + @Override + public synchronized int read(ByteBuffer buf) throws IOException { + int bytesRead = seaweedInputStream.read(buf); + + if (bytesRead > 0) { + if (statistics != null) { + statistics.incrementBytesRead(bytesRead); + } + } + + return bytesRead; + } + + /** + * Seek to given position in stream. + * + * @param n position to seek to + * @throws IOException if there is an error + * @throws EOFException if attempting to seek past end of file + */ + @Override + public synchronized void seek(long n) throws IOException { + seaweedInputStream.seek(n); + } + + @Override + public synchronized long skip(long n) throws IOException { + return seaweedInputStream.skip(n); + } + + /** + * Return the size of the remaining available bytes + * if the size is less than or equal to {@link Integer#MAX_VALUE}, + * otherwise, return {@link Integer#MAX_VALUE}. + *

+ * This is to match the behavior of DFSInputStream.available(), + * which some clients may rely on (HBase write-ahead log reading in + * particular). + */ + @Override + public synchronized int available() throws IOException { + return seaweedInputStream.available(); + } + + /** + * Returns the length of the file that this stream refers to. Note that the length returned is the length + * as of the time the Stream was opened. Specifically, if there have been subsequent appends to the file, + * they wont be reflected in the returned length. + * + * @return length of the file. + * @throws IOException if the stream is closed + */ + public long length() throws IOException { + return seaweedInputStream.length(); + } + + /** + * Return the current offset from the start of the file + * + * @throws IOException throws {@link IOException} if there is an error + */ + @Override + public synchronized long getPos() throws IOException { + return seaweedInputStream.getPos(); + } + + /** + * Seeks a different copy of the data. Returns true if + * found a new source, false otherwise. + * + * @throws IOException throws {@link IOException} if there is an error + */ + @Override + public boolean seekToNewSource(long l) throws IOException { + return false; + } + + @Override + public synchronized void close() throws IOException { + seaweedInputStream.close(); + } + + /** + * Not supported by this stream. Throws {@link UnsupportedOperationException} + * + * @param readlimit ignored + */ + @Override + public synchronized void mark(int readlimit) { + throw new UnsupportedOperationException("mark()/reset() not supported on this stream"); + } + + /** + * Not supported by this stream. Throws {@link UnsupportedOperationException} + */ + @Override + public synchronized void reset() throws IOException { + throw new UnsupportedOperationException("mark()/reset() not supported on this stream"); + } + + /** + * gets whether mark and reset are supported by {@code ADLFileInputStream}. Always returns false. + * + * @return always {@code false} + */ + @Override + public boolean markSupported() { + return false; + } +} diff --git a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedInputStream.java b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedInputStream.java deleted file mode 100644 index 690366849..000000000 --- a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedInputStream.java +++ /dev/null @@ -1,259 +0,0 @@ -package seaweed.hdfs; - -// based on org.apache.hadoop.fs.azurebfs.services.AbfsInputStream - -import org.apache.hadoop.fs.ByteBufferReadable; -import org.apache.hadoop.fs.FSExceptionMessages; -import org.apache.hadoop.fs.FSInputStream; -import org.apache.hadoop.fs.FileSystem.Statistics; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import seaweedfs.client.FilerGrpcClient; -import seaweedfs.client.FilerProto; -import seaweedfs.client.SeaweedRead; - -import java.io.EOFException; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.List; - -public class SeaweedInputStream extends FSInputStream implements ByteBufferReadable { - - private static final Logger LOG = LoggerFactory.getLogger(SeaweedInputStream.class); - - private final FilerGrpcClient filerGrpcClient; - private final Statistics statistics; - private final String path; - private final FilerProto.Entry entry; - private final List visibleIntervalList; - private final long contentLength; - - private long position = 0; // cursor of the file - - private boolean closed = false; - - public SeaweedInputStream( - final FilerGrpcClient filerGrpcClient, - final Statistics statistics, - final String path, - final FilerProto.Entry entry) throws IOException { - this.filerGrpcClient = filerGrpcClient; - this.statistics = statistics; - this.path = path; - this.entry = entry; - this.contentLength = SeaweedRead.fileSize(entry); - - this.visibleIntervalList = SeaweedRead.nonOverlappingVisibleIntervals(filerGrpcClient, entry.getChunksList()); - - LOG.debug("new path:{} entry:{} visibleIntervalList:{}", path, entry, visibleIntervalList); - - } - - public String getPath() { - return path; - } - - @Override - public int read() throws IOException { - byte[] b = new byte[1]; - int numberOfBytesRead = read(b, 0, 1); - if (numberOfBytesRead < 0) { - return -1; - } else { - return (b[0] & 0xFF); - } - } - - @Override - public int read(final byte[] b, final int off, final int len) throws IOException { - - if (b == null) { - throw new IllegalArgumentException("null byte array passed in to read() method"); - } - if (off >= b.length) { - throw new IllegalArgumentException("offset greater than length of array"); - } - if (len < 0) { - throw new IllegalArgumentException("requested read length is less than zero"); - } - if (len > (b.length - off)) { - throw new IllegalArgumentException("requested read length is more than will fit after requested offset in buffer"); - } - - ByteBuffer buf = ByteBuffer.wrap(b, off, len); - return read(buf); - - } - - // implement ByteBufferReadable - @Override - public synchronized int read(ByteBuffer buf) throws IOException { - - if (position < 0) { - throw new IllegalArgumentException("attempting to read from negative offset"); - } - if (position >= contentLength) { - return -1; // Hadoop prefers -1 to EOFException - } - - long bytesRead = 0; - int len = buf.remaining(); - int start = (int) this.position; - if (start+len <= entry.getContent().size()) { - entry.getContent().substring(start, start+len).copyTo(buf); - } else { - bytesRead = SeaweedRead.read(this.filerGrpcClient, this.visibleIntervalList, this.position, buf, SeaweedRead.fileSize(entry)); - } - - if (bytesRead > Integer.MAX_VALUE) { - throw new IOException("Unexpected Content-Length"); - } - - if (bytesRead > 0) { - this.position += bytesRead; - if (statistics != null) { - statistics.incrementBytesRead(bytesRead); - } - } - - return (int) bytesRead; - } - - /** - * Seek to given position in stream. - * - * @param n position to seek to - * @throws IOException if there is an error - * @throws EOFException if attempting to seek past end of file - */ - @Override - public synchronized void seek(long n) throws IOException { - if (closed) { - throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED); - } - if (n < 0) { - throw new EOFException(FSExceptionMessages.NEGATIVE_SEEK); - } - if (n > contentLength) { - throw new EOFException(FSExceptionMessages.CANNOT_SEEK_PAST_EOF); - } - - this.position = n; - - } - - @Override - public synchronized long skip(long n) throws IOException { - if (closed) { - throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED); - } - if (this.position == contentLength) { - if (n > 0) { - throw new EOFException(FSExceptionMessages.CANNOT_SEEK_PAST_EOF); - } - } - long newPos = this.position + n; - if (newPos < 0) { - newPos = 0; - n = newPos - this.position; - } - if (newPos > contentLength) { - newPos = contentLength; - n = newPos - this.position; - } - seek(newPos); - return n; - } - - /** - * Return the size of the remaining available bytes - * if the size is less than or equal to {@link Integer#MAX_VALUE}, - * otherwise, return {@link Integer#MAX_VALUE}. - *

- * This is to match the behavior of DFSInputStream.available(), - * which some clients may rely on (HBase write-ahead log reading in - * particular). - */ - @Override - public synchronized int available() throws IOException { - if (closed) { - throw new IOException( - FSExceptionMessages.STREAM_IS_CLOSED); - } - final long remaining = this.contentLength - this.getPos(); - return remaining <= Integer.MAX_VALUE - ? (int) remaining : Integer.MAX_VALUE; - } - - /** - * Returns the length of the file that this stream refers to. Note that the length returned is the length - * as of the time the Stream was opened. Specifically, if there have been subsequent appends to the file, - * they wont be reflected in the returned length. - * - * @return length of the file. - * @throws IOException if the stream is closed - */ - public long length() throws IOException { - if (closed) { - throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED); - } - return contentLength; - } - - /** - * Return the current offset from the start of the file - * - * @throws IOException throws {@link IOException} if there is an error - */ - @Override - public synchronized long getPos() throws IOException { - if (closed) { - throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED); - } - return position; - } - - /** - * Seeks a different copy of the data. Returns true if - * found a new source, false otherwise. - * - * @throws IOException throws {@link IOException} if there is an error - */ - @Override - public boolean seekToNewSource(long l) throws IOException { - return false; - } - - @Override - public synchronized void close() throws IOException { - closed = true; - } - - /** - * Not supported by this stream. Throws {@link UnsupportedOperationException} - * - * @param readlimit ignored - */ - @Override - public synchronized void mark(int readlimit) { - throw new UnsupportedOperationException("mark()/reset() not supported on this stream"); - } - - /** - * Not supported by this stream. Throws {@link UnsupportedOperationException} - */ - @Override - public synchronized void reset() throws IOException { - throw new UnsupportedOperationException("mark()/reset() not supported on this stream"); - } - - /** - * gets whether mark and reset are supported by {@code ADLFileInputStream}. Always returns false. - * - * @return always {@code false} - */ - @Override - public boolean markSupported() { - return false; - } -} From 87d1bfa862e60b53eda7487dd523a04bf4c730c4 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 30 Jan 2021 06:18:41 -0800 Subject: [PATCH 218/260] Hadoop Compatible FS: 1.5.7 --- other/java/client/pom.xml | 2 +- other/java/client/pom.xml.deploy | 2 +- other/java/client/pom_debug.xml | 2 +- other/java/examples/pom.xml | 4 ++-- other/java/hdfs2/dependency-reduced-pom.xml | 2 +- other/java/hdfs2/pom.xml | 2 +- other/java/hdfs3/dependency-reduced-pom.xml | 2 +- other/java/hdfs3/pom.xml | 2 +- 8 files changed, 9 insertions(+), 9 deletions(-) diff --git a/other/java/client/pom.xml b/other/java/client/pom.xml index 4bfc5ab8f..60fb84082 100644 --- a/other/java/client/pom.xml +++ b/other/java/client/pom.xml @@ -5,7 +5,7 @@ com.github.chrislusf seaweedfs-client - 1.5.6 + 1.5.7 org.sonatype.oss diff --git a/other/java/client/pom.xml.deploy b/other/java/client/pom.xml.deploy index c3c960a28..b45e0bee7 100644 --- a/other/java/client/pom.xml.deploy +++ b/other/java/client/pom.xml.deploy @@ -5,7 +5,7 @@ com.github.chrislusf seaweedfs-client - 1.5.6 + 1.5.7 org.sonatype.oss diff --git a/other/java/client/pom_debug.xml b/other/java/client/pom_debug.xml index 078778b6c..26bef1655 100644 --- a/other/java/client/pom_debug.xml +++ b/other/java/client/pom_debug.xml @@ -5,7 +5,7 @@ com.github.chrislusf seaweedfs-client - 1.5.6 + 1.5.7 org.sonatype.oss diff --git a/other/java/examples/pom.xml b/other/java/examples/pom.xml index f7c48d0ab..b1f416941 100644 --- a/other/java/examples/pom.xml +++ b/other/java/examples/pom.xml @@ -11,13 +11,13 @@ com.github.chrislusf seaweedfs-client - 1.5.6 + 1.5.7 compile com.github.chrislusf seaweedfs-hadoop2-client - 1.5.6 + 1.5.7 compile diff --git a/other/java/hdfs2/dependency-reduced-pom.xml b/other/java/hdfs2/dependency-reduced-pom.xml index f7873a435..47b58b4c0 100644 --- a/other/java/hdfs2/dependency-reduced-pom.xml +++ b/other/java/hdfs2/dependency-reduced-pom.xml @@ -301,7 +301,7 @@ - 1.5.6 + 1.5.7 2.9.2 diff --git a/other/java/hdfs2/pom.xml b/other/java/hdfs2/pom.xml index bda0eba56..de6d42d98 100644 --- a/other/java/hdfs2/pom.xml +++ b/other/java/hdfs2/pom.xml @@ -5,7 +5,7 @@ 4.0.0 - 1.5.6 + 1.5.7 2.9.2 diff --git a/other/java/hdfs3/dependency-reduced-pom.xml b/other/java/hdfs3/dependency-reduced-pom.xml index 20b52e20f..78f4e38e8 100644 --- a/other/java/hdfs3/dependency-reduced-pom.xml +++ b/other/java/hdfs3/dependency-reduced-pom.xml @@ -309,7 +309,7 @@ - 1.5.6 + 1.5.7 3.1.1 diff --git a/other/java/hdfs3/pom.xml b/other/java/hdfs3/pom.xml index 85d8db859..4f097d648 100644 --- a/other/java/hdfs3/pom.xml +++ b/other/java/hdfs3/pom.xml @@ -5,7 +5,7 @@ 4.0.0 - 1.5.6 + 1.5.7 3.1.1 From 314dc1c957041501349fa731e21148c7e93d0006 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 30 Jan 2021 13:38:44 -0800 Subject: [PATCH 219/260] filer: etcd store fix listing fix https://github.com/chrislusf/seaweedfs/issues/1767 --- docker/Makefile | 3 ++ docker/swarm-etcd.yml | 84 +++++++++++++++++++++++++++++++++++ weed/filer/etcd/etcd_store.go | 12 ++++- 3 files changed, 97 insertions(+), 2 deletions(-) create mode 100644 docker/swarm-etcd.yml diff --git a/docker/Makefile b/docker/Makefile index 5949842f1..c3c541738 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -30,5 +30,8 @@ cluster: build 2clusters: build docker-compose -f local-clusters-compose.yml -p seaweedfs up +filer_etcd: build + docker stack deploy -c swarm-etcd.yml fs + clean: rm ./weed diff --git a/docker/swarm-etcd.yml b/docker/swarm-etcd.yml new file mode 100644 index 000000000..186b24790 --- /dev/null +++ b/docker/swarm-etcd.yml @@ -0,0 +1,84 @@ +# 2021-01-30 16:25:30 +version: '3.8' + +services: + + etcd: + image: gasparekatapy/etcd + networks: + - net + deploy: + mode: replicated + replicas: 3 + + master: + image: chrislusf/seaweedfs:local + environment: + WEED_MASTER_FILER_DEFAULT: "filer:8888" + WEED_MASTER_SEQUENCER_TYPE: "raft" + ports: + - "9333:9333" + - "19333:19333" + networks: + - net + command: + - 'master' + - '-resumeState=true' + - '-ip=master' + - '-port=9333' + deploy: + mode: replicated + replicas: 1 + + filer: + image: chrislusf/seaweedfs:local + environment: + WEED_LEVELDB2_ENABLED: "false" + WEED_ETCD_ENABLED: "true" + WEED_ETCD_SERVERS: "etcd:2379" + ports: + - target: 8888 + published: 8888 + protocol: tcp + mode: host + - target: 18888 + published: 18888 + protocol: tcp + mode: host + networks: + - net + command: + - 'filer' + - '-ip=filer' + - '-port=8888' + - '-port.readonly=28888' + - '-master=master:9333' + - '-disableDirListing=true' + deploy: + mode: replicated + replicas: 1 + + volume: + image: chrislusf/seaweedfs:local + ports: + - target: 8080 + published: 8080 + protocol: tcp + mode: host + - target: 18080 + published: 18080 + protocol: tcp + mode: host + networks: + - net + command: + - 'volume' + - '-mserver=master:9333' + - '-port=8080' + deploy: + mode: global + + ########################################################################### + +networks: + net: diff --git a/weed/filer/etcd/etcd_store.go b/weed/filer/etcd/etcd_store.go index 8159c634d..71ed738f9 100644 --- a/weed/filer/etcd/etcd_store.go +++ b/weed/filer/etcd/etcd_store.go @@ -1,6 +1,7 @@ package etcd import ( + "bytes" "context" "fmt" "strings" @@ -145,14 +146,21 @@ func (store *EtcdStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPat func (store *EtcdStore) ListDirectoryEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { directoryPrefix := genDirectoryKeyPrefix(dirPath, "") + lastFileStart := directoryPrefix + if startFileName != "" { + lastFileStart = genDirectoryKeyPrefix(dirPath, startFileName) + } - resp, err := store.client.Get(ctx, string(directoryPrefix), + resp, err := store.client.Get(ctx, string(lastFileStart), clientv3.WithPrefix(), clientv3.WithSort(clientv3.SortByKey, clientv3.SortDescend)) if err != nil { return lastFileName, fmt.Errorf("list %s : %v", dirPath, err) } for _, kv := range resp.Kvs { + if !bytes.HasPrefix(kv.Key, directoryPrefix) { + break + } fileName := getNameFromKey(kv.Key) if fileName == "" { continue @@ -160,7 +168,6 @@ func (store *EtcdStore) ListDirectoryEntries(ctx context.Context, dirPath weed_u if fileName == startFileName && !includeStartFile { continue } - lastFileName = fileName limit-- if limit < 0 { break @@ -176,6 +183,7 @@ func (store *EtcdStore) ListDirectoryEntries(ctx context.Context, dirPath weed_u if !eachEntryFunc(entry) { break } + lastFileName = fileName } return lastFileName, err From 1102ae32c4012f37fe4ce6d31758f8802f9e644d Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 31 Jan 2021 18:26:26 -0800 Subject: [PATCH 220/260] fix concurrent map reads --- weed/topology/volume_layout.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/topology/volume_layout.go b/weed/topology/volume_layout.go index 0db95d289..e2d6c170c 100644 --- a/weed/topology/volume_layout.go +++ b/weed/topology/volume_layout.go @@ -218,7 +218,7 @@ func (vl *VolumeLayout) ensureCorrectWritables(vid needle.VolumeId) { func (vl *VolumeLayout) isAllWritable(vid needle.VolumeId) bool { for _, dn := range vl.vid2location[vid].list { - if v, found := dn.volumes[vid]; found { + if v, getError := dn.GetVolumesById(vid); getError == nil { if v.ReadOnly { return false } From 4be51c07014b1072a4e9dbbf9303a13e94fefe5e Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 31 Jan 2021 20:11:44 -0800 Subject: [PATCH 221/260] filer: leveldb and hbase may miss files when listing large directories more than 1024 fix https://github.com/chrislusf/seaweedfs/issues/1768 --- weed/filer/hbase/hbase_store.go | 5 +++-- weed/filer/leveldb/leveldb_store.go | 2 +- weed/filer/leveldb2/leveldb2_store.go | 2 +- weed/filer/leveldb3/leveldb3_store.go | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/weed/filer/hbase/hbase_store.go b/weed/filer/hbase/hbase_store.go index 2e4491515..e0d878ca7 100644 --- a/weed/filer/hbase/hbase_store.go +++ b/weed/filer/hbase/hbase_store.go @@ -185,8 +185,6 @@ func (store *HbaseStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPa continue } - lastFileName = fileName - value := cell.Value if fileName == startFileName && !includeStartFile { @@ -197,6 +195,9 @@ func (store *HbaseStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPa if limit < 0 { break } + + lastFileName = fileName + entry := &filer.Entry{ FullPath: fullpath, } diff --git a/weed/filer/leveldb/leveldb_store.go b/weed/filer/leveldb/leveldb_store.go index f0ae64769..ac0ad4ba6 100644 --- a/weed/filer/leveldb/leveldb_store.go +++ b/weed/filer/leveldb/leveldb_store.go @@ -187,11 +187,11 @@ func (store *LevelDBStore) ListDirectoryPrefixedEntries(ctx context.Context, dir if fileName == startFileName && !includeStartFile { continue } - lastFileName = fileName limit-- if limit < 0 { break } + lastFileName = fileName entry := &filer.Entry{ FullPath: weed_util.NewFullPath(string(dirPath), fileName), } diff --git a/weed/filer/leveldb2/leveldb2_store.go b/weed/filer/leveldb2/leveldb2_store.go index 965721460..c76340da4 100644 --- a/weed/filer/leveldb2/leveldb2_store.go +++ b/weed/filer/leveldb2/leveldb2_store.go @@ -196,11 +196,11 @@ func (store *LevelDB2Store) ListDirectoryPrefixedEntries(ctx context.Context, di if fileName == startFileName && !includeStartFile { continue } - lastFileName = fileName limit-- if limit < 0 { break } + lastFileName = fileName entry := &filer.Entry{ FullPath: weed_util.NewFullPath(string(dirPath), fileName), } diff --git a/weed/filer/leveldb3/leveldb3_store.go b/weed/filer/leveldb3/leveldb3_store.go index 24e00edc7..c81ea7bbe 100644 --- a/weed/filer/leveldb3/leveldb3_store.go +++ b/weed/filer/leveldb3/leveldb3_store.go @@ -316,11 +316,11 @@ func (store *LevelDB3Store) ListDirectoryPrefixedEntries(ctx context.Context, di if fileName == startFileName && !includeStartFile { continue } - lastFileName = fileName limit-- if limit < 0 { break } + lastFileName = fileName entry := &filer.Entry{ FullPath: weed_util.NewFullPath(string(dirPath), fileName), } From 318a3d2efcacb41e56ee4a027638f4be48dc14fa Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 31 Jan 2021 20:18:06 -0800 Subject: [PATCH 222/260] 2.23 --- k8s/seaweedfs/Chart.yaml | 4 ++-- k8s/seaweedfs/values.yaml | 2 +- weed/util/constants.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/k8s/seaweedfs/Chart.yaml b/k8s/seaweedfs/Chart.yaml index 112648a60..4febe5129 100644 --- a/k8s/seaweedfs/Chart.yaml +++ b/k8s/seaweedfs/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v1 description: SeaweedFS name: seaweedfs -appVersion: "2.22" -version: 2.22 +appVersion: "2.23" +version: 2.23 diff --git a/k8s/seaweedfs/values.yaml b/k8s/seaweedfs/values.yaml index 4b4833da3..168c0dbcd 100644 --- a/k8s/seaweedfs/values.yaml +++ b/k8s/seaweedfs/values.yaml @@ -4,7 +4,7 @@ global: registry: "" repository: "" imageName: chrislusf/seaweedfs - # imageTag: "2.22" - started using {.Chart.appVersion} + # imageTag: "2.23" - started using {.Chart.appVersion} imagePullPolicy: IfNotPresent imagePullSecrets: imagepullsecret restartPolicy: Always diff --git a/weed/util/constants.go b/weed/util/constants.go index b9a9bd492..ccc0ef1fb 100644 --- a/weed/util/constants.go +++ b/weed/util/constants.go @@ -5,7 +5,7 @@ import ( ) var ( - VERSION = fmt.Sprintf("%s %d.%02d", sizeLimit, 2, 22) + VERSION = fmt.Sprintf("%s %d.%02d", sizeLimit, 2, 23) COMMIT = "" ) From 76430790b996b868dc8e9bdcbd4026b9e0a0c499 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 31 Jan 2021 22:02:03 -0800 Subject: [PATCH 223/260] webdav: cache to version specific folder --- weed/server/webdav_server.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/weed/server/webdav_server.go b/weed/server/webdav_server.go index feb193230..1afb71aee 100644 --- a/weed/server/webdav_server.go +++ b/weed/server/webdav_server.go @@ -105,7 +105,10 @@ type WebDavFile struct { func NewWebDavFileSystem(option *WebDavOption) (webdav.FileSystem, error) { - chunkCache := chunk_cache.NewTieredChunkCache(256, option.CacheDir, option.CacheSizeMB, 1024*1024) + cacheUniqueId := util.Md5String([]byte("webdav" + option.FilerGrpcAddress + util.Version()))[0:8] + cacheDir := path.Join(option.CacheDir, cacheUniqueId) + + chunkCache := chunk_cache.NewTieredChunkCache(256, cacheDir, option.CacheSizeMB, 1024*1024) return &WebDavFileSystem{ option: option, chunkCache: chunkCache, From d67ccb66c0525a60315a90869aee5f858f841371 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 31 Jan 2021 22:16:52 -0800 Subject: [PATCH 224/260] webdav: can start together with "weed server" or "weed filer" --- weed/command/filer.go | 27 ++++++++++++++++++++++++--- weed/command/server.go | 22 ++++++++++++++++++++++ weed/server/webdav_server.go | 1 + 3 files changed, 47 insertions(+), 3 deletions(-) diff --git a/weed/command/filer.go b/weed/command/filer.go index 633c25cac..7f665cec8 100644 --- a/weed/command/filer.go +++ b/weed/command/filer.go @@ -3,6 +3,7 @@ package command import ( "fmt" "net/http" + "os" "strconv" "strings" "time" @@ -19,9 +20,11 @@ import ( ) var ( - f FilerOptions - filerStartS3 *bool - filerS3Options S3Options + f FilerOptions + filerStartS3 *bool + filerS3Options S3Options + filerStartWebDav *bool + filerWebDavOptions WebDavOption ) type FilerOptions struct { @@ -75,6 +78,15 @@ func init() { filerS3Options.tlsCertificate = cmdFiler.Flag.String("s3.cert.file", "", "path to the TLS certificate file") filerS3Options.config = cmdFiler.Flag.String("s3.config", "", "path to the config file") filerS3Options.allowEmptyFolder = cmdFiler.Flag.Bool("s3.allowEmptyFolder", false, "allow empty folders") + + // start webdav on filer + filerStartWebDav = cmdFiler.Flag.Bool("webdav", false, "whether to start webdav gateway") + filerWebDavOptions.port = cmdFiler.Flag.Int("webdav.port", 7333, "webdav server http listen port") + filerWebDavOptions.collection = cmdFiler.Flag.String("webdav.collection", "", "collection to create the files") + filerWebDavOptions.tlsPrivateKey = cmdFiler.Flag.String("webdav.key.file", "", "path to the TLS private key file") + filerWebDavOptions.tlsCertificate = cmdFiler.Flag.String("webdav.cert.file", "", "path to the TLS certificate file") + filerWebDavOptions.cacheDir = cmdFiler.Flag.String("webdav.cacheDir", os.TempDir(), "local cache directory for file chunks") + filerWebDavOptions.cacheSizeMB = cmdFiler.Flag.Int64("webdav.cacheCapacityMB", 1000, "local cache capacity in MB") } var cmdFiler = &Command{ @@ -114,6 +126,15 @@ func runFiler(cmd *Command, args []string) bool { }() } + if *filerStartWebDav { + filerAddress := fmt.Sprintf("%s:%d", *f.ip, *f.port) + filerWebDavOptions.filer = &filerAddress + go func() { + time.Sleep(2 * time.Second) + filerWebDavOptions.startWebDav() + }() + } + f.startFiler() return true diff --git a/weed/command/server.go b/weed/command/server.go index 9976db2ea..206de34d1 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -24,6 +24,7 @@ var ( masterOptions MasterOptions filerOptions FilerOptions s3Options S3Options + webdavOptions WebDavOption msgBrokerOptions MessageBrokerOptions ) @@ -65,6 +66,7 @@ var ( isStartingVolumeServer = cmdServer.Flag.Bool("volume", true, "whether to start volume server") isStartingFiler = cmdServer.Flag.Bool("filer", false, "whether to start filer") isStartingS3 = cmdServer.Flag.Bool("s3", false, "whether to start S3 gateway") + isStartingWebDav = cmdServer.Flag.Bool("webdav", false, "whether to start WebDAV gateway") isStartingMsgBroker = cmdServer.Flag.Bool("msgBroker", false, "whether to start message broker") serverWhiteList []string @@ -116,6 +118,13 @@ func init() { s3Options.config = cmdServer.Flag.String("s3.config", "", "path to the config file") s3Options.allowEmptyFolder = cmdServer.Flag.Bool("s3.allowEmptyFolder", false, "allow empty folders") + webdavOptions.port = cmdServer.Flag.Int("webdav.port", 7333, "webdav server http listen port") + webdavOptions.collection = cmdServer.Flag.String("webdav.collection", "", "collection to create the files") + webdavOptions.tlsPrivateKey = cmdServer.Flag.String("webdav.key.file", "", "path to the TLS private key file") + webdavOptions.tlsCertificate = cmdServer.Flag.String("webdav.cert.file", "", "path to the TLS certificate file") + webdavOptions.cacheDir = cmdServer.Flag.String("webdav.cacheDir", os.TempDir(), "local cache directory for file chunks") + webdavOptions.cacheSizeMB = cmdServer.Flag.Int64("webdav.cacheCapacityMB", 1000, "local cache capacity in MB") + msgBrokerOptions.port = cmdServer.Flag.Int("msgBroker.port", 17777, "broker gRPC listen port") } @@ -137,6 +146,9 @@ func runServer(cmd *Command, args []string) bool { if *isStartingS3 { *isStartingFiler = true } + if *isStartingWebDav { + *isStartingFiler = true + } if *isStartingMsgBroker { *isStartingFiler = true } @@ -171,6 +183,7 @@ func runServer(cmd *Command, args []string) bool { filerAddress := fmt.Sprintf("%s:%d", *serverIp, *filerOptions.port) s3Options.filer = &filerAddress + webdavOptions.filer = &filerAddress msgBrokerOptions.filer = &filerAddress runtime.GOMAXPROCS(runtime.NumCPU()) @@ -212,6 +225,15 @@ func runServer(cmd *Command, args []string) bool { }() } + if *isStartingWebDav { + go func() { + time.Sleep(2 * time.Second) + + webdavOptions.startWebDav() + + }() + } + if *isStartingMsgBroker { go func() { time.Sleep(2 * time.Second) diff --git a/weed/server/webdav_server.go b/weed/server/webdav_server.go index 1afb71aee..ec9798997 100644 --- a/weed/server/webdav_server.go +++ b/weed/server/webdav_server.go @@ -108,6 +108,7 @@ func NewWebDavFileSystem(option *WebDavOption) (webdav.FileSystem, error) { cacheUniqueId := util.Md5String([]byte("webdav" + option.FilerGrpcAddress + util.Version()))[0:8] cacheDir := path.Join(option.CacheDir, cacheUniqueId) + os.MkdirAll(cacheDir, os.FileMode(0755)) chunkCache := chunk_cache.NewTieredChunkCache(256, cacheDir, option.CacheSizeMB, 1024*1024) return &WebDavFileSystem{ option: option, From 9c56b46886491c62054368206c0ca3e4e8014a17 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 31 Jan 2021 22:57:30 -0800 Subject: [PATCH 225/260] webdav: adds to docker file --- docker/Dockerfile | 2 ++ docker/Dockerfile.go_build | 2 ++ docker/Dockerfile.go_build_large | 2 ++ docker/Dockerfile.local | 2 ++ docker/seaweedfs-compose.yml | 9 +++++++++ docker/seaweedfs-dev-compose.yml | 9 +++++++++ 6 files changed, 26 insertions(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index be7414d0b..528a52609 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -37,6 +37,8 @@ EXPOSE 19333 EXPOSE 9333 # s3 server http port EXPOSE 8333 +# webdav server http port +EXPOSE 7333 RUN mkdir -p /data/filerldb2 diff --git a/docker/Dockerfile.go_build b/docker/Dockerfile.go_build index 36d9fd07a..64105ee29 100644 --- a/docker/Dockerfile.go_build +++ b/docker/Dockerfile.go_build @@ -30,6 +30,8 @@ EXPOSE 19333 EXPOSE 9333 # s3 server http port EXPOSE 8333 +# webdav server http port +EXPOSE 7333 RUN mkdir -p /data/filerldb2 diff --git a/docker/Dockerfile.go_build_large b/docker/Dockerfile.go_build_large index 08cfbd547..39f45cbde 100644 --- a/docker/Dockerfile.go_build_large +++ b/docker/Dockerfile.go_build_large @@ -30,6 +30,8 @@ EXPOSE 19333 EXPOSE 9333 # s3 server http port EXPOSE 8333 +# webdav server http port +EXPOSE 7333 RUN mkdir -p /data/filerldb2 diff --git a/docker/Dockerfile.local b/docker/Dockerfile.local index 199c21e81..0a85c56f0 100644 --- a/docker/Dockerfile.local +++ b/docker/Dockerfile.local @@ -20,6 +20,8 @@ EXPOSE 19333 EXPOSE 9333 # s3 server http port EXPOSE 8333 +# webdav server http port +EXPOSE 7333 RUN mkdir -p /data/filerldb2 diff --git a/docker/seaweedfs-compose.yml b/docker/seaweedfs-compose.yml index 3c8a8e3e3..f7d02a105 100644 --- a/docker/seaweedfs-compose.yml +++ b/docker/seaweedfs-compose.yml @@ -48,6 +48,15 @@ services: - master - volume - filer + webdav: + image: chrislusf/seaweedfs # use a remote image + ports: + - 7333:7333 + command: 'webdav -filer="filer:8888"' + depends_on: + - master + - volume + - filer prometheus: image: prom/prometheus:v2.21.0 ports: diff --git a/docker/seaweedfs-dev-compose.yml b/docker/seaweedfs-dev-compose.yml index 75801102e..2382fb17d 100644 --- a/docker/seaweedfs-dev-compose.yml +++ b/docker/seaweedfs-dev-compose.yml @@ -33,3 +33,12 @@ services: - master - volume - filer + webdav: + image: chrislusf/seaweedfs:dev # use a remote dev image + ports: + - 7333:7333 + command: 'webdav -filer="filer:8888"' + depends_on: + - master + - volume + - filer From 609daaf3878927f86a4d4a59eb7f8d9786232a5c Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 1 Feb 2021 10:49:17 -0800 Subject: [PATCH 226/260] s3: DeleteMultipleObjectsHandler clean up leftover empty folders fix https://github.com/chrislusf/seaweedfs/issues/1772 --- weed/s3api/s3api_object_handlers.go | 34 +++++++++++++++++++---------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/weed/s3api/s3api_object_handlers.go b/weed/s3api/s3api_object_handlers.go index 1c0b290c9..ffe46aa85 100644 --- a/weed/s3api/s3api_object_handlers.go +++ b/weed/s3api/s3api_object_handlers.go @@ -206,18 +206,8 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h } // purge empty folders, only checking folders with deletions - var allDirs []string - for dir, _ := range directoriesWithDeletion { - allDirs = append(allDirs, dir) - } - sort.Slice(allDirs, func(i, j int) bool { - return len(allDirs[i]) > len(allDirs[j]) - }) - for _, dir := range allDirs { - parentDir, dirName := util.FullPath(dir).DirAndName() - if err := doDeleteEntry(client, parentDir, dirName, false, false); err != nil { - glog.V(4).Infof("directory %s has %d deletion but still not empty: %v", dir, directoriesWithDeletion[dir], err) - } + for len(directoriesWithDeletion) > 0 { + directoriesWithDeletion = doDeleteEmptyDirectories(client, directoriesWithDeletion) } return nil @@ -233,6 +223,26 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h } +func doDeleteEmptyDirectories(client filer_pb.SeaweedFilerClient, directoriesWithDeletion map[string]int) (newDirectoriesWithDeletion map[string]int){ + var allDirs []string + for dir, _ := range directoriesWithDeletion { + allDirs = append(allDirs, dir) + } + sort.Slice(allDirs, func(i, j int) bool { + return len(allDirs[i]) > len(allDirs[j]) + }) + newDirectoriesWithDeletion = make(map[string]int) + for _, dir := range allDirs { + parentDir, dirName := util.FullPath(dir).DirAndName() + if err := doDeleteEntry(client, parentDir, dirName, false, false); err != nil { + glog.V(4).Infof("directory %s has %d deletion but still not empty: %v", dir, directoriesWithDeletion[dir], err) + } else { + newDirectoriesWithDeletion[parentDir]++ + } + } + return +} + var passThroughHeaders = []string{ "response-cache-control", "response-content-disposition", From 2396ac234c060774ef0640f513fee2e424fbb706 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 2 Feb 2021 11:34:12 -0800 Subject: [PATCH 227/260] filer: fix elastic search pagination possible fix for https://github.com/chrislusf/seaweedfs/issues/1774 --- weed/filer/elastic/v7/elastic_store.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/weed/filer/elastic/v7/elastic_store.go b/weed/filer/elastic/v7/elastic_store.go index 1e7f55599..14719e6ed 100644 --- a/weed/filer/elastic/v7/elastic_store.go +++ b/weed/filer/elastic/v7/elastic_store.go @@ -218,7 +218,6 @@ func (store *ElasticStore) listRootDirectoryEntries(ctx context.Context, startFi if entry, err := store.FindEntry(ctx, weed_util.FullPath("/"+strings.Replace(index.Index, indexPrefix, "", 1))); err == nil { fileName := getFileName(entry.FullPath) - lastFileName = fileName if fileName == startFileName && !inclusive { continue } @@ -229,6 +228,7 @@ func (store *ElasticStore) listRootDirectoryEntries(ctx context.Context, startFi if !eachEntryFunc(entry) { break } + lastFileName = fileName } } } @@ -278,13 +278,13 @@ func (store *ElasticStore) listDirectoryEntries( } nextStart = string(esEntry.Entry.FullPath) fileName := getFileName(esEntry.Entry.FullPath) - lastFileName = fileName if fileName == startFileName && !inclusive { continue } if !eachEntryFunc(esEntry.Entry) { break } + lastFileName = fileName } } } From 5353e384696764af171e9479bd2cf488c0422be3 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 3 Feb 2021 00:35:44 -0800 Subject: [PATCH 228/260] S3: should return 204 on DELETE to nonexistent file fix https://github.com/chrislusf/seaweedfs/issues/1776 --- weed/s3api/s3api_object_handlers.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/weed/s3api/s3api_object_handlers.go b/weed/s3api/s3api_object_handlers.go index ffe46aa85..c9e124328 100644 --- a/weed/s3api/s3api_object_handlers.go +++ b/weed/s3api/s3api_object_handlers.go @@ -296,8 +296,10 @@ func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, des defer util.CloseResponse(resp) if (resp.ContentLength == -1 || resp.StatusCode == 404) && !strings.HasSuffix(destUrl, "/") { - writeErrorResponse(w, s3err.ErrNoSuchKey, r.URL) - return + if r.Method != "DELETE" { + writeErrorResponse(w, s3err.ErrNoSuchKey, r.URL) + return + } } responseFn(resp, w) From a331bbb3ae7b75025850f67a5758812fddfe0ecd Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 3 Feb 2021 00:40:31 -0800 Subject: [PATCH 229/260] filer: should return 204 on DELETE to nonexistent file related to https://github.com/chrislusf/seaweedfs/issues/1776 https://github.com/chrislusf/seaweedfs/issues/1160 --- weed/server/filer_server_handlers_write.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/server/filer_server_handlers_write.go b/weed/server/filer_server_handlers_write.go index 9131b042b..6f33e8494 100644 --- a/weed/server/filer_server_handlers_write.go +++ b/weed/server/filer_server_handlers_write.go @@ -95,7 +95,7 @@ func (fs *FilerServer) DeleteHandler(w http.ResponseWriter, r *http.Request) { glog.V(1).Infoln("deleting", objectPath, ":", err.Error()) httpStatus := http.StatusInternalServerError if err == filer_pb.ErrNotFound { - httpStatus = http.StatusNotFound + httpStatus = http.StatusNoContent } writeJsonError(w, r, httpStatus, err) return From 24ec17219be00f8c00e1f56412b5dd929028d8b5 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev Date: Wed, 3 Feb 2021 21:32:30 +0500 Subject: [PATCH 230/260] RabbitMQ delay retry with Dead Letter Exchange https://github.com/chrislusf/seaweedfs/issues/1773 https://github.com/google/go-cloud/issues/2952 --- docker/local-replicate-compose.yml | 4 +- go.mod | 23 +- go.sum | 224 ++++++++++++++++++ .../replication/repl_util/replication_utli.go | 10 +- .../sub/notification_gocdk_pub_sub.go | 94 +++++++- 5 files changed, 335 insertions(+), 20 deletions(-) diff --git a/docker/local-replicate-compose.yml b/docker/local-replicate-compose.yml index a8e0f808e..8240d45a7 100644 --- a/docker/local-replicate-compose.yml +++ b/docker/local-replicate-compose.yml @@ -56,4 +56,6 @@ services: ports: - 5672:5672 - 15671:15671 - - 15672:15672 \ No newline at end of file + - 15672:15672 + environment: + RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS: "-rabbit log_levels [{connection,error},{queue,debug}]" \ No newline at end of file diff --git a/go.mod b/go.mod index cbd806799..e689ed4a5 100644 --- a/go.mod +++ b/go.mod @@ -3,9 +3,12 @@ module github.com/chrislusf/seaweedfs go 1.12 require ( - cloud.google.com/go v0.44.3 + cloud.google.com/go v0.58.0 + cloud.google.com/go/pubsub v1.3.1 + cloud.google.com/go/storage v1.9.0 + github.com/Azure/azure-amqp-common-go/v2 v2.1.0 // indirect github.com/Azure/azure-pipeline-go v0.2.2 // indirect - github.com/Azure/azure-storage-blob-go v0.8.0 + github.com/Azure/azure-storage-blob-go v0.9.0 github.com/OneOfOne/xxhash v1.2.2 github.com/Shopify/sarama v1.23.1 github.com/aws/aws-sdk-go v1.34.30 @@ -29,7 +32,7 @@ require ( github.com/go-sql-driver/mysql v1.5.0 github.com/gocql/gocql v0.0.0-20190829130954-e163eff7a8c6 github.com/gogo/protobuf v1.2.2-0.20190730201129-28a6bbf47e48 // indirect - github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 + github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e github.com/golang/protobuf v1.4.2 github.com/google/btree v1.0.0 github.com/google/uuid v1.1.1 @@ -48,7 +51,7 @@ require ( github.com/lib/pq v1.2.0 github.com/lunixbochs/vtclean v1.0.0 // indirect github.com/magiconair/properties v1.8.1 // indirect - github.com/mattn/go-ieproxy v0.0.0-20190805055040-f9202b1cfdeb // indirect + github.com/mattn/go-colorable v0.1.2 // indirect github.com/mattn/go-runewidth v0.0.4 // indirect github.com/olivere/elastic/v7 v7.0.19 github.com/peterh/liner v1.1.0 @@ -62,6 +65,7 @@ require ( github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/viper v1.4.0 + github.com/streadway/amqp v0.0.0-20200108173154-1c71cc93ed71 github.com/stretchr/testify v1.6.1 github.com/syndtr/goleveldb v1.0.0 github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c @@ -77,16 +81,15 @@ require ( github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 // indirect go.etcd.io/etcd v3.3.15+incompatible go.mongodb.org/mongo-driver v1.3.2 - gocloud.dev v0.16.0 - gocloud.dev/pubsub/natspubsub v0.16.0 - gocloud.dev/pubsub/rabbitpubsub v0.16.0 + gocloud.dev v0.20.0 + gocloud.dev/pubsub/natspubsub v0.20.0 + gocloud.dev/pubsub/rabbitpubsub v0.20.0 golang.org/x/image v0.0.0-20200119044424-58c23975cae1 // indirect golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb golang.org/x/sync v0.0.0-20200930132711-30421366ff76 // indirect golang.org/x/sys v0.0.0-20201022201747-fb209a7c41cd - golang.org/x/tools v0.0.0-20200103221440-774c71fcf114 - google.golang.org/api v0.9.0 - google.golang.org/appengine v1.6.2 // indirect + golang.org/x/tools v0.0.0-20200608174601-1b747fd94509 + google.golang.org/api v0.26.0 google.golang.org/grpc v1.29.1 google.golang.org/protobuf v1.24.0 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect diff --git a/go.sum b/go.sum index db7d2748a..27fbf029a 100644 --- a/go.sum +++ b/go.sum @@ -1,18 +1,52 @@ +bazil.org/fuse v0.0.0-20180421153158-65cc252bf669/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.39.0/go.mod h1:rVLT6fkc8chs9sfPtFc1SBH6em7n+ZoXaG+87tDISts= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.44.3 h1:0sMegbmn/8uTwpNkB0q9cLEpZ2W5a6kl+wtBQgPWBJQ= cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.55.0/go.mod h1:ZHmoY+/lIMNkN2+fBmuTiqZ4inFhvQad8ft7MT8IV5Y= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.58.0 h1:vtAfVc723K3xKq1BQydk/FyCldnaNFhGhpJxaJzgRMQ= +cloud.google.com/go v0.58.0/go.mod h1:W+9FnSUw6nhVwXlFcp1eL+krq5+HQUJeUogSeJZZiWg= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/firestore v1.2.0/go.mod h1:iISCjWnTpnoJT1R287xRdjvQHJrxQOpeah4phb5D3h0= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1 h1:ukjixP1wl0LpnZ6LWtZJ0mX5tBmjp1f8Sqer8Z2OMUU= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.9.0 h1:oXnZyBjHB6hC8TnSle0AWW6pGJ29EuSo5ww+SFmdNBg= +cloud.google.com/go/storage v1.9.0/go.mod h1:m+/etGaqZbylxaNT876QGXqEHp4PR2Rq5GMqICWb9bU= contrib.go.opencensus.io/exporter/aws v0.0.0-20181029163544-2befc13012d0/go.mod h1:uu1P0UCM/6RbsMrgPa98ll8ZcHM858i/AD06a9aLRCA= contrib.go.opencensus.io/exporter/ocagent v0.5.0 h1:TKXjQSRS0/cCDrP7KvkgU6SmILtF/yV2TOs/02K/WZQ= contrib.go.opencensus.io/exporter/ocagent v0.5.0/go.mod h1:ImxhfLRpxoYiSq891pBrLVhN+qmP8BTVvdH2YLs7Gl0= contrib.go.opencensus.io/exporter/stackdriver v0.12.1/go.mod h1:iwB6wGarfphGGe/e5CWqyUk/cLzKnWsOKPVW3no6OTw= contrib.go.opencensus.io/integrations/ocsql v0.1.4/go.mod h1:8DsSdjz3F+APR+0z0WkU1aRorQCFfRxvqjUUPMbF3fE= contrib.go.opencensus.io/resource v0.1.1/go.mod h1:F361eGI91LCmW1I/Saf+rX0+OFcigGlFvXwEGEnkRLA= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-amqp-common-go/v2 v2.1.0/go.mod h1:R8rea+gJRuJR6QxTir/XuEd+YuKoUiazDC/N96FiDEU= +github.com/Azure/azure-amqp-common-go/v3 v3.0.0/go.mod h1:SY08giD/XbhTz07tJdpw1SoxQXHPN30+DI3Z04SYqyg= github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= github.com/Azure/azure-pipeline-go v0.1.9/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= github.com/Azure/azure-pipeline-go v0.2.1 h1:OLBdZJ3yvOn2MezlWvbrBMTEUQC72zAftRZOMdj5HYo= @@ -21,18 +55,42 @@ github.com/Azure/azure-pipeline-go v0.2.2 h1:6oiIS9yaG6XCCzhgAgKFfIWyo4LLCiDhZot github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= github.com/Azure/azure-sdk-for-go v29.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go v30.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v37.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-service-bus-go v0.9.1/go.mod h1:yzBx6/BUGfjfeqbRZny9AQIbIe3AcV9WZbAdpkoXOa0= +github.com/Azure/azure-service-bus-go v0.10.1/go.mod h1:E/FOceuKAFUfpbIJDKWz/May6guE+eGibfGT6q+n1to= github.com/Azure/azure-storage-blob-go v0.6.0/go.mod h1:oGfmITT1V6x//CswqY2gtAHND+xIP64/qL7a5QJix0Y= github.com/Azure/azure-storage-blob-go v0.8.0 h1:53qhf0Oxa0nOjgbDeeYPUeyiNmafAFEY95rZLK0Tj6o= github.com/Azure/azure-storage-blob-go v0.8.0/go.mod h1:lPI3aLPpuLTeUwh1sViKXFxwl2B6teiRqI0deQUvsw0= +github.com/Azure/azure-storage-blob-go v0.9.0 h1:kORqvzXP8ORhKbW13FflGUaSE5CMyDWun9UwMxY8gPs= +github.com/Azure/azure-storage-blob-go v0.9.0/go.mod h1:8UBPbiOhrMQ4pLPi3gA1tXnpjrS76UYE/fo5A40vf4g= +github.com/Azure/go-amqp v0.12.6/go.mod h1:qApuH6OFTSKZFmCOxccvAv5rLizBQf4v8pRmG138DPo= +github.com/Azure/go-amqp v0.12.7/go.mod h1:qApuH6OFTSKZFmCOxccvAv5rLizBQf4v8pRmG138DPo= github.com/Azure/go-autorest v12.0.0+incompatible h1:N+VqClcomLGD/sHb3smbSYYtNMgKpVV3Cd5r5i8z6bQ= github.com/Azure/go-autorest v12.0.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= +github.com/Azure/go-autorest/autorest v0.9.3/go.mod h1:GsRuLYvwzLjjjRoWEIyMUaYq8GNUx2nRB378IPt/1p0= +github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= +github.com/Azure/go-autorest/autorest/adal v0.8.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc= +github.com/Azure/go-autorest/autorest/adal v0.8.1/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= +github.com/Azure/go-autorest/autorest/adal v0.8.3/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= +github.com/Azure/go-autorest/autorest/azure/auth v0.4.2/go.mod h1:90gmfKdlmKgfjUpnCEpOJzsUEjrWDSLwHIG73tSXddM= +github.com/Azure/go-autorest/autorest/azure/cli v0.3.1/go.mod h1:ZG5p860J94/0kI9mNJVoIoLgXcirM2gF5i2kWloofxw= +github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= +github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g= +github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= +github.com/Azure/go-autorest/autorest/to v0.3.0/go.mod h1:MgwOyqaIuKdG4TL/2ywSsIWKAfJfgHDo8ObuUk3t5sA= +github.com/Azure/go-autorest/autorest/validation v0.2.0/go.mod h1:3EEqHnBxQGHXRYq3HT1WyXAvT7LLY3tl70hw6tQIbjI= +github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= +github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798 h1:2T/jmrHeTezcCM58lvEQXs0UpQJCo5SoGAcg+mbSTIg= github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/GoogleCloudPlatform/cloudsql-proxy v0.0.0-20190605020000-c4ba1fdf4d36/go.mod h1:aJ4qN3TfrelA6NZ6AXsXRfmEVaYin3EDbSPJrKS8OXo= +github.com/GoogleCloudPlatform/cloudsql-proxy v0.0.0-20191009163259-e802c2cb94ae/go.mod h1:mjwGPas4yKduTyubHvD1Atl9r1rUq8DfVy+gkVvZ+oo= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= @@ -59,6 +117,7 @@ github.com/aws/aws-sdk-go v1.15.27/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZo github.com/aws/aws-sdk-go v1.19.18/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.19.45/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.31.13/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/aws/aws-sdk-go v1.33.5 h1:p2fr1ryvNTU6avUWLI+/H7FGv0TBIjzVM5WDgXBBv4U= github.com/aws/aws-sdk-go v1.33.5/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/aws/aws-sdk-go v1.34.30 h1:izATc/E0+HcT5YHmaQVjn7GHCoqaBxn0PGo6Zq5UNFA= @@ -88,6 +147,9 @@ github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chrislusf/raft v1.0.4 h1:THhbsVik2hxdE0/VXX834f64Wn9RzgVPp+E+XCWZdKM= github.com/chrislusf/raft v1.0.4/go.mod h1:Ep5DP+mJSosjfKiix1uU7Lc2Df/SX4oGJEpZlXH5l68= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= @@ -167,6 +229,9 @@ github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-errors/errors v1.1.1 h1:ljK/pL5ltg3qoN+OtN6yCv9HWSfMwxSx90GJCZQxYNg= github.com/go-errors/errors v1.1.1/go.mod h1:psDX2osz5VnTOnFWbDeWwS7yejl+uV3FEWEp4lssFEs= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= @@ -228,9 +293,14 @@ github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3 h1:GV+pQPG/EUUbkh47niozDcADz6go/dUwhVzdUQHIVRw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -238,6 +308,8 @@ github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -261,6 +333,7 @@ github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= @@ -275,6 +348,11 @@ github.com/google/martian v2.1.1-0.20190517191504-25dcb96d9e51+incompatible h1:x github.com/google/martian v2.1.1-0.20190517191504-25dcb96d9e51+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200507031123-427632fa3b1c/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/subcommands v1.0.1/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -282,6 +360,8 @@ github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/wire v0.3.0 h1:imGQZGEVEHpje5056+K+cgdO72p0LQv2xIIFXNGUf60= github.com/google/wire v0.3.0/go.mod h1:i1DMg/Lu8Sz5yYl25iOdmc5CT5qusaa+zmRWs16741s= +github.com/google/wire v0.4.0 h1:kXcsA/rIGzJImVqPdhfnr6q0xsS9gU0515q1EPpJ9fE= +github.com/google/wire v0.4.0/go.mod h1:ngWDr9Qvq3yZA10YrxfyGELY/AFWGVpy9c1LTRi1EoU= github.com/googleapis/gax-go v2.0.2+incompatible h1:silFMLAnr330+NRuag/VjIGF7TLp/LBrV2CJKFLWEww= github.com/googleapis/gax-go v2.0.2+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= @@ -340,6 +420,7 @@ github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/J github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03 h1:FUwcHNlEqkqLjLBdCp5PRlCFijNjvcYANOZXzCfXwCM= @@ -365,6 +446,7 @@ github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/karlseguin/ccache v2.0.3+incompatible h1:j68C9tWOROiOLWTS/kCGg9IcJG+ACqn5+0+t8Oh83UU= @@ -425,6 +507,8 @@ github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HN github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-ieproxy v0.0.0-20190805055040-f9202b1cfdeb h1:hXqqXzQtJbENrsb+rsIqkVqcg4FUJL0SQFGw08Dgivw= github.com/mattn/go-ieproxy v0.0.0-20190805055040-f9202b1cfdeb/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= +github.com/mattn/go-ieproxy v0.0.1 h1:qiyop7gCflfhwCzGyeT0gro3sF9AIg9HU98JORTkqfI= +github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= @@ -457,6 +541,8 @@ github.com/nats-io/jwt v0.2.6/go.mod h1:mQxQ0uHQ9FhEVPIcTSKwx2lqZEpXWWcCgA7R6NrW github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= github.com/nats-io/jwt v0.3.2 h1:+RB5hMpXUUA2dfxuhBTEkMOrYmM+gKIZYS1KjSostMI= github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= +github.com/nats-io/jwt v1.0.1 h1:71ivoESdfT2K/qDiw5YwX/3W9/dR7c+m83xiGOj/EZ4= +github.com/nats-io/jwt v1.0.1/go.mod h1:n3cvmLfBfnpV4JJRN7lRYCyZnw48ksGsbThGXEk4w9M= github.com/nats-io/nats-server/v2 v2.0.0/go.mod h1:RyVdsHHvY4B6c9pWG+uRLpZ0h0XsqiuKp2XCTurP5LI= github.com/nats-io/nats-server/v2 v2.1.2 h1:i2Ly0B+1+rzNZHHWtD4ZwKi+OU5l+uQo1iDHZ2PmiIc= github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= @@ -464,12 +550,17 @@ github.com/nats-io/nats.go v1.8.1 h1:6lF/f1/NN6kzUDBz6pyvQDEXO39jqXcWRLu/tKjtOUQ github.com/nats-io/nats.go v1.8.1/go.mod h1:BrFz9vVn0fU3AcH9Vn4Kd7W0NpJ651tD5omQ3M8LwxM= github.com/nats-io/nats.go v1.9.1 h1:ik3HbLhZ0YABLto7iX80pZLPw/6dx3T+++MZJwLnMrQ= github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= +github.com/nats-io/nats.go v1.10.0 h1:L8qnKaofSfNFbXg0C5F71LdjPRnmQwSsA4ukmkt1TvY= +github.com/nats-io/nats.go v1.10.0/go.mod h1:AjGArbfyR50+afOUotNX2Xs5SYHf+CoOa5HH1eEl2HE= github.com/nats-io/nkeys v0.0.2 h1:+qM7QpgXnvDDixitZtQUBDY9w/s9mu1ghS+JIbsrx6M= github.com/nats-io/nkeys v0.0.2/go.mod h1:dab7URMsZm6Z/jp9Z5UGa87Uutgc2mVpXLC4B7TDb/4= github.com/nats-io/nkeys v0.1.0 h1:qMd4+pRHgdr1nAClu+2h/2a5F2TmKcCzjCDazVgRoX4= github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nkeys v0.1.3 h1:6JrEfig+HzTH85yxzhSVbjHRJv9cn0p6n3IngIcM5/k= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nkeys v0.1.4/go.mod h1:XdZpAbhgyyODYqjTawOnIOI7VlbKSarI9Gfy1tqEu/s= +github.com/nats-io/nkeys v0.2.0 h1:WXKF7diOaPU9cJdLD7nuzwasQy9vT1tBqzXZZf3AMJM= +github.com/nats-io/nkeys v0.2.0/go.mod h1:XdZpAbhgyyODYqjTawOnIOI7VlbKSarI9Gfy1tqEu/s= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= @@ -627,6 +718,8 @@ github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94 h1:0ngsPmuP6XIjiFRN github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271 h1:WhxRHzgeVGETMlmVfqhRn8RIeeNoPr2Czh33I4Zdccw= github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= +github.com/streadway/amqp v0.0.0-20200108173154-1c71cc93ed71 h1:2MR0pKUzlP3SGgj5NYJe/zRYDwOu9ku6YHy+Iw7l5DM= +github.com/streadway/amqp v0.0.0-20200108173154-1c71cc93ed71/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= @@ -683,6 +776,8 @@ github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.2 h1:Z/90sZLPOeCy2PwprqkFa25PdkusRzaj9P8zm/KNyvk= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.4 h1:hi1bXHMVrlQh6WwxAy+qZCV/SYIlqo+Ushwdpa4tAKg= @@ -698,6 +793,7 @@ go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0 h1:C9hSCOW830chIVkdja34wa6Ky+IzWllkUinR+BtRZd4= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4 h1:LYy1Hy3MJdrCdMwwzxA/dRok4ejH+RwNGbuoD9fCjto= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opentelemetry.io/otel v0.15.0 h1:CZFy2lPhxd4HlhZnYK8gRyDotksO3Ip9rBweY1vVYJw= @@ -721,10 +817,16 @@ go.uber.org/zap v1.14.1 h1:nYDKopTbvAPq/NrUVZwT15y2lpROBiLLyoRTbXOYWOo= go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= gocloud.dev v0.16.0 h1:hWeaQWxamGerwsU7B9xSWvUjx0p7TwG8fcHro2TzbbM= gocloud.dev v0.16.0/go.mod h1:xWGXD8t7bEhqPIuyAUFyXV9qHn+PvEY2F2GKPh7i/O0= +gocloud.dev v0.20.0 h1:mbEKMfnyPV7W1Rj35R1xXfjszs9dXkwSOq2KoFr25g8= +gocloud.dev v0.20.0/go.mod h1:+Y/RpSXrJthIOM8uFNzWp6MRu9pFPNFEEZrQMxpkfIc= gocloud.dev/pubsub/natspubsub v0.16.0 h1:MoBGXULDzb1fVaZsGWO5cUCgr6yoI/DHhau8OPGaGEI= gocloud.dev/pubsub/natspubsub v0.16.0/go.mod h1:0n7pT7PkLMClBUHDrOkHfOFVr/o/6kawNMwsyAbwadI= +gocloud.dev/pubsub/natspubsub v0.20.0 h1:DsOXYKfcSTh0SHKwuhpQAJmPLDj3+ARvYgBIupVPClE= +gocloud.dev/pubsub/natspubsub v0.20.0/go.mod h1:Zh7v7Q1DZjAoBwsavZLdvinMIO1eYE0PJTllMuX3VGA= gocloud.dev/pubsub/rabbitpubsub v0.16.0 h1:Bkv2njMSl2tmT3tGbvbwpiIDAXBIpqzP9dmts+rhD4E= gocloud.dev/pubsub/rabbitpubsub v0.16.0/go.mod h1:JJVdUUIqwgaaMJg/1xHQza0g4sI/4KHHSNiGE+pn4JM= +gocloud.dev/pubsub/rabbitpubsub v0.20.0 h1:hwupxLvWG8jTPNQ+9Q/YWZzyMagL9blTwWYYhoW4pco= +gocloud.dev/pubsub/rabbitpubsub v0.20.0/go.mod h1:xYCXmI3ixWuCW4s1KqyZpgKT90MMjdXdMlb0Kgmd7TM= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -741,12 +843,24 @@ golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc h1:c0o/qxkaO2LF5t6fQrT4b5 golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067 h1:KYGJGHOQy8oSi1fDlSpcZF0+juKwk/hEMv5SiwHogR0= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20200119044424-58c23975cae1 h1:5h3ngYt7+vXCDZCup/HkCQgW5XwmSvR/nA2JmJ0RErg= golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -755,11 +869,20 @@ golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvx golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -779,10 +902,23 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb h1:eBmm0M9fYhWpKZLjQUUKka/LtIxf46G4fxeEz5KJr9U= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -790,6 +926,9 @@ golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -798,6 +937,7 @@ golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200930132711-30421366ff76 h1:JnxiSYT3Nm0BT2a8CyvYyM6cnrWpidecD1UuSYbhKm0= golang.org/x/sync v0.0.0-20200930132711-30421366ff76/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -824,14 +964,30 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0 h1:HyfiK1WMnHj5FXFXatD+Qs1A/ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200317113312-5766fd39f98d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201022201747-fb209a7c41cd h1:WgqgiQvkiZWz7XLhphjt2GI2GcGCTIZs9jqXMWmH+oc= golang.org/x/sys v0.0.0-20201022201747-fb209a7c41cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -869,11 +1025,40 @@ golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5 h1:hKsoRgsbwY1NafxrwTs+k64bikrLBkAgPir1TNCj3Zs= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200103221440-774c71fcf114 h1:DnSr2mCsxyCE6ZgIkmcWUQY2R5cH/6wL7eIxEmQOMSE= golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200317043434-63da46f3035e/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200325010219-a49f79bcc224/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200601175630-2caf76543d99/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200606014950-c42cb6316fb6/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200608174601-1b747fd94509/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -889,6 +1074,17 @@ google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.9.0 h1:jbyannxz0XFD3zdjgrSUsaJbgpH4eTrkdhRChkHPfO8= google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.26.0 h1:VJZ8h6E8ip82FRpQl848c5vAadxlTXrUh8RzQzSRm08= +google.golang.org/api v0.26.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -897,6 +1093,8 @@ google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.2 h1:j8RI1yW0SkI+paT6uGwMlrMI/6zwYA6/CFil8rxOzGI= google.golang.org/appengine v1.6.2/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -908,8 +1106,30 @@ google.golang.org/genproto v0.0.0-20190620144150-6af8c5fc6601/go.mod h1:z3L6/3dT google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200317114155-1f3552e48f24/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200325114520-5b2d0af7952b/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200603110839-e855014d5736/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200608115520-7c474a2e3482 h1:i+Aiej6cta/Frzp13/swvwz5O00kYcSe0A/C5Wd7zX8= +google.golang.org/genproto v0.0.0-20200608115520-7c474a2e3482/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= @@ -924,6 +1144,8 @@ google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0 h1:rRYRFMVgRv6E0D70Skyfsr28tDXIuuPZyWGMPdMcnXg= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -988,6 +1210,8 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= modernc.org/b v1.0.0 h1:vpvqeyp17ddcQWF29Czawql4lDdABCDRbXRAS4+aF2o= modernc.org/b v1.0.0/go.mod h1:uZWcZfRj1BpYzfN9JTerzlNUnnPsV9O2ZA8JsRcubNg= modernc.org/mathutil v1.1.1 h1:FeylZSVX8S+58VsyJlkEj2bcpdytmp9MmDKZkKx8OIE= diff --git a/weed/replication/repl_util/replication_utli.go b/weed/replication/repl_util/replication_utli.go index 42777f4ad..c5b8ab4e1 100644 --- a/weed/replication/repl_util/replication_utli.go +++ b/weed/replication/repl_util/replication_utli.go @@ -17,9 +17,10 @@ func CopyFromChunkViews(chunkViews []*filer.ChunkView, filerSource *source.Filer } var writeErr error + var shouldRetry bool for _, fileUrl := range fileUrls { - _, err = util.ReadUrlAsStream(fileUrl+"?readDeleted=true", nil, false, chunk.IsFullChunk(), chunk.Offset, int(chunk.Size), func(data []byte) { + shouldRetry, err = util.ReadUrlAsStream(fileUrl+"?readDeleted=true", nil, false, chunk.IsFullChunk(), chunk.Offset, int(chunk.Size), func(data []byte) { writeErr = writeFunc(data) }) if err != nil { @@ -30,11 +31,12 @@ func CopyFromChunkViews(chunkViews []*filer.ChunkView, filerSource *source.Filer break } } - - if err != nil { + if shouldRetry && err != nil { return err } - + if writeErr != nil { + return writeErr + } } return nil } diff --git a/weed/replication/sub/notification_gocdk_pub_sub.go b/weed/replication/sub/notification_gocdk_pub_sub.go index 09a96d5d5..432f61745 100644 --- a/weed/replication/sub/notification_gocdk_pub_sub.go +++ b/weed/replication/sub/notification_gocdk_pub_sub.go @@ -2,13 +2,17 @@ package sub import ( "context" - "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/util" "github.com/golang/protobuf/proto" + "github.com/streadway/amqp" "gocloud.dev/pubsub" _ "gocloud.dev/pubsub/awssnssqs" + "net/url" + "path" + "strings" + // _ "gocloud.dev/pubsub/azuresb" _ "gocloud.dev/pubsub/gcppubsub" _ "gocloud.dev/pubsub/natspubsub" @@ -19,6 +23,55 @@ func init() { NotificationInputs = append(NotificationInputs, &GoCDKPubSubInput{}) } +func getPath(rawUrl string) string { + parsedUrl, _ := url.Parse(rawUrl) + return path.Join(parsedUrl.Host, parsedUrl.Path) +} + +func QueueDeclareAndBind(conn *amqp.Connection, exchangeUrl string, queueUrl string) error { + exchangeName := getPath(exchangeUrl) + queueName := getPath(queueUrl) + exchangeNameDLX := "DLX." + exchangeName + queueNameDLX := "DLX." + queueName + ch, err := conn.Channel() + if err != nil { + glog.Error(err) + return err + } + defer ch.Close() + if err := ch.ExchangeDeclare( + exchangeNameDLX, "fanout", false, false, false, false, nil); err != nil { + glog.Error(err) + return err + } + if err := ch.ExchangeDeclare( + exchangeName, "fanout", false, false, false, false, nil); err != nil { + glog.Error(err) + return err + } + if _, err := ch.QueueDeclare( + queueName, false, false, false, false, + amqp.Table{"x-dead-letter-exchange": exchangeNameDLX}); err != nil { + glog.Error(err) + return err + } + if err := ch.QueueBind(queueName, "", exchangeName, false, nil); err != nil { + glog.Error(err) + return err + } + if _, err := ch.QueueDeclare( + queueNameDLX, false, false, false, false, + amqp.Table{"x-dead-letter-exchange": exchangeName, "x-message-ttl": 600000}); err != nil { + glog.Error(err) + return err + } + if err := ch.QueueBind(queueNameDLX, "", exchangeNameDLX, false, nil); err != nil { + glog.Error(err) + return err + } + return nil +} + type GoCDKPubSubInput struct { sub *pubsub.Subscription } @@ -28,12 +81,31 @@ func (k *GoCDKPubSubInput) GetName() string { } func (k *GoCDKPubSubInput) Initialize(configuration util.Configuration, prefix string) error { + topicUrl := configuration.GetString(prefix + "topic_url") subURL := configuration.GetString(prefix + "sub_url") glog.V(0).Infof("notification.gocdk_pub_sub.sub_url: %v", subURL) sub, err := pubsub.OpenSubscription(context.Background(), subURL) if err != nil { return err } + var conn *amqp.Connection + if sub.As(&conn) { + ch, err := conn.Channel() + if err != nil { + return err + } + defer ch.Close() + _, err = ch.QueueInspect(getPath(subURL)) + if err != nil { + if strings.HasPrefix(err.Error(), "Exception (404) Reason") { + if err := QueueDeclareAndBind(conn, topicUrl, subURL); err != nil { + return err + } + } else { + return err + } + } + } k.sub = sub return nil } @@ -43,14 +115,26 @@ func (k *GoCDKPubSubInput) ReceiveMessage() (key string, message *filer_pb.Event if err != nil { return } - onSuccessFn = func() { - msg.Ack() - } onFailureFn = func() { if msg.Nackable() { - msg.Nack() + isRedelivered := false + var delivery amqp.Delivery + if msg.As(&delivery) { + isRedelivered = delivery.Redelivered + glog.Warning("onFailureFn() metadata: %+v, redelivered: %v", msg.Metadata, delivery.Redelivered) + } + if isRedelivered { + if err := delivery.Nack(false, false); err != nil { + glog.Error(err) + } + } else { + msg.Nack() + } } } + onSuccessFn = func() { + msg.Ack() + } key = msg.Metadata["key"] message = &filer_pb.EventNotification{} err = proto.Unmarshal(msg.Body, message) From 2fb4c605403275067a2837ce54735b1d09ed9883 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev Date: Wed, 3 Feb 2021 21:34:32 +0500 Subject: [PATCH 231/260] fix log warn --- weed/replication/sub/notification_gocdk_pub_sub.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/replication/sub/notification_gocdk_pub_sub.go b/weed/replication/sub/notification_gocdk_pub_sub.go index 432f61745..413c0e3cf 100644 --- a/weed/replication/sub/notification_gocdk_pub_sub.go +++ b/weed/replication/sub/notification_gocdk_pub_sub.go @@ -121,7 +121,7 @@ func (k *GoCDKPubSubInput) ReceiveMessage() (key string, message *filer_pb.Event var delivery amqp.Delivery if msg.As(&delivery) { isRedelivered = delivery.Redelivered - glog.Warning("onFailureFn() metadata: %+v, redelivered: %v", msg.Metadata, delivery.Redelivered) + glog.Warningf("onFailureFn() metadata: %+v, redelivered: %v", msg.Metadata, delivery.Redelivered) } if isRedelivered { if err := delivery.Nack(false, false); err != nil { From 430aca474ca89a4e69660f2f19b50dccabd860c0 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 3 Feb 2021 13:09:28 -0800 Subject: [PATCH 232/260] relocating local docker compose files --- docker/Makefile | 16 ++++++++-------- docker/{ => compose}/local-cluster-compose.yml | 0 docker/{ => compose}/local-clusters-compose.yml | 0 docker/{ => compose}/local-dev-compose.yml | 0 docker/{ => compose}/local-k8s-compose.yml | 0 docker/{ => compose}/local-mount-compose.yml | 0 docker/{ => compose}/local-registry-compose.yml | 0 docker/{ => compose}/local-replicate-compose.yml | 0 docker/{ => compose}/swarm-etcd.yml | 0 9 files changed, 8 insertions(+), 8 deletions(-) rename docker/{ => compose}/local-cluster-compose.yml (100%) rename docker/{ => compose}/local-clusters-compose.yml (100%) rename docker/{ => compose}/local-dev-compose.yml (100%) rename docker/{ => compose}/local-k8s-compose.yml (100%) rename docker/{ => compose}/local-mount-compose.yml (100%) rename docker/{ => compose}/local-registry-compose.yml (100%) rename docker/{ => compose}/local-replicate-compose.yml (100%) rename docker/{ => compose}/swarm-etcd.yml (100%) diff --git a/docker/Makefile b/docker/Makefile index c3c541738..b402147ed 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -10,28 +10,28 @@ build: rm ./weed dev: build - docker-compose -f local-dev-compose.yml -p seaweedfs up + docker-compose -f compose/local-dev-compose.yml -p seaweedfs up dev_mount: build - docker-compose -f local-mount-compose.yml -p seaweedfs up + docker-compose -f compose/local-mount-compose.yml -p seaweedfs up k8s: build - docker-compose -f local-k8s-compose.yml -p seaweedfs up + docker-compose -f compose/local-k8s-compose.yml -p seaweedfs up dev_registry: build - docker-compose -f local-registry-compose.yml -p seaweedfs up + docker-compose -f compose/local-registry-compose.yml -p seaweedfs up dev_replicate: build - docker-compose -f local-replicate-compose.yml -p seaweedfs up + docker-compose -f compose/local-replicate-compose.yml -p seaweedfs up cluster: build - docker-compose -f local-cluster-compose.yml -p seaweedfs up + docker-compose -f compose/local-cluster-compose.yml -p seaweedfs up 2clusters: build - docker-compose -f local-clusters-compose.yml -p seaweedfs up + docker-compose -f compose/local-clusters-compose.yml -p seaweedfs up filer_etcd: build - docker stack deploy -c swarm-etcd.yml fs + docker stack deploy -c compose/swarm-etcd.yml fs clean: rm ./weed diff --git a/docker/local-cluster-compose.yml b/docker/compose/local-cluster-compose.yml similarity index 100% rename from docker/local-cluster-compose.yml rename to docker/compose/local-cluster-compose.yml diff --git a/docker/local-clusters-compose.yml b/docker/compose/local-clusters-compose.yml similarity index 100% rename from docker/local-clusters-compose.yml rename to docker/compose/local-clusters-compose.yml diff --git a/docker/local-dev-compose.yml b/docker/compose/local-dev-compose.yml similarity index 100% rename from docker/local-dev-compose.yml rename to docker/compose/local-dev-compose.yml diff --git a/docker/local-k8s-compose.yml b/docker/compose/local-k8s-compose.yml similarity index 100% rename from docker/local-k8s-compose.yml rename to docker/compose/local-k8s-compose.yml diff --git a/docker/local-mount-compose.yml b/docker/compose/local-mount-compose.yml similarity index 100% rename from docker/local-mount-compose.yml rename to docker/compose/local-mount-compose.yml diff --git a/docker/local-registry-compose.yml b/docker/compose/local-registry-compose.yml similarity index 100% rename from docker/local-registry-compose.yml rename to docker/compose/local-registry-compose.yml diff --git a/docker/local-replicate-compose.yml b/docker/compose/local-replicate-compose.yml similarity index 100% rename from docker/local-replicate-compose.yml rename to docker/compose/local-replicate-compose.yml diff --git a/docker/swarm-etcd.yml b/docker/compose/swarm-etcd.yml similarity index 100% rename from docker/swarm-etcd.yml rename to docker/compose/swarm-etcd.yml From ee4b208b8a92e1e3720827569e9bf768eb5580c8 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 3 Feb 2021 13:55:19 -0800 Subject: [PATCH 233/260] increase to 3 volumes for local cluster --- docker/compose/local-cluster-compose.yml | 30 ++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/docker/compose/local-cluster-compose.yml b/docker/compose/local-cluster-compose.yml index bf12c4639..89a1ed0cb 100644 --- a/docker/compose/local-cluster-compose.yml +++ b/docker/compose/local-cluster-compose.yml @@ -19,12 +19,32 @@ services: - 9335:9335 - 19335:19335 command: "master -ip=master2 -port=9335 -peers=master0:9333,master1:9334,master2:9335 -mdir=/data/m3" - volume: + volume1: image: chrislusf/seaweedfs:local ports: - 8080:8080 - 18080:18080 - command: 'volume -mserver="master0:9333,master1:9334,master2:9335" -port=8080 -ip=volume -publicUrl=localhost:8080' + command: 'volume -mserver="master0:9333,master1:9334,master2:9335" -port=8080 -ip=volume1 -publicUrl=localhost:8080' + depends_on: + - master0 + - master1 + - master2 + volume2: + image: chrislusf/seaweedfs:local + ports: + - 8082:8082 + - 18082:18082 + command: 'volume -mserver="master0:9333,master1:9334,master2:9335" -port=8082 -ip=volume2 -publicUrl=localhost:8082' + depends_on: + - master0 + - master1 + - master2 + volume3: + image: chrislusf/seaweedfs:local + ports: + - 8083:8083 + - 18083:18083 + command: 'volume -mserver="master0:9333,master1:9334,master2:9335" -port=8083 -ip=volume3 -publicUrl=localhost:8083' depends_on: - master0 - master1 @@ -39,7 +59,8 @@ services: - master0 - master1 - master2 - - volume + - volume1 + - volume2 s3: image: chrislusf/seaweedfs:local ports: @@ -49,5 +70,6 @@ services: - master0 - master1 - master2 - - volume + - volume1 + - volume2 - filer From c3af72d950d93bfa0d42f9e2bf68d183c6285d73 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 4 Feb 2021 01:30:14 -0800 Subject: [PATCH 234/260] fier store: fix elastic search regression fix https://github.com/chrislusf/seaweedfs/issues/1774 --- weed/filer/elastic/v7/elastic_store.go | 65 +++++++------------------- 1 file changed, 16 insertions(+), 49 deletions(-) diff --git a/weed/filer/elastic/v7/elastic_store.go b/weed/filer/elastic/v7/elastic_store.go index 14719e6ed..a16e5ebca 100644 --- a/weed/filer/elastic/v7/elastic_store.go +++ b/weed/filer/elastic/v7/elastic_store.go @@ -101,7 +101,7 @@ func (store *ElasticStore) ListDirectoryPrefixedEntries(ctx context.Context, dir } func (store *ElasticStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) { - index := getIndex(entry.FullPath) + index := getIndex(entry.FullPath, false) dir, _ := entry.FullPath.DirAndName() id := weed_util.Md5String([]byte(entry.FullPath)) esEntry := &ESEntry{ @@ -131,7 +131,7 @@ func (store *ElasticStore) UpdateEntry(ctx context.Context, entry *filer.Entry) } func (store *ElasticStore) FindEntry(ctx context.Context, fullpath weed_util.FullPath) (entry *filer.Entry, err error) { - index := getIndex(fullpath) + index := getIndex(fullpath, false) id := weed_util.Md5String([]byte(fullpath)) searchResult, err := store.client.Get(). Index(index). @@ -154,7 +154,7 @@ func (store *ElasticStore) FindEntry(ctx context.Context, fullpath weed_util.Ful } func (store *ElasticStore) DeleteEntry(ctx context.Context, fullpath weed_util.FullPath) (err error) { - index := getIndex(fullpath) + index := getIndex(fullpath, false) id := weed_util.Md5String([]byte(fullpath)) if strings.Count(string(fullpath), "/") == 1 { return store.deleteIndex(ctx, index) @@ -198,47 +198,13 @@ func (store *ElasticStore) DeleteFolderChildren(ctx context.Context, fullpath we } func (store *ElasticStore) ListDirectoryEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { - if string(dirPath) == "/" { - return store.listRootDirectoryEntries(ctx, startFileName, includeStartFile, limit, eachEntryFunc) - } return store.listDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit, eachEntryFunc) } -func (store *ElasticStore) listRootDirectoryEntries(ctx context.Context, startFileName string, inclusive bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { - indexResult, err := store.client.CatIndices().Do(ctx) - if err != nil { - glog.Errorf("list indices %v.", err) - return - } - for _, index := range indexResult { - if index.Index == indexKV { - continue - } - if strings.HasPrefix(index.Index, indexPrefix) { - if entry, err := store.FindEntry(ctx, - weed_util.FullPath("/"+strings.Replace(index.Index, indexPrefix, "", 1))); err == nil { - fileName := getFileName(entry.FullPath) - if fileName == startFileName && !inclusive { - continue - } - limit-- - if limit < 0 { - break - } - if !eachEntryFunc(entry) { - break - } - lastFileName = fileName - } - } - } - return -} - func (store *ElasticStore) listDirectoryEntries( ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { first := true - index := getIndex(fullpath) + index := getIndex(fullpath, true) nextStart := "" parentId := weed_util.Md5String([]byte(fullpath)) if _, err = store.client.Refresh(index).Do(ctx); err != nil { @@ -277,7 +243,7 @@ func (store *ElasticStore) listDirectoryEntries( return lastFileName, nil } nextStart = string(esEntry.Entry.FullPath) - fileName := getFileName(esEntry.Entry.FullPath) + fileName := esEntry.Entry.FullPath.Name() if fileName == startFileName && !inclusive { continue } @@ -287,6 +253,9 @@ func (store *ElasticStore) listDirectoryEntries( lastFileName = fileName } } + if len(result.Hits.Hits) < store.maxPageSize { + break + } } return } @@ -323,18 +292,16 @@ func (store *ElasticStore) Shutdown() { store.client.Stop() } -func getIndex(fullpath weed_util.FullPath) string { +func getIndex(fullpath weed_util.FullPath, isDirectory bool) string { path := strings.Split(string(fullpath), "/") - if len(path) > 1 { - return indexPrefix + path[1] + if isDirectory && len(path) >= 2 { + return indexPrefix + strings.ToLower(path[1]) } - return "" -} - -func getFileName(fullpath weed_util.FullPath) string { - path := strings.Split(string(fullpath), "/") - if len(path) > 1 { - return path[len(path)-1] + if len(path) > 2 { + return indexPrefix + strings.ToLower(path[1]) + } + if len(path) == 2 { + return indexPrefix } return "" } From 502554887f58915c077462372db4e2813eac3f92 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 4 Feb 2021 18:44:57 -0800 Subject: [PATCH 235/260] Java: add SeaweedInputStream example --- .../java/seaweedfs/client/SeaweedInputStream.java | 15 +++++++++++++++ .../java/com/seaweedfs/examples/UnzipFile.java | 12 +++--------- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/other/java/client/src/main/java/seaweedfs/client/SeaweedInputStream.java b/other/java/client/src/main/java/seaweedfs/client/SeaweedInputStream.java index 312e77aa2..519ff0fd9 100644 --- a/other/java/client/src/main/java/seaweedfs/client/SeaweedInputStream.java +++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedInputStream.java @@ -26,6 +26,21 @@ public class SeaweedInputStream extends InputStream { private boolean closed = false; + public SeaweedInputStream( + final FilerGrpcClient filerGrpcClient, + final String dir, final String name) throws IOException { + this.filerGrpcClient = filerGrpcClient; + this.path = dir; + FilerClient filerClient = new FilerClient(filerGrpcClient); + this.entry = filerClient.lookupEntry(dir, name); + this.contentLength = SeaweedRead.fileSize(entry); + + this.visibleIntervalList = SeaweedRead.nonOverlappingVisibleIntervals(filerGrpcClient, entry.getChunksList()); + + LOG.debug("new path:{} entry:{} visibleIntervalList:{}", path, entry, visibleIntervalList); + + } + public SeaweedInputStream( final FilerGrpcClient filerGrpcClient, final String path, diff --git a/other/java/examples/src/main/java/com/seaweedfs/examples/UnzipFile.java b/other/java/examples/src/main/java/com/seaweedfs/examples/UnzipFile.java index 317327be4..fad1471b6 100644 --- a/other/java/examples/src/main/java/com/seaweedfs/examples/UnzipFile.java +++ b/other/java/examples/src/main/java/com/seaweedfs/examples/UnzipFile.java @@ -1,8 +1,7 @@ package com.seaweedfs.examples; -import seaweed.hdfs.SeaweedHadoopInputStream; -import seaweedfs.client.FilerClient; import seaweedfs.client.FilerGrpcClient; +import seaweedfs.client.SeaweedInputStream; import java.io.FileInputStream; import java.io.IOException; @@ -15,7 +14,6 @@ public class UnzipFile { public static void main(String[] args) throws IOException { FilerGrpcClient filerGrpcClient = new FilerGrpcClient("localhost", 18888); - FilerClient filerClient = new FilerClient(filerGrpcClient); long startTime = System.currentTimeMillis(); parseZip("/Users/chris/tmp/test.zip"); @@ -24,12 +22,8 @@ public class UnzipFile { long localProcessTime = startTime2 - startTime; - SeaweedHadoopInputStream seaweedInputStream = new SeaweedHadoopInputStream( - filerGrpcClient, - new org.apache.hadoop.fs.FileSystem.Statistics(""), - "/", - filerClient.lookupEntry("/", "test.zip") - ); + SeaweedInputStream seaweedInputStream = new SeaweedInputStream( + filerGrpcClient, "/", "test.zip"); parseZip(seaweedInputStream); long swProcessTime = System.currentTimeMillis() - startTime2; From 7f90d14f100f9ce69b6b05f6b8f80823f4c69fdf Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 4 Feb 2021 20:16:08 -0800 Subject: [PATCH 236/260] Java: add SeaweedOutputStream example --- .../seaweedfs/client/SeaweedInputStream.java | 8 ++- .../seaweedfs/client/SeaweedOutputStream.java | 58 ++++++++++++++++++- .../com/seaweedfs/examples/UnzipFile.java | 2 +- .../com/seaweedfs/examples/WriteFile.java | 48 +++++++++++++++ 4 files changed, 110 insertions(+), 6 deletions(-) create mode 100644 other/java/examples/src/main/java/com/seaweedfs/examples/WriteFile.java diff --git a/other/java/client/src/main/java/seaweedfs/client/SeaweedInputStream.java b/other/java/client/src/main/java/seaweedfs/client/SeaweedInputStream.java index 519ff0fd9..8b26c242c 100644 --- a/other/java/client/src/main/java/seaweedfs/client/SeaweedInputStream.java +++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedInputStream.java @@ -28,11 +28,13 @@ public class SeaweedInputStream extends InputStream { public SeaweedInputStream( final FilerGrpcClient filerGrpcClient, - final String dir, final String name) throws IOException { + final String fullpath) throws IOException { this.filerGrpcClient = filerGrpcClient; - this.path = dir; + this.path = fullpath; FilerClient filerClient = new FilerClient(filerGrpcClient); - this.entry = filerClient.lookupEntry(dir, name); + this.entry = filerClient.lookupEntry( + SeaweedOutputStream.getParentDirectory(fullpath), + SeaweedOutputStream.getFileName(fullpath)); this.contentLength = SeaweedRead.fileSize(entry); this.visibleIntervalList = SeaweedRead.nonOverlappingVisibleIntervals(filerGrpcClient, entry.getChunksList()); diff --git a/other/java/client/src/main/java/seaweedfs/client/SeaweedOutputStream.java b/other/java/client/src/main/java/seaweedfs/client/SeaweedOutputStream.java index b09a15a5c..a98bbd1ab 100644 --- a/other/java/client/src/main/java/seaweedfs/client/SeaweedOutputStream.java +++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedOutputStream.java @@ -14,7 +14,7 @@ import java.util.concurrent.*; public class SeaweedOutputStream extends OutputStream { private static final Logger LOG = LoggerFactory.getLogger(SeaweedOutputStream.class); - + protected final boolean supportFlush = false; // true; private final FilerGrpcClient filerGrpcClient; private final String path; private final int bufferSize; @@ -22,7 +22,6 @@ public class SeaweedOutputStream extends OutputStream { private final ThreadPoolExecutor threadExecutor; private final ExecutorCompletionService completionService; private final FilerProto.Entry.Builder entry; - protected final boolean supportFlush = false; // true; private final ConcurrentLinkedDeque writeOperations; private long position; private boolean closed; @@ -32,6 +31,45 @@ public class SeaweedOutputStream extends OutputStream { private ByteBuffer buffer; private long outputIndex; private String replication = "000"; + private boolean shouldSaveMetadata = false; + + public SeaweedOutputStream(FilerGrpcClient filerGrpcClient, final String fullpath) { + this.filerGrpcClient = filerGrpcClient; + this.path = fullpath; + this.position = 0; + this.closed = false; + this.lastError = null; + this.lastFlushOffset = 0; + this.bufferSize = 8 * 1024 * 1024; + this.buffer = ByteBufferPool.request(bufferSize); + this.outputIndex = 0; + this.writeOperations = new ConcurrentLinkedDeque<>(); + + this.maxConcurrentRequestCount = Runtime.getRuntime().availableProcessors(); + + this.threadExecutor + = new ThreadPoolExecutor(maxConcurrentRequestCount, + maxConcurrentRequestCount, + 120L, + TimeUnit.SECONDS, + new LinkedBlockingQueue()); + this.completionService = new ExecutorCompletionService<>(this.threadExecutor); + + long now = System.currentTimeMillis() / 1000L; + + this.entry = FilerProto.Entry.newBuilder() + .setName(getFileName(path)) + .setIsDirectory(false) + .setAttributes(FilerProto.FuseAttributes.newBuilder() + .setFileMode(0755) + .setReplication(replication) + .setCrtime(now) + .setMtime(now) + .clearGroupName() + ); + this.shouldSaveMetadata = true; + + } public SeaweedOutputStream(FilerGrpcClient filerGrpcClient, final String path, FilerProto.Entry.Builder entry, final long position, final int bufferSize, final String replication) { @@ -66,9 +104,20 @@ public class SeaweedOutputStream extends OutputStream { return path; } int lastSlashIndex = path.lastIndexOf("/"); + if (lastSlashIndex == 0) { + return "/"; + } return path.substring(0, lastSlashIndex); } + public static String getFileName(String path) { + if (path.indexOf("/") < 0) { + return path; + } + int lastSlashIndex = path.lastIndexOf("/"); + return path.substring(lastSlashIndex+1); + } + private synchronized void flushWrittenBytesToServiceInternal(final long offset) throws IOException { try { SeaweedWrite.writeMeta(filerGrpcClient, getParentDirectory(path), entry); @@ -163,6 +212,11 @@ public class SeaweedOutputStream extends OutputStream { threadExecutor.shutdownNow(); } } + + if (shouldSaveMetadata) { + SeaweedWrite.writeMeta(filerGrpcClient, getParentDirectory(path), entry); + } + } private synchronized void writeCurrentBufferToService() throws IOException { diff --git a/other/java/examples/src/main/java/com/seaweedfs/examples/UnzipFile.java b/other/java/examples/src/main/java/com/seaweedfs/examples/UnzipFile.java index fad1471b6..12eab1a2c 100644 --- a/other/java/examples/src/main/java/com/seaweedfs/examples/UnzipFile.java +++ b/other/java/examples/src/main/java/com/seaweedfs/examples/UnzipFile.java @@ -23,7 +23,7 @@ public class UnzipFile { long localProcessTime = startTime2 - startTime; SeaweedInputStream seaweedInputStream = new SeaweedInputStream( - filerGrpcClient, "/", "test.zip"); + filerGrpcClient, "/test.zip"); parseZip(seaweedInputStream); long swProcessTime = System.currentTimeMillis() - startTime2; diff --git a/other/java/examples/src/main/java/com/seaweedfs/examples/WriteFile.java b/other/java/examples/src/main/java/com/seaweedfs/examples/WriteFile.java new file mode 100644 index 000000000..b0bd54997 --- /dev/null +++ b/other/java/examples/src/main/java/com/seaweedfs/examples/WriteFile.java @@ -0,0 +1,48 @@ +package com.seaweedfs.examples; + +import seaweedfs.client.FilerGrpcClient; +import seaweedfs.client.SeaweedInputStream; +import seaweedfs.client.SeaweedOutputStream; + +import java.io.IOException; +import java.io.InputStream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +public class WriteFile { + + public static void main(String[] args) throws IOException { + + FilerGrpcClient filerGrpcClient = new FilerGrpcClient("localhost", 18888); + + SeaweedInputStream seaweedInputStream = new SeaweedInputStream( + filerGrpcClient, "/test.zip"); + unZipFiles(filerGrpcClient, seaweedInputStream); + + } + + public static void unZipFiles(FilerGrpcClient filerGrpcClient, InputStream is) throws IOException { + ZipInputStream zin = new ZipInputStream(is); + ZipEntry ze; + while ((ze = zin.getNextEntry()) != null) { + + String filename = ze.getName(); + if (filename.indexOf("/") >= 0) { + filename = filename.substring(filename.lastIndexOf("/") + 1); + } + if (filename.length()==0) { + continue; + } + + SeaweedOutputStream seaweedOutputStream = new SeaweedOutputStream(filerGrpcClient, "/test/"+filename); + byte[] bytesIn = new byte[16 * 1024]; + int read = 0; + while ((read = zin.read(bytesIn))!=-1) { + seaweedOutputStream.write(bytesIn,0,read); + } + seaweedOutputStream.close(); + + System.out.println(ze.getName()); + } + } +} From 82c167aaca8716443a84a338a9e181f661ca1c79 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 4 Feb 2021 20:18:33 -0800 Subject: [PATCH 237/260] Java: supportFlush set to true --- .../src/main/java/seaweedfs/client/SeaweedOutputStream.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/other/java/client/src/main/java/seaweedfs/client/SeaweedOutputStream.java b/other/java/client/src/main/java/seaweedfs/client/SeaweedOutputStream.java index a98bbd1ab..925a56bdd 100644 --- a/other/java/client/src/main/java/seaweedfs/client/SeaweedOutputStream.java +++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedOutputStream.java @@ -14,7 +14,7 @@ import java.util.concurrent.*; public class SeaweedOutputStream extends OutputStream { private static final Logger LOG = LoggerFactory.getLogger(SeaweedOutputStream.class); - protected final boolean supportFlush = false; // true; + protected final boolean supportFlush = true; private final FilerGrpcClient filerGrpcClient; private final String path; private final int bufferSize; From ce416d765f42f673e84723703f75c8d0ab151889 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 4 Feb 2021 20:22:40 -0800 Subject: [PATCH 238/260] Java: SeaweedOutputStream add replication option --- .../src/main/java/seaweedfs/client/SeaweedOutputStream.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/other/java/client/src/main/java/seaweedfs/client/SeaweedOutputStream.java b/other/java/client/src/main/java/seaweedfs/client/SeaweedOutputStream.java index 925a56bdd..94f34b221 100644 --- a/other/java/client/src/main/java/seaweedfs/client/SeaweedOutputStream.java +++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedOutputStream.java @@ -34,6 +34,11 @@ public class SeaweedOutputStream extends OutputStream { private boolean shouldSaveMetadata = false; public SeaweedOutputStream(FilerGrpcClient filerGrpcClient, final String fullpath) { + this(filerGrpcClient, fullpath, "000"); + } + + public SeaweedOutputStream(FilerGrpcClient filerGrpcClient, final String fullpath, final String replication) { + this.replication = replication; this.filerGrpcClient = filerGrpcClient; this.path = fullpath; this.position = 0; From 9fa7977714d99e546cee32e02b5f7fdf3528078b Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 4 Feb 2021 20:30:49 -0800 Subject: [PATCH 239/260] Java: SeaweedOutputStream refactoring --- .../seaweedfs/client/SeaweedOutputStream.java | 61 ++++++------------- 1 file changed, 18 insertions(+), 43 deletions(-) diff --git a/other/java/client/src/main/java/seaweedfs/client/SeaweedOutputStream.java b/other/java/client/src/main/java/seaweedfs/client/SeaweedOutputStream.java index 94f34b221..f9df22c9b 100644 --- a/other/java/client/src/main/java/seaweedfs/client/SeaweedOutputStream.java +++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedOutputStream.java @@ -21,8 +21,9 @@ public class SeaweedOutputStream extends OutputStream { private final int maxConcurrentRequestCount; private final ThreadPoolExecutor threadExecutor; private final ExecutorCompletionService completionService; - private final FilerProto.Entry.Builder entry; private final ConcurrentLinkedDeque writeOperations; + private final boolean shouldSaveMetadata = false; + private FilerProto.Entry.Builder entry; private long position; private boolean closed; private volatile IOException lastError; @@ -31,49 +32,13 @@ public class SeaweedOutputStream extends OutputStream { private ByteBuffer buffer; private long outputIndex; private String replication = "000"; - private boolean shouldSaveMetadata = false; public SeaweedOutputStream(FilerGrpcClient filerGrpcClient, final String fullpath) { this(filerGrpcClient, fullpath, "000"); } public SeaweedOutputStream(FilerGrpcClient filerGrpcClient, final String fullpath, final String replication) { - this.replication = replication; - this.filerGrpcClient = filerGrpcClient; - this.path = fullpath; - this.position = 0; - this.closed = false; - this.lastError = null; - this.lastFlushOffset = 0; - this.bufferSize = 8 * 1024 * 1024; - this.buffer = ByteBufferPool.request(bufferSize); - this.outputIndex = 0; - this.writeOperations = new ConcurrentLinkedDeque<>(); - - this.maxConcurrentRequestCount = Runtime.getRuntime().availableProcessors(); - - this.threadExecutor - = new ThreadPoolExecutor(maxConcurrentRequestCount, - maxConcurrentRequestCount, - 120L, - TimeUnit.SECONDS, - new LinkedBlockingQueue()); - this.completionService = new ExecutorCompletionService<>(this.threadExecutor); - - long now = System.currentTimeMillis() / 1000L; - - this.entry = FilerProto.Entry.newBuilder() - .setName(getFileName(path)) - .setIsDirectory(false) - .setAttributes(FilerProto.FuseAttributes.newBuilder() - .setFileMode(0755) - .setReplication(replication) - .setCrtime(now) - .setMtime(now) - .clearGroupName() - ); - this.shouldSaveMetadata = true; - + this(filerGrpcClient, fullpath, null, 0, 8 * 1024 * 1024, "000"); } public SeaweedOutputStream(FilerGrpcClient filerGrpcClient, final String path, FilerProto.Entry.Builder entry, @@ -101,6 +66,20 @@ public class SeaweedOutputStream extends OutputStream { this.completionService = new ExecutorCompletionService<>(this.threadExecutor); this.entry = entry; + if (this.entry == null) { + long now = System.currentTimeMillis() / 1000L; + + this.entry = FilerProto.Entry.newBuilder() + .setName(getFileName(path)) + .setIsDirectory(false) + .setAttributes(FilerProto.FuseAttributes.newBuilder() + .setFileMode(0755) + .setReplication(replication) + .setCrtime(now) + .setMtime(now) + .clearGroupName() + ); + } } @@ -120,7 +99,7 @@ public class SeaweedOutputStream extends OutputStream { return path; } int lastSlashIndex = path.lastIndexOf("/"); - return path.substring(lastSlashIndex+1); + return path.substring(lastSlashIndex + 1); } private synchronized void flushWrittenBytesToServiceInternal(final long offset) throws IOException { @@ -218,10 +197,6 @@ public class SeaweedOutputStream extends OutputStream { } } - if (shouldSaveMetadata) { - SeaweedWrite.writeMeta(filerGrpcClient, getParentDirectory(path), entry); - } - } private synchronized void writeCurrentBufferToService() throws IOException { From 42e5ef4b0150d339befedb06f6ed23a6c9890296 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 4 Feb 2021 21:02:54 -0800 Subject: [PATCH 240/260] Java: adjust examples --- .../seaweedfs/examples/{UnzipFile.java => ExampleReadFile.java} | 2 +- .../examples/{WatchFiles.java => ExampleWatchFileChanges.java} | 2 +- .../examples/{WriteFile.java => ExampleWriteFile.java} | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename other/java/examples/src/main/java/com/seaweedfs/examples/{UnzipFile.java => ExampleReadFile.java} (97%) rename other/java/examples/src/main/java/com/seaweedfs/examples/{WatchFiles.java => ExampleWatchFileChanges.java} (98%) rename other/java/examples/src/main/java/com/seaweedfs/examples/{WriteFile.java => ExampleWriteFile.java} (97%) diff --git a/other/java/examples/src/main/java/com/seaweedfs/examples/UnzipFile.java b/other/java/examples/src/main/java/com/seaweedfs/examples/ExampleReadFile.java similarity index 97% rename from other/java/examples/src/main/java/com/seaweedfs/examples/UnzipFile.java rename to other/java/examples/src/main/java/com/seaweedfs/examples/ExampleReadFile.java index 12eab1a2c..bd73df802 100644 --- a/other/java/examples/src/main/java/com/seaweedfs/examples/UnzipFile.java +++ b/other/java/examples/src/main/java/com/seaweedfs/examples/ExampleReadFile.java @@ -9,7 +9,7 @@ import java.io.InputStream; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; -public class UnzipFile { +public class ExampleReadFile { public static void main(String[] args) throws IOException { diff --git a/other/java/examples/src/main/java/com/seaweedfs/examples/WatchFiles.java b/other/java/examples/src/main/java/com/seaweedfs/examples/ExampleWatchFileChanges.java similarity index 98% rename from other/java/examples/src/main/java/com/seaweedfs/examples/WatchFiles.java rename to other/java/examples/src/main/java/com/seaweedfs/examples/ExampleWatchFileChanges.java index e489cb3b1..72c572d31 100644 --- a/other/java/examples/src/main/java/com/seaweedfs/examples/WatchFiles.java +++ b/other/java/examples/src/main/java/com/seaweedfs/examples/ExampleWatchFileChanges.java @@ -7,7 +7,7 @@ import java.io.IOException; import java.util.Date; import java.util.Iterator; -public class WatchFiles { +public class ExampleWatchFileChanges { public static void main(String[] args) throws IOException { diff --git a/other/java/examples/src/main/java/com/seaweedfs/examples/WriteFile.java b/other/java/examples/src/main/java/com/seaweedfs/examples/ExampleWriteFile.java similarity index 97% rename from other/java/examples/src/main/java/com/seaweedfs/examples/WriteFile.java rename to other/java/examples/src/main/java/com/seaweedfs/examples/ExampleWriteFile.java index b0bd54997..228a3c0b7 100644 --- a/other/java/examples/src/main/java/com/seaweedfs/examples/WriteFile.java +++ b/other/java/examples/src/main/java/com/seaweedfs/examples/ExampleWriteFile.java @@ -9,7 +9,7 @@ import java.io.InputStream; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; -public class WriteFile { +public class ExampleWriteFile { public static void main(String[] args) throws IOException { From 8c3177d835bb86eed6127b390e2f39ca63ba1a04 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 4 Feb 2021 21:41:19 -0800 Subject: [PATCH 241/260] java: resolve parent directory if started with seaweedfs:// --- .../seaweedfs/client/SeaweedOutputStream.java | 5 ++++ .../com/seaweedfs/examples/HdfsCopyFile.java | 25 +++++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 other/java/examples/src/main/java/com/seaweedfs/examples/HdfsCopyFile.java diff --git a/other/java/client/src/main/java/seaweedfs/client/SeaweedOutputStream.java b/other/java/client/src/main/java/seaweedfs/client/SeaweedOutputStream.java index f9df22c9b..92dc59f61 100644 --- a/other/java/client/src/main/java/seaweedfs/client/SeaweedOutputStream.java +++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedOutputStream.java @@ -84,6 +84,11 @@ public class SeaweedOutputStream extends OutputStream { } public static String getParentDirectory(String path) { + int protoIndex = path.indexOf("://"); + if (protoIndex >= 0) { + int pathStart = path.indexOf("/", protoIndex+3); + path = path.substring(pathStart); + } if (path.equals("/")) { return path; } diff --git a/other/java/examples/src/main/java/com/seaweedfs/examples/HdfsCopyFile.java b/other/java/examples/src/main/java/com/seaweedfs/examples/HdfsCopyFile.java new file mode 100644 index 000000000..006c581c9 --- /dev/null +++ b/other/java/examples/src/main/java/com/seaweedfs/examples/HdfsCopyFile.java @@ -0,0 +1,25 @@ +package com.seaweedfs.examples; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IOUtils; + +import java.io.*; + +public class HdfsCopyFile { + public static void main(String[] args) throws IOException { + Configuration configuration = new Configuration(); + + configuration.set("fs.defaultFS", "seaweedfs://localhost:8888"); + configuration.set("fs.seaweedfs.impl", "seaweed.hdfs.SeaweedFileSystem"); + + FileSystem fs = FileSystem.get(configuration); + String source = "/Users/chris/tmp/test.zip"; + String destination = "/buckets/spark/test01.zip"; + InputStream in = new BufferedInputStream(new FileInputStream(source)); + + OutputStream out = fs.create(new Path(destination)); + IOUtils.copyBytes(in, out, 4096, true); + } +} From 694df8933139a1b498eefb5f2f501e2d5912d58c Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 4 Feb 2021 22:21:55 -0800 Subject: [PATCH 242/260] java: add configurable volume access mode --- .../seaweedfs/client/FileChunkManifest.java | 2 +- .../seaweedfs/client/FilerGrpcClient.java | 32 ++++++++++++++++++- .../java/seaweedfs/client/SeaweedRead.java | 16 +++++++--- .../java/seaweedfs/client/SeaweedWrite.java | 10 ++++-- .../java/seaweed/hdfs/SeaweedFileSystem.java | 1 + .../seaweed/hdfs/SeaweedFileSystemStore.java | 10 ++++-- 6 files changed, 60 insertions(+), 11 deletions(-) diff --git a/other/java/client/src/main/java/seaweedfs/client/FileChunkManifest.java b/other/java/client/src/main/java/seaweedfs/client/FileChunkManifest.java index 3293db2ca..3d7da91d5 100644 --- a/other/java/client/src/main/java/seaweedfs/client/FileChunkManifest.java +++ b/other/java/client/src/main/java/seaweedfs/client/FileChunkManifest.java @@ -74,7 +74,7 @@ public class FileChunkManifest { byte[] chunkData = SeaweedRead.chunkCache.getChunk(chunkView.fileId); if (chunkData == null) { LOG.debug("doFetchFullChunkData:{}", chunkView); - chunkData = SeaweedRead.doFetchFullChunkData(chunkView, locations); + chunkData = SeaweedRead.doFetchFullChunkData(filerGrpcClient, chunkView, locations); } if (chunk.getIsChunkManifest()){ LOG.debug("chunk {} size {}", chunkView.fileId, chunkData.length); diff --git a/other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java b/other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java index 1a719f3c0..8a37827f1 100644 --- a/other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java +++ b/other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java @@ -15,6 +15,10 @@ import java.util.concurrent.TimeUnit; public class FilerGrpcClient { + public final int VOLUME_SERVER_ACCESS_DIRECT = 0; + public final int VOLUME_SERVER_ACCESS_PUBLIC_URL = 1; + public final int VOLUME_SERVER_ACCESS_FILER_PROXY = 2; + private static final Logger logger = LoggerFactory.getLogger(FilerGrpcClient.class); static SslContext sslContext; @@ -34,6 +38,8 @@ public class FilerGrpcClient { private boolean cipher = false; private String collection = ""; private String replication = ""; + private int volumeServerAccess = VOLUME_SERVER_ACCESS_DIRECT; + private String filerAddress; public FilerGrpcClient(String host, int grpcPort) { this(host, grpcPort, sslContext); @@ -49,6 +55,8 @@ public class FilerGrpcClient { .negotiationType(NegotiationType.TLS) .sslContext(sslContext)); + filerAddress = String.format("%s:%d", host, grpcPort-10000); + FilerProto.GetFilerConfigurationResponse filerConfigurationResponse = this.getBlockingStub().getFilerConfiguration( FilerProto.GetFilerConfigurationRequest.newBuilder().build()); @@ -58,7 +66,7 @@ public class FilerGrpcClient { } - public FilerGrpcClient(ManagedChannelBuilder channelBuilder) { + private FilerGrpcClient(ManagedChannelBuilder channelBuilder) { channel = channelBuilder.build(); blockingStub = SeaweedFilerGrpc.newBlockingStub(channel); asyncStub = SeaweedFilerGrpc.newStub(channel); @@ -93,4 +101,26 @@ public class FilerGrpcClient { return futureStub; } + public void setAccessVolumeServerDirectly() { + this.volumeServerAccess = VOLUME_SERVER_ACCESS_DIRECT; + } + public boolean isAccessVolumeServerDirectly() { + return this.volumeServerAccess == VOLUME_SERVER_ACCESS_DIRECT; + } + public void setAccessVolumeServerByPublicUrl() { + this.volumeServerAccess = VOLUME_SERVER_ACCESS_PUBLIC_URL; + } + public boolean isAccessVolumeServerByPublicUrl() { + return this.volumeServerAccess == VOLUME_SERVER_ACCESS_PUBLIC_URL; + } + public void setAccessVolumeServerByFilerProxy() { + this.volumeServerAccess = VOLUME_SERVER_ACCESS_FILER_PROXY; + } + public boolean isAccessVolumeServerByFilerProxy() { + return this.volumeServerAccess == VOLUME_SERVER_ACCESS_FILER_PROXY; + } + public String getFilerAddress() { + return this.filerAddress; + } + } diff --git a/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java b/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java index c45987bed..3df832d7d 100644 --- a/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java +++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java @@ -71,7 +71,7 @@ public class SeaweedRead { return 0; } - int len = readChunkView(startOffset, buf, chunkView, locations); + int len = readChunkView(filerGrpcClient, startOffset, buf, chunkView, locations); LOG.debug("read [{},{}) {} size {}", startOffset, startOffset + len, chunkView.fileId, chunkView.size); @@ -93,12 +93,12 @@ public class SeaweedRead { return readCount; } - private static int readChunkView(long startOffset, ByteBuffer buf, ChunkView chunkView, FilerProto.Locations locations) throws IOException { + private static int readChunkView(FilerGrpcClient filerGrpcClient, long startOffset, ByteBuffer buf, ChunkView chunkView, FilerProto.Locations locations) throws IOException { byte[] chunkData = chunkCache.getChunk(chunkView.fileId); if (chunkData == null) { - chunkData = doFetchFullChunkData(chunkView, locations); + chunkData = doFetchFullChunkData(filerGrpcClient, chunkView, locations); chunkCache.setChunk(chunkView.fileId, chunkData); } @@ -110,12 +110,18 @@ public class SeaweedRead { return len; } - public static byte[] doFetchFullChunkData(ChunkView chunkView, FilerProto.Locations locations) throws IOException { + public static byte[] doFetchFullChunkData(FilerGrpcClient filerGrpcClient, ChunkView chunkView, FilerProto.Locations locations) throws IOException { byte[] data = null; IOException lastException = null; for (long waitTime = 1000L; waitTime < 10 * 1000; waitTime += waitTime / 2) { for (FilerProto.Location location : locations.getLocationsList()) { + String host = location.getUrl(); + if (filerGrpcClient.isAccessVolumeServerByPublicUrl()) { + host = location.getPublicUrl(); + } else if (filerGrpcClient.isAccessVolumeServerByFilerProxy()) { + host = filerGrpcClient.getFilerAddress(); + } String url = String.format("http://%s/%s", location.getUrl(), chunkView.fileId); try { data = doFetchOneFullChunkData(chunkView, url); @@ -145,7 +151,7 @@ public class SeaweedRead { } - public static byte[] doFetchOneFullChunkData(ChunkView chunkView, String url) throws IOException { + private static byte[] doFetchOneFullChunkData(ChunkView chunkView, String url) throws IOException { HttpGet request = new HttpGet(url); diff --git a/other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java b/other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java index b8fd3e299..3cc11e21c 100644 --- a/other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java +++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java @@ -51,9 +51,15 @@ public class SeaweedWrite { .setPath(path) .build()); String fileId = response.getFileId(); - String url = response.getUrl(); String auth = response.getAuth(); - String targetUrl = String.format("http://%s/%s", url, fileId); + + String host = response.getUrl(); + if (filerGrpcClient.isAccessVolumeServerByPublicUrl()) { + host = response.getPublicUrl(); + } else if (filerGrpcClient.isAccessVolumeServerByFilerProxy()) { + host = filerGrpcClient.getFilerAddress(); + } + String targetUrl = String.format("http://%s/%s", host, fileId); ByteString cipherKeyString = com.google.protobuf.ByteString.EMPTY; byte[] cipherKey = null; diff --git a/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java b/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java index 84f11e846..6072d3ec8 100644 --- a/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java +++ b/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java @@ -26,6 +26,7 @@ public class SeaweedFileSystem extends FileSystem { public static final int FS_SEAWEED_DEFAULT_PORT = 8888; public static final String FS_SEAWEED_BUFFER_SIZE = "fs.seaweed.buffer.size"; public static final String FS_SEAWEED_REPLICATION = "fs.seaweed.replication"; + public static final String FS_SEAWEED_VOLUME_SERVER_ACCESS = "fs.seaweed.volumeServerAccess"; public static final int FS_SEAWEED_DEFAULT_BUFFER_SIZE = 4 * 1024 * 1024; private static final Logger LOG = LoggerFactory.getLogger(SeaweedFileSystem.class); diff --git a/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java b/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java index 223036c13..8147f3efe 100644 --- a/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java +++ b/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java @@ -18,8 +18,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import static seaweed.hdfs.SeaweedFileSystem.FS_SEAWEED_BUFFER_SIZE; -import static seaweed.hdfs.SeaweedFileSystem.FS_SEAWEED_DEFAULT_BUFFER_SIZE; +import static seaweed.hdfs.SeaweedFileSystem.*; public class SeaweedFileSystemStore { @@ -34,6 +33,13 @@ public class SeaweedFileSystemStore { filerGrpcClient = new FilerGrpcClient(host, grpcPort); filerClient = new FilerClient(filerGrpcClient); this.conf = conf; + String volumeServerAccessMode = this.conf.get(FS_SEAWEED_VOLUME_SERVER_ACCESS, "direct"); + if (volumeServerAccessMode.equals("publicUrl")) { + filerGrpcClient.setAccessVolumeServerByPublicUrl(); + } else if (volumeServerAccessMode.equals("filerProxy")) { + filerGrpcClient.setAccessVolumeServerByFilerProxy(); + } + } public void close() { From 6a2a9b67e84a179cb7fa9cdadec9bfdbe4051f7e Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 4 Feb 2021 22:22:15 -0800 Subject: [PATCH 243/260] Java: 1.5.8 --- other/java/client/pom.xml | 2 +- other/java/client/pom.xml.deploy | 2 +- other/java/client/pom_debug.xml | 2 +- other/java/examples/pom.xml | 4 ++-- other/java/hdfs2/dependency-reduced-pom.xml | 2 +- other/java/hdfs2/pom.xml | 2 +- other/java/hdfs3/dependency-reduced-pom.xml | 2 +- other/java/hdfs3/pom.xml | 2 +- 8 files changed, 9 insertions(+), 9 deletions(-) diff --git a/other/java/client/pom.xml b/other/java/client/pom.xml index 60fb84082..7e7a5d1f9 100644 --- a/other/java/client/pom.xml +++ b/other/java/client/pom.xml @@ -5,7 +5,7 @@ com.github.chrislusf seaweedfs-client - 1.5.7 + 1.5.8 org.sonatype.oss diff --git a/other/java/client/pom.xml.deploy b/other/java/client/pom.xml.deploy index b45e0bee7..7d845f439 100644 --- a/other/java/client/pom.xml.deploy +++ b/other/java/client/pom.xml.deploy @@ -5,7 +5,7 @@ com.github.chrislusf seaweedfs-client - 1.5.7 + 1.5.8 org.sonatype.oss diff --git a/other/java/client/pom_debug.xml b/other/java/client/pom_debug.xml index 26bef1655..a4e2d694e 100644 --- a/other/java/client/pom_debug.xml +++ b/other/java/client/pom_debug.xml @@ -5,7 +5,7 @@ com.github.chrislusf seaweedfs-client - 1.5.7 + 1.5.8 org.sonatype.oss diff --git a/other/java/examples/pom.xml b/other/java/examples/pom.xml index b1f416941..12196324e 100644 --- a/other/java/examples/pom.xml +++ b/other/java/examples/pom.xml @@ -11,13 +11,13 @@ com.github.chrislusf seaweedfs-client - 1.5.7 + 1.5.8 compile com.github.chrislusf seaweedfs-hadoop2-client - 1.5.7 + 1.5.8 compile diff --git a/other/java/hdfs2/dependency-reduced-pom.xml b/other/java/hdfs2/dependency-reduced-pom.xml index 47b58b4c0..a9c062a75 100644 --- a/other/java/hdfs2/dependency-reduced-pom.xml +++ b/other/java/hdfs2/dependency-reduced-pom.xml @@ -301,7 +301,7 @@ - 1.5.7 + 1.5.8 2.9.2 diff --git a/other/java/hdfs2/pom.xml b/other/java/hdfs2/pom.xml index de6d42d98..65ce0766a 100644 --- a/other/java/hdfs2/pom.xml +++ b/other/java/hdfs2/pom.xml @@ -5,7 +5,7 @@ 4.0.0 - 1.5.7 + 1.5.8 2.9.2 diff --git a/other/java/hdfs3/dependency-reduced-pom.xml b/other/java/hdfs3/dependency-reduced-pom.xml index 78f4e38e8..007e7115b 100644 --- a/other/java/hdfs3/dependency-reduced-pom.xml +++ b/other/java/hdfs3/dependency-reduced-pom.xml @@ -309,7 +309,7 @@ - 1.5.7 + 1.5.8 3.1.1 diff --git a/other/java/hdfs3/pom.xml b/other/java/hdfs3/pom.xml index 4f097d648..4a0e2603b 100644 --- a/other/java/hdfs3/pom.xml +++ b/other/java/hdfs3/pom.xml @@ -5,7 +5,7 @@ 4.0.0 - 1.5.7 + 1.5.8 3.1.1 From 8f3a51f2b8896254c74040b0fed9ed307d8efb34 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 5 Feb 2021 10:42:20 -0800 Subject: [PATCH 244/260] Java: 1.5.8 additional fixes --- .../src/main/java/seaweedfs/client/SeaweedRead.java | 2 +- .../src/main/java/seaweed/hdfs/SeaweedFileSystem.java | 1 + .../main/java/seaweed/hdfs/SeaweedFileSystemStore.java | 10 ++++++++-- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java b/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java index 3df832d7d..a70553762 100644 --- a/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java +++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java @@ -122,7 +122,7 @@ public class SeaweedRead { } else if (filerGrpcClient.isAccessVolumeServerByFilerProxy()) { host = filerGrpcClient.getFilerAddress(); } - String url = String.format("http://%s/%s", location.getUrl(), chunkView.fileId); + String url = String.format("http://%s/%s", host, chunkView.fileId); try { data = doFetchOneFullChunkData(chunkView, url); lastException = null; diff --git a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java index 84f11e846..6072d3ec8 100644 --- a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java +++ b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java @@ -26,6 +26,7 @@ public class SeaweedFileSystem extends FileSystem { public static final int FS_SEAWEED_DEFAULT_PORT = 8888; public static final String FS_SEAWEED_BUFFER_SIZE = "fs.seaweed.buffer.size"; public static final String FS_SEAWEED_REPLICATION = "fs.seaweed.replication"; + public static final String FS_SEAWEED_VOLUME_SERVER_ACCESS = "fs.seaweed.volumeServerAccess"; public static final int FS_SEAWEED_DEFAULT_BUFFER_SIZE = 4 * 1024 * 1024; private static final Logger LOG = LoggerFactory.getLogger(SeaweedFileSystem.class); diff --git a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java index 223036c13..8147f3efe 100644 --- a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java +++ b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java @@ -18,8 +18,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import static seaweed.hdfs.SeaweedFileSystem.FS_SEAWEED_BUFFER_SIZE; -import static seaweed.hdfs.SeaweedFileSystem.FS_SEAWEED_DEFAULT_BUFFER_SIZE; +import static seaweed.hdfs.SeaweedFileSystem.*; public class SeaweedFileSystemStore { @@ -34,6 +33,13 @@ public class SeaweedFileSystemStore { filerGrpcClient = new FilerGrpcClient(host, grpcPort); filerClient = new FilerClient(filerGrpcClient); this.conf = conf; + String volumeServerAccessMode = this.conf.get(FS_SEAWEED_VOLUME_SERVER_ACCESS, "direct"); + if (volumeServerAccessMode.equals("publicUrl")) { + filerGrpcClient.setAccessVolumeServerByPublicUrl(); + } else if (volumeServerAccessMode.equals("filerProxy")) { + filerGrpcClient.setAccessVolumeServerByFilerProxy(); + } + } public void close() { From 9c1efdf11bc17c9881ac4d3e9aad74d8a6448677 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 5 Feb 2021 11:47:56 -0800 Subject: [PATCH 245/260] HCFS: 1.5.9 --- other/java/client/pom.xml | 2 +- other/java/client/pom.xml.deploy | 2 +- other/java/client/pom_debug.xml | 2 +- other/java/examples/pom.xml | 4 ++-- other/java/hdfs2/dependency-reduced-pom.xml | 2 +- other/java/hdfs2/pom.xml | 2 +- .../hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java | 2 +- other/java/hdfs3/dependency-reduced-pom.xml | 2 +- other/java/hdfs3/pom.xml | 2 +- .../hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java | 2 +- 10 files changed, 11 insertions(+), 11 deletions(-) diff --git a/other/java/client/pom.xml b/other/java/client/pom.xml index 7e7a5d1f9..4606ef5d9 100644 --- a/other/java/client/pom.xml +++ b/other/java/client/pom.xml @@ -5,7 +5,7 @@ com.github.chrislusf seaweedfs-client - 1.5.8 + 1.5.9 org.sonatype.oss diff --git a/other/java/client/pom.xml.deploy b/other/java/client/pom.xml.deploy index 7d845f439..bc55291fb 100644 --- a/other/java/client/pom.xml.deploy +++ b/other/java/client/pom.xml.deploy @@ -5,7 +5,7 @@ com.github.chrislusf seaweedfs-client - 1.5.8 + 1.5.9 org.sonatype.oss diff --git a/other/java/client/pom_debug.xml b/other/java/client/pom_debug.xml index a4e2d694e..7a9019176 100644 --- a/other/java/client/pom_debug.xml +++ b/other/java/client/pom_debug.xml @@ -5,7 +5,7 @@ com.github.chrislusf seaweedfs-client - 1.5.8 + 1.5.9 org.sonatype.oss diff --git a/other/java/examples/pom.xml b/other/java/examples/pom.xml index 12196324e..22159f114 100644 --- a/other/java/examples/pom.xml +++ b/other/java/examples/pom.xml @@ -11,13 +11,13 @@ com.github.chrislusf seaweedfs-client - 1.5.8 + 1.5.9 compile com.github.chrislusf seaweedfs-hadoop2-client - 1.5.8 + 1.5.9 compile diff --git a/other/java/hdfs2/dependency-reduced-pom.xml b/other/java/hdfs2/dependency-reduced-pom.xml index a9c062a75..699a2c241 100644 --- a/other/java/hdfs2/dependency-reduced-pom.xml +++ b/other/java/hdfs2/dependency-reduced-pom.xml @@ -301,7 +301,7 @@ - 1.5.8 + 1.5.9 2.9.2 diff --git a/other/java/hdfs2/pom.xml b/other/java/hdfs2/pom.xml index 65ce0766a..d31b5aa5b 100644 --- a/other/java/hdfs2/pom.xml +++ b/other/java/hdfs2/pom.xml @@ -5,7 +5,7 @@ 4.0.0 - 1.5.8 + 1.5.9 2.9.2 diff --git a/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java b/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java index 6072d3ec8..25395db7a 100644 --- a/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java +++ b/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java @@ -26,7 +26,7 @@ public class SeaweedFileSystem extends FileSystem { public static final int FS_SEAWEED_DEFAULT_PORT = 8888; public static final String FS_SEAWEED_BUFFER_SIZE = "fs.seaweed.buffer.size"; public static final String FS_SEAWEED_REPLICATION = "fs.seaweed.replication"; - public static final String FS_SEAWEED_VOLUME_SERVER_ACCESS = "fs.seaweed.volumeServerAccess"; + public static final String FS_SEAWEED_VOLUME_SERVER_ACCESS = "fs.seaweed.volume.server.access"; public static final int FS_SEAWEED_DEFAULT_BUFFER_SIZE = 4 * 1024 * 1024; private static final Logger LOG = LoggerFactory.getLogger(SeaweedFileSystem.class); diff --git a/other/java/hdfs3/dependency-reduced-pom.xml b/other/java/hdfs3/dependency-reduced-pom.xml index 007e7115b..162b1f64d 100644 --- a/other/java/hdfs3/dependency-reduced-pom.xml +++ b/other/java/hdfs3/dependency-reduced-pom.xml @@ -309,7 +309,7 @@ - 1.5.8 + 1.5.9 3.1.1 diff --git a/other/java/hdfs3/pom.xml b/other/java/hdfs3/pom.xml index 4a0e2603b..0a39185fe 100644 --- a/other/java/hdfs3/pom.xml +++ b/other/java/hdfs3/pom.xml @@ -5,7 +5,7 @@ 4.0.0 - 1.5.8 + 1.5.9 3.1.1 diff --git a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java index 6072d3ec8..25395db7a 100644 --- a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java +++ b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java @@ -26,7 +26,7 @@ public class SeaweedFileSystem extends FileSystem { public static final int FS_SEAWEED_DEFAULT_PORT = 8888; public static final String FS_SEAWEED_BUFFER_SIZE = "fs.seaweed.buffer.size"; public static final String FS_SEAWEED_REPLICATION = "fs.seaweed.replication"; - public static final String FS_SEAWEED_VOLUME_SERVER_ACCESS = "fs.seaweed.volumeServerAccess"; + public static final String FS_SEAWEED_VOLUME_SERVER_ACCESS = "fs.seaweed.volume.server.access"; public static final int FS_SEAWEED_DEFAULT_BUFFER_SIZE = 4 * 1024 * 1024; private static final Logger LOG = LoggerFactory.getLogger(SeaweedFileSystem.class); From 35ba277a976b583f486e4beb0a1dde3ccff5c433 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 5 Feb 2021 22:43:56 -0800 Subject: [PATCH 246/260] Java: fix filerProxy mode --- .../seaweedfs/client/FilerGrpcClient.java | 28 +++++++++++++------ .../java/seaweedfs/client/SeaweedRead.java | 8 +----- .../java/seaweedfs/client/SeaweedWrite.java | 8 +----- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java b/other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java index 8a37827f1..6c57e2e0d 100644 --- a/other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java +++ b/other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java @@ -9,16 +9,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.net.ssl.SSLException; -import java.util.Map; import java.util.HashMap; +import java.util.Map; import java.util.concurrent.TimeUnit; public class FilerGrpcClient { - public final int VOLUME_SERVER_ACCESS_DIRECT = 0; - public final int VOLUME_SERVER_ACCESS_PUBLIC_URL = 1; - public final int VOLUME_SERVER_ACCESS_FILER_PROXY = 2; - private static final Logger logger = LoggerFactory.getLogger(FilerGrpcClient.class); static SslContext sslContext; @@ -30,6 +26,9 @@ public class FilerGrpcClient { } } + public final int VOLUME_SERVER_ACCESS_DIRECT = 0; + public final int VOLUME_SERVER_ACCESS_PUBLIC_URL = 1; + public final int VOLUME_SERVER_ACCESS_FILER_PROXY = 2; public final Map vidLocations = new HashMap<>(); private final ManagedChannel channel; private final SeaweedFilerGrpc.SeaweedFilerBlockingStub blockingStub; @@ -55,7 +54,7 @@ public class FilerGrpcClient { .negotiationType(NegotiationType.TLS) .sslContext(sslContext)); - filerAddress = String.format("%s:%d", host, grpcPort-10000); + filerAddress = String.format("%s:%d", host, grpcPort - 10000); FilerProto.GetFilerConfigurationResponse filerConfigurationResponse = this.getBlockingStub().getFilerConfiguration( @@ -104,23 +103,36 @@ public class FilerGrpcClient { public void setAccessVolumeServerDirectly() { this.volumeServerAccess = VOLUME_SERVER_ACCESS_DIRECT; } + public boolean isAccessVolumeServerDirectly() { return this.volumeServerAccess == VOLUME_SERVER_ACCESS_DIRECT; } + public void setAccessVolumeServerByPublicUrl() { this.volumeServerAccess = VOLUME_SERVER_ACCESS_PUBLIC_URL; } + public boolean isAccessVolumeServerByPublicUrl() { return this.volumeServerAccess == VOLUME_SERVER_ACCESS_PUBLIC_URL; } + public void setAccessVolumeServerByFilerProxy() { this.volumeServerAccess = VOLUME_SERVER_ACCESS_FILER_PROXY; } + public boolean isAccessVolumeServerByFilerProxy() { return this.volumeServerAccess == VOLUME_SERVER_ACCESS_FILER_PROXY; } - public String getFilerAddress() { - return this.filerAddress; + + public String getChunkUrl(String chunkId, String url, String publicUrl) { + switch (this.volumeServerAccess) { + case VOLUME_SERVER_ACCESS_PUBLIC_URL: + return String.format("http://%s/%s", publicUrl, chunkId); + case VOLUME_SERVER_ACCESS_FILER_PROXY: + return String.format("http://%s/?proxyChunkId=%s", this.filerAddress, chunkId); + default: + return String.format("http://%s/%s", url, chunkId); + } } } diff --git a/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java b/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java index a70553762..e55c5b7aa 100644 --- a/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java +++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java @@ -116,13 +116,7 @@ public class SeaweedRead { IOException lastException = null; for (long waitTime = 1000L; waitTime < 10 * 1000; waitTime += waitTime / 2) { for (FilerProto.Location location : locations.getLocationsList()) { - String host = location.getUrl(); - if (filerGrpcClient.isAccessVolumeServerByPublicUrl()) { - host = location.getPublicUrl(); - } else if (filerGrpcClient.isAccessVolumeServerByFilerProxy()) { - host = filerGrpcClient.getFilerAddress(); - } - String url = String.format("http://%s/%s", host, chunkView.fileId); + String url = filerGrpcClient.getChunkUrl(chunkView.fileId, location.getUrl(), location.getPublicUrl()); try { data = doFetchOneFullChunkData(chunkView, url); lastException = null; diff --git a/other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java b/other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java index 3cc11e21c..db3cc3931 100644 --- a/other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java +++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java @@ -53,13 +53,7 @@ public class SeaweedWrite { String fileId = response.getFileId(); String auth = response.getAuth(); - String host = response.getUrl(); - if (filerGrpcClient.isAccessVolumeServerByPublicUrl()) { - host = response.getPublicUrl(); - } else if (filerGrpcClient.isAccessVolumeServerByFilerProxy()) { - host = filerGrpcClient.getFilerAddress(); - } - String targetUrl = String.format("http://%s/%s", host, fileId); + String targetUrl = filerGrpcClient.getChunkUrl(fileId, response.getUrl(), response.getPublicUrl()); ByteString cipherKeyString = com.google.protobuf.ByteString.EMPTY; byte[] cipherKey = null; From 5138d3954fea88cc2131f0d86b14e3a777de29d7 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 6 Feb 2021 00:09:12 -0800 Subject: [PATCH 247/260] Java 1.6.0 fix filerProxy mode --- other/java/client/pom.xml | 2 +- other/java/client/pom.xml.deploy | 2 +- other/java/client/pom_debug.xml | 2 +- other/java/examples/pom.xml | 4 ++-- other/java/hdfs2/dependency-reduced-pom.xml | 2 +- other/java/hdfs2/pom.xml | 2 +- other/java/hdfs3/dependency-reduced-pom.xml | 2 +- other/java/hdfs3/pom.xml | 2 +- 8 files changed, 9 insertions(+), 9 deletions(-) diff --git a/other/java/client/pom.xml b/other/java/client/pom.xml index 4606ef5d9..aaffe0aa8 100644 --- a/other/java/client/pom.xml +++ b/other/java/client/pom.xml @@ -5,7 +5,7 @@ com.github.chrislusf seaweedfs-client - 1.5.9 + 1.6.0 org.sonatype.oss diff --git a/other/java/client/pom.xml.deploy b/other/java/client/pom.xml.deploy index bc55291fb..f32cdc427 100644 --- a/other/java/client/pom.xml.deploy +++ b/other/java/client/pom.xml.deploy @@ -5,7 +5,7 @@ com.github.chrislusf seaweedfs-client - 1.5.9 + 1.6.0 org.sonatype.oss diff --git a/other/java/client/pom_debug.xml b/other/java/client/pom_debug.xml index 7a9019176..06b8e0ed5 100644 --- a/other/java/client/pom_debug.xml +++ b/other/java/client/pom_debug.xml @@ -5,7 +5,7 @@ com.github.chrislusf seaweedfs-client - 1.5.9 + 1.6.0 org.sonatype.oss diff --git a/other/java/examples/pom.xml b/other/java/examples/pom.xml index 22159f114..eb1e73a92 100644 --- a/other/java/examples/pom.xml +++ b/other/java/examples/pom.xml @@ -11,13 +11,13 @@ com.github.chrislusf seaweedfs-client - 1.5.9 + 1.6.0 compile com.github.chrislusf seaweedfs-hadoop2-client - 1.5.9 + 1.6.0 compile diff --git a/other/java/hdfs2/dependency-reduced-pom.xml b/other/java/hdfs2/dependency-reduced-pom.xml index 699a2c241..10b95db45 100644 --- a/other/java/hdfs2/dependency-reduced-pom.xml +++ b/other/java/hdfs2/dependency-reduced-pom.xml @@ -301,7 +301,7 @@ - 1.5.9 + 1.6.0 2.9.2 diff --git a/other/java/hdfs2/pom.xml b/other/java/hdfs2/pom.xml index d31b5aa5b..48b7750f4 100644 --- a/other/java/hdfs2/pom.xml +++ b/other/java/hdfs2/pom.xml @@ -5,7 +5,7 @@ 4.0.0 - 1.5.9 + 1.6.0 2.9.2 diff --git a/other/java/hdfs3/dependency-reduced-pom.xml b/other/java/hdfs3/dependency-reduced-pom.xml index 162b1f64d..71a2f739d 100644 --- a/other/java/hdfs3/dependency-reduced-pom.xml +++ b/other/java/hdfs3/dependency-reduced-pom.xml @@ -309,7 +309,7 @@ - 1.5.9 + 1.6.0 3.1.1 diff --git a/other/java/hdfs3/pom.xml b/other/java/hdfs3/pom.xml index 0a39185fe..ffe2d319a 100644 --- a/other/java/hdfs3/pom.xml +++ b/other/java/hdfs3/pom.xml @@ -5,7 +5,7 @@ 4.0.0 - 1.5.9 + 1.6.0 3.1.1 From 94ea3bd3a56dd16583c33a3baec17a952b58c6bb Mon Sep 17 00:00:00 2001 From: bingoohuang Date: Sun, 7 Feb 2021 09:00:03 +0800 Subject: [PATCH 248/260] renaming NeedleMapType to NeedleMapKind --- weed/server/volume_server.go | 4 ++-- weed/storage/disk_location.go | 8 ++++---- weed/storage/needle_map.go | 4 ++-- weed/storage/store.go | 12 ++++++------ weed/storage/volume.go | 4 ++-- weed/storage/volume_loading.go | 4 ++-- weed/storage/volume_read_write.go | 2 +- 7 files changed, 19 insertions(+), 19 deletions(-) diff --git a/weed/server/volume_server.go b/weed/server/volume_server.go index 468f75890..0443309fb 100644 --- a/weed/server/volume_server.go +++ b/weed/server/volume_server.go @@ -24,7 +24,7 @@ type VolumeServer struct { guard *security.Guard grpcDialOption grpc.DialOption - needleMapKind storage.NeedleMapType + needleMapKind storage.NeedleMapKind FixJpgOrientation bool ReadRedirect bool compactionBytePerSecond int64 @@ -39,7 +39,7 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string, port int, publicUrl string, folders []string, maxCounts []int, minFreeSpacePercents []float32, idxFolder string, - needleMapKind storage.NeedleMapType, + needleMapKind storage.NeedleMapKind, masterNodes []string, pulseSeconds int, dataCenter string, rack string, whiteList []string, diff --git a/weed/storage/disk_location.go b/weed/storage/disk_location.go index 9b2ab69fe..76eab64a3 100644 --- a/weed/storage/disk_location.go +++ b/weed/storage/disk_location.go @@ -82,7 +82,7 @@ func getValidVolumeName(basename string) string { return "" } -func (l *DiskLocation) loadExistingVolume(fileInfo os.FileInfo, needleMapKind NeedleMapType) bool { +func (l *DiskLocation) loadExistingVolume(fileInfo os.FileInfo, needleMapKind NeedleMapKind) bool { basename := fileInfo.Name() if fileInfo.IsDir() { return false @@ -133,7 +133,7 @@ func (l *DiskLocation) loadExistingVolume(fileInfo os.FileInfo, needleMapKind Ne return true } -func (l *DiskLocation) concurrentLoadingVolumes(needleMapKind NeedleMapType, concurrency int) { +func (l *DiskLocation) concurrentLoadingVolumes(needleMapKind NeedleMapKind, concurrency int) { task_queue := make(chan os.FileInfo, 10*concurrency) go func() { @@ -167,7 +167,7 @@ func (l *DiskLocation) concurrentLoadingVolumes(needleMapKind NeedleMapType, con } -func (l *DiskLocation) loadExistingVolumes(needleMapKind NeedleMapType) { +func (l *DiskLocation) loadExistingVolumes(needleMapKind NeedleMapKind) { l.concurrentLoadingVolumes(needleMapKind, 10) glog.V(0).Infof("Store started on dir: %s with %d volumes max %d", l.Directory, len(l.volumes), l.MaxVolumeCount) @@ -237,7 +237,7 @@ func (l *DiskLocation) deleteVolumeById(vid needle.VolumeId) (found bool, e erro return } -func (l *DiskLocation) LoadVolume(vid needle.VolumeId, needleMapKind NeedleMapType) bool { +func (l *DiskLocation) LoadVolume(vid needle.VolumeId, needleMapKind NeedleMapKind) bool { if fileInfo, found := l.LocateVolume(vid); found { return l.loadExistingVolume(fileInfo, needleMapKind) } diff --git a/weed/storage/needle_map.go b/weed/storage/needle_map.go index 9f331267d..5b41286ea 100644 --- a/weed/storage/needle_map.go +++ b/weed/storage/needle_map.go @@ -11,10 +11,10 @@ import ( . "github.com/chrislusf/seaweedfs/weed/storage/types" ) -type NeedleMapType int +type NeedleMapKind int const ( - NeedleMapInMemory NeedleMapType = iota + NeedleMapInMemory NeedleMapKind = iota NeedleMapLevelDb // small memory footprint, 4MB total, 1 write buffer, 3 block buffer NeedleMapLevelDbMedium // medium memory footprint, 8MB total, 3 write buffer, 5 block buffer NeedleMapLevelDbLarge // large memory footprint, 12MB total, 4write buffer, 8 block buffer diff --git a/weed/storage/store.go b/weed/storage/store.go index ff28be47c..bd66e23f4 100644 --- a/weed/storage/store.go +++ b/weed/storage/store.go @@ -40,7 +40,7 @@ type Store struct { dataCenter string // optional informaton, overwriting master setting if exists rack string // optional information, overwriting master setting if exists connected bool - NeedleMapType NeedleMapType + NeedleMapKind NeedleMapKind NewVolumesChan chan master_pb.VolumeShortInformationMessage DeletedVolumesChan chan master_pb.VolumeShortInformationMessage NewEcShardsChan chan master_pb.VolumeEcShardInformationMessage @@ -52,8 +52,8 @@ func (s *Store) String() (str string) { return } -func NewStore(grpcDialOption grpc.DialOption, port int, ip, publicUrl string, dirnames []string, maxVolumeCounts []int, minFreeSpacePercents []float32, idxFolder string, needleMapKind NeedleMapType) (s *Store) { - s = &Store{grpcDialOption: grpcDialOption, Port: port, Ip: ip, PublicUrl: publicUrl, NeedleMapType: needleMapKind} +func NewStore(grpcDialOption grpc.DialOption, port int, ip, publicUrl string, dirnames []string, maxVolumeCounts []int, minFreeSpacePercents []float32, idxFolder string, needleMapKind NeedleMapKind) (s *Store) { + s = &Store{grpcDialOption: grpcDialOption, Port: port, Ip: ip, PublicUrl: publicUrl, NeedleMapKind: needleMapKind} s.Locations = make([]*DiskLocation, 0) for i := 0; i < len(dirnames); i++ { location := NewDiskLocation(dirnames[i], maxVolumeCounts[i], minFreeSpacePercents[i], idxFolder) @@ -69,7 +69,7 @@ func NewStore(grpcDialOption grpc.DialOption, port int, ip, publicUrl string, di return } -func (s *Store) AddVolume(volumeId needle.VolumeId, collection string, needleMapKind NeedleMapType, replicaPlacement string, ttlString string, preallocate int64, MemoryMapMaxSizeMb uint32) error { +func (s *Store) AddVolume(volumeId needle.VolumeId, collection string, needleMapKind NeedleMapKind, replicaPlacement string, ttlString string, preallocate int64, MemoryMapMaxSizeMb uint32) error { rt, e := super_block.NewReplicaPlacementFromString(replicaPlacement) if e != nil { return e @@ -114,7 +114,7 @@ func (s *Store) FindFreeLocation() (ret *DiskLocation) { } return ret } -func (s *Store) addVolume(vid needle.VolumeId, collection string, needleMapKind NeedleMapType, replicaPlacement *super_block.ReplicaPlacement, ttl *needle.TTL, preallocate int64, memoryMapMaxSizeMb uint32) error { +func (s *Store) addVolume(vid needle.VolumeId, collection string, needleMapKind NeedleMapKind, replicaPlacement *super_block.ReplicaPlacement, ttl *needle.TTL, preallocate int64, memoryMapMaxSizeMb uint32) error { if s.findVolume(vid) != nil { return fmt.Errorf("Volume Id %d already exists!", vid) } @@ -362,7 +362,7 @@ func (s *Store) MarkVolumeWritable(i needle.VolumeId) error { func (s *Store) MountVolume(i needle.VolumeId) error { for _, location := range s.Locations { - if found := location.LoadVolume(i, s.NeedleMapType); found == true { + if found := location.LoadVolume(i, s.NeedleMapKind); found == true { glog.V(0).Infof("mount volume %d", i) v := s.findVolume(i) s.NewVolumesChan <- master_pb.VolumeShortInformationMessage{ diff --git a/weed/storage/volume.go b/weed/storage/volume.go index 80a74c3e3..d86e25885 100644 --- a/weed/storage/volume.go +++ b/weed/storage/volume.go @@ -25,7 +25,7 @@ type Volume struct { Collection string DataBackend backend.BackendStorageFile nm NeedleMapper - needleMapKind NeedleMapType + needleMapKind NeedleMapKind noWriteOrDelete bool // if readonly, either noWriteOrDelete or noWriteCanDelete noWriteCanDelete bool // if readonly, either noWriteOrDelete or noWriteCanDelete noWriteLock sync.RWMutex @@ -50,7 +50,7 @@ type Volume struct { lastIoError error } -func NewVolume(dirname string, dirIdx string, collection string, id needle.VolumeId, needleMapKind NeedleMapType, replicaPlacement *super_block.ReplicaPlacement, ttl *needle.TTL, preallocate int64, memoryMapMaxSizeMb uint32) (v *Volume, e error) { +func NewVolume(dirname string, dirIdx string, collection string, id needle.VolumeId, needleMapKind NeedleMapKind, replicaPlacement *super_block.ReplicaPlacement, ttl *needle.TTL, preallocate int64, memoryMapMaxSizeMb uint32) (v *Volume, e error) { // if replicaPlacement is nil, the superblock will be loaded from disk v = &Volume{dir: dirname, dirIdx: dirIdx, Collection: collection, Id: id, MemoryMapMaxSizeMb: memoryMapMaxSizeMb, asyncRequestsChan: make(chan *needle.AsyncRequest, 128)} diff --git a/weed/storage/volume_loading.go b/weed/storage/volume_loading.go index a6efc630d..52a50a98c 100644 --- a/weed/storage/volume_loading.go +++ b/weed/storage/volume_loading.go @@ -14,7 +14,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/util" ) -func loadVolumeWithoutIndex(dirname string, collection string, id needle.VolumeId, needleMapKind NeedleMapType) (v *Volume, err error) { +func loadVolumeWithoutIndex(dirname string, collection string, id needle.VolumeId, needleMapKind NeedleMapKind) (v *Volume, err error) { v = &Volume{dir: dirname, Collection: collection, Id: id} v.SuperBlock = super_block.SuperBlock{} v.needleMapKind = needleMapKind @@ -22,7 +22,7 @@ func loadVolumeWithoutIndex(dirname string, collection string, id needle.VolumeI return } -func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind NeedleMapType, preallocate int64) (err error) { +func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind NeedleMapKind, preallocate int64) (err error) { alreadyHasSuperBlock := false hasLoadedVolume := false diff --git a/weed/storage/volume_read_write.go b/weed/storage/volume_read_write.go index f28ee50e6..b40414ad4 100644 --- a/weed/storage/volume_read_write.go +++ b/weed/storage/volume_read_write.go @@ -410,7 +410,7 @@ type VolumeFileScanner interface { } func ScanVolumeFile(dirname string, collection string, id needle.VolumeId, - needleMapKind NeedleMapType, + needleMapKind NeedleMapKind, volumeFileScanner VolumeFileScanner) (err error) { var v *Volume if v, err = loadVolumeWithoutIndex(dirname, collection, id, needleMapKind); err != nil { From 7256902fb0c3ac637d40d8ee91508216b42a6d60 Mon Sep 17 00:00:00 2001 From: bingoohuang Date: Sun, 7 Feb 2021 12:11:51 +0800 Subject: [PATCH 249/260] fix typo offset.ToAcutalOffset to offset.ToActualOffset --- .../diff_volume_servers/diff_volume_servers.go | 2 +- weed/command/export.go | 2 +- weed/server/volume_grpc_copy_incremental.go | 2 +- weed/server/volume_grpc_tail.go | 2 +- weed/storage/erasure_coding/ec_decoder.go | 2 +- weed/storage/erasure_coding/ec_test.go | 4 ++-- weed/storage/erasure_coding/ec_volume.go | 2 +- weed/storage/erasure_coding/ec_volume_test.go | 6 +++--- weed/storage/store_ec.go | 4 ++-- weed/storage/types/offset_4bytes.go | 2 +- weed/storage/volume_backup.go | 8 ++++---- weed/storage/volume_checking.go | 2 +- weed/storage/volume_read_write.go | 16 ++++++++-------- weed/storage/volume_vacuum.go | 10 +++++----- weed/util/chunk_cache/chunk_cache_on_disk.go | 4 ++-- 15 files changed, 34 insertions(+), 34 deletions(-) diff --git a/unmaintained/diff_volume_servers/diff_volume_servers.go b/unmaintained/diff_volume_servers/diff_volume_servers.go index 6107f3d48..27a537617 100644 --- a/unmaintained/diff_volume_servers/diff_volume_servers.go +++ b/unmaintained/diff_volume_servers/diff_volume_servers.go @@ -168,7 +168,7 @@ func getVolumeFiles(v uint32, addr string) (map[types.NeedleId]needleState, int6 size: size, } } - if actual := offset.ToAcutalOffset(); actual > maxOffset { + if actual := offset.ToActualOffset(); actual > maxOffset { maxOffset = actual } return nil diff --git a/weed/command/export.go b/weed/command/export.go index 78d75ef52..f100f3af5 100644 --- a/weed/command/export.go +++ b/weed/command/export.go @@ -113,7 +113,7 @@ func (scanner *VolumeFileScanner4Export) VisitNeedle(n *needle.Needle, offset in nv, ok := needleMap.Get(n.Id) glog.V(3).Infof("key %d offset %d size %d disk_size %d compressed %v ok %v nv %+v", n.Id, offset, n.Size, n.DiskSize(scanner.version), n.IsCompressed(), ok, nv) - if *showDeleted && n.Size > 0 || ok && nv.Size.IsValid() && nv.Offset.ToAcutalOffset() == offset { + if *showDeleted && n.Size > 0 || ok && nv.Size.IsValid() && nv.Offset.ToActualOffset() == offset { if newerThanUnix >= 0 && n.HasLastModifiedDate() && n.LastModified < uint64(newerThanUnix) { glog.V(3).Infof("Skipping this file, as it's old enough: LastModified %d vs %d", n.LastModified, newerThanUnix) diff --git a/weed/server/volume_grpc_copy_incremental.go b/weed/server/volume_grpc_copy_incremental.go index 6d6c3daa3..82b143c3d 100644 --- a/weed/server/volume_grpc_copy_incremental.go +++ b/weed/server/volume_grpc_copy_incremental.go @@ -27,7 +27,7 @@ func (vs *VolumeServer) VolumeIncrementalCopy(req *volume_server_pb.VolumeIncrem return nil } - startOffset := foundOffset.ToAcutalOffset() + startOffset := foundOffset.ToActualOffset() buf := make([]byte, 1024*1024*2) return sendFileContent(v.DataBackend, buf, startOffset, int64(stopOffset), stream) diff --git a/weed/server/volume_grpc_tail.go b/weed/server/volume_grpc_tail.go index 2dde5b69c..6c039ebf5 100644 --- a/weed/server/volume_grpc_tail.go +++ b/weed/server/volume_grpc_tail.go @@ -72,7 +72,7 @@ func sendNeedlesSince(stream volume_server_pb.VolumeServer_VolumeTailSenderServe stream: stream, } - err = storage.ScanVolumeFileFrom(v.Version(), v.DataBackend, foundOffset.ToAcutalOffset(), scanner) + err = storage.ScanVolumeFileFrom(v.Version(), v.DataBackend, foundOffset.ToActualOffset(), scanner) return scanner.lastProcessedTimestampNs, err diff --git a/weed/storage/erasure_coding/ec_decoder.go b/weed/storage/erasure_coding/ec_decoder.go index bc86d9c04..47d3c6550 100644 --- a/weed/storage/erasure_coding/ec_decoder.go +++ b/weed/storage/erasure_coding/ec_decoder.go @@ -58,7 +58,7 @@ func FindDatFileSize(dataBaseFileName, indexBaseFileName string) (datSize int64, return nil } - entryStopOffset := offset.ToAcutalOffset() + needle.GetActualSize(size, version) + entryStopOffset := offset.ToActualOffset() + needle.GetActualSize(size, version) if datSize < entryStopOffset { datSize = entryStopOffset } diff --git a/weed/storage/erasure_coding/ec_test.go b/weed/storage/erasure_coding/ec_test.go index 63cc2c352..0d48bec02 100644 --- a/weed/storage/erasure_coding/ec_test.go +++ b/weed/storage/erasure_coding/ec_test.go @@ -93,7 +93,7 @@ func assertSame(datFile *os.File, datSize int64, ecFiles []*os.File, offset type func readDatFile(datFile *os.File, offset types.Offset, size types.Size) ([]byte, error) { data := make([]byte, size) - n, err := datFile.ReadAt(data, offset.ToAcutalOffset()) + n, err := datFile.ReadAt(data, offset.ToActualOffset()) if err != nil { return nil, fmt.Errorf("failed to ReadAt dat file: %v", err) } @@ -105,7 +105,7 @@ func readDatFile(datFile *os.File, offset types.Offset, size types.Size) ([]byte func readEcFile(datSize int64, ecFiles []*os.File, offset types.Offset, size types.Size) (data []byte, err error) { - intervals := LocateData(largeBlockSize, smallBlockSize, datSize, offset.ToAcutalOffset(), size) + intervals := LocateData(largeBlockSize, smallBlockSize, datSize, offset.ToActualOffset(), size) for i, interval := range intervals { if d, e := readOneInterval(interval, ecFiles); e != nil { diff --git a/weed/storage/erasure_coding/ec_volume.go b/weed/storage/erasure_coding/ec_volume.go index 2183e43d6..a9d08ed0e 100644 --- a/weed/storage/erasure_coding/ec_volume.go +++ b/weed/storage/erasure_coding/ec_volume.go @@ -211,7 +211,7 @@ func (ev *EcVolume) LocateEcShardNeedle(needleId types.NeedleId, version needle. shard := ev.Shards[0] // calculate the locations in the ec shards - intervals = LocateData(ErasureCodingLargeBlockSize, ErasureCodingSmallBlockSize, DataShardsCount*shard.ecdFileSize, offset.ToAcutalOffset(), types.Size(needle.GetActualSize(size, version))) + intervals = LocateData(ErasureCodingLargeBlockSize, ErasureCodingSmallBlockSize, DataShardsCount*shard.ecdFileSize, offset.ToActualOffset(), types.Size(needle.GetActualSize(size, version))) return } diff --git a/weed/storage/erasure_coding/ec_volume_test.go b/weed/storage/erasure_coding/ec_volume_test.go index fe45bf722..747ef4aab 100644 --- a/weed/storage/erasure_coding/ec_volume_test.go +++ b/weed/storage/erasure_coding/ec_volume_test.go @@ -35,16 +35,16 @@ func TestPositioning(t *testing.T) { needleId, _ := types.ParseNeedleId(test.needleId) offset, size, err := SearchNeedleFromSortedIndex(ecxFile, fileSize, needleId, nil) assert.Equal(t, nil, err, "SearchNeedleFromSortedIndex") - fmt.Printf("offset: %d size: %d\n", offset.ToAcutalOffset(), size) + fmt.Printf("offset: %d size: %d\n", offset.ToActualOffset(), size) } needleId, _ := types.ParseNeedleId("0f087622") offset, size, err := SearchNeedleFromSortedIndex(ecxFile, fileSize, needleId, nil) assert.Equal(t, nil, err, "SearchNeedleFromSortedIndex") - fmt.Printf("offset: %d size: %d\n", offset.ToAcutalOffset(), size) + fmt.Printf("offset: %d size: %d\n", offset.ToActualOffset(), size) var shardEcdFileSize int64 = 1118830592 // 1024*1024*1024*3 - intervals := LocateData(ErasureCodingLargeBlockSize, ErasureCodingSmallBlockSize, DataShardsCount*shardEcdFileSize, offset.ToAcutalOffset(), types.Size(needle.GetActualSize(size, needle.CurrentVersion))) + intervals := LocateData(ErasureCodingLargeBlockSize, ErasureCodingSmallBlockSize, DataShardsCount*shardEcdFileSize, offset.ToActualOffset(), types.Size(needle.GetActualSize(size, needle.CurrentVersion))) for _, interval := range intervals { shardId, shardOffset := interval.ToShardIdAndOffset(ErasureCodingLargeBlockSize, ErasureCodingSmallBlockSize) diff --git a/weed/storage/store_ec.go b/weed/storage/store_ec.go index 853757ce3..ab4e96634 100644 --- a/weed/storage/store_ec.go +++ b/weed/storage/store_ec.go @@ -131,7 +131,7 @@ func (s *Store) ReadEcShardNeedle(vid needle.VolumeId, n *needle.Needle) (int, e return 0, ErrorDeleted } - glog.V(3).Infof("read ec volume %d offset %d size %d intervals:%+v", vid, offset.ToAcutalOffset(), size, intervals) + glog.V(3).Infof("read ec volume %d offset %d size %d intervals:%+v", vid, offset.ToActualOffset(), size, intervals) if len(intervals) > 1 { glog.V(3).Infof("ReadEcShardNeedle needle id %s intervals:%+v", n.String(), intervals) @@ -144,7 +144,7 @@ func (s *Store) ReadEcShardNeedle(vid needle.VolumeId, n *needle.Needle) (int, e return 0, ErrorDeleted } - err = n.ReadBytes(bytes, offset.ToAcutalOffset(), size, localEcVolume.Version) + err = n.ReadBytes(bytes, offset.ToActualOffset(), size, localEcVolume.Version) if err != nil { return 0, fmt.Errorf("readbytes: %v", err) } diff --git a/weed/storage/types/offset_4bytes.go b/weed/storage/types/offset_4bytes.go index d53147e21..5348d5b36 100644 --- a/weed/storage/types/offset_4bytes.go +++ b/weed/storage/types/offset_4bytes.go @@ -54,7 +54,7 @@ func ToOffset(offset int64) Offset { return Uint32ToOffset(smaller) } -func (offset Offset) ToAcutalOffset() (actualOffset int64) { +func (offset Offset) ToActualOffset() (actualOffset int64) { return (int64(offset.b0) + int64(offset.b1)<<8 + int64(offset.b2)<<16 + int64(offset.b3)<<24) * int64(NeedlePaddingSize) } diff --git a/weed/storage/volume_backup.go b/weed/storage/volume_backup.go index 9aeb10f69..82ea12a89 100644 --- a/weed/storage/volume_backup.go +++ b/weed/storage/volume_backup.go @@ -154,13 +154,13 @@ func (v *Volume) locateLastAppendEntry() (Offset, error) { func (v *Volume) readAppendAtNs(offset Offset) (uint64, error) { - n, _, bodyLength, err := needle.ReadNeedleHeader(v.DataBackend, v.SuperBlock.Version, offset.ToAcutalOffset()) + n, _, bodyLength, err := needle.ReadNeedleHeader(v.DataBackend, v.SuperBlock.Version, offset.ToActualOffset()) if err != nil { - return 0, fmt.Errorf("ReadNeedleHeader %s [%d,%d): %v", v.DataBackend.Name(), offset.ToAcutalOffset(), offset.ToAcutalOffset()+NeedleHeaderSize, err) + return 0, fmt.Errorf("ReadNeedleHeader %s [%d,%d): %v", v.DataBackend.Name(), offset.ToActualOffset(), offset.ToActualOffset()+NeedleHeaderSize, err) } - _, err = n.ReadNeedleBody(v.DataBackend, v.SuperBlock.Version, offset.ToAcutalOffset()+NeedleHeaderSize, bodyLength) + _, err = n.ReadNeedleBody(v.DataBackend, v.SuperBlock.Version, offset.ToActualOffset()+NeedleHeaderSize, bodyLength) if err != nil { - return 0, fmt.Errorf("ReadNeedleBody offset %d, bodyLength %d: %v", offset.ToAcutalOffset(), bodyLength, err) + return 0, fmt.Errorf("ReadNeedleBody offset %d, bodyLength %d: %v", offset.ToActualOffset(), bodyLength, err) } return n.AppendAtNs, nil diff --git a/weed/storage/volume_checking.go b/weed/storage/volume_checking.go index 00e04047f..8d63c39c1 100644 --- a/weed/storage/volume_checking.go +++ b/weed/storage/volume_checking.go @@ -58,7 +58,7 @@ func doCheckAndFixVolumeData(v *Volume, indexFile *os.File, indexOffset int64) ( return lastAppendAtNs, fmt.Errorf("verifyNeedleIntegrity %s failed: %v", indexFile.Name(), err) } } else { - if lastAppendAtNs, err = verifyNeedleIntegrity(v.DataBackend, v.Version(), offset.ToAcutalOffset(), key, size); err != nil { + if lastAppendAtNs, err = verifyNeedleIntegrity(v.DataBackend, v.Version(), offset.ToActualOffset(), key, size); err != nil { return lastAppendAtNs, err } } diff --git a/weed/storage/volume_read_write.go b/weed/storage/volume_read_write.go index b40414ad4..07376bc88 100644 --- a/weed/storage/volume_read_write.go +++ b/weed/storage/volume_read_write.go @@ -41,9 +41,9 @@ func (v *Volume) isFileUnchanged(n *needle.Needle) bool { nv, ok := v.nm.Get(n.Id) if ok && !nv.Offset.IsZero() && nv.Size.IsValid() { oldNeedle := new(needle.Needle) - err := oldNeedle.ReadData(v.DataBackend, nv.Offset.ToAcutalOffset(), nv.Size, v.Version()) + err := oldNeedle.ReadData(v.DataBackend, nv.Offset.ToActualOffset(), nv.Size, v.Version()) if err != nil { - glog.V(0).Infof("Failed to check updated file at offset %d size %d: %v", nv.Offset.ToAcutalOffset(), nv.Size, err) + glog.V(0).Infof("Failed to check updated file at offset %d size %d: %v", nv.Offset.ToActualOffset(), nv.Size, err) return false } if oldNeedle.Cookie == n.Cookie && oldNeedle.Checksum == n.Checksum && bytes.Equal(oldNeedle.Data, n.Data) { @@ -113,7 +113,7 @@ func (v *Volume) syncWrite(n *needle.Needle) (offset uint64, size Size, isUnchan // check whether existing needle cookie matches nv, ok := v.nm.Get(n.Id) if ok { - existingNeedle, _, _, existingNeedleReadErr := needle.ReadNeedleHeader(v.DataBackend, v.Version(), nv.Offset.ToAcutalOffset()) + existingNeedle, _, _, existingNeedleReadErr := needle.ReadNeedleHeader(v.DataBackend, v.Version(), nv.Offset.ToActualOffset()) if existingNeedleReadErr != nil { err = fmt.Errorf("reading existing needle: %v", existingNeedleReadErr) return @@ -136,7 +136,7 @@ func (v *Volume) syncWrite(n *needle.Needle) (offset uint64, size Size, isUnchan v.lastAppendAtNs = n.AppendAtNs // add to needle map - if !ok || uint64(nv.Offset.ToAcutalOffset()) < offset { + if !ok || uint64(nv.Offset.ToActualOffset()) < offset { if err = v.nm.Put(n.Id, ToOffset(int64(offset)), n.Size); err != nil { glog.V(4).Infof("failed to save in needle map %d: %v", n.Id, err) } @@ -179,7 +179,7 @@ func (v *Volume) doWriteRequest(n *needle.Needle) (offset uint64, size Size, isU // check whether existing needle cookie matches nv, ok := v.nm.Get(n.Id) if ok { - existingNeedle, _, _, existingNeedleReadErr := needle.ReadNeedleHeader(v.DataBackend, v.Version(), nv.Offset.ToAcutalOffset()) + existingNeedle, _, _, existingNeedleReadErr := needle.ReadNeedleHeader(v.DataBackend, v.Version(), nv.Offset.ToActualOffset()) if existingNeedleReadErr != nil { err = fmt.Errorf("reading existing needle: %v", existingNeedleReadErr) return @@ -201,7 +201,7 @@ func (v *Volume) doWriteRequest(n *needle.Needle) (offset uint64, size Size, isU v.lastAppendAtNs = n.AppendAtNs // add to needle map - if !ok || uint64(nv.Offset.ToAcutalOffset()) < offset { + if !ok || uint64(nv.Offset.ToActualOffset()) < offset { if err = v.nm.Put(n.Id, ToOffset(int64(offset)), n.Size); err != nil { glog.V(4).Infof("failed to save in needle map %d: %v", n.Id, err) } @@ -303,9 +303,9 @@ func (v *Volume) readNeedle(n *needle.Needle, readOption *ReadOption) (int, erro if readSize == 0 { return 0, nil } - err := n.ReadData(v.DataBackend, nv.Offset.ToAcutalOffset(), readSize, v.Version()) + err := n.ReadData(v.DataBackend, nv.Offset.ToActualOffset(), readSize, v.Version()) if err == needle.ErrorSizeMismatch && OffsetSize == 4 { - err = n.ReadData(v.DataBackend, nv.Offset.ToAcutalOffset()+int64(MaxPossibleVolumeSize), readSize, v.Version()) + err = n.ReadData(v.DataBackend, nv.Offset.ToActualOffset()+int64(MaxPossibleVolumeSize), readSize, v.Version()) } v.checkReadWriteError(err) if err != nil { diff --git a/weed/storage/volume_vacuum.go b/weed/storage/volume_vacuum.go index 5884eca87..c17c9c937 100644 --- a/weed/storage/volume_vacuum.go +++ b/weed/storage/volume_vacuum.go @@ -280,11 +280,11 @@ func (v *Volume) makeupDiff(newDatFileName, newIdxFileName, oldDatFileName, oldI //updated needle if !increIdxEntry.offset.IsZero() && increIdxEntry.size != 0 && increIdxEntry.size.IsValid() { //even the needle cache in memory is hit, the need_bytes is correct - glog.V(4).Infof("file %d offset %d size %d", key, increIdxEntry.offset.ToAcutalOffset(), increIdxEntry.size) + glog.V(4).Infof("file %d offset %d size %d", key, increIdxEntry.offset.ToActualOffset(), increIdxEntry.size) var needleBytes []byte - needleBytes, err = needle.ReadNeedleBlob(oldDatBackend, increIdxEntry.offset.ToAcutalOffset(), increIdxEntry.size, v.Version()) + needleBytes, err = needle.ReadNeedleBlob(oldDatBackend, increIdxEntry.offset.ToActualOffset(), increIdxEntry.size, v.Version()) if err != nil { - return fmt.Errorf("ReadNeedleBlob %s key %d offset %d size %d failed: %v", oldDatFile.Name(), key, increIdxEntry.offset.ToAcutalOffset(), increIdxEntry.size, err) + return fmt.Errorf("ReadNeedleBlob %s key %d offset %d size %d failed: %v", oldDatFile.Name(), key, increIdxEntry.offset.ToActualOffset(), increIdxEntry.size, err) } dst.Write(needleBytes) util.Uint32toBytes(idxEntryBytes[8:12], uint32(offset/NeedlePaddingSize)) @@ -339,7 +339,7 @@ func (scanner *VolumeFileScanner4Vacuum) VisitNeedle(n *needle.Needle, offset in } nv, ok := scanner.v.nm.Get(n.Id) glog.V(4).Infoln("needle expected offset ", offset, "ok", ok, "nv", nv) - if ok && nv.Offset.ToAcutalOffset() == offset && nv.Size > 0 && nv.Size.IsValid() { + if ok && nv.Offset.ToActualOffset() == offset && nv.Size > 0 && nv.Size.IsValid() { if err := scanner.nm.Set(n.Id, ToOffset(scanner.newOffset), n.Size); err != nil { return fmt.Errorf("cannot put needle: %s", err) } @@ -422,7 +422,7 @@ func copyDataBasedOnIndexFile(srcDatName, srcIdxName, dstDatName, datIdxName str } n := new(needle.Needle) - err := n.ReadData(srcDatBackend, offset.ToAcutalOffset(), size, version) + err := n.ReadData(srcDatBackend, offset.ToActualOffset(), size, version) if err != nil { return nil } diff --git a/weed/util/chunk_cache/chunk_cache_on_disk.go b/weed/util/chunk_cache/chunk_cache_on_disk.go index 356dfe188..d724e925e 100644 --- a/weed/util/chunk_cache/chunk_cache_on_disk.go +++ b/weed/util/chunk_cache/chunk_cache_on_disk.go @@ -107,9 +107,9 @@ func (v *ChunkCacheVolume) GetNeedle(key types.NeedleId) ([]byte, error) { return nil, storage.ErrorNotFound } data := make([]byte, nv.Size) - if readSize, readErr := v.DataBackend.ReadAt(data, nv.Offset.ToAcutalOffset()); readErr != nil { + if readSize, readErr := v.DataBackend.ReadAt(data, nv.Offset.ToActualOffset()); readErr != nil { return nil, fmt.Errorf("read %s.dat [%d,%d): %v", - v.fileName, nv.Offset.ToAcutalOffset(), nv.Offset.ToAcutalOffset()+int64(nv.Size), readErr) + v.fileName, nv.Offset.ToActualOffset(), nv.Offset.ToActualOffset()+int64(nv.Size), readErr) } else { if readSize != int(nv.Size) { return nil, fmt.Errorf("read %d, expected %d", readSize, nv.Size) From 15c60cbb2660d60d2714d3386731044fcfeac473 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 7 Feb 2021 03:50:01 -0800 Subject: [PATCH 250/260] close the grpc connection after 10 hours related to https://github.com/chrislusf/seaweedfs/issues/1782 --- weed/pb/grpc_client_server.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/weed/pb/grpc_client_server.go b/weed/pb/grpc_client_server.go index 4d78d769f..9ffda9b04 100644 --- a/weed/pb/grpc_client_server.go +++ b/weed/pb/grpc_client_server.go @@ -35,8 +35,9 @@ func NewGrpcServer(opts ...grpc.ServerOption) *grpc.Server { var options []grpc.ServerOption options = append(options, grpc.KeepaliveParams(keepalive.ServerParameters{ - Time: 10 * time.Second, // wait time before ping if no activity - Timeout: 20 * time.Second, // ping timeout + Time: 10 * time.Second, // wait time before ping if no activity + Timeout: 20 * time.Second, // ping timeout + MaxConnectionAge: 10 * time.Hour, }), grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{ MinTime: 60 * time.Second, // min time a client should wait before sending a ping From 5e4b5109ddaff1fc7c09cc107081c04aaf0056f6 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 7 Feb 2021 22:29:43 -0800 Subject: [PATCH 251/260] 2.24 --- k8s/seaweedfs/Chart.yaml | 4 ++-- k8s/seaweedfs/values.yaml | 2 +- weed/util/constants.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/k8s/seaweedfs/Chart.yaml b/k8s/seaweedfs/Chart.yaml index 4febe5129..b890cc666 100644 --- a/k8s/seaweedfs/Chart.yaml +++ b/k8s/seaweedfs/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v1 description: SeaweedFS name: seaweedfs -appVersion: "2.23" -version: 2.23 +appVersion: "2.24" +version: 2.24 diff --git a/k8s/seaweedfs/values.yaml b/k8s/seaweedfs/values.yaml index 168c0dbcd..2b33a6149 100644 --- a/k8s/seaweedfs/values.yaml +++ b/k8s/seaweedfs/values.yaml @@ -4,7 +4,7 @@ global: registry: "" repository: "" imageName: chrislusf/seaweedfs - # imageTag: "2.23" - started using {.Chart.appVersion} + # imageTag: "2.24" - started using {.Chart.appVersion} imagePullPolicy: IfNotPresent imagePullSecrets: imagepullsecret restartPolicy: Always diff --git a/weed/util/constants.go b/weed/util/constants.go index ccc0ef1fb..6001ae78e 100644 --- a/weed/util/constants.go +++ b/weed/util/constants.go @@ -5,7 +5,7 @@ import ( ) var ( - VERSION = fmt.Sprintf("%s %d.%02d", sizeLimit, 2, 23) + VERSION = fmt.Sprintf("%s %d.%02d", sizeLimit, 2, 24) COMMIT = "" ) From a83302113214748fe3600510c8835b6d88f3496c Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 7 Feb 2021 23:03:03 -0800 Subject: [PATCH 252/260] fix refactoring left over --- weed/storage/types/offset_5bytes.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/storage/types/offset_5bytes.go b/weed/storage/types/offset_5bytes.go index 05c6d2f39..b6181fc11 100644 --- a/weed/storage/types/offset_5bytes.go +++ b/weed/storage/types/offset_5bytes.go @@ -71,7 +71,7 @@ func ToOffset(offset int64) Offset { } } -func (offset Offset) ToAcutalOffset() (actualOffset int64) { +func (offset Offset) ToActualOffset() (actualOffset int64) { return (int64(offset.b0) + int64(offset.b1)<<8 + int64(offset.b2)<<16 + int64(offset.b3)<<24 + int64(offset.b4)<<32) * int64(NeedlePaddingSize) } From ad36c7b0d76f870a5cb0c7c68f5c81ec5340a79e Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 8 Feb 2021 02:28:45 -0800 Subject: [PATCH 253/260] refactoring: only expose FilerClient class --- .../seaweedfs/client/FileChunkManifest.java | 26 +++++++++---------- .../java/seaweedfs/client/FilerClient.java | 24 +++++++---------- .../seaweedfs/client/SeaweedInputStream.java | 17 ++++++------ .../seaweedfs/client/SeaweedOutputStream.java | 18 ++++++------- .../java/seaweedfs/client/SeaweedRead.java | 18 ++++++------- .../java/seaweedfs/client/SeaweedWrite.java | 22 ++++++++-------- .../seaweedfs/examples/ExampleReadFile.java | 6 ++--- .../seaweedfs/examples/ExampleWriteFile.java | 13 +++++----- .../seaweed/hdfs/SeaweedFileSystemStore.java | 16 +++++------- .../hdfs/SeaweedHadoopInputStream.java | 6 ++--- .../hdfs/SeaweedHadoopOutputStream.java | 6 ++--- .../seaweed/hdfs/SeaweedFileSystemStore.java | 16 +++++------- .../hdfs/SeaweedHadoopInputStream.java | 6 ++--- .../hdfs/SeaweedHadoopOutputStream.java | 6 ++--- 14 files changed, 94 insertions(+), 106 deletions(-) diff --git a/other/java/client/src/main/java/seaweedfs/client/FileChunkManifest.java b/other/java/client/src/main/java/seaweedfs/client/FileChunkManifest.java index 3d7da91d5..9b6ba5dfc 100644 --- a/other/java/client/src/main/java/seaweedfs/client/FileChunkManifest.java +++ b/other/java/client/src/main/java/seaweedfs/client/FileChunkManifest.java @@ -23,7 +23,7 @@ public class FileChunkManifest { } public static List resolveChunkManifest( - final FilerGrpcClient filerGrpcClient, List chunks) throws IOException { + final FilerClient filerClient, List chunks) throws IOException { List dataChunks = new ArrayList<>(); @@ -35,30 +35,30 @@ public class FileChunkManifest { // IsChunkManifest LOG.debug("fetching chunk manifest:{}", chunk); - byte[] data = fetchChunk(filerGrpcClient, chunk); + byte[] data = fetchChunk(filerClient, chunk); FilerProto.FileChunkManifest m = FilerProto.FileChunkManifest.newBuilder().mergeFrom(data).build(); List resolvedChunks = new ArrayList<>(); for (FilerProto.FileChunk t : m.getChunksList()) { // avoid deprecated chunk.getFileId() resolvedChunks.add(t.toBuilder().setFileId(FilerClient.toFileId(t.getFid())).build()); } - dataChunks.addAll(resolveChunkManifest(filerGrpcClient, resolvedChunks)); + dataChunks.addAll(resolveChunkManifest(filerClient, resolvedChunks)); } return dataChunks; } - private static byte[] fetchChunk(final FilerGrpcClient filerGrpcClient, FilerProto.FileChunk chunk) throws IOException { + private static byte[] fetchChunk(final FilerClient filerClient, FilerProto.FileChunk chunk) throws IOException { String vid = "" + chunk.getFid().getVolumeId(); - FilerProto.Locations locations = filerGrpcClient.vidLocations.get(vid); + FilerProto.Locations locations = filerClient.vidLocations.get(vid); if (locations == null) { FilerProto.LookupVolumeRequest.Builder lookupRequest = FilerProto.LookupVolumeRequest.newBuilder(); lookupRequest.addVolumeIds(vid); - FilerProto.LookupVolumeResponse lookupResponse = filerGrpcClient + FilerProto.LookupVolumeResponse lookupResponse = filerClient .getBlockingStub().lookupVolume(lookupRequest.build()); locations = lookupResponse.getLocationsMapMap().get(vid); - filerGrpcClient.vidLocations.put(vid, locations); + filerClient.vidLocations.put(vid, locations); LOG.debug("fetchChunk vid:{} locations:{}", vid, locations); } @@ -74,7 +74,7 @@ public class FileChunkManifest { byte[] chunkData = SeaweedRead.chunkCache.getChunk(chunkView.fileId); if (chunkData == null) { LOG.debug("doFetchFullChunkData:{}", chunkView); - chunkData = SeaweedRead.doFetchFullChunkData(filerGrpcClient, chunkView, locations); + chunkData = SeaweedRead.doFetchFullChunkData(filerClient, chunkView, locations); } if (chunk.getIsChunkManifest()){ LOG.debug("chunk {} size {}", chunkView.fileId, chunkData.length); @@ -86,7 +86,7 @@ public class FileChunkManifest { } public static List maybeManifestize( - final FilerGrpcClient filerGrpcClient, List inputChunks, String parentDirectory) throws IOException { + final FilerClient filerClient, List inputChunks, String parentDirectory) throws IOException { // the return variable List chunks = new ArrayList<>(); @@ -101,7 +101,7 @@ public class FileChunkManifest { int remaining = dataChunks.size(); for (int i = 0; i + mergeFactor < dataChunks.size(); i += mergeFactor) { - FilerProto.FileChunk chunk = mergeIntoManifest(filerGrpcClient, dataChunks.subList(i, i + mergeFactor), parentDirectory); + FilerProto.FileChunk chunk = mergeIntoManifest(filerClient, dataChunks.subList(i, i + mergeFactor), parentDirectory); chunks.add(chunk); remaining -= mergeFactor; } @@ -113,7 +113,7 @@ public class FileChunkManifest { return chunks; } - private static FilerProto.FileChunk mergeIntoManifest(final FilerGrpcClient filerGrpcClient, List dataChunks, String parentDirectory) throws IOException { + private static FilerProto.FileChunk mergeIntoManifest(final FilerClient filerClient, List dataChunks, String parentDirectory) throws IOException { // create and serialize the manifest dataChunks = FilerClient.beforeEntrySerialization(dataChunks); FilerProto.FileChunkManifest.Builder m = FilerProto.FileChunkManifest.newBuilder().addAllChunks(dataChunks); @@ -127,8 +127,8 @@ public class FileChunkManifest { } FilerProto.FileChunk.Builder manifestChunk = SeaweedWrite.writeChunk( - filerGrpcClient.getReplication(), - filerGrpcClient, + filerClient.getReplication(), + filerClient, minOffset, data, 0, data.length, parentDirectory); manifestChunk.setIsChunkManifest(true); diff --git a/other/java/client/src/main/java/seaweedfs/client/FilerClient.java b/other/java/client/src/main/java/seaweedfs/client/FilerClient.java index 7338d5bee..58269d41f 100644 --- a/other/java/client/src/main/java/seaweedfs/client/FilerClient.java +++ b/other/java/client/src/main/java/seaweedfs/client/FilerClient.java @@ -11,18 +11,12 @@ import java.util.Arrays; import java.util.Iterator; import java.util.List; -public class FilerClient { +public class FilerClient extends FilerGrpcClient { private static final Logger LOG = LoggerFactory.getLogger(FilerClient.class); - private final FilerGrpcClient filerGrpcClient; - public FilerClient(String host, int grpcPort) { - filerGrpcClient = new FilerGrpcClient(host, grpcPort); - } - - public FilerClient(FilerGrpcClient filerGrpcClient) { - this.filerGrpcClient = filerGrpcClient; + super(host, grpcPort); } public static String toFileId(FilerProto.FileId fid) { @@ -236,7 +230,7 @@ public class FilerClient { } public List listEntries(String path, String entryPrefix, String lastEntryName, int limit, boolean includeLastEntry) { - Iterator iter = filerGrpcClient.getBlockingStub().listEntries(FilerProto.ListEntriesRequest.newBuilder() + Iterator iter = this.getBlockingStub().listEntries(FilerProto.ListEntriesRequest.newBuilder() .setDirectory(path) .setPrefix(entryPrefix) .setStartFromFileName(lastEntryName) @@ -253,7 +247,7 @@ public class FilerClient { public FilerProto.Entry lookupEntry(String directory, String entryName) { try { - FilerProto.Entry entry = filerGrpcClient.getBlockingStub().lookupDirectoryEntry( + FilerProto.Entry entry = this.getBlockingStub().lookupDirectoryEntry( FilerProto.LookupDirectoryEntryRequest.newBuilder() .setDirectory(directory) .setName(entryName) @@ -274,7 +268,7 @@ public class FilerClient { public boolean createEntry(String parent, FilerProto.Entry entry) { try { FilerProto.CreateEntryResponse createEntryResponse = - filerGrpcClient.getBlockingStub().createEntry(FilerProto.CreateEntryRequest.newBuilder() + this.getBlockingStub().createEntry(FilerProto.CreateEntryRequest.newBuilder() .setDirectory(parent) .setEntry(entry) .build()); @@ -291,7 +285,7 @@ public class FilerClient { public boolean updateEntry(String parent, FilerProto.Entry entry) { try { - filerGrpcClient.getBlockingStub().updateEntry(FilerProto.UpdateEntryRequest.newBuilder() + this.getBlockingStub().updateEntry(FilerProto.UpdateEntryRequest.newBuilder() .setDirectory(parent) .setEntry(entry) .build()); @@ -304,7 +298,7 @@ public class FilerClient { public boolean deleteEntry(String parent, String entryName, boolean isDeleteFileChunk, boolean isRecursive, boolean ignoreRecusiveError) { try { - filerGrpcClient.getBlockingStub().deleteEntry(FilerProto.DeleteEntryRequest.newBuilder() + this.getBlockingStub().deleteEntry(FilerProto.DeleteEntryRequest.newBuilder() .setDirectory(parent) .setName(entryName) .setIsDeleteData(isDeleteFileChunk) @@ -320,7 +314,7 @@ public class FilerClient { public boolean atomicRenameEntry(String oldParent, String oldName, String newParent, String newName) { try { - filerGrpcClient.getBlockingStub().atomicRenameEntry(FilerProto.AtomicRenameEntryRequest.newBuilder() + this.getBlockingStub().atomicRenameEntry(FilerProto.AtomicRenameEntryRequest.newBuilder() .setOldDirectory(oldParent) .setOldName(oldName) .setNewDirectory(newParent) @@ -334,7 +328,7 @@ public class FilerClient { } public Iterator watch(String prefix, String clientName, long sinceNs) { - return filerGrpcClient.getBlockingStub().subscribeMetadata(FilerProto.SubscribeMetadataRequest.newBuilder() + return this.getBlockingStub().subscribeMetadata(FilerProto.SubscribeMetadataRequest.newBuilder() .setPathPrefix(prefix) .setClientName(clientName) .setSinceNs(sinceNs) diff --git a/other/java/client/src/main/java/seaweedfs/client/SeaweedInputStream.java b/other/java/client/src/main/java/seaweedfs/client/SeaweedInputStream.java index 8b26c242c..4e40ce1b6 100644 --- a/other/java/client/src/main/java/seaweedfs/client/SeaweedInputStream.java +++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedInputStream.java @@ -16,7 +16,7 @@ public class SeaweedInputStream extends InputStream { private static final Logger LOG = LoggerFactory.getLogger(SeaweedInputStream.class); private static final IOException EXCEPTION_STREAM_IS_CLOSED = new IOException("Stream is closed!"); - private final FilerGrpcClient filerGrpcClient; + private final FilerClient filerClient; private final String path; private final FilerProto.Entry entry; private final List visibleIntervalList; @@ -27,32 +27,31 @@ public class SeaweedInputStream extends InputStream { private boolean closed = false; public SeaweedInputStream( - final FilerGrpcClient filerGrpcClient, + final FilerClient filerClient, final String fullpath) throws IOException { - this.filerGrpcClient = filerGrpcClient; this.path = fullpath; - FilerClient filerClient = new FilerClient(filerGrpcClient); + this.filerClient = filerClient; this.entry = filerClient.lookupEntry( SeaweedOutputStream.getParentDirectory(fullpath), SeaweedOutputStream.getFileName(fullpath)); this.contentLength = SeaweedRead.fileSize(entry); - this.visibleIntervalList = SeaweedRead.nonOverlappingVisibleIntervals(filerGrpcClient, entry.getChunksList()); + this.visibleIntervalList = SeaweedRead.nonOverlappingVisibleIntervals(filerClient, entry.getChunksList()); LOG.debug("new path:{} entry:{} visibleIntervalList:{}", path, entry, visibleIntervalList); } public SeaweedInputStream( - final FilerGrpcClient filerGrpcClient, + final FilerClient filerClient, final String path, final FilerProto.Entry entry) throws IOException { - this.filerGrpcClient = filerGrpcClient; + this.filerClient = filerClient; this.path = path; this.entry = entry; this.contentLength = SeaweedRead.fileSize(entry); - this.visibleIntervalList = SeaweedRead.nonOverlappingVisibleIntervals(filerGrpcClient, entry.getChunksList()); + this.visibleIntervalList = SeaweedRead.nonOverlappingVisibleIntervals(filerClient, entry.getChunksList()); LOG.debug("new path:{} entry:{} visibleIntervalList:{}", path, entry, visibleIntervalList); @@ -110,7 +109,7 @@ public class SeaweedInputStream extends InputStream { if (start+len <= entry.getContent().size()) { entry.getContent().substring(start, start+len).copyTo(buf); } else { - bytesRead = SeaweedRead.read(this.filerGrpcClient, this.visibleIntervalList, this.position, buf, SeaweedRead.fileSize(entry)); + bytesRead = SeaweedRead.read(this.filerClient, this.visibleIntervalList, this.position, buf, SeaweedRead.fileSize(entry)); } if (bytesRead > Integer.MAX_VALUE) { diff --git a/other/java/client/src/main/java/seaweedfs/client/SeaweedOutputStream.java b/other/java/client/src/main/java/seaweedfs/client/SeaweedOutputStream.java index 92dc59f61..b73e99e69 100644 --- a/other/java/client/src/main/java/seaweedfs/client/SeaweedOutputStream.java +++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedOutputStream.java @@ -15,7 +15,7 @@ public class SeaweedOutputStream extends OutputStream { private static final Logger LOG = LoggerFactory.getLogger(SeaweedOutputStream.class); protected final boolean supportFlush = true; - private final FilerGrpcClient filerGrpcClient; + private final FilerClient filerClient; private final String path; private final int bufferSize; private final int maxConcurrentRequestCount; @@ -33,17 +33,17 @@ public class SeaweedOutputStream extends OutputStream { private long outputIndex; private String replication = "000"; - public SeaweedOutputStream(FilerGrpcClient filerGrpcClient, final String fullpath) { - this(filerGrpcClient, fullpath, "000"); + public SeaweedOutputStream(FilerClient filerClient, final String fullpath) { + this(filerClient, fullpath, "000"); } - public SeaweedOutputStream(FilerGrpcClient filerGrpcClient, final String fullpath, final String replication) { - this(filerGrpcClient, fullpath, null, 0, 8 * 1024 * 1024, "000"); + public SeaweedOutputStream(FilerClient filerClient, final String fullpath, final String replication) { + this(filerClient, fullpath, null, 0, 8 * 1024 * 1024, "000"); } - public SeaweedOutputStream(FilerGrpcClient filerGrpcClient, final String path, FilerProto.Entry.Builder entry, + public SeaweedOutputStream(FilerClient filerClient, final String path, FilerProto.Entry.Builder entry, final long position, final int bufferSize, final String replication) { - this.filerGrpcClient = filerGrpcClient; + this.filerClient = filerClient; this.replication = replication; this.path = path; this.position = position; @@ -109,7 +109,7 @@ public class SeaweedOutputStream extends OutputStream { private synchronized void flushWrittenBytesToServiceInternal(final long offset) throws IOException { try { - SeaweedWrite.writeMeta(filerGrpcClient, getParentDirectory(path), entry); + SeaweedWrite.writeMeta(filerClient, getParentDirectory(path), entry); } catch (Exception ex) { throw new IOException(ex); } @@ -225,7 +225,7 @@ public class SeaweedOutputStream extends OutputStream { } final Future job = completionService.submit(() -> { // System.out.println(path + " is going to save [" + (writePosition) + "," + ((writePosition) + bytesLength) + ")"); - SeaweedWrite.writeData(entry, replication, filerGrpcClient, writePosition, bufferToWrite.array(), bufferToWrite.position(), bufferToWrite.limit(), path); + SeaweedWrite.writeData(entry, replication, filerClient, writePosition, bufferToWrite.array(), bufferToWrite.position(), bufferToWrite.limit(), path); // System.out.println(path + " saved [" + (writePosition) + "," + ((writePosition) + bytesLength) + ")"); ByteBufferPool.release(bufferToWrite); return null; diff --git a/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java b/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java index e55c5b7aa..384636601 100644 --- a/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java +++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java @@ -23,7 +23,7 @@ public class SeaweedRead { static VolumeIdCache volumeIdCache = new VolumeIdCache(4 * 1024); // returns bytesRead - public static long read(FilerGrpcClient filerGrpcClient, List visibleIntervals, + public static long read(FilerClient filerClient, List visibleIntervals, final long position, final ByteBuffer buf, final long fileSize) throws IOException { List chunkViews = viewFromVisibles(visibleIntervals, position, buf.remaining()); @@ -42,7 +42,7 @@ public class SeaweedRead { } if (lookupRequest.getVolumeIdsCount() > 0) { - FilerProto.LookupVolumeResponse lookupResponse = filerGrpcClient + FilerProto.LookupVolumeResponse lookupResponse = filerClient .getBlockingStub().lookupVolume(lookupRequest.build()); Map vid2Locations = lookupResponse.getLocationsMapMap(); for (Map.Entry entry : vid2Locations.entrySet()) { @@ -71,7 +71,7 @@ public class SeaweedRead { return 0; } - int len = readChunkView(filerGrpcClient, startOffset, buf, chunkView, locations); + int len = readChunkView(filerClient, startOffset, buf, chunkView, locations); LOG.debug("read [{},{}) {} size {}", startOffset, startOffset + len, chunkView.fileId, chunkView.size); @@ -93,12 +93,12 @@ public class SeaweedRead { return readCount; } - private static int readChunkView(FilerGrpcClient filerGrpcClient, long startOffset, ByteBuffer buf, ChunkView chunkView, FilerProto.Locations locations) throws IOException { + private static int readChunkView(FilerClient filerClient, long startOffset, ByteBuffer buf, ChunkView chunkView, FilerProto.Locations locations) throws IOException { byte[] chunkData = chunkCache.getChunk(chunkView.fileId); if (chunkData == null) { - chunkData = doFetchFullChunkData(filerGrpcClient, chunkView, locations); + chunkData = doFetchFullChunkData(filerClient, chunkView, locations); chunkCache.setChunk(chunkView.fileId, chunkData); } @@ -110,13 +110,13 @@ public class SeaweedRead { return len; } - public static byte[] doFetchFullChunkData(FilerGrpcClient filerGrpcClient, ChunkView chunkView, FilerProto.Locations locations) throws IOException { + public static byte[] doFetchFullChunkData(FilerClient filerClient, ChunkView chunkView, FilerProto.Locations locations) throws IOException { byte[] data = null; IOException lastException = null; for (long waitTime = 1000L; waitTime < 10 * 1000; waitTime += waitTime / 2) { for (FilerProto.Location location : locations.getLocationsList()) { - String url = filerGrpcClient.getChunkUrl(chunkView.fileId, location.getUrl(), location.getPublicUrl()); + String url = filerClient.getChunkUrl(chunkView.fileId, location.getUrl(), location.getPublicUrl()); try { data = doFetchOneFullChunkData(chunkView, url); lastException = null; @@ -221,9 +221,9 @@ public class SeaweedRead { } public static List nonOverlappingVisibleIntervals( - final FilerGrpcClient filerGrpcClient, List chunkList) throws IOException { + final FilerClient filerClient, List chunkList) throws IOException { - chunkList = FileChunkManifest.resolveChunkManifest(filerGrpcClient, chunkList); + chunkList = FileChunkManifest.resolveChunkManifest(filerClient, chunkList); FilerProto.FileChunk[] chunks = chunkList.toArray(new FilerProto.FileChunk[0]); Arrays.sort(chunks, new Comparator() { diff --git a/other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java b/other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java index db3cc3931..f8c0c76b6 100644 --- a/other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java +++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java @@ -23,29 +23,29 @@ public class SeaweedWrite { public static void writeData(FilerProto.Entry.Builder entry, final String replication, - final FilerGrpcClient filerGrpcClient, + final FilerClient filerClient, final long offset, final byte[] bytes, final long bytesOffset, final long bytesLength, final String path) throws IOException { FilerProto.FileChunk.Builder chunkBuilder = writeChunk( - replication, filerGrpcClient, offset, bytes, bytesOffset, bytesLength, path); + replication, filerClient, offset, bytes, bytesOffset, bytesLength, path); synchronized (entry) { entry.addChunks(chunkBuilder); } } public static FilerProto.FileChunk.Builder writeChunk(final String replication, - final FilerGrpcClient filerGrpcClient, + final FilerClient filerClient, final long offset, final byte[] bytes, final long bytesOffset, final long bytesLength, final String path) throws IOException { - FilerProto.AssignVolumeResponse response = filerGrpcClient.getBlockingStub().assignVolume( + FilerProto.AssignVolumeResponse response = filerClient.getBlockingStub().assignVolume( FilerProto.AssignVolumeRequest.newBuilder() - .setCollection(filerGrpcClient.getCollection()) - .setReplication(replication == null ? filerGrpcClient.getReplication() : replication) + .setCollection(filerClient.getCollection()) + .setReplication(replication == null ? filerClient.getReplication() : replication) .setDataCenter("") .setTtlSec(0) .setPath(path) @@ -53,11 +53,11 @@ public class SeaweedWrite { String fileId = response.getFileId(); String auth = response.getAuth(); - String targetUrl = filerGrpcClient.getChunkUrl(fileId, response.getUrl(), response.getPublicUrl()); + String targetUrl = filerClient.getChunkUrl(fileId, response.getUrl(), response.getPublicUrl()); ByteString cipherKeyString = com.google.protobuf.ByteString.EMPTY; byte[] cipherKey = null; - if (filerGrpcClient.isCipher()) { + if (filerClient.isCipher()) { cipherKey = genCipherKey(); cipherKeyString = ByteString.copyFrom(cipherKey); } @@ -75,15 +75,15 @@ public class SeaweedWrite { .setCipherKey(cipherKeyString); } - public static void writeMeta(final FilerGrpcClient filerGrpcClient, + public static void writeMeta(final FilerClient filerClient, final String parentDirectory, final FilerProto.Entry.Builder entry) throws IOException { synchronized (entry) { - List chunks = FileChunkManifest.maybeManifestize(filerGrpcClient, entry.getChunksList(), parentDirectory); + List chunks = FileChunkManifest.maybeManifestize(filerClient, entry.getChunksList(), parentDirectory); entry.clearChunks(); entry.addAllChunks(chunks); - filerGrpcClient.getBlockingStub().createEntry( + filerClient.getBlockingStub().createEntry( FilerProto.CreateEntryRequest.newBuilder() .setDirectory(parentDirectory) .setEntry(entry) diff --git a/other/java/examples/src/main/java/com/seaweedfs/examples/ExampleReadFile.java b/other/java/examples/src/main/java/com/seaweedfs/examples/ExampleReadFile.java index bd73df802..d2eb94135 100644 --- a/other/java/examples/src/main/java/com/seaweedfs/examples/ExampleReadFile.java +++ b/other/java/examples/src/main/java/com/seaweedfs/examples/ExampleReadFile.java @@ -1,6 +1,6 @@ package com.seaweedfs.examples; -import seaweedfs.client.FilerGrpcClient; +import seaweedfs.client.FilerClient; import seaweedfs.client.SeaweedInputStream; import java.io.FileInputStream; @@ -13,7 +13,7 @@ public class ExampleReadFile { public static void main(String[] args) throws IOException { - FilerGrpcClient filerGrpcClient = new FilerGrpcClient("localhost", 18888); + FilerClient filerClient = new FilerClient("localhost", 18888); long startTime = System.currentTimeMillis(); parseZip("/Users/chris/tmp/test.zip"); @@ -23,7 +23,7 @@ public class ExampleReadFile { long localProcessTime = startTime2 - startTime; SeaweedInputStream seaweedInputStream = new SeaweedInputStream( - filerGrpcClient, "/test.zip"); + filerClient, "/test.zip"); parseZip(seaweedInputStream); long swProcessTime = System.currentTimeMillis() - startTime2; diff --git a/other/java/examples/src/main/java/com/seaweedfs/examples/ExampleWriteFile.java b/other/java/examples/src/main/java/com/seaweedfs/examples/ExampleWriteFile.java index 228a3c0b7..26b74028f 100644 --- a/other/java/examples/src/main/java/com/seaweedfs/examples/ExampleWriteFile.java +++ b/other/java/examples/src/main/java/com/seaweedfs/examples/ExampleWriteFile.java @@ -1,6 +1,6 @@ package com.seaweedfs.examples; -import seaweedfs.client.FilerGrpcClient; +import seaweedfs.client.FilerClient; import seaweedfs.client.SeaweedInputStream; import seaweedfs.client.SeaweedOutputStream; @@ -13,15 +13,14 @@ public class ExampleWriteFile { public static void main(String[] args) throws IOException { - FilerGrpcClient filerGrpcClient = new FilerGrpcClient("localhost", 18888); + FilerClient filerClient = new FilerClient("localhost", 18888); - SeaweedInputStream seaweedInputStream = new SeaweedInputStream( - filerGrpcClient, "/test.zip"); - unZipFiles(filerGrpcClient, seaweedInputStream); + SeaweedInputStream seaweedInputStream = new SeaweedInputStream(filerClient, "/test.zip"); + unZipFiles(filerClient, seaweedInputStream); } - public static void unZipFiles(FilerGrpcClient filerGrpcClient, InputStream is) throws IOException { + public static void unZipFiles(FilerClient filerClient, InputStream is) throws IOException { ZipInputStream zin = new ZipInputStream(is); ZipEntry ze; while ((ze = zin.getNextEntry()) != null) { @@ -34,7 +33,7 @@ public class ExampleWriteFile { continue; } - SeaweedOutputStream seaweedOutputStream = new SeaweedOutputStream(filerGrpcClient, "/test/"+filename); + SeaweedOutputStream seaweedOutputStream = new SeaweedOutputStream(filerClient, "/test/"+filename); byte[] bytesIn = new byte[16 * 1024]; int read = 0; while ((read = zin.read(bytesIn))!=-1) { diff --git a/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java b/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java index 8147f3efe..f4e8c9349 100644 --- a/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java +++ b/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java @@ -24,27 +24,25 @@ public class SeaweedFileSystemStore { private static final Logger LOG = LoggerFactory.getLogger(SeaweedFileSystemStore.class); - private FilerGrpcClient filerGrpcClient; private FilerClient filerClient; private Configuration conf; public SeaweedFileSystemStore(String host, int port, Configuration conf) { int grpcPort = 10000 + port; - filerGrpcClient = new FilerGrpcClient(host, grpcPort); - filerClient = new FilerClient(filerGrpcClient); + filerClient = new FilerClient(host, grpcPort); this.conf = conf; String volumeServerAccessMode = this.conf.get(FS_SEAWEED_VOLUME_SERVER_ACCESS, "direct"); if (volumeServerAccessMode.equals("publicUrl")) { - filerGrpcClient.setAccessVolumeServerByPublicUrl(); + filerClient.setAccessVolumeServerByPublicUrl(); } else if (volumeServerAccessMode.equals("filerProxy")) { - filerGrpcClient.setAccessVolumeServerByFilerProxy(); + filerClient.setAccessVolumeServerByFilerProxy(); } } public void close() { try { - this.filerGrpcClient.shutdown(); + this.filerClient.shutdown(); } catch (InterruptedException e) { e.printStackTrace(); } @@ -219,10 +217,10 @@ public class SeaweedFileSystemStore { .clearGroupName() .addAllGroupName(Arrays.asList(userGroupInformation.getGroupNames())) ); - SeaweedWrite.writeMeta(filerGrpcClient, getParentDirectory(path), entry); + SeaweedWrite.writeMeta(filerClient, getParentDirectory(path), entry); } - return new SeaweedHadoopOutputStream(filerGrpcClient, path.toString(), entry, writePosition, bufferSize, replication); + return new SeaweedHadoopOutputStream(filerClient, path.toString(), entry, writePosition, bufferSize, replication); } @@ -236,7 +234,7 @@ public class SeaweedFileSystemStore { throw new FileNotFoundException("read non-exist file " + path); } - return new SeaweedHadoopInputStream(filerGrpcClient, + return new SeaweedHadoopInputStream(filerClient, statistics, path.toUri().getPath(), entry); diff --git a/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedHadoopInputStream.java b/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedHadoopInputStream.java index dd9bf4032..f26eae597 100644 --- a/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedHadoopInputStream.java +++ b/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedHadoopInputStream.java @@ -5,7 +5,7 @@ package seaweed.hdfs; import org.apache.hadoop.fs.ByteBufferReadable; import org.apache.hadoop.fs.FSInputStream; import org.apache.hadoop.fs.FileSystem.Statistics; -import seaweedfs.client.FilerGrpcClient; +import seaweedfs.client.FilerClient; import seaweedfs.client.FilerProto; import seaweedfs.client.SeaweedInputStream; @@ -19,11 +19,11 @@ public class SeaweedHadoopInputStream extends FSInputStream implements ByteBuffe private final Statistics statistics; public SeaweedHadoopInputStream( - final FilerGrpcClient filerGrpcClient, + final FilerClient filerClient, final Statistics statistics, final String path, final FilerProto.Entry entry) throws IOException { - this.seaweedInputStream = new SeaweedInputStream(filerGrpcClient, path, entry); + this.seaweedInputStream = new SeaweedInputStream(filerClient, path, entry); this.statistics = statistics; } diff --git a/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedHadoopOutputStream.java b/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedHadoopOutputStream.java index f7a6225d8..da5b56bbc 100644 --- a/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedHadoopOutputStream.java +++ b/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedHadoopOutputStream.java @@ -2,15 +2,15 @@ package seaweed.hdfs; // adapted from org.apache.hadoop.fs.azurebfs.services.AbfsOutputStream -import seaweedfs.client.FilerGrpcClient; +import seaweedfs.client.FilerClient; import seaweedfs.client.FilerProto; import seaweedfs.client.SeaweedOutputStream; public class SeaweedHadoopOutputStream extends SeaweedOutputStream { - public SeaweedHadoopOutputStream(FilerGrpcClient filerGrpcClient, final String path, FilerProto.Entry.Builder entry, + public SeaweedHadoopOutputStream(FilerClient filerClient, final String path, FilerProto.Entry.Builder entry, final long position, final int bufferSize, final String replication) { - super(filerGrpcClient, path.toString(), entry, position, bufferSize, replication); + super(filerClient, path, entry, position, bufferSize, replication); } } diff --git a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java index 8147f3efe..f4e8c9349 100644 --- a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java +++ b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java @@ -24,27 +24,25 @@ public class SeaweedFileSystemStore { private static final Logger LOG = LoggerFactory.getLogger(SeaweedFileSystemStore.class); - private FilerGrpcClient filerGrpcClient; private FilerClient filerClient; private Configuration conf; public SeaweedFileSystemStore(String host, int port, Configuration conf) { int grpcPort = 10000 + port; - filerGrpcClient = new FilerGrpcClient(host, grpcPort); - filerClient = new FilerClient(filerGrpcClient); + filerClient = new FilerClient(host, grpcPort); this.conf = conf; String volumeServerAccessMode = this.conf.get(FS_SEAWEED_VOLUME_SERVER_ACCESS, "direct"); if (volumeServerAccessMode.equals("publicUrl")) { - filerGrpcClient.setAccessVolumeServerByPublicUrl(); + filerClient.setAccessVolumeServerByPublicUrl(); } else if (volumeServerAccessMode.equals("filerProxy")) { - filerGrpcClient.setAccessVolumeServerByFilerProxy(); + filerClient.setAccessVolumeServerByFilerProxy(); } } public void close() { try { - this.filerGrpcClient.shutdown(); + this.filerClient.shutdown(); } catch (InterruptedException e) { e.printStackTrace(); } @@ -219,10 +217,10 @@ public class SeaweedFileSystemStore { .clearGroupName() .addAllGroupName(Arrays.asList(userGroupInformation.getGroupNames())) ); - SeaweedWrite.writeMeta(filerGrpcClient, getParentDirectory(path), entry); + SeaweedWrite.writeMeta(filerClient, getParentDirectory(path), entry); } - return new SeaweedHadoopOutputStream(filerGrpcClient, path.toString(), entry, writePosition, bufferSize, replication); + return new SeaweedHadoopOutputStream(filerClient, path.toString(), entry, writePosition, bufferSize, replication); } @@ -236,7 +234,7 @@ public class SeaweedFileSystemStore { throw new FileNotFoundException("read non-exist file " + path); } - return new SeaweedHadoopInputStream(filerGrpcClient, + return new SeaweedHadoopInputStream(filerClient, statistics, path.toUri().getPath(), entry); diff --git a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedHadoopInputStream.java b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedHadoopInputStream.java index dd9bf4032..f26eae597 100644 --- a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedHadoopInputStream.java +++ b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedHadoopInputStream.java @@ -5,7 +5,7 @@ package seaweed.hdfs; import org.apache.hadoop.fs.ByteBufferReadable; import org.apache.hadoop.fs.FSInputStream; import org.apache.hadoop.fs.FileSystem.Statistics; -import seaweedfs.client.FilerGrpcClient; +import seaweedfs.client.FilerClient; import seaweedfs.client.FilerProto; import seaweedfs.client.SeaweedInputStream; @@ -19,11 +19,11 @@ public class SeaweedHadoopInputStream extends FSInputStream implements ByteBuffe private final Statistics statistics; public SeaweedHadoopInputStream( - final FilerGrpcClient filerGrpcClient, + final FilerClient filerClient, final Statistics statistics, final String path, final FilerProto.Entry entry) throws IOException { - this.seaweedInputStream = new SeaweedInputStream(filerGrpcClient, path, entry); + this.seaweedInputStream = new SeaweedInputStream(filerClient, path, entry); this.statistics = statistics; } diff --git a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedHadoopOutputStream.java b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedHadoopOutputStream.java index f65aef619..1740312fe 100644 --- a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedHadoopOutputStream.java +++ b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedHadoopOutputStream.java @@ -4,7 +4,7 @@ package seaweed.hdfs; import org.apache.hadoop.fs.StreamCapabilities; import org.apache.hadoop.fs.Syncable; -import seaweedfs.client.FilerGrpcClient; +import seaweedfs.client.FilerClient; import seaweedfs.client.FilerProto; import seaweedfs.client.SeaweedOutputStream; @@ -13,9 +13,9 @@ import java.util.Locale; public class SeaweedHadoopOutputStream extends SeaweedOutputStream implements Syncable, StreamCapabilities { - public SeaweedHadoopOutputStream(FilerGrpcClient filerGrpcClient, final String path, FilerProto.Entry.Builder entry, + public SeaweedHadoopOutputStream(FilerClient filerClient, final String path, FilerProto.Entry.Builder entry, final long position, final int bufferSize, final String replication) { - super(filerGrpcClient, path, entry, position, bufferSize, replication); + super(filerClient, path, entry, position, bufferSize, replication); } /** From 5b1def9080b44913280af52821798a9850b42eff Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 8 Feb 2021 02:42:01 -0800 Subject: [PATCH 254/260] Java: 1.6.1 refacoring API --- other/java/client/pom.xml | 2 +- other/java/client/pom.xml.deploy | 2 +- other/java/client/pom_debug.xml | 2 +- other/java/examples/pom.xml | 4 ++-- other/java/hdfs2/dependency-reduced-pom.xml | 2 +- other/java/hdfs2/pom.xml | 2 +- other/java/hdfs3/dependency-reduced-pom.xml | 2 +- other/java/hdfs3/pom.xml | 2 +- 8 files changed, 9 insertions(+), 9 deletions(-) diff --git a/other/java/client/pom.xml b/other/java/client/pom.xml index aaffe0aa8..056904ebe 100644 --- a/other/java/client/pom.xml +++ b/other/java/client/pom.xml @@ -5,7 +5,7 @@ com.github.chrislusf seaweedfs-client - 1.6.0 + 1.6.1 org.sonatype.oss diff --git a/other/java/client/pom.xml.deploy b/other/java/client/pom.xml.deploy index f32cdc427..69b900017 100644 --- a/other/java/client/pom.xml.deploy +++ b/other/java/client/pom.xml.deploy @@ -5,7 +5,7 @@ com.github.chrislusf seaweedfs-client - 1.6.0 + 1.6.1 org.sonatype.oss diff --git a/other/java/client/pom_debug.xml b/other/java/client/pom_debug.xml index 06b8e0ed5..1447401b7 100644 --- a/other/java/client/pom_debug.xml +++ b/other/java/client/pom_debug.xml @@ -5,7 +5,7 @@ com.github.chrislusf seaweedfs-client - 1.6.0 + 1.6.1 org.sonatype.oss diff --git a/other/java/examples/pom.xml b/other/java/examples/pom.xml index eb1e73a92..2456113d0 100644 --- a/other/java/examples/pom.xml +++ b/other/java/examples/pom.xml @@ -11,13 +11,13 @@ com.github.chrislusf seaweedfs-client - 1.6.0 + 1.6.1 compile com.github.chrislusf seaweedfs-hadoop2-client - 1.6.0 + 1.6.1 compile diff --git a/other/java/hdfs2/dependency-reduced-pom.xml b/other/java/hdfs2/dependency-reduced-pom.xml index 10b95db45..0680d86bb 100644 --- a/other/java/hdfs2/dependency-reduced-pom.xml +++ b/other/java/hdfs2/dependency-reduced-pom.xml @@ -301,7 +301,7 @@ - 1.6.0 + 1.6.1 2.9.2 diff --git a/other/java/hdfs2/pom.xml b/other/java/hdfs2/pom.xml index 48b7750f4..897477066 100644 --- a/other/java/hdfs2/pom.xml +++ b/other/java/hdfs2/pom.xml @@ -5,7 +5,7 @@ 4.0.0 - 1.6.0 + 1.6.1 2.9.2 diff --git a/other/java/hdfs3/dependency-reduced-pom.xml b/other/java/hdfs3/dependency-reduced-pom.xml index 71a2f739d..2b4a1a494 100644 --- a/other/java/hdfs3/dependency-reduced-pom.xml +++ b/other/java/hdfs3/dependency-reduced-pom.xml @@ -309,7 +309,7 @@ - 1.6.0 + 1.6.1 3.1.1 diff --git a/other/java/hdfs3/pom.xml b/other/java/hdfs3/pom.xml index ffe2d319a..49ff8f926 100644 --- a/other/java/hdfs3/pom.xml +++ b/other/java/hdfs3/pom.xml @@ -5,7 +5,7 @@ 4.0.0 - 1.6.0 + 1.6.1 3.1.1 From b6fba0c03feeb39c7db36c05da70280f1ca270fb Mon Sep 17 00:00:00 2001 From: LazyDBA247-Anyvision Date: Tue, 9 Feb 2021 09:43:55 +0200 Subject: [PATCH 255/260] filer: make k8s service ClusterIP type easier to patch/expose ( probably LoadBalancer will be better, but that also works) --- k8s/seaweedfs/templates/filer-service-client.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/k8s/seaweedfs/templates/filer-service-client.yaml b/k8s/seaweedfs/templates/filer-service-client.yaml index 65568418f..f509086e3 100644 --- a/k8s/seaweedfs/templates/filer-service-client.yaml +++ b/k8s/seaweedfs/templates/filer-service-client.yaml @@ -10,7 +10,6 @@ metadata: monitoring: "true" {{- end }} spec: - clusterIP: None ports: - name: "swfs-filer" port: {{ .Values.filer.port }} @@ -28,4 +27,4 @@ spec: {{- end }} selector: app: {{ template "seaweedfs.name" . }} - component: filer \ No newline at end of file + component: filer From 2646f5a7847ffc2117861eaaab749a63f5c471e5 Mon Sep 17 00:00:00 2001 From: LazyDBA247-Anyvision Date: Tue, 9 Feb 2021 09:46:21 +0200 Subject: [PATCH 256/260] cronjob: fix and update enable/disable fix.replication & support for CollectionPrefix filtering fix toleration and nodeSelector helm templating issues --- k8s/seaweedfs/templates/cronjob.yaml | 15 +++++++++------ k8s/seaweedfs/values.yaml | 16 ++++++++++------ 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/k8s/seaweedfs/templates/cronjob.yaml b/k8s/seaweedfs/templates/cronjob.yaml index 6f4ed8c70..4caf4bad1 100644 --- a/k8s/seaweedfs/templates/cronjob.yaml +++ b/k8s/seaweedfs/templates/cronjob.yaml @@ -15,13 +15,13 @@ spec: backoffLimit: 2 template: spec: - {{- with .Values.cronjob.nodeSelector }} + {{- if .Values.cronjob.nodeSelector }} nodeSelector: - {{- toYaml . | nindent 12 }} + {{ tpl .Values.cronjob.nodeSelector . | indent 12 | trim }} {{- end }} - {{- with .Values.cronjob.tolerations }} + {{- if .Values.cronjob.tolerations }} tolerations: - {{- toYaml . | nindent 12 }} + {{ tpl .Values.cronjob.tolerations . | nindent 12 | trim }} {{- end }} restartPolicy: OnFailure containers: @@ -36,10 +36,13 @@ spec: - | set -ex echo -e "lock\n\ - volume.balance -force\ + volume.balance -force \ {{ if .Values.volume.dataCenter }} -dataCenter {{ .Values.volume.dataCenter }}{{ end }}\ {{ if .Values.cronjob.collection }} -collection {{ .Values.cronjob.collection }}{{ end }}\n\ - volume.fix.replication\nunlock\n" | \ + {{- if .Values.cronjob.enableFixReplication }} + volume.fix.replication -collectionPattern={{ .Values.cronjob.collectionPattern }} \n\ + {{- end }} + unlock\n" | \ /usr/bin/weed shell \ {{- if .Values.cronjob.master }} -master {{ .Values.cronjob.master }} \ diff --git a/k8s/seaweedfs/values.yaml b/k8s/seaweedfs/values.yaml index 2b33a6149..301c9f236 100644 --- a/k8s/seaweedfs/values.yaml +++ b/k8s/seaweedfs/values.yaml @@ -359,17 +359,21 @@ s3: storageClass: "" cronjob: - enabled: false + enabled: true + master: "seaweedfs-master:9333" + filer: "seaweedfs-filer-client:8888" + tolerations: "" + nodeSelector: | + sw-backend: "true" + replication: + enable: true + collectionPattern: "" schedule: "*/7 * * * *" resources: null # balance all volumes among volume servers # ALL|EACH_COLLECTION| collection: "" - master: "" - filer: "" - tolerations: "" - nodeSelector: | - sw-backend: "true" + certificates: commonName: "SeaweedFS CA" From 91a3314d36445c84d337963147d0ed30bde21599 Mon Sep 17 00:00:00 2001 From: LazyDBA247-Anyvision Date: Tue, 9 Feb 2021 09:55:58 +0200 Subject: [PATCH 257/260] filer/s3: enable/disable s3 and peers enable/disable s3 pods launch s3 in filer pod, to reduce network/latency between pods when using s3 disable/enable the peers option (with external backend DB is not needed) added option to create s3 admin+read users --- .../templates/filer-statefulset.yaml | 29 +++++++++++++++++++ k8s/seaweedfs/templates/s3-deployment.yaml | 10 +++++++ k8s/seaweedfs/templates/s3-service.yaml | 10 +++---- k8s/seaweedfs/values.yaml | 18 +++++++++++- 4 files changed, 61 insertions(+), 6 deletions(-) diff --git a/k8s/seaweedfs/templates/filer-statefulset.yaml b/k8s/seaweedfs/templates/filer-statefulset.yaml index d284e9992..fc1253479 100644 --- a/k8s/seaweedfs/templates/filer-statefulset.yaml +++ b/k8s/seaweedfs/templates/filer-statefulset.yaml @@ -133,14 +133,36 @@ spec: -encryptVolumeData \ {{- end }} -ip=${POD_IP} \ + {{- if .Values.filer.enable_peers }} {{- if gt (.Values.filer.replicas | int) 1 }} -peers=$(echo -n "{{ range $index := until (.Values.filer.replicas | int) }}${SEAWEEDFS_FULLNAME}-filer-{{ $index }}.${SEAWEEDFS_FULLNAME}-filer:{{ $.Values.filer.port }}{{ if lt $index (sub ($.Values.filer.replicas | int) 1) }},{{ end }}{{ end }}" | sed "s/$HOSTNAME.${SEAWEEDFS_FULLNAME}-filer:{{ $.Values.filer.port }}//" | sed 's/,$//; 's/^,//'; s/,,/,/;' ) \ {{- end }} + {{- end }} + {{- if .Values.filer.s3.enabled }} + -s3 \ + -s3.port={{ .Values.filer.s3.port }} \ + {{- if .Values.filer.s3.domainName }} + -s3.domainName={{ .Values.filer.s3.domainName }} \ + {{- end }} + {{- if .Values.global.enableSecurity }} + -s3.cert.file=/usr/local/share/ca-certificates/client/tls.crt \ + -s3.key.file=/usr/local/share/ca-certificates/client/tls.key \ + {{- end }} + {{- if .Values.filer.s3.allowEmptyFolder }} + -s3.allowEmptyFolder={{ .Values.filer.s3.allowEmptyFolder }} \ + {{- end }} + {{- if .Values.filer.s3.enableAuth }} + -s3.config=/etc/sw/seaweedfs_s3_config \ + {{- end }} + {{- end }} -master={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }} {{- if or (.Values.global.enableSecurity) (.Values.filer.extraVolumeMounts) }} volumeMounts: - name: seaweedfs-filer-log-volume mountPath: "/logs/" + - mountPath: /etc/sw + name: config-users + readOnly: true {{- if .Values.global.enableSecurity }} - name: security-config readOnly: true @@ -198,6 +220,13 @@ spec: hostPath: path: /storage/logs/seaweedfs/filer type: DirectoryOrCreate + - name: db-schema-config-volume + configMap: + name: seaweedfs-db-init-config + - name: config-users + secret: + defaultMode: 420 + secretName: seaweedfs-s3-secret {{- if .Values.global.enableSecurity }} - name: security-config configMap: diff --git a/k8s/seaweedfs/templates/s3-deployment.yaml b/k8s/seaweedfs/templates/s3-deployment.yaml index ed01758b8..b513e937b 100644 --- a/k8s/seaweedfs/templates/s3-deployment.yaml +++ b/k8s/seaweedfs/templates/s3-deployment.yaml @@ -90,10 +90,16 @@ spec: {{- if .Values.s3.allowEmptyFolder }} -allowEmptyFolder={{ .Values.s3.allowEmptyFolder }} \ {{- end }} + {{- if .Values.s3.enableAuth }} + -config=/etc/sw/seaweedfs_s3_config \ + {{- end }} -filer={{ template "seaweedfs.name" . }}-filer-client:{{ .Values.filer.port }} volumeMounts: - name: logs mountPath: "/logs/" + - mountPath: /etc/sw + name: config-users + readOnly: true {{- if .Values.global.enableSecurity }} - name: security-config readOnly: true @@ -144,6 +150,10 @@ spec: {{ tpl .Values.s3.resources . | nindent 12 | trim }} {{- end }} volumes: + - name: config-users + secret: + defaultMode: 420 + secretName: seaweedfs-s3-secret {{- if eq .Values.s3.logs.type "hostPath" }} - name: logs hostPath: diff --git a/k8s/seaweedfs/templates/s3-service.yaml b/k8s/seaweedfs/templates/s3-service.yaml index 4a68c7976..122b33298 100644 --- a/k8s/seaweedfs/templates/s3-service.yaml +++ b/k8s/seaweedfs/templates/s3-service.yaml @@ -9,15 +9,15 @@ metadata: spec: ports: - name: "swfs-s3" - port: {{ .Values.s3.port }} - targetPort: {{ .Values.s3.port }} + port: {{ if .Values.s3.enabled }}{{ .Values.s3.port }}{{ else }}{{ .Values.filer.s3.port }}{{ end }} + targetPort: {{ if .Values.s3.enabled }}{{ .Values.s3.port }}{{ else }}{{ .Values.filer.s3.port }}{{ end }} protocol: TCP -{{- if .Values.s3.metricsPort }} - - name: "swfs-s3-metrics" +{{- if and .Values.s3.enabled .Values.s3.metricsPort }} + - name: "metrics" port: {{ .Values.s3.metricsPort }} targetPort: {{ .Values.s3.metricsPort }} protocol: TCP {{- end }} selector: app: {{ template "seaweedfs.name" . }} - component: s3 \ No newline at end of file + component: {{ if .Values.s3.enabled }}s3{{ else }}filer{{ end }} diff --git a/k8s/seaweedfs/values.yaml b/k8s/seaweedfs/values.yaml index 301c9f236..43783d3b7 100644 --- a/k8s/seaweedfs/values.yaml +++ b/k8s/seaweedfs/values.yaml @@ -229,6 +229,8 @@ filer: maxMB: null # encrypt data on volume servers encryptVolumeData: false + # enable peers sync metadata, for leveldb (localdb for filer but with sync across) + enable_peers: false # Whether proxy or redirect to volume server during file GET request redirectOnRead: false @@ -311,8 +313,19 @@ filer: # directories under this folder will be automatically creating a separate bucket WEED_FILER_BUCKETS_FOLDER: "/buckets" + s3: + enabled: true + port: 8333 + #allow empty folders + allowEmptyFolder: false + # Suffix of the host name, {bucket}.{domainName} + domainName: "" + # enable user & permission to s3 (need to inject to all services) + enableAuth: false + skipAuthSecretCreation: false + s3: - enabled: true + enabled: false repository: null imageName: null imageTag: null @@ -323,6 +336,9 @@ s3: loggingOverrideLevel: null #allow empty folders allowEmptyFolder: true + # enable user & permission to s3 (need to inject to all services) + enableAuth: false + skipAuthSecretCreation: false # Suffix of the host name, {bucket}.{domainName} domainName: "" From 259c66d978f7d296325fcc20ab7eceb127c377c8 Mon Sep 17 00:00:00 2001 From: LazyDBA247-Anyvision Date: Tue, 9 Feb 2021 09:57:09 +0200 Subject: [PATCH 258/260] volume: set minFreeSpacePercent to 7% when k8s node have less than 5% free space, the k8s will start killing pods... --- k8s/seaweedfs/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/seaweedfs/values.yaml b/k8s/seaweedfs/values.yaml index 43783d3b7..273ab21ad 100644 --- a/k8s/seaweedfs/values.yaml +++ b/k8s/seaweedfs/values.yaml @@ -136,7 +136,7 @@ volume: # limit file size to avoid out of memory, default 256mb fileSizeLimitMB: null # minimum free disk space(in percents). If free disk space lower this value - all volumes marks as ReadOnly - minFreeSpacePercent: 1 + minFreeSpacePercent: 7 # limit background compaction or copying speed in mega bytes per second From b373c1de93ab172ca8f0c3b15329ccea74f5b3c9 Mon Sep 17 00:00:00 2001 From: LazyDBA247-Anyvision Date: Tue, 9 Feb 2021 10:07:25 +0200 Subject: [PATCH 259/260] remove collectionPattern if is empty string / null --- k8s/seaweedfs/templates/cronjob.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/seaweedfs/templates/cronjob.yaml b/k8s/seaweedfs/templates/cronjob.yaml index 4caf4bad1..c7dcd52b1 100644 --- a/k8s/seaweedfs/templates/cronjob.yaml +++ b/k8s/seaweedfs/templates/cronjob.yaml @@ -40,7 +40,7 @@ spec: {{ if .Values.volume.dataCenter }} -dataCenter {{ .Values.volume.dataCenter }}{{ end }}\ {{ if .Values.cronjob.collection }} -collection {{ .Values.cronjob.collection }}{{ end }}\n\ {{- if .Values.cronjob.enableFixReplication }} - volume.fix.replication -collectionPattern={{ .Values.cronjob.collectionPattern }} \n\ + volume.fix.replication {{ if .Values.cronjob.collectionPattern }} -collectionPattern={{ .Values.cronjob.collectionPattern }} {{ end }} \n\ {{- end }} unlock\n" | \ /usr/bin/weed shell \ From a6e8d606b47e5f3e8cd8a57d2769d6f1404fbc8f Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 9 Feb 2021 00:30:16 -0800 Subject: [PATCH 260/260] Update README.md --- k8s/README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/k8s/README.md b/k8s/README.md index 5ec3ab407..6a1eb7a05 100644 --- a/k8s/README.md +++ b/k8s/README.md @@ -12,9 +12,12 @@ with ENV. ### current instances config (AIO): 1 instance for each type (master/filer/volume/s3) -instances need node labels: +To avoid multiple volume servers on the same node, apply these node labels: * sw-volume: true (for volume instance, specific tag) * sw-backend: true (for all others, as they less resource demanding) +``` +kubectl label node YOUR_NODE_NAME sw-volume=true,sw-backend=true +``` you can update the replicas count for each node type in values.yaml, need to add more nodes with the corresponding label.