From a34438384e478c23aa817357c2e392626fbcba34 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Mon, 23 May 2022 18:53:35 +0500 Subject: [PATCH 1/6] docker compose file for nextcloud testing https://github.com/chrislusf/seaweedfs/issues/3086 --- docker/Makefile | 3 ++ docker/compose/local-nextcloud-compose.yml | 55 ++++++++++++++++++++++ docker/seaweedfs.sql | 2 +- 3 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 docker/compose/local-nextcloud-compose.yml diff --git a/docker/Makefile b/docker/Makefile index dbc82fde4..3afea17c1 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -52,6 +52,9 @@ dev_replicate: build dev_auditlog: build docker-compose -f compose/local-auditlog-compose.yml -p seaweedfs up +dev_nextcloud: build + docker-compose -f compose/local-nextcloud-compose.yml -p seaweedfs up + cluster: build docker-compose -f compose/local-cluster-compose.yml -p seaweedfs up diff --git a/docker/compose/local-nextcloud-compose.yml b/docker/compose/local-nextcloud-compose.yml new file mode 100644 index 000000000..e6f706c26 --- /dev/null +++ b/docker/compose/local-nextcloud-compose.yml @@ -0,0 +1,55 @@ +version: '2' + +services: + master: + image: chrislusf/seaweedfs:local + ports: + - 9333:9333 + - 19333:19333 + command: "master -ip=master" + volume: + image: chrislusf/seaweedfs:local + ports: + - 8080:8080 + - 18080:18080 + command: "volume -mserver=master:9333 -port=8080 -ip=volume" + depends_on: + - master + mysql: + image: percona/percona-server:5.7 + ports: + - 3306:3306 + environment: + - MYSQL_ROOT_PASSWORD=secret + - MYSQL_DATABASE=nextcloud + - MYSQL_PASSWORD=secret + - MYSQL_USER=nextcloud + s3: + image: chrislusf/seaweedfs:local + ports: + - 8888:8888 + - 18888:18888 + - 8333:8333 + command: '-v 9 filer -master="master:9333" -s3' + depends_on: + - master + - volume + nextcloud: + image: nextcloud:23.0.5-apache + environment: + - OBJECTSTORE_S3_HOST=s3 + - OBJECTSTORE_S3_BUCKET=nextcloud + - OBJECTSTORE_S3_KEY=some_access_key1 + - OBJECTSTORE_S3_SECRET=some_secret_key1 + - OBJECTSTORE_S3_PORT=8333 + - OBJECTSTORE_S3_SSL=false + - OBJECTSTORE_S3_USEPATH_STYLE=true + - MYSQL_PASSWORD=secret + - MYSQL_DATABASE=nextcloud + - MYSQL_USER=nextcloud + - MYSQL_HOST=mysql + ports: + - 80:80 + depends_on: + - s3 + - mysql diff --git a/docker/seaweedfs.sql b/docker/seaweedfs.sql index 38ebc575c..a27eb7081 100644 --- a/docker/seaweedfs.sql +++ b/docker/seaweedfs.sql @@ -1,6 +1,6 @@ CREATE DATABASE IF NOT EXISTS seaweedfs; CREATE USER IF NOT EXISTS 'seaweedfs'@'%' IDENTIFIED BY 'secret'; -GRANT ALL PRIVILEGES ON seaweedfs_fast.* TO 'seaweedfs'@'%'; +GRANT ALL PRIVILEGES ON seaweedfs.* TO 'seaweedfs'@'%'; FLUSH PRIVILEGES; USE seaweedfs; CREATE TABLE IF NOT EXISTS filemeta ( From a14728d508666a50addc51d645c792044927c675 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Mon, 23 May 2022 20:11:32 +0500 Subject: [PATCH 2/6] docker compose nextcloud rm mysql --- docker/compose/local-nextcloud-compose.yml | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/docker/compose/local-nextcloud-compose.yml b/docker/compose/local-nextcloud-compose.yml index e6f706c26..80c3fca53 100644 --- a/docker/compose/local-nextcloud-compose.yml +++ b/docker/compose/local-nextcloud-compose.yml @@ -15,15 +15,6 @@ services: command: "volume -mserver=master:9333 -port=8080 -ip=volume" depends_on: - master - mysql: - image: percona/percona-server:5.7 - ports: - - 3306:3306 - environment: - - MYSQL_ROOT_PASSWORD=secret - - MYSQL_DATABASE=nextcloud - - MYSQL_PASSWORD=secret - - MYSQL_USER=nextcloud s3: image: chrislusf/seaweedfs:local ports: @@ -44,12 +35,10 @@ services: - OBJECTSTORE_S3_PORT=8333 - OBJECTSTORE_S3_SSL=false - OBJECTSTORE_S3_USEPATH_STYLE=true - - MYSQL_PASSWORD=secret - - MYSQL_DATABASE=nextcloud - - MYSQL_USER=nextcloud - - MYSQL_HOST=mysql + - SQLITE_DATABASE=nextcloud + - NEXTCLOUD_ADMIN_USER=admin + - NEXTCLOUD_ADMIN_PASSWORD=admin ports: - 80:80 depends_on: - - s3 - - mysql + - s3 \ No newline at end of file From 54356211c2f93924e484e5049b9ed7755dab660c Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Fri, 27 May 2022 13:17:18 +0500 Subject: [PATCH 3/6] list empty folder --- weed/s3api/s3api_objects_list_handlers.go | 25 +++++++++++++---------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/weed/s3api/s3api_objects_list_handlers.go b/weed/s3api/s3api_objects_list_handlers.go index a3b858dcb..8c48645a4 100644 --- a/weed/s3api/s3api_objects_list_handlers.go +++ b/weed/s3api/s3api_objects_list_handlers.go @@ -134,6 +134,9 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m } bucketPrefix := fmt.Sprintf("%s/%s/", s3a.option.BucketsPath, bucket) reqDir = fmt.Sprintf("%s%s", bucketPrefix, reqDir) + if prefix == "" && strings.HasSuffix(reqDir, "/") { + reqDir, prefix = filepath.Split(reqDir[:len(reqDir)-1]) + } if strings.HasSuffix(reqDir, "/") { // remove trailing "/" reqDir = reqDir[:len(reqDir)-1] @@ -149,19 +152,21 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m err = s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { _, isTruncated, nextMarker, doErr = s3a.doListFilerEntries(client, reqDir, prefix, maxKeys, marker, delimiter, func(dir string, entry *filer_pb.Entry) { - if entry.IsDirectory { - if delimiter == "/" { - commonPrefixes = append(commonPrefixes, PrefixEntry{ - Prefix: fmt.Sprintf("%s/%s/", dir, entry.Name)[len(bucketPrefix):], - }) - } + if entry.IsDirectory && delimiter == "/" { + commonPrefixes = append(commonPrefixes, PrefixEntry{ + Prefix: fmt.Sprintf("%s/%s/", dir, entry.Name)[len(bucketPrefix):], + }) } else { storageClass := "STANDARD" if v, ok := entry.Extended[xhttp.AmzStorageClass]; ok { storageClass = string(v) } + entryName := entry.Name + if entry.IsDirectory { + entryName += "/" + } contents = append(contents, ListEntry{ - Key: fmt.Sprintf("%s/%s", dir, entry.Name)[len(bucketPrefix):], + Key: fmt.Sprintf("%s/%s", dir, entryName)[len(bucketPrefix):], LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(), ETag: "\"" + filer.ETag(entry) + "\"", Size: int64(filer.FileSize(entry)), @@ -289,10 +294,8 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d glog.Errorf("check empty folder %s: %v", dir, err) } } - if !isEmpty { - eachEntryFn(dir, entry) - counter++ - } + eachEntryFn(dir, entry) + counter++ } } } else { From 26f36469615e55580f48473daeba28cb7ab13f59 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Mon, 30 May 2022 15:03:27 +0500 Subject: [PATCH 4/6] show fake dirs https://github.com/chrislusf/seaweedfs/issues/3086 --- weed/s3api/s3api_object_handlers.go | 5 ++++- weed/s3api/s3api_objects_list_handlers.go | 19 ++++++++++--------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/weed/s3api/s3api_object_handlers.go b/weed/s3api/s3api_object_handlers.go index 3d26d395e..ef56ba85d 100644 --- a/weed/s3api/s3api_object_handlers.go +++ b/weed/s3api/s3api_object_handlers.go @@ -93,7 +93,10 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request) defer dataReader.Close() if strings.HasSuffix(object, "/") { - if err := s3a.mkdir(s3a.option.BucketsPath, bucket+object, nil); err != nil { + if err := s3a.mkdir(s3a.option.BucketsPath, bucket+strings.TrimSuffix(object, "/"), func(entry *filer_pb.Entry) { + glog.V(0).Infof("make file %s", bucket+object) + entry.Attributes.Mime = r.Header.Get("Content-Type") + }); err != nil { s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) return } diff --git a/weed/s3api/s3api_objects_list_handlers.go b/weed/s3api/s3api_objects_list_handlers.go index 8c48645a4..d491964d1 100644 --- a/weed/s3api/s3api_objects_list_handlers.go +++ b/weed/s3api/s3api_objects_list_handlers.go @@ -134,9 +134,6 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m } bucketPrefix := fmt.Sprintf("%s/%s/", s3a.option.BucketsPath, bucket) reqDir = fmt.Sprintf("%s%s", bucketPrefix, reqDir) - if prefix == "" && strings.HasSuffix(reqDir, "/") { - reqDir, prefix = filepath.Split(reqDir[:len(reqDir)-1]) - } if strings.HasSuffix(reqDir, "/") { // remove trailing "/" reqDir = reqDir[:len(reqDir)-1] @@ -152,10 +149,12 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m err = s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { _, isTruncated, nextMarker, doErr = s3a.doListFilerEntries(client, reqDir, prefix, maxKeys, marker, delimiter, func(dir string, entry *filer_pb.Entry) { - if entry.IsDirectory && delimiter == "/" { - commonPrefixes = append(commonPrefixes, PrefixEntry{ - Prefix: fmt.Sprintf("%s/%s/", dir, entry.Name)[len(bucketPrefix):], - }) + if entry.IsDirectory && entry.Attributes.Mime == "" { + if delimiter == "/" { + commonPrefixes = append(commonPrefixes, PrefixEntry{ + Prefix: fmt.Sprintf("%s/%s/", dir, entry.Name)[len(bucketPrefix):], + }) + } } else { storageClass := "STANDARD" if v, ok := entry.Extended[xhttp.AmzStorageClass]; ok { @@ -294,8 +293,10 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d glog.Errorf("check empty folder %s: %v", dir, err) } } - eachEntryFn(dir, entry) - counter++ + if !isEmpty || entry.Attributes.Mime != "" { + eachEntryFn(dir, entry) + counter++ + } } } } else { From 62e5e3822ceb400e8c72ea9087f3b6ca40f76246 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Mon, 30 May 2022 18:09:08 +0500 Subject: [PATCH 5/6] list self dir https://github.com/chrislusf/seaweedfs/issues/3086 --- weed/s3api/s3api_objects_list_handlers.go | 130 +++++++++++++--------- 1 file changed, 76 insertions(+), 54 deletions(-) diff --git a/weed/s3api/s3api_objects_list_handlers.go b/weed/s3api/s3api_objects_list_handlers.go index d491964d1..d898061fa 100644 --- a/weed/s3api/s3api_objects_list_handlers.go +++ b/weed/s3api/s3api_objects_list_handlers.go @@ -135,8 +135,7 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m bucketPrefix := fmt.Sprintf("%s/%s/", s3a.option.BucketsPath, bucket) reqDir = fmt.Sprintf("%s%s", bucketPrefix, reqDir) if strings.HasSuffix(reqDir, "/") { - // remove trailing "/" - reqDir = reqDir[:len(reqDir)-1] + reqDir = strings.TrimSuffix(reqDir, "/") } var contents []ListEntry @@ -148,34 +147,30 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m // check filer err = s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { - _, isTruncated, nextMarker, doErr = s3a.doListFilerEntries(client, reqDir, prefix, maxKeys, marker, delimiter, func(dir string, entry *filer_pb.Entry) { - if entry.IsDirectory && entry.Attributes.Mime == "" { + _, isTruncated, nextMarker, doErr = s3a.doListFilerEntries(client, reqDir, prefix, maxKeys, marker, delimiter, false, func(dir string, entry *filer_pb.Entry) { + if entry.IsDirectory { if delimiter == "/" { commonPrefixes = append(commonPrefixes, PrefixEntry{ Prefix: fmt.Sprintf("%s/%s/", dir, entry.Name)[len(bucketPrefix):], }) } - } else { - storageClass := "STANDARD" - if v, ok := entry.Extended[xhttp.AmzStorageClass]; ok { - storageClass = string(v) - } - entryName := entry.Name - if entry.IsDirectory { - entryName += "/" - } - contents = append(contents, ListEntry{ - Key: fmt.Sprintf("%s/%s", dir, entryName)[len(bucketPrefix):], - LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(), - ETag: "\"" + filer.ETag(entry) + "\"", - Size: int64(filer.FileSize(entry)), - Owner: CanonicalUser{ - ID: fmt.Sprintf("%x", entry.Attributes.Uid), - DisplayName: entry.Attributes.UserName, - }, - StorageClass: StorageClass(storageClass), - }) + return + } + storageClass := "STANDARD" + if v, ok := entry.Extended[xhttp.AmzStorageClass]; ok { + storageClass = string(v) } + contents = append(contents, ListEntry{ + Key: fmt.Sprintf("%s/%s", dir, entry.Name)[len(bucketPrefix):], + LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(), + ETag: "\"" + filer.ETag(entry) + "\"", + Size: int64(filer.FileSize(entry)), + Owner: CanonicalUser{ + ID: fmt.Sprintf("%x", entry.Attributes.Uid), + DisplayName: entry.Attributes.UserName, + }, + StorageClass: StorageClass(storageClass), + }) }) if doErr != nil { return doErr @@ -185,6 +180,32 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m nextMarker = "" } + if len(contents) == 0 && maxKeys > 0 { + if strings.HasSuffix(originalPrefix, "/") && prefix == "" { + reqDir, prefix = filepath.Split(strings.TrimSuffix(reqDir, "/")) + reqDir = strings.TrimSuffix(reqDir, "/") + } + _, _, _, doErr = s3a.doListFilerEntries(client, reqDir, prefix, 1, prefix, delimiter, true, func(dir string, entry *filer_pb.Entry) { + if entry.IsDirectory && entry.Attributes.Mime != "" && entry.Name == prefix { + storageClass := "STANDARD" + if v, ok := entry.Extended[xhttp.AmzStorageClass]; ok { + storageClass = string(v) + } + contents = append(contents, ListEntry{ + Key: fmt.Sprintf("%s/%s", dir, entry.Name+"/")[len(bucketPrefix):], + LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(), + ETag: "\"" + filer.ETag(entry) + "\"", + Size: int64(filer.FileSize(entry)), + Owner: CanonicalUser{ + ID: fmt.Sprintf("%x", entry.Attributes.Uid), + DisplayName: entry.Attributes.UserName, + }, + StorageClass: StorageClass(storageClass), + }) + } + }) + } + response = ListBucketResult{ Name: bucket, Prefix: originalPrefix, @@ -203,7 +224,7 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m return } -func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, dir, prefix string, maxKeys int, marker, delimiter string, eachEntryFn func(dir string, entry *filer_pb.Entry)) (counter int, isTruncated bool, nextMarker string, err error) { +func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, dir, prefix string, maxKeys int, marker, delimiter string, inclusiveStartFrom bool, eachEntryFn func(dir string, entry *filer_pb.Entry)) (counter int, isTruncated bool, nextMarker string, err error) { // invariants // prefix and marker should be under dir, marker may contain "/" // maxKeys should be updated for each recursion @@ -218,8 +239,8 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d if strings.Contains(marker, "/") { sepIndex := strings.Index(marker, "/") subDir, subMarker := marker[0:sepIndex], marker[sepIndex+1:] - // println("doListFilerEntries dir", dir+"/"+subDir, "subMarker", subMarker, "maxKeys", maxKeys) - subCounter, subIsTruncated, subNextMarker, subErr := s3a.doListFilerEntries(client, dir+"/"+subDir, "", maxKeys, subMarker, delimiter, eachEntryFn) + glog.V(4).Infoln("doListFilerEntries dir", dir+"/"+subDir, "subMarker", subMarker, "maxKeys", maxKeys) + subCounter, subIsTruncated, subNextMarker, subErr := s3a.doListFilerEntries(client, dir+"/"+subDir, "", maxKeys, subMarker, delimiter, false, eachEntryFn) if subErr != nil { err = subErr return @@ -241,7 +262,7 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d Prefix: prefix, Limit: uint32(maxKeys + 1), StartFromFileName: marker, - InclusiveStartFrom: false, + InclusiveStartFrom: inclusiveStartFrom, } ctx, cancel := context.WithCancel(context.Background()) @@ -270,34 +291,35 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d nextMarker = entry.Name if entry.IsDirectory { // println("ListEntries", dir, "dir:", entry.Name) - if entry.Name != ".uploads" { // FIXME no need to apply to all directories. this extra also affects maxKeys - if delimiter != "/" { - eachEntryFn(dir, entry) - // println("doListFilerEntries2 dir", dir+"/"+entry.Name, "maxKeys", maxKeys-counter) - subCounter, subIsTruncated, subNextMarker, subErr := s3a.doListFilerEntries(client, dir+"/"+entry.Name, "", maxKeys-counter, "", delimiter, eachEntryFn) - if subErr != nil { - err = fmt.Errorf("doListFilerEntries2: %v", subErr) - return - } - // println("doListFilerEntries2 dir", dir+"/"+entry.Name, "maxKeys", maxKeys-counter, "subCounter", subCounter, "subNextMarker", subNextMarker, "subIsTruncated", subIsTruncated) - counter += subCounter - nextMarker = entry.Name + "/" + subNextMarker - if subIsTruncated { - isTruncated = true - return - } - } else { - var isEmpty bool - if !s3a.option.AllowEmptyFolder { - if isEmpty, err = s3a.isDirectoryAllEmpty(client, dir, entry.Name); err != nil { - glog.Errorf("check empty folder %s: %v", dir, err) - } - } - if !isEmpty || entry.Attributes.Mime != "" { - eachEntryFn(dir, entry) - counter++ + if entry.Name == ".uploads" { // FIXME no need to apply to all directories. this extra also affects maxKeys + continue + } + if delimiter != "/" { + eachEntryFn(dir, entry) + // println("doListFilerEntries2 dir", dir+"/"+entry.Name, "maxKeys", maxKeys-counter) + subCounter, subIsTruncated, subNextMarker, subErr := s3a.doListFilerEntries(client, dir+"/"+entry.Name, "", maxKeys-counter, "", delimiter, false, eachEntryFn) + if subErr != nil { + err = fmt.Errorf("doListFilerEntries2: %v", subErr) + return + } + // println("doListFilerEntries2 dir", dir+"/"+entry.Name, "maxKeys", maxKeys-counter, "subCounter", subCounter, "subNextMarker", subNextMarker, "subIsTruncated", subIsTruncated) + counter += subCounter + nextMarker = entry.Name + "/" + subNextMarker + if subIsTruncated { + isTruncated = true + return + } + } else { + var isEmpty bool + if !s3a.option.AllowEmptyFolder { + if isEmpty, err = s3a.isDirectoryAllEmpty(client, dir, entry.Name); err != nil { + glog.Errorf("check empty folder %s: %v", dir, err) } } + if !isEmpty { + eachEntryFn(dir, entry) + counter++ + } } } else { // println("ListEntries", dir, "file:", entry.Name) From af562e1a208b973e187a60a20629d7bfde8809fb Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Mon, 30 May 2022 23:26:26 +0500 Subject: [PATCH 6/6] rm log info --- weed/s3api/s3api_object_handlers.go | 1 - 1 file changed, 1 deletion(-) diff --git a/weed/s3api/s3api_object_handlers.go b/weed/s3api/s3api_object_handlers.go index ef56ba85d..91086fec8 100644 --- a/weed/s3api/s3api_object_handlers.go +++ b/weed/s3api/s3api_object_handlers.go @@ -94,7 +94,6 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request) if strings.HasSuffix(object, "/") { if err := s3a.mkdir(s3a.option.BucketsPath, bucket+strings.TrimSuffix(object, "/"), func(entry *filer_pb.Entry) { - glog.V(0).Infof("make file %s", bucket+object) entry.Attributes.Mime = r.Header.Get("Content-Type") }); err != nil { s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)