From dac862a3040d0b927eb86357a066b84f34d0f93a Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Wed, 22 May 2024 19:17:46 +0500 Subject: [PATCH] pass test_bucket_listv2_encoding_basic and test_bucket_listv2_fetchowner_defaultempty --- .github/workflows/s3tests.yml | 4 + Makefile | 2 +- weed/filer/abstract_sql/abstract_sql_store.go | 14 +- weed/filer/filer.go | 12 +- weed/filer/filer_search.go | 26 ++-- weed/filer/filerstore_translate_path.go | 2 + weed/filer/filerstore_wrapper.go | 19 +++ weed/filer/mysql/mysql_sql_gen.go | 2 +- weed/filer/sqlite/sqlite_store.go | 2 +- weed/s3api/s3api_object_handlers.go | 37 ++++- weed/s3api/s3api_object_handlers_list.go | 132 ++++++++---------- weed/s3api/s3api_xsd_generated.go | 23 +++ weed/server/filer_grpc_server.go | 3 +- weed/server/filer_server.go | 11 +- 14 files changed, 189 insertions(+), 100 deletions(-) diff --git a/.github/workflows/s3tests.yml b/.github/workflows/s3tests.yml index 866f2d888..b7670d358 100644 --- a/.github/workflows/s3tests.yml +++ b/.github/workflows/s3tests.yml @@ -54,6 +54,8 @@ jobs: s3tests_boto3/functional/test_s3.py::test_bucket_list_distinct \ s3tests_boto3/functional/test_s3.py::test_bucket_list_many \ s3tests_boto3/functional/test_s3.py::test_bucket_listv2_many \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_basic \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_encoding_basic \ s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_prefix \ s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_prefix_ends_with_delimiter \ s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_alt \ @@ -76,6 +78,7 @@ jobs: s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_delimiter_prefix_delimiter_not_exist \ s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist \ s3tests_boto3/functional/test_s3.py::test_bucket_listv2_fetchowner_notempty \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_fetchowner_defaultempty \ s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_basic \ s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_basic \ s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_alt \ @@ -94,6 +97,7 @@ jobs: s3tests_boto3/functional/test_s3.py::test_bucket_listv2_maxkeys_zero \ s3tests_boto3/functional/test_s3.py::test_bucket_list_marker_none \ s3tests_boto3/functional/test_s3.py::test_bucket_list_marker_empty \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_continuationtoken_empty \ s3tests_boto3/functional/test_s3.py::test_bucket_listv2_continuationtoken \ s3tests_boto3/functional/test_s3.py::test_bucket_listv2_both_continuationtoken_startafter \ s3tests_boto3/functional/test_s3.py::test_bucket_list_marker_unreadable \ diff --git a/Makefile b/Makefile index 4827acd3c..37abadfae 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ full_install: cd weed; go install -tags "elastic gocdk sqlite ydb tikv rclone" server: install - export WEED_LEVELDB2_ENABLED="false";export WEED_SQLITE_ENABLED="true"; \ + export WEED_LEVELDB2_ENABLED="false";export WEED_SQLITE_ENABLED="true"; export WEED_SQLITE_DBFILE="/tmp/filer.db"; \ weed -v 0 server -s3.allowListRecursive=true -dir /tmp -master.volumeSizeLimitMB=1024 -s3 -filer -filer.maxMB=64 -filer.port.public=7777 -volume.max=100 -volume.preStopSeconds=1 -s3.port=8000 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config=./docker/compose/s3.json -metricsPort=9324 benchmark: install warp_install diff --git a/weed/filer/abstract_sql/abstract_sql_store.go b/weed/filer/abstract_sql/abstract_sql_store.go index 916ffb8bb..fe2d7fe66 100644 --- a/weed/filer/abstract_sql/abstract_sql_store.go +++ b/weed/filer/abstract_sql/abstract_sql_store.go @@ -289,6 +289,7 @@ func (store *AbstractSqlStore) DeleteFolderChildren(ctx context.Context, fullpat } func (store *AbstractSqlStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { + glog.V(5).Infof("ListDirectoryPrefixedEntries dirPath %v, includeStartFile %v", dirPath, includeStartFile) db, bucket, shortPath, err := store.getTxOrDB(ctx, dirPath, true) if err != nil { @@ -337,24 +338,27 @@ func (store *AbstractSqlStore) ListRecursivePrefixedEntries(ctx context.Context, if err != nil { return lastFileName, fmt.Errorf("findDB %s : %v", dirPath, err) } - rows, err := db.QueryContext(ctx, store.GetSqlListRecursive(bucket), util.HashStringToLong(string(shortPath)), startFileName, string(shortPath), prefix+"%", prefix+"%", limit+1) + glog.V(5).Infof("ListRecursivePrefixedEntries lastFileName %s shortPath %v, prefix %v, sql %s", lastFileName, string(shortPath), prefix, store.GetSqlListRecursive(bucket)) + rows, err := db.QueryContext(ctx, store.GetSqlListRecursive(bucket), startFileName, util.HashStringToLong(string(shortPath)), prefix+"%", string(shortPath)+prefix+"%", limit+1) if err != nil { return lastFileName, fmt.Errorf("list %s : %v", dirPath, err) } defer rows.Close() for rows.Next() { - var name string + var dir, name string var data []byte - if err = rows.Scan(&name, &data); err != nil { + if err = rows.Scan(&dir, &name, &data); err != nil { glog.V(0).Infof("scan %s : %v", dirPath, err) return lastFileName, fmt.Errorf("scan %s: %v", dirPath, err) } - lastFileName = name + glog.V(0).Infof("scan dir %s name %v", dir, name) entry := &filer.Entry{ - FullPath: util.NewFullPath(string(dirPath), name), + FullPath: util.NewFullPath(dir, name), } + lastFileName = string(entry.FullPath) + if err = entry.DecodeAttributesAndChunks(util.MaybeDecompressData(data)); err != nil { glog.V(0).Infof("scan decode %s : %v", entry.FullPath, err) return lastFileName, fmt.Errorf("scan decode %s : %v", entry.FullPath, err) diff --git a/weed/filer/filer.go b/weed/filer/filer.go index 68c4c90d2..1f20b2684 100644 --- a/weed/filer/filer.go +++ b/weed/filer/filer.go @@ -354,8 +354,8 @@ func (f *Filer) FindEntry(ctx context.Context, p util.FullPath) (entry *Entry, e } -func (f *Filer) doListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int64, prefix string, eachEntryFunc ListEachEntryFunc) (expiredCount int64, lastFileName string, err error) { - lastFileName, err = f.Store.ListDirectoryPrefixedEntries(ctx, p, startFileName, inclusive, limit, prefix, func(entry *Entry) bool { +func (f *Filer) doListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, recursive bool, limit int64, prefix string, eachEntryFunc ListEachEntryFunc) (expiredCount int64, lastFileName string, err error) { + listFn := func(entry *Entry) bool { select { case <-ctx.Done(): return false @@ -369,7 +369,13 @@ func (f *Filer) doListDirectoryEntries(ctx context.Context, p util.FullPath, sta } return eachEntryFunc(entry) } - }) + } + glog.V(5).Infof("doListDirectoryEntries recursive %v path: %+v, prefix %s", recursive, p, prefix) + if recursive { + lastFileName, err = f.Store.ListRecursivePrefixedEntries(ctx, p, startFileName, inclusive, limit, prefix, listFn) + } else { + lastFileName, err = f.Store.ListDirectoryPrefixedEntries(ctx, p, startFileName, inclusive, limit, prefix, listFn) + } if err != nil { return expiredCount, lastFileName, err } diff --git a/weed/filer/filer_search.go b/weed/filer/filer_search.go index 6c7ba0747..c32756f39 100644 --- a/weed/filer/filer_search.go +++ b/weed/filer/filer_search.go @@ -2,6 +2,7 @@ package filer import ( "context" + "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/util" "math" "path/filepath" @@ -27,7 +28,7 @@ func (f *Filer) ListDirectoryEntries(ctx context.Context, p util.FullPath, start limit = math.MaxInt32 - 1 } - _, err = f.StreamListDirectoryEntries(ctx, p, startFileName, inclusive, limit+1, prefix, namePattern, namePatternExclude, func(entry *Entry) bool { + _, err = f.StreamListDirectoryEntries(ctx, p, startFileName, inclusive, false, limit+1, prefix, namePattern, namePatternExclude, func(entry *Entry) bool { entries = append(entries, entry) return true }) @@ -41,7 +42,9 @@ func (f *Filer) ListDirectoryEntries(ctx context.Context, p util.FullPath, start } // For now, prefix and namePattern are mutually exclusive -func (f *Filer) StreamListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int64, prefix string, namePattern string, namePatternExclude string, eachEntryFunc ListEachEntryFunc) (lastFileName string, err error) { +func (f *Filer) StreamListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, recursive bool, limit int64, prefix string, namePattern string, namePatternExclude string, eachEntryFunc ListEachEntryFunc) (lastFileName string, err error) { + glog.V(5).Infof("StreamListDirectoryEntries p %v startFileName %s prefix %s namePattern %v, recursive %v", p, startFileName, prefix, namePattern, recursive) + if strings.HasSuffix(string(p), "/") && len(p) > 1 { p = p[0 : len(p)-1] } @@ -52,23 +55,24 @@ func (f *Filer) StreamListDirectoryEntries(ctx context.Context, p util.FullPath, } var missedCount int64 - missedCount, lastFileName, err = f.doListPatternMatchedEntries(ctx, p, startFileName, inclusive, limit, prefix, restNamePattern, namePatternExclude, eachEntryFunc) + missedCount, lastFileName, err = f.doListPatternMatchedEntries(ctx, p, startFileName, inclusive, recursive, limit, prefix, restNamePattern, namePatternExclude, eachEntryFunc) for missedCount > 0 && err == nil { - missedCount, lastFileName, err = f.doListPatternMatchedEntries(ctx, p, lastFileName, false, missedCount, prefix, restNamePattern, namePatternExclude, eachEntryFunc) + missedCount, lastFileName, err = f.doListPatternMatchedEntries(ctx, p, lastFileName, false, recursive, missedCount, prefix, restNamePattern, namePatternExclude, eachEntryFunc) } return } -func (f *Filer) doListPatternMatchedEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int64, prefix, restNamePattern string, namePatternExclude string, eachEntryFunc ListEachEntryFunc) (missedCount int64, lastFileName string, err error) { +func (f *Filer) doListPatternMatchedEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, recursive bool, limit int64, prefix, restNamePattern string, namePatternExclude string, eachEntryFunc ListEachEntryFunc) (missedCount int64, lastFileName string, err error) { + glog.V(5).Infof("doListPatternMatchedEntries startFileName %v, recursive %v", startFileName, recursive) if len(restNamePattern) == 0 && len(namePatternExclude) == 0 { - lastFileName, err = f.doListValidEntries(ctx, p, startFileName, inclusive, limit, prefix, eachEntryFunc) + lastFileName, err = f.doListValidEntries(ctx, p, startFileName, inclusive, recursive, limit, prefix, eachEntryFunc) return 0, lastFileName, err } - lastFileName, err = f.doListValidEntries(ctx, p, startFileName, inclusive, limit, prefix, func(entry *Entry) bool { + lastFileName, err = f.doListValidEntries(ctx, p, startFileName, inclusive, recursive, limit, prefix, func(entry *Entry) bool { nameToTest := entry.Name() if len(namePatternExclude) > 0 { if matched, matchErr := filepath.Match(namePatternExclude, nameToTest); matchErr == nil && matched { @@ -93,11 +97,13 @@ func (f *Filer) doListPatternMatchedEntries(ctx context.Context, p util.FullPath return } -func (f *Filer) doListValidEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int64, prefix string, eachEntryFunc ListEachEntryFunc) (lastFileName string, err error) { +func (f *Filer) doListValidEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, recursive bool, limit int64, prefix string, eachEntryFunc ListEachEntryFunc) (lastFileName string, err error) { + glog.V(5).Infof("doListValidEntries p %v startFileName %v, recursive %v", p, startFileName, recursive) + var expiredCount int64 - expiredCount, lastFileName, err = f.doListDirectoryEntries(ctx, p, startFileName, inclusive, limit, prefix, eachEntryFunc) + expiredCount, lastFileName, err = f.doListDirectoryEntries(ctx, p, startFileName, inclusive, recursive, limit, prefix, eachEntryFunc) for expiredCount > 0 && err == nil { - expiredCount, lastFileName, err = f.doListDirectoryEntries(ctx, p, lastFileName, false, expiredCount, prefix, eachEntryFunc) + expiredCount, lastFileName, err = f.doListDirectoryEntries(ctx, p, lastFileName, false, recursive, expiredCount, prefix, eachEntryFunc) } return } diff --git a/weed/filer/filerstore_translate_path.go b/weed/filer/filerstore_translate_path.go index 002df0d14..34169d449 100644 --- a/weed/filer/filerstore_translate_path.go +++ b/weed/filer/filerstore_translate_path.go @@ -2,6 +2,7 @@ package filer import ( "context" + "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/util" "math" "strings" @@ -118,6 +119,7 @@ func (t *FilerStorePathTranslator) ListDirectoryEntries(ctx context.Context, dir } func (t *FilerStorePathTranslator) ListRecursivePrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc ListEachEntryFunc) (string, error) { + glog.V(5).Infof("ListRecursivePrefixedEntries dirPath %v", dirPath) newFullPath := t.translatePath(dirPath) diff --git a/weed/filer/filerstore_wrapper.go b/weed/filer/filerstore_wrapper.go index 38642a5c2..639fa6385 100644 --- a/weed/filer/filerstore_wrapper.go +++ b/weed/filer/filerstore_wrapper.go @@ -275,6 +275,25 @@ func (fsw *FilerStoreWrapper) ListDirectoryPrefixedEntries(ctx context.Context, } func (fsw *FilerStoreWrapper) ListRecursivePrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc ListEachEntryFunc) (lastFileName string, err error) { + actualStore := fsw.getActualStore(dirPath + "/") + stats.FilerStoreCounter.WithLabelValues(actualStore.GetName(), "prefixRecursiveList").Inc() + start := time.Now() + defer func() { + stats.FilerStoreHistogram.WithLabelValues(actualStore.GetName(), "prefixRecursiveList").Observe(time.Since(start).Seconds()) + }() + if limit > math.MaxInt32-1 { + limit = math.MaxInt32 - 1 + } + glog.V(5).Infof("ListRecursivePrefixedEntries %s from %s prefix %s limit %d", dirPath, startFileName, prefix, limit) + adjustedEntryFunc := func(entry *Entry) bool { + fsw.maybeReadHardLink(ctx, entry) + filer_pb.AfterEntryDeserialization(entry.GetChunks()) + return eachEntryFunc(entry) + } + lastFileName, err = actualStore.ListRecursivePrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, prefix, adjustedEntryFunc) + if err == ErrUnsupportedListDirectoryPrefixed { + lastFileName, err = fsw.prefixFilterEntries(ctx, dirPath, startFileName, includeStartFile, limit, prefix, adjustedEntryFunc) + } return lastFileName, err } diff --git a/weed/filer/mysql/mysql_sql_gen.go b/weed/filer/mysql/mysql_sql_gen.go index 2655e794f..cc8bef3b2 100644 --- a/weed/filer/mysql/mysql_sql_gen.go +++ b/weed/filer/mysql/mysql_sql_gen.go @@ -50,7 +50,7 @@ func (gen *SqlGenMysql) GetSqlListInclusive(tableName string) string { } func (gen *SqlGenMysql) GetSqlListRecursive(tableName string) string { - return fmt.Sprintf("SELECT `name`, `meta` FROM `%s` WHERE `dirhash` > ? AND `name` > ? AND ((`directory` = ? AND `name` LIKE ?) OR `directory` LIKE ?) ORDER BY `directory,name` ASC LIMIT ?", tableName) + return fmt.Sprintf("SELECT `directory`, `name`, `meta` FROM `%s` WHERE `directory` || '/' || `name` > ? AND ((dirhash == ? AND `name` like ?) OR `directory` like ?) ORDER BY `directory`,`name` ASC LIMIT ?", tableName) } func (gen *SqlGenMysql) GetSqlCreateTable(tableName string) string { diff --git a/weed/filer/sqlite/sqlite_store.go b/weed/filer/sqlite/sqlite_store.go index 6c9ca4ecc..8e30d72ac 100644 --- a/weed/filer/sqlite/sqlite_store.go +++ b/weed/filer/sqlite/sqlite_store.go @@ -61,7 +61,7 @@ func (store *SqliteStore) initialize(dbFile, createTable, upsertQuery string) (e } var dbErr error - store.DB, dbErr = sql.Open("sqlite", dbFile) + store.DB, dbErr = sql.Open("sqlite", "file:"+dbFile) if dbErr != nil { if store.DB != nil { store.DB.Close() diff --git a/weed/s3api/s3api_object_handlers.go b/weed/s3api/s3api_object_handlers.go index 1d58af8bc..a86904c9c 100644 --- a/weed/s3api/s3api_object_handlers.go +++ b/weed/s3api/s3api_object_handlers.go @@ -3,6 +3,8 @@ package s3api import ( "bytes" "fmt" + "github.com/seaweedfs/seaweedfs/weed/filer" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "io" "net/http" "net/url" @@ -35,10 +37,17 @@ func urlEscapeObject(object string) string { return "/" + t } +func entryUrlEncode(dir string, entry string, encodingTypeUrl bool) (dirName string, entryName string, prefix string) { + if !encodingTypeUrl { + return dir, entry, entry + } + return urlPathEscape(dir), url.QueryEscape(entry), urlPathEscape(entry) +} + func urlPathEscape(object string) string { var escapedParts []string for _, part := range strings.Split(object, "/") { - escapedParts = append(escapedParts, url.PathEscape(part)) + escapedParts = append(escapedParts, strings.ReplaceAll(url.PathEscape(part), "+", "%2B")) } return strings.Join(escapedParts, "/") } @@ -63,6 +72,32 @@ func removeDuplicateSlashes(object string) string { return result.String() } +func newListEntry(entry *filer_pb.Entry, dir string, name string, bucketPrefix string, fetchOwner bool, isDirectory bool) (listEntry ListEntry) { + storageClass := "STANDARD" + if v, ok := entry.Extended[s3_constants.AmzStorageClass]; ok { + storageClass = string(v) + } + keyFormat := "%s/%s" + if isDirectory { + keyFormat += "/" + } + listEntry = ListEntry{ + Key: fmt.Sprintf(keyFormat, dir, name)[len(bucketPrefix):], + LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(), + ETag: "\"" + filer.ETag(entry) + "\"", + Size: int64(filer.FileSize(entry)), + StorageClass: StorageClass(storageClass), + } + if fetchOwner { + listEntry.Owner = CanonicalUser{ + ID: fmt.Sprintf("%x", entry.Attributes.Uid), + DisplayName: entry.Attributes.UserName, + set: true, + } + } + return listEntry +} + func (s3a *S3ApiServer) toFilerUrl(bucket, object string) string { object = urlPathEscape(removeDuplicateSlashes(object)) destUrl := fmt.Sprintf("http://%s%s/%s%s", diff --git a/weed/s3api/s3api_object_handlers_list.go b/weed/s3api/s3api_object_handlers_list.go index a85affe4a..1d9faa407 100644 --- a/weed/s3api/s3api_object_handlers_list.go +++ b/weed/s3api/s3api_object_handlers_list.go @@ -4,33 +4,44 @@ import ( "context" "encoding/xml" "fmt" + "github.com/aws/aws-sdk-go/service/s3" "github.com/seaweedfs/seaweedfs/weed/glog" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" "io" "net/http" "net/url" "strconv" "strings" - "time" - - "github.com/seaweedfs/seaweedfs/weed/filer" - "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" - "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" ) +type OptionalString struct { + string + set bool +} + +func (o OptionalString) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error { + if !o.set { + return nil + } + return e.EncodeElement(o.string, startElement) +} + type ListBucketResultV2 struct { - XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult"` - Name string `xml:"Name"` - Prefix string `xml:"Prefix"` - MaxKeys int `xml:"MaxKeys"` - Delimiter string `xml:"Delimiter,omitempty"` - IsTruncated bool `xml:"IsTruncated"` - Contents []ListEntry `xml:"Contents,omitempty"` - CommonPrefixes []PrefixEntry `xml:"CommonPrefixes,omitempty"` - ContinuationToken string `xml:"ContinuationToken,omitempty"` - NextContinuationToken string `xml:"NextContinuationToken,omitempty"` - KeyCount int `xml:"KeyCount"` - StartAfter string `xml:"StartAfter,omitempty"` + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult"` + Name string `xml:"Name"` + Prefix string `xml:"Prefix"` + MaxKeys int `xml:"MaxKeys"` + Delimiter string `xml:"Delimiter,omitempty"` + IsTruncated bool `xml:"IsTruncated"` + Contents []ListEntry `xml:"Contents,omitempty"` + CommonPrefixes []PrefixEntry `xml:"CommonPrefixes,omitempty"` + ContinuationToken OptionalString `xml:"ContinuationToken,omitempty"` + NextContinuationToken string `xml:"NextContinuationToken,omitempty"` + EncodingType string `xml:"EncodingType,omitempty"` + KeyCount int `xml:"KeyCount"` + StartAfter string `xml:"StartAfter,omitempty"` } func (s3a *S3ApiServer) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) { @@ -41,19 +52,19 @@ func (s3a *S3ApiServer) ListObjectsV2Handler(w http.ResponseWriter, r *http.Requ bucket, _ := s3_constants.GetBucketAndObject(r) glog.V(3).Infof("ListObjectsV2Handler %s", bucket) - originalPrefix, continuationToken, startAfter, delimiter, _, maxKeys := getListObjectsV2Args(r.URL.Query()) + originalPrefix, startAfter, delimiter, continuationToken, encodingTypeUrl, fetchOwner, maxKeys := getListObjectsV2Args(r.URL.Query()) if maxKeys < 0 { s3err.WriteErrorResponse(w, r, s3err.ErrInvalidMaxKeys) return } - marker := continuationToken - if continuationToken == "" { + marker := continuationToken.string + if !continuationToken.set { marker = startAfter } - response, err := s3a.listFilerEntries(bucket, originalPrefix, maxKeys, marker, delimiter) + response, err := s3a.listFilerEntries(bucket, originalPrefix, maxKeys, marker, delimiter, encodingTypeUrl, fetchOwner) if err != nil { s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) @@ -81,6 +92,9 @@ func (s3a *S3ApiServer) ListObjectsV2Handler(w http.ResponseWriter, r *http.Requ Prefix: response.Prefix, StartAfter: startAfter, } + if encodingTypeUrl { + responseV2.EncodingType = s3.EncodingTypeUrl + } writeSuccessResponseXML(w, r, responseV2) } @@ -93,14 +107,14 @@ func (s3a *S3ApiServer) ListObjectsV1Handler(w http.ResponseWriter, r *http.Requ bucket, _ := s3_constants.GetBucketAndObject(r) glog.V(3).Infof("ListObjectsV1Handler %s", bucket) - originalPrefix, marker, delimiter, maxKeys := getListObjectsV1Args(r.URL.Query()) + originalPrefix, marker, delimiter, encodingTypeUrl, maxKeys := getListObjectsV1Args(r.URL.Query()) if maxKeys < 0 { s3err.WriteErrorResponse(w, r, s3err.ErrInvalidMaxKeys) return } - response, err := s3a.listFilerEntries(bucket, originalPrefix, maxKeys, marker, delimiter) + response, err := s3a.listFilerEntries(bucket, originalPrefix, maxKeys, marker, delimiter, encodingTypeUrl, true) if err != nil { s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) @@ -117,7 +131,7 @@ func (s3a *S3ApiServer) ListObjectsV1Handler(w http.ResponseWriter, r *http.Requ writeSuccessResponseXML(w, r, response) } -func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, maxKeys int, originalMarker string, delimiter string) (response ListBucketResult, err error) { +func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, maxKeys int, originalMarker string, delimiter string, encodingTypeUrl bool, fetchOwner bool) (response ListBucketResult, err error) { // convert full path prefix into directory name and prefix for entry name requestDir, prefix, marker := normalizePrefixMarker(originalPrefix, originalMarker) bucketPrefix := fmt.Sprintf("%s/%s/", s3a.option.BucketsPath, bucket) @@ -134,36 +148,22 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m maxKeys: maxKeys, prefixEndsOnDelimiter: strings.HasSuffix(originalPrefix, "/") && len(originalMarker) == 0, } - - if s3a.option.AllowListRecursive { + // Todo remove force disable + if s3a.option.AllowListRecursive && prefix == "force_disable" { err = s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { glog.V(0).Infof("doListFilerRecursiveEntries reqDir: %s, prefix: %s, delimiter: %s, cursor: %+v", reqDir, prefix, delimiter, cursor) nextMarker, doErr = s3a.doListFilerRecursiveEntries(client, reqDir, prefix, cursor, marker, delimiter, false, func(dir string, entry *filer_pb.Entry) { + dirName, entryName, prefixName := entryUrlEncode(dir, entry.Name, encodingTypeUrl) if entry.IsDirectory { if delimiter == "/" { // A response can contain CommonPrefixes only if you specify a delimiter. commonPrefixes = append(commonPrefixes, PrefixEntry{ - Prefix: fmt.Sprintf("%s/%s/", dir, entry.Name)[len(bucketPrefix):], + Prefix: fmt.Sprintf("%s/%s/", dirName, prefixName)[len(bucketPrefix):], }) } return } - storageClass := "STANDARD" - if v, ok := entry.Extended[s3_constants.AmzStorageClass]; ok { - storageClass = string(v) - } - - contents = append(contents, ListEntry{ - Key: fmt.Sprintf("%s/%s", dir, entry.Name)[len(bucketPrefix):], - LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(), - ETag: "\"" + filer.ETag(entry) + "\"", - Size: int64(filer.FileSize(entry)), - Owner: CanonicalUser{ - ID: fmt.Sprintf("%x", entry.Attributes.Uid), - DisplayName: entry.Attributes.UserName, - }, - StorageClass: StorageClass(storageClass), - }) + contents = append(contents, newListEntry(entry, dirName, entryName, bucketPrefix, fetchOwner, entry.IsDirectoryKeyObject())) cursor.maxKeys-- }, ) @@ -180,6 +180,9 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m Contents: contents, CommonPrefixes: commonPrefixes, } + if encodingTypeUrl { + response.EncodingType = s3.EncodingTypeUrl + } return } @@ -189,23 +192,16 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m empty := true nextMarker, doErr = s3a.doListFilerEntries(client, reqDir, prefix, cursor, marker, delimiter, false, func(dir string, entry *filer_pb.Entry) { empty = false + glog.V(5).Infof("doListFilerEntries dir: %s entry: %+v", dir, entry) + dirName, entryName, prefixName := entryUrlEncode(dir, entry.Name, encodingTypeUrl) if entry.IsDirectory { if entry.IsDirectoryKeyObject() { - contents = append(contents, ListEntry{ - Key: fmt.Sprintf("%s/%s/", dir, entry.Name)[len(bucketPrefix):], - LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(), - ETag: "\"" + filer.ETag(entry) + "\"", - Owner: CanonicalUser{ - ID: fmt.Sprintf("%x", entry.Attributes.Uid), - DisplayName: entry.Attributes.UserName, - }, - StorageClass: "STANDARD", - }) + contents = append(contents, newListEntry(entry, dirName, entryName, bucketPrefix, fetchOwner, true)) cursor.maxKeys-- // https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html } else if delimiter == "/" { // A response can contain CommonPrefixes only if you specify a delimiter. commonPrefixes = append(commonPrefixes, PrefixEntry{ - Prefix: fmt.Sprintf("%s/%s/", dir, entry.Name)[len(bucketPrefix):], + Prefix: fmt.Sprintf("%s/%s/", dirName, prefixName)[len(bucketPrefix):], }) //All of the keys (up to 1,000) rolled up into a common prefix count as a single return when calculating the number of returns. cursor.maxKeys-- @@ -243,21 +239,7 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m } } if !delimiterFound { - storageClass := "STANDARD" - if v, ok := entry.Extended[s3_constants.AmzStorageClass]; ok { - storageClass = string(v) - } - contents = append(contents, ListEntry{ - Key: fmt.Sprintf("%s/%s", dir, entry.Name)[len(bucketPrefix):], - LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(), - ETag: "\"" + filer.ETag(entry) + "\"", - Size: int64(filer.FileSize(entry)), - Owner: CanonicalUser{ - ID: fmt.Sprintf("%x", entry.Attributes.Uid), - DisplayName: entry.Attributes.UserName, - }, - StorageClass: StorageClass(storageClass), - }) + contents = append(contents, newListEntry(entry, dirName, entryName, bucketPrefix, fetchOwner, false)) cursor.maxKeys-- } } @@ -291,7 +273,9 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m Contents: contents, CommonPrefixes: commonPrefixes, } - + if encodingTypeUrl { + response.EncodingType = s3.EncodingTypeUrl + } return nil }) @@ -514,11 +498,12 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d return } -func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimiter string, fetchOwner bool, maxkeys int) { +func getListObjectsV2Args(values url.Values) (prefix, startAfter, delimiter string, token OptionalString, encodingTypeUrl bool, fetchOwner bool, maxkeys int) { prefix = values.Get("prefix") - token = values.Get("continuation-token") + token = OptionalString{set: values.Has("continuation-token"), string: values.Get("continuation-token")} startAfter = values.Get("start-after") delimiter = values.Get("delimiter") + encodingTypeUrl = values.Get("encoding-type") == s3.EncodingTypeUrl if values.Get("max-keys") != "" { maxkeys, _ = strconv.Atoi(values.Get("max-keys")) } else { @@ -528,10 +513,11 @@ func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimit return } -func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string, maxkeys int) { +func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string, encodingTypeUrl bool, maxkeys int) { prefix = values.Get("prefix") marker = values.Get("marker") delimiter = values.Get("delimiter") + encodingTypeUrl = values.Get("encoding-type") == "url" if values.Get("max-keys") != "" { maxkeys, _ = strconv.Atoi(values.Get("max-keys")) } else { diff --git a/weed/s3api/s3api_xsd_generated.go b/weed/s3api/s3api_xsd_generated.go index dd6a32ff2..b349764a5 100644 --- a/weed/s3api/s3api_xsd_generated.go +++ b/weed/s3api/s3api_xsd_generated.go @@ -27,6 +27,7 @@ type BucketLoggingStatus struct { type CanonicalUser struct { ID string `xml:"ID"` DisplayName string `xml:"DisplayName,omitempty"` + set bool } type CopyObject struct { @@ -62,6 +63,7 @@ func (t *CopyObject) MarshalXML(e *xml.Encoder, start xml.StartElement) error { layout.Timestamp = (*xsdDateTime)(&layout.T.Timestamp) return e.EncodeElement(layout, start) } + func (t *CopyObject) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { type T CopyObject var overlay struct { @@ -96,6 +98,7 @@ func (t *CopyObjectResult) MarshalXML(e *xml.Encoder, start xml.StartElement) er layout.LastModified = (*xsdDateTime)(&layout.T.LastModified) return e.EncodeElement(layout, start) } + func (t *CopyObjectResult) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { type T CopyObjectResult var overlay struct { @@ -125,6 +128,7 @@ func (t *CreateBucket) MarshalXML(e *xml.Encoder, start xml.StartElement) error layout.Timestamp = (*xsdDateTime)(&layout.T.Timestamp) return e.EncodeElement(layout, start) } + func (t *CreateBucket) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { type T CreateBucket var overlay struct { @@ -591,6 +595,7 @@ type ListBucketResult struct { NextMarker string `xml:"NextMarker,omitempty"` MaxKeys int `xml:"MaxKeys"` Delimiter string `xml:"Delimiter,omitempty"` + EncodingType string `xml:"EncodingType,omitempty"` IsTruncated bool `xml:"IsTruncated"` Contents []ListEntry `xml:"Contents,omitempty"` CommonPrefixes []PrefixEntry `xml:"CommonPrefixes,omitempty"` @@ -615,6 +620,14 @@ func (t *ListEntry) MarshalXML(e *xml.Encoder, start xml.StartElement) error { layout.LastModified = (*xsdDateTime)(&layout.T.LastModified) return e.EncodeElement(layout, start) } +func (c CanonicalUser) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + if !c.set { + return nil + } + type canonicalUserWrapper CanonicalUser + return e.EncodeElement(canonicalUserWrapper(c), start) +} + func (t *ListEntry) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { type T ListEntry var overlay struct { @@ -784,6 +797,7 @@ func (t *PutObjectResult) MarshalXML(e *xml.Encoder, start xml.StartElement) err layout.LastModified = (*xsdDateTime)(&layout.T.LastModified) return e.EncodeElement(layout, start) } + func (t *PutObjectResult) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { type T PutObjectResult var overlay struct { @@ -822,6 +836,7 @@ func (t *SetBucketAccessControlPolicy) MarshalXML(e *xml.Encoder, start xml.Star layout.Timestamp = (*xsdDateTime)(&layout.T.Timestamp) return e.EncodeElement(layout, start) } + func (t *SetBucketAccessControlPolicy) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { type T SetBucketAccessControlPolicy var overlay struct { @@ -855,6 +870,7 @@ func (t *SetBucketLoggingStatus) MarshalXML(e *xml.Encoder, start xml.StartEleme layout.Timestamp = (*xsdDateTime)(&layout.T.Timestamp) return e.EncodeElement(layout, start) } + func (t *SetBucketLoggingStatus) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { type T SetBucketLoggingStatus var overlay struct { @@ -889,6 +905,7 @@ func (t *SetObjectAccessControlPolicy) MarshalXML(e *xml.Encoder, start xml.Star layout.Timestamp = (*xsdDateTime)(&layout.T.Timestamp) return e.EncodeElement(layout, start) } + func (t *SetObjectAccessControlPolicy) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { type T SetObjectAccessControlPolicy var overlay struct { @@ -940,6 +957,7 @@ func (t *VersionEntry) MarshalXML(e *xml.Encoder, start xml.StartElement) error layout.LastModified = (*xsdDateTime)(&layout.T.LastModified) return e.EncodeElement(layout, start) } + func (t *VersionEntry) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { type T VersionEntry var overlay struct { @@ -965,6 +983,7 @@ func (b *xsdBase64Binary) UnmarshalText(text []byte) (err error) { *b, err = base64.StdEncoding.DecodeString(string(text)) return } + func (b xsdBase64Binary) MarshalText() ([]byte, error) { var buf bytes.Buffer enc := base64.NewEncoder(base64.StdEncoding, &buf) @@ -978,9 +997,11 @@ type xsdDateTime time.Time func (t *xsdDateTime) UnmarshalText(text []byte) error { return _unmarshalTime(text, (*time.Time)(t), s3TimeFormat) } + func (t xsdDateTime) MarshalText() ([]byte, error) { return []byte((time.Time)(t).Format(s3TimeFormat)), nil } + func (t xsdDateTime) MarshalXML(e *xml.Encoder, start xml.StartElement) error { if (time.Time)(t).IsZero() { return nil @@ -991,6 +1012,7 @@ func (t xsdDateTime) MarshalXML(e *xml.Encoder, start xml.StartElement) error { } return e.EncodeElement(m, start) } + func (t xsdDateTime) MarshalXMLAttr(name xml.Name) (xml.Attr, error) { if (time.Time)(t).IsZero() { return xml.Attr{}, nil @@ -998,6 +1020,7 @@ func (t xsdDateTime) MarshalXMLAttr(name xml.Name) (xml.Attr, error) { m, err := t.MarshalText() return xml.Attr{Name: name, Value: string(m)}, err } + func _unmarshalTime(text []byte, t *time.Time, format string) (err error) { s := string(bytes.TrimSpace(text)) *t, err = time.Parse(format, s) diff --git a/weed/server/filer_grpc_server.go b/weed/server/filer_grpc_server.go index b9571710d..819ddfcb5 100644 --- a/weed/server/filer_grpc_server.go +++ b/weed/server/filer_grpc_server.go @@ -56,8 +56,9 @@ func (fs *FilerServer) ListEntries(req *filer_pb.ListEntriesRequest, stream file var listErr error for limit > 0 { var hasEntries bool - lastFileName, listErr = fs.filer.StreamListDirectoryEntries(stream.Context(), util.FullPath(req.Directory), lastFileName, includeLastFile, int64(paginationLimit), req.Prefix, "", "", func(entry *filer.Entry) bool { + lastFileName, listErr = fs.filer.StreamListDirectoryEntries(stream.Context(), util.FullPath(req.Directory), lastFileName, includeLastFile, req.Recursive, int64(paginationLimit), req.Prefix, "", "", func(entry *filer.Entry) bool { hasEntries = true + glog.V(5).Infof("StreamListDirectoryEntries recursive %v, entry: %+v", req.Recursive, entry) if err = stream.Send(&filer_pb.ListEntriesResponse{ Entry: entry.ToProtoEntry(), }); err != nil { diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index ec1dff445..4832c5685 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -164,11 +164,14 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption) if !util.LoadConfiguration("filer", false) { v.SetDefault("leveldb2.enabled", true) v.SetDefault("leveldb2.dir", option.DefaultLevelDbDir) - _, err := os.Stat(option.DefaultLevelDbDir) - if os.IsNotExist(err) { - os.MkdirAll(option.DefaultLevelDbDir, 0755) + if v.GetBool("leveldb2.enabled") { + levelDbDir := v.GetString("leveldb2.dir") + _, err := os.Stat(levelDbDir) + if os.IsNotExist(err) { + os.MkdirAll(levelDbDir, 0755) + } + glog.V(0).Infof("create filer store dir in %s", levelDbDir) } - glog.V(0).Infof("default to create filer store dir in %s", option.DefaultLevelDbDir) } else { glog.Warningf("skipping default store dir in %s", option.DefaultLevelDbDir) }