pass test_bucket_listv2_encoding_basic and test_bucket_listv2_fetchowner_defaultempty

.github/workflows/s3tests.yml

@@ -54,6 +54,8 @@ jobs:
           s3tests_boto3/functional/test_s3.py::test_bucket_list_distinct \
           s3tests_boto3/functional/test_s3.py::test_bucket_list_many \
           s3tests_boto3/functional/test_s3.py::test_bucket_listv2_many \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_basic \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_encoding_basic \
           s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_prefix \
           s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_prefix_ends_with_delimiter \
           s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_alt \
@@ -76,6 +78,7 @@ jobs:
           s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_delimiter_prefix_delimiter_not_exist \
           s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist \
           s3tests_boto3/functional/test_s3.py::test_bucket_listv2_fetchowner_notempty \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_fetchowner_defaultempty \
           s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_basic \
           s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_basic \
           s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_alt \
@@ -94,6 +97,7 @@ jobs:
           s3tests_boto3/functional/test_s3.py::test_bucket_listv2_maxkeys_zero \
           s3tests_boto3/functional/test_s3.py::test_bucket_list_marker_none \
           s3tests_boto3/functional/test_s3.py::test_bucket_list_marker_empty \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_continuationtoken_empty \
           s3tests_boto3/functional/test_s3.py::test_bucket_listv2_continuationtoken \
           s3tests_boto3/functional/test_s3.py::test_bucket_listv2_both_continuationtoken_startafter \
           s3tests_boto3/functional/test_s3.py::test_bucket_list_marker_unreadable \

Makefile

@@ -15,7 +15,7 @@ full_install:
 	cd weed; go install -tags "elastic gocdk sqlite ydb tikv rclone"
 
 server: install
-	export WEED_LEVELDB2_ENABLED="false";export WEED_SQLITE_ENABLED="true"; \
+	export WEED_LEVELDB2_ENABLED="false";export WEED_SQLITE_ENABLED="true"; export WEED_SQLITE_DBFILE="/tmp/filer.db"; \
 	weed -v 0 server -s3.allowListRecursive=true -dir /tmp -master.volumeSizeLimitMB=1024 -s3 -filer -filer.maxMB=64 -filer.port.public=7777 -volume.max=100 -volume.preStopSeconds=1 -s3.port=8000 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config=./docker/compose/s3.json -metricsPort=9324
 
 benchmark: install warp_install

weed/filer/abstract_sql/abstract_sql_store.go

@@ -289,6 +289,7 @@ func (store *AbstractSqlStore) DeleteFolderChildren(ctx context.Context, fullpat
 }
 
 func (store *AbstractSqlStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
+	glog.V(5).Infof("ListDirectoryPrefixedEntries dirPath %v, includeStartFile %v", dirPath, includeStartFile)
 
 	db, bucket, shortPath, err := store.getTxOrDB(ctx, dirPath, true)
 	if err != nil {
@@ -337,24 +338,27 @@ func (store *AbstractSqlStore) ListRecursivePrefixedEntries(ctx context.Context,
 	if err != nil {
 		return lastFileName, fmt.Errorf("findDB %s : %v", dirPath, err)
 	}
-	rows, err := db.QueryContext(ctx, store.GetSqlListRecursive(bucket), util.HashStringToLong(string(shortPath)), startFileName, string(shortPath), prefix+"%", prefix+"%", limit+1)
+	glog.V(5).Infof("ListRecursivePrefixedEntries lastFileName %s shortPath %v, prefix %v, sql %s", lastFileName, string(shortPath), prefix, store.GetSqlListRecursive(bucket))
+	rows, err := db.QueryContext(ctx, store.GetSqlListRecursive(bucket), startFileName, util.HashStringToLong(string(shortPath)), prefix+"%", string(shortPath)+prefix+"%", limit+1)
 	if err != nil {
 		return lastFileName, fmt.Errorf("list %s : %v", dirPath, err)
 	}
 	defer rows.Close()
 
 	for rows.Next() {
-		var name string
+		var dir, name string
 		var data []byte
-		if err = rows.Scan(&name, &data); err != nil {
+		if err = rows.Scan(&dir, &name, &data); err != nil {
 			glog.V(0).Infof("scan %s : %v", dirPath, err)
 			return lastFileName, fmt.Errorf("scan %s: %v", dirPath, err)
 		}
-		lastFileName = name
+		glog.V(0).Infof("scan dir %s name %v", dir, name)
 
 		entry := &filer.Entry{
-			FullPath: util.NewFullPath(string(dirPath), name),
+			FullPath: util.NewFullPath(dir, name),
 		}
+		lastFileName = string(entry.FullPath)
+
 		if err = entry.DecodeAttributesAndChunks(util.MaybeDecompressData(data)); err != nil {
 			glog.V(0).Infof("scan decode %s : %v", entry.FullPath, err)
 			return lastFileName, fmt.Errorf("scan decode %s : %v", entry.FullPath, err)

weed/filer/filer.go

@@ -354,8 +354,8 @@ func (f *Filer) FindEntry(ctx context.Context, p util.FullPath) (entry *Entry, e
 
 }
 
-func (f *Filer) doListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int64, prefix string, eachEntryFunc ListEachEntryFunc) (expiredCount int64, lastFileName string, err error) {
-	lastFileName, err = f.Store.ListDirectoryPrefixedEntries(ctx, p, startFileName, inclusive, limit, prefix, func(entry *Entry) bool {
+func (f *Filer) doListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, recursive bool, limit int64, prefix string, eachEntryFunc ListEachEntryFunc) (expiredCount int64, lastFileName string, err error) {
+	listFn := func(entry *Entry) bool {
 		select {
 		case <-ctx.Done():
 			return false
@@ -369,7 +369,13 @@ func (f *Filer) doListDirectoryEntries(ctx context.Context, p util.FullPath, sta
 			}
 			return eachEntryFunc(entry)
 		}
-	})
+	}
+	glog.V(5).Infof("doListDirectoryEntries recursive %v  path: %+v, prefix %s", recursive, p, prefix)
+	if recursive {
+		lastFileName, err = f.Store.ListRecursivePrefixedEntries(ctx, p, startFileName, inclusive, limit, prefix, listFn)
+	} else {
+		lastFileName, err = f.Store.ListDirectoryPrefixedEntries(ctx, p, startFileName, inclusive, limit, prefix, listFn)
+	}
 	if err != nil {
 		return expiredCount, lastFileName, err
 	}

weed/filer/filer_search.go

@@ -2,6 +2,7 @@ package filer
 
 import (
 	"context"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/util"
 	"math"
 	"path/filepath"
@@ -27,7 +28,7 @@ func (f *Filer) ListDirectoryEntries(ctx context.Context, p util.FullPath, start
 		limit = math.MaxInt32 - 1
 	}
 
-	_, err = f.StreamListDirectoryEntries(ctx, p, startFileName, inclusive, limit+1, prefix, namePattern, namePatternExclude, func(entry *Entry) bool {
+	_, err = f.StreamListDirectoryEntries(ctx, p, startFileName, inclusive, false, limit+1, prefix, namePattern, namePatternExclude, func(entry *Entry) bool {
 		entries = append(entries, entry)
 		return true
 	})
@@ -41,7 +42,9 @@ func (f *Filer) ListDirectoryEntries(ctx context.Context, p util.FullPath, start
 }
 
 // For now, prefix and namePattern are mutually exclusive
-func (f *Filer) StreamListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int64, prefix string, namePattern string, namePatternExclude string, eachEntryFunc ListEachEntryFunc) (lastFileName string, err error) {
+func (f *Filer) StreamListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, recursive bool, limit int64, prefix string, namePattern string, namePatternExclude string, eachEntryFunc ListEachEntryFunc) (lastFileName string, err error) {
+	glog.V(5).Infof("StreamListDirectoryEntries p %v startFileName %s prefix %s namePattern %v, recursive %v", p, startFileName, prefix, namePattern, recursive)
+
 	if strings.HasSuffix(string(p), "/") && len(p) > 1 {
 		p = p[0 : len(p)-1]
 	}
@@ -52,23 +55,24 @@ func (f *Filer) StreamListDirectoryEntries(ctx context.Context, p util.FullPath,
 	}
 	var missedCount int64
 
-	missedCount, lastFileName, err = f.doListPatternMatchedEntries(ctx, p, startFileName, inclusive, limit, prefix, restNamePattern, namePatternExclude, eachEntryFunc)
+	missedCount, lastFileName, err = f.doListPatternMatchedEntries(ctx, p, startFileName, inclusive, recursive, limit, prefix, restNamePattern, namePatternExclude, eachEntryFunc)
 
 	for missedCount > 0 && err == nil {
-		missedCount, lastFileName, err = f.doListPatternMatchedEntries(ctx, p, lastFileName, false, missedCount, prefix, restNamePattern, namePatternExclude, eachEntryFunc)
+		missedCount, lastFileName, err = f.doListPatternMatchedEntries(ctx, p, lastFileName, false, recursive, missedCount, prefix, restNamePattern, namePatternExclude, eachEntryFunc)
 	}
 
 	return
 }
 
-func (f *Filer) doListPatternMatchedEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int64, prefix, restNamePattern string, namePatternExclude string, eachEntryFunc ListEachEntryFunc) (missedCount int64, lastFileName string, err error) {
+func (f *Filer) doListPatternMatchedEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, recursive bool, limit int64, prefix, restNamePattern string, namePatternExclude string, eachEntryFunc ListEachEntryFunc) (missedCount int64, lastFileName string, err error) {
+	glog.V(5).Infof("doListPatternMatchedEntries startFileName %v, recursive %v", startFileName, recursive)
 
 	if len(restNamePattern) == 0 && len(namePatternExclude) == 0 {
-		lastFileName, err = f.doListValidEntries(ctx, p, startFileName, inclusive, limit, prefix, eachEntryFunc)
+		lastFileName, err = f.doListValidEntries(ctx, p, startFileName, inclusive, recursive, limit, prefix, eachEntryFunc)
 		return 0, lastFileName, err
 	}
 
-	lastFileName, err = f.doListValidEntries(ctx, p, startFileName, inclusive, limit, prefix, func(entry *Entry) bool {
+	lastFileName, err = f.doListValidEntries(ctx, p, startFileName, inclusive, recursive, limit, prefix, func(entry *Entry) bool {
 		nameToTest := entry.Name()
 		if len(namePatternExclude) > 0 {
 			if matched, matchErr := filepath.Match(namePatternExclude, nameToTest); matchErr == nil && matched {
@@ -93,11 +97,13 @@ func (f *Filer) doListPatternMatchedEntries(ctx context.Context, p util.FullPath
 	return
 }
 
-func (f *Filer) doListValidEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int64, prefix string, eachEntryFunc ListEachEntryFunc) (lastFileName string, err error) {
+func (f *Filer) doListValidEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, recursive bool, limit int64, prefix string, eachEntryFunc ListEachEntryFunc) (lastFileName string, err error) {
+	glog.V(5).Infof("doListValidEntries p %v startFileName %v, recursive %v", p, startFileName, recursive)
+
 	var expiredCount int64
-	expiredCount, lastFileName, err = f.doListDirectoryEntries(ctx, p, startFileName, inclusive, limit, prefix, eachEntryFunc)
+	expiredCount, lastFileName, err = f.doListDirectoryEntries(ctx, p, startFileName, inclusive, recursive, limit, prefix, eachEntryFunc)
 	for expiredCount > 0 && err == nil {
-		expiredCount, lastFileName, err = f.doListDirectoryEntries(ctx, p, lastFileName, false, expiredCount, prefix, eachEntryFunc)
+		expiredCount, lastFileName, err = f.doListDirectoryEntries(ctx, p, lastFileName, false, recursive, expiredCount, prefix, eachEntryFunc)
 	}
 	return
 }

weed/filer/filerstore_translate_path.go

@@ -2,6 +2,7 @@ package filer
 
 import (
 	"context"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/util"
 	"math"
 	"strings"
@@ -118,6 +119,7 @@ func (t *FilerStorePathTranslator) ListDirectoryEntries(ctx context.Context, dir
 }
 
 func (t *FilerStorePathTranslator) ListRecursivePrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc ListEachEntryFunc) (string, error) {
+	glog.V(5).Infof("ListRecursivePrefixedEntries dirPath %v", dirPath)
 
 	newFullPath := t.translatePath(dirPath)
 

weed/filer/filerstore_wrapper.go

@@ -275,6 +275,25 @@ func (fsw *FilerStoreWrapper) ListDirectoryPrefixedEntries(ctx context.Context,
 }
 
 func (fsw *FilerStoreWrapper) ListRecursivePrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc ListEachEntryFunc) (lastFileName string, err error) {
+	actualStore := fsw.getActualStore(dirPath + "/")
+	stats.FilerStoreCounter.WithLabelValues(actualStore.GetName(), "prefixRecursiveList").Inc()
+	start := time.Now()
+	defer func() {
+		stats.FilerStoreHistogram.WithLabelValues(actualStore.GetName(), "prefixRecursiveList").Observe(time.Since(start).Seconds())
+	}()
+	if limit > math.MaxInt32-1 {
+		limit = math.MaxInt32 - 1
+	}
+	glog.V(5).Infof("ListRecursivePrefixedEntries %s from %s prefix %s limit %d", dirPath, startFileName, prefix, limit)
+	adjustedEntryFunc := func(entry *Entry) bool {
+		fsw.maybeReadHardLink(ctx, entry)
+		filer_pb.AfterEntryDeserialization(entry.GetChunks())
+		return eachEntryFunc(entry)
+	}
+	lastFileName, err = actualStore.ListRecursivePrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, prefix, adjustedEntryFunc)
+	if err == ErrUnsupportedListDirectoryPrefixed {
+		lastFileName, err = fsw.prefixFilterEntries(ctx, dirPath, startFileName, includeStartFile, limit, prefix, adjustedEntryFunc)
+	}
 	return lastFileName, err
 }
 

weed/filer/mysql/mysql_sql_gen.go

@@ -50,7 +50,7 @@ func (gen *SqlGenMysql) GetSqlListInclusive(tableName string) string {
 }
 
 func (gen *SqlGenMysql) GetSqlListRecursive(tableName string) string {
-	return fmt.Sprintf("SELECT `name`, `meta` FROM `%s` WHERE `dirhash` > ? AND `name` > ? AND ((`directory` = ? AND `name` LIKE ?) OR `directory` LIKE ?) ORDER BY `directory,name` ASC LIMIT ?", tableName)
+	return fmt.Sprintf("SELECT `directory`, `name`, `meta` FROM `%s` WHERE `directory` || '/'  || `name` > ? AND ((dirhash == ? AND `name` like ?) OR `directory` like ?) ORDER BY `directory`,`name` ASC LIMIT ?", tableName)
 }
 
 func (gen *SqlGenMysql) GetSqlCreateTable(tableName string) string {

weed/filer/sqlite/sqlite_store.go

@@ -61,7 +61,7 @@ func (store *SqliteStore) initialize(dbFile, createTable, upsertQuery string) (e
 	}
 
 	var dbErr error
-	store.DB, dbErr = sql.Open("sqlite", dbFile)
+	store.DB, dbErr = sql.Open("sqlite", "file:"+dbFile)
 	if dbErr != nil {
 		if store.DB != nil {
 			store.DB.Close()

weed/s3api/s3api_object_handlers.go

@@ -3,6 +3,8 @@ package s3api
 import (
 	"bytes"
 	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"io"
 	"net/http"
 	"net/url"
@@ -35,10 +37,17 @@ func urlEscapeObject(object string) string {
 	return "/" + t
 }
 
+func entryUrlEncode(dir string, entry string, encodingTypeUrl bool) (dirName string, entryName string, prefix string) {
+	if !encodingTypeUrl {
+		return dir, entry, entry
+	}
+	return urlPathEscape(dir), url.QueryEscape(entry), urlPathEscape(entry)
+}
+
 func urlPathEscape(object string) string {
 	var escapedParts []string
 	for _, part := range strings.Split(object, "/") {
-		escapedParts = append(escapedParts, url.PathEscape(part))
+		escapedParts = append(escapedParts, strings.ReplaceAll(url.PathEscape(part), "+", "%2B"))
 	}
 	return strings.Join(escapedParts, "/")
 }
@@ -63,6 +72,32 @@ func removeDuplicateSlashes(object string) string {
 	return result.String()
 }
 
+func newListEntry(entry *filer_pb.Entry, dir string, name string, bucketPrefix string, fetchOwner bool, isDirectory bool) (listEntry ListEntry) {
+	storageClass := "STANDARD"
+	if v, ok := entry.Extended[s3_constants.AmzStorageClass]; ok {
+		storageClass = string(v)
+	}
+	keyFormat := "%s/%s"
+	if isDirectory {
+		keyFormat += "/"
+	}
+	listEntry = ListEntry{
+		Key:          fmt.Sprintf(keyFormat, dir, name)[len(bucketPrefix):],
+		LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(),
+		ETag:         "\"" + filer.ETag(entry) + "\"",
+		Size:         int64(filer.FileSize(entry)),
+		StorageClass: StorageClass(storageClass),
+	}
+	if fetchOwner {
+		listEntry.Owner = CanonicalUser{
+			ID:          fmt.Sprintf("%x", entry.Attributes.Uid),
+			DisplayName: entry.Attributes.UserName,
+			set:         true,
+		}
+	}
+	return listEntry
+}
+
 func (s3a *S3ApiServer) toFilerUrl(bucket, object string) string {
 	object = urlPathEscape(removeDuplicateSlashes(object))
 	destUrl := fmt.Sprintf("http://%s%s/%s%s",

weed/s3api/s3api_object_handlers_list.go

@@ -4,33 +4,44 @@ import (
 	"context"
 	"encoding/xml"
 	"fmt"
+	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
 	"io"
 	"net/http"
 	"net/url"
 	"strconv"
 	"strings"
-	"time"
-
-	"github.com/seaweedfs/seaweedfs/weed/filer"
-	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
-	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
 )
 
+type OptionalString struct {
+	string
+	set bool
+}
+
+func (o OptionalString) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error {
+	if !o.set {
+		return nil
+	}
+	return e.EncodeElement(o.string, startElement)
+}
+
 type ListBucketResultV2 struct {
-	XMLName               xml.Name      `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult"`
-	Name                  string        `xml:"Name"`
-	Prefix                string        `xml:"Prefix"`
-	MaxKeys               int           `xml:"MaxKeys"`
-	Delimiter             string        `xml:"Delimiter,omitempty"`
-	IsTruncated           bool          `xml:"IsTruncated"`
-	Contents              []ListEntry   `xml:"Contents,omitempty"`
-	CommonPrefixes        []PrefixEntry `xml:"CommonPrefixes,omitempty"`
-	ContinuationToken     string        `xml:"ContinuationToken,omitempty"`
-	NextContinuationToken string        `xml:"NextContinuationToken,omitempty"`
-	KeyCount              int           `xml:"KeyCount"`
-	StartAfter            string        `xml:"StartAfter,omitempty"`
+	XMLName               xml.Name       `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult"`
+	Name                  string         `xml:"Name"`
+	Prefix                string         `xml:"Prefix"`
+	MaxKeys               int            `xml:"MaxKeys"`
+	Delimiter             string         `xml:"Delimiter,omitempty"`
+	IsTruncated           bool           `xml:"IsTruncated"`
+	Contents              []ListEntry    `xml:"Contents,omitempty"`
+	CommonPrefixes        []PrefixEntry  `xml:"CommonPrefixes,omitempty"`
+	ContinuationToken     OptionalString `xml:"ContinuationToken,omitempty"`
+	NextContinuationToken string         `xml:"NextContinuationToken,omitempty"`
+	EncodingType          string         `xml:"EncodingType,omitempty"`
+	KeyCount              int            `xml:"KeyCount"`
+	StartAfter            string         `xml:"StartAfter,omitempty"`
 }
 
 func (s3a *S3ApiServer) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {
@@ -41,19 +52,19 @@ func (s3a *S3ApiServer) ListObjectsV2Handler(w http.ResponseWriter, r *http.Requ
 	bucket, _ := s3_constants.GetBucketAndObject(r)
 	glog.V(3).Infof("ListObjectsV2Handler %s", bucket)
 
-	originalPrefix, continuationToken, startAfter, delimiter, _, maxKeys := getListObjectsV2Args(r.URL.Query())
+	originalPrefix, startAfter, delimiter, continuationToken, encodingTypeUrl, fetchOwner, maxKeys := getListObjectsV2Args(r.URL.Query())
 
 	if maxKeys < 0 {
 		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidMaxKeys)
 		return
 	}
 
-	marker := continuationToken
-	if continuationToken == "" {
+	marker := continuationToken.string
+	if !continuationToken.set {
 		marker = startAfter
 	}
 
-	response, err := s3a.listFilerEntries(bucket, originalPrefix, maxKeys, marker, delimiter)
+	response, err := s3a.listFilerEntries(bucket, originalPrefix, maxKeys, marker, delimiter, encodingTypeUrl, fetchOwner)
 
 	if err != nil {
 		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
@@ -81,6 +92,9 @@ func (s3a *S3ApiServer) ListObjectsV2Handler(w http.ResponseWriter, r *http.Requ
 		Prefix:                response.Prefix,
 		StartAfter:            startAfter,
 	}
+	if encodingTypeUrl {
+		responseV2.EncodingType = s3.EncodingTypeUrl
+	}
 
 	writeSuccessResponseXML(w, r, responseV2)
 }
@@ -93,14 +107,14 @@ func (s3a *S3ApiServer) ListObjectsV1Handler(w http.ResponseWriter, r *http.Requ
 	bucket, _ := s3_constants.GetBucketAndObject(r)
 	glog.V(3).Infof("ListObjectsV1Handler %s", bucket)
 
-	originalPrefix, marker, delimiter, maxKeys := getListObjectsV1Args(r.URL.Query())
+	originalPrefix, marker, delimiter, encodingTypeUrl, maxKeys := getListObjectsV1Args(r.URL.Query())
 
 	if maxKeys < 0 {
 		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidMaxKeys)
 		return
 	}
 
-	response, err := s3a.listFilerEntries(bucket, originalPrefix, maxKeys, marker, delimiter)
+	response, err := s3a.listFilerEntries(bucket, originalPrefix, maxKeys, marker, delimiter, encodingTypeUrl, true)
 
 	if err != nil {
 		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
@@ -117,7 +131,7 @@ func (s3a *S3ApiServer) ListObjectsV1Handler(w http.ResponseWriter, r *http.Requ
 	writeSuccessResponseXML(w, r, response)
 }
 
-func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, maxKeys int, originalMarker string, delimiter string) (response ListBucketResult, err error) {
+func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, maxKeys int, originalMarker string, delimiter string, encodingTypeUrl bool, fetchOwner bool) (response ListBucketResult, err error) {
 	// convert full path prefix into directory name and prefix for entry name
 	requestDir, prefix, marker := normalizePrefixMarker(originalPrefix, originalMarker)
 	bucketPrefix := fmt.Sprintf("%s/%s/", s3a.option.BucketsPath, bucket)
@@ -134,36 +148,22 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
 		maxKeys:               maxKeys,
 		prefixEndsOnDelimiter: strings.HasSuffix(originalPrefix, "/") && len(originalMarker) == 0,
 	}
-
-	if s3a.option.AllowListRecursive {
+	// Todo remove force disable
+	if s3a.option.AllowListRecursive && prefix == "force_disable" {
 		err = s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
 			glog.V(0).Infof("doListFilerRecursiveEntries reqDir: %s, prefix: %s, delimiter: %s, cursor: %+v", reqDir, prefix, delimiter, cursor)
 			nextMarker, doErr = s3a.doListFilerRecursiveEntries(client, reqDir, prefix, cursor, marker, delimiter, false,
 				func(dir string, entry *filer_pb.Entry) {
+					dirName, entryName, prefixName := entryUrlEncode(dir, entry.Name, encodingTypeUrl)
 					if entry.IsDirectory {
 						if delimiter == "/" { // A response can contain CommonPrefixes only if you specify a delimiter.
 							commonPrefixes = append(commonPrefixes, PrefixEntry{
-								Prefix: fmt.Sprintf("%s/%s/", dir, entry.Name)[len(bucketPrefix):],
+								Prefix: fmt.Sprintf("%s/%s/", dirName, prefixName)[len(bucketPrefix):],
 							})
 						}
 						return
 					}
-					storageClass := "STANDARD"
-					if v, ok := entry.Extended[s3_constants.AmzStorageClass]; ok {
-						storageClass = string(v)
-					}
-
-					contents = append(contents, ListEntry{
-						Key:          fmt.Sprintf("%s/%s", dir, entry.Name)[len(bucketPrefix):],
-						LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(),
-						ETag:         "\"" + filer.ETag(entry) + "\"",
-						Size:         int64(filer.FileSize(entry)),
-						Owner: CanonicalUser{
-							ID:          fmt.Sprintf("%x", entry.Attributes.Uid),
-							DisplayName: entry.Attributes.UserName,
-						},
-						StorageClass: StorageClass(storageClass),
-					})
+					contents = append(contents, newListEntry(entry, dirName, entryName, bucketPrefix, fetchOwner, entry.IsDirectoryKeyObject()))
 					cursor.maxKeys--
 				},
 			)
@@ -180,6 +180,9 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
 			Contents:       contents,
 			CommonPrefixes: commonPrefixes,
 		}
+		if encodingTypeUrl {
+			response.EncodingType = s3.EncodingTypeUrl
+		}
 		return
 	}
 
@@ -189,23 +192,16 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
 			empty := true
 			nextMarker, doErr = s3a.doListFilerEntries(client, reqDir, prefix, cursor, marker, delimiter, false, func(dir string, entry *filer_pb.Entry) {
 				empty = false
+				glog.V(5).Infof("doListFilerEntries dir: %s entry: %+v", dir, entry)
+				dirName, entryName, prefixName := entryUrlEncode(dir, entry.Name, encodingTypeUrl)
 				if entry.IsDirectory {
 					if entry.IsDirectoryKeyObject() {
-						contents = append(contents, ListEntry{
-							Key:          fmt.Sprintf("%s/%s/", dir, entry.Name)[len(bucketPrefix):],
-							LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(),
-							ETag:         "\"" + filer.ETag(entry) + "\"",
-							Owner: CanonicalUser{
-								ID:          fmt.Sprintf("%x", entry.Attributes.Uid),
-								DisplayName: entry.Attributes.UserName,
-							},
-							StorageClass: "STANDARD",
-						})
+						contents = append(contents, newListEntry(entry, dirName, entryName, bucketPrefix, fetchOwner, true))
 						cursor.maxKeys--
 						// https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html
 					} else if delimiter == "/" { // A response can contain CommonPrefixes only if you specify a delimiter.
 						commonPrefixes = append(commonPrefixes, PrefixEntry{
-							Prefix: fmt.Sprintf("%s/%s/", dir, entry.Name)[len(bucketPrefix):],
+							Prefix: fmt.Sprintf("%s/%s/", dirName, prefixName)[len(bucketPrefix):],
 						})
 						//All of the keys (up to 1,000) rolled up into a common prefix count as a single return when calculating the number of returns.
 						cursor.maxKeys--
@@ -243,21 +239,7 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
 						}
 					}
 					if !delimiterFound {
-						storageClass := "STANDARD"
-						if v, ok := entry.Extended[s3_constants.AmzStorageClass]; ok {
-							storageClass = string(v)
-						}
-						contents = append(contents, ListEntry{
-							Key:          fmt.Sprintf("%s/%s", dir, entry.Name)[len(bucketPrefix):],
-							LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(),
-							ETag:         "\"" + filer.ETag(entry) + "\"",
-							Size:         int64(filer.FileSize(entry)),
-							Owner: CanonicalUser{
-								ID:          fmt.Sprintf("%x", entry.Attributes.Uid),
-								DisplayName: entry.Attributes.UserName,
-							},
-							StorageClass: StorageClass(storageClass),
-						})
+						contents = append(contents, newListEntry(entry, dirName, entryName, bucketPrefix, fetchOwner, false))
 						cursor.maxKeys--
 					}
 				}
@@ -291,7 +273,9 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
 			Contents:       contents,
 			CommonPrefixes: commonPrefixes,
 		}
-
+		if encodingTypeUrl {
+			response.EncodingType = s3.EncodingTypeUrl
+		}
 		return nil
 	})
 
@@ -514,11 +498,12 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
 	return
 }
 
-func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimiter string, fetchOwner bool, maxkeys int) {
+func getListObjectsV2Args(values url.Values) (prefix, startAfter, delimiter string, token OptionalString, encodingTypeUrl bool, fetchOwner bool, maxkeys int) {
 	prefix = values.Get("prefix")
-	token = values.Get("continuation-token")
+	token = OptionalString{set: values.Has("continuation-token"), string: values.Get("continuation-token")}
 	startAfter = values.Get("start-after")
 	delimiter = values.Get("delimiter")
+	encodingTypeUrl = values.Get("encoding-type") == s3.EncodingTypeUrl
 	if values.Get("max-keys") != "" {
 		maxkeys, _ = strconv.Atoi(values.Get("max-keys"))
 	} else {
@@ -528,10 +513,11 @@ func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimit
 	return
 }
 
-func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string, maxkeys int) {
+func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string, encodingTypeUrl bool, maxkeys int) {
 	prefix = values.Get("prefix")
 	marker = values.Get("marker")
 	delimiter = values.Get("delimiter")
+	encodingTypeUrl = values.Get("encoding-type") == "url"
 	if values.Get("max-keys") != "" {
 		maxkeys, _ = strconv.Atoi(values.Get("max-keys"))
 	} else {

weed/s3api/s3api_xsd_generated.go

@@ -27,6 +27,7 @@ type BucketLoggingStatus struct {
 type CanonicalUser struct {
 	ID          string `xml:"ID"`
 	DisplayName string `xml:"DisplayName,omitempty"`
+	set         bool
 }
 
 type CopyObject struct {
@@ -62,6 +63,7 @@ func (t *CopyObject) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 	layout.Timestamp = (*xsdDateTime)(&layout.T.Timestamp)
 	return e.EncodeElement(layout, start)
 }
+
 func (t *CopyObject) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
 	type T CopyObject
 	var overlay struct {
@@ -96,6 +98,7 @@ func (t *CopyObjectResult) MarshalXML(e *xml.Encoder, start xml.StartElement) er
 	layout.LastModified = (*xsdDateTime)(&layout.T.LastModified)
 	return e.EncodeElement(layout, start)
 }
+
 func (t *CopyObjectResult) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
 	type T CopyObjectResult
 	var overlay struct {
@@ -125,6 +128,7 @@ func (t *CreateBucket) MarshalXML(e *xml.Encoder, start xml.StartElement) error
 	layout.Timestamp = (*xsdDateTime)(&layout.T.Timestamp)
 	return e.EncodeElement(layout, start)
 }
+
 func (t *CreateBucket) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
 	type T CreateBucket
 	var overlay struct {
@@ -591,6 +595,7 @@ type ListBucketResult struct {
 	NextMarker     string          `xml:"NextMarker,omitempty"`
 	MaxKeys        int             `xml:"MaxKeys"`
 	Delimiter      string          `xml:"Delimiter,omitempty"`
+	EncodingType   string          `xml:"EncodingType,omitempty"`
 	IsTruncated    bool            `xml:"IsTruncated"`
 	Contents       []ListEntry     `xml:"Contents,omitempty"`
 	CommonPrefixes []PrefixEntry   `xml:"CommonPrefixes,omitempty"`
@@ -615,6 +620,14 @@ func (t *ListEntry) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 	layout.LastModified = (*xsdDateTime)(&layout.T.LastModified)
 	return e.EncodeElement(layout, start)
 }
+func (c CanonicalUser) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
+	if !c.set {
+		return nil
+	}
+	type canonicalUserWrapper CanonicalUser
+	return e.EncodeElement(canonicalUserWrapper(c), start)
+}
+
 func (t *ListEntry) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
 	type T ListEntry
 	var overlay struct {
@@ -784,6 +797,7 @@ func (t *PutObjectResult) MarshalXML(e *xml.Encoder, start xml.StartElement) err
 	layout.LastModified = (*xsdDateTime)(&layout.T.LastModified)
 	return e.EncodeElement(layout, start)
 }
+
 func (t *PutObjectResult) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
 	type T PutObjectResult
 	var overlay struct {
@@ -822,6 +836,7 @@ func (t *SetBucketAccessControlPolicy) MarshalXML(e *xml.Encoder, start xml.Star
 	layout.Timestamp = (*xsdDateTime)(&layout.T.Timestamp)
 	return e.EncodeElement(layout, start)
 }
+
 func (t *SetBucketAccessControlPolicy) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
 	type T SetBucketAccessControlPolicy
 	var overlay struct {
@@ -855,6 +870,7 @@ func (t *SetBucketLoggingStatus) MarshalXML(e *xml.Encoder, start xml.StartEleme
 	layout.Timestamp = (*xsdDateTime)(&layout.T.Timestamp)
 	return e.EncodeElement(layout, start)
 }
+
 func (t *SetBucketLoggingStatus) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
 	type T SetBucketLoggingStatus
 	var overlay struct {
@@ -889,6 +905,7 @@ func (t *SetObjectAccessControlPolicy) MarshalXML(e *xml.Encoder, start xml.Star
 	layout.Timestamp = (*xsdDateTime)(&layout.T.Timestamp)
 	return e.EncodeElement(layout, start)
 }
+
 func (t *SetObjectAccessControlPolicy) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
 	type T SetObjectAccessControlPolicy
 	var overlay struct {
@@ -940,6 +957,7 @@ func (t *VersionEntry) MarshalXML(e *xml.Encoder, start xml.StartElement) error
 	layout.LastModified = (*xsdDateTime)(&layout.T.LastModified)
 	return e.EncodeElement(layout, start)
 }
+
 func (t *VersionEntry) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
 	type T VersionEntry
 	var overlay struct {
@@ -965,6 +983,7 @@ func (b *xsdBase64Binary) UnmarshalText(text []byte) (err error) {
 	*b, err = base64.StdEncoding.DecodeString(string(text))
 	return
 }
+
 func (b xsdBase64Binary) MarshalText() ([]byte, error) {
 	var buf bytes.Buffer
 	enc := base64.NewEncoder(base64.StdEncoding, &buf)
@@ -978,9 +997,11 @@ type xsdDateTime time.Time
 func (t *xsdDateTime) UnmarshalText(text []byte) error {
 	return _unmarshalTime(text, (*time.Time)(t), s3TimeFormat)
 }
+
 func (t xsdDateTime) MarshalText() ([]byte, error) {
 	return []byte((time.Time)(t).Format(s3TimeFormat)), nil
 }
+
 func (t xsdDateTime) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 	if (time.Time)(t).IsZero() {
 		return nil
@@ -991,6 +1012,7 @@ func (t xsdDateTime) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 	}
 	return e.EncodeElement(m, start)
 }
+
 func (t xsdDateTime) MarshalXMLAttr(name xml.Name) (xml.Attr, error) {
 	if (time.Time)(t).IsZero() {
 		return xml.Attr{}, nil
@@ -998,6 +1020,7 @@ func (t xsdDateTime) MarshalXMLAttr(name xml.Name) (xml.Attr, error) {
 	m, err := t.MarshalText()
 	return xml.Attr{Name: name, Value: string(m)}, err
 }
+
 func _unmarshalTime(text []byte, t *time.Time, format string) (err error) {
 	s := string(bytes.TrimSpace(text))
 	*t, err = time.Parse(format, s)

weed/server/filer_grpc_server.go

@@ -56,8 +56,9 @@ func (fs *FilerServer) ListEntries(req *filer_pb.ListEntriesRequest, stream file
 	var listErr error
 	for limit > 0 {
 		var hasEntries bool
-		lastFileName, listErr = fs.filer.StreamListDirectoryEntries(stream.Context(), util.FullPath(req.Directory), lastFileName, includeLastFile, int64(paginationLimit), req.Prefix, "", "", func(entry *filer.Entry) bool {
+		lastFileName, listErr = fs.filer.StreamListDirectoryEntries(stream.Context(), util.FullPath(req.Directory), lastFileName, includeLastFile, req.Recursive, int64(paginationLimit), req.Prefix, "", "", func(entry *filer.Entry) bool {
 			hasEntries = true
+			glog.V(5).Infof("StreamListDirectoryEntries recursive %v, entry: %+v", req.Recursive, entry)
 			if err = stream.Send(&filer_pb.ListEntriesResponse{
 				Entry: entry.ToProtoEntry(),
 			}); err != nil {

weed/server/filer_server.go

@@ -164,11 +164,14 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption)
 	if !util.LoadConfiguration("filer", false) {
 		v.SetDefault("leveldb2.enabled", true)
 		v.SetDefault("leveldb2.dir", option.DefaultLevelDbDir)
-		_, err := os.Stat(option.DefaultLevelDbDir)
-		if os.IsNotExist(err) {
-			os.MkdirAll(option.DefaultLevelDbDir, 0755)
+		if v.GetBool("leveldb2.enabled") {
+			levelDbDir := v.GetString("leveldb2.dir")
+			_, err := os.Stat(levelDbDir)
+			if os.IsNotExist(err) {
+				os.MkdirAll(levelDbDir, 0755)
+			}
+			glog.V(0).Infof("create filer store dir in %s", levelDbDir)
 		}
-		glog.V(0).Infof("default to create filer store dir in %s", option.DefaultLevelDbDir)
 	} else {
 		glog.Warningf("skipping default store dir in %s", option.DefaultLevelDbDir)
 	}