add sql list tests

8 months ago · a7b64c1fab
7 changed files with 757 additions and 645 deletions
--- a/.github/workflows/s3tests.yml
+++ b/.github/workflows/s3tests.yml
@ -34,7 +34,7 @@ jobs:
          go-version-file: 'go.mod'
        id: go

-      - name: Run Ceph S3 tests
+      - name: Run Ceph S3 tests with KV store
        timeout-minutes: 15
        env:
          S3TEST_CONF: /__w/seaweedfs/seaweedfs/docker/compose/s3tests.conf
@ -202,3 +202,81 @@ jobs:
          s3tests_boto3/functional/test_s3.py::test_lifecycle_set \
          s3tests_boto3/functional/test_s3.py::test_lifecycle_get \
          s3tests_boto3/functional/test_s3.py::test_lifecycle_set_filter
+
+      - name: Run Ceph S3 tests with SQL store
+        timeout-minutes: 15
+        env:
+          S3TEST_CONF: /__w/seaweedfs/seaweedfs/docker/compose/s3tests.conf
+        shell: bash
+        run: |
+          cd /__w/seaweedfs/seaweedfs/weed
+          go install -tags "sqlite" -buildvcs=false
+          export WEED_LEVELDB2_ENABLED="false" WEED_SQLITE_ENABLED="true" WEED_SQLITE_DBFILE="./filer.db"
+          set -x
+          nohup weed -v 0 server -filer -filer.maxMB=64 -s3 -ip.bind 0.0.0.0 -s3.allowListRecursive=true \
+            -master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=1024 \
+            -volume.max=100 -volume.preStopSeconds=1 -s3.port=8000 -metricsPort=9324 \
+            -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config=../docker/compose/s3.json &
+          sleep 10          
+          cd /s3-tests
+          tox -- \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_empty \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_distinct \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_many \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_many \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_basic \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_encoding_basic \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_prefix \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_prefix_ends_with_delimiter \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_alt \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_prefix_underscore \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_percentage \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_whitespace \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_dot \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_unreadable \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_empty \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_none \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_not_exist \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_delimiter_basic \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_basic \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_delimiter_alt \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_alt \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_delimiter_prefix_not_exist \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_prefix_not_exist \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_delimiter_delimiter_not_exist \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_delimiter_not_exist \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_delimiter_prefix_delimiter_not_exist \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_fetchowner_notempty \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_fetchowner_defaultempty \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_basic \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_basic \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_alt \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_alt \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_empty \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_empty \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_none \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_none \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_not_exist \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_not_exist \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_unreadable \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_unreadable \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_maxkeys_one \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_maxkeys_one \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_maxkeys_zero \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_maxkeys_zero \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_marker_none \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_marker_empty \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_continuationtoken_empty \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_continuationtoken \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_both_continuationtoken_startafter \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_marker_unreadable \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_startafter_unreadable \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_marker_not_in_list \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_startafter_not_in_list \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_marker_after_list \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_startafter_after_list \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_objects_anonymous_fail \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_objects_anonymous_fail \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_long_name \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_special_prefix
--- a/other/java/client/src/main/proto/filer.proto
+++ b/other/java/client/src/main/proto/filer.proto
@ -102,6 +102,7 @@ message ListEntriesRequest {

 message ListEntriesResponse {
    Entry entry = 1;
+    string dir = 2;
 }

 message RemoteEntry {
--- a/weed/filer/abstract_sql/abstract_sql_store.go
+++ b/weed/filer/abstract_sql/abstract_sql_store.go
@ -354,10 +354,10 @@ func (store *AbstractSqlStore) ListRecursivePrefixedEntries(ctx context.Context,
 		}
 		glog.V(0).Infof("scan dir %s name %v", dir, name)

+		lastFileName = fmt.Sprintf("%s/%s", dir, name)
 		entry := &filer.Entry{
-			FullPath: util.NewFullPath(dir, name),
+			FullPath: util.NewFullPath(string(dirPath), lastFileName),
 		}
-		lastFileName = string(entry.FullPath)

 		if err = entry.DecodeAttributesAndChunks(util.MaybeDecompressData(data)); err != nil {
 			glog.V(0).Infof("scan decode %s : %v", entry.FullPath, err)
--- a/weed/pb/filer.proto
+++ b/weed/pb/filer.proto
@ -102,6 +102,7 @@ message ListEntriesRequest {

 message ListEntriesResponse {
    Entry entry = 1;
+    string dir = 2;
 }

 message RemoteEntry {
--- a/weed/pb/filer_pb/filer.pb.go
+++ b/weed/pb/filer_pb/filer.pb.go
--- a/weed/s3api/s3api_object_handlers_list.go
+++ b/weed/s3api/s3api_object_handlers_list.go
@ -151,22 +151,33 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
 	}

 	// Todo remove force disable
-	if s3a.option.AllowListRecursive && prefix == "force_disable" {
+	if s3a.option.AllowListRecursive && prefix != "" && (delimiter == "" || delimiter == "/") {
 		err = s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
 			glog.V(0).Infof("doListFilerRecursiveEntries reqDir: %s, prefix: %s, delimiter: %s, cursor: %+v", reqDir, prefix, delimiter, cursor)
 			nextMarker, doErr = s3a.doListFilerRecursiveEntries(client, reqDir, prefix, cursor, marker, delimiter, false,
 				func(dir string, entry *filer_pb.Entry) {
+					glog.V(5).Infof("doListFilerRecursiveEntries dir %s, shortDir %s, entry: %+v, cursor: %+v", dir, dir[len(bucketPrefix):], entry, cursor)
+					if cursor.isTruncated {
+						return
+					}
 					dirName, entryName, prefixName := entryUrlEncode(dir, entry.Name, encodingTypeUrl)
-					if entry.IsDirectory {
-						if delimiter == "/" { // A response can contain CommonPrefixes only if you specify a delimiter.
+					isCommonDir := strings.Index(dir[len(bucketPrefix):], "/") != -1
+					if cursor.prefixEndsOnDelimiter && !isCommonDir && entry.Name == prefix {
+						return
+					}
+					if delimiter == "/" {
+						if entry.IsDirectory {
 							commonPrefixes = append(commonPrefixes, PrefixEntry{
 								Prefix: fmt.Sprintf("%s/%s/", dirName, prefixName)[len(bucketPrefix):],
 							})
+							cursor.Decrease()
+							return
+						} else if isCommonDir {
+							return
 						}
-						return
 					}
 					contents = append(contents, newListEntry(entry, dirName, entryName, bucketPrefix, fetchOwner, entry.IsDirectoryKeyObject()))
-					cursor.maxKeys--
+					cursor.Decrease()
 				},
 			)
 			return nil
@ -293,6 +304,13 @@ type ListingCursor struct {
 	prefixEndsOnDelimiter bool
 }

+func (l *ListingCursor) Decrease() {
+	l.maxKeys--
+	if l.maxKeys == 0 {
+		l.isTruncated = true
+	}
+}
+
 // the prefix and marker may be in different directories
 // normalizePrefixMarker ensures the prefix and marker both starts from the same directory
 func normalizePrefixMarker(prefix, marker string) (alignedDir, alignedPrefix, alignedMarker string) {
@ -358,11 +376,14 @@ func (s3a *S3ApiServer) doListFilerRecursiveEntries(client filer_pb.SeaweedFiler
 	request := &filer_pb.ListEntriesRequest{
 		Directory:          dir,
 		Prefix:             prefix,
-		Limit:              uint32(cursor.maxKeys + 2),
+		Limit:              uint32(cursor.maxKeys),
 		StartFromFileName:  marker,
 		InclusiveStartFrom: inclusiveStartFrom,
 		Recursive:          true,
 	}
+	if cursor.prefixEndsOnDelimiter {
+		request.Limit += 1
+	}
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 	stream, listErr := client.ListEntries(ctx, request)
@ -378,7 +399,7 @@ func (s3a *S3ApiServer) doListFilerRecursiveEntries(client filer_pb.SeaweedFiler
 				return "", fmt.Errorf("iterating entires %+v: %v", request, recvErr)
 			}
 		}
-		eachEntryFn(dir, resp.Entry)
+		eachEntryFn(resp.Dir, resp.Entry)
 	}
 	return
 }
--- a/weed/server/filer_grpc_server.go
+++ b/weed/server/filer_grpc_server.go
@ -59,8 +59,10 @@ func (fs *FilerServer) ListEntries(req *filer_pb.ListEntriesRequest, stream file
 		lastFileName, listErr = fs.filer.StreamListDirectoryEntries(stream.Context(), util.FullPath(req.Directory), lastFileName, includeLastFile, req.Recursive, int64(paginationLimit), req.Prefix, "", "", func(entry *filer.Entry) bool {
 			hasEntries = true
 			glog.V(5).Infof("StreamListDirectoryEntries recursive %v, entry: %+v", req.Recursive, entry)
+			dir, _ := entry.FullPath.DirAndName()
 			if err = stream.Send(&filer_pb.ListEntriesResponse{
 				Entry: entry.ToProtoEntry(),
+				Dir:   dir,
 			}); err != nil {
 				return false
 			}