fix test_bucket_listv2_delimiter_prefi

x

weed/filer/abstract_sql/abstract_sql_store.go

@@ -341,13 +341,13 @@ func (store *AbstractSqlStore) ListRecursivePrefixedEntries(ctx context.Context,
 	}
 	shortDir := string(shortPath)
 	var dirPrefix string
-	if shortDir == "/" {
-		dirPrefix = fmt.Sprintf("/%s%%", prefix)
+	if shortDir == "/" || prefix == "" {
+		dirPrefix = fmt.Sprintf("%s%s%%", shortDir, prefix)
 	} else {
 		dirPrefix = fmt.Sprintf("%s/%s%%", shortDir, prefix)
 	}
 	glog.V(0).Infof("ListRecursivePrefixedEntries %s lastFileName %s shortPath %v, prefix %v, startFileName %s, limit %d, delimiter %v, dirPrefix %s", string(dirPath), lastFileName, string(shortPath), prefix, startFileName, limit, delimiter, dirPrefix)
-	rows, err := db.QueryContext(ctx, store.GetSqlListRecursive(bucket), startFileName, util.HashStringToLong(shortDir), prefix+"%", dirPrefix, limit+1)
+	rows, err := db.QueryContext(ctx, store.GetSqlListRecursive(bucket), startFileName, util.HashStringToLong(shortDir), prefix+"%", dirPrefix, limit+2)
 	if err != nil {
 		glog.Errorf("list %s : %v", dirPath, err)
 		return lastFileName, fmt.Errorf("list %s : %v", dirPath, err)
@@ -382,9 +382,15 @@ func (store *AbstractSqlStore) ListRecursivePrefixedEntries(ctx context.Context,
 			continue
 		}
 		// Todo test_bucket_listv2_delimiter_prefix move start from prefix to SQL because in extreme cases, where there are more keys that need to be skipped than the limit
-		if delimiter && shortDir != dir && (!entry.IsDirectory() || (len(startFileName) > 0 && strings.HasPrefix(dir, startFileName))) {
-			glog.V(0).Infof("scan isDir %v skip %v", entry.IsDirectory(), entry.FullPath)
-			continue
+		if delimiter {
+			if shortDir == fileName && !(entry.IsDirectory() && entry.Attr.Mime != "") {
+				// glog.V(0).Infof("scan is not DirKey %v skip %v", entry.IsDirectory(), entry.FullPath)
+				continue
+			}
+			if shortDir != dir && (!entry.IsDirectory() || (len(startFileName) > 0 && strings.HasPrefix(dir, startFileName))) {
+				// glog.V(0).Infof("scan isDir %v skip %v", entry.IsDirectory(), entry.FullPath)
+				continue
+			}
 		}
 		if !eachEntryFunc(entry) {
 			break

weed/filer/mysql/mysql_sql_gen.go

@@ -50,7 +50,7 @@ func (gen *SqlGenMysql) GetSqlListInclusive(tableName string) string {
 }
 
 func (gen *SqlGenMysql) GetSqlListRecursive(tableName string) string {
-	return fmt.Sprintf("SELECT `directory`, `name`, `meta` FROM `%s` WHERE `directory` || `name` > ? AND ((`dirhash` == ? AND `name` like ?) OR `directory` like ?) ORDER BY `directory` || `name` ASC LIMIT ?", tableName)
+	return fmt.Sprintf("SELECT `directory`, `name`, `meta` FROM `%s` WHERE `directory` || `name` > ? AND ((`dirhash` == ? AND `name` like ?) OR `directory` || `name` like ?) ORDER BY `directory` || `name` ASC LIMIT ?", tableName)
 }
 
 func (gen *SqlGenMysql) GetSqlCreateTable(tableName string) string {

weed/s3api/s3api_object_handlers_list.go

@@ -151,9 +151,6 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
 
 	if s3a.option.AllowListRecursive && (delimiter == "" || delimiter == "/") {
 		reqDir = bucketPrefix
-		if cursor.prefixEndsOnDelimiter && delimiter == "/" {
-			originalPrefix = originalPrefix[0 : len(originalPrefix)-1]
-		}
 		if idx := strings.LastIndex(originalPrefix, "/"); idx > 0 {
 			reqDir += originalPrefix[:idx]
 			prefix = originalPrefix[idx+1:]
@@ -193,21 +190,23 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
 							cursor.nextMarker = getStartFileFromKey(key)
 						}
 					}()
-					if cursor.prefixEndsOnDelimiter && originalPrefix == key && entry.IsDirectoryKeyObject() {
-						contents = append(contents, newListEntry(entry, key+"/", "", "", bucketPrefix, fetchOwner, true))
-						cursor.maxKeys--
-						cursor.prefixEndsOnDelimiter = false
-						return
-					}
-					if delimiter == "/" && entry.IsDirectory {
-						glog.V(0).Infof("append commonPrefixes %s", path[len(bucketPrefix):]+"/")
-						commonPrefixes = append(commonPrefixes, PrefixEntry{
-							Prefix: key + "/",
-						})
-						cursor.maxKeys--
-						return
+					if delimiter == "/" {
+						if entry.IsDirectoryKeyObject() {
+							// glog.V(0).Infof("append IsDirectoryKeyObject %s", key+"/")
+							contents = append(contents, newListEntry(entry, key+"/", "", "", bucketPrefix, fetchOwner, false))
+							cursor.maxKeys--
+							return
+						}
+						if entry.IsDirectory {
+							// glog.V(0).Infof("append commonPrefixes %s", key+"/")
+							commonPrefixes = append(commonPrefixes, PrefixEntry{
+								Prefix: key + "/",
+							})
+							cursor.maxKeys--
+							return
+						}
 					}
-					contents = append(contents, newListEntry(entry, key, "", "", bucketPrefix, fetchOwner, entry.IsDirectoryKeyObject()))
+					contents = append(contents, newListEntry(entry, key, "", "", bucketPrefix, fetchOwner, false))
 					cursor.maxKeys--
 				},
 			)