diff --git a/weed/filer/abstract_sql/abstract_sql_store.go b/weed/filer/abstract_sql/abstract_sql_store.go index 67f14f459..1f51e1030 100644 --- a/weed/filer/abstract_sql/abstract_sql_store.go +++ b/weed/filer/abstract_sql/abstract_sql_store.go @@ -341,13 +341,13 @@ func (store *AbstractSqlStore) ListRecursivePrefixedEntries(ctx context.Context, } shortDir := string(shortPath) var dirPrefix string - if shortDir == "/" { - dirPrefix = fmt.Sprintf("/%s%%", prefix) + if shortDir == "/" || prefix == "" { + dirPrefix = fmt.Sprintf("%s%s%%", shortDir, prefix) } else { dirPrefix = fmt.Sprintf("%s/%s%%", shortDir, prefix) } glog.V(0).Infof("ListRecursivePrefixedEntries %s lastFileName %s shortPath %v, prefix %v, startFileName %s, limit %d, delimiter %v, dirPrefix %s", string(dirPath), lastFileName, string(shortPath), prefix, startFileName, limit, delimiter, dirPrefix) - rows, err := db.QueryContext(ctx, store.GetSqlListRecursive(bucket), startFileName, util.HashStringToLong(shortDir), prefix+"%", dirPrefix, limit+1) + rows, err := db.QueryContext(ctx, store.GetSqlListRecursive(bucket), startFileName, util.HashStringToLong(shortDir), prefix+"%", dirPrefix, limit+2) if err != nil { glog.Errorf("list %s : %v", dirPath, err) return lastFileName, fmt.Errorf("list %s : %v", dirPath, err) @@ -382,9 +382,15 @@ func (store *AbstractSqlStore) ListRecursivePrefixedEntries(ctx context.Context, continue } // Todo test_bucket_listv2_delimiter_prefix move start from prefix to SQL because in extreme cases, where there are more keys that need to be skipped than the limit - if delimiter && shortDir != dir && (!entry.IsDirectory() || (len(startFileName) > 0 && strings.HasPrefix(dir, startFileName))) { - glog.V(0).Infof("scan isDir %v skip %v", entry.IsDirectory(), entry.FullPath) - continue + if delimiter { + if shortDir == fileName && !(entry.IsDirectory() && entry.Attr.Mime != "") { + // glog.V(0).Infof("scan is not DirKey %v skip %v", entry.IsDirectory(), entry.FullPath) + continue + } + if shortDir != dir && (!entry.IsDirectory() || (len(startFileName) > 0 && strings.HasPrefix(dir, startFileName))) { + // glog.V(0).Infof("scan isDir %v skip %v", entry.IsDirectory(), entry.FullPath) + continue + } } if !eachEntryFunc(entry) { break diff --git a/weed/filer/mysql/mysql_sql_gen.go b/weed/filer/mysql/mysql_sql_gen.go index db31df180..f242a1559 100644 --- a/weed/filer/mysql/mysql_sql_gen.go +++ b/weed/filer/mysql/mysql_sql_gen.go @@ -50,7 +50,7 @@ func (gen *SqlGenMysql) GetSqlListInclusive(tableName string) string { } func (gen *SqlGenMysql) GetSqlListRecursive(tableName string) string { - return fmt.Sprintf("SELECT `directory`, `name`, `meta` FROM `%s` WHERE `directory` || `name` > ? AND ((`dirhash` == ? AND `name` like ?) OR `directory` like ?) ORDER BY `directory` || `name` ASC LIMIT ?", tableName) + return fmt.Sprintf("SELECT `directory`, `name`, `meta` FROM `%s` WHERE `directory` || `name` > ? AND ((`dirhash` == ? AND `name` like ?) OR `directory` || `name` like ?) ORDER BY `directory` || `name` ASC LIMIT ?", tableName) } func (gen *SqlGenMysql) GetSqlCreateTable(tableName string) string { diff --git a/weed/s3api/s3api_object_handlers_list.go b/weed/s3api/s3api_object_handlers_list.go index 19d72887d..9fce40176 100644 --- a/weed/s3api/s3api_object_handlers_list.go +++ b/weed/s3api/s3api_object_handlers_list.go @@ -151,9 +151,6 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m if s3a.option.AllowListRecursive && (delimiter == "" || delimiter == "/") { reqDir = bucketPrefix - if cursor.prefixEndsOnDelimiter && delimiter == "/" { - originalPrefix = originalPrefix[0 : len(originalPrefix)-1] - } if idx := strings.LastIndex(originalPrefix, "/"); idx > 0 { reqDir += originalPrefix[:idx] prefix = originalPrefix[idx+1:] @@ -193,21 +190,23 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m cursor.nextMarker = getStartFileFromKey(key) } }() - if cursor.prefixEndsOnDelimiter && originalPrefix == key && entry.IsDirectoryKeyObject() { - contents = append(contents, newListEntry(entry, key+"/", "", "", bucketPrefix, fetchOwner, true)) - cursor.maxKeys-- - cursor.prefixEndsOnDelimiter = false - return - } - if delimiter == "/" && entry.IsDirectory { - glog.V(0).Infof("append commonPrefixes %s", path[len(bucketPrefix):]+"/") - commonPrefixes = append(commonPrefixes, PrefixEntry{ - Prefix: key + "/", - }) - cursor.maxKeys-- - return + if delimiter == "/" { + if entry.IsDirectoryKeyObject() { + // glog.V(0).Infof("append IsDirectoryKeyObject %s", key+"/") + contents = append(contents, newListEntry(entry, key+"/", "", "", bucketPrefix, fetchOwner, false)) + cursor.maxKeys-- + return + } + if entry.IsDirectory { + // glog.V(0).Infof("append commonPrefixes %s", key+"/") + commonPrefixes = append(commonPrefixes, PrefixEntry{ + Prefix: key + "/", + }) + cursor.maxKeys-- + return + } } - contents = append(contents, newListEntry(entry, key, "", "", bucketPrefix, fetchOwner, entry.IsDirectoryKeyObject())) + contents = append(contents, newListEntry(entry, key, "", "", bucketPrefix, fetchOwner, false)) cursor.maxKeys-- }, )