Browse Source

Fix s3 versioning listing bugs (#7705)

* fix: add pagination to list-object-versions for buckets with >1000 objects

The findVersionsRecursively() function used a fixed limit of 1000 entries
without pagination. This caused objects beyond the first 1000 entries
(sorted alphabetically) to never appear in list-object-versions responses.

Changes:
- Add pagination loop using filer.PaginationSize (1024)
- Use isLast flag from s3a.list() to detect end of pagination
- Track startFrom marker for each page

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

* fix: prevent infinite loop in ListObjects when processing .versions directories

The doListFilerEntries() function processes .versions directories in a
secondary loop after the main entry loop, but failed to update nextMarker.
This caused infinite pagination loops when results were truncated, as the
same .versions directories would be reprocessed on each page.

Bug introduced by: c196d03951
("fix listing object versions (#7006)")

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
pull/7709/head
jfburdet 6 days ago
committed by GitHub
parent
commit
27a28faf49
No known key found for this signature in database GPG Key ID: B5690EEEBB952194
  1. 4
      weed/s3api/s3api_object_handlers_list.go
  2. 15
      weed/s3api/s3api_object_versioning.go

4
weed/s3api/s3api_object_handlers_list.go

@ -573,6 +573,10 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
break
}
// Update nextMarker to ensure pagination advances past this .versions directory
// This is critical to prevent infinite loops when results are truncated
nextMarker = versionsDir
// Extract object name from .versions directory name (remove .versions suffix)
baseObjectName := strings.TrimSuffix(versionsDir, s3_constants.VersionsFolder)

15
weed/s3api/s3api_object_versioning.go

@ -264,13 +264,18 @@ func (s3a *S3ApiServer) listObjectVersions(bucket, prefix, keyMarker, versionIdM
// findVersionsRecursively searches for all .versions directories and regular files recursively
func (s3a *S3ApiServer) findVersionsRecursively(currentPath, relativePath string, allVersions *[]interface{}, processedObjects map[string]bool, seenVersionIds map[string]bool, bucket, prefix string) error {
// List entries in current directory
entries, _, err := s3a.list(currentPath, "", "", false, 1000)
// List entries in current directory with pagination
startFrom := ""
for {
entries, isLast, err := s3a.list(currentPath, "", startFrom, false, filer.PaginationSize)
if err != nil {
return err
}
for _, entry := range entries {
// Track last entry name for pagination
startFrom = entry.Name
entryPath := path.Join(relativePath, entry.Name)
// Skip if this doesn't match the prefix filter
@ -486,6 +491,12 @@ func (s3a *S3ApiServer) findVersionsRecursively(currentPath, relativePath string
}
}
// If we've reached the last page, stop pagination
if isLast {
break
}
}
return nil
}

Loading…
Cancel
Save