Browse Source

s3: improve retry logic for filer consistency with exponential backoff

- Increase retry attempts from 3 to 5 for both updateLatestVersionInDirectory
  and getLatestObjectVersion functions
- Implement exponential backoff: 50ms, 100ms, 200ms, 400ms delays
- Addresses persistent CI failures where .versions directories are not
  immediately visible after creation in filer store
- Based on CI log analysis showing 50ms fixed delays were insufficient
- Maintains CI debug logging to track improved retry behavior
pull/7231/head
chrislu 2 months ago
parent
commit
b0afb12b16
  1. 7
      weed/s3api/s3api_object_handlers_put.go
  2. 7
      weed/s3api/s3api_object_versioning.go

7
weed/s3api/s3api_object_handlers_put.go

@ -643,7 +643,7 @@ func (s3a *S3ApiServer) updateLatestVersionInDirectory(bucket, object, versionId
// Get the current .versions directory entry with retry logic for filer consistency
var versionsEntry *filer_pb.Entry
var err error
maxRetries := 3
maxRetries := 5
for attempt := 1; attempt <= maxRetries; attempt++ {
versionsEntry, err = s3a.getEntry(bucketDir, versionsObjectPath)
if err == nil {
@ -653,8 +653,9 @@ func (s3a *S3ApiServer) updateLatestVersionInDirectory(bucket, object, versionId
glog.V(0).Infof("CI-DEBUG: updateLatestVersionInDirectory: attempt %d/%d failed to get .versions entry for %s/%s: %v", attempt, maxRetries, bucket, object, err)
if attempt < maxRetries {
// Brief wait before retry to allow filer consistency
time.Sleep(time.Millisecond * 50)
// Exponential backoff: 50ms, 100ms, 200ms, 400ms
delay := time.Millisecond * time.Duration(50 * (1 << (attempt - 1)))
time.Sleep(delay)
}
}

7
weed/s3api/s3api_object_versioning.go

@ -785,7 +785,7 @@ func (s3a *S3ApiServer) getLatestObjectVersion(bucket, object string) (*filer_pb
// Get the .versions directory entry to read latest version metadata with retry logic for filer consistency
var versionsEntry *filer_pb.Entry
var err error
maxRetries := 3
maxRetries := 5
for attempt := 1; attempt <= maxRetries; attempt++ {
versionsEntry, err = s3a.getEntry(bucketDir, versionsObjectPath)
if err == nil {
@ -795,8 +795,9 @@ func (s3a *S3ApiServer) getLatestObjectVersion(bucket, object string) (*filer_pb
glog.V(0).Infof("CI-DEBUG: getLatestObjectVersion: attempt %d/%d failed to get .versions directory for %s/%s: %v", attempt, maxRetries, bucket, object, err)
if attempt < maxRetries {
// Brief wait before retry to allow filer consistency
time.Sleep(time.Millisecond * 50)
// Exponential backoff: 50ms, 100ms, 200ms, 400ms
delay := time.Millisecond * time.Duration(50 * (1 << (attempt - 1)))
time.Sleep(delay)
}
}

Loading…
Cancel
Save