From b0afb12b1656c3693d09de76db554c249aac2643 Mon Sep 17 00:00:00 2001 From: chrislu Date: Sun, 14 Sep 2025 22:40:12 -0700 Subject: [PATCH] s3: improve retry logic for filer consistency with exponential backoff - Increase retry attempts from 3 to 5 for both updateLatestVersionInDirectory and getLatestObjectVersion functions - Implement exponential backoff: 50ms, 100ms, 200ms, 400ms delays - Addresses persistent CI failures where .versions directories are not immediately visible after creation in filer store - Based on CI log analysis showing 50ms fixed delays were insufficient - Maintains CI debug logging to track improved retry behavior --- weed/s3api/s3api_object_handlers_put.go | 7 ++++--- weed/s3api/s3api_object_versioning.go | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/weed/s3api/s3api_object_handlers_put.go b/weed/s3api/s3api_object_handlers_put.go index c61b6f8a7..0bb9d2795 100644 --- a/weed/s3api/s3api_object_handlers_put.go +++ b/weed/s3api/s3api_object_handlers_put.go @@ -643,7 +643,7 @@ func (s3a *S3ApiServer) updateLatestVersionInDirectory(bucket, object, versionId // Get the current .versions directory entry with retry logic for filer consistency var versionsEntry *filer_pb.Entry var err error - maxRetries := 3 + maxRetries := 5 for attempt := 1; attempt <= maxRetries; attempt++ { versionsEntry, err = s3a.getEntry(bucketDir, versionsObjectPath) if err == nil { @@ -653,8 +653,9 @@ func (s3a *S3ApiServer) updateLatestVersionInDirectory(bucket, object, versionId glog.V(0).Infof("CI-DEBUG: updateLatestVersionInDirectory: attempt %d/%d failed to get .versions entry for %s/%s: %v", attempt, maxRetries, bucket, object, err) if attempt < maxRetries { - // Brief wait before retry to allow filer consistency - time.Sleep(time.Millisecond * 50) + // Exponential backoff: 50ms, 100ms, 200ms, 400ms + delay := time.Millisecond * time.Duration(50 * (1 << (attempt - 1))) + time.Sleep(delay) } } diff --git a/weed/s3api/s3api_object_versioning.go b/weed/s3api/s3api_object_versioning.go index d3ced793a..df8ada047 100644 --- a/weed/s3api/s3api_object_versioning.go +++ b/weed/s3api/s3api_object_versioning.go @@ -785,7 +785,7 @@ func (s3a *S3ApiServer) getLatestObjectVersion(bucket, object string) (*filer_pb // Get the .versions directory entry to read latest version metadata with retry logic for filer consistency var versionsEntry *filer_pb.Entry var err error - maxRetries := 3 + maxRetries := 5 for attempt := 1; attempt <= maxRetries; attempt++ { versionsEntry, err = s3a.getEntry(bucketDir, versionsObjectPath) if err == nil { @@ -795,8 +795,9 @@ func (s3a *S3ApiServer) getLatestObjectVersion(bucket, object string) (*filer_pb glog.V(0).Infof("CI-DEBUG: getLatestObjectVersion: attempt %d/%d failed to get .versions directory for %s/%s: %v", attempt, maxRetries, bucket, object, err) if attempt < maxRetries { - // Brief wait before retry to allow filer consistency - time.Sleep(time.Millisecond * 50) + // Exponential backoff: 50ms, 100ms, 200ms, 400ms + delay := time.Millisecond * time.Duration(50 * (1 << (attempt - 1))) + time.Sleep(delay) } }