s3: warm bucket config cache on startup for multi-filer consistency (#7772)

* s3: warm bucket config cache on startup for multi-filer consistency

In multi-filer clusters, the bucket configuration cache (storing Object Lock,
versioning, and other settings) was not being pre-populated on S3 API server
startup. This caused issues where:

1. After server restart, Object Lock and versioning settings appeared lost
   until the bucket was accessed (lazy loading)
2. In multi-filer clusters, race conditions during bucket creation could
   result in inconsistent Object Lock configuration

This fix warms the bucketConfigCache during BucketRegistry initialization,
ensuring all bucket configurations (including Object Lock and versioning)
are immediately available after restart without waiting for first access.

The fix piggybacks on the existing BucketRegistry.init() which already
iterates through all buckets, adding a call to update the config cache
with each bucket's extended attributes.

* s3: add visibility logging for bucket config cache warming

- Add bucket count tracking during initialization
- Log error if bucket listing fails
- Log INFO message with count of warmed buckets on successful init

This improves observability for the cache warming process and addresses
review feedback about error handling visibility.

* s3: fix bucket deletion not invalidating config cache

Bug fix: The metadata subscription handler had an early return when
NewEntry was nil, which skipped the onBucketMetadataChange call for
bucket deletions. This caused deleted buckets to remain in the config
cache.

The fix moves onBucketMetadataChange before the nil check so it's
called for all events (create, update, delete). The IAM and circuit
breaker updates still require NewEntry content, so they remain after
the check.

* s3: handle config file deletions for IAM and circuit breaker

Refactored the metadata subscription handlers to properly handle all
event types (create, update, delete) for IAM and circuit breaker configs:

- Renamed onIamConfigUpdate -> onIamConfigChange
- Renamed onCircuitBreakerConfigUpdate -> onCircuitBreakerConfigChange
- Both handlers now check for deletions (newEntry == nil && oldEntry != nil)
- On config file deletion, reset to empty config by loading empty bytes
- Simplified processEventFn to call all handlers unconditionally
- Each handler checks for nil entries internally

This ensures that deleting identity.json or circuit_breaker.json will
clear the in-memory config rather than leaving stale data.

* s3: restore NewParentPath handling for rename/move operations

The directory resolution logic was accidentally removed. This restores
the check for NewParentPath which is needed when files are renamed or
moved - in such cases, NewParentPath contains the destination directory
which should be used for directory matching in the handlers.

weed/s3api/auth_credentials_subscribe.go

@@ -17,21 +17,18 @@ func (s3a *S3ApiServer) subscribeMetaEvents(clientName string, lastTsNs int64, p
 	processEventFn := func(resp *filer_pb.SubscribeMetadataResponse) error {
 
 		message := resp.EventNotification
-		if message.NewEntry == nil {
-			return nil
-		}
 
+		// For rename/move operations, NewParentPath contains the destination directory
 		dir := resp.Directory
-
 		if message.NewParentPath != "" {
 			dir = message.NewParentPath
 		}
-		fileName := message.NewEntry.Name
-		content := message.NewEntry.Content
 
-		_ = s3a.onIamConfigUpdate(dir, fileName, content)
-		_ = s3a.onCircuitBreakerConfigUpdate(dir, fileName, content)
+		// Handle all metadata changes (create, update, delete, rename)
+		// These handlers check for nil entries internally
 		_ = s3a.onBucketMetadataChange(dir, message.OldEntry, message.NewEntry)
+		_ = s3a.onIamConfigChange(dir, message.OldEntry, message.NewEntry)
+		_ = s3a.onCircuitBreakerConfigChange(dir, message.OldEntry, message.NewEntry)
 
 		return nil
 	}
@@ -57,24 +54,54 @@ func (s3a *S3ApiServer) subscribeMetaEvents(clientName string, lastTsNs int64, p
 	})
 }
 
-// reload iam config
-func (s3a *S3ApiServer) onIamConfigUpdate(dir, filename string, content []byte) error {
-	if dir == filer.IamConfigDirectory && filename == filer.IamIdentityFile {
-		if err := s3a.iam.LoadS3ApiConfigurationFromBytes(content); err != nil {
+// onIamConfigChange handles IAM config file changes (create, update, delete)
+func (s3a *S3ApiServer) onIamConfigChange(dir string, oldEntry *filer_pb.Entry, newEntry *filer_pb.Entry) error {
+	if dir != filer.IamConfigDirectory {
+		return nil
+	}
+
+	// Handle deletion: reset to empty config
+	if newEntry == nil && oldEntry != nil && oldEntry.Name == filer.IamIdentityFile {
+		glog.V(0).Infof("IAM config file deleted, clearing identities")
+		if err := s3a.iam.LoadS3ApiConfigurationFromBytes([]byte{}); err != nil {
+			glog.Warningf("failed to clear IAM config on deletion: %v", err)
+			return err
+		}
+		return nil
+	}
+
+	// Handle create/update
+	if newEntry != nil && newEntry.Name == filer.IamIdentityFile {
+		if err := s3a.iam.LoadS3ApiConfigurationFromBytes(newEntry.Content); err != nil {
 			return err
 		}
-		glog.V(1).Infof("updated %s/%s", dir, filename)
+		glog.V(1).Infof("updated %s/%s", dir, newEntry.Name)
 	}
 	return nil
 }
 
-// reload circuit breaker config
-func (s3a *S3ApiServer) onCircuitBreakerConfigUpdate(dir, filename string, content []byte) error {
-	if dir == s3_constants.CircuitBreakerConfigDir && filename == s3_constants.CircuitBreakerConfigFile {
-		if err := s3a.cb.LoadS3ApiConfigurationFromBytes(content); err != nil {
+// onCircuitBreakerConfigChange handles circuit breaker config file changes (create, update, delete)
+func (s3a *S3ApiServer) onCircuitBreakerConfigChange(dir string, oldEntry *filer_pb.Entry, newEntry *filer_pb.Entry) error {
+	if dir != s3_constants.CircuitBreakerConfigDir {
+		return nil
+	}
+
+	// Handle deletion: reset to empty config
+	if newEntry == nil && oldEntry != nil && oldEntry.Name == s3_constants.CircuitBreakerConfigFile {
+		glog.V(0).Infof("Circuit breaker config file deleted, resetting to defaults")
+		if err := s3a.cb.LoadS3ApiConfigurationFromBytes([]byte{}); err != nil {
+			glog.Warningf("failed to reset circuit breaker config on deletion: %v", err)
+			return err
+		}
+		return nil
+	}
+
+	// Handle create/update
+	if newEntry != nil && newEntry.Name == s3_constants.CircuitBreakerConfigFile {
+		if err := s3a.cb.LoadS3ApiConfigurationFromBytes(newEntry.Content); err != nil {
 			return err
 		}
-		glog.V(1).Infof("updated %s/%s", dir, filename)
+		glog.V(1).Infof("updated %s/%s", dir, newEntry.Name)
 	}
 	return nil
 }

weed/s3api/bucket_metadata.go

@@ -65,11 +65,21 @@ func NewBucketRegistry(s3a *S3ApiServer) *BucketRegistry {
 }
 
 func (r *BucketRegistry) init() error {
+	var bucketCount int
 	err := filer_pb.List(context.Background(), r.s3a, r.s3a.option.BucketsPath, "", func(entry *filer_pb.Entry, isLast bool) error {
 		r.LoadBucketMetadata(entry)
+		// Also warm the bucket config cache with Object Lock and versioning settings
+		// This ensures cache consistency across multi-filer clusters after restart
+		r.s3a.updateBucketConfigCacheFromEntry(entry)
+		bucketCount++
 		return nil
 	}, "", false, math.MaxUint32)
-	return err
+	if err != nil {
+		glog.Errorf("BucketRegistry.init: failed to list buckets: %v", err)
+		return err
+	}
+	glog.V(0).Infof("BucketRegistry.init: warmed config cache for %d buckets", bucketCount)
+	return nil
 }
 
 func (r *BucketRegistry) LoadBucketMetadata(entry *filer_pb.Entry) {