diff --git a/test/foundationdb/foundationdb_concurrent_test.go b/test/foundationdb/foundationdb_concurrent_test.go index b0ecaf742..de49ecc61 100644 --- a/test/foundationdb/foundationdb_concurrent_test.go +++ b/test/foundationdb/foundationdb_concurrent_test.go @@ -65,9 +65,9 @@ func TestFoundationDBStore_ConcurrentInserts(t *testing.T) { expectedTotal := numGoroutines * entriesPerGoroutine actualCount := 0 - _, err := store.ListDirectoryEntries(ctx, "/concurrent", "", true, 10000, func(entry *filer.Entry) bool { + _, err := store.ListDirectoryEntries(ctx, "/concurrent", "", true, 10000, func(entry *filer.Entry) (bool, error) { actualCount++ - return true + return true, nil }) if err != nil { t.Fatalf("ListDirectoryEntries failed: %v", err) @@ -265,9 +265,9 @@ func TestFoundationDBStore_ConcurrentTransactions(t *testing.T) { totalExpectedEntries := successCount * entriesPerTransaction actualCount := 0 - _, err := store.ListDirectoryEntries(ctx, "/transactions", "", true, 10000, func(entry *filer.Entry) bool { + _, err := store.ListDirectoryEntries(ctx, "/transactions", "", true, 10000, func(entry *filer.Entry) (bool, error) { actualCount++ - return true + return true, nil }) if err != nil { t.Fatalf("ListDirectoryEntries failed: %v", err) @@ -335,9 +335,9 @@ func TestFoundationDBStore_ConcurrentDirectoryOperations(t *testing.T) { dirPath := fmt.Sprintf("/worker%d/dir%d", w, d) fileCount := 0 - _, err := store.ListDirectoryEntries(ctx, dirPath, "", true, 1000, func(entry *filer.Entry) bool { + _, err := store.ListDirectoryEntries(ctx, dirPath, "", true, 1000, func(entry *filer.Entry) (bool, error) { fileCount++ - return true + return true, nil }) if err != nil { t.Errorf("ListDirectoryEntries failed for %s: %v", dirPath, err) diff --git a/test/foundationdb/foundationdb_integration_test.go b/test/foundationdb/foundationdb_integration_test.go index 5fdf993d7..63ed41ef9 100644 --- a/test/foundationdb/foundationdb_integration_test.go +++ b/test/foundationdb/foundationdb_integration_test.go @@ -115,9 +115,9 @@ func TestFoundationDBStore_DirectoryOperations(t *testing.T) { // Test ListDirectoryEntries var listedFiles []string - lastFileName, err := store.ListDirectoryEntries(ctx, testDir, "", true, 100, func(entry *filer.Entry) bool { + lastFileName, err := store.ListDirectoryEntries(ctx, testDir, "", true, 100, func(entry *filer.Entry) (bool, error) { listedFiles = append(listedFiles, entry.Name()) - return true + return true, nil }) if err != nil { t.Fatalf("ListDirectoryEntries failed: %v", err) @@ -132,9 +132,9 @@ func TestFoundationDBStore_DirectoryOperations(t *testing.T) { // Test ListDirectoryPrefixedEntries var prefixedFiles []string - _, err = store.ListDirectoryPrefixedEntries(ctx, testDir, "", true, 100, "file", func(entry *filer.Entry) bool { + _, err = store.ListDirectoryPrefixedEntries(ctx, testDir, "", true, 100, "file", func(entry *filer.Entry) (bool, error) { prefixedFiles = append(prefixedFiles, entry.Name()) - return true + return true, nil }) if err != nil { t.Fatalf("ListDirectoryPrefixedEntries failed: %v", err) @@ -153,9 +153,9 @@ func TestFoundationDBStore_DirectoryOperations(t *testing.T) { // Verify children are deleted var remainingFiles []string - _, err = store.ListDirectoryEntries(ctx, testDir, "", true, 100, func(entry *filer.Entry) bool { + _, err = store.ListDirectoryEntries(ctx, testDir, "", true, 100, func(entry *filer.Entry) (bool, error) { remainingFiles = append(remainingFiles, entry.Name()) - return true + return true, nil }) if err != nil { t.Fatalf("ListDirectoryEntries after delete failed: %v", err) diff --git a/test/foundationdb/mock_integration_test.go b/test/foundationdb/mock_integration_test.go index 5073ba5b3..fd1893eba 100644 --- a/test/foundationdb/mock_integration_test.go +++ b/test/foundationdb/mock_integration_test.go @@ -390,9 +390,9 @@ func TestMockFoundationDBStore_DirectoryOperations(t *testing.T) { // Test ListDirectoryEntries var listedFiles []string - lastFileName, err := store.ListDirectoryEntries(ctx, testDir, "", true, 100, func(entry *filer.Entry) bool { + lastFileName, err := store.ListDirectoryEntries(ctx, testDir, "", true, 100, func(entry *filer.Entry) (bool, error) { listedFiles = append(listedFiles, entry.Name()) - return true + return true, nil }) if err != nil { t.Fatalf("ListDirectoryEntries failed: %v", err) @@ -409,9 +409,9 @@ func TestMockFoundationDBStore_DirectoryOperations(t *testing.T) { // Verify children are deleted var remainingFiles []string - _, err = store.ListDirectoryEntries(ctx, testDir, "", true, 100, func(entry *filer.Entry) bool { + _, err = store.ListDirectoryEntries(ctx, testDir, "", true, 100, func(entry *filer.Entry) (bool, error) { remainingFiles = append(remainingFiles, entry.Name()) - return true + return true, nil }) if err != nil { t.Fatalf("ListDirectoryEntries after delete failed: %v", err) diff --git a/weed/filer/foundationdb/foundationdb_store.go b/weed/filer/foundationdb/foundationdb_store.go index 509ee4b86..f96574107 100644 --- a/weed/filer/foundationdb/foundationdb_store.go +++ b/weed/filer/foundationdb/foundationdb_store.go @@ -318,12 +318,12 @@ func (store *FoundationDBStore) deleteFolderChildrenInBatches(ctx context.Contex var subDirectories []util.FullPath // List entries - we'll process BATCH_SIZE at a time - _, err := store.ListDirectoryEntries(ctxNoTxn, fullpath, "", true, int64(BATCH_SIZE), func(entry *filer.Entry) bool { + _, err := store.ListDirectoryEntries(ctxNoTxn, fullpath, "", true, int64(BATCH_SIZE), func(entry *filer.Entry) (bool, error) { entriesToDelete = append(entriesToDelete, entry.FullPath) if entry.IsDirectory() { subDirectories = append(subDirectories, entry.FullPath) } - return true + return true, nil }) if err != nil { diff --git a/weed/filer/foundationdb/foundationdb_store_test.go b/weed/filer/foundationdb/foundationdb_store_test.go index 215c98c76..73255d67d 100644 --- a/weed/filer/foundationdb/foundationdb_store_test.go +++ b/weed/filer/foundationdb/foundationdb_store_test.go @@ -372,16 +372,16 @@ func containsString(s, substr string) bool { func TestFoundationDBStore_DeleteFolderChildrenWithBatching(t *testing.T) { // This test validates that DeleteFolderChildren always uses batching // to safely handle large directories, regardless of transaction context - + store := getTestStore(t) defer store.Shutdown() - + ctx := context.Background() testDir := util.FullPath(fmt.Sprintf("/test_batch_delete_%d", time.Now().UnixNano())) - + // Create a large directory (> 100 entries to trigger batching) const NUM_ENTRIES = 250 - + t.Logf("Creating %d test entries...", NUM_ENTRIES) for i := 0; i < NUM_ENTRIES; i++ { entry := &filer.Entry{ @@ -397,11 +397,11 @@ func TestFoundationDBStore_DeleteFolderChildrenWithBatching(t *testing.T) { t.Fatalf("Failed to insert test entry %d: %v", i, err) } } - + // Test 1: DeleteFolderChildren outside transaction should succeed t.Run("OutsideTransaction", func(t *testing.T) { testDir1 := util.FullPath(fmt.Sprintf("/test_batch_1_%d", time.Now().UnixNano())) - + // Create entries for i := 0; i < NUM_ENTRIES; i++ { entry := &filer.Entry{ @@ -415,28 +415,28 @@ func TestFoundationDBStore_DeleteFolderChildrenWithBatching(t *testing.T) { } store.InsertEntry(ctx, entry) } - + // Delete with batching err := store.DeleteFolderChildren(ctx, testDir1) if err != nil { t.Errorf("DeleteFolderChildren outside transaction should succeed, got error: %v", err) } - + // Verify all entries deleted var count int - store.ListDirectoryEntries(ctx, testDir1, "", true, 1000, func(entry *filer.Entry) bool { + store.ListDirectoryEntries(ctx, testDir1, "", true, 1000, func(entry *filer.Entry) (bool, error) { count++ - return true + return true, nil }) if count != 0 { t.Errorf("Expected all entries to be deleted, found %d", count) } }) - + // Test 2: DeleteFolderChildren with transaction context - uses its own batched transactions t.Run("WithTransactionContext", func(t *testing.T) { testDir2 := util.FullPath(fmt.Sprintf("/test_batch_2_%d", time.Now().UnixNano())) - + // Create entries for i := 0; i < NUM_ENTRIES; i++ { entry := &filer.Entry{ @@ -450,38 +450,38 @@ func TestFoundationDBStore_DeleteFolderChildrenWithBatching(t *testing.T) { } store.InsertEntry(ctx, entry) } - + // Start a transaction (DeleteFolderChildren will ignore it and use its own batching) txCtx, err := store.BeginTransaction(ctx) if err != nil { t.Fatalf("BeginTransaction failed: %v", err) } - + // Delete large directory - should succeed with batching err = store.DeleteFolderChildren(txCtx, testDir2) if err != nil { t.Errorf("DeleteFolderChildren should succeed with batching even when transaction context present, got: %v", err) } - + // Rollback transaction (DeleteFolderChildren used its own transactions, so this doesn't affect deletions) store.RollbackTransaction(txCtx) - + // Verify entries are still deleted (because DeleteFolderChildren managed its own transactions) var count int - store.ListDirectoryEntries(ctx, testDir2, "", true, 1000, func(entry *filer.Entry) bool { + store.ListDirectoryEntries(ctx, testDir2, "", true, 1000, func(entry *filer.Entry) (bool, error) { count++ - return true + return true, nil }) - + if count != 0 { t.Errorf("Expected all entries to be deleted, found %d (DeleteFolderChildren uses its own transactions)", count) } }) - + // Test 3: Nested directories with batching t.Run("NestedDirectories", func(t *testing.T) { testDir3 := util.FullPath(fmt.Sprintf("/test_batch_3_%d", time.Now().UnixNano())) - + // Create nested structure for i := 0; i < 50; i++ { // Files in root @@ -495,7 +495,7 @@ func TestFoundationDBStore_DeleteFolderChildrenWithBatching(t *testing.T) { }, } store.InsertEntry(ctx, entry) - + // Subdirectory subDir := &filer.Entry{ FullPath: util.NewFullPath(string(testDir3), fmt.Sprintf("dir_%02d", i)), @@ -507,7 +507,7 @@ func TestFoundationDBStore_DeleteFolderChildrenWithBatching(t *testing.T) { }, } store.InsertEntry(ctx, subDir) - + // Files in subdirectory for j := 0; j < 3; j++ { subEntry := &filer.Entry{ @@ -522,24 +522,24 @@ func TestFoundationDBStore_DeleteFolderChildrenWithBatching(t *testing.T) { store.InsertEntry(ctx, subEntry) } } - + // Delete all with batching err := store.DeleteFolderChildren(ctx, testDir3) if err != nil { t.Errorf("DeleteFolderChildren should handle nested directories, got: %v", err) } - + // Verify all deleted var count int - store.ListDirectoryEntries(ctx, testDir3, "", true, 1000, func(entry *filer.Entry) bool { + store.ListDirectoryEntries(ctx, testDir3, "", true, 1000, func(entry *filer.Entry) (bool, error) { count++ - return true + return true, nil }) if count != 0 { t.Errorf("Expected all nested entries to be deleted, found %d", count) } }) - + // Cleanup store.DeleteFolderChildren(ctx, testDir) }