seaweedfs/test/fuse_integration/writeback_cache_test.go

package fuse_test

import (
	"bytes"
	"crypto/rand"
	"fmt"
	"os"
	"path/filepath"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

// writebackConfig returns a TestConfig with writebackCache enabled
func writebackConfig() *TestConfig {
	return &TestConfig{
		Collection:  "",
		Replication: "000",
		ChunkSizeMB: 2,
		CacheSizeMB: 100,
		NumVolumes:  3,
		EnableDebug: false,
		MountOptions: []string{
			"-writebackCache",
		},
		SkipCleanup: false,
	}
}

// waitForFileContent polls until a file has the expected content or timeout expires.
// This is needed because writebackCache defers data upload to background goroutines,
// so there is a brief window after close() where the file may not yet be readable.
func waitForFileContent(t *testing.T, path string, expected []byte, timeout time.Duration) {
	t.Helper()
	deadline := time.Now().Add(timeout)
	var lastErr error
	for time.Now().Before(deadline) {
		actual, err := os.ReadFile(path)
		if err == nil && bytes.Equal(expected, actual) {
			return
		}
		if err != nil {
			lastErr = err
		} else {
			lastErr = fmt.Errorf("content mismatch: got %d bytes, want %d bytes", len(actual), len(expected))
		}
		time.Sleep(200 * time.Millisecond)
	}
	t.Fatalf("file %s did not have expected content within %v: %v", path, timeout, lastErr)
}

// waitForFileSize polls until a file reports the expected size or timeout expires.
func waitForFileSize(t *testing.T, path string, expectedSize int64, timeout time.Duration) {
	t.Helper()
	deadline := time.Now().Add(timeout)
	for time.Now().Before(deadline) {
		info, err := os.Stat(path)
		if err == nil && info.Size() == expectedSize {
			return
		}
		time.Sleep(200 * time.Millisecond)
	}
	t.Fatalf("file %s did not reach expected size %d within %v", path, expectedSize, timeout)
}

// TestWritebackCacheBasicOperations tests fundamental file I/O with writebackCache enabled
func TestWritebackCacheBasicOperations(t *testing.T) {
	config := writebackConfig()
	framework := NewFuseTestFramework(t, config)
	defer framework.Cleanup()

	require.NoError(t, framework.Setup(config))

	t.Run("WriteAndReadBack", func(t *testing.T) {
		testWritebackWriteAndReadBack(t, framework)
	})

	t.Run("MultipleFilesSequential", func(t *testing.T) {
		testWritebackMultipleFilesSequential(t, framework)
	})

	t.Run("LargeFile", func(t *testing.T) {
		testWritebackLargeFile(t, framework)
	})

	t.Run("EmptyFile", func(t *testing.T) {
		testWritebackEmptyFile(t, framework)
	})

	t.Run("OverwriteExistingFile", func(t *testing.T) {
		testWritebackOverwriteFile(t, framework)
	})
}

// testWritebackWriteAndReadBack writes a file and verifies it can be read back
// after the async flush completes.
func testWritebackWriteAndReadBack(t *testing.T, framework *FuseTestFramework) {
	filename := "writeback_basic.txt"
	content := []byte("Hello from writebackCache test!")
	mountPath := filepath.Join(framework.GetMountPoint(), filename)

	// Write file — close() returns immediately with async flush
	require.NoError(t, os.WriteFile(mountPath, content, 0644))

	// Wait for async flush to complete and verify content
	waitForFileContent(t, mountPath, content, 30*time.Second)
}

// testWritebackMultipleFilesSequential writes multiple files sequentially
// and verifies all are readable after async flushes complete.
func testWritebackMultipleFilesSequential(t *testing.T, framework *FuseTestFramework) {
	dir := "writeback_sequential"
	framework.CreateTestDir(dir)

	numFiles := 50
	files := make(map[string][]byte, numFiles)

	// Write files sequentially — each close() returns immediately
	for i := 0; i < numFiles; i++ {
		filename := fmt.Sprintf("file_%03d.txt", i)
		content := []byte(fmt.Sprintf("Sequential file %d content: %s", i, time.Now().Format(time.RFC3339Nano)))
		path := filepath.Join(framework.GetMountPoint(), dir, filename)
		require.NoError(t, os.WriteFile(path, content, 0644))
		files[filename] = content
	}

	// Verify all files after a brief wait for async flushes
	for filename, expectedContent := range files {
		path := filepath.Join(framework.GetMountPoint(), dir, filename)
		waitForFileContent(t, path, expectedContent, 30*time.Second)
	}
}

// testWritebackLargeFile writes a large file (multi-chunk) with writebackCache
func testWritebackLargeFile(t *testing.T, framework *FuseTestFramework) {
	filename := "writeback_large.bin"
	mountPath := filepath.Join(framework.GetMountPoint(), filename)

	// 8MB file (spans multiple 2MB chunks)
	content := make([]byte, 8*1024*1024)
	_, err := rand.Read(content)
	require.NoError(t, err)

	require.NoError(t, os.WriteFile(mountPath, content, 0644))

	// Wait for file to be fully flushed
	waitForFileContent(t, mountPath, content, 60*time.Second)
}

// testWritebackEmptyFile creates an empty file with writebackCache
func testWritebackEmptyFile(t *testing.T, framework *FuseTestFramework) {
	filename := "writeback_empty.txt"
	mountPath := filepath.Join(framework.GetMountPoint(), filename)

	// Create empty file
	f, err := os.Create(mountPath)
	require.NoError(t, err)
	require.NoError(t, f.Close())

	// Should exist and be empty
	info, err := os.Stat(mountPath)
	require.NoError(t, err)
	assert.Equal(t, int64(0), info.Size())
}

// testWritebackOverwriteFile tests overwriting an existing file
func testWritebackOverwriteFile(t *testing.T, framework *FuseTestFramework) {
	filename := "writeback_overwrite.txt"
	mountPath := filepath.Join(framework.GetMountPoint(), filename)

	// First write
	content1 := []byte("First version of the file")
	require.NoError(t, os.WriteFile(mountPath, content1, 0644))
	waitForFileContent(t, mountPath, content1, 30*time.Second)

	// Overwrite with different content
	content2 := []byte("Second version — overwritten content that is longer than the first")
	require.NoError(t, os.WriteFile(mountPath, content2, 0644))
	waitForFileContent(t, mountPath, content2, 30*time.Second)
}

// TestWritebackCacheFsync tests that fsync still forces synchronous flush
// even when writebackCache is enabled
func TestWritebackCacheFsync(t *testing.T) {
	config := writebackConfig()
	framework := NewFuseTestFramework(t, config)
	defer framework.Cleanup()

	require.NoError(t, framework.Setup(config))

	t.Run("FsyncForcesFlush", func(t *testing.T) {
		testFsyncForcesFlush(t, framework)
	})

	t.Run("FsyncThenRead", func(t *testing.T) {
		testFsyncThenRead(t, framework)
	})
}

// testFsyncForcesFlush verifies that calling fsync before close ensures
// data is immediately available for reading, bypassing the async path.
func testFsyncForcesFlush(t *testing.T, framework *FuseTestFramework) {
	filename := "writeback_fsync.txt"
	mountPath := filepath.Join(framework.GetMountPoint(), filename)

	content := []byte("Data that must be flushed synchronously via fsync")

	// Open, write, fsync, close
	f, err := os.OpenFile(mountPath, os.O_CREATE|os.O_WRONLY, 0644)
	require.NoError(t, err)

	_, err = f.Write(content)
	require.NoError(t, err)

	// fsync forces synchronous data+metadata flush
	require.NoError(t, f.Sync())
	require.NoError(t, f.Close())

	// Data should be immediately available — no wait needed
	actual, err := os.ReadFile(mountPath)
	require.NoError(t, err)
	assert.Equal(t, content, actual)
}

// testFsyncThenRead verifies that after fsync, a freshly opened read
// returns the correct data without any delay.
func testFsyncThenRead(t *testing.T, framework *FuseTestFramework) {
	filename := "writeback_fsync_read.txt"
	mountPath := filepath.Join(framework.GetMountPoint(), filename)

	content := make([]byte, 64*1024) // 64KB
	_, err := rand.Read(content)
	require.NoError(t, err)

	// Write with explicit fsync
	f, err := os.OpenFile(mountPath, os.O_CREATE|os.O_WRONLY, 0644)
	require.NoError(t, err)
	_, err = f.Write(content)
	require.NoError(t, err)
	require.NoError(t, f.Sync())
	require.NoError(t, f.Close())

	// Immediate read should succeed
	actual, err := os.ReadFile(mountPath)
	require.NoError(t, err)
	assert.Equal(t, content, actual)
}

// TestWritebackCacheConcurrentSmallFiles is the primary test for issue #8718:
// many small files written concurrently should all be eventually readable.
func TestWritebackCacheConcurrentSmallFiles(t *testing.T) {
	config := writebackConfig()
	framework := NewFuseTestFramework(t, config)
	defer framework.Cleanup()

	require.NoError(t, framework.Setup(config))

	t.Run("ConcurrentSmallFiles", func(t *testing.T) {
		testWritebackConcurrentSmallFiles(t, framework)
	})

	t.Run("ConcurrentSmallFilesMultiDir", func(t *testing.T) {
		testWritebackConcurrentSmallFilesMultiDir(t, framework)
	})

	t.Run("RapidCreateCloseSequence", func(t *testing.T) {
		testWritebackRapidCreateClose(t, framework)
	})
}

// testWritebackConcurrentSmallFiles simulates the rsync workload from #8718:
// multiple workers creating many small files in parallel.
func testWritebackConcurrentSmallFiles(t *testing.T, framework *FuseTestFramework) {
	dir := "writeback_concurrent_small"
	framework.CreateTestDir(dir)

	numWorkers := 8
	filesPerWorker := 20
	totalFiles := numWorkers * filesPerWorker

	type fileRecord struct {
		path    string
		content []byte
	}

	var mu sync.Mutex
	var writeErrors []error
	records := make([]fileRecord, 0, totalFiles)

	// Phase 1: Write files concurrently (simulating rsync workers)
	var wg sync.WaitGroup
	for w := 0; w < numWorkers; w++ {
		wg.Add(1)
		go func(workerID int) {
			defer wg.Done()

			for f := 0; f < filesPerWorker; f++ {
				filename := fmt.Sprintf("w%02d_f%03d.dat", workerID, f)
				path := filepath.Join(framework.GetMountPoint(), dir, filename)

				// Vary sizes: 100B to 100KB
				size := 100 + (workerID*filesPerWorker+f)*500
				if size > 100*1024 {
					size = 100*1024
				}
				content := make([]byte, size)
				if _, err := rand.Read(content); err != nil {
					mu.Lock()
					writeErrors = append(writeErrors, fmt.Errorf("worker %d file %d rand: %v", workerID, f, err))
					mu.Unlock()
					return
				}

				if err := os.WriteFile(path, content, 0644); err != nil {
					mu.Lock()
					writeErrors = append(writeErrors, fmt.Errorf("worker %d file %d: %v", workerID, f, err))
					mu.Unlock()
					return
				}

				mu.Lock()
				records = append(records, fileRecord{path: path, content: content})
				mu.Unlock()
			}
		}(w)
	}
	wg.Wait()

	require.Empty(t, writeErrors, "write errors: %v", writeErrors)
	assert.Equal(t, totalFiles, len(records))

	// Phase 2: Wait for async flushes and verify all files
	for _, rec := range records {
		waitForFileContent(t, rec.path, rec.content, 60*time.Second)
	}

	// Phase 3: Verify directory listing has correct count
	entries, err := os.ReadDir(filepath.Join(framework.GetMountPoint(), dir))
	require.NoError(t, err)
	assert.Equal(t, totalFiles, len(entries))
}

// testWritebackConcurrentSmallFilesMultiDir tests concurrent writes across
// multiple directories — a common pattern for parallel copy tools.
func testWritebackConcurrentSmallFilesMultiDir(t *testing.T, framework *FuseTestFramework) {
	baseDir := "writeback_multidir"
	framework.CreateTestDir(baseDir)

	numDirs := 4
	filesPerDir := 25

	type fileRecord struct {
		path    string
		content []byte
	}
	var mu sync.Mutex
	var records []fileRecord
	var writeErrors []error

	var wg sync.WaitGroup
	for d := 0; d < numDirs; d++ {
		subDir := filepath.Join(baseDir, fmt.Sprintf("dir_%02d", d))
		framework.CreateTestDir(subDir)

		wg.Add(1)
		go func(dirID int, dirPath string) {
			defer wg.Done()

			for f := 0; f < filesPerDir; f++ {
				filename := fmt.Sprintf("file_%03d.txt", f)
				path := filepath.Join(framework.GetMountPoint(), dirPath, filename)
				content := []byte(fmt.Sprintf("dir=%d file=%d data=%s", dirID, f, time.Now().Format(time.RFC3339Nano)))

				if err := os.WriteFile(path, content, 0644); err != nil {
					mu.Lock()
					writeErrors = append(writeErrors, fmt.Errorf("dir %d file %d: %v", dirID, f, err))
					mu.Unlock()
					return
				}

				mu.Lock()
				records = append(records, fileRecord{path: path, content: content})
				mu.Unlock()
			}
		}(d, subDir)
	}
	wg.Wait()

	require.Empty(t, writeErrors, "write errors: %v", writeErrors)

	// Verify all files
	for _, rec := range records {
		waitForFileContent(t, rec.path, rec.content, 60*time.Second)
	}
}

// testWritebackRapidCreateClose rapidly creates and closes files to stress
// the async flush goroutine pool.
func testWritebackRapidCreateClose(t *testing.T, framework *FuseTestFramework) {
	dir := "writeback_rapid"
	framework.CreateTestDir(dir)

	numFiles := 200
	type fileRecord struct {
		path    string
		content []byte
	}
	records := make([]fileRecord, numFiles)

	// Rapidly create files without pausing
	for i := 0; i < numFiles; i++ {
		path := filepath.Join(framework.GetMountPoint(), dir, fmt.Sprintf("rapid_%04d.bin", i))
		content := []byte(fmt.Sprintf("rapid-file-%d", i))
		require.NoError(t, os.WriteFile(path, content, 0644))
		records[i] = fileRecord{path: path, content: content}
	}

	// Verify all files eventually appear with correct content
	for _, rec := range records {
		waitForFileContent(t, rec.path, rec.content, 60*time.Second)
	}
}

// TestWritebackCacheDataIntegrity tests that data integrity is preserved
// across various write patterns with writebackCache enabled.
func TestWritebackCacheDataIntegrity(t *testing.T) {
	config := writebackConfig()
	framework := NewFuseTestFramework(t, config)
	defer framework.Cleanup()

	require.NoError(t, framework.Setup(config))

	t.Run("AppendAfterClose", func(t *testing.T) {
		testWritebackAppendAfterClose(t, framework)
	})

	t.Run("PartialWrites", func(t *testing.T) {
		testWritebackPartialWrites(t, framework)
	})

	t.Run("FileSizeCorrectness", func(t *testing.T) {
		testWritebackFileSizeCorrectness(t, framework)
	})

	t.Run("BinaryData", func(t *testing.T) {
		testWritebackBinaryData(t, framework)
	})
}

// testWritebackAppendAfterClose writes a file, closes it (triggering async flush),
// waits for flush, then reopens and appends more data.
func testWritebackAppendAfterClose(t *testing.T, framework *FuseTestFramework) {
	filename := "writeback_append.txt"
	mountPath := filepath.Join(framework.GetMountPoint(), filename)

	// First write
	part1 := []byte("First part of the data.\n")
	require.NoError(t, os.WriteFile(mountPath, part1, 0644))

	// Wait for first async flush
	waitForFileContent(t, mountPath, part1, 30*time.Second)

	// Append more data
	part2 := []byte("Second part appended.\n")
	f, err := os.OpenFile(mountPath, os.O_APPEND|os.O_WRONLY, 0644)
	require.NoError(t, err)
	_, err = f.Write(part2)
	require.NoError(t, err)
	require.NoError(t, f.Close())

	// Verify combined content
	expected := append(part1, part2...)
	waitForFileContent(t, mountPath, expected, 30*time.Second)
}

// testWritebackPartialWrites tests writing to specific offsets within a file
func testWritebackPartialWrites(t *testing.T, framework *FuseTestFramework) {
	filename := "writeback_partial.bin"
	mountPath := filepath.Join(framework.GetMountPoint(), filename)

	// Create file with initial content
	initial := bytes.Repeat([]byte("A"), 4096)
	require.NoError(t, os.WriteFile(mountPath, initial, 0644))
	waitForFileContent(t, mountPath, initial, 30*time.Second)

	// Open and write at specific offset
	f, err := os.OpenFile(mountPath, os.O_WRONLY, 0644)
	require.NoError(t, err)
	patch := []byte("PATCHED")
	_, err = f.WriteAt(patch, 100)
	require.NoError(t, err)
	require.NoError(t, f.Close())

	// Build expected content
	expected := make([]byte, 4096)
	copy(expected, initial)
	copy(expected[100:], patch)

	waitForFileContent(t, mountPath, expected, 30*time.Second)
}

// testWritebackFileSizeCorrectness verifies that file sizes are correct
// after async flush completes.
func testWritebackFileSizeCorrectness(t *testing.T, framework *FuseTestFramework) {
	sizes := []int{0, 1, 100, 4096, 65536, 1024 * 1024}

	for _, size := range sizes {
		filename := fmt.Sprintf("writeback_size_%d.bin", size)
		mountPath := filepath.Join(framework.GetMountPoint(), filename)

		content := make([]byte, size)
		if size > 0 {
			_, err := rand.Read(content)
			require.NoError(t, err, "rand.Read failed for size %d", size)
		}

		require.NoError(t, os.WriteFile(mountPath, content, 0644), "failed to write file of size %d", size)

		if size > 0 {
			waitForFileSize(t, mountPath, int64(size), 30*time.Second)
			waitForFileContent(t, mountPath, content, 30*time.Second)
		}
	}
}

// testWritebackBinaryData verifies that arbitrary binary data (including null bytes)
// is preserved correctly through the async flush path.
func testWritebackBinaryData(t *testing.T, framework *FuseTestFramework) {
	filename := "writeback_binary.bin"
	mountPath := filepath.Join(framework.GetMountPoint(), filename)

	// Generate data with all byte values including nulls
	content := make([]byte, 256*100)
	for i := range content {
		content[i] = byte(i % 256)
	}

	require.NoError(t, os.WriteFile(mountPath, content, 0644))
	waitForFileContent(t, mountPath, content, 30*time.Second)
}

// TestWritebackCachePerformance measures whether writebackCache actually
// improves throughput for small file workloads compared to synchronous flush.
func TestWritebackCachePerformance(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping performance test in short mode")
	}

	numFiles := 200
	fileSize := 4096 // 4KB files

	// Generate test data upfront
	testData := make([][]byte, numFiles)
	for i := range testData {
		testData[i] = make([]byte, fileSize)
		rand.Read(testData[i])
	}

	// Benchmark with writebackCache enabled
	t.Run("WithWritebackCache", func(t *testing.T) {
		config := writebackConfig()
		framework := NewFuseTestFramework(t, config)
		defer framework.Cleanup()
		require.NoError(t, framework.Setup(config))

		dir := "perf_writeback"
		framework.CreateTestDir(dir)

		start := time.Now()
		for i := 0; i < numFiles; i++ {
			path := filepath.Join(framework.GetMountPoint(), dir, fmt.Sprintf("file_%04d.bin", i))
			require.NoError(t, os.WriteFile(path, testData[i], 0644))
		}
		writebackDuration := time.Since(start)

		// Wait for all files to be flushed
		for i := 0; i < numFiles; i++ {
			path := filepath.Join(framework.GetMountPoint(), dir, fmt.Sprintf("file_%04d.bin", i))
			waitForFileContent(t, path, testData[i], 60*time.Second)
		}

		t.Logf("writebackCache: wrote %d files in %v (%.0f files/sec)",
			numFiles, writebackDuration, float64(numFiles)/writebackDuration.Seconds())
	})

	// Benchmark without writebackCache (synchronous flush)
	t.Run("WithoutWritebackCache", func(t *testing.T) {
		config := DefaultTestConfig()
		framework := NewFuseTestFramework(t, config)
		defer framework.Cleanup()
		require.NoError(t, framework.Setup(config))

		dir := "perf_sync"
		framework.CreateTestDir(dir)

		start := time.Now()
		for i := 0; i < numFiles; i++ {
			path := filepath.Join(framework.GetMountPoint(), dir, fmt.Sprintf("file_%04d.bin", i))
			require.NoError(t, os.WriteFile(path, testData[i], 0644))
		}
		syncDuration := time.Since(start)

		t.Logf("synchronous: wrote %d files in %v (%.0f files/sec)",
			numFiles, syncDuration, float64(numFiles)/syncDuration.Seconds())
	})
}

// TestWritebackCacheConcurrentMixedOps tests a mix of operations happening
// concurrently with writebackCache: creates, reads, overwrites, and deletes.
func TestWritebackCacheConcurrentMixedOps(t *testing.T) {
	config := writebackConfig()
	framework := NewFuseTestFramework(t, config)
	defer framework.Cleanup()

	require.NoError(t, framework.Setup(config))

	dir := "writeback_mixed"
	framework.CreateTestDir(dir)

	numFiles := 50
	var mu sync.Mutex
	var errors []error
	var completedWrites int64

	addError := func(err error) {
		mu.Lock()
		defer mu.Unlock()
		errors = append(errors, err)
	}

	// Phase 1: Create initial files and wait for async flushes
	initialContents := make(map[int][]byte, numFiles)
	for i := 0; i < numFiles; i++ {
		path := filepath.Join(framework.GetMountPoint(), dir, fmt.Sprintf("file_%03d.txt", i))
		content := []byte(fmt.Sprintf("initial content %d", i))
		require.NoError(t, os.WriteFile(path, content, 0644))
		initialContents[i] = content
	}

	// Poll until initial files are flushed (instead of fixed sleep)
	for i := 0; i < numFiles; i++ {
		path := filepath.Join(framework.GetMountPoint(), dir, fmt.Sprintf("file_%03d.txt", i))
		waitForFileContent(t, path, initialContents[i], 30*time.Second)
	}

	// Phase 2: Concurrent mixed operations
	var wg sync.WaitGroup

	// Writers: overwrite existing files
	for i := 0; i < 4; i++ {
		wg.Add(1)
		go func(workerID int) {
			defer wg.Done()
			for j := 0; j < numFiles; j++ {
				if j%4 != workerID {
					continue // each worker handles a subset
				}
				path := filepath.Join(framework.GetMountPoint(), dir, fmt.Sprintf("file_%03d.txt", j))
				content := []byte(fmt.Sprintf("overwritten by worker %d at %s", workerID, time.Now().Format(time.RFC3339Nano)))
				if err := os.WriteFile(path, content, 0644); err != nil {
					addError(fmt.Errorf("writer %d file %d: %v", workerID, j, err))
					return
				}
				atomic.AddInt64(&completedWrites, 1)
			}
		}(i)
	}

	// Readers: read files (may see old or new content, but should not error)
	for i := 0; i < 4; i++ {
		wg.Add(1)
		go func(readerID int) {
			defer wg.Done()
			for j := 0; j < numFiles; j++ {
				path := filepath.Join(framework.GetMountPoint(), dir, fmt.Sprintf("file_%03d.txt", j))
				_, err := os.ReadFile(path)
				if err != nil && !os.IsNotExist(err) {
					addError(fmt.Errorf("reader %d file %d: %v", readerID, j, err))
					return
				}
			}
		}(i)
	}

	// New file creators
	wg.Add(1)
	go func() {
		defer wg.Done()
		for i := 0; i < 20; i++ {
			path := filepath.Join(framework.GetMountPoint(), dir, fmt.Sprintf("new_file_%03d.txt", i))
			content := []byte(fmt.Sprintf("new file %d", i))
			if err := os.WriteFile(path, content, 0644); err != nil {
				addError(fmt.Errorf("creator file %d: %v", i, err))
				return
			}
		}
	}()

	wg.Wait()

	require.Empty(t, errors, "mixed operation errors: %v", errors)
	assert.True(t, atomic.LoadInt64(&completedWrites) > 0, "should have completed some writes")

	// Verify new files exist after async flushes complete (poll instead of fixed sleep)
	for i := 0; i < 20; i++ {
		path := filepath.Join(framework.GetMountPoint(), dir, fmt.Sprintf("new_file_%03d.txt", i))
		expected := []byte(fmt.Sprintf("new file %d", i))
		waitForFileContent(t, path, expected, 30*time.Second)
	}
}

// TestWritebackCacheStressSmallFiles is a focused stress test for the
// async flush path with many small files — the core scenario from #8718.
func TestWritebackCacheStressSmallFiles(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping stress test in short mode")
	}

	config := writebackConfig()
	framework := NewFuseTestFramework(t, config)
	defer framework.Cleanup()

	require.NoError(t, framework.Setup(config))

	dir := "writeback_stress"
	framework.CreateTestDir(dir)

	numWorkers := 16
	filesPerWorker := 100
	totalFiles := numWorkers * filesPerWorker

	type fileRecord struct {
		path    string
		content []byte
	}

	var mu sync.Mutex
	var writeErrors []error
	records := make([]fileRecord, 0, totalFiles)

	start := time.Now()

	// Simulate rsync-like workload: many workers each writing small files
	var wg sync.WaitGroup
	for w := 0; w < numWorkers; w++ {
		wg.Add(1)
		go func(workerID int) {
			defer wg.Done()
			for f := 0; f < filesPerWorker; f++ {
				filename := fmt.Sprintf("w%02d/f%04d.dat", workerID, f)
				path := filepath.Join(framework.GetMountPoint(), dir, filename)

				// Ensure subdirectory exists
				if f == 0 {
					subDir := filepath.Join(framework.GetMountPoint(), dir, fmt.Sprintf("w%02d", workerID))
					if err := os.MkdirAll(subDir, 0755); err != nil {
						mu.Lock()
						writeErrors = append(writeErrors, fmt.Errorf("worker %d mkdir: %v", workerID, err))
						mu.Unlock()
						return
					}
				}

				// Small file: 1KB-10KB (typical for rsync of config/source files)
				size := 1024 + (f%10)*1024
				content := make([]byte, size)
				if _, err := rand.Read(content); err != nil {
					mu.Lock()
					writeErrors = append(writeErrors, fmt.Errorf("worker %d file %d rand: %v", workerID, f, err))
					mu.Unlock()
					return
				}

				if err := os.WriteFile(path, content, 0644); err != nil {
					mu.Lock()
					writeErrors = append(writeErrors, fmt.Errorf("worker %d file %d: %v", workerID, f, err))
					mu.Unlock()
					return
				}

				mu.Lock()
				records = append(records, fileRecord{path: path, content: content})
				mu.Unlock()
			}
		}(w)
	}
	wg.Wait()

	writeDuration := time.Since(start)
	t.Logf("wrote %d files in %v (%.0f files/sec)",
		totalFiles, writeDuration, float64(totalFiles)/writeDuration.Seconds())

	require.Empty(t, writeErrors, "write errors: %v", writeErrors)
	assert.Equal(t, totalFiles, len(records))

	// Verify all files are eventually readable with correct content
	var verifyErrors []error
	for _, rec := range records {
		deadline := time.Now().Add(120 * time.Second)
		var lastErr error
		for time.Now().Before(deadline) {
			actual, err := os.ReadFile(rec.path)
			if err == nil && bytes.Equal(rec.content, actual) {
				lastErr = nil
				break
			}
			if err != nil {
				lastErr = err
			} else {
				lastErr = fmt.Errorf("content mismatch for %s: got %d bytes, want %d", rec.path, len(actual), len(rec.content))
			}
			time.Sleep(500 * time.Millisecond)
		}
		if lastErr != nil {
			verifyErrors = append(verifyErrors, lastErr)
		}
	}
	require.Empty(t, verifyErrors, "verification errors after stress test: %v", verifyErrors)

	t.Logf("all %d files verified successfully", totalFiles)
}