Merge branch 'master' of https://github.com/seaweedfs/seaweedfs

test/fuse_integration/git_operations_test.go

@@ -1,6 +1,7 @@
 package fuse_test
 
 import (
+	"fmt"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -132,17 +133,10 @@ func testGitCloneAndPull(t *testing.T, mountPoint, localDir string) {
 	// ---- Phase 6: Pull with real changes ----
 	t.Log("Phase 6: pull with real fast-forward changes")
 
-	ensureMountClone(t, bareRepo, mountClone)
-
-	// After git reset --hard, give the FUSE mount a moment to settle its
-	// metadata cache. On slow CI, the working directory can briefly appear
-	// missing to a new subprocess (git pull → unpack-objects).
-	waitForDir(t, mountClone)
-
-	oldHead := gitOutput(t, mountClone, "rev-parse", "HEAD")
-	assert.Equal(t, commit2, oldHead, "should be at commit 2 before pull")
-
-	gitRun(t, mountClone, "pull")
+	// After git reset --hard on FUSE (Phase 5), the kernel dcache can
+	// permanently lose the directory entry. Wrap the pull in a recovery
+	// loop that re-clones from the bare repo if the clone has vanished.
+	pullFromCommitWithRecovery(t, bareRepo, localClone, mountClone, commit2)
 
 	newHead := gitOutput(t, mountClone, "rev-parse", "HEAD")
 	assert.Equal(t, commit5, newHead, "HEAD should be commit 5 after pull")
@@ -321,13 +315,110 @@ func isBareRepo(bareRepo string) bool {
 
 func ensureMountClone(t *testing.T, bareRepo, mountClone string) {
 	t.Helper()
-	if _, err := os.Stat(mountClone); err == nil {
-		return
-	} else if !os.IsNotExist(err) {
-		require.NoError(t, err)
+	require.NoError(t, tryEnsureMountClone(bareRepo, mountClone))
+}
+
+// tryEnsureBareRepo verifies the bare repo on the FUSE mount exists.
+// If it has vanished, it re-creates it from the local clone.
+func tryEnsureBareRepo(bareRepo, localClone string) error {
+	if _, err := os.Stat(filepath.Join(bareRepo, "HEAD")); err == nil {
+		return nil
 	}
-	t.Logf("mount clone missing, re-cloning from %s", bareRepo)
-	gitRun(t, "", "clone", bareRepo, mountClone)
+	os.RemoveAll(bareRepo)
+	time.Sleep(500 * time.Millisecond)
+	if _, err := tryGitCommand("", "init", "--bare", bareRepo); err != nil {
+		return fmt.Errorf("re-init bare repo: %w", err)
+	}
+	if _, err := tryGitCommand(localClone, "push", "--force", bareRepo, "master"); err != nil {
+		return fmt.Errorf("re-push to bare repo: %w", err)
+	}
+	return nil
+}
+
+// tryEnsureMountClone is like ensureMountClone but returns an error instead
+// of failing the test, for use in recovery loops.
+func tryEnsureMountClone(bareRepo, mountClone string) error {
+	// Verify .git/HEAD exists — just checking the top-level dir is
+	// insufficient because FUSE may cache a stale directory entry.
+	if _, err := os.Stat(filepath.Join(mountClone, ".git", "HEAD")); err == nil {
+		return nil
+	}
+	os.RemoveAll(mountClone)
+	time.Sleep(500 * time.Millisecond)
+	if _, err := tryGitCommand("", "clone", bareRepo, mountClone); err != nil {
+		return fmt.Errorf("re-clone: %w", err)
+	}
+	return nil
+}
+
+// tryGitCommand runs a git command and returns (output, error) without
+// failing the test, for use in recovery loops.
+func tryGitCommand(dir string, args ...string) (string, error) {
+	cmd := exec.Command("git", args...)
+	if dir != "" {
+		cmd.Dir = dir
+	}
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return strings.TrimSpace(string(out)), fmt.Errorf("%s: %w", strings.TrimSpace(string(out)), err)
+	}
+	return strings.TrimSpace(string(out)), nil
+}
+
+// pullFromCommitWithRecovery resets to fromCommit and runs git pull. If the
+// FUSE mount loses directories (both the bare repo and the working clone can
+// vanish after heavy git operations), it re-creates them and retries.
+func pullFromCommitWithRecovery(t *testing.T, bareRepo, localClone, cloneDir, fromCommit string) {
+	t.Helper()
+	const maxAttempts = 3
+	var lastErr error
+	for attempt := 1; attempt <= maxAttempts; attempt++ {
+		if lastErr = tryPullFromCommit(t, bareRepo, localClone, cloneDir, fromCommit); lastErr == nil {
+			return
+		}
+		if attempt == maxAttempts {
+			require.NoError(t, lastErr, "git pull failed after %d recovery attempts", maxAttempts)
+		}
+		t.Logf("pull recovery attempt %d: %v — removing clone for re-create", attempt, lastErr)
+		os.RemoveAll(cloneDir)
+		time.Sleep(2 * time.Second)
+	}
+}
+
+func tryPullFromCommit(t *testing.T, bareRepo, localClone, cloneDir, fromCommit string) error {
+	t.Helper()
+	// The bare repo lives on the FUSE mount and can also vanish.
+	// Re-create it from the local clone (which is on local disk).
+	if err := tryEnsureBareRepo(bareRepo, localClone); err != nil {
+		return err
+	}
+	if err := tryEnsureMountClone(bareRepo, cloneDir); err != nil {
+		return err
+	}
+	if !waitForDirEventually(t, cloneDir, 10*time.Second) {
+		return fmt.Errorf("clone dir %s did not appear", cloneDir)
+	}
+
+	if _, err := tryGitCommand(cloneDir, "reset", "--hard", fromCommit); err != nil {
+		return fmt.Errorf("reset --hard: %w", err)
+	}
+	if !waitForDirEventually(t, cloneDir, 5*time.Second) {
+		return fmt.Errorf("clone dir %s did not recover after reset", cloneDir)
+	}
+	refreshDirEntry(t, cloneDir)
+
+	head, err := tryGitCommand(cloneDir, "rev-parse", "HEAD")
+	if err != nil {
+		return fmt.Errorf("rev-parse after reset: %w", err)
+	}
+	if head != fromCommit {
+		return fmt.Errorf("expected HEAD at %s after reset, got %s", fromCommit, head)
+	}
+
+	if _, err := tryGitCommand(cloneDir, "pull"); err != nil {
+		return fmt.Errorf("pull: %w", err)
+	}
+	return nil
 }
 
 var gitRepoPathRe = regexp.MustCompile(`'([^']+)' does not appear to be a git repository`)