fix(fuse-test): recover from FUSE directory loss in git pull test (#8789)

* fix(fuse-test): harden ensureMountClone to verify .git/HEAD

On FUSE, the kernel dcache can retain a stale directory entry after
heavy git operations. Checking only os.Stat(mountClone) may find
the top-level directory but the clone internals (.git/HEAD) are gone.

Now ensureMountClone verifies .git/HEAD exists, cleans up stale
remnants with os.RemoveAll before re-cloning, and adds a brief
settle delay for the FUSE metadata cache.

* fix(fuse-test): add pull recovery loop for Phase 6 git operations

After git reset --hard on a FUSE mount, the kernel dcache can
permanently lose the clone directory entry. The existing retry logic
polls for 60+ seconds but the directory never recovers.

Add pullFromCommitWithRecovery which wraps the Phase 6 pull in a
recovery loop: if the clone directory vanishes, it removes the stale
clone, re-creates it from the bare repo, resets to the target commit,
and retries the pull (up to 3 attempts).

Also adds tryGitCommand, a non-fatal git runner that returns
(output, error) instead of calling require.NoError, enabling the
recovery loop to handle failures gracefully without aborting the test.

Fixes flaky TestGitOperations/CloneAndPull on CI.

* fix(fuse-test): make recovery loop fully non-fatal

Address review feedback:

- Extract tryEnsureMountClone (returns error) so a transient FUSE
  failure during re-clone doesn't abort the test — the recovery loop
  can retry instead.
- Check waitForDirEventually return after git reset --hard and return
  a specific error if the directory doesn't recover.

* style(fuse-test): simplify ensureMountClone error check

* fix(fuse-test): recover bare repo when FUSE mount loses it

CI showed that not just the working clone but also the bare repo on
the FUSE mount can vanish after heavy git operations. The recovery
loop now re-creates the bare repo from the local clone (which lives
on local disk and is always available) before attempting to re-clone.

Adds tryEnsureBareRepo: checks HEAD exists in the bare repo, and if
not, re-inits and force-pushes from the local clone.

test/fuse_integration/git_operations_test.go

@@ -1,6 +1,7 @@
 package fuse_test
 
 import (
+	"fmt"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -132,17 +133,10 @@ func testGitCloneAndPull(t *testing.T, mountPoint, localDir string) {
 	// ---- Phase 6: Pull with real changes ----
 	t.Log("Phase 6: pull with real fast-forward changes")
 
-	ensureMountClone(t, bareRepo, mountClone)
-
-	// After git reset --hard, give the FUSE mount a moment to settle its
-	// metadata cache. On slow CI, the working directory can briefly appear
-	// missing to a new subprocess (git pull → unpack-objects).
-	waitForDir(t, mountClone)
-
-	oldHead := gitOutput(t, mountClone, "rev-parse", "HEAD")
-	assert.Equal(t, commit2, oldHead, "should be at commit 2 before pull")
-
-	gitRun(t, mountClone, "pull")
+	// After git reset --hard on FUSE (Phase 5), the kernel dcache can
+	// permanently lose the directory entry. Wrap the pull in a recovery
+	// loop that re-clones from the bare repo if the clone has vanished.
+	pullFromCommitWithRecovery(t, bareRepo, localClone, mountClone, commit2)
 
 	newHead := gitOutput(t, mountClone, "rev-parse", "HEAD")
 	assert.Equal(t, commit5, newHead, "HEAD should be commit 5 after pull")
@@ -321,13 +315,110 @@ func isBareRepo(bareRepo string) bool {
 
 func ensureMountClone(t *testing.T, bareRepo, mountClone string) {
 	t.Helper()
-	if _, err := os.Stat(mountClone); err == nil {
-		return
-	} else if !os.IsNotExist(err) {
-		require.NoError(t, err)
+	require.NoError(t, tryEnsureMountClone(bareRepo, mountClone))
+}
+
+// tryEnsureBareRepo verifies the bare repo on the FUSE mount exists.
+// If it has vanished, it re-creates it from the local clone.
+func tryEnsureBareRepo(bareRepo, localClone string) error {
+	if _, err := os.Stat(filepath.Join(bareRepo, "HEAD")); err == nil {
+		return nil
 	}
-	t.Logf("mount clone missing, re-cloning from %s", bareRepo)
-	gitRun(t, "", "clone", bareRepo, mountClone)
+	os.RemoveAll(bareRepo)
+	time.Sleep(500 * time.Millisecond)
+	if _, err := tryGitCommand("", "init", "--bare", bareRepo); err != nil {
+		return fmt.Errorf("re-init bare repo: %w", err)
+	}
+	if _, err := tryGitCommand(localClone, "push", "--force", bareRepo, "master"); err != nil {
+		return fmt.Errorf("re-push to bare repo: %w", err)
+	}
+	return nil
+}
+
+// tryEnsureMountClone is like ensureMountClone but returns an error instead
+// of failing the test, for use in recovery loops.
+func tryEnsureMountClone(bareRepo, mountClone string) error {
+	// Verify .git/HEAD exists — just checking the top-level dir is
+	// insufficient because FUSE may cache a stale directory entry.
+	if _, err := os.Stat(filepath.Join(mountClone, ".git", "HEAD")); err == nil {
+		return nil
+	}
+	os.RemoveAll(mountClone)
+	time.Sleep(500 * time.Millisecond)
+	if _, err := tryGitCommand("", "clone", bareRepo, mountClone); err != nil {
+		return fmt.Errorf("re-clone: %w", err)
+	}
+	return nil
+}
+
+// tryGitCommand runs a git command and returns (output, error) without
+// failing the test, for use in recovery loops.
+func tryGitCommand(dir string, args ...string) (string, error) {
+	cmd := exec.Command("git", args...)
+	if dir != "" {
+		cmd.Dir = dir
+	}
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return strings.TrimSpace(string(out)), fmt.Errorf("%s: %w", strings.TrimSpace(string(out)), err)
+	}
+	return strings.TrimSpace(string(out)), nil
+}
+
+// pullFromCommitWithRecovery resets to fromCommit and runs git pull. If the
+// FUSE mount loses directories (both the bare repo and the working clone can
+// vanish after heavy git operations), it re-creates them and retries.
+func pullFromCommitWithRecovery(t *testing.T, bareRepo, localClone, cloneDir, fromCommit string) {
+	t.Helper()
+	const maxAttempts = 3
+	var lastErr error
+	for attempt := 1; attempt <= maxAttempts; attempt++ {
+		if lastErr = tryPullFromCommit(t, bareRepo, localClone, cloneDir, fromCommit); lastErr == nil {
+			return
+		}
+		if attempt == maxAttempts {
+			require.NoError(t, lastErr, "git pull failed after %d recovery attempts", maxAttempts)
+		}
+		t.Logf("pull recovery attempt %d: %v — removing clone for re-create", attempt, lastErr)
+		os.RemoveAll(cloneDir)
+		time.Sleep(2 * time.Second)
+	}
+}
+
+func tryPullFromCommit(t *testing.T, bareRepo, localClone, cloneDir, fromCommit string) error {
+	t.Helper()
+	// The bare repo lives on the FUSE mount and can also vanish.
+	// Re-create it from the local clone (which is on local disk).
+	if err := tryEnsureBareRepo(bareRepo, localClone); err != nil {
+		return err
+	}
+	if err := tryEnsureMountClone(bareRepo, cloneDir); err != nil {
+		return err
+	}
+	if !waitForDirEventually(t, cloneDir, 10*time.Second) {
+		return fmt.Errorf("clone dir %s did not appear", cloneDir)
+	}
+
+	if _, err := tryGitCommand(cloneDir, "reset", "--hard", fromCommit); err != nil {
+		return fmt.Errorf("reset --hard: %w", err)
+	}
+	if !waitForDirEventually(t, cloneDir, 5*time.Second) {
+		return fmt.Errorf("clone dir %s did not recover after reset", cloneDir)
+	}
+	refreshDirEntry(t, cloneDir)
+
+	head, err := tryGitCommand(cloneDir, "rev-parse", "HEAD")
+	if err != nil {
+		return fmt.Errorf("rev-parse after reset: %w", err)
+	}
+	if head != fromCommit {
+		return fmt.Errorf("expected HEAD at %s after reset, got %s", fromCommit, head)
+	}
+
+	if _, err := tryGitCommand(cloneDir, "pull"); err != nil {
+		return fmt.Errorf("pull: %w", err)
+	}
+	return nil
 }
 
 var gitRepoPathRe = regexp.MustCompile(`'([^']+)' does not appear to be a git repository`)