admin/plugin: delete job_detail files when jobs are pruned from memory (#8722)

* admin/plugin: delete job_detail files when jobs are pruned from memory

pruneTrackedJobsLocked evicts the oldest terminal jobs from the in-memory
tracker when the total exceeds maxTrackedJobsTotal (1000). However the
dedicated per-job detail files in jobs/job_details/ were never removed,
causing them to accumulate indefinitely on disk.

Add ConfigStore.DeleteJobDetail and call it from pruneTrackedJobsLocked so
that the file is cleaned up together with the in-memory entry. Deletion
errors are logged at verbosity level 2 and do not abort the prune.

* admin/plugin: add test for DeleteJobDetail

---------

Co-authored-by: Anton Ustyugov <anton@devops>
Co-authored-by: Chris Lu <chris.lu@gmail.com>

weed/admin/plugin/config_store.go

@@ -446,6 +446,30 @@ func (s *ConfigStore) LoadJobDetail(jobID string) (*TrackedJob, error) {
 	return &clone, nil
 }
 
+// DeleteJobDetail removes the persisted detail snapshot for a job.
+// It is a no-op when the file does not exist.
+func (s *ConfigStore) DeleteJobDetail(jobID string) error {
+	jobID, err := sanitizeJobID(jobID)
+	if err != nil {
+		return err
+	}
+
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	if !s.configured {
+		delete(s.memJobDetails, jobID)
+		return nil
+	}
+
+	path := filepath.Join(s.baseDir, jobsDirName, jobDetailsDirName, jobDetailFileName(jobID))
+	err = os.Remove(path)
+	if err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("delete job detail: %w", err)
+	}
+	return nil
+}
+
 func (s *ConfigStore) SaveActivities(activities []JobActivity) error {
 	s.mu.Lock()
 	defer s.mu.Unlock()

weed/admin/plugin/config_store_test.go

@@ -330,3 +330,50 @@ func TestConfigStoreJobDetailRoundTrip(t *testing.T) {
 		t.Fatalf("expected result output values")
 	}
 }
+
+func TestConfigStoreDeleteJobDetail(t *testing.T) {
+	t.Parallel()
+
+	store, err := NewConfigStore(t.TempDir())
+	if err != nil {
+		t.Fatalf("NewConfigStore: %v", err)
+	}
+
+	input := TrackedJob{
+		JobID:   "job-to-delete",
+		JobType: "vacuum",
+		Summary: "will be deleted",
+	}
+
+	if err := store.SaveJobDetail(input); err != nil {
+		t.Fatalf("SaveJobDetail: %v", err)
+	}
+
+	// Verify it was persisted.
+	got, err := store.LoadJobDetail(input.JobID)
+	if err != nil {
+		t.Fatalf("LoadJobDetail before delete: %v", err)
+	}
+	if got == nil {
+		t.Fatalf("expected saved job detail, got nil")
+	}
+
+	// Delete it.
+	if err := store.DeleteJobDetail(input.JobID); err != nil {
+		t.Fatalf("DeleteJobDetail: %v", err)
+	}
+
+	// Verify it is gone.
+	got, err = store.LoadJobDetail(input.JobID)
+	if err != nil {
+		t.Fatalf("LoadJobDetail after delete: %v", err)
+	}
+	if got != nil {
+		t.Fatalf("expected nil after delete, got %+v", got)
+	}
+
+	// Deleting again should be a no-op, not an error.
+	if err := store.DeleteJobDetail(input.JobID); err != nil {
+		t.Fatalf("second DeleteJobDetail: %v", err)
+	}
+}

weed/admin/plugin/plugin_monitor.go

@@ -1134,7 +1134,11 @@ func (r *Plugin) pruneTrackedJobsLocked() {
 	}
 
 	for i := 0; i < toDelete; i++ {
-		delete(r.jobs, terminalJobs[i].jobID)
+		jobID := terminalJobs[i].jobID
+		delete(r.jobs, jobID)
+		if err := r.store.DeleteJobDetail(jobID); err != nil {
+			glog.V(2).Infof("Plugin failed to delete job detail for pruned job %s: %v", jobID, err)
+		}
 	}
 }