From 479e72b5ab964f533b29e3ec9d57430520952f5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Br=C3=A5ten?= <97455552+hexahigh@users.noreply.github.com> Date: Sun, 29 Mar 2026 22:33:17 +0200 Subject: [PATCH] mount: add option to show system entries (#8829) * mount: add option to show system entries * address gemini code review's suggested changes * rename flag from -showSystemEntries to -includeSystemEntries * meta_cache: purge hidden system entries on filer events --------- Co-authored-by: Chris Lu --- weed/command/mount.go | 2 + weed/command/mount_std.go | 1 + weed/mount/meta_cache/meta_cache.go | 78 +++++++++---- .../mount/meta_cache/meta_cache_apply_test.go | 110 ++++++++++++++++++ weed/mount/meta_cache/meta_cache_init.go | 2 +- weed/mount/weedfs.go | 2 + weed/mount/weedfs_dir_read.go | 8 +- weed/mount/weedfs_dir_read_test.go | 24 +++- weed/mount/weedfs_file_copy_range_test.go | 1 + weed/mount/weedfs_file_mkrm_test.go | 1 + weed/mount/weedfs_rename_test.go | 1 + 11 files changed, 203 insertions(+), 27 deletions(-) diff --git a/weed/command/mount.go b/weed/command/mount.go index f1bacae8b..37bf2d15a 100644 --- a/weed/command/mount.go +++ b/weed/command/mount.go @@ -31,6 +31,7 @@ type MountOptions struct { uidMap *string gidMap *string readOnly *bool + includeSystemEntries *bool debug *bool debugPort *int localSocket *string @@ -99,6 +100,7 @@ func init() { mountOptions.uidMap = cmdMount.Flag.String("map.uid", "", "map local uid to uid on filer, comma-separated :") mountOptions.gidMap = cmdMount.Flag.String("map.gid", "", "map local gid to gid on filer, comma-separated :") mountOptions.readOnly = cmdMount.Flag.Bool("readOnly", false, "read only") + mountOptions.includeSystemEntries = cmdMount.Flag.Bool("includeSystemEntries", false, "show filer system entries (e.g. /topics, /etc) in directory listings") mountOptions.debug = cmdMount.Flag.Bool("debug", false, "serves runtime profiling data, e.g., http://localhost:/debug/pprof/goroutine?debug=2") mountOptions.debugPort = cmdMount.Flag.Int("debug.port", 6061, "http port for debugging") mountOptions.localSocket = cmdMount.Flag.String("localSocket", "", "default to /tmp/seaweedfs-mount-.sock") diff --git a/weed/command/mount_std.go b/weed/command/mount_std.go index 331820ef4..8cb617fbf 100644 --- a/weed/command/mount_std.go +++ b/weed/command/mount_std.go @@ -340,6 +340,7 @@ func RunMount(option *MountOptions, umask os.FileMode) bool { VolumeServerAccess: *mountOptions.volumeServerAccess, Cipher: cipher, UidGidMapper: uidGidMapper, + IncludeSystemEntries: *option.includeSystemEntries, DisableXAttr: *option.disableXAttr, IsMacOs: runtime.GOOS == "darwin", MetadataFlushSeconds: *option.metadataFlushSeconds, diff --git a/weed/mount/meta_cache/meta_cache.go b/weed/mount/meta_cache/meta_cache.go index 4925a23b6..ffa9ba3a6 100644 --- a/weed/mount/meta_cache/meta_cache.go +++ b/weed/mount/meta_cache/meta_cache.go @@ -27,18 +27,19 @@ type MetaCache struct { localStore filer.VirtualFilerStore leveldbStore *leveldb.LevelDBStore // direct reference for batch operations sync.RWMutex - uidGidMapper *UidGidMapper - markCachedFn func(fullpath util.FullPath) - isCachedFn func(fullpath util.FullPath) bool - invalidateFunc func(fullpath util.FullPath, entry *filer_pb.Entry) - onDirectoryUpdate func(dir util.FullPath) - visitGroup singleflight.Group // deduplicates concurrent EnsureVisited calls for the same path - applyCh chan metadataApplyRequest - applyDone chan struct{} - applyStateMu sync.Mutex - applyClosed bool - buildingDirs map[util.FullPath]*directoryBuildState - dedupRing dedupRingBuffer + uidGidMapper *UidGidMapper + markCachedFn func(fullpath util.FullPath) + isCachedFn func(fullpath util.FullPath) bool + invalidateFunc func(fullpath util.FullPath, entry *filer_pb.Entry) + onDirectoryUpdate func(dir util.FullPath) + visitGroup singleflight.Group // deduplicates concurrent EnsureVisited calls for the same path + applyCh chan metadataApplyRequest + applyDone chan struct{} + applyStateMu sync.Mutex + applyClosed bool + buildingDirs map[util.FullPath]*directoryBuildState + dedupRing dedupRingBuffer + includeSystemEntries bool } var errMetaCacheClosed = errors.New("metadata cache is shut down") @@ -84,17 +85,18 @@ type metadataApplyRequest struct { done chan error } -func NewMetaCache(dbFolder string, uidGidMapper *UidGidMapper, root util.FullPath, +func NewMetaCache(dbFolder string, uidGidMapper *UidGidMapper, root util.FullPath, includeSystemEntries bool, markCachedFn func(path util.FullPath), isCachedFn func(path util.FullPath) bool, invalidateFunc func(util.FullPath, *filer_pb.Entry), onDirectoryUpdate func(dir util.FullPath)) *MetaCache { leveldbStore, virtualStore := openMetaStore(dbFolder) mc := &MetaCache{ - root: root, - localStore: virtualStore, - leveldbStore: leveldbStore, - markCachedFn: markCachedFn, - isCachedFn: isCachedFn, - uidGidMapper: uidGidMapper, - onDirectoryUpdate: onDirectoryUpdate, + root: root, + localStore: virtualStore, + leveldbStore: leveldbStore, + markCachedFn: markCachedFn, + isCachedFn: isCachedFn, + uidGidMapper: uidGidMapper, + onDirectoryUpdate: onDirectoryUpdate, + includeSystemEntries: includeSystemEntries, invalidateFunc: func(fullpath util.FullPath, entry *filer_pb.Entry) { invalidateFunc(fullpath, entry) }, @@ -182,6 +184,29 @@ func (mc *MetaCache) atomicUpdateEntryFromFilerLocked(ctx context.Context, oldPa return nil } +func (mc *MetaCache) shouldHideEntry(fullpath util.FullPath) bool { + if mc.includeSystemEntries { + return false + } + dir, name := fullpath.DirAndName() + return IsHiddenSystemEntry(dir, name) +} + +func (mc *MetaCache) purgeEntryLocked(ctx context.Context, fullpath util.FullPath, isDirectory bool) error { + if fullpath == "" { + return nil + } + if err := mc.localStore.DeleteEntry(ctx, fullpath); err != nil { + return err + } + if isDirectory { + if err := mc.localStore.DeleteFolderChildren(ctx, fullpath); err != nil { + return err + } + } + return nil +} + func (mc *MetaCache) ApplyMetadataResponse(ctx context.Context, resp *filer_pb.SubscribeMetadataResponse, options MetadataResponseApplyOptions) error { if resp == nil || resp.EventNotification == nil { return nil @@ -520,7 +545,9 @@ func (mc *MetaCache) applyMetadataResponseLocked(ctx context.Context, resp *file } var oldPath util.FullPath + var newPath util.FullPath var newEntry *filer.Entry + hideNewPath := false if message.OldEntry != nil { oldPath = util.NewFullPath(resp.Directory, message.OldEntry.Name) } @@ -530,11 +557,20 @@ func (mc *MetaCache) applyMetadataResponseLocked(ctx context.Context, resp *file if message.NewParentPath != "" { dir = message.NewParentPath } - newEntry = filer.FromPbEntry(dir, message.NewEntry) + newPath = util.NewFullPath(dir, message.NewEntry.Name) + hideNewPath = mc.shouldHideEntry(newPath) + if !hideNewPath { + newEntry = filer.FromPbEntry(dir, message.NewEntry) + } } mc.Lock() err := mc.atomicUpdateEntryFromFilerLocked(ctx, oldPath, newEntry, allowUncachedInsert) + if err == nil && hideNewPath { + if purgeErr := mc.purgeEntryLocked(ctx, newPath, message.NewEntry.IsDirectory); purgeErr != nil { + err = purgeErr + } + } // When a directory is deleted or moved, remove its cached descendants // so stale children cannot be served from the local cache. if err == nil && oldPath != "" && message.OldEntry != nil && message.OldEntry.IsDirectory { diff --git a/weed/mount/meta_cache/meta_cache_apply_test.go b/weed/mount/meta_cache/meta_cache_apply_test.go index d30fab90d..214403f2c 100644 --- a/weed/mount/meta_cache/meta_cache_apply_test.go +++ b/weed/mount/meta_cache/meta_cache_apply_test.go @@ -2,6 +2,7 @@ package meta_cache import ( "context" + "os" "path/filepath" "sync" "testing" @@ -296,6 +297,114 @@ func TestApplyMetadataResponseDeduplicatesRepeatedFilerEvent(t *testing.T) { } } +func TestApplyMetadataResponseSkipsHiddenSystemEntryWhenDisabled(t *testing.T) { + mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{ + "/": true, + }) + defer mc.Shutdown() + + createResp := &filer_pb.SubscribeMetadataResponse{ + Directory: "/", + EventNotification: &filer_pb.EventNotification{ + NewEntry: &filer_pb.Entry{ + Name: "topics", + Attributes: &filer_pb.FuseAttributes{ + Crtime: 1, + Mtime: 1, + FileMode: uint32(os.ModeDir | 0o755), + }, + IsDirectory: true, + }, + }, + } + + if err := mc.ApplyMetadataResponse(context.Background(), createResp, SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply create: %v", err) + } + + entry, err := mc.FindEntry(context.Background(), util.FullPath("/topics")) + if err != filer_pb.ErrNotFound { + t.Fatalf("find hidden entry error = %v, want %v", err, filer_pb.ErrNotFound) + } + if entry != nil { + t.Fatalf("hidden entry still cached: %+v", entry) + } +} + +func TestApplyMetadataResponsePurgesHiddenDestinationPath(t *testing.T) { + mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{ + "/": true, + "/src": true, + }) + defer mc.Shutdown() + + if err := mc.InsertEntry(context.Background(), &filer.Entry{ + FullPath: "/topics", + Attr: filer.Attr{ + Crtime: time.Unix(1, 0), + Mtime: time.Unix(1, 0), + Mode: os.ModeDir | 0o755, + }, + }); err != nil { + t.Fatalf("insert stale hidden dir: %v", err) + } + if err := mc.InsertEntry(context.Background(), &filer.Entry{ + FullPath: "/topics/leaked.txt", + Attr: filer.Attr{ + Crtime: time.Unix(1, 0), + Mtime: time.Unix(1, 0), + Mode: 0o644, + FileSize: 7, + }, + }); err != nil { + t.Fatalf("insert leaked hidden child: %v", err) + } + if err := mc.InsertEntry(context.Background(), &filer.Entry{ + FullPath: "/src/visible", + Attr: filer.Attr{ + Crtime: time.Unix(1, 0), + Mtime: time.Unix(1, 0), + Mode: os.ModeDir | 0o755, + }, + }); err != nil { + t.Fatalf("insert source dir: %v", err) + } + + renameResp := &filer_pb.SubscribeMetadataResponse{ + Directory: "/src", + EventNotification: &filer_pb.EventNotification{ + OldEntry: &filer_pb.Entry{ + Name: "visible", + IsDirectory: true, + }, + NewEntry: &filer_pb.Entry{ + Name: "topics", + Attributes: &filer_pb.FuseAttributes{ + Crtime: 2, + Mtime: 2, + FileMode: uint32(os.ModeDir | 0o755), + }, + IsDirectory: true, + }, + NewParentPath: "/", + }, + } + + if err := mc.ApplyMetadataResponse(context.Background(), renameResp, SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply rename: %v", err) + } + + if entry, err := mc.FindEntry(context.Background(), util.FullPath("/src/visible")); err != filer_pb.ErrNotFound || entry != nil { + t.Fatalf("source dir after rename = %+v, %v; want nil, %v", entry, err, filer_pb.ErrNotFound) + } + if entry, err := mc.FindEntry(context.Background(), util.FullPath("/topics")); err != filer_pb.ErrNotFound || entry != nil { + t.Fatalf("hidden destination after rename = %+v, %v; want nil, %v", entry, err, filer_pb.ErrNotFound) + } + if entry, err := mc.FindEntry(context.Background(), util.FullPath("/topics/leaked.txt")); err != filer_pb.ErrNotFound || entry != nil { + t.Fatalf("hidden child after rename = %+v, %v; want nil, %v", entry, err, filer_pb.ErrNotFound) + } +} + func newTestMetaCache(t *testing.T, cached map[util.FullPath]bool) (*MetaCache, map[util.FullPath]bool, *recordedPaths, *recordedPaths) { t.Helper() @@ -312,6 +421,7 @@ func newTestMetaCache(t *testing.T, cached map[util.FullPath]bool) (*MetaCache, filepath.Join(t.TempDir(), "meta"), mapper, util.FullPath("/"), + false, func(path util.FullPath) { cachedMu.Lock() defer cachedMu.Unlock() diff --git a/weed/mount/meta_cache/meta_cache_init.go b/weed/mount/meta_cache/meta_cache_init.go index 81aad780f..60af3481c 100644 --- a/weed/mount/meta_cache/meta_cache_init.go +++ b/weed/mount/meta_cache/meta_cache_init.go @@ -107,7 +107,7 @@ func doEnsureVisited(ctx context.Context, mc *MetaCache, client filer_pb.FilerCl var err error snapshotTsNs, err = filer_pb.ReadDirAllEntriesWithSnapshot(ctx, client, path, "", func(pbEntry *filer_pb.Entry, isLast bool) error { entry := filer.FromPbEntry(string(path), pbEntry) - if IsHiddenSystemEntry(string(path), entry.Name()) { + if !mc.includeSystemEntries && IsHiddenSystemEntry(string(path), entry.Name()) { return nil } diff --git a/weed/mount/weedfs.go b/weed/mount/weedfs.go index c5a1acef1..877d73f48 100644 --- a/weed/mount/weedfs.go +++ b/weed/mount/weedfs.go @@ -65,6 +65,7 @@ type Option struct { VolumeServerAccess string // how to access volume servers Cipher bool // whether encrypt data on volume server UidGidMapper *meta_cache.UidGidMapper + IncludeSystemEntries bool // Periodic metadata flush interval in seconds (0 to disable) // This protects chunks from being purged by volume.fsck for long-running writes @@ -203,6 +204,7 @@ func NewSeaweedFileSystem(option *Option) *WFS { wfs.metaCache = meta_cache.NewMetaCache(path.Join(option.getUniqueCacheDirForRead(), "meta"), option.UidGidMapper, util.FullPath(option.FilerMountRootPath), + option.IncludeSystemEntries, func(path util.FullPath) { wfs.inodeToPath.MarkChildrenCached(path) }, func(path util.FullPath) bool { diff --git a/weed/mount/weedfs_dir_read.go b/weed/mount/weedfs_dir_read.go index 9488f9aff..488a69f38 100644 --- a/weed/mount/weedfs_dir_read.go +++ b/weed/mount/weedfs_dir_read.go @@ -302,7 +302,7 @@ func (wfs *WFS) readDirectoryDirect(input *fuse.ReadIn, out *fuse.DirEntryList, if input.Offset >= dh.entryStreamOffset { if len(dh.entryStream) == 0 && input.Offset > dh.entryStreamOffset { skipCount := uint32(input.Offset-dh.entryStreamOffset) + batchSize - entries, snapshotTs, err := loadDirectoryEntriesDirect(context.Background(), wfs, wfs.option.UidGidMapper, dirPath, "", false, skipCount, dh.snapshotTsNs) + entries, snapshotTs, err := loadDirectoryEntriesDirect(context.Background(), wfs, wfs.option.UidGidMapper, dirPath, "", false, skipCount, dh.snapshotTsNs, wfs.option.IncludeSystemEntries) if err != nil { glog.Errorf("list filer directory: %v", err) return fuse.EIO @@ -331,7 +331,7 @@ func (wfs *WFS) readDirectoryDirect(input *fuse.ReadIn, out *fuse.DirEntryList, } } - entries, snapshotTs, err := loadDirectoryEntriesDirect(context.Background(), wfs, wfs.option.UidGidMapper, dirPath, lastEntryName, false, batchSize, dh.snapshotTsNs) + entries, snapshotTs, err := loadDirectoryEntriesDirect(context.Background(), wfs, wfs.option.UidGidMapper, dirPath, lastEntryName, false, batchSize, dh.snapshotTsNs, wfs.option.IncludeSystemEntries) if err != nil { glog.Errorf("list filer directory: %v", err) return fuse.EIO @@ -360,13 +360,13 @@ func (wfs *WFS) readDirectoryDirect(input *fuse.ReadIn, out *fuse.DirEntryList, return fuse.OK } -func loadDirectoryEntriesDirect(ctx context.Context, client filer_pb.FilerClient, uidGidMapper *meta_cache.UidGidMapper, dirPath util.FullPath, startFileName string, includeStart bool, limit uint32, snapshotTsNs int64) ([]*filer.Entry, int64, error) { +func loadDirectoryEntriesDirect(ctx context.Context, client filer_pb.FilerClient, uidGidMapper *meta_cache.UidGidMapper, dirPath util.FullPath, startFileName string, includeStart bool, limit uint32, snapshotTsNs int64, includeSystemEntries bool) ([]*filer.Entry, int64, error) { entries := make([]*filer.Entry, 0, limit) var actualSnapshotTsNs int64 err := client.WithFilerClient(false, func(sc filer_pb.SeaweedFilerClient) error { var innerErr error actualSnapshotTsNs, innerErr = filer_pb.DoSeaweedListWithSnapshot(ctx, sc, dirPath, "", func(entry *filer_pb.Entry, isLast bool) error { - if meta_cache.IsHiddenSystemEntry(string(dirPath), entry.Name) { + if !includeSystemEntries && meta_cache.IsHiddenSystemEntry(string(dirPath), entry.Name) { return nil } if uidGidMapper != nil && entry.Attributes != nil { diff --git a/weed/mount/weedfs_dir_read_test.go b/weed/mount/weedfs_dir_read_test.go index e8e8e7d79..b2c792da0 100644 --- a/weed/mount/weedfs_dir_read_test.go +++ b/weed/mount/weedfs_dir_read_test.go @@ -84,7 +84,7 @@ func TestLoadDirectoryEntriesDirectFiltersHiddenEntriesAndMapsIds(t *testing.T) }, } - entries, _, err := loadDirectoryEntriesDirect(context.Background(), client, mapper, util.FullPath("/"), "", false, 10, 0) + entries, _, err := loadDirectoryEntriesDirect(context.Background(), client, mapper, util.FullPath("/"), "", false, 10, 0, false) if err != nil { t.Fatalf("loadDirectoryEntriesDirect: %v", err) } @@ -98,3 +98,25 @@ func TestLoadDirectoryEntriesDirectFiltersHiddenEntriesAndMapsIds(t *testing.T) t.Fatalf("mapped uid/gid = %d/%d, want 10/20", entries[0].Attr.Uid, entries[0].Attr.Gid) } } + +func TestLoadDirectoryEntriesDirectShowsSystemEntriesWhenEnabled(t *testing.T) { + client := &directoryFilerAccessor{ + client: &directoryListClient{ + responses: []*filer_pb.ListEntriesResponse{ + {Entry: &filer_pb.Entry{Name: "topics"}}, + {Entry: &filer_pb.Entry{Name: "visible"}}, + }, + }, + } + + entries, _, err := loadDirectoryEntriesDirect(context.Background(), client, nil, util.FullPath("/"), "", false, 10, 0, true) + if err != nil { + t.Fatalf("loadDirectoryEntriesDirect: %v", err) + } + if got := len(entries); got != 2 { + t.Fatalf("entry count = %d, want 2", got) + } + if entries[0].Name() != "topics" || entries[1].Name() != "visible" { + t.Fatalf("entry names = %q, %q, want topics, visible", entries[0].Name(), entries[1].Name()) + } +} diff --git a/weed/mount/weedfs_file_copy_range_test.go b/weed/mount/weedfs_file_copy_range_test.go index a596eadf8..5142f3f50 100644 --- a/weed/mount/weedfs_file_copy_range_test.go +++ b/weed/mount/weedfs_file_copy_range_test.go @@ -338,6 +338,7 @@ func newCopyRangeTestWFSWithMetaCache(t *testing.T) *WFS { filepath.Join(t.TempDir(), "meta"), uidGidMapper, root, + false, func(path util.FullPath) { wfs.inodeToPath.MarkChildrenCached(path) }, diff --git a/weed/mount/weedfs_file_mkrm_test.go b/weed/mount/weedfs_file_mkrm_test.go index 03506b48f..abfb93d20 100644 --- a/weed/mount/weedfs_file_mkrm_test.go +++ b/weed/mount/weedfs_file_mkrm_test.go @@ -120,6 +120,7 @@ func newCreateTestWFS(t *testing.T) (*WFS, *createEntryTestServer) { filepath.Join(t.TempDir(), "meta"), uidGidMapper, root, + false, func(path util.FullPath) { wfs.inodeToPath.MarkChildrenCached(path) }, diff --git a/weed/mount/weedfs_rename_test.go b/weed/mount/weedfs_rename_test.go index 4b79cc709..75b1e19eb 100644 --- a/weed/mount/weedfs_rename_test.go +++ b/weed/mount/weedfs_rename_test.go @@ -23,6 +23,7 @@ func TestHandleRenameResponseLeavesUncachedTargetOutOfCache(t *testing.T) { filepath.Join(t.TempDir(), "meta"), uidGidMapper, root, + false, func(path util.FullPath) { inodeToPath.MarkChildrenCached(path) },