Browse Source
filer.meta.backup: add -excludePaths flag to skip paths from backup (#7916)
filer.meta.backup: add -excludePaths flag to skip paths from backup (#7916)
* filer.meta.backup: add -excludePaths flag to skip paths from backup Add a new -excludePaths flag that accepts comma-separated path prefixes to exclude from backup operations. This enables selective backup when certain directories (e.g., legacy buckets) should be skipped. Usage: weed filer.meta.backup -filerDir=/buckets -excludePaths=/buckets/legacy1,/buckets/legacy2 -config=backup.toml 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> * filer.meta.backup: address code review feedback for -excludePaths Fixes based on CodeRabbit and Gemini review: - Cache parsed exclude paths in struct (performance) - TrimSpace and skip empty entries (handles "a,,b" and "a, b") - Add trailing slash for directory boundary matching (prevents /buckets/legacy matching /buckets/legacy_backup) - Validate paths start with '/' and warn if not - Log excluded paths at startup for debugging - Fix rename handling: check both old and new paths, handle all four combinations correctly - Add docstring to shouldExclude() - Update UsageLine and Long description with new flag 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> * filer.meta.backup: address nitpick feedback - Clarify directory boundary matching behavior in help text - Add warning when root path '/' is excluded (would exclude everything) 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> * includePrefixes and excludePrefixes --------- Co-authored-by: C Shaw <cliffshaw@users.noreply.github.com> Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com> Co-authored-by: Chris Lu <chris.lu@gmail.com>pull/7920/head
ai8future
1 day ago
committed by
GitHub
GitHub
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 405 additions and 13 deletions
@ -0,0 +1,128 @@ |
|||
package util |
|||
|
|||
import ( |
|||
"strings" |
|||
) |
|||
|
|||
// PathPrefixFilter provides filtering based on include and exclude path prefixes.
|
|||
// When both include and exclude prefixes match a path, the deepest matching prefix wins.
|
|||
// This enables fine-grained control like: exclude /buckets/legacy but include /buckets/legacy/important
|
|||
type PathPrefixFilter struct { |
|||
includePrefixes []string // normalized with trailing /
|
|||
excludePrefixes []string // normalized with trailing /
|
|||
} |
|||
|
|||
// NewPathPrefixFilter creates a new PathPrefixFilter from comma-separated include and exclude prefix strings.
|
|||
// Each prefix is normalized to have a trailing slash for directory boundary matching.
|
|||
// Invalid prefixes (empty or not starting with /) are skipped with a warning via the provided warn function.
|
|||
func NewPathPrefixFilter(includePrefixes, excludePrefixes string, warn func(format string, args ...interface{})) *PathPrefixFilter { |
|||
pf := &PathPrefixFilter{} |
|||
|
|||
pf.includePrefixes = parsePrefixes(includePrefixes, warn) |
|||
pf.excludePrefixes = parsePrefixes(excludePrefixes, warn) |
|||
|
|||
return pf |
|||
} |
|||
|
|||
// parsePrefixes parses a comma-separated list of prefixes and normalizes them.
|
|||
func parsePrefixes(prefixList string, warn func(format string, args ...interface{})) []string { |
|||
if prefixList == "" { |
|||
return nil |
|||
} |
|||
|
|||
var result []string |
|||
for _, p := range strings.Split(prefixList, ",") { |
|||
p = strings.TrimSpace(p) |
|||
if p == "" { |
|||
continue |
|||
} |
|||
if !strings.HasPrefix(p, "/") { |
|||
if warn != nil { |
|||
warn("prefix %q does not start with '/', skipping", p) |
|||
} |
|||
continue |
|||
} |
|||
// Normalize: ensure trailing slash for directory boundary matching
|
|||
if !strings.HasSuffix(p, "/") { |
|||
p = p + "/" |
|||
} |
|||
result = append(result, p) |
|||
} |
|||
return result |
|||
} |
|||
|
|||
// HasFilters returns true if any include or exclude prefixes are configured.
|
|||
func (pf *PathPrefixFilter) HasFilters() bool { |
|||
return len(pf.includePrefixes) > 0 || len(pf.excludePrefixes) > 0 |
|||
} |
|||
|
|||
// ShouldInclude returns true if the path should be included based on the configured prefixes.
|
|||
//
|
|||
// Logic:
|
|||
// - If no filters are configured, include everything.
|
|||
// - Find the deepest matching prefix from either include or exclude list.
|
|||
// - If the deepest match is in includePrefixes, include the path.
|
|||
// - If the deepest match is in excludePrefixes, exclude the path.
|
|||
// - If no match is found and includePrefixes is non-empty, exclude (explicit include required).
|
|||
// - If no match is found and includePrefixes is empty, include (default allow with excludes).
|
|||
func (pf *PathPrefixFilter) ShouldInclude(fullpath string) bool { |
|||
if !pf.HasFilters() { |
|||
return true |
|||
} |
|||
|
|||
// Normalize path for matching
|
|||
checkPath := fullpath |
|||
if !strings.HasSuffix(checkPath, "/") { |
|||
checkPath = checkPath + "/" |
|||
} |
|||
|
|||
// Find deepest matching prefix from each list
|
|||
includeMatch := findDeepestMatch(checkPath, pf.includePrefixes) |
|||
excludeMatch := findDeepestMatch(checkPath, pf.excludePrefixes) |
|||
|
|||
// Determine result based on which match is deeper
|
|||
if includeMatch != "" && excludeMatch != "" { |
|||
// Both matched - deeper prefix wins
|
|||
return len(includeMatch) >= len(excludeMatch) |
|||
} |
|||
|
|||
if includeMatch != "" { |
|||
return true |
|||
} |
|||
|
|||
if excludeMatch != "" { |
|||
return false |
|||
} |
|||
|
|||
// No match found
|
|||
if len(pf.includePrefixes) > 0 { |
|||
// If includes are specified, require explicit include
|
|||
return false |
|||
} |
|||
|
|||
// Default: include if only excludes are specified
|
|||
return true |
|||
} |
|||
|
|||
// findDeepestMatch finds the longest prefix that matches the path.
|
|||
func findDeepestMatch(path string, prefixes []string) string { |
|||
var deepest string |
|||
for _, prefix := range prefixes { |
|||
if strings.HasPrefix(path, prefix) { |
|||
if len(prefix) > len(deepest) { |
|||
deepest = prefix |
|||
} |
|||
} |
|||
} |
|||
return deepest |
|||
} |
|||
|
|||
// GetIncludePrefixes returns the configured include prefixes.
|
|||
func (pf *PathPrefixFilter) GetIncludePrefixes() []string { |
|||
return pf.includePrefixes |
|||
} |
|||
|
|||
// GetExcludePrefixes returns the configured exclude prefixes.
|
|||
func (pf *PathPrefixFilter) GetExcludePrefixes() []string { |
|||
return pf.excludePrefixes |
|||
} |
|||
@ -0,0 +1,201 @@ |
|||
package util |
|||
|
|||
import ( |
|||
"testing" |
|||
) |
|||
|
|||
func TestPathPrefixFilter_Empty(t *testing.T) { |
|||
pf := NewPathPrefixFilter("", "", nil) |
|||
|
|||
if pf.HasFilters() { |
|||
t.Error("empty filter should have no filters") |
|||
} |
|||
|
|||
// Should include everything when no filters
|
|||
tests := []string{"/", "/foo", "/foo/bar", "/buckets/test"} |
|||
for _, path := range tests { |
|||
if !pf.ShouldInclude(path) { |
|||
t.Errorf("empty filter should include %q", path) |
|||
} |
|||
} |
|||
} |
|||
|
|||
func TestPathPrefixFilter_ExcludeOnly(t *testing.T) { |
|||
pf := NewPathPrefixFilter("", "/buckets/legacy,/buckets/old", nil) |
|||
|
|||
tests := []struct { |
|||
path string |
|||
include bool |
|||
}{ |
|||
{"/buckets/active", true}, |
|||
{"/buckets/active/file.txt", true}, |
|||
{"/buckets/legacy", false}, |
|||
{"/buckets/legacy/file.txt", false}, |
|||
{"/buckets/legacy_new", true}, // boundary check: not a prefix match
|
|||
{"/buckets/old", false}, |
|||
{"/buckets/old/data", false}, |
|||
{"/other", true}, |
|||
} |
|||
|
|||
for _, tc := range tests { |
|||
got := pf.ShouldInclude(tc.path) |
|||
if got != tc.include { |
|||
t.Errorf("ShouldInclude(%q) = %v, want %v", tc.path, got, tc.include) |
|||
} |
|||
} |
|||
} |
|||
|
|||
func TestPathPrefixFilter_IncludeOnly(t *testing.T) { |
|||
pf := NewPathPrefixFilter("/buckets/important,/data", "", nil) |
|||
|
|||
tests := []struct { |
|||
path string |
|||
include bool |
|||
}{ |
|||
{"/buckets/important", true}, |
|||
{"/buckets/important/file.txt", true}, |
|||
{"/data", true}, |
|||
{"/data/file.txt", true}, |
|||
{"/buckets/other", false}, // not in include list
|
|||
{"/other", false}, |
|||
} |
|||
|
|||
for _, tc := range tests { |
|||
got := pf.ShouldInclude(tc.path) |
|||
if got != tc.include { |
|||
t.Errorf("ShouldInclude(%q) = %v, want %v", tc.path, got, tc.include) |
|||
} |
|||
} |
|||
} |
|||
|
|||
func TestPathPrefixFilter_DeeperPrefixWins(t *testing.T) { |
|||
// Exclude /buckets/keep but include /buckets/keep/important
|
|||
pf := NewPathPrefixFilter("/buckets/keep/important", "/buckets/keep", nil) |
|||
|
|||
tests := []struct { |
|||
path string |
|||
include bool |
|||
}{ |
|||
{"/buckets/keep", false}, |
|||
{"/buckets/keep/other", false}, |
|||
{"/buckets/keep/important", true}, // deeper include wins
|
|||
{"/buckets/keep/important/file.txt", true}, // deeper include wins
|
|||
{"/buckets/other", false}, // not matched, include required
|
|||
} |
|||
|
|||
for _, tc := range tests { |
|||
got := pf.ShouldInclude(tc.path) |
|||
if got != tc.include { |
|||
t.Errorf("ShouldInclude(%q) = %v, want %v", tc.path, got, tc.include) |
|||
} |
|||
} |
|||
} |
|||
|
|||
func TestPathPrefixFilter_DeeperExcludeWins(t *testing.T) { |
|||
// Include /buckets but exclude /buckets/legacy
|
|||
pf := NewPathPrefixFilter("/buckets", "/buckets/legacy", nil) |
|||
|
|||
tests := []struct { |
|||
path string |
|||
include bool |
|||
}{ |
|||
{"/buckets", true}, |
|||
{"/buckets/active", true}, |
|||
{"/buckets/legacy", false}, // deeper exclude wins
|
|||
{"/buckets/legacy/file.txt", false}, // deeper exclude wins
|
|||
{"/other", false}, // not in include list
|
|||
} |
|||
|
|||
for _, tc := range tests { |
|||
got := pf.ShouldInclude(tc.path) |
|||
if got != tc.include { |
|||
t.Errorf("ShouldInclude(%q) = %v, want %v", tc.path, got, tc.include) |
|||
} |
|||
} |
|||
} |
|||
|
|||
func TestPathPrefixFilter_MultipleOverlappingPrefixes(t *testing.T) { |
|||
// Complex scenario with multiple overlapping prefixes
|
|||
pf := NewPathPrefixFilter( |
|||
"/a,/a/b/c/d", // includes
|
|||
"/a/b,/a/b/c/d/e", // excludes
|
|||
nil, |
|||
) |
|||
|
|||
tests := []struct { |
|||
path string |
|||
include bool |
|||
}{ |
|||
{"/a", true}, // direct include match
|
|||
{"/a/x", true}, // under include /a
|
|||
{"/a/b", false}, // deeper exclude /a/b beats /a
|
|||
{"/a/b/x", false}, // under exclude /a/b
|
|||
{"/a/b/c", false}, // under exclude /a/b
|
|||
{"/a/b/c/d", true}, // deeper include /a/b/c/d beats /a/b
|
|||
{"/a/b/c/d/x", true}, // under include /a/b/c/d
|
|||
{"/a/b/c/d/e", false}, // deeper exclude /a/b/c/d/e beats /a/b/c/d
|
|||
{"/a/b/c/d/e/f", false}, // under exclude /a/b/c/d/e
|
|||
} |
|||
|
|||
for _, tc := range tests { |
|||
got := pf.ShouldInclude(tc.path) |
|||
if got != tc.include { |
|||
t.Errorf("ShouldInclude(%q) = %v, want %v", tc.path, got, tc.include) |
|||
} |
|||
} |
|||
} |
|||
|
|||
func TestPathPrefixFilter_InvalidPrefixes(t *testing.T) { |
|||
var warnings []string |
|||
warn := func(format string, args ...interface{}) { |
|||
warnings = append(warnings, format) |
|||
} |
|||
|
|||
pf := NewPathPrefixFilter("invalid,/valid", "also_invalid", warn) |
|||
|
|||
if len(warnings) != 2 { |
|||
t.Errorf("expected 2 warnings, got %d", len(warnings)) |
|||
} |
|||
|
|||
// Only valid prefix should be stored
|
|||
if len(pf.includePrefixes) != 1 { |
|||
t.Errorf("expected 1 include prefix, got %d", len(pf.includePrefixes)) |
|||
} |
|||
if len(pf.excludePrefixes) != 0 { |
|||
t.Errorf("expected 0 exclude prefixes, got %d", len(pf.excludePrefixes)) |
|||
} |
|||
} |
|||
|
|||
func TestPathPrefixFilter_TrailingSlashNormalization(t *testing.T) { |
|||
pf := NewPathPrefixFilter("/path/to/dir", "/exclude/this/", nil) |
|||
|
|||
// Both should be normalized with trailing slash
|
|||
if pf.includePrefixes[0] != "/path/to/dir/" { |
|||
t.Errorf("include prefix not normalized: %q", pf.includePrefixes[0]) |
|||
} |
|||
if pf.excludePrefixes[0] != "/exclude/this/" { |
|||
t.Errorf("exclude prefix not normalized: %q", pf.excludePrefixes[0]) |
|||
} |
|||
} |
|||
|
|||
func TestPathPrefixFilter_BoundaryMatching(t *testing.T) { |
|||
pf := NewPathPrefixFilter("", "/buckets/legacy1", nil) |
|||
|
|||
tests := []struct { |
|||
path string |
|||
include bool |
|||
}{ |
|||
{"/buckets/legacy1", false}, |
|||
{"/buckets/legacy1/file", false}, |
|||
{"/buckets/legacy1_backup", true}, // not a prefix match due to boundary
|
|||
{"/buckets/legacy10", true}, // not a prefix match due to boundary
|
|||
{"/buckets/legacy", true}, |
|||
} |
|||
|
|||
for _, tc := range tests { |
|||
got := pf.ShouldInclude(tc.path) |
|||
if got != tc.include { |
|||
t.Errorf("ShouldInclude(%q) = %v, want %v", tc.path, got, tc.include) |
|||
} |
|||
} |
|||
} |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue