Browse Source
Worker set its working directory (#8461)
Worker set its working directory (#8461)
* set working directory * consolidate to worker directory * working directory * correct directory name * refactoring to use wildcard matcher * simplify * cleaning ec working directory * fix reference * clean * adjust testpull/8360/merge
Chris Lu
2 days ago
committed by
GitHub
GitHub
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
23 changed files with 559 additions and 815 deletions
-
30weed/command/mini.go
-
6weed/command/plugin_worker_test.go
-
2weed/command/worker.go
-
30weed/command/worker_runtime.go
-
13weed/plugin/worker/erasure_coding_handler.go
-
147weed/plugin/worker/vacuum_handler.go
-
168weed/plugin/worker/volume_metrics.go
-
6weed/s3api/auth_credentials.go
-
6weed/s3api/auth_credentials_test.go
-
7weed/s3api/policy_engine/conditions.go
-
3weed/s3api/policy_engine/engine_test.go
-
23weed/s3api/policy_engine/types.go
-
469weed/s3api/policy_engine/wildcard_matcher_test.go
-
11weed/s3api/s3tables/permissions.go
-
39weed/util/wildcard/filter.go
-
116weed/util/wildcard/wildcard_matcher.go
-
211weed/util/wildcard/wildcard_matcher_test.go
-
27weed/worker/tasks/erasure_coding/detection.go
-
18weed/worker/tasks/erasure_coding/ec_task.go
-
8weed/worker/tasks/vacuum/detection.go
-
11weed/worker/types/base/task.go
-
15weed/worker/types/task.go
-
8weed/worker/worker.go
@ -0,0 +1,168 @@ |
|||||
|
package pluginworker |
||||
|
|
||||
|
import ( |
||||
|
"context" |
||||
|
"fmt" |
||||
|
"sort" |
||||
|
"strings" |
||||
|
"time" |
||||
|
|
||||
|
"github.com/seaweedfs/seaweedfs/weed/admin/topology" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/glog" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/pb" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/util/wildcard" |
||||
|
workertypes "github.com/seaweedfs/seaweedfs/weed/worker/types" |
||||
|
"google.golang.org/grpc" |
||||
|
) |
||||
|
|
||||
|
func collectVolumeMetricsFromMasters( |
||||
|
ctx context.Context, |
||||
|
masterAddresses []string, |
||||
|
collectionFilter string, |
||||
|
grpcDialOption grpc.DialOption, |
||||
|
) ([]*workertypes.VolumeHealthMetrics, *topology.ActiveTopology, error) { |
||||
|
if grpcDialOption == nil { |
||||
|
return nil, nil, fmt.Errorf("grpc dial option is not configured") |
||||
|
} |
||||
|
if len(masterAddresses) == 0 { |
||||
|
return nil, nil, fmt.Errorf("no master addresses provided in cluster context") |
||||
|
} |
||||
|
|
||||
|
for _, masterAddress := range masterAddresses { |
||||
|
response, err := fetchVolumeList(ctx, masterAddress, grpcDialOption) |
||||
|
if err != nil { |
||||
|
glog.Warningf("Plugin worker failed master volume list at %s: %v", masterAddress, err) |
||||
|
continue |
||||
|
} |
||||
|
|
||||
|
metrics, activeTopology, buildErr := buildVolumeMetrics(response, collectionFilter) |
||||
|
if buildErr != nil { |
||||
|
glog.Warningf("Plugin worker failed to build metrics from master %s: %v", masterAddress, buildErr) |
||||
|
continue |
||||
|
} |
||||
|
return metrics, activeTopology, nil |
||||
|
} |
||||
|
|
||||
|
return nil, nil, fmt.Errorf("failed to load topology from all provided masters") |
||||
|
} |
||||
|
|
||||
|
func fetchVolumeList(ctx context.Context, address string, grpcDialOption grpc.DialOption) (*master_pb.VolumeListResponse, error) { |
||||
|
var lastErr error |
||||
|
for _, candidate := range masterAddressCandidates(address) { |
||||
|
if ctx.Err() != nil { |
||||
|
return nil, ctx.Err() |
||||
|
} |
||||
|
|
||||
|
dialCtx, cancelDial := context.WithTimeout(ctx, 5*time.Second) |
||||
|
conn, err := pb.GrpcDial(dialCtx, candidate, false, grpcDialOption) |
||||
|
cancelDial() |
||||
|
if err != nil { |
||||
|
lastErr = err |
||||
|
continue |
||||
|
} |
||||
|
|
||||
|
client := master_pb.NewSeaweedClient(conn) |
||||
|
callCtx, cancelCall := context.WithTimeout(ctx, 10*time.Second) |
||||
|
response, callErr := client.VolumeList(callCtx, &master_pb.VolumeListRequest{}) |
||||
|
cancelCall() |
||||
|
_ = conn.Close() |
||||
|
|
||||
|
if callErr == nil { |
||||
|
return response, nil |
||||
|
} |
||||
|
lastErr = callErr |
||||
|
} |
||||
|
|
||||
|
if lastErr == nil { |
||||
|
lastErr = fmt.Errorf("no valid master address candidate") |
||||
|
} |
||||
|
return nil, lastErr |
||||
|
} |
||||
|
|
||||
|
func buildVolumeMetrics( |
||||
|
response *master_pb.VolumeListResponse, |
||||
|
collectionFilter string, |
||||
|
) ([]*workertypes.VolumeHealthMetrics, *topology.ActiveTopology, error) { |
||||
|
if response == nil || response.TopologyInfo == nil { |
||||
|
return nil, nil, fmt.Errorf("volume list response has no topology info") |
||||
|
} |
||||
|
|
||||
|
activeTopology := topology.NewActiveTopology(10) |
||||
|
if err := activeTopology.UpdateTopology(response.TopologyInfo); err != nil { |
||||
|
return nil, nil, err |
||||
|
} |
||||
|
|
||||
|
patterns := wildcard.CompileWildcardMatchers(collectionFilter) |
||||
|
volumeSizeLimitBytes := uint64(response.VolumeSizeLimitMb) * 1024 * 1024 |
||||
|
now := time.Now() |
||||
|
metrics := make([]*workertypes.VolumeHealthMetrics, 0, 256) |
||||
|
|
||||
|
for _, dc := range response.TopologyInfo.DataCenterInfos { |
||||
|
for _, rack := range dc.RackInfos { |
||||
|
for _, node := range rack.DataNodeInfos { |
||||
|
for diskType, diskInfo := range node.DiskInfos { |
||||
|
for _, volume := range diskInfo.VolumeInfos { |
||||
|
if !wildcard.MatchesAnyWildcard(patterns, volume.Collection) { |
||||
|
continue |
||||
|
} |
||||
|
|
||||
|
metric := &workertypes.VolumeHealthMetrics{ |
||||
|
VolumeID: volume.Id, |
||||
|
Server: node.Id, |
||||
|
ServerAddress: string(pb.NewServerAddressFromDataNode(node)), |
||||
|
DiskType: diskType, |
||||
|
DiskId: volume.DiskId, |
||||
|
DataCenter: dc.Id, |
||||
|
Rack: rack.Id, |
||||
|
Collection: volume.Collection, |
||||
|
Size: volume.Size, |
||||
|
DeletedBytes: volume.DeletedByteCount, |
||||
|
LastModified: time.Unix(volume.ModifiedAtSecond, 0), |
||||
|
ReplicaCount: 1, |
||||
|
ExpectedReplicas: int(volume.ReplicaPlacement), |
||||
|
IsReadOnly: volume.ReadOnly, |
||||
|
} |
||||
|
if metric.Size > 0 { |
||||
|
metric.GarbageRatio = float64(metric.DeletedBytes) / float64(metric.Size) |
||||
|
} |
||||
|
if volumeSizeLimitBytes > 0 { |
||||
|
metric.FullnessRatio = float64(metric.Size) / float64(volumeSizeLimitBytes) |
||||
|
} |
||||
|
metric.Age = now.Sub(metric.LastModified) |
||||
|
metrics = append(metrics, metric) |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
replicaCounts := make(map[uint32]int) |
||||
|
for _, metric := range metrics { |
||||
|
replicaCounts[metric.VolumeID]++ |
||||
|
} |
||||
|
for _, metric := range metrics { |
||||
|
metric.ReplicaCount = replicaCounts[metric.VolumeID] |
||||
|
} |
||||
|
|
||||
|
return metrics, activeTopology, nil |
||||
|
} |
||||
|
|
||||
|
func masterAddressCandidates(address string) []string { |
||||
|
trimmed := strings.TrimSpace(address) |
||||
|
if trimmed == "" { |
||||
|
return nil |
||||
|
} |
||||
|
candidateSet := map[string]struct{}{ |
||||
|
trimmed: {}, |
||||
|
} |
||||
|
converted := pb.ServerToGrpcAddress(trimmed) |
||||
|
candidateSet[converted] = struct{}{} |
||||
|
|
||||
|
candidates := make([]string, 0, len(candidateSet)) |
||||
|
for candidate := range candidateSet { |
||||
|
candidates = append(candidates, candidate) |
||||
|
} |
||||
|
sort.Strings(candidates) |
||||
|
return candidates |
||||
|
} |
||||
@ -1,469 +0,0 @@ |
|||||
package policy_engine |
|
||||
|
|
||||
import ( |
|
||||
"testing" |
|
||||
) |
|
||||
|
|
||||
func TestMatchesWildcard(t *testing.T) { |
|
||||
tests := []struct { |
|
||||
name string |
|
||||
pattern string |
|
||||
str string |
|
||||
expected bool |
|
||||
}{ |
|
||||
// Basic functionality tests
|
|
||||
{ |
|
||||
name: "Exact match", |
|
||||
pattern: "test", |
|
||||
str: "test", |
|
||||
expected: true, |
|
||||
}, |
|
||||
{ |
|
||||
name: "Single wildcard", |
|
||||
pattern: "*", |
|
||||
str: "anything", |
|
||||
expected: true, |
|
||||
}, |
|
||||
{ |
|
||||
name: "Empty string with wildcard", |
|
||||
pattern: "*", |
|
||||
str: "", |
|
||||
expected: true, |
|
||||
}, |
|
||||
|
|
||||
// Star (*) wildcard tests
|
|
||||
{ |
|
||||
name: "Prefix wildcard", |
|
||||
pattern: "test*", |
|
||||
str: "test123", |
|
||||
expected: true, |
|
||||
}, |
|
||||
{ |
|
||||
name: "Suffix wildcard", |
|
||||
pattern: "*test", |
|
||||
str: "123test", |
|
||||
expected: true, |
|
||||
}, |
|
||||
{ |
|
||||
name: "Middle wildcard", |
|
||||
pattern: "test*123", |
|
||||
str: "testABC123", |
|
||||
expected: true, |
|
||||
}, |
|
||||
{ |
|
||||
name: "Multiple wildcards", |
|
||||
pattern: "test*abc*123", |
|
||||
str: "testXYZabcDEF123", |
|
||||
expected: true, |
|
||||
}, |
|
||||
{ |
|
||||
name: "No match", |
|
||||
pattern: "test*", |
|
||||
str: "other", |
|
||||
expected: false, |
|
||||
}, |
|
||||
|
|
||||
// Question mark (?) wildcard tests
|
|
||||
{ |
|
||||
name: "Single question mark", |
|
||||
pattern: "test?", |
|
||||
str: "test1", |
|
||||
expected: true, |
|
||||
}, |
|
||||
{ |
|
||||
name: "Multiple question marks", |
|
||||
pattern: "test??", |
|
||||
str: "test12", |
|
||||
expected: true, |
|
||||
}, |
|
||||
{ |
|
||||
name: "Question mark no match", |
|
||||
pattern: "test?", |
|
||||
str: "test12", |
|
||||
expected: false, |
|
||||
}, |
|
||||
{ |
|
||||
name: "Mixed wildcards", |
|
||||
pattern: "test*abc?def", |
|
||||
str: "testXYZabc1def", |
|
||||
expected: true, |
|
||||
}, |
|
||||
|
|
||||
// Edge cases
|
|
||||
{ |
|
||||
name: "Empty pattern", |
|
||||
pattern: "", |
|
||||
str: "", |
|
||||
expected: true, |
|
||||
}, |
|
||||
{ |
|
||||
name: "Empty pattern with string", |
|
||||
pattern: "", |
|
||||
str: "test", |
|
||||
expected: false, |
|
||||
}, |
|
||||
{ |
|
||||
name: "Pattern with string empty", |
|
||||
pattern: "test", |
|
||||
str: "", |
|
||||
expected: false, |
|
||||
}, |
|
||||
|
|
||||
// Special characters
|
|
||||
{ |
|
||||
name: "Pattern with regex special chars", |
|
||||
pattern: "test[abc]", |
|
||||
str: "test[abc]", |
|
||||
expected: true, |
|
||||
}, |
|
||||
{ |
|
||||
name: "Pattern with dots", |
|
||||
pattern: "test.txt", |
|
||||
str: "test.txt", |
|
||||
expected: true, |
|
||||
}, |
|
||||
{ |
|
||||
name: "Pattern with dots and wildcard", |
|
||||
pattern: "*.txt", |
|
||||
str: "test.txt", |
|
||||
expected: true, |
|
||||
}, |
|
||||
} |
|
||||
|
|
||||
for _, tt := range tests { |
|
||||
t.Run(tt.name, func(t *testing.T) { |
|
||||
result := MatchesWildcard(tt.pattern, tt.str) |
|
||||
if result != tt.expected { |
|
||||
t.Errorf("Pattern %s against %s: expected %v, got %v", tt.pattern, tt.str, tt.expected, result) |
|
||||
} |
|
||||
}) |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
func TestWildcardMatcher(t *testing.T) { |
|
||||
tests := []struct { |
|
||||
name string |
|
||||
pattern string |
|
||||
strings []string |
|
||||
expected []bool |
|
||||
}{ |
|
||||
{ |
|
||||
name: "Simple star pattern", |
|
||||
pattern: "test*", |
|
||||
strings: []string{"test", "test123", "testing", "other"}, |
|
||||
expected: []bool{true, true, true, false}, |
|
||||
}, |
|
||||
{ |
|
||||
name: "Question mark pattern", |
|
||||
pattern: "test?", |
|
||||
strings: []string{"test1", "test2", "test", "test12"}, |
|
||||
expected: []bool{true, true, false, false}, |
|
||||
}, |
|
||||
{ |
|
||||
name: "Mixed pattern", |
|
||||
pattern: "*.txt", |
|
||||
strings: []string{"file.txt", "test.txt", "file.doc", "txt"}, |
|
||||
expected: []bool{true, true, false, false}, |
|
||||
}, |
|
||||
} |
|
||||
|
|
||||
for _, tt := range tests { |
|
||||
t.Run(tt.name, func(t *testing.T) { |
|
||||
matcher, err := NewWildcardMatcher(tt.pattern) |
|
||||
if err != nil { |
|
||||
t.Fatalf("Failed to create matcher: %v", err) |
|
||||
} |
|
||||
|
|
||||
for i, str := range tt.strings { |
|
||||
result := matcher.Match(str) |
|
||||
if result != tt.expected[i] { |
|
||||
t.Errorf("Pattern %s against %s: expected %v, got %v", tt.pattern, str, tt.expected[i], result) |
|
||||
} |
|
||||
} |
|
||||
}) |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
func TestCompileWildcardPattern(t *testing.T) { |
|
||||
tests := []struct { |
|
||||
name string |
|
||||
pattern string |
|
||||
input string |
|
||||
want bool |
|
||||
}{ |
|
||||
{"Star wildcard", "s3:Get*", "s3:GetObject", true}, |
|
||||
{"Question mark wildcard", "s3:Get?bject", "s3:GetObject", true}, |
|
||||
{"Mixed wildcards", "s3:*Object*", "s3:GetObjectAcl", true}, |
|
||||
} |
|
||||
|
|
||||
for _, tt := range tests { |
|
||||
t.Run(tt.name, func(t *testing.T) { |
|
||||
regex, err := CompileWildcardPattern(tt.pattern) |
|
||||
if err != nil { |
|
||||
t.Errorf("CompileWildcardPattern() error = %v", err) |
|
||||
return |
|
||||
} |
|
||||
got := regex.MatchString(tt.input) |
|
||||
if got != tt.want { |
|
||||
t.Errorf("CompileWildcardPattern() = %v, want %v", got, tt.want) |
|
||||
} |
|
||||
}) |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
// BenchmarkWildcardMatchingPerformance demonstrates the performance benefits of caching
|
|
||||
func BenchmarkWildcardMatchingPerformance(b *testing.B) { |
|
||||
patterns := []string{ |
|
||||
"s3:Get*", |
|
||||
"s3:Put*", |
|
||||
"s3:Delete*", |
|
||||
"s3:List*", |
|
||||
"arn:aws:s3:::bucket/*", |
|
||||
"arn:aws:s3:::bucket/prefix*", |
|
||||
"user:*", |
|
||||
"user:admin-*", |
|
||||
} |
|
||||
|
|
||||
inputs := []string{ |
|
||||
"s3:GetObject", |
|
||||
"s3:PutObject", |
|
||||
"s3:DeleteObject", |
|
||||
"s3:ListBucket", |
|
||||
"arn:aws:s3:::bucket/file.txt", |
|
||||
"arn:aws:s3:::bucket/prefix/file.txt", |
|
||||
"user:admin", |
|
||||
"user:admin-john", |
|
||||
} |
|
||||
|
|
||||
b.Run("WithoutCache", func(b *testing.B) { |
|
||||
for i := 0; i < b.N; i++ { |
|
||||
for _, pattern := range patterns { |
|
||||
for _, input := range inputs { |
|
||||
MatchesWildcard(pattern, input) |
|
||||
} |
|
||||
} |
|
||||
} |
|
||||
}) |
|
||||
|
|
||||
b.Run("WithCache", func(b *testing.B) { |
|
||||
for i := 0; i < b.N; i++ { |
|
||||
for _, pattern := range patterns { |
|
||||
for _, input := range inputs { |
|
||||
FastMatchesWildcard(pattern, input) |
|
||||
} |
|
||||
} |
|
||||
} |
|
||||
}) |
|
||||
} |
|
||||
|
|
||||
// BenchmarkWildcardMatcherReuse demonstrates the performance benefits of reusing WildcardMatcher instances
|
|
||||
func BenchmarkWildcardMatcherReuse(b *testing.B) { |
|
||||
pattern := "s3:Get*" |
|
||||
input := "s3:GetObject" |
|
||||
|
|
||||
b.Run("NewMatcherEveryTime", func(b *testing.B) { |
|
||||
for i := 0; i < b.N; i++ { |
|
||||
matcher, _ := NewWildcardMatcher(pattern) |
|
||||
matcher.Match(input) |
|
||||
} |
|
||||
}) |
|
||||
|
|
||||
b.Run("CachedMatcher", func(b *testing.B) { |
|
||||
for i := 0; i < b.N; i++ { |
|
||||
matcher, _ := GetCachedWildcardMatcher(pattern) |
|
||||
matcher.Match(input) |
|
||||
} |
|
||||
}) |
|
||||
} |
|
||||
|
|
||||
// TestWildcardMatcherCaching verifies that caching works correctly
|
|
||||
func TestWildcardMatcherCaching(t *testing.T) { |
|
||||
pattern := "s3:Get*" |
|
||||
|
|
||||
// Get the first matcher
|
|
||||
matcher1, err := GetCachedWildcardMatcher(pattern) |
|
||||
if err != nil { |
|
||||
t.Fatalf("Failed to get cached matcher: %v", err) |
|
||||
} |
|
||||
|
|
||||
// Get the second matcher - should be the same instance
|
|
||||
matcher2, err := GetCachedWildcardMatcher(pattern) |
|
||||
if err != nil { |
|
||||
t.Fatalf("Failed to get cached matcher: %v", err) |
|
||||
} |
|
||||
|
|
||||
// Check that they're the same instance (same pointer)
|
|
||||
if matcher1 != matcher2 { |
|
||||
t.Errorf("Expected same matcher instance, got different instances") |
|
||||
} |
|
||||
|
|
||||
// Test that both matchers work correctly
|
|
||||
testInput := "s3:GetObject" |
|
||||
if !matcher1.Match(testInput) { |
|
||||
t.Errorf("First matcher failed to match %s", testInput) |
|
||||
} |
|
||||
if !matcher2.Match(testInput) { |
|
||||
t.Errorf("Second matcher failed to match %s", testInput) |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
// TestFastMatchesWildcard verifies that the fast matching function works correctly
|
|
||||
func TestFastMatchesWildcard(t *testing.T) { |
|
||||
tests := []struct { |
|
||||
pattern string |
|
||||
input string |
|
||||
want bool |
|
||||
}{ |
|
||||
{"s3:Get*", "s3:GetObject", true}, |
|
||||
{"s3:Put*", "s3:GetObject", false}, |
|
||||
{"arn:aws:s3:::bucket/*", "arn:aws:s3:::bucket/file.txt", true}, |
|
||||
{"user:admin-*", "user:admin-john", true}, |
|
||||
{"user:admin-*", "user:guest-john", false}, |
|
||||
} |
|
||||
|
|
||||
for _, tt := range tests { |
|
||||
t.Run(tt.pattern+"_"+tt.input, func(t *testing.T) { |
|
||||
got := FastMatchesWildcard(tt.pattern, tt.input) |
|
||||
if got != tt.want { |
|
||||
t.Errorf("FastMatchesWildcard(%q, %q) = %v, want %v", tt.pattern, tt.input, got, tt.want) |
|
||||
} |
|
||||
}) |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
// TestWildcardMatcherCacheBounding tests the bounded cache functionality
|
|
||||
func TestWildcardMatcherCacheBounding(t *testing.T) { |
|
||||
// Clear cache before test
|
|
||||
wildcardMatcherCache.ClearCache() |
|
||||
|
|
||||
// Get original max size
|
|
||||
originalMaxSize := wildcardMatcherCache.maxSize |
|
||||
|
|
||||
// Set a small max size for testing
|
|
||||
wildcardMatcherCache.maxSize = 3 |
|
||||
defer func() { |
|
||||
wildcardMatcherCache.maxSize = originalMaxSize |
|
||||
wildcardMatcherCache.ClearCache() |
|
||||
}() |
|
||||
|
|
||||
// Add patterns up to max size
|
|
||||
patterns := []string{"pattern1", "pattern2", "pattern3"} |
|
||||
for _, pattern := range patterns { |
|
||||
_, err := GetCachedWildcardMatcher(pattern) |
|
||||
if err != nil { |
|
||||
t.Fatalf("Failed to get cached matcher for %s: %v", pattern, err) |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
// Verify cache size
|
|
||||
size, maxSize := wildcardMatcherCache.GetCacheStats() |
|
||||
if size != 3 { |
|
||||
t.Errorf("Expected cache size 3, got %d", size) |
|
||||
} |
|
||||
if maxSize != 3 { |
|
||||
t.Errorf("Expected max size 3, got %d", maxSize) |
|
||||
} |
|
||||
|
|
||||
// Add another pattern, should evict the least recently used
|
|
||||
_, err := GetCachedWildcardMatcher("pattern4") |
|
||||
if err != nil { |
|
||||
t.Fatalf("Failed to get cached matcher for pattern4: %v", err) |
|
||||
} |
|
||||
|
|
||||
// Cache should still be at max size
|
|
||||
size, _ = wildcardMatcherCache.GetCacheStats() |
|
||||
if size != 3 { |
|
||||
t.Errorf("Expected cache size 3 after eviction, got %d", size) |
|
||||
} |
|
||||
|
|
||||
// The first pattern should have been evicted
|
|
||||
wildcardMatcherCache.mu.RLock() |
|
||||
if _, exists := wildcardMatcherCache.matchers["pattern1"]; exists { |
|
||||
t.Errorf("Expected pattern1 to be evicted, but it still exists") |
|
||||
} |
|
||||
if _, exists := wildcardMatcherCache.matchers["pattern4"]; !exists { |
|
||||
t.Errorf("Expected pattern4 to be in cache, but it doesn't exist") |
|
||||
} |
|
||||
wildcardMatcherCache.mu.RUnlock() |
|
||||
} |
|
||||
|
|
||||
// TestWildcardMatcherCacheLRU tests the LRU eviction policy
|
|
||||
func TestWildcardMatcherCacheLRU(t *testing.T) { |
|
||||
// Clear cache before test
|
|
||||
wildcardMatcherCache.ClearCache() |
|
||||
|
|
||||
// Get original max size
|
|
||||
originalMaxSize := wildcardMatcherCache.maxSize |
|
||||
|
|
||||
// Set a small max size for testing
|
|
||||
wildcardMatcherCache.maxSize = 3 |
|
||||
defer func() { |
|
||||
wildcardMatcherCache.maxSize = originalMaxSize |
|
||||
wildcardMatcherCache.ClearCache() |
|
||||
}() |
|
||||
|
|
||||
// Add patterns to fill cache
|
|
||||
patterns := []string{"pattern1", "pattern2", "pattern3"} |
|
||||
for _, pattern := range patterns { |
|
||||
_, err := GetCachedWildcardMatcher(pattern) |
|
||||
if err != nil { |
|
||||
t.Fatalf("Failed to get cached matcher for %s: %v", pattern, err) |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
// Access pattern1 to make it most recently used
|
|
||||
_, err := GetCachedWildcardMatcher("pattern1") |
|
||||
if err != nil { |
|
||||
t.Fatalf("Failed to access pattern1: %v", err) |
|
||||
} |
|
||||
|
|
||||
// Add another pattern, should evict pattern2 (now least recently used)
|
|
||||
_, err = GetCachedWildcardMatcher("pattern4") |
|
||||
if err != nil { |
|
||||
t.Fatalf("Failed to get cached matcher for pattern4: %v", err) |
|
||||
} |
|
||||
|
|
||||
// pattern1 should still be in cache (was accessed recently)
|
|
||||
// pattern2 should be evicted (was least recently used)
|
|
||||
wildcardMatcherCache.mu.RLock() |
|
||||
if _, exists := wildcardMatcherCache.matchers["pattern1"]; !exists { |
|
||||
t.Errorf("Expected pattern1 to remain in cache (most recently used)") |
|
||||
} |
|
||||
if _, exists := wildcardMatcherCache.matchers["pattern2"]; exists { |
|
||||
t.Errorf("Expected pattern2 to be evicted (least recently used)") |
|
||||
} |
|
||||
if _, exists := wildcardMatcherCache.matchers["pattern3"]; !exists { |
|
||||
t.Errorf("Expected pattern3 to remain in cache") |
|
||||
} |
|
||||
if _, exists := wildcardMatcherCache.matchers["pattern4"]; !exists { |
|
||||
t.Errorf("Expected pattern4 to be in cache") |
|
||||
} |
|
||||
wildcardMatcherCache.mu.RUnlock() |
|
||||
} |
|
||||
|
|
||||
// TestWildcardMatcherCacheClear tests the cache clearing functionality
|
|
||||
func TestWildcardMatcherCacheClear(t *testing.T) { |
|
||||
// Add some patterns to cache
|
|
||||
patterns := []string{"pattern1", "pattern2", "pattern3"} |
|
||||
for _, pattern := range patterns { |
|
||||
_, err := GetCachedWildcardMatcher(pattern) |
|
||||
if err != nil { |
|
||||
t.Fatalf("Failed to get cached matcher for %s: %v", pattern, err) |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
// Verify cache has patterns
|
|
||||
size, _ := wildcardMatcherCache.GetCacheStats() |
|
||||
if size == 0 { |
|
||||
t.Errorf("Expected cache to have patterns before clearing") |
|
||||
} |
|
||||
|
|
||||
// Clear cache
|
|
||||
wildcardMatcherCache.ClearCache() |
|
||||
|
|
||||
// Verify cache is empty
|
|
||||
size, _ = wildcardMatcherCache.GetCacheStats() |
|
||||
if size != 0 { |
|
||||
t.Errorf("Expected cache to be empty after clearing, got size %d", size) |
|
||||
} |
|
||||
} |
|
||||
@ -0,0 +1,39 @@ |
|||||
|
package wildcard |
||||
|
|
||||
|
import "strings" |
||||
|
|
||||
|
// CompileWildcardMatchers parses comma-separated wildcard patterns and compiles them.
|
||||
|
// Empty tokens are ignored. Invalid patterns are skipped.
|
||||
|
func CompileWildcardMatchers(filter string) []*WildcardMatcher { |
||||
|
parts := strings.Split(filter, ",") |
||||
|
matchers := make([]*WildcardMatcher, 0, len(parts)) |
||||
|
for _, part := range parts { |
||||
|
trimmed := strings.TrimSpace(part) |
||||
|
if trimmed == "" { |
||||
|
continue |
||||
|
} |
||||
|
matcher, err := NewWildcardMatcher(trimmed) |
||||
|
if err != nil { |
||||
|
continue |
||||
|
} |
||||
|
matchers = append(matchers, matcher) |
||||
|
} |
||||
|
if len(matchers) == 0 { |
||||
|
return nil |
||||
|
} |
||||
|
return matchers |
||||
|
} |
||||
|
|
||||
|
// MatchesAnyWildcard returns true when no matcher is provided,
|
||||
|
// or when any matcher matches the given value.
|
||||
|
func MatchesAnyWildcard(matchers []*WildcardMatcher, value string) bool { |
||||
|
if len(matchers) == 0 { |
||||
|
return true |
||||
|
} |
||||
|
for _, matcher := range matchers { |
||||
|
if matcher != nil && matcher.Match(value) { |
||||
|
return true |
||||
|
} |
||||
|
} |
||||
|
return false |
||||
|
} |
||||
@ -0,0 +1,211 @@ |
|||||
|
package wildcard |
||||
|
|
||||
|
import "testing" |
||||
|
|
||||
|
func TestMatchesWildcard(t *testing.T) { |
||||
|
tests := []struct { |
||||
|
name string |
||||
|
pattern string |
||||
|
str string |
||||
|
expected bool |
||||
|
}{ |
||||
|
{"Exact match", "test", "test", true}, |
||||
|
{"Single wildcard", "*", "anything", true}, |
||||
|
{"Empty string with wildcard", "*", "", true}, |
||||
|
{"Prefix wildcard", "test*", "test123", true}, |
||||
|
{"Suffix wildcard", "*test", "123test", true}, |
||||
|
{"Middle wildcard", "test*123", "testABC123", true}, |
||||
|
{"Multiple wildcards", "test*abc*123", "testXYZabcDEF123", true}, |
||||
|
{"No match", "test*", "other", false}, |
||||
|
{"Single question mark", "test?", "test1", true}, |
||||
|
{"Multiple question marks", "test??", "test12", true}, |
||||
|
{"Question mark no match", "test?", "test12", false}, |
||||
|
{"Mixed wildcards", "test*abc?def", "testXYZabc1def", true}, |
||||
|
{"Empty pattern", "", "", true}, |
||||
|
{"Empty pattern with string", "", "test", false}, |
||||
|
{"Pattern with string empty", "test", "", false}, |
||||
|
{"Pattern with regex special chars", "test[abc]", "test[abc]", true}, |
||||
|
{"Pattern with dots", "test.txt", "test.txt", true}, |
||||
|
{"Pattern with dots and wildcard", "*.txt", "test.txt", true}, |
||||
|
} |
||||
|
for _, tt := range tests { |
||||
|
t.Run(tt.name, func(t *testing.T) { |
||||
|
got := MatchesWildcard(tt.pattern, tt.str) |
||||
|
if got != tt.expected { |
||||
|
t.Errorf("MatchesWildcard(%q, %q) = %v, want %v", tt.pattern, tt.str, got, tt.expected) |
||||
|
} |
||||
|
}) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func TestWildcardMatcherMatch(t *testing.T) { |
||||
|
tests := []struct { |
||||
|
name string |
||||
|
pattern string |
||||
|
inputs []string |
||||
|
expected []bool |
||||
|
}{ |
||||
|
{"Simple star", "test*", []string{"test", "test123", "testing", "other"}, []bool{true, true, true, false}}, |
||||
|
{"Question mark", "test?", []string{"test1", "test2", "test", "test12"}, []bool{true, true, false, false}}, |
||||
|
{"Extension filter", "*.txt", []string{"file.txt", "test.txt", "file.doc", "txt"}, []bool{true, true, false, false}}, |
||||
|
} |
||||
|
for _, tt := range tests { |
||||
|
t.Run(tt.name, func(t *testing.T) { |
||||
|
m, err := NewWildcardMatcher(tt.pattern) |
||||
|
if err != nil { |
||||
|
t.Fatalf("NewWildcardMatcher: %v", err) |
||||
|
} |
||||
|
for i, s := range tt.inputs { |
||||
|
got := m.Match(s) |
||||
|
if got != tt.expected[i] { |
||||
|
t.Errorf("Match(%q) = %v, want %v", s, got, tt.expected[i]) |
||||
|
} |
||||
|
} |
||||
|
}) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func TestCompileWildcardPattern(t *testing.T) { |
||||
|
tests := []struct { |
||||
|
pattern string |
||||
|
input string |
||||
|
want bool |
||||
|
}{ |
||||
|
{"s3:Get*", "s3:GetObject", true}, |
||||
|
{"s3:Get?bject", "s3:GetObject", true}, |
||||
|
{"s3:*Object*", "s3:GetObjectAcl", true}, |
||||
|
} |
||||
|
for _, tt := range tests { |
||||
|
t.Run(tt.pattern, func(t *testing.T) { |
||||
|
re, err := CompileWildcardPattern(tt.pattern) |
||||
|
if err != nil { |
||||
|
t.Fatalf("CompileWildcardPattern: %v", err) |
||||
|
} |
||||
|
if got := re.MatchString(tt.input); got != tt.want { |
||||
|
t.Errorf("got %v, want %v", got, tt.want) |
||||
|
} |
||||
|
}) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func TestFastMatchesWildcard(t *testing.T) { |
||||
|
tests := []struct { |
||||
|
pattern string |
||||
|
input string |
||||
|
want bool |
||||
|
}{ |
||||
|
{"s3:Get*", "s3:GetObject", true}, |
||||
|
{"s3:Put*", "s3:GetObject", false}, |
||||
|
{"arn:aws:s3:::bucket/*", "arn:aws:s3:::bucket/file.txt", true}, |
||||
|
{"user:admin-*", "user:admin-john", true}, |
||||
|
{"user:admin-*", "user:guest-john", false}, |
||||
|
} |
||||
|
for _, tt := range tests { |
||||
|
t.Run(tt.pattern+"_"+tt.input, func(t *testing.T) { |
||||
|
got := FastMatchesWildcard(tt.pattern, tt.input) |
||||
|
if got != tt.want { |
||||
|
t.Errorf("FastMatchesWildcard(%q, %q) = %v, want %v", tt.pattern, tt.input, got, tt.want) |
||||
|
} |
||||
|
}) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func TestWildcardMatcherCaching(t *testing.T) { |
||||
|
m1, err := GetCachedWildcardMatcher("s3:Get*") |
||||
|
if err != nil { |
||||
|
t.Fatal(err) |
||||
|
} |
||||
|
m2, err := GetCachedWildcardMatcher("s3:Get*") |
||||
|
if err != nil { |
||||
|
t.Fatal(err) |
||||
|
} |
||||
|
if m1 != m2 { |
||||
|
t.Error("expected same cached instance") |
||||
|
} |
||||
|
if !m1.Match("s3:GetObject") { |
||||
|
t.Error("expected match") |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func TestWildcardMatcherCacheBounding(t *testing.T) { |
||||
|
wildcardMatcherCache.ClearCache() |
||||
|
orig := wildcardMatcherCache.maxSize |
||||
|
wildcardMatcherCache.maxSize = 3 |
||||
|
defer func() { |
||||
|
wildcardMatcherCache.maxSize = orig |
||||
|
wildcardMatcherCache.ClearCache() |
||||
|
}() |
||||
|
for _, p := range []string{"p1", "p2", "p3"} { |
||||
|
GetCachedWildcardMatcher(p) |
||||
|
} |
||||
|
size, maxSize := wildcardMatcherCache.GetCacheStats() |
||||
|
if size != 3 { |
||||
|
t.Errorf("expected size 3, got %d", size) |
||||
|
} |
||||
|
if maxSize != 3 { |
||||
|
t.Errorf("expected maxSize 3, got %d", maxSize) |
||||
|
} |
||||
|
GetCachedWildcardMatcher("p4") |
||||
|
size, _ = wildcardMatcherCache.GetCacheStats() |
||||
|
if size != 3 { |
||||
|
t.Errorf("expected size 3 after eviction, got %d", size) |
||||
|
} |
||||
|
wildcardMatcherCache.mu.RLock() |
||||
|
defer wildcardMatcherCache.mu.RUnlock() |
||||
|
if _, ok := wildcardMatcherCache.matchers["p1"]; ok { |
||||
|
t.Error("p1 should have been evicted (LRU)") |
||||
|
} |
||||
|
if _, ok := wildcardMatcherCache.matchers["p4"]; !ok { |
||||
|
t.Error("p4 should be in cache") |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func TestWildcardMatcherCacheLRU(t *testing.T) { |
||||
|
wildcardMatcherCache.ClearCache() |
||||
|
orig := wildcardMatcherCache.maxSize |
||||
|
wildcardMatcherCache.maxSize = 3 |
||||
|
defer func() { |
||||
|
wildcardMatcherCache.maxSize = orig |
||||
|
wildcardMatcherCache.ClearCache() |
||||
|
}() |
||||
|
for _, p := range []string{"p1", "p2", "p3"} { |
||||
|
GetCachedWildcardMatcher(p) |
||||
|
} |
||||
|
GetCachedWildcardMatcher("p1") // access p1 to make it most-recently used
|
||||
|
GetCachedWildcardMatcher("p4") // should evict p2 (now LRU)
|
||||
|
wildcardMatcherCache.mu.RLock() |
||||
|
defer wildcardMatcherCache.mu.RUnlock() |
||||
|
if _, ok := wildcardMatcherCache.matchers["p2"]; ok { |
||||
|
t.Error("p2 should be evicted (least recently used)") |
||||
|
} |
||||
|
if _, ok := wildcardMatcherCache.matchers["p1"]; !ok { |
||||
|
t.Error("p1 should remain (recently accessed)") |
||||
|
} |
||||
|
if _, ok := wildcardMatcherCache.matchers["p3"]; !ok { |
||||
|
t.Error("p3 should remain") |
||||
|
} |
||||
|
if _, ok := wildcardMatcherCache.matchers["p4"]; !ok { |
||||
|
t.Error("p4 should be in cache") |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func TestWildcardMatcherCacheClear(t *testing.T) { |
||||
|
GetCachedWildcardMatcher("test") |
||||
|
wildcardMatcherCache.ClearCache() |
||||
|
size, _ := wildcardMatcherCache.GetCacheStats() |
||||
|
if size != 0 { |
||||
|
t.Errorf("expected 0 after clear, got %d", size) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func BenchmarkMatchesWildcard(b *testing.B) { |
||||
|
for i := 0; i < b.N; i++ { |
||||
|
MatchesWildcard("s3:Get*", "s3:GetObject") |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func BenchmarkFastMatchesWildcard(b *testing.B) { |
||||
|
for i := 0; i < b.N; i++ { |
||||
|
FastMatchesWildcard("s3:Get*", "s3:GetObject") |
||||
|
} |
||||
|
} |
||||
Write
Preview
Loading…
Cancel
Save
Reference in new issue