Mirror
/
seaweedfs
mirror of https://github.com/seaweedfs/seaweedfs.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

fmt

improve-fuse-mount
chrislu 3 months ago
parent
e7f5fff989
commit
63b94321ec
6 changed files with 485 additions and 487 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 44
      weed/mount/ml/cache_policy.go
  2. 46
      weed/mount/ml/cache_policy_test.go
  3. 36
      weed/mount/ml/fuse_integration.go
  4. 124
      weed/mount/ml/open_file_cache.go
  5. 46
      weed/mount/ml/open_file_cache_test.go
  6. 6
      weed/mount/ml_integration.go

44
weed/mount/ml/cache_policy.go
View File

@ -19,10 +19,10 @@ type CacheEntry struct {
IsHot bool // Whether this is a hot chunk IsHot bool // Whether this is a hot chunk
// ML-specific metadata // ML-specific metadata
IsTrainingData bool // Whether this is training data
IsModel bool // Whether this is a model file
PredictedReuse float64 // Predicted reuse probability (0.0-1.0)
EpochRelevance float64 // Relevance for current training epoch
IsTrainingData bool // Whether this is training data
IsModel bool // Whether this is a model file
PredictedReuse float64 // Predicted reuse probability (0.0-1.0)
EpochRelevance float64 // Relevance for current training epoch
} }
// MLCachePolicy implements ML-aware cache eviction policy // MLCachePolicy implements ML-aware cache eviction policy
@ -34,18 +34,18 @@ type MLCachePolicy struct {
mlWeight float64 // Weight for ML-specific factors mlWeight float64 // Weight for ML-specific factors
// ML-specific parameters // ML-specific parameters
trainingDataBoost float64 // Boost factor for training data
modelFileBoost float64 // Boost factor for model files
sequentialBoost float64 // Boost factor for sequential access
epochRelevanceBoost float64 // Boost factor for epoch-relevant data
trainingDataBoost float64 // Boost factor for training data
modelFileBoost float64 // Boost factor for model files
sequentialBoost float64 // Boost factor for sequential access
epochRelevanceBoost float64 // Boost factor for epoch-relevant data
// Time-based parameters // Time-based parameters
hotThreshold time.Duration // Threshold for considering item "hot"
coldThreshold time.Duration // Threshold for considering item "cold"
hotThreshold time.Duration // Threshold for considering item "hot"
coldThreshold time.Duration // Threshold for considering item "cold"
// Size-based parameters // Size-based parameters
largeFileThreshold uint64 // Threshold for large files
smallFilePreference float64 // Preference for keeping small files
largeFileThreshold uint64 // Threshold for large files
smallFilePreference float64 // Preference for keeping small files
// Statistics // Statistics
totalEvictions int64 totalEvictions int64
@ -60,8 +60,8 @@ func NewMLCachePolicy() *MLCachePolicy {
// Balanced weights // Balanced weights
accessFrequencyWeight: 0.3, accessFrequencyWeight: 0.3,
recencyWeight: 0.3, recencyWeight: 0.3,
sizeWeight: 0.2,
mlWeight: 0.2,
sizeWeight: 0.2,
mlWeight: 0.2,
// ML-specific boosts // ML-specific boosts
trainingDataBoost: 1.5, // 50% boost for training data trainingDataBoost: 1.5, // 50% boost for training data
@ -264,23 +264,23 @@ func (policy *MLCachePolicy) GetEvictionMetrics() MLCachePolicyMetrics {
// Configuration // Configuration
AccessFrequencyWeight: policy.accessFrequencyWeight, AccessFrequencyWeight: policy.accessFrequencyWeight,
RecencyWeight: policy.recencyWeight, RecencyWeight: policy.recencyWeight,
SizeWeight: policy.sizeWeight,
MLWeight: policy.mlWeight,
SizeWeight: policy.sizeWeight,
MLWeight: policy.mlWeight,
} }
} }
// MLCachePolicyMetrics holds metrics for the ML cache policy // MLCachePolicyMetrics holds metrics for the ML cache policy
type MLCachePolicyMetrics struct { type MLCachePolicyMetrics struct {
TotalEvictions int64 `json:"total_evictions"`
MLFileEvictions int64 `json:"ml_file_evictions"`
TrainingDataEvictions int64 `json:"training_data_evictions"`
ModelFileEvictions int64 `json:"model_file_evictions"`
TotalEvictions int64 `json:"total_evictions"`
MLFileEvictions int64 `json:"ml_file_evictions"`
TrainingDataEvictions int64 `json:"training_data_evictions"`
ModelFileEvictions int64 `json:"model_file_evictions"`
// Configuration weights // Configuration weights
AccessFrequencyWeight float64 `json:"access_frequency_weight"` AccessFrequencyWeight float64 `json:"access_frequency_weight"`
RecencyWeight float64 `json:"recency_weight"` RecencyWeight float64 `json:"recency_weight"`
SizeWeight float64 `json:"size_weight"`
MLWeight float64 `json:"ml_weight"`
SizeWeight float64 `json:"size_weight"`
MLWeight float64 `json:"ml_weight"`
} }
// SetWeights updates the eviction policy weights // SetWeights updates the eviction policy weights

46
weed/mount/ml/cache_policy_test.go
View File

@ -79,20 +79,20 @@ func TestMLCachePolicy_TrainingDataBoost(t *testing.T) {
policy := NewMLCachePolicy() policy := NewMLCachePolicy()
regularEntry := &CacheEntry{ regularEntry := &CacheEntry{
Inode: 1,
Size: 1024,
LastAccess: time.Now().Add(-2 * time.Minute),
AccessCount: 10,
FileType: MLFileUnknown,
Inode: 1,
Size: 1024,
LastAccess: time.Now().Add(-2 * time.Minute),
AccessCount: 10,
FileType: MLFileUnknown,
IsTrainingData: false, IsTrainingData: false,
} }
trainingEntry := &CacheEntry{ trainingEntry := &CacheEntry{
Inode: 2,
Size: 1024,
LastAccess: time.Now().Add(-2 * time.Minute),
AccessCount: 10,
FileType: MLFileDataset,
Inode: 2,
Size: 1024,
LastAccess: time.Now().Add(-2 * time.Minute),
AccessCount: 10,
FileType: MLFileDataset,
IsTrainingData: true, IsTrainingData: true,
} }
@ -186,7 +186,7 @@ func TestMLCachePolicy_RecencyDecay(t *testing.T) {
// Create entries with different access times // Create entries with different access times
recentEntry := &CacheEntry{ recentEntry := &CacheEntry{
Inode: 1,
Inode: 1,
Size: 1024, Size: 1024,
LastAccess: time.Now(), LastAccess: time.Now(),
@ -195,7 +195,7 @@ func TestMLCachePolicy_RecencyDecay(t *testing.T) {
} }
oldEntry := &CacheEntry{ oldEntry := &CacheEntry{
Inode: 2,
Inode: 2,
Size: 1024, Size: 1024,
LastAccess: time.Now().Add(-20 * time.Minute), LastAccess: time.Now().Add(-20 * time.Minute),
@ -216,7 +216,7 @@ func TestMLCachePolicy_EpochRelevance(t *testing.T) {
policy := NewMLCachePolicy() policy := NewMLCachePolicy()
lowRelevanceEntry := &CacheEntry{ lowRelevanceEntry := &CacheEntry{
Inode: 1,
Inode: 1,
Size: 1024, Size: 1024,
LastAccess: time.Now(), LastAccess: time.Now(),
@ -226,7 +226,7 @@ func TestMLCachePolicy_EpochRelevance(t *testing.T) {
} }
highRelevanceEntry := &CacheEntry{ highRelevanceEntry := &CacheEntry{
Inode: 2,
Inode: 2,
Size: 1024, Size: 1024,
LastAccess: time.Now(), LastAccess: time.Now(),
@ -249,7 +249,7 @@ func TestMLCachePolicy_DifferentThresholds(t *testing.T) {
// Create entries for different file types with same base score // Create entries for different file types with same base score
unknownEntry := &CacheEntry{ unknownEntry := &CacheEntry{
Inode: 1,
Inode: 1,
Size: 1024, Size: 1024,
LastAccess: time.Now().Add(-15 * time.Minute), // Old enough to potentially evict LastAccess: time.Now().Add(-15 * time.Minute), // Old enough to potentially evict
@ -258,7 +258,7 @@ func TestMLCachePolicy_DifferentThresholds(t *testing.T) {
} }
modelEntry := &CacheEntry{ modelEntry := &CacheEntry{
Inode: 2,
Inode: 2,
Size: 1024, Size: 1024,
LastAccess: time.Now().Add(-15 * time.Minute), LastAccess: time.Now().Add(-15 * time.Minute),
@ -268,7 +268,7 @@ func TestMLCachePolicy_DifferentThresholds(t *testing.T) {
} }
datasetEntry := &CacheEntry{ datasetEntry := &CacheEntry{
Inode: 3,
Inode: 3,
Size: 1024, Size: 1024,
LastAccess: time.Now().Add(-15 * time.Minute), LastAccess: time.Now().Add(-15 * time.Minute),
@ -316,7 +316,7 @@ func TestMLCachePolicy_SetWeights(t *testing.T) {
policy.SetWeights(2.0, 2.0, 1.0, 1.0) // Total = 6.0 policy.SetWeights(2.0, 2.0, 1.0, 1.0) // Total = 6.0
expectedFreq := 2.0 / 6.0 expectedFreq := 2.0 / 6.0
if abs(policy.accessFrequencyWeight - expectedFreq) > 0.001 {
if abs(policy.accessFrequencyWeight-expectedFreq) > 0.001 {
t.Errorf("Expected normalized frequency weight %.3f, got %.3f", t.Errorf("Expected normalized frequency weight %.3f, got %.3f",
expectedFreq, policy.accessFrequencyWeight) expectedFreq, policy.accessFrequencyWeight)
} }
@ -381,7 +381,7 @@ func TestMLCachePolicy_HotChunkPreference(t *testing.T) {
policy := NewMLCachePolicy() policy := NewMLCachePolicy()
coldEntry := &CacheEntry{ coldEntry := &CacheEntry{
Inode: 1,
Inode: 1,
Size: 1024, Size: 1024,
LastAccess: time.Now(), LastAccess: time.Now(),
@ -391,7 +391,7 @@ func TestMLCachePolicy_HotChunkPreference(t *testing.T) {
} }
hotEntry := &CacheEntry{ hotEntry := &CacheEntry{
Inode: 2,
Inode: 2,
Size: 1024, Size: 1024,
LastAccess: time.Now(), LastAccess: time.Now(),
@ -456,7 +456,7 @@ func TestMLCachePolicy_RecencyThresholds(t *testing.T) {
func TestMLCachePolicy_SizeScore(t *testing.T) { func TestMLCachePolicy_SizeScore(t *testing.T) {
policy := NewMLCachePolicy() policy := NewMLCachePolicy()
smallSize := uint64(1024) // 1KB
smallSize := uint64(1024) // 1KB
largeSize := uint64(100 * 1024 * 1024) // 100MB largeSize := uint64(100 * 1024 * 1024) // 100MB
smallScore := policy.calculateSizeScore(smallSize) smallScore := policy.calculateSizeScore(smallSize)
@ -511,7 +511,7 @@ func BenchmarkMLCachePolicy_CalculateEvictionScore(b *testing.B) {
policy := NewMLCachePolicy() policy := NewMLCachePolicy()
entry := &CacheEntry{ entry := &CacheEntry{
Inode: 1,
Inode: 1,
Size: 1024, Size: 1024,
LastAccess: time.Now().Add(-5 * time.Minute), LastAccess: time.Now().Add(-5 * time.Minute),
@ -533,7 +533,7 @@ func BenchmarkMLCachePolicy_ShouldEvict(b *testing.B) {
policy := NewMLCachePolicy() policy := NewMLCachePolicy()
entry := &CacheEntry{ entry := &CacheEntry{
Inode: 1,
Inode: 1,
Size: 1024, Size: 1024,
LastAccess: time.Now().Add(-5 * time.Minute), LastAccess: time.Now().Add(-5 * time.Minute),

36
weed/mount/ml/fuse_integration.go
View File

@ -16,10 +16,10 @@ type FUSEMLIntegration struct {
mlOptimization *MLOptimization mlOptimization *MLOptimization
// FUSE-specific configuration // FUSE-specific configuration
enableKeepCache bool // Enable FOPEN_KEEP_CACHE for ML files
enableWriteback bool // Enable writeback caching
attrCacheTimeout time.Duration // Attribute cache timeout for ML files
entryCacheTimeout time.Duration // Entry cache timeout for ML files
enableKeepCache bool // Enable FOPEN_KEEP_CACHE for ML files
enableWriteback bool // Enable writeback caching
attrCacheTimeout time.Duration // Attribute cache timeout for ML files
entryCacheTimeout time.Duration // Entry cache timeout for ML files
// ML-specific FUSE optimizations // ML-specific FUSE optimizations
mlAttrTimeout time.Duration // Extended attribute timeout for ML files mlAttrTimeout time.Duration // Extended attribute timeout for ML files
@ -27,9 +27,9 @@ type FUSEMLIntegration struct {
modelAttrTimeout time.Duration // Longest timeout for model files modelAttrTimeout time.Duration // Longest timeout for model files
// Statistics // Statistics
keepCacheEnabled int64 // Number of times keep cache was enabled
writebackEnabled int64 // Number of times writeback was enabled
mlAttrCacheHits int64 // ML-specific attribute cache hits
keepCacheEnabled int64 // Number of times keep cache was enabled
writebackEnabled int64 // Number of times writeback was enabled
mlAttrCacheHits int64 // ML-specific attribute cache hits
} }
// NewFUSEMLIntegration creates a new FUSE ML integration // NewFUSEMLIntegration creates a new FUSE ML integration
@ -242,14 +242,14 @@ func (fmi *FUSEMLIntegration) GetOptimizationMetrics() FUSEMLMetrics {
} }
return FUSEMLMetrics{ return FUSEMLMetrics{
MLOptimizationMetrics: mlMetrics,
OpenFileCacheMetrics: fmi.openFileCache.GetMetrics(),
CachePolicyMetrics: fmi.cachePolicy.GetEvictionMetrics(),
KeepCacheEnabled: fmi.keepCacheEnabled,
WritebackEnabled: fmi.writebackEnabled,
MLAttrCacheHits: fmi.mlAttrCacheHits,
EnableKeepCache: fmi.enableKeepCache,
EnableWriteback: fmi.enableWriteback,
MLOptimizationMetrics: mlMetrics,
OpenFileCacheMetrics: fmi.openFileCache.GetMetrics(),
CachePolicyMetrics: fmi.cachePolicy.GetEvictionMetrics(),
KeepCacheEnabled: fmi.keepCacheEnabled,
WritebackEnabled: fmi.writebackEnabled,
MLAttrCacheHits: fmi.mlAttrCacheHits,
EnableKeepCache: fmi.enableKeepCache,
EnableWriteback: fmi.enableWriteback,
} }
} }
@ -260,9 +260,9 @@ type FUSEMLMetrics struct {
CachePolicyMetrics MLCachePolicyMetrics `json:"cache_policy"` CachePolicyMetrics MLCachePolicyMetrics `json:"cache_policy"`
// FUSE-specific metrics // FUSE-specific metrics
KeepCacheEnabled int64 `json:"keep_cache_enabled"`
WritebackEnabled int64 `json:"writeback_enabled"`
MLAttrCacheHits int64 `json:"ml_attr_cache_hits"`
KeepCacheEnabled int64 `json:"keep_cache_enabled"`
WritebackEnabled int64 `json:"writeback_enabled"`
MLAttrCacheHits int64 `json:"ml_attr_cache_hits"`
// Configuration // Configuration
EnableKeepCache bool `json:"enable_keep_cache"` EnableKeepCache bool `json:"enable_keep_cache"`

124
weed/mount/ml/open_file_cache.go
View File

@ -10,14 +10,14 @@ import (
// ChunkMetadata contains metadata about a cached chunk // ChunkMetadata contains metadata about a cached chunk
type ChunkMetadata struct { type ChunkMetadata struct {
FileId string // Chunk file ID
Offset uint64 // Offset within the file
Size uint64 // Size of the chunk
CacheLevel int // 0=memory, 1=disk, 2=not cached
LastAccess time.Time // Last access time
AccessCount int64 // Number of times accessed
IsHot bool // Whether this chunk is frequently accessed
Pattern AccessPattern // Access pattern for this chunk
FileId string // Chunk file ID
Offset uint64 // Offset within the file
Size uint64 // Size of the chunk
CacheLevel int // 0=memory, 1=disk, 2=not cached
LastAccess time.Time // Last access time
AccessCount int64 // Number of times accessed
IsHot bool // Whether this chunk is frequently accessed
Pattern AccessPattern // Access pattern for this chunk
} }
// OpenFileInfo contains comprehensive information about an open file // OpenFileInfo contains comprehensive information about an open file
@ -25,33 +25,33 @@ type OpenFileInfo struct {
sync.RWMutex sync.RWMutex
// Basic file information // Basic file information
Inode uint64 // File inode
Entry *filer_pb.Entry // File entry from filer
OpenCount int // Number of open handles
OpenTime time.Time // When file was first opened
LastAccess time.Time // Last access time
Inode uint64 // File inode
Entry *filer_pb.Entry // File entry from filer
OpenCount int // Number of open handles
OpenTime time.Time // When file was first opened
LastAccess time.Time // Last access time
// Chunk-level caching // Chunk-level caching
ChunkCache map[uint32]*ChunkMetadata // chunk index -> metadata
ChunkCount uint32 // Total number of chunks in file
ChunkSize int64 // Size of each chunk
ChunkCache map[uint32]*ChunkMetadata // chunk index -> metadata
ChunkCount uint32 // Total number of chunks in file
ChunkSize int64 // Size of each chunk
// Access pattern tracking // Access pattern tracking
AccessInfo *AccessInfo // Access pattern information
ReadPattern AccessPattern // Overall file access pattern
PrefetchState PrefetchState // Current prefetch state
AccessInfo *AccessInfo // Access pattern information
ReadPattern AccessPattern // Overall file access pattern
PrefetchState PrefetchState // Current prefetch state
// ML-specific optimizations // ML-specific optimizations
IsMLFile bool // Whether this is likely an ML-related file
FileType MLFileType // Type of ML file (dataset, model, etc.)
BatchSize int // Detected batch size for training data
EpochCount int // Number of epochs detected
IsMLFile bool // Whether this is likely an ML-related file
FileType MLFileType // Type of ML file (dataset, model, etc.)
BatchSize int // Detected batch size for training data
EpochCount int // Number of epochs detected
// Performance tracking // Performance tracking
TotalBytesRead int64 // Total bytes read from this file
CacheHitCount int64 // Number of cache hits
CacheMissCount int64 // Number of cache misses
PrefetchHitCount int64 // Number of prefetch hits
TotalBytesRead int64 // Total bytes read from this file
CacheHitCount int64 // Number of cache hits
CacheMissCount int64 // Number of cache misses
PrefetchHitCount int64 // Number of prefetch hits
} }
// PrefetchState represents the current prefetch state for a file // PrefetchState represents the current prefetch state for a file
@ -69,11 +69,11 @@ type MLFileType int
const ( const (
MLFileUnknown MLFileType = iota MLFileUnknown MLFileType = iota
MLFileDataset // Training/validation dataset
MLFileModel // Model checkpoint/weights
MLFileConfig // Configuration files
MLFileTensor // Individual tensor files
MLFileLog // Training logs
MLFileDataset // Training/validation dataset
MLFileModel // Model checkpoint/weights
MLFileConfig // Configuration files
MLFileTensor // Individual tensor files
MLFileLog // Training logs
) )
// OpenFileCache manages open file information with ML-aware optimizations // OpenFileCache manages open file information with ML-aware optimizations
@ -86,18 +86,18 @@ type OpenFileCache struct {
cleanupInterval time.Duration // Cleanup interval cleanupInterval time.Duration // Cleanup interval
// File tracking // File tracking
files map[uint64]*OpenFileInfo // inode -> file info
accessOrder []uint64 // LRU order for eviction
files map[uint64]*OpenFileInfo // inode -> file info
accessOrder []uint64 // LRU order for eviction
// ML-specific configuration // ML-specific configuration
enableMLOptimization bool enableMLOptimization bool
mlFileDetector *MLFileDetector
mlFileDetector *MLFileDetector
// Metrics // Metrics
totalFiles int64
evictedFiles int64
cacheHits int64
cacheMisses int64
totalFiles int64
evictedFiles int64
cacheHits int64
cacheMisses int64
// Background cleanup // Background cleanup
shutdown chan struct{} shutdown chan struct{}
@ -130,15 +130,15 @@ func NewOpenFileCache(maxFiles int, ttl time.Duration) *OpenFileCache {
} }
ofc := &OpenFileCache{ ofc := &OpenFileCache{
maxFiles: maxFiles,
ttl: ttl,
cleanupInterval: 5 * time.Minute,
files: make(map[uint64]*OpenFileInfo),
accessOrder: make([]uint64, 0, maxFiles),
maxFiles: maxFiles,
ttl: ttl,
cleanupInterval: 5 * time.Minute,
files: make(map[uint64]*OpenFileInfo),
accessOrder: make([]uint64, 0, maxFiles),
enableMLOptimization: true, enableMLOptimization: true,
mlFileDetector: newMLFileDetector(),
shutdown: make(chan struct{}),
done: make(chan struct{}),
mlFileDetector: newMLFileDetector(),
shutdown: make(chan struct{}),
done: make(chan struct{}),
} }
// Start background cleanup // Start background cleanup
@ -415,27 +415,27 @@ func (ofc *OpenFileCache) GetMetrics() OpenFileCacheMetrics {
} }
return OpenFileCacheMetrics{ return OpenFileCacheMetrics{
TotalFiles: int64(len(ofc.files)),
MLFiles: mlFiles,
TotalChunks: totalChunks,
CacheHits: ofc.cacheHits,
CacheMisses: ofc.cacheMisses,
EvictedFiles: ofc.evictedFiles,
FileTypes: fileTypes,
TotalFiles: int64(len(ofc.files)),
MLFiles: mlFiles,
TotalChunks: totalChunks,
CacheHits: ofc.cacheHits,
CacheMisses: ofc.cacheMisses,
EvictedFiles: ofc.evictedFiles,
FileTypes: fileTypes,
AccessPatterns: patterns, AccessPatterns: patterns,
} }
} }
// OpenFileCacheMetrics holds metrics for the open file cache // OpenFileCacheMetrics holds metrics for the open file cache
type OpenFileCacheMetrics struct { type OpenFileCacheMetrics struct {
TotalFiles int64 `json:"total_files"`
MLFiles int64 `json:"ml_files"`
TotalChunks int64 `json:"total_chunks"`
CacheHits int64 `json:"cache_hits"`
CacheMisses int64 `json:"cache_misses"`
EvictedFiles int64 `json:"evicted_files"`
FileTypes map[MLFileType]int `json:"file_types"`
AccessPatterns map[AccessPattern]int `json:"access_patterns"`
TotalFiles int64 `json:"total_files"`
MLFiles int64 `json:"ml_files"`
TotalChunks int64 `json:"total_chunks"`
CacheHits int64 `json:"cache_hits"`
CacheMisses int64 `json:"cache_misses"`
EvictedFiles int64 `json:"evicted_files"`
FileTypes map[MLFileType]int `json:"file_types"`
AccessPatterns map[AccessPattern]int `json:"access_patterns"`
} }
// Shutdown gracefully shuts down the open file cache // Shutdown gracefully shuts down the open file cache

46
weed/mount/ml/open_file_cache_test.go
View File

@ -13,7 +13,7 @@ func TestOpenFileCache_Basic(t *testing.T) {
// Test opening a file // Test opening a file
entry := &filer_pb.Entry{ entry := &filer_pb.Entry{
Name: "test.txt",
Name: "test.txt",
Attributes: &filer_pb.FuseAttributes{ Attributes: &filer_pb.FuseAttributes{
FileSize: 1024, FileSize: 1024,
}, },
@ -47,18 +47,18 @@ func TestOpenFileCache_MLFileDetection(t *testing.T) {
size uint64 size uint64
expected MLFileType expected MLFileType
}{ }{
{"PyTorch model", "/models/checkpoint.pt", "checkpoint.pt", 100*1024*1024, MLFileModel},
{"Dataset image", "/datasets/train/image001.jpg", "image001.jpg", 2*1024*1024, MLFileDataset},
{"PyTorch model", "/models/checkpoint.pt", "checkpoint.pt", 100 * 1024 * 1024, MLFileModel},
{"Dataset image", "/datasets/train/image001.jpg", "image001.jpg", 2 * 1024 * 1024, MLFileDataset},
{"Config file", "/config/training.yaml", "training.yaml", 1024, MLFileConfig}, {"Config file", "/config/training.yaml", "training.yaml", 1024, MLFileConfig},
{"Tensor file", "/tensors/weights.safetensors", "weights.safetensors", 50*1024*1024, MLFileModel},
{"Log file", "/logs/training.log", "training.log", 10*1024, MLFileLog},
{"Regular file", "/documents/readme.txt", "readme.txt", 5*1024, MLFileUnknown},
{"Tensor file", "/tensors/weights.safetensors", "weights.safetensors", 50 * 1024 * 1024, MLFileModel},
{"Log file", "/logs/training.log", "training.log", 10 * 1024, MLFileLog},
{"Regular file", "/documents/readme.txt", "readme.txt", 5 * 1024, MLFileUnknown},
} }
for _, tc := range testCases { for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
entry := &filer_pb.Entry{ entry := &filer_pb.Entry{
Name: tc.filename,
Name: tc.filename,
Attributes: &filer_pb.FuseAttributes{ Attributes: &filer_pb.FuseAttributes{
FileSize: tc.size, FileSize: tc.size,
}, },
@ -90,7 +90,7 @@ func TestOpenFileCache_ChunkMetadata(t *testing.T) {
inode := uint64(1) inode := uint64(1)
entry := &filer_pb.Entry{ entry := &filer_pb.Entry{
Name: "data.bin",
Name: "data.bin",
Attributes: &filer_pb.FuseAttributes{ Attributes: &filer_pb.FuseAttributes{
FileSize: 10240, FileSize: 10240,
}, },
@ -135,7 +135,7 @@ func TestOpenFileCache_LRUEviction(t *testing.T) {
// Fill cache to capacity // Fill cache to capacity
for i := 1; i <= 3; i++ { for i := 1; i <= 3; i++ {
entry := &filer_pb.Entry{ entry := &filer_pb.Entry{
Name: "file" + string(rune('0'+i)) + ".txt",
Name: "file" + string(rune('0'+i)) + ".txt",
Attributes: &filer_pb.FuseAttributes{ Attributes: &filer_pb.FuseAttributes{
FileSize: 1024, FileSize: 1024,
}, },
@ -147,7 +147,7 @@ func TestOpenFileCache_LRUEviction(t *testing.T) {
// Add one more file - should trigger eviction // Add one more file - should trigger eviction
entry4 := &filer_pb.Entry{ entry4 := &filer_pb.Entry{
Name: "file4.txt",
Name: "file4.txt",
Attributes: &filer_pb.FuseAttributes{ Attributes: &filer_pb.FuseAttributes{
FileSize: 1024, FileSize: 1024,
}, },
@ -178,7 +178,7 @@ func TestOpenFileCache_TTLCleanup(t *testing.T) {
inode := uint64(1) inode := uint64(1)
entry := &filer_pb.Entry{ entry := &filer_pb.Entry{
Name: "test.txt",
Name: "test.txt",
Attributes: &filer_pb.FuseAttributes{ Attributes: &filer_pb.FuseAttributes{
FileSize: 1024, FileSize: 1024,
}, },
@ -208,7 +208,7 @@ func TestOpenFileCache_MultipleOpens(t *testing.T) {
inode := uint64(1) inode := uint64(1)
entry := &filer_pb.Entry{ entry := &filer_pb.Entry{
Name: "shared.txt",
Name: "shared.txt",
Attributes: &filer_pb.FuseAttributes{ Attributes: &filer_pb.FuseAttributes{
FileSize: 1024, FileSize: 1024,
}, },
@ -263,7 +263,7 @@ func TestOpenFileCache_Metrics(t *testing.T) {
for _, file := range files { for _, file := range files {
entry := &filer_pb.Entry{ entry := &filer_pb.Entry{
Name: file.filename,
Name: file.filename,
Attributes: &filer_pb.FuseAttributes{ Attributes: &filer_pb.FuseAttributes{
FileSize: file.size, FileSize: file.size,
}, },
@ -318,7 +318,7 @@ func TestOpenFileCache_ConcurrentAccess(t *testing.T) {
inode := uint64(id) inode := uint64(id)
entry := &filer_pb.Entry{ entry := &filer_pb.Entry{
Name: "file" + string(rune('0'+id)) + ".txt",
Name: "file" + string(rune('0'+id)) + ".txt",
Attributes: &filer_pb.FuseAttributes{ Attributes: &filer_pb.FuseAttributes{
FileSize: 1024, FileSize: 1024,
}, },
@ -375,7 +375,7 @@ func TestMLFileDetector_Extensions(t *testing.T) {
for _, tc := range testCases { for _, tc := range testCases {
t.Run(tc.filename, func(t *testing.T) { t.Run(tc.filename, func(t *testing.T) {
entry := &filer_pb.Entry{ entry := &filer_pb.Entry{
Name: tc.filename,
Name: tc.filename,
Attributes: &filer_pb.FuseAttributes{ Attributes: &filer_pb.FuseAttributes{
FileSize: 1024, FileSize: 1024,
}, },
@ -417,7 +417,7 @@ func TestMLFileDetector_PathPatterns(t *testing.T) {
for _, tc := range testCases { for _, tc := range testCases {
t.Run(tc.path, func(t *testing.T) { t.Run(tc.path, func(t *testing.T) {
entry := &filer_pb.Entry{ entry := &filer_pb.Entry{
Name: tc.filename,
Name: tc.filename,
Attributes: &filer_pb.FuseAttributes{ Attributes: &filer_pb.FuseAttributes{
FileSize: 1024, FileSize: 1024,
}, },
@ -445,7 +445,7 @@ func TestMLFileDetector_SizeHeuristics(t *testing.T) {
// Large file with model-related name should be detected as model // Large file with model-related name should be detected as model
largeModelEntry := &filer_pb.Entry{ largeModelEntry := &filer_pb.Entry{
Name: "large_model.bin",
Name: "large_model.bin",
Attributes: &filer_pb.FuseAttributes{ Attributes: &filer_pb.FuseAttributes{
FileSize: 500 * 1024 * 1024, // 500MB FileSize: 500 * 1024 * 1024, // 500MB
}, },
@ -469,7 +469,7 @@ func TestOpenFileCache_EvictionProtection(t *testing.T) {
// Open two files and keep them open // Open two files and keep them open
for i := 1; i <= 2; i++ { for i := 1; i <= 2; i++ {
entry := &filer_pb.Entry{ entry := &filer_pb.Entry{
Name: "file" + string(rune('0'+i)) + ".txt",
Name: "file" + string(rune('0'+i)) + ".txt",
Attributes: &filer_pb.FuseAttributes{ Attributes: &filer_pb.FuseAttributes{
FileSize: 1024, FileSize: 1024,
}, },
@ -481,7 +481,7 @@ func TestOpenFileCache_EvictionProtection(t *testing.T) {
// Try to open a third file - should not evict open files // Try to open a third file - should not evict open files
entry3 := &filer_pb.Entry{ entry3 := &filer_pb.Entry{
Name: "file3.txt",
Name: "file3.txt",
Attributes: &filer_pb.FuseAttributes{ Attributes: &filer_pb.FuseAttributes{
FileSize: 1024, FileSize: 1024,
}, },
@ -516,7 +516,7 @@ func TestOpenFileCache_GetFileInfo_CacheHitMiss(t *testing.T) {
// Add file to cache // Add file to cache
entry := &filer_pb.Entry{ entry := &filer_pb.Entry{
Name: "test.txt",
Name: "test.txt",
Attributes: &filer_pb.FuseAttributes{ Attributes: &filer_pb.FuseAttributes{
FileSize: 1024, FileSize: 1024,
}, },
@ -545,7 +545,7 @@ func TestOpenFileCache_Shutdown(t *testing.T) {
// Add some files // Add some files
for i := 1; i <= 3; i++ { for i := 1; i <= 3; i++ {
entry := &filer_pb.Entry{ entry := &filer_pb.Entry{
Name: "file" + string(rune('0'+i)) + ".txt",
Name: "file" + string(rune('0'+i)) + ".txt",
Attributes: &filer_pb.FuseAttributes{ Attributes: &filer_pb.FuseAttributes{
FileSize: 1024, FileSize: 1024,
}, },
@ -576,7 +576,7 @@ func BenchmarkOpenFileCache_OpenFile(b *testing.B) {
defer cache.Shutdown() defer cache.Shutdown()
entry := &filer_pb.Entry{ entry := &filer_pb.Entry{
Name: "benchmark.txt",
Name: "benchmark.txt",
Attributes: &filer_pb.FuseAttributes{ Attributes: &filer_pb.FuseAttributes{
FileSize: 1024, FileSize: 1024,
}, },
@ -597,7 +597,7 @@ func BenchmarkOpenFileCache_GetFileInfo(b *testing.B) {
// Pre-populate cache // Pre-populate cache
entry := &filer_pb.Entry{ entry := &filer_pb.Entry{
Name: "benchmark.txt",
Name: "benchmark.txt",
Attributes: &filer_pb.FuseAttributes{ Attributes: &filer_pb.FuseAttributes{
FileSize: 1024, FileSize: 1024,
}, },

6
weed/mount/ml_integration.go
View File

@ -1,8 +1,6 @@
package mount package mount
import ( import (
"time"
"github.com/hanwen/go-fuse/v2/fuse" "github.com/hanwen/go-fuse/v2/fuse"
"github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/mount/ml" "github.com/seaweedfs/seaweedfs/weed/mount/ml"
@ -13,9 +11,9 @@ import (
// MLIntegrationManager manages ML optimization integration for the main WFS // MLIntegrationManager manages ML optimization integration for the main WFS
type MLIntegrationManager struct { type MLIntegrationManager struct {
mlOptimization *ml.MLOptimization
mlOptimization *ml.MLOptimization
fuseIntegration *ml.FUSEMLIntegration fuseIntegration *ml.FUSEMLIntegration
enabled bool
enabled bool
} }
// NewMLIntegrationManager creates a new ML integration manager // NewMLIntegrationManager creates a new ML integration manager

Write Preview
Loading…
Cancel
Save