|
|
|
@ -10,14 +10,14 @@ import ( |
|
|
|
|
|
|
|
// ChunkMetadata contains metadata about a cached chunk
|
|
|
|
type ChunkMetadata struct { |
|
|
|
FileId string // Chunk file ID
|
|
|
|
Offset uint64 // Offset within the file
|
|
|
|
Size uint64 // Size of the chunk
|
|
|
|
CacheLevel int // 0=memory, 1=disk, 2=not cached
|
|
|
|
LastAccess time.Time // Last access time
|
|
|
|
AccessCount int64 // Number of times accessed
|
|
|
|
IsHot bool // Whether this chunk is frequently accessed
|
|
|
|
Pattern AccessPattern // Access pattern for this chunk
|
|
|
|
FileId string // Chunk file ID
|
|
|
|
Offset uint64 // Offset within the file
|
|
|
|
Size uint64 // Size of the chunk
|
|
|
|
CacheLevel int // 0=memory, 1=disk, 2=not cached
|
|
|
|
LastAccess time.Time // Last access time
|
|
|
|
AccessCount int64 // Number of times accessed
|
|
|
|
IsHot bool // Whether this chunk is frequently accessed
|
|
|
|
Pattern AccessPattern // Access pattern for this chunk
|
|
|
|
} |
|
|
|
|
|
|
|
// OpenFileInfo contains comprehensive information about an open file
|
|
|
|
@ -25,33 +25,33 @@ type OpenFileInfo struct { |
|
|
|
sync.RWMutex |
|
|
|
|
|
|
|
// Basic file information
|
|
|
|
Inode uint64 // File inode
|
|
|
|
Entry *filer_pb.Entry // File entry from filer
|
|
|
|
OpenCount int // Number of open handles
|
|
|
|
OpenTime time.Time // When file was first opened
|
|
|
|
LastAccess time.Time // Last access time
|
|
|
|
Inode uint64 // File inode
|
|
|
|
Entry *filer_pb.Entry // File entry from filer
|
|
|
|
OpenCount int // Number of open handles
|
|
|
|
OpenTime time.Time // When file was first opened
|
|
|
|
LastAccess time.Time // Last access time
|
|
|
|
|
|
|
|
// Chunk-level caching
|
|
|
|
ChunkCache map[uint32]*ChunkMetadata // chunk index -> metadata
|
|
|
|
ChunkCount uint32 // Total number of chunks in file
|
|
|
|
ChunkSize int64 // Size of each chunk
|
|
|
|
ChunkCache map[uint32]*ChunkMetadata // chunk index -> metadata
|
|
|
|
ChunkCount uint32 // Total number of chunks in file
|
|
|
|
ChunkSize int64 // Size of each chunk
|
|
|
|
|
|
|
|
// Access pattern tracking
|
|
|
|
AccessInfo *AccessInfo // Access pattern information
|
|
|
|
ReadPattern AccessPattern // Overall file access pattern
|
|
|
|
PrefetchState PrefetchState // Current prefetch state
|
|
|
|
AccessInfo *AccessInfo // Access pattern information
|
|
|
|
ReadPattern AccessPattern // Overall file access pattern
|
|
|
|
PrefetchState PrefetchState // Current prefetch state
|
|
|
|
|
|
|
|
// ML-specific optimizations
|
|
|
|
IsMLFile bool // Whether this is likely an ML-related file
|
|
|
|
FileType MLFileType // Type of ML file (dataset, model, etc.)
|
|
|
|
BatchSize int // Detected batch size for training data
|
|
|
|
EpochCount int // Number of epochs detected
|
|
|
|
IsMLFile bool // Whether this is likely an ML-related file
|
|
|
|
FileType MLFileType // Type of ML file (dataset, model, etc.)
|
|
|
|
BatchSize int // Detected batch size for training data
|
|
|
|
EpochCount int // Number of epochs detected
|
|
|
|
|
|
|
|
// Performance tracking
|
|
|
|
TotalBytesRead int64 // Total bytes read from this file
|
|
|
|
CacheHitCount int64 // Number of cache hits
|
|
|
|
CacheMissCount int64 // Number of cache misses
|
|
|
|
PrefetchHitCount int64 // Number of prefetch hits
|
|
|
|
TotalBytesRead int64 // Total bytes read from this file
|
|
|
|
CacheHitCount int64 // Number of cache hits
|
|
|
|
CacheMissCount int64 // Number of cache misses
|
|
|
|
PrefetchHitCount int64 // Number of prefetch hits
|
|
|
|
} |
|
|
|
|
|
|
|
// PrefetchState represents the current prefetch state for a file
|
|
|
|
@ -69,11 +69,11 @@ type MLFileType int |
|
|
|
|
|
|
|
const ( |
|
|
|
MLFileUnknown MLFileType = iota |
|
|
|
MLFileDataset // Training/validation dataset
|
|
|
|
MLFileModel // Model checkpoint/weights
|
|
|
|
MLFileConfig // Configuration files
|
|
|
|
MLFileTensor // Individual tensor files
|
|
|
|
MLFileLog // Training logs
|
|
|
|
MLFileDataset // Training/validation dataset
|
|
|
|
MLFileModel // Model checkpoint/weights
|
|
|
|
MLFileConfig // Configuration files
|
|
|
|
MLFileTensor // Individual tensor files
|
|
|
|
MLFileLog // Training logs
|
|
|
|
) |
|
|
|
|
|
|
|
// OpenFileCache manages open file information with ML-aware optimizations
|
|
|
|
@ -86,18 +86,18 @@ type OpenFileCache struct { |
|
|
|
cleanupInterval time.Duration // Cleanup interval
|
|
|
|
|
|
|
|
// File tracking
|
|
|
|
files map[uint64]*OpenFileInfo // inode -> file info
|
|
|
|
accessOrder []uint64 // LRU order for eviction
|
|
|
|
files map[uint64]*OpenFileInfo // inode -> file info
|
|
|
|
accessOrder []uint64 // LRU order for eviction
|
|
|
|
|
|
|
|
// ML-specific configuration
|
|
|
|
enableMLOptimization bool |
|
|
|
mlFileDetector *MLFileDetector |
|
|
|
mlFileDetector *MLFileDetector |
|
|
|
|
|
|
|
// Metrics
|
|
|
|
totalFiles int64 |
|
|
|
evictedFiles int64 |
|
|
|
cacheHits int64 |
|
|
|
cacheMisses int64 |
|
|
|
totalFiles int64 |
|
|
|
evictedFiles int64 |
|
|
|
cacheHits int64 |
|
|
|
cacheMisses int64 |
|
|
|
|
|
|
|
// Background cleanup
|
|
|
|
shutdown chan struct{} |
|
|
|
@ -130,15 +130,15 @@ func NewOpenFileCache(maxFiles int, ttl time.Duration) *OpenFileCache { |
|
|
|
} |
|
|
|
|
|
|
|
ofc := &OpenFileCache{ |
|
|
|
maxFiles: maxFiles, |
|
|
|
ttl: ttl, |
|
|
|
cleanupInterval: 5 * time.Minute, |
|
|
|
files: make(map[uint64]*OpenFileInfo), |
|
|
|
accessOrder: make([]uint64, 0, maxFiles), |
|
|
|
maxFiles: maxFiles, |
|
|
|
ttl: ttl, |
|
|
|
cleanupInterval: 5 * time.Minute, |
|
|
|
files: make(map[uint64]*OpenFileInfo), |
|
|
|
accessOrder: make([]uint64, 0, maxFiles), |
|
|
|
enableMLOptimization: true, |
|
|
|
mlFileDetector: newMLFileDetector(), |
|
|
|
shutdown: make(chan struct{}), |
|
|
|
done: make(chan struct{}), |
|
|
|
mlFileDetector: newMLFileDetector(), |
|
|
|
shutdown: make(chan struct{}), |
|
|
|
done: make(chan struct{}), |
|
|
|
} |
|
|
|
|
|
|
|
// Start background cleanup
|
|
|
|
@ -415,27 +415,27 @@ func (ofc *OpenFileCache) GetMetrics() OpenFileCacheMetrics { |
|
|
|
} |
|
|
|
|
|
|
|
return OpenFileCacheMetrics{ |
|
|
|
TotalFiles: int64(len(ofc.files)), |
|
|
|
MLFiles: mlFiles, |
|
|
|
TotalChunks: totalChunks, |
|
|
|
CacheHits: ofc.cacheHits, |
|
|
|
CacheMisses: ofc.cacheMisses, |
|
|
|
EvictedFiles: ofc.evictedFiles, |
|
|
|
FileTypes: fileTypes, |
|
|
|
TotalFiles: int64(len(ofc.files)), |
|
|
|
MLFiles: mlFiles, |
|
|
|
TotalChunks: totalChunks, |
|
|
|
CacheHits: ofc.cacheHits, |
|
|
|
CacheMisses: ofc.cacheMisses, |
|
|
|
EvictedFiles: ofc.evictedFiles, |
|
|
|
FileTypes: fileTypes, |
|
|
|
AccessPatterns: patterns, |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
// OpenFileCacheMetrics holds metrics for the open file cache
|
|
|
|
type OpenFileCacheMetrics struct { |
|
|
|
TotalFiles int64 `json:"total_files"` |
|
|
|
MLFiles int64 `json:"ml_files"` |
|
|
|
TotalChunks int64 `json:"total_chunks"` |
|
|
|
CacheHits int64 `json:"cache_hits"` |
|
|
|
CacheMisses int64 `json:"cache_misses"` |
|
|
|
EvictedFiles int64 `json:"evicted_files"` |
|
|
|
FileTypes map[MLFileType]int `json:"file_types"` |
|
|
|
AccessPatterns map[AccessPattern]int `json:"access_patterns"` |
|
|
|
TotalFiles int64 `json:"total_files"` |
|
|
|
MLFiles int64 `json:"ml_files"` |
|
|
|
TotalChunks int64 `json:"total_chunks"` |
|
|
|
CacheHits int64 `json:"cache_hits"` |
|
|
|
CacheMisses int64 `json:"cache_misses"` |
|
|
|
EvictedFiles int64 `json:"evicted_files"` |
|
|
|
FileTypes map[MLFileType]int `json:"file_types"` |
|
|
|
AccessPatterns map[AccessPattern]int `json:"access_patterns"` |
|
|
|
} |
|
|
|
|
|
|
|
// Shutdown gracefully shuts down the open file cache
|
|
|
|
|
chrislu
3 months ago