From 54282293475a3a25800d96ed0c90cfb1de4ad778 Mon Sep 17 00:00:00 2001 From: Bruce Date: Sat, 14 Sep 2024 23:33:35 +0800 Subject: [PATCH] fix file read crash (#6021) --- weed/filer/reader_at.go | 3 ++- weed/filer/reader_cache.go | 3 ++- weed/util/chunk_cache/chunk_cache.go | 11 +++++++---- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/weed/filer/reader_at.go b/weed/filer/reader_at.go index b70942edc..24162995e 100644 --- a/weed/filer/reader_at.go +++ b/weed/filer/reader_at.go @@ -199,7 +199,8 @@ func (c *ChunkReadAt) readChunkSliceAt(buffer []byte, chunkView *ChunkView, next return fetchChunkRange(buffer, c.readerCache.lookupFileIdFn, chunkView.FileId, chunkView.CipherKey, chunkView.IsGzipped, int64(offset)) } - n, err = c.readerCache.ReadChunkAt(buffer, chunkView.FileId, chunkView.CipherKey, chunkView.IsGzipped, int64(offset), int(chunkView.ChunkSize), (uint64(chunkView.ViewOffset)+chunkView.ChunkSize) <= c.readerCache.chunkCache.GetMaxFilePartSizeInCache()) + shouldCache := (uint64(chunkView.ViewOffset) + chunkView.ChunkSize) <= c.readerCache.chunkCache.GetMaxFilePartSizeInCache() + n, err = c.readerCache.ReadChunkAt(buffer, chunkView.FileId, chunkView.CipherKey, chunkView.IsGzipped, int64(offset), int(chunkView.ChunkSize), shouldCache) if c.lastChunkFid != chunkView.FileId { if chunkView.OffsetInChunk == 0 { // start of a new chunk if c.lastChunkFid != "" { diff --git a/weed/filer/reader_cache.go b/weed/filer/reader_cache.go index 716e796c9..2ef81a931 100644 --- a/weed/filer/reader_cache.go +++ b/weed/filer/reader_cache.go @@ -74,7 +74,8 @@ func (rc *ReaderCache) MaybeCache(chunkViews *Interval[*ChunkView]) { // glog.V(4).Infof("prefetch %s offset %d", chunkView.FileId, chunkView.ViewOffset) // cache this chunk if not yet - cacher := newSingleChunkCacher(rc, chunkView.FileId, chunkView.CipherKey, chunkView.IsGzipped, int(chunkView.ChunkSize), (uint64(chunkView.ViewOffset)+chunkView.ChunkSize) <= rc.chunkCache.GetMaxFilePartSizeInCache()) + shouldCache := (uint64(chunkView.ViewOffset) + chunkView.ChunkSize) <= rc.chunkCache.GetMaxFilePartSizeInCache() + cacher := newSingleChunkCacher(rc, chunkView.FileId, chunkView.CipherKey, chunkView.IsGzipped, int(chunkView.ChunkSize), shouldCache) go cacher.startCaching() <-cacher.cacheStartedCh rc.downloaders[chunkView.FileId] = cacher diff --git a/weed/util/chunk_cache/chunk_cache.go b/weed/util/chunk_cache/chunk_cache.go index 37dde1950..7eee41b9b 100644 --- a/weed/util/chunk_cache/chunk_cache.go +++ b/weed/util/chunk_cache/chunk_cache.go @@ -22,9 +22,9 @@ type TieredChunkCache struct { memCache *ChunkCacheInMemory diskCaches []*OnDiskCacheLayer sync.RWMutex - onDiskCacheSizeLimit0 uint64 - onDiskCacheSizeLimit1 uint64 - onDiskCacheSizeLimit2 uint64 + onDiskCacheSizeLimit0 uint64 + onDiskCacheSizeLimit1 uint64 + onDiskCacheSizeLimit2 uint64 maxFilePartSizeInCache uint64 } @@ -42,12 +42,15 @@ func NewTieredChunkCache(maxEntries int64, dir string, diskSizeInUnit int64, uni c.diskCaches[0] = NewOnDiskCacheLayer(dir, "c0_2", diskSizeInUnit*unitSize/8, 2) c.diskCaches[1] = NewOnDiskCacheLayer(dir, "c1_3", diskSizeInUnit*unitSize/4+diskSizeInUnit*unitSize/8, 3) c.diskCaches[2] = NewOnDiskCacheLayer(dir, "c2_2", diskSizeInUnit*unitSize/2, 2) - c.maxFilePartSizeInCache = uint64(unitSize*diskSizeInUnit)/4 + c.maxFilePartSizeInCache = uint64(unitSize*diskSizeInUnit) / 4 return c } func (c *TieredChunkCache) GetMaxFilePartSizeInCache() (answer uint64) { + if c == nil { + return 0 + } return c.maxFilePartSizeInCache }