Browse Source

fix file read crash (#6021)

pull/5631/merge
Bruce 3 months ago
committed by GitHub
parent
commit
5428229347
No known key found for this signature in database GPG Key ID: B5690EEEBB952194
  1. 3
      weed/filer/reader_at.go
  2. 3
      weed/filer/reader_cache.go
  3. 11
      weed/util/chunk_cache/chunk_cache.go

3
weed/filer/reader_at.go

@ -199,7 +199,8 @@ func (c *ChunkReadAt) readChunkSliceAt(buffer []byte, chunkView *ChunkView, next
return fetchChunkRange(buffer, c.readerCache.lookupFileIdFn, chunkView.FileId, chunkView.CipherKey, chunkView.IsGzipped, int64(offset)) return fetchChunkRange(buffer, c.readerCache.lookupFileIdFn, chunkView.FileId, chunkView.CipherKey, chunkView.IsGzipped, int64(offset))
} }
n, err = c.readerCache.ReadChunkAt(buffer, chunkView.FileId, chunkView.CipherKey, chunkView.IsGzipped, int64(offset), int(chunkView.ChunkSize), (uint64(chunkView.ViewOffset)+chunkView.ChunkSize) <= c.readerCache.chunkCache.GetMaxFilePartSizeInCache())
shouldCache := (uint64(chunkView.ViewOffset) + chunkView.ChunkSize) <= c.readerCache.chunkCache.GetMaxFilePartSizeInCache()
n, err = c.readerCache.ReadChunkAt(buffer, chunkView.FileId, chunkView.CipherKey, chunkView.IsGzipped, int64(offset), int(chunkView.ChunkSize), shouldCache)
if c.lastChunkFid != chunkView.FileId { if c.lastChunkFid != chunkView.FileId {
if chunkView.OffsetInChunk == 0 { // start of a new chunk if chunkView.OffsetInChunk == 0 { // start of a new chunk
if c.lastChunkFid != "" { if c.lastChunkFid != "" {

3
weed/filer/reader_cache.go

@ -74,7 +74,8 @@ func (rc *ReaderCache) MaybeCache(chunkViews *Interval[*ChunkView]) {
// glog.V(4).Infof("prefetch %s offset %d", chunkView.FileId, chunkView.ViewOffset) // glog.V(4).Infof("prefetch %s offset %d", chunkView.FileId, chunkView.ViewOffset)
// cache this chunk if not yet // cache this chunk if not yet
cacher := newSingleChunkCacher(rc, chunkView.FileId, chunkView.CipherKey, chunkView.IsGzipped, int(chunkView.ChunkSize), (uint64(chunkView.ViewOffset)+chunkView.ChunkSize) <= rc.chunkCache.GetMaxFilePartSizeInCache())
shouldCache := (uint64(chunkView.ViewOffset) + chunkView.ChunkSize) <= rc.chunkCache.GetMaxFilePartSizeInCache()
cacher := newSingleChunkCacher(rc, chunkView.FileId, chunkView.CipherKey, chunkView.IsGzipped, int(chunkView.ChunkSize), shouldCache)
go cacher.startCaching() go cacher.startCaching()
<-cacher.cacheStartedCh <-cacher.cacheStartedCh
rc.downloaders[chunkView.FileId] = cacher rc.downloaders[chunkView.FileId] = cacher

11
weed/util/chunk_cache/chunk_cache.go

@ -22,9 +22,9 @@ type TieredChunkCache struct {
memCache *ChunkCacheInMemory memCache *ChunkCacheInMemory
diskCaches []*OnDiskCacheLayer diskCaches []*OnDiskCacheLayer
sync.RWMutex sync.RWMutex
onDiskCacheSizeLimit0 uint64
onDiskCacheSizeLimit1 uint64
onDiskCacheSizeLimit2 uint64
onDiskCacheSizeLimit0 uint64
onDiskCacheSizeLimit1 uint64
onDiskCacheSizeLimit2 uint64
maxFilePartSizeInCache uint64 maxFilePartSizeInCache uint64
} }
@ -42,12 +42,15 @@ func NewTieredChunkCache(maxEntries int64, dir string, diskSizeInUnit int64, uni
c.diskCaches[0] = NewOnDiskCacheLayer(dir, "c0_2", diskSizeInUnit*unitSize/8, 2) c.diskCaches[0] = NewOnDiskCacheLayer(dir, "c0_2", diskSizeInUnit*unitSize/8, 2)
c.diskCaches[1] = NewOnDiskCacheLayer(dir, "c1_3", diskSizeInUnit*unitSize/4+diskSizeInUnit*unitSize/8, 3) c.diskCaches[1] = NewOnDiskCacheLayer(dir, "c1_3", diskSizeInUnit*unitSize/4+diskSizeInUnit*unitSize/8, 3)
c.diskCaches[2] = NewOnDiskCacheLayer(dir, "c2_2", diskSizeInUnit*unitSize/2, 2) c.diskCaches[2] = NewOnDiskCacheLayer(dir, "c2_2", diskSizeInUnit*unitSize/2, 2)
c.maxFilePartSizeInCache = uint64(unitSize*diskSizeInUnit)/4
c.maxFilePartSizeInCache = uint64(unitSize*diskSizeInUnit) / 4
return c return c
} }
func (c *TieredChunkCache) GetMaxFilePartSizeInCache() (answer uint64) { func (c *TieredChunkCache) GetMaxFilePartSizeInCache() (answer uint64) {
if c == nil {
return 0
}
return c.maxFilePartSizeInCache return c.maxFilePartSizeInCache
} }

Loading…
Cancel
Save