You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

133 lines
2.7 KiB

5 years ago
5 years ago
5 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
4 years ago
5 years ago
5 years ago
4 years ago
5 years ago
  1. package chunk_cache
  2. import (
  3. "sync"
  4. "github.com/chrislusf/seaweedfs/weed/glog"
  5. "github.com/chrislusf/seaweedfs/weed/storage/needle"
  6. )
  7. const (
  8. memCacheSizeLimit = 1024 * 1024
  9. onDiskCacheSizeLimit0 = memCacheSizeLimit
  10. onDiskCacheSizeLimit1 = 4 * memCacheSizeLimit
  11. )
  12. type ChunkCache interface {
  13. GetChunk(fileId string, minSize uint64) (data []byte)
  14. SetChunk(fileId string, data []byte)
  15. }
  16. // a global cache for recently accessed file chunks
  17. type TieredChunkCache struct {
  18. memCache *ChunkCacheInMemory
  19. diskCaches []*OnDiskCacheLayer
  20. sync.RWMutex
  21. }
  22. func NewTieredChunkCache(maxEntries int64, dir string, diskSizeMB int64) *TieredChunkCache {
  23. c := &TieredChunkCache{
  24. memCache: NewChunkCacheInMemory(maxEntries),
  25. }
  26. c.diskCaches = make([]*OnDiskCacheLayer, 3)
  27. c.diskCaches[0] = NewOnDiskCacheLayer(dir, "c0_1", diskSizeMB/4, 4)
  28. c.diskCaches[1] = NewOnDiskCacheLayer(dir, "c1_4", diskSizeMB/4, 4)
  29. c.diskCaches[2] = NewOnDiskCacheLayer(dir, "cache", diskSizeMB/2, 4)
  30. return c
  31. }
  32. func (c *TieredChunkCache) GetChunk(fileId string, minSize uint64) (data []byte) {
  33. if c == nil {
  34. return
  35. }
  36. c.RLock()
  37. defer c.RUnlock()
  38. return c.doGetChunk(fileId, minSize)
  39. }
  40. func (c *TieredChunkCache) doGetChunk(fileId string, minSize uint64) (data []byte) {
  41. if minSize < memCacheSizeLimit {
  42. data = c.memCache.GetChunk(fileId)
  43. if len(data) >= int(minSize) {
  44. return data
  45. }
  46. }
  47. fid, err := needle.ParseFileIdFromString(fileId)
  48. if err != nil {
  49. glog.Errorf("failed to parse file id %s", fileId)
  50. return nil
  51. }
  52. if minSize < onDiskCacheSizeLimit0 {
  53. data = c.diskCaches[0].getChunk(fid.Key)
  54. if len(data) >= int(minSize) {
  55. return data
  56. }
  57. }
  58. if minSize < onDiskCacheSizeLimit1 {
  59. data = c.diskCaches[1].getChunk(fid.Key)
  60. if len(data) >= int(minSize) {
  61. return data
  62. }
  63. }
  64. {
  65. data = c.diskCaches[2].getChunk(fid.Key)
  66. if len(data) >= int(minSize) {
  67. return data
  68. }
  69. }
  70. return nil
  71. }
  72. func (c *TieredChunkCache) SetChunk(fileId string, data []byte) {
  73. if c == nil {
  74. return
  75. }
  76. c.Lock()
  77. defer c.Unlock()
  78. glog.V(4).Infof("SetChunk %s size %d\n", fileId, len(data))
  79. c.doSetChunk(fileId, data)
  80. }
  81. func (c *TieredChunkCache) doSetChunk(fileId string, data []byte) {
  82. if len(data) < memCacheSizeLimit {
  83. c.memCache.SetChunk(fileId, data)
  84. }
  85. fid, err := needle.ParseFileIdFromString(fileId)
  86. if err != nil {
  87. glog.Errorf("failed to parse file id %s", fileId)
  88. return
  89. }
  90. if len(data) < onDiskCacheSizeLimit0 {
  91. c.diskCaches[0].setChunk(fid.Key, data)
  92. } else if len(data) < onDiskCacheSizeLimit1 {
  93. c.diskCaches[1].setChunk(fid.Key, data)
  94. } else {
  95. c.diskCaches[2].setChunk(fid.Key, data)
  96. }
  97. }
  98. func (c *TieredChunkCache) Shutdown() {
  99. if c == nil {
  100. return
  101. }
  102. c.Lock()
  103. defer c.Unlock()
  104. for _, diskCache := range c.diskCaches {
  105. diskCache.shutdown()
  106. }
  107. }