You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

231 lines
5.1 KiB

7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
  1. package filer2
  2. import (
  3. "fmt"
  4. "github.com/chrislusf/seaweedfs/weed/glog"
  5. "github.com/chrislusf/seaweedfs/weed/operation"
  6. "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
  7. "github.com/karlseguin/ccache"
  8. "os"
  9. "path/filepath"
  10. "strings"
  11. "time"
  12. )
  13. type Filer struct {
  14. masters []string
  15. store FilerStore
  16. directoryCache *ccache.Cache
  17. currentMaster string
  18. }
  19. func NewFiler(masters []string) *Filer {
  20. return &Filer{
  21. masters: masters,
  22. directoryCache: ccache.New(ccache.Configure().MaxSize(1000).ItemsToPrune(100)),
  23. }
  24. }
  25. func (f *Filer) SetStore(store FilerStore) {
  26. f.store = store
  27. }
  28. func (f *Filer) DisableDirectoryCache() {
  29. f.directoryCache = nil
  30. }
  31. func (f *Filer) CreateEntry(entry *Entry) error {
  32. dirParts := strings.Split(string(entry.FullPath), "/")
  33. // fmt.Printf("directory parts: %+v\n", dirParts)
  34. var lastDirectoryEntry *Entry
  35. for i := 1; i < len(dirParts); i++ {
  36. dirPath := "/" + filepath.Join(dirParts[:i]...)
  37. // fmt.Printf("%d directory: %+v\n", i, dirPath)
  38. // first check local cache
  39. dirEntry := f.cacheGetDirectory(dirPath)
  40. // not found, check the store directly
  41. if dirEntry == nil {
  42. glog.V(4).Infof("find uncached directory: %s", dirPath)
  43. dirEntry, _ = f.FindEntry(FullPath(dirPath))
  44. } else {
  45. glog.V(4).Infof("found cached directory: %s", dirPath)
  46. }
  47. // no such existing directory
  48. if dirEntry == nil {
  49. // create the directory
  50. now := time.Now()
  51. dirEntry = &Entry{
  52. FullPath: FullPath(dirPath),
  53. Attr: Attr{
  54. Mtime: now,
  55. Crtime: now,
  56. Mode: os.ModeDir | 0770,
  57. Uid: entry.Uid,
  58. Gid: entry.Gid,
  59. },
  60. }
  61. glog.V(2).Infof("create directory: %s %v", dirPath, dirEntry.Mode)
  62. mkdirErr := f.store.InsertEntry(dirEntry)
  63. if mkdirErr != nil {
  64. return fmt.Errorf("mkdir %s: %v", dirPath, mkdirErr)
  65. }
  66. } else if !dirEntry.IsDirectory() {
  67. return fmt.Errorf("%s is a file", dirPath)
  68. }
  69. // cache the directory entry
  70. f.cacheSetDirectory(dirPath, dirEntry, i)
  71. // remember the direct parent directory entry
  72. if i == len(dirParts)-1 {
  73. lastDirectoryEntry = dirEntry
  74. }
  75. }
  76. if lastDirectoryEntry == nil {
  77. return fmt.Errorf("parent folder not found: %v", entry.FullPath)
  78. }
  79. /*
  80. if !hasWritePermission(lastDirectoryEntry, entry) {
  81. glog.V(0).Infof("directory %s: %v, entry: uid=%d gid=%d",
  82. lastDirectoryEntry.FullPath, lastDirectoryEntry.Attr, entry.Uid, entry.Gid)
  83. return fmt.Errorf("no write permission in folder %v", lastDirectoryEntry.FullPath)
  84. }
  85. */
  86. oldEntry, _ := f.FindEntry(entry.FullPath)
  87. if err := f.store.InsertEntry(entry); err != nil {
  88. return fmt.Errorf("insert entry %s: %v", entry.FullPath, err)
  89. }
  90. f.deleteChunksIfNotNew(oldEntry, entry)
  91. return nil
  92. }
  93. func (f *Filer) UpdateEntry(entry *Entry) (err error) {
  94. return f.store.UpdateEntry(entry)
  95. }
  96. func (f *Filer) FindEntry(p FullPath) (entry *Entry, err error) {
  97. return f.store.FindEntry(p)
  98. }
  99. func (f *Filer) DeleteEntryMetaAndData(p FullPath, isRecursive bool, shouldDeleteChunks bool) (err error) {
  100. entry, err := f.FindEntry(p)
  101. if err != nil {
  102. return err
  103. }
  104. if entry.IsDirectory() {
  105. entries, err := f.ListDirectoryEntries(p, "", false, 1)
  106. if err != nil {
  107. return fmt.Errorf("list folder %s: %v", p, err)
  108. }
  109. if isRecursive {
  110. for _, sub := range entries {
  111. f.DeleteEntryMetaAndData(sub.FullPath, isRecursive, shouldDeleteChunks)
  112. }
  113. } else {
  114. if len(entries) > 0 {
  115. return fmt.Errorf("folder %s is not empty", p)
  116. }
  117. }
  118. f.cacheDelDirectory(string(p))
  119. }
  120. if shouldDeleteChunks {
  121. f.deleteChunks(entry.Chunks)
  122. }
  123. return f.store.DeleteEntry(p)
  124. }
  125. func (f *Filer) ListDirectoryEntries(p FullPath, startFileName string, inclusive bool, limit int) ([]*Entry, error) {
  126. if strings.HasSuffix(string(p), "/") && len(p) > 1 {
  127. p = p[0 : len(p)-1]
  128. }
  129. return f.store.ListDirectoryEntries(p, startFileName, inclusive, limit)
  130. }
  131. func (f *Filer) cacheDelDirectory(dirpath string) {
  132. if f.directoryCache == nil {
  133. return
  134. }
  135. f.directoryCache.Delete(dirpath)
  136. return
  137. }
  138. func (f *Filer) cacheGetDirectory(dirpath string) *Entry {
  139. if f.directoryCache == nil {
  140. return nil
  141. }
  142. item := f.directoryCache.Get(dirpath)
  143. if item == nil {
  144. return nil
  145. }
  146. return item.Value().(*Entry)
  147. }
  148. func (f *Filer) cacheSetDirectory(dirpath string, dirEntry *Entry, level int) {
  149. if f.directoryCache == nil {
  150. return
  151. }
  152. minutes := 60
  153. if level < 10 {
  154. minutes -= level * 6
  155. }
  156. f.directoryCache.Set(dirpath, dirEntry, time.Duration(minutes)*time.Minute)
  157. }
  158. func (f *Filer) deleteChunks(chunks []*filer_pb.FileChunk) {
  159. for _, chunk := range chunks {
  160. if err := operation.DeleteFile(f.GetMaster(), chunk.FileId, ""); err != nil {
  161. glog.V(0).Infof("deleting file %s: %v", chunk.FileId, err)
  162. }
  163. }
  164. }
  165. func (f *Filer) deleteChunksIfNotNew(oldEntry, newEntry *Entry) {
  166. if oldEntry == nil {
  167. return
  168. }
  169. if newEntry == nil {
  170. f.deleteChunks(oldEntry.Chunks)
  171. }
  172. var toDelete []*filer_pb.FileChunk
  173. for _, oldChunk := range oldEntry.Chunks {
  174. found := false
  175. for _, newChunk := range newEntry.Chunks {
  176. if oldChunk.FileId == newChunk.FileId {
  177. found = true
  178. break
  179. }
  180. }
  181. if !found {
  182. toDelete = append(toDelete, oldChunk)
  183. }
  184. }
  185. f.deleteChunks(toDelete)
  186. }