You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

311 lines
7.1 KiB

7 years ago
7 years ago
7 years ago
6 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
6 years ago
6 years ago
7 years ago
6 years ago
6 years ago
7 years ago
7 years ago
6 years ago
6 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
  1. package filer2
  2. import (
  3. "context"
  4. "fmt"
  5. "google.golang.org/grpc"
  6. "math"
  7. "os"
  8. "path/filepath"
  9. "strings"
  10. "time"
  11. "github.com/chrislusf/seaweedfs/weed/glog"
  12. "github.com/chrislusf/seaweedfs/weed/wdclient"
  13. "github.com/karlseguin/ccache"
  14. )
  15. var (
  16. OS_UID = uint32(os.Getuid())
  17. OS_GID = uint32(os.Getgid())
  18. )
  19. type Filer struct {
  20. store *FilerStoreWrapper
  21. directoryCache *ccache.Cache
  22. MasterClient *wdclient.MasterClient
  23. fileIdDeletionChan chan string
  24. GrpcDialOption grpc.DialOption
  25. }
  26. func NewFiler(masters []string, grpcDialOption grpc.DialOption) *Filer {
  27. f := &Filer{
  28. directoryCache: ccache.New(ccache.Configure().MaxSize(1000).ItemsToPrune(100)),
  29. MasterClient: wdclient.NewMasterClient(context.Background(), grpcDialOption, "filer", masters),
  30. fileIdDeletionChan: make(chan string, 4096),
  31. GrpcDialOption: grpcDialOption,
  32. }
  33. go f.loopProcessingDeletion()
  34. return f
  35. }
  36. func (f *Filer) SetStore(store FilerStore) {
  37. f.store = NewFilerStoreWrapper(store)
  38. }
  39. func (f *Filer) DisableDirectoryCache() {
  40. f.directoryCache = nil
  41. }
  42. func (fs *Filer) GetMaster() string {
  43. return fs.MasterClient.GetMaster()
  44. }
  45. func (fs *Filer) KeepConnectedToMaster() {
  46. fs.MasterClient.KeepConnectedToMaster()
  47. }
  48. func (f *Filer) BeginTransaction(ctx context.Context) (context.Context, error) {
  49. return f.store.BeginTransaction(ctx)
  50. }
  51. func (f *Filer) CommitTransaction(ctx context.Context) error {
  52. return f.store.CommitTransaction(ctx)
  53. }
  54. func (f *Filer) RollbackTransaction(ctx context.Context) error {
  55. return f.store.RollbackTransaction(ctx)
  56. }
  57. func (f *Filer) CreateEntry(ctx context.Context, entry *Entry) error {
  58. if string(entry.FullPath) == "/" {
  59. return nil
  60. }
  61. dirParts := strings.Split(string(entry.FullPath), "/")
  62. // fmt.Printf("directory parts: %+v\n", dirParts)
  63. var lastDirectoryEntry *Entry
  64. for i := 1; i < len(dirParts); i++ {
  65. dirPath := "/" + filepath.ToSlash(filepath.Join(dirParts[:i]...))
  66. // fmt.Printf("%d directory: %+v\n", i, dirPath)
  67. // first check local cache
  68. dirEntry := f.cacheGetDirectory(dirPath)
  69. // not found, check the store directly
  70. if dirEntry == nil {
  71. glog.V(4).Infof("find uncached directory: %s", dirPath)
  72. dirEntry, _ = f.FindEntry(ctx, FullPath(dirPath))
  73. } else {
  74. glog.V(4).Infof("found cached directory: %s", dirPath)
  75. }
  76. // no such existing directory
  77. if dirEntry == nil {
  78. // create the directory
  79. now := time.Now()
  80. dirEntry = &Entry{
  81. FullPath: FullPath(dirPath),
  82. Attr: Attr{
  83. Mtime: now,
  84. Crtime: now,
  85. Mode: os.ModeDir | 0770,
  86. Uid: entry.Uid,
  87. Gid: entry.Gid,
  88. },
  89. }
  90. glog.V(2).Infof("create directory: %s %v", dirPath, dirEntry.Mode)
  91. mkdirErr := f.store.InsertEntry(ctx, dirEntry)
  92. if mkdirErr != nil {
  93. if _, err := f.FindEntry(ctx, FullPath(dirPath)); err == ErrNotFound {
  94. return fmt.Errorf("mkdir %s: %v", dirPath, mkdirErr)
  95. }
  96. } else {
  97. f.NotifyUpdateEvent(nil, dirEntry, false)
  98. }
  99. } else if !dirEntry.IsDirectory() {
  100. return fmt.Errorf("%s is a file", dirPath)
  101. }
  102. // cache the directory entry
  103. f.cacheSetDirectory(dirPath, dirEntry, i)
  104. // remember the direct parent directory entry
  105. if i == len(dirParts)-1 {
  106. lastDirectoryEntry = dirEntry
  107. }
  108. }
  109. if lastDirectoryEntry == nil {
  110. return fmt.Errorf("parent folder not found: %v", entry.FullPath)
  111. }
  112. /*
  113. if !hasWritePermission(lastDirectoryEntry, entry) {
  114. glog.V(0).Infof("directory %s: %v, entry: uid=%d gid=%d",
  115. lastDirectoryEntry.FullPath, lastDirectoryEntry.Attr, entry.Uid, entry.Gid)
  116. return fmt.Errorf("no write permission in folder %v", lastDirectoryEntry.FullPath)
  117. }
  118. */
  119. oldEntry, _ := f.FindEntry(ctx, entry.FullPath)
  120. if oldEntry == nil {
  121. if err := f.store.InsertEntry(ctx, entry); err != nil {
  122. glog.Errorf("insert entry %s: %v", entry.FullPath, err)
  123. return fmt.Errorf("insert entry %s: %v", entry.FullPath, err)
  124. }
  125. } else {
  126. if err := f.UpdateEntry(ctx, oldEntry, entry); err != nil {
  127. glog.Errorf("update entry %s: %v", entry.FullPath, err)
  128. return fmt.Errorf("update entry %s: %v", entry.FullPath, err)
  129. }
  130. }
  131. f.NotifyUpdateEvent(oldEntry, entry, true)
  132. f.deleteChunksIfNotNew(oldEntry, entry)
  133. return nil
  134. }
  135. func (f *Filer) UpdateEntry(ctx context.Context, oldEntry, entry *Entry) (err error) {
  136. if oldEntry != nil {
  137. if oldEntry.IsDirectory() && !entry.IsDirectory() {
  138. glog.Errorf("existing %s is a directory", entry.FullPath)
  139. return fmt.Errorf("existing %s is a directory", entry.FullPath)
  140. }
  141. if !oldEntry.IsDirectory() && entry.IsDirectory() {
  142. glog.Errorf("existing %s is a file", entry.FullPath)
  143. return fmt.Errorf("existing %s is a file", entry.FullPath)
  144. }
  145. }
  146. return f.store.UpdateEntry(ctx, entry)
  147. }
  148. func (f *Filer) FindEntry(ctx context.Context, p FullPath) (entry *Entry, err error) {
  149. now := time.Now()
  150. if string(p) == "/" {
  151. return &Entry{
  152. FullPath: p,
  153. Attr: Attr{
  154. Mtime: now,
  155. Crtime: now,
  156. Mode: os.ModeDir | 0755,
  157. Uid: OS_UID,
  158. Gid: OS_GID,
  159. },
  160. }, nil
  161. }
  162. return f.store.FindEntry(ctx, p)
  163. }
  164. func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p FullPath, isRecursive bool, shouldDeleteChunks bool) (err error) {
  165. entry, err := f.FindEntry(ctx, p)
  166. if err != nil {
  167. return err
  168. }
  169. if entry.IsDirectory() {
  170. limit := int(1)
  171. if isRecursive {
  172. limit = math.MaxInt32
  173. }
  174. lastFileName := ""
  175. includeLastFile := false
  176. for limit > 0 {
  177. entries, err := f.ListDirectoryEntries(ctx, p, lastFileName, includeLastFile, 1024)
  178. if err != nil {
  179. glog.Errorf("list folder %s: %v", p, err)
  180. return fmt.Errorf("list folder %s: %v", p, err)
  181. }
  182. if len(entries) == 0 {
  183. break
  184. }
  185. if isRecursive {
  186. for _, sub := range entries {
  187. lastFileName = sub.Name()
  188. err = f.DeleteEntryMetaAndData(ctx, sub.FullPath, isRecursive, shouldDeleteChunks)
  189. if err != nil {
  190. return err
  191. }
  192. limit--
  193. if limit <= 0 {
  194. break
  195. }
  196. }
  197. }
  198. if len(entries) < 1024 {
  199. break
  200. }
  201. }
  202. f.cacheDelDirectory(string(p))
  203. }
  204. if shouldDeleteChunks {
  205. f.DeleteChunks(p, entry.Chunks)
  206. }
  207. if p == "/" {
  208. return nil
  209. }
  210. glog.V(3).Infof("deleting entry %v", p)
  211. f.NotifyUpdateEvent(entry, nil, shouldDeleteChunks)
  212. return f.store.DeleteEntry(ctx, p)
  213. }
  214. func (f *Filer) ListDirectoryEntries(ctx context.Context, p FullPath, startFileName string, inclusive bool, limit int) ([]*Entry, error) {
  215. if strings.HasSuffix(string(p), "/") && len(p) > 1 {
  216. p = p[0 : len(p)-1]
  217. }
  218. return f.store.ListDirectoryEntries(ctx, p, startFileName, inclusive, limit)
  219. }
  220. func (f *Filer) cacheDelDirectory(dirpath string) {
  221. if dirpath == "/" {
  222. return
  223. }
  224. if f.directoryCache == nil {
  225. return
  226. }
  227. f.directoryCache.Delete(dirpath)
  228. return
  229. }
  230. func (f *Filer) cacheGetDirectory(dirpath string) *Entry {
  231. if f.directoryCache == nil {
  232. return nil
  233. }
  234. item := f.directoryCache.Get(dirpath)
  235. if item == nil {
  236. return nil
  237. }
  238. return item.Value().(*Entry)
  239. }
  240. func (f *Filer) cacheSetDirectory(dirpath string, dirEntry *Entry, level int) {
  241. if f.directoryCache == nil {
  242. return
  243. }
  244. minutes := 60
  245. if level < 10 {
  246. minutes -= level * 6
  247. }
  248. f.directoryCache.Set(dirpath, dirEntry, time.Duration(minutes)*time.Minute)
  249. }