You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

353 lines
10 KiB

7 years ago
6 years ago
7 years ago
7 years ago
7 years ago
6 years ago
7 years ago
4 years ago
7 years ago
2 years ago
5 years ago
5 years ago
7 years ago
3 years ago
7 years ago
4 years ago
7 years ago
2 years ago
5 years ago
3 years ago
5 years ago
4 years ago
5 years ago
5 years ago
7 years ago
7 years ago
6 years ago
5 years ago
6 years ago
5 years ago
7 years ago
4 years ago
4 years ago
7 years ago
4 years ago
4 years ago
4 years ago
7 years ago
7 years ago
5 years ago
  1. package filer
  2. import (
  3. "context"
  4. "fmt"
  5. "os"
  6. "sort"
  7. "strings"
  8. "time"
  9. "github.com/seaweedfs/seaweedfs/weed/cluster"
  10. "github.com/seaweedfs/seaweedfs/weed/pb"
  11. "github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
  12. "google.golang.org/grpc"
  13. "github.com/seaweedfs/seaweedfs/weed/glog"
  14. "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
  15. "github.com/seaweedfs/seaweedfs/weed/util"
  16. "github.com/seaweedfs/seaweedfs/weed/util/log_buffer"
  17. "github.com/seaweedfs/seaweedfs/weed/wdclient"
  18. )
  19. const (
  20. LogFlushInterval = time.Minute
  21. PaginationSize = 1024
  22. FilerStoreId = "filer.store.id"
  23. )
  24. var (
  25. OS_UID = uint32(os.Getuid())
  26. OS_GID = uint32(os.Getgid())
  27. )
  28. type Filer struct {
  29. UniqueFilerId int32
  30. UniqueFilerEpoch int32
  31. Store VirtualFilerStore
  32. MasterClient *wdclient.MasterClient
  33. fileIdDeletionQueue *util.UnboundedQueue
  34. GrpcDialOption grpc.DialOption
  35. DirBucketsPath string
  36. Cipher bool
  37. LocalMetaLogBuffer *log_buffer.LogBuffer
  38. metaLogCollection string
  39. metaLogReplication string
  40. MetaAggregator *MetaAggregator
  41. Signature int32
  42. FilerConf *FilerConf
  43. RemoteStorage *FilerRemoteStorage
  44. }
  45. func NewFiler(masters map[string]pb.ServerAddress, grpcDialOption grpc.DialOption, filerHost pb.ServerAddress,
  46. filerGroup string, collection string, replication string, dataCenter string, notifyFn func()) *Filer {
  47. f := &Filer{
  48. MasterClient: wdclient.NewMasterClient(grpcDialOption, filerGroup, cluster.FilerType, filerHost, dataCenter, "", masters),
  49. fileIdDeletionQueue: util.NewUnboundedQueue(),
  50. GrpcDialOption: grpcDialOption,
  51. FilerConf: NewFilerConf(),
  52. RemoteStorage: NewFilerRemoteStorage(),
  53. UniqueFilerId: util.RandomInt32(),
  54. }
  55. if f.UniqueFilerId < 0 {
  56. f.UniqueFilerId = -f.UniqueFilerId
  57. }
  58. f.LocalMetaLogBuffer = log_buffer.NewLogBuffer("local", LogFlushInterval, f.logFlushFunc, notifyFn)
  59. f.metaLogCollection = collection
  60. f.metaLogReplication = replication
  61. go f.loopProcessingDeletion()
  62. return f
  63. }
  64. func (f *Filer) MaybeBootstrapFromPeers(self pb.ServerAddress, existingNodes []*master_pb.ClusterNodeUpdate, snapshotTime time.Time) (err error) {
  65. if len(existingNodes) == 0 {
  66. return
  67. }
  68. sort.Slice(existingNodes, func(i, j int) bool {
  69. return existingNodes[i].CreatedAtNs < existingNodes[j].CreatedAtNs
  70. })
  71. earliestNode := existingNodes[0]
  72. if earliestNode.Address == string(self) {
  73. return
  74. }
  75. glog.V(0).Infof("bootstrap from %v clientId:%d", earliestNode.Address, f.UniqueFilerId)
  76. f.UniqueFilerEpoch++
  77. metadataFollowOption := &pb.MetadataFollowOption{
  78. ClientName: "bootstrap",
  79. ClientId: f.UniqueFilerId,
  80. ClientEpoch: f.UniqueFilerEpoch,
  81. SelfSignature: f.Signature,
  82. PathPrefix: "/",
  83. AdditionalPathPrefixes: nil,
  84. DirectoriesToWatch: nil,
  85. StartTsNs: snapshotTime.UnixNano(),
  86. StopTsNs: 0,
  87. EventErrorType: pb.FatalOnError,
  88. }
  89. err = pb.FollowMetadata(pb.ServerAddress(earliestNode.Address), f.GrpcDialOption, metadataFollowOption, func(resp *filer_pb.SubscribeMetadataResponse) error {
  90. return Replay(f.Store, resp)
  91. })
  92. return
  93. }
  94. func (f *Filer) AggregateFromPeers(self pb.ServerAddress, existingNodes []*master_pb.ClusterNodeUpdate, startFrom time.Time) {
  95. f.MetaAggregator = NewMetaAggregator(f, self, f.GrpcDialOption)
  96. f.MasterClient.SetOnPeerUpdateFn(f.MetaAggregator.OnPeerUpdate)
  97. for _, peerUpdate := range existingNodes {
  98. f.MetaAggregator.OnPeerUpdate(peerUpdate, startFrom)
  99. }
  100. }
  101. func (f *Filer) ListExistingPeerUpdates() (existingNodes []*master_pb.ClusterNodeUpdate) {
  102. return cluster.ListExistingPeerUpdates(f.GetMaster(), f.GrpcDialOption, f.MasterClient.FilerGroup, cluster.FilerType)
  103. }
  104. func (f *Filer) SetStore(store FilerStore) (isFresh bool) {
  105. f.Store = NewFilerStoreWrapper(store)
  106. return f.setOrLoadFilerStoreSignature(store)
  107. }
  108. func (f *Filer) setOrLoadFilerStoreSignature(store FilerStore) (isFresh bool) {
  109. storeIdBytes, err := store.KvGet(context.Background(), []byte(FilerStoreId))
  110. if err == ErrKvNotFound || err == nil && len(storeIdBytes) == 0 {
  111. f.Signature = util.RandomInt32()
  112. storeIdBytes = make([]byte, 4)
  113. util.Uint32toBytes(storeIdBytes, uint32(f.Signature))
  114. if err = store.KvPut(context.Background(), []byte(FilerStoreId), storeIdBytes); err != nil {
  115. glog.Fatalf("set %s=%d : %v", FilerStoreId, f.Signature, err)
  116. }
  117. glog.V(0).Infof("create %s to %d", FilerStoreId, f.Signature)
  118. return true
  119. } else if err == nil && len(storeIdBytes) == 4 {
  120. f.Signature = int32(util.BytesToUint32(storeIdBytes))
  121. glog.V(0).Infof("existing %s = %d", FilerStoreId, f.Signature)
  122. } else {
  123. glog.Fatalf("read %v=%v : %v", FilerStoreId, string(storeIdBytes), err)
  124. }
  125. return false
  126. }
  127. func (f *Filer) GetStore() (store FilerStore) {
  128. return f.Store
  129. }
  130. func (fs *Filer) GetMaster() pb.ServerAddress {
  131. return fs.MasterClient.GetMaster()
  132. }
  133. func (fs *Filer) KeepMasterClientConnected() {
  134. fs.MasterClient.KeepConnectedToMaster()
  135. }
  136. func (f *Filer) BeginTransaction(ctx context.Context) (context.Context, error) {
  137. return f.Store.BeginTransaction(ctx)
  138. }
  139. func (f *Filer) CommitTransaction(ctx context.Context) error {
  140. return f.Store.CommitTransaction(ctx)
  141. }
  142. func (f *Filer) RollbackTransaction(ctx context.Context) error {
  143. return f.Store.RollbackTransaction(ctx)
  144. }
  145. func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, o_excl bool, isFromOtherCluster bool, signatures []int32, skipCreateParentDir bool) error {
  146. if string(entry.FullPath) == "/" {
  147. return nil
  148. }
  149. oldEntry, _ := f.FindEntry(ctx, entry.FullPath)
  150. /*
  151. if !hasWritePermission(lastDirectoryEntry, entry) {
  152. glog.V(0).Infof("directory %s: %v, entry: uid=%d gid=%d",
  153. lastDirectoryEntry.FullPath, lastDirectoryEntry.Attr, entry.Uid, entry.Gid)
  154. return fmt.Errorf("no write permission in folder %v", lastDirectoryEntry.FullPath)
  155. }
  156. */
  157. if oldEntry == nil {
  158. if !skipCreateParentDir {
  159. dirParts := strings.Split(string(entry.FullPath), "/")
  160. if err := f.ensureParentDirectoryEntry(ctx, entry, dirParts, len(dirParts)-1, isFromOtherCluster); err != nil {
  161. return err
  162. }
  163. }
  164. glog.V(4).Infof("InsertEntry %s: new entry: %v", entry.FullPath, entry.Name())
  165. if err := f.Store.InsertEntry(ctx, entry); err != nil {
  166. glog.Errorf("insert entry %s: %v", entry.FullPath, err)
  167. return fmt.Errorf("insert entry %s: %v", entry.FullPath, err)
  168. }
  169. } else {
  170. if o_excl {
  171. glog.V(3).Infof("EEXIST: entry %s already exists", entry.FullPath)
  172. return fmt.Errorf("EEXIST: entry %s already exists", entry.FullPath)
  173. }
  174. glog.V(4).Infof("UpdateEntry %s: old entry: %v", entry.FullPath, oldEntry.Name())
  175. if err := f.UpdateEntry(ctx, oldEntry, entry); err != nil {
  176. glog.Errorf("update entry %s: %v", entry.FullPath, err)
  177. return fmt.Errorf("update entry %s: %v", entry.FullPath, err)
  178. }
  179. }
  180. f.NotifyUpdateEvent(ctx, oldEntry, entry, true, isFromOtherCluster, signatures)
  181. f.deleteChunksIfNotNew(oldEntry, entry)
  182. glog.V(4).Infof("CreateEntry %s: created", entry.FullPath)
  183. return nil
  184. }
  185. func (f *Filer) ensureParentDirectoryEntry(ctx context.Context, entry *Entry, dirParts []string, level int, isFromOtherCluster bool) (err error) {
  186. if level == 0 {
  187. return nil
  188. }
  189. dirPath := "/" + util.Join(dirParts[:level]...)
  190. // fmt.Printf("%d directory: %+v\n", i, dirPath)
  191. // check the store directly
  192. glog.V(4).Infof("find uncached directory: %s", dirPath)
  193. dirEntry, _ := f.FindEntry(ctx, util.FullPath(dirPath))
  194. // no such existing directory
  195. if dirEntry == nil {
  196. // ensure parent directory
  197. if err = f.ensureParentDirectoryEntry(ctx, entry, dirParts, level-1, isFromOtherCluster); err != nil {
  198. return err
  199. }
  200. // create the directory
  201. now := time.Now()
  202. dirEntry = &Entry{
  203. FullPath: util.FullPath(dirPath),
  204. Attr: Attr{
  205. Mtime: now,
  206. Crtime: now,
  207. Mode: os.ModeDir | entry.Mode | 0111,
  208. Uid: entry.Uid,
  209. Gid: entry.Gid,
  210. UserName: entry.UserName,
  211. GroupNames: entry.GroupNames,
  212. },
  213. }
  214. glog.V(2).Infof("create directory: %s %v", dirPath, dirEntry.Mode)
  215. mkdirErr := f.Store.InsertEntry(ctx, dirEntry)
  216. if mkdirErr != nil {
  217. if _, err := f.FindEntry(ctx, util.FullPath(dirPath)); err == filer_pb.ErrNotFound {
  218. glog.V(3).Infof("mkdir %s: %v", dirPath, mkdirErr)
  219. return fmt.Errorf("mkdir %s: %v", dirPath, mkdirErr)
  220. }
  221. } else {
  222. if !strings.HasPrefix("/"+util.Join(dirParts[:]...), SystemLogDir) {
  223. f.NotifyUpdateEvent(ctx, nil, dirEntry, false, isFromOtherCluster, nil)
  224. }
  225. }
  226. } else if !dirEntry.IsDirectory() {
  227. glog.Errorf("CreateEntry %s: %s should be a directory", entry.FullPath, dirPath)
  228. return fmt.Errorf("%s is a file", dirPath)
  229. }
  230. return nil
  231. }
  232. func (f *Filer) UpdateEntry(ctx context.Context, oldEntry, entry *Entry) (err error) {
  233. if oldEntry != nil {
  234. entry.Attr.Crtime = oldEntry.Attr.Crtime
  235. if oldEntry.IsDirectory() && !entry.IsDirectory() {
  236. glog.Errorf("existing %s is a directory", oldEntry.FullPath)
  237. return fmt.Errorf("existing %s is a directory", oldEntry.FullPath)
  238. }
  239. if !oldEntry.IsDirectory() && entry.IsDirectory() {
  240. glog.Errorf("existing %s is a file", oldEntry.FullPath)
  241. return fmt.Errorf("existing %s is a file", oldEntry.FullPath)
  242. }
  243. }
  244. return f.Store.UpdateEntry(ctx, entry)
  245. }
  246. var (
  247. Root = &Entry{
  248. FullPath: "/",
  249. Attr: Attr{
  250. Mtime: time.Now(),
  251. Crtime: time.Now(),
  252. Mode: os.ModeDir | 0755,
  253. Uid: OS_UID,
  254. Gid: OS_GID,
  255. },
  256. }
  257. )
  258. func (f *Filer) FindEntry(ctx context.Context, p util.FullPath) (entry *Entry, err error) {
  259. if string(p) == "/" {
  260. return Root, nil
  261. }
  262. entry, err = f.Store.FindEntry(ctx, p)
  263. if entry != nil && entry.TtlSec > 0 {
  264. if entry.Crtime.Add(time.Duration(entry.TtlSec) * time.Second).Before(time.Now()) {
  265. f.Store.DeleteOneEntry(ctx, entry)
  266. return nil, filer_pb.ErrNotFound
  267. }
  268. }
  269. return
  270. }
  271. func (f *Filer) doListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int64, prefix string, eachEntryFunc ListEachEntryFunc) (expiredCount int64, lastFileName string, err error) {
  272. lastFileName, err = f.Store.ListDirectoryPrefixedEntries(ctx, p, startFileName, inclusive, limit, prefix, func(entry *Entry) bool {
  273. select {
  274. case <-ctx.Done():
  275. return false
  276. default:
  277. if entry.TtlSec > 0 {
  278. if entry.Crtime.Add(time.Duration(entry.TtlSec) * time.Second).Before(time.Now()) {
  279. f.Store.DeleteOneEntry(ctx, entry)
  280. expiredCount++
  281. return true
  282. }
  283. }
  284. return eachEntryFunc(entry)
  285. }
  286. })
  287. if err != nil {
  288. return expiredCount, lastFileName, err
  289. }
  290. return
  291. }
  292. func (f *Filer) Shutdown() {
  293. f.LocalMetaLogBuffer.Shutdown()
  294. f.Store.Shutdown()
  295. }