You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

382 lines
11 KiB

7 years ago
6 years ago
7 years ago
7 years ago
7 years ago
6 years ago
7 years ago
4 years ago
7 years ago
2 years ago
5 years ago
5 years ago
2 years ago
7 years ago
3 years ago
7 years ago
4 years ago
7 years ago
2 years ago
2 years ago
5 years ago
2 years ago
2 years ago
2 years ago
3 years ago
5 years ago
4 years ago
5 years ago
5 years ago
7 years ago
7 years ago
6 years ago
5 years ago
6 years ago
5 years ago
7 years ago
4 years ago
4 years ago
7 years ago
4 years ago
4 years ago
4 years ago
7 years ago
7 years ago
5 years ago
  1. package filer
  2. import (
  3. "context"
  4. "fmt"
  5. "os"
  6. "sort"
  7. "strings"
  8. "time"
  9. "github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
  10. "github.com/seaweedfs/seaweedfs/weed/cluster"
  11. "github.com/seaweedfs/seaweedfs/weed/pb"
  12. "github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
  13. "google.golang.org/grpc"
  14. "github.com/seaweedfs/seaweedfs/weed/glog"
  15. "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
  16. "github.com/seaweedfs/seaweedfs/weed/util"
  17. "github.com/seaweedfs/seaweedfs/weed/util/log_buffer"
  18. "github.com/seaweedfs/seaweedfs/weed/wdclient"
  19. )
  20. const (
  21. LogFlushInterval = time.Minute
  22. PaginationSize = 1024
  23. FilerStoreId = "filer.store.id"
  24. )
  25. var (
  26. OS_UID = uint32(os.Getuid())
  27. OS_GID = uint32(os.Getgid())
  28. )
  29. type Filer struct {
  30. UniqueFilerId int32
  31. UniqueFilerEpoch int32
  32. Store VirtualFilerStore
  33. MasterClient *wdclient.MasterClient
  34. fileIdDeletionQueue *util.UnboundedQueue
  35. GrpcDialOption grpc.DialOption
  36. DirBucketsPath string
  37. Cipher bool
  38. LocalMetaLogBuffer *log_buffer.LogBuffer
  39. metaLogCollection string
  40. metaLogReplication string
  41. MetaAggregator *MetaAggregator
  42. Signature int32
  43. FilerConf *FilerConf
  44. RemoteStorage *FilerRemoteStorage
  45. Dlm *lock_manager.DistributedLockManager
  46. MaxFilenameLength uint32
  47. }
  48. func NewFiler(masters pb.ServerDiscovery, grpcDialOption grpc.DialOption, filerHost pb.ServerAddress, filerGroup string, collection string, replication string, dataCenter string, maxFilenameLength uint32, notifyFn func()) *Filer {
  49. f := &Filer{
  50. MasterClient: wdclient.NewMasterClient(grpcDialOption, filerGroup, cluster.FilerType, filerHost, dataCenter, "", masters),
  51. fileIdDeletionQueue: util.NewUnboundedQueue(),
  52. GrpcDialOption: grpcDialOption,
  53. FilerConf: NewFilerConf(),
  54. RemoteStorage: NewFilerRemoteStorage(),
  55. UniqueFilerId: util.RandomInt32(),
  56. Dlm: lock_manager.NewDistributedLockManager(filerHost),
  57. MaxFilenameLength: maxFilenameLength,
  58. }
  59. if f.UniqueFilerId < 0 {
  60. f.UniqueFilerId = -f.UniqueFilerId
  61. }
  62. f.LocalMetaLogBuffer = log_buffer.NewLogBuffer("local", LogFlushInterval, f.logFlushFunc, notifyFn)
  63. f.metaLogCollection = collection
  64. f.metaLogReplication = replication
  65. go f.loopProcessingDeletion()
  66. return f
  67. }
  68. func (f *Filer) MaybeBootstrapFromPeers(self pb.ServerAddress, existingNodes []*master_pb.ClusterNodeUpdate, snapshotTime time.Time) (err error) {
  69. if len(existingNodes) == 0 {
  70. return
  71. }
  72. sort.Slice(existingNodes, func(i, j int) bool {
  73. return existingNodes[i].CreatedAtNs < existingNodes[j].CreatedAtNs
  74. })
  75. earliestNode := existingNodes[0]
  76. if earliestNode.Address == string(self) {
  77. return
  78. }
  79. glog.V(0).Infof("bootstrap from %v clientId:%d", earliestNode.Address, f.UniqueFilerId)
  80. f.UniqueFilerEpoch++
  81. metadataFollowOption := &pb.MetadataFollowOption{
  82. ClientName: "bootstrap",
  83. ClientId: f.UniqueFilerId,
  84. ClientEpoch: f.UniqueFilerEpoch,
  85. SelfSignature: f.Signature,
  86. PathPrefix: "/",
  87. AdditionalPathPrefixes: nil,
  88. DirectoriesToWatch: nil,
  89. StartTsNs: 0,
  90. StopTsNs: snapshotTime.UnixNano(),
  91. EventErrorType: pb.FatalOnError,
  92. }
  93. err = pb.FollowMetadata(pb.ServerAddress(earliestNode.Address), f.GrpcDialOption, metadataFollowOption, func(resp *filer_pb.SubscribeMetadataResponse) error {
  94. return Replay(f.Store, resp)
  95. })
  96. return
  97. }
  98. func (f *Filer) AggregateFromPeers(self pb.ServerAddress, existingNodes []*master_pb.ClusterNodeUpdate, startFrom time.Time) {
  99. var snapshot []pb.ServerAddress
  100. for _, node := range existingNodes {
  101. address := pb.ServerAddress(node.Address)
  102. snapshot = append(snapshot, address)
  103. }
  104. f.Dlm.LockRing.SetSnapshot(snapshot)
  105. glog.V(0).Infof("%s aggregate from peers %+v", self, snapshot)
  106. f.MetaAggregator = NewMetaAggregator(f, self, f.GrpcDialOption)
  107. f.MasterClient.SetOnPeerUpdateFn(func(update *master_pb.ClusterNodeUpdate, startFrom time.Time) {
  108. if update.NodeType != cluster.FilerType {
  109. return
  110. }
  111. address := pb.ServerAddress(update.Address)
  112. if update.IsAdd {
  113. f.Dlm.LockRing.AddServer(address)
  114. } else {
  115. f.Dlm.LockRing.RemoveServer(address)
  116. }
  117. f.MetaAggregator.OnPeerUpdate(update, startFrom)
  118. })
  119. for _, peerUpdate := range existingNodes {
  120. f.MetaAggregator.OnPeerUpdate(peerUpdate, startFrom)
  121. }
  122. }
  123. func (f *Filer) ListExistingPeerUpdates() (existingNodes []*master_pb.ClusterNodeUpdate) {
  124. return cluster.ListExistingPeerUpdates(f.GetMaster(), f.GrpcDialOption, f.MasterClient.FilerGroup, cluster.FilerType)
  125. }
  126. func (f *Filer) SetStore(store FilerStore) (isFresh bool) {
  127. f.Store = NewFilerStoreWrapper(store)
  128. return f.setOrLoadFilerStoreSignature(store)
  129. }
  130. func (f *Filer) setOrLoadFilerStoreSignature(store FilerStore) (isFresh bool) {
  131. storeIdBytes, err := store.KvGet(context.Background(), []byte(FilerStoreId))
  132. if err == ErrKvNotFound || err == nil && len(storeIdBytes) == 0 {
  133. f.Signature = util.RandomInt32()
  134. storeIdBytes = make([]byte, 4)
  135. util.Uint32toBytes(storeIdBytes, uint32(f.Signature))
  136. if err = store.KvPut(context.Background(), []byte(FilerStoreId), storeIdBytes); err != nil {
  137. glog.Fatalf("set %s=%d : %v", FilerStoreId, f.Signature, err)
  138. }
  139. glog.V(0).Infof("create %s to %d", FilerStoreId, f.Signature)
  140. return true
  141. } else if err == nil && len(storeIdBytes) == 4 {
  142. f.Signature = int32(util.BytesToUint32(storeIdBytes))
  143. glog.V(0).Infof("existing %s = %d", FilerStoreId, f.Signature)
  144. } else {
  145. glog.Fatalf("read %v=%v : %v", FilerStoreId, string(storeIdBytes), err)
  146. }
  147. return false
  148. }
  149. func (f *Filer) GetStore() (store FilerStore) {
  150. return f.Store
  151. }
  152. func (fs *Filer) GetMaster() pb.ServerAddress {
  153. return fs.MasterClient.GetMaster()
  154. }
  155. func (fs *Filer) KeepMasterClientConnected() {
  156. fs.MasterClient.KeepConnectedToMaster()
  157. }
  158. func (f *Filer) BeginTransaction(ctx context.Context) (context.Context, error) {
  159. return f.Store.BeginTransaction(ctx)
  160. }
  161. func (f *Filer) CommitTransaction(ctx context.Context) error {
  162. return f.Store.CommitTransaction(ctx)
  163. }
  164. func (f *Filer) RollbackTransaction(ctx context.Context) error {
  165. return f.Store.RollbackTransaction(ctx)
  166. }
  167. func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, o_excl bool, isFromOtherCluster bool, signatures []int32, skipCreateParentDir bool, maxFilenameLength uint32) error {
  168. if string(entry.FullPath) == "/" {
  169. return nil
  170. }
  171. if entry.FullPath.IsLongerFileName(maxFilenameLength) {
  172. return fmt.Errorf("entry name too long")
  173. }
  174. oldEntry, _ := f.FindEntry(ctx, entry.FullPath)
  175. /*
  176. if !hasWritePermission(lastDirectoryEntry, entry) {
  177. glog.V(0).Infof("directory %s: %v, entry: uid=%d gid=%d",
  178. lastDirectoryEntry.FullPath, lastDirectoryEntry.Attr, entry.Uid, entry.Gid)
  179. return fmt.Errorf("no write permission in folder %v", lastDirectoryEntry.FullPath)
  180. }
  181. */
  182. if oldEntry == nil {
  183. if !skipCreateParentDir {
  184. dirParts := strings.Split(string(entry.FullPath), "/")
  185. if err := f.ensureParentDirectoryEntry(ctx, entry, dirParts, len(dirParts)-1, isFromOtherCluster); err != nil {
  186. return err
  187. }
  188. }
  189. glog.V(4).Infof("InsertEntry %s: new entry: %v", entry.FullPath, entry.Name())
  190. if err := f.Store.InsertEntry(ctx, entry); err != nil {
  191. glog.Errorf("insert entry %s: %v", entry.FullPath, err)
  192. return fmt.Errorf("insert entry %s: %v", entry.FullPath, err)
  193. }
  194. } else {
  195. if o_excl {
  196. glog.V(3).Infof("EEXIST: entry %s already exists", entry.FullPath)
  197. return fmt.Errorf("EEXIST: entry %s already exists", entry.FullPath)
  198. }
  199. glog.V(4).Infof("UpdateEntry %s: old entry: %v", entry.FullPath, oldEntry.Name())
  200. if err := f.UpdateEntry(ctx, oldEntry, entry); err != nil {
  201. glog.Errorf("update entry %s: %v", entry.FullPath, err)
  202. return fmt.Errorf("update entry %s: %v", entry.FullPath, err)
  203. }
  204. }
  205. f.NotifyUpdateEvent(ctx, oldEntry, entry, true, isFromOtherCluster, signatures)
  206. f.deleteChunksIfNotNew(oldEntry, entry)
  207. glog.V(4).Infof("CreateEntry %s: created", entry.FullPath)
  208. return nil
  209. }
  210. func (f *Filer) ensureParentDirectoryEntry(ctx context.Context, entry *Entry, dirParts []string, level int, isFromOtherCluster bool) (err error) {
  211. if level == 0 {
  212. return nil
  213. }
  214. dirPath := "/" + util.Join(dirParts[:level]...)
  215. // fmt.Printf("%d directory: %+v\n", i, dirPath)
  216. // check the store directly
  217. glog.V(4).Infof("find uncached directory: %s", dirPath)
  218. dirEntry, _ := f.FindEntry(ctx, util.FullPath(dirPath))
  219. // no such existing directory
  220. if dirEntry == nil {
  221. // ensure parent directory
  222. if err = f.ensureParentDirectoryEntry(ctx, entry, dirParts, level-1, isFromOtherCluster); err != nil {
  223. return err
  224. }
  225. // create the directory
  226. now := time.Now()
  227. dirEntry = &Entry{
  228. FullPath: util.FullPath(dirPath),
  229. Attr: Attr{
  230. Mtime: now,
  231. Crtime: now,
  232. Mode: os.ModeDir | entry.Mode | 0111,
  233. Uid: entry.Uid,
  234. Gid: entry.Gid,
  235. UserName: entry.UserName,
  236. GroupNames: entry.GroupNames,
  237. },
  238. }
  239. glog.V(2).Infof("create directory: %s %v", dirPath, dirEntry.Mode)
  240. mkdirErr := f.Store.InsertEntry(ctx, dirEntry)
  241. if mkdirErr != nil {
  242. if _, err := f.FindEntry(ctx, util.FullPath(dirPath)); err == filer_pb.ErrNotFound {
  243. glog.V(3).Infof("mkdir %s: %v", dirPath, mkdirErr)
  244. return fmt.Errorf("mkdir %s: %v", dirPath, mkdirErr)
  245. }
  246. } else {
  247. if !strings.HasPrefix("/"+util.Join(dirParts[:]...), SystemLogDir) {
  248. f.NotifyUpdateEvent(ctx, nil, dirEntry, false, isFromOtherCluster, nil)
  249. }
  250. }
  251. } else if !dirEntry.IsDirectory() {
  252. glog.Errorf("CreateEntry %s: %s should be a directory", entry.FullPath, dirPath)
  253. return fmt.Errorf("%s is a file", dirPath)
  254. }
  255. return nil
  256. }
  257. func (f *Filer) UpdateEntry(ctx context.Context, oldEntry, entry *Entry) (err error) {
  258. if oldEntry != nil {
  259. entry.Attr.Crtime = oldEntry.Attr.Crtime
  260. if oldEntry.IsDirectory() && !entry.IsDirectory() {
  261. glog.Errorf("existing %s is a directory", oldEntry.FullPath)
  262. return fmt.Errorf("existing %s is a directory", oldEntry.FullPath)
  263. }
  264. if !oldEntry.IsDirectory() && entry.IsDirectory() {
  265. glog.Errorf("existing %s is a file", oldEntry.FullPath)
  266. return fmt.Errorf("existing %s is a file", oldEntry.FullPath)
  267. }
  268. }
  269. return f.Store.UpdateEntry(ctx, entry)
  270. }
  271. var (
  272. Root = &Entry{
  273. FullPath: "/",
  274. Attr: Attr{
  275. Mtime: time.Now(),
  276. Crtime: time.Now(),
  277. Mode: os.ModeDir | 0755,
  278. Uid: OS_UID,
  279. Gid: OS_GID,
  280. },
  281. }
  282. )
  283. func (f *Filer) FindEntry(ctx context.Context, p util.FullPath) (entry *Entry, err error) {
  284. if string(p) == "/" {
  285. return Root, nil
  286. }
  287. entry, err = f.Store.FindEntry(ctx, p)
  288. if entry != nil && entry.TtlSec > 0 {
  289. if entry.Crtime.Add(time.Duration(entry.TtlSec) * time.Second).Before(time.Now()) {
  290. f.Store.DeleteOneEntry(ctx, entry)
  291. return nil, filer_pb.ErrNotFound
  292. }
  293. }
  294. return
  295. }
  296. func (f *Filer) doListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int64, prefix string, eachEntryFunc ListEachEntryFunc) (expiredCount int64, lastFileName string, err error) {
  297. lastFileName, err = f.Store.ListDirectoryPrefixedEntries(ctx, p, startFileName, inclusive, limit, prefix, func(entry *Entry) bool {
  298. select {
  299. case <-ctx.Done():
  300. return false
  301. default:
  302. if entry.TtlSec > 0 {
  303. if entry.Crtime.Add(time.Duration(entry.TtlSec) * time.Second).Before(time.Now()) {
  304. f.Store.DeleteOneEntry(ctx, entry)
  305. expiredCount++
  306. return true
  307. }
  308. }
  309. return eachEntryFunc(entry)
  310. }
  311. })
  312. if err != nil {
  313. return expiredCount, lastFileName, err
  314. }
  315. return
  316. }
  317. func (f *Filer) Shutdown() {
  318. f.LocalMetaLogBuffer.Shutdown()
  319. f.Store.Shutdown()
  320. }