You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

364 lines
10 KiB

7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
6 years ago
5 years ago
5 years ago
5 years ago
6 years ago
5 years ago
  1. package weed_server
  2. import (
  3. "context"
  4. "fmt"
  5. "os"
  6. "path/filepath"
  7. "strconv"
  8. "strings"
  9. "time"
  10. "github.com/chrislusf/seaweedfs/weed/filer2"
  11. "github.com/chrislusf/seaweedfs/weed/glog"
  12. "github.com/chrislusf/seaweedfs/weed/operation"
  13. "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
  14. "github.com/chrislusf/seaweedfs/weed/pb/master_pb"
  15. "github.com/chrislusf/seaweedfs/weed/util"
  16. )
  17. func (fs *FilerServer) LookupDirectoryEntry(ctx context.Context, req *filer_pb.LookupDirectoryEntryRequest) (*filer_pb.LookupDirectoryEntryResponse, error) {
  18. entry, err := fs.filer.FindEntry(ctx, util.JoinPath(req.Directory, req.Name))
  19. if err == filer_pb.ErrNotFound {
  20. return &filer_pb.LookupDirectoryEntryResponse{}, nil
  21. }
  22. if err != nil {
  23. glog.V(3).Infof("LookupDirectoryEntry %s: %+v, ", filepath.Join(req.Directory, req.Name), err)
  24. return nil, err
  25. }
  26. return &filer_pb.LookupDirectoryEntryResponse{
  27. Entry: &filer_pb.Entry{
  28. Name: req.Name,
  29. IsDirectory: entry.IsDirectory(),
  30. Attributes: filer2.EntryAttributeToPb(entry),
  31. Chunks: entry.Chunks,
  32. Extended: entry.Extended,
  33. },
  34. }, nil
  35. }
  36. func (fs *FilerServer) ListEntries(req *filer_pb.ListEntriesRequest, stream filer_pb.SeaweedFiler_ListEntriesServer) error {
  37. limit := int(req.Limit)
  38. if limit == 0 {
  39. limit = fs.option.DirListingLimit
  40. }
  41. paginationLimit := filer2.PaginationSize
  42. if limit < paginationLimit {
  43. paginationLimit = limit
  44. }
  45. lastFileName := req.StartFromFileName
  46. includeLastFile := req.InclusiveStartFrom
  47. for limit > 0 {
  48. entries, err := fs.filer.ListDirectoryEntries(stream.Context(), util.FullPath(req.Directory), lastFileName, includeLastFile, paginationLimit)
  49. if err != nil {
  50. return err
  51. }
  52. if len(entries) == 0 {
  53. return nil
  54. }
  55. includeLastFile = false
  56. for _, entry := range entries {
  57. lastFileName = entry.Name()
  58. if req.Prefix != "" {
  59. if !strings.HasPrefix(entry.Name(), req.Prefix) {
  60. continue
  61. }
  62. }
  63. if err := stream.Send(&filer_pb.ListEntriesResponse{
  64. Entry: &filer_pb.Entry{
  65. Name: entry.Name(),
  66. IsDirectory: entry.IsDirectory(),
  67. Chunks: entry.Chunks,
  68. Attributes: filer2.EntryAttributeToPb(entry),
  69. Extended: entry.Extended,
  70. },
  71. }); err != nil {
  72. return err
  73. }
  74. limit--
  75. if limit == 0 {
  76. return nil
  77. }
  78. }
  79. if len(entries) < paginationLimit {
  80. break
  81. }
  82. }
  83. return nil
  84. }
  85. func (fs *FilerServer) LookupVolume(ctx context.Context, req *filer_pb.LookupVolumeRequest) (*filer_pb.LookupVolumeResponse, error) {
  86. resp := &filer_pb.LookupVolumeResponse{
  87. LocationsMap: make(map[string]*filer_pb.Locations),
  88. }
  89. for _, vidString := range req.VolumeIds {
  90. vid, err := strconv.Atoi(vidString)
  91. if err != nil {
  92. glog.V(1).Infof("Unknown volume id %d", vid)
  93. return nil, err
  94. }
  95. var locs []*filer_pb.Location
  96. locations, found := fs.filer.MasterClient.GetLocations(uint32(vid))
  97. if !found {
  98. continue
  99. }
  100. for _, loc := range locations {
  101. locs = append(locs, &filer_pb.Location{
  102. Url: loc.Url,
  103. PublicUrl: loc.PublicUrl,
  104. })
  105. }
  106. resp.LocationsMap[vidString] = &filer_pb.Locations{
  107. Locations: locs,
  108. }
  109. }
  110. return resp, nil
  111. }
  112. func (fs *FilerServer) CreateEntry(ctx context.Context, req *filer_pb.CreateEntryRequest) (resp *filer_pb.CreateEntryResponse, err error) {
  113. resp = &filer_pb.CreateEntryResponse{}
  114. chunks, garbages := filer2.CompactFileChunks(req.Entry.Chunks)
  115. if req.Entry.Attributes == nil {
  116. glog.V(3).Infof("CreateEntry %s: nil attributes", filepath.Join(req.Directory, req.Entry.Name))
  117. resp.Error = fmt.Sprintf("can not create entry with empty attributes")
  118. return
  119. }
  120. createErr := fs.filer.CreateEntry(ctx, &filer2.Entry{
  121. FullPath: util.JoinPath(req.Directory, req.Entry.Name),
  122. Attr: filer2.PbToEntryAttribute(req.Entry.Attributes),
  123. Chunks: chunks,
  124. }, req.OExcl)
  125. if createErr == nil {
  126. fs.filer.DeleteChunks(garbages)
  127. } else {
  128. glog.V(3).Infof("CreateEntry %s: %v", filepath.Join(req.Directory, req.Entry.Name), createErr)
  129. resp.Error = createErr.Error()
  130. }
  131. return
  132. }
  133. func (fs *FilerServer) UpdateEntry(ctx context.Context, req *filer_pb.UpdateEntryRequest) (*filer_pb.UpdateEntryResponse, error) {
  134. fullpath := util.Join(req.Directory, req.Entry.Name)
  135. entry, err := fs.filer.FindEntry(ctx, util.FullPath(fullpath))
  136. if err != nil {
  137. return &filer_pb.UpdateEntryResponse{}, fmt.Errorf("not found %s: %v", fullpath, err)
  138. }
  139. // remove old chunks if not included in the new ones
  140. unusedChunks := filer2.MinusChunks(entry.Chunks, req.Entry.Chunks)
  141. chunks, garbages := filer2.CompactFileChunks(req.Entry.Chunks)
  142. newEntry := &filer2.Entry{
  143. FullPath: util.JoinPath(req.Directory, req.Entry.Name),
  144. Attr: entry.Attr,
  145. Extended: req.Entry.Extended,
  146. Chunks: chunks,
  147. }
  148. glog.V(3).Infof("updating %s: %+v, chunks %d: %v => %+v, chunks %d: %v, extended: %v => %v",
  149. fullpath, entry.Attr, len(entry.Chunks), entry.Chunks,
  150. req.Entry.Attributes, len(req.Entry.Chunks), req.Entry.Chunks,
  151. entry.Extended, req.Entry.Extended)
  152. if req.Entry.Attributes != nil {
  153. if req.Entry.Attributes.Mtime != 0 {
  154. newEntry.Attr.Mtime = time.Unix(req.Entry.Attributes.Mtime, 0)
  155. }
  156. if req.Entry.Attributes.FileMode != 0 {
  157. newEntry.Attr.Mode = os.FileMode(req.Entry.Attributes.FileMode)
  158. }
  159. newEntry.Attr.Uid = req.Entry.Attributes.Uid
  160. newEntry.Attr.Gid = req.Entry.Attributes.Gid
  161. newEntry.Attr.Mime = req.Entry.Attributes.Mime
  162. newEntry.Attr.UserName = req.Entry.Attributes.UserName
  163. newEntry.Attr.GroupNames = req.Entry.Attributes.GroupName
  164. }
  165. if filer2.EqualEntry(entry, newEntry) {
  166. return &filer_pb.UpdateEntryResponse{}, err
  167. }
  168. if err = fs.filer.UpdateEntry(ctx, entry, newEntry); err == nil {
  169. fs.filer.DeleteChunks(unusedChunks)
  170. fs.filer.DeleteChunks(garbages)
  171. } else {
  172. glog.V(3).Infof("UpdateEntry %s: %v", filepath.Join(req.Directory, req.Entry.Name), err)
  173. }
  174. fs.filer.NotifyUpdateEvent(entry, newEntry, true)
  175. return &filer_pb.UpdateEntryResponse{}, err
  176. }
  177. func (fs *FilerServer) AppendToEntry(ctx context.Context, req *filer_pb.AppendToEntryRequest) (*filer_pb.AppendToEntryResponse, error) {
  178. fullpath := util.NewFullPath(req.Directory, req.EntryName)
  179. var offset int64 = 0
  180. entry, err := fs.filer.FindEntry(ctx, util.FullPath(fullpath))
  181. if err == filer_pb.ErrNotFound {
  182. entry = &filer2.Entry{
  183. FullPath: fullpath,
  184. Attr: filer2.Attr{
  185. Crtime: time.Now(),
  186. Mtime: time.Now(),
  187. Mode: os.FileMode(0644),
  188. Uid: OS_UID,
  189. Gid: OS_GID,
  190. },
  191. }
  192. } else {
  193. offset = int64(filer2.TotalSize(entry.Chunks))
  194. }
  195. for _, chunk := range req.Chunks {
  196. chunk.Offset = offset
  197. offset += int64(chunk.Size)
  198. }
  199. entry.Chunks = append(entry.Chunks, req.Chunks...)
  200. err = fs.filer.CreateEntry(context.Background(), entry, false)
  201. return &filer_pb.AppendToEntryResponse{}, err
  202. }
  203. func (fs *FilerServer) DeleteEntry(ctx context.Context, req *filer_pb.DeleteEntryRequest) (resp *filer_pb.DeleteEntryResponse, err error) {
  204. err = fs.filer.DeleteEntryMetaAndData(ctx, util.JoinPath(req.Directory, req.Name), req.IsRecursive, req.IgnoreRecursiveError, req.IsDeleteData)
  205. resp = &filer_pb.DeleteEntryResponse{}
  206. if err != nil {
  207. resp.Error = err.Error()
  208. }
  209. return resp, nil
  210. }
  211. func (fs *FilerServer) AssignVolume(ctx context.Context, req *filer_pb.AssignVolumeRequest) (resp *filer_pb.AssignVolumeResponse, err error) {
  212. ttlStr := ""
  213. if req.TtlSec > 0 {
  214. ttlStr = strconv.Itoa(int(req.TtlSec))
  215. }
  216. collection, replication, _ := fs.detectCollection(req.ParentPath, req.Collection, req.Replication)
  217. var altRequest *operation.VolumeAssignRequest
  218. dataCenter := req.DataCenter
  219. if dataCenter == "" {
  220. dataCenter = fs.option.DataCenter
  221. }
  222. assignRequest := &operation.VolumeAssignRequest{
  223. Count: uint64(req.Count),
  224. Replication: replication,
  225. Collection: collection,
  226. Ttl: ttlStr,
  227. DataCenter: dataCenter,
  228. }
  229. if dataCenter != "" {
  230. altRequest = &operation.VolumeAssignRequest{
  231. Count: uint64(req.Count),
  232. Replication: replication,
  233. Collection: collection,
  234. Ttl: ttlStr,
  235. DataCenter: "",
  236. }
  237. }
  238. assignResult, err := operation.Assign(fs.filer.GetMaster(), fs.grpcDialOption, assignRequest, altRequest)
  239. if err != nil {
  240. glog.V(3).Infof("AssignVolume: %v", err)
  241. return &filer_pb.AssignVolumeResponse{Error: fmt.Sprintf("assign volume: %v", err)}, nil
  242. }
  243. if assignResult.Error != "" {
  244. glog.V(3).Infof("AssignVolume error: %v", assignResult.Error)
  245. return &filer_pb.AssignVolumeResponse{Error: fmt.Sprintf("assign volume result: %v", assignResult.Error)}, nil
  246. }
  247. return &filer_pb.AssignVolumeResponse{
  248. FileId: assignResult.Fid,
  249. Count: int32(assignResult.Count),
  250. Url: assignResult.Url,
  251. PublicUrl: assignResult.PublicUrl,
  252. Auth: string(assignResult.Auth),
  253. Collection: collection,
  254. Replication: replication,
  255. }, nil
  256. }
  257. func (fs *FilerServer) DeleteCollection(ctx context.Context, req *filer_pb.DeleteCollectionRequest) (resp *filer_pb.DeleteCollectionResponse, err error) {
  258. err = fs.filer.MasterClient.WithClient(func(client master_pb.SeaweedClient) error {
  259. _, err := client.CollectionDelete(context.Background(), &master_pb.CollectionDeleteRequest{
  260. Name: req.GetCollection(),
  261. })
  262. return err
  263. })
  264. return &filer_pb.DeleteCollectionResponse{}, err
  265. }
  266. func (fs *FilerServer) Statistics(ctx context.Context, req *filer_pb.StatisticsRequest) (resp *filer_pb.StatisticsResponse, err error) {
  267. var output *master_pb.StatisticsResponse
  268. err = fs.filer.MasterClient.WithClient(func(masterClient master_pb.SeaweedClient) error {
  269. grpcResponse, grpcErr := masterClient.Statistics(context.Background(), &master_pb.StatisticsRequest{
  270. Replication: req.Replication,
  271. Collection: req.Collection,
  272. Ttl: req.Ttl,
  273. })
  274. if grpcErr != nil {
  275. return grpcErr
  276. }
  277. output = grpcResponse
  278. return nil
  279. })
  280. if err != nil {
  281. return nil, err
  282. }
  283. return &filer_pb.StatisticsResponse{
  284. TotalSize: output.TotalSize,
  285. UsedSize: output.UsedSize,
  286. FileCount: output.FileCount,
  287. }, nil
  288. }
  289. func (fs *FilerServer) GetFilerConfiguration(ctx context.Context, req *filer_pb.GetFilerConfigurationRequest) (resp *filer_pb.GetFilerConfigurationResponse, err error) {
  290. return &filer_pb.GetFilerConfigurationResponse{
  291. Masters: fs.option.Masters,
  292. Collection: fs.option.Collection,
  293. Replication: fs.option.DefaultReplication,
  294. MaxMb: uint32(fs.option.MaxMB),
  295. DirBuckets: fs.filer.DirBucketsPath,
  296. Cipher: fs.filer.Cipher,
  297. }, nil
  298. }