You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

575 lines
14 KiB

5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
  1. package weed_server
  2. import (
  3. "context"
  4. "fmt"
  5. "io"
  6. "math"
  7. "os"
  8. "path"
  9. "strings"
  10. "time"
  11. "golang.org/x/net/webdav"
  12. "google.golang.org/grpc"
  13. "github.com/chrislusf/seaweedfs/weed/operation"
  14. "github.com/chrislusf/seaweedfs/weed/pb"
  15. "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
  16. "github.com/chrislusf/seaweedfs/weed/util"
  17. "github.com/chrislusf/seaweedfs/weed/util/chunk_cache"
  18. "github.com/chrislusf/seaweedfs/weed/filer2"
  19. "github.com/chrislusf/seaweedfs/weed/glog"
  20. "github.com/chrislusf/seaweedfs/weed/security"
  21. )
  22. type WebDavOption struct {
  23. Filer string
  24. FilerGrpcAddress string
  25. DomainName string
  26. BucketsPath string
  27. GrpcDialOption grpc.DialOption
  28. Collection string
  29. Uid uint32
  30. Gid uint32
  31. Cipher bool
  32. CacheDir string
  33. CacheSizeMB int64
  34. }
  35. type WebDavServer struct {
  36. option *WebDavOption
  37. secret security.SigningKey
  38. filer *filer2.Filer
  39. grpcDialOption grpc.DialOption
  40. Handler *webdav.Handler
  41. }
  42. func NewWebDavServer(option *WebDavOption) (ws *WebDavServer, err error) {
  43. fs, _ := NewWebDavFileSystem(option)
  44. ws = &WebDavServer{
  45. option: option,
  46. grpcDialOption: security.LoadClientTLS(util.GetViper(), "grpc.filer"),
  47. Handler: &webdav.Handler{
  48. FileSystem: fs,
  49. LockSystem: webdav.NewMemLS(),
  50. },
  51. }
  52. return ws, nil
  53. }
  54. // adapted from https://github.com/mattn/davfs/blob/master/plugin/mysql/mysql.go
  55. type WebDavFileSystem struct {
  56. option *WebDavOption
  57. secret security.SigningKey
  58. filer *filer2.Filer
  59. grpcDialOption grpc.DialOption
  60. chunkCache *chunk_cache.ChunkCache
  61. }
  62. type FileInfo struct {
  63. name string
  64. size int64
  65. mode os.FileMode
  66. modifiledTime time.Time
  67. isDirectory bool
  68. }
  69. func (fi *FileInfo) Name() string { return fi.name }
  70. func (fi *FileInfo) Size() int64 { return fi.size }
  71. func (fi *FileInfo) Mode() os.FileMode { return fi.mode }
  72. func (fi *FileInfo) ModTime() time.Time { return fi.modifiledTime }
  73. func (fi *FileInfo) IsDir() bool { return fi.isDirectory }
  74. func (fi *FileInfo) Sys() interface{} { return nil }
  75. type WebDavFile struct {
  76. fs *WebDavFileSystem
  77. name string
  78. isDirectory bool
  79. off int64
  80. entry *filer_pb.Entry
  81. entryViewCache []filer2.VisibleInterval
  82. reader io.ReaderAt
  83. }
  84. func NewWebDavFileSystem(option *WebDavOption) (webdav.FileSystem, error) {
  85. chunkCache := chunk_cache.NewChunkCache(256, option.CacheDir, option.CacheSizeMB)
  86. util.OnInterrupt(func() {
  87. chunkCache.Shutdown()
  88. })
  89. return &WebDavFileSystem{
  90. option: option,
  91. chunkCache: chunkCache,
  92. }, nil
  93. }
  94. func (fs *WebDavFileSystem) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) error) error {
  95. return pb.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
  96. client := filer_pb.NewSeaweedFilerClient(grpcConnection)
  97. return fn(client)
  98. }, fs.option.FilerGrpcAddress, fs.option.GrpcDialOption)
  99. }
  100. func (fs *WebDavFileSystem) AdjustedUrl(hostAndPort string) string {
  101. return hostAndPort
  102. }
  103. func clearName(name string) (string, error) {
  104. slashed := strings.HasSuffix(name, "/")
  105. name = path.Clean(name)
  106. if !strings.HasSuffix(name, "/") && slashed {
  107. name += "/"
  108. }
  109. if !strings.HasPrefix(name, "/") {
  110. return "", os.ErrInvalid
  111. }
  112. return name, nil
  113. }
  114. func (fs *WebDavFileSystem) Mkdir(ctx context.Context, fullDirPath string, perm os.FileMode) error {
  115. glog.V(2).Infof("WebDavFileSystem.Mkdir %v", fullDirPath)
  116. if !strings.HasSuffix(fullDirPath, "/") {
  117. fullDirPath += "/"
  118. }
  119. var err error
  120. if fullDirPath, err = clearName(fullDirPath); err != nil {
  121. return err
  122. }
  123. _, err = fs.stat(ctx, fullDirPath)
  124. if err == nil {
  125. return os.ErrExist
  126. }
  127. return fs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
  128. dir, name := util.FullPath(fullDirPath).DirAndName()
  129. request := &filer_pb.CreateEntryRequest{
  130. Directory: dir,
  131. Entry: &filer_pb.Entry{
  132. Name: name,
  133. IsDirectory: true,
  134. Attributes: &filer_pb.FuseAttributes{
  135. Mtime: time.Now().Unix(),
  136. Crtime: time.Now().Unix(),
  137. FileMode: uint32(perm | os.ModeDir),
  138. Uid: fs.option.Uid,
  139. Gid: fs.option.Gid,
  140. },
  141. },
  142. }
  143. glog.V(1).Infof("mkdir: %v", request)
  144. if err := filer_pb.CreateEntry(client, request); err != nil {
  145. return fmt.Errorf("mkdir %s/%s: %v", dir, name, err)
  146. }
  147. return nil
  148. })
  149. }
  150. func (fs *WebDavFileSystem) OpenFile(ctx context.Context, fullFilePath string, flag int, perm os.FileMode) (webdav.File, error) {
  151. glog.V(2).Infof("WebDavFileSystem.OpenFile %v %x", fullFilePath, flag)
  152. var err error
  153. if fullFilePath, err = clearName(fullFilePath); err != nil {
  154. return nil, err
  155. }
  156. if flag&os.O_CREATE != 0 {
  157. // file should not have / suffix.
  158. if strings.HasSuffix(fullFilePath, "/") {
  159. return nil, os.ErrInvalid
  160. }
  161. _, err = fs.stat(ctx, fullFilePath)
  162. if err == nil {
  163. if flag&os.O_EXCL != 0 {
  164. return nil, os.ErrExist
  165. }
  166. fs.removeAll(ctx, fullFilePath)
  167. }
  168. dir, name := util.FullPath(fullFilePath).DirAndName()
  169. err = fs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
  170. if err := filer_pb.CreateEntry(client, &filer_pb.CreateEntryRequest{
  171. Directory: dir,
  172. Entry: &filer_pb.Entry{
  173. Name: name,
  174. IsDirectory: perm&os.ModeDir > 0,
  175. Attributes: &filer_pb.FuseAttributes{
  176. Mtime: time.Now().Unix(),
  177. Crtime: time.Now().Unix(),
  178. FileMode: uint32(perm),
  179. Uid: fs.option.Uid,
  180. Gid: fs.option.Gid,
  181. Collection: fs.option.Collection,
  182. Replication: "000",
  183. TtlSec: 0,
  184. },
  185. },
  186. }); err != nil {
  187. return fmt.Errorf("create %s: %v", fullFilePath, err)
  188. }
  189. return nil
  190. })
  191. if err != nil {
  192. return nil, err
  193. }
  194. return &WebDavFile{
  195. fs: fs,
  196. name: fullFilePath,
  197. isDirectory: false,
  198. }, nil
  199. }
  200. fi, err := fs.stat(ctx, fullFilePath)
  201. if err != nil {
  202. return nil, os.ErrNotExist
  203. }
  204. if !strings.HasSuffix(fullFilePath, "/") && fi.IsDir() {
  205. fullFilePath += "/"
  206. }
  207. return &WebDavFile{
  208. fs: fs,
  209. name: fullFilePath,
  210. isDirectory: false,
  211. }, nil
  212. }
  213. func (fs *WebDavFileSystem) removeAll(ctx context.Context, fullFilePath string) error {
  214. var err error
  215. if fullFilePath, err = clearName(fullFilePath); err != nil {
  216. return err
  217. }
  218. dir, name := util.FullPath(fullFilePath).DirAndName()
  219. return filer_pb.Remove(fs, dir, name, true, false, false)
  220. }
  221. func (fs *WebDavFileSystem) RemoveAll(ctx context.Context, name string) error {
  222. glog.V(2).Infof("WebDavFileSystem.RemoveAll %v", name)
  223. return fs.removeAll(ctx, name)
  224. }
  225. func (fs *WebDavFileSystem) Rename(ctx context.Context, oldName, newName string) error {
  226. glog.V(2).Infof("WebDavFileSystem.Rename %v to %v", oldName, newName)
  227. var err error
  228. if oldName, err = clearName(oldName); err != nil {
  229. return err
  230. }
  231. if newName, err = clearName(newName); err != nil {
  232. return err
  233. }
  234. of, err := fs.stat(ctx, oldName)
  235. if err != nil {
  236. return os.ErrExist
  237. }
  238. if of.IsDir() {
  239. if strings.HasSuffix(oldName, "/") {
  240. oldName = strings.TrimRight(oldName, "/")
  241. }
  242. if strings.HasSuffix(newName, "/") {
  243. newName = strings.TrimRight(newName, "/")
  244. }
  245. }
  246. _, err = fs.stat(ctx, newName)
  247. if err == nil {
  248. return os.ErrExist
  249. }
  250. oldDir, oldBaseName := util.FullPath(oldName).DirAndName()
  251. newDir, newBaseName := util.FullPath(newName).DirAndName()
  252. return fs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
  253. request := &filer_pb.AtomicRenameEntryRequest{
  254. OldDirectory: oldDir,
  255. OldName: oldBaseName,
  256. NewDirectory: newDir,
  257. NewName: newBaseName,
  258. }
  259. _, err := client.AtomicRenameEntry(ctx, request)
  260. if err != nil {
  261. return fmt.Errorf("renaming %s/%s => %s/%s: %v", oldDir, oldBaseName, newDir, newBaseName, err)
  262. }
  263. return nil
  264. })
  265. }
  266. func (fs *WebDavFileSystem) stat(ctx context.Context, fullFilePath string) (os.FileInfo, error) {
  267. var err error
  268. if fullFilePath, err = clearName(fullFilePath); err != nil {
  269. return nil, err
  270. }
  271. fullpath := util.FullPath(fullFilePath)
  272. var fi FileInfo
  273. entry, err := filer_pb.GetEntry(fs, fullpath)
  274. if entry == nil {
  275. return nil, os.ErrNotExist
  276. }
  277. if err != nil {
  278. return nil, err
  279. }
  280. fi.size = int64(filer2.TotalSize(entry.GetChunks()))
  281. fi.name = string(fullpath)
  282. fi.mode = os.FileMode(entry.Attributes.FileMode)
  283. fi.modifiledTime = time.Unix(entry.Attributes.Mtime, 0)
  284. fi.isDirectory = entry.IsDirectory
  285. if fi.name == "/" {
  286. fi.modifiledTime = time.Now()
  287. fi.isDirectory = true
  288. }
  289. return &fi, nil
  290. }
  291. func (fs *WebDavFileSystem) Stat(ctx context.Context, name string) (os.FileInfo, error) {
  292. glog.V(2).Infof("WebDavFileSystem.Stat %v", name)
  293. return fs.stat(ctx, name)
  294. }
  295. func (f *WebDavFile) Write(buf []byte) (int, error) {
  296. glog.V(2).Infof("WebDavFileSystem.Write %v", f.name)
  297. dir, _ := util.FullPath(f.name).DirAndName()
  298. var err error
  299. ctx := context.Background()
  300. if f.entry == nil {
  301. f.entry, err = filer_pb.GetEntry(f.fs, util.FullPath(f.name))
  302. }
  303. if f.entry == nil {
  304. return 0, err
  305. }
  306. if err != nil {
  307. return 0, err
  308. }
  309. var fileId, host string
  310. var auth security.EncodedJwt
  311. var collection, replication string
  312. if err = f.fs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
  313. request := &filer_pb.AssignVolumeRequest{
  314. Count: 1,
  315. Replication: "",
  316. Collection: f.fs.option.Collection,
  317. ParentPath: dir,
  318. }
  319. resp, err := client.AssignVolume(ctx, request)
  320. if err != nil {
  321. glog.V(0).Infof("assign volume failure %v: %v", request, err)
  322. return err
  323. }
  324. if resp.Error != "" {
  325. return fmt.Errorf("assign volume failure %v: %v", request, resp.Error)
  326. }
  327. fileId, host, auth = resp.FileId, resp.Url, security.EncodedJwt(resp.Auth)
  328. collection, replication = resp.Collection, resp.Replication
  329. return nil
  330. }); err != nil {
  331. return 0, fmt.Errorf("filerGrpcAddress assign volume: %v", err)
  332. }
  333. fileUrl := fmt.Sprintf("http://%s/%s", host, fileId)
  334. uploadResult, err := operation.UploadData(fileUrl, f.name, f.fs.option.Cipher, buf, false, "", nil, auth)
  335. if err != nil {
  336. glog.V(0).Infof("upload data %v to %s: %v", f.name, fileUrl, err)
  337. return 0, fmt.Errorf("upload data: %v", err)
  338. }
  339. if uploadResult.Error != "" {
  340. glog.V(0).Infof("upload failure %v to %s: %v", f.name, fileUrl, err)
  341. return 0, fmt.Errorf("upload result: %v", uploadResult.Error)
  342. }
  343. chunk := &filer_pb.FileChunk{
  344. FileId: fileId,
  345. Offset: f.off,
  346. Size: uint64(len(buf)),
  347. Mtime: time.Now().UnixNano(),
  348. ETag: uploadResult.ETag,
  349. CipherKey: uploadResult.CipherKey,
  350. IsGzipped: uploadResult.Gzip > 0,
  351. }
  352. f.entry.Chunks = append(f.entry.Chunks, chunk)
  353. err = f.fs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
  354. f.entry.Attributes.Mtime = time.Now().Unix()
  355. f.entry.Attributes.Collection = collection
  356. f.entry.Attributes.Replication = replication
  357. request := &filer_pb.UpdateEntryRequest{
  358. Directory: dir,
  359. Entry: f.entry,
  360. }
  361. if _, err := client.UpdateEntry(ctx, request); err != nil {
  362. return fmt.Errorf("update %s: %v", f.name, err)
  363. }
  364. return nil
  365. })
  366. if err == nil {
  367. glog.V(3).Infof("WebDavFileSystem.Write %v: written [%d,%d)", f.name, f.off, f.off+int64(len(buf)))
  368. f.off += int64(len(buf))
  369. }
  370. return len(buf), err
  371. }
  372. func (f *WebDavFile) Close() error {
  373. glog.V(2).Infof("WebDavFileSystem.Close %v", f.name)
  374. if f.entry != nil {
  375. f.entry = nil
  376. f.entryViewCache = nil
  377. }
  378. return nil
  379. }
  380. func (f *WebDavFile) Read(p []byte) (readSize int, err error) {
  381. glog.V(2).Infof("WebDavFileSystem.Read %v", f.name)
  382. if f.entry == nil {
  383. f.entry, err = filer_pb.GetEntry(f.fs, util.FullPath(f.name))
  384. }
  385. if f.entry == nil {
  386. return 0, err
  387. }
  388. if err != nil {
  389. return 0, err
  390. }
  391. if len(f.entry.Chunks) == 0 {
  392. return 0, io.EOF
  393. }
  394. if f.entryViewCache == nil {
  395. f.entryViewCache = filer2.NonOverlappingVisibleIntervals(f.entry.Chunks)
  396. f.reader = nil
  397. }
  398. if f.reader == nil {
  399. chunkViews := filer2.ViewFromVisibleIntervals(f.entryViewCache, 0, math.MaxInt32)
  400. f.reader = filer2.NewChunkReaderAtFromClient(f.fs, chunkViews, f.fs.chunkCache)
  401. }
  402. readSize, err = f.reader.ReadAt(p, f.off)
  403. glog.V(3).Infof("WebDavFileSystem.Read %v: [%d,%d)", f.name, f.off, f.off+int64(readSize))
  404. f.off += int64(readSize)
  405. if err != nil {
  406. glog.Errorf("file read %s: %v", f.name, err)
  407. }
  408. return
  409. }
  410. func (f *WebDavFile) Readdir(count int) (ret []os.FileInfo, err error) {
  411. glog.V(2).Infof("WebDavFileSystem.Readdir %v count %d", f.name, count)
  412. dir, _ := util.FullPath(f.name).DirAndName()
  413. err = filer_pb.ReadDirAllEntries(f.fs, util.FullPath(dir), "", func(entry *filer_pb.Entry, isLast bool) {
  414. fi := FileInfo{
  415. size: int64(filer2.TotalSize(entry.GetChunks())),
  416. name: entry.Name,
  417. mode: os.FileMode(entry.Attributes.FileMode),
  418. modifiledTime: time.Unix(entry.Attributes.Mtime, 0),
  419. isDirectory: entry.IsDirectory,
  420. }
  421. if !strings.HasSuffix(fi.name, "/") && fi.IsDir() {
  422. fi.name += "/"
  423. }
  424. glog.V(4).Infof("entry: %v", fi.name)
  425. ret = append(ret, &fi)
  426. })
  427. old := f.off
  428. if old >= int64(len(ret)) {
  429. if count > 0 {
  430. return nil, io.EOF
  431. }
  432. return nil, nil
  433. }
  434. if count > 0 {
  435. f.off += int64(count)
  436. if f.off > int64(len(ret)) {
  437. f.off = int64(len(ret))
  438. }
  439. } else {
  440. f.off = int64(len(ret))
  441. old = 0
  442. }
  443. return ret[old:f.off], nil
  444. }
  445. func (f *WebDavFile) Seek(offset int64, whence int) (int64, error) {
  446. glog.V(2).Infof("WebDavFile.Seek %v %v %v", f.name, offset, whence)
  447. ctx := context.Background()
  448. var err error
  449. switch whence {
  450. case 0:
  451. f.off = 0
  452. case 2:
  453. if fi, err := f.fs.stat(ctx, f.name); err != nil {
  454. return 0, err
  455. } else {
  456. f.off = fi.Size()
  457. }
  458. }
  459. f.off += offset
  460. return f.off, err
  461. }
  462. func (f *WebDavFile) Stat() (os.FileInfo, error) {
  463. glog.V(2).Infof("WebDavFile.Stat %v", f.name)
  464. ctx := context.Background()
  465. return f.fs.stat(ctx, f.name)
  466. }