You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

630 lines
16 KiB

5 years ago
5 years ago
5 years ago
5 years ago
3 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
3 years ago
4 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
4 years ago
4 years ago
  1. package weed_server
  2. import (
  3. "bytes"
  4. "context"
  5. "fmt"
  6. "io"
  7. "os"
  8. "path"
  9. "strings"
  10. "time"
  11. "github.com/chrislusf/seaweedfs/weed/util/buffered_writer"
  12. "golang.org/x/net/webdav"
  13. "google.golang.org/grpc"
  14. "github.com/chrislusf/seaweedfs/weed/operation"
  15. "github.com/chrislusf/seaweedfs/weed/pb"
  16. "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
  17. "github.com/chrislusf/seaweedfs/weed/util"
  18. "github.com/chrislusf/seaweedfs/weed/util/chunk_cache"
  19. "github.com/chrislusf/seaweedfs/weed/filer"
  20. "github.com/chrislusf/seaweedfs/weed/glog"
  21. "github.com/chrislusf/seaweedfs/weed/security"
  22. )
  23. type WebDavOption struct {
  24. Filer pb.ServerAddress
  25. DomainName string
  26. BucketsPath string
  27. GrpcDialOption grpc.DialOption
  28. Collection string
  29. Replication string
  30. DiskType string
  31. Uid uint32
  32. Gid uint32
  33. Cipher bool
  34. CacheDir string
  35. CacheSizeMB int64
  36. }
  37. type WebDavServer struct {
  38. option *WebDavOption
  39. secret security.SigningKey
  40. filer *filer.Filer
  41. grpcDialOption grpc.DialOption
  42. Handler *webdav.Handler
  43. }
  44. func NewWebDavServer(option *WebDavOption) (ws *WebDavServer, err error) {
  45. fs, _ := NewWebDavFileSystem(option)
  46. ws = &WebDavServer{
  47. option: option,
  48. grpcDialOption: security.LoadClientTLS(util.GetViper(), "grpc.filer"),
  49. Handler: &webdav.Handler{
  50. FileSystem: fs,
  51. LockSystem: webdav.NewMemLS(),
  52. },
  53. }
  54. return ws, nil
  55. }
  56. // adapted from https://github.com/mattn/davfs/blob/master/plugin/mysql/mysql.go
  57. type WebDavFileSystem struct {
  58. option *WebDavOption
  59. secret security.SigningKey
  60. filer *filer.Filer
  61. grpcDialOption grpc.DialOption
  62. chunkCache *chunk_cache.TieredChunkCache
  63. signature int32
  64. }
  65. type FileInfo struct {
  66. name string
  67. size int64
  68. mode os.FileMode
  69. modifiledTime time.Time
  70. isDirectory bool
  71. }
  72. func (fi *FileInfo) Name() string { return fi.name }
  73. func (fi *FileInfo) Size() int64 { return fi.size }
  74. func (fi *FileInfo) Mode() os.FileMode { return fi.mode }
  75. func (fi *FileInfo) ModTime() time.Time { return fi.modifiledTime }
  76. func (fi *FileInfo) IsDir() bool { return fi.isDirectory }
  77. func (fi *FileInfo) Sys() interface{} { return nil }
  78. type WebDavFile struct {
  79. fs *WebDavFileSystem
  80. name string
  81. isDirectory bool
  82. off int64
  83. entry *filer_pb.Entry
  84. entryViewCache []filer.VisibleInterval
  85. reader io.ReaderAt
  86. bufWriter *buffered_writer.BufferedWriteCloser
  87. collection string
  88. replication string
  89. }
  90. func NewWebDavFileSystem(option *WebDavOption) (webdav.FileSystem, error) {
  91. cacheUniqueId := util.Md5String([]byte("webdav" + string(option.Filer) + util.Version()))[0:8]
  92. cacheDir := path.Join(option.CacheDir, cacheUniqueId)
  93. os.MkdirAll(cacheDir, os.FileMode(0755))
  94. chunkCache := chunk_cache.NewTieredChunkCache(256, cacheDir, option.CacheSizeMB, 1024*1024)
  95. return &WebDavFileSystem{
  96. option: option,
  97. chunkCache: chunkCache,
  98. signature: util.RandomInt32(),
  99. }, nil
  100. }
  101. var _ = filer_pb.FilerClient(&WebDavFileSystem{})
  102. func (fs *WebDavFileSystem) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error {
  103. return pb.WithGrpcClient(streamingMode, func(grpcConnection *grpc.ClientConn) error {
  104. client := filer_pb.NewSeaweedFilerClient(grpcConnection)
  105. return fn(client)
  106. }, fs.option.Filer.ToGrpcAddress(), fs.option.GrpcDialOption)
  107. }
  108. func (fs *WebDavFileSystem) AdjustedUrl(location *filer_pb.Location) string {
  109. return location.Url
  110. }
  111. func clearName(name string) (string, error) {
  112. slashed := strings.HasSuffix(name, "/")
  113. name = path.Clean(name)
  114. if !strings.HasSuffix(name, "/") && slashed {
  115. name += "/"
  116. }
  117. if !strings.HasPrefix(name, "/") {
  118. return "", os.ErrInvalid
  119. }
  120. return name, nil
  121. }
  122. func (fs *WebDavFileSystem) Mkdir(ctx context.Context, fullDirPath string, perm os.FileMode) error {
  123. glog.V(2).Infof("WebDavFileSystem.Mkdir %v", fullDirPath)
  124. if !strings.HasSuffix(fullDirPath, "/") {
  125. fullDirPath += "/"
  126. }
  127. var err error
  128. if fullDirPath, err = clearName(fullDirPath); err != nil {
  129. return err
  130. }
  131. _, err = fs.stat(ctx, fullDirPath)
  132. if err == nil {
  133. return os.ErrExist
  134. }
  135. return fs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
  136. dir, name := util.FullPath(fullDirPath).DirAndName()
  137. request := &filer_pb.CreateEntryRequest{
  138. Directory: dir,
  139. Entry: &filer_pb.Entry{
  140. Name: name,
  141. IsDirectory: true,
  142. Attributes: &filer_pb.FuseAttributes{
  143. Mtime: time.Now().Unix(),
  144. Crtime: time.Now().Unix(),
  145. FileMode: uint32(perm | os.ModeDir),
  146. Uid: fs.option.Uid,
  147. Gid: fs.option.Gid,
  148. },
  149. },
  150. Signatures: []int32{fs.signature},
  151. }
  152. glog.V(1).Infof("mkdir: %v", request)
  153. if err := filer_pb.CreateEntry(client, request); err != nil {
  154. return fmt.Errorf("mkdir %s/%s: %v", dir, name, err)
  155. }
  156. return nil
  157. })
  158. }
  159. func (fs *WebDavFileSystem) OpenFile(ctx context.Context, fullFilePath string, flag int, perm os.FileMode) (webdav.File, error) {
  160. glog.V(2).Infof("WebDavFileSystem.OpenFile %v %x", fullFilePath, flag)
  161. var err error
  162. if fullFilePath, err = clearName(fullFilePath); err != nil {
  163. return nil, err
  164. }
  165. if flag&os.O_CREATE != 0 {
  166. // file should not have / suffix.
  167. if strings.HasSuffix(fullFilePath, "/") {
  168. return nil, os.ErrInvalid
  169. }
  170. _, err = fs.stat(ctx, fullFilePath)
  171. if err == nil {
  172. if flag&os.O_EXCL != 0 {
  173. return nil, os.ErrExist
  174. }
  175. fs.removeAll(ctx, fullFilePath)
  176. }
  177. dir, name := util.FullPath(fullFilePath).DirAndName()
  178. err = fs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
  179. if err := filer_pb.CreateEntry(client, &filer_pb.CreateEntryRequest{
  180. Directory: dir,
  181. Entry: &filer_pb.Entry{
  182. Name: name,
  183. IsDirectory: perm&os.ModeDir > 0,
  184. Attributes: &filer_pb.FuseAttributes{
  185. Mtime: time.Now().Unix(),
  186. Crtime: time.Now().Unix(),
  187. FileMode: uint32(perm),
  188. Uid: fs.option.Uid,
  189. Gid: fs.option.Gid,
  190. TtlSec: 0,
  191. },
  192. },
  193. Signatures: []int32{fs.signature},
  194. }); err != nil {
  195. return fmt.Errorf("create %s: %v", fullFilePath, err)
  196. }
  197. return nil
  198. })
  199. if err != nil {
  200. return nil, err
  201. }
  202. return &WebDavFile{
  203. fs: fs,
  204. name: fullFilePath,
  205. isDirectory: false,
  206. bufWriter: buffered_writer.NewBufferedWriteCloser(4 * 1024 * 1024),
  207. }, nil
  208. }
  209. fi, err := fs.stat(ctx, fullFilePath)
  210. if err != nil {
  211. return nil, os.ErrNotExist
  212. }
  213. if !strings.HasSuffix(fullFilePath, "/") && fi.IsDir() {
  214. fullFilePath += "/"
  215. }
  216. return &WebDavFile{
  217. fs: fs,
  218. name: fullFilePath,
  219. isDirectory: false,
  220. bufWriter: buffered_writer.NewBufferedWriteCloser(4 * 1024 * 1024),
  221. }, nil
  222. }
  223. func (fs *WebDavFileSystem) removeAll(ctx context.Context, fullFilePath string) error {
  224. var err error
  225. if fullFilePath, err = clearName(fullFilePath); err != nil {
  226. return err
  227. }
  228. dir, name := util.FullPath(fullFilePath).DirAndName()
  229. return filer_pb.Remove(fs, dir, name, true, false, false, false, []int32{fs.signature})
  230. }
  231. func (fs *WebDavFileSystem) RemoveAll(ctx context.Context, name string) error {
  232. glog.V(2).Infof("WebDavFileSystem.RemoveAll %v", name)
  233. return fs.removeAll(ctx, name)
  234. }
  235. func (fs *WebDavFileSystem) Rename(ctx context.Context, oldName, newName string) error {
  236. glog.V(2).Infof("WebDavFileSystem.Rename %v to %v", oldName, newName)
  237. var err error
  238. if oldName, err = clearName(oldName); err != nil {
  239. return err
  240. }
  241. if newName, err = clearName(newName); err != nil {
  242. return err
  243. }
  244. of, err := fs.stat(ctx, oldName)
  245. if err != nil {
  246. return os.ErrExist
  247. }
  248. if of.IsDir() {
  249. if strings.HasSuffix(oldName, "/") {
  250. oldName = strings.TrimRight(oldName, "/")
  251. }
  252. if strings.HasSuffix(newName, "/") {
  253. newName = strings.TrimRight(newName, "/")
  254. }
  255. }
  256. _, err = fs.stat(ctx, newName)
  257. if err == nil {
  258. return os.ErrExist
  259. }
  260. oldDir, oldBaseName := util.FullPath(oldName).DirAndName()
  261. newDir, newBaseName := util.FullPath(newName).DirAndName()
  262. return fs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
  263. request := &filer_pb.AtomicRenameEntryRequest{
  264. OldDirectory: oldDir,
  265. OldName: oldBaseName,
  266. NewDirectory: newDir,
  267. NewName: newBaseName,
  268. }
  269. _, err := client.AtomicRenameEntry(ctx, request)
  270. if err != nil {
  271. return fmt.Errorf("renaming %s/%s => %s/%s: %v", oldDir, oldBaseName, newDir, newBaseName, err)
  272. }
  273. return nil
  274. })
  275. }
  276. func (fs *WebDavFileSystem) stat(ctx context.Context, fullFilePath string) (os.FileInfo, error) {
  277. var err error
  278. if fullFilePath, err = clearName(fullFilePath); err != nil {
  279. return nil, err
  280. }
  281. fullpath := util.FullPath(fullFilePath)
  282. var fi FileInfo
  283. entry, err := filer_pb.GetEntry(fs, fullpath)
  284. if entry == nil {
  285. return nil, os.ErrNotExist
  286. }
  287. if err != nil {
  288. return nil, err
  289. }
  290. fi.size = int64(filer.FileSize(entry))
  291. fi.name = string(fullpath)
  292. fi.mode = os.FileMode(entry.Attributes.FileMode)
  293. fi.modifiledTime = time.Unix(entry.Attributes.Mtime, 0)
  294. fi.isDirectory = entry.IsDirectory
  295. if fi.name == "/" {
  296. fi.modifiledTime = time.Now()
  297. fi.isDirectory = true
  298. }
  299. return &fi, nil
  300. }
  301. func (fs *WebDavFileSystem) Stat(ctx context.Context, name string) (os.FileInfo, error) {
  302. glog.V(2).Infof("WebDavFileSystem.Stat %v", name)
  303. return fs.stat(ctx, name)
  304. }
  305. func (f *WebDavFile) saveDataAsChunk(reader io.Reader, name string, offset int64) (chunk *filer_pb.FileChunk, collection, replication string, err error) {
  306. var fileId, host string
  307. var auth security.EncodedJwt
  308. if flushErr := f.fs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
  309. ctx := context.Background()
  310. assignErr := util.Retry("assignVolume", func() error {
  311. request := &filer_pb.AssignVolumeRequest{
  312. Count: 1,
  313. Replication: f.fs.option.Replication,
  314. Collection: f.fs.option.Collection,
  315. DiskType: f.fs.option.DiskType,
  316. Path: name,
  317. }
  318. resp, err := client.AssignVolume(ctx, request)
  319. if err != nil {
  320. glog.V(0).Infof("assign volume failure %v: %v", request, err)
  321. return err
  322. }
  323. if resp.Error != "" {
  324. return fmt.Errorf("assign volume failure %v: %v", request, resp.Error)
  325. }
  326. fileId, host, auth = resp.FileId, resp.Location.Url, security.EncodedJwt(resp.Auth)
  327. f.collection, f.replication = resp.Collection, resp.Replication
  328. return nil
  329. })
  330. if assignErr != nil {
  331. return assignErr
  332. }
  333. return nil
  334. }); flushErr != nil {
  335. return nil, f.collection, f.replication, fmt.Errorf("filerGrpcAddress assign volume: %v", flushErr)
  336. }
  337. fileUrl := fmt.Sprintf("http://%s/%s", host, fileId)
  338. uploadOption := &operation.UploadOption{
  339. UploadUrl: fileUrl,
  340. Filename: f.name,
  341. Cipher: f.fs.option.Cipher,
  342. IsInputCompressed: false,
  343. MimeType: "",
  344. PairMap: nil,
  345. Jwt: auth,
  346. }
  347. uploadResult, flushErr, _ := operation.Upload(reader, uploadOption)
  348. if flushErr != nil {
  349. glog.V(0).Infof("upload data %v to %s: %v", f.name, fileUrl, flushErr)
  350. return nil, f.collection, f.replication, fmt.Errorf("upload data: %v", flushErr)
  351. }
  352. if uploadResult.Error != "" {
  353. glog.V(0).Infof("upload failure %v to %s: %v", f.name, fileUrl, flushErr)
  354. return nil, f.collection, f.replication, fmt.Errorf("upload result: %v", uploadResult.Error)
  355. }
  356. return uploadResult.ToPbFileChunk(fileId, offset), f.collection, f.replication, nil
  357. }
  358. func (f *WebDavFile) Write(buf []byte) (int, error) {
  359. glog.V(2).Infof("WebDavFileSystem.Write %v", f.name)
  360. dir, _ := util.FullPath(f.name).DirAndName()
  361. var getErr error
  362. ctx := context.Background()
  363. if f.entry == nil {
  364. f.entry, getErr = filer_pb.GetEntry(f.fs, util.FullPath(f.name))
  365. }
  366. if f.entry == nil {
  367. return 0, getErr
  368. }
  369. if getErr != nil {
  370. return 0, getErr
  371. }
  372. if f.bufWriter.FlushFunc == nil {
  373. f.bufWriter.FlushFunc = func(data []byte, offset int64) (flushErr error) {
  374. var chunk *filer_pb.FileChunk
  375. chunk, f.collection, f.replication, flushErr = f.saveDataAsChunk(bytes.NewReader(data), f.name, offset)
  376. if flushErr != nil {
  377. return fmt.Errorf("%s upload result: %v", f.name, flushErr)
  378. }
  379. f.entry.Content = nil
  380. f.entry.Chunks = append(f.entry.Chunks, chunk)
  381. return flushErr
  382. }
  383. f.bufWriter.CloseFunc = func() error {
  384. manifestedChunks, manifestErr := filer.MaybeManifestize(f.saveDataAsChunk, f.entry.Chunks)
  385. if manifestErr != nil {
  386. // not good, but should be ok
  387. glog.V(0).Infof("file %s close MaybeManifestize: %v", f.name, manifestErr)
  388. } else {
  389. f.entry.Chunks = manifestedChunks
  390. }
  391. flushErr := f.fs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
  392. f.entry.Attributes.Mtime = time.Now().Unix()
  393. request := &filer_pb.UpdateEntryRequest{
  394. Directory: dir,
  395. Entry: f.entry,
  396. Signatures: []int32{f.fs.signature},
  397. }
  398. if _, err := client.UpdateEntry(ctx, request); err != nil {
  399. return fmt.Errorf("update %s: %v", f.name, err)
  400. }
  401. return nil
  402. })
  403. return flushErr
  404. }
  405. }
  406. written, err := f.bufWriter.Write(buf)
  407. if err == nil {
  408. glog.V(3).Infof("WebDavFileSystem.Write %v: written [%d,%d)", f.name, f.off, f.off+int64(len(buf)))
  409. f.off += int64(written)
  410. }
  411. return written, err
  412. }
  413. func (f *WebDavFile) Close() error {
  414. glog.V(2).Infof("WebDavFileSystem.Close %v", f.name)
  415. err := f.bufWriter.Close()
  416. if f.entry != nil {
  417. f.entry = nil
  418. f.entryViewCache = nil
  419. }
  420. return err
  421. }
  422. func (f *WebDavFile) Read(p []byte) (readSize int, err error) {
  423. glog.V(2).Infof("WebDavFileSystem.Read %v", f.name)
  424. if f.entry == nil {
  425. f.entry, err = filer_pb.GetEntry(f.fs, util.FullPath(f.name))
  426. }
  427. if f.entry == nil {
  428. return 0, err
  429. }
  430. if err != nil {
  431. return 0, err
  432. }
  433. fileSize := int64(filer.FileSize(f.entry))
  434. if fileSize == 0 {
  435. return 0, io.EOF
  436. }
  437. if f.entryViewCache == nil {
  438. f.entryViewCache, _ = filer.NonOverlappingVisibleIntervals(filer.LookupFn(f.fs), f.entry.Chunks, 0, fileSize)
  439. f.reader = nil
  440. }
  441. if f.reader == nil {
  442. chunkViews := filer.ViewFromVisibleIntervals(f.entryViewCache, 0, fileSize)
  443. f.reader = filer.NewChunkReaderAtFromClient(filer.LookupFn(f.fs), chunkViews, f.fs.chunkCache, fileSize)
  444. }
  445. readSize, err = f.reader.ReadAt(p, f.off)
  446. glog.V(3).Infof("WebDavFileSystem.Read %v: [%d,%d)", f.name, f.off, f.off+int64(readSize))
  447. f.off += int64(readSize)
  448. if err != nil && err != io.EOF {
  449. glog.Errorf("file read %s: %v", f.name, err)
  450. }
  451. return
  452. }
  453. func (f *WebDavFile) Readdir(count int) (ret []os.FileInfo, err error) {
  454. glog.V(2).Infof("WebDavFileSystem.Readdir %v count %d", f.name, count)
  455. dir, _ := util.FullPath(f.name).DirAndName()
  456. err = filer_pb.ReadDirAllEntries(f.fs, util.FullPath(dir), "", func(entry *filer_pb.Entry, isLast bool) error {
  457. fi := FileInfo{
  458. size: int64(filer.FileSize(entry)),
  459. name: entry.Name,
  460. mode: os.FileMode(entry.Attributes.FileMode),
  461. modifiledTime: time.Unix(entry.Attributes.Mtime, 0),
  462. isDirectory: entry.IsDirectory,
  463. }
  464. if !strings.HasSuffix(fi.name, "/") && fi.IsDir() {
  465. fi.name += "/"
  466. }
  467. glog.V(4).Infof("entry: %v", fi.name)
  468. ret = append(ret, &fi)
  469. return nil
  470. })
  471. old := f.off
  472. if old >= int64(len(ret)) {
  473. if count > 0 {
  474. return nil, io.EOF
  475. }
  476. return nil, nil
  477. }
  478. if count > 0 {
  479. f.off += int64(count)
  480. if f.off > int64(len(ret)) {
  481. f.off = int64(len(ret))
  482. }
  483. } else {
  484. f.off = int64(len(ret))
  485. old = 0
  486. }
  487. return ret[old:f.off], nil
  488. }
  489. func (f *WebDavFile) Seek(offset int64, whence int) (int64, error) {
  490. glog.V(2).Infof("WebDavFile.Seek %v %v %v", f.name, offset, whence)
  491. ctx := context.Background()
  492. var err error
  493. switch whence {
  494. case io.SeekStart:
  495. f.off = 0
  496. case io.SeekEnd:
  497. if fi, err := f.fs.stat(ctx, f.name); err != nil {
  498. return 0, err
  499. } else {
  500. f.off = fi.Size()
  501. }
  502. }
  503. f.off += offset
  504. return f.off, err
  505. }
  506. func (f *WebDavFile) Stat() (os.FileInfo, error) {
  507. glog.V(2).Infof("WebDavFile.Stat %v", f.name)
  508. ctx := context.Background()
  509. return f.fs.stat(ctx, f.name)
  510. }