You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

99 lines
2.7 KiB

3 years ago
3 years ago
3 years ago
3 years ago
  1. package mount
  2. import (
  3. "fmt"
  4. "github.com/chrislusf/seaweedfs/weed/glog"
  5. "github.com/chrislusf/seaweedfs/weed/mount/page_writer"
  6. "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
  7. "io"
  8. "sync"
  9. "time"
  10. )
  11. type ChunkedDirtyPages struct {
  12. fh *FileHandle
  13. writeWaitGroup sync.WaitGroup
  14. lastErr error
  15. collection string
  16. replication string
  17. uploadPipeline *page_writer.UploadPipeline
  18. hasWrites bool
  19. }
  20. var (
  21. _ = page_writer.DirtyPages(&ChunkedDirtyPages{})
  22. )
  23. func newMemoryChunkPages(fh *FileHandle, chunkSize int64) *ChunkedDirtyPages {
  24. dirtyPages := &ChunkedDirtyPages{
  25. fh: fh,
  26. }
  27. dirtyPages.uploadPipeline = page_writer.NewUploadPipeline(fh.wfs.concurrentWriters, chunkSize, dirtyPages.saveChunkedFileIntevalToStorage, fh.wfs.option.ConcurrentWriters)
  28. return dirtyPages
  29. }
  30. func (pages *ChunkedDirtyPages) AddPage(offset int64, data []byte) {
  31. pages.hasWrites = true
  32. glog.V(4).Infof("%v memory AddPage [%d, %d)", pages.fh.fh, offset, offset+int64(len(data)))
  33. pages.uploadPipeline.SaveDataAt(data, offset)
  34. return
  35. }
  36. func (pages *ChunkedDirtyPages) FlushData() error {
  37. if !pages.hasWrites {
  38. return nil
  39. }
  40. pages.uploadPipeline.FlushAll()
  41. if pages.lastErr != nil {
  42. return fmt.Errorf("flush data: %v", pages.lastErr)
  43. }
  44. return nil
  45. }
  46. func (pages *ChunkedDirtyPages) ReadDirtyDataAt(data []byte, startOffset int64) (maxStop int64) {
  47. if !pages.hasWrites {
  48. return
  49. }
  50. return pages.uploadPipeline.MaybeReadDataAt(data, startOffset)
  51. }
  52. func (pages *ChunkedDirtyPages) GetStorageOptions() (collection, replication string) {
  53. return pages.collection, pages.replication
  54. }
  55. func (pages *ChunkedDirtyPages) saveChunkedFileIntevalToStorage(reader io.Reader, offset int64, size int64, cleanupFn func()) {
  56. mtime := time.Now().UnixNano()
  57. defer cleanupFn()
  58. fileFullPath := pages.fh.FullPath()
  59. fileName := fileFullPath.Name()
  60. chunk, collection, replication, err := pages.fh.wfs.saveDataAsChunk(fileFullPath)(reader, fileName, offset)
  61. if err != nil {
  62. glog.V(0).Infof("%v saveToStorage [%d,%d): %v", fileFullPath, offset, offset+size, err)
  63. pages.lastErr = err
  64. return
  65. }
  66. chunk.Mtime = mtime
  67. pages.collection, pages.replication = collection, replication
  68. pages.fh.addChunks([]*filer_pb.FileChunk{chunk})
  69. pages.fh.entryViewCache = nil
  70. glog.V(3).Infof("%v saveToStorage %s [%d,%d)", fileFullPath, chunk.FileId, offset, offset+size)
  71. }
  72. func (pages ChunkedDirtyPages) Destroy() {
  73. pages.uploadPipeline.Shutdown()
  74. }
  75. func (pages *ChunkedDirtyPages) LockForRead(startOffset, stopOffset int64) {
  76. pages.uploadPipeline.LockForRead(startOffset, stopOffset)
  77. }
  78. func (pages *ChunkedDirtyPages) UnlockForRead(startOffset, stopOffset int64) {
  79. pages.uploadPipeline.UnlockForRead(startOffset, stopOffset)
  80. }