You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

153 lines
3.8 KiB

5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
  1. package filer2
  2. import (
  3. "bytes"
  4. "io"
  5. "math"
  6. "strings"
  7. "github.com/chrislusf/seaweedfs/weed/glog"
  8. "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
  9. "github.com/chrislusf/seaweedfs/weed/util"
  10. "github.com/chrislusf/seaweedfs/weed/wdclient"
  11. )
  12. func StreamContent(masterClient *wdclient.MasterClient, w io.Writer, chunks []*filer_pb.FileChunk, offset int64, size int64) error {
  13. chunkViews := ViewFromChunks(chunks, offset, size)
  14. fileId2Url := make(map[string]string)
  15. for _, chunkView := range chunkViews {
  16. urlString, err := masterClient.LookupFileId(chunkView.FileId)
  17. if err != nil {
  18. glog.V(1).Infof("operation LookupFileId %s failed, err: %v", chunkView.FileId, err)
  19. return err
  20. }
  21. fileId2Url[chunkView.FileId] = urlString
  22. }
  23. for _, chunkView := range chunkViews {
  24. urlString := fileId2Url[chunkView.FileId]
  25. err := util.ReadUrlAsStream(urlString, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk, chunkView.Offset, int(chunkView.Size), func(data []byte) {
  26. w.Write(data)
  27. })
  28. if err != nil {
  29. glog.V(1).Infof("read %s failed, err: %v", chunkView.FileId, err)
  30. return err
  31. }
  32. }
  33. return nil
  34. }
  35. type ChunkStreamReader struct {
  36. masterClient *wdclient.MasterClient
  37. chunkViews []*ChunkView
  38. logicOffset int64
  39. buffer []byte
  40. bufferOffset int64
  41. bufferPos int
  42. chunkIndex int
  43. lookupFileId func(fileId string) (targetUrl string, err error)
  44. }
  45. var _ = io.ReadSeeker(&ChunkStreamReader{})
  46. func NewChunkStreamReaderFromFiler(masterClient *wdclient.MasterClient, chunks []*filer_pb.FileChunk) *ChunkStreamReader {
  47. chunkViews := ViewFromChunks(chunks, 0, math.MaxInt32)
  48. return &ChunkStreamReader{
  49. chunkViews: chunkViews,
  50. lookupFileId: func(fileId string) (targetUrl string, err error) {
  51. return masterClient.LookupFileId(fileId)
  52. },
  53. }
  54. }
  55. func (c *ChunkStreamReader) Read(p []byte) (n int, err error) {
  56. if c.isBufferEmpty() {
  57. if c.chunkIndex >= len(c.chunkViews) {
  58. return 0, io.EOF
  59. }
  60. chunkView := c.chunkViews[c.chunkIndex]
  61. c.fetchChunkToBuffer(chunkView)
  62. c.chunkIndex++
  63. }
  64. n = copy(p, c.buffer[c.bufferPos:])
  65. c.bufferPos += n
  66. return
  67. }
  68. func (c *ChunkStreamReader) isBufferEmpty() bool {
  69. return len(c.buffer) <= c.bufferPos
  70. }
  71. func (c *ChunkStreamReader) Seek(offset int64, whence int) (int64, error) {
  72. var totalSize int64
  73. for _, chunk := range c.chunkViews {
  74. totalSize += int64(chunk.Size)
  75. }
  76. var err error
  77. switch whence {
  78. case io.SeekStart:
  79. case io.SeekCurrent:
  80. offset += c.bufferOffset + int64(c.bufferPos)
  81. case io.SeekEnd:
  82. offset = totalSize + offset
  83. }
  84. if offset > totalSize {
  85. err = io.ErrUnexpectedEOF
  86. }
  87. for i, chunk := range c.chunkViews {
  88. if chunk.LogicOffset <= offset && offset < chunk.LogicOffset+int64(chunk.Size) {
  89. if c.isBufferEmpty() || c.bufferOffset != chunk.LogicOffset {
  90. c.fetchChunkToBuffer(chunk)
  91. c.chunkIndex = i + 1
  92. break
  93. }
  94. }
  95. }
  96. c.bufferPos = int(offset - c.bufferOffset)
  97. return offset, err
  98. }
  99. func (c *ChunkStreamReader) fetchChunkToBuffer(chunkView *ChunkView) error {
  100. urlString, err := c.lookupFileId(chunkView.FileId)
  101. if err != nil {
  102. glog.V(1).Infof("operation LookupFileId %s failed, err: %v", chunkView.FileId, err)
  103. return err
  104. }
  105. var buffer bytes.Buffer
  106. err = util.ReadUrlAsStream(urlString, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk, chunkView.Offset, int(chunkView.Size), func(data []byte) {
  107. buffer.Write(data)
  108. })
  109. if err != nil {
  110. glog.V(1).Infof("read %s failed, err: %v", chunkView.FileId, err)
  111. return err
  112. }
  113. c.buffer = buffer.Bytes()
  114. c.bufferPos = 0
  115. c.bufferOffset = chunkView.LogicOffset
  116. // glog.V(0).Infof("read %s [%d,%d)", chunkView.FileId, chunkView.LogicOffset, chunkView.LogicOffset+int64(chunkView.Size))
  117. return nil
  118. }
  119. func VolumeId(fileId string) string {
  120. lastCommaIndex := strings.LastIndex(fileId, ",")
  121. if lastCommaIndex > 0 {
  122. return fileId[:lastCommaIndex]
  123. }
  124. return fileId
  125. }