You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

237 lines
7.0 KiB

6 years ago
6 years ago
5 years ago
6 years ago
6 years ago
6 years ago
5 years ago
  1. package storage
  2. import (
  3. "bytes"
  4. "errors"
  5. "fmt"
  6. "io"
  7. "os"
  8. "time"
  9. "github.com/chrislusf/seaweedfs/weed/glog"
  10. "github.com/chrislusf/seaweedfs/weed/storage/backend"
  11. "github.com/chrislusf/seaweedfs/weed/storage/needle"
  12. "github.com/chrislusf/seaweedfs/weed/storage/super_block"
  13. . "github.com/chrislusf/seaweedfs/weed/storage/types"
  14. )
  15. var ErrorNotFound = errors.New("not found")
  16. // isFileUnchanged checks whether this needle to write is same as last one.
  17. // It requires serialized access in the same volume.
  18. func (v *Volume) isFileUnchanged(n *needle.Needle) bool {
  19. if v.Ttl.String() != "" {
  20. return false
  21. }
  22. nv, ok := v.nm.Get(n.Id)
  23. if ok && !nv.Offset.IsZero() && nv.Size != TombstoneFileSize {
  24. oldNeedle := new(needle.Needle)
  25. err := oldNeedle.ReadData(v.DataBackend, nv.Offset.ToAcutalOffset(), nv.Size, v.Version())
  26. if err != nil {
  27. glog.V(0).Infof("Failed to check updated file at offset %d size %d: %v", nv.Offset.ToAcutalOffset(), nv.Size, err)
  28. return false
  29. }
  30. if oldNeedle.Cookie == n.Cookie && oldNeedle.Checksum == n.Checksum && bytes.Equal(oldNeedle.Data, n.Data) {
  31. n.DataSize = oldNeedle.DataSize
  32. return true
  33. }
  34. }
  35. return false
  36. }
  37. // Destroy removes everything related to this volume
  38. func (v *Volume) Destroy() (err error) {
  39. if v.isCompacting {
  40. err = fmt.Errorf("volume %d is compacting", v.Id)
  41. return
  42. }
  43. storageName, storageKey := v.RemoteStorageNameKey()
  44. if v.HasRemoteFile() && storageName != "" && storageKey != "" {
  45. if backendStorage, found := backend.BackendStorages[storageName]; found {
  46. backendStorage.DeleteFile(storageKey)
  47. }
  48. }
  49. v.Close()
  50. os.Remove(v.FileName() + ".dat")
  51. os.Remove(v.FileName() + ".idx")
  52. os.Remove(v.FileName() + ".vif")
  53. os.Remove(v.FileName() + ".sdx")
  54. os.Remove(v.FileName() + ".cpd")
  55. os.Remove(v.FileName() + ".cpx")
  56. os.RemoveAll(v.FileName() + ".ldb")
  57. return
  58. }
  59. func (v *Volume) writeNeedle(n *needle.Needle) (offset uint64, size uint32, isUnchanged bool, err error) {
  60. // glog.V(4).Infof("writing needle %s", needle.NewFileIdFromNeedle(v.Id, n).String())
  61. v.dataFileAccessLock.Lock()
  62. defer v.dataFileAccessLock.Unlock()
  63. if v.isFileUnchanged(n) {
  64. size = n.DataSize
  65. isUnchanged = true
  66. return
  67. }
  68. if n.Ttl == needle.EMPTY_TTL && v.Ttl != needle.EMPTY_TTL {
  69. n.SetHasTtl()
  70. n.Ttl = v.Ttl
  71. }
  72. // check whether existing needle cookie matches
  73. nv, ok := v.nm.Get(n.Id)
  74. if ok {
  75. existingNeedle, _, _, existingNeedleReadErr := needle.ReadNeedleHeader(v.DataBackend, v.Version(), nv.Offset.ToAcutalOffset())
  76. if existingNeedleReadErr != nil {
  77. err = fmt.Errorf("reading existing needle: %v", existingNeedleReadErr)
  78. return
  79. }
  80. if existingNeedle.Cookie != n.Cookie {
  81. glog.V(0).Infof("write cookie mismatch: existing %x, new %x", existingNeedle.Cookie, n.Cookie)
  82. err = fmt.Errorf("mismatching cookie %x", n.Cookie)
  83. return
  84. }
  85. }
  86. // append to dat file
  87. n.AppendAtNs = uint64(time.Now().UnixNano())
  88. if offset, size, _, err = n.Append(v.DataBackend, v.Version()); err != nil {
  89. return
  90. }
  91. v.lastAppendAtNs = n.AppendAtNs
  92. // add to needle map
  93. if !ok || uint64(nv.Offset.ToAcutalOffset()) < offset {
  94. if err = v.nm.Put(n.Id, ToOffset(int64(offset)), n.Size); err != nil {
  95. glog.V(4).Infof("failed to save in needle map %d: %v", n.Id, err)
  96. }
  97. }
  98. if v.lastModifiedTsSeconds < n.LastModified {
  99. v.lastModifiedTsSeconds = n.LastModified
  100. }
  101. return
  102. }
  103. func (v *Volume) deleteNeedle(n *needle.Needle) (uint32, error) {
  104. glog.V(4).Infof("delete needle %s", needle.NewFileIdFromNeedle(v.Id, n).String())
  105. v.dataFileAccessLock.Lock()
  106. defer v.dataFileAccessLock.Unlock()
  107. nv, ok := v.nm.Get(n.Id)
  108. //fmt.Println("key", n.Id, "volume offset", nv.Offset, "data_size", n.Size, "cached size", nv.Size)
  109. if ok && nv.Size != TombstoneFileSize {
  110. size := nv.Size
  111. n.Data = nil
  112. n.AppendAtNs = uint64(time.Now().UnixNano())
  113. offset, _, _, err := n.Append(v.DataBackend, v.Version())
  114. if err != nil {
  115. return size, err
  116. }
  117. v.lastAppendAtNs = n.AppendAtNs
  118. if err = v.nm.Delete(n.Id, ToOffset(int64(offset))); err != nil {
  119. return size, err
  120. }
  121. return size, err
  122. }
  123. return 0, nil
  124. }
  125. // read fills in Needle content by looking up n.Id from NeedleMapper
  126. func (v *Volume) readNeedle(n *needle.Needle) (int, error) {
  127. v.dataFileAccessLock.RLock()
  128. defer v.dataFileAccessLock.RUnlock()
  129. nv, ok := v.nm.Get(n.Id)
  130. if !ok || nv.Offset.IsZero() {
  131. return -1, ErrorNotFound
  132. }
  133. if nv.Size == TombstoneFileSize {
  134. return -1, errors.New("already deleted")
  135. }
  136. if nv.Size == 0 {
  137. return 0, nil
  138. }
  139. err := n.ReadData(v.DataBackend, nv.Offset.ToAcutalOffset(), nv.Size, v.Version())
  140. if err != nil {
  141. return 0, err
  142. }
  143. bytesRead := len(n.Data)
  144. if !n.HasTtl() {
  145. return bytesRead, nil
  146. }
  147. ttlMinutes := n.Ttl.Minutes()
  148. if ttlMinutes == 0 {
  149. return bytesRead, nil
  150. }
  151. if !n.HasLastModifiedDate() {
  152. return bytesRead, nil
  153. }
  154. if uint64(time.Now().Unix()) < n.LastModified+uint64(ttlMinutes*60) {
  155. return bytesRead, nil
  156. }
  157. return -1, ErrorNotFound
  158. }
  159. type VolumeFileScanner interface {
  160. VisitSuperBlock(super_block.SuperBlock) error
  161. ReadNeedleBody() bool
  162. VisitNeedle(n *needle.Needle, offset int64, needleHeader, needleBody []byte) error
  163. }
  164. func ScanVolumeFile(dirname string, collection string, id needle.VolumeId,
  165. needleMapKind NeedleMapType,
  166. volumeFileScanner VolumeFileScanner) (err error) {
  167. var v *Volume
  168. if v, err = loadVolumeWithoutIndex(dirname, collection, id, needleMapKind); err != nil {
  169. return fmt.Errorf("failed to load volume %d: %v", id, err)
  170. }
  171. if v.volumeInfo.Version == 0 {
  172. if err = volumeFileScanner.VisitSuperBlock(v.SuperBlock); err != nil {
  173. return fmt.Errorf("failed to process volume %d super block: %v", id, err)
  174. }
  175. }
  176. defer v.Close()
  177. version := v.Version()
  178. offset := int64(v.SuperBlock.BlockSize())
  179. return ScanVolumeFileFrom(version, v.DataBackend, offset, volumeFileScanner)
  180. }
  181. func ScanVolumeFileFrom(version needle.Version, datBackend backend.BackendStorageFile, offset int64, volumeFileScanner VolumeFileScanner) (err error) {
  182. n, nh, rest, e := needle.ReadNeedleHeader(datBackend, version, offset)
  183. if e != nil {
  184. if e == io.EOF {
  185. return nil
  186. }
  187. return fmt.Errorf("cannot read %s at offset %d: %v", datBackend.Name(), offset, e)
  188. }
  189. for n != nil {
  190. var needleBody []byte
  191. if volumeFileScanner.ReadNeedleBody() {
  192. if needleBody, err = n.ReadNeedleBody(datBackend, version, offset+NeedleHeaderSize, rest); err != nil {
  193. glog.V(0).Infof("cannot read needle body: %v", err)
  194. //err = fmt.Errorf("cannot read needle body: %v", err)
  195. //return
  196. }
  197. }
  198. err := volumeFileScanner.VisitNeedle(n, offset, nh, needleBody)
  199. if err == io.EOF {
  200. return nil
  201. }
  202. if err != nil {
  203. glog.V(0).Infof("visit needle error: %v", err)
  204. return fmt.Errorf("visit needle error: %v", err)
  205. }
  206. offset += NeedleHeaderSize + rest
  207. glog.V(4).Infof("==> new entry offset %d", offset)
  208. if n, nh, rest, err = needle.ReadNeedleHeader(datBackend, version, offset); err != nil {
  209. if err == io.EOF {
  210. return nil
  211. }
  212. return fmt.Errorf("cannot read needle header at offset %d: %v", offset, err)
  213. }
  214. glog.V(4).Infof("new entry needle size:%d rest:%d", n.Size, rest)
  215. }
  216. return nil
  217. }