You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

476 lines
15 KiB

5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
2 years ago
6 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
6 years ago
4 years ago
4 years ago
6 years ago
4 years ago
2 years ago
6 years ago
8 years ago
5 years ago
5 years ago
4 years ago
6 years ago
  1. package storage
  2. import (
  3. "fmt"
  4. "os"
  5. "runtime"
  6. "time"
  7. "github.com/seaweedfs/seaweedfs/weed/glog"
  8. "github.com/seaweedfs/seaweedfs/weed/stats"
  9. "github.com/seaweedfs/seaweedfs/weed/storage/backend"
  10. idx2 "github.com/seaweedfs/seaweedfs/weed/storage/idx"
  11. "github.com/seaweedfs/seaweedfs/weed/storage/needle"
  12. "github.com/seaweedfs/seaweedfs/weed/storage/needle_map"
  13. "github.com/seaweedfs/seaweedfs/weed/storage/super_block"
  14. . "github.com/seaweedfs/seaweedfs/weed/storage/types"
  15. "github.com/seaweedfs/seaweedfs/weed/util"
  16. )
  17. type ProgressFunc func(processed int64) bool
  18. func (v *Volume) garbageLevel() float64 {
  19. if v.ContentSize() == 0 {
  20. return 0
  21. }
  22. deletedSize := v.DeletedSize()
  23. fileSize := v.ContentSize()
  24. if v.DeletedCount() > 0 && v.DeletedSize() == 0 {
  25. // this happens for .sdx converted back to normal .idx
  26. // where deleted entry size is missing
  27. datFileSize, _, _ := v.FileStat()
  28. deletedSize = datFileSize - fileSize - super_block.SuperBlockSize
  29. fileSize = datFileSize
  30. }
  31. return float64(deletedSize) / float64(fileSize)
  32. }
  33. // compact a volume based on deletions in .dat files
  34. func (v *Volume) Compact(preallocate int64, compactionBytePerSecond int64) error {
  35. if v.MemoryMapMaxSizeMb != 0 { //it makes no sense to compact in memory
  36. return nil
  37. }
  38. glog.V(3).Infof("Compacting volume %d ...", v.Id)
  39. //no need to lock for copy on write
  40. //v.accessLock.Lock()
  41. //defer v.accessLock.Unlock()
  42. //glog.V(3).Infof("Got Compaction lock...")
  43. v.isCompacting = true
  44. defer func() {
  45. v.isCompacting = false
  46. }()
  47. v.lastCompactIndexOffset = v.IndexFileSize()
  48. v.lastCompactRevision = v.SuperBlock.CompactionRevision
  49. glog.V(3).Infof("creating copies for volume %d ,last offset %d...", v.Id, v.lastCompactIndexOffset)
  50. if err := v.DataBackend.Sync(); err != nil {
  51. glog.V(0).Infof("compact fail to sync volume %d", v.Id)
  52. }
  53. if err := v.nm.Sync(); err != nil {
  54. glog.V(0).Infof("compact fail to sync volume idx %d", v.Id)
  55. }
  56. return v.copyDataAndGenerateIndexFile(v.FileName(".cpd"), v.FileName(".cpx"), preallocate, compactionBytePerSecond)
  57. }
  58. // compact a volume based on deletions in .idx files
  59. func (v *Volume) Compact2(preallocate int64, compactionBytePerSecond int64, progressFn ProgressFunc) error {
  60. if v.MemoryMapMaxSizeMb != 0 { //it makes no sense to compact in memory
  61. return nil
  62. }
  63. glog.V(3).Infof("Compact2 volume %d ...", v.Id)
  64. v.isCompacting = true
  65. defer func() {
  66. v.isCompacting = false
  67. }()
  68. v.lastCompactIndexOffset = v.IndexFileSize()
  69. v.lastCompactRevision = v.SuperBlock.CompactionRevision
  70. glog.V(3).Infof("creating copies for volume %d ...", v.Id)
  71. if v.DataBackend == nil {
  72. return fmt.Errorf("volume %d backend is empty remote:%v", v.Id, v.HasRemoteFile())
  73. }
  74. if err := v.DataBackend.Sync(); err != nil {
  75. glog.V(0).Infof("compact2 fail to sync volume dat %d: %v", v.Id, err)
  76. }
  77. if err := v.nm.Sync(); err != nil {
  78. glog.V(0).Infof("compact2 fail to sync volume idx %d: %v", v.Id, err)
  79. }
  80. return copyDataBasedOnIndexFile(
  81. v.FileName(".dat"), v.FileName(".idx"),
  82. v.FileName(".cpd"), v.FileName(".cpx"),
  83. v.SuperBlock,
  84. v.Version(),
  85. preallocate,
  86. compactionBytePerSecond,
  87. progressFn,
  88. )
  89. }
  90. func (v *Volume) CommitCompact() error {
  91. if v.MemoryMapMaxSizeMb != 0 { //it makes no sense to compact in memory
  92. return nil
  93. }
  94. glog.V(0).Infof("Committing volume %d vacuuming...", v.Id)
  95. v.isCommitCompacting = true
  96. defer func() {
  97. v.isCommitCompacting = false
  98. }()
  99. v.dataFileAccessLock.Lock()
  100. defer v.dataFileAccessLock.Unlock()
  101. glog.V(3).Infof("Got volume %d committing lock...", v.Id)
  102. v.nm.Close()
  103. if v.DataBackend != nil {
  104. if err := v.DataBackend.Close(); err != nil {
  105. glog.V(0).Infof("fail to close volume %d", v.Id)
  106. }
  107. }
  108. v.DataBackend = nil
  109. stats.VolumeServerVolumeCounter.WithLabelValues(v.Collection, "volume").Dec()
  110. var e error
  111. if e = v.makeupDiff(v.FileName(".cpd"), v.FileName(".cpx"), v.FileName(".dat"), v.FileName(".idx")); e != nil {
  112. glog.V(0).Infof("makeupDiff in CommitCompact volume %d failed %v", v.Id, e)
  113. e = os.Remove(v.FileName(".cpd"))
  114. if e != nil {
  115. return e
  116. }
  117. e = os.Remove(v.FileName(".cpx"))
  118. if e != nil {
  119. return e
  120. }
  121. } else {
  122. if runtime.GOOS == "windows" {
  123. e = os.RemoveAll(v.FileName(".dat"))
  124. if e != nil {
  125. return e
  126. }
  127. e = os.RemoveAll(v.FileName(".idx"))
  128. if e != nil {
  129. return e
  130. }
  131. }
  132. var e error
  133. if e = os.Rename(v.FileName(".cpd"), v.FileName(".dat")); e != nil {
  134. return fmt.Errorf("rename %s: %v", v.FileName(".cpd"), e)
  135. }
  136. if e = os.Rename(v.FileName(".cpx"), v.FileName(".idx")); e != nil {
  137. return fmt.Errorf("rename %s: %v", v.FileName(".cpx"), e)
  138. }
  139. }
  140. //glog.V(3).Infof("Pretending to be vacuuming...")
  141. //time.Sleep(20 * time.Second)
  142. os.RemoveAll(v.FileName(".ldb"))
  143. glog.V(3).Infof("Loading volume %d commit file...", v.Id)
  144. if e = v.load(true, false, v.needleMapKind, 0); e != nil {
  145. return e
  146. }
  147. return nil
  148. }
  149. func (v *Volume) cleanupCompact() error {
  150. glog.V(0).Infof("Cleaning up volume %d vacuuming...", v.Id)
  151. e1 := os.Remove(v.FileName(".cpd"))
  152. e2 := os.Remove(v.FileName(".cpx"))
  153. if e1 != nil && !os.IsNotExist(e1) {
  154. return e1
  155. }
  156. if e2 != nil && !os.IsNotExist(e2) {
  157. return e2
  158. }
  159. return nil
  160. }
  161. func fetchCompactRevisionFromDatFile(datBackend backend.BackendStorageFile) (compactRevision uint16, err error) {
  162. superBlock, err := super_block.ReadSuperBlock(datBackend)
  163. if err != nil {
  164. return 0, err
  165. }
  166. return superBlock.CompactionRevision, nil
  167. }
  168. // if old .dat and .idx files are updated, this func tries to apply the same changes to new files accordingly
  169. func (v *Volume) makeupDiff(newDatFileName, newIdxFileName, oldDatFileName, oldIdxFileName string) (err error) {
  170. var indexSize int64
  171. oldIdxFile, err := os.Open(oldIdxFileName)
  172. if err != nil {
  173. return fmt.Errorf("makeupDiff open %s failed: %v", oldIdxFileName, err)
  174. }
  175. defer oldIdxFile.Close()
  176. oldDatFile, err := os.Open(oldDatFileName)
  177. if err != nil {
  178. return fmt.Errorf("makeupDiff open %s failed: %v", oldDatFileName, err)
  179. }
  180. oldDatBackend := backend.NewDiskFile(oldDatFile)
  181. defer oldDatBackend.Close()
  182. // skip if the old .idx file has not changed
  183. if indexSize, err = verifyIndexFileIntegrity(oldIdxFile); err != nil {
  184. return fmt.Errorf("verifyIndexFileIntegrity %s failed: %v", oldIdxFileName, err)
  185. }
  186. if indexSize == 0 || uint64(indexSize) <= v.lastCompactIndexOffset {
  187. return nil
  188. }
  189. // fail if the old .dat file has changed to a new revision
  190. oldDatCompactRevision, err := fetchCompactRevisionFromDatFile(oldDatBackend)
  191. if err != nil {
  192. return fmt.Errorf("fetchCompactRevisionFromDatFile src %s failed: %v", oldDatFile.Name(), err)
  193. }
  194. if oldDatCompactRevision != v.lastCompactRevision {
  195. return fmt.Errorf("current old dat file's compact revision %d is not the expected one %d", oldDatCompactRevision, v.lastCompactRevision)
  196. }
  197. type keyField struct {
  198. offset Offset
  199. size Size
  200. }
  201. incrementedHasUpdatedIndexEntry := make(map[NeedleId]keyField)
  202. for idxOffset := indexSize - NeedleMapEntrySize; uint64(idxOffset) >= v.lastCompactIndexOffset; idxOffset -= NeedleMapEntrySize {
  203. var IdxEntry []byte
  204. if IdxEntry, err = readIndexEntryAtOffset(oldIdxFile, idxOffset); err != nil {
  205. return fmt.Errorf("readIndexEntry %s at offset %d failed: %v", oldIdxFileName, idxOffset, err)
  206. }
  207. key, offset, size := idx2.IdxFileEntry(IdxEntry)
  208. glog.V(4).Infof("key %d offset %d size %d", key, offset, size)
  209. if _, found := incrementedHasUpdatedIndexEntry[key]; !found {
  210. incrementedHasUpdatedIndexEntry[key] = keyField{
  211. offset: offset,
  212. size: size,
  213. }
  214. }
  215. }
  216. // no updates during commit step
  217. if len(incrementedHasUpdatedIndexEntry) == 0 {
  218. return nil
  219. }
  220. // deal with updates during commit step
  221. var (
  222. dst, idx *os.File
  223. )
  224. if dst, err = os.OpenFile(newDatFileName, os.O_RDWR, 0644); err != nil {
  225. return fmt.Errorf("open dat file %s failed: %v", newDatFileName, err)
  226. }
  227. dstDatBackend := backend.NewDiskFile(dst)
  228. defer dstDatBackend.Close()
  229. if idx, err = os.OpenFile(newIdxFileName, os.O_RDWR, 0644); err != nil {
  230. return fmt.Errorf("open idx file %s failed: %v", newIdxFileName, err)
  231. }
  232. defer idx.Close()
  233. var newDatCompactRevision uint16
  234. newDatCompactRevision, err = fetchCompactRevisionFromDatFile(dstDatBackend)
  235. if err != nil {
  236. return fmt.Errorf("fetchCompactRevisionFromDatFile dst %s failed: %v", dst.Name(), err)
  237. }
  238. if oldDatCompactRevision+1 != newDatCompactRevision {
  239. return fmt.Errorf("oldDatFile %s 's compact revision is %d while newDatFile %s 's compact revision is %d", oldDatFileName, oldDatCompactRevision, newDatFileName, newDatCompactRevision)
  240. }
  241. for key, increIdxEntry := range incrementedHasUpdatedIndexEntry {
  242. idxEntryBytes := needle_map.ToBytes(key, increIdxEntry.offset, increIdxEntry.size)
  243. var offset int64
  244. if offset, err = dst.Seek(0, 2); err != nil {
  245. glog.V(0).Infof("failed to seek the end of file: %v", err)
  246. return
  247. }
  248. //ensure file writing starting from aligned positions
  249. if offset%NeedlePaddingSize != 0 {
  250. offset = offset + (NeedlePaddingSize - offset%NeedlePaddingSize)
  251. if offset, err = dst.Seek(offset, 0); err != nil {
  252. glog.V(0).Infof("failed to align in datafile %s: %v", dst.Name(), err)
  253. return
  254. }
  255. }
  256. //updated needle
  257. if !increIdxEntry.offset.IsZero() && increIdxEntry.size != 0 && increIdxEntry.size.IsValid() {
  258. //even the needle cache in memory is hit, the need_bytes is correct
  259. glog.V(4).Infof("file %d offset %d size %d", key, increIdxEntry.offset.ToActualOffset(), increIdxEntry.size)
  260. var needleBytes []byte
  261. needleBytes, err = needle.ReadNeedleBlob(oldDatBackend, increIdxEntry.offset.ToActualOffset(), increIdxEntry.size, v.Version())
  262. if err != nil {
  263. return fmt.Errorf("ReadNeedleBlob %s key %d offset %d size %d failed: %v", oldDatFile.Name(), key, increIdxEntry.offset.ToActualOffset(), increIdxEntry.size, err)
  264. }
  265. dstDatBackend.Write(needleBytes)
  266. if err := dstDatBackend.Sync(); err != nil {
  267. return fmt.Errorf("cannot sync needle %s: %v", dstDatBackend.File.Name(), err)
  268. }
  269. util.Uint32toBytes(idxEntryBytes[8:12], uint32(offset/NeedlePaddingSize))
  270. } else { //deleted needle
  271. //fakeDelNeedle's default Data field is nil
  272. fakeDelNeedle := new(needle.Needle)
  273. fakeDelNeedle.Id = key
  274. fakeDelNeedle.Cookie = 0x12345678
  275. fakeDelNeedle.AppendAtNs = uint64(time.Now().UnixNano())
  276. _, _, _, err = fakeDelNeedle.Append(dstDatBackend, v.Version())
  277. if err != nil {
  278. return fmt.Errorf("append deleted %d failed: %v", key, err)
  279. }
  280. util.Uint32toBytes(idxEntryBytes[8:12], uint32(0))
  281. }
  282. if _, err := idx.Seek(0, 2); err != nil {
  283. return fmt.Errorf("cannot seek end of indexfile %s: %v",
  284. newIdxFileName, err)
  285. }
  286. _, err = idx.Write(idxEntryBytes)
  287. if err != nil {
  288. return fmt.Errorf("cannot write indexfile %s: %v", newIdxFileName, err)
  289. }
  290. }
  291. return nil
  292. }
  293. type VolumeFileScanner4Vacuum struct {
  294. version needle.Version
  295. v *Volume
  296. dstBackend backend.BackendStorageFile
  297. nm *needle_map.MemDb
  298. newOffset int64
  299. now uint64
  300. writeThrottler *util.WriteThrottler
  301. }
  302. func (scanner *VolumeFileScanner4Vacuum) VisitSuperBlock(superBlock super_block.SuperBlock) error {
  303. scanner.version = superBlock.Version
  304. superBlock.CompactionRevision++
  305. _, err := scanner.dstBackend.WriteAt(superBlock.Bytes(), 0)
  306. scanner.newOffset = int64(superBlock.BlockSize())
  307. return err
  308. }
  309. func (scanner *VolumeFileScanner4Vacuum) ReadNeedleBody() bool {
  310. return true
  311. }
  312. func (scanner *VolumeFileScanner4Vacuum) VisitNeedle(n *needle.Needle, offset int64, needleHeader, needleBody []byte) error {
  313. if n.HasTtl() && scanner.now >= n.LastModified+uint64(scanner.v.Ttl.Minutes()*60) {
  314. return nil
  315. }
  316. nv, ok := scanner.v.nm.Get(n.Id)
  317. glog.V(4).Infoln("needle expected offset ", offset, "ok", ok, "nv", nv)
  318. if ok && nv.Offset.ToActualOffset() == offset && nv.Size > 0 && nv.Size.IsValid() {
  319. if err := scanner.nm.Set(n.Id, ToOffset(scanner.newOffset), n.Size); err != nil {
  320. return fmt.Errorf("cannot put needle: %s", err)
  321. }
  322. if _, _, _, err := n.Append(scanner.dstBackend, scanner.v.Version()); err != nil {
  323. return fmt.Errorf("cannot append needle: %s", err)
  324. }
  325. delta := n.DiskSize(scanner.version)
  326. scanner.newOffset += delta
  327. scanner.writeThrottler.MaybeSlowdown(delta)
  328. glog.V(4).Infoln("saving key", n.Id, "volume offset", offset, "=>", scanner.newOffset, "data_size", n.Size)
  329. }
  330. return nil
  331. }
  332. func (v *Volume) copyDataAndGenerateIndexFile(dstName, idxName string, preallocate int64, compactionBytePerSecond int64) (err error) {
  333. var (
  334. dst backend.BackendStorageFile
  335. )
  336. if dst, err = backend.CreateVolumeFile(dstName, preallocate, 0); err != nil {
  337. return err
  338. }
  339. defer dst.Close()
  340. nm := needle_map.NewMemDb()
  341. defer nm.Close()
  342. scanner := &VolumeFileScanner4Vacuum{
  343. v: v,
  344. now: uint64(time.Now().Unix()),
  345. nm: nm,
  346. dstBackend: dst,
  347. writeThrottler: util.NewWriteThrottler(compactionBytePerSecond),
  348. }
  349. err = ScanVolumeFile(v.dir, v.Collection, v.Id, v.needleMapKind, scanner)
  350. if err != nil {
  351. return err
  352. }
  353. return nm.SaveToIdx(idxName)
  354. }
  355. func copyDataBasedOnIndexFile(srcDatName, srcIdxName, dstDatName, datIdxName string, sb super_block.SuperBlock, version needle.Version, preallocate, compactionBytePerSecond int64, progressFn ProgressFunc) (err error) {
  356. var (
  357. srcDatBackend, dstDatBackend backend.BackendStorageFile
  358. dataFile *os.File
  359. )
  360. if dstDatBackend, err = backend.CreateVolumeFile(dstDatName, preallocate, 0); err != nil {
  361. return err
  362. }
  363. defer dstDatBackend.Close()
  364. oldNm := needle_map.NewMemDb()
  365. defer oldNm.Close()
  366. newNm := needle_map.NewMemDb()
  367. defer newNm.Close()
  368. if err = oldNm.LoadFromIdx(srcIdxName); err != nil {
  369. return err
  370. }
  371. if dataFile, err = os.Open(srcDatName); err != nil {
  372. return err
  373. }
  374. srcDatBackend = backend.NewDiskFile(dataFile)
  375. defer srcDatBackend.Close()
  376. now := uint64(time.Now().Unix())
  377. sb.CompactionRevision++
  378. dstDatBackend.WriteAt(sb.Bytes(), 0)
  379. newOffset := int64(sb.BlockSize())
  380. writeThrottler := util.NewWriteThrottler(compactionBytePerSecond)
  381. err = oldNm.AscendingVisit(func(value needle_map.NeedleValue) error {
  382. offset, size := value.Offset, value.Size
  383. if offset.IsZero() || size.IsDeleted() {
  384. return nil
  385. }
  386. if progressFn != nil {
  387. if !progressFn(offset.ToActualOffset()) {
  388. return fmt.Errorf("interrupted")
  389. }
  390. }
  391. n := new(needle.Needle)
  392. if err := n.ReadData(srcDatBackend, offset.ToActualOffset(), size, version); err != nil {
  393. return fmt.Errorf("cannot hydrate needle from file: %s", err)
  394. }
  395. if n.HasTtl() && now >= n.LastModified+uint64(sb.Ttl.Minutes()*60) {
  396. return nil
  397. }
  398. if err = newNm.Set(n.Id, ToOffset(newOffset), n.Size); err != nil {
  399. return fmt.Errorf("cannot put needle: %s", err)
  400. }
  401. if _, _, _, err = n.Append(dstDatBackend, sb.Version); err != nil {
  402. return fmt.Errorf("cannot append needle: %s", err)
  403. }
  404. delta := n.DiskSize(version)
  405. newOffset += delta
  406. writeThrottler.MaybeSlowdown(delta)
  407. glog.V(4).Infoln("saving key", n.Id, "volume offset", offset, "=>", newOffset, "data_size", n.Size)
  408. return nil
  409. })
  410. if err != nil {
  411. return err
  412. }
  413. return newNm.SaveToIdx(datIdxName)
  414. }