You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

90 lines
2.4 KiB

  1. package storage
  2. import (
  3. "fmt"
  4. "os"
  5. "time"
  6. "github.com/chrislusf/weed-fs/go/glog"
  7. )
  8. func (v *Volume) garbageLevel() float64 {
  9. return float64(v.nm.DeletedSize()) / float64(v.ContentSize())
  10. }
  11. func (v *Volume) Compact() error {
  12. glog.V(3).Infof("Compacting ...")
  13. //no need to lock for copy on write
  14. //v.accessLock.Lock()
  15. //defer v.accessLock.Unlock()
  16. //glog.V(3).Infof("Got Compaction lock...")
  17. filePath := v.FileName()
  18. glog.V(3).Infof("creating copies for volume %d ...", v.Id)
  19. return v.copyDataAndGenerateIndexFile(filePath+".cpd", filePath+".cpx")
  20. }
  21. func (v *Volume) commitCompact() error {
  22. glog.V(3).Infof("Committing vacuuming...")
  23. v.accessLock.Lock()
  24. defer v.accessLock.Unlock()
  25. glog.V(3).Infof("Got Committing lock...")
  26. _ = v.dataFile.Close()
  27. var e error
  28. if e = os.Rename(v.FileName()+".cpd", v.FileName()+".dat"); e != nil {
  29. return e
  30. }
  31. if e = os.Rename(v.FileName()+".cpx", v.FileName()+".idx"); e != nil {
  32. return e
  33. }
  34. //glog.V(3).Infof("Pretending to be vacuuming...")
  35. //time.Sleep(20 * time.Second)
  36. if e = v.load(true, false, v.needleMapKind); e != nil {
  37. return e
  38. }
  39. return nil
  40. }
  41. func (v *Volume) copyDataAndGenerateIndexFile(dstName, idxName string) (err error) {
  42. var (
  43. dst, idx *os.File
  44. )
  45. if dst, err = os.OpenFile(dstName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644); err != nil {
  46. return
  47. }
  48. defer dst.Close()
  49. if idx, err = os.OpenFile(idxName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644); err != nil {
  50. return
  51. }
  52. defer idx.Close()
  53. nm := NewNeedleMap(idx)
  54. new_offset := int64(SuperBlockSize)
  55. now := uint64(time.Now().Unix())
  56. err = ScanVolumeFile(v.dir, v.Collection, v.Id, v.needleMapKind,
  57. func(superBlock SuperBlock) error {
  58. _, err = dst.Write(superBlock.Bytes())
  59. return err
  60. }, true, func(n *Needle, offset int64) error {
  61. if n.HasTtl() && now >= n.LastModified+uint64(v.Ttl.Minutes()*60) {
  62. return nil
  63. }
  64. nv, ok := v.nm.Get(n.Id)
  65. glog.V(4).Infoln("needle expected offset ", offset, "ok", ok, "nv", nv)
  66. if ok && int64(nv.Offset)*NeedlePaddingSize == offset && nv.Size > 0 {
  67. if err = nm.Put(n.Id, uint32(new_offset/NeedlePaddingSize), n.Size); err != nil {
  68. return fmt.Errorf("cannot put needle: %s", err)
  69. }
  70. if _, err = n.Append(dst, v.Version()); err != nil {
  71. return fmt.Errorf("cannot append needle: %s", err)
  72. }
  73. new_offset += n.DiskSize()
  74. glog.V(3).Infoln("saving key", n.Id, "volume offset", offset, "=>", new_offset, "data_size", n.Size)
  75. }
  76. return nil
  77. })
  78. return
  79. }