You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

373 lines
9.0 KiB

5 years ago
5 years ago
5 years ago
6 years ago
5 years ago
6 years ago
6 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
5 years ago
4 years ago
6 years ago
6 years ago
6 years ago
6 years ago
5 years ago
6 years ago
6 years ago
6 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
  1. package storage
  2. import (
  3. "fmt"
  4. "io/ioutil"
  5. "os"
  6. "path/filepath"
  7. "strings"
  8. "sync"
  9. "time"
  10. "github.com/chrislusf/seaweedfs/weed/glog"
  11. "github.com/chrislusf/seaweedfs/weed/stats"
  12. "github.com/chrislusf/seaweedfs/weed/storage/erasure_coding"
  13. "github.com/chrislusf/seaweedfs/weed/storage/needle"
  14. "github.com/chrislusf/seaweedfs/weed/util"
  15. )
  16. type DiskLocation struct {
  17. Directory string
  18. IdxDirectory string
  19. MaxVolumeCount int
  20. OriginalMaxVolumeCount int
  21. MinFreeSpacePercent float32
  22. volumes map[needle.VolumeId]*Volume
  23. volumesLock sync.RWMutex
  24. // erasure coding
  25. ecVolumes map[needle.VolumeId]*erasure_coding.EcVolume
  26. ecVolumesLock sync.RWMutex
  27. isDiskSpaceLow bool
  28. }
  29. func NewDiskLocation(dir string, maxVolumeCount int, minFreeSpacePercent float32, idxDir string) *DiskLocation {
  30. dir = util.ResolvePath(dir)
  31. if idxDir == "" {
  32. idxDir = dir
  33. } else {
  34. idxDir = util.ResolvePath(idxDir)
  35. }
  36. location := &DiskLocation{
  37. Directory: dir,
  38. IdxDirectory: idxDir,
  39. MaxVolumeCount: maxVolumeCount,
  40. OriginalMaxVolumeCount: maxVolumeCount,
  41. MinFreeSpacePercent: minFreeSpacePercent,
  42. }
  43. location.volumes = make(map[needle.VolumeId]*Volume)
  44. location.ecVolumes = make(map[needle.VolumeId]*erasure_coding.EcVolume)
  45. go location.CheckDiskSpace()
  46. return location
  47. }
  48. func volumeIdFromFileName(filename string) (needle.VolumeId, string, error) {
  49. if isValidVolume(filename) {
  50. base := filename[:len(filename)-4]
  51. collection, volumeId, err := parseCollectionVolumeId(base)
  52. return volumeId, collection, err
  53. }
  54. return 0, "", fmt.Errorf("file is not a volume: %s", filename)
  55. }
  56. func parseCollectionVolumeId(base string) (collection string, vid needle.VolumeId, err error) {
  57. i := strings.LastIndex(base, "_")
  58. if i > 0 {
  59. collection, base = base[0:i], base[i+1:]
  60. }
  61. vol, err := needle.NewVolumeId(base)
  62. return collection, vol, err
  63. }
  64. func isValidVolume(basename string) bool {
  65. return strings.HasSuffix(basename, ".idx") || strings.HasSuffix(basename, ".vif")
  66. }
  67. func getValidVolumeName(basename string) string {
  68. if isValidVolume(basename) {
  69. return basename[:len(basename)-4]
  70. }
  71. return ""
  72. }
  73. func (l *DiskLocation) loadExistingVolume(fileInfo os.FileInfo, needleMapKind NeedleMapKind) bool {
  74. basename := fileInfo.Name()
  75. if fileInfo.IsDir() {
  76. return false
  77. }
  78. volumeName := getValidVolumeName(basename)
  79. if volumeName == "" {
  80. return false
  81. }
  82. // check for incomplete volume
  83. noteFile := l.Directory + "/" + volumeName + ".note"
  84. if util.FileExists(noteFile) {
  85. note, _ := ioutil.ReadFile(noteFile)
  86. glog.Warningf("volume %s was not completed: %s", volumeName, string(note))
  87. removeVolumeFiles(l.Directory + "/" + volumeName)
  88. removeVolumeFiles(l.IdxDirectory + "/" + volumeName)
  89. return false
  90. }
  91. // parse out collection, volume id
  92. vid, collection, err := volumeIdFromFileName(basename)
  93. if err != nil {
  94. glog.Warningf("get volume id failed, %s, err : %s", volumeName, err)
  95. return false
  96. }
  97. // avoid loading one volume more than once
  98. l.volumesLock.RLock()
  99. _, found := l.volumes[vid]
  100. l.volumesLock.RUnlock()
  101. if found {
  102. glog.V(1).Infof("loaded volume, %v", vid)
  103. return true
  104. }
  105. // load the volume
  106. v, e := NewVolume(l.Directory, l.IdxDirectory, collection, vid, needleMapKind, nil, nil, 0, 0)
  107. if e != nil {
  108. glog.V(0).Infof("new volume %s error %s", volumeName, e)
  109. return false
  110. }
  111. l.SetVolume(vid, v)
  112. size, _, _ := v.FileStat()
  113. glog.V(0).Infof("data file %s, replicaPlacement=%s v=%d size=%d ttl=%s",
  114. l.Directory+"/"+volumeName+".dat", v.ReplicaPlacement, v.Version(), size, v.Ttl.String())
  115. return true
  116. }
  117. func (l *DiskLocation) concurrentLoadingVolumes(needleMapKind NeedleMapKind, concurrency int) {
  118. task_queue := make(chan os.FileInfo, 10*concurrency)
  119. go func() {
  120. foundVolumeNames := make(map[string]bool)
  121. if fileInfos, err := ioutil.ReadDir(l.Directory); err == nil {
  122. for _, fi := range fileInfos {
  123. volumeName := getValidVolumeName(fi.Name())
  124. if volumeName == "" {
  125. continue
  126. }
  127. if _, found := foundVolumeNames[volumeName]; !found {
  128. foundVolumeNames[volumeName] = true
  129. task_queue <- fi
  130. }
  131. }
  132. }
  133. close(task_queue)
  134. }()
  135. var wg sync.WaitGroup
  136. for workerNum := 0; workerNum < concurrency; workerNum++ {
  137. wg.Add(1)
  138. go func() {
  139. defer wg.Done()
  140. for fi := range task_queue {
  141. _ = l.loadExistingVolume(fi, needleMapKind)
  142. }
  143. }()
  144. }
  145. wg.Wait()
  146. }
  147. func (l *DiskLocation) loadExistingVolumes(needleMapKind NeedleMapKind) {
  148. l.concurrentLoadingVolumes(needleMapKind, 10)
  149. glog.V(0).Infof("Store started on dir: %s with %d volumes max %d", l.Directory, len(l.volumes), l.MaxVolumeCount)
  150. l.loadAllEcShards()
  151. glog.V(0).Infof("Store started on dir: %s with %d ec shards", l.Directory, len(l.ecVolumes))
  152. }
  153. func (l *DiskLocation) DeleteCollectionFromDiskLocation(collection string) (e error) {
  154. l.volumesLock.Lock()
  155. delVolsMap := l.unmountVolumeByCollection(collection)
  156. l.volumesLock.Unlock()
  157. l.ecVolumesLock.Lock()
  158. delEcVolsMap := l.unmountEcVolumeByCollection(collection)
  159. l.ecVolumesLock.Unlock()
  160. errChain := make(chan error, 2)
  161. var wg sync.WaitGroup
  162. wg.Add(2)
  163. go func() {
  164. for _, v := range delVolsMap {
  165. if err := v.Destroy(); err != nil {
  166. errChain <- err
  167. }
  168. }
  169. wg.Done()
  170. }()
  171. go func() {
  172. for _, v := range delEcVolsMap {
  173. v.Destroy()
  174. }
  175. wg.Done()
  176. }()
  177. go func() {
  178. wg.Wait()
  179. close(errChain)
  180. }()
  181. errBuilder := strings.Builder{}
  182. for err := range errChain {
  183. errBuilder.WriteString(err.Error())
  184. errBuilder.WriteString("; ")
  185. }
  186. if errBuilder.Len() > 0 {
  187. e = fmt.Errorf(errBuilder.String())
  188. }
  189. return
  190. }
  191. func (l *DiskLocation) deleteVolumeById(vid needle.VolumeId) (found bool, e error) {
  192. v, ok := l.volumes[vid]
  193. if !ok {
  194. return
  195. }
  196. e = v.Destroy()
  197. if e != nil {
  198. return
  199. }
  200. found = true
  201. delete(l.volumes, vid)
  202. return
  203. }
  204. func (l *DiskLocation) LoadVolume(vid needle.VolumeId, needleMapKind NeedleMapKind) bool {
  205. if fileInfo, found := l.LocateVolume(vid); found {
  206. return l.loadExistingVolume(fileInfo, needleMapKind)
  207. }
  208. return false
  209. }
  210. func (l *DiskLocation) DeleteVolume(vid needle.VolumeId) error {
  211. l.volumesLock.Lock()
  212. defer l.volumesLock.Unlock()
  213. _, ok := l.volumes[vid]
  214. if !ok {
  215. return fmt.Errorf("Volume not found, VolumeId: %d", vid)
  216. }
  217. _, err := l.deleteVolumeById(vid)
  218. return err
  219. }
  220. func (l *DiskLocation) UnloadVolume(vid needle.VolumeId) error {
  221. l.volumesLock.Lock()
  222. defer l.volumesLock.Unlock()
  223. v, ok := l.volumes[vid]
  224. if !ok {
  225. return fmt.Errorf("Volume not loaded, VolumeId: %d", vid)
  226. }
  227. v.Close()
  228. delete(l.volumes, vid)
  229. return nil
  230. }
  231. func (l *DiskLocation) unmountVolumeByCollection(collectionName string) map[needle.VolumeId]*Volume {
  232. deltaVols := make(map[needle.VolumeId]*Volume, 0)
  233. for k, v := range l.volumes {
  234. if v.Collection == collectionName && !v.isCompacting {
  235. deltaVols[k] = v
  236. }
  237. }
  238. for k := range deltaVols {
  239. delete(l.volumes, k)
  240. }
  241. return deltaVols
  242. }
  243. func (l *DiskLocation) SetVolume(vid needle.VolumeId, volume *Volume) {
  244. l.volumesLock.Lock()
  245. defer l.volumesLock.Unlock()
  246. l.volumes[vid] = volume
  247. volume.location = l
  248. }
  249. func (l *DiskLocation) FindVolume(vid needle.VolumeId) (*Volume, bool) {
  250. l.volumesLock.RLock()
  251. defer l.volumesLock.RUnlock()
  252. v, ok := l.volumes[vid]
  253. return v, ok
  254. }
  255. func (l *DiskLocation) VolumesLen() int {
  256. l.volumesLock.RLock()
  257. defer l.volumesLock.RUnlock()
  258. return len(l.volumes)
  259. }
  260. func (l *DiskLocation) Close() {
  261. l.volumesLock.Lock()
  262. for _, v := range l.volumes {
  263. v.Close()
  264. }
  265. l.volumesLock.Unlock()
  266. l.ecVolumesLock.Lock()
  267. for _, ecVolume := range l.ecVolumes {
  268. ecVolume.Close()
  269. }
  270. l.ecVolumesLock.Unlock()
  271. return
  272. }
  273. func (l *DiskLocation) LocateVolume(vid needle.VolumeId) (os.FileInfo, bool) {
  274. if fileInfos, err := ioutil.ReadDir(l.Directory); err == nil {
  275. for _, fileInfo := range fileInfos {
  276. volId, _, err := volumeIdFromFileName(fileInfo.Name())
  277. if vid == volId && err == nil {
  278. return fileInfo, true
  279. }
  280. }
  281. }
  282. return nil, false
  283. }
  284. func (l *DiskLocation) UnUsedSpace(volumeSizeLimit uint64) (unUsedSpace uint64) {
  285. l.volumesLock.RLock()
  286. defer l.volumesLock.RUnlock()
  287. for _, vol := range l.volumes {
  288. if vol.IsReadOnly() {
  289. continue
  290. }
  291. datSize, idxSize, _ := vol.FileStat()
  292. unUsedSpace += volumeSizeLimit - (datSize + idxSize)
  293. }
  294. return
  295. }
  296. func (l *DiskLocation) CheckDiskSpace() {
  297. for {
  298. if dir, e := filepath.Abs(l.Directory); e == nil {
  299. s := stats.NewDiskStatus(dir)
  300. stats.VolumeServerResourceGauge.WithLabelValues(l.Directory, "all").Set(float64(s.All))
  301. stats.VolumeServerResourceGauge.WithLabelValues(l.Directory, "used").Set(float64(s.Used))
  302. stats.VolumeServerResourceGauge.WithLabelValues(l.Directory, "free").Set(float64(s.Free))
  303. if (s.PercentFree < l.MinFreeSpacePercent) != l.isDiskSpaceLow {
  304. l.isDiskSpaceLow = !l.isDiskSpaceLow
  305. }
  306. if l.isDiskSpaceLow {
  307. glog.V(0).Infof("dir %s freePercent %.2f%% < min %.2f%%, isLowDiskSpace: %v", dir, s.PercentFree, l.MinFreeSpacePercent, l.isDiskSpaceLow)
  308. } else {
  309. glog.V(4).Infof("dir %s freePercent %.2f%% < min %.2f%%, isLowDiskSpace: %v", dir, s.PercentFree, l.MinFreeSpacePercent, l.isDiskSpaceLow)
  310. }
  311. }
  312. time.Sleep(time.Minute)
  313. }
  314. }