You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

382 lines
9.2 KiB

5 years ago
5 years ago
5 years ago
4 years ago
6 years ago
5 years ago
4 years ago
6 years ago
6 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
5 years ago
4 years ago
6 years ago
6 years ago
6 years ago
6 years ago
5 years ago
6 years ago
6 years ago
6 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
  1. package storage
  2. import (
  3. "fmt"
  4. "io/ioutil"
  5. "os"
  6. "path/filepath"
  7. "strings"
  8. "sync"
  9. "time"
  10. "github.com/chrislusf/seaweedfs/weed/glog"
  11. "github.com/chrislusf/seaweedfs/weed/stats"
  12. "github.com/chrislusf/seaweedfs/weed/storage/erasure_coding"
  13. "github.com/chrislusf/seaweedfs/weed/storage/needle"
  14. "github.com/chrislusf/seaweedfs/weed/util"
  15. )
  16. type DiskLocation struct {
  17. Directory string
  18. IdxDirectory string
  19. DiskType DiskType
  20. MaxVolumeCount int
  21. OriginalMaxVolumeCount int
  22. MinFreeSpacePercent float32
  23. volumes map[needle.VolumeId]*Volume
  24. volumesLock sync.RWMutex
  25. // erasure coding
  26. ecVolumes map[needle.VolumeId]*erasure_coding.EcVolume
  27. ecVolumesLock sync.RWMutex
  28. isDiskSpaceLow bool
  29. }
  30. func NewDiskLocation(dir string, maxVolumeCount int, minFreeSpacePercent float32, idxDir string, diskType DiskType) *DiskLocation {
  31. dir = util.ResolvePath(dir)
  32. if idxDir == "" {
  33. idxDir = dir
  34. } else {
  35. idxDir = util.ResolvePath(idxDir)
  36. }
  37. location := &DiskLocation{
  38. Directory: dir,
  39. IdxDirectory: idxDir,
  40. DiskType: diskType,
  41. MaxVolumeCount: maxVolumeCount,
  42. OriginalMaxVolumeCount: maxVolumeCount,
  43. MinFreeSpacePercent: minFreeSpacePercent,
  44. }
  45. location.volumes = make(map[needle.VolumeId]*Volume)
  46. location.ecVolumes = make(map[needle.VolumeId]*erasure_coding.EcVolume)
  47. go location.CheckDiskSpace()
  48. return location
  49. }
  50. func volumeIdFromFileName(filename string) (needle.VolumeId, string, error) {
  51. if isValidVolume(filename) {
  52. base := filename[:len(filename)-4]
  53. collection, volumeId, err := parseCollectionVolumeId(base)
  54. return volumeId, collection, err
  55. }
  56. return 0, "", fmt.Errorf("file is not a volume: %s", filename)
  57. }
  58. func parseCollectionVolumeId(base string) (collection string, vid needle.VolumeId, err error) {
  59. i := strings.LastIndex(base, "_")
  60. if i > 0 {
  61. collection, base = base[0:i], base[i+1:]
  62. }
  63. vol, err := needle.NewVolumeId(base)
  64. return collection, vol, err
  65. }
  66. func isValidVolume(basename string) bool {
  67. return strings.HasSuffix(basename, ".idx") || strings.HasSuffix(basename, ".vif")
  68. }
  69. func getValidVolumeName(basename string) string {
  70. if isValidVolume(basename) {
  71. return basename[:len(basename)-4]
  72. }
  73. return ""
  74. }
  75. func (l *DiskLocation) loadExistingVolume(fileInfo os.FileInfo, needleMapKind NeedleMapKind) bool {
  76. basename := fileInfo.Name()
  77. if fileInfo.IsDir() {
  78. return false
  79. }
  80. volumeName := getValidVolumeName(basename)
  81. if volumeName == "" {
  82. return false
  83. }
  84. // check for incomplete volume
  85. noteFile := l.Directory + "/" + volumeName + ".note"
  86. if util.FileExists(noteFile) {
  87. note, _ := ioutil.ReadFile(noteFile)
  88. glog.Warningf("volume %s was not completed: %s", volumeName, string(note))
  89. removeVolumeFiles(l.Directory + "/" + volumeName)
  90. removeVolumeFiles(l.IdxDirectory + "/" + volumeName)
  91. return false
  92. }
  93. // parse out collection, volume id
  94. vid, collection, err := volumeIdFromFileName(basename)
  95. if err != nil {
  96. glog.Warningf("get volume id failed, %s, err : %s", volumeName, err)
  97. return false
  98. }
  99. // avoid loading one volume more than once
  100. l.volumesLock.RLock()
  101. _, found := l.volumes[vid]
  102. l.volumesLock.RUnlock()
  103. if found {
  104. glog.V(1).Infof("loaded volume, %v", vid)
  105. return true
  106. }
  107. // load the volume
  108. v, e := NewVolume(l.Directory, l.IdxDirectory, collection, vid, needleMapKind, nil, nil, 0, 0)
  109. if e != nil {
  110. glog.V(0).Infof("new volume %s error %s", volumeName, e)
  111. return false
  112. }
  113. l.SetVolume(vid, v)
  114. size, _, _ := v.FileStat()
  115. glog.V(0).Infof("data file %s, replicaPlacement=%s v=%d size=%d ttl=%s",
  116. l.Directory+"/"+volumeName+".dat", v.ReplicaPlacement, v.Version(), size, v.Ttl.String())
  117. return true
  118. }
  119. func (l *DiskLocation) concurrentLoadingVolumes(needleMapKind NeedleMapKind, concurrency int) {
  120. task_queue := make(chan os.FileInfo, 10*concurrency)
  121. go func() {
  122. foundVolumeNames := make(map[string]bool)
  123. if fileInfos, err := ioutil.ReadDir(l.Directory); err == nil {
  124. for _, fi := range fileInfos {
  125. volumeName := getValidVolumeName(fi.Name())
  126. if volumeName == "" {
  127. continue
  128. }
  129. if _, found := foundVolumeNames[volumeName]; !found {
  130. foundVolumeNames[volumeName] = true
  131. task_queue <- fi
  132. }
  133. }
  134. }
  135. close(task_queue)
  136. }()
  137. var wg sync.WaitGroup
  138. for workerNum := 0; workerNum < concurrency; workerNum++ {
  139. wg.Add(1)
  140. go func() {
  141. defer wg.Done()
  142. for fi := range task_queue {
  143. _ = l.loadExistingVolume(fi, needleMapKind)
  144. }
  145. }()
  146. }
  147. wg.Wait()
  148. }
  149. func (l *DiskLocation) loadExistingVolumes(needleMapKind NeedleMapKind) {
  150. l.concurrentLoadingVolumes(needleMapKind, 10)
  151. glog.V(0).Infof("Store started on dir: %s with %d volumes max %d", l.Directory, len(l.volumes), l.MaxVolumeCount)
  152. l.loadAllEcShards()
  153. glog.V(0).Infof("Store started on dir: %s with %d ec shards", l.Directory, len(l.ecVolumes))
  154. }
  155. func (l *DiskLocation) DeleteCollectionFromDiskLocation(collection string) (e error) {
  156. l.volumesLock.Lock()
  157. delVolsMap := l.unmountVolumeByCollection(collection)
  158. l.volumesLock.Unlock()
  159. l.ecVolumesLock.Lock()
  160. delEcVolsMap := l.unmountEcVolumeByCollection(collection)
  161. l.ecVolumesLock.Unlock()
  162. errChain := make(chan error, 2)
  163. var wg sync.WaitGroup
  164. wg.Add(2)
  165. go func() {
  166. for _, v := range delVolsMap {
  167. if err := v.Destroy(); err != nil {
  168. errChain <- err
  169. }
  170. }
  171. wg.Done()
  172. }()
  173. go func() {
  174. for _, v := range delEcVolsMap {
  175. v.Destroy()
  176. }
  177. wg.Done()
  178. }()
  179. go func() {
  180. wg.Wait()
  181. close(errChain)
  182. }()
  183. errBuilder := strings.Builder{}
  184. for err := range errChain {
  185. errBuilder.WriteString(err.Error())
  186. errBuilder.WriteString("; ")
  187. }
  188. if errBuilder.Len() > 0 {
  189. e = fmt.Errorf(errBuilder.String())
  190. }
  191. return
  192. }
  193. func (l *DiskLocation) deleteVolumeById(vid needle.VolumeId) (found bool, e error) {
  194. v, ok := l.volumes[vid]
  195. if !ok {
  196. return
  197. }
  198. e = v.Destroy()
  199. if e != nil {
  200. return
  201. }
  202. found = true
  203. delete(l.volumes, vid)
  204. return
  205. }
  206. func (l *DiskLocation) LoadVolume(vid needle.VolumeId, needleMapKind NeedleMapKind) bool {
  207. if fileInfo, found := l.LocateVolume(vid); found {
  208. return l.loadExistingVolume(fileInfo, needleMapKind)
  209. }
  210. return false
  211. }
  212. func (l *DiskLocation) DeleteVolume(vid needle.VolumeId) error {
  213. l.volumesLock.Lock()
  214. defer l.volumesLock.Unlock()
  215. _, ok := l.volumes[vid]
  216. if !ok {
  217. return fmt.Errorf("Volume not found, VolumeId: %d", vid)
  218. }
  219. _, err := l.deleteVolumeById(vid)
  220. return err
  221. }
  222. func (l *DiskLocation) UnloadVolume(vid needle.VolumeId) error {
  223. l.volumesLock.Lock()
  224. defer l.volumesLock.Unlock()
  225. v, ok := l.volumes[vid]
  226. if !ok {
  227. return fmt.Errorf("Volume not loaded, VolumeId: %d", vid)
  228. }
  229. v.Close()
  230. delete(l.volumes, vid)
  231. return nil
  232. }
  233. func (l *DiskLocation) unmountVolumeByCollection(collectionName string) map[needle.VolumeId]*Volume {
  234. deltaVols := make(map[needle.VolumeId]*Volume, 0)
  235. for k, v := range l.volumes {
  236. if v.Collection == collectionName && !v.isCompacting {
  237. deltaVols[k] = v
  238. }
  239. }
  240. for k := range deltaVols {
  241. delete(l.volumes, k)
  242. }
  243. return deltaVols
  244. }
  245. func (l *DiskLocation) SetVolume(vid needle.VolumeId, volume *Volume) {
  246. l.volumesLock.Lock()
  247. defer l.volumesLock.Unlock()
  248. l.volumes[vid] = volume
  249. volume.location = l
  250. }
  251. func (l *DiskLocation) FindVolume(vid needle.VolumeId) (*Volume, bool) {
  252. l.volumesLock.RLock()
  253. defer l.volumesLock.RUnlock()
  254. v, ok := l.volumes[vid]
  255. return v, ok
  256. }
  257. func (l *DiskLocation) VolumesLen() int {
  258. l.volumesLock.RLock()
  259. defer l.volumesLock.RUnlock()
  260. return len(l.volumes)
  261. }
  262. func (l *DiskLocation) Close() {
  263. l.volumesLock.Lock()
  264. for _, v := range l.volumes {
  265. v.Close()
  266. }
  267. l.volumesLock.Unlock()
  268. l.ecVolumesLock.Lock()
  269. for _, ecVolume := range l.ecVolumes {
  270. ecVolume.Close()
  271. }
  272. l.ecVolumesLock.Unlock()
  273. return
  274. }
  275. func (l *DiskLocation) LocateVolume(vid needle.VolumeId) (os.FileInfo, bool) {
  276. if fileInfos, err := ioutil.ReadDir(l.Directory); err == nil {
  277. for _, fileInfo := range fileInfos {
  278. volId, _, err := volumeIdFromFileName(fileInfo.Name())
  279. if vid == volId && err == nil {
  280. return fileInfo, true
  281. }
  282. }
  283. }
  284. return nil, false
  285. }
  286. func (l *DiskLocation) UnUsedSpace(volumeSizeLimit uint64) (unUsedSpace uint64) {
  287. l.volumesLock.RLock()
  288. defer l.volumesLock.RUnlock()
  289. for _, vol := range l.volumes {
  290. if vol.IsReadOnly() {
  291. continue
  292. }
  293. datSize, idxSize, _ := vol.FileStat()
  294. unUsedSpace += volumeSizeLimit - (datSize + idxSize)
  295. }
  296. return
  297. }
  298. func (l *DiskLocation) CheckDiskSpace() {
  299. for {
  300. if dir, e := filepath.Abs(l.Directory); e == nil {
  301. s := stats.NewDiskStatus(dir)
  302. stats.VolumeServerResourceGauge.WithLabelValues(l.Directory, "all").Set(float64(s.All))
  303. stats.VolumeServerResourceGauge.WithLabelValues(l.Directory, "used").Set(float64(s.Used))
  304. stats.VolumeServerResourceGauge.WithLabelValues(l.Directory, "free").Set(float64(s.Free))
  305. if (s.PercentFree < l.MinFreeSpacePercent) != l.isDiskSpaceLow {
  306. l.isDiskSpaceLow = !l.isDiskSpaceLow
  307. }
  308. if l.isDiskSpaceLow {
  309. glog.V(0).Infof("dir %s freePercent %.2f%% < min %.2f%%, isLowDiskSpace: %v", dir, s.PercentFree, l.MinFreeSpacePercent, l.isDiskSpaceLow)
  310. } else {
  311. glog.V(4).Infof("dir %s freePercent %.2f%% < min %.2f%%, isLowDiskSpace: %v", dir, s.PercentFree, l.MinFreeSpacePercent, l.isDiskSpaceLow)
  312. }
  313. }
  314. time.Sleep(time.Minute)
  315. }
  316. }
  317. func (l *DiskLocation) GetDiskType() string {
  318. if l.DiskType == SsdType {
  319. return "SSD"
  320. }
  321. return "HDD"
  322. }