You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

354 lines
11 KiB

6 years ago
6 years ago
6 years ago
12 years ago
12 years ago
6 years ago
5 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
  1. package storage
  2. import (
  3. "fmt"
  4. "sync/atomic"
  5. "google.golang.org/grpc"
  6. "github.com/chrislusf/seaweedfs/weed/glog"
  7. "github.com/chrislusf/seaweedfs/weed/pb/master_pb"
  8. "github.com/chrislusf/seaweedfs/weed/stats"
  9. "github.com/chrislusf/seaweedfs/weed/storage/needle"
  10. . "github.com/chrislusf/seaweedfs/weed/storage/types"
  11. )
  12. const (
  13. MAX_TTL_VOLUME_REMOVAL_DELAY = 10 // 10 minutes
  14. )
  15. /*
  16. * A VolumeServer contains one Store
  17. */
  18. type Store struct {
  19. MasterAddress string
  20. grpcDialOption grpc.DialOption
  21. volumeSizeLimit uint64 //read from the master
  22. Ip string
  23. Port int
  24. PublicUrl string
  25. Locations []*DiskLocation
  26. dataCenter string //optional informaton, overwriting master setting if exists
  27. rack string //optional information, overwriting master setting if exists
  28. connected bool
  29. NeedleMapType NeedleMapType
  30. NewVolumesChan chan master_pb.VolumeShortInformationMessage
  31. DeletedVolumesChan chan master_pb.VolumeShortInformationMessage
  32. NewEcShardsChan chan master_pb.VolumeEcShardInformationMessage
  33. DeletedEcShardsChan chan master_pb.VolumeEcShardInformationMessage
  34. }
  35. func (s *Store) String() (str string) {
  36. str = fmt.Sprintf("Ip:%s, Port:%d, PublicUrl:%s, dataCenter:%s, rack:%s, connected:%v, volumeSizeLimit:%d", s.Ip, s.Port, s.PublicUrl, s.dataCenter, s.rack, s.connected, s.GetVolumeSizeLimit())
  37. return
  38. }
  39. func NewStore(grpcDialOption grpc.DialOption, port int, ip, publicUrl string, dirnames []string, maxVolumeCounts []int, needleMapKind NeedleMapType) (s *Store) {
  40. s = &Store{grpcDialOption: grpcDialOption, Port: port, Ip: ip, PublicUrl: publicUrl, NeedleMapType: needleMapKind}
  41. s.Locations = make([]*DiskLocation, 0)
  42. for i := 0; i < len(dirnames); i++ {
  43. location := NewDiskLocation(dirnames[i], maxVolumeCounts[i])
  44. location.loadExistingVolumes(needleMapKind)
  45. s.Locations = append(s.Locations, location)
  46. stats.VolumeServerMaxVolumeCounter.Add(float64(maxVolumeCounts[i]))
  47. }
  48. s.NewVolumesChan = make(chan master_pb.VolumeShortInformationMessage, 3)
  49. s.DeletedVolumesChan = make(chan master_pb.VolumeShortInformationMessage, 3)
  50. s.NewEcShardsChan = make(chan master_pb.VolumeEcShardInformationMessage, 3)
  51. s.DeletedEcShardsChan = make(chan master_pb.VolumeEcShardInformationMessage, 3)
  52. return
  53. }
  54. func (s *Store) AddVolume(volumeId needle.VolumeId, collection string, needleMapKind NeedleMapType, replicaPlacement string, ttlString string, preallocate int64, MemoryMapMaxSizeMb uint32) error {
  55. rt, e := NewReplicaPlacementFromString(replicaPlacement)
  56. if e != nil {
  57. return e
  58. }
  59. ttl, e := needle.ReadTTL(ttlString)
  60. if e != nil {
  61. return e
  62. }
  63. e = s.addVolume(volumeId, collection, needleMapKind, rt, ttl, preallocate, MemoryMapMaxSizeMb)
  64. return e
  65. }
  66. func (s *Store) DeleteCollection(collection string) (e error) {
  67. for _, location := range s.Locations {
  68. e = location.DeleteCollectionFromDiskLocation(collection)
  69. if e != nil {
  70. return
  71. }
  72. // let the heartbeat send the list of volumes, instead of sending the deleted volume ids to DeletedVolumesChan
  73. }
  74. return
  75. }
  76. func (s *Store) findVolume(vid needle.VolumeId) *Volume {
  77. for _, location := range s.Locations {
  78. if v, found := location.FindVolume(vid); found {
  79. return v
  80. }
  81. }
  82. return nil
  83. }
  84. func (s *Store) FindFreeLocation() (ret *DiskLocation) {
  85. max := 0
  86. for _, location := range s.Locations {
  87. currentFreeCount := location.MaxVolumeCount - location.VolumesLen()
  88. if currentFreeCount > max {
  89. max = currentFreeCount
  90. ret = location
  91. }
  92. }
  93. return ret
  94. }
  95. func (s *Store) addVolume(vid needle.VolumeId, collection string, needleMapKind NeedleMapType, replicaPlacement *ReplicaPlacement, ttl *needle.TTL, preallocate int64, memoryMapMaxSizeMb uint32) error {
  96. if s.findVolume(vid) != nil {
  97. return fmt.Errorf("Volume Id %d already exists!", vid)
  98. }
  99. if location := s.FindFreeLocation(); location != nil {
  100. glog.V(0).Infof("In dir %s adds volume:%v collection:%s replicaPlacement:%v ttl:%v",
  101. location.Directory, vid, collection, replicaPlacement, ttl)
  102. if volume, err := NewVolume(location.Directory, collection, vid, needleMapKind, replicaPlacement, ttl, preallocate, memoryMapMaxSizeMb); err == nil {
  103. location.SetVolume(vid, volume)
  104. glog.V(0).Infof("add volume %d", vid)
  105. s.NewVolumesChan <- master_pb.VolumeShortInformationMessage{
  106. Id: uint32(vid),
  107. Collection: collection,
  108. ReplicaPlacement: uint32(replicaPlacement.Byte()),
  109. Version: uint32(volume.Version()),
  110. Ttl: ttl.ToUint32(),
  111. }
  112. return nil
  113. } else {
  114. return err
  115. }
  116. }
  117. return fmt.Errorf("No more free space left")
  118. }
  119. func (s *Store) VolumeInfos() []*VolumeInfo {
  120. var stats []*VolumeInfo
  121. for _, location := range s.Locations {
  122. location.volumesLock.RLock()
  123. for k, v := range location.volumes {
  124. s := &VolumeInfo{
  125. Id: needle.VolumeId(k),
  126. Size: v.ContentSize(),
  127. Collection: v.Collection,
  128. ReplicaPlacement: v.ReplicaPlacement,
  129. Version: v.Version(),
  130. FileCount: int(v.FileCount()),
  131. DeleteCount: int(v.DeletedCount()),
  132. DeletedByteCount: v.DeletedSize(),
  133. ReadOnly: v.noWriteOrDelete || v.noWriteCanDelete,
  134. Ttl: v.Ttl,
  135. CompactRevision: uint32(v.CompactionRevision),
  136. }
  137. s.RemoteStorageName, s.RemoteStorageKey = v.RemoteStorageNameKey()
  138. stats = append(stats, s)
  139. }
  140. location.volumesLock.RUnlock()
  141. }
  142. sortVolumeInfos(stats)
  143. return stats
  144. }
  145. func (s *Store) SetDataCenter(dataCenter string) {
  146. s.dataCenter = dataCenter
  147. }
  148. func (s *Store) SetRack(rack string) {
  149. s.rack = rack
  150. }
  151. func (s *Store) CollectHeartbeat() *master_pb.Heartbeat {
  152. var volumeMessages []*master_pb.VolumeInformationMessage
  153. maxVolumeCount := 0
  154. var maxFileKey NeedleId
  155. collectionVolumeSize := make(map[string]uint64)
  156. for _, location := range s.Locations {
  157. var deleteVids []needle.VolumeId
  158. maxVolumeCount = maxVolumeCount + location.MaxVolumeCount
  159. location.volumesLock.RLock()
  160. for _, v := range location.volumes {
  161. if maxFileKey < v.MaxFileKey() {
  162. maxFileKey = v.MaxFileKey()
  163. }
  164. if !v.expired(s.GetVolumeSizeLimit()) {
  165. volumeMessages = append(volumeMessages, v.ToVolumeInformationMessage())
  166. } else {
  167. if v.expiredLongEnough(MAX_TTL_VOLUME_REMOVAL_DELAY) {
  168. deleteVids = append(deleteVids, v.Id)
  169. } else {
  170. glog.V(0).Infoln("volume", v.Id, "is expired.")
  171. }
  172. }
  173. fileSize, _, _ := v.FileStat()
  174. collectionVolumeSize[v.Collection] += fileSize
  175. }
  176. location.volumesLock.RUnlock()
  177. if len(deleteVids) > 0 {
  178. // delete expired volumes.
  179. location.volumesLock.Lock()
  180. for _, vid := range deleteVids {
  181. location.deleteVolumeById(vid)
  182. glog.V(0).Infoln("volume", vid, "is deleted.")
  183. }
  184. location.volumesLock.Unlock()
  185. }
  186. }
  187. for col, size := range collectionVolumeSize {
  188. stats.VolumeServerDiskSizeGauge.WithLabelValues(col, "normal").Set(float64(size))
  189. }
  190. return &master_pb.Heartbeat{
  191. Ip: s.Ip,
  192. Port: uint32(s.Port),
  193. PublicUrl: s.PublicUrl,
  194. MaxVolumeCount: uint32(maxVolumeCount),
  195. MaxFileKey: NeedleIdToUint64(maxFileKey),
  196. DataCenter: s.dataCenter,
  197. Rack: s.rack,
  198. Volumes: volumeMessages,
  199. HasNoVolumes: len(volumeMessages) == 0,
  200. }
  201. }
  202. func (s *Store) Close() {
  203. for _, location := range s.Locations {
  204. location.Close()
  205. }
  206. }
  207. func (s *Store) WriteVolumeNeedle(i needle.VolumeId, n *needle.Needle) (size uint32, isUnchanged bool, err error) {
  208. if v := s.findVolume(i); v != nil {
  209. if v.noWriteOrDelete || v.noWriteCanDelete {
  210. err = fmt.Errorf("volume %d is read only", i)
  211. return
  212. }
  213. if MaxPossibleVolumeSize >= v.ContentSize()+uint64(needle.GetActualSize(size, v.version)) {
  214. _, size, isUnchanged, err = v.writeNeedle(n)
  215. } else {
  216. err = fmt.Errorf("volume size limit %d exceeded! current size is %d", s.GetVolumeSizeLimit(), v.ContentSize())
  217. }
  218. return
  219. }
  220. glog.V(0).Infoln("volume", i, "not found!")
  221. err = fmt.Errorf("volume %d not found on %s:%d", i, s.Ip, s.Port)
  222. return
  223. }
  224. func (s *Store) DeleteVolumeNeedle(i needle.VolumeId, n *needle.Needle) (uint32, error) {
  225. if v := s.findVolume(i); v != nil {
  226. if v.noWriteOrDelete {
  227. return 0, fmt.Errorf("volume %d is read only", i)
  228. }
  229. if MaxPossibleVolumeSize >= v.ContentSize()+uint64(needle.GetActualSize(0, v.version)) {
  230. return v.deleteNeedle(n)
  231. } else {
  232. return 0, fmt.Errorf("volume size limit %d exceeded! current size is %d", s.GetVolumeSizeLimit(), v.ContentSize())
  233. }
  234. }
  235. return 0, fmt.Errorf("volume %d not found on %s:%d", i, s.Ip, s.Port)
  236. }
  237. func (s *Store) ReadVolumeNeedle(i needle.VolumeId, n *needle.Needle) (int, error) {
  238. if v := s.findVolume(i); v != nil {
  239. return v.readNeedle(n)
  240. }
  241. return 0, fmt.Errorf("volume %d not found", i)
  242. }
  243. func (s *Store) GetVolume(i needle.VolumeId) *Volume {
  244. return s.findVolume(i)
  245. }
  246. func (s *Store) HasVolume(i needle.VolumeId) bool {
  247. v := s.findVolume(i)
  248. return v != nil
  249. }
  250. func (s *Store) MarkVolumeReadonly(i needle.VolumeId) error {
  251. v := s.findVolume(i)
  252. if v == nil {
  253. return fmt.Errorf("volume %d not found", i)
  254. }
  255. v.noWriteOrDelete = true
  256. return nil
  257. }
  258. func (s *Store) MountVolume(i needle.VolumeId) error {
  259. for _, location := range s.Locations {
  260. if found := location.LoadVolume(i, s.NeedleMapType); found == true {
  261. glog.V(0).Infof("mount volume %d", i)
  262. v := s.findVolume(i)
  263. s.NewVolumesChan <- master_pb.VolumeShortInformationMessage{
  264. Id: uint32(v.Id),
  265. Collection: v.Collection,
  266. ReplicaPlacement: uint32(v.ReplicaPlacement.Byte()),
  267. Version: uint32(v.Version()),
  268. Ttl: v.Ttl.ToUint32(),
  269. }
  270. return nil
  271. }
  272. }
  273. return fmt.Errorf("volume %d not found on disk", i)
  274. }
  275. func (s *Store) UnmountVolume(i needle.VolumeId) error {
  276. v := s.findVolume(i)
  277. if v == nil {
  278. return nil
  279. }
  280. message := master_pb.VolumeShortInformationMessage{
  281. Id: uint32(v.Id),
  282. Collection: v.Collection,
  283. ReplicaPlacement: uint32(v.ReplicaPlacement.Byte()),
  284. Version: uint32(v.Version()),
  285. Ttl: v.Ttl.ToUint32(),
  286. }
  287. for _, location := range s.Locations {
  288. if err := location.UnloadVolume(i); err == nil {
  289. glog.V(0).Infof("UnmountVolume %d", i)
  290. s.DeletedVolumesChan <- message
  291. return nil
  292. }
  293. }
  294. return fmt.Errorf("volume %d not found on disk", i)
  295. }
  296. func (s *Store) DeleteVolume(i needle.VolumeId) error {
  297. v := s.findVolume(i)
  298. if v == nil {
  299. return nil
  300. }
  301. message := master_pb.VolumeShortInformationMessage{
  302. Id: uint32(v.Id),
  303. Collection: v.Collection,
  304. ReplicaPlacement: uint32(v.ReplicaPlacement.Byte()),
  305. Version: uint32(v.Version()),
  306. Ttl: v.Ttl.ToUint32(),
  307. }
  308. for _, location := range s.Locations {
  309. if error := location.deleteVolumeById(i); error == nil {
  310. glog.V(0).Infof("DeleteVolume %d", i)
  311. s.DeletedVolumesChan <- message
  312. return nil
  313. }
  314. }
  315. return fmt.Errorf("volume %d not found on disk", i)
  316. }
  317. func (s *Store) SetVolumeSizeLimit(x uint64) {
  318. atomic.StoreUint64(&s.volumeSizeLimit, x)
  319. }
  320. func (s *Store) GetVolumeSizeLimit() uint64 {
  321. return atomic.LoadUint64(&s.volumeSizeLimit)
  322. }