You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

410 lines
14 KiB

6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
  1. package storage
  2. import (
  3. "context"
  4. "fmt"
  5. "io"
  6. "sort"
  7. "sync"
  8. "time"
  9. "github.com/joeslay/seaweedfs/weed/glog"
  10. "github.com/joeslay/seaweedfs/weed/operation"
  11. "github.com/joeslay/seaweedfs/weed/pb/master_pb"
  12. "github.com/joeslay/seaweedfs/weed/pb/volume_server_pb"
  13. "github.com/joeslay/seaweedfs/weed/stats"
  14. "github.com/joeslay/seaweedfs/weed/storage/erasure_coding"
  15. "github.com/joeslay/seaweedfs/weed/storage/needle"
  16. "github.com/joeslay/seaweedfs/weed/storage/types"
  17. "github.com/klauspost/reedsolomon"
  18. )
  19. func (s *Store) CollectErasureCodingHeartbeat() *master_pb.Heartbeat {
  20. var ecShardMessages []*master_pb.VolumeEcShardInformationMessage
  21. collectionEcShardSize := make(map[string]int64)
  22. for _, location := range s.Locations {
  23. location.ecVolumesLock.RLock()
  24. for _, ecShards := range location.ecVolumes {
  25. ecShardMessages = append(ecShardMessages, ecShards.ToVolumeEcShardInformationMessage()...)
  26. for _, ecShard := range ecShards.Shards {
  27. collectionEcShardSize[ecShards.Collection] += ecShard.Size()
  28. }
  29. }
  30. location.ecVolumesLock.RUnlock()
  31. }
  32. for col, size := range collectionEcShardSize {
  33. stats.VolumeServerDiskSizeGauge.WithLabelValues(col, "ec").Set(float64(size))
  34. }
  35. return &master_pb.Heartbeat{
  36. EcShards: ecShardMessages,
  37. HasNoEcShards: len(ecShardMessages) == 0,
  38. }
  39. }
  40. func (s *Store) MountEcShards(collection string, vid needle.VolumeId, shardId erasure_coding.ShardId) error {
  41. for _, location := range s.Locations {
  42. if err := location.LoadEcShard(collection, vid, shardId); err == nil {
  43. glog.V(0).Infof("MountEcShards %d.%d", vid, shardId)
  44. var shardBits erasure_coding.ShardBits
  45. s.NewEcShardsChan <- master_pb.VolumeEcShardInformationMessage{
  46. Id: uint32(vid),
  47. Collection: collection,
  48. EcIndexBits: uint32(shardBits.AddShardId(shardId)),
  49. }
  50. return nil
  51. } else {
  52. return fmt.Errorf("%s load ec shard %d.%d: %v", location.Directory, vid, shardId, err)
  53. }
  54. }
  55. return fmt.Errorf("MountEcShards %d.%d not found on disk", vid, shardId)
  56. }
  57. func (s *Store) UnmountEcShards(vid needle.VolumeId, shardId erasure_coding.ShardId) error {
  58. ecShard, found := s.findEcShard(vid, shardId)
  59. if !found {
  60. return nil
  61. }
  62. var shardBits erasure_coding.ShardBits
  63. message := master_pb.VolumeEcShardInformationMessage{
  64. Id: uint32(vid),
  65. Collection: ecShard.Collection,
  66. EcIndexBits: uint32(shardBits.AddShardId(shardId)),
  67. }
  68. for _, location := range s.Locations {
  69. if deleted := location.UnloadEcShard(vid, shardId); deleted {
  70. glog.V(0).Infof("UnmountEcShards %d.%d", vid, shardId)
  71. s.DeletedEcShardsChan <- message
  72. return nil
  73. }
  74. }
  75. return fmt.Errorf("UnmountEcShards %d.%d not found on disk", vid, shardId)
  76. }
  77. func (s *Store) findEcShard(vid needle.VolumeId, shardId erasure_coding.ShardId) (*erasure_coding.EcVolumeShard, bool) {
  78. for _, location := range s.Locations {
  79. if v, found := location.FindEcShard(vid, shardId); found {
  80. return v, found
  81. }
  82. }
  83. return nil, false
  84. }
  85. func (s *Store) FindEcVolume(vid needle.VolumeId) (*erasure_coding.EcVolume, bool) {
  86. for _, location := range s.Locations {
  87. if s, found := location.FindEcVolume(vid); found {
  88. return s, true
  89. }
  90. }
  91. return nil, false
  92. }
  93. func (s *Store) DestroyEcVolume(vid needle.VolumeId) {
  94. for _, location := range s.Locations {
  95. location.DestroyEcVolume(vid)
  96. }
  97. }
  98. func (s *Store) ReadEcShardNeedle(ctx context.Context, vid needle.VolumeId, n *needle.Needle) (int, error) {
  99. for _, location := range s.Locations {
  100. if localEcVolume, found := location.FindEcVolume(vid); found {
  101. // read the volume version
  102. for localEcVolume.Version == 0 {
  103. err := s.readEcVolumeVersion(ctx, vid, localEcVolume)
  104. time.Sleep(1357 * time.Millisecond)
  105. glog.V(0).Infof("ReadEcShardNeedle vid %d version:%v: %v", vid, localEcVolume.Version, err)
  106. }
  107. version := localEcVolume.Version
  108. offset, size, intervals, err := localEcVolume.LocateEcShardNeedle(n.Id, version)
  109. if err != nil {
  110. return 0, fmt.Errorf("locate in local ec volume: %v", err)
  111. }
  112. if size == types.TombstoneFileSize {
  113. return 0, fmt.Errorf("entry %s is deleted", n.Id)
  114. }
  115. glog.V(3).Infof("read ec volume %d offset %d size %d intervals:%+v", vid, offset.ToAcutalOffset(), size, intervals)
  116. if len(intervals) > 1 {
  117. glog.V(3).Infof("ReadEcShardNeedle needle id %s intervals:%+v", n.String(), intervals)
  118. }
  119. bytes, isDeleted, err := s.readEcShardIntervals(ctx, vid, n.Id, localEcVolume, intervals)
  120. if err != nil {
  121. return 0, fmt.Errorf("ReadEcShardIntervals: %v", err)
  122. }
  123. if isDeleted {
  124. return 0, fmt.Errorf("ec entry %s is deleted", n.Id)
  125. }
  126. err = n.ReadBytes(bytes, offset.ToAcutalOffset(), size, version)
  127. if err != nil {
  128. return 0, fmt.Errorf("readbytes: %v", err)
  129. }
  130. return len(bytes), nil
  131. }
  132. }
  133. return 0, fmt.Errorf("ec shard %d not found", vid)
  134. }
  135. func (s *Store) readEcVolumeVersion(ctx context.Context, vid needle.VolumeId, ecVolume *erasure_coding.EcVolume) (err error) {
  136. interval := erasure_coding.Interval{
  137. BlockIndex: 0,
  138. InnerBlockOffset: 0,
  139. Size: _SuperBlockSize,
  140. IsLargeBlock: true, // it could be large block, but ok in this place
  141. LargeBlockRowsCount: 0,
  142. }
  143. data, _, err := s.readEcShardIntervals(ctx, vid, 0, ecVolume, []erasure_coding.Interval{interval})
  144. if err == nil {
  145. ecVolume.Version = needle.Version(data[0])
  146. }
  147. return
  148. }
  149. func (s *Store) readEcShardIntervals(ctx context.Context, vid needle.VolumeId, needleId types.NeedleId, ecVolume *erasure_coding.EcVolume, intervals []erasure_coding.Interval) (data []byte, is_deleted bool, err error) {
  150. if err = s.cachedLookupEcShardLocations(ctx, ecVolume); err != nil {
  151. return nil, false, fmt.Errorf("failed to locate shard via master grpc %s: %v", s.MasterAddress, err)
  152. }
  153. for i, interval := range intervals {
  154. if d, isDeleted, e := s.readOneEcShardInterval(ctx, needleId, ecVolume, interval); e != nil {
  155. return nil, isDeleted, e
  156. } else {
  157. if isDeleted {
  158. is_deleted = true
  159. }
  160. if i == 0 {
  161. data = d
  162. } else {
  163. data = append(data, d...)
  164. }
  165. }
  166. }
  167. return
  168. }
  169. func (s *Store) readOneEcShardInterval(ctx context.Context, needleId types.NeedleId, ecVolume *erasure_coding.EcVolume, interval erasure_coding.Interval) (data []byte, is_deleted bool, err error) {
  170. shardId, actualOffset := interval.ToShardIdAndOffset(erasure_coding.ErasureCodingLargeBlockSize, erasure_coding.ErasureCodingSmallBlockSize)
  171. data = make([]byte, interval.Size)
  172. if shard, found := ecVolume.FindEcVolumeShard(shardId); found {
  173. if _, err = shard.ReadAt(data, actualOffset); err != nil {
  174. glog.V(0).Infof("read local ec shard %d.%d: %v", ecVolume.VolumeId, shardId, err)
  175. return
  176. }
  177. } else {
  178. ecVolume.ShardLocationsLock.RLock()
  179. sourceDataNodes, hasShardIdLocation := ecVolume.ShardLocations[shardId]
  180. ecVolume.ShardLocationsLock.RUnlock()
  181. // try reading directly
  182. if hasShardIdLocation {
  183. _, is_deleted, err = s.readRemoteEcShardInterval(ctx, sourceDataNodes, needleId, ecVolume.VolumeId, shardId, data, actualOffset)
  184. if err == nil {
  185. return
  186. }
  187. glog.V(0).Infof("clearing ec shard %d.%d locations: %v", ecVolume.VolumeId, shardId, err)
  188. forgetShardId(ecVolume, shardId)
  189. }
  190. // try reading by recovering from other shards
  191. _, is_deleted, err = s.recoverOneRemoteEcShardInterval(ctx, needleId, ecVolume, shardId, data, actualOffset)
  192. if err == nil {
  193. return
  194. }
  195. glog.V(0).Infof("recover ec shard %d.%d : %v", ecVolume.VolumeId, shardId, err)
  196. }
  197. return
  198. }
  199. func forgetShardId(ecVolume *erasure_coding.EcVolume, shardId erasure_coding.ShardId) {
  200. // failed to access the source data nodes, clear it up
  201. ecVolume.ShardLocationsLock.Lock()
  202. delete(ecVolume.ShardLocations, shardId)
  203. ecVolume.ShardLocationsLock.Unlock()
  204. }
  205. func (s *Store) cachedLookupEcShardLocations(ctx context.Context, ecVolume *erasure_coding.EcVolume) (err error) {
  206. shardCount := len(ecVolume.ShardLocations)
  207. if shardCount < erasure_coding.DataShardsCount &&
  208. ecVolume.ShardLocationsRefreshTime.Add(11*time.Second).After(time.Now()) ||
  209. shardCount == erasure_coding.TotalShardsCount &&
  210. ecVolume.ShardLocationsRefreshTime.Add(37*time.Minute).After(time.Now()) ||
  211. shardCount >= erasure_coding.DataShardsCount &&
  212. ecVolume.ShardLocationsRefreshTime.Add(7*time.Minute).After(time.Now()) {
  213. // still fresh
  214. return nil
  215. }
  216. glog.V(3).Infof("lookup and cache ec volume %d locations", ecVolume.VolumeId)
  217. err = operation.WithMasterServerClient(s.MasterAddress, s.grpcDialOption, func(masterClient master_pb.SeaweedClient) error {
  218. req := &master_pb.LookupEcVolumeRequest{
  219. VolumeId: uint32(ecVolume.VolumeId),
  220. }
  221. resp, err := masterClient.LookupEcVolume(ctx, req)
  222. if err != nil {
  223. return fmt.Errorf("lookup ec volume %d: %v", ecVolume.VolumeId, err)
  224. }
  225. if len(resp.ShardIdLocations) < erasure_coding.DataShardsCount {
  226. return fmt.Errorf("only %d shards found but %d required", len(resp.ShardIdLocations), erasure_coding.DataShardsCount)
  227. }
  228. ecVolume.ShardLocationsLock.Lock()
  229. for _, shardIdLocations := range resp.ShardIdLocations {
  230. shardId := erasure_coding.ShardId(shardIdLocations.ShardId)
  231. delete(ecVolume.ShardLocations, shardId)
  232. for _, loc := range shardIdLocations.Locations {
  233. ecVolume.ShardLocations[shardId] = append(ecVolume.ShardLocations[shardId], loc.Url)
  234. }
  235. }
  236. ecVolume.ShardLocationsRefreshTime = time.Now()
  237. ecVolume.ShardLocationsLock.Unlock()
  238. return nil
  239. })
  240. return
  241. }
  242. func (s *Store) readRemoteEcShardInterval(ctx context.Context, sourceDataNodes []string, needleId types.NeedleId, vid needle.VolumeId, shardId erasure_coding.ShardId, buf []byte, offset int64) (n int, is_deleted bool, err error) {
  243. if len(sourceDataNodes) == 0 {
  244. return 0, false, fmt.Errorf("failed to find ec shard %d.%d", vid, shardId)
  245. }
  246. for _, sourceDataNode := range sourceDataNodes {
  247. glog.V(4).Infof("read remote ec shard %d.%d from %s", vid, shardId, sourceDataNode)
  248. n, is_deleted, err = s.doReadRemoteEcShardInterval(ctx, sourceDataNode, needleId, vid, shardId, buf, offset)
  249. if err == nil {
  250. return
  251. }
  252. glog.V(1).Infof("read remote ec shard %d.%d from %s: %v", vid, shardId, sourceDataNode, err)
  253. }
  254. return
  255. }
  256. func (s *Store) doReadRemoteEcShardInterval(ctx context.Context, sourceDataNode string, needleId types.NeedleId, vid needle.VolumeId, shardId erasure_coding.ShardId, buf []byte, offset int64) (n int, is_deleted bool, err error) {
  257. err = operation.WithVolumeServerClient(sourceDataNode, s.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
  258. // copy data slice
  259. shardReadClient, err := client.VolumeEcShardRead(ctx, &volume_server_pb.VolumeEcShardReadRequest{
  260. VolumeId: uint32(vid),
  261. ShardId: uint32(shardId),
  262. Offset: offset,
  263. Size: int64(len(buf)),
  264. FileKey: uint64(needleId),
  265. })
  266. if err != nil {
  267. return fmt.Errorf("failed to start reading ec shard %d.%d from %s: %v", vid, shardId, sourceDataNode, err)
  268. }
  269. for {
  270. resp, receiveErr := shardReadClient.Recv()
  271. if receiveErr == io.EOF {
  272. break
  273. }
  274. if receiveErr != nil {
  275. return fmt.Errorf("receiving ec shard %d.%d from %s: %v", vid, shardId, sourceDataNode, err)
  276. }
  277. if resp.IsDeleted {
  278. is_deleted = true
  279. }
  280. copy(buf[n:n+len(resp.Data)], resp.Data)
  281. n += len(resp.Data)
  282. }
  283. return nil
  284. })
  285. if err != nil {
  286. return 0, is_deleted, fmt.Errorf("read ec shard %d.%d from %s: %v", vid, shardId, sourceDataNode, err)
  287. }
  288. return
  289. }
  290. func (s *Store) recoverOneRemoteEcShardInterval(ctx context.Context, needleId types.NeedleId, ecVolume *erasure_coding.EcVolume, shardIdToRecover erasure_coding.ShardId, buf []byte, offset int64) (n int, is_deleted bool, err error) {
  291. glog.V(4).Infof("recover ec shard %d.%d from other locations", ecVolume.VolumeId, shardIdToRecover)
  292. enc, err := reedsolomon.New(erasure_coding.DataShardsCount, erasure_coding.ParityShardsCount)
  293. if err != nil {
  294. return 0, false, fmt.Errorf("failed to create encoder: %v", err)
  295. }
  296. bufs := make([][]byte, erasure_coding.TotalShardsCount)
  297. var wg sync.WaitGroup
  298. ecVolume.ShardLocationsLock.RLock()
  299. for shardId, locations := range ecVolume.ShardLocations {
  300. // skip currnent shard or empty shard
  301. if shardId == shardIdToRecover {
  302. continue
  303. }
  304. if len(locations) == 0 {
  305. glog.V(3).Infof("readRemoteEcShardInterval missing %d.%d from %+v", ecVolume.VolumeId, shardId, locations)
  306. continue
  307. }
  308. // read from remote locations
  309. wg.Add(1)
  310. go func(shardId erasure_coding.ShardId, locations []string) {
  311. defer wg.Done()
  312. data := make([]byte, len(buf))
  313. nRead, isDeleted, readErr := s.readRemoteEcShardInterval(ctx, locations, needleId, ecVolume.VolumeId, shardId, data, offset)
  314. if readErr != nil {
  315. glog.V(3).Infof("recover: readRemoteEcShardInterval %d.%d %d bytes from %+v: %v", ecVolume.VolumeId, shardId, nRead, locations, readErr)
  316. forgetShardId(ecVolume, shardId)
  317. }
  318. if isDeleted {
  319. is_deleted = true
  320. }
  321. if nRead == len(buf) {
  322. bufs[shardId] = data
  323. }
  324. }(shardId, locations)
  325. }
  326. ecVolume.ShardLocationsLock.RUnlock()
  327. wg.Wait()
  328. if err = enc.ReconstructData(bufs); err != nil {
  329. glog.V(3).Infof("recovered ec shard %d.%d failed: %v", ecVolume.VolumeId, shardIdToRecover, err)
  330. return 0, false, err
  331. }
  332. glog.V(4).Infof("recovered ec shard %d.%d from other locations", ecVolume.VolumeId, shardIdToRecover)
  333. copy(buf, bufs[shardIdToRecover])
  334. return len(buf), is_deleted, nil
  335. }
  336. func (s *Store) EcVolumes() (ecVolumes []*erasure_coding.EcVolume) {
  337. for _, location := range s.Locations {
  338. location.ecVolumesLock.RLock()
  339. for _, v := range location.ecVolumes {
  340. ecVolumes = append(ecVolumes, v)
  341. }
  342. location.ecVolumesLock.RUnlock()
  343. }
  344. sort.Slice(ecVolumes, func(i, j int) bool {
  345. return ecVolumes[i].VolumeId > ecVolumes[j].VolumeId
  346. })
  347. return ecVolumes
  348. }