You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

366 lines
11 KiB

8 months ago
4 months ago
6 years ago
6 years ago
3 years ago
6 years ago
  1. package weed_server
  2. import (
  3. "context"
  4. "fmt"
  5. "math"
  6. "math/rand/v2"
  7. "strings"
  8. "sync"
  9. "time"
  10. "github.com/seaweedfs/seaweedfs/weed/stats"
  11. "github.com/seaweedfs/seaweedfs/weed/topology"
  12. "github.com/seaweedfs/raft"
  13. "github.com/seaweedfs/seaweedfs/weed/glog"
  14. "github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
  15. "github.com/seaweedfs/seaweedfs/weed/security"
  16. "github.com/seaweedfs/seaweedfs/weed/storage/needle"
  17. "github.com/seaweedfs/seaweedfs/weed/storage/super_block"
  18. "github.com/seaweedfs/seaweedfs/weed/storage/types"
  19. )
  20. const (
  21. volumeGrowStepCount = 2
  22. )
  23. func (ms *MasterServer) DoAutomaticVolumeGrow(req *topology.VolumeGrowRequest) {
  24. glog.V(1).Infoln("starting automatic volume grow")
  25. start := time.Now()
  26. newVidLocations, err := ms.vg.AutomaticGrowByType(req.Option, ms.grpcDialOption, ms.Topo, req.Count)
  27. glog.V(1).Infoln("finished automatic volume grow, cost ", time.Now().Sub(start))
  28. if err != nil {
  29. glog.V(1).Infof("automatic volume grow failed: %+v", err)
  30. return
  31. }
  32. for _, newVidLocation := range newVidLocations {
  33. ms.broadcastToClients(&master_pb.KeepConnectedResponse{VolumeLocation: newVidLocation})
  34. }
  35. }
  36. func (ms *MasterServer) ProcessGrowRequest() {
  37. go func() {
  38. ctx := context.Background()
  39. firstRun := true
  40. for {
  41. if firstRun {
  42. firstRun = false
  43. } else {
  44. time.Sleep(5*time.Minute + time.Duration(30*rand.Float32())*time.Second)
  45. }
  46. if !ms.Topo.IsLeader() {
  47. continue
  48. }
  49. dcs := ms.Topo.ListDCAndRacks()
  50. var err error
  51. for _, vlc := range ms.Topo.ListVolumeLayoutCollections() {
  52. vl := vlc.VolumeLayout
  53. lastGrowCount := vl.GetLastGrowCount()
  54. if vl.HasGrowRequest() {
  55. continue
  56. }
  57. writable, crowded := vl.GetWritableVolumeCount()
  58. mustGrow := int(lastGrowCount) - writable
  59. vgr := vlc.ToVolumeGrowRequest()
  60. stats.MasterVolumeLayoutWritable.WithLabelValues(vlc.Collection, vgr.DiskType, vgr.Replication, vgr.Ttl).Set(float64(writable))
  61. stats.MasterVolumeLayoutCrowded.WithLabelValues(vlc.Collection, vgr.DiskType, vgr.Replication, vgr.Ttl).Set(float64(crowded))
  62. switch {
  63. case mustGrow > 0:
  64. vgr.WritableVolumeCount = uint32(mustGrow)
  65. _, err = ms.VolumeGrow(ctx, vgr)
  66. case lastGrowCount > 0 && writable < int(lastGrowCount*2) && float64(crowded+volumeGrowStepCount) > float64(writable)*topology.VolumeGrowStrategy.Threshold:
  67. vgr.WritableVolumeCount = volumeGrowStepCount
  68. _, err = ms.VolumeGrow(ctx, vgr)
  69. }
  70. if err != nil {
  71. glog.V(0).Infof("volume grow request failed: %+v", err)
  72. }
  73. writableVolumes := vl.CloneWritableVolumes()
  74. for dcId, racks := range dcs {
  75. for _, rackId := range racks {
  76. if vl.ShouldGrowVolumesByDcAndRack(&writableVolumes, dcId, rackId) {
  77. vgr.DataCenter = string(dcId)
  78. vgr.Rack = string(rackId)
  79. if lastGrowCount > 0 {
  80. vgr.WritableVolumeCount = uint32(math.Ceil(float64(lastGrowCount) / float64(len(dcs)*len(racks))))
  81. } else {
  82. vgr.WritableVolumeCount = volumeGrowStepCount
  83. }
  84. if _, err = ms.VolumeGrow(ctx, vgr); err != nil {
  85. glog.V(0).Infof("volume grow request for dc:%s rack:%s failed: %+v", dcId, rackId, err)
  86. }
  87. }
  88. }
  89. }
  90. }
  91. }
  92. }()
  93. go func() {
  94. filter := sync.Map{}
  95. for {
  96. req, ok := <-ms.volumeGrowthRequestChan
  97. if !ok {
  98. break
  99. }
  100. option := req.Option
  101. vl := ms.Topo.GetVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl, option.DiskType)
  102. if !ms.Topo.IsLeader() {
  103. //discard buffered requests
  104. time.Sleep(time.Second * 1)
  105. vl.DoneGrowRequest()
  106. continue
  107. }
  108. // filter out identical requests being processed
  109. found := false
  110. filter.Range(func(k, v interface{}) bool {
  111. existingReq := k.(*topology.VolumeGrowRequest)
  112. if existingReq.Equals(req) {
  113. found = true
  114. }
  115. return !found
  116. })
  117. // not atomic but it's okay
  118. if found || (!req.Force && !vl.ShouldGrowVolumes()) {
  119. glog.V(4).Infoln("discard volume grow request")
  120. time.Sleep(time.Millisecond * 211)
  121. vl.DoneGrowRequest()
  122. continue
  123. }
  124. filter.Store(req, nil)
  125. // we have lock called inside vg
  126. glog.V(0).Infof("volume grow %+v", req)
  127. go func(req *topology.VolumeGrowRequest, vl *topology.VolumeLayout) {
  128. ms.DoAutomaticVolumeGrow(req)
  129. vl.DoneGrowRequest()
  130. filter.Delete(req)
  131. }(req, vl)
  132. }
  133. }()
  134. }
  135. func (ms *MasterServer) LookupVolume(ctx context.Context, req *master_pb.LookupVolumeRequest) (*master_pb.LookupVolumeResponse, error) {
  136. resp := &master_pb.LookupVolumeResponse{}
  137. volumeLocations := ms.lookupVolumeId(req.VolumeOrFileIds, req.Collection)
  138. for _, volumeOrFileId := range req.VolumeOrFileIds {
  139. vid := volumeOrFileId
  140. commaSep := strings.Index(vid, ",")
  141. if commaSep > 0 {
  142. vid = vid[0:commaSep]
  143. }
  144. if result, found := volumeLocations[vid]; found {
  145. var locations []*master_pb.Location
  146. for _, loc := range result.Locations {
  147. locations = append(locations, &master_pb.Location{
  148. Url: loc.Url,
  149. PublicUrl: loc.PublicUrl,
  150. DataCenter: loc.DataCenter,
  151. GrpcPort: uint32(loc.GrpcPort),
  152. })
  153. }
  154. var auth string
  155. if commaSep > 0 { // this is a file id
  156. auth = string(security.GenJwtForVolumeServer(ms.guard.SigningKey, ms.guard.ExpiresAfterSec, result.VolumeOrFileId))
  157. }
  158. resp.VolumeIdLocations = append(resp.VolumeIdLocations, &master_pb.LookupVolumeResponse_VolumeIdLocation{
  159. VolumeOrFileId: result.VolumeOrFileId,
  160. Locations: locations,
  161. Error: result.Error,
  162. Auth: auth,
  163. })
  164. }
  165. }
  166. return resp, nil
  167. }
  168. func (ms *MasterServer) Statistics(ctx context.Context, req *master_pb.StatisticsRequest) (*master_pb.StatisticsResponse, error) {
  169. if !ms.Topo.IsLeader() {
  170. return nil, raft.NotLeaderError
  171. }
  172. if req.Replication == "" {
  173. req.Replication = ms.option.DefaultReplicaPlacement
  174. }
  175. replicaPlacement, err := super_block.NewReplicaPlacementFromString(req.Replication)
  176. if err != nil {
  177. return nil, err
  178. }
  179. ttl, err := needle.ReadTTL(req.Ttl)
  180. if err != nil {
  181. return nil, err
  182. }
  183. volumeLayout := ms.Topo.GetVolumeLayout(req.Collection, replicaPlacement, ttl, types.ToDiskType(req.DiskType))
  184. stats := volumeLayout.Stats()
  185. totalSize := ms.Topo.GetDiskUsages().GetMaxVolumeCount() * int64(ms.option.VolumeSizeLimitMB) * 1024 * 1024
  186. resp := &master_pb.StatisticsResponse{
  187. TotalSize: uint64(totalSize),
  188. UsedSize: stats.UsedSize,
  189. FileCount: stats.FileCount,
  190. }
  191. return resp, nil
  192. }
  193. func (ms *MasterServer) VolumeList(ctx context.Context, req *master_pb.VolumeListRequest) (*master_pb.VolumeListResponse, error) {
  194. if !ms.Topo.IsLeader() {
  195. return nil, raft.NotLeaderError
  196. }
  197. resp := &master_pb.VolumeListResponse{
  198. TopologyInfo: ms.Topo.ToTopologyInfo(),
  199. VolumeSizeLimitMb: uint64(ms.option.VolumeSizeLimitMB),
  200. }
  201. return resp, nil
  202. }
  203. func (ms *MasterServer) LookupEcVolume(ctx context.Context, req *master_pb.LookupEcVolumeRequest) (*master_pb.LookupEcVolumeResponse, error) {
  204. if !ms.Topo.IsLeader() {
  205. return nil, raft.NotLeaderError
  206. }
  207. resp := &master_pb.LookupEcVolumeResponse{}
  208. ecLocations, found := ms.Topo.LookupEcShards(needle.VolumeId(req.VolumeId))
  209. if !found {
  210. return resp, fmt.Errorf("ec volume %d not found", req.VolumeId)
  211. }
  212. resp.VolumeId = req.VolumeId
  213. for shardId, shardLocations := range ecLocations.Locations {
  214. var locations []*master_pb.Location
  215. for _, dn := range shardLocations {
  216. locations = append(locations, &master_pb.Location{
  217. Url: string(dn.Id()),
  218. PublicUrl: dn.PublicUrl,
  219. DataCenter: dn.GetDataCenterId(),
  220. })
  221. }
  222. resp.ShardIdLocations = append(resp.ShardIdLocations, &master_pb.LookupEcVolumeResponse_EcShardIdLocation{
  223. ShardId: uint32(shardId),
  224. Locations: locations,
  225. })
  226. }
  227. return resp, nil
  228. }
  229. func (ms *MasterServer) VacuumVolume(ctx context.Context, req *master_pb.VacuumVolumeRequest) (*master_pb.VacuumVolumeResponse, error) {
  230. if !ms.Topo.IsLeader() {
  231. return nil, raft.NotLeaderError
  232. }
  233. resp := &master_pb.VacuumVolumeResponse{}
  234. ms.Topo.Vacuum(ms.grpcDialOption, float64(req.GarbageThreshold), ms.option.MaxParallelVacuumPerServer, req.VolumeId, req.Collection, ms.preallocateSize, false)
  235. return resp, nil
  236. }
  237. func (ms *MasterServer) DisableVacuum(ctx context.Context, req *master_pb.DisableVacuumRequest) (*master_pb.DisableVacuumResponse, error) {
  238. ms.Topo.DisableVacuum()
  239. resp := &master_pb.DisableVacuumResponse{}
  240. return resp, nil
  241. }
  242. func (ms *MasterServer) EnableVacuum(ctx context.Context, req *master_pb.EnableVacuumRequest) (*master_pb.EnableVacuumResponse, error) {
  243. ms.Topo.EnableVacuum()
  244. resp := &master_pb.EnableVacuumResponse{}
  245. return resp, nil
  246. }
  247. func (ms *MasterServer) VolumeMarkReadonly(ctx context.Context, req *master_pb.VolumeMarkReadonlyRequest) (*master_pb.VolumeMarkReadonlyResponse, error) {
  248. if !ms.Topo.IsLeader() {
  249. return nil, raft.NotLeaderError
  250. }
  251. resp := &master_pb.VolumeMarkReadonlyResponse{}
  252. replicaPlacement, _ := super_block.NewReplicaPlacementFromByte(byte(req.ReplicaPlacement))
  253. vl := ms.Topo.GetVolumeLayout(req.Collection, replicaPlacement, needle.LoadTTLFromUint32(req.Ttl), types.ToDiskType(req.DiskType))
  254. dataNodes := ms.Topo.Lookup(req.Collection, needle.VolumeId(req.VolumeId))
  255. for _, dn := range dataNodes {
  256. if dn.Ip == req.Ip && dn.Port == int(req.Port) {
  257. if req.IsReadonly {
  258. vl.SetVolumeReadOnly(dn, needle.VolumeId(req.VolumeId))
  259. } else {
  260. vl.SetVolumeWritable(dn, needle.VolumeId(req.VolumeId))
  261. }
  262. }
  263. }
  264. return resp, nil
  265. }
  266. func (ms *MasterServer) VolumeGrow(ctx context.Context, req *master_pb.VolumeGrowRequest) (*master_pb.VolumeGrowResponse, error) {
  267. if !ms.Topo.IsLeader() {
  268. return nil, raft.NotLeaderError
  269. }
  270. if req.Replication == "" {
  271. req.Replication = ms.option.DefaultReplicaPlacement
  272. }
  273. replicaPlacement, err := super_block.NewReplicaPlacementFromString(req.Replication)
  274. if err != nil {
  275. return nil, err
  276. }
  277. ttl, err := needle.ReadTTL(req.Ttl)
  278. if err != nil {
  279. return nil, err
  280. }
  281. if req.DataCenter != "" && !ms.Topo.DataCenterExists(req.DataCenter) {
  282. return nil, fmt.Errorf("data center not exists")
  283. }
  284. volumeGrowOption := topology.VolumeGrowOption{
  285. Collection: req.Collection,
  286. ReplicaPlacement: replicaPlacement,
  287. Ttl: ttl,
  288. DiskType: types.ToDiskType(req.DiskType),
  289. Preallocate: ms.preallocateSize,
  290. DataCenter: req.DataCenter,
  291. Rack: req.Rack,
  292. DataNode: req.DataNode,
  293. MemoryMapMaxSizeMb: req.MemoryMapMaxSizeMb,
  294. }
  295. volumeGrowRequest := topology.VolumeGrowRequest{
  296. Option: &volumeGrowOption,
  297. Count: req.WritableVolumeCount,
  298. Force: true,
  299. Reason: "grpc volume grow",
  300. }
  301. replicaCount := int64(req.WritableVolumeCount * uint32(replicaPlacement.GetCopyCount()))
  302. if ms.Topo.AvailableSpaceFor(&volumeGrowOption) < replicaCount {
  303. return nil, fmt.Errorf("only %d volumes left, not enough for %d", ms.Topo.AvailableSpaceFor(&volumeGrowOption), replicaCount)
  304. }
  305. if !ms.Topo.DataCenterExists(volumeGrowOption.DataCenter) {
  306. err = fmt.Errorf("data center %v not found in topology", volumeGrowOption.DataCenter)
  307. }
  308. ms.DoAutomaticVolumeGrow(&volumeGrowRequest)
  309. return &master_pb.VolumeGrowResponse{}, nil
  310. }