Mirror
/
seaweedfs
mirror of https://github.com/seaweedfs/seaweedfs.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

refactor volumeSizeLimit

pull/843/head
bingoohuang 7 years ago
parent
44e899464f
commit
2023e4eb47
5 changed files with 51 additions and 29 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 8
      weed/command/master.go
  2. 36
      weed/command/server.go
  3. 2
      weed/server/master_grpc_server.go
  4. 12
      weed/server/master_server.go
  5. 22
      weed/util/arg.go

8
weed/command/master.go
View File

@ -37,6 +37,7 @@ var (
metaFolder = cmdMaster.Flag.String("mdir", os.TempDir(), "data directory to store meta data")
masterPeers = cmdMaster.Flag.String("peers", "", "all master nodes in comma separated ip:port list, example: 127.0.0.1:9093,127.0.0.1:9094")
volumeSizeLimitMiB = cmdMaster.Flag.Uint("volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes. (MiB)")
volumeSizeLimitArg = cmdMaster.Flag.String("volumeSizeLimit", "", "Master stops directing writes to oversized volumes. (eg: 30G, 20G)")
volumePreallocate = cmdMaster.Flag.Bool("volumePreallocate", false, "Preallocate disk space for volumes.")
mpulse = cmdMaster.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
defaultReplicaPlacement = cmdMaster.Flag.String("defaultReplication", "000", "Default replication type if not specified.")
@ -64,13 +65,12 @@ func runMaster(cmd *Command, args []string) bool {
if *masterWhiteListOption != "" {
masterWhiteList = strings.Split(*masterWhiteListOption, ",")
}
if *volumeSizeLimitMiB > 30*1000 {
glog.Fatalf("volumeSizeLimitMB should be smaller than 30000")
}
volumeSizeLimit := util.ParseVolumeSizeLimit(*volumeSizeLimitMiB, *volumeSizeLimitArg)
r := mux.NewRouter()
ms := weed_server.NewMasterServer(r, *mport, *metaFolder,
*volumeSizeLimitMiB, *volumePreallocate,
volumeSizeLimit, *volumePreallocate,
*mpulse, *defaultReplicaPlacement, *garbageThreshold,
masterWhiteList, *masterSecureKey,
)

36
weed/command/server.go
View File

@ -50,20 +50,22 @@ var cmdServer = &Command{
}
var (
serverIp = cmdServer.Flag.String("ip", "localhost", "ip or server name")
serverBindIp = cmdServer.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
serverMaxCpu = cmdServer.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
serverTimeout = cmdServer.Flag.Int("idleTimeout", 30, "connection idle seconds")
serverDataCenter = cmdServer.Flag.String("dataCenter", "", "current volume server's data center name")
serverRack = cmdServer.Flag.String("rack", "", "current volume server's rack name")
serverWhiteListOption = cmdServer.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
serverPeers = cmdServer.Flag.String("master.peers", "", "all master nodes in comma separated ip:masterPort list")
serverSecureKey = cmdServer.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
serverGarbageThreshold = cmdServer.Flag.Float64("garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
masterPort = cmdServer.Flag.Int("master.port", 9333, "master server http listen port")
masterGrpcPort = cmdServer.Flag.Int("master.port.grpc", 0, "master grpc server listen port, default to http port + 10000")
masterMetaFolder = cmdServer.Flag.String("master.dir", "", "data directory to store meta data, default to same as -dir specified")
masterVolumeSizeLimitMB = cmdServer.Flag.Uint("master.volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
serverIp = cmdServer.Flag.String("ip", "localhost", "ip or server name")
serverBindIp = cmdServer.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
serverMaxCpu = cmdServer.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
serverTimeout = cmdServer.Flag.Int("idleTimeout", 30, "connection idle seconds")
serverDataCenter = cmdServer.Flag.String("dataCenter", "", "current volume server's data center name")
serverRack = cmdServer.Flag.String("rack", "", "current volume server's rack name")
serverWhiteListOption = cmdServer.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
serverPeers = cmdServer.Flag.String("master.peers", "", "all master nodes in comma separated ip:masterPort list")
serverSecureKey = cmdServer.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
serverGarbageThreshold = cmdServer.Flag.Float64("garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
masterPort = cmdServer.Flag.Int("master.port", 9333, "master server http listen port")
masterGrpcPort = cmdServer.Flag.Int("master.port.grpc", 0, "master grpc server listen port, default to http port + 10000")
masterMetaFolder = cmdServer.Flag.String("master.dir", "", "data directory to store meta data, default to same as -dir specified")
masterVolumeSizeLimitMib = cmdServer.Flag.Uint("master.volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
masterVolumeSizeLimitArg = cmdMaster.Flag.String("master.volumeSizeLimit", "", "Master stops directing writes to oversized volumes. (eg: 30G, 20G)")
masterVolumePreallocate = cmdServer.Flag.Bool("master.volumePreallocate", false, "Preallocate disk space for volumes.")
masterDefaultReplicaPlacement = cmdServer.Flag.String("master.defaultReplicaPlacement", "000", "Default replication type if not specified.")
volumeDataFolders = cmdServer.Flag.String("dir", os.TempDir(), "directories to store data files. dir[,dir]...")
@ -134,9 +136,7 @@ func runServer(cmd *Command, args []string) bool {
folders := strings.Split(*volumeDataFolders, ",")
if *masterVolumeSizeLimitMB > 30*1000 {
glog.Fatalf("masterVolumeSizeLimitMB should be less than 30000")
}
volumeSizeLimit := util.ParseVolumeSizeLimit(*masterVolumeSizeLimitMib, *masterVolumeSizeLimitArg)
if *masterMetaFolder == "" {
*masterMetaFolder = folders[0]
@ -168,7 +168,7 @@ func runServer(cmd *Command, args []string) bool {
go func() {
r := mux.NewRouter()
ms := weed_server.NewMasterServer(r, *masterPort, *masterMetaFolder,
*masterVolumeSizeLimitMB, *masterVolumePreallocate,
volumeSizeLimit, *masterVolumePreallocate,
*pulseSeconds, *masterDefaultReplicaPlacement, *serverGarbageThreshold,
serverWhiteList, *serverSecureKey,
)

2
weed/server/master_grpc_server.go
View File

@ -64,7 +64,7 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ
dn = rack.GetOrCreateDataNode(heartbeat.Ip, int(heartbeat.Port), heartbeat.PublicUrl, int(heartbeat.MaxVolumeCount))
glog.V(0).Infof("added volume server %v:%d", heartbeat.GetIp(), heartbeat.GetPort())
if err := stream.Send(&master_pb.HeartbeatResponse{
VolumeSizeLimit: uint64(ms.volumeSizeLimitMiB) * 1024 * 1024,
VolumeSizeLimit: ms.volumeSizeLimit,
SecretKey: string(ms.guard.SecretKey),
}); err != nil {
return err

12
weed/server/master_server.go
View File

@ -20,7 +20,7 @@ import (
type MasterServer struct {
port int
metaFolder string
volumeSizeLimitMiB uint
volumeSizeLimit uint64
preallocate int64
pulseSeconds int
defaultReplicaPlacement string
@ -39,7 +39,7 @@ type MasterServer struct {
}
func NewMasterServer(r *mux.Router, port int, metaFolder string,
volumeSizeLimitMiB uint,
volumeSizeLimit uint64,
preallocate bool,
pulseSeconds int,
defaultReplicaPlacement string,
@ -50,11 +50,11 @@ func NewMasterServer(r *mux.Router, port int, metaFolder string,
var preallocateSize int64
if preallocate {
preallocateSize = int64(volumeSizeLimitMiB) * (1 << 20)
preallocateSize = int64(volumeSizeLimit) * (1 << 20)
}
ms := &MasterServer{
port: port,
volumeSizeLimitMiB: volumeSizeLimitMiB,
volumeSizeLimit: volumeSizeLimit,
preallocate: preallocateSize,
pulseSeconds: pulseSeconds,
defaultReplicaPlacement: defaultReplicaPlacement,
@ -63,9 +63,9 @@ func NewMasterServer(r *mux.Router, port int, metaFolder string,
}
ms.bounedLeaderChan = make(chan int, 16)
seq := sequence.NewMemorySequencer()
ms.Topo = topology.NewTopology("topo", seq, uint64(volumeSizeLimitMiB)*1024*1024, pulseSeconds)
ms.Topo = topology.NewTopology("topo", seq, uint64(volumeSizeLimit), pulseSeconds)
ms.vg = topology.NewDefaultVolumeGrowth()
glog.V(0).Infoln("Volume Size Limit is", volumeSizeLimitMiB, "MiB")
glog.V(0).Infoln("Volume Size Limit is", volumeSizeLimit)
ms.guard = security.NewGuard(whiteList, secureKey)

22
weed/util/arg.go
View File

@ -0,0 +1,22 @@
package util
import (
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/dustin/go-humanize"
)
func ParseVolumeSizeLimit(volumeSizeLimitMiBValue uint, volumeSizeLimitArgValue string) uint64 {
volumeSizeLimit := uint64(volumeSizeLimitMiBValue) * 1024 * 1024
if volumeSizeLimitArgValue != "" {
var err error
volumeSizeLimit, err = humanize.ParseBytes(volumeSizeLimitArgValue)
if err != nil {
glog.Fatalf("Parse volumeSizeLimit %s : %s", volumeSizeLimitMiBValue, err)
}
}
if volumeSizeLimit > uint64(30*1000)*1024*1024 {
glog.Fatalf("volumeSizeLimitMB should be smaller than 30000")
}
return volumeSizeLimit
}
Write Preview
Loading…
Cancel
Save