Browse Source

Adjust cli option (#7418)

* adjust "weed benchmark" CLI to use readOnly/writeOnly

* consistently use "-master" CLI option

* If both -readOnly and -writeOnly are specified, the current logic silently allows it with -writeOnly taking precedence. This is confusing and could lead to unexpected behavior.
pull/7421/head
Chris Lu 4 days ago
committed by GitHub
parent
commit
5ab49e2971
No known key found for this signature in database GPG Key ID: B5690EEEBB952194
  1. 37
      weed/command/admin.go
  2. 14
      weed/command/backup.go
  3. 34
      weed/command/benchmark.go
  4. 18
      weed/command/download.go
  5. 1
      weed/command/master.go
  6. 10
      weed/command/master_follower.go
  7. 11
      weed/command/volume.go

37
weed/command/admin.go

@ -34,7 +34,8 @@ var (
type AdminOptions struct { type AdminOptions struct {
port *int port *int
grpcPort *int grpcPort *int
masters *string
master *string
masters *string // deprecated, for backward compatibility
adminUser *string adminUser *string
adminPassword *string adminPassword *string
dataDir *string dataDir *string
@ -44,7 +45,8 @@ func init() {
cmdAdmin.Run = runAdmin // break init cycle cmdAdmin.Run = runAdmin // break init cycle
a.port = cmdAdmin.Flag.Int("port", 23646, "admin server port") a.port = cmdAdmin.Flag.Int("port", 23646, "admin server port")
a.grpcPort = cmdAdmin.Flag.Int("port.grpc", 0, "gRPC server port for worker connections (default: http port + 10000)") a.grpcPort = cmdAdmin.Flag.Int("port.grpc", 0, "gRPC server port for worker connections (default: http port + 10000)")
a.masters = cmdAdmin.Flag.String("masters", "localhost:9333", "comma-separated master servers")
a.master = cmdAdmin.Flag.String("master", "localhost:9333", "comma-separated master servers")
a.masters = cmdAdmin.Flag.String("masters", "", "comma-separated master servers (deprecated, use -master instead)")
a.dataDir = cmdAdmin.Flag.String("dataDir", "", "directory to store admin configuration and data files") a.dataDir = cmdAdmin.Flag.String("dataDir", "", "directory to store admin configuration and data files")
a.adminUser = cmdAdmin.Flag.String("adminUser", "admin", "admin interface username") a.adminUser = cmdAdmin.Flag.String("adminUser", "admin", "admin interface username")
@ -52,7 +54,7 @@ func init() {
} }
var cmdAdmin = &Command{ var cmdAdmin = &Command{
UsageLine: "admin -port=23646 -masters=localhost:9333 [-port.grpc=33646] [-dataDir=/path/to/data]",
UsageLine: "admin -port=23646 -master=localhost:9333 [-port.grpc=33646] [-dataDir=/path/to/data]",
Short: "start SeaweedFS web admin interface", Short: "start SeaweedFS web admin interface",
Long: `Start a web admin interface for SeaweedFS cluster management. Long: `Start a web admin interface for SeaweedFS cluster management.
@ -68,10 +70,10 @@ var cmdAdmin = &Command{
A gRPC server for worker connections runs on the configured gRPC port (default: HTTP port + 10000). A gRPC server for worker connections runs on the configured gRPC port (default: HTTP port + 10000).
Example Usage: Example Usage:
weed admin -port=23646 -masters="master1:9333,master2:9333"
weed admin -port=23646 -masters="localhost:9333" -dataDir="/var/lib/seaweedfs-admin"
weed admin -port=23646 -port.grpc=33646 -masters="localhost:9333" -dataDir="~/seaweedfs-admin"
weed admin -port=9900 -port.grpc=19900 -masters="localhost:9333"
weed admin -port=23646 -master="master1:9333,master2:9333"
weed admin -port=23646 -master="localhost:9333" -dataDir="/var/lib/seaweedfs-admin"
weed admin -port=23646 -port.grpc=33646 -master="localhost:9333" -dataDir="~/seaweedfs-admin"
weed admin -port=9900 -port.grpc=19900 -master="localhost:9333"
Data Directory: Data Directory:
- If dataDir is specified, admin configuration and maintenance data is persisted - If dataDir is specified, admin configuration and maintenance data is persisted
@ -116,18 +118,23 @@ func runAdmin(cmd *Command, args []string) bool {
// Load security configuration // Load security configuration
util.LoadSecurityConfiguration() util.LoadSecurityConfiguration()
// Backward compatibility: if -masters is provided, use it
if *a.masters != "" {
*a.master = *a.masters
}
// Validate required parameters // Validate required parameters
if *a.masters == "" {
fmt.Println("Error: masters parameter is required")
fmt.Println("Usage: weed admin -masters=master1:9333,master2:9333")
if *a.master == "" {
fmt.Println("Error: master parameter is required")
fmt.Println("Usage: weed admin -master=master1:9333,master2:9333")
return false return false
} }
// Validate that masters string can be parsed
masterAddresses := pb.ServerAddresses(*a.masters).ToAddresses()
// Validate that master string can be parsed
masterAddresses := pb.ServerAddresses(*a.master).ToAddresses()
if len(masterAddresses) == 0 { if len(masterAddresses) == 0 {
fmt.Println("Error: no valid master addresses found") fmt.Println("Error: no valid master addresses found")
fmt.Println("Usage: weed admin -masters=master1:9333,master2:9333")
fmt.Println("Usage: weed admin -master=master1:9333,master2:9333")
return false return false
} }
@ -144,7 +151,7 @@ func runAdmin(cmd *Command, args []string) bool {
fmt.Printf("Starting SeaweedFS Admin Interface on port %d\n", *a.port) fmt.Printf("Starting SeaweedFS Admin Interface on port %d\n", *a.port)
fmt.Printf("Worker gRPC server will run on port %d\n", *a.grpcPort) fmt.Printf("Worker gRPC server will run on port %d\n", *a.grpcPort)
fmt.Printf("Masters: %s\n", *a.masters)
fmt.Printf("Masters: %s\n", *a.master)
fmt.Printf("Filers will be discovered automatically from masters\n") fmt.Printf("Filers will be discovered automatically from masters\n")
if *a.dataDir != "" { if *a.dataDir != "" {
fmt.Printf("Data Directory: %s\n", *a.dataDir) fmt.Printf("Data Directory: %s\n", *a.dataDir)
@ -242,7 +249,7 @@ func startAdminServer(ctx context.Context, options AdminOptions) error {
} }
// Create admin server // Create admin server
adminServer := dash.NewAdminServer(*options.masters, nil, dataDir)
adminServer := dash.NewAdminServer(*options.master, nil, dataDir)
// Show discovered filers // Show discovered filers
filers := adminServer.GetAllFilers() filers := adminServer.GetAllFilers()

14
weed/command/backup.go

@ -21,6 +21,7 @@ var (
type BackupOptions struct { type BackupOptions struct {
master *string master *string
server *string // deprecated, for backward compatibility
collection *string collection *string
dir *string dir *string
volumeId *int volumeId *int
@ -30,7 +31,8 @@ type BackupOptions struct {
func init() { func init() {
cmdBackup.Run = runBackup // break init cycle cmdBackup.Run = runBackup // break init cycle
s.master = cmdBackup.Flag.String("server", "localhost:9333", "SeaweedFS master location")
s.master = cmdBackup.Flag.String("master", "localhost:9333", "SeaweedFS master location")
s.server = cmdBackup.Flag.String("server", "", "SeaweedFS master location (deprecated, use -master instead)")
s.collection = cmdBackup.Flag.String("collection", "", "collection name") s.collection = cmdBackup.Flag.String("collection", "", "collection name")
s.dir = cmdBackup.Flag.String("dir", ".", "directory to store volume data files") s.dir = cmdBackup.Flag.String("dir", ".", "directory to store volume data files")
s.volumeId = cmdBackup.Flag.Int("volumeId", -1, "a volume id. The volume .dat and .idx files should already exist in the dir.") s.volumeId = cmdBackup.Flag.Int("volumeId", -1, "a volume id. The volume .dat and .idx files should already exist in the dir.")
@ -46,7 +48,7 @@ func init() {
} }
var cmdBackup = &Command{ var cmdBackup = &Command{
UsageLine: "backup -dir=. -volumeId=234 -server=localhost:9333",
UsageLine: "backup -dir=. -volumeId=234 -master=localhost:9333",
Short: "incrementally backup a volume to local folder", Short: "incrementally backup a volume to local folder",
Long: `Incrementally backup volume data. Long: `Incrementally backup volume data.
@ -69,13 +71,19 @@ func runBackup(cmd *Command, args []string) bool {
util.LoadSecurityConfiguration() util.LoadSecurityConfiguration()
grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client") grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
// Backward compatibility: if -server is provided, use it
masterServer := *s.master
if *s.server != "" {
masterServer = *s.server
}
if *s.volumeId == -1 { if *s.volumeId == -1 {
return false return false
} }
vid := needle.VolumeId(*s.volumeId) vid := needle.VolumeId(*s.volumeId)
// find volume location, replication, ttl info // find volume location, replication, ttl info
lookup, err := operation.LookupVolumeId(func(_ context.Context) pb.ServerAddress { return pb.ServerAddress(*s.master) }, grpcDialOption, vid.String())
lookup, err := operation.LookupVolumeId(func(_ context.Context) pb.ServerAddress { return pb.ServerAddress(masterServer) }, grpcDialOption, vid.String())
if err != nil { if err != nil {
fmt.Printf("Error looking up volume %d: %v\n", vid, err) fmt.Printf("Error looking up volume %d: %v\n", vid, err)
return true return true

34
weed/command/benchmark.go

@ -32,9 +32,9 @@ type BenchmarkOptions struct {
numberOfFiles *int numberOfFiles *int
fileSize *int fileSize *int
idListFile *string idListFile *string
write *bool
deletePercentage *int deletePercentage *int
read *bool
readOnly *bool
writeOnly *bool
sequentialRead *bool sequentialRead *bool
collection *string collection *string
replication *string replication *string
@ -60,9 +60,9 @@ func init() {
b.fileSize = cmdBenchmark.Flag.Int("size", 1024, "simulated file size in bytes, with random(0~63) bytes padding") b.fileSize = cmdBenchmark.Flag.Int("size", 1024, "simulated file size in bytes, with random(0~63) bytes padding")
b.numberOfFiles = cmdBenchmark.Flag.Int("n", 1024*1024, "number of files to write for each thread") b.numberOfFiles = cmdBenchmark.Flag.Int("n", 1024*1024, "number of files to write for each thread")
b.idListFile = cmdBenchmark.Flag.String("list", os.TempDir()+"/benchmark_list.txt", "list of uploaded file ids") b.idListFile = cmdBenchmark.Flag.String("list", os.TempDir()+"/benchmark_list.txt", "list of uploaded file ids")
b.write = cmdBenchmark.Flag.Bool("write", true, "enable write")
b.deletePercentage = cmdBenchmark.Flag.Int("deletePercent", 0, "the percent of writes that are deletes") b.deletePercentage = cmdBenchmark.Flag.Int("deletePercent", 0, "the percent of writes that are deletes")
b.read = cmdBenchmark.Flag.Bool("read", true, "enable read")
b.readOnly = cmdBenchmark.Flag.Bool("readOnly", false, "only benchmark read operations")
b.writeOnly = cmdBenchmark.Flag.Bool("writeOnly", false, "only benchmark write operations")
b.sequentialRead = cmdBenchmark.Flag.Bool("readSequentially", false, "randomly read by ids from \"-list\" specified file") b.sequentialRead = cmdBenchmark.Flag.Bool("readSequentially", false, "randomly read by ids from \"-list\" specified file")
b.collection = cmdBenchmark.Flag.String("collection", "benchmark", "write data to this collection") b.collection = cmdBenchmark.Flag.String("collection", "benchmark", "write data to this collection")
b.replication = cmdBenchmark.Flag.String("replication", "000", "replication type") b.replication = cmdBenchmark.Flag.String("replication", "000", "replication type")
@ -84,7 +84,10 @@ var cmdBenchmark = &Command{
The file content is mostly zeros, but no compression is done. The file content is mostly zeros, but no compression is done.
You can choose to only benchmark read or write.
You can choose to only benchmark read or write:
-readOnly only benchmark read operations
-writeOnly only benchmark write operations
During write, the list of uploaded file ids is stored in "-list" specified file. During write, the list of uploaded file ids is stored in "-list" specified file.
You can also use your own list of file ids to run read test. You can also use your own list of file ids to run read test.
@ -130,16 +133,33 @@ func runBenchmark(cmd *Command, args []string) bool {
defer pprof.StopCPUProfile() defer pprof.StopCPUProfile()
} }
// Determine what operations to perform
// Default: both write and read
// -readOnly: only read
// -writeOnly: only write
if *b.readOnly && *b.writeOnly {
fmt.Fprintln(os.Stderr, "Error: -readOnly and -writeOnly are mutually exclusive.")
return false
}
doWrite := true
doRead := true
if *b.readOnly {
doWrite = false
} else if *b.writeOnly {
doRead = false
}
b.masterClient = wdclient.NewMasterClient(b.grpcDialOption, "", "client", "", "", "", *pb.ServerAddresses(*b.masters).ToServiceDiscovery()) b.masterClient = wdclient.NewMasterClient(b.grpcDialOption, "", "client", "", "", "", *pb.ServerAddresses(*b.masters).ToServiceDiscovery())
ctx := context.Background() ctx := context.Background()
go b.masterClient.KeepConnectedToMaster(ctx) go b.masterClient.KeepConnectedToMaster(ctx)
b.masterClient.WaitUntilConnected(ctx) b.masterClient.WaitUntilConnected(ctx)
if *b.write {
if doWrite {
benchWrite() benchWrite()
} }
if *b.read {
if doRead {
benchRead() benchRead()
} }

18
weed/command/download.go

@ -23,18 +23,20 @@ var (
) )
type DownloadOptions struct { type DownloadOptions struct {
server *string
dir *string
master *string
server *string // deprecated, for backward compatibility
dir *string
} }
func init() { func init() {
cmdDownload.Run = runDownload // break init cycle cmdDownload.Run = runDownload // break init cycle
d.server = cmdDownload.Flag.String("server", "localhost:9333", "SeaweedFS master location")
d.master = cmdDownload.Flag.String("master", "localhost:9333", "SeaweedFS master location")
d.server = cmdDownload.Flag.String("server", "", "SeaweedFS master location (deprecated, use -master instead)")
d.dir = cmdDownload.Flag.String("dir", ".", "Download the whole folder recursively if specified.") d.dir = cmdDownload.Flag.String("dir", ".", "Download the whole folder recursively if specified.")
} }
var cmdDownload = &Command{ var cmdDownload = &Command{
UsageLine: "download -server=localhost:9333 -dir=one_directory fid1 [fid2 fid3 ...]",
UsageLine: "download -master=localhost:9333 -dir=one_directory fid1 [fid2 fid3 ...]",
Short: "download files by file id", Short: "download files by file id",
Long: `download files by file id. Long: `download files by file id.
@ -51,8 +53,14 @@ func runDownload(cmd *Command, args []string) bool {
util.LoadSecurityConfiguration() util.LoadSecurityConfiguration()
grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client") grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
// Backward compatibility: if -server is provided, use it
masterServer := *d.master
if *d.server != "" {
masterServer = *d.server
}
for _, fid := range args { for _, fid := range args {
if e := downloadToFile(func(_ context.Context) pb.ServerAddress { return pb.ServerAddress(*d.server) }, grpcDialOption, fid, util.ResolvePath(*d.dir)); e != nil {
if e := downloadToFile(func(_ context.Context) pb.ServerAddress { return pb.ServerAddress(masterServer) }, grpcDialOption, fid, util.ResolvePath(*d.dir)); e != nil {
fmt.Println("Download Error: ", fid, e) fmt.Println("Download Error: ", fid, e)
} }
} }

1
weed/command/master.go

@ -45,6 +45,7 @@ type MasterOptions struct {
ipBind *string ipBind *string
metaFolder *string metaFolder *string
peers *string peers *string
mastersDeprecated *string // deprecated, for backward compatibility in master.follower
volumeSizeLimitMB *uint volumeSizeLimitMB *uint
volumePreallocate *bool volumePreallocate *bool
maxParallelVacuumPerServer *int maxParallelVacuumPerServer *int

10
weed/command/master_follower.go

@ -27,7 +27,8 @@ func init() {
mf.port = cmdMasterFollower.Flag.Int("port", 9334, "http listen port") mf.port = cmdMasterFollower.Flag.Int("port", 9334, "http listen port")
mf.portGrpc = cmdMasterFollower.Flag.Int("port.grpc", 0, "grpc listen port") mf.portGrpc = cmdMasterFollower.Flag.Int("port.grpc", 0, "grpc listen port")
mf.ipBind = cmdMasterFollower.Flag.String("ip.bind", "", "ip address to bind to. Default to localhost.") mf.ipBind = cmdMasterFollower.Flag.String("ip.bind", "", "ip address to bind to. Default to localhost.")
mf.peers = cmdMasterFollower.Flag.String("masters", "localhost:9333", "all master nodes in comma separated ip:port list, example: 127.0.0.1:9093,127.0.0.1:9094,127.0.0.1:9095")
mf.peers = cmdMasterFollower.Flag.String("master", "localhost:9333", "all master nodes in comma separated ip:port list, example: 127.0.0.1:9093,127.0.0.1:9094,127.0.0.1:9095")
mf.mastersDeprecated = cmdMasterFollower.Flag.String("masters", "", "all master nodes in comma separated ip:port list (deprecated, use -master instead)")
mf.ip = aws.String(util.DetectedHostAddress()) mf.ip = aws.String(util.DetectedHostAddress())
mf.metaFolder = aws.String("") mf.metaFolder = aws.String("")
@ -43,7 +44,7 @@ func init() {
} }
var cmdMasterFollower = &Command{ var cmdMasterFollower = &Command{
UsageLine: "master.follower -port=9333 -masters=<master1Host>:<master1Port>",
UsageLine: "master.follower -port=9333 -master=<master1Host>:<master1Port>",
Short: "start a master follower", Short: "start a master follower",
Long: `start a master follower to provide volume=>location mapping service Long: `start a master follower to provide volume=>location mapping service
@ -72,6 +73,11 @@ func runMasterFollower(cmd *Command, args []string) bool {
util.LoadSecurityConfiguration() util.LoadSecurityConfiguration()
util.LoadConfiguration("master", false) util.LoadConfiguration("master", false)
// Backward compatibility: if -masters is provided, use it
if *mf.mastersDeprecated != "" {
*mf.peers = *mf.mastersDeprecated
}
if *mf.portGrpc == 0 { if *mf.portGrpc == 0 {
*mf.portGrpc = 10000 + *mf.port *mf.portGrpc = 10000 + *mf.port
} }

11
weed/command/volume.go

@ -44,6 +44,7 @@ type VolumeServerOptions struct {
publicUrl *string publicUrl *string
bindIp *string bindIp *string
mastersString *string mastersString *string
mserverString *string // deprecated, for backward compatibility
masters []pb.ServerAddress masters []pb.ServerAddress
idleConnectionTimeout *int idleConnectionTimeout *int
dataCenter *string dataCenter *string
@ -79,7 +80,8 @@ func init() {
v.ip = cmdVolume.Flag.String("ip", util.DetectedHostAddress(), "ip or server name, also used as identifier") v.ip = cmdVolume.Flag.String("ip", util.DetectedHostAddress(), "ip or server name, also used as identifier")
v.publicUrl = cmdVolume.Flag.String("publicUrl", "", "Publicly accessible address") v.publicUrl = cmdVolume.Flag.String("publicUrl", "", "Publicly accessible address")
v.bindIp = cmdVolume.Flag.String("ip.bind", "", "ip address to bind to. If empty, default to same as -ip option.") v.bindIp = cmdVolume.Flag.String("ip.bind", "", "ip address to bind to. If empty, default to same as -ip option.")
v.mastersString = cmdVolume.Flag.String("mserver", "localhost:9333", "comma-separated master servers")
v.mastersString = cmdVolume.Flag.String("master", "localhost:9333", "comma-separated master servers")
v.mserverString = cmdVolume.Flag.String("mserver", "", "comma-separated master servers (deprecated, use -master instead)")
v.preStopSeconds = cmdVolume.Flag.Int("preStopSeconds", 10, "number of seconds between stop send heartbeats and stop volume server") v.preStopSeconds = cmdVolume.Flag.Int("preStopSeconds", 10, "number of seconds between stop send heartbeats and stop volume server")
// v.pulseSeconds = cmdVolume.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats, must be smaller than or equal to the master's setting") // v.pulseSeconds = cmdVolume.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats, must be smaller than or equal to the master's setting")
v.idleConnectionTimeout = cmdVolume.Flag.Int("idleTimeout", 30, "connection idle seconds") v.idleConnectionTimeout = cmdVolume.Flag.Int("idleTimeout", 30, "connection idle seconds")
@ -107,7 +109,7 @@ func init() {
} }
var cmdVolume = &Command{ var cmdVolume = &Command{
UsageLine: "volume -port=8080 -dir=/tmp -max=5 -ip=server_name -mserver=localhost:9333",
UsageLine: "volume -port=8080 -dir=/tmp -max=5 -ip=server_name -master=localhost:9333",
Short: "start a volume server", Short: "start a volume server",
Long: `start a volume server to provide storage spaces Long: `start a volume server to provide storage spaces
@ -142,6 +144,11 @@ func runVolume(cmd *Command, args []string) bool {
} }
go stats_collect.StartMetricsServer(*v.metricsHttpIp, *v.metricsHttpPort) go stats_collect.StartMetricsServer(*v.metricsHttpIp, *v.metricsHttpPort)
// Backward compatibility: if -mserver is provided, use it
if *v.mserverString != "" {
*v.mastersString = *v.mserverString
}
minFreeSpaces := util.MustParseMinFreeSpace(*minFreeSpace, *minFreeSpacePercent) minFreeSpaces := util.MustParseMinFreeSpace(*minFreeSpace, *minFreeSpacePercent)
v.masters = pb.ServerAddresses(*v.mastersString).ToAddresses() v.masters = pb.ServerAddresses(*v.mastersString).ToAddresses()
v.startVolumeServer(*volumeFolders, *maxVolumeCounts, *volumeWhiteListOption, minFreeSpaces) v.startVolumeServer(*volumeFolders, *maxVolumeCounts, *volumeWhiteListOption, minFreeSpaces)

Loading…
Cancel
Save