Mirror
/
seaweedfs
mirror of https://github.com/seaweedfs/seaweedfs.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

faster master startup

pull/7510/head
chrislu 1 week ago
parent
0299e78de7
commit
aef5121c36
13 changed files with 23 additions and 15 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 4
      .github/workflows/kafka-tests.yml
  2. 9
      .github/workflows/s3tests.yml
  3. 2
      .github/workflows/test-s3-over-https-using-awscli.yml
  4. 1
      test/erasure_coding/ec_integration_test.go
  5. 1
      test/fuse_integration/framework.go
  6. 2
      test/kafka/README.md
  7. 2
      test/kafka/loadtest/run_million_record_test.sh
  8. 2
      test/kafka/loadtest/setup_seaweed_infrastructure.sh
  9. 2
      test/kafka/scripts/test_schema_registry.sh
  10. 4
      test/mq/README.md
  11. 2
      test/s3/copying/Makefile
  12. 3
      test/s3/iam/Makefile
  13. 4
      test/s3/sse/Makefile

4
.github/workflows/kafka-tests.yml
View File

@ -176,6 +176,7 @@ jobs:
-filer.port=8888 \ -filer.port=8888 \
-filer=true \ -filer=true \
-metricsPort=9325 \ -metricsPort=9325 \
-master.peers=none \
> /tmp/weed-server.log 2>&1 & > /tmp/weed-server.log 2>&1 &
# Wait for master to be ready # Wait for master to be ready
@ -353,6 +354,7 @@ jobs:
-filer.port=8888 \ -filer.port=8888 \
-filer=true \ -filer=true \
-metricsPort=9325 \ -metricsPort=9325 \
-master.peers=none \
> /tmp/weed-server.log 2>&1 & > /tmp/weed-server.log 2>&1 &
# Wait for master to be ready # Wait for master to be ready
@ -512,6 +514,7 @@ jobs:
-filer.port=8888 \ -filer.port=8888 \
-filer=true \ -filer=true \
-metricsPort=9325 \ -metricsPort=9325 \
-master.peers=none \
> /tmp/weed-server.log 2>&1 & > /tmp/weed-server.log 2>&1 &
# Wait for master to be ready # Wait for master to be ready
@ -668,6 +671,7 @@ jobs:
-filer.port=8888 \ -filer.port=8888 \
-filer=true \ -filer=true \
-metricsPort=9325 \ -metricsPort=9325 \
-master.peers=none \
> /tmp/weed-server.log 2>&1 & > /tmp/weed-server.log 2>&1 &
# Wait for master to be ready # Wait for master to be ready

9
.github/workflows/s3tests.yml
View File

@ -64,7 +64,7 @@ jobs:
-master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 \ -master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 \
-volume.max=100 -volume.preStopSeconds=1 \ -volume.max=100 -volume.preStopSeconds=1 \
-master.port=9333 -volume.port=8080 -filer.port=8888 -s3.port=8000 -metricsPort=9324 \ -master.port=9333 -volume.port=8080 -filer.port=8888 -s3.port=8000 -metricsPort=9324 \
-s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config="$GITHUB_WORKSPACE/docker/compose/s3.json" &
-s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config="$GITHUB_WORKSPACE/docker/compose/s3.json" -master.peers=none &
pid=$! pid=$!
# Wait for all SeaweedFS components to be ready # Wait for all SeaweedFS components to be ready
@ -368,7 +368,7 @@ jobs:
-master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 \ -master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 \
-volume.max=100 -volume.preStopSeconds=1 \ -volume.max=100 -volume.preStopSeconds=1 \
-master.port=9334 -volume.port=8081 -filer.port=8889 -s3.port=8001 -metricsPort=9325 \ -master.port=9334 -volume.port=8081 -filer.port=8889 -s3.port=8001 -metricsPort=9325 \
-s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config="$GITHUB_WORKSPACE/docker/compose/s3.json" &
-s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config="$GITHUB_WORKSPACE/docker/compose/s3.json" -master.peers=none &
pid=$! pid=$!
# Wait for all SeaweedFS components to be ready # Wait for all SeaweedFS components to be ready
@ -526,7 +526,7 @@ jobs:
-master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 \ -master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 \
-volume.max=100 -volume.preStopSeconds=1 \ -volume.max=100 -volume.preStopSeconds=1 \
-master.port=9335 -volume.port=8082 -filer.port=8890 -s3.port=8002 -metricsPort=9326 \ -master.port=9335 -volume.port=8082 -filer.port=8890 -s3.port=8002 -metricsPort=9326 \
-s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config="$GITHUB_WORKSPACE/docker/compose/s3.json" &
-s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config="$GITHUB_WORKSPACE/docker/compose/s3.json" -master.peers=none &
pid=$! pid=$!
# Wait for all SeaweedFS components to be ready # Wait for all SeaweedFS components to be ready
@ -636,7 +636,7 @@ jobs:
-master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 \ -master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 \
-volume.max=100 -volume.preStopSeconds=1 \ -volume.max=100 -volume.preStopSeconds=1 \
-master.port=9336 -volume.port=8083 -filer.port=8891 -s3.port=8003 -metricsPort=9327 \ -master.port=9336 -volume.port=8083 -filer.port=8891 -s3.port=8003 -metricsPort=9327 \
-s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config="$GITHUB_WORKSPACE/docker/compose/s3.json" &
-s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config="$GITHUB_WORKSPACE/docker/compose/s3.json" -master.peers=none &
pid=$! pid=$!
# Wait for all SeaweedFS components to be ready # Wait for all SeaweedFS components to be ready
@ -818,6 +818,7 @@ jobs:
-volume.max=100 -volume.preStopSeconds=1 \ -volume.max=100 -volume.preStopSeconds=1 \
-master.port=9337 -volume.port=8085 -filer.port=8892 -s3.port=8004 -metricsPort=9328 \ -master.port=9337 -volume.port=8085 -filer.port=8892 -s3.port=8004 -metricsPort=9328 \
-s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config="$GITHUB_WORKSPACE/docker/compose/s3.json" \ -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config="$GITHUB_WORKSPACE/docker/compose/s3.json" \
-master.peers=none \
> /tmp/seaweedfs-sql-server.log 2>&1 & > /tmp/seaweedfs-sql-server.log 2>&1 &
pid=$! pid=$!

2
.github/workflows/test-s3-over-https-using-awscli.yml
View File

@ -34,7 +34,7 @@ jobs:
run: | run: |
set -e set -e
mkdir -p /tmp/data mkdir -p /tmp/data
./weed server -s3 -dir=/tmp/data -s3.config=../docker/compose/s3.json &
./weed -v=3 server -s3 -dir=/tmp/data -s3.config=../docker/compose/s3.json -master.peers=none > weed.log 2>&1 &
until curl -s http://localhost:8333/ > /dev/null; do sleep 1; done until curl -s http://localhost:8333/ > /dev/null; do sleep 1; done
- name: Setup Caddy - name: Setup Caddy

1
test/erasure_coding/ec_integration_test.go
View File

@ -391,6 +391,7 @@ func startSeaweedFSCluster(ctx context.Context, dataDir string) (*TestCluster, e
"-mdir", masterDir, "-mdir", masterDir,
"-volumeSizeLimitMB", "10", // Small volumes for testing "-volumeSizeLimitMB", "10", // Small volumes for testing
"-ip", "127.0.0.1", "-ip", "127.0.0.1",
"-peers", "none", // Faster startup when no multiple masters needed
) )
masterLogFile, err := os.Create(filepath.Join(masterDir, "master.log")) masterLogFile, err := os.Create(filepath.Join(masterDir, "master.log"))

1
test/fuse_integration/framework.go
View File

@ -175,6 +175,7 @@ func (f *FuseTestFramework) startMaster(config *TestConfig) error {
"-port=19333", "-port=19333",
"-mdir=" + filepath.Join(f.dataDir, "master"), "-mdir=" + filepath.Join(f.dataDir, "master"),
"-raftBootstrap", "-raftBootstrap",
"-peers=none", // Faster startup when no multiple masters needed
} }
if config.EnableDebug { if config.EnableDebug {
args = append(args, "-v=4") args = append(args, "-v=4")

2
test/kafka/README.md
View File

@ -37,7 +37,7 @@ Requires running SeaweedFS instance:
1. **Start SeaweedFS with MQ support:** 1. **Start SeaweedFS with MQ support:**
```bash ```bash
# Terminal 1: Start SeaweedFS server # Terminal 1: Start SeaweedFS server
weed server -ip="127.0.0.1" -ip.bind="0.0.0.0" -dir=/tmp/seaweedfs-data -master.port=9333 -volume.port=8081 -filer.port=8888 -filer=true
weed server -ip="127.0.0.1" -ip.bind="0.0.0.0" -dir=/tmp/seaweedfs-data -master.port=9333 -volume.port=8081 -filer.port=8888 -filer=true -master.peers=none
# Terminal 2: Start MQ broker # Terminal 2: Start MQ broker
weed mq.broker -master="127.0.0.1:9333" -ip="127.0.0.1" -port=17777 weed mq.broker -master="127.0.0.1:9333" -ip="127.0.0.1" -port=17777

2
test/kafka/loadtest/run_million_record_test.sh
View File

@ -55,7 +55,7 @@ if [ "$MASTERS_OK" = false ]; then
echo "" echo ""
echo "Example commands to start SeaweedFS:" echo "Example commands to start SeaweedFS:"
echo " # Terminal 1: Start Master" echo " # Terminal 1: Start Master"
echo " weed master -defaultReplication=001 -mdir=/tmp/seaweedfs/master"
echo " weed master -defaultReplication=001 -mdir=/tmp/seaweedfs/master -peers=none"
echo "" echo ""
echo " # Terminal 2: Start Filer" echo " # Terminal 2: Start Filer"
echo " weed filer -master=localhost:9333 -filer.dir=/tmp/seaweedfs/filer" echo " weed filer -master=localhost:9333 -filer.dir=/tmp/seaweedfs/filer"

2
test/kafka/loadtest/setup_seaweed_infrastructure.sh
View File

@ -76,7 +76,7 @@ echo "=== Starting SeaweedFS Components ==="
# Start Master # Start Master
if ! check_service "localhost:9333" "SeaweedFS Master"; then if ! check_service "localhost:9333" "SeaweedFS Master"; then
start_service \ start_service \
"weed master -defaultReplication=001 -mdir=$BASE_DIR/master" \
"weed master -defaultReplication=001 -mdir=$BASE_DIR/master -peers=none" \
"SeaweedFS Master" \ "SeaweedFS Master" \
"$LOG_DIR/master.log" \ "$LOG_DIR/master.log" \
"localhost:9333" "localhost:9333"

2
test/kafka/scripts/test_schema_registry.sh
View File

@ -29,7 +29,7 @@ sleep 2
# Start SeaweedFS server # Start SeaweedFS server
echo "🗄️ Starting SeaweedFS server..." echo "🗄️ Starting SeaweedFS server..."
/tmp/weed server -dir=/tmp/seaweedfs-test -master.port=9333 -volume.port=8080 -filer.port=8888 -ip=localhost > /tmp/seaweed-server.log 2>&1 &
/tmp/weed server -dir=/tmp/seaweedfs-test -master.port=9333 -volume.port=8080 -filer.port=8888 -ip=localhost -master.peers=none > /tmp/seaweed-server.log 2>&1 &
SERVER_PID=$! SERVER_PID=$!
# Wait for server to be ready # Wait for server to be ready

4
test/mq/README.md
View File

@ -13,10 +13,10 @@ This directory contains test programs for SeaweedFS Message Queue (MQ) functiona
```bash ```bash
# Start SeaweedFS server with MQ broker and agent # Start SeaweedFS server with MQ broker and agent
weed server -mq.broker -mq.agent -filer -volume
weed server -mq.broker -mq.agent -filer -volume -master.peers=none
# Or start components separately # Or start components separately
weed master
weed master -peers=none
weed volume -mserver=localhost:9333 weed volume -mserver=localhost:9333
weed filer -master=localhost:9333 weed filer -master=localhost:9333
weed mq.broker -filer=localhost:8888 weed mq.broker -filer=localhost:8888

2
test/s3/copying/Makefile
View File

@ -70,7 +70,7 @@ start-seaweedfs: check-binary
@mkdir -p /tmp/seaweedfs-test-copying-volume @mkdir -p /tmp/seaweedfs-test-copying-volume
# Start master server with volume size limit # Start master server with volume size limit
@nohup $(SEAWEEDFS_BINARY) master -port=$(MASTER_PORT) -mdir=/tmp/seaweedfs-test-copying-master -volumeSizeLimitMB=$(VOLUME_MAX_SIZE_MB) -ip=127.0.0.1 > /tmp/seaweedfs-master.log 2>&1 &
@nohup $(SEAWEEDFS_BINARY) master -port=$(MASTER_PORT) -mdir=/tmp/seaweedfs-test-copying-master -volumeSizeLimitMB=$(VOLUME_MAX_SIZE_MB) -ip=127.0.0.1 -peers=none > /tmp/seaweedfs-master.log 2>&1 &
@sleep 3 @sleep 3
# Start volume server # Start volume server

3
test/s3/iam/Makefile
View File

@ -61,7 +61,8 @@ start-services: ## Start SeaweedFS services for testing
@echo "🚀 Starting SeaweedFS services..." @echo "🚀 Starting SeaweedFS services..."
@echo "Starting master server..." @echo "Starting master server..."
@$(WEED_BINARY) master -port=$(MASTER_PORT) \ @$(WEED_BINARY) master -port=$(MASTER_PORT) \
-mdir=test-volume-data/m9333 > weed-master.log 2>&1 & \
-mdir=test-volume-data/m9333 \
-peers=none > weed-master.log 2>&1 & \
echo $$! > $(MASTER_PID_FILE) echo $$! > $(MASTER_PID_FILE)
@echo "Waiting for master server to be ready..." @echo "Waiting for master server to be ready..."

4
test/s3/sse/Makefile
View File

@ -98,7 +98,7 @@ start-seaweedfs: check-binary
@mkdir -p /tmp/seaweedfs-test-sse-filer @mkdir -p /tmp/seaweedfs-test-sse-filer
# Start master server with volume size limit and explicit gRPC port # Start master server with volume size limit and explicit gRPC port
@nohup $(SEAWEEDFS_BINARY) master -port=$(MASTER_PORT) -port.grpc=$$(( $(MASTER_PORT) + 10000 )) -mdir=/tmp/seaweedfs-test-sse-master -volumeSizeLimitMB=$(VOLUME_MAX_SIZE_MB) -ip=127.0.0.1 > /tmp/seaweedfs-sse-master.log 2>&1 &
@nohup $(SEAWEEDFS_BINARY) master -port=$(MASTER_PORT) -port.grpc=$$(( $(MASTER_PORT) + 10000 )) -mdir=/tmp/seaweedfs-test-sse-master -volumeSizeLimitMB=$(VOLUME_MAX_SIZE_MB) -ip=127.0.0.1 -peers=none > /tmp/seaweedfs-sse-master.log 2>&1 &
@sleep 3 @sleep 3
# Start volume server with master HTTP port and increased capacity # Start volume server with master HTTP port and increased capacity
@ -354,7 +354,7 @@ start-seaweedfs-ci: check-binary
# Start master server with volume size limit and explicit gRPC port # Start master server with volume size limit and explicit gRPC port
@echo "Starting master server..." @echo "Starting master server..."
@nohup $(SEAWEEDFS_BINARY) master -port=$(MASTER_PORT) -port.grpc=$$(( $(MASTER_PORT) + 10000 )) -mdir=/tmp/seaweedfs-test-sse-master -volumeSizeLimitMB=$(VOLUME_MAX_SIZE_MB) -ip=127.0.0.1 > /tmp/seaweedfs-sse-master.log 2>&1 &
@nohup $(SEAWEEDFS_BINARY) master -port=$(MASTER_PORT) -port.grpc=$$(( $(MASTER_PORT) + 10000 )) -mdir=/tmp/seaweedfs-test-sse-master -volumeSizeLimitMB=$(VOLUME_MAX_SIZE_MB) -ip=127.0.0.1 -peers=none > /tmp/seaweedfs-sse-master.log 2>&1 &
@sleep 3 @sleep 3
# Start volume server with master HTTP port and increased capacity # Start volume server with master HTTP port and increased capacity

Write Preview
Loading…
Cancel
Save