You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
187 lines
7.3 KiB
187 lines
7.3 KiB
# Makefile for EC integration testing
|
|
# Usage:
|
|
# make start - Start the test cluster (master + 6 volume servers + filer)
|
|
# make stop - Stop the test cluster
|
|
# make populate - Populate test data (~300MB across 7 volumes)
|
|
# make shell - Open weed shell connected to the test cluster
|
|
# make clean - Stop cluster and remove all test data
|
|
# make setup - Start cluster and populate data (one command)
|
|
#
|
|
# Requirements: curl, jq
|
|
|
|
WEED_BINARY := $(shell pwd)/../../weed/weed
|
|
TEST_DIR := /tmp/ec_manual_test
|
|
# Use non-standard ports to avoid conflicts with existing SeaweedFS servers
|
|
MASTER_PORT := 29333
|
|
FILER_PORT := 28888
|
|
VOLUME_BASE_PORT := 28080
|
|
NUM_VOLUME_SERVERS := 6
|
|
VOLUME_SIZE_LIMIT_MB := 30
|
|
MAX_VOLUMES_PER_SERVER := 10
|
|
|
|
# Build weed binary if it doesn't exist
|
|
$(WEED_BINARY):
|
|
cd ../../weed && go build -o weed .
|
|
|
|
.PHONY: build
|
|
build: $(WEED_BINARY)
|
|
|
|
.PHONY: start
|
|
start: build
|
|
@echo "=== Starting SeaweedFS test cluster ==="
|
|
@mkdir -p $(TEST_DIR)/master $(TEST_DIR)/filer
|
|
@for i in $$(seq 0 $$(($(NUM_VOLUME_SERVERS)-1))); do mkdir -p $(TEST_DIR)/volume$$i; done
|
|
@# Create security.toml with JWT disabled
|
|
@echo "# Disable JWT for testing" > $(TEST_DIR)/security.toml
|
|
@echo '[jwt.signing]' >> $(TEST_DIR)/security.toml
|
|
@echo 'key = ""' >> $(TEST_DIR)/security.toml
|
|
@echo 'expires_after_seconds = 0' >> $(TEST_DIR)/security.toml
|
|
@echo '' >> $(TEST_DIR)/security.toml
|
|
@echo '[jwt.signing.read]' >> $(TEST_DIR)/security.toml
|
|
@echo 'key = ""' >> $(TEST_DIR)/security.toml
|
|
@echo 'expires_after_seconds = 0' >> $(TEST_DIR)/security.toml
|
|
@# Create filer.toml with leveldb2
|
|
@echo '[leveldb2]' > $(TEST_DIR)/filer.toml
|
|
@echo 'enabled = true' >> $(TEST_DIR)/filer.toml
|
|
@echo 'dir = "$(TEST_DIR)/filer/filerldb2"' >> $(TEST_DIR)/filer.toml
|
|
@# Start master
|
|
@echo "Starting master on port $(MASTER_PORT)..."
|
|
@cd $(TEST_DIR) && $(WEED_BINARY) master \
|
|
-port=$(MASTER_PORT) \
|
|
-mdir=$(TEST_DIR)/master \
|
|
-volumeSizeLimitMB=$(VOLUME_SIZE_LIMIT_MB) \
|
|
-ip=127.0.0.1 \
|
|
> $(TEST_DIR)/master/master.log 2>&1 & echo $$! > $(TEST_DIR)/master.pid
|
|
@sleep 3
|
|
@# Start volume servers (run from TEST_DIR to find security.toml)
|
|
@for i in $$(seq 0 $$(($(NUM_VOLUME_SERVERS)-1))); do \
|
|
port=$$(($(VOLUME_BASE_PORT) + $$i)); \
|
|
echo "Starting volume server $$i on port $$port (rack$$i)..."; \
|
|
cd $(TEST_DIR) && $(WEED_BINARY) volume \
|
|
-port=$$port \
|
|
-dir=$(TEST_DIR)/volume$$i \
|
|
-max=$(MAX_VOLUMES_PER_SERVER) \
|
|
-master=127.0.0.1:$(MASTER_PORT) \
|
|
-ip=127.0.0.1 \
|
|
-dataCenter=dc1 \
|
|
-rack=rack$$i \
|
|
> $(TEST_DIR)/volume$$i/volume.log 2>&1 & echo $$! > $(TEST_DIR)/volume$$i.pid; \
|
|
done
|
|
@sleep 3
|
|
@# Start filer (run from TEST_DIR to find security.toml)
|
|
@echo "Starting filer on port $(FILER_PORT)..."
|
|
@cd $(TEST_DIR) && $(WEED_BINARY) filer \
|
|
-port=$(FILER_PORT) \
|
|
-master=127.0.0.1:$(MASTER_PORT) \
|
|
-ip=127.0.0.1 \
|
|
> $(TEST_DIR)/filer/filer.log 2>&1 & echo $$! > $(TEST_DIR)/filer.pid
|
|
@sleep 3
|
|
@echo ""
|
|
@echo "=== Cluster started ==="
|
|
@echo "Master: http://127.0.0.1:$(MASTER_PORT)"
|
|
@echo "Filer: http://127.0.0.1:$(FILER_PORT)"
|
|
@echo "Volume servers: http://127.0.0.1:$(VOLUME_BASE_PORT) - http://127.0.0.1:$$(($(VOLUME_BASE_PORT) + $(NUM_VOLUME_SERVERS) - 1))"
|
|
@echo ""
|
|
@echo "Run 'make shell' to open weed shell"
|
|
@echo "Run 'make populate' to add test data"
|
|
|
|
.PHONY: stop
|
|
stop:
|
|
@echo "=== Stopping SeaweedFS test cluster ==="
|
|
@# Stop filer by PID
|
|
@-[ -f $(TEST_DIR)/filer.pid ] && kill $$(cat $(TEST_DIR)/filer.pid) 2>/dev/null && rm -f $(TEST_DIR)/filer.pid || true
|
|
@# Stop volume servers by PID
|
|
@for i in $$(seq 0 $$(($(NUM_VOLUME_SERVERS)-1))); do \
|
|
[ -f $(TEST_DIR)/volume$$i.pid ] && kill $$(cat $(TEST_DIR)/volume$$i.pid) 2>/dev/null && rm -f $(TEST_DIR)/volume$$i.pid || true; \
|
|
done
|
|
@# Stop master by PID
|
|
@-[ -f $(TEST_DIR)/master.pid ] && kill $$(cat $(TEST_DIR)/master.pid) 2>/dev/null && rm -f $(TEST_DIR)/master.pid || true
|
|
@# Fallback: use pkill with specific patterns to ensure cleanup
|
|
@-pkill -f "weed filer.*-master=127.0.0.1:$(MASTER_PORT)" 2>/dev/null || true
|
|
@-pkill -f "weed volume.*-dir=$(TEST_DIR)/volume" 2>/dev/null || true
|
|
@-pkill -f "weed master.*-mdir=$(TEST_DIR)/master" 2>/dev/null || true
|
|
@echo "Cluster stopped."
|
|
|
|
.PHONY: clean
|
|
clean: stop
|
|
@echo "Removing test data..."
|
|
@rm -rf $(TEST_DIR)
|
|
@echo "Clean complete."
|
|
|
|
.PHONY: populate
|
|
populate:
|
|
@echo "=== Populating test data (~300MB) ==="
|
|
@# Create a 500KB test file template using mktemp for isolation
|
|
@tmpfile=$$(mktemp) && \
|
|
dd if=/dev/urandom bs=1024 count=500 of=$$tmpfile 2>/dev/null && \
|
|
uploaded=0; \
|
|
for i in $$(seq 1 600); do \
|
|
response=$$(curl -s "http://127.0.0.1:$(MASTER_PORT)/dir/assign?collection=ectest&replication=000"); \
|
|
fid=$$(echo $$response | jq -r '.fid'); \
|
|
url=$$(echo $$response | jq -r '.url'); \
|
|
if [ "$$fid" != "null" ] && [ -n "$$fid" ]; then \
|
|
curl -s -F "file=@$$tmpfile;filename=file_$$i.bin" "http://$$url/$$fid" > /dev/null; \
|
|
uploaded=$$((uploaded + 1)); \
|
|
fi; \
|
|
if [ $$((i % 100)) -eq 0 ]; then \
|
|
echo "Uploaded $$uploaded files..."; \
|
|
fi; \
|
|
done; \
|
|
rm -f $$tmpfile; \
|
|
echo ""; \
|
|
echo "=== Data population complete ==="; \
|
|
echo "Uploaded $$uploaded files (~$$((uploaded * 500 / 1024))MB)"
|
|
@echo ""
|
|
@echo "Volume status:"
|
|
@curl -s "http://127.0.0.1:$(MASTER_PORT)/vol/status" | jq -r \
|
|
'.Volumes.DataCenters.dc1 | to_entries[] | .key as $$rack | .value | to_entries[] | select(.value != null) | .key as $$server | .value[] | select(.Collection == "ectest") | " Volume \(.Id): \(.FileCount) files, \((.Size/1048576*10|floor)/10)MB - \($$rack)"' 2>/dev/null || true
|
|
|
|
.PHONY: shell
|
|
shell: build
|
|
@echo "Opening weed shell..."
|
|
@echo "Commands to try:"
|
|
@echo " lock"
|
|
@echo " volume.list"
|
|
@echo " ec.encode -collection=ectest -quietFor=1s -force"
|
|
@echo " ec.balance -collection=ectest"
|
|
@echo " unlock"
|
|
@echo ""
|
|
@$(WEED_BINARY) shell -master=127.0.0.1:$(MASTER_PORT) -filer=127.0.0.1:$(FILER_PORT)
|
|
|
|
.PHONY: setup
|
|
setup: clean start
|
|
@sleep 2
|
|
@$(MAKE) populate
|
|
|
|
.PHONY: status
|
|
status:
|
|
@echo "=== Cluster Status ==="
|
|
@curl -s "http://127.0.0.1:$(MASTER_PORT)/vol/status" | jq -r \
|
|
'.Volumes.DataCenters.dc1 | to_entries[] | .key as $$rack | .value | to_entries[] | select(.value != null) | .key as $$server | .value[] | select(.Collection == "ectest") | "Volume \(.Id): \(.FileCount) files, \((.Size/1048576*10|floor)/10)MB - \($$rack) (\($$server))"' 2>/dev/null | sort -t: -k1 -n || echo "Cluster not running"
|
|
@echo ""
|
|
@echo "=== EC Shards ==="
|
|
@for i in $$(seq 0 $$(($(NUM_VOLUME_SERVERS)-1))); do \
|
|
count=$$(ls $(TEST_DIR)/volume$$i/*.ec[0-9]* 2>/dev/null | wc -l | tr -d ' '); \
|
|
if [ "$$count" != "0" ]; then \
|
|
echo " volume$$i (port $$(($(VOLUME_BASE_PORT) + $$i))): $$count EC shard files"; \
|
|
fi; \
|
|
done
|
|
|
|
.PHONY: help
|
|
help:
|
|
@echo "EC Integration Test Makefile"
|
|
@echo ""
|
|
@echo "Targets:"
|
|
@echo " make start - Start test cluster (master + 6 volume servers + filer)"
|
|
@echo " make stop - Stop test cluster"
|
|
@echo " make populate - Populate ~300MB of test data"
|
|
@echo " make shell - Open weed shell"
|
|
@echo " make setup - Clean, start, and populate (all-in-one)"
|
|
@echo " make status - Show cluster and EC shard status"
|
|
@echo " make clean - Stop cluster and remove all test data"
|
|
@echo " make help - Show this help"
|
|
@echo ""
|
|
@echo "Quick start:"
|
|
@echo " make setup # Start cluster and populate data"
|
|
@echo " make shell # Open shell to run EC commands"
|
|
|