chrislu
2 years ago
58 changed files with 584 additions and 209 deletions
-
4.github/workflows/codeql.yml
-
89.github/workflows/e2e.yml
-
2.github/workflows/go.yml
-
30docker/Dockerfile.e2e
-
2docker/Dockerfile.gccgo_build
-
2docker/Dockerfile.go_build
-
1docker/Dockerfile.local
-
16docker/Dockerfile.rocksdb_dev_env
-
4docker/Dockerfile.rocksdb_large
-
45docker/Dockerfile.rocksdb_large_local
-
20docker/Makefile
-
53docker/compose/e2e-mount.yml
-
33go.mod
-
73go.sum
-
4k8s/helm_charts2/Chart.yaml
-
2other/java/hdfs2/pom.xml
-
10unmaintained/s3/presigned_put/presigned_put.go
-
31weed/command/filer.go
-
22weed/command/filer_sync.go
-
4weed/command/mount_linux.go
-
2weed/command/mount_std.go
-
2weed/command/s3.go
-
15weed/command/update.go
-
2weed/filer/arangodb/arangodb_store.go
-
3weed/filer/arangodb/arangodb_store_bucket.go
-
2weed/filer/arangodb/helpers.go
-
4weed/filer/filer.go
-
2weed/filer/filer_deletion.go
-
5weed/filer/meta_aggregator.go
-
2weed/filer/rocksdb/rocksdb_store.go
-
2weed/filer/s3iam_conf.go
-
35weed/filer/s3iam_conf_test.go
-
11weed/mount/filehandle.go
-
4weed/mount/weedfs_file_read.go
-
31weed/notification/gocdk_pub_sub/gocdk_pub_sub.go
-
5weed/operation/chunked_file.go
-
3weed/operation/upload_content.go
-
1weed/replication/sink/filersink/filer_sink.go
-
2weed/replication/sub/notification_gocdk_pub_sub.go
-
3weed/s3api/s3api_object_copy_handlers.go
-
6weed/s3api/s3api_server.go
-
1weed/server/common.go
-
2weed/server/filer_server_handlers_read.go
-
2weed/server/filer_server_handlers_write_autochunk.go
-
12weed/server/filer_server_handlers_write_upload.go
-
4weed/server/raft_hashicorp.go
-
14weed/shell/command_fs_meta_load.go
-
4weed/shell/command_volume_check_disk.go
-
35weed/shell/command_volume_tier_move.go
-
15weed/storage/backend/s3_backend/s3_backend.go
-
2weed/storage/store.go
-
5weed/storage/volume.go
-
9weed/storage/volume_read.go
-
12weed/topology/volume_layout.go
-
2weed/util/constants.go
-
6weed/util/grace/signal_handling.go
-
19weed/util/http_util.go
-
36weed/wdclient/masterclient.go
@ -0,0 +1,89 @@ |
|||
name: "End to End" |
|||
|
|||
on: |
|||
push: |
|||
branches: [ master ] |
|||
pull_request: |
|||
branches: [ master ] |
|||
|
|||
concurrency: |
|||
group: ${{ github.head_ref }}/e2e |
|||
cancel-in-progress: true |
|||
|
|||
permissions: |
|||
contents: read |
|||
|
|||
defaults: |
|||
run: |
|||
working-directory: docker |
|||
|
|||
jobs: |
|||
e2e: |
|||
name: FUSE Mount |
|||
runs-on: ubuntu-22.04 |
|||
timeout-minutes: 15 |
|||
steps: |
|||
- name: Set up Go 1.x |
|||
uses: actions/setup-go@268d8c0ca0432bb2cf416faae41297df9d262d7f # v2 |
|||
with: |
|||
go-version: ^1.13 |
|||
id: go |
|||
|
|||
- name: Check out code into the Go module directory |
|||
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2 |
|||
|
|||
- name: Install dependencies |
|||
run: | |
|||
sudo apt-get update |
|||
sudo apt-get install -y fuse |
|||
|
|||
- name: Start SeaweedFS |
|||
timeout-minutes: 5 |
|||
run: make build_e2e && docker compose -f ./compose/e2e-mount.yml up --wait |
|||
|
|||
- name: Run FIO |
|||
timeout-minutes: 5 |
|||
run: | |
|||
echo "Starting FIO at: $(date)" |
|||
# Concurrent r/w |
|||
echo 'Run randrw with size=16M bs=4k' |
|||
docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 40 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randrw --bs=4k --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1 |
|||
|
|||
echo 'Run randrw with size=16M bs=128k' |
|||
docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 40 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randrw --bs=128k --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1 |
|||
|
|||
echo 'Run randrw with size=16M bs=1m' |
|||
docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 40 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randrw --bs=1m --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1 |
|||
|
|||
# Verified write |
|||
echo 'Run randwrite with size=16M bs=4k' |
|||
docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 40 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randwrite --bs=4k --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1 --do_verify=0 --verify=crc32c --verify_backlog=1 |
|||
|
|||
echo 'Run randwrite with size=16M bs=128k' |
|||
docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 40 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randwrite --bs=128k --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1 --do_verify=0 --verify=crc32c --verify_backlog=1 |
|||
|
|||
echo 'Run randwrite with size=16M bs=1m' |
|||
docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 40 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randwrite --bs=1m --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1 --do_verify=0 --verify=crc32c --verify_backlog=1 |
|||
|
|||
- name: Save logs |
|||
if: always() |
|||
run: | |
|||
docker compose -f ./compose/e2e-mount.yml logs > output.log |
|||
echo 'Showing last 500 log lines of mount service:' |
|||
docker compose -f ./compose/e2e-mount.yml logs --tail 500 mount |
|||
|
|||
- name: Check for data races |
|||
if: always() |
|||
continue-on-error: true # TODO: remove this comment to enable build failure on data races (after all are fixed) |
|||
run: grep -A50 'DATA RACE' output.log && exit 1 || exit 0 |
|||
|
|||
- name: Archive logs |
|||
if: always() |
|||
uses: actions/upload-artifact@v3 |
|||
with: |
|||
name: output-logs |
|||
path: docker/output.log |
|||
|
|||
- name: Cleanup |
|||
if: always() |
|||
run: docker compose -f ./compose/e2e-mount.yml down --volumes --remove-orphans --rmi all |
|||
@ -0,0 +1,30 @@ |
|||
FROM ubuntu:22.04 |
|||
|
|||
LABEL author="Chris Lu" |
|||
|
|||
RUN apt-get update && apt-get install -y curl fio fuse |
|||
RUN mkdir -p /etc/seaweedfs /data/filerldb2 |
|||
|
|||
COPY ./weed /usr/bin/ |
|||
COPY ./filer.toml /etc/seaweedfs/filer.toml |
|||
COPY ./entrypoint.sh /entrypoint.sh |
|||
|
|||
# volume server grpc port |
|||
EXPOSE 18080 |
|||
# volume server http port |
|||
EXPOSE 8080 |
|||
# filer server grpc port |
|||
EXPOSE 18888 |
|||
# filer server http port |
|||
EXPOSE 8888 |
|||
# master server shared grpc port |
|||
EXPOSE 19333 |
|||
# master server shared http port |
|||
EXPOSE 9333 |
|||
|
|||
VOLUME /data |
|||
WORKDIR /data |
|||
|
|||
RUN chmod +x /entrypoint.sh |
|||
|
|||
ENTRYPOINT ["/entrypoint.sh"] |
|||
@ -0,0 +1,16 @@ |
|||
FROM golang:1.19-buster as builder |
|||
|
|||
RUN apt-get update |
|||
RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev |
|||
|
|||
ENV ROCKSDB_VERSION v7.5.3 |
|||
|
|||
# build RocksDB |
|||
RUN cd /tmp && \ |
|||
git clone https://github.com/facebook/rocksdb.git /tmp/rocksdb --depth 1 --single-branch --branch $ROCKSDB_VERSION && \ |
|||
cd rocksdb && \ |
|||
PORTABLE=1 make static_lib && \ |
|||
make install-static |
|||
|
|||
ENV CGO_CFLAGS "-I/tmp/rocksdb/include" |
|||
ENV CGO_LDFLAGS "-L/tmp/rocksdb -lrocksdb -lstdc++ -lm -lz -lbz2 -lsnappy -llz4 -lzstd" |
|||
@ -0,0 +1,45 @@ |
|||
FROM chrislusf/rocksdb_dev_env as builder |
|||
|
|||
# build SeaweedFS |
|||
RUN mkdir -p /go/src/github.com/seaweedfs/ |
|||
ADD . /go/src/github.com/seaweedfs/seaweedfs |
|||
RUN ls -al /go/src/github.com/seaweedfs/ && \ |
|||
cd /go/src/github.com/seaweedfs/seaweedfs/weed \ |
|||
&& export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \ |
|||
&& go install -tags "5BytesOffset rocksdb" -ldflags "-extldflags -static ${LDFLAGS}" |
|||
|
|||
|
|||
FROM alpine AS final |
|||
LABEL author="Chris Lu" |
|||
COPY --from=builder /go/bin/weed /usr/bin/ |
|||
RUN mkdir -p /etc/seaweedfs |
|||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer_rocksdb.toml /etc/seaweedfs/filer.toml |
|||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh |
|||
RUN apk add fuse snappy gflags tmux |
|||
|
|||
# volume server gprc port |
|||
EXPOSE 18080 |
|||
# volume server http port |
|||
EXPOSE 8080 |
|||
# filer server gprc port |
|||
EXPOSE 18888 |
|||
# filer server http port |
|||
EXPOSE 8888 |
|||
# master server shared gprc port |
|||
EXPOSE 19333 |
|||
# master server shared http port |
|||
EXPOSE 9333 |
|||
# s3 server http port |
|||
EXPOSE 8333 |
|||
# webdav server http port |
|||
EXPOSE 7333 |
|||
|
|||
RUN mkdir -p /data/filer_rocksdb |
|||
|
|||
VOLUME /data |
|||
|
|||
WORKDIR /data |
|||
|
|||
RUN chmod +x /entrypoint.sh |
|||
|
|||
ENTRYPOINT ["/entrypoint.sh"] |
|||
@ -0,0 +1,53 @@ |
|||
version: '3.9' |
|||
|
|||
services: |
|||
master: |
|||
image: chrislusf/seaweedfs:e2e |
|||
command: "-v=4 master -ip=master -ip.bind=0.0.0.0 -raftBootstrap" |
|||
healthcheck: |
|||
test: [ "CMD", "curl", "--fail", "-I", "http://localhost:9333/cluster/healthz" ] |
|||
interval: 1s |
|||
timeout: 60s |
|||
|
|||
volume: |
|||
image: chrislusf/seaweedfs:e2e |
|||
command: "-v=4 volume -mserver=master:9333 -ip=volume -ip.bind=0.0.0.0 -preStopSeconds=1" |
|||
healthcheck: |
|||
test: [ "CMD", "curl", "--fail", "-I", "http://localhost:8080/healthz" ] |
|||
interval: 1s |
|||
timeout: 30s |
|||
depends_on: |
|||
master: |
|||
condition: service_healthy |
|||
|
|||
filer: |
|||
image: chrislusf/seaweedfs:e2e |
|||
command: "-v=4 filer -master=master:9333 -ip=filer -ip.bind=0.0.0.0" |
|||
healthcheck: |
|||
test: [ "CMD", "curl", "--fail", "-I", "http://localhost:8888" ] |
|||
interval: 1s |
|||
timeout: 30s |
|||
depends_on: |
|||
volume: |
|||
condition: service_healthy |
|||
|
|||
mount: |
|||
image: chrislusf/seaweedfs:e2e |
|||
command: "-v=4 mount -filer=filer:8888 -filer.path=/ -dirAutoCreate -dir=/mnt/seaweedfs" |
|||
cap_add: |
|||
- SYS_ADMIN |
|||
devices: |
|||
- /dev/fuse |
|||
security_opt: |
|||
- apparmor:unconfined |
|||
deploy: |
|||
resources: |
|||
limits: |
|||
memory: 4096m |
|||
healthcheck: |
|||
test: [ "CMD", "mountpoint", "-q", "--", "/mnt/seaweedfs" ] |
|||
interval: 1s |
|||
timeout: 30s |
|||
depends_on: |
|||
filer: |
|||
condition: service_healthy |
|||
@ -1,5 +1,5 @@ |
|||
apiVersion: v1 |
|||
description: SeaweedFS |
|||
name: seaweedfs |
|||
appVersion: "3.24" |
|||
version: "3.24" |
|||
appVersion: "3.25" |
|||
version: "3.25" |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue