From bf398d2266a32229d9b63ab95de7afbf2e7a4227 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Sun, 31 Mar 2024 20:50:30 +0500 Subject: [PATCH 1/8] fix volume Encode --- weed/operation/upload_content.go | 5 ++--- weed/s3api/s3api_object_handlers_postpolicy.go | 2 +- weed/server/filer_server_handlers_write_autochunk.go | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/weed/operation/upload_content.go b/weed/operation/upload_content.go index b48d73deb..f644265ca 100644 --- a/weed/operation/upload_content.go +++ b/weed/operation/upload_content.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/rclone/rclone/lib/encoder" "io" "mime" "mime/multipart" @@ -128,8 +129,6 @@ func UploadWithRetry(filerClient filer_pb.FilerClient, assignRequest *filer_pb.A return } -var fileNameEscaper = strings.NewReplacer(`\`, `\\`, `"`, `\"`, "\n", "") - // Upload sends a POST request to a volume server to upload the content with adjustable compression level func UploadData(data []byte, option *UploadOption) (uploadResult *UploadResult, err error) { uploadResult, err = retriedUploadData(data, option) @@ -279,7 +278,7 @@ func upload_content(fillBufferFunction func(w io.Writer) error, originalDataSize defer PutBuffer(buf) body_writer := multipart.NewWriter(buf) h := make(textproto.MIMEHeader) - filename := fileNameEscaper.Replace(option.Filename) + filename := encoder.Standard.Encode(option.Filename) h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, filename)) h.Set("Idempotency-Key", option.UploadUrl) if option.MimeType == "" { diff --git a/weed/s3api/s3api_object_handlers_postpolicy.go b/weed/s3api/s3api_object_handlers_postpolicy.go index cd80e0ad3..d7915cde1 100644 --- a/weed/s3api/s3api_object_handlers_postpolicy.go +++ b/weed/s3api/s3api_object_handlers_postpolicy.go @@ -126,7 +126,7 @@ func (s3a *S3ApiServer) PostPolicyBucketHandler(w http.ResponseWriter, r *http.R // Add s3 postpolicy support header for k, _ := range formValues { - if k == "Cache-Control" || k == "Expires" || k == "Content-Disposition" { + if k == "Cache-Control" || k == "Expires" || k == "Content-Disposition" || k == "Content-Language" { r.Header.Set(k, formValues.Get(k)) continue } diff --git a/weed/server/filer_server_handlers_write_autochunk.go b/weed/server/filer_server_handlers_write_autochunk.go index f4428be61..c0192f39e 100644 --- a/weed/server/filer_server_handlers_write_autochunk.go +++ b/weed/server/filer_server_handlers_write_autochunk.go @@ -257,7 +257,7 @@ func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileNa for k, v := range r.Header { if len(v) > 0 && len(v[0]) > 0 { - if strings.HasPrefix(k, needle.PairNamePrefix) || k == "Cache-Control" || k == "Expires" || k == "Content-Disposition" { + if strings.HasPrefix(k, needle.PairNamePrefix) || k == "Cache-Control" || k == "Expires" || k == "Content-Disposition" || k == "Content-Language" { entry.Extended[k] = []byte(v[0]) } if k == "Response-Content-Disposition" { From 9ac05672078345a1e2242784a74265ba621cb029 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Sun, 31 Mar 2024 21:41:17 +0500 Subject: [PATCH 2/8] rm import rclone and add test action --- .github/workflows/s3tests.yml | 34 ++++++++++++++++++++++++++++++-- weed/operation/upload_content.go | 6 +++--- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/.github/workflows/s3tests.yml b/.github/workflows/s3tests.yml index dfd3d4acf..70ed73614 100644 --- a/.github/workflows/s3tests.yml +++ b/.github/workflows/s3tests.yml @@ -34,14 +34,18 @@ jobs: go-version-file: 'go.mod' id: go + - name: Build weed + shell: bash + run: | + cd /__w/seaweedfs/seaweedfs/weed + go install -buildvcs=false + - name: Run Ceph S3 tests timeout-minutes: 15 env: S3TEST_CONF: /__w/seaweedfs/seaweedfs/docker/compose/s3tests.conf shell: bash run: | - cd /__w/seaweedfs/seaweedfs/weed - go install -buildvcs=false set -x nohup weed -v 0 server -filer -s3 -ip.bind 0.0.0.0 \ -master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=1024 \ @@ -187,3 +191,29 @@ jobs: s3tests_boto3/functional/test_s3.py::test_ranged_request_return_trailing_bytes_response_code \ s3tests_boto3/functional/test_s3.py::test_copy_object_ifmatch_good \ s3tests_boto3/functional/test_s3.py::test_copy_object_ifnonematch_failed + + - uses: actions/checkout@v4 + with: + repository: "rclone/rclone" + + - name: Run Rclone S3 tests + timeout-minutes: 15 + env: + RCLONE_CONFIG_SWFS_TYPE: s3 + RCLONE_CONFIG_SWFS_PROVIDER: Other + RCLONE_CONFIG_SWFS_ACCESS_KEY_ID: some_access_key1 + RCLONE_CONFIG_SWFS_SECRET_ACCESS_KEY: some_secret_key1 + RCLONE_CONFIG_SWFS_ENDPOINT: http://localhost:8000 + RCLONE_CONFIG_SWFS_UPLOAD_CUTOFF: 50Mi + RCLONE_CONFIG_SWFS_CHUNK_SIZE: 50Mi + RCLONE_CONFIG_SWFS_FORCE_PATH_STYLE: true + shell: bash + run: | + set -x + nohup weed -v 0 server -filer -s3 -ip.bind 0.0.0.0 \ + -master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=1024 \ + -volume.max=100 -volume.preStopSeconds=1 -s3.port=8000 -metricsPort=9324 \ + -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config=../docker/compose/s3.json & + sleep 10 + cd /__w/seaweedfs/seaweedfs/rclone/rclone/backend/s3 + go test -v -remote swfs: diff --git a/weed/operation/upload_content.go b/weed/operation/upload_content.go index f644265ca..aa40eff14 100644 --- a/weed/operation/upload_content.go +++ b/weed/operation/upload_content.go @@ -5,7 +5,6 @@ import ( "context" "encoding/json" "fmt" - "github.com/rclone/rclone/lib/encoder" "io" "mime" "mime/multipart" @@ -67,7 +66,8 @@ type HTTPClient interface { } var ( - HttpClient HTTPClient + HttpClient HTTPClient + fileNameEscaper = strings.NewReplacer(`\`, `\\`, `"`, `\"`) ) func init() { @@ -278,7 +278,7 @@ func upload_content(fillBufferFunction func(w io.Writer) error, originalDataSize defer PutBuffer(buf) body_writer := multipart.NewWriter(buf) h := make(textproto.MIMEHeader) - filename := encoder.Standard.Encode(option.Filename) + filename := fileNameEscaper.Replace(strings.Trim(option.Filename, "\n\r\t\v")) h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, filename)) h.Set("Idempotency-Key", option.UploadUrl) if option.MimeType == "" { From 3d0c5ab0bea6fa44f18c1ea5e39e1854329b3c7b Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Sun, 31 Mar 2024 21:45:10 +0500 Subject: [PATCH 3/8] fix action step Run Rclone S3 tests --- .github/workflows/s3tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/s3tests.yml b/.github/workflows/s3tests.yml index 70ed73614..aa1b2a1d8 100644 --- a/.github/workflows/s3tests.yml +++ b/.github/workflows/s3tests.yml @@ -19,7 +19,7 @@ defaults: jobs: s3tests: - name: Ceph S3 tests + name: S3 tests runs-on: ubuntu-22.04 container: image: docker.io/kmlebedev/ceph-s3-tests:0.0.2 @@ -215,5 +215,5 @@ jobs: -volume.max=100 -volume.preStopSeconds=1 -s3.port=8000 -metricsPort=9324 \ -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config=../docker/compose/s3.json & sleep 10 - cd /__w/seaweedfs/seaweedfs/rclone/rclone/backend/s3 + cd /__w/rclone/rclone/backend/s3 go test -v -remote swfs: From 8e00c02e6ebe0979315ba31b5af2acaa1cde1946 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Mon, 1 Apr 2024 00:23:03 +0500 Subject: [PATCH 4/8] fix path for checkout rclone --- .github/workflows/s3tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/s3tests.yml b/.github/workflows/s3tests.yml index aa1b2a1d8..b33878510 100644 --- a/.github/workflows/s3tests.yml +++ b/.github/workflows/s3tests.yml @@ -195,6 +195,7 @@ jobs: - uses: actions/checkout@v4 with: repository: "rclone/rclone" + path: "/__w/rclone/rclone" - name: Run Rclone S3 tests timeout-minutes: 15 From a9fcb2f7088213a898b745b08a76aa099e709bf8 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Mon, 1 Apr 2024 00:27:31 +0500 Subject: [PATCH 5/8] fix chdir --- .github/workflows/s3tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/s3tests.yml b/.github/workflows/s3tests.yml index b33878510..9934d0561 100644 --- a/.github/workflows/s3tests.yml +++ b/.github/workflows/s3tests.yml @@ -195,7 +195,7 @@ jobs: - uses: actions/checkout@v4 with: repository: "rclone/rclone" - path: "/__w/rclone/rclone" + path: "/__w/seaweedfs/seaweedfs/rclone" - name: Run Rclone S3 tests timeout-minutes: 15 @@ -216,5 +216,5 @@ jobs: -volume.max=100 -volume.preStopSeconds=1 -s3.port=8000 -metricsPort=9324 \ -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config=../docker/compose/s3.json & sleep 10 - cd /__w/rclone/rclone/backend/s3 + cd /__w/seaweedfs/seaweedfs/rclone/backend/s3 go test -v -remote swfs: From b7a889bad83dfeecd206329c8da6033705e4b945 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Mon, 1 Apr 2024 01:06:51 +0500 Subject: [PATCH 6/8] cp config --- .github/workflows/s3tests.yml | 17 ++--------------- docker/compose/rclone.conf | 9 +++++++++ 2 files changed, 11 insertions(+), 15 deletions(-) create mode 100644 docker/compose/rclone.conf diff --git a/.github/workflows/s3tests.yml b/.github/workflows/s3tests.yml index 9934d0561..5911450a2 100644 --- a/.github/workflows/s3tests.yml +++ b/.github/workflows/s3tests.yml @@ -199,22 +199,9 @@ jobs: - name: Run Rclone S3 tests timeout-minutes: 15 - env: - RCLONE_CONFIG_SWFS_TYPE: s3 - RCLONE_CONFIG_SWFS_PROVIDER: Other - RCLONE_CONFIG_SWFS_ACCESS_KEY_ID: some_access_key1 - RCLONE_CONFIG_SWFS_SECRET_ACCESS_KEY: some_secret_key1 - RCLONE_CONFIG_SWFS_ENDPOINT: http://localhost:8000 - RCLONE_CONFIG_SWFS_UPLOAD_CUTOFF: 50Mi - RCLONE_CONFIG_SWFS_CHUNK_SIZE: 50Mi - RCLONE_CONFIG_SWFS_FORCE_PATH_STYLE: true shell: bash run: | - set -x - nohup weed -v 0 server -filer -s3 -ip.bind 0.0.0.0 \ - -master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=1024 \ - -volume.max=100 -volume.preStopSeconds=1 -s3.port=8000 -metricsPort=9324 \ - -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config=../docker/compose/s3.json & - sleep 10 + mkdir -p ~/.config/rclone + cp /__w/seaweedfs/seaweedfs/docker/compose/rclone.conf ~/.config/rclone/rclone.conf cd /__w/seaweedfs/seaweedfs/rclone/backend/s3 go test -v -remote swfs: diff --git a/docker/compose/rclone.conf b/docker/compose/rclone.conf new file mode 100644 index 000000000..27ba0c772 --- /dev/null +++ b/docker/compose/rclone.conf @@ -0,0 +1,9 @@ +[swfs] +type = s3 +provider = Other +access_key_id = some_access_key1 +secret_access_key = some_secret_key1 +endpoint = http://localhost:8000 +upload_cutoff = 50Mi +chunk_size = 50Mi +force_path_style = true \ No newline at end of file From 073931d943c7ac772079d7c02c935a03f495f1b5 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Mon, 1 Apr 2024 01:12:04 +0500 Subject: [PATCH 7/8] kill weed --- .github/workflows/s3tests.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/s3tests.yml b/.github/workflows/s3tests.yml index 5911450a2..c2e9a2529 100644 --- a/.github/workflows/s3tests.yml +++ b/.github/workflows/s3tests.yml @@ -191,6 +191,7 @@ jobs: s3tests_boto3/functional/test_s3.py::test_ranged_request_return_trailing_bytes_response_code \ s3tests_boto3/functional/test_s3.py::test_copy_object_ifmatch_good \ s3tests_boto3/functional/test_s3.py::test_copy_object_ifnonematch_failed + pkill weed - uses: actions/checkout@v4 with: @@ -201,7 +202,14 @@ jobs: timeout-minutes: 15 shell: bash run: | + set -x + nohup weed -v 0 server -filer -s3 -ip.bind 0.0.0.0 \ + -master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=1024 \ + -volume.max=100 -volume.preStopSeconds=1 -s3.port=8000 -metricsPort=9324 \ + -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config=../docker/compose/s3.json & + sleep 10 mkdir -p ~/.config/rclone cp /__w/seaweedfs/seaweedfs/docker/compose/rclone.conf ~/.config/rclone/rclone.conf cd /__w/seaweedfs/seaweedfs/rclone/backend/s3 go test -v -remote swfs: + pkill weed From d8f29aec7d8fcd2c6a8db717d20d54132f3bbe3c Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Mon, 1 Apr 2024 01:18:00 +0500 Subject: [PATCH 8/8] try run weed once --- .github/workflows/s3tests.yml | 22 +++++++--------------- Makefile | 2 +- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/.github/workflows/s3tests.yml b/.github/workflows/s3tests.yml index c2e9a2529..6fa2ba5bb 100644 --- a/.github/workflows/s3tests.yml +++ b/.github/workflows/s3tests.yml @@ -34,11 +34,17 @@ jobs: go-version-file: 'go.mod' id: go - - name: Build weed + - name: Build an run weed shell: bash run: | cd /__w/seaweedfs/seaweedfs/weed go install -buildvcs=false + set -x + nohup weed -v 0 server -filer -s3 -ip.bind 0.0.0.0 \ + -master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=1024 \ + -volume.max=100 -volume.preStopSeconds=1 -s3.port=8000 -metricsPort=9324 \ + -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config=../docker/compose/s3.json & + sleep 10 - name: Run Ceph S3 tests timeout-minutes: 15 @@ -46,12 +52,6 @@ jobs: S3TEST_CONF: /__w/seaweedfs/seaweedfs/docker/compose/s3tests.conf shell: bash run: | - set -x - nohup weed -v 0 server -filer -s3 -ip.bind 0.0.0.0 \ - -master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=1024 \ - -volume.max=100 -volume.preStopSeconds=1 -s3.port=8000 -metricsPort=9324 \ - -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config=../docker/compose/s3.json & - sleep 10 cd /s3-tests tox -- \ s3tests_boto3/functional/test_s3.py::test_bucket_list_empty \ @@ -191,7 +191,6 @@ jobs: s3tests_boto3/functional/test_s3.py::test_ranged_request_return_trailing_bytes_response_code \ s3tests_boto3/functional/test_s3.py::test_copy_object_ifmatch_good \ s3tests_boto3/functional/test_s3.py::test_copy_object_ifnonematch_failed - pkill weed - uses: actions/checkout@v4 with: @@ -202,14 +201,7 @@ jobs: timeout-minutes: 15 shell: bash run: | - set -x - nohup weed -v 0 server -filer -s3 -ip.bind 0.0.0.0 \ - -master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=1024 \ - -volume.max=100 -volume.preStopSeconds=1 -s3.port=8000 -metricsPort=9324 \ - -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config=../docker/compose/s3.json & - sleep 10 mkdir -p ~/.config/rclone cp /__w/seaweedfs/seaweedfs/docker/compose/rclone.conf ~/.config/rclone/rclone.conf cd /__w/seaweedfs/seaweedfs/rclone/backend/s3 go test -v -remote swfs: - pkill weed diff --git a/Makefile b/Makefile index 269b6ec91..21d1ed45e 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ full_install: cd weed; go install -tags "elastic gocdk sqlite ydb tikv rclone" server: install - weed -v 4 server -s3 -filer -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1 -s3.port=8000 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=false -s3.config=./docker/compose/s3.json -metricsPort=9324 + weed -v 0 server -s3 -filer -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1 -s3.port=8000 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=false -s3.config=./docker/compose/s3.json -metricsPort=9324 benchmark: install warp_install pkill weed || true