hilimd
4 years ago
committed by
GitHub
GitHub
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
203 changed files with 5921 additions and 3233 deletions
-
4.travis.yml
-
4docker/Dockerfile
-
31docker/Dockerfile.s3tests
-
6docker/Makefile
-
6docker/compose/local-cluster-compose.yml
-
4docker/compose/local-mount-compose.yml
-
45docker/compose/local-s3tests-compose.yml
-
0docker/compose/notification.toml
-
0docker/compose/replication.toml
-
105docker/compose/s3.json
-
70docker/compose/s3tests.conf
-
5go.mod
-
18go.sum
-
29k8s/README.md
-
4k8s/seaweedfs/Chart.yaml
-
13k8s/seaweedfs/templates/cronjob.yaml
-
1k8s/seaweedfs/templates/filer-service-client.yaml
-
29k8s/seaweedfs/templates/filer-statefulset.yaml
-
10k8s/seaweedfs/templates/s3-deployment.yaml
-
10k8s/seaweedfs/templates/s3-service.yaml
-
21k8s/seaweedfs/templates/seaweedfs-s3-secret.yaml
-
36k8s/seaweedfs/values.yaml
-
2other/java/client/pom.xml
-
2other/java/client/pom.xml.deploy
-
2other/java/client/pom_debug.xml
-
26other/java/client/src/main/java/seaweedfs/client/FileChunkManifest.java
-
57other/java/client/src/main/java/seaweedfs/client/FilerClient.java
-
17other/java/client/src/main/java/seaweedfs/client/SeaweedInputStream.java
-
18other/java/client/src/main/java/seaweedfs/client/SeaweedOutputStream.java
-
18other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java
-
22other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java
-
13other/java/client/src/main/proto/filer.proto
-
4other/java/examples/pom.xml
-
6other/java/examples/src/main/java/com/seaweedfs/examples/ExampleReadFile.java
-
13other/java/examples/src/main/java/com/seaweedfs/examples/ExampleWriteFile.java
-
2other/java/hdfs2/dependency-reduced-pom.xml
-
2other/java/hdfs2/pom.xml
-
16other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java
-
6other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedHadoopInputStream.java
-
6other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedHadoopOutputStream.java
-
2other/java/hdfs3/dependency-reduced-pom.xml
-
2other/java/hdfs3/pom.xml
-
16other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java
-
6other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedHadoopInputStream.java
-
6other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedHadoopOutputStream.java
-
2unmaintained/change_superblock/change_superblock.go
-
2unmaintained/diff_volume_servers/diff_volume_servers.go
-
2unmaintained/stress_filer_upload/bench_filer_upload/bench_filer_upload.go
-
2unmaintained/volume_tailer/volume_tailer.go
-
2weed/command/backup.go
-
9weed/command/benchmark.go
-
2weed/command/command.go
-
12weed/command/download.go
-
2weed/command/export.go
-
4weed/command/filer.go
-
8weed/command/filer_copy.go
-
18weed/command/filer_sync.go
-
2weed/command/mount.go
-
4weed/command/mount_std.go
-
6weed/command/scaffold.go
-
3weed/command/server.go
-
6weed/command/upload.go
-
20weed/command/volume.go
-
6weed/command/webdav.go
-
2weed/filer/abstract_sql/abstract_sql_store.go
-
1weed/filer/entry.go
-
2weed/filer/entry_codec.go
-
2weed/filer/filechunk_manifest.go
-
12weed/filer/filer_conf.go
-
5weed/filer/filer_notify.go
-
2weed/filer/filer_notify_append.go
-
2weed/filer/filer_on_meta_event.go
-
14weed/filer/mysql/mysql_sql_gen.go
-
6weed/filer/mysql/mysql_store.go
-
6weed/filer/mysql2/mysql2_store.go
-
14weed/filer/postgres/postgres_sql_gen.go
-
13weed/filer/postgres/postgres_store.go
-
13weed/filer/postgres2/postgres2_store.go
-
2weed/filer/read_write.go
-
2weed/filer/stream.go
-
7weed/filesys/filehandle.go
-
3weed/filesys/wfs.go
-
1weed/filesys/wfs_write.go
-
2weed/glog/glog.go
-
2weed/glog/glog_file.go
-
35weed/notification/gocdk_pub_sub/gocdk_pub_sub.go
-
9weed/operation/assign_file_id.go
-
8weed/operation/chunked_file.go
-
4weed/operation/delete_content.go
-
15weed/operation/lookup.go
-
29weed/operation/submit.go
-
4weed/operation/tail_volume.go
-
21weed/operation/upload_content.go
-
13weed/pb/filer.proto
-
1068weed/pb/filer_pb/filer.pb.go
-
3weed/pb/grpc_client_server.go
-
43weed/pb/master.proto
-
1586weed/pb/master_pb/master.pb.go
-
4weed/pb/volume_server.proto
-
1428weed/pb/volume_server_pb/volume_server.pb.go
@ -0,0 +1,31 @@ |
|||||
|
FROM ubuntu:20.04 |
||||
|
|
||||
|
RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ |
||||
|
DEBIAN_FRONTEND=noninteractive apt-get upgrade -y && \ |
||||
|
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ |
||||
|
git \ |
||||
|
sudo \ |
||||
|
debianutils \ |
||||
|
python3-pip \ |
||||
|
python3-virtualenv \ |
||||
|
python3-dev \ |
||||
|
libevent-dev \ |
||||
|
libffi-dev \ |
||||
|
libxml2-dev \ |
||||
|
libxslt-dev \ |
||||
|
zlib1g-dev && \ |
||||
|
DEBIAN_FRONTEND=noninteractive apt-get clean && \ |
||||
|
rm -rf /var/lib/apt/lists/* && \ |
||||
|
git clone https://github.com/ceph/s3-tests.git /opt/s3-tests |
||||
|
|
||||
|
WORKDIR /opt/s3-tests |
||||
|
RUN ./bootstrap |
||||
|
|
||||
|
ENV \ |
||||
|
NOSETESTS_EXCLUDE="" \ |
||||
|
NOSETESTS_ATTR="" \ |
||||
|
NOSETESTS_OPTIONS="" \ |
||||
|
S3TEST_CONF="/s3test.conf" |
||||
|
|
||||
|
ENTRYPOINT ["/bin/bash", "-c"] |
||||
|
CMD ["exec ./virtualenv/bin/nosetests ${NOSETESTS_OPTIONS-} ${NOSETESTS_ATTR:+-a $NOSETESTS_ATTR} ${NOSETESTS_EXCLUDE:+-e $NOSETESTS_EXCLUDE}"] |
||||
@ -0,0 +1,45 @@ |
|||||
|
version: '2' |
||||
|
|
||||
|
services: |
||||
|
master: |
||||
|
image: chrislusf/seaweedfs:local |
||||
|
ports: |
||||
|
- 9333:9333 |
||||
|
- 19333:19333 |
||||
|
command: "master -ip=master -volumeSizeLimitMB=16" |
||||
|
environment: |
||||
|
WEED_MASTER_VOLUME_GROWTH_COPY_1: 1 |
||||
|
WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1 |
||||
|
volume: |
||||
|
image: chrislusf/seaweedfs:local |
||||
|
ports: |
||||
|
- 8080:8080 |
||||
|
- 18080:18080 |
||||
|
command: "volume -mserver=master:9333 -port=8080 -ip=volume -preStopSeconds=1" |
||||
|
depends_on: |
||||
|
- master |
||||
|
s3: |
||||
|
image: chrislusf/seaweedfs:local |
||||
|
ports: |
||||
|
- 8888:8888 |
||||
|
- 18888:18888 |
||||
|
- 8000:8000 |
||||
|
command: 'filer -master="master:9333" -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8000' |
||||
|
volumes: |
||||
|
- ./s3.json:/etc/seaweedfs/s3.json |
||||
|
depends_on: |
||||
|
- master |
||||
|
- volume |
||||
|
s3tests: |
||||
|
image: chrislusf/ceph-s3-tests:local |
||||
|
volumes: |
||||
|
- ./s3tests.conf:/opt/s3-tests/s3tests.conf |
||||
|
environment: |
||||
|
S3TEST_CONF: "s3tests.conf" |
||||
|
NOSETESTS_OPTIONS: "--verbose --logging-level=ERROR --with-xunit --failure-detail s3tests_boto3.functional.test_s3" |
||||
|
NOSETESTS_ATTR: "!tagging,!fails_on_aws,!encryption,!bucket-policy,!versioning,!fails_on_rgw,!bucket-policy,!fails_with_subdomain,!policy_status,!object-lock,!lifecycle,!cors,!user-policy" |
||||
|
NOSETESTS_EXCLUDE: "(bucket_list_delimiter_basic|bucket_listv2_delimiter_basic|bucket_listv2_encoding_basic|bucket_list_encoding_basic|bucket_list_delimiter_prefix|bucket_listv2_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_alt|bucket_listv2_delimiter_alt|bucket_list_delimiter_prefix_underscore|bucket_list_delimiter_percentage|bucket_listv2_delimiter_percentage|bucket_list_delimiter_whitespace|bucket_listv2_delimiter_whitespace|bucket_list_delimiter_dot|bucket_listv2_delimiter_dot|bucket_list_delimiter_unreadable|bucket_listv2_delimiter_unreadable|bucket_listv2_fetchowner_defaultempty|bucket_listv2_fetchowner_empty|bucket_list_delimiter_not_skip_special|bucket_list_prefix_delimiter_alt|bucket_listv2_prefix_delimiter_alt|bucket_list_prefix_delimiter_prefix_not_exist|bucket_listv2_prefix_delimiter_prefix_not_exist|bucket_list_prefix_delimiter_delimiter_not_exist|bucket_listv2_prefix_delimiter_delimiter_not_exist|bucket_list_prefix_delimiter_prefix_delimiter_not_exist|bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist|bucket_list_maxkeys_none|bucket_listv2_maxkeys_none|bucket_list_maxkeys_invalid|bucket_listv2_continuationtoken_empty|bucket_list_return_data|bucket_list_objects_anonymous|bucket_listv2_objects_anonymous|bucket_notexist|bucketv2_notexist|bucket_delete_nonempty|bucket_concurrent_set_canned_acl|object_write_to_nonexist_bucket|object_requestid_matches_header_on_error|object_head_zero_bytes|object_write_cache_control|object_write_expires|object_set_get_metadata_none_to_good|object_set_get_metadata_none_to_empty|object_set_get_metadata_overwrite_to_empty|post_object_anonymous_request|post_object_authenticated_request|post_object_authenticated_no_content_type|post_object_authenticated_request_bad_access_key|post_object_set_success_code|post_object_set_invalid_success_code|post_object_upload_larger_than_chunk|post_object_set_key_from_filename|post_object_ignored_header|post_object_case_insensitive_condition_fields|post_object_escaped_field_values|post_object_success_redirect_action|post_object_invalid_signature|post_object_invalid_access_key|post_object_missing_policy_condition|post_object_user_specified_header|post_object_request_missing_policy_specified_field|post_object_expired_policy|post_object_invalid_request_field_value|get_object_ifmatch_failed|get_object_ifunmodifiedsince_good|put_object_ifmatch_failed|object_raw_get|object_raw_get_bucket_gone|object_delete_key_bucket_gone|object_raw_get_bucket_acl|object_raw_get_object_acl|object_raw_authenticated|object_raw_response_headers|object_raw_authenticated_bucket_acl|object_raw_authenticated_object_acl|object_raw_authenticated_bucket_gone|object_raw_get_x_amz_expires_not_expired|object_raw_get_x_amz_expires_out_max_range|object_raw_get_x_amz_expires_out_positive_range|object_anon_put_write_access|object_raw_put_authenticated_expired|bucket_create_naming_bad_short_one|bucket_create_naming_bad_short_two|bucket_create_exists|bucket_get_location|bucket_acl_default|bucket_acl_canned|bucket_acl_canned_publicreadwrite|bucket_acl_canned_authenticatedread|object_acl_default|object_acl_canned_during_create|object_acl_canned|object_acl_canned_publicreadwrite|object_acl_canned_authenticatedread|object_acl_canned_bucketownerread|object_acl_canned_bucketownerfullcontrol|object_acl_full_control_verify_attributes|bucket_acl_canned_private_to_private|bucket_acl_grant_nonexist_user|bucket_acl_no_grants|bucket_acl_grant_email_not_exist|bucket_acl_revoke_all|bucket_recreate_not_overriding|bucket_create_special_key_names|object_copy_zero_size|object_copy_verify_contenttype|object_copy_to_itself|object_copy_to_itself_with_metadata|object_copy_not_owned_bucket|object_copy_not_owned_object_bucket|object_copy_retaining_metadata|object_copy_replacing_metadata|multipart_upload_empty|multipart_copy_invalid_range|multipart_copy_special_names|multipart_upload_resend_part|multipart_upload_size_too_small|abort_multipart_upload_not_found|multipart_upload_missing_part|multipart_upload_incorrect_etag|100_continue|ranged_request_invalid_range|ranged_request_empty_object|access_bucket)" |
||||
|
depends_on: |
||||
|
- master |
||||
|
- volume |
||||
|
- s3 |
||||
@ -0,0 +1,105 @@ |
|||||
|
{ |
||||
|
"identities": [ |
||||
|
{ |
||||
|
"name": "anonymous", |
||||
|
"actions": [ |
||||
|
"Read" |
||||
|
] |
||||
|
}, |
||||
|
{ |
||||
|
"name": "some_admin_user", |
||||
|
"credentials": [ |
||||
|
{ |
||||
|
"accessKey": "some_access_key1", |
||||
|
"secretKey": "some_secret_key1" |
||||
|
} |
||||
|
], |
||||
|
"actions": [ |
||||
|
"Admin", |
||||
|
"Read", |
||||
|
"List", |
||||
|
"Tagging", |
||||
|
"Write" |
||||
|
] |
||||
|
}, |
||||
|
{ |
||||
|
"name": "s3_tests", |
||||
|
"credentials": [ |
||||
|
{ |
||||
|
"accessKey": "ABCDEFGHIJKLMNOPQRST", |
||||
|
"secretKey": "abcdefghijklmnopqrstuvwxyzabcdefghijklmn" |
||||
|
}, |
||||
|
{ |
||||
|
"accessKey": "0555b35654ad1656d804", |
||||
|
"secretKey": "h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q==" |
||||
|
} |
||||
|
], |
||||
|
"actions": [ |
||||
|
"Admin", |
||||
|
"Read", |
||||
|
"List", |
||||
|
"Tagging", |
||||
|
"Write" |
||||
|
] |
||||
|
}, |
||||
|
{ |
||||
|
"name": "s3_tests_alt", |
||||
|
"credentials": [ |
||||
|
{ |
||||
|
"accessKey": "NOPQRSTUVWXYZABCDEFG", |
||||
|
"secretKey": "nopqrstuvwxyzabcdefghijklmnabcdefghijklm" |
||||
|
} |
||||
|
], |
||||
|
"actions": [ |
||||
|
"Admin", |
||||
|
"Read", |
||||
|
"List", |
||||
|
"Tagging", |
||||
|
"Write" |
||||
|
] |
||||
|
}, |
||||
|
{ |
||||
|
"name": "s3_tests_tenant", |
||||
|
"credentials": [ |
||||
|
{ |
||||
|
"accessKey": "HIJKLMNOPQRSTUVWXYZA", |
||||
|
"secretKey": "opqrstuvwxyzabcdefghijklmnopqrstuvwxyzab" |
||||
|
} |
||||
|
], |
||||
|
"actions": [ |
||||
|
"Admin", |
||||
|
"Read", |
||||
|
"List", |
||||
|
"Tagging", |
||||
|
"Write" |
||||
|
] |
||||
|
}, |
||||
|
{ |
||||
|
"name": "some_read_only_user", |
||||
|
"credentials": [ |
||||
|
{ |
||||
|
"accessKey": "some_access_key2", |
||||
|
"secretKey": "some_secret_key2" |
||||
|
} |
||||
|
], |
||||
|
"actions": [ |
||||
|
"Read" |
||||
|
] |
||||
|
}, |
||||
|
{ |
||||
|
"name": "some_normal_user", |
||||
|
"credentials": [ |
||||
|
{ |
||||
|
"accessKey": "some_access_key3", |
||||
|
"secretKey": "some_secret_key3" |
||||
|
} |
||||
|
], |
||||
|
"actions": [ |
||||
|
"Read", |
||||
|
"List", |
||||
|
"Tagging", |
||||
|
"Write" |
||||
|
] |
||||
|
} |
||||
|
] |
||||
|
} |
||||
@ -0,0 +1,70 @@ |
|||||
|
[DEFAULT] |
||||
|
## this section is just used for host, port and bucket_prefix |
||||
|
|
||||
|
# host set for rgw in vstart.sh |
||||
|
host = s3 |
||||
|
|
||||
|
# port set for rgw in vstart.sh |
||||
|
port = 8000 |
||||
|
|
||||
|
## say "False" to disable TLS |
||||
|
is_secure = False |
||||
|
|
||||
|
[fixtures] |
||||
|
## all the buckets created will start with this prefix; |
||||
|
## {random} will be filled with random characters to pad |
||||
|
## the prefix to 30 characters long, and avoid collisions |
||||
|
bucket prefix = yournamehere-{random}- |
||||
|
|
||||
|
[s3 main] |
||||
|
# main display_name set in vstart.sh |
||||
|
display_name = M. Tester |
||||
|
|
||||
|
# main user_idname set in vstart.sh |
||||
|
user_id = testid |
||||
|
|
||||
|
# main email set in vstart.sh |
||||
|
email = tester@ceph.com |
||||
|
|
||||
|
# zonegroup api_name for bucket location |
||||
|
api_name = default |
||||
|
|
||||
|
## main AWS access key |
||||
|
access_key = 0555b35654ad1656d804 |
||||
|
|
||||
|
## main AWS secret key |
||||
|
secret_key = h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q== |
||||
|
|
||||
|
## replace with key id obtained when secret is created, or delete if KMS not tested |
||||
|
#kms_keyid = 01234567-89ab-cdef-0123-456789abcdef |
||||
|
|
||||
|
[s3 alt] |
||||
|
# alt display_name set in vstart.sh |
||||
|
display_name = john.doe |
||||
|
## alt email set in vstart.sh |
||||
|
email = john.doe@example.com |
||||
|
|
||||
|
# alt user_id set in vstart.sh |
||||
|
user_id = 56789abcdef0123456789abcdef0123456789abcdef0123456789abcdef01234 |
||||
|
|
||||
|
# alt AWS access key set in vstart.sh |
||||
|
access_key = NOPQRSTUVWXYZABCDEFG |
||||
|
|
||||
|
# alt AWS secret key set in vstart.sh |
||||
|
secret_key = nopqrstuvwxyzabcdefghijklmnabcdefghijklm |
||||
|
|
||||
|
[s3 tenant] |
||||
|
# tenant display_name set in vstart.sh |
||||
|
display_name = testx$tenanteduser |
||||
|
|
||||
|
# tenant user_id set in vstart.sh |
||||
|
user_id = 9876543210abcdef0123456789abcdef0123456789abcdef0123456789abcdef |
||||
|
|
||||
|
# tenant AWS secret key set in vstart.sh |
||||
|
access_key = HIJKLMNOPQRSTUVWXYZA |
||||
|
|
||||
|
# tenant AWS secret key set in vstart.sh |
||||
|
secret_key = opqrstuvwxyzabcdefghijklmnopqrstuvwxyzab |
||||
|
|
||||
|
# tenant email set in vstart.sh |
||||
|
email = tenanteduser@example.com |
||||
@ -1,5 +1,5 @@ |
|||||
apiVersion: v1 |
apiVersion: v1 |
||||
description: SeaweedFS |
description: SeaweedFS |
||||
name: seaweedfs |
name: seaweedfs |
||||
appVersion: "2.23" |
|
||||
version: 2.23 |
|
||||
|
appVersion: "2.27" |
||||
|
version: 2.27 |
||||
@ -0,0 +1,21 @@ |
|||||
|
{{- if not (or .Values.filer.s3.skipAuthSecretCreation .Values.s3.skipAuthSecretCreation) }} |
||||
|
{{- $access_key_admin := randAlphaNum 16 -}} |
||||
|
{{- $secret_key_admin := randAlphaNum 32 -}} |
||||
|
{{- $access_key_read := randAlphaNum 16 -}} |
||||
|
{{- $secret_key_read := randAlphaNum 32 -}} |
||||
|
apiVersion: v1 |
||||
|
kind: Secret |
||||
|
type: Opaque |
||||
|
metadata: |
||||
|
name: seaweedfs-s3-secret |
||||
|
namespace: {{ .Release.Namespace }} |
||||
|
annotations: |
||||
|
"helm.sh/resource-policy": keep |
||||
|
"helm.sh/hook": "pre-install" |
||||
|
stringData: |
||||
|
admin_access_key_id: {{ $access_key_admin }} |
||||
|
admin_secret_access_key: {{ $secret_key_admin }} |
||||
|
read_access_key_id: {{ $access_key_read }} |
||||
|
read_secret_access_key: {{ $secret_key_read }} |
||||
|
seaweedfs_s3_config: '{"identities":[{"name":"anvAdmin","credentials":[{"accessKey":"{{ $access_key_admin }}","secretKey":"{{ $secret_key_admin }}"}],"actions":["Admin","Read","Write"]},{"name":"anvReadOnly","credentials":[{"accessKey":"{{ $access_key_read }}","secretKey":"{{ $secret_key_read }}"}],"actions":["Read"]}]}' |
||||
|
{{- end }} |
||||
1068
weed/pb/filer_pb/filer.pb.go
File diff suppressed because it is too large
View File
File diff suppressed because it is too large
View File
1586
weed/pb/master_pb/master.pb.go
File diff suppressed because it is too large
View File
File diff suppressed because it is too large
View File
1428
weed/pb/volume_server_pb/volume_server.pb.go
File diff suppressed because it is too large
View File
File diff suppressed because it is too large
View File
Some files were not shown because too many files changed in this diff
Write
Preview
Loading…
Cancel
Save
Reference in new issue