hilimd
4 years ago
committed by
GitHub
GitHub
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
203 changed files with 5921 additions and 3233 deletions
-
4.travis.yml
-
4docker/Dockerfile
-
31docker/Dockerfile.s3tests
-
6docker/Makefile
-
6docker/compose/local-cluster-compose.yml
-
4docker/compose/local-mount-compose.yml
-
45docker/compose/local-s3tests-compose.yml
-
0docker/compose/notification.toml
-
0docker/compose/replication.toml
-
105docker/compose/s3.json
-
70docker/compose/s3tests.conf
-
5go.mod
-
18go.sum
-
29k8s/README.md
-
4k8s/seaweedfs/Chart.yaml
-
13k8s/seaweedfs/templates/cronjob.yaml
-
1k8s/seaweedfs/templates/filer-service-client.yaml
-
29k8s/seaweedfs/templates/filer-statefulset.yaml
-
10k8s/seaweedfs/templates/s3-deployment.yaml
-
10k8s/seaweedfs/templates/s3-service.yaml
-
21k8s/seaweedfs/templates/seaweedfs-s3-secret.yaml
-
36k8s/seaweedfs/values.yaml
-
2other/java/client/pom.xml
-
2other/java/client/pom.xml.deploy
-
2other/java/client/pom_debug.xml
-
26other/java/client/src/main/java/seaweedfs/client/FileChunkManifest.java
-
57other/java/client/src/main/java/seaweedfs/client/FilerClient.java
-
17other/java/client/src/main/java/seaweedfs/client/SeaweedInputStream.java
-
18other/java/client/src/main/java/seaweedfs/client/SeaweedOutputStream.java
-
18other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java
-
22other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java
-
13other/java/client/src/main/proto/filer.proto
-
4other/java/examples/pom.xml
-
6other/java/examples/src/main/java/com/seaweedfs/examples/ExampleReadFile.java
-
13other/java/examples/src/main/java/com/seaweedfs/examples/ExampleWriteFile.java
-
2other/java/hdfs2/dependency-reduced-pom.xml
-
2other/java/hdfs2/pom.xml
-
16other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java
-
6other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedHadoopInputStream.java
-
6other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedHadoopOutputStream.java
-
2other/java/hdfs3/dependency-reduced-pom.xml
-
2other/java/hdfs3/pom.xml
-
16other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java
-
6other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedHadoopInputStream.java
-
6other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedHadoopOutputStream.java
-
2unmaintained/change_superblock/change_superblock.go
-
2unmaintained/diff_volume_servers/diff_volume_servers.go
-
2unmaintained/stress_filer_upload/bench_filer_upload/bench_filer_upload.go
-
2unmaintained/volume_tailer/volume_tailer.go
-
2weed/command/backup.go
-
9weed/command/benchmark.go
-
2weed/command/command.go
-
12weed/command/download.go
-
2weed/command/export.go
-
4weed/command/filer.go
-
8weed/command/filer_copy.go
-
18weed/command/filer_sync.go
-
2weed/command/mount.go
-
4weed/command/mount_std.go
-
6weed/command/scaffold.go
-
3weed/command/server.go
-
6weed/command/upload.go
-
20weed/command/volume.go
-
6weed/command/webdav.go
-
2weed/filer/abstract_sql/abstract_sql_store.go
-
1weed/filer/entry.go
-
2weed/filer/entry_codec.go
-
2weed/filer/filechunk_manifest.go
-
12weed/filer/filer_conf.go
-
5weed/filer/filer_notify.go
-
2weed/filer/filer_notify_append.go
-
2weed/filer/filer_on_meta_event.go
-
14weed/filer/mysql/mysql_sql_gen.go
-
6weed/filer/mysql/mysql_store.go
-
6weed/filer/mysql2/mysql2_store.go
-
14weed/filer/postgres/postgres_sql_gen.go
-
13weed/filer/postgres/postgres_store.go
-
13weed/filer/postgres2/postgres2_store.go
-
2weed/filer/read_write.go
-
2weed/filer/stream.go
-
7weed/filesys/filehandle.go
-
3weed/filesys/wfs.go
-
1weed/filesys/wfs_write.go
-
2weed/glog/glog.go
-
2weed/glog/glog_file.go
-
35weed/notification/gocdk_pub_sub/gocdk_pub_sub.go
-
9weed/operation/assign_file_id.go
-
8weed/operation/chunked_file.go
-
4weed/operation/delete_content.go
-
15weed/operation/lookup.go
-
29weed/operation/submit.go
-
4weed/operation/tail_volume.go
-
21weed/operation/upload_content.go
-
13weed/pb/filer.proto
-
1068weed/pb/filer_pb/filer.pb.go
-
3weed/pb/grpc_client_server.go
-
43weed/pb/master.proto
-
1586weed/pb/master_pb/master.pb.go
-
4weed/pb/volume_server.proto
-
1428weed/pb/volume_server_pb/volume_server.pb.go
@ -0,0 +1,31 @@ |
|||
FROM ubuntu:20.04 |
|||
|
|||
RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ |
|||
DEBIAN_FRONTEND=noninteractive apt-get upgrade -y && \ |
|||
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ |
|||
git \ |
|||
sudo \ |
|||
debianutils \ |
|||
python3-pip \ |
|||
python3-virtualenv \ |
|||
python3-dev \ |
|||
libevent-dev \ |
|||
libffi-dev \ |
|||
libxml2-dev \ |
|||
libxslt-dev \ |
|||
zlib1g-dev && \ |
|||
DEBIAN_FRONTEND=noninteractive apt-get clean && \ |
|||
rm -rf /var/lib/apt/lists/* && \ |
|||
git clone https://github.com/ceph/s3-tests.git /opt/s3-tests |
|||
|
|||
WORKDIR /opt/s3-tests |
|||
RUN ./bootstrap |
|||
|
|||
ENV \ |
|||
NOSETESTS_EXCLUDE="" \ |
|||
NOSETESTS_ATTR="" \ |
|||
NOSETESTS_OPTIONS="" \ |
|||
S3TEST_CONF="/s3test.conf" |
|||
|
|||
ENTRYPOINT ["/bin/bash", "-c"] |
|||
CMD ["exec ./virtualenv/bin/nosetests ${NOSETESTS_OPTIONS-} ${NOSETESTS_ATTR:+-a $NOSETESTS_ATTR} ${NOSETESTS_EXCLUDE:+-e $NOSETESTS_EXCLUDE}"] |
|||
@ -0,0 +1,45 @@ |
|||
version: '2' |
|||
|
|||
services: |
|||
master: |
|||
image: chrislusf/seaweedfs:local |
|||
ports: |
|||
- 9333:9333 |
|||
- 19333:19333 |
|||
command: "master -ip=master -volumeSizeLimitMB=16" |
|||
environment: |
|||
WEED_MASTER_VOLUME_GROWTH_COPY_1: 1 |
|||
WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1 |
|||
volume: |
|||
image: chrislusf/seaweedfs:local |
|||
ports: |
|||
- 8080:8080 |
|||
- 18080:18080 |
|||
command: "volume -mserver=master:9333 -port=8080 -ip=volume -preStopSeconds=1" |
|||
depends_on: |
|||
- master |
|||
s3: |
|||
image: chrislusf/seaweedfs:local |
|||
ports: |
|||
- 8888:8888 |
|||
- 18888:18888 |
|||
- 8000:8000 |
|||
command: 'filer -master="master:9333" -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8000' |
|||
volumes: |
|||
- ./s3.json:/etc/seaweedfs/s3.json |
|||
depends_on: |
|||
- master |
|||
- volume |
|||
s3tests: |
|||
image: chrislusf/ceph-s3-tests:local |
|||
volumes: |
|||
- ./s3tests.conf:/opt/s3-tests/s3tests.conf |
|||
environment: |
|||
S3TEST_CONF: "s3tests.conf" |
|||
NOSETESTS_OPTIONS: "--verbose --logging-level=ERROR --with-xunit --failure-detail s3tests_boto3.functional.test_s3" |
|||
NOSETESTS_ATTR: "!tagging,!fails_on_aws,!encryption,!bucket-policy,!versioning,!fails_on_rgw,!bucket-policy,!fails_with_subdomain,!policy_status,!object-lock,!lifecycle,!cors,!user-policy" |
|||
NOSETESTS_EXCLUDE: "(bucket_list_delimiter_basic|bucket_listv2_delimiter_basic|bucket_listv2_encoding_basic|bucket_list_encoding_basic|bucket_list_delimiter_prefix|bucket_listv2_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_alt|bucket_listv2_delimiter_alt|bucket_list_delimiter_prefix_underscore|bucket_list_delimiter_percentage|bucket_listv2_delimiter_percentage|bucket_list_delimiter_whitespace|bucket_listv2_delimiter_whitespace|bucket_list_delimiter_dot|bucket_listv2_delimiter_dot|bucket_list_delimiter_unreadable|bucket_listv2_delimiter_unreadable|bucket_listv2_fetchowner_defaultempty|bucket_listv2_fetchowner_empty|bucket_list_delimiter_not_skip_special|bucket_list_prefix_delimiter_alt|bucket_listv2_prefix_delimiter_alt|bucket_list_prefix_delimiter_prefix_not_exist|bucket_listv2_prefix_delimiter_prefix_not_exist|bucket_list_prefix_delimiter_delimiter_not_exist|bucket_listv2_prefix_delimiter_delimiter_not_exist|bucket_list_prefix_delimiter_prefix_delimiter_not_exist|bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist|bucket_list_maxkeys_none|bucket_listv2_maxkeys_none|bucket_list_maxkeys_invalid|bucket_listv2_continuationtoken_empty|bucket_list_return_data|bucket_list_objects_anonymous|bucket_listv2_objects_anonymous|bucket_notexist|bucketv2_notexist|bucket_delete_nonempty|bucket_concurrent_set_canned_acl|object_write_to_nonexist_bucket|object_requestid_matches_header_on_error|object_head_zero_bytes|object_write_cache_control|object_write_expires|object_set_get_metadata_none_to_good|object_set_get_metadata_none_to_empty|object_set_get_metadata_overwrite_to_empty|post_object_anonymous_request|post_object_authenticated_request|post_object_authenticated_no_content_type|post_object_authenticated_request_bad_access_key|post_object_set_success_code|post_object_set_invalid_success_code|post_object_upload_larger_than_chunk|post_object_set_key_from_filename|post_object_ignored_header|post_object_case_insensitive_condition_fields|post_object_escaped_field_values|post_object_success_redirect_action|post_object_invalid_signature|post_object_invalid_access_key|post_object_missing_policy_condition|post_object_user_specified_header|post_object_request_missing_policy_specified_field|post_object_expired_policy|post_object_invalid_request_field_value|get_object_ifmatch_failed|get_object_ifunmodifiedsince_good|put_object_ifmatch_failed|object_raw_get|object_raw_get_bucket_gone|object_delete_key_bucket_gone|object_raw_get_bucket_acl|object_raw_get_object_acl|object_raw_authenticated|object_raw_response_headers|object_raw_authenticated_bucket_acl|object_raw_authenticated_object_acl|object_raw_authenticated_bucket_gone|object_raw_get_x_amz_expires_not_expired|object_raw_get_x_amz_expires_out_max_range|object_raw_get_x_amz_expires_out_positive_range|object_anon_put_write_access|object_raw_put_authenticated_expired|bucket_create_naming_bad_short_one|bucket_create_naming_bad_short_two|bucket_create_exists|bucket_get_location|bucket_acl_default|bucket_acl_canned|bucket_acl_canned_publicreadwrite|bucket_acl_canned_authenticatedread|object_acl_default|object_acl_canned_during_create|object_acl_canned|object_acl_canned_publicreadwrite|object_acl_canned_authenticatedread|object_acl_canned_bucketownerread|object_acl_canned_bucketownerfullcontrol|object_acl_full_control_verify_attributes|bucket_acl_canned_private_to_private|bucket_acl_grant_nonexist_user|bucket_acl_no_grants|bucket_acl_grant_email_not_exist|bucket_acl_revoke_all|bucket_recreate_not_overriding|bucket_create_special_key_names|object_copy_zero_size|object_copy_verify_contenttype|object_copy_to_itself|object_copy_to_itself_with_metadata|object_copy_not_owned_bucket|object_copy_not_owned_object_bucket|object_copy_retaining_metadata|object_copy_replacing_metadata|multipart_upload_empty|multipart_copy_invalid_range|multipart_copy_special_names|multipart_upload_resend_part|multipart_upload_size_too_small|abort_multipart_upload_not_found|multipart_upload_missing_part|multipart_upload_incorrect_etag|100_continue|ranged_request_invalid_range|ranged_request_empty_object|access_bucket)" |
|||
depends_on: |
|||
- master |
|||
- volume |
|||
- s3 |
|||
@ -0,0 +1,105 @@ |
|||
{ |
|||
"identities": [ |
|||
{ |
|||
"name": "anonymous", |
|||
"actions": [ |
|||
"Read" |
|||
] |
|||
}, |
|||
{ |
|||
"name": "some_admin_user", |
|||
"credentials": [ |
|||
{ |
|||
"accessKey": "some_access_key1", |
|||
"secretKey": "some_secret_key1" |
|||
} |
|||
], |
|||
"actions": [ |
|||
"Admin", |
|||
"Read", |
|||
"List", |
|||
"Tagging", |
|||
"Write" |
|||
] |
|||
}, |
|||
{ |
|||
"name": "s3_tests", |
|||
"credentials": [ |
|||
{ |
|||
"accessKey": "ABCDEFGHIJKLMNOPQRST", |
|||
"secretKey": "abcdefghijklmnopqrstuvwxyzabcdefghijklmn" |
|||
}, |
|||
{ |
|||
"accessKey": "0555b35654ad1656d804", |
|||
"secretKey": "h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q==" |
|||
} |
|||
], |
|||
"actions": [ |
|||
"Admin", |
|||
"Read", |
|||
"List", |
|||
"Tagging", |
|||
"Write" |
|||
] |
|||
}, |
|||
{ |
|||
"name": "s3_tests_alt", |
|||
"credentials": [ |
|||
{ |
|||
"accessKey": "NOPQRSTUVWXYZABCDEFG", |
|||
"secretKey": "nopqrstuvwxyzabcdefghijklmnabcdefghijklm" |
|||
} |
|||
], |
|||
"actions": [ |
|||
"Admin", |
|||
"Read", |
|||
"List", |
|||
"Tagging", |
|||
"Write" |
|||
] |
|||
}, |
|||
{ |
|||
"name": "s3_tests_tenant", |
|||
"credentials": [ |
|||
{ |
|||
"accessKey": "HIJKLMNOPQRSTUVWXYZA", |
|||
"secretKey": "opqrstuvwxyzabcdefghijklmnopqrstuvwxyzab" |
|||
} |
|||
], |
|||
"actions": [ |
|||
"Admin", |
|||
"Read", |
|||
"List", |
|||
"Tagging", |
|||
"Write" |
|||
] |
|||
}, |
|||
{ |
|||
"name": "some_read_only_user", |
|||
"credentials": [ |
|||
{ |
|||
"accessKey": "some_access_key2", |
|||
"secretKey": "some_secret_key2" |
|||
} |
|||
], |
|||
"actions": [ |
|||
"Read" |
|||
] |
|||
}, |
|||
{ |
|||
"name": "some_normal_user", |
|||
"credentials": [ |
|||
{ |
|||
"accessKey": "some_access_key3", |
|||
"secretKey": "some_secret_key3" |
|||
} |
|||
], |
|||
"actions": [ |
|||
"Read", |
|||
"List", |
|||
"Tagging", |
|||
"Write" |
|||
] |
|||
} |
|||
] |
|||
} |
|||
@ -0,0 +1,70 @@ |
|||
[DEFAULT] |
|||
## this section is just used for host, port and bucket_prefix |
|||
|
|||
# host set for rgw in vstart.sh |
|||
host = s3 |
|||
|
|||
# port set for rgw in vstart.sh |
|||
port = 8000 |
|||
|
|||
## say "False" to disable TLS |
|||
is_secure = False |
|||
|
|||
[fixtures] |
|||
## all the buckets created will start with this prefix; |
|||
## {random} will be filled with random characters to pad |
|||
## the prefix to 30 characters long, and avoid collisions |
|||
bucket prefix = yournamehere-{random}- |
|||
|
|||
[s3 main] |
|||
# main display_name set in vstart.sh |
|||
display_name = M. Tester |
|||
|
|||
# main user_idname set in vstart.sh |
|||
user_id = testid |
|||
|
|||
# main email set in vstart.sh |
|||
email = tester@ceph.com |
|||
|
|||
# zonegroup api_name for bucket location |
|||
api_name = default |
|||
|
|||
## main AWS access key |
|||
access_key = 0555b35654ad1656d804 |
|||
|
|||
## main AWS secret key |
|||
secret_key = h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q== |
|||
|
|||
## replace with key id obtained when secret is created, or delete if KMS not tested |
|||
#kms_keyid = 01234567-89ab-cdef-0123-456789abcdef |
|||
|
|||
[s3 alt] |
|||
# alt display_name set in vstart.sh |
|||
display_name = john.doe |
|||
## alt email set in vstart.sh |
|||
email = john.doe@example.com |
|||
|
|||
# alt user_id set in vstart.sh |
|||
user_id = 56789abcdef0123456789abcdef0123456789abcdef0123456789abcdef01234 |
|||
|
|||
# alt AWS access key set in vstart.sh |
|||
access_key = NOPQRSTUVWXYZABCDEFG |
|||
|
|||
# alt AWS secret key set in vstart.sh |
|||
secret_key = nopqrstuvwxyzabcdefghijklmnabcdefghijklm |
|||
|
|||
[s3 tenant] |
|||
# tenant display_name set in vstart.sh |
|||
display_name = testx$tenanteduser |
|||
|
|||
# tenant user_id set in vstart.sh |
|||
user_id = 9876543210abcdef0123456789abcdef0123456789abcdef0123456789abcdef |
|||
|
|||
# tenant AWS secret key set in vstart.sh |
|||
access_key = HIJKLMNOPQRSTUVWXYZA |
|||
|
|||
# tenant AWS secret key set in vstart.sh |
|||
secret_key = opqrstuvwxyzabcdefghijklmnopqrstuvwxyzab |
|||
|
|||
# tenant email set in vstart.sh |
|||
email = tenanteduser@example.com |
|||
@ -1,5 +1,5 @@ |
|||
apiVersion: v1 |
|||
description: SeaweedFS |
|||
name: seaweedfs |
|||
appVersion: "2.23" |
|||
version: 2.23 |
|||
appVersion: "2.27" |
|||
version: 2.27 |
|||
@ -0,0 +1,21 @@ |
|||
{{- if not (or .Values.filer.s3.skipAuthSecretCreation .Values.s3.skipAuthSecretCreation) }} |
|||
{{- $access_key_admin := randAlphaNum 16 -}} |
|||
{{- $secret_key_admin := randAlphaNum 32 -}} |
|||
{{- $access_key_read := randAlphaNum 16 -}} |
|||
{{- $secret_key_read := randAlphaNum 32 -}} |
|||
apiVersion: v1 |
|||
kind: Secret |
|||
type: Opaque |
|||
metadata: |
|||
name: seaweedfs-s3-secret |
|||
namespace: {{ .Release.Namespace }} |
|||
annotations: |
|||
"helm.sh/resource-policy": keep |
|||
"helm.sh/hook": "pre-install" |
|||
stringData: |
|||
admin_access_key_id: {{ $access_key_admin }} |
|||
admin_secret_access_key: {{ $secret_key_admin }} |
|||
read_access_key_id: {{ $access_key_read }} |
|||
read_secret_access_key: {{ $secret_key_read }} |
|||
seaweedfs_s3_config: '{"identities":[{"name":"anvAdmin","credentials":[{"accessKey":"{{ $access_key_admin }}","secretKey":"{{ $secret_key_admin }}"}],"actions":["Admin","Read","Write"]},{"name":"anvReadOnly","credentials":[{"accessKey":"{{ $access_key_read }}","secretKey":"{{ $secret_key_read }}"}],"actions":["Read"]}]}' |
|||
{{- end }} |
|||
1068
weed/pb/filer_pb/filer.pb.go
File diff suppressed because it is too large
View File
File diff suppressed because it is too large
View File
1586
weed/pb/master_pb/master.pb.go
File diff suppressed because it is too large
View File
File diff suppressed because it is too large
View File
1428
weed/pb/volume_server_pb/volume_server.pb.go
File diff suppressed because it is too large
View File
File diff suppressed because it is too large
View File
Some files were not shown because too many files changed in this diff
Write
Preview
Loading…
Cancel
Save
Reference in new issue