Rain Li
3 years ago
committed by
GitHub
GitHub
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
66 changed files with 4876 additions and 3749 deletions
-
50.github/workflows/binary_test.yml
-
2.github/workflows/cleanup.yml
-
13.github/workflows/container_latest.yml
-
13.github/workflows/container_release.yml
-
53.github/workflows/container_test.yml
-
6.github/workflows/go.yml
-
14.github/workflows/release.yml
-
16.github/workflows/release_binaries.yml
-
4docker/Dockerfile
-
15go.mod
-
26go.sum
-
4k8s/helm_charts2/Chart.yaml
-
4other/java/client/src/main/java/seaweedfs/client/FilerClient.java
-
5other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java
-
7other/java/client/src/main/java/seaweedfs/client/VolumeIdCache.java
-
45other/java/client/src/main/proto/filer.proto
-
644other/metrics/grafana_seaweedfs.json
-
30weed/command/filer_remote_sync.go
-
1weed/command/imports.go
-
41weed/filer/filer_remote_storage.go
-
6weed/filer/filer_remote_storage_test.go
-
23weed/filer/read_remote.go
-
68weed/filer/stream.go
-
4weed/filesys/meta_cache/meta_cache_subscribe.go
-
1weed/pb/Makefile
-
45weed/pb/filer.proto
-
1044weed/pb/filer_pb/filer.pb.go
-
4weed/pb/filer_pb/filer_pb_helper.go
-
64weed/pb/remote.proto
-
653weed/pb/remote_pb/remote.pb.go
-
8weed/pb/remote_pb/remote_pb_helper.go
-
12weed/pb/volume_server.proto
-
1899weed/pb/volume_server_pb/volume_server.pb.go
-
27weed/remote_storage/azure/azure_storage_client.go
-
27weed/remote_storage/gcs/gcs_storage_client.go
-
55weed/remote_storage/hdfs/hdfs_kerberos.go
-
178weed/remote_storage/hdfs/hdfs_storage_client.go
-
66weed/remote_storage/remote_storage.go
-
10weed/remote_storage/s3/aliyun.go
-
10weed/remote_storage/s3/backblaze.go
-
10weed/remote_storage/s3/baidu.go
-
33weed/remote_storage/s3/s3_storage_client.go
-
11weed/remote_storage/s3/tencent.go
-
61weed/remote_storage/s3/wasabi.go
-
62weed/remote_storage/traverse_bfs.go
-
1weed/replication/sink/s3sink/s3_sink.go
-
1weed/replication/sub/notification_aws_sqs.go
-
3weed/server/filer_grpc_server.go
-
26weed/server/filer_grpc_server_remote.go
-
4weed/server/volume_grpc_erasure_coding.go
-
17weed/server/volume_grpc_remote.go
-
36weed/shell/command_remote_cache.go
-
45weed/shell/command_remote_configure.go
-
11weed/shell/command_remote_meta_sync.go
-
22weed/shell/command_remote_mount.go
-
4weed/shell/command_remote_uncache.go
-
14weed/shell/command_remote_unmount.go
-
2weed/shell/command_volume_fix_replication.go
-
20weed/shell/shell_liner.go
-
1weed/storage/backend/s3_backend/s3_sessions.go
-
6weed/storage/erasure_coding/ec_volume.go
-
6weed/storage/needle_map_metric.go
-
6weed/storage/store.go
-
2weed/storage/volume_info/volume_info.go
-
6weed/storage/volume_tier.go
-
2weed/util/constants.go
@ -0,0 +1,50 @@ |
|||||
|
name: "go: test building cross-platform binary" |
||||
|
|
||||
|
on: |
||||
|
pull_request: |
||||
|
workflow_dispatch: [] |
||||
|
|
||||
|
jobs: |
||||
|
|
||||
|
build: |
||||
|
name: Build |
||||
|
runs-on: ubuntu-latest |
||||
|
strategy: |
||||
|
matrix: |
||||
|
goos: [linux, windows, darwin, freebsd, netbsd, openbsd] |
||||
|
goarch: [amd64, arm, arm64, 386] |
||||
|
exclude: |
||||
|
- goarch: arm |
||||
|
goos: darwin |
||||
|
- goarch: 386 |
||||
|
goos: darwin |
||||
|
- goarch: arm |
||||
|
goos: windows |
||||
|
- goarch: arm64 |
||||
|
goos: windows |
||||
|
|
||||
|
concurrency: |
||||
|
group: ${{ github.head_ref }}/binary_test/${{ matrix.goos }}/${{ matrix.goarch }} |
||||
|
cancel-in-progress: true |
||||
|
|
||||
|
steps: |
||||
|
|
||||
|
- name: Set up Go 1.x |
||||
|
uses: actions/setup-go@v2 |
||||
|
with: |
||||
|
go-version: ^1.13 |
||||
|
id: go |
||||
|
|
||||
|
- name: Check out code into the Go module directory |
||||
|
uses: actions/checkout@v2 |
||||
|
|
||||
|
- name: Get dependencies |
||||
|
run: | |
||||
|
cd weed; go get -v -t -d ./... |
||||
|
if [ -f Gopkg.toml ]; then |
||||
|
curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh |
||||
|
dep ensure |
||||
|
fi |
||||
|
|
||||
|
- name: Build |
||||
|
run: cd weed; GOOS=${{ matrix.goos }} GOARCH=${{ matrix.goarch }} go build -v . |
||||
@ -0,0 +1,53 @@ |
|||||
|
name: "docker: test building container images" |
||||
|
|
||||
|
on: |
||||
|
pull_request: |
||||
|
workflow_dispatch: [] |
||||
|
|
||||
|
concurrency: |
||||
|
group: ${{ github.head_ref }}/container_test |
||||
|
cancel-in-progress: true |
||||
|
|
||||
|
jobs: |
||||
|
build-test: |
||||
|
runs-on: [ubuntu-latest] |
||||
|
strategy: |
||||
|
matrix: |
||||
|
platform: [ linux ] |
||||
|
arch: [ amd64, arm, arm64, 386 ] |
||||
|
|
||||
|
steps: |
||||
|
- |
||||
|
name: Checkout |
||||
|
uses: actions/checkout@v2 |
||||
|
- |
||||
|
name: Docker meta |
||||
|
id: docker_meta |
||||
|
uses: docker/metadata-action@v3 |
||||
|
with: |
||||
|
images: | |
||||
|
chrislusf/seaweedfs |
||||
|
ghcr.io/chrislusf/seaweedfs |
||||
|
tags: | |
||||
|
type=raw,value=latest |
||||
|
labels: | |
||||
|
org.opencontainers.image.title=seaweedfs |
||||
|
org.opencontainers.image.vendor=Chris Lu |
||||
|
- |
||||
|
name: Set up QEMU |
||||
|
uses: docker/setup-qemu-action@v1 |
||||
|
- |
||||
|
name: Set up Docker Buildx |
||||
|
uses: docker/setup-buildx-action@v1 |
||||
|
with: |
||||
|
buildkitd-flags: "--debug" |
||||
|
- |
||||
|
name: Build |
||||
|
uses: docker/build-push-action@v2 |
||||
|
with: |
||||
|
context: ./docker |
||||
|
push: false |
||||
|
file: ./docker/Dockerfile |
||||
|
platforms: ${{ matrix.platform }}/${{ matrix.arch }} |
||||
|
tags: ${{ steps.docker_meta.outputs.tags }} |
||||
|
labels: ${{ steps.docker_meta.outputs.labels }} |
||||
@ -1,5 +1,5 @@ |
|||||
apiVersion: v1 |
apiVersion: v1 |
||||
description: SeaweedFS |
description: SeaweedFS |
||||
name: seaweedfs |
name: seaweedfs |
||||
appVersion: "2.64" |
|
||||
version: "2.64" |
|
||||
|
appVersion: "2.65" |
||||
|
version: "2.65" |
||||
644
other/metrics/grafana_seaweedfs.json
File diff suppressed because it is too large
View File
File diff suppressed because it is too large
View File
1044
weed/pb/filer_pb/filer.pb.go
File diff suppressed because it is too large
View File
File diff suppressed because it is too large
View File
@ -0,0 +1,64 @@ |
|||||
|
syntax = "proto3"; |
||||
|
|
||||
|
package remote_pb; |
||||
|
|
||||
|
option go_package = "github.com/chrislusf/seaweedfs/weed/pb/remote_pb"; |
||||
|
option java_package = "seaweedfs.client"; |
||||
|
option java_outer_classname = "FilerProto"; |
||||
|
|
||||
|
///////////////////////// |
||||
|
// Remote Storage related |
||||
|
///////////////////////// |
||||
|
message RemoteConf { |
||||
|
string type = 1; |
||||
|
string name = 2; |
||||
|
string s3_access_key = 4; |
||||
|
string s3_secret_key = 5; |
||||
|
string s3_region = 6; |
||||
|
string s3_endpoint = 7; |
||||
|
string s3_storage_class = 8; |
||||
|
bool s3_force_path_style = 9; |
||||
|
|
||||
|
string gcs_google_application_credentials = 10; |
||||
|
|
||||
|
string azure_account_name = 15; |
||||
|
string azure_account_key = 16; |
||||
|
|
||||
|
string backblaze_key_id = 20; |
||||
|
string backblaze_application_key = 21; |
||||
|
string backblaze_endpoint = 22; |
||||
|
|
||||
|
string aliyun_access_key = 25; |
||||
|
string aliyun_secret_key = 26; |
||||
|
string aliyun_endpoint = 27; |
||||
|
string aliyun_region = 28; |
||||
|
|
||||
|
string tencent_secret_id = 30; |
||||
|
string tencent_secret_key = 31; |
||||
|
string tencent_endpoint = 32; |
||||
|
|
||||
|
string baidu_access_key = 35; |
||||
|
string baidu_secret_key = 36; |
||||
|
string baidu_endpoint = 37; |
||||
|
string baidu_region = 38; |
||||
|
|
||||
|
string wasabi_access_key = 40; |
||||
|
string wasabi_secret_key = 41; |
||||
|
string wasabi_endpoint = 42; |
||||
|
string wasabi_region = 43; |
||||
|
|
||||
|
repeated string hdfs_namenodes = 50; |
||||
|
string hdfs_username = 51; |
||||
|
string hdfs_service_principal_name = 52; |
||||
|
string hdfs_data_transfer_protection = 53; |
||||
|
|
||||
|
} |
||||
|
|
||||
|
message RemoteStorageMapping { |
||||
|
map<string,RemoteStorageLocation> mappings = 1; |
||||
|
} |
||||
|
message RemoteStorageLocation { |
||||
|
string name = 1; |
||||
|
string bucket = 2; |
||||
|
string path = 3; |
||||
|
} |
||||
@ -0,0 +1,653 @@ |
|||||
|
// Code generated by protoc-gen-go. DO NOT EDIT.
|
||||
|
// versions:
|
||||
|
// protoc-gen-go v1.25.0
|
||||
|
// protoc v3.12.3
|
||||
|
// source: remote.proto
|
||||
|
|
||||
|
package remote_pb |
||||
|
|
||||
|
import ( |
||||
|
proto "github.com/golang/protobuf/proto" |
||||
|
protoreflect "google.golang.org/protobuf/reflect/protoreflect" |
||||
|
protoimpl "google.golang.org/protobuf/runtime/protoimpl" |
||||
|
reflect "reflect" |
||||
|
sync "sync" |
||||
|
) |
||||
|
|
||||
|
const ( |
||||
|
// Verify that this generated code is sufficiently up-to-date.
|
||||
|
_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) |
||||
|
// Verify that runtime/protoimpl is sufficiently up-to-date.
|
||||
|
_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) |
||||
|
) |
||||
|
|
||||
|
// This is a compile-time assertion that a sufficiently up-to-date version
|
||||
|
// of the legacy proto package is being used.
|
||||
|
const _ = proto.ProtoPackageIsVersion4 |
||||
|
|
||||
|
/////////////////////////
|
||||
|
// Remote Storage related
|
||||
|
/////////////////////////
|
||||
|
type RemoteConf struct { |
||||
|
state protoimpl.MessageState |
||||
|
sizeCache protoimpl.SizeCache |
||||
|
unknownFields protoimpl.UnknownFields |
||||
|
|
||||
|
Type string `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"` |
||||
|
Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` |
||||
|
S3AccessKey string `protobuf:"bytes,4,opt,name=s3_access_key,json=s3AccessKey,proto3" json:"s3_access_key,omitempty"` |
||||
|
S3SecretKey string `protobuf:"bytes,5,opt,name=s3_secret_key,json=s3SecretKey,proto3" json:"s3_secret_key,omitempty"` |
||||
|
S3Region string `protobuf:"bytes,6,opt,name=s3_region,json=s3Region,proto3" json:"s3_region,omitempty"` |
||||
|
S3Endpoint string `protobuf:"bytes,7,opt,name=s3_endpoint,json=s3Endpoint,proto3" json:"s3_endpoint,omitempty"` |
||||
|
S3StorageClass string `protobuf:"bytes,8,opt,name=s3_storage_class,json=s3StorageClass,proto3" json:"s3_storage_class,omitempty"` |
||||
|
S3ForcePathStyle bool `protobuf:"varint,9,opt,name=s3_force_path_style,json=s3ForcePathStyle,proto3" json:"s3_force_path_style,omitempty"` |
||||
|
GcsGoogleApplicationCredentials string `protobuf:"bytes,10,opt,name=gcs_google_application_credentials,json=gcsGoogleApplicationCredentials,proto3" json:"gcs_google_application_credentials,omitempty"` |
||||
|
AzureAccountName string `protobuf:"bytes,15,opt,name=azure_account_name,json=azureAccountName,proto3" json:"azure_account_name,omitempty"` |
||||
|
AzureAccountKey string `protobuf:"bytes,16,opt,name=azure_account_key,json=azureAccountKey,proto3" json:"azure_account_key,omitempty"` |
||||
|
BackblazeKeyId string `protobuf:"bytes,20,opt,name=backblaze_key_id,json=backblazeKeyId,proto3" json:"backblaze_key_id,omitempty"` |
||||
|
BackblazeApplicationKey string `protobuf:"bytes,21,opt,name=backblaze_application_key,json=backblazeApplicationKey,proto3" json:"backblaze_application_key,omitempty"` |
||||
|
BackblazeEndpoint string `protobuf:"bytes,22,opt,name=backblaze_endpoint,json=backblazeEndpoint,proto3" json:"backblaze_endpoint,omitempty"` |
||||
|
AliyunAccessKey string `protobuf:"bytes,25,opt,name=aliyun_access_key,json=aliyunAccessKey,proto3" json:"aliyun_access_key,omitempty"` |
||||
|
AliyunSecretKey string `protobuf:"bytes,26,opt,name=aliyun_secret_key,json=aliyunSecretKey,proto3" json:"aliyun_secret_key,omitempty"` |
||||
|
AliyunEndpoint string `protobuf:"bytes,27,opt,name=aliyun_endpoint,json=aliyunEndpoint,proto3" json:"aliyun_endpoint,omitempty"` |
||||
|
AliyunRegion string `protobuf:"bytes,28,opt,name=aliyun_region,json=aliyunRegion,proto3" json:"aliyun_region,omitempty"` |
||||
|
TencentSecretId string `protobuf:"bytes,30,opt,name=tencent_secret_id,json=tencentSecretId,proto3" json:"tencent_secret_id,omitempty"` |
||||
|
TencentSecretKey string `protobuf:"bytes,31,opt,name=tencent_secret_key,json=tencentSecretKey,proto3" json:"tencent_secret_key,omitempty"` |
||||
|
TencentEndpoint string `protobuf:"bytes,32,opt,name=tencent_endpoint,json=tencentEndpoint,proto3" json:"tencent_endpoint,omitempty"` |
||||
|
BaiduAccessKey string `protobuf:"bytes,35,opt,name=baidu_access_key,json=baiduAccessKey,proto3" json:"baidu_access_key,omitempty"` |
||||
|
BaiduSecretKey string `protobuf:"bytes,36,opt,name=baidu_secret_key,json=baiduSecretKey,proto3" json:"baidu_secret_key,omitempty"` |
||||
|
BaiduEndpoint string `protobuf:"bytes,37,opt,name=baidu_endpoint,json=baiduEndpoint,proto3" json:"baidu_endpoint,omitempty"` |
||||
|
BaiduRegion string `protobuf:"bytes,38,opt,name=baidu_region,json=baiduRegion,proto3" json:"baidu_region,omitempty"` |
||||
|
WasabiAccessKey string `protobuf:"bytes,40,opt,name=wasabi_access_key,json=wasabiAccessKey,proto3" json:"wasabi_access_key,omitempty"` |
||||
|
WasabiSecretKey string `protobuf:"bytes,41,opt,name=wasabi_secret_key,json=wasabiSecretKey,proto3" json:"wasabi_secret_key,omitempty"` |
||||
|
WasabiEndpoint string `protobuf:"bytes,42,opt,name=wasabi_endpoint,json=wasabiEndpoint,proto3" json:"wasabi_endpoint,omitempty"` |
||||
|
WasabiRegion string `protobuf:"bytes,43,opt,name=wasabi_region,json=wasabiRegion,proto3" json:"wasabi_region,omitempty"` |
||||
|
HdfsNamenodes []string `protobuf:"bytes,50,rep,name=hdfs_namenodes,json=hdfsNamenodes,proto3" json:"hdfs_namenodes,omitempty"` |
||||
|
HdfsUsername string `protobuf:"bytes,51,opt,name=hdfs_username,json=hdfsUsername,proto3" json:"hdfs_username,omitempty"` |
||||
|
HdfsServicePrincipalName string `protobuf:"bytes,52,opt,name=hdfs_service_principal_name,json=hdfsServicePrincipalName,proto3" json:"hdfs_service_principal_name,omitempty"` |
||||
|
HdfsDataTransferProtection string `protobuf:"bytes,53,opt,name=hdfs_data_transfer_protection,json=hdfsDataTransferProtection,proto3" json:"hdfs_data_transfer_protection,omitempty"` |
||||
|
} |
||||
|
|
||||
|
func (x *RemoteConf) Reset() { |
||||
|
*x = RemoteConf{} |
||||
|
if protoimpl.UnsafeEnabled { |
||||
|
mi := &file_remote_proto_msgTypes[0] |
||||
|
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) |
||||
|
ms.StoreMessageInfo(mi) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func (x *RemoteConf) String() string { |
||||
|
return protoimpl.X.MessageStringOf(x) |
||||
|
} |
||||
|
|
||||
|
func (*RemoteConf) ProtoMessage() {} |
||||
|
|
||||
|
func (x *RemoteConf) ProtoReflect() protoreflect.Message { |
||||
|
mi := &file_remote_proto_msgTypes[0] |
||||
|
if protoimpl.UnsafeEnabled && x != nil { |
||||
|
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) |
||||
|
if ms.LoadMessageInfo() == nil { |
||||
|
ms.StoreMessageInfo(mi) |
||||
|
} |
||||
|
return ms |
||||
|
} |
||||
|
return mi.MessageOf(x) |
||||
|
} |
||||
|
|
||||
|
// Deprecated: Use RemoteConf.ProtoReflect.Descriptor instead.
|
||||
|
func (*RemoteConf) Descriptor() ([]byte, []int) { |
||||
|
return file_remote_proto_rawDescGZIP(), []int{0} |
||||
|
} |
||||
|
|
||||
|
func (x *RemoteConf) GetType() string { |
||||
|
if x != nil { |
||||
|
return x.Type |
||||
|
} |
||||
|
return "" |
||||
|
} |
||||
|
|
||||
|
func (x *RemoteConf) GetName() string { |
||||
|
if x != nil { |
||||
|
return x.Name |
||||
|
} |
||||
|
return "" |
||||
|
} |
||||
|
|
||||
|
func (x *RemoteConf) GetS3AccessKey() string { |
||||
|
if x != nil { |
||||
|
return x.S3AccessKey |
||||
|
} |
||||
|
return "" |
||||
|
} |
||||
|
|
||||
|
func (x *RemoteConf) GetS3SecretKey() string { |
||||
|
if x != nil { |
||||
|
return x.S3SecretKey |
||||
|
} |
||||
|
return "" |
||||
|
} |
||||
|
|
||||
|
func (x *RemoteConf) GetS3Region() string { |
||||
|
if x != nil { |
||||
|
return x.S3Region |
||||
|
} |
||||
|
return "" |
||||
|
} |
||||
|
|
||||
|
func (x *RemoteConf) GetS3Endpoint() string { |
||||
|
if x != nil { |
||||
|
return x.S3Endpoint |
||||
|
} |
||||
|
return "" |
||||
|
} |
||||
|
|
||||
|
func (x *RemoteConf) GetS3StorageClass() string { |
||||
|
if x != nil { |
||||
|
return x.S3StorageClass |
||||
|
} |
||||
|
return "" |
||||
|
} |
||||
|
|
||||
|
func (x *RemoteConf) GetS3ForcePathStyle() bool { |
||||
|
if x != nil { |
||||
|
return x.S3ForcePathStyle |
||||
|
} |
||||
|
return false |
||||
|
} |
||||
|
|
||||
|
func (x *RemoteConf) GetGcsGoogleApplicationCredentials() string { |
||||
|
if x != nil { |
||||
|
return x.GcsGoogleApplicationCredentials |
||||
|
} |
||||
|
return "" |
||||
|
} |
||||
|
|
||||
|
func (x *RemoteConf) GetAzureAccountName() string { |
||||
|
if x != nil { |
||||
|
return x.AzureAccountName |
||||
|
} |
||||
|
return "" |
||||
|
} |
||||
|
|
||||
|
func (x *RemoteConf) GetAzureAccountKey() string { |
||||
|
if x != nil { |
||||
|
return x.AzureAccountKey |
||||
|
} |
||||
|
return "" |
||||
|
} |
||||
|
|
||||
|
func (x *RemoteConf) GetBackblazeKeyId() string { |
||||
|
if x != nil { |
||||
|
return x.BackblazeKeyId |
||||
|
} |
||||
|
return "" |
||||
|
} |
||||
|
|
||||
|
func (x *RemoteConf) GetBackblazeApplicationKey() string { |
||||
|
if x != nil { |
||||
|
return x.BackblazeApplicationKey |
||||
|
} |
||||
|
return "" |
||||
|
} |
||||
|
|
||||
|
func (x *RemoteConf) GetBackblazeEndpoint() string { |
||||
|
if x != nil { |
||||
|
return x.BackblazeEndpoint |
||||
|
} |
||||
|
return "" |
||||
|
} |
||||
|
|
||||
|
func (x *RemoteConf) GetAliyunAccessKey() string { |
||||
|
if x != nil { |
||||
|
return x.AliyunAccessKey |
||||
|
} |
||||
|
return "" |
||||
|
} |
||||
|
|
||||
|
func (x *RemoteConf) GetAliyunSecretKey() string { |
||||
|
if x != nil { |
||||
|
return x.AliyunSecretKey |
||||
|
} |
||||
|
return "" |
||||
|
} |
||||
|
|
||||
|
func (x *RemoteConf) GetAliyunEndpoint() string { |
||||
|
if x != nil { |
||||
|
return x.AliyunEndpoint |
||||
|
} |
||||
|
return "" |
||||
|
} |
||||
|
|
||||
|
func (x *RemoteConf) GetAliyunRegion() string { |
||||
|
if x != nil { |
||||
|
return x.AliyunRegion |
||||
|
} |
||||
|
return "" |
||||
|
} |
||||
|
|
||||
|
func (x *RemoteConf) GetTencentSecretId() string { |
||||
|
if x != nil { |
||||
|
return x.TencentSecretId |
||||
|
} |
||||
|
return "" |
||||
|
} |
||||
|
|
||||
|
func (x *RemoteConf) GetTencentSecretKey() string { |
||||
|
if x != nil { |
||||
|
return x.TencentSecretKey |
||||
|
} |
||||
|
return "" |
||||
|
} |
||||
|
|
||||
|
func (x *RemoteConf) GetTencentEndpoint() string { |
||||
|
if x != nil { |
||||
|
return x.TencentEndpoint |
||||
|
} |
||||
|
return "" |
||||
|
} |
||||
|
|
||||
|
func (x *RemoteConf) GetBaiduAccessKey() string { |
||||
|
if x != nil { |
||||
|
return x.BaiduAccessKey |
||||
|
} |
||||
|
return "" |
||||
|
} |
||||
|
|
||||
|
func (x *RemoteConf) GetBaiduSecretKey() string { |
||||
|
if x != nil { |
||||
|
return x.BaiduSecretKey |
||||
|
} |
||||
|
return "" |
||||
|
} |
||||
|
|
||||
|
func (x *RemoteConf) GetBaiduEndpoint() string { |
||||
|
if x != nil { |
||||
|
return x.BaiduEndpoint |
||||
|
} |
||||
|
return "" |
||||
|
} |
||||
|
|
||||
|
func (x *RemoteConf) GetBaiduRegion() string { |
||||
|
if x != nil { |
||||
|
return x.BaiduRegion |
||||
|
} |
||||
|
return "" |
||||
|
} |
||||
|
|
||||
|
func (x *RemoteConf) GetWasabiAccessKey() string { |
||||
|
if x != nil { |
||||
|
return x.WasabiAccessKey |
||||
|
} |
||||
|
return "" |
||||
|
} |
||||
|
|
||||
|
func (x *RemoteConf) GetWasabiSecretKey() string { |
||||
|
if x != nil { |
||||
|
return x.WasabiSecretKey |
||||
|
} |
||||
|
return "" |
||||
|
} |
||||
|
|
||||
|
func (x *RemoteConf) GetWasabiEndpoint() string { |
||||
|
if x != nil { |
||||
|
return x.WasabiEndpoint |
||||
|
} |
||||
|
return "" |
||||
|
} |
||||
|
|
||||
|
func (x *RemoteConf) GetWasabiRegion() string { |
||||
|
if x != nil { |
||||
|
return x.WasabiRegion |
||||
|
} |
||||
|
return "" |
||||
|
} |
||||
|
|
||||
|
func (x *RemoteConf) GetHdfsNamenodes() []string { |
||||
|
if x != nil { |
||||
|
return x.HdfsNamenodes |
||||
|
} |
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
func (x *RemoteConf) GetHdfsUsername() string { |
||||
|
if x != nil { |
||||
|
return x.HdfsUsername |
||||
|
} |
||||
|
return "" |
||||
|
} |
||||
|
|
||||
|
func (x *RemoteConf) GetHdfsServicePrincipalName() string { |
||||
|
if x != nil { |
||||
|
return x.HdfsServicePrincipalName |
||||
|
} |
||||
|
return "" |
||||
|
} |
||||
|
|
||||
|
func (x *RemoteConf) GetHdfsDataTransferProtection() string { |
||||
|
if x != nil { |
||||
|
return x.HdfsDataTransferProtection |
||||
|
} |
||||
|
return "" |
||||
|
} |
||||
|
|
||||
|
type RemoteStorageMapping struct { |
||||
|
state protoimpl.MessageState |
||||
|
sizeCache protoimpl.SizeCache |
||||
|
unknownFields protoimpl.UnknownFields |
||||
|
|
||||
|
Mappings map[string]*RemoteStorageLocation `protobuf:"bytes,1,rep,name=mappings,proto3" json:"mappings,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` |
||||
|
} |
||||
|
|
||||
|
func (x *RemoteStorageMapping) Reset() { |
||||
|
*x = RemoteStorageMapping{} |
||||
|
if protoimpl.UnsafeEnabled { |
||||
|
mi := &file_remote_proto_msgTypes[1] |
||||
|
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) |
||||
|
ms.StoreMessageInfo(mi) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func (x *RemoteStorageMapping) String() string { |
||||
|
return protoimpl.X.MessageStringOf(x) |
||||
|
} |
||||
|
|
||||
|
func (*RemoteStorageMapping) ProtoMessage() {} |
||||
|
|
||||
|
func (x *RemoteStorageMapping) ProtoReflect() protoreflect.Message { |
||||
|
mi := &file_remote_proto_msgTypes[1] |
||||
|
if protoimpl.UnsafeEnabled && x != nil { |
||||
|
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) |
||||
|
if ms.LoadMessageInfo() == nil { |
||||
|
ms.StoreMessageInfo(mi) |
||||
|
} |
||||
|
return ms |
||||
|
} |
||||
|
return mi.MessageOf(x) |
||||
|
} |
||||
|
|
||||
|
// Deprecated: Use RemoteStorageMapping.ProtoReflect.Descriptor instead.
|
||||
|
func (*RemoteStorageMapping) Descriptor() ([]byte, []int) { |
||||
|
return file_remote_proto_rawDescGZIP(), []int{1} |
||||
|
} |
||||
|
|
||||
|
func (x *RemoteStorageMapping) GetMappings() map[string]*RemoteStorageLocation { |
||||
|
if x != nil { |
||||
|
return x.Mappings |
||||
|
} |
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
type RemoteStorageLocation struct { |
||||
|
state protoimpl.MessageState |
||||
|
sizeCache protoimpl.SizeCache |
||||
|
unknownFields protoimpl.UnknownFields |
||||
|
|
||||
|
Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` |
||||
|
Bucket string `protobuf:"bytes,2,opt,name=bucket,proto3" json:"bucket,omitempty"` |
||||
|
Path string `protobuf:"bytes,3,opt,name=path,proto3" json:"path,omitempty"` |
||||
|
} |
||||
|
|
||||
|
func (x *RemoteStorageLocation) Reset() { |
||||
|
*x = RemoteStorageLocation{} |
||||
|
if protoimpl.UnsafeEnabled { |
||||
|
mi := &file_remote_proto_msgTypes[2] |
||||
|
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) |
||||
|
ms.StoreMessageInfo(mi) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func (x *RemoteStorageLocation) String() string { |
||||
|
return protoimpl.X.MessageStringOf(x) |
||||
|
} |
||||
|
|
||||
|
func (*RemoteStorageLocation) ProtoMessage() {} |
||||
|
|
||||
|
func (x *RemoteStorageLocation) ProtoReflect() protoreflect.Message { |
||||
|
mi := &file_remote_proto_msgTypes[2] |
||||
|
if protoimpl.UnsafeEnabled && x != nil { |
||||
|
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) |
||||
|
if ms.LoadMessageInfo() == nil { |
||||
|
ms.StoreMessageInfo(mi) |
||||
|
} |
||||
|
return ms |
||||
|
} |
||||
|
return mi.MessageOf(x) |
||||
|
} |
||||
|
|
||||
|
// Deprecated: Use RemoteStorageLocation.ProtoReflect.Descriptor instead.
|
||||
|
func (*RemoteStorageLocation) Descriptor() ([]byte, []int) { |
||||
|
return file_remote_proto_rawDescGZIP(), []int{2} |
||||
|
} |
||||
|
|
||||
|
func (x *RemoteStorageLocation) GetName() string { |
||||
|
if x != nil { |
||||
|
return x.Name |
||||
|
} |
||||
|
return "" |
||||
|
} |
||||
|
|
||||
|
func (x *RemoteStorageLocation) GetBucket() string { |
||||
|
if x != nil { |
||||
|
return x.Bucket |
||||
|
} |
||||
|
return "" |
||||
|
} |
||||
|
|
||||
|
func (x *RemoteStorageLocation) GetPath() string { |
||||
|
if x != nil { |
||||
|
return x.Path |
||||
|
} |
||||
|
return "" |
||||
|
} |
||||
|
|
||||
|
var File_remote_proto protoreflect.FileDescriptor |
||||
|
|
||||
|
var file_remote_proto_rawDesc = []byte{ |
||||
|
0x0a, 0x0c, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x09, |
||||
|
0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x70, 0x62, 0x22, 0x8c, 0x0b, 0x0a, 0x0a, 0x52, 0x65, |
||||
|
0x6d, 0x6f, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, |
||||
|
0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, |
||||
|
0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, |
||||
|
0x12, 0x22, 0x0a, 0x0d, 0x73, 0x33, 0x5f, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x6b, 0x65, |
||||
|
0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x73, 0x33, 0x41, 0x63, 0x63, 0x65, 0x73, |
||||
|
0x73, 0x4b, 0x65, 0x79, 0x12, 0x22, 0x0a, 0x0d, 0x73, 0x33, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, |
||||
|
0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x73, 0x33, 0x53, |
||||
|
0x65, 0x63, 0x72, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x33, 0x5f, 0x72, |
||||
|
0x65, 0x67, 0x69, 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x73, 0x33, 0x52, |
||||
|
0x65, 0x67, 0x69, 0x6f, 0x6e, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x33, 0x5f, 0x65, 0x6e, 0x64, 0x70, |
||||
|
0x6f, 0x69, 0x6e, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x33, 0x45, 0x6e, |
||||
|
0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x28, 0x0a, 0x10, 0x73, 0x33, 0x5f, 0x73, 0x74, 0x6f, |
||||
|
0x72, 0x61, 0x67, 0x65, 0x5f, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, |
||||
|
0x52, 0x0e, 0x73, 0x33, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, |
||||
|
0x12, 0x2d, 0x0a, 0x13, 0x73, 0x33, 0x5f, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x5f, 0x70, 0x61, 0x74, |
||||
|
0x68, 0x5f, 0x73, 0x74, 0x79, 0x6c, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x73, |
||||
|
0x33, 0x46, 0x6f, 0x72, 0x63, 0x65, 0x50, 0x61, 0x74, 0x68, 0x53, 0x74, 0x79, 0x6c, 0x65, 0x12, |
||||
|
0x4b, 0x0a, 0x22, 0x67, 0x63, 0x73, 0x5f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x5f, 0x61, 0x70, |
||||
|
0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, |
||||
|
0x74, 0x69, 0x61, 0x6c, 0x73, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x1f, 0x67, 0x63, 0x73, |
||||
|
0x47, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, |
||||
|
0x6e, 0x43, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x12, 0x2c, 0x0a, 0x12, |
||||
|
0x61, 0x7a, 0x75, 0x72, 0x65, 0x5f, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x6e, 0x61, |
||||
|
0x6d, 0x65, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x61, 0x7a, 0x75, 0x72, 0x65, 0x41, |
||||
|
0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x2a, 0x0a, 0x11, 0x61, 0x7a, |
||||
|
0x75, 0x72, 0x65, 0x5f, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, |
||||
|
0x10, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x61, 0x7a, 0x75, 0x72, 0x65, 0x41, 0x63, 0x63, 0x6f, |
||||
|
0x75, 0x6e, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x28, 0x0a, 0x10, 0x62, 0x61, 0x63, 0x6b, 0x62, 0x6c, |
||||
|
0x61, 0x7a, 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x5f, 0x69, 0x64, 0x18, 0x14, 0x20, 0x01, 0x28, 0x09, |
||||
|
0x52, 0x0e, 0x62, 0x61, 0x63, 0x6b, 0x62, 0x6c, 0x61, 0x7a, 0x65, 0x4b, 0x65, 0x79, 0x49, 0x64, |
||||
|
0x12, 0x3a, 0x0a, 0x19, 0x62, 0x61, 0x63, 0x6b, 0x62, 0x6c, 0x61, 0x7a, 0x65, 0x5f, 0x61, 0x70, |
||||
|
0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x15, 0x20, |
||||
|
0x01, 0x28, 0x09, 0x52, 0x17, 0x62, 0x61, 0x63, 0x6b, 0x62, 0x6c, 0x61, 0x7a, 0x65, 0x41, 0x70, |
||||
|
0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x12, 0x2d, 0x0a, 0x12, |
||||
|
0x62, 0x61, 0x63, 0x6b, 0x62, 0x6c, 0x61, 0x7a, 0x65, 0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, |
||||
|
0x6e, 0x74, 0x18, 0x16, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x62, 0x61, 0x63, 0x6b, 0x62, 0x6c, |
||||
|
0x61, 0x7a, 0x65, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x2a, 0x0a, 0x11, 0x61, |
||||
|
0x6c, 0x69, 0x79, 0x75, 0x6e, 0x5f, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x6b, 0x65, 0x79, |
||||
|
0x18, 0x19, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x61, 0x6c, 0x69, 0x79, 0x75, 0x6e, 0x41, 0x63, |
||||
|
0x63, 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79, 0x12, 0x2a, 0x0a, 0x11, 0x61, 0x6c, 0x69, 0x79, 0x75, |
||||
|
0x6e, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x1a, 0x20, 0x01, |
||||
|
0x28, 0x09, 0x52, 0x0f, 0x61, 0x6c, 0x69, 0x79, 0x75, 0x6e, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, |
||||
|
0x4b, 0x65, 0x79, 0x12, 0x27, 0x0a, 0x0f, 0x61, 0x6c, 0x69, 0x79, 0x75, 0x6e, 0x5f, 0x65, 0x6e, |
||||
|
0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x1b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x61, 0x6c, |
||||
|
0x69, 0x79, 0x75, 0x6e, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x23, 0x0a, 0x0d, |
||||
|
0x61, 0x6c, 0x69, 0x79, 0x75, 0x6e, 0x5f, 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x18, 0x1c, 0x20, |
||||
|
0x01, 0x28, 0x09, 0x52, 0x0c, 0x61, 0x6c, 0x69, 0x79, 0x75, 0x6e, 0x52, 0x65, 0x67, 0x69, 0x6f, |
||||
|
0x6e, 0x12, 0x2a, 0x0a, 0x11, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x6e, 0x74, 0x5f, 0x73, 0x65, 0x63, |
||||
|
0x72, 0x65, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x1e, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x74, 0x65, |
||||
|
0x6e, 0x63, 0x65, 0x6e, 0x74, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x49, 0x64, 0x12, 0x2c, 0x0a, |
||||
|
0x12, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x6e, 0x74, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x5f, |
||||
|
0x6b, 0x65, 0x79, 0x18, 0x1f, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x74, 0x65, 0x6e, 0x63, 0x65, |
||||
|
0x6e, 0x74, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x29, 0x0a, 0x10, 0x74, |
||||
|
0x65, 0x6e, 0x63, 0x65, 0x6e, 0x74, 0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, |
||||
|
0x20, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x6e, 0x74, 0x45, 0x6e, |
||||
|
0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x28, 0x0a, 0x10, 0x62, 0x61, 0x69, 0x64, 0x75, 0x5f, |
||||
|
0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x23, 0x20, 0x01, 0x28, 0x09, |
||||
|
0x52, 0x0e, 0x62, 0x61, 0x69, 0x64, 0x75, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79, |
||||
|
0x12, 0x28, 0x0a, 0x10, 0x62, 0x61, 0x69, 0x64, 0x75, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, |
||||
|
0x5f, 0x6b, 0x65, 0x79, 0x18, 0x24, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x62, 0x61, 0x69, 0x64, |
||||
|
0x75, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x25, 0x0a, 0x0e, 0x62, 0x61, |
||||
|
0x69, 0x64, 0x75, 0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x25, 0x20, 0x01, |
||||
|
0x28, 0x09, 0x52, 0x0d, 0x62, 0x61, 0x69, 0x64, 0x75, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, |
||||
|
0x74, 0x12, 0x21, 0x0a, 0x0c, 0x62, 0x61, 0x69, 0x64, 0x75, 0x5f, 0x72, 0x65, 0x67, 0x69, 0x6f, |
||||
|
0x6e, 0x18, 0x26, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x62, 0x61, 0x69, 0x64, 0x75, 0x52, 0x65, |
||||
|
0x67, 0x69, 0x6f, 0x6e, 0x12, 0x2a, 0x0a, 0x11, 0x77, 0x61, 0x73, 0x61, 0x62, 0x69, 0x5f, 0x61, |
||||
|
0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x28, 0x20, 0x01, 0x28, 0x09, 0x52, |
||||
|
0x0f, 0x77, 0x61, 0x73, 0x61, 0x62, 0x69, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79, |
||||
|
0x12, 0x2a, 0x0a, 0x11, 0x77, 0x61, 0x73, 0x61, 0x62, 0x69, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, |
||||
|
0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x29, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x77, 0x61, 0x73, |
||||
|
0x61, 0x62, 0x69, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x27, 0x0a, 0x0f, |
||||
|
0x77, 0x61, 0x73, 0x61, 0x62, 0x69, 0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, |
||||
|
0x2a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x77, 0x61, 0x73, 0x61, 0x62, 0x69, 0x45, 0x6e, 0x64, |
||||
|
0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x23, 0x0a, 0x0d, 0x77, 0x61, 0x73, 0x61, 0x62, 0x69, 0x5f, |
||||
|
0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x18, 0x2b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x77, 0x61, |
||||
|
0x73, 0x61, 0x62, 0x69, 0x52, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x12, 0x25, 0x0a, 0x0e, 0x68, 0x64, |
||||
|
0x66, 0x73, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x6e, 0x6f, 0x64, 0x65, 0x73, 0x18, 0x32, 0x20, 0x03, |
||||
|
0x28, 0x09, 0x52, 0x0d, 0x68, 0x64, 0x66, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x6e, 0x6f, 0x64, 0x65, |
||||
|
0x73, 0x12, 0x23, 0x0a, 0x0d, 0x68, 0x64, 0x66, 0x73, 0x5f, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, |
||||
|
0x6d, 0x65, 0x18, 0x33, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x68, 0x64, 0x66, 0x73, 0x55, 0x73, |
||||
|
0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x3d, 0x0a, 0x1b, 0x68, 0x64, 0x66, 0x73, 0x5f, 0x73, |
||||
|
0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x70, 0x72, 0x69, 0x6e, 0x63, 0x69, 0x70, 0x61, 0x6c, |
||||
|
0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x34, 0x20, 0x01, 0x28, 0x09, 0x52, 0x18, 0x68, 0x64, 0x66, |
||||
|
0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x50, 0x72, 0x69, 0x6e, 0x63, 0x69, 0x70, 0x61, |
||||
|
0x6c, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x41, 0x0a, 0x1d, 0x68, 0x64, 0x66, 0x73, 0x5f, 0x64, 0x61, |
||||
|
0x74, 0x61, 0x5f, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x5f, 0x70, 0x72, 0x6f, 0x74, |
||||
|
0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x35, 0x20, 0x01, 0x28, 0x09, 0x52, 0x1a, 0x68, 0x64, |
||||
|
0x66, 0x73, 0x44, 0x61, 0x74, 0x61, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x50, 0x72, |
||||
|
0x6f, 0x74, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0xc0, 0x01, 0x0a, 0x14, 0x52, 0x65, 0x6d, |
||||
|
0x6f, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, |
||||
|
0x67, 0x12, 0x49, 0x0a, 0x08, 0x6d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x01, 0x20, |
||||
|
0x03, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x70, 0x62, 0x2e, |
||||
|
0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4d, 0x61, 0x70, |
||||
|
0x70, 0x69, 0x6e, 0x67, 0x2e, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x45, 0x6e, 0x74, |
||||
|
0x72, 0x79, 0x52, 0x08, 0x6d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x1a, 0x5d, 0x0a, 0x0d, |
||||
|
0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, |
||||
|
0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, |
||||
|
0x36, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, |
||||
|
0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x74, |
||||
|
0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, |
||||
|
0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x57, 0x0a, 0x15, 0x52, |
||||
|
0x65, 0x6d, 0x6f, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61, |
||||
|
0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, |
||||
|
0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x75, 0x63, 0x6b, |
||||
|
0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, |
||||
|
0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, |
||||
|
0x70, 0x61, 0x74, 0x68, 0x42, 0x50, 0x0a, 0x10, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, |
||||
|
0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x42, 0x0a, 0x46, 0x69, 0x6c, 0x65, 0x72, 0x50, |
||||
|
0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x30, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, |
||||
|
0x2f, 0x63, 0x68, 0x72, 0x69, 0x73, 0x6c, 0x75, 0x73, 0x66, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, |
||||
|
0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x70, 0x62, 0x2f, 0x72, 0x65, 0x6d, |
||||
|
0x6f, 0x74, 0x65, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, |
||||
|
} |
||||
|
|
||||
|
var ( |
||||
|
file_remote_proto_rawDescOnce sync.Once |
||||
|
file_remote_proto_rawDescData = file_remote_proto_rawDesc |
||||
|
) |
||||
|
|
||||
|
func file_remote_proto_rawDescGZIP() []byte { |
||||
|
file_remote_proto_rawDescOnce.Do(func() { |
||||
|
file_remote_proto_rawDescData = protoimpl.X.CompressGZIP(file_remote_proto_rawDescData) |
||||
|
}) |
||||
|
return file_remote_proto_rawDescData |
||||
|
} |
||||
|
|
||||
|
var file_remote_proto_msgTypes = make([]protoimpl.MessageInfo, 4) |
||||
|
var file_remote_proto_goTypes = []interface{}{ |
||||
|
(*RemoteConf)(nil), // 0: remote_pb.RemoteConf
|
||||
|
(*RemoteStorageMapping)(nil), // 1: remote_pb.RemoteStorageMapping
|
||||
|
(*RemoteStorageLocation)(nil), // 2: remote_pb.RemoteStorageLocation
|
||||
|
nil, // 3: remote_pb.RemoteStorageMapping.MappingsEntry
|
||||
|
} |
||||
|
var file_remote_proto_depIdxs = []int32{ |
||||
|
3, // 0: remote_pb.RemoteStorageMapping.mappings:type_name -> remote_pb.RemoteStorageMapping.MappingsEntry
|
||||
|
2, // 1: remote_pb.RemoteStorageMapping.MappingsEntry.value:type_name -> remote_pb.RemoteStorageLocation
|
||||
|
2, // [2:2] is the sub-list for method output_type
|
||||
|
2, // [2:2] is the sub-list for method input_type
|
||||
|
2, // [2:2] is the sub-list for extension type_name
|
||||
|
2, // [2:2] is the sub-list for extension extendee
|
||||
|
0, // [0:2] is the sub-list for field type_name
|
||||
|
} |
||||
|
|
||||
|
func init() { file_remote_proto_init() } |
||||
|
func file_remote_proto_init() { |
||||
|
if File_remote_proto != nil { |
||||
|
return |
||||
|
} |
||||
|
if !protoimpl.UnsafeEnabled { |
||||
|
file_remote_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { |
||||
|
switch v := v.(*RemoteConf); i { |
||||
|
case 0: |
||||
|
return &v.state |
||||
|
case 1: |
||||
|
return &v.sizeCache |
||||
|
case 2: |
||||
|
return &v.unknownFields |
||||
|
default: |
||||
|
return nil |
||||
|
} |
||||
|
} |
||||
|
file_remote_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { |
||||
|
switch v := v.(*RemoteStorageMapping); i { |
||||
|
case 0: |
||||
|
return &v.state |
||||
|
case 1: |
||||
|
return &v.sizeCache |
||||
|
case 2: |
||||
|
return &v.unknownFields |
||||
|
default: |
||||
|
return nil |
||||
|
} |
||||
|
} |
||||
|
file_remote_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { |
||||
|
switch v := v.(*RemoteStorageLocation); i { |
||||
|
case 0: |
||||
|
return &v.state |
||||
|
case 1: |
||||
|
return &v.sizeCache |
||||
|
case 2: |
||||
|
return &v.unknownFields |
||||
|
default: |
||||
|
return nil |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
type x struct{} |
||||
|
out := protoimpl.TypeBuilder{ |
||||
|
File: protoimpl.DescBuilder{ |
||||
|
GoPackagePath: reflect.TypeOf(x{}).PkgPath(), |
||||
|
RawDescriptor: file_remote_proto_rawDesc, |
||||
|
NumEnums: 0, |
||||
|
NumMessages: 4, |
||||
|
NumExtensions: 0, |
||||
|
NumServices: 0, |
||||
|
}, |
||||
|
GoTypes: file_remote_proto_goTypes, |
||||
|
DependencyIndexes: file_remote_proto_depIdxs, |
||||
|
MessageInfos: file_remote_proto_msgTypes, |
||||
|
}.Build() |
||||
|
File_remote_proto = out.File |
||||
|
file_remote_proto_rawDesc = nil |
||||
|
file_remote_proto_goTypes = nil |
||||
|
file_remote_proto_depIdxs = nil |
||||
|
} |
||||
@ -0,0 +1,8 @@ |
|||||
|
package remote_pb |
||||
|
|
||||
|
import "github.com/golang/protobuf/proto" |
||||
|
|
||||
|
func (fp *RemoteStorageLocation) Key() interface{} { |
||||
|
key, _ := proto.Marshal(fp) |
||||
|
return string(key) |
||||
|
} |
||||
1899
weed/pb/volume_server_pb/volume_server.pb.go
File diff suppressed because it is too large
View File
File diff suppressed because it is too large
View File
@ -0,0 +1,55 @@ |
|||||
|
package hdfs |
||||
|
|
||||
|
import ( |
||||
|
"fmt" |
||||
|
"os" |
||||
|
"os/user" |
||||
|
"strings" |
||||
|
|
||||
|
krb "github.com/jcmturner/gokrb5/v8/client" |
||||
|
"github.com/jcmturner/gokrb5/v8/config" |
||||
|
"github.com/jcmturner/gokrb5/v8/credentials" |
||||
|
) |
||||
|
|
||||
|
// copy-paste from https://github.com/colinmarc/hdfs/blob/master/cmd/hdfs/kerberos.go
|
||||
|
func getKerberosClient() (*krb.Client, error) { |
||||
|
configPath := os.Getenv("KRB5_CONFIG") |
||||
|
if configPath == "" { |
||||
|
configPath = "/etc/krb5.conf" |
||||
|
} |
||||
|
|
||||
|
cfg, err := config.Load(configPath) |
||||
|
if err != nil { |
||||
|
return nil, err |
||||
|
} |
||||
|
|
||||
|
// Determine the ccache location from the environment, falling back to the
|
||||
|
// default location.
|
||||
|
ccachePath := os.Getenv("KRB5CCNAME") |
||||
|
if strings.Contains(ccachePath, ":") { |
||||
|
if strings.HasPrefix(ccachePath, "FILE:") { |
||||
|
ccachePath = strings.SplitN(ccachePath, ":", 2)[1] |
||||
|
} else { |
||||
|
return nil, fmt.Errorf("unusable ccache: %s", ccachePath) |
||||
|
} |
||||
|
} else if ccachePath == "" { |
||||
|
u, err := user.Current() |
||||
|
if err != nil { |
||||
|
return nil, err |
||||
|
} |
||||
|
|
||||
|
ccachePath = fmt.Sprintf("/tmp/krb5cc_%s", u.Uid) |
||||
|
} |
||||
|
|
||||
|
ccache, err := credentials.LoadCCache(ccachePath) |
||||
|
if err != nil { |
||||
|
return nil, err |
||||
|
} |
||||
|
|
||||
|
client, err := krb.NewFromCCache(ccache, cfg) |
||||
|
if err != nil { |
||||
|
return nil, err |
||||
|
} |
||||
|
|
||||
|
return client, nil |
||||
|
} |
||||
@ -0,0 +1,178 @@ |
|||||
|
package hdfs |
||||
|
|
||||
|
import ( |
||||
|
"fmt" |
||||
|
"github.com/chrislusf/seaweedfs/weed/glog" |
||||
|
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" |
||||
|
"github.com/chrislusf/seaweedfs/weed/pb/remote_pb" |
||||
|
"github.com/chrislusf/seaweedfs/weed/remote_storage" |
||||
|
"github.com/chrislusf/seaweedfs/weed/util" |
||||
|
"github.com/colinmarc/hdfs/v2" |
||||
|
"io" |
||||
|
"os" |
||||
|
"path" |
||||
|
) |
||||
|
|
||||
|
func init() { |
||||
|
remote_storage.RemoteStorageClientMakers["hdfs"] = new(hdfsRemoteStorageMaker) |
||||
|
} |
||||
|
|
||||
|
type hdfsRemoteStorageMaker struct{} |
||||
|
|
||||
|
func (s hdfsRemoteStorageMaker) HasBucket() bool { |
||||
|
return false |
||||
|
} |
||||
|
|
||||
|
func (s hdfsRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) { |
||||
|
client := &hdfsRemoteStorageClient{ |
||||
|
conf: conf, |
||||
|
} |
||||
|
|
||||
|
options := hdfs.ClientOptions{ |
||||
|
Addresses: conf.HdfsNamenodes, |
||||
|
UseDatanodeHostname: false, |
||||
|
} |
||||
|
|
||||
|
if conf.HdfsServicePrincipalName != "" { |
||||
|
var err error |
||||
|
options.KerberosClient, err = getKerberosClient() |
||||
|
if err != nil { |
||||
|
return nil, fmt.Errorf("get kerberos authentication: %s", err) |
||||
|
} |
||||
|
options.KerberosServicePrincipleName = conf.HdfsServicePrincipalName |
||||
|
|
||||
|
if conf.HdfsDataTransferProtection != "" { |
||||
|
options.DataTransferProtection = conf.HdfsDataTransferProtection |
||||
|
} |
||||
|
} else { |
||||
|
options.User = conf.HdfsUsername |
||||
|
} |
||||
|
|
||||
|
c, err := hdfs.NewClient(options) |
||||
|
if err != nil { |
||||
|
return nil, err |
||||
|
} |
||||
|
|
||||
|
client.client = c |
||||
|
return client, nil |
||||
|
} |
||||
|
|
||||
|
type hdfsRemoteStorageClient struct { |
||||
|
conf *remote_pb.RemoteConf |
||||
|
client *hdfs.Client |
||||
|
} |
||||
|
|
||||
|
var _ = remote_storage.RemoteStorageClient(&hdfsRemoteStorageClient{}) |
||||
|
|
||||
|
func (c *hdfsRemoteStorageClient) Traverse(loc *remote_pb.RemoteStorageLocation, visitFn remote_storage.VisitFunc) (err error) { |
||||
|
|
||||
|
return remote_storage.TraverseBfs(func(parentDir util.FullPath, visitFn remote_storage.VisitFunc) error { |
||||
|
children, err := c.client.ReadDir(string(parentDir)) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
for _, child := range children { |
||||
|
if err := visitFn(string(parentDir), child.Name(), child.IsDir(), &filer_pb.RemoteEntry{ |
||||
|
StorageName: c.conf.Name, |
||||
|
LastLocalSyncTsNs: 0, |
||||
|
RemoteETag: "", |
||||
|
RemoteMtime: child.ModTime().Unix(), |
||||
|
RemoteSize: child.Size(), |
||||
|
}); err != nil { |
||||
|
return nil |
||||
|
} |
||||
|
} |
||||
|
return nil |
||||
|
}, util.FullPath(loc.Path), visitFn) |
||||
|
|
||||
|
} |
||||
|
func (c *hdfsRemoteStorageClient) ReadFile(loc *remote_pb.RemoteStorageLocation, offset int64, size int64) (data []byte, err error) { |
||||
|
|
||||
|
f, err := c.client.Open(loc.Path) |
||||
|
if err != nil { |
||||
|
return |
||||
|
} |
||||
|
defer f.Close() |
||||
|
data = make([]byte, size) |
||||
|
_, err = f.ReadAt(data, offset) |
||||
|
|
||||
|
return |
||||
|
|
||||
|
} |
||||
|
|
||||
|
func (c *hdfsRemoteStorageClient) WriteDirectory(loc *remote_pb.RemoteStorageLocation, entry *filer_pb.Entry) (err error) { |
||||
|
return c.client.MkdirAll(loc.Path, os.FileMode(entry.Attributes.FileMode)) |
||||
|
} |
||||
|
|
||||
|
func (c *hdfsRemoteStorageClient) RemoveDirectory(loc *remote_pb.RemoteStorageLocation) (err error) { |
||||
|
return c.client.RemoveAll(loc.Path) |
||||
|
} |
||||
|
|
||||
|
func (c *hdfsRemoteStorageClient) WriteFile(loc *remote_pb.RemoteStorageLocation, entry *filer_pb.Entry, reader io.Reader) (remoteEntry *filer_pb.RemoteEntry, err error) { |
||||
|
|
||||
|
dirname := path.Dir(loc.Path) |
||||
|
|
||||
|
// ensure parent directory
|
||||
|
if err = c.client.MkdirAll(dirname, 0755); err != nil { |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
// remove existing file
|
||||
|
info, err := c.client.Stat(loc.Path) |
||||
|
if err == nil { |
||||
|
err = c.client.Remove(loc.Path) |
||||
|
if err != nil { |
||||
|
return |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// create new file
|
||||
|
out, err := c.client.Create(loc.Path) |
||||
|
if err != nil { |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
cleanup := func() { |
||||
|
if removeErr := c.client.Remove(loc.Path); removeErr != nil { |
||||
|
glog.Errorf("clean up %s%s: %v", loc.Name, loc.Path, removeErr) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
if _, err = io.Copy(out, reader); err != nil { |
||||
|
cleanup() |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
if err = out.Close(); err != nil { |
||||
|
cleanup() |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
info, err = c.client.Stat(loc.Path) |
||||
|
if err != nil { |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
return &filer_pb.RemoteEntry{ |
||||
|
RemoteMtime: info.ModTime().Unix(), |
||||
|
RemoteSize: info.Size(), |
||||
|
RemoteETag: "", |
||||
|
StorageName: c.conf.Name, |
||||
|
}, nil |
||||
|
|
||||
|
} |
||||
|
|
||||
|
func (c *hdfsRemoteStorageClient) UpdateFileMetadata(loc *remote_pb.RemoteStorageLocation, oldEntry *filer_pb.Entry, newEntry *filer_pb.Entry) error { |
||||
|
if oldEntry.Attributes.FileMode != newEntry.Attributes.FileMode { |
||||
|
if err := c.client.Chmod(loc.Path, os.FileMode(newEntry.Attributes.FileMode)); err != nil { |
||||
|
return err |
||||
|
} |
||||
|
} |
||||
|
return nil |
||||
|
} |
||||
|
func (c *hdfsRemoteStorageClient) DeleteFile(loc *remote_pb.RemoteStorageLocation) (err error) { |
||||
|
if err = c.client.Remove(loc.Path); err != nil { |
||||
|
return fmt.Errorf("hdfs delete %s: %v", loc.Path, err) |
||||
|
} |
||||
|
return |
||||
|
} |
||||
@ -0,0 +1,61 @@ |
|||||
|
package s3 |
||||
|
|
||||
|
import ( |
||||
|
"fmt" |
||||
|
"github.com/aws/aws-sdk-go/aws" |
||||
|
"github.com/aws/aws-sdk-go/aws/credentials" |
||||
|
"github.com/aws/aws-sdk-go/aws/request" |
||||
|
"github.com/aws/aws-sdk-go/aws/session" |
||||
|
"github.com/aws/aws-sdk-go/service/s3" |
||||
|
"github.com/chrislusf/seaweedfs/weed/pb/remote_pb" |
||||
|
"github.com/chrislusf/seaweedfs/weed/remote_storage" |
||||
|
"github.com/chrislusf/seaweedfs/weed/util" |
||||
|
) |
||||
|
|
||||
|
func init() { |
||||
|
remote_storage.RemoteStorageClientMakers["wasabi"] = new(WasabiRemoteStorageMaker) |
||||
|
} |
||||
|
|
||||
|
type WasabiRemoteStorageMaker struct{} |
||||
|
|
||||
|
func (s WasabiRemoteStorageMaker) HasBucket() bool { |
||||
|
return true |
||||
|
} |
||||
|
|
||||
|
func (s WasabiRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) { |
||||
|
client := &s3RemoteStorageClient{ |
||||
|
conf: conf, |
||||
|
} |
||||
|
accessKey := util.Nvl(conf.WasabiAccessKey) |
||||
|
secretKey := util.Nvl(conf.WasabiSecretKey) |
||||
|
|
||||
|
config := &aws.Config{ |
||||
|
Endpoint: aws.String(conf.WasabiEndpoint), |
||||
|
Region: aws.String(conf.WasabiRegion), |
||||
|
S3ForcePathStyle: aws.Bool(true), |
||||
|
S3DisableContentMD5Validation: aws.Bool(true), |
||||
|
} |
||||
|
if accessKey != "" && secretKey != "" { |
||||
|
config.Credentials = credentials.NewStaticCredentials(accessKey, secretKey, "") |
||||
|
} |
||||
|
|
||||
|
sess, err := session.NewSession(config) |
||||
|
if err != nil { |
||||
|
return nil, fmt.Errorf("create wasabi session: %v", err) |
||||
|
} |
||||
|
sess.Handlers.Build.PushFront(skipSha256PayloadSigning) |
||||
|
client.conn = s3.New(sess) |
||||
|
return client, nil |
||||
|
} |
||||
|
|
||||
|
var skipSha256PayloadSigning = func(r *request.Request) { |
||||
|
// see https://github.com/ceph/ceph/pull/15965/files
|
||||
|
if r.ClientInfo.ServiceID != "S3" { |
||||
|
return |
||||
|
} |
||||
|
if r.Operation.Name == "PutObject" || r.Operation.Name == "UploadPart" { |
||||
|
if len(r.HTTPRequest.Header.Get("X-Amz-Content-Sha256")) == 0 { |
||||
|
r.HTTPRequest.Header.Set("X-Amz-Content-Sha256", "UNSIGNED-PAYLOAD") |
||||
|
} |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,62 @@ |
|||||
|
package remote_storage |
||||
|
|
||||
|
import ( |
||||
|
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" |
||||
|
"github.com/chrislusf/seaweedfs/weed/util" |
||||
|
"sync" |
||||
|
"time" |
||||
|
) |
||||
|
|
||||
|
type ListDirectoryFunc func(parentDir util.FullPath, visitFn VisitFunc) error |
||||
|
|
||||
|
func TraverseBfs(listDirFn ListDirectoryFunc, parentPath util.FullPath, visitFn VisitFunc) (err error) { |
||||
|
K := 5 |
||||
|
|
||||
|
var dirQueueWg sync.WaitGroup |
||||
|
dirQueue := util.NewQueue() |
||||
|
dirQueueWg.Add(1) |
||||
|
dirQueue.Enqueue(parentPath) |
||||
|
var isTerminating bool |
||||
|
|
||||
|
for i := 0; i < K; i++ { |
||||
|
go func() { |
||||
|
for { |
||||
|
if isTerminating { |
||||
|
break |
||||
|
} |
||||
|
t := dirQueue.Dequeue() |
||||
|
if t == nil { |
||||
|
time.Sleep(329 * time.Millisecond) |
||||
|
continue |
||||
|
} |
||||
|
dir := t.(util.FullPath) |
||||
|
processErr := processOneDirectory(listDirFn, dir, visitFn, dirQueue, &dirQueueWg) |
||||
|
if processErr != nil { |
||||
|
err = processErr |
||||
|
} |
||||
|
dirQueueWg.Done() |
||||
|
} |
||||
|
}() |
||||
|
} |
||||
|
|
||||
|
dirQueueWg.Wait() |
||||
|
isTerminating = true |
||||
|
return |
||||
|
|
||||
|
} |
||||
|
|
||||
|
func processOneDirectory(listDirFn ListDirectoryFunc, parentPath util.FullPath, visitFn VisitFunc, dirQueue *util.Queue, dirQueueWg *sync.WaitGroup) (error) { |
||||
|
|
||||
|
return listDirFn(parentPath, func(dir string, name string, isDirectory bool, remoteEntry *filer_pb.RemoteEntry) error { |
||||
|
if err := visitFn(dir, name, isDirectory, remoteEntry); err != nil { |
||||
|
return err |
||||
|
} |
||||
|
if !isDirectory { |
||||
|
return nil |
||||
|
} |
||||
|
dirQueueWg.Add(1) |
||||
|
dirQueue.Enqueue(parentPath.Child(name)) |
||||
|
return nil |
||||
|
}) |
||||
|
|
||||
|
} |
||||
@ -1,4 +1,4 @@ |
|||||
package pb |
|
||||
|
package volume_info |
||||
|
|
||||
import ( |
import ( |
||||
"bytes" |
"bytes" |
||||
Write
Preview
Loading…
Cancel
Save
Reference in new issue