Rain Li
3 years ago
committed by
GitHub
GitHub
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
66 changed files with 4876 additions and 3749 deletions
-
50.github/workflows/binary_test.yml
-
2.github/workflows/cleanup.yml
-
13.github/workflows/container_latest.yml
-
13.github/workflows/container_release.yml
-
53.github/workflows/container_test.yml
-
6.github/workflows/go.yml
-
12.github/workflows/release.yml
-
16.github/workflows/release_binaries.yml
-
4docker/Dockerfile
-
15go.mod
-
26go.sum
-
4k8s/helm_charts2/Chart.yaml
-
4other/java/client/src/main/java/seaweedfs/client/FilerClient.java
-
5other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java
-
7other/java/client/src/main/java/seaweedfs/client/VolumeIdCache.java
-
45other/java/client/src/main/proto/filer.proto
-
644other/metrics/grafana_seaweedfs.json
-
30weed/command/filer_remote_sync.go
-
1weed/command/imports.go
-
41weed/filer/filer_remote_storage.go
-
6weed/filer/filer_remote_storage_test.go
-
23weed/filer/read_remote.go
-
66weed/filer/stream.go
-
4weed/filesys/meta_cache/meta_cache_subscribe.go
-
1weed/pb/Makefile
-
45weed/pb/filer.proto
-
1044weed/pb/filer_pb/filer.pb.go
-
4weed/pb/filer_pb/filer_pb_helper.go
-
64weed/pb/remote.proto
-
653weed/pb/remote_pb/remote.pb.go
-
8weed/pb/remote_pb/remote_pb_helper.go
-
12weed/pb/volume_server.proto
-
1899weed/pb/volume_server_pb/volume_server.pb.go
-
27weed/remote_storage/azure/azure_storage_client.go
-
27weed/remote_storage/gcs/gcs_storage_client.go
-
55weed/remote_storage/hdfs/hdfs_kerberos.go
-
178weed/remote_storage/hdfs/hdfs_storage_client.go
-
66weed/remote_storage/remote_storage.go
-
10weed/remote_storage/s3/aliyun.go
-
10weed/remote_storage/s3/backblaze.go
-
10weed/remote_storage/s3/baidu.go
-
33weed/remote_storage/s3/s3_storage_client.go
-
11weed/remote_storage/s3/tencent.go
-
61weed/remote_storage/s3/wasabi.go
-
62weed/remote_storage/traverse_bfs.go
-
1weed/replication/sink/s3sink/s3_sink.go
-
1weed/replication/sub/notification_aws_sqs.go
-
3weed/server/filer_grpc_server.go
-
26weed/server/filer_grpc_server_remote.go
-
4weed/server/volume_grpc_erasure_coding.go
-
17weed/server/volume_grpc_remote.go
-
36weed/shell/command_remote_cache.go
-
45weed/shell/command_remote_configure.go
-
7weed/shell/command_remote_meta_sync.go
-
22weed/shell/command_remote_mount.go
-
4weed/shell/command_remote_uncache.go
-
14weed/shell/command_remote_unmount.go
-
2weed/shell/command_volume_fix_replication.go
-
20weed/shell/shell_liner.go
-
1weed/storage/backend/s3_backend/s3_sessions.go
-
6weed/storage/erasure_coding/ec_volume.go
-
6weed/storage/needle_map_metric.go
-
6weed/storage/store.go
-
2weed/storage/volume_info/volume_info.go
-
6weed/storage/volume_tier.go
-
2weed/util/constants.go
@ -0,0 +1,50 @@ |
|||
name: "go: test building cross-platform binary" |
|||
|
|||
on: |
|||
pull_request: |
|||
workflow_dispatch: [] |
|||
|
|||
jobs: |
|||
|
|||
build: |
|||
name: Build |
|||
runs-on: ubuntu-latest |
|||
strategy: |
|||
matrix: |
|||
goos: [linux, windows, darwin, freebsd, netbsd, openbsd] |
|||
goarch: [amd64, arm, arm64, 386] |
|||
exclude: |
|||
- goarch: arm |
|||
goos: darwin |
|||
- goarch: 386 |
|||
goos: darwin |
|||
- goarch: arm |
|||
goos: windows |
|||
- goarch: arm64 |
|||
goos: windows |
|||
|
|||
concurrency: |
|||
group: ${{ github.head_ref }}/binary_test/${{ matrix.goos }}/${{ matrix.goarch }} |
|||
cancel-in-progress: true |
|||
|
|||
steps: |
|||
|
|||
- name: Set up Go 1.x |
|||
uses: actions/setup-go@v2 |
|||
with: |
|||
go-version: ^1.13 |
|||
id: go |
|||
|
|||
- name: Check out code into the Go module directory |
|||
uses: actions/checkout@v2 |
|||
|
|||
- name: Get dependencies |
|||
run: | |
|||
cd weed; go get -v -t -d ./... |
|||
if [ -f Gopkg.toml ]; then |
|||
curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh |
|||
dep ensure |
|||
fi |
|||
|
|||
- name: Build |
|||
run: cd weed; GOOS=${{ matrix.goos }} GOARCH=${{ matrix.goarch }} go build -v . |
|||
@ -0,0 +1,53 @@ |
|||
name: "docker: test building container images" |
|||
|
|||
on: |
|||
pull_request: |
|||
workflow_dispatch: [] |
|||
|
|||
concurrency: |
|||
group: ${{ github.head_ref }}/container_test |
|||
cancel-in-progress: true |
|||
|
|||
jobs: |
|||
build-test: |
|||
runs-on: [ubuntu-latest] |
|||
strategy: |
|||
matrix: |
|||
platform: [ linux ] |
|||
arch: [ amd64, arm, arm64, 386 ] |
|||
|
|||
steps: |
|||
- |
|||
name: Checkout |
|||
uses: actions/checkout@v2 |
|||
- |
|||
name: Docker meta |
|||
id: docker_meta |
|||
uses: docker/metadata-action@v3 |
|||
with: |
|||
images: | |
|||
chrislusf/seaweedfs |
|||
ghcr.io/chrislusf/seaweedfs |
|||
tags: | |
|||
type=raw,value=latest |
|||
labels: | |
|||
org.opencontainers.image.title=seaweedfs |
|||
org.opencontainers.image.vendor=Chris Lu |
|||
- |
|||
name: Set up QEMU |
|||
uses: docker/setup-qemu-action@v1 |
|||
- |
|||
name: Set up Docker Buildx |
|||
uses: docker/setup-buildx-action@v1 |
|||
with: |
|||
buildkitd-flags: "--debug" |
|||
- |
|||
name: Build |
|||
uses: docker/build-push-action@v2 |
|||
with: |
|||
context: ./docker |
|||
push: false |
|||
file: ./docker/Dockerfile |
|||
platforms: ${{ matrix.platform }}/${{ matrix.arch }} |
|||
tags: ${{ steps.docker_meta.outputs.tags }} |
|||
labels: ${{ steps.docker_meta.outputs.labels }} |
|||
@ -1,5 +1,5 @@ |
|||
apiVersion: v1 |
|||
description: SeaweedFS |
|||
name: seaweedfs |
|||
appVersion: "2.64" |
|||
version: "2.64" |
|||
appVersion: "2.65" |
|||
version: "2.65" |
|||
644
other/metrics/grafana_seaweedfs.json
File diff suppressed because it is too large
View File
File diff suppressed because it is too large
View File
1044
weed/pb/filer_pb/filer.pb.go
File diff suppressed because it is too large
View File
File diff suppressed because it is too large
View File
@ -0,0 +1,64 @@ |
|||
syntax = "proto3"; |
|||
|
|||
package remote_pb; |
|||
|
|||
option go_package = "github.com/chrislusf/seaweedfs/weed/pb/remote_pb"; |
|||
option java_package = "seaweedfs.client"; |
|||
option java_outer_classname = "FilerProto"; |
|||
|
|||
///////////////////////// |
|||
// Remote Storage related |
|||
///////////////////////// |
|||
message RemoteConf { |
|||
string type = 1; |
|||
string name = 2; |
|||
string s3_access_key = 4; |
|||
string s3_secret_key = 5; |
|||
string s3_region = 6; |
|||
string s3_endpoint = 7; |
|||
string s3_storage_class = 8; |
|||
bool s3_force_path_style = 9; |
|||
|
|||
string gcs_google_application_credentials = 10; |
|||
|
|||
string azure_account_name = 15; |
|||
string azure_account_key = 16; |
|||
|
|||
string backblaze_key_id = 20; |
|||
string backblaze_application_key = 21; |
|||
string backblaze_endpoint = 22; |
|||
|
|||
string aliyun_access_key = 25; |
|||
string aliyun_secret_key = 26; |
|||
string aliyun_endpoint = 27; |
|||
string aliyun_region = 28; |
|||
|
|||
string tencent_secret_id = 30; |
|||
string tencent_secret_key = 31; |
|||
string tencent_endpoint = 32; |
|||
|
|||
string baidu_access_key = 35; |
|||
string baidu_secret_key = 36; |
|||
string baidu_endpoint = 37; |
|||
string baidu_region = 38; |
|||
|
|||
string wasabi_access_key = 40; |
|||
string wasabi_secret_key = 41; |
|||
string wasabi_endpoint = 42; |
|||
string wasabi_region = 43; |
|||
|
|||
repeated string hdfs_namenodes = 50; |
|||
string hdfs_username = 51; |
|||
string hdfs_service_principal_name = 52; |
|||
string hdfs_data_transfer_protection = 53; |
|||
|
|||
} |
|||
|
|||
message RemoteStorageMapping { |
|||
map<string,RemoteStorageLocation> mappings = 1; |
|||
} |
|||
message RemoteStorageLocation { |
|||
string name = 1; |
|||
string bucket = 2; |
|||
string path = 3; |
|||
} |
|||
@ -0,0 +1,653 @@ |
|||
// Code generated by protoc-gen-go. DO NOT EDIT.
|
|||
// versions:
|
|||
// protoc-gen-go v1.25.0
|
|||
// protoc v3.12.3
|
|||
// source: remote.proto
|
|||
|
|||
package remote_pb |
|||
|
|||
import ( |
|||
proto "github.com/golang/protobuf/proto" |
|||
protoreflect "google.golang.org/protobuf/reflect/protoreflect" |
|||
protoimpl "google.golang.org/protobuf/runtime/protoimpl" |
|||
reflect "reflect" |
|||
sync "sync" |
|||
) |
|||
|
|||
const ( |
|||
// Verify that this generated code is sufficiently up-to-date.
|
|||
_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) |
|||
// Verify that runtime/protoimpl is sufficiently up-to-date.
|
|||
_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) |
|||
) |
|||
|
|||
// This is a compile-time assertion that a sufficiently up-to-date version
|
|||
// of the legacy proto package is being used.
|
|||
const _ = proto.ProtoPackageIsVersion4 |
|||
|
|||
/////////////////////////
|
|||
// Remote Storage related
|
|||
/////////////////////////
|
|||
type RemoteConf struct { |
|||
state protoimpl.MessageState |
|||
sizeCache protoimpl.SizeCache |
|||
unknownFields protoimpl.UnknownFields |
|||
|
|||
Type string `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"` |
|||
Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` |
|||
S3AccessKey string `protobuf:"bytes,4,opt,name=s3_access_key,json=s3AccessKey,proto3" json:"s3_access_key,omitempty"` |
|||
S3SecretKey string `protobuf:"bytes,5,opt,name=s3_secret_key,json=s3SecretKey,proto3" json:"s3_secret_key,omitempty"` |
|||
S3Region string `protobuf:"bytes,6,opt,name=s3_region,json=s3Region,proto3" json:"s3_region,omitempty"` |
|||
S3Endpoint string `protobuf:"bytes,7,opt,name=s3_endpoint,json=s3Endpoint,proto3" json:"s3_endpoint,omitempty"` |
|||
S3StorageClass string `protobuf:"bytes,8,opt,name=s3_storage_class,json=s3StorageClass,proto3" json:"s3_storage_class,omitempty"` |
|||
S3ForcePathStyle bool `protobuf:"varint,9,opt,name=s3_force_path_style,json=s3ForcePathStyle,proto3" json:"s3_force_path_style,omitempty"` |
|||
GcsGoogleApplicationCredentials string `protobuf:"bytes,10,opt,name=gcs_google_application_credentials,json=gcsGoogleApplicationCredentials,proto3" json:"gcs_google_application_credentials,omitempty"` |
|||
AzureAccountName string `protobuf:"bytes,15,opt,name=azure_account_name,json=azureAccountName,proto3" json:"azure_account_name,omitempty"` |
|||
AzureAccountKey string `protobuf:"bytes,16,opt,name=azure_account_key,json=azureAccountKey,proto3" json:"azure_account_key,omitempty"` |
|||
BackblazeKeyId string `protobuf:"bytes,20,opt,name=backblaze_key_id,json=backblazeKeyId,proto3" json:"backblaze_key_id,omitempty"` |
|||
BackblazeApplicationKey string `protobuf:"bytes,21,opt,name=backblaze_application_key,json=backblazeApplicationKey,proto3" json:"backblaze_application_key,omitempty"` |
|||
BackblazeEndpoint string `protobuf:"bytes,22,opt,name=backblaze_endpoint,json=backblazeEndpoint,proto3" json:"backblaze_endpoint,omitempty"` |
|||
AliyunAccessKey string `protobuf:"bytes,25,opt,name=aliyun_access_key,json=aliyunAccessKey,proto3" json:"aliyun_access_key,omitempty"` |
|||
AliyunSecretKey string `protobuf:"bytes,26,opt,name=aliyun_secret_key,json=aliyunSecretKey,proto3" json:"aliyun_secret_key,omitempty"` |
|||
AliyunEndpoint string `protobuf:"bytes,27,opt,name=aliyun_endpoint,json=aliyunEndpoint,proto3" json:"aliyun_endpoint,omitempty"` |
|||
AliyunRegion string `protobuf:"bytes,28,opt,name=aliyun_region,json=aliyunRegion,proto3" json:"aliyun_region,omitempty"` |
|||
TencentSecretId string `protobuf:"bytes,30,opt,name=tencent_secret_id,json=tencentSecretId,proto3" json:"tencent_secret_id,omitempty"` |
|||
TencentSecretKey string `protobuf:"bytes,31,opt,name=tencent_secret_key,json=tencentSecretKey,proto3" json:"tencent_secret_key,omitempty"` |
|||
TencentEndpoint string `protobuf:"bytes,32,opt,name=tencent_endpoint,json=tencentEndpoint,proto3" json:"tencent_endpoint,omitempty"` |
|||
BaiduAccessKey string `protobuf:"bytes,35,opt,name=baidu_access_key,json=baiduAccessKey,proto3" json:"baidu_access_key,omitempty"` |
|||
BaiduSecretKey string `protobuf:"bytes,36,opt,name=baidu_secret_key,json=baiduSecretKey,proto3" json:"baidu_secret_key,omitempty"` |
|||
BaiduEndpoint string `protobuf:"bytes,37,opt,name=baidu_endpoint,json=baiduEndpoint,proto3" json:"baidu_endpoint,omitempty"` |
|||
BaiduRegion string `protobuf:"bytes,38,opt,name=baidu_region,json=baiduRegion,proto3" json:"baidu_region,omitempty"` |
|||
WasabiAccessKey string `protobuf:"bytes,40,opt,name=wasabi_access_key,json=wasabiAccessKey,proto3" json:"wasabi_access_key,omitempty"` |
|||
WasabiSecretKey string `protobuf:"bytes,41,opt,name=wasabi_secret_key,json=wasabiSecretKey,proto3" json:"wasabi_secret_key,omitempty"` |
|||
WasabiEndpoint string `protobuf:"bytes,42,opt,name=wasabi_endpoint,json=wasabiEndpoint,proto3" json:"wasabi_endpoint,omitempty"` |
|||
WasabiRegion string `protobuf:"bytes,43,opt,name=wasabi_region,json=wasabiRegion,proto3" json:"wasabi_region,omitempty"` |
|||
HdfsNamenodes []string `protobuf:"bytes,50,rep,name=hdfs_namenodes,json=hdfsNamenodes,proto3" json:"hdfs_namenodes,omitempty"` |
|||
HdfsUsername string `protobuf:"bytes,51,opt,name=hdfs_username,json=hdfsUsername,proto3" json:"hdfs_username,omitempty"` |
|||
HdfsServicePrincipalName string `protobuf:"bytes,52,opt,name=hdfs_service_principal_name,json=hdfsServicePrincipalName,proto3" json:"hdfs_service_principal_name,omitempty"` |
|||
HdfsDataTransferProtection string `protobuf:"bytes,53,opt,name=hdfs_data_transfer_protection,json=hdfsDataTransferProtection,proto3" json:"hdfs_data_transfer_protection,omitempty"` |
|||
} |
|||
|
|||
func (x *RemoteConf) Reset() { |
|||
*x = RemoteConf{} |
|||
if protoimpl.UnsafeEnabled { |
|||
mi := &file_remote_proto_msgTypes[0] |
|||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) |
|||
ms.StoreMessageInfo(mi) |
|||
} |
|||
} |
|||
|
|||
func (x *RemoteConf) String() string { |
|||
return protoimpl.X.MessageStringOf(x) |
|||
} |
|||
|
|||
func (*RemoteConf) ProtoMessage() {} |
|||
|
|||
func (x *RemoteConf) ProtoReflect() protoreflect.Message { |
|||
mi := &file_remote_proto_msgTypes[0] |
|||
if protoimpl.UnsafeEnabled && x != nil { |
|||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) |
|||
if ms.LoadMessageInfo() == nil { |
|||
ms.StoreMessageInfo(mi) |
|||
} |
|||
return ms |
|||
} |
|||
return mi.MessageOf(x) |
|||
} |
|||
|
|||
// Deprecated: Use RemoteConf.ProtoReflect.Descriptor instead.
|
|||
func (*RemoteConf) Descriptor() ([]byte, []int) { |
|||
return file_remote_proto_rawDescGZIP(), []int{0} |
|||
} |
|||
|
|||
func (x *RemoteConf) GetType() string { |
|||
if x != nil { |
|||
return x.Type |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (x *RemoteConf) GetName() string { |
|||
if x != nil { |
|||
return x.Name |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (x *RemoteConf) GetS3AccessKey() string { |
|||
if x != nil { |
|||
return x.S3AccessKey |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (x *RemoteConf) GetS3SecretKey() string { |
|||
if x != nil { |
|||
return x.S3SecretKey |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (x *RemoteConf) GetS3Region() string { |
|||
if x != nil { |
|||
return x.S3Region |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (x *RemoteConf) GetS3Endpoint() string { |
|||
if x != nil { |
|||
return x.S3Endpoint |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (x *RemoteConf) GetS3StorageClass() string { |
|||
if x != nil { |
|||
return x.S3StorageClass |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (x *RemoteConf) GetS3ForcePathStyle() bool { |
|||
if x != nil { |
|||
return x.S3ForcePathStyle |
|||
} |
|||
return false |
|||
} |
|||
|
|||
func (x *RemoteConf) GetGcsGoogleApplicationCredentials() string { |
|||
if x != nil { |
|||
return x.GcsGoogleApplicationCredentials |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (x *RemoteConf) GetAzureAccountName() string { |
|||
if x != nil { |
|||
return x.AzureAccountName |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (x *RemoteConf) GetAzureAccountKey() string { |
|||
if x != nil { |
|||
return x.AzureAccountKey |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (x *RemoteConf) GetBackblazeKeyId() string { |
|||
if x != nil { |
|||
return x.BackblazeKeyId |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (x *RemoteConf) GetBackblazeApplicationKey() string { |
|||
if x != nil { |
|||
return x.BackblazeApplicationKey |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (x *RemoteConf) GetBackblazeEndpoint() string { |
|||
if x != nil { |
|||
return x.BackblazeEndpoint |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (x *RemoteConf) GetAliyunAccessKey() string { |
|||
if x != nil { |
|||
return x.AliyunAccessKey |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (x *RemoteConf) GetAliyunSecretKey() string { |
|||
if x != nil { |
|||
return x.AliyunSecretKey |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (x *RemoteConf) GetAliyunEndpoint() string { |
|||
if x != nil { |
|||
return x.AliyunEndpoint |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (x *RemoteConf) GetAliyunRegion() string { |
|||
if x != nil { |
|||
return x.AliyunRegion |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (x *RemoteConf) GetTencentSecretId() string { |
|||
if x != nil { |
|||
return x.TencentSecretId |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (x *RemoteConf) GetTencentSecretKey() string { |
|||
if x != nil { |
|||
return x.TencentSecretKey |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (x *RemoteConf) GetTencentEndpoint() string { |
|||
if x != nil { |
|||
return x.TencentEndpoint |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (x *RemoteConf) GetBaiduAccessKey() string { |
|||
if x != nil { |
|||
return x.BaiduAccessKey |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (x *RemoteConf) GetBaiduSecretKey() string { |
|||
if x != nil { |
|||
return x.BaiduSecretKey |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (x *RemoteConf) GetBaiduEndpoint() string { |
|||
if x != nil { |
|||
return x.BaiduEndpoint |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (x *RemoteConf) GetBaiduRegion() string { |
|||
if x != nil { |
|||
return x.BaiduRegion |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (x *RemoteConf) GetWasabiAccessKey() string { |
|||
if x != nil { |
|||
return x.WasabiAccessKey |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (x *RemoteConf) GetWasabiSecretKey() string { |
|||
if x != nil { |
|||
return x.WasabiSecretKey |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (x *RemoteConf) GetWasabiEndpoint() string { |
|||
if x != nil { |
|||
return x.WasabiEndpoint |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (x *RemoteConf) GetWasabiRegion() string { |
|||
if x != nil { |
|||
return x.WasabiRegion |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (x *RemoteConf) GetHdfsNamenodes() []string { |
|||
if x != nil { |
|||
return x.HdfsNamenodes |
|||
} |
|||
return nil |
|||
} |
|||
|
|||
func (x *RemoteConf) GetHdfsUsername() string { |
|||
if x != nil { |
|||
return x.HdfsUsername |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (x *RemoteConf) GetHdfsServicePrincipalName() string { |
|||
if x != nil { |
|||
return x.HdfsServicePrincipalName |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (x *RemoteConf) GetHdfsDataTransferProtection() string { |
|||
if x != nil { |
|||
return x.HdfsDataTransferProtection |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
type RemoteStorageMapping struct { |
|||
state protoimpl.MessageState |
|||
sizeCache protoimpl.SizeCache |
|||
unknownFields protoimpl.UnknownFields |
|||
|
|||
Mappings map[string]*RemoteStorageLocation `protobuf:"bytes,1,rep,name=mappings,proto3" json:"mappings,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` |
|||
} |
|||
|
|||
func (x *RemoteStorageMapping) Reset() { |
|||
*x = RemoteStorageMapping{} |
|||
if protoimpl.UnsafeEnabled { |
|||
mi := &file_remote_proto_msgTypes[1] |
|||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) |
|||
ms.StoreMessageInfo(mi) |
|||
} |
|||
} |
|||
|
|||
func (x *RemoteStorageMapping) String() string { |
|||
return protoimpl.X.MessageStringOf(x) |
|||
} |
|||
|
|||
func (*RemoteStorageMapping) ProtoMessage() {} |
|||
|
|||
func (x *RemoteStorageMapping) ProtoReflect() protoreflect.Message { |
|||
mi := &file_remote_proto_msgTypes[1] |
|||
if protoimpl.UnsafeEnabled && x != nil { |
|||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) |
|||
if ms.LoadMessageInfo() == nil { |
|||
ms.StoreMessageInfo(mi) |
|||
} |
|||
return ms |
|||
} |
|||
return mi.MessageOf(x) |
|||
} |
|||
|
|||
// Deprecated: Use RemoteStorageMapping.ProtoReflect.Descriptor instead.
|
|||
func (*RemoteStorageMapping) Descriptor() ([]byte, []int) { |
|||
return file_remote_proto_rawDescGZIP(), []int{1} |
|||
} |
|||
|
|||
func (x *RemoteStorageMapping) GetMappings() map[string]*RemoteStorageLocation { |
|||
if x != nil { |
|||
return x.Mappings |
|||
} |
|||
return nil |
|||
} |
|||
|
|||
type RemoteStorageLocation struct { |
|||
state protoimpl.MessageState |
|||
sizeCache protoimpl.SizeCache |
|||
unknownFields protoimpl.UnknownFields |
|||
|
|||
Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` |
|||
Bucket string `protobuf:"bytes,2,opt,name=bucket,proto3" json:"bucket,omitempty"` |
|||
Path string `protobuf:"bytes,3,opt,name=path,proto3" json:"path,omitempty"` |
|||
} |
|||
|
|||
func (x *RemoteStorageLocation) Reset() { |
|||
*x = RemoteStorageLocation{} |
|||
if protoimpl.UnsafeEnabled { |
|||
mi := &file_remote_proto_msgTypes[2] |
|||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) |
|||
ms.StoreMessageInfo(mi) |
|||
} |
|||
} |
|||
|
|||
func (x *RemoteStorageLocation) String() string { |
|||
return protoimpl.X.MessageStringOf(x) |
|||
} |
|||
|
|||
func (*RemoteStorageLocation) ProtoMessage() {} |
|||
|
|||
func (x *RemoteStorageLocation) ProtoReflect() protoreflect.Message { |
|||
mi := &file_remote_proto_msgTypes[2] |
|||
if protoimpl.UnsafeEnabled && x != nil { |
|||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) |
|||
if ms.LoadMessageInfo() == nil { |
|||
ms.StoreMessageInfo(mi) |
|||
} |
|||
return ms |
|||
} |
|||
return mi.MessageOf(x) |
|||
} |
|||
|
|||
// Deprecated: Use RemoteStorageLocation.ProtoReflect.Descriptor instead.
|
|||
func (*RemoteStorageLocation) Descriptor() ([]byte, []int) { |
|||
return file_remote_proto_rawDescGZIP(), []int{2} |
|||
} |
|||
|
|||
func (x *RemoteStorageLocation) GetName() string { |
|||
if x != nil { |
|||
return x.Name |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (x *RemoteStorageLocation) GetBucket() string { |
|||
if x != nil { |
|||
return x.Bucket |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (x *RemoteStorageLocation) GetPath() string { |
|||
if x != nil { |
|||
return x.Path |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
var File_remote_proto protoreflect.FileDescriptor |
|||
|
|||
var file_remote_proto_rawDesc = []byte{ |
|||
0x0a, 0x0c, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x09, |
|||
0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x70, 0x62, 0x22, 0x8c, 0x0b, 0x0a, 0x0a, 0x52, 0x65, |
|||
0x6d, 0x6f, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, |
|||
0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, |
|||
0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, |
|||
0x12, 0x22, 0x0a, 0x0d, 0x73, 0x33, 0x5f, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x6b, 0x65, |
|||
0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x73, 0x33, 0x41, 0x63, 0x63, 0x65, 0x73, |
|||
0x73, 0x4b, 0x65, 0x79, 0x12, 0x22, 0x0a, 0x0d, 0x73, 0x33, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, |
|||
0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x73, 0x33, 0x53, |
|||
0x65, 0x63, 0x72, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x33, 0x5f, 0x72, |
|||
0x65, 0x67, 0x69, 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x73, 0x33, 0x52, |
|||
0x65, 0x67, 0x69, 0x6f, 0x6e, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x33, 0x5f, 0x65, 0x6e, 0x64, 0x70, |
|||
0x6f, 0x69, 0x6e, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x33, 0x45, 0x6e, |
|||
0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x28, 0x0a, 0x10, 0x73, 0x33, 0x5f, 0x73, 0x74, 0x6f, |
|||
0x72, 0x61, 0x67, 0x65, 0x5f, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, |
|||
0x52, 0x0e, 0x73, 0x33, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, |
|||
0x12, 0x2d, 0x0a, 0x13, 0x73, 0x33, 0x5f, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x5f, 0x70, 0x61, 0x74, |
|||
0x68, 0x5f, 0x73, 0x74, 0x79, 0x6c, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x73, |
|||
0x33, 0x46, 0x6f, 0x72, 0x63, 0x65, 0x50, 0x61, 0x74, 0x68, 0x53, 0x74, 0x79, 0x6c, 0x65, 0x12, |
|||
0x4b, 0x0a, 0x22, 0x67, 0x63, 0x73, 0x5f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x5f, 0x61, 0x70, |
|||
0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, |
|||
0x74, 0x69, 0x61, 0x6c, 0x73, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x1f, 0x67, 0x63, 0x73, |
|||
0x47, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, |
|||
0x6e, 0x43, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x12, 0x2c, 0x0a, 0x12, |
|||
0x61, 0x7a, 0x75, 0x72, 0x65, 0x5f, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x6e, 0x61, |
|||
0x6d, 0x65, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x61, 0x7a, 0x75, 0x72, 0x65, 0x41, |
|||
0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x2a, 0x0a, 0x11, 0x61, 0x7a, |
|||
0x75, 0x72, 0x65, 0x5f, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, |
|||
0x10, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x61, 0x7a, 0x75, 0x72, 0x65, 0x41, 0x63, 0x63, 0x6f, |
|||
0x75, 0x6e, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x28, 0x0a, 0x10, 0x62, 0x61, 0x63, 0x6b, 0x62, 0x6c, |
|||
0x61, 0x7a, 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x5f, 0x69, 0x64, 0x18, 0x14, 0x20, 0x01, 0x28, 0x09, |
|||
0x52, 0x0e, 0x62, 0x61, 0x63, 0x6b, 0x62, 0x6c, 0x61, 0x7a, 0x65, 0x4b, 0x65, 0x79, 0x49, 0x64, |
|||
0x12, 0x3a, 0x0a, 0x19, 0x62, 0x61, 0x63, 0x6b, 0x62, 0x6c, 0x61, 0x7a, 0x65, 0x5f, 0x61, 0x70, |
|||
0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x15, 0x20, |
|||
0x01, 0x28, 0x09, 0x52, 0x17, 0x62, 0x61, 0x63, 0x6b, 0x62, 0x6c, 0x61, 0x7a, 0x65, 0x41, 0x70, |
|||
0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x12, 0x2d, 0x0a, 0x12, |
|||
0x62, 0x61, 0x63, 0x6b, 0x62, 0x6c, 0x61, 0x7a, 0x65, 0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, |
|||
0x6e, 0x74, 0x18, 0x16, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x62, 0x61, 0x63, 0x6b, 0x62, 0x6c, |
|||
0x61, 0x7a, 0x65, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x2a, 0x0a, 0x11, 0x61, |
|||
0x6c, 0x69, 0x79, 0x75, 0x6e, 0x5f, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x6b, 0x65, 0x79, |
|||
0x18, 0x19, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x61, 0x6c, 0x69, 0x79, 0x75, 0x6e, 0x41, 0x63, |
|||
0x63, 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79, 0x12, 0x2a, 0x0a, 0x11, 0x61, 0x6c, 0x69, 0x79, 0x75, |
|||
0x6e, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x1a, 0x20, 0x01, |
|||
0x28, 0x09, 0x52, 0x0f, 0x61, 0x6c, 0x69, 0x79, 0x75, 0x6e, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, |
|||
0x4b, 0x65, 0x79, 0x12, 0x27, 0x0a, 0x0f, 0x61, 0x6c, 0x69, 0x79, 0x75, 0x6e, 0x5f, 0x65, 0x6e, |
|||
0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x1b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x61, 0x6c, |
|||
0x69, 0x79, 0x75, 0x6e, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x23, 0x0a, 0x0d, |
|||
0x61, 0x6c, 0x69, 0x79, 0x75, 0x6e, 0x5f, 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x18, 0x1c, 0x20, |
|||
0x01, 0x28, 0x09, 0x52, 0x0c, 0x61, 0x6c, 0x69, 0x79, 0x75, 0x6e, 0x52, 0x65, 0x67, 0x69, 0x6f, |
|||
0x6e, 0x12, 0x2a, 0x0a, 0x11, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x6e, 0x74, 0x5f, 0x73, 0x65, 0x63, |
|||
0x72, 0x65, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x1e, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x74, 0x65, |
|||
0x6e, 0x63, 0x65, 0x6e, 0x74, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x49, 0x64, 0x12, 0x2c, 0x0a, |
|||
0x12, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x6e, 0x74, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x5f, |
|||
0x6b, 0x65, 0x79, 0x18, 0x1f, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x74, 0x65, 0x6e, 0x63, 0x65, |
|||
0x6e, 0x74, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x29, 0x0a, 0x10, 0x74, |
|||
0x65, 0x6e, 0x63, 0x65, 0x6e, 0x74, 0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, |
|||
0x20, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x6e, 0x74, 0x45, 0x6e, |
|||
0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x28, 0x0a, 0x10, 0x62, 0x61, 0x69, 0x64, 0x75, 0x5f, |
|||
0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x23, 0x20, 0x01, 0x28, 0x09, |
|||
0x52, 0x0e, 0x62, 0x61, 0x69, 0x64, 0x75, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79, |
|||
0x12, 0x28, 0x0a, 0x10, 0x62, 0x61, 0x69, 0x64, 0x75, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, |
|||
0x5f, 0x6b, 0x65, 0x79, 0x18, 0x24, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x62, 0x61, 0x69, 0x64, |
|||
0x75, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x25, 0x0a, 0x0e, 0x62, 0x61, |
|||
0x69, 0x64, 0x75, 0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x25, 0x20, 0x01, |
|||
0x28, 0x09, 0x52, 0x0d, 0x62, 0x61, 0x69, 0x64, 0x75, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, |
|||
0x74, 0x12, 0x21, 0x0a, 0x0c, 0x62, 0x61, 0x69, 0x64, 0x75, 0x5f, 0x72, 0x65, 0x67, 0x69, 0x6f, |
|||
0x6e, 0x18, 0x26, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x62, 0x61, 0x69, 0x64, 0x75, 0x52, 0x65, |
|||
0x67, 0x69, 0x6f, 0x6e, 0x12, 0x2a, 0x0a, 0x11, 0x77, 0x61, 0x73, 0x61, 0x62, 0x69, 0x5f, 0x61, |
|||
0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x28, 0x20, 0x01, 0x28, 0x09, 0x52, |
|||
0x0f, 0x77, 0x61, 0x73, 0x61, 0x62, 0x69, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79, |
|||
0x12, 0x2a, 0x0a, 0x11, 0x77, 0x61, 0x73, 0x61, 0x62, 0x69, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, |
|||
0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x29, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x77, 0x61, 0x73, |
|||
0x61, 0x62, 0x69, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x27, 0x0a, 0x0f, |
|||
0x77, 0x61, 0x73, 0x61, 0x62, 0x69, 0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, |
|||
0x2a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x77, 0x61, 0x73, 0x61, 0x62, 0x69, 0x45, 0x6e, 0x64, |
|||
0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x23, 0x0a, 0x0d, 0x77, 0x61, 0x73, 0x61, 0x62, 0x69, 0x5f, |
|||
0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x18, 0x2b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x77, 0x61, |
|||
0x73, 0x61, 0x62, 0x69, 0x52, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x12, 0x25, 0x0a, 0x0e, 0x68, 0x64, |
|||
0x66, 0x73, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x6e, 0x6f, 0x64, 0x65, 0x73, 0x18, 0x32, 0x20, 0x03, |
|||
0x28, 0x09, 0x52, 0x0d, 0x68, 0x64, 0x66, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x6e, 0x6f, 0x64, 0x65, |
|||
0x73, 0x12, 0x23, 0x0a, 0x0d, 0x68, 0x64, 0x66, 0x73, 0x5f, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, |
|||
0x6d, 0x65, 0x18, 0x33, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x68, 0x64, 0x66, 0x73, 0x55, 0x73, |
|||
0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x3d, 0x0a, 0x1b, 0x68, 0x64, 0x66, 0x73, 0x5f, 0x73, |
|||
0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x70, 0x72, 0x69, 0x6e, 0x63, 0x69, 0x70, 0x61, 0x6c, |
|||
0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x34, 0x20, 0x01, 0x28, 0x09, 0x52, 0x18, 0x68, 0x64, 0x66, |
|||
0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x50, 0x72, 0x69, 0x6e, 0x63, 0x69, 0x70, 0x61, |
|||
0x6c, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x41, 0x0a, 0x1d, 0x68, 0x64, 0x66, 0x73, 0x5f, 0x64, 0x61, |
|||
0x74, 0x61, 0x5f, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x5f, 0x70, 0x72, 0x6f, 0x74, |
|||
0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x35, 0x20, 0x01, 0x28, 0x09, 0x52, 0x1a, 0x68, 0x64, |
|||
0x66, 0x73, 0x44, 0x61, 0x74, 0x61, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x50, 0x72, |
|||
0x6f, 0x74, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0xc0, 0x01, 0x0a, 0x14, 0x52, 0x65, 0x6d, |
|||
0x6f, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, |
|||
0x67, 0x12, 0x49, 0x0a, 0x08, 0x6d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x01, 0x20, |
|||
0x03, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x70, 0x62, 0x2e, |
|||
0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4d, 0x61, 0x70, |
|||
0x70, 0x69, 0x6e, 0x67, 0x2e, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x45, 0x6e, 0x74, |
|||
0x72, 0x79, 0x52, 0x08, 0x6d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x1a, 0x5d, 0x0a, 0x0d, |
|||
0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, |
|||
0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, |
|||
0x36, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, |
|||
0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x74, |
|||
0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, |
|||
0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x57, 0x0a, 0x15, 0x52, |
|||
0x65, 0x6d, 0x6f, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61, |
|||
0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, |
|||
0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x75, 0x63, 0x6b, |
|||
0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, |
|||
0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, |
|||
0x70, 0x61, 0x74, 0x68, 0x42, 0x50, 0x0a, 0x10, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, |
|||
0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x42, 0x0a, 0x46, 0x69, 0x6c, 0x65, 0x72, 0x50, |
|||
0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x30, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, |
|||
0x2f, 0x63, 0x68, 0x72, 0x69, 0x73, 0x6c, 0x75, 0x73, 0x66, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, |
|||
0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x70, 0x62, 0x2f, 0x72, 0x65, 0x6d, |
|||
0x6f, 0x74, 0x65, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, |
|||
} |
|||
|
|||
var ( |
|||
file_remote_proto_rawDescOnce sync.Once |
|||
file_remote_proto_rawDescData = file_remote_proto_rawDesc |
|||
) |
|||
|
|||
func file_remote_proto_rawDescGZIP() []byte { |
|||
file_remote_proto_rawDescOnce.Do(func() { |
|||
file_remote_proto_rawDescData = protoimpl.X.CompressGZIP(file_remote_proto_rawDescData) |
|||
}) |
|||
return file_remote_proto_rawDescData |
|||
} |
|||
|
|||
var file_remote_proto_msgTypes = make([]protoimpl.MessageInfo, 4) |
|||
var file_remote_proto_goTypes = []interface{}{ |
|||
(*RemoteConf)(nil), // 0: remote_pb.RemoteConf
|
|||
(*RemoteStorageMapping)(nil), // 1: remote_pb.RemoteStorageMapping
|
|||
(*RemoteStorageLocation)(nil), // 2: remote_pb.RemoteStorageLocation
|
|||
nil, // 3: remote_pb.RemoteStorageMapping.MappingsEntry
|
|||
} |
|||
var file_remote_proto_depIdxs = []int32{ |
|||
3, // 0: remote_pb.RemoteStorageMapping.mappings:type_name -> remote_pb.RemoteStorageMapping.MappingsEntry
|
|||
2, // 1: remote_pb.RemoteStorageMapping.MappingsEntry.value:type_name -> remote_pb.RemoteStorageLocation
|
|||
2, // [2:2] is the sub-list for method output_type
|
|||
2, // [2:2] is the sub-list for method input_type
|
|||
2, // [2:2] is the sub-list for extension type_name
|
|||
2, // [2:2] is the sub-list for extension extendee
|
|||
0, // [0:2] is the sub-list for field type_name
|
|||
} |
|||
|
|||
func init() { file_remote_proto_init() } |
|||
func file_remote_proto_init() { |
|||
if File_remote_proto != nil { |
|||
return |
|||
} |
|||
if !protoimpl.UnsafeEnabled { |
|||
file_remote_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { |
|||
switch v := v.(*RemoteConf); i { |
|||
case 0: |
|||
return &v.state |
|||
case 1: |
|||
return &v.sizeCache |
|||
case 2: |
|||
return &v.unknownFields |
|||
default: |
|||
return nil |
|||
} |
|||
} |
|||
file_remote_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { |
|||
switch v := v.(*RemoteStorageMapping); i { |
|||
case 0: |
|||
return &v.state |
|||
case 1: |
|||
return &v.sizeCache |
|||
case 2: |
|||
return &v.unknownFields |
|||
default: |
|||
return nil |
|||
} |
|||
} |
|||
file_remote_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { |
|||
switch v := v.(*RemoteStorageLocation); i { |
|||
case 0: |
|||
return &v.state |
|||
case 1: |
|||
return &v.sizeCache |
|||
case 2: |
|||
return &v.unknownFields |
|||
default: |
|||
return nil |
|||
} |
|||
} |
|||
} |
|||
type x struct{} |
|||
out := protoimpl.TypeBuilder{ |
|||
File: protoimpl.DescBuilder{ |
|||
GoPackagePath: reflect.TypeOf(x{}).PkgPath(), |
|||
RawDescriptor: file_remote_proto_rawDesc, |
|||
NumEnums: 0, |
|||
NumMessages: 4, |
|||
NumExtensions: 0, |
|||
NumServices: 0, |
|||
}, |
|||
GoTypes: file_remote_proto_goTypes, |
|||
DependencyIndexes: file_remote_proto_depIdxs, |
|||
MessageInfos: file_remote_proto_msgTypes, |
|||
}.Build() |
|||
File_remote_proto = out.File |
|||
file_remote_proto_rawDesc = nil |
|||
file_remote_proto_goTypes = nil |
|||
file_remote_proto_depIdxs = nil |
|||
} |
|||
@ -0,0 +1,8 @@ |
|||
package remote_pb |
|||
|
|||
import "github.com/golang/protobuf/proto" |
|||
|
|||
func (fp *RemoteStorageLocation) Key() interface{} { |
|||
key, _ := proto.Marshal(fp) |
|||
return string(key) |
|||
} |
|||
1899
weed/pb/volume_server_pb/volume_server.pb.go
File diff suppressed because it is too large
View File
File diff suppressed because it is too large
View File
@ -0,0 +1,55 @@ |
|||
package hdfs |
|||
|
|||
import ( |
|||
"fmt" |
|||
"os" |
|||
"os/user" |
|||
"strings" |
|||
|
|||
krb "github.com/jcmturner/gokrb5/v8/client" |
|||
"github.com/jcmturner/gokrb5/v8/config" |
|||
"github.com/jcmturner/gokrb5/v8/credentials" |
|||
) |
|||
|
|||
// copy-paste from https://github.com/colinmarc/hdfs/blob/master/cmd/hdfs/kerberos.go
|
|||
func getKerberosClient() (*krb.Client, error) { |
|||
configPath := os.Getenv("KRB5_CONFIG") |
|||
if configPath == "" { |
|||
configPath = "/etc/krb5.conf" |
|||
} |
|||
|
|||
cfg, err := config.Load(configPath) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
|
|||
// Determine the ccache location from the environment, falling back to the
|
|||
// default location.
|
|||
ccachePath := os.Getenv("KRB5CCNAME") |
|||
if strings.Contains(ccachePath, ":") { |
|||
if strings.HasPrefix(ccachePath, "FILE:") { |
|||
ccachePath = strings.SplitN(ccachePath, ":", 2)[1] |
|||
} else { |
|||
return nil, fmt.Errorf("unusable ccache: %s", ccachePath) |
|||
} |
|||
} else if ccachePath == "" { |
|||
u, err := user.Current() |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
|
|||
ccachePath = fmt.Sprintf("/tmp/krb5cc_%s", u.Uid) |
|||
} |
|||
|
|||
ccache, err := credentials.LoadCCache(ccachePath) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
|
|||
client, err := krb.NewFromCCache(ccache, cfg) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
|
|||
return client, nil |
|||
} |
|||
@ -0,0 +1,178 @@ |
|||
package hdfs |
|||
|
|||
import ( |
|||
"fmt" |
|||
"github.com/chrislusf/seaweedfs/weed/glog" |
|||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" |
|||
"github.com/chrislusf/seaweedfs/weed/pb/remote_pb" |
|||
"github.com/chrislusf/seaweedfs/weed/remote_storage" |
|||
"github.com/chrislusf/seaweedfs/weed/util" |
|||
"github.com/colinmarc/hdfs/v2" |
|||
"io" |
|||
"os" |
|||
"path" |
|||
) |
|||
|
|||
func init() { |
|||
remote_storage.RemoteStorageClientMakers["hdfs"] = new(hdfsRemoteStorageMaker) |
|||
} |
|||
|
|||
type hdfsRemoteStorageMaker struct{} |
|||
|
|||
func (s hdfsRemoteStorageMaker) HasBucket() bool { |
|||
return false |
|||
} |
|||
|
|||
func (s hdfsRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) { |
|||
client := &hdfsRemoteStorageClient{ |
|||
conf: conf, |
|||
} |
|||
|
|||
options := hdfs.ClientOptions{ |
|||
Addresses: conf.HdfsNamenodes, |
|||
UseDatanodeHostname: false, |
|||
} |
|||
|
|||
if conf.HdfsServicePrincipalName != "" { |
|||
var err error |
|||
options.KerberosClient, err = getKerberosClient() |
|||
if err != nil { |
|||
return nil, fmt.Errorf("get kerberos authentication: %s", err) |
|||
} |
|||
options.KerberosServicePrincipleName = conf.HdfsServicePrincipalName |
|||
|
|||
if conf.HdfsDataTransferProtection != "" { |
|||
options.DataTransferProtection = conf.HdfsDataTransferProtection |
|||
} |
|||
} else { |
|||
options.User = conf.HdfsUsername |
|||
} |
|||
|
|||
c, err := hdfs.NewClient(options) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
|
|||
client.client = c |
|||
return client, nil |
|||
} |
|||
|
|||
type hdfsRemoteStorageClient struct { |
|||
conf *remote_pb.RemoteConf |
|||
client *hdfs.Client |
|||
} |
|||
|
|||
var _ = remote_storage.RemoteStorageClient(&hdfsRemoteStorageClient{}) |
|||
|
|||
func (c *hdfsRemoteStorageClient) Traverse(loc *remote_pb.RemoteStorageLocation, visitFn remote_storage.VisitFunc) (err error) { |
|||
|
|||
return remote_storage.TraverseBfs(func(parentDir util.FullPath, visitFn remote_storage.VisitFunc) error { |
|||
children, err := c.client.ReadDir(string(parentDir)) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
for _, child := range children { |
|||
if err := visitFn(string(parentDir), child.Name(), child.IsDir(), &filer_pb.RemoteEntry{ |
|||
StorageName: c.conf.Name, |
|||
LastLocalSyncTsNs: 0, |
|||
RemoteETag: "", |
|||
RemoteMtime: child.ModTime().Unix(), |
|||
RemoteSize: child.Size(), |
|||
}); err != nil { |
|||
return nil |
|||
} |
|||
} |
|||
return nil |
|||
}, util.FullPath(loc.Path), visitFn) |
|||
|
|||
} |
|||
func (c *hdfsRemoteStorageClient) ReadFile(loc *remote_pb.RemoteStorageLocation, offset int64, size int64) (data []byte, err error) { |
|||
|
|||
f, err := c.client.Open(loc.Path) |
|||
if err != nil { |
|||
return |
|||
} |
|||
defer f.Close() |
|||
data = make([]byte, size) |
|||
_, err = f.ReadAt(data, offset) |
|||
|
|||
return |
|||
|
|||
} |
|||
|
|||
func (c *hdfsRemoteStorageClient) WriteDirectory(loc *remote_pb.RemoteStorageLocation, entry *filer_pb.Entry) (err error) { |
|||
return c.client.MkdirAll(loc.Path, os.FileMode(entry.Attributes.FileMode)) |
|||
} |
|||
|
|||
func (c *hdfsRemoteStorageClient) RemoveDirectory(loc *remote_pb.RemoteStorageLocation) (err error) { |
|||
return c.client.RemoveAll(loc.Path) |
|||
} |
|||
|
|||
func (c *hdfsRemoteStorageClient) WriteFile(loc *remote_pb.RemoteStorageLocation, entry *filer_pb.Entry, reader io.Reader) (remoteEntry *filer_pb.RemoteEntry, err error) { |
|||
|
|||
dirname := path.Dir(loc.Path) |
|||
|
|||
// ensure parent directory
|
|||
if err = c.client.MkdirAll(dirname, 0755); err != nil { |
|||
return |
|||
} |
|||
|
|||
// remove existing file
|
|||
info, err := c.client.Stat(loc.Path) |
|||
if err == nil { |
|||
err = c.client.Remove(loc.Path) |
|||
if err != nil { |
|||
return |
|||
} |
|||
} |
|||
|
|||
// create new file
|
|||
out, err := c.client.Create(loc.Path) |
|||
if err != nil { |
|||
return |
|||
} |
|||
|
|||
cleanup := func() { |
|||
if removeErr := c.client.Remove(loc.Path); removeErr != nil { |
|||
glog.Errorf("clean up %s%s: %v", loc.Name, loc.Path, removeErr) |
|||
} |
|||
} |
|||
|
|||
if _, err = io.Copy(out, reader); err != nil { |
|||
cleanup() |
|||
return |
|||
} |
|||
|
|||
if err = out.Close(); err != nil { |
|||
cleanup() |
|||
return |
|||
} |
|||
|
|||
info, err = c.client.Stat(loc.Path) |
|||
if err != nil { |
|||
return |
|||
} |
|||
|
|||
return &filer_pb.RemoteEntry{ |
|||
RemoteMtime: info.ModTime().Unix(), |
|||
RemoteSize: info.Size(), |
|||
RemoteETag: "", |
|||
StorageName: c.conf.Name, |
|||
}, nil |
|||
|
|||
} |
|||
|
|||
func (c *hdfsRemoteStorageClient) UpdateFileMetadata(loc *remote_pb.RemoteStorageLocation, oldEntry *filer_pb.Entry, newEntry *filer_pb.Entry) error { |
|||
if oldEntry.Attributes.FileMode != newEntry.Attributes.FileMode { |
|||
if err := c.client.Chmod(loc.Path, os.FileMode(newEntry.Attributes.FileMode)); err != nil { |
|||
return err |
|||
} |
|||
} |
|||
return nil |
|||
} |
|||
func (c *hdfsRemoteStorageClient) DeleteFile(loc *remote_pb.RemoteStorageLocation) (err error) { |
|||
if err = c.client.Remove(loc.Path); err != nil { |
|||
return fmt.Errorf("hdfs delete %s: %v", loc.Path, err) |
|||
} |
|||
return |
|||
} |
|||
@ -0,0 +1,61 @@ |
|||
package s3 |
|||
|
|||
import ( |
|||
"fmt" |
|||
"github.com/aws/aws-sdk-go/aws" |
|||
"github.com/aws/aws-sdk-go/aws/credentials" |
|||
"github.com/aws/aws-sdk-go/aws/request" |
|||
"github.com/aws/aws-sdk-go/aws/session" |
|||
"github.com/aws/aws-sdk-go/service/s3" |
|||
"github.com/chrislusf/seaweedfs/weed/pb/remote_pb" |
|||
"github.com/chrislusf/seaweedfs/weed/remote_storage" |
|||
"github.com/chrislusf/seaweedfs/weed/util" |
|||
) |
|||
|
|||
func init() { |
|||
remote_storage.RemoteStorageClientMakers["wasabi"] = new(WasabiRemoteStorageMaker) |
|||
} |
|||
|
|||
type WasabiRemoteStorageMaker struct{} |
|||
|
|||
func (s WasabiRemoteStorageMaker) HasBucket() bool { |
|||
return true |
|||
} |
|||
|
|||
func (s WasabiRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) { |
|||
client := &s3RemoteStorageClient{ |
|||
conf: conf, |
|||
} |
|||
accessKey := util.Nvl(conf.WasabiAccessKey) |
|||
secretKey := util.Nvl(conf.WasabiSecretKey) |
|||
|
|||
config := &aws.Config{ |
|||
Endpoint: aws.String(conf.WasabiEndpoint), |
|||
Region: aws.String(conf.WasabiRegion), |
|||
S3ForcePathStyle: aws.Bool(true), |
|||
S3DisableContentMD5Validation: aws.Bool(true), |
|||
} |
|||
if accessKey != "" && secretKey != "" { |
|||
config.Credentials = credentials.NewStaticCredentials(accessKey, secretKey, "") |
|||
} |
|||
|
|||
sess, err := session.NewSession(config) |
|||
if err != nil { |
|||
return nil, fmt.Errorf("create wasabi session: %v", err) |
|||
} |
|||
sess.Handlers.Build.PushFront(skipSha256PayloadSigning) |
|||
client.conn = s3.New(sess) |
|||
return client, nil |
|||
} |
|||
|
|||
var skipSha256PayloadSigning = func(r *request.Request) { |
|||
// see https://github.com/ceph/ceph/pull/15965/files
|
|||
if r.ClientInfo.ServiceID != "S3" { |
|||
return |
|||
} |
|||
if r.Operation.Name == "PutObject" || r.Operation.Name == "UploadPart" { |
|||
if len(r.HTTPRequest.Header.Get("X-Amz-Content-Sha256")) == 0 { |
|||
r.HTTPRequest.Header.Set("X-Amz-Content-Sha256", "UNSIGNED-PAYLOAD") |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,62 @@ |
|||
package remote_storage |
|||
|
|||
import ( |
|||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" |
|||
"github.com/chrislusf/seaweedfs/weed/util" |
|||
"sync" |
|||
"time" |
|||
) |
|||
|
|||
type ListDirectoryFunc func(parentDir util.FullPath, visitFn VisitFunc) error |
|||
|
|||
func TraverseBfs(listDirFn ListDirectoryFunc, parentPath util.FullPath, visitFn VisitFunc) (err error) { |
|||
K := 5 |
|||
|
|||
var dirQueueWg sync.WaitGroup |
|||
dirQueue := util.NewQueue() |
|||
dirQueueWg.Add(1) |
|||
dirQueue.Enqueue(parentPath) |
|||
var isTerminating bool |
|||
|
|||
for i := 0; i < K; i++ { |
|||
go func() { |
|||
for { |
|||
if isTerminating { |
|||
break |
|||
} |
|||
t := dirQueue.Dequeue() |
|||
if t == nil { |
|||
time.Sleep(329 * time.Millisecond) |
|||
continue |
|||
} |
|||
dir := t.(util.FullPath) |
|||
processErr := processOneDirectory(listDirFn, dir, visitFn, dirQueue, &dirQueueWg) |
|||
if processErr != nil { |
|||
err = processErr |
|||
} |
|||
dirQueueWg.Done() |
|||
} |
|||
}() |
|||
} |
|||
|
|||
dirQueueWg.Wait() |
|||
isTerminating = true |
|||
return |
|||
|
|||
} |
|||
|
|||
func processOneDirectory(listDirFn ListDirectoryFunc, parentPath util.FullPath, visitFn VisitFunc, dirQueue *util.Queue, dirQueueWg *sync.WaitGroup) (error) { |
|||
|
|||
return listDirFn(parentPath, func(dir string, name string, isDirectory bool, remoteEntry *filer_pb.RemoteEntry) error { |
|||
if err := visitFn(dir, name, isDirectory, remoteEntry); err != nil { |
|||
return err |
|||
} |
|||
if !isDirectory { |
|||
return nil |
|||
} |
|||
dirQueueWg.Add(1) |
|||
dirQueue.Enqueue(parentPath.Child(name)) |
|||
return nil |
|||
}) |
|||
|
|||
} |
|||
@ -1,4 +1,4 @@ |
|||
package pb |
|||
package volume_info |
|||
|
|||
import ( |
|||
"bytes" |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue