From 001a472057f01b3ac2d3edb59b3d5fb0a141cddd Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 29 Aug 2021 18:41:29 -0700 Subject: [PATCH] cloud mount: remote storage support hdfs --- go.mod | 11 +- go.sum | 22 ++- weed/pb/remote.proto | 6 + weed/pb/remote_pb/remote.pb.go | 153 +++++++++------ .../azure/azure_storage_client.go | 4 + weed/remote_storage/gcs/gcs_storage_client.go | 4 + weed/remote_storage/hdfs/hdfs_kerberos.go | 55 ++++++ .../hdfs/hdfs_storage_client.go | 174 ++++++++++++++++++ weed/remote_storage/hdfs/traverse_bfs.go | 63 +++++++ weed/remote_storage/remote_storage.go | 42 ++++- weed/remote_storage/s3/aliyun.go | 4 + weed/remote_storage/s3/backblaze.go | 4 + weed/remote_storage/s3/baidu.go | 4 + weed/remote_storage/s3/s3_storage_client.go | 4 + weed/remote_storage/s3/tencent.go | 4 + weed/remote_storage/s3/wasabi.go | 4 + weed/shell/command_remote_configure.go | 30 ++- weed/shell/command_remote_meta_sync.go | 6 +- weed/shell/command_remote_mount.go | 10 +- 19 files changed, 539 insertions(+), 65 deletions(-) create mode 100644 weed/remote_storage/hdfs/hdfs_kerberos.go create mode 100644 weed/remote_storage/hdfs/hdfs_storage_client.go create mode 100644 weed/remote_storage/hdfs/traverse_bfs.go diff --git a/go.mod b/go.mod index c57af848d..daaec16b1 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( github.com/cespare/xxhash v1.1.0 github.com/cespare/xxhash/v2 v2.1.1 // indirect github.com/chrislusf/raft v1.0.7 + github.com/colinmarc/hdfs/v2 v2.2.0 github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd/v22 v22.0.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect @@ -60,7 +61,7 @@ require ( github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-multierror v1.0.0 // indirect - github.com/hashicorp/go-uuid v1.0.1 // indirect + github.com/hashicorp/go-uuid v1.0.2 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/jcmturner/gofork v1.0.0 // indirect github.com/jinzhu/copier v0.2.8 @@ -177,6 +178,14 @@ require ( modernc.org/token v1.0.0 // indirect ) +require ( + github.com/jcmturner/aescts/v2 v2.0.0 // indirect + github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect + github.com/jcmturner/goidentity/v6 v6.0.1 // indirect + github.com/jcmturner/gokrb5/v8 v8.4.1 // indirect + github.com/jcmturner/rpc/v2 v2.0.2 // indirect +) + // replace github.com/seaweedfs/fuse => /Users/chris/go/src/github.com/seaweedfs/fuse // replace github.com/chrislusf/raft => /Users/chris/go/src/github.com/chrislusf/raft diff --git a/go.sum b/go.sum index e18da414c..49994f764 100644 --- a/go.sum +++ b/go.sum @@ -154,6 +154,8 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= +github.com/colinmarc/hdfs/v2 v2.2.0 h1:4AaIlTq+/sWmeqYhI0dX8bD4YrMQM990tRjm636FkGM= +github.com/colinmarc/hdfs/v2 v2.2.0/go.mod h1:Wss6n3mtaZyRwWaqtSH+6ge01qT0rw9dJJmvoUnIQ/E= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -292,6 +294,7 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3 h1:GV+pQPG/EUUbkh47niozDcADz6go/dUwhVzdUQHIVRw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -360,6 +363,10 @@ github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2z github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc= github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ= +github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= +github.com/gorilla/sessions v1.2.0 h1:S7P+1Hm5V/AT9cjEcUD5uDaQSX0OE577aCXgoaKpYbQ= +github.com/gorilla/sessions v1.2.0/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM= @@ -388,8 +395,9 @@ github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE= +github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -405,9 +413,19 @@ github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmK github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= +github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= +github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= +github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= +github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM= github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8= github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= +github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o= +github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg= +github.com/jcmturner/gokrb5/v8 v8.4.1 h1:IGSJfqBzMS6TA0oJ7DxXdyzPK563QHa8T2IqER2ggyQ= +github.com/jcmturner/gokrb5/v8 v8.4.1/go.mod h1:T1hnNppQsBtxW0tCHMHTkAt8n/sABdzZgZdoFrZaZNM= +github.com/jcmturner/rpc/v2 v2.0.2 h1:gMB4IwRXYsWw4Bc6o/az2HJgFUA1ffSh90i26ZJ6Xl0= +github.com/jcmturner/rpc/v2 v2.0.2/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= github.com/jinzhu/copier v0.2.8 h1:N8MbL5niMwE3P4dOwurJixz5rMkKfujmMRFmAanSzWE= github.com/jinzhu/copier v0.2.8/go.mod h1:24xnZezI2Yqac9J61UC6/dG/k76ttpq0DdJI3QmUvro= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -564,6 +582,7 @@ github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnh github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pborman/getopt v0.0.0-20180729010549-6fdd0a2c7117/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.7.0 h1:7utD74fnzVc/cpcyy8sjrlFr5vYpypUixARcHIMIGuI= @@ -792,6 +811,7 @@ golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200117160349-530e935923ad/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= diff --git a/weed/pb/remote.proto b/weed/pb/remote.proto index 20f4f9934..1f1c37343 100644 --- a/weed/pb/remote.proto +++ b/weed/pb/remote.proto @@ -46,6 +46,12 @@ message RemoteConf { string wasabi_secret_key = 41; string wasabi_endpoint = 42; string wasabi_region = 43; + + repeated string hdfs_namenodes = 50; + string hdfs_username = 51; + string hdfs_service_principal_name = 52; + string hdfs_data_transfer_protection = 53; + } message RemoteStorageMapping { diff --git a/weed/pb/remote_pb/remote.pb.go b/weed/pb/remote_pb/remote.pb.go index 99e9db026..cc513fe26 100644 --- a/weed/pb/remote_pb/remote.pb.go +++ b/weed/pb/remote_pb/remote.pb.go @@ -33,35 +33,39 @@ type RemoteConf struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Type string `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"` - Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` - S3AccessKey string `protobuf:"bytes,4,opt,name=s3_access_key,json=s3AccessKey,proto3" json:"s3_access_key,omitempty"` - S3SecretKey string `protobuf:"bytes,5,opt,name=s3_secret_key,json=s3SecretKey,proto3" json:"s3_secret_key,omitempty"` - S3Region string `protobuf:"bytes,6,opt,name=s3_region,json=s3Region,proto3" json:"s3_region,omitempty"` - S3Endpoint string `protobuf:"bytes,7,opt,name=s3_endpoint,json=s3Endpoint,proto3" json:"s3_endpoint,omitempty"` - S3StorageClass string `protobuf:"bytes,8,opt,name=s3_storage_class,json=s3StorageClass,proto3" json:"s3_storage_class,omitempty"` - S3ForcePathStyle bool `protobuf:"varint,9,opt,name=s3_force_path_style,json=s3ForcePathStyle,proto3" json:"s3_force_path_style,omitempty"` - GcsGoogleApplicationCredentials string `protobuf:"bytes,10,opt,name=gcs_google_application_credentials,json=gcsGoogleApplicationCredentials,proto3" json:"gcs_google_application_credentials,omitempty"` - AzureAccountName string `protobuf:"bytes,15,opt,name=azure_account_name,json=azureAccountName,proto3" json:"azure_account_name,omitempty"` - AzureAccountKey string `protobuf:"bytes,16,opt,name=azure_account_key,json=azureAccountKey,proto3" json:"azure_account_key,omitempty"` - BackblazeKeyId string `protobuf:"bytes,20,opt,name=backblaze_key_id,json=backblazeKeyId,proto3" json:"backblaze_key_id,omitempty"` - BackblazeApplicationKey string `protobuf:"bytes,21,opt,name=backblaze_application_key,json=backblazeApplicationKey,proto3" json:"backblaze_application_key,omitempty"` - BackblazeEndpoint string `protobuf:"bytes,22,opt,name=backblaze_endpoint,json=backblazeEndpoint,proto3" json:"backblaze_endpoint,omitempty"` - AliyunAccessKey string `protobuf:"bytes,25,opt,name=aliyun_access_key,json=aliyunAccessKey,proto3" json:"aliyun_access_key,omitempty"` - AliyunSecretKey string `protobuf:"bytes,26,opt,name=aliyun_secret_key,json=aliyunSecretKey,proto3" json:"aliyun_secret_key,omitempty"` - AliyunEndpoint string `protobuf:"bytes,27,opt,name=aliyun_endpoint,json=aliyunEndpoint,proto3" json:"aliyun_endpoint,omitempty"` - AliyunRegion string `protobuf:"bytes,28,opt,name=aliyun_region,json=aliyunRegion,proto3" json:"aliyun_region,omitempty"` - TencentSecretId string `protobuf:"bytes,30,opt,name=tencent_secret_id,json=tencentSecretId,proto3" json:"tencent_secret_id,omitempty"` - TencentSecretKey string `protobuf:"bytes,31,opt,name=tencent_secret_key,json=tencentSecretKey,proto3" json:"tencent_secret_key,omitempty"` - TencentEndpoint string `protobuf:"bytes,32,opt,name=tencent_endpoint,json=tencentEndpoint,proto3" json:"tencent_endpoint,omitempty"` - BaiduAccessKey string `protobuf:"bytes,35,opt,name=baidu_access_key,json=baiduAccessKey,proto3" json:"baidu_access_key,omitempty"` - BaiduSecretKey string `protobuf:"bytes,36,opt,name=baidu_secret_key,json=baiduSecretKey,proto3" json:"baidu_secret_key,omitempty"` - BaiduEndpoint string `protobuf:"bytes,37,opt,name=baidu_endpoint,json=baiduEndpoint,proto3" json:"baidu_endpoint,omitempty"` - BaiduRegion string `protobuf:"bytes,38,opt,name=baidu_region,json=baiduRegion,proto3" json:"baidu_region,omitempty"` - WasabiAccessKey string `protobuf:"bytes,40,opt,name=wasabi_access_key,json=wasabiAccessKey,proto3" json:"wasabi_access_key,omitempty"` - WasabiSecretKey string `protobuf:"bytes,41,opt,name=wasabi_secret_key,json=wasabiSecretKey,proto3" json:"wasabi_secret_key,omitempty"` - WasabiEndpoint string `protobuf:"bytes,42,opt,name=wasabi_endpoint,json=wasabiEndpoint,proto3" json:"wasabi_endpoint,omitempty"` - WasabiRegion string `protobuf:"bytes,43,opt,name=wasabi_region,json=wasabiRegion,proto3" json:"wasabi_region,omitempty"` + Type string `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"` + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` + S3AccessKey string `protobuf:"bytes,4,opt,name=s3_access_key,json=s3AccessKey,proto3" json:"s3_access_key,omitempty"` + S3SecretKey string `protobuf:"bytes,5,opt,name=s3_secret_key,json=s3SecretKey,proto3" json:"s3_secret_key,omitempty"` + S3Region string `protobuf:"bytes,6,opt,name=s3_region,json=s3Region,proto3" json:"s3_region,omitempty"` + S3Endpoint string `protobuf:"bytes,7,opt,name=s3_endpoint,json=s3Endpoint,proto3" json:"s3_endpoint,omitempty"` + S3StorageClass string `protobuf:"bytes,8,opt,name=s3_storage_class,json=s3StorageClass,proto3" json:"s3_storage_class,omitempty"` + S3ForcePathStyle bool `protobuf:"varint,9,opt,name=s3_force_path_style,json=s3ForcePathStyle,proto3" json:"s3_force_path_style,omitempty"` + GcsGoogleApplicationCredentials string `protobuf:"bytes,10,opt,name=gcs_google_application_credentials,json=gcsGoogleApplicationCredentials,proto3" json:"gcs_google_application_credentials,omitempty"` + AzureAccountName string `protobuf:"bytes,15,opt,name=azure_account_name,json=azureAccountName,proto3" json:"azure_account_name,omitempty"` + AzureAccountKey string `protobuf:"bytes,16,opt,name=azure_account_key,json=azureAccountKey,proto3" json:"azure_account_key,omitempty"` + BackblazeKeyId string `protobuf:"bytes,20,opt,name=backblaze_key_id,json=backblazeKeyId,proto3" json:"backblaze_key_id,omitempty"` + BackblazeApplicationKey string `protobuf:"bytes,21,opt,name=backblaze_application_key,json=backblazeApplicationKey,proto3" json:"backblaze_application_key,omitempty"` + BackblazeEndpoint string `protobuf:"bytes,22,opt,name=backblaze_endpoint,json=backblazeEndpoint,proto3" json:"backblaze_endpoint,omitempty"` + AliyunAccessKey string `protobuf:"bytes,25,opt,name=aliyun_access_key,json=aliyunAccessKey,proto3" json:"aliyun_access_key,omitempty"` + AliyunSecretKey string `protobuf:"bytes,26,opt,name=aliyun_secret_key,json=aliyunSecretKey,proto3" json:"aliyun_secret_key,omitempty"` + AliyunEndpoint string `protobuf:"bytes,27,opt,name=aliyun_endpoint,json=aliyunEndpoint,proto3" json:"aliyun_endpoint,omitempty"` + AliyunRegion string `protobuf:"bytes,28,opt,name=aliyun_region,json=aliyunRegion,proto3" json:"aliyun_region,omitempty"` + TencentSecretId string `protobuf:"bytes,30,opt,name=tencent_secret_id,json=tencentSecretId,proto3" json:"tencent_secret_id,omitempty"` + TencentSecretKey string `protobuf:"bytes,31,opt,name=tencent_secret_key,json=tencentSecretKey,proto3" json:"tencent_secret_key,omitempty"` + TencentEndpoint string `protobuf:"bytes,32,opt,name=tencent_endpoint,json=tencentEndpoint,proto3" json:"tencent_endpoint,omitempty"` + BaiduAccessKey string `protobuf:"bytes,35,opt,name=baidu_access_key,json=baiduAccessKey,proto3" json:"baidu_access_key,omitempty"` + BaiduSecretKey string `protobuf:"bytes,36,opt,name=baidu_secret_key,json=baiduSecretKey,proto3" json:"baidu_secret_key,omitempty"` + BaiduEndpoint string `protobuf:"bytes,37,opt,name=baidu_endpoint,json=baiduEndpoint,proto3" json:"baidu_endpoint,omitempty"` + BaiduRegion string `protobuf:"bytes,38,opt,name=baidu_region,json=baiduRegion,proto3" json:"baidu_region,omitempty"` + WasabiAccessKey string `protobuf:"bytes,40,opt,name=wasabi_access_key,json=wasabiAccessKey,proto3" json:"wasabi_access_key,omitempty"` + WasabiSecretKey string `protobuf:"bytes,41,opt,name=wasabi_secret_key,json=wasabiSecretKey,proto3" json:"wasabi_secret_key,omitempty"` + WasabiEndpoint string `protobuf:"bytes,42,opt,name=wasabi_endpoint,json=wasabiEndpoint,proto3" json:"wasabi_endpoint,omitempty"` + WasabiRegion string `protobuf:"bytes,43,opt,name=wasabi_region,json=wasabiRegion,proto3" json:"wasabi_region,omitempty"` + HdfsNamenodes []string `protobuf:"bytes,50,rep,name=hdfs_namenodes,json=hdfsNamenodes,proto3" json:"hdfs_namenodes,omitempty"` + HdfsUsername string `protobuf:"bytes,51,opt,name=hdfs_username,json=hdfsUsername,proto3" json:"hdfs_username,omitempty"` + HdfsServicePrincipalName string `protobuf:"bytes,52,opt,name=hdfs_service_principal_name,json=hdfsServicePrincipalName,proto3" json:"hdfs_service_principal_name,omitempty"` + HdfsDataTransferProtection string `protobuf:"bytes,53,opt,name=hdfs_data_transfer_protection,json=hdfsDataTransferProtection,proto3" json:"hdfs_data_transfer_protection,omitempty"` } func (x *RemoteConf) Reset() { @@ -299,6 +303,34 @@ func (x *RemoteConf) GetWasabiRegion() string { return "" } +func (x *RemoteConf) GetHdfsNamenodes() []string { + if x != nil { + return x.HdfsNamenodes + } + return nil +} + +func (x *RemoteConf) GetHdfsUsername() string { + if x != nil { + return x.HdfsUsername + } + return "" +} + +func (x *RemoteConf) GetHdfsServicePrincipalName() string { + if x != nil { + return x.HdfsServicePrincipalName + } + return "" +} + +func (x *RemoteConf) GetHdfsDataTransferProtection() string { + if x != nil { + return x.HdfsDataTransferProtection + } + return "" +} + type RemoteStorageMapping struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -413,7 +445,7 @@ var File_remote_proto protoreflect.FileDescriptor var file_remote_proto_rawDesc = []byte{ 0x0a, 0x0c, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x09, - 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x70, 0x62, 0x22, 0xbe, 0x09, 0x0a, 0x0a, 0x52, 0x65, + 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x70, 0x62, 0x22, 0x8c, 0x0b, 0x0a, 0x0a, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, @@ -489,30 +521,43 @@ var file_remote_proto_rawDesc = []byte{ 0x2a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x77, 0x61, 0x73, 0x61, 0x62, 0x69, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x23, 0x0a, 0x0d, 0x77, 0x61, 0x73, 0x61, 0x62, 0x69, 0x5f, 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x18, 0x2b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x77, 0x61, - 0x73, 0x61, 0x62, 0x69, 0x52, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x22, 0xc0, 0x01, 0x0a, 0x14, 0x52, - 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4d, 0x61, 0x70, 0x70, - 0x69, 0x6e, 0x67, 0x12, 0x49, 0x0a, 0x08, 0x6d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x18, - 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x70, - 0x62, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4d, - 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x2e, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x45, - 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x6d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x1a, 0x5d, - 0x0a, 0x0d, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, - 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, - 0x79, 0x12, 0x36, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x20, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x6d, - 0x6f, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x57, 0x0a, - 0x15, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4c, 0x6f, - 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x75, - 0x63, 0x6b, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x62, 0x75, 0x63, 0x6b, - 0x65, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x42, 0x50, 0x0a, 0x10, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, - 0x64, 0x66, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x42, 0x0a, 0x46, 0x69, 0x6c, 0x65, - 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x30, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, - 0x6f, 0x6d, 0x2f, 0x63, 0x68, 0x72, 0x69, 0x73, 0x6c, 0x75, 0x73, 0x66, 0x2f, 0x73, 0x65, 0x61, - 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x70, 0x62, 0x2f, 0x72, - 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x73, 0x61, 0x62, 0x69, 0x52, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x12, 0x25, 0x0a, 0x0e, 0x68, 0x64, + 0x66, 0x73, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x6e, 0x6f, 0x64, 0x65, 0x73, 0x18, 0x32, 0x20, 0x03, + 0x28, 0x09, 0x52, 0x0d, 0x68, 0x64, 0x66, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x6e, 0x6f, 0x64, 0x65, + 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x68, 0x64, 0x66, 0x73, 0x5f, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, + 0x6d, 0x65, 0x18, 0x33, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x68, 0x64, 0x66, 0x73, 0x55, 0x73, + 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x3d, 0x0a, 0x1b, 0x68, 0x64, 0x66, 0x73, 0x5f, 0x73, + 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x70, 0x72, 0x69, 0x6e, 0x63, 0x69, 0x70, 0x61, 0x6c, + 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x34, 0x20, 0x01, 0x28, 0x09, 0x52, 0x18, 0x68, 0x64, 0x66, + 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x50, 0x72, 0x69, 0x6e, 0x63, 0x69, 0x70, 0x61, + 0x6c, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x41, 0x0a, 0x1d, 0x68, 0x64, 0x66, 0x73, 0x5f, 0x64, 0x61, + 0x74, 0x61, 0x5f, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x5f, 0x70, 0x72, 0x6f, 0x74, + 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x35, 0x20, 0x01, 0x28, 0x09, 0x52, 0x1a, 0x68, 0x64, + 0x66, 0x73, 0x44, 0x61, 0x74, 0x61, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x50, 0x72, + 0x6f, 0x74, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0xc0, 0x01, 0x0a, 0x14, 0x52, 0x65, 0x6d, + 0x6f, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, + 0x67, 0x12, 0x49, 0x0a, 0x08, 0x6d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x01, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x70, 0x62, 0x2e, + 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4d, 0x61, 0x70, + 0x70, 0x69, 0x6e, 0x67, 0x2e, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x52, 0x08, 0x6d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x1a, 0x5d, 0x0a, 0x0d, + 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, + 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, + 0x36, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, + 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x74, + 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x57, 0x0a, 0x15, 0x52, + 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x75, 0x63, 0x6b, + 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, + 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, + 0x70, 0x61, 0x74, 0x68, 0x42, 0x50, 0x0a, 0x10, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, + 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x42, 0x0a, 0x46, 0x69, 0x6c, 0x65, 0x72, 0x50, + 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x30, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, + 0x2f, 0x63, 0x68, 0x72, 0x69, 0x73, 0x6c, 0x75, 0x73, 0x66, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, + 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x70, 0x62, 0x2f, 0x72, 0x65, 0x6d, + 0x6f, 0x74, 0x65, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/weed/remote_storage/azure/azure_storage_client.go b/weed/remote_storage/azure/azure_storage_client.go index c2e1416a5..21b8606c3 100644 --- a/weed/remote_storage/azure/azure_storage_client.go +++ b/weed/remote_storage/azure/azure_storage_client.go @@ -22,6 +22,10 @@ func init() { type azureRemoteStorageMaker struct{} +func (s azureRemoteStorageMaker) HasBucket() bool { + return true +} + func (s azureRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) { client := &azureRemoteStorageClient{ diff --git a/weed/remote_storage/gcs/gcs_storage_client.go b/weed/remote_storage/gcs/gcs_storage_client.go index 44d41f4fd..828d62978 100644 --- a/weed/remote_storage/gcs/gcs_storage_client.go +++ b/weed/remote_storage/gcs/gcs_storage_client.go @@ -22,6 +22,10 @@ func init() { type gcsRemoteStorageMaker struct{} +func (s gcsRemoteStorageMaker) HasBucket() bool { + return true +} + func (s gcsRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) { client := &gcsRemoteStorageClient{ conf: conf, diff --git a/weed/remote_storage/hdfs/hdfs_kerberos.go b/weed/remote_storage/hdfs/hdfs_kerberos.go new file mode 100644 index 000000000..50abc0ad5 --- /dev/null +++ b/weed/remote_storage/hdfs/hdfs_kerberos.go @@ -0,0 +1,55 @@ +package hdfs + +import ( + "fmt" + "os" + "os/user" + "strings" + + krb "github.com/jcmturner/gokrb5/v8/client" + "github.com/jcmturner/gokrb5/v8/config" + "github.com/jcmturner/gokrb5/v8/credentials" +) + +// copy-paste from https://github.com/colinmarc/hdfs/blob/master/cmd/hdfs/kerberos.go +func getKerberosClient() (*krb.Client, error) { + configPath := os.Getenv("KRB5_CONFIG") + if configPath == "" { + configPath = "/etc/krb5.conf" + } + + cfg, err := config.Load(configPath) + if err != nil { + return nil, err + } + + // Determine the ccache location from the environment, falling back to the + // default location. + ccachePath := os.Getenv("KRB5CCNAME") + if strings.Contains(ccachePath, ":") { + if strings.HasPrefix(ccachePath, "FILE:") { + ccachePath = strings.SplitN(ccachePath, ":", 2)[1] + } else { + return nil, fmt.Errorf("unusable ccache: %s", ccachePath) + } + } else if ccachePath == "" { + u, err := user.Current() + if err != nil { + return nil, err + } + + ccachePath = fmt.Sprintf("/tmp/krb5cc_%s", u.Uid) + } + + ccache, err := credentials.LoadCCache(ccachePath) + if err != nil { + return nil, err + } + + client, err := krb.NewFromCCache(ccache, cfg) + if err != nil { + return nil, err + } + + return client, nil +} diff --git a/weed/remote_storage/hdfs/hdfs_storage_client.go b/weed/remote_storage/hdfs/hdfs_storage_client.go new file mode 100644 index 000000000..0c5f5a45d --- /dev/null +++ b/weed/remote_storage/hdfs/hdfs_storage_client.go @@ -0,0 +1,174 @@ +package hdfs + +import ( + "fmt" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/pb/remote_pb" + "github.com/chrislusf/seaweedfs/weed/remote_storage" + "github.com/chrislusf/seaweedfs/weed/util" + "github.com/colinmarc/hdfs/v2" + "io" + "os" + "path" +) + +func init() { + remote_storage.RemoteStorageClientMakers["hdfs"] = new(hdfsRemoteStorageMaker) +} + +type hdfsRemoteStorageMaker struct{} + +func (s hdfsRemoteStorageMaker) HasBucket() bool { + return false +} + +func (s hdfsRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) { + client := &hdfsRemoteStorageClient{ + conf: conf, + } + + options := hdfs.ClientOptions{ + Addresses: conf.HdfsNamenodes, + UseDatanodeHostname: false, + } + + if conf.HdfsServicePrincipalName != "" { + var err error + options.KerberosClient, err = getKerberosClient() + if err != nil { + return nil, fmt.Errorf("get kerberos authentication: %s", err) + } + options.KerberosServicePrincipleName = conf.HdfsServicePrincipalName + + if conf.HdfsDataTransferProtection != "" { + options.DataTransferProtection = conf.HdfsDataTransferProtection + } + } else { + options.User = conf.HdfsUsername + } + + c, err := hdfs.NewClient(options) + if err != nil { + return nil, err + } + + client.client = c + return client, nil +} + +type hdfsRemoteStorageClient struct { + conf *remote_pb.RemoteConf + client *hdfs.Client +} + +var _ = remote_storage.RemoteStorageClient(&hdfsRemoteStorageClient{}) + +func (c *hdfsRemoteStorageClient) Traverse(loc *remote_pb.RemoteStorageLocation, visitFn remote_storage.VisitFunc) (err error) { + + return TraverseBfs(func(parentDir util.FullPath, visitFn remote_storage.VisitFunc) error { + children, err := c.client.ReadDir(string(parentDir)) + if err != nil { + return err + } + for _, child := range children { + if err := visitFn(string(parentDir), child.Name(), child.IsDir(), &filer_pb.RemoteEntry{ + StorageName: c.conf.Name, + LastLocalSyncTsNs: 0, + RemoteETag: "", + RemoteMtime: child.ModTime().Unix(), + RemoteSize: child.Size(), + }); err != nil { + return nil + } + } + return nil + }, util.FullPath(loc.Path), visitFn) + +} +func (c *hdfsRemoteStorageClient) ReadFile(loc *remote_pb.RemoteStorageLocation, offset int64, size int64) (data []byte, err error) { + + f, err := c.client.Open(loc.Path) + if err != nil { + return + } + defer f.Close() + data = make([]byte, size) + _, err = f.ReadAt(data, offset) + + return + +} + +func (c *hdfsRemoteStorageClient) WriteDirectory(loc *remote_pb.RemoteStorageLocation, entry *filer_pb.Entry) (err error) { + return c.client.MkdirAll(loc.Path, os.FileMode(entry.Attributes.FileMode)) +} + +func (c *hdfsRemoteStorageClient) WriteFile(loc *remote_pb.RemoteStorageLocation, entry *filer_pb.Entry, reader io.Reader) (remoteEntry *filer_pb.RemoteEntry, err error) { + + dirname := path.Dir(loc.Path) + + // ensure parent directory + if err = c.client.MkdirAll(dirname, 0755); err != nil { + return + } + + // remove existing file + info, err := c.client.Stat(loc.Path) + if err == nil { + err = c.client.Remove(loc.Path) + if err != nil { + return + } + } + + // create new file + out, err := c.client.Create(loc.Path) + if err != nil { + return + } + + cleanup := func() { + if removeErr := c.client.Remove(loc.Path); removeErr != nil { + glog.Errorf("clean up %s%s: %v", loc.Name, loc.Path, removeErr) + } + } + + if _, err = io.Copy(out, reader); err != nil { + cleanup() + return + } + + if err = out.Close(); err != nil { + cleanup() + return + } + + info, err = c.client.Stat(loc.Path) + if err != nil { + return + } + + return &filer_pb.RemoteEntry{ + RemoteMtime: info.ModTime().Unix(), + RemoteSize: info.Size(), + RemoteETag: "", + StorageName: c.conf.Name, + }, nil + +} + +func (c *hdfsRemoteStorageClient) UpdateFileMetadata(loc *remote_pb.RemoteStorageLocation, oldEntry *filer_pb.Entry, newEntry *filer_pb.Entry) error { + if oldEntry.Attributes.FileMode != newEntry.Attributes.FileMode { + if err := c.client.Chmod(loc.Path, os.FileMode(newEntry.Attributes.FileMode)); err != nil { + return err + } + } + return nil +} +func (c *hdfsRemoteStorageClient) DeleteFile(loc *remote_pb.RemoteStorageLocation) (err error) { + if err = c.client.Remove(loc.Path); err != nil { + return fmt.Errorf("hdfs delete %s: %v", loc.Path, err) + } + return +} diff --git a/weed/remote_storage/hdfs/traverse_bfs.go b/weed/remote_storage/hdfs/traverse_bfs.go new file mode 100644 index 000000000..755771283 --- /dev/null +++ b/weed/remote_storage/hdfs/traverse_bfs.go @@ -0,0 +1,63 @@ +package hdfs + +import ( + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/remote_storage" + "github.com/chrislusf/seaweedfs/weed/util" + "sync" + "time" +) + +type ListDirectoryFunc func(parentDir util.FullPath, visitFn remote_storage.VisitFunc) error + +func TraverseBfs(listDirFn ListDirectoryFunc, parentPath util.FullPath, visitFn remote_storage.VisitFunc) (err error) { + K := 5 + + var dirQueueWg sync.WaitGroup + dirQueue := util.NewQueue() + dirQueueWg.Add(1) + dirQueue.Enqueue(parentPath) + var isTerminating bool + + for i := 0; i < K; i++ { + go func() { + for { + if isTerminating { + break + } + t := dirQueue.Dequeue() + if t == nil { + time.Sleep(329 * time.Millisecond) + continue + } + dir := t.(util.FullPath) + processErr := processOneDirectory(listDirFn, dir, visitFn, dirQueue, &dirQueueWg) + if processErr != nil { + err = processErr + } + dirQueueWg.Done() + } + }() + } + + dirQueueWg.Wait() + isTerminating = true + return + +} + +func processOneDirectory(listDirFn ListDirectoryFunc, parentPath util.FullPath, visitFn remote_storage.VisitFunc, dirQueue *util.Queue, dirQueueWg *sync.WaitGroup) (error) { + + return listDirFn(parentPath, func(dir string, name string, isDirectory bool, remoteEntry *filer_pb.RemoteEntry) error { + if err := visitFn(dir, name, isDirectory, remoteEntry); err != nil { + return err + } + if !isDirectory { + return nil + } + dirQueueWg.Add(1) + dirQueue.Enqueue(parentPath.Child(name)) + return nil + }) + +} diff --git a/weed/remote_storage/remote_storage.go b/weed/remote_storage/remote_storage.go index c9bef8c9b..a4caef0d4 100644 --- a/weed/remote_storage/remote_storage.go +++ b/weed/remote_storage/remote_storage.go @@ -9,7 +9,18 @@ import ( "sync" ) -func ParseLocation(remote string) (loc *remote_pb.RemoteStorageLocation) { +func ParseLocationName(remote string) (locationName string) { + if strings.HasSuffix(string(remote), "/") { + remote = remote[:len(remote)-1] + } + parts := strings.SplitN(string(remote), "/", 2) + if len(parts) >= 1 { + return parts[0] + } + return +} + +func parseBucketLocation(remote string) (loc *remote_pb.RemoteStorageLocation) { loc = &remote_pb.RemoteStorageLocation{} if strings.HasSuffix(string(remote), "/") { remote = remote[:len(remote)-1] @@ -28,6 +39,22 @@ func ParseLocation(remote string) (loc *remote_pb.RemoteStorageLocation) { return } +func parseNoBucketLocation(remote string) (loc *remote_pb.RemoteStorageLocation) { + loc = &remote_pb.RemoteStorageLocation{} + if strings.HasSuffix(string(remote), "/") { + remote = remote[:len(remote)-1] + } + parts := strings.SplitN(string(remote), "/", 2) + if len(parts) >= 1 { + loc.Name = parts[0] + } + loc.Path = string(remote[len(loc.Name):]) + if loc.Path == "" { + loc.Path = "/" + } + return +} + func FormatLocation(loc *remote_pb.RemoteStorageLocation) string { return fmt.Sprintf("%s/%s%s", loc.Name, loc.Bucket, loc.Path) } @@ -45,6 +72,7 @@ type RemoteStorageClient interface { type RemoteStorageClientMaker interface { Make(remoteConf *remote_pb.RemoteConf) (RemoteStorageClient, error) + HasBucket() bool } var ( @@ -53,6 +81,18 @@ var ( remoteStorageClientsLock sync.Mutex ) +func ParseRemoteLocation(remoteConfType string, remote string) (remoteStorageLocation *remote_pb.RemoteStorageLocation, err error) { + maker, found := RemoteStorageClientMakers[remoteConfType] + if !found { + return nil, fmt.Errorf("remote storage type %s not found", remoteConfType) + } + + if !maker.HasBucket() { + return parseNoBucketLocation(remote), nil + } + return parseBucketLocation(remote), nil +} + func makeRemoteStorageClient(remoteConf *remote_pb.RemoteConf) (RemoteStorageClient, error) { maker, found := RemoteStorageClientMakers[remoteConf.Type] if !found { diff --git a/weed/remote_storage/s3/aliyun.go b/weed/remote_storage/s3/aliyun.go index 3a681369a..567c74299 100644 --- a/weed/remote_storage/s3/aliyun.go +++ b/weed/remote_storage/s3/aliyun.go @@ -18,6 +18,10 @@ func init() { type AliyunRemoteStorageMaker struct{} +func (s AliyunRemoteStorageMaker) HasBucket() bool { + return true +} + func (s AliyunRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) { client := &s3RemoteStorageClient{ conf: conf, diff --git a/weed/remote_storage/s3/backblaze.go b/weed/remote_storage/s3/backblaze.go index e4833999e..914f0ca44 100644 --- a/weed/remote_storage/s3/backblaze.go +++ b/weed/remote_storage/s3/backblaze.go @@ -16,6 +16,10 @@ func init() { type BackBlazeRemoteStorageMaker struct{} +func (s BackBlazeRemoteStorageMaker) HasBucket() bool { + return true +} + func (s BackBlazeRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) { client := &s3RemoteStorageClient{ conf: conf, diff --git a/weed/remote_storage/s3/baidu.go b/weed/remote_storage/s3/baidu.go index 8c5bf7d1b..dfcf32512 100644 --- a/weed/remote_storage/s3/baidu.go +++ b/weed/remote_storage/s3/baidu.go @@ -18,6 +18,10 @@ func init() { type BaiduRemoteStorageMaker struct{} +func (s BaiduRemoteStorageMaker) HasBucket() bool { + return true +} + func (s BaiduRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) { client := &s3RemoteStorageClient{ conf: conf, diff --git a/weed/remote_storage/s3/s3_storage_client.go b/weed/remote_storage/s3/s3_storage_client.go index 5fadcbc3b..a210683aa 100644 --- a/weed/remote_storage/s3/s3_storage_client.go +++ b/weed/remote_storage/s3/s3_storage_client.go @@ -23,6 +23,10 @@ func init() { type s3RemoteStorageMaker struct{} +func (s s3RemoteStorageMaker) HasBucket() bool { + return true +} + func (s s3RemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) { client := &s3RemoteStorageClient{ conf: conf, diff --git a/weed/remote_storage/s3/tencent.go b/weed/remote_storage/s3/tencent.go index e2591ca8c..9df72a7e2 100644 --- a/weed/remote_storage/s3/tencent.go +++ b/weed/remote_storage/s3/tencent.go @@ -18,6 +18,10 @@ func init() { type TencentRemoteStorageMaker struct{} +func (s TencentRemoteStorageMaker) HasBucket() bool { + return true +} + func (s TencentRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) { client := &s3RemoteStorageClient{ conf: conf, diff --git a/weed/remote_storage/s3/wasabi.go b/weed/remote_storage/s3/wasabi.go index 6f8fc3ca8..29cdf7395 100644 --- a/weed/remote_storage/s3/wasabi.go +++ b/weed/remote_storage/s3/wasabi.go @@ -18,6 +18,10 @@ func init() { type WasabiRemoteStorageMaker struct{} +func (s WasabiRemoteStorageMaker) HasBucket() bool { + return true +} + func (s WasabiRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) { client := &s3RemoteStorageClient{ conf: conf, diff --git a/weed/shell/command_remote_configure.go b/weed/shell/command_remote_configure.go index 1bfe82673..b6c4af178 100644 --- a/weed/shell/command_remote_configure.go +++ b/weed/shell/command_remote_configure.go @@ -58,7 +58,7 @@ func (c *commandRemoteConfigure) Do(args []string, commandEnv *CommandEnv, write isDelete := remoteConfigureCommand.Bool("delete", false, "delete one remote storage by its name") remoteConfigureCommand.StringVar(&conf.Name, "name", "", "a short name to identify the remote storage") - remoteConfigureCommand.StringVar(&conf.Type, "type", "s3", "[s3|gcs|azure|b2|aliyun|tencent|baidu|wasabi] storage type") + remoteConfigureCommand.StringVar(&conf.Type, "type", "s3", "[s3|gcs|azure|b2|aliyun|tencent|baidu|wasabi|hdfs] storage type") remoteConfigureCommand.StringVar(&conf.S3AccessKey, "s3.access_key", "", "s3 access key") remoteConfigureCommand.StringVar(&conf.S3SecretKey, "s3.secret_key", "", "s3 secret key") @@ -95,10 +95,27 @@ func (c *commandRemoteConfigure) Do(args []string, commandEnv *CommandEnv, write remoteConfigureCommand.StringVar(&conf.WasabiEndpoint, "wasabi.endpoint", "", "Wasabi endpoint, see https://wasabi.com/wp-content/themes/wasabi/docs/API_Guide/index.html#t=topics%2Fapidiff-intro.htm") remoteConfigureCommand.StringVar(&conf.WasabiRegion, "wasabi.region", "", "Wasabi region") + var namenodes arrayFlags + remoteConfigureCommand.Var(&namenodes, "hdfs.namenodes", "hdfs name node and port, example: namenode1:8020,namenode2:8020") + remoteConfigureCommand.StringVar(&conf.HdfsUsername, "hdfs.username", "", "hdfs user name") + remoteConfigureCommand.StringVar(&conf.HdfsServicePrincipalName, "hdfs.servicePrincipalName", "", `Kerberos service principal name for the namenode + +Example: hdfs/namenode.hadoop.docker +Namenode running as service 'hdfs' with FQDN 'namenode.hadoop.docker'. +`) + remoteConfigureCommand.StringVar(&conf.HdfsDataTransferProtection, "hdfs.dataTransferProtection", "", "[authentication|integrity|privacy] Kerberos data transfer protection") + + if err = remoteConfigureCommand.Parse(args); err != nil { return nil } + if conf.Type != "s3" { + // clear out the default values + conf.S3Region = "" + conf.S3ForcePathStyle = false + } + if conf.Name == "" { return c.listExistingRemoteStorages(commandEnv, writer) } @@ -187,3 +204,14 @@ func (c *commandRemoteConfigure) saveRemoteStorage(commandEnv *CommandEnv, write return nil } + +type arrayFlags []string + +func (i *arrayFlags) String() string { + return "my string representation" +} + +func (i *arrayFlags) Set(value string) error { + *i = append(*i, value) + return nil +} diff --git a/weed/shell/command_remote_meta_sync.go b/weed/shell/command_remote_meta_sync.go index b4e3534fc..17b9abdb8 100644 --- a/weed/shell/command_remote_meta_sync.go +++ b/weed/shell/command_remote_meta_sync.go @@ -55,7 +55,7 @@ func (c *commandRemoteMetaSync) Do(args []string, commandEnv *CommandEnv, writer } mappings, localMountedDir, remoteStorageMountedLocation, remoteStorageConf, detectErr := detectMountInfo(commandEnv, writer, *dir) - if detectErr != nil{ + if detectErr != nil { jsonPrintln(writer, mappings) return detectErr } @@ -106,7 +106,7 @@ func detectMountInfo(commandEnv *CommandEnv, writer io.Writer, dir string) (*rem If entry.RemoteEntry.RemoteTag != remoteEntry.RemoteTag { the remote version is updated, need to pull meta } - */ +*/ func pullMetadata(commandEnv *CommandEnv, writer io.Writer, localMountedDir util.FullPath, remoteMountedLocation *remote_pb.RemoteStorageLocation, dirToCache util.FullPath, remoteConf *remote_pb.RemoteConf) error { // visit remote storage @@ -158,7 +158,7 @@ func pullMetadata(commandEnv *CommandEnv, writer io.Writer, localMountedDir util fmt.Fprintln(writer, " (skip)") return nil } - if existingEntry.RemoteEntry.RemoteETag != remoteEntry.RemoteETag { + if existingEntry.RemoteEntry.RemoteETag != remoteEntry.RemoteETag || existingEntry.RemoteEntry.RemoteMtime < remoteEntry.RemoteMtime { // the remote version is updated, need to pull meta fmt.Fprintln(writer, " (update)") return doSaveRemoteEntry(client, string(localDir), existingEntry, remoteEntry) diff --git a/weed/shell/command_remote_mount.go b/weed/shell/command_remote_mount.go index d1b282d9c..3e92428d9 100644 --- a/weed/shell/command_remote_mount.go +++ b/weed/shell/command_remote_mount.go @@ -60,15 +60,17 @@ func (c *commandRemoteMount) Do(args []string, commandEnv *CommandEnv, writer io return err } - remoteStorageLocation := remote_storage.ParseLocation(*remote) - // find configuration for remote storage - // remotePath is //path/to/dir - remoteConf, err := c.findRemoteStorageConfiguration(commandEnv, writer, remoteStorageLocation) + remoteConf, err := filer.ReadRemoteStorageConf(commandEnv.option.GrpcDialOption, commandEnv.option.FilerAddress, remote_storage.ParseLocationName(*remote)) if err != nil { return fmt.Errorf("find configuration for %s: %v", *remote, err) } + remoteStorageLocation, err := remote_storage.ParseRemoteLocation(remoteConf.Type, *remote) + if err != nil { + return err + } + // sync metadata from remote if err = c.syncMetadata(commandEnv, writer, *dir, *nonEmpty, remoteConf, remoteStorageLocation); err != nil { return fmt.Errorf("pull metadata: %v", err)