From 572cc440ce50eef590fd9d2de536e6ec5ed4b34d Mon Sep 17 00:00:00 2001 From: chrislu Date: Sat, 17 Jun 2023 12:01:50 -0700 Subject: [PATCH] a simple example of using aws-sdk-go-v2 --- go.mod | 12 +++- go.sum | 6 ++ test/s3/s3client/s3client.go | 114 +++++++++++++++++++++++++++++++++++ 3 files changed, 129 insertions(+), 3 deletions(-) create mode 100644 test/s3/s3client/s3client.go diff --git a/go.mod b/go.mod index d0af8fab6..c5e34f753 100644 --- a/go.mod +++ b/go.mod @@ -143,6 +143,10 @@ require ( github.com/Jille/raft-grpc-transport v1.4.0 github.com/arangodb/go-driver v1.6.0 github.com/armon/go-metrics v0.4.1 + github.com/aws/aws-sdk-go-v2 v1.17.4 + github.com/aws/aws-sdk-go-v2/config v1.18.12 + github.com/aws/aws-sdk-go-v2/credentials v1.13.12 + github.com/aws/aws-sdk-go-v2/service/s3 v1.30.2 github.com/fluent/fluent-logger-golang v1.9.0 github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50 github.com/hanwen/go-fuse/v2 v2.3.0 @@ -173,14 +177,16 @@ require ( github.com/Unknwon/goconfig v1.0.0 // indirect github.com/abbot/go-http-auth v0.4.0 // indirect github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect - github.com/aws/aws-sdk-go-v2 v1.17.4 // indirect - github.com/aws/aws-sdk-go-v2/config v1.18.12 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.13.12 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.22 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.28 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.22 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.3.29 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.19 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.23 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.22 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.22 // indirect github.com/aws/aws-sdk-go-v2/service/sns v1.20.2 // indirect github.com/aws/aws-sdk-go-v2/service/sqs v1.20.2 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.12.1 // indirect diff --git a/go.sum b/go.sum index 566eca93b..3ef8a2a75 100644 --- a/go.sum +++ b/go.sum @@ -593,6 +593,7 @@ github.com/aws/aws-sdk-go v1.44.280/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8 github.com/aws/aws-sdk-go-v2 v1.9.1/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4= github.com/aws/aws-sdk-go-v2 v1.17.4 h1:wyC6p9Yfq6V2y98wfDsj6OnNQa4w2BLGCLIxzNhwOGY= github.com/aws/aws-sdk-go-v2 v1.17.4/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 h1:dK82zF6kkPeCo8J1e+tGx4JdvDIQzj7ygIoLg8WMuGs= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10/go.mod h1:VeTZetY5KRJLuD/7fkQXMU6Mw7H5m/KP2J5Iy9osMno= github.com/aws/aws-sdk-go-v2/config v1.18.12 h1:fKs/I4wccmfrNRO9rdrbMO1NgLxct6H9rNMiPdBxHWw= github.com/aws/aws-sdk-go-v2/config v1.18.12/go.mod h1:J36fOhj1LQBr+O4hJCiT8FwVvieeoSGOtPuvhKlsNu8= @@ -607,14 +608,19 @@ github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.22 h1:7AwGYXDdqRQYsluvKF github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.22/go.mod h1:EqK7gVrIGAHyZItrD1D8B0ilgwMD1GiWAmbU4u/JHNk= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.29 h1:J4xhFd6zHhdF9jPP0FQJ6WknzBboGMBNjKOv4iTuw4A= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.29/go.mod h1:TwuqRBGzxjQJIwH16/fOZodwXt2Zxa9/cwJC5ke4j7s= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.19 h1:FGvpyTg2LKEmMrLlpjOgkoNp9XF5CGeyAyo33LdqZW8= github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.19/go.mod h1:8W88sW3PjamQpKFUQvHWWKay6ARsNvZnzU7+a4apubw= github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.8.1/go.mod h1:CM+19rL1+4dFWnOQKwDc7H1KwXTz+h61oUSHyhV0b3o= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 h1:y2+VQzC6Zh2ojtV2LoC0MNwHWc6qXv/j2vrQtlftkdA= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11/go.mod h1:iV4q2hsqtNECrfmlXyord9u4zyuFEJX9eLgLpSPzWA8= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.23 h1:c5+bNdV8E4fIPteWx4HZSkqI07oY9exbfQ7JH7Yx4PI= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.23/go.mod h1:1jcUfF+FAOEwtIcNiHPaV4TSoZqkUIPzrohmD7fb95c= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.22 h1:LjFQf8hFuMO22HkV5VWGLBvmCLBCLPivUAmpdpnp4Vs= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.22/go.mod h1:xt0Au8yPIwYXf/GYPy/vl4K3CgwhfQMYbrH7DlUUIws= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.22 h1:ISLJ2BKXe4zzyZ7mp5ewKECiw0U7KpLgS3S6OxY9Cm0= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.22/go.mod h1:QFVbqK54XArazLvn2wvWMRBi/jGrWii46qbr5DyPGjc= github.com/aws/aws-sdk-go-v2/service/kms v1.20.2/go.mod h1:vdqtUOdVuf5ooy+hJ2GnzqNo94xiAA9s1xbZ1hQgRE0= +github.com/aws/aws-sdk-go-v2/service/s3 v1.30.2 h1:5EQWIFO+Hc8E2hFcXQJ1vm6ufl/PMt/6RVRDZRju2vM= github.com/aws/aws-sdk-go-v2/service/s3 v1.30.2/go.mod h1:SXDHd6fI2RhqB7vmAzyYQCTQnpZrIprVJvYxpzW3JAM= github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.18.3/go.mod h1:hqPcyOuLU6yWIbLy3qMnQnmidgKuIEwqIlW6+chYnog= github.com/aws/aws-sdk-go-v2/service/sns v1.20.2 h1:MU/v2qtfGjKexJ09BMqE8pXo9xYMhT13FXjKgFc0cFw= diff --git a/test/s3/s3client/s3client.go b/test/s3/s3client/s3client.go new file mode 100644 index 000000000..e789c57c6 --- /dev/null +++ b/test/s3/s3client/s3client.go @@ -0,0 +1,114 @@ +package main + +import ( + "context" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/aws/retry" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + "time" +) + +func main() { + cfg := MyConfig{ + Key: "any", + Secret: "any", + Region: "US", + Endpoint: MyEndpoint{ + URL: "http://localhost:8333", + }, + Bucket: MyBucketConfig{ + Name: "newbucket", + Versioning: false, + }, + MaxBackoffDelay: aws.Int(int(time.Second * 5)), + MaxRetryAttempts: aws.Int(1), + } + + awsCfg, err := MyAwsConfig(cfg) + if err != nil { + panic(err) + } + svc := s3.NewFromConfig(*awsCfg, func(o *s3.Options) { + o.UsePathStyle = true + }) + + // Use the S3 client to interact with SeaweedFS + // ... + // Example: List all buckets + result, err := svc.ListBuckets(context.Background(), &s3.ListBucketsInput{}) + // no errors - got list of buckets + if err != nil { + panic(err) + } + + // Print the list of buckets + for _, bucket := range result.Buckets { + println(*bucket.Name) + } + + bucket := "bucket1" + _, err = svc.HeadBucket(context.Background(), &s3.HeadBucketInput{Bucket: &bucket}) + // ERROR HERE + if err != nil { + println(err) + } + +} + +// === helpers + +func MyAwsConfig(cfg MyConfig) (*aws.Config, error) { + + cred := aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider(cfg.Key, cfg.Secret, "")) + customResolver := aws.EndpointResolverWithOptionsFunc( + func(service, region string, options ...interface{}) (aws.Endpoint, error) { + return aws.Endpoint{ + URL: cfg.Endpoint.URL, + SigningRegion: cfg.Region, + }, nil + }) + + awsCfg, err := config.LoadDefaultConfig(context.TODO(), + config.WithRegion(cfg.Region), + config.WithCredentialsProvider(cred), + config.WithEndpointResolverWithOptions(customResolver), + config.WithRetryer(func() aws.Retryer { + r := retry.AddWithMaxAttempts(retry.NewStandard(), *cfg.MaxRetryAttempts) + return retry.AddWithMaxBackoffDelay(r, time.Duration(*cfg.MaxBackoffDelay*1000*1000)) + })) + return &awsCfg, err +} + +type MyConfig struct { + // Access key of S3 AWS. + Key string + // Access secret of S3 AWS. + Secret string + // Region. + Region string + // AWS endpoint. + Endpoint MyEndpoint + // Bucket configuration. + Bucket MyBucketConfig + // File access. + FileAccess MyFileAccessType + // Maximum backoff delay (ms, default: 20 sec). + MaxBackoffDelay *int + // Maximum attempts to retry operation on error (default: 5). + MaxRetryAttempts *int +} + +type MyBucketConfig struct { + // Name of bucket + Name string + // Enable or not versioning + Versioning bool +} + +type MyEndpoint struct { + URL string +} + +type MyFileAccessType byte