cloud tier: support for Alibaba Cloud OSS (#6466)

weed/storage/backend/s3_backend/s3_backend.go

@@ -2,12 +2,13 @@ package s3_backend
 
 import (
 	"fmt"
-	"github.com/seaweedfs/seaweedfs/weed/util"
 	"io"
 	"os"
 	"strings"
 	"time"
 
+	"github.com/seaweedfs/seaweedfs/weed/util"
+
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/aws/aws-sdk-go/service/s3/s3iface"
 	"github.com/google/uuid"
@@ -39,6 +40,7 @@ type S3BackendStorage struct {
 	bucket                string
 	endpoint              string
 	storageClass          string
+	forcePathStyle        bool
 	conn                  s3iface.S3API
 }
 
@@ -51,11 +53,12 @@ func newS3BackendStorage(configuration backend.StringProperties, configPrefix st
 	s.bucket = configuration.GetString(configPrefix + "bucket")
 	s.endpoint = configuration.GetString(configPrefix + "endpoint")
 	s.storageClass = configuration.GetString(configPrefix + "storage_class")
+	s.forcePathStyle = util.ParseBool(configuration.GetString(configPrefix+"force_path_style"), true)
 	if s.storageClass == "" {
 		s.storageClass = "STANDARD_IA"
 	}
 
-	s.conn, err = createSession(s.aws_access_key_id, s.aws_secret_access_key, s.region, s.endpoint)
+	s.conn, err = createSession(s.aws_access_key_id, s.aws_secret_access_key, s.region, s.endpoint, s.forcePathStyle)
 
 	glog.V(0).Infof("created backend storage s3.%s for region %s bucket %s", s.id, s.region, s.bucket)
 	return
@@ -69,6 +72,7 @@ func (s *S3BackendStorage) ToProperties() map[string]string {
 	m["bucket"] = s.bucket
 	m["endpoint"] = s.endpoint
 	m["storage_class"] = s.storageClass
+	m["force_path_style"] = util.BoolToString(s.forcePathStyle)
 	return m
 }
 

weed/storage/backend/s3_backend/s3_sessions.go

@@ -2,9 +2,10 @@ package s3_backend
 
 import (
 	"fmt"
+	"sync"
+
 	"github.com/aws/aws-sdk-go/aws/request"
 	"github.com/seaweedfs/seaweedfs/weed/util"
-	"sync"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/credentials"
@@ -26,7 +27,7 @@ func getSession(region string) (s3iface.S3API, bool) {
 	return sess, found
 }
 
-func createSession(awsAccessKeyId, awsSecretAccessKey, region, endpoint string) (s3iface.S3API, error) {
+func createSession(awsAccessKeyId, awsSecretAccessKey, region, endpoint string, forcePathStyle bool) (s3iface.S3API, error) {
 
 	sessionsLock.Lock()
 	defer sessionsLock.Unlock()
@@ -38,7 +39,7 @@ func createSession(awsAccessKeyId, awsSecretAccessKey, region, endpoint string)
 	config := &aws.Config{
 		Region:                        aws.String(region),
 		Endpoint:                      aws.String(endpoint),
-		S3ForcePathStyle:              aws.Bool(true),
+		S3ForcePathStyle:              aws.Bool(forcePathStyle),
 		S3DisableContentMD5Validation: aws.Bool(true),
 	}
 	if awsAccessKeyId != "" && awsSecretAccessKey != "" {

weed/util/parse.go

@@ -28,6 +28,18 @@ func ParseUint64(text string, defaultValue uint64) uint64 {
 	return count
 }
 
+func ParseBool(s string, defaultValue bool) bool {
+	value, err := strconv.ParseBool(s)
+	if err != nil {
+		return defaultValue
+	}
+	return value
+}
+
+func BoolToString(b bool) string {
+	return strconv.FormatBool(b)
+}
+
 func ParseFilerUrl(entryPath string) (filerServer string, filerPort int64, path string, err error) {
 	if !strings.HasPrefix(entryPath, "http://") && !strings.HasPrefix(entryPath, "https://") {
 		entryPath = "http://" + entryPath