Browse Source
s3: ListParts output xml format
s3: ListParts output xml format
fix https://github.com/chrislusf/seaweedfs/issues/1461pull/1475/head
Chris Lu
4 years ago
5 changed files with 230 additions and 15 deletions
-
175test/s3/multipart/aws_upload.go
-
28weed/s3api/filer_multipart.go
-
23weed/s3api/filer_multipart_test.go
-
7weed/s3api/filer_util.go
-
2weed/s3api/s3api_bucket_handlers.go
@ -0,0 +1,175 @@ |
|||
package main |
|||
|
|||
// copied from https://github.com/apoorvam/aws-s3-multipart-upload
|
|||
|
|||
import ( |
|||
"bytes" |
|||
"flag" |
|||
"fmt" |
|||
"net/http" |
|||
"os" |
|||
|
|||
"github.com/aws/aws-sdk-go/aws" |
|||
"github.com/aws/aws-sdk-go/aws/awserr" |
|||
"github.com/aws/aws-sdk-go/aws/credentials" |
|||
"github.com/aws/aws-sdk-go/aws/session" |
|||
"github.com/aws/aws-sdk-go/service/s3" |
|||
) |
|||
|
|||
const ( |
|||
maxPartSize = int64(5 * 1024 * 1024) |
|||
maxRetries = 3 |
|||
awsAccessKeyID = "Your access key" |
|||
awsSecretAccessKey = "Your secret key" |
|||
awsBucketRegion = "S3 bucket region" |
|||
awsBucketName = "newBucket" |
|||
) |
|||
|
|||
var ( |
|||
filename = flag.String("f", "", "the file name") |
|||
) |
|||
|
|||
func main() { |
|||
flag.Parse() |
|||
|
|||
creds := credentials.NewStaticCredentials(awsAccessKeyID, awsSecretAccessKey, "") |
|||
_, err := creds.Get() |
|||
if err != nil { |
|||
fmt.Printf("bad credentials: %s", err) |
|||
} |
|||
cfg := aws.NewConfig().WithRegion(awsBucketRegion).WithCredentials(creds).WithDisableSSL(true).WithEndpoint("localhost:8333") |
|||
svc := s3.New(session.New(), cfg) |
|||
|
|||
file, err := os.Open(*filename) |
|||
if err != nil { |
|||
fmt.Printf("err opening file: %s", err) |
|||
return |
|||
} |
|||
defer file.Close() |
|||
fileInfo, _ := file.Stat() |
|||
size := fileInfo.Size() |
|||
buffer := make([]byte, size) |
|||
fileType := http.DetectContentType(buffer) |
|||
file.Read(buffer) |
|||
|
|||
path := "/media/" + file.Name() |
|||
input := &s3.CreateMultipartUploadInput{ |
|||
Bucket: aws.String(awsBucketName), |
|||
Key: aws.String(path), |
|||
ContentType: aws.String(fileType), |
|||
} |
|||
|
|||
resp, err := svc.CreateMultipartUpload(input) |
|||
if err != nil { |
|||
fmt.Println(err.Error()) |
|||
return |
|||
} |
|||
fmt.Println("Created multipart upload request") |
|||
|
|||
var curr, partLength int64 |
|||
var remaining = size |
|||
var completedParts []*s3.CompletedPart |
|||
partNumber := 1 |
|||
for curr = 0; remaining != 0; curr += partLength { |
|||
if remaining < maxPartSize { |
|||
partLength = remaining |
|||
} else { |
|||
partLength = maxPartSize |
|||
} |
|||
completedPart, err := uploadPart(svc, resp, buffer[curr:curr+partLength], partNumber) |
|||
if err != nil { |
|||
fmt.Println(err.Error()) |
|||
err := abortMultipartUpload(svc, resp) |
|||
if err != nil { |
|||
fmt.Println(err.Error()) |
|||
} |
|||
return |
|||
} |
|||
remaining -= partLength |
|||
partNumber++ |
|||
completedParts = append(completedParts, completedPart) |
|||
} |
|||
|
|||
// list parts
|
|||
parts, err := svc.ListParts(&s3.ListPartsInput{ |
|||
Bucket: input.Bucket, |
|||
Key: input.Key, |
|||
MaxParts: nil, |
|||
PartNumberMarker: nil, |
|||
RequestPayer: nil, |
|||
UploadId: resp.UploadId, |
|||
}) |
|||
if err != nil { |
|||
fmt.Println(err.Error()) |
|||
return |
|||
} |
|||
fmt.Printf("list parts: %d\n", len(parts.Parts)) |
|||
for i, part := range parts.Parts { |
|||
fmt.Printf("part %d: %v\n", i, part) |
|||
} |
|||
|
|||
|
|||
completeResponse, err := completeMultipartUpload(svc, resp, completedParts) |
|||
if err != nil { |
|||
fmt.Println(err.Error()) |
|||
return |
|||
} |
|||
|
|||
fmt.Printf("Successfully uploaded file: %s\n", completeResponse.String()) |
|||
} |
|||
|
|||
func completeMultipartUpload(svc *s3.S3, resp *s3.CreateMultipartUploadOutput, completedParts []*s3.CompletedPart) (*s3.CompleteMultipartUploadOutput, error) { |
|||
completeInput := &s3.CompleteMultipartUploadInput{ |
|||
Bucket: resp.Bucket, |
|||
Key: resp.Key, |
|||
UploadId: resp.UploadId, |
|||
MultipartUpload: &s3.CompletedMultipartUpload{ |
|||
Parts: completedParts, |
|||
}, |
|||
} |
|||
return svc.CompleteMultipartUpload(completeInput) |
|||
} |
|||
|
|||
func uploadPart(svc *s3.S3, resp *s3.CreateMultipartUploadOutput, fileBytes []byte, partNumber int) (*s3.CompletedPart, error) { |
|||
tryNum := 1 |
|||
partInput := &s3.UploadPartInput{ |
|||
Body: bytes.NewReader(fileBytes), |
|||
Bucket: resp.Bucket, |
|||
Key: resp.Key, |
|||
PartNumber: aws.Int64(int64(partNumber)), |
|||
UploadId: resp.UploadId, |
|||
ContentLength: aws.Int64(int64(len(fileBytes))), |
|||
} |
|||
|
|||
for tryNum <= maxRetries { |
|||
uploadResult, err := svc.UploadPart(partInput) |
|||
if err != nil { |
|||
if tryNum == maxRetries { |
|||
if aerr, ok := err.(awserr.Error); ok { |
|||
return nil, aerr |
|||
} |
|||
return nil, err |
|||
} |
|||
fmt.Printf("Retrying to upload part #%v\n", partNumber) |
|||
tryNum++ |
|||
} else { |
|||
fmt.Printf("Uploaded part #%v\n", partNumber) |
|||
return &s3.CompletedPart{ |
|||
ETag: uploadResult.ETag, |
|||
PartNumber: aws.Int64(int64(partNumber)), |
|||
}, nil |
|||
} |
|||
} |
|||
return nil, nil |
|||
} |
|||
|
|||
func abortMultipartUpload(svc *s3.S3, resp *s3.CreateMultipartUploadOutput) error { |
|||
fmt.Println("Aborting multipart upload for UploadId#" + *resp.UploadId) |
|||
abortInput := &s3.AbortMultipartUploadInput{ |
|||
Bucket: resp.Bucket, |
|||
Key: resp.Key, |
|||
UploadId: resp.UploadId, |
|||
} |
|||
_, err := svc.AbortMultipartUpload(abortInput) |
|||
return err |
|||
} |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue