ustuzhanin
4 years ago
committed by
GitHub
GitHub
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 592 additions and 55 deletions
-
4go.mod
-
4go.sum
-
82test/s3/basic/object_tagging_test.go
-
8weed/command/master.go
-
27weed/command/mount_std.go
-
1weed/command/server.go
-
91weed/filer/reader_at.go
-
6weed/filesys/dir.go
-
3weed/filesys/wfs.go
-
104weed/s3api/filer_util_tags.go
-
117weed/s3api/s3api_object_tagging_handlers.go
-
7weed/s3api/s3api_server.go
-
6weed/s3api/s3err/s3api_errors.go
-
38weed/s3api/tags.go
-
50weed/s3api/tags_test.go
-
53weed/server/raft_server.go
-
2weed/storage/backend/volume_create_linux.go
-
4weed/util/chunk_cache/chunk_cache.go
@ -0,0 +1,82 @@ |
|||
package basic |
|||
|
|||
import ( |
|||
"fmt" |
|||
"github.com/aws/aws-sdk-go/aws" |
|||
"github.com/aws/aws-sdk-go/service/s3" |
|||
"testing" |
|||
) |
|||
|
|||
func TestObjectTagging(t *testing.T) { |
|||
|
|||
input := &s3.PutObjectInput{ |
|||
Bucket: aws.String("theBucket"), |
|||
Key: aws.String("testDir/testObject"), |
|||
} |
|||
|
|||
svc.PutObject(input) |
|||
|
|||
printTags() |
|||
|
|||
setTags() |
|||
|
|||
printTags() |
|||
|
|||
clearTags() |
|||
|
|||
printTags() |
|||
|
|||
} |
|||
|
|||
func printTags() { |
|||
response, err := svc.GetObjectTagging( |
|||
&s3.GetObjectTaggingInput{ |
|||
Bucket: aws.String("theBucket"), |
|||
Key: aws.String("testDir/testObject"), |
|||
}) |
|||
|
|||
fmt.Println("printTags") |
|||
if err != nil { |
|||
fmt.Println(err.Error()) |
|||
} |
|||
|
|||
fmt.Println(response.TagSet) |
|||
} |
|||
|
|||
func setTags() { |
|||
|
|||
response, err := svc.PutObjectTagging(&s3.PutObjectTaggingInput{ |
|||
Bucket: aws.String("theBucket"), |
|||
Key: aws.String("testDir/testObject"), |
|||
Tagging: &s3.Tagging{ |
|||
TagSet: []*s3.Tag{ |
|||
{ |
|||
Key: aws.String("kye2"), |
|||
Value: aws.String("value2"), |
|||
}, |
|||
}, |
|||
}, |
|||
}) |
|||
|
|||
fmt.Println("setTags") |
|||
if err != nil { |
|||
fmt.Println(err.Error()) |
|||
} |
|||
|
|||
fmt.Println(response.String()) |
|||
} |
|||
|
|||
func clearTags() { |
|||
|
|||
response, err := svc.DeleteObjectTagging(&s3.DeleteObjectTaggingInput{ |
|||
Bucket: aws.String("theBucket"), |
|||
Key: aws.String("testDir/testObject"), |
|||
}) |
|||
|
|||
fmt.Println("clearTags") |
|||
if err != nil { |
|||
fmt.Println(err.Error()) |
|||
} |
|||
|
|||
fmt.Println(response.String()) |
|||
} |
|||
@ -0,0 +1,104 @@ |
|||
package s3api |
|||
|
|||
import ( |
|||
"strings" |
|||
|
|||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" |
|||
) |
|||
|
|||
const( |
|||
S3TAG_PREFIX = "s3-" |
|||
) |
|||
|
|||
func (s3a *S3ApiServer) getTags(parentDirectoryPath string, entryName string) (tags map[string]string, err error) { |
|||
|
|||
err = s3a.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { |
|||
|
|||
resp, err := filer_pb.LookupEntry(client, &filer_pb.LookupDirectoryEntryRequest{ |
|||
Directory: parentDirectoryPath, |
|||
Name: entryName, |
|||
}) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
tags = make(map[string]string) |
|||
for k, v := range resp.Entry.Extended { |
|||
if strings.HasPrefix(k, S3TAG_PREFIX) { |
|||
tags[k[len(S3TAG_PREFIX):]] = string(v) |
|||
} |
|||
} |
|||
return nil |
|||
}) |
|||
return |
|||
} |
|||
|
|||
func (s3a *S3ApiServer) setTags(parentDirectoryPath string, entryName string, tags map[string]string) (err error) { |
|||
|
|||
return s3a.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { |
|||
|
|||
resp, err := filer_pb.LookupEntry(client, &filer_pb.LookupDirectoryEntryRequest{ |
|||
Directory: parentDirectoryPath, |
|||
Name: entryName, |
|||
}) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
|
|||
for k, _ := range resp.Entry.Extended { |
|||
if strings.HasPrefix(k, S3TAG_PREFIX) { |
|||
delete(resp.Entry.Extended, k) |
|||
} |
|||
} |
|||
|
|||
if resp.Entry.Extended == nil { |
|||
resp.Entry.Extended = make(map[string][]byte) |
|||
} |
|||
for k, v := range tags { |
|||
resp.Entry.Extended[S3TAG_PREFIX+k] = []byte(v) |
|||
} |
|||
|
|||
return filer_pb.UpdateEntry(client, &filer_pb.UpdateEntryRequest{ |
|||
Directory: parentDirectoryPath, |
|||
Entry: resp.Entry, |
|||
IsFromOtherCluster: false, |
|||
Signatures: nil, |
|||
}) |
|||
|
|||
}) |
|||
|
|||
} |
|||
|
|||
func (s3a *S3ApiServer) rmTags(parentDirectoryPath string, entryName string) (err error) { |
|||
|
|||
return s3a.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { |
|||
|
|||
resp, err := filer_pb.LookupEntry(client, &filer_pb.LookupDirectoryEntryRequest{ |
|||
Directory: parentDirectoryPath, |
|||
Name: entryName, |
|||
}) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
|
|||
hasDeletion := false |
|||
for k, _ := range resp.Entry.Extended { |
|||
if strings.HasPrefix(k, S3TAG_PREFIX) { |
|||
delete(resp.Entry.Extended, k) |
|||
hasDeletion = true |
|||
} |
|||
} |
|||
|
|||
if !hasDeletion { |
|||
return nil |
|||
} |
|||
|
|||
return filer_pb.UpdateEntry(client, &filer_pb.UpdateEntryRequest{ |
|||
Directory: parentDirectoryPath, |
|||
Entry: resp.Entry, |
|||
IsFromOtherCluster: false, |
|||
Signatures: nil, |
|||
}) |
|||
|
|||
}) |
|||
|
|||
} |
|||
@ -0,0 +1,117 @@ |
|||
package s3api |
|||
|
|||
import ( |
|||
"encoding/xml" |
|||
"fmt" |
|||
"github.com/chrislusf/seaweedfs/weed/glog" |
|||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" |
|||
"github.com/chrislusf/seaweedfs/weed/s3api/s3err" |
|||
"github.com/chrislusf/seaweedfs/weed/util" |
|||
"io" |
|||
"io/ioutil" |
|||
"net/http" |
|||
) |
|||
|
|||
// GetObjectTaggingHandler - GET object tagging
|
|||
// API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObjectTagging.html
|
|||
func (s3a *S3ApiServer) GetObjectTaggingHandler(w http.ResponseWriter, r *http.Request) { |
|||
|
|||
bucket, object := getBucketAndObject(r) |
|||
|
|||
target := util.FullPath(fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, bucket, object)) |
|||
dir, name := target.DirAndName() |
|||
|
|||
tags, err := s3a.getTags(dir, name) |
|||
if err != nil { |
|||
if err == filer_pb.ErrNotFound { |
|||
glog.Errorf("GetObjectTaggingHandler %s: %v", r.URL, err) |
|||
writeErrorResponse(w, s3err.ErrNoSuchKey, r.URL) |
|||
} else { |
|||
glog.Errorf("GetObjectTaggingHandler %s: %v", r.URL, err) |
|||
writeErrorResponse(w, s3err.ErrInternalError, r.URL) |
|||
} |
|||
return |
|||
} |
|||
|
|||
writeSuccessResponseXML(w, encodeResponse(FromTags(tags))) |
|||
|
|||
} |
|||
|
|||
// PutObjectTaggingHandler Put object tagging
|
|||
// API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObjectTagging.html
|
|||
func (s3a *S3ApiServer) PutObjectTaggingHandler(w http.ResponseWriter, r *http.Request) { |
|||
|
|||
bucket, object := getBucketAndObject(r) |
|||
|
|||
target := util.FullPath(fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, bucket, object)) |
|||
dir, name := target.DirAndName() |
|||
|
|||
tagging := &Tagging{} |
|||
input, err := ioutil.ReadAll(io.LimitReader(r.Body, r.ContentLength)) |
|||
if err != nil { |
|||
glog.Errorf("PutObjectTaggingHandler read input %s: %v", r.URL, err) |
|||
writeErrorResponse(w, s3err.ErrInternalError, r.URL) |
|||
return |
|||
} |
|||
if err = xml.Unmarshal(input, tagging); err != nil { |
|||
glog.Errorf("PutObjectTaggingHandler Unmarshal %s: %v", r.URL, err) |
|||
writeErrorResponse(w, s3err.ErrMalformedXML, r.URL) |
|||
return |
|||
} |
|||
tags := tagging.ToTags() |
|||
if len(tags) > 10 { |
|||
glog.Errorf("PutObjectTaggingHandler tags %s: %d tags more than 10", r.URL, len(tags)) |
|||
writeErrorResponse(w, s3err.ErrInvalidTag, r.URL) |
|||
return |
|||
} |
|||
for k, v := range tags { |
|||
if len(k) > 128 { |
|||
glog.Errorf("PutObjectTaggingHandler tags %s: tag key %s longer than 128", r.URL, k) |
|||
writeErrorResponse(w, s3err.ErrInvalidTag, r.URL) |
|||
return |
|||
} |
|||
if len(v) > 256 { |
|||
glog.Errorf("PutObjectTaggingHandler tags %s: tag value %s longer than 256", r.URL, v) |
|||
writeErrorResponse(w, s3err.ErrInvalidTag, r.URL) |
|||
return |
|||
} |
|||
} |
|||
|
|||
if err = s3a.setTags(dir, name, tagging.ToTags()); err != nil { |
|||
if err == filer_pb.ErrNotFound { |
|||
glog.Errorf("PutObjectTaggingHandler setTags %s: %v", r.URL, err) |
|||
writeErrorResponse(w, s3err.ErrNoSuchKey, r.URL) |
|||
} else { |
|||
glog.Errorf("PutObjectTaggingHandler setTags %s: %v", r.URL, err) |
|||
writeErrorResponse(w, s3err.ErrInternalError, r.URL) |
|||
} |
|||
return |
|||
} |
|||
|
|||
w.WriteHeader(http.StatusNoContent) |
|||
|
|||
} |
|||
|
|||
// DeleteObjectTaggingHandler Delete object tagging
|
|||
// API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjectTagging.html
|
|||
func (s3a *S3ApiServer) DeleteObjectTaggingHandler(w http.ResponseWriter, r *http.Request) { |
|||
|
|||
bucket, object := getBucketAndObject(r) |
|||
|
|||
target := util.FullPath(fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, bucket, object)) |
|||
dir, name := target.DirAndName() |
|||
|
|||
err := s3a.rmTags(dir, name) |
|||
if err != nil { |
|||
if err == filer_pb.ErrNotFound { |
|||
glog.Errorf("DeleteObjectTaggingHandler %s: %v", r.URL, err) |
|||
writeErrorResponse(w, s3err.ErrNoSuchKey, r.URL) |
|||
} else { |
|||
glog.Errorf("DeleteObjectTaggingHandler %s: %v", r.URL, err) |
|||
writeErrorResponse(w, s3err.ErrInternalError, r.URL) |
|||
} |
|||
return |
|||
} |
|||
|
|||
w.WriteHeader(http.StatusNoContent) |
|||
} |
|||
@ -0,0 +1,38 @@ |
|||
package s3api |
|||
|
|||
import ( |
|||
"encoding/xml" |
|||
) |
|||
|
|||
type Tag struct { |
|||
Key string `xml:"Key"` |
|||
Value string `xml:"Value"` |
|||
} |
|||
|
|||
type TagSet struct { |
|||
Tag []Tag `xml:"Tag"` |
|||
} |
|||
|
|||
type Tagging struct { |
|||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Tagging"` |
|||
TagSet TagSet `xml:"TagSet"` |
|||
} |
|||
|
|||
func (t *Tagging) ToTags() map[string]string { |
|||
output := make(map[string]string) |
|||
for _, tag := range t.TagSet.Tag { |
|||
output[tag.Key] = tag.Value |
|||
} |
|||
return output |
|||
} |
|||
|
|||
func FromTags(tags map[string]string) (t *Tagging) { |
|||
t = &Tagging{} |
|||
for k, v := range tags { |
|||
t.TagSet.Tag = append(t.TagSet.Tag, Tag{ |
|||
Key: k, |
|||
Value: v, |
|||
}) |
|||
} |
|||
return |
|||
} |
|||
@ -0,0 +1,50 @@ |
|||
package s3api |
|||
|
|||
import ( |
|||
"encoding/xml" |
|||
"github.com/stretchr/testify/assert" |
|||
"testing" |
|||
) |
|||
|
|||
func TestXMLUnmarshall(t *testing.T) { |
|||
|
|||
input := `<?xml version="1.0" encoding="UTF-8"?> |
|||
<Tagging xmlns="http://s3.amazonaws.com/doc/2006-03-01/"> |
|||
<TagSet> |
|||
<Tag> |
|||
<Key>key1</Key> |
|||
<Value>value1</Value> |
|||
</Tag> |
|||
</TagSet> |
|||
</Tagging> |
|||
` |
|||
|
|||
tags := &Tagging{} |
|||
|
|||
xml.Unmarshal([]byte(input), tags) |
|||
|
|||
assert.Equal(t, len(tags.TagSet.Tag), 1) |
|||
assert.Equal(t, tags.TagSet.Tag[0].Key, "key1") |
|||
assert.Equal(t, tags.TagSet.Tag[0].Value, "value1") |
|||
|
|||
} |
|||
|
|||
func TestXMLMarshall(t *testing.T) { |
|||
tags := &Tagging{ |
|||
TagSet: TagSet{ |
|||
[]Tag{ |
|||
{ |
|||
Key: "key1", |
|||
Value: "value1", |
|||
}, |
|||
}, |
|||
}, |
|||
} |
|||
|
|||
actual := string(encodeResponse(tags)) |
|||
|
|||
expected := `<?xml version="1.0" encoding="UTF-8"?> |
|||
<Tagging xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><TagSet><Tag><Key>key1</Key><Value>value1</Value></Tag></TagSet></Tagging>` |
|||
assert.Equal(t, expected, actual) |
|||
|
|||
} |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue