From ce6a2140a2173ec4090a18d07b85e8c553bcb3a8 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 4 Sep 2018 00:42:44 -0700 Subject: [PATCH] preparing to support S3 multipart uploads --- weed/pb/filer.proto | 1 + weed/pb/filer_pb/filer.pb.go | 163 ++++++------ weed/s3api/filer_multipart.go | 70 +++++ weed/s3api/filer_util.go | 55 +++- weed/s3api/s3api_errors.go | 42 +++ weed/s3api/s3api_object_handlers.go | 102 +++++--- weed/s3api/s3api_object_multipart_handlers.go | 240 +++++++++++++++++- weed/s3api/s3api_server.go | 6 +- 8 files changed, 541 insertions(+), 138 deletions(-) create mode 100644 weed/s3api/filer_multipart.go diff --git a/weed/pb/filer.proto b/weed/pb/filer.proto index b7e95d238..7681c6c5e 100644 --- a/weed/pb/filer.proto +++ b/weed/pb/filer.proto @@ -63,6 +63,7 @@ message Entry { bool is_directory = 2; repeated FileChunk chunks = 3; FuseAttributes attributes = 4; + map extended = 5; } message EventNotification { diff --git a/weed/pb/filer_pb/filer.pb.go b/weed/pb/filer_pb/filer.pb.go index ecce7ef76..70d63957b 100644 --- a/weed/pb/filer_pb/filer.pb.go +++ b/weed/pb/filer_pb/filer.pb.go @@ -163,10 +163,11 @@ func (m *ListEntriesResponse) GetEntries() []*Entry { } type Entry struct { - Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` - IsDirectory bool `protobuf:"varint,2,opt,name=is_directory,json=isDirectory" json:"is_directory,omitempty"` - Chunks []*FileChunk `protobuf:"bytes,3,rep,name=chunks" json:"chunks,omitempty"` - Attributes *FuseAttributes `protobuf:"bytes,4,opt,name=attributes" json:"attributes,omitempty"` + Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` + IsDirectory bool `protobuf:"varint,2,opt,name=is_directory,json=isDirectory" json:"is_directory,omitempty"` + Chunks []*FileChunk `protobuf:"bytes,3,rep,name=chunks" json:"chunks,omitempty"` + Attributes *FuseAttributes `protobuf:"bytes,4,opt,name=attributes" json:"attributes,omitempty"` + Extended map[string]string `protobuf:"bytes,5,rep,name=extended" json:"extended,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` } func (m *Entry) Reset() { *m = Entry{} } @@ -202,6 +203,13 @@ func (m *Entry) GetAttributes() *FuseAttributes { return nil } +func (m *Entry) GetExtended() map[string]string { + if m != nil { + return m.Extended + } + return nil +} + type EventNotification struct { OldEntry *Entry `protobuf:"bytes,2,opt,name=old_entry,json=oldEntry" json:"old_entry,omitempty"` NewEntry *Entry `protobuf:"bytes,3,opt,name=new_entry,json=newEntry" json:"new_entry,omitempty"` @@ -1114,77 +1122,80 @@ var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{ func init() { proto.RegisterFile("filer.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ - // 1152 bytes of a gzipped FileDescriptorProto + // 1187 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x57, 0xcd, 0x6e, 0xdb, 0x46, - 0x10, 0x36, 0x25, 0x4b, 0x36, 0x47, 0x72, 0x1a, 0xaf, 0x9c, 0x84, 0x55, 0xec, 0x54, 0xd9, 0x26, - 0x85, 0x83, 0x1a, 0x86, 0xe1, 0xf6, 0x90, 0xb6, 0x28, 0xd0, 0xc0, 0x76, 0x82, 0x00, 0x4e, 0x02, - 0xd0, 0x75, 0x81, 0xa2, 0x07, 0x81, 0x26, 0x47, 0xee, 0xc2, 0x14, 0xa9, 0x72, 0x97, 0x76, 0xdd, - 0x6b, 0x8f, 0x3d, 0xf5, 0x21, 0x8a, 0xde, 0xfb, 0x00, 0xbd, 0xf4, 0xc5, 0x8a, 0xfd, 0xa1, 0xb4, - 0x14, 0x29, 0x27, 0x39, 0xe4, 0xb6, 0x3b, 0x33, 0x3b, 0x33, 0xdf, 0xec, 0xcc, 0xb7, 0x24, 0x74, - 0x46, 0x2c, 0xc6, 0x6c, 0x77, 0x92, 0xa5, 0x22, 0x25, 0xab, 0x6a, 0x33, 0x9c, 0x9c, 0xd1, 0x37, - 0x70, 0xff, 0x38, 0x4d, 0x2f, 0xf2, 0xc9, 0x21, 0xcb, 0x30, 0x14, 0x69, 0x76, 0x7d, 0x94, 0x88, - 0xec, 0xda, 0xc7, 0x5f, 0x72, 0xe4, 0x82, 0x6c, 0x82, 0x1b, 0x15, 0x0a, 0xcf, 0x19, 0x38, 0xdb, - 0xae, 0x3f, 0x13, 0x10, 0x02, 0xcb, 0x49, 0x30, 0x46, 0xaf, 0xa1, 0x14, 0x6a, 0x4d, 0x8f, 0x60, - 0xb3, 0xde, 0x21, 0x9f, 0xa4, 0x09, 0x47, 0xf2, 0x18, 0x5a, 0x28, 0x05, 0xca, 0x5b, 0x67, 0xff, - 0xa3, 0xdd, 0x22, 0x95, 0x5d, 0x6d, 0xa7, 0xb5, 0xf4, 0x5f, 0x07, 0xc8, 0x31, 0xe3, 0x42, 0x0a, - 0x19, 0xf2, 0x77, 0xcb, 0xe7, 0x2e, 0xb4, 0x27, 0x19, 0x8e, 0xd8, 0xaf, 0x26, 0x23, 0xb3, 0x23, - 0x3b, 0xb0, 0xce, 0x45, 0x90, 0x89, 0xe7, 0x59, 0x3a, 0x7e, 0xce, 0x62, 0x7c, 0x2d, 0x93, 0x6e, - 0x2a, 0x93, 0xaa, 0x82, 0xec, 0x02, 0x61, 0x49, 0x18, 0xe7, 0x9c, 0x5d, 0xe2, 0x49, 0xa1, 0xf5, - 0x96, 0x07, 0xce, 0xf6, 0xaa, 0x5f, 0xa3, 0x21, 0x1b, 0xd0, 0x8a, 0xd9, 0x98, 0x09, 0xaf, 0x35, - 0x70, 0xb6, 0xd7, 0x7c, 0xbd, 0xa1, 0xdf, 0x41, 0xaf, 0x94, 0xbf, 0x81, 0xff, 0x04, 0x56, 0x50, - 0x8b, 0x3c, 0x67, 0xd0, 0xac, 0x2b, 0x40, 0xa1, 0xa7, 0x7f, 0x39, 0xd0, 0x52, 0xa2, 0x69, 0x9d, - 0x9d, 0x59, 0x9d, 0xc9, 0x43, 0xe8, 0x32, 0x3e, 0x9c, 0x15, 0xa3, 0xa1, 0xf2, 0xeb, 0x30, 0x3e, - 0xad, 0x3b, 0xf9, 0x1c, 0xda, 0xe1, 0xcf, 0x79, 0x72, 0xc1, 0xbd, 0xa6, 0x0a, 0xd5, 0x9b, 0x85, - 0x92, 0x60, 0x0f, 0xa4, 0xce, 0x37, 0x26, 0xe4, 0x29, 0x40, 0x20, 0x44, 0xc6, 0xce, 0x72, 0x81, - 0x5c, 0xa1, 0xed, 0xec, 0x7b, 0xd6, 0x81, 0x9c, 0xe3, 0xb3, 0xa9, 0xde, 0xb7, 0x6c, 0x69, 0x0a, - 0xeb, 0x47, 0x97, 0x98, 0x88, 0xd7, 0xa9, 0x60, 0x23, 0x16, 0x06, 0x82, 0xa5, 0x09, 0xd9, 0x01, - 0x37, 0x8d, 0xa3, 0xa1, 0xbe, 0xea, 0x46, 0xfd, 0x55, 0xaf, 0xa6, 0x71, 0xa4, 0x01, 0xee, 0x80, - 0x9b, 0xe0, 0x95, 0xb1, 0x6e, 0x2e, 0xb0, 0x4e, 0xf0, 0x4a, 0xad, 0xe8, 0x08, 0xdc, 0x69, 0xfe, - 0xe4, 0x1e, 0xac, 0x48, 0xc3, 0x21, 0x8b, 0x4c, 0x79, 0xda, 0x72, 0xfb, 0x32, 0x92, 0xcd, 0x90, - 0x8e, 0x46, 0x1c, 0x85, 0x0a, 0xdf, 0xf4, 0xcd, 0x4e, 0x16, 0x93, 0xb3, 0xdf, 0xf4, 0xfd, 0x2f, - 0xfb, 0x6a, 0x2d, 0xaf, 0x70, 0x2c, 0xd8, 0x18, 0x15, 0xee, 0xa6, 0xaf, 0x37, 0xf4, 0x8f, 0x06, - 0xdc, 0x2a, 0xe3, 0x26, 0xf7, 0xc1, 0x55, 0xd1, 0x94, 0x07, 0x47, 0x79, 0x50, 0xb3, 0x74, 0x52, - 0xf2, 0xd2, 0xb0, 0xbc, 0x4c, 0x8f, 0x8c, 0xd3, 0x48, 0x07, 0x5d, 0xd3, 0x47, 0x5e, 0xa5, 0x11, - 0x92, 0xdb, 0xd0, 0xcc, 0x59, 0xa4, 0xc2, 0xae, 0xf9, 0x72, 0x29, 0x25, 0xe7, 0x2c, 0x32, 0xbd, - 0x24, 0x97, 0x12, 0x48, 0x98, 0x29, 0xbf, 0x6d, 0x0d, 0x44, 0xef, 0x24, 0x90, 0xb1, 0x94, 0xae, - 0xe8, 0xae, 0x90, 0x6b, 0x32, 0x80, 0x4e, 0x86, 0x93, 0xd8, 0xdc, 0x82, 0xb7, 0xaa, 0x54, 0xb6, - 0x88, 0x3c, 0x00, 0x08, 0xd3, 0x38, 0xc6, 0x50, 0x19, 0xb8, 0xca, 0xc0, 0x92, 0xc8, 0x7a, 0x0a, - 0x11, 0x0f, 0x39, 0x86, 0x1e, 0x0c, 0x9c, 0xed, 0x96, 0xdf, 0x16, 0x22, 0x3e, 0xc1, 0x90, 0x9e, - 0xc3, 0xc7, 0x2f, 0x50, 0xf5, 0xf3, 0xb5, 0xd5, 0x08, 0x66, 0x2e, 0xeb, 0x3a, 0x74, 0x0b, 0x60, - 0x12, 0x64, 0x98, 0x08, 0xd9, 0xa5, 0x66, 0x22, 0x5d, 0x2d, 0x39, 0x64, 0x99, 0x7d, 0x71, 0x4d, - 0xfb, 0xe2, 0xe8, 0xef, 0x0e, 0xf4, 0xeb, 0x22, 0x99, 0x09, 0x2a, 0x37, 0xaa, 0xf3, 0xee, 0x8d, - 0x6a, 0xcd, 0x43, 0xe3, 0xad, 0xf3, 0x40, 0xf7, 0xe0, 0xce, 0x0b, 0x14, 0x4a, 0x9e, 0x26, 0x02, - 0x13, 0x51, 0x40, 0x5d, 0xd4, 0x70, 0x74, 0x1f, 0xee, 0xce, 0x9f, 0x30, 0x29, 0x7b, 0xb0, 0x12, - 0x6a, 0x91, 0x3a, 0xd2, 0xf5, 0x8b, 0x2d, 0xfd, 0x11, 0xc8, 0x41, 0x86, 0x81, 0xc0, 0xf7, 0x60, - 0xdd, 0x29, 0x83, 0x36, 0x6e, 0x64, 0xd0, 0x3b, 0xd0, 0x2b, 0xb9, 0xd6, 0xb9, 0xc8, 0x88, 0xa7, - 0x93, 0xe8, 0x43, 0x45, 0x2c, 0xb9, 0x36, 0x11, 0xff, 0x71, 0x80, 0x1c, 0x62, 0x8c, 0xef, 0x15, - 0xb2, 0xe6, 0x69, 0xa9, 0x50, 0x5e, 0xb3, 0x4a, 0x79, 0x8f, 0xe0, 0x96, 0x34, 0x51, 0xd1, 0x86, - 0x51, 0x20, 0x02, 0xc3, 0xdb, 0x5d, 0xc6, 0x75, 0x0a, 0x87, 0x81, 0x08, 0x8c, 0xa3, 0x0c, 0xc3, - 0x3c, 0x93, 0x54, 0xae, 0x86, 0x4d, 0x39, 0xf2, 0x0b, 0x91, 0xc4, 0x52, 0xca, 0xd9, 0x60, 0xf9, - 0xdb, 0x81, 0xde, 0x33, 0xce, 0xd9, 0x79, 0xf2, 0x43, 0x1a, 0xe7, 0x63, 0x2c, 0xc0, 0x6c, 0x40, - 0x2b, 0x4c, 0x73, 0x73, 0xbf, 0x2d, 0x5f, 0x6f, 0xe6, 0x66, 0xad, 0x51, 0x99, 0xb5, 0xb9, 0x69, - 0x6d, 0x56, 0xa7, 0xd5, 0x9a, 0xc6, 0x65, 0x7b, 0x1a, 0xc9, 0x27, 0xd0, 0x91, 0xf0, 0x86, 0x21, - 0x26, 0x02, 0x33, 0x85, 0xc0, 0xf5, 0x41, 0x8a, 0x0e, 0x94, 0x84, 0x5e, 0xc2, 0x46, 0x39, 0x51, - 0xd3, 0x8b, 0x0b, 0xf9, 0x52, 0x52, 0x51, 0x16, 0x9b, 0x2c, 0xe5, 0x52, 0x0d, 0x70, 0x7e, 0x16, - 0xb3, 0x70, 0x28, 0x15, 0x4d, 0x33, 0xc0, 0x4a, 0x72, 0x9a, 0xc5, 0x33, 0xcc, 0xcb, 0x16, 0x66, - 0xfa, 0x25, 0xf4, 0xf4, 0xfb, 0x5f, 0x2e, 0xd0, 0x16, 0xc0, 0xa5, 0x12, 0x0c, 0x59, 0xa4, 0x9f, - 0x3e, 0xd7, 0x77, 0xb5, 0xe4, 0x65, 0xc4, 0xe9, 0xb7, 0xe0, 0x1e, 0xa7, 0x1a, 0x33, 0x27, 0x7b, - 0xe0, 0xc6, 0xc5, 0xc6, 0xbc, 0x92, 0x64, 0xd6, 0x72, 0x85, 0x9d, 0x3f, 0x33, 0xa2, 0xdf, 0xc0, - 0x6a, 0x21, 0x2e, 0x70, 0x38, 0x8b, 0x70, 0x34, 0xe6, 0x70, 0xd0, 0xff, 0x1c, 0xd8, 0x28, 0xa7, - 0x6c, 0x4a, 0x75, 0x0a, 0x6b, 0xd3, 0x10, 0xc3, 0x71, 0x30, 0x31, 0xb9, 0xec, 0xd9, 0xb9, 0x54, - 0x8f, 0x4d, 0x13, 0xe4, 0xaf, 0x82, 0x89, 0xee, 0x9e, 0x6e, 0x6c, 0x89, 0xfa, 0xdf, 0xc3, 0x7a, - 0xc5, 0x44, 0x66, 0x7d, 0x81, 0xc5, 0x1c, 0xc8, 0x25, 0x79, 0x02, 0xad, 0xcb, 0x20, 0xce, 0xd1, - 0x0c, 0x5d, 0xaf, 0x5a, 0x01, 0xee, 0x6b, 0x8b, 0xaf, 0x1b, 0x4f, 0x1d, 0xfa, 0x15, 0xdc, 0xd3, - 0x0d, 0x7b, 0x30, 0xed, 0xaf, 0xa2, 0xf6, 0xe5, 0x36, 0x74, 0xe6, 0xdb, 0x90, 0xf6, 0xc1, 0xab, - 0x1e, 0xd5, 0x60, 0xf6, 0xff, 0x6c, 0x43, 0xf7, 0x04, 0x83, 0x2b, 0xc4, 0x48, 0x32, 0x5b, 0x46, - 0xce, 0x8b, 0x62, 0x95, 0xbf, 0xef, 0xc8, 0xe3, 0xf9, 0xaa, 0xd4, 0x7e, 0x50, 0xf6, 0x3f, 0x7b, - 0x9b, 0x99, 0x19, 0xb4, 0x25, 0x72, 0x0c, 0x1d, 0xeb, 0x03, 0x8a, 0x6c, 0x5a, 0x07, 0x2b, 0xdf, - 0x85, 0xfd, 0xad, 0x05, 0xda, 0xa9, 0xb7, 0x00, 0x48, 0xf5, 0x4d, 0x21, 0x9f, 0xce, 0x8e, 0x2d, - 0x7c, 0xdb, 0xfa, 0x8f, 0x6e, 0x36, 0xb2, 0x13, 0xb6, 0x08, 0xd7, 0x4e, 0xb8, 0x4a, 0xf1, 0x76, - 0xc2, 0x75, 0x2c, 0xad, 0xbc, 0x59, 0x64, 0x6a, 0x7b, 0xab, 0xd2, 0xb7, 0xed, 0xad, 0x8e, 0x81, - 0x95, 0x37, 0x8b, 0xce, 0x6c, 0x6f, 0x55, 0x66, 0xb6, 0xbd, 0xd5, 0x71, 0xe0, 0x12, 0x79, 0x03, - 0x5d, 0x9b, 0x5b, 0x88, 0x75, 0xa0, 0x86, 0x1c, 0xfb, 0x0f, 0x16, 0xa9, 0x6d, 0x87, 0xf6, 0x28, - 0xd9, 0x0e, 0x6b, 0xc8, 0xc4, 0x76, 0x58, 0x37, 0x81, 0x74, 0x89, 0xfc, 0x04, 0xb7, 0xe7, 0x5b, - 0x9a, 0x3c, 0x9c, 0x87, 0x55, 0x99, 0x94, 0x3e, 0xbd, 0xc9, 0xa4, 0x70, 0x7e, 0xd6, 0x56, 0x3f, - 0x51, 0x5f, 0xfc, 0x1f, 0x00, 0x00, 0xff, 0xff, 0x81, 0x71, 0x63, 0x3e, 0x53, 0x0d, 0x00, 0x00, + 0x10, 0x36, 0x49, 0x4b, 0x16, 0x47, 0x72, 0x1a, 0xaf, 0x9d, 0x84, 0x55, 0xec, 0x54, 0x61, 0x93, + 0xc2, 0x41, 0x0d, 0xc3, 0x70, 0x7b, 0x48, 0x1a, 0x14, 0x68, 0x60, 0x3b, 0x41, 0x00, 0x27, 0x01, + 0xe8, 0xba, 0x40, 0xd1, 0x83, 0x40, 0x93, 0x23, 0x77, 0x61, 0x8a, 0x54, 0xc9, 0xa5, 0x1c, 0xf7, + 0xda, 0x63, 0x4f, 0x7d, 0x80, 0x9e, 0x7b, 0xef, 0x03, 0xf4, 0xd2, 0x17, 0x2b, 0xf6, 0x87, 0xd4, + 0x52, 0xa4, 0x9c, 0xfa, 0x90, 0xdb, 0xee, 0xcc, 0xec, 0xcc, 0x7c, 0xb3, 0x33, 0xdf, 0x92, 0xd0, + 0x1d, 0xd1, 0x08, 0xd3, 0xdd, 0x49, 0x9a, 0xb0, 0x84, 0x74, 0xc4, 0x66, 0x38, 0x39, 0x73, 0xdf, + 0xc1, 0xfd, 0xe3, 0x24, 0xb9, 0xc8, 0x27, 0x87, 0x34, 0xc5, 0x80, 0x25, 0xe9, 0xd5, 0x51, 0xcc, + 0xd2, 0x2b, 0x0f, 0x7f, 0xc9, 0x31, 0x63, 0x64, 0x13, 0xec, 0xb0, 0x50, 0x38, 0xc6, 0xc0, 0xd8, + 0xb6, 0xbd, 0x99, 0x80, 0x10, 0x58, 0x8e, 0xfd, 0x31, 0x3a, 0xa6, 0x50, 0x88, 0xb5, 0x7b, 0x04, + 0x9b, 0xcd, 0x0e, 0xb3, 0x49, 0x12, 0x67, 0x48, 0x1e, 0x43, 0x0b, 0xb9, 0x40, 0x78, 0xeb, 0xee, + 0x7f, 0xb2, 0x5b, 0xa4, 0xb2, 0x2b, 0xed, 0xa4, 0xd6, 0xfd, 0xc7, 0x00, 0x72, 0x4c, 0x33, 0xc6, + 0x85, 0x14, 0xb3, 0xff, 0x97, 0xcf, 0x5d, 0x68, 0x4f, 0x52, 0x1c, 0xd1, 0xf7, 0x2a, 0x23, 0xb5, + 0x23, 0x3b, 0xb0, 0x96, 0x31, 0x3f, 0x65, 0x2f, 0xd3, 0x64, 0xfc, 0x92, 0x46, 0xf8, 0x96, 0x27, + 0x6d, 0x09, 0x93, 0xba, 0x82, 0xec, 0x02, 0xa1, 0x71, 0x10, 0xe5, 0x19, 0x9d, 0xe2, 0x49, 0xa1, + 0x75, 0x96, 0x07, 0xc6, 0x76, 0xc7, 0x6b, 0xd0, 0x90, 0x0d, 0x68, 0x45, 0x74, 0x4c, 0x99, 0xd3, + 0x1a, 0x18, 0xdb, 0xab, 0x9e, 0xdc, 0xb8, 0xdf, 0xc1, 0x7a, 0x25, 0x7f, 0x05, 0xff, 0x09, 0xac, + 0xa0, 0x14, 0x39, 0xc6, 0xc0, 0x6a, 0x2a, 0x40, 0xa1, 0x77, 0xff, 0x34, 0xa1, 0x25, 0x44, 0x65, + 0x9d, 0x8d, 0x59, 0x9d, 0xc9, 0x43, 0xe8, 0xd1, 0x6c, 0x38, 0x2b, 0x86, 0x29, 0xf2, 0xeb, 0xd2, + 0xac, 0xac, 0x3b, 0xf9, 0x12, 0xda, 0xc1, 0xcf, 0x79, 0x7c, 0x91, 0x39, 0x96, 0x08, 0xb5, 0x3e, + 0x0b, 0xc5, 0xc1, 0x1e, 0x70, 0x9d, 0xa7, 0x4c, 0xc8, 0x53, 0x00, 0x9f, 0xb1, 0x94, 0x9e, 0xe5, + 0x0c, 0x33, 0x81, 0xb6, 0xbb, 0xef, 0x68, 0x07, 0xf2, 0x0c, 0x5f, 0x94, 0x7a, 0x4f, 0xb3, 0x25, + 0xcf, 0xa0, 0x83, 0xef, 0x19, 0xc6, 0x21, 0x86, 0x4e, 0x4b, 0x04, 0xda, 0x9a, 0xc3, 0xb4, 0x7b, + 0xa4, 0xf4, 0x12, 0x61, 0x69, 0xde, 0x7f, 0x0e, 0xab, 0x15, 0x15, 0xb9, 0x0d, 0xd6, 0x05, 0x16, + 0x37, 0xcb, 0x97, 0xbc, 0xba, 0x53, 0x3f, 0xca, 0x8b, 0x26, 0x93, 0x9b, 0x6f, 0xcc, 0xa7, 0x86, + 0x9b, 0xc0, 0xda, 0xd1, 0x14, 0x63, 0xf6, 0x36, 0x61, 0x74, 0x44, 0x03, 0x9f, 0xd1, 0x24, 0x26, + 0x3b, 0x60, 0x27, 0x51, 0x38, 0x94, 0x2d, 0x66, 0x36, 0xb7, 0x58, 0x27, 0x89, 0x54, 0xb8, 0x1d, + 0xb0, 0x63, 0xbc, 0x54, 0xd6, 0xd6, 0x02, 0xeb, 0x18, 0x2f, 0xc5, 0xca, 0x1d, 0x81, 0x5d, 0xd6, + 0x8d, 0xdc, 0x83, 0x15, 0x6e, 0x38, 0xa4, 0xa1, 0xca, 0xb6, 0xcd, 0xb7, 0xaf, 0x43, 0xde, 0x84, + 0xc9, 0x68, 0x94, 0x21, 0x13, 0xe1, 0x2d, 0x4f, 0xed, 0xf8, 0x25, 0x66, 0xf4, 0x57, 0xd9, 0x77, + 0xcb, 0x9e, 0x58, 0x73, 0x70, 0x63, 0x46, 0xc7, 0x28, 0xea, 0x6d, 0x79, 0x72, 0xe3, 0xfe, 0x6e, + 0xc2, 0xad, 0x6a, 0xbd, 0xc9, 0x7d, 0xb0, 0x45, 0x34, 0xe1, 0xc1, 0x10, 0x1e, 0xc4, 0x0c, 0x9f, + 0x54, 0xbc, 0x98, 0x9a, 0x97, 0xf2, 0xc8, 0x38, 0x09, 0x65, 0xd0, 0x55, 0x79, 0xe4, 0x4d, 0x12, + 0x22, 0xaf, 0x73, 0x4e, 0x43, 0x11, 0x76, 0xd5, 0xe3, 0x4b, 0x2e, 0x39, 0xa7, 0xa1, 0xea, 0x61, + 0xbe, 0xe4, 0x40, 0x82, 0x54, 0xf8, 0x6d, 0x4b, 0x20, 0x72, 0xc7, 0x81, 0x8c, 0xb9, 0x74, 0x45, + 0x76, 0x23, 0x5f, 0x93, 0x01, 0x74, 0x53, 0x9c, 0x44, 0xea, 0x16, 0x9c, 0x8e, 0x50, 0xe9, 0x22, + 0xf2, 0x00, 0x20, 0x48, 0xa2, 0x08, 0x03, 0x61, 0x60, 0x0b, 0x03, 0x4d, 0xc2, 0xeb, 0xc9, 0x58, + 0x34, 0xcc, 0x30, 0x70, 0x60, 0x60, 0x6c, 0xb7, 0xbc, 0x36, 0x63, 0xd1, 0x09, 0x06, 0xee, 0x39, + 0x7c, 0xfa, 0x0a, 0xc5, 0x1c, 0x5d, 0x69, 0x0d, 0xa8, 0xf8, 0xa0, 0x69, 0x32, 0xb6, 0x00, 0x26, + 0x7e, 0x8a, 0x31, 0xe3, 0xd3, 0xa1, 0xda, 0xc6, 0x96, 0x92, 0x43, 0x9a, 0xea, 0x17, 0x67, 0xe9, + 0x17, 0xe7, 0xfe, 0x66, 0x40, 0xbf, 0x29, 0x92, 0x9a, 0xdc, 0xea, 0x80, 0x18, 0x37, 0x18, 0x90, + 0xd9, 0x1c, 0x9a, 0x1f, 0x9c, 0x43, 0x77, 0x0f, 0xee, 0xbc, 0x42, 0x26, 0xe4, 0x49, 0xcc, 0x30, + 0x66, 0x05, 0xd4, 0x45, 0x0d, 0xe7, 0xee, 0xc3, 0xdd, 0xf9, 0x13, 0x2a, 0x65, 0x07, 0x56, 0x02, + 0x29, 0x12, 0x47, 0x7a, 0x5e, 0xb1, 0x75, 0x7f, 0x04, 0x72, 0x90, 0xa2, 0xcf, 0xf0, 0x06, 0x6c, + 0x5f, 0x32, 0xb7, 0x79, 0x2d, 0x73, 0xdf, 0x81, 0xf5, 0x8a, 0x6b, 0x99, 0x0b, 0x8f, 0x78, 0x3a, + 0x09, 0x3f, 0x56, 0xc4, 0x8a, 0x6b, 0x15, 0xf1, 0x6f, 0x03, 0xc8, 0x21, 0x46, 0x78, 0xa3, 0x90, + 0x0d, 0x4f, 0x5a, 0x8d, 0x6a, 0xad, 0x3a, 0xd5, 0x3e, 0x82, 0x5b, 0xdc, 0x44, 0x44, 0x1b, 0x86, + 0x3e, 0xf3, 0xd5, 0x7b, 0xd1, 0xa3, 0x99, 0x4c, 0xe1, 0xd0, 0x67, 0xbe, 0x72, 0x94, 0x62, 0x90, + 0xa7, 0xfc, 0x09, 0x11, 0xc3, 0x26, 0x1c, 0x79, 0x85, 0x88, 0x63, 0xa9, 0xe4, 0xac, 0xb0, 0xfc, + 0x65, 0xc0, 0xfa, 0x8b, 0x2c, 0xa3, 0xe7, 0xf1, 0x0f, 0x49, 0x94, 0x8f, 0xb1, 0x00, 0xb3, 0x01, + 0xad, 0x20, 0xc9, 0xd5, 0xfd, 0xb6, 0x3c, 0xb9, 0x99, 0x9b, 0x35, 0xb3, 0x36, 0x6b, 0x73, 0xd3, + 0x6a, 0xd5, 0xa7, 0x55, 0x9b, 0xc6, 0x65, 0x7d, 0x1a, 0xc9, 0x67, 0xd0, 0xe5, 0xf0, 0x86, 0x01, + 0xc6, 0x0c, 0x53, 0x81, 0xc0, 0xf6, 0x80, 0x8b, 0x0e, 0x84, 0xc4, 0x9d, 0xc2, 0x46, 0x35, 0x51, + 0xd5, 0x8b, 0x0b, 0xf9, 0x92, 0x53, 0x51, 0x1a, 0xa9, 0x2c, 0xf9, 0x52, 0x0c, 0x70, 0x7e, 0x16, + 0xd1, 0x60, 0xc8, 0x15, 0x96, 0x1a, 0x60, 0x21, 0x39, 0x4d, 0xa3, 0x19, 0xe6, 0x65, 0x0d, 0xb3, + 0xfb, 0x35, 0xac, 0xcb, 0xef, 0x8e, 0x6a, 0x81, 0xb6, 0x00, 0xa6, 0x42, 0x30, 0xa4, 0xa1, 0x7c, + 0x72, 0x6d, 0xcf, 0x96, 0x92, 0xd7, 0x61, 0xe6, 0x7e, 0x0b, 0xf6, 0x71, 0x22, 0x31, 0x67, 0x64, + 0x0f, 0xec, 0xa8, 0xd8, 0xa8, 0xd7, 0x99, 0xcc, 0x5a, 0xae, 0xb0, 0xf3, 0x66, 0x46, 0xee, 0x73, + 0xe8, 0x14, 0xe2, 0x02, 0x87, 0xb1, 0x08, 0x87, 0x39, 0x87, 0xc3, 0xfd, 0xd7, 0x80, 0x8d, 0x6a, + 0xca, 0xaa, 0x54, 0xa7, 0xb0, 0x5a, 0x86, 0x18, 0x8e, 0xfd, 0x89, 0xca, 0x65, 0x4f, 0xcf, 0xa5, + 0x7e, 0xac, 0x4c, 0x30, 0x7b, 0xe3, 0x4f, 0x64, 0xf7, 0xf4, 0x22, 0x4d, 0xd4, 0xff, 0x1e, 0xd6, + 0x6a, 0x26, 0x0d, 0x0f, 0xee, 0x13, 0xfd, 0xc1, 0xad, 0x90, 0x55, 0x79, 0x5a, 0x7f, 0x85, 0x9f, + 0xc1, 0x3d, 0xd9, 0xb0, 0x07, 0x65, 0x7f, 0x15, 0xb5, 0xaf, 0xb6, 0xa1, 0x31, 0xdf, 0x86, 0x6e, + 0x1f, 0x9c, 0xfa, 0x51, 0x09, 0x66, 0xff, 0x8f, 0x36, 0xf4, 0x4e, 0xd0, 0xbf, 0x44, 0x0c, 0x39, + 0xb3, 0xa5, 0xe4, 0xbc, 0x28, 0x56, 0xf5, 0xbb, 0x92, 0x3c, 0x9e, 0xaf, 0x4a, 0xe3, 0x87, 0x6c, + 0xff, 0x8b, 0x0f, 0x99, 0xa9, 0x41, 0x5b, 0x22, 0xc7, 0xd0, 0xd5, 0x3e, 0xdc, 0xc8, 0xa6, 0x76, + 0xb0, 0xf6, 0x3d, 0xda, 0xdf, 0x5a, 0xa0, 0x2d, 0xbd, 0xf9, 0x40, 0xea, 0x6f, 0x0a, 0xf9, 0x7c, + 0x76, 0x6c, 0xe1, 0xdb, 0xd6, 0x7f, 0x74, 0xbd, 0x91, 0x9e, 0xb0, 0x46, 0xb8, 0x7a, 0xc2, 0x75, + 0x8a, 0xd7, 0x13, 0x6e, 0x62, 0x69, 0xe1, 0x4d, 0x23, 0x53, 0xdd, 0x5b, 0x9d, 0xbe, 0x75, 0x6f, + 0x4d, 0x0c, 0x2c, 0xbc, 0x69, 0x74, 0xa6, 0x7b, 0xab, 0x33, 0xb3, 0xee, 0xad, 0x89, 0x03, 0x97, + 0xc8, 0x3b, 0xe8, 0xe9, 0xdc, 0x42, 0xb4, 0x03, 0x0d, 0xe4, 0xd8, 0x7f, 0xb0, 0x48, 0xad, 0x3b, + 0xd4, 0x47, 0x49, 0x77, 0xd8, 0x40, 0x26, 0xba, 0xc3, 0xa6, 0x09, 0x74, 0x97, 0xc8, 0x4f, 0x70, + 0x7b, 0xbe, 0xa5, 0xc9, 0xc3, 0x79, 0x58, 0xb5, 0x49, 0xe9, 0xbb, 0xd7, 0x99, 0x14, 0xce, 0xcf, + 0xda, 0xe2, 0xe7, 0xed, 0xab, 0xff, 0x02, 0x00, 0x00, 0xff, 0xff, 0x8a, 0x18, 0xac, 0xec, 0xcb, + 0x0d, 0x00, 0x00, } diff --git a/weed/s3api/filer_multipart.go b/weed/s3api/filer_multipart.go new file mode 100644 index 000000000..d85a94326 --- /dev/null +++ b/weed/s3api/filer_multipart.go @@ -0,0 +1,70 @@ +package s3api + +import ( + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/aws" + "github.com/satori/go.uuid" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" +) + +func (s3a *S3ApiServer) createMultipartUpload(input *s3.CreateMultipartUploadInput) (output *s3.CreateMultipartUploadOutput, code ErrorCode) { + uploadId, _ := uuid.NewV4() + uploadIdString := uploadId.String() + + if err := s3a.mkdir(s3a.genUploadsFolder(*input.Bucket), uploadIdString, func(entry *filer_pb.Entry) { + if entry.Extended == nil { + entry.Extended = make(map[string]string) + } + entry.Extended["key"] = *input.Key + }); err != nil { + glog.Errorf("NewMultipartUpload error: %v", err) + return nil, ErrInternalError + } + + output = &s3.CreateMultipartUploadOutput{ + Bucket: input.Bucket, + Key: input.Key, + UploadId: aws.String(uploadIdString), + } + + return +} + +func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploadInput) (output *s3.CompleteMultipartUploadOutput, code ErrorCode) { + return +} + +func (s3a *S3ApiServer) abortMultipartUpload(input *s3.AbortMultipartUploadInput) (output *s3.AbortMultipartUploadOutput, code ErrorCode) { + return +} + +func (s3a *S3ApiServer) listMultipartUploads(input *s3.ListMultipartUploadsInput) (output *s3.ListMultipartUploadsOutput, code ErrorCode) { + entries, err := s3a.list(s3a.genUploadsFolder(*input.Bucket)) + if err != nil { + glog.Errorf("listMultipartUploads %s error: %v", *input.Bucket, err) + return nil, ErrNoSuchUpload + } + output = &s3.ListMultipartUploadsOutput{ + Bucket: input.Bucket, + Delimiter: input.Delimiter, + EncodingType: input.EncodingType, + KeyMarker: input.KeyMarker, + MaxUploads: input.MaxUploads, + Prefix: input.Prefix, + } + for _, entry := range entries { + if entry.Extended != nil { + key := entry.Extended["key"] + output.Uploads = append(output.Uploads, &s3.MultipartUpload{ + Key: aws.String(key), + UploadId: aws.String(entry.Name), + }) + } + } + return +} + +func (s3a *S3ApiServer) listObjectParts(input *s3.ListPartsInput) (output *s3.ListPartsOutput, code ErrorCode) { + return +} diff --git a/weed/s3api/filer_util.go b/weed/s3api/filer_util.go index be807fb82..a44305505 100644 --- a/weed/s3api/filer_util.go +++ b/weed/s3api/filer_util.go @@ -10,22 +10,28 @@ import ( "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" ) -func (s3a *S3ApiServer) mkdir(parentDirectoryPath string, dirName string) error { +func (s3a *S3ApiServer) mkdir(parentDirectoryPath string, dirName string, fn func(entry *filer_pb.Entry)) error { return s3a.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { + entry := &filer_pb.Entry{ + Name: dirName, + IsDirectory: true, + Attributes: &filer_pb.FuseAttributes{ + Mtime: time.Now().Unix(), + Crtime: time.Now().Unix(), + FileMode: uint32(0777 | os.ModeDir), + Uid: OS_UID, + Gid: OS_GID, + }, + } + + if fn != nil { + fn(entry) + } + request := &filer_pb.CreateEntryRequest{ Directory: parentDirectoryPath, - Entry: &filer_pb.Entry{ - Name: dirName, - IsDirectory: true, - Attributes: &filer_pb.FuseAttributes{ - Mtime: time.Now().Unix(), - Crtime: time.Now().Unix(), - FileMode: uint32(0777 | os.ModeDir), - Uid: OS_UID, - Gid: OS_GID, - }, - }, + Entry: entry, } glog.V(1).Infof("create bucket: %v", request) @@ -83,3 +89,28 @@ func (s3a *S3ApiServer) rm(parentDirectoryPath string, entryName string, isDirec }) } + +func (s3a *S3ApiServer) exists(parentDirectoryPath string, entryName string, isDirectory bool) (exists bool, err error) { + + err = s3a.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { + + ctx := context.Background() + + request := &filer_pb.LookupDirectoryEntryRequest{ + Directory: parentDirectoryPath, + Name: entryName, + } + + glog.V(1).Infof("delete entry %v/%v: %v", parentDirectoryPath, entryName, request) + resp, err := client.LookupDirectoryEntry(ctx, request) + if err != nil { + return fmt.Errorf("delete entry %s/%s: %v", parentDirectoryPath, entryName, err) + } + + exists = resp.Entry.IsDirectory == isDirectory + + return nil + }) + + return +} diff --git a/weed/s3api/s3api_errors.go b/weed/s3api/s3api_errors.go index e5ce8df5a..e9975dbb6 100644 --- a/weed/s3api/s3api_errors.go +++ b/weed/s3api/s3api_errors.go @@ -32,10 +32,17 @@ const ( ErrBucketAlreadyExists ErrBucketAlreadyOwnedByYou ErrNoSuchBucket + ErrNoSuchUpload ErrInvalidBucketName ErrInvalidDigest ErrInvalidMaxKeys + ErrInvalidMaxUploads + ErrInvalidMaxParts + ErrInvalidPartNumberMarker + ErrInvalidPart + ErrInvalidPartOrder ErrInternalError + ErrMalformedXML ErrNotImplemented ) @@ -72,21 +79,56 @@ var errorCodeResponse = map[ErrorCode]APIError{ Description: "The Content-Md5 you specified is not valid.", HTTPStatusCode: http.StatusBadRequest, }, + ErrInvalidMaxUploads: { + Code: "InvalidArgument", + Description: "Argument max-uploads must be an integer between 0 and 2147483647", + HTTPStatusCode: http.StatusBadRequest, + }, ErrInvalidMaxKeys: { Code: "InvalidArgument", Description: "Argument maxKeys must be an integer between 0 and 2147483647", HTTPStatusCode: http.StatusBadRequest, }, + ErrInvalidMaxParts: { + Code: "InvalidArgument", + Description: "Argument max-parts must be an integer between 0 and 2147483647", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidPartNumberMarker: { + Code: "InvalidArgument", + Description: "Argument partNumberMarker must be an integer.", + HTTPStatusCode: http.StatusBadRequest, + }, ErrNoSuchBucket: { Code: "NoSuchBucket", Description: "The specified bucket does not exist", HTTPStatusCode: http.StatusNotFound, }, + ErrNoSuchUpload: { + Code: "NoSuchUpload", + Description: "The specified multipart upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed.", + HTTPStatusCode: http.StatusNotFound, + }, ErrInternalError: { Code: "InternalError", Description: "We encountered an internal error, please try again.", HTTPStatusCode: http.StatusInternalServerError, }, + ErrMalformedXML: { + Code: "MalformedXML", + Description: "The XML you provided was not well-formed or did not validate against our published schema.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidPart: { + Code: "InvalidPart", + Description: "One or more of the specified parts could not be found. The part may not have been uploaded, or the specified entity tag may not match the part's entity tag.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidPartOrder: { + Code: "InvalidPartOrder", + Description: "The list of parts was not in ascending order. The parts list must be specified in order by part number.", + HTTPStatusCode: http.StatusBadRequest, + }, ErrNotImplemented: { Code: "NotImplemented", Description: "A header you provided implies functionality that is not implemented", diff --git a/weed/s3api/s3api_object_handlers.go b/weed/s3api/s3api_object_handlers.go index 912b76230..83a9186a9 100644 --- a/weed/s3api/s3api_object_handlers.go +++ b/weed/s3api/s3api_object_handlers.go @@ -49,50 +49,15 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request) uploadUrl := fmt.Sprintf("http://%s%s/%s/%s?collection=%s", s3a.option.Filer, s3a.option.BucketsPath, bucket, object, bucket) - proxyReq, err := http.NewRequest("PUT", uploadUrl, dataReader) - if err != nil { - glog.Errorf("NewRequest %s: %v", uploadUrl, err) - writeErrorResponse(w, ErrInternalError, r.URL) - return - } + etag, errCode := s3a.putToFiler(r, uploadUrl, dataReader) - proxyReq.Header.Set("Host", s3a.option.Filer) - proxyReq.Header.Set("X-Forwarded-For", r.RemoteAddr) - - for header, values := range r.Header { - for _, value := range values { - proxyReq.Header.Add(header, value) - } - } - - resp, postErr := client.Do(proxyReq) - - if postErr != nil { - glog.Errorf("post to filer: %v", postErr) - writeErrorResponse(w, ErrInternalError, r.URL) + if errCode != ErrNone { + writeErrorResponse(w, errCode, r.URL) return } - defer resp.Body.Close() - resp_body, ra_err := ioutil.ReadAll(resp.Body) - if ra_err != nil { - glog.Errorf("upload to filer response read: %v", ra_err) - writeErrorResponse(w, ErrInternalError, r.URL) - return - } - var ret UploadResult - unmarshal_err := json.Unmarshal(resp_body, &ret) - if unmarshal_err != nil { - glog.Errorf("failing to read upload to %s : %v", uploadUrl, string(resp_body)) - writeErrorResponse(w, ErrInternalError, r.URL) - return - } - if ret.Error != "" { - glog.Errorf("upload to filer error: %v", ret.Error) - writeErrorResponse(w, ErrInternalError, r.URL) - return - } + setEtag(w, etag) writeSuccessResponseEmpty(w) } @@ -134,6 +99,12 @@ func (s3a *S3ApiServer) DeleteObjectHandler(w http.ResponseWriter, r *http.Reque } +// DeleteMultipleObjectsHandler - Delete multiple objects +func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) { + // TODO + writeErrorResponse(w, ErrNotImplemented, r.URL) +} + func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, destUrl string, responseFn func(proxyResonse *http.Response, w http.ResponseWriter)) { glog.V(2).Infof("s3 proxying %s to %s", r.Method, destUrl) @@ -173,3 +144,56 @@ func passThroghResponse(proxyResonse *http.Response, w http.ResponseWriter) { w.WriteHeader(proxyResonse.StatusCode) io.Copy(w, proxyResonse.Body) } + +func (s3a *S3ApiServer) putToFiler(r *http.Request, uploadUrl string, dataReader io.ReadCloser) (etag string, code ErrorCode) { + + proxyReq, err := http.NewRequest("PUT", uploadUrl, dataReader) + + if err != nil { + glog.Errorf("NewRequest %s: %v", uploadUrl, err) + return "", ErrInternalError + } + + proxyReq.Header.Set("Host", s3a.option.Filer) + proxyReq.Header.Set("X-Forwarded-For", r.RemoteAddr) + + for header, values := range r.Header { + for _, value := range values { + proxyReq.Header.Add(header, value) + } + } + + resp, postErr := client.Do(proxyReq) + + if postErr != nil { + glog.Errorf("post to filer: %v", postErr) + return "", ErrInternalError + } + defer resp.Body.Close() + + etag = resp.Header.Get("ETag") + + resp_body, ra_err := ioutil.ReadAll(resp.Body) + if ra_err != nil { + glog.Errorf("upload to filer response read: %v", ra_err) + return etag, ErrInternalError + } + var ret UploadResult + unmarshal_err := json.Unmarshal(resp_body, &ret) + if unmarshal_err != nil { + glog.Errorf("failing to read upload to %s : %v", uploadUrl, string(resp_body)) + return etag, ErrInternalError + } + if ret.Error != "" { + glog.Errorf("upload to filer error: %v", ret.Error) + return etag, ErrInternalError + } + + return etag, ErrNone +} + +func setEtag(w http.ResponseWriter, etag string) { + if etag != "" { + w.Header().Set("ETag", "\""+etag+"\"") + } +} diff --git a/weed/s3api/s3api_object_multipart_handlers.go b/weed/s3api/s3api_object_multipart_handlers.go index 6e2c96ddd..62b702a39 100644 --- a/weed/s3api/s3api_object_multipart_handlers.go +++ b/weed/s3api/s3api_object_multipart_handlers.go @@ -2,39 +2,263 @@ package s3api import ( "net/http" + "github.com/gorilla/mux" + "fmt" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/aws" + "net/url" + "strconv" + "io/ioutil" + "encoding/xml" + "sort" + "strings" +) + +const ( + maxObjectList = 1000 // Limit number of objects in a listObjectsResponse. + maxUploadsList = 1000 // Limit number of uploads in a listUploadsResponse. + maxPartsList = 1000 // Limit number of parts in a listPartsResponse. + globalMaxPartID = 10000 ) // NewMultipartUploadHandler - New multipart upload. -func (api *S3ApiServer) NewMultipartUploadHandler(w http.ResponseWriter, r *http.Request) { +func (s3a *S3ApiServer) NewMultipartUploadHandler(w http.ResponseWriter, r *http.Request) { + var object, bucket string + vars := mux.Vars(r) + bucket = vars["bucket"] + object = vars["object"] + + response, errCode := s3a.createMultipartUpload(&s3.CreateMultipartUploadInput{ + Bucket: aws.String(bucket), + Key: aws.String(object), + }) + + if errCode != ErrNone { + writeErrorResponse(w, errCode, r.URL) + return + } + + writeSuccessResponseXML(w, encodeResponse(response)) } // CompleteMultipartUploadHandler - Completes multipart upload. -func (api *S3ApiServer) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.Request) { +func (s3a *S3ApiServer) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + bucket := vars["bucket"] + object := vars["object"] + + // Get upload id. + uploadID, _, _, _ := getObjectResources(r.URL.Query()) + + completeMultipartBytes, err := ioutil.ReadAll(r.Body) + if err != nil { + writeErrorResponse(w, ErrInternalError, r.URL) + return + } + completedMultipartUpload := &s3.CompletedMultipartUpload{} + if err = xml.Unmarshal(completeMultipartBytes, completedMultipartUpload); err != nil { + writeErrorResponse(w, ErrMalformedXML, r.URL) + return + } + if len(completedMultipartUpload.Parts) == 0 { + writeErrorResponse(w, ErrMalformedXML, r.URL) + return + } + if !sort.IsSorted(byCompletedPartNumber(completedMultipartUpload.Parts)) { + writeErrorResponse(w, ErrInvalidPartOrder, r.URL) + return + } + + response, errCode := s3a.completeMultipartUpload(&s3.CompleteMultipartUploadInput{ + Bucket: aws.String(bucket), + Key: aws.String(object), + MultipartUpload: completedMultipartUpload, + UploadId: aws.String(uploadID), + }) + + if errCode != ErrNone { + writeErrorResponse(w, errCode, r.URL) + return + } + + writeSuccessResponseXML(w, encodeResponse(response)) } // AbortMultipartUploadHandler - Aborts multipart upload. -func (api *S3ApiServer) AbortMultipartUploadHandler(w http.ResponseWriter, r *http.Request) { +func (s3a *S3ApiServer) AbortMultipartUploadHandler(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + bucket := vars["bucket"] + object := vars["object"] + + // Get upload id. + uploadID, _, _, _ := getObjectResources(r.URL.Query()) + + response, errCode := s3a.abortMultipartUpload(&s3.AbortMultipartUploadInput{ + Bucket: aws.String(bucket), + Key: aws.String(object), + UploadId: aws.String(uploadID), + }) + + if errCode != ErrNone { + writeErrorResponse(w, errCode, r.URL) + return + } + + writeSuccessResponseXML(w, encodeResponse(response)) } // ListMultipartUploadsHandler - Lists multipart uploads. -func (api *S3ApiServer) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) { +func (s3a *S3ApiServer) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + bucket := vars["bucket"] + prefix, keyMarker, uploadIDMarker, delimiter, maxUploads, encodingType := getBucketMultipartResources(r.URL.Query()) + if maxUploads < 0 { + writeErrorResponse(w, ErrInvalidMaxUploads, r.URL) + return + } + if keyMarker != "" { + // Marker not common with prefix is not implemented. + if !strings.HasPrefix(keyMarker, prefix) { + writeErrorResponse(w, ErrNotImplemented, r.URL) + return + } + } + + response, errCode := s3a.listMultipartUploads(&s3.ListMultipartUploadsInput{ + Bucket: aws.String(bucket), + Delimiter: aws.String(delimiter), + EncodingType: aws.String(encodingType), + KeyMarker: aws.String(keyMarker), + MaxUploads: aws.Int64(int64(maxUploads)), + Prefix: aws.String(prefix), + UploadIdMarker: aws.String(uploadIDMarker), + }) + + if errCode != ErrNone { + writeErrorResponse(w, errCode, r.URL) + return + } + + writeSuccessResponseXML(w, encodeResponse(response)) } // ListObjectPartsHandler - Lists object parts in a multipart upload. -func (api *S3ApiServer) ListObjectPartsHandler(w http.ResponseWriter, r *http.Request) { +func (s3a *S3ApiServer) ListObjectPartsHandler(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + bucket := vars["bucket"] + object := vars["object"] + + uploadID, partNumberMarker, maxParts, _ := getObjectResources(r.URL.Query()) + if partNumberMarker < 0 { + writeErrorResponse(w, ErrInvalidPartNumberMarker, r.URL) + return + } + if maxParts < 0 { + writeErrorResponse(w, ErrInvalidMaxParts, r.URL) + return + } + + response, errCode := s3a.listObjectParts(&s3.ListPartsInput{ + Bucket: aws.String(bucket), + Key: aws.String(object), + MaxParts: aws.Int64(int64(maxParts)), + PartNumberMarker: aws.Int64(int64(partNumberMarker)), + UploadId: aws.String(uploadID), + }) + + if errCode != ErrNone { + writeErrorResponse(w, errCode, r.URL) + return + } + + writeSuccessResponseXML(w, encodeResponse(response)) } // PutObjectPartHandler - Put an object part in a multipart upload. -func (api *S3ApiServer) PutObjectPartHandler(w http.ResponseWriter, r *http.Request) { +func (s3a *S3ApiServer) PutObjectPartHandler(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + bucket := vars["bucket"] + + rAuthType := getRequestAuthType(r) + + uploadID := r.URL.Query().Get("uploadId") + exists, err := s3a.exists(s3a.genUploadsFolder(bucket), uploadID, true) + if !exists { + writeErrorResponse(w, ErrNoSuchUpload, r.URL) + return + } + + partIDString := r.URL.Query().Get("partNumber") + partID, err := strconv.Atoi(partIDString) + if err != nil { + writeErrorResponse(w, ErrInvalidPart, r.URL) + return + } + if partID > globalMaxPartID { + writeErrorResponse(w, ErrInvalidMaxParts, r.URL) + return + } + + dataReader := r.Body + if rAuthType == authTypeStreamingSigned { + dataReader = newSignV4ChunkedReader(r) + } + + uploadUrl := fmt.Sprintf("http://%s%s/%s/%04d.part", + s3a.option.Filer, s3a.genUploadsFolder(bucket), uploadID, partID-1) + + etag, errCode := s3a.putToFiler(r, uploadUrl, dataReader) + if errCode != ErrNone { + writeErrorResponse(w, errCode, r.URL) + return + } + + setEtag(w, etag) + + writeSuccessResponseEmpty(w) + +} + +func (s3a *S3ApiServer) genUploadsFolder(bucket string) string { + return fmt.Sprintf("%s/%s/_uploads", s3a.option.BucketsPath, bucket) } -// DeleteMultipleObjectsHandler - Delete an object part in a multipart upload. -func (api *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) { +// Parse bucket url queries for ?uploads +func getBucketMultipartResources(values url.Values) (prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int, encodingType string) { + prefix = values.Get("prefix") + keyMarker = values.Get("key-marker") + uploadIDMarker = values.Get("upload-id-marker") + delimiter = values.Get("delimiter") + if values.Get("max-uploads") != "" { + maxUploads, _ = strconv.Atoi(values.Get("max-uploads")) + } else { + maxUploads = maxUploadsList + } + encodingType = values.Get("encoding-type") + return +} +// Parse object url queries +func getObjectResources(values url.Values) (uploadID string, partNumberMarker, maxParts int, encodingType string) { + uploadID = values.Get("uploadId") + partNumberMarker, _ = strconv.Atoi(values.Get("part-number-marker")) + if values.Get("max-parts") != "" { + maxParts, _ = strconv.Atoi(values.Get("max-parts")) + } else { + maxParts = maxPartsList + } + encodingType = values.Get("encoding-type") + return } + +type byCompletedPartNumber []*s3.CompletedPart + +func (a byCompletedPartNumber) Len() int { return len(a) } +func (a byCompletedPartNumber) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a byCompletedPartNumber) Less(i, j int) bool { return *a[i].PartNumber < *a[j].PartNumber } diff --git a/weed/s3api/s3api_server.go b/weed/s3api/s3api_server.go index 2cc063098..efeeb34ce 100644 --- a/weed/s3api/s3api_server.go +++ b/weed/s3api/s3api_server.go @@ -37,7 +37,7 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) { apiRouter := router.PathPrefix("/").Subrouter() var routers []*mux.Router if s3a.option.DomainName != "" { - routers = append(routers, apiRouter.Host("{bucket:.+}."+s3a.option.DomainName).Subrouter()) + routers = append(routers, apiRouter.Host("{bucket:.+}."+ s3a.option.DomainName).Subrouter()) } routers = append(routers, apiRouter.PathPrefix("/{bucket}").Subrouter()) @@ -59,8 +59,6 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) { bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(s3a.CompleteMultipartUploadHandler).Queries("uploadId", "{uploadId:.*}") // NewMultipartUpload bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(s3a.NewMultipartUploadHandler).Queries("uploads", "") - // DeleteMultipleObjects - bucket.Methods("POST").HandlerFunc(s3a.DeleteMultipleObjectsHandler).Queries("delete", "") // AbortMultipartUpload bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(s3a.AbortMultipartUploadHandler).Queries("uploadId", "{uploadId:.*}") // ListObjectParts @@ -80,6 +78,8 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) { // ListObjectsV1 (Legacy) bucket.Methods("GET").HandlerFunc(s3a.ListObjectsV1Handler) + // DeleteMultipleObjects + bucket.Methods("POST").HandlerFunc(s3a.DeleteMultipleObjectsHandler).Queries("delete", "") /* // CopyObject bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(s3a.CopyObjectHandler)