Mirror
/
seaweedfs
mirror of https://github.com/seaweedfs/seaweedfs.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

some code optimizations

pull/3189/head
石昌林 3 years ago
parent
37df209195
commit
3dd60529c5
8 changed files with 257 additions and 129 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 6
      weed/pb/s3.proto
  2. 128
      weed/pb/s3_pb/s3.pb.go
  3. 4
      weed/s3api/auth_credentials_subscribe.go
  4. 5
      weed/s3api/s3_constants/s3_config.go
  5. 16
      weed/s3api/s3api_circuit_breaker.go
  6. 24
      weed/s3api/s3api_circuit_breaker_test.go
  7. 58
      weed/shell/command_s3_circuitbreaker.go
  8. 145
      weed/shell/command_s3_circuitbreaker_test.go

6
weed/pb/s3.proto
View File

@ -25,11 +25,11 @@ message S3ConfigureResponse {
}
message S3CircuitBreakerConfig {
CbOptions global=1;
map<string, CbOptions> buckets= 2;
S3CircuitBreakerOptions global=1;
map<string, S3CircuitBreakerOptions> buckets= 2;
}
message CbOptions {
message S3CircuitBreakerOptions {
bool enabled=1;
map<string, int64> actions = 2;
}

128
weed/pb/s3_pb/s3.pb.go
View File

@ -1,7 +1,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.26.0
// protoc v3.17.3
// protoc-gen-go v1.28.0
// protoc v3.21.1
// source: s3.proto
package s3_pb
@ -110,8 +110,8 @@ type S3CircuitBreakerConfig struct {
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Global *CbOptions `protobuf:"bytes,1,opt,name=global,proto3" json:"global,omitempty"`
Buckets map[string]*CbOptions `protobuf:"bytes,2,rep,name=buckets,proto3" json:"buckets,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
Global *S3CircuitBreakerOptions `protobuf:"bytes,1,opt,name=global,proto3" json:"global,omitempty"`
Buckets map[string]*S3CircuitBreakerOptions `protobuf:"bytes,2,rep,name=buckets,proto3" json:"buckets,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
}
func (x *S3CircuitBreakerConfig) Reset() {
@ -146,21 +146,21 @@ func (*S3CircuitBreakerConfig) Descriptor() ([]byte, []int) {
return file_s3_proto_rawDescGZIP(), []int{2}
}
func (x *S3CircuitBreakerConfig) GetGlobal() *CbOptions {
func (x *S3CircuitBreakerConfig) GetGlobal() *S3CircuitBreakerOptions {
if x != nil {
return x.Global
}
return nil
}
func (x *S3CircuitBreakerConfig) GetBuckets() map[string]*CbOptions {
func (x *S3CircuitBreakerConfig) GetBuckets() map[string]*S3CircuitBreakerOptions {
if x != nil {
return x.Buckets
}
return nil
}
type CbOptions struct {
type S3CircuitBreakerOptions struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
@ -169,8 +169,8 @@ type CbOptions struct {
Actions map[string]int64 `protobuf:"bytes,2,rep,name=actions,proto3" json:"actions,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"`
}
func (x *CbOptions) Reset() {
*x = CbOptions{}
func (x *S3CircuitBreakerOptions) Reset() {
*x = S3CircuitBreakerOptions{}
if protoimpl.UnsafeEnabled {
mi := &file_s3_proto_msgTypes[3]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
@ -178,13 +178,13 @@ func (x *CbOptions) Reset() {
}
}
func (x *CbOptions) String() string {
func (x *S3CircuitBreakerOptions) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*CbOptions) ProtoMessage() {}
func (*S3CircuitBreakerOptions) ProtoMessage() {}
func (x *CbOptions) ProtoReflect() protoreflect.Message {
func (x *S3CircuitBreakerOptions) ProtoReflect() protoreflect.Message {
mi := &file_s3_proto_msgTypes[3]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
@ -196,19 +196,19 @@ func (x *CbOptions) ProtoReflect() protoreflect.Message {
return mi.MessageOf(x)
}
// Deprecated: Use CbOptions.ProtoReflect.Descriptor instead.
func (*CbOptions) Descriptor() ([]byte, []int) {
// Deprecated: Use S3CircuitBreakerOptions.ProtoReflect.Descriptor instead.
func (*S3CircuitBreakerOptions) Descriptor() ([]byte, []int) {
return file_s3_proto_rawDescGZIP(), []int{3}
}
func (x *CbOptions) GetEnabled() bool {
func (x *S3CircuitBreakerOptions) GetEnabled() bool {
if x != nil {
return x.Enabled
}
return false
}
func (x *CbOptions) GetActions() map[string]int64 {
func (x *S3CircuitBreakerOptions) GetActions() map[string]int64 {
if x != nil {
return x.Actions
}
@ -226,43 +226,47 @@ var file_s3_proto_rawDesc = []byte{
0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x1a, 0x73, 0x33, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75,
0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x69, 0x6c, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e,
0x74, 0x22, 0x15, 0x0a, 0x13, 0x53, 0x33, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65,
0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0xeb, 0x01, 0x0a, 0x16, 0x53, 0x33, 0x43,
0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x87, 0x02, 0x0a, 0x16, 0x53, 0x33, 0x43,
0x69, 0x72, 0x63, 0x75, 0x69, 0x74, 0x42, 0x72, 0x65, 0x61, 0x6b, 0x65, 0x72, 0x43, 0x6f, 0x6e,
0x66, 0x69, 0x67, 0x12, 0x2f, 0x0a, 0x06, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x18, 0x01, 0x20,
0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f,
0x70, 0x62, 0x2e, 0x43, 0x62, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x06, 0x67, 0x6c,
0x6f, 0x62, 0x61, 0x6c, 0x12, 0x4b, 0x0a, 0x07, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x18,
0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e,
0x67, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x33, 0x43, 0x69, 0x72, 0x63, 0x75, 0x69, 0x74, 0x42, 0x72,
0x65, 0x61, 0x6b, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x42, 0x75, 0x63, 0x6b,
0x65, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74,
0x73, 0x1a, 0x53, 0x0a, 0x0c, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72,
0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03,
0x6b, 0x65, 0x79, 0x12, 0x2d, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01,
0x28, 0x0b, 0x32, 0x17, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70,
0x62, 0x2e, 0x43, 0x62, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x05, 0x76, 0x61, 0x6c,
0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xa1, 0x01, 0x0a, 0x09, 0x43, 0x62, 0x4f, 0x70, 0x74,
0x69, 0x6f, 0x6e, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18,
0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x3e,
0x0a, 0x07, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32,
0x24, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x62, 0x2e, 0x43,
0x62, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73,
0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0x3a,
0x0a, 0x0c, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10,
0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79,
0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52,
0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x32, 0x5f, 0x0a, 0x09, 0x53, 0x65,
0x61, 0x77, 0x65, 0x65, 0x64, 0x53, 0x33, 0x12, 0x52, 0x0a, 0x09, 0x43, 0x6f, 0x6e, 0x66, 0x69,
0x67, 0x75, 0x72, 0x65, 0x12, 0x20, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67,
0x5f, 0x70, 0x62, 0x2e, 0x53, 0x33, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52,
0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69,
0x6e, 0x67, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x33, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72,
0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x49, 0x0a, 0x10, 0x73,
0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x42,
0x07, 0x53, 0x33, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x2c, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62,
0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x68, 0x72, 0x69, 0x73, 0x6c, 0x75, 0x73, 0x66, 0x2f, 0x73,
0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x70, 0x62,
0x2f, 0x73, 0x33, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
0x66, 0x69, 0x67, 0x12, 0x3d, 0x0a, 0x06, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x18, 0x01, 0x20,
0x01, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f,
0x70, 0x62, 0x2e, 0x53, 0x33, 0x43, 0x69, 0x72, 0x63, 0x75, 0x69, 0x74, 0x42, 0x72, 0x65, 0x61,
0x6b, 0x65, 0x72, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x06, 0x67, 0x6c, 0x6f, 0x62,
0x61, 0x6c, 0x12, 0x4b, 0x0a, 0x07, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x18, 0x02, 0x20,
0x03, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f,
0x70, 0x62, 0x2e, 0x53, 0x33, 0x43, 0x69, 0x72, 0x63, 0x75, 0x69, 0x74, 0x42, 0x72, 0x65, 0x61,
0x6b, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74,
0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x1a,
0x61, 0x0a, 0x0c, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12,
0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65,
0x79, 0x12, 0x3b, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b,
0x32, 0x25, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x62, 0x2e,
0x53, 0x33, 0x43, 0x69, 0x72, 0x63, 0x75, 0x69, 0x74, 0x42, 0x72, 0x65, 0x61, 0x6b, 0x65, 0x72,
0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02,
0x38, 0x01, 0x22, 0xbd, 0x01, 0x0a, 0x17, 0x53, 0x33, 0x43, 0x69, 0x72, 0x63, 0x75, 0x69, 0x74,
0x42, 0x72, 0x65, 0x61, 0x6b, 0x65, 0x72, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x18,
0x0a, 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52,
0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x4c, 0x0a, 0x07, 0x61, 0x63, 0x74, 0x69,
0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x32, 0x2e, 0x6d, 0x65, 0x73, 0x73,
0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x33, 0x43, 0x69, 0x72, 0x63, 0x75,
0x69, 0x74, 0x42, 0x72, 0x65, 0x61, 0x6b, 0x65, 0x72, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73,
0x2e, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x61,
0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0x3a, 0x0a, 0x0c, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e,
0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20,
0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75,
0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02,
0x38, 0x01, 0x32, 0x5f, 0x0a, 0x09, 0x53, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x53, 0x33, 0x12,
0x52, 0x0a, 0x09, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x12, 0x20, 0x2e, 0x6d,
0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x33, 0x43, 0x6f,
0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21,
0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x33,
0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
0x65, 0x22, 0x00, 0x42, 0x49, 0x0a, 0x10, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73,
0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x42, 0x07, 0x53, 0x33, 0x50, 0x72, 0x6f, 0x74, 0x6f,
0x5a, 0x2c, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x68, 0x72,
0x69, 0x73, 0x6c, 0x75, 0x73, 0x66, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73,
0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x70, 0x62, 0x2f, 0x73, 0x33, 0x5f, 0x70, 0x62, 0x62, 0x06,
0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (
@ -279,18 +283,18 @@ func file_s3_proto_rawDescGZIP() []byte {
var file_s3_proto_msgTypes = make([]protoimpl.MessageInfo, 6)
var file_s3_proto_goTypes = []interface{}{
(*S3ConfigureRequest)(nil), // 0: messaging_pb.S3ConfigureRequest
(*S3ConfigureResponse)(nil), // 1: messaging_pb.S3ConfigureResponse
(*S3CircuitBreakerConfig)(nil), // 2: messaging_pb.S3CircuitBreakerConfig
(*CbOptions)(nil), // 3: messaging_pb.CbOptions
nil, // 4: messaging_pb.S3CircuitBreakerConfig.BucketsEntry
nil, // 5: messaging_pb.CbOptions.ActionsEntry
(*S3ConfigureRequest)(nil), // 0: messaging_pb.S3ConfigureRequest
(*S3ConfigureResponse)(nil), // 1: messaging_pb.S3ConfigureResponse
(*S3CircuitBreakerConfig)(nil), // 2: messaging_pb.S3CircuitBreakerConfig
(*S3CircuitBreakerOptions)(nil), // 3: messaging_pb.S3CircuitBreakerOptions
nil, // 4: messaging_pb.S3CircuitBreakerConfig.BucketsEntry
nil, // 5: messaging_pb.S3CircuitBreakerOptions.ActionsEntry
}
var file_s3_proto_depIdxs = []int32{
3, // 0: messaging_pb.S3CircuitBreakerConfig.global:type_name -> messaging_pb.CbOptions
3, // 0: messaging_pb.S3CircuitBreakerConfig.global:type_name -> messaging_pb.S3CircuitBreakerOptions
4, // 1: messaging_pb.S3CircuitBreakerConfig.buckets:type_name -> messaging_pb.S3CircuitBreakerConfig.BucketsEntry
5, // 2: messaging_pb.CbOptions.actions:type_name -> messaging_pb.CbOptions.ActionsEntry
3, // 3: messaging_pb.S3CircuitBreakerConfig.BucketsEntry.value:type_name -> messaging_pb.CbOptions
5, // 2: messaging_pb.S3CircuitBreakerOptions.actions:type_name -> messaging_pb.S3CircuitBreakerOptions.ActionsEntry
3, // 3: messaging_pb.S3CircuitBreakerConfig.BucketsEntry.value:type_name -> messaging_pb.S3CircuitBreakerOptions
0, // 4: messaging_pb.SeaweedS3.Configure:input_type -> messaging_pb.S3ConfigureRequest
1, // 5: messaging_pb.SeaweedS3.Configure:output_type -> messaging_pb.S3ConfigureResponse
5, // [5:6] is the sub-list for method output_type
@ -343,7 +347,7 @@ func file_s3_proto_init() {
}
}
file_s3_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*CbOptions); i {
switch v := v.(*S3CircuitBreakerOptions); i {
case 0:
return &v.state
case 1:

4
weed/s3api/auth_credentials_subscribe.go
View File

@ -5,7 +5,7 @@ import (
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/s3api/s3_config"
"github.com/chrislusf/seaweedfs/weed/s3api/s3_constants"
"github.com/chrislusf/seaweedfs/weed/util"
)
@ -53,7 +53,7 @@ func (s3a *S3ApiServer) onIamConfigUpdate(dir, filename string, content []byte)
//reload circuit breaker config
func (s3a *S3ApiServer) onCircuitBreakerConfigUpdate(dir, filename string, content []byte) error {
if dir == s3_config.CircuitBreakerConfigDir && filename == s3_config.CircuitBreakerConfigFile {
if dir == s3_constants.CircuitBreakerConfigDir && filename == s3_constants.CircuitBreakerConfigFile {
if err := s3a.cb.LoadS3ApiConfigurationFromBytes(content); err != nil {
return err
}

5
weed/s3api/s3_config/s3_config.go → weed/s3api/s3_constants/s3_config.go
View File

@ -1,14 +1,13 @@
package s3_config
package s3_constants
import (
"github.com/chrislusf/seaweedfs/weed/s3api/s3_constants"
"strings"
)
var (
CircuitBreakerConfigDir = "/etc/s3"
CircuitBreakerConfigFile = "circuit_breaker.json"
AllowedActions = []string{s3_constants.ACTION_READ, s3_constants.ACTION_WRITE, s3_constants.ACTION_LIST, s3_constants.ACTION_TAGGING, s3_constants.ACTION_ADMIN}
AllowedActions = []string{ACTION_READ, ACTION_WRITE, ACTION_LIST, ACTION_TAGGING, ACTION_ADMIN}
LimitTypeCount = "count"
LimitTypeBytes = "bytes"
Separator = ":"

16
weed/s3api/s3api_circuit_breaker.go
View File

@ -7,7 +7,7 @@ import (
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/pb/s3_pb"
"github.com/chrislusf/seaweedfs/weed/s3api/s3_config"
"github.com/chrislusf/seaweedfs/weed/s3api/s3_constants"
"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
"github.com/gorilla/mux"
"go.uber.org/atomic"
@ -29,7 +29,7 @@ func NewCircuitBreaker(option *S3ApiServerOption) *CircuitBreaker {
}
err := pb.WithFilerClient(false, option.Filer, option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
content, err := filer.ReadInsideFiler(client, s3_config.CircuitBreakerConfigDir, s3_config.CircuitBreakerConfigFile)
content, err := filer.ReadInsideFiler(client, s3_constants.CircuitBreakerConfigDir, s3_constants.CircuitBreakerConfigFile)
if err != nil {
return fmt.Errorf("read S3 circuit breaker config: %v", err)
}
@ -73,7 +73,7 @@ func (cb *CircuitBreaker) loadCircuitBreakerConfig(cfg *s3_pb.S3CircuitBreakerCo
for bucket, cbOptions := range cfg.Buckets {
if cbOptions.Enabled {
for action, limit := range cbOptions.Actions {
limitations[s3_config.Concat(bucket, action)] = limit
limitations[s3_constants.Concat(bucket, action)] = limit
}
}
}
@ -110,7 +110,7 @@ func (cb *CircuitBreaker) Limit(f func(w http.ResponseWriter, r *http.Request),
func (cb *CircuitBreaker) limit(r *http.Request, bucket string, action string) (rollback []func(), errCode s3err.ErrorCode) {
//bucket simultaneous request count
bucketCountRollBack, errCode := cb.loadCounterAndCompare(bucket, action, s3_config.LimitTypeCount, 1, s3err.ErrTooManyRequest)
bucketCountRollBack, errCode := cb.loadCounterAndCompare(bucket, action, s3_constants.LimitTypeCount, 1, s3err.ErrTooManyRequest)
if bucketCountRollBack != nil {
rollback = append(rollback, bucketCountRollBack)
}
@ -119,7 +119,7 @@ func (cb *CircuitBreaker) limit(r *http.Request, bucket string, action string) (
}
//bucket simultaneous request content bytes
bucketContentLengthRollBack, errCode := cb.loadCounterAndCompare(bucket, action, s3_config.LimitTypeBytes, r.ContentLength, s3err.ErrRequestBytesExceed)
bucketContentLengthRollBack, errCode := cb.loadCounterAndCompare(bucket, action, s3_constants.LimitTypeBytes, r.ContentLength, s3err.ErrRequestBytesExceed)
if bucketContentLengthRollBack != nil {
rollback = append(rollback, bucketContentLengthRollBack)
}
@ -128,7 +128,7 @@ func (cb *CircuitBreaker) limit(r *http.Request, bucket string, action string) (
}
//global simultaneous request count
globalCountRollBack, errCode := cb.loadCounterAndCompare("", action, s3_config.LimitTypeCount, 1, s3err.ErrTooManyRequest)
globalCountRollBack, errCode := cb.loadCounterAndCompare("", action, s3_constants.LimitTypeCount, 1, s3err.ErrTooManyRequest)
if globalCountRollBack != nil {
rollback = append(rollback, globalCountRollBack)
}
@ -137,7 +137,7 @@ func (cb *CircuitBreaker) limit(r *http.Request, bucket string, action string) (
}
//global simultaneous request content bytes
globalContentLengthRollBack, errCode := cb.loadCounterAndCompare("", action, s3_config.LimitTypeBytes, r.ContentLength, s3err.ErrRequestBytesExceed)
globalContentLengthRollBack, errCode := cb.loadCounterAndCompare("", action, s3_constants.LimitTypeBytes, r.ContentLength, s3err.ErrRequestBytesExceed)
if globalContentLengthRollBack != nil {
rollback = append(rollback, globalContentLengthRollBack)
}
@ -148,7 +148,7 @@ func (cb *CircuitBreaker) limit(r *http.Request, bucket string, action string) (
}
func (cb *CircuitBreaker) loadCounterAndCompare(bucket, action, limitType string, inc int64, errCode s3err.ErrorCode) (f func(), e s3err.ErrorCode) {
key := s3_config.Concat(bucket, action, limitType)
key := s3_constants.Concat(bucket, action, limitType)
e = s3err.ErrNone
if max, ok := cb.limitations[key]; ok {
counter, exists := cb.counters[key]

24
weed/s3api/s3api_circuit_breaker_test.go
View File

@ -2,7 +2,6 @@ package s3api
import (
"github.com/chrislusf/seaweedfs/weed/pb/s3_pb"
"github.com/chrislusf/seaweedfs/weed/s3api/s3_config"
"github.com/chrislusf/seaweedfs/weed/s3api/s3_constants"
"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
"go.uber.org/atomic"
@ -27,30 +26,31 @@ var (
bucket = "/test"
action = s3_constants.ACTION_READ
TestLimitCases = []*TestLimitCase{
{action, s3_config.LimitTypeCount, 5, 5, 6, 1024, 5},
{action, s3_config.LimitTypeCount, 6, 6, 6, 1024, 6},
{action, s3_config.LimitTypeCount, 5, 6, 6, 1024, 5},
{action, s3_config.LimitTypeBytes, 1024, 1024, 6, 200, 5},
{action, s3_config.LimitTypeBytes, 1200, 1200, 6, 200, 6},
{action, s3_config.LimitTypeBytes, 11990, 11990, 60, 200, 59},
{action, s3_config.LimitTypeBytes, 11790, 11990, 60, 200, 58},
{action, s3_constants.LimitTypeCount, 5, 5, 6, 1024, 5},
{action, s3_constants.LimitTypeCount, 6, 6, 6, 1024, 6},
{action, s3_constants.LimitTypeCount, 5, 6, 6, 1024, 5},
{action, s3_constants.LimitTypeBytes, 1024, 1024, 6, 200, 5},
{action, s3_constants.LimitTypeBytes, 1200, 1200, 6, 200, 6},
{action, s3_constants.LimitTypeBytes, 11990, 11990, 60, 200, 59},
{action, s3_constants.LimitTypeBytes, 11790, 11990, 60, 200, 58},
}
)
func TestLimit(t *testing.T) {
for _, tc := range TestLimitCases {
circuitBreakerConfig := &s3_pb.S3CircuitBreakerConfig{
Global: &s3_pb.CbOptions{
Global: &s3_pb.S3CircuitBreakerOptions{
Enabled: true,
Actions: map[string]int64{
s3_config.Concat(tc.actionName, tc.limitType): tc.globalLimitValue,
s3_constants.Concat(tc.actionName, tc.limitType): tc.globalLimitValue,
s3_constants.Concat(tc.actionName, tc.limitType): tc.globalLimitValue,
},
},
Buckets: map[string]*s3_pb.CbOptions{
Buckets: map[string]*s3_pb.S3CircuitBreakerOptions{
bucket: {
Enabled: true,
Actions: map[string]int64{
s3_config.Concat(tc.actionName, tc.limitType): tc.bucketLimitValue,
s3_constants.Concat(tc.actionName, tc.limitType): tc.bucketLimitValue,
},
},
},

58
weed/shell/command_s3_circuitbreaker.go
View File

@ -7,7 +7,7 @@ import (
"github.com/alecthomas/units"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/pb/s3_pb"
"github.com/chrislusf/seaweedfs/weed/s3api/s3_config"
"github.com/chrislusf/seaweedfs/weed/s3api/s3_constants"
"io"
"strconv"
"strings"
@ -32,21 +32,29 @@ func (c *commandS3CircuitBreaker) Help() string {
return `configure and apply s3 circuit breaker options for each bucket
# examples
# add
s3.circuitBreaker -actions Read,Write -values 500,200 -global -enable -apply -type count
s3.circuitBreaker -actions Write -values 200MiB -global -enable -apply -type bytes
s3.circuitBreaker -actions Write -values 200MiB -bucket x,y,z -enable -apply -type bytes
#delete
s3.circuitBreaker -actions Write -bucket x,y,z -delete -apply -type bytes
s3.circuitBreaker -actions Write -bucket x,y,z -delete -apply
s3.circuitBreaker -actions Write -delete -apply
# add circuit breaker config for global
s3.circuitBreaker -global -type count -actions Read,Write -values 500,200 -apply
# disable global config
s3.circuitBreaker -global -disable -apply
# add circuit breaker config for buckets x,y,z
s3.circuitBreaker -buckets x,y,z -type count -actions Read,Write -values 200,100 -apply
# disable circuit breaker config of x
s3.circuitBreaker -buckets x -disable -apply
# delete circuit breaker config of x
s3.circuitBreaker -buckets x -delete -apply
# clear all circuit breaker config
s3.circuitBreaker -delete -apply
`
}
func (c *commandS3CircuitBreaker) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
dir := s3_config.CircuitBreakerConfigDir
file := s3_config.CircuitBreakerConfigFile
dir := s3_constants.CircuitBreakerConfigDir
file := s3_constants.CircuitBreakerConfigFile
s3CircuitBreakerCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
buckets := s3CircuitBreakerCommand.String("buckets", "", "the bucket name(s) to configure, eg: -buckets x,y,z")
@ -73,7 +81,7 @@ func (c *commandS3CircuitBreaker) Do(args []string, commandEnv *CommandEnv, writ
}
cbCfg := &s3_pb.S3CircuitBreakerConfig{
Buckets: make(map[string]*s3_pb.CbOptions),
Buckets: make(map[string]*s3_pb.S3CircuitBreakerOptions),
}
if buf.Len() > 0 {
if err = filer.ParseS3ConfigurationFromBytes(buf.Bytes(), cbCfg); err != nil {
@ -121,10 +129,10 @@ func (c *commandS3CircuitBreaker) Do(args []string, commandEnv *CommandEnv, writ
if len(*buckets) > 0 {
for _, bucket := range cmdBuckets {
var cbOptions *s3_pb.CbOptions
var cbOptions *s3_pb.S3CircuitBreakerOptions
var exists bool
if cbOptions, exists = cbCfg.Buckets[bucket]; !exists {
cbOptions = &s3_pb.CbOptions{}
cbOptions = &s3_pb.S3CircuitBreakerOptions{}
cbCfg.Buckets[bucket] = cbOptions
}
cbOptions.Enabled = !*disabled
@ -145,7 +153,7 @@ func (c *commandS3CircuitBreaker) Do(args []string, commandEnv *CommandEnv, writ
if *global {
globalOptions := cbCfg.Global
if globalOptions == nil {
globalOptions = &s3_pb.CbOptions{Actions: make(map[string]int64, len(cmdActions))}
globalOptions = &s3_pb.S3CircuitBreakerOptions{Actions: make(map[string]int64, len(cmdActions))}
cbCfg.Global = globalOptions
}
globalOptions.Enabled = !*disabled
@ -192,7 +200,7 @@ func loadConfig(commandEnv *CommandEnv, dir string, file string, buf *bytes.Buff
return nil
}
func insertOrUpdateValues(cbOptions *s3_pb.CbOptions, cmdActions []string, cmdValues []int64, limitType *string) error {
func insertOrUpdateValues(cbOptions *s3_pb.S3CircuitBreakerOptions, cmdActions []string, cmdValues []int64, limitType *string) error {
if len(*limitType) == 0 {
return fmt.Errorf("type not valid, only 'count' and 'bytes' are allowed")
}
@ -203,7 +211,7 @@ func insertOrUpdateValues(cbOptions *s3_pb.CbOptions, cmdActions []string, cmdVa
if len(cmdValues) > 0 {
for i, action := range cmdActions {
cbOptions.Actions[s3_config.Concat(action, *limitType)] = cmdValues[i]
cbOptions.Actions[s3_constants.Concat(action, *limitType)] = cmdValues[i]
}
}
return nil
@ -223,7 +231,7 @@ func deleteBucketsActions(cmdBuckets []string, cbCfg *s3_pb.S3CircuitBreakerConf
if cbOption, ok := cbCfg.Buckets[bucket]; ok {
if len(cmdActions) > 0 && cbOption.Actions != nil {
for _, action := range cmdActions {
delete(cbOption.Actions, s3_config.Concat(action, *limitType))
delete(cbOption.Actions, s3_constants.Concat(action, *limitType))
}
}
@ -250,7 +258,7 @@ func deleteGlobalActions(cbCfg *s3_pb.S3CircuitBreakerConfig, cmdActions []strin
return
} else {
for _, action := range cmdActions {
delete(globalOptions.Actions, s3_config.Concat(action, *limitType))
delete(globalOptions.Actions, s3_constants.Concat(action, *limitType))
}
}
@ -270,27 +278,27 @@ func (c *commandS3CircuitBreaker) initActionsAndValues(buckets, actions, limitTy
//check action valid
for _, action := range cmdActions {
var found bool
for _, allowedAction := range s3_config.AllowedActions {
for _, allowedAction := range s3_constants.AllowedActions {
if allowedAction == action {
found = true
}
}
if !found {
return nil, nil, nil, fmt.Errorf("value(%s) of flag[-action] not valid, allowed actions: %v", *actions, s3_config.AllowedActions)
return nil, nil, nil, fmt.Errorf("value(%s) of flag[-action] not valid, allowed actions: %v", *actions, s3_constants.AllowedActions)
}
}
}
if !parseValues {
if len(cmdActions) < 0 {
for _, action := range s3_config.AllowedActions {
for _, action := range s3_constants.AllowedActions {
cmdActions = append(cmdActions, action)
}
}
if len(*limitType) > 0 {
switch *limitType {
case s3_config.LimitTypeCount:
case s3_constants.LimitTypeCount:
elements := strings.Split(*values, ",")
if len(cmdActions) != len(elements) {
if len(elements) != 1 || len(elements) == 0 {
@ -312,7 +320,7 @@ func (c *commandS3CircuitBreaker) initActionsAndValues(buckets, actions, limitTy
cmdValues = append(cmdValues, int64(v))
}
}
case s3_config.LimitTypeBytes:
case s3_constants.LimitTypeBytes:
elements := strings.Split(*values, ",")
if len(cmdActions) != len(elements) {
if len(elements) != 1 || len(elements) == 0 {

145
weed/shell/command_s3_circuitbreaker_test.go
View File

@ -2,6 +2,8 @@ package shell
import (
"bytes"
"encoding/json"
"reflect"
"strings"
"testing"
)
@ -15,33 +17,137 @@ var (
TestCases = []*Case{
//add circuit breaker config for global
{
args: strings.Split("-global -type count -actions Read,Write -values 500,200", " "),
result: "{\n \"global\": {\n \"enabled\": true,\n \"actions\": {\n \"Read:count\": \"500\",\n \"Write:count\": \"200\"\n }\n }\n}\n",
args: strings.Split("-global -type count -actions Read,Write -values 500,200", " "),
result: `{
"global": {
"enabled": true,
"actions": {
"Read:count": "500",
"Write:count": "200"
}
}
}`,
},
//disable global config
{
args: strings.Split("-global -disable", " "),
result: "{\n \"global\": {\n \"actions\": {\n \"Read:count\": \"500\",\n \"Write:count\": \"200\"\n }\n }\n}\n",
args: strings.Split("-global -disable", " "),
result: `{
"global": {
"actions": {
"Read:count": "500",
"Write:count": "200"
}
}
}`,
},
//add circuit breaker config for buckets x,y,z
{
args: strings.Split("-buckets x,y,z -type count -actions Read,Write -values 200,100", " "),
result: "{\n \"global\": {\n \"actions\": {\n \"Read:count\": \"500\",\n \"Write:count\": \"200\"\n }\n },\n \"buckets\": {\n \"x\": {\n \"enabled\": true,\n \"actions\": {\n \"Read:count\": \"200\",\n \"Write:count\": \"100\"\n }\n },\n \"y\": {\n \"enabled\": true,\n \"actions\": {\n \"Read:count\": \"200\",\n \"Write:count\": \"100\"\n }\n },\n \"z\": {\n \"enabled\": true,\n \"actions\": {\n \"Read:count\": \"200\",\n \"Write:count\": \"100\"\n }\n }\n }\n}\n",
args: strings.Split("-buckets x,y,z -type count -actions Read,Write -values 200,100", " "),
result: `{
"global": {
"actions": {
"Read:count": "500",
"Write:count": "200"
}
},
"buckets": {
"x": {
"enabled": true,
"actions": {
"Read:count": "200",
"Write:count": "100"
}
},
"y": {
"enabled": true,
"actions": {
"Read:count": "200",
"Write:count": "100"
}
},
"z": {
"enabled": true,
"actions": {
"Read:count": "200",
"Write:count": "100"
}
}
}
}`,
},
//disable circuit breaker config of x
{
args: strings.Split("-buckets x -disable", " "),
result: "{\n \"global\": {\n \"actions\": {\n \"Read:count\": \"500\",\n \"Write:count\": \"200\"\n }\n },\n \"buckets\": {\n \"x\": {\n \"actions\": {\n \"Read:count\": \"200\",\n \"Write:count\": \"100\"\n }\n },\n \"y\": {\n \"enabled\": true,\n \"actions\": {\n \"Read:count\": \"200\",\n \"Write:count\": \"100\"\n }\n },\n \"z\": {\n \"enabled\": true,\n \"actions\": {\n \"Read:count\": \"200\",\n \"Write:count\": \"100\"\n }\n }\n }\n}\n",
args: strings.Split("-buckets x -disable", " "),
result: `{
"global": {
"actions": {
"Read:count": "500",
"Write:count": "200"
}
},
"buckets": {
"x": {
"actions": {
"Read:count": "200",
"Write:count": "100"
}
},
"y": {
"enabled": true,
"actions": {
"Read:count": "200",
"Write:count": "100"
}
},
"z": {
"enabled": true,
"actions": {
"Read:count": "200",
"Write:count": "100"
}
}
}
}`,
},
//delete circuit breaker config of x
{
args: strings.Split("-buckets x -delete", " "),
result: "{\n \"global\": {\n \"actions\": {\n \"Read:count\": \"500\",\n \"Write:count\": \"200\"\n }\n },\n \"buckets\": {\n \"y\": {\n \"enabled\": true,\n \"actions\": {\n \"Read:count\": \"200\",\n \"Write:count\": \"100\"\n }\n },\n \"z\": {\n \"enabled\": true,\n \"actions\": {\n \"Read:count\": \"200\",\n \"Write:count\": \"100\"\n }\n }\n }\n}\n",
args: strings.Split("-buckets x -delete", " "),
result: `{
"global": {
"actions": {
"Read:count": "500",
"Write:count": "200"
}
},
"buckets": {
"y": {
"enabled": true,
"actions": {
"Read:count": "200",
"Write:count": "100"
}
},
"z": {
"enabled": true,
"actions": {
"Read:count": "200",
"Write:count": "100"
}
}
}
}`,
},
//clear all circuit breaker config
{
args: strings.Split("-delete", " "),
result: "{\n\n}\n",
args: strings.Split("-delete", " "),
result: `{
}`,
},
}
)
@ -65,10 +171,21 @@ func TestCircuitBreakerShell(t *testing.T) {
}
if i != 0 {
result := writeBuf.String()
if result != tc.result {
t.Fatal("result of s3 circuit breaker shell command is unexpect!")
actual := make(map[string]interface{})
err := json.Unmarshal([]byte(result), &actual)
if err != nil {
t.Error(err)
}
expect := make(map[string]interface{})
err = json.Unmarshal([]byte(result), &expect)
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(actual, expect) {
t.Fatal("result of s3 circuit breaker shell command is unexpect!")
}
}
}
}
Write Preview
Loading…
Cancel
Save