chrislu
3 years ago
19 changed files with 1318 additions and 108 deletions
-
2.github/workflows/depsreview.yml
-
12go.mod
-
36go.sum
-
2weed/command/update.go
-
7weed/filer/filechunk_manifest.go
-
8weed/filer/filechunks.go
-
3weed/filer/s3iam_conf.go
-
10weed/pb/s3.proto
-
217weed/pb/s3_pb/s3.pb.go
-
33weed/s3api/auth_credentials_subscribe.go
-
18weed/s3api/s3_constants/s3_config.go
-
183weed/s3api/s3api_circuit_breaker.go
-
107weed/s3api/s3api_circuit_breaker_test.go
-
13weed/s3api/s3api_object_multipart_handlers.go
-
84weed/s3api/s3api_server.go
-
13weed/s3api/s3err/s3api_errors.go
-
24weed/server/volume_grpc_copy.go
-
358weed/shell/command_s3_circuitbreaker.go
-
292weed/shell/command_s3_circuitbreaker_test.go
@ -0,0 +1,18 @@ |
|||||
|
package s3_constants |
||||
|
|
||||
|
import ( |
||||
|
"strings" |
||||
|
) |
||||
|
|
||||
|
var ( |
||||
|
CircuitBreakerConfigDir = "/etc/s3" |
||||
|
CircuitBreakerConfigFile = "circuit_breaker.json" |
||||
|
AllowedActions = []string{ACTION_READ, ACTION_WRITE, ACTION_LIST, ACTION_TAGGING, ACTION_ADMIN} |
||||
|
LimitTypeCount = "Count" |
||||
|
LimitTypeBytes = "MB" |
||||
|
Separator = ":" |
||||
|
) |
||||
|
|
||||
|
func Concat(elements ...string) string { |
||||
|
return strings.Join(elements, Separator) |
||||
|
} |
||||
@ -0,0 +1,183 @@ |
|||||
|
package s3api |
||||
|
|
||||
|
import ( |
||||
|
"fmt" |
||||
|
"github.com/chrislusf/seaweedfs/weed/filer" |
||||
|
"github.com/chrislusf/seaweedfs/weed/glog" |
||||
|
"github.com/chrislusf/seaweedfs/weed/pb" |
||||
|
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" |
||||
|
"github.com/chrislusf/seaweedfs/weed/pb/s3_pb" |
||||
|
"github.com/chrislusf/seaweedfs/weed/s3api/s3_constants" |
||||
|
"github.com/chrislusf/seaweedfs/weed/s3api/s3err" |
||||
|
"github.com/gorilla/mux" |
||||
|
"net/http" |
||||
|
"sync" |
||||
|
"sync/atomic" |
||||
|
) |
||||
|
|
||||
|
type CircuitBreaker struct { |
||||
|
sync.RWMutex |
||||
|
Enabled bool |
||||
|
counters map[string]*int64 |
||||
|
limitations map[string]int64 |
||||
|
} |
||||
|
|
||||
|
func NewCircuitBreaker(option *S3ApiServerOption) *CircuitBreaker { |
||||
|
cb := &CircuitBreaker{ |
||||
|
counters: make(map[string]*int64), |
||||
|
limitations: make(map[string]int64), |
||||
|
} |
||||
|
|
||||
|
err := pb.WithFilerClient(false, option.Filer, option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error { |
||||
|
content, err := filer.ReadInsideFiler(client, s3_constants.CircuitBreakerConfigDir, s3_constants.CircuitBreakerConfigFile) |
||||
|
if err != nil { |
||||
|
return fmt.Errorf("read S3 circuit breaker config: %v", err) |
||||
|
} |
||||
|
return cb.LoadS3ApiConfigurationFromBytes(content) |
||||
|
}) |
||||
|
|
||||
|
if err != nil { |
||||
|
glog.Warningf("fail to load config: %v", err) |
||||
|
} |
||||
|
|
||||
|
return cb |
||||
|
} |
||||
|
|
||||
|
func (cb *CircuitBreaker) LoadS3ApiConfigurationFromBytes(content []byte) error { |
||||
|
cbCfg := &s3_pb.S3CircuitBreakerConfig{} |
||||
|
if err := filer.ParseS3ConfigurationFromBytes(content, cbCfg); err != nil { |
||||
|
glog.Warningf("unmarshal error: %v", err) |
||||
|
return fmt.Errorf("unmarshal error: %v", err) |
||||
|
} |
||||
|
if err := cb.loadCircuitBreakerConfig(cbCfg); err != nil { |
||||
|
return err |
||||
|
} |
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
func (cb *CircuitBreaker) loadCircuitBreakerConfig(cfg *s3_pb.S3CircuitBreakerConfig) error { |
||||
|
|
||||
|
//global
|
||||
|
globalEnabled := false |
||||
|
globalOptions := cfg.Global |
||||
|
limitations := make(map[string]int64) |
||||
|
if globalOptions != nil && globalOptions.Enabled && len(globalOptions.Actions) > 0 { |
||||
|
globalEnabled = globalOptions.Enabled |
||||
|
for action, limit := range globalOptions.Actions { |
||||
|
limitations[action] = limit |
||||
|
} |
||||
|
} |
||||
|
cb.Enabled = globalEnabled |
||||
|
|
||||
|
//buckets
|
||||
|
for bucket, cbOptions := range cfg.Buckets { |
||||
|
if cbOptions.Enabled { |
||||
|
for action, limit := range cbOptions.Actions { |
||||
|
limitations[s3_constants.Concat(bucket, action)] = limit |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
cb.limitations = limitations |
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
func (cb *CircuitBreaker) Limit(f func(w http.ResponseWriter, r *http.Request), action string) (http.HandlerFunc, Action) { |
||||
|
return func(w http.ResponseWriter, r *http.Request) { |
||||
|
if !cb.Enabled { |
||||
|
f(w, r) |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
vars := mux.Vars(r) |
||||
|
bucket := vars["bucket"] |
||||
|
|
||||
|
rollback, errCode := cb.limit(r, bucket, action) |
||||
|
defer func() { |
||||
|
for _, rf := range rollback { |
||||
|
rf() |
||||
|
} |
||||
|
}() |
||||
|
|
||||
|
if errCode == s3err.ErrNone { |
||||
|
f(w, r) |
||||
|
return |
||||
|
} |
||||
|
s3err.WriteErrorResponse(w, r, errCode) |
||||
|
}, Action(action) |
||||
|
} |
||||
|
|
||||
|
func (cb *CircuitBreaker) limit(r *http.Request, bucket string, action string) (rollback []func(), errCode s3err.ErrorCode) { |
||||
|
|
||||
|
//bucket simultaneous request count
|
||||
|
bucketCountRollBack, errCode := cb.loadCounterAndCompare(s3_constants.Concat(bucket, action, s3_constants.LimitTypeCount), 1, s3err.ErrTooManyRequest) |
||||
|
if bucketCountRollBack != nil { |
||||
|
rollback = append(rollback, bucketCountRollBack) |
||||
|
} |
||||
|
if errCode != s3err.ErrNone { |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
//bucket simultaneous request content bytes
|
||||
|
bucketContentLengthRollBack, errCode := cb.loadCounterAndCompare(s3_constants.Concat(bucket, action, s3_constants.LimitTypeBytes), r.ContentLength, s3err.ErrRequestBytesExceed) |
||||
|
if bucketContentLengthRollBack != nil { |
||||
|
rollback = append(rollback, bucketContentLengthRollBack) |
||||
|
} |
||||
|
if errCode != s3err.ErrNone { |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
//global simultaneous request count
|
||||
|
globalCountRollBack, errCode := cb.loadCounterAndCompare(s3_constants.Concat(action, s3_constants.LimitTypeCount), 1, s3err.ErrTooManyRequest) |
||||
|
if globalCountRollBack != nil { |
||||
|
rollback = append(rollback, globalCountRollBack) |
||||
|
} |
||||
|
if errCode != s3err.ErrNone { |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
//global simultaneous request content bytes
|
||||
|
globalContentLengthRollBack, errCode := cb.loadCounterAndCompare(s3_constants.Concat(action, s3_constants.LimitTypeBytes), r.ContentLength, s3err.ErrRequestBytesExceed) |
||||
|
if globalContentLengthRollBack != nil { |
||||
|
rollback = append(rollback, globalContentLengthRollBack) |
||||
|
} |
||||
|
if errCode != s3err.ErrNone { |
||||
|
return |
||||
|
} |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
func (cb *CircuitBreaker) loadCounterAndCompare(key string, inc int64, errCode s3err.ErrorCode) (f func(), e s3err.ErrorCode) { |
||||
|
e = s3err.ErrNone |
||||
|
if max, ok := cb.limitations[key]; ok { |
||||
|
cb.RLock() |
||||
|
counter, exists := cb.counters[key] |
||||
|
cb.RUnlock() |
||||
|
|
||||
|
if !exists { |
||||
|
cb.Lock() |
||||
|
counter, exists = cb.counters[key] |
||||
|
if !exists { |
||||
|
var newCounter int64 |
||||
|
counter = &newCounter |
||||
|
cb.counters[key] = counter |
||||
|
} |
||||
|
cb.Unlock() |
||||
|
} |
||||
|
current := atomic.LoadInt64(counter) |
||||
|
if current+inc > max { |
||||
|
e = errCode |
||||
|
return |
||||
|
} else { |
||||
|
current := atomic.AddInt64(counter, inc) |
||||
|
f = func() { |
||||
|
atomic.AddInt64(counter, -inc) |
||||
|
} |
||||
|
if current > max { |
||||
|
e = errCode |
||||
|
return |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
return |
||||
|
} |
||||
@ -0,0 +1,107 @@ |
|||||
|
package s3api |
||||
|
|
||||
|
import ( |
||||
|
"github.com/chrislusf/seaweedfs/weed/pb/s3_pb" |
||||
|
"github.com/chrislusf/seaweedfs/weed/s3api/s3_constants" |
||||
|
"github.com/chrislusf/seaweedfs/weed/s3api/s3err" |
||||
|
"net/http" |
||||
|
"sync" |
||||
|
"sync/atomic" |
||||
|
"testing" |
||||
|
) |
||||
|
|
||||
|
type TestLimitCase struct { |
||||
|
actionName string |
||||
|
|
||||
|
limitType string |
||||
|
bucketLimitValue int64 |
||||
|
globalLimitValue int64 |
||||
|
|
||||
|
routineCount int |
||||
|
successCount int64 |
||||
|
} |
||||
|
|
||||
|
var ( |
||||
|
bucket = "/test" |
||||
|
action = s3_constants.ACTION_WRITE |
||||
|
fileSize int64 = 200 |
||||
|
|
||||
|
TestLimitCases = []*TestLimitCase{ |
||||
|
|
||||
|
//bucket-LimitTypeCount
|
||||
|
{action, s3_constants.LimitTypeCount, 5, 6, 60, 5}, |
||||
|
{action, s3_constants.LimitTypeCount, 0, 6, 6, 0}, |
||||
|
|
||||
|
//global-LimitTypeCount
|
||||
|
{action, s3_constants.LimitTypeCount, 6, 5, 6, 5}, |
||||
|
{action, s3_constants.LimitTypeCount, 6, 0, 6, 0}, |
||||
|
|
||||
|
//bucket-LimitTypeBytes
|
||||
|
{action, s3_constants.LimitTypeBytes, 1000, 1020, 6, 5}, |
||||
|
{action, s3_constants.LimitTypeBytes, 0, 1020, 6, 0}, |
||||
|
|
||||
|
//global-LimitTypeBytes
|
||||
|
{action, s3_constants.LimitTypeBytes, 1020, 1000, 6, 5}, |
||||
|
{action, s3_constants.LimitTypeBytes, 1020, 0, 6, 0}, |
||||
|
} |
||||
|
) |
||||
|
|
||||
|
func TestLimit(t *testing.T) { |
||||
|
for _, tc := range TestLimitCases { |
||||
|
circuitBreakerConfig := &s3_pb.S3CircuitBreakerConfig{ |
||||
|
Global: &s3_pb.S3CircuitBreakerOptions{ |
||||
|
Enabled: true, |
||||
|
Actions: map[string]int64{ |
||||
|
s3_constants.Concat(tc.actionName, tc.limitType): tc.globalLimitValue, |
||||
|
s3_constants.Concat(tc.actionName, tc.limitType): tc.globalLimitValue, |
||||
|
}, |
||||
|
}, |
||||
|
Buckets: map[string]*s3_pb.S3CircuitBreakerOptions{ |
||||
|
bucket: { |
||||
|
Enabled: true, |
||||
|
Actions: map[string]int64{ |
||||
|
s3_constants.Concat(tc.actionName, tc.limitType): tc.bucketLimitValue, |
||||
|
}, |
||||
|
}, |
||||
|
}, |
||||
|
} |
||||
|
circuitBreaker := &CircuitBreaker{ |
||||
|
counters: make(map[string]*int64), |
||||
|
limitations: make(map[string]int64), |
||||
|
} |
||||
|
err := circuitBreaker.loadCircuitBreakerConfig(circuitBreakerConfig) |
||||
|
if err != nil { |
||||
|
t.Fatal(err) |
||||
|
} |
||||
|
|
||||
|
successCount := doLimit(circuitBreaker, tc.routineCount, &http.Request{ContentLength: fileSize}, tc.actionName) |
||||
|
if successCount != tc.successCount { |
||||
|
t.Errorf("successCount not equal, expect=%d, actual=%d, case: %v", tc.successCount, successCount, tc) |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func doLimit(circuitBreaker *CircuitBreaker, routineCount int, r *http.Request, action string) int64 { |
||||
|
var successCounter int64 |
||||
|
resultCh := make(chan []func(), routineCount) |
||||
|
var wg sync.WaitGroup |
||||
|
for i := 0; i < routineCount; i++ { |
||||
|
wg.Add(1) |
||||
|
go func() { |
||||
|
defer wg.Done() |
||||
|
rollbackFn, errCode := circuitBreaker.limit(r, bucket, action) |
||||
|
if errCode == s3err.ErrNone { |
||||
|
atomic.AddInt64(&successCounter, 1) |
||||
|
} |
||||
|
resultCh <- rollbackFn |
||||
|
}() |
||||
|
} |
||||
|
wg.Wait() |
||||
|
close(resultCh) |
||||
|
for fns := range resultCh { |
||||
|
for _, fn := range fns { |
||||
|
fn() |
||||
|
} |
||||
|
} |
||||
|
return successCounter |
||||
|
} |
||||
@ -0,0 +1,358 @@ |
|||||
|
package shell |
||||
|
|
||||
|
import ( |
||||
|
"bytes" |
||||
|
"flag" |
||||
|
"fmt" |
||||
|
"github.com/chrislusf/seaweedfs/weed/filer" |
||||
|
"github.com/chrislusf/seaweedfs/weed/pb/s3_pb" |
||||
|
"github.com/chrislusf/seaweedfs/weed/s3api/s3_constants" |
||||
|
"io" |
||||
|
"strconv" |
||||
|
"strings" |
||||
|
|
||||
|
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" |
||||
|
) |
||||
|
|
||||
|
var LoadConfig = loadConfig |
||||
|
|
||||
|
func init() { |
||||
|
Commands = append(Commands, &commandS3CircuitBreaker{}) |
||||
|
} |
||||
|
|
||||
|
type commandS3CircuitBreaker struct { |
||||
|
} |
||||
|
|
||||
|
func (c *commandS3CircuitBreaker) Name() string { |
||||
|
return "s3.circuitBreaker" |
||||
|
} |
||||
|
|
||||
|
func (c *commandS3CircuitBreaker) Help() string { |
||||
|
return `configure and apply s3 circuit breaker options for each bucket |
||||
|
|
||||
|
# examples |
||||
|
# add circuit breaker config for global |
||||
|
s3.circuitBreaker -global -type count -actions Read,Write -values 500,200 -apply |
||||
|
|
||||
|
# disable global config |
||||
|
s3.circuitBreaker -global -disable -apply |
||||
|
|
||||
|
# add circuit breaker config for buckets x,y,z |
||||
|
s3.circuitBreaker -buckets x,y,z -type count -actions Read,Write -values 200,100 -apply |
||||
|
|
||||
|
# disable circuit breaker config of x |
||||
|
s3.circuitBreaker -buckets x -disable -apply |
||||
|
|
||||
|
# delete circuit breaker config of x |
||||
|
s3.circuitBreaker -buckets x -delete -apply |
||||
|
|
||||
|
# clear all circuit breaker config |
||||
|
s3.circuitBreaker -delete -apply |
||||
|
` |
||||
|
} |
||||
|
|
||||
|
func (c *commandS3CircuitBreaker) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { |
||||
|
dir := s3_constants.CircuitBreakerConfigDir |
||||
|
file := s3_constants.CircuitBreakerConfigFile |
||||
|
|
||||
|
s3CircuitBreakerCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) |
||||
|
buckets := s3CircuitBreakerCommand.String("buckets", "", "the bucket name(s) to configure, eg: -buckets x,y,z") |
||||
|
global := s3CircuitBreakerCommand.Bool("global", false, "configure global circuit breaker") |
||||
|
|
||||
|
actions := s3CircuitBreakerCommand.String("actions", "", "comma separated actions names: Read,Write,List,Tagging,Admin") |
||||
|
limitType := s3CircuitBreakerCommand.String("type", "", "'Count' or 'MB'; Count represents the number of simultaneous requests, and MB represents the content size of all simultaneous requests") |
||||
|
values := s3CircuitBreakerCommand.String("values", "", "comma separated values") |
||||
|
|
||||
|
disabled := s3CircuitBreakerCommand.Bool("disable", false, "disable global or buckets circuit breaker") |
||||
|
deleted := s3CircuitBreakerCommand.Bool("delete", false, "delete circuit breaker config") |
||||
|
|
||||
|
apply := s3CircuitBreakerCommand.Bool("apply", false, "update and apply current configuration") |
||||
|
|
||||
|
if err = s3CircuitBreakerCommand.Parse(args); err != nil { |
||||
|
return nil |
||||
|
|
||||
|
} |
||||
|
|
||||
|
var buf bytes.Buffer |
||||
|
err = LoadConfig(commandEnv, dir, file, &buf) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
cbCfg := &s3_pb.S3CircuitBreakerConfig{ |
||||
|
Buckets: make(map[string]*s3_pb.S3CircuitBreakerOptions), |
||||
|
} |
||||
|
if buf.Len() > 0 { |
||||
|
if err = filer.ParseS3ConfigurationFromBytes(buf.Bytes(), cbCfg); err != nil { |
||||
|
return err |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
if *deleted { |
||||
|
cmdBuckets, cmdActions, _, err := c.initActionsAndValues(buckets, actions, limitType, values, true) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
if len(cmdBuckets) <= 0 && !*global { |
||||
|
if len(cmdActions) > 0 { |
||||
|
deleteGlobalActions(cbCfg, cmdActions, limitType) |
||||
|
if cbCfg.Buckets != nil { |
||||
|
var allBuckets []string |
||||
|
for bucket := range cbCfg.Buckets { |
||||
|
allBuckets = append(allBuckets, bucket) |
||||
|
} |
||||
|
deleteBucketsActions(allBuckets, cbCfg, cmdActions, limitType) |
||||
|
} |
||||
|
} else { |
||||
|
cbCfg.Global = nil |
||||
|
cbCfg.Buckets = nil |
||||
|
} |
||||
|
} else { |
||||
|
if len(cmdBuckets) > 0 { |
||||
|
deleteBucketsActions(cmdBuckets, cbCfg, cmdActions, limitType) |
||||
|
} |
||||
|
if *global { |
||||
|
deleteGlobalActions(cbCfg, cmdActions, nil) |
||||
|
} |
||||
|
} |
||||
|
} else { |
||||
|
cmdBuckets, cmdActions, cmdValues, err := c.initActionsAndValues(buckets, actions, limitType, values, *disabled) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
if len(cmdActions) > 0 && len(*buckets) <= 0 && !*global { |
||||
|
return fmt.Errorf("one of -global and -buckets must be specified") |
||||
|
} |
||||
|
|
||||
|
if len(*buckets) > 0 { |
||||
|
for _, bucket := range cmdBuckets { |
||||
|
var cbOptions *s3_pb.S3CircuitBreakerOptions |
||||
|
var exists bool |
||||
|
if cbOptions, exists = cbCfg.Buckets[bucket]; !exists { |
||||
|
cbOptions = &s3_pb.S3CircuitBreakerOptions{} |
||||
|
cbCfg.Buckets[bucket] = cbOptions |
||||
|
} |
||||
|
cbOptions.Enabled = !*disabled |
||||
|
|
||||
|
if len(cmdActions) > 0 { |
||||
|
err = insertOrUpdateValues(cbOptions, cmdActions, cmdValues, limitType) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
if len(cbOptions.Actions) <= 0 && !cbOptions.Enabled { |
||||
|
delete(cbCfg.Buckets, bucket) |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
if *global { |
||||
|
globalOptions := cbCfg.Global |
||||
|
if globalOptions == nil { |
||||
|
globalOptions = &s3_pb.S3CircuitBreakerOptions{Actions: make(map[string]int64, len(cmdActions))} |
||||
|
cbCfg.Global = globalOptions |
||||
|
} |
||||
|
globalOptions.Enabled = !*disabled |
||||
|
|
||||
|
if len(cmdActions) > 0 { |
||||
|
err = insertOrUpdateValues(globalOptions, cmdActions, cmdValues, limitType) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
if len(globalOptions.Actions) <= 0 && !globalOptions.Enabled { |
||||
|
cbCfg.Global = nil |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
buf.Reset() |
||||
|
err = filer.ProtoToText(&buf, cbCfg) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
_, _ = fmt.Fprintf(writer, string(buf.Bytes())) |
||||
|
_, _ = fmt.Fprintln(writer) |
||||
|
|
||||
|
if *apply { |
||||
|
if err := commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { |
||||
|
return filer.SaveInsideFiler(client, dir, file, buf.Bytes()) |
||||
|
}); err != nil { |
||||
|
return err |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
func loadConfig(commandEnv *CommandEnv, dir string, file string, buf *bytes.Buffer) error { |
||||
|
if err := commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { |
||||
|
return filer.ReadEntry(commandEnv.MasterClient, client, dir, file, buf) |
||||
|
}); err != nil && err != filer_pb.ErrNotFound { |
||||
|
return err |
||||
|
} |
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
func insertOrUpdateValues(cbOptions *s3_pb.S3CircuitBreakerOptions, cmdActions []string, cmdValues []int64, limitType *string) error { |
||||
|
if len(*limitType) == 0 { |
||||
|
return fmt.Errorf("type not valid, only 'count' and 'bytes' are allowed") |
||||
|
} |
||||
|
|
||||
|
if cbOptions.Actions == nil { |
||||
|
cbOptions.Actions = make(map[string]int64, len(cmdActions)) |
||||
|
} |
||||
|
|
||||
|
if len(cmdValues) > 0 { |
||||
|
for i, action := range cmdActions { |
||||
|
cbOptions.Actions[s3_constants.Concat(action, *limitType)] = cmdValues[i] |
||||
|
} |
||||
|
} |
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
func deleteBucketsActions(cmdBuckets []string, cbCfg *s3_pb.S3CircuitBreakerConfig, cmdActions []string, limitType *string) { |
||||
|
if cbCfg.Buckets == nil { |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
if len(cmdActions) == 0 { |
||||
|
for _, bucket := range cmdBuckets { |
||||
|
delete(cbCfg.Buckets, bucket) |
||||
|
} |
||||
|
} else { |
||||
|
for _, bucket := range cmdBuckets { |
||||
|
if cbOption, ok := cbCfg.Buckets[bucket]; ok { |
||||
|
if len(cmdActions) > 0 && cbOption.Actions != nil { |
||||
|
for _, action := range cmdActions { |
||||
|
delete(cbOption.Actions, s3_constants.Concat(action, *limitType)) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
if len(cbOption.Actions) == 0 && !cbOption.Enabled { |
||||
|
delete(cbCfg.Buckets, bucket) |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
if len(cbCfg.Buckets) == 0 { |
||||
|
cbCfg.Buckets = nil |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func deleteGlobalActions(cbCfg *s3_pb.S3CircuitBreakerConfig, cmdActions []string, limitType *string) { |
||||
|
globalOptions := cbCfg.Global |
||||
|
if globalOptions == nil { |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
if len(cmdActions) == 0 && globalOptions.Actions != nil { |
||||
|
globalOptions.Actions = nil |
||||
|
return |
||||
|
} else { |
||||
|
for _, action := range cmdActions { |
||||
|
delete(globalOptions.Actions, s3_constants.Concat(action, *limitType)) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
if len(globalOptions.Actions) == 0 && !globalOptions.Enabled { |
||||
|
cbCfg.Global = nil |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func (c *commandS3CircuitBreaker) initActionsAndValues(buckets, actions, limitType, values *string, parseValues bool) (cmdBuckets, cmdActions []string, cmdValues []int64, err error) { |
||||
|
if len(*buckets) > 0 { |
||||
|
cmdBuckets = strings.Split(*buckets, ",") |
||||
|
} |
||||
|
|
||||
|
if len(*actions) > 0 { |
||||
|
cmdActions = strings.Split(*actions, ",") |
||||
|
|
||||
|
//check action valid
|
||||
|
for _, action := range cmdActions { |
||||
|
var found bool |
||||
|
for _, allowedAction := range s3_constants.AllowedActions { |
||||
|
if allowedAction == action { |
||||
|
found = true |
||||
|
} |
||||
|
} |
||||
|
if !found { |
||||
|
return nil, nil, nil, fmt.Errorf("value(%s) of flag[-action] not valid, allowed actions: %v", *actions, s3_constants.AllowedActions) |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
if !parseValues { |
||||
|
if len(cmdActions) < 0 { |
||||
|
for _, action := range s3_constants.AllowedActions { |
||||
|
cmdActions = append(cmdActions, action) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
if len(*limitType) > 0 { |
||||
|
switch *limitType { |
||||
|
case s3_constants.LimitTypeCount: |
||||
|
elements := strings.Split(*values, ",") |
||||
|
if len(cmdActions) != len(elements) { |
||||
|
if len(elements) != 1 || len(elements) == 0 { |
||||
|
return nil, nil, nil, fmt.Errorf("count of flag[-actions] and flag[-counts] not equal") |
||||
|
} |
||||
|
v, err := strconv.Atoi(elements[0]) |
||||
|
if err != nil { |
||||
|
return nil, nil, nil, fmt.Errorf("value of -values must be a legal number(s)") |
||||
|
} |
||||
|
for range cmdActions { |
||||
|
cmdValues = append(cmdValues, int64(v)) |
||||
|
} |
||||
|
} else { |
||||
|
for _, value := range elements { |
||||
|
v, err := strconv.Atoi(value) |
||||
|
if err != nil { |
||||
|
return nil, nil, nil, fmt.Errorf("value of -values must be a legal number(s)") |
||||
|
} |
||||
|
cmdValues = append(cmdValues, int64(v)) |
||||
|
} |
||||
|
} |
||||
|
case s3_constants.LimitTypeBytes: |
||||
|
elements := strings.Split(*values, ",") |
||||
|
if len(cmdActions) != len(elements) { |
||||
|
if len(elements) != 1 || len(elements) == 0 { |
||||
|
return nil, nil, nil, fmt.Errorf("values count of -actions and -values not equal") |
||||
|
} |
||||
|
v, err := parseMBToBytes(elements[0]) |
||||
|
if err != nil { |
||||
|
return nil, nil, nil, fmt.Errorf("value of -max must be a legal number(s)") |
||||
|
} |
||||
|
for range cmdActions { |
||||
|
cmdValues = append(cmdValues, v) |
||||
|
} |
||||
|
} else { |
||||
|
for _, value := range elements { |
||||
|
v, err := parseMBToBytes(value) |
||||
|
if err != nil { |
||||
|
return nil, nil, nil, fmt.Errorf("value of -max must be a legal number(s)") |
||||
|
} |
||||
|
cmdValues = append(cmdValues, v) |
||||
|
} |
||||
|
} |
||||
|
default: |
||||
|
return nil, nil, nil, fmt.Errorf("type not valid, only 'count' and 'bytes' are allowed") |
||||
|
} |
||||
|
} else { |
||||
|
*limitType = "" |
||||
|
} |
||||
|
} |
||||
|
return cmdBuckets, cmdActions, cmdValues, nil |
||||
|
} |
||||
|
|
||||
|
func parseMBToBytes(valStr string) (int64, error) { |
||||
|
v, err := strconv.Atoi(valStr) |
||||
|
v *= 1024 * 1024 |
||||
|
return int64(v), err |
||||
|
} |
||||
@ -0,0 +1,292 @@ |
|||||
|
package shell |
||||
|
|
||||
|
import ( |
||||
|
"bytes" |
||||
|
"encoding/json" |
||||
|
"reflect" |
||||
|
"strings" |
||||
|
"testing" |
||||
|
) |
||||
|
|
||||
|
type Case struct { |
||||
|
args []string |
||||
|
result string |
||||
|
} |
||||
|
|
||||
|
var ( |
||||
|
TestCases = []*Case{ |
||||
|
//add circuit breaker config for global
|
||||
|
{ |
||||
|
args: strings.Split("-global -type Count -actions Read,Write -values 500,200", " "), |
||||
|
result: `{ |
||||
|
"global": { |
||||
|
"enabled": true, |
||||
|
"actions": { |
||||
|
"Read:Count": "500", |
||||
|
"Write:Count": "200" |
||||
|
} |
||||
|
} |
||||
|
}`, |
||||
|
}, |
||||
|
|
||||
|
//disable global config
|
||||
|
{ |
||||
|
args: strings.Split("-global -disable", " "), |
||||
|
result: `{ |
||||
|
"global": { |
||||
|
"actions": { |
||||
|
"Read:Count": "500", |
||||
|
"Write:Count": "200" |
||||
|
} |
||||
|
} |
||||
|
}`, |
||||
|
}, |
||||
|
|
||||
|
//add circuit breaker config for buckets x,y,z
|
||||
|
{ |
||||
|
args: strings.Split("-buckets x,y,z -type Count -actions Read,Write -values 200,100", " "), |
||||
|
result: `{ |
||||
|
"global": { |
||||
|
"actions": { |
||||
|
"Read:Count": "500", |
||||
|
"Write:Count": "200" |
||||
|
} |
||||
|
}, |
||||
|
"buckets": { |
||||
|
"x": { |
||||
|
"enabled": true, |
||||
|
"actions": { |
||||
|
"Read:Count": "200", |
||||
|
"Write:Count": "100" |
||||
|
} |
||||
|
}, |
||||
|
"y": { |
||||
|
"enabled": true, |
||||
|
"actions": { |
||||
|
"Read:Count": "200", |
||||
|
"Write:Count": "100" |
||||
|
} |
||||
|
}, |
||||
|
"z": { |
||||
|
"enabled": true, |
||||
|
"actions": { |
||||
|
"Read:Count": "200", |
||||
|
"Write:Count": "100" |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
}`, |
||||
|
}, |
||||
|
|
||||
|
//disable circuit breaker config of x
|
||||
|
{ |
||||
|
args: strings.Split("-buckets x -disable", " "), |
||||
|
result: `{ |
||||
|
"global": { |
||||
|
"actions": { |
||||
|
"Read:Count": "500", |
||||
|
"Write:Count": "200" |
||||
|
} |
||||
|
}, |
||||
|
"buckets": { |
||||
|
"x": { |
||||
|
"actions": { |
||||
|
"Read:Count": "200", |
||||
|
"Write:Count": "100" |
||||
|
} |
||||
|
}, |
||||
|
"y": { |
||||
|
"enabled": true, |
||||
|
"actions": { |
||||
|
"Read:Count": "200", |
||||
|
"Write:Count": "100" |
||||
|
} |
||||
|
}, |
||||
|
"z": { |
||||
|
"enabled": true, |
||||
|
"actions": { |
||||
|
"Read:Count": "200", |
||||
|
"Write:Count": "100" |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
}`, |
||||
|
}, |
||||
|
|
||||
|
//delete circuit breaker config of x
|
||||
|
{ |
||||
|
args: strings.Split("-buckets x -delete", " "), |
||||
|
result: `{ |
||||
|
"global": { |
||||
|
"actions": { |
||||
|
"Read:Count": "500", |
||||
|
"Write:Count": "200" |
||||
|
} |
||||
|
}, |
||||
|
"buckets": { |
||||
|
"y": { |
||||
|
"enabled": true, |
||||
|
"actions": { |
||||
|
"Read:Count": "200", |
||||
|
"Write:Count": "100" |
||||
|
} |
||||
|
}, |
||||
|
"z": { |
||||
|
"enabled": true, |
||||
|
"actions": { |
||||
|
"Read:Count": "200", |
||||
|
"Write:Count": "100" |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
}`, |
||||
|
}, |
||||
|
|
||||
|
//configure the circuit breaker for the size of the uploaded file for bucket x,y
|
||||
|
{ |
||||
|
args: strings.Split("-buckets x,y -type MB -actions Write -values 1024", " "), |
||||
|
result: `{ |
||||
|
"global": { |
||||
|
"actions": { |
||||
|
"Read:Count": "500", |
||||
|
"Write:Count": "200" |
||||
|
} |
||||
|
}, |
||||
|
"buckets": { |
||||
|
"x": { |
||||
|
"enabled": true, |
||||
|
"actions": { |
||||
|
"Write:MB": "1073741824" |
||||
|
} |
||||
|
}, |
||||
|
"y": { |
||||
|
"enabled": true, |
||||
|
"actions": { |
||||
|
"Read:Count": "200", |
||||
|
"Write:Count": "100", |
||||
|
"Write:MB": "1073741824" |
||||
|
} |
||||
|
}, |
||||
|
"z": { |
||||
|
"enabled": true, |
||||
|
"actions": { |
||||
|
"Read:Count": "200", |
||||
|
"Write:Count": "100" |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
}`, |
||||
|
}, |
||||
|
|
||||
|
//delete the circuit breaker configuration for the size of the uploaded file of bucket x,y
|
||||
|
{ |
||||
|
args: strings.Split("-buckets x,y -type MB -actions Write -delete", " "), |
||||
|
result: `{ |
||||
|
"global": { |
||||
|
"actions": { |
||||
|
"Read:Count": "500", |
||||
|
"Write:Count": "200" |
||||
|
} |
||||
|
}, |
||||
|
"buckets": { |
||||
|
"x": { |
||||
|
"enabled": true |
||||
|
}, |
||||
|
"y": { |
||||
|
"enabled": true, |
||||
|
"actions": { |
||||
|
"Read:Count": "200", |
||||
|
"Write:Count": "100" |
||||
|
} |
||||
|
}, |
||||
|
"z": { |
||||
|
"enabled": true, |
||||
|
"actions": { |
||||
|
"Read:Count": "200", |
||||
|
"Write:Count": "100" |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
}`, |
||||
|
}, |
||||
|
|
||||
|
//enable global circuit breaker config (without -disable flag)
|
||||
|
{ |
||||
|
args: strings.Split("-global", " "), |
||||
|
result: `{ |
||||
|
"global": { |
||||
|
"enabled": true, |
||||
|
"actions": { |
||||
|
"Read:Count": "500", |
||||
|
"Write:Count": "200" |
||||
|
} |
||||
|
}, |
||||
|
"buckets": { |
||||
|
"x": { |
||||
|
"enabled": true |
||||
|
}, |
||||
|
"y": { |
||||
|
"enabled": true, |
||||
|
"actions": { |
||||
|
"Read:Count": "200", |
||||
|
"Write:Count": "100" |
||||
|
} |
||||
|
}, |
||||
|
"z": { |
||||
|
"enabled": true, |
||||
|
"actions": { |
||||
|
"Read:Count": "200", |
||||
|
"Write:Count": "100" |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
}`, |
||||
|
}, |
||||
|
|
||||
|
//clear all circuit breaker config
|
||||
|
{ |
||||
|
args: strings.Split("-delete", " "), |
||||
|
result: `{ |
||||
|
|
||||
|
}`, |
||||
|
}, |
||||
|
} |
||||
|
) |
||||
|
|
||||
|
func TestCircuitBreakerShell(t *testing.T) { |
||||
|
var writeBuf bytes.Buffer |
||||
|
cmd := &commandS3CircuitBreaker{} |
||||
|
LoadConfig = func(commandEnv *CommandEnv, dir string, file string, buf *bytes.Buffer) error { |
||||
|
_, err := buf.Write(writeBuf.Bytes()) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
writeBuf.Reset() |
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
for i, tc := range TestCases { |
||||
|
err := cmd.Do(tc.args, nil, &writeBuf) |
||||
|
if err != nil { |
||||
|
t.Fatal(err) |
||||
|
} |
||||
|
if i != 0 { |
||||
|
result := writeBuf.String() |
||||
|
|
||||
|
actual := make(map[string]interface{}) |
||||
|
err := json.Unmarshal([]byte(result), &actual) |
||||
|
if err != nil { |
||||
|
t.Error(err) |
||||
|
} |
||||
|
|
||||
|
expect := make(map[string]interface{}) |
||||
|
err = json.Unmarshal([]byte(result), &expect) |
||||
|
if err != nil { |
||||
|
t.Error(err) |
||||
|
} |
||||
|
if !reflect.DeepEqual(actual, expect) { |
||||
|
t.Fatal("result of s3 circuit breaker shell command is unexpect!") |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
Write
Preview
Loading…
Cancel
Save
Reference in new issue