lsm/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2025-11-24 08:46:54 +08:00
Code Issues Actions 17 Packages Projects Releases Wiki Activity

some code optimizations

Browse Source
This commit is contained in:
石昌林
2022-06-17 19:07:39 +08:00
parent 37df209195
commit 3dd60529c5
8 changed files with 256 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
weed/s3api/auth_credentials_subscribe.go
View File

@@ -5,7 +5,7 @@ import (
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/s3api/s3_config"
"github.com/chrislusf/seaweedfs/weed/s3api/s3_constants"
"github.com/chrislusf/seaweedfs/weed/util"
)
@@ -53,7 +53,7 @@ func (s3a *S3ApiServer) onIamConfigUpdate(dir, filename string, content []byte)
//reload circuit breaker config
func (s3a *S3ApiServer) onCircuitBreakerConfigUpdate(dir, filename string, content []byte) error {
if dir == s3_config.CircuitBreakerConfigDir && filename == s3_config.CircuitBreakerConfigFile {
if dir == s3_constants.CircuitBreakerConfigDir && filename == s3_constants.CircuitBreakerConfigFile {
if err := s3a.cb.LoadS3ApiConfigurationFromBytes(content); err != nil {
return err
}

5
weed/s3api/s3_config/s3_config.go → weed/s3api/s3_constants/s3_config.go
View File

@@ -1,14 +1,13 @@
package s3_config
package s3_constants
import (
"github.com/chrislusf/seaweedfs/weed/s3api/s3_constants"
"strings"
)
var (
CircuitBreakerConfigDir = "/etc/s3"
CircuitBreakerConfigFile = "circuit_breaker.json"
AllowedActions = []string{s3_constants.ACTION_READ, s3_constants.ACTION_WRITE, s3_constants.ACTION_LIST, s3_constants.ACTION_TAGGING, s3_constants.ACTION_ADMIN}
AllowedActions = []string{ACTION_READ, ACTION_WRITE, ACTION_LIST, ACTION_TAGGING, ACTION_ADMIN}
LimitTypeCount = "count"
LimitTypeBytes = "bytes"
Separator = ":"

16
weed/s3api/s3api_circuit_breaker.go
View File

@@ -7,7 +7,7 @@ import (
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/pb/s3_pb"
"github.com/chrislusf/seaweedfs/weed/s3api/s3_config"
"github.com/chrislusf/seaweedfs/weed/s3api/s3_constants"
"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
"github.com/gorilla/mux"
"go.uber.org/atomic"
@@ -29,7 +29,7 @@ func NewCircuitBreaker(option *S3ApiServerOption) *CircuitBreaker {
}
err := pb.WithFilerClient(false, option.Filer, option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
content, err := filer.ReadInsideFiler(client, s3_config.CircuitBreakerConfigDir, s3_config.CircuitBreakerConfigFile)
content, err := filer.ReadInsideFiler(client, s3_constants.CircuitBreakerConfigDir, s3_constants.CircuitBreakerConfigFile)
if err != nil {
return fmt.Errorf("read S3 circuit breaker config: %v", err)
}
@@ -73,7 +73,7 @@ func (cb *CircuitBreaker) loadCircuitBreakerConfig(cfg *s3_pb.S3CircuitBreakerCo
for bucket, cbOptions := range cfg.Buckets {
if cbOptions.Enabled {
for action, limit := range cbOptions.Actions {
limitations[s3_config.Concat(bucket, action)] = limit
limitations[s3_constants.Concat(bucket, action)] = limit
}
}
}
@@ -110,7 +110,7 @@ func (cb *CircuitBreaker) Limit(f func(w http.ResponseWriter, r *http.Request),
func (cb *CircuitBreaker) limit(r *http.Request, bucket string, action string) (rollback []func(), errCode s3err.ErrorCode) {
//bucket simultaneous request count
bucketCountRollBack, errCode := cb.loadCounterAndCompare(bucket, action, s3_config.LimitTypeCount, 1, s3err.ErrTooManyRequest)
bucketCountRollBack, errCode := cb.loadCounterAndCompare(bucket, action, s3_constants.LimitTypeCount, 1, s3err.ErrTooManyRequest)
if bucketCountRollBack != nil {
rollback = append(rollback, bucketCountRollBack)
}
@@ -119,7 +119,7 @@ func (cb *CircuitBreaker) limit(r *http.Request, bucket string, action string) (
}
//bucket simultaneous request content bytes
bucketContentLengthRollBack, errCode := cb.loadCounterAndCompare(bucket, action, s3_config.LimitTypeBytes, r.ContentLength, s3err.ErrRequestBytesExceed)
bucketContentLengthRollBack, errCode := cb.loadCounterAndCompare(bucket, action, s3_constants.LimitTypeBytes, r.ContentLength, s3err.ErrRequestBytesExceed)
if bucketContentLengthRollBack != nil {
rollback = append(rollback, bucketContentLengthRollBack)
}
@@ -128,7 +128,7 @@ func (cb *CircuitBreaker) limit(r *http.Request, bucket string, action string) (
}
//global simultaneous request count
globalCountRollBack, errCode := cb.loadCounterAndCompare("", action, s3_config.LimitTypeCount, 1, s3err.ErrTooManyRequest)
globalCountRollBack, errCode := cb.loadCounterAndCompare("", action, s3_constants.LimitTypeCount, 1, s3err.ErrTooManyRequest)
if globalCountRollBack != nil {
rollback = append(rollback, globalCountRollBack)
}
@@ -137,7 +137,7 @@ func (cb *CircuitBreaker) limit(r *http.Request, bucket string, action string) (
}
//global simultaneous request content bytes
globalContentLengthRollBack, errCode := cb.loadCounterAndCompare("", action, s3_config.LimitTypeBytes, r.ContentLength, s3err.ErrRequestBytesExceed)
globalContentLengthRollBack, errCode := cb.loadCounterAndCompare("", action, s3_constants.LimitTypeBytes, r.ContentLength, s3err.ErrRequestBytesExceed)
if globalContentLengthRollBack != nil {
rollback = append(rollback, globalContentLengthRollBack)
}
@@ -148,7 +148,7 @@ func (cb *CircuitBreaker) limit(r *http.Request, bucket string, action string) (
}
func (cb *CircuitBreaker) loadCounterAndCompare(bucket, action, limitType string, inc int64, errCode s3err.ErrorCode) (f func(), e s3err.ErrorCode) {
key := s3_config.Concat(bucket, action, limitType)
key := s3_constants.Concat(bucket, action, limitType)
e = s3err.ErrNone
if max, ok := cb.limitations[key]; ok {
counter, exists := cb.counters[key]

24
weed/s3api/s3api_circuit_breaker_test.go
View File

@@ -2,7 +2,6 @@ package s3api
import (
"github.com/chrislusf/seaweedfs/weed/pb/s3_pb"
"github.com/chrislusf/seaweedfs/weed/s3api/s3_config"
"github.com/chrislusf/seaweedfs/weed/s3api/s3_constants"
"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
"go.uber.org/atomic"
@@ -27,30 +26,31 @@ var (
bucket = "/test"
action = s3_constants.ACTION_READ
TestLimitCases = []*TestLimitCase{
{action, s3_config.LimitTypeCount, 5, 5, 6, 1024, 5},
{action, s3_config.LimitTypeCount, 6, 6, 6, 1024, 6},
{action, s3_config.LimitTypeCount, 5, 6, 6, 1024, 5},
{action, s3_config.LimitTypeBytes, 1024, 1024, 6, 200, 5},
{action, s3_config.LimitTypeBytes, 1200, 1200, 6, 200, 6},
{action, s3_config.LimitTypeBytes, 11990, 11990, 60, 200, 59},
{action, s3_config.LimitTypeBytes, 11790, 11990, 60, 200, 58},
{action, s3_constants.LimitTypeCount, 5, 5, 6, 1024, 5},
{action, s3_constants.LimitTypeCount, 6, 6, 6, 1024, 6},
{action, s3_constants.LimitTypeCount, 5, 6, 6, 1024, 5},
{action, s3_constants.LimitTypeBytes, 1024, 1024, 6, 200, 5},
{action, s3_constants.LimitTypeBytes, 1200, 1200, 6, 200, 6},
{action, s3_constants.LimitTypeBytes, 11990, 11990, 60, 200, 59},
{action, s3_constants.LimitTypeBytes, 11790, 11990, 60, 200, 58},
}
)
func TestLimit(t *testing.T) {
for _, tc := range TestLimitCases {
circuitBreakerConfig := &s3_pb.S3CircuitBreakerConfig{
Global: &s3_pb.CbOptions{
Global: &s3_pb.S3CircuitBreakerOptions{
Enabled: true,
Actions: map[string]int64{
s3_config.Concat(tc.actionName, tc.limitType): tc.globalLimitValue,
s3_constants.Concat(tc.actionName, tc.limitType): tc.globalLimitValue,
s3_constants.Concat(tc.actionName, tc.limitType): tc.globalLimitValue,
},
},
Buckets: map[string]*s3_pb.CbOptions{
Buckets: map[string]*s3_pb.S3CircuitBreakerOptions{
bucket: {
Enabled: true,
Actions: map[string]int64{
s3_config.Concat(tc.actionName, tc.limitType): tc.bucketLimitValue,
s3_constants.Concat(tc.actionName, tc.limitType): tc.bucketLimitValue,
},
},
},