lsm/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2025-11-24 16:53:14 +08:00
Code Issues Actions 17 Packages Projects Releases Wiki Activity

Use filerGroup for s3 buckets collection prefix (#4465)

Browse Source 
* Use filerGroup for s3 buckets collection prefix

* Fix templates

* Remove flags

* Remove s3CollectionPrefix
This commit is contained in:
SmsS4
2023-05-16 20:09:43 +03:30
committed by GitHub
parent b7f011f777
commit 17e91d2917
14 changed files with 40 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
weed/s3api/filer_util.go
View File

@@ -107,6 +107,13 @@ func (s3a *S3ApiServer) updateEntry(parentDirectoryPath string, newEntry *filer_
return err
}
func (s3a *S3ApiServer) getCollectionName(bucket string) string {
if s3a.option.FilerGroup != "" {
return fmt.Sprintf("%s_%s", s3a.option.FilerGroup, bucket)
}
return bucket
}
func objectKey(key *string) *string {
if strings.HasPrefix(*key, "/") {
t := (*key)[1:]

6
weed/s3api/s3api_bucket_handlers.go
View File

@@ -104,7 +104,7 @@ func (s3a *S3ApiServer) PutBucketHandler(w http.ResponseWriter, r *http.Request)
return fmt.Errorf("list collections: %v", err)
} else {
for _, c := range resp.Collections {
if bucket == c.Name {
if s3a.getCollectionName(bucket) == c.Name {
errCode = s3err.ErrBucketAlreadyExists
break
}
@@ -174,7 +174,7 @@ func (s3a *S3ApiServer) DeleteBucketHandler(w http.ResponseWriter, r *http.Reque
// delete collection
deleteCollectionRequest := &filer_pb.DeleteCollectionRequest{
Collection: bucket,
Collection: s3a.getCollectionName(bucket),
}
glog.V(1).Infof("delete collection: %v", deleteCollectionRequest)
@@ -304,7 +304,7 @@ func (s3a *S3ApiServer) GetBucketLifecycleConfigurationHandler(w http.ResponseWr
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
return
}
ttls := fc.GetCollectionTtls(bucket)
ttls := fc.GetCollectionTtls(s3a.getCollectionName(bucket))
if len(ttls) == 0 {
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchLifecycleConfiguration)
return

4
weed/s3api/s3api_object_copy_handlers.go
View File

@@ -100,7 +100,7 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request
}
glog.V(2).Infof("copy from %s to %s", srcUrl, dstUrl)
destination := fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, dstBucket, dstObject)
etag, errCode := s3a.putToFiler(r, dstUrl, resp.Body, destination)
etag, errCode := s3a.putToFiler(r, dstUrl, resp.Body, destination, dstBucket)
if errCode != s3err.ErrNone {
s3err.WriteErrorResponse(w, r, errCode)
@@ -185,7 +185,7 @@ func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Req
glog.V(2).Infof("copy from %s to %s", srcUrl, dstUrl)
destination := fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, dstBucket, dstObject)
etag, errCode := s3a.putToFiler(r, dstUrl, dataReader, destination)
etag, errCode := s3a.putToFiler(r, dstUrl, dataReader, destination, dstBucket)
if errCode != s3err.ErrNone {
s3err.WriteErrorResponse(w, r, errCode)

10
weed/s3api/s3api_object_handlers.go
View File

@@ -115,7 +115,7 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request)
dataReader = mimeDetect(r, dataReader)
}
etag, errCode := s3a.putToFiler(r, uploadUrl, dataReader, "")
etag, errCode := s3a.putToFiler(r, uploadUrl, dataReader, "", bucket)
if errCode != s3err.ErrNone {
s3err.WriteErrorResponse(w, r, errCode)
@@ -457,7 +457,7 @@ func passThroughResponse(proxyResponse *http.Response, w http.ResponseWriter) (s
return statusCode
}
func (s3a *S3ApiServer) putToFiler(r *http.Request, uploadUrl string, dataReader io.Reader, destination string) (etag string, code s3err.ErrorCode) {
func (s3a *S3ApiServer) putToFiler(r *http.Request, uploadUrl string, dataReader io.Reader, destination string, bucket string) (etag string, code s3err.ErrorCode) {
hash := md5.New()
var body = io.TeeReader(dataReader, hash)
@@ -474,6 +474,12 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, uploadUrl string, dataReader
proxyReq.Header.Set(s3_constants.SeaweedStorageDestinationHeader, destination)
}
if s3a.option.FilerGroup != "" {
query := proxyReq.URL.Query()
query.Add("collection", s3a.getCollectionName(bucket))
proxyReq.URL.RawQuery = query.Encode()
}
for header, values := range r.Header {
for _, value := range values {
proxyReq.Header.Add(header, value)

2
weed/s3api/s3api_object_handlers_postpolicy.go
View File

@@ -115,7 +115,7 @@ func (s3a *S3ApiServer) PostPolicyBucketHandler(w http.ResponseWriter, r *http.R
uploadUrl := fmt.Sprintf("http://%s%s/%s%s", s3a.option.Filer.ToHttpAddress(), s3a.option.BucketsPath, bucket, urlEscapeObject(object))
etag, errCode := s3a.putToFiler(r, uploadUrl, fileBody, "")
etag, errCode := s3a.putToFiler(r, uploadUrl, fileBody, "", bucket)
if errCode != s3err.ErrNone {
s3err.WriteErrorResponse(w, r, errCode)

2
weed/s3api/s3api_object_multipart_handlers.go
View File

@@ -255,7 +255,7 @@ func (s3a *S3ApiServer) PutObjectPartHandler(w http.ResponseWriter, r *http.Requ
}
destination := fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, bucket, object)
etag, errCode := s3a.putToFiler(r, uploadUrl, dataReader, destination)
etag, errCode := s3a.putToFiler(r, uploadUrl, dataReader, destination, bucket)
if errCode != s3err.ErrNone {
s3err.WriteErrorResponse(w, r, errCode)
return

1
weed/s3api/s3api_server.go
View File

@@ -31,6 +31,7 @@ type S3ApiServerOption struct {
AllowDeleteBucketNotEmpty bool
LocalFilerSocket string
DataCenter string
FilerGroup string
}
type S3ApiServer struct {