lsm/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2025-09-23 03:33:34 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Improve S3 request signing performance

Browse Source 
This change is caching HMAC hashers for repeated use in subsequent
requests and chunks, so they don't have to be initialized from
scratch every time.
On my local computer this gives me ~5-6 times faster signature
calculation and ~5-6.5% more throughput in S3 requests. The smaller
the payload the better the throughput gets.
This commit is contained in:
Patrick Schmidt
2023-01-20 13:12:30 +01:00
committed by Chris Lu
parent f07876cb23
commit cdd817edf9
4 changed files with 112 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

78
weed/s3api/auth_signature_v4.go
View File

@@ -23,6 +23,7 @@ import (
"crypto/sha256"
"crypto/subtle"
"encoding/hex"
"hash"
"io"
"net/http"
"net/url"
@@ -30,6 +31,7 @@ import (
"sort"
"strconv"
"strings"
"sync"
"time"
"unicode/utf8"
@@ -151,14 +153,14 @@ func (iam *IdentityAccessManagement) doesSignatureMatch(hashedPayload string, r
// Get string to sign from canonical request.
stringToSign := getStringToSign(canonicalRequest, t, signV4Values.Credential.getScope())
// Get hmac signing key.
signingKey := getSigningKey(cred.SecretKey,
// Calculate signature.
newSignature := iam.getSignature(
cred.SecretKey,
signV4Values.Credential.scope.date,
signV4Values.Credential.scope.region,
signV4Values.Credential.scope.service)
// Calculate signature.
newSignature := getSignature(signingKey, stringToSign)
signV4Values.Credential.scope.service,
stringToSign,
)
// Verify if signature match.
if !compareSignatureV4(newSignature, signV4Values.Signature) {
@@ -325,11 +327,14 @@ func (iam *IdentityAccessManagement) doesPolicySignatureV4Match(formValues http.
return s3err.ErrInvalidAccessKeyID
}
// Get signing key.
signingKey := getSigningKey(cred.SecretKey, credHeader.scope.date, credHeader.scope.region, credHeader.scope.service)
// Get signature.
newSignature := getSignature(signingKey, formValues.Get("Policy"))
newSignature := iam.getSignature(
cred.SecretKey,
credHeader.scope.date,
credHeader.scope.region,
credHeader.scope.service,
formValues.Get("Policy"),
)
// Verify signature.
if !compareSignatureV4(newSignature, formValues.Get("X-Amz-Signature")) {
@@ -442,14 +447,14 @@ func (iam *IdentityAccessManagement) doesPresignedSignatureMatch(hashedPayload s
// Get string to sign from canonical request.
presignedStringToSign := getStringToSign(presignedCanonicalReq, t, pSignValues.Credential.getScope())
// Get hmac presigned signing key.
presignedSigningKey := getSigningKey(cred.SecretKey,
// Get new signature.
newSignature := iam.getSignature(
cred.SecretKey,
pSignValues.Credential.scope.date,
pSignValues.Credential.scope.region,
pSignValues.Credential.scope.service)
// Get new signature.
newSignature := getSignature(presignedSigningKey, presignedStringToSign)
pSignValues.Credential.scope.service,
presignedStringToSign,
)
// Verify signature.
if !compareSignatureV4(req.URL.Query().Get("X-Amz-Signature"), newSignature) {
@@ -458,6 +463,38 @@ func (iam *IdentityAccessManagement) doesPresignedSignatureMatch(hashedPayload s
return identity, s3err.ErrNone
}
// getSignature
func (iam *IdentityAccessManagement) getSignature(secretKey string, t time.Time, region string, service string, stringToSign string) string {
date := t.Format(yyyymmdd)
hashID := "AWS4" + secretKey + "/" + date + "/" + region + "/" + service + "/" + "aws4_request"
iam.hashMu.RLock()
pool, ok := iam.hashes[hashID]
iam.hashMu.RUnlock()
if !ok {
iam.hashMu.Lock()
if pool, ok = iam.hashes[hashID]; !ok {
pool = &sync.Pool{
New: func() any {
signingKey := getSigningKey(secretKey, date, region, service)
return hmac.New(sha256.New, signingKey)
},
}
iam.hashes[hashID] = pool
}
iam.hashMu.Unlock()
}
h := pool.Get().(hash.Hash)
h.Reset()
h.Write([]byte(stringToSign))
sig := hex.EncodeToString(h.Sum(nil))
pool.Put(h)
return sig
}
func contains(list []string, elem string) bool {
for _, t := range list {
if t == elem {
@@ -674,19 +711,14 @@ func sumHMAC(key []byte, data []byte) []byte {
}
// getSigningKey hmac seed to calculate final signature.
func getSigningKey(secretKey string, t time.Time, region string, service string) []byte {
date := sumHMAC([]byte("AWS4"+secretKey), []byte(t.Format(yyyymmdd)))
func getSigningKey(secretKey string, time string, region string, service string) []byte {
date := sumHMAC([]byte("AWS4"+secretKey), []byte(time))
regionBytes := sumHMAC(date, []byte(region))
serviceBytes := sumHMAC(regionBytes, []byte(service))
signingKey := sumHMAC(serviceBytes, []byte("aws4_request"))
return signingKey
}
// getSignature final signature in hexadecimal form.
func getSignature(signingKey []byte, stringToSign string) string {
return hex.EncodeToString(sumHMAC(signingKey, []byte(stringToSign)))
}
// getCanonicalHeaders generate a list of request headers with their values
func getCanonicalHeaders(signedHeaders http.Header) string {
var headers []string