lsm/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2025-08-20 06:35:56 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

S3: support for the X-Forwarded-Prefix header (#7068)

Browse Source 
* support for the X-Forwarded-Prefix header

* remove comments

* refactoring

* refactoring

* path.Clean
This commit is contained in:
Chris Lu 2025-08-01 13:07:54 -07:00 committed by GitHub
parent 52d87f1d29
commit f1eb4dd427
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 328 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

96
weed/s3api/auth_signature_v4.go
View File

@ -24,6 +24,7 @@ import (
"crypto/subtle"
"encoding/hex"
"net/http"
"path"
"regexp"
"sort"
"strconv"
@ -154,13 +155,14 @@ func (iam *IdentityAccessManagement) doesSignatureMatch(hashedPayload string, r
}
bucket, object := s3_constants.GetBucketAndObject(r)
if !identity.canDo(s3_constants.ACTION_WRITE, bucket, object) {
canDoResult := identity.canDo(s3_constants.ACTION_WRITE, bucket, object)
if !canDoResult {
return nil, s3err.ErrAccessDenied
}
// Extract date, if not present throw error.
var dateStr string
if dateStr = req.Header.Get(http.CanonicalHeaderKey("x-amz-date")); dateStr == "" {
if dateStr = req.Header.Get("x-amz-date"); dateStr == "" {
if dateStr = r.Header.Get("Date"); dateStr == "" {
return nil, s3err.ErrMissingDateHeader
}
@ -174,25 +176,67 @@ func (iam *IdentityAccessManagement) doesSignatureMatch(hashedPayload string, r
// Query string.
queryStr := req.URL.Query().Encode()
// Check if reverse proxy is forwarding with prefix
if forwardedPrefix := r.Header.Get("X-Forwarded-Prefix"); forwardedPrefix != "" {
// Try signature verification with the forwarded prefix first.
// This handles cases where reverse proxies strip URL prefixes and add the X-Forwarded-Prefix header.
errCode = iam.verifySignatureWithPath(extractedSignedHeaders, hashedPayload, queryStr, path.Clean(forwardedPrefix+req.URL.Path), req.Method, foundCred.SecretKey, t, signV4Values)
if errCode == s3err.ErrNone {
return identity, errCode
}
}
// Try normal signature verification (without prefix)
errCode = iam.verifySignatureWithPath(extractedSignedHeaders, hashedPayload, queryStr, req.URL.Path, req.Method, foundCred.SecretKey, t, signV4Values)
if errCode == s3err.ErrNone {
return identity, errCode
}
return nil, errCode
}
// verifySignatureWithPath verifies signature with a given path (used for both normal and prefixed paths).
func (iam *IdentityAccessManagement) verifySignatureWithPath(extractedSignedHeaders http.Header, hashedPayload, queryStr, urlPath, method, secretKey string, t time.Time, signV4Values signValues) s3err.ErrorCode {
// Get canonical request.
canonicalRequest := getCanonicalRequest(extractedSignedHeaders, hashedPayload, queryStr, req.URL.Path, req.Method)
canonicalRequest := getCanonicalRequest(extractedSignedHeaders, hashedPayload, queryStr, urlPath, method)
// Get string to sign from canonical request.
stringToSign := getStringToSign(canonicalRequest, t, signV4Values.Credential.getScope())
// Get hmac signing key.
signingKey := getSigningKey(foundCred.SecretKey, signV4Values.Credential.scope.date.Format(yyyymmdd), signV4Values.Credential.scope.region, "s3")
signingKey := getSigningKey(secretKey, signV4Values.Credential.scope.date.Format(yyyymmdd), signV4Values.Credential.scope.region, "s3")
// Calculate signature.
newSignature := getSignature(signingKey, stringToSign)
// Verify if signature match.
if !compareSignatureV4(newSignature, signV4Values.Signature) {
return nil, s3err.ErrSignatureDoesNotMatch
return s3err.ErrSignatureDoesNotMatch
}
// Return error none.
return identity, s3err.ErrNone
return s3err.ErrNone
}
// verifyPresignedSignatureWithPath verifies presigned signature with a given path (used for both normal and prefixed paths).
func (iam *IdentityAccessManagement) verifyPresignedSignatureWithPath(extractedSignedHeaders http.Header, hashedPayload, queryStr, urlPath, method, secretKey string, t time.Time, credHeader credentialHeader, signature string) s3err.ErrorCode {
// Get canonical request.
canonicalRequest := getCanonicalRequest(extractedSignedHeaders, hashedPayload, queryStr, urlPath, method)
// Get string to sign from canonical request.
stringToSign := getStringToSign(canonicalRequest, t, credHeader.getScope())
// Get hmac signing key.
signingKey := getSigningKey(secretKey, credHeader.scope.date.Format(yyyymmdd), credHeader.scope.region, "s3")
// Calculate expected signature.
expectedSignature := getSignature(signingKey, stringToSign)
// Verify if signature match.
if !compareSignatureV4(expectedSignature, signature) {
return s3err.ErrSignatureDoesNotMatch
}
return s3err.ErrNone
}
// Simple implementation for presigned signature verification
@ -284,24 +328,24 @@ func (iam *IdentityAccessManagement) doesPresignedSignatureMatch(hashedPayload s
queryForCanonical.Del("X-Amz-Signature")
queryStr := strings.Replace(queryForCanonical.Encode(), "+", "%20", -1)
// Get canonical request
canonicalRequest := getCanonicalRequest(extractedSignedHeaders, hashedPayload, queryStr, r.URL.Path, r.Method)
// Get string to sign
stringToSign := getStringToSign(canonicalRequest, t, credHeader.getScope())
// Get signing key
signingKey := getSigningKey(foundCred.SecretKey, credHeader.scope.date.Format(yyyymmdd), credHeader.scope.region, "s3")
// Calculate expected signature
expectedSignature := getSignature(signingKey, stringToSign)
// Verify signature
if !compareSignatureV4(expectedSignature, signature) {
return nil, s3err.ErrSignatureDoesNotMatch
var errCode s3err.ErrorCode
// Check if reverse proxy is forwarding with prefix for presigned URLs
if forwardedPrefix := r.Header.Get("X-Forwarded-Prefix"); forwardedPrefix != "" {
// Try signature verification with the forwarded prefix first.
// This handles cases where reverse proxies strip URL prefixes and add the X-Forwarded-Prefix header.
errCode = iam.verifyPresignedSignatureWithPath(extractedSignedHeaders, hashedPayload, queryStr, path.Clean(forwardedPrefix+r.URL.Path), r.Method, foundCred.SecretKey, t, credHeader, signature)
if errCode == s3err.ErrNone {
return identity, errCode
}
}
return identity, s3err.ErrNone
// Try normal signature verification (without prefix)
errCode = iam.verifyPresignedSignatureWithPath(extractedSignedHeaders, hashedPayload, queryStr, r.URL.Path, r.Method, foundCred.SecretKey, t, credHeader, signature)
if errCode == s3err.ErrNone {
return identity, errCode
}
return nil, errCode
}
// credentialHeader data type represents structured form of Credential
@ -444,6 +488,12 @@ func extractSignedHeaders(signedHeaders []string, r *http.Request) (http.Header,
// extractHostHeader returns the value of host header if available.
func extractHostHeader(r *http.Request) string {
// Check for X-Forwarded-Host header first, which is set by reverse proxies
if forwardedHost := r.Header.Get("X-Forwarded-Host"); forwardedHost != "" {
// Using reverse proxy with X-Forwarded-Host.
return forwardedHost
}
hostHeaderValue := r.Host
// For standard requests, this should be fine.
if r.Host != "" {

255
weed/s3api/auto_signature_v4_test.go
View File

@ -16,6 +16,7 @@ import (
"time"
"unicode/utf8"
"github.com/gorilla/mux"
"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
@ -254,6 +255,260 @@ func preSignV4(iam *IdentityAccessManagement, req *http.Request, accessKey, secr
return nil
}
// newTestIAM creates a test IAM with a standard test user
func newTestIAM() *IdentityAccessManagement {
iam := &IdentityAccessManagement{}
iam.identities = []*Identity{
{
Name: "testuser",
Credentials: []*Credential{{AccessKey: "AKIAIOSFODNN7EXAMPLE", SecretKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"}},
Actions: []Action{s3_constants.ACTION_ADMIN, s3_constants.ACTION_READ, s3_constants.ACTION_WRITE},
},
}
// Initialize the access key map for lookup
iam.accessKeyIdent = make(map[string]*Identity)
iam.accessKeyIdent["AKIAIOSFODNN7EXAMPLE"] = iam.identities[0]
return iam
}
// Test X-Forwarded-Prefix support for reverse proxy scenarios
func TestSignatureV4WithForwardedPrefix(t *testing.T) {
tests := []struct {
name string
forwardedPrefix string
expectedPath string
}{
{
name: "prefix without trailing slash",
forwardedPrefix: "/s3",
expectedPath: "/s3/test-bucket/test-object",
},
{
name: "prefix with trailing slash",
forwardedPrefix: "/s3/",
expectedPath: "/s3/test-bucket/test-object",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
iam := newTestIAM()
// Create a request with X-Forwarded-Prefix header
r, err := newTestRequest("GET", "https://example.com/test-bucket/test-object", 0, nil)
if err != nil {
t.Fatalf("Failed to create test request: %v", err)
}
// Set the mux variables manually since we're not going through the actual router
r = mux.SetURLVars(r, map[string]string{
"bucket": "test-bucket",
"object": "test-object",
})
r.Header.Set("X-Forwarded-Prefix", tt.forwardedPrefix)
r.Header.Set("Host", "example.com")
r.Header.Set("X-Forwarded-Host", "example.com")
// Sign the request with the expected normalized path
signV4WithPath(r, "AKIAIOSFODNN7EXAMPLE", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", tt.expectedPath)
// Test signature verification
_, errCode := iam.doesSignatureMatch(getContentSha256Cksum(r), r)
if errCode != s3err.ErrNone {
t.Errorf("Expected successful signature validation with X-Forwarded-Prefix %q, got error: %v (code: %d)", tt.forwardedPrefix, errCode, int(errCode))
}
})
}
}
// Test basic presigned URL functionality without prefix
func TestPresignedSignatureV4Basic(t *testing.T) {
iam := newTestIAM()
// Create a presigned request without X-Forwarded-Prefix header
r, err := newTestRequest("GET", "https://example.com/test-bucket/test-object", 0, nil)
if err != nil {
t.Fatalf("Failed to create test request: %v", err)
}
// Set the mux variables manually since we're not going through the actual router
r = mux.SetURLVars(r, map[string]string{
"bucket": "test-bucket",
"object": "test-object",
})
r.Header.Set("Host", "example.com")
// Create presigned URL with the normal path (no prefix)
err = preSignV4WithPath(iam, r, "AKIAIOSFODNN7EXAMPLE", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", 3600, r.URL.Path)
if err != nil {
t.Errorf("Failed to presign request: %v", err)
}
// Test presigned signature verification
_, errCode := iam.doesPresignedSignatureMatch(getContentSha256Cksum(r), r)
if errCode != s3err.ErrNone {
t.Errorf("Expected successful presigned signature validation, got error: %v (code: %d)", errCode, int(errCode))
}
}
// Test X-Forwarded-Prefix support for presigned URLs
func TestPresignedSignatureV4WithForwardedPrefix(t *testing.T) {
tests := []struct {
name string
forwardedPrefix string
originalPath string
expectedPath string
}{
{
name: "prefix without trailing slash",
forwardedPrefix: "/s3",
originalPath: "/s3/test-bucket/test-object",
expectedPath: "/s3/test-bucket/test-object",
},
{
name: "prefix with trailing slash",
forwardedPrefix: "/s3/",
originalPath: "/s3/test-bucket/test-object",
expectedPath: "/s3/test-bucket/test-object",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
iam := newTestIAM()
// Create a presigned request that simulates reverse proxy scenario:
// 1. Client generates presigned URL with prefixed path
// 2. Proxy strips prefix and forwards to SeaweedFS with X-Forwarded-Prefix header
// Start with the original request URL (what client sees)
r, err := newTestRequest("GET", "https://example.com"+tt.originalPath, 0, nil)
if err != nil {
t.Fatalf("Failed to create test request: %v", err)
}
// Generate presigned URL with the original prefixed path
err = preSignV4WithPath(iam, r, "AKIAIOSFODNN7EXAMPLE", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", 3600, tt.originalPath)
if err != nil {
t.Errorf("Failed to presign request: %v", err)
return
}
// Now simulate what the reverse proxy does:
// 1. Strip the prefix from the URL path
r.URL.Path = "/test-bucket/test-object"
// 2. Set the mux variables for the stripped path
r = mux.SetURLVars(r, map[string]string{
"bucket": "test-bucket",
"object": "test-object",
})
// 3. Add the forwarded headers
r.Header.Set("X-Forwarded-Prefix", tt.forwardedPrefix)
r.Header.Set("Host", "example.com")
r.Header.Set("X-Forwarded-Host", "example.com")
// Test presigned signature verification
_, errCode := iam.doesPresignedSignatureMatch(getContentSha256Cksum(r), r)
if errCode != s3err.ErrNone {
t.Errorf("Expected successful presigned signature validation with X-Forwarded-Prefix %q, got error: %v (code: %d)", tt.forwardedPrefix, errCode, int(errCode))
}
})
}
}
// preSignV4WithPath adds presigned URL parameters to the request with a custom path
func preSignV4WithPath(iam *IdentityAccessManagement, req *http.Request, accessKey, secretKey string, expires int64, urlPath string) error {
// Create credential scope
now := time.Now().UTC()
dateStr := now.Format(iso8601Format)
// Create credential header
scope := fmt.Sprintf("%s/%s/%s/%s", now.Format(yyyymmdd), "us-east-1", "s3", "aws4_request")
credential := fmt.Sprintf("%s/%s", accessKey, scope)
// Get the query parameters
query := req.URL.Query()
query.Set("X-Amz-Algorithm", signV4Algorithm)
query.Set("X-Amz-Credential", credential)
query.Set("X-Amz-Date", dateStr)
query.Set("X-Amz-Expires", fmt.Sprintf("%d", expires))
query.Set("X-Amz-SignedHeaders", "host")
// Set the query on the URL (without signature yet)
req.URL.RawQuery = query.Encode()
// Get the payload hash
hashedPayload := getContentSha256Cksum(req)
// Extract signed headers
extractedSignedHeaders := make(http.Header)
extractedSignedHeaders["host"] = []string{extractHostHeader(req)}
// Get canonical request with custom path
canonicalRequest := getCanonicalRequest(extractedSignedHeaders, hashedPayload, req.URL.RawQuery, urlPath, req.Method)
// Get string to sign
stringToSign := getStringToSign(canonicalRequest, now, scope)
// Get signing key
signingKey := getSigningKey(secretKey, now.Format(yyyymmdd), "us-east-1", "s3")
// Calculate signature
signature := getSignature(signingKey, stringToSign)
// Add signature to query
query.Set("X-Amz-Signature", signature)
req.URL.RawQuery = query.Encode()
return nil
}
// signV4WithPath signs a request with a custom path
func signV4WithPath(req *http.Request, accessKey, secretKey, urlPath string) {
// Create credential scope
now := time.Now().UTC()
dateStr := now.Format(iso8601Format)
// Set required headers
req.Header.Set("X-Amz-Date", dateStr)
// Create credential header
scope := fmt.Sprintf("%s/%s/%s/%s", now.Format(yyyymmdd), "us-east-1", "s3", "aws4_request")
credential := fmt.Sprintf("%s/%s", accessKey, scope)
// Get signed headers
signedHeaders := "host;x-amz-date"
// Extract signed headers
extractedSignedHeaders := make(http.Header)
extractedSignedHeaders["host"] = []string{extractHostHeader(req)}
extractedSignedHeaders["x-amz-date"] = []string{dateStr}
// Get the payload hash
hashedPayload := getContentSha256Cksum(req)
// Get canonical request with custom path
canonicalRequest := getCanonicalRequest(extractedSignedHeaders, hashedPayload, req.URL.RawQuery, urlPath, req.Method)
// Get string to sign
stringToSign := getStringToSign(canonicalRequest, now, scope)
// Get signing key
signingKey := getSigningKey(secretKey, now.Format(yyyymmdd), "us-east-1", "s3")
// Calculate signature
signature := getSignature(signingKey, stringToSign)
// Set Authorization header
authorization := fmt.Sprintf("%s Credential=%s, SignedHeaders=%s, Signature=%s",
signV4Algorithm, credential, signedHeaders, signature)
req.Header.Set("Authorization", authorization)
}
// Returns new HTTP request object.
func newTestRequest(method, urlStr string, contentLength int64, body io.ReadSeeker) (*http.Request, error) {
if method == "" {