rewrite, simplify, avoid unused functions (#6989)

* adding cors support

* address some comments

* optimize matchesWildcard

* address comments

* fix for tests

* address comments

* address comments

* address comments

* path building

* refactor

* Update weed/s3api/s3api_bucket_config.go

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* address comment

Service-level responses need both Access-Control-Allow-Methods and Access-Control-Allow-Headers. After setting Access-Control-Allow-Origin and Access-Control-Expose-Headers, also set Access-Control-Allow-Methods: * and Access-Control-Allow-Headers: * so service endpoints satisfy CORS preflight requirements.

* Update weed/s3api/s3api_bucket_config.go

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update weed/s3api/s3api_object_handlers.go

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update weed/s3api/s3api_object_handlers.go

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* fix

* refactor

* Update weed/s3api/s3api_bucket_config.go

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update weed/s3api/s3api_object_handlers.go

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update weed/s3api/s3api_server.go

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* simplify

* add cors tests

* fix tests

* fix tests

* remove unused functions

* fix tests

* simplify

* address comments

* fix

* Update weed/s3api/auth_signature_v4.go

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Apply suggestion from @Copilot

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* rename variable

* Revert "Apply suggestion from @Copilot"

This reverts commit fce2d4e57e.

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

weed/s3api/auth_signature_v2.go

@@ -22,16 +22,14 @@ import (
 	"crypto/sha1"
 	"crypto/subtle"
 	"encoding/base64"
-	"fmt"
-	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
-	"net"
 	"net/http"
-	"net/url"
-	"path"
 	"sort"
 	"strconv"
 	"strings"
 	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
 )
 
 // Whitelist resource list that will be used in query string for signature-V2 calculation.
@@ -61,7 +59,7 @@ var resourceList = []string{
 	"website",
 }
 
-// Verify if request has valid AWS Signature Version '2'.
+// Verify if request has AWS Signature Version '2'.
 func (iam *IdentityAccessManagement) isReqAuthenticatedV2(r *http.Request) (*Identity, s3err.ErrorCode) {
 	if isRequestSignatureV2(r) {
 		return iam.doesSignV2Match(r)
@@ -70,277 +68,186 @@ func (iam *IdentityAccessManagement) isReqAuthenticatedV2(r *http.Request) (*Ide
 }
 
 func (iam *IdentityAccessManagement) doesPolicySignatureV2Match(formValues http.Header) s3err.ErrorCode {
+
 	accessKey := formValues.Get("AWSAccessKeyId")
-	_, cred, found := iam.lookupByAccessKey(accessKey)
+	if accessKey == "" {
+		return s3err.ErrMissingFields
+	}
+
+	identity, cred, found := iam.lookupByAccessKey(accessKey)
 	if !found {
 		return s3err.ErrInvalidAccessKeyID
 	}
+
+	bucket := formValues.Get("bucket")
+	if !identity.canDo(s3_constants.ACTION_WRITE, bucket, "") {
+		return s3err.ErrAccessDenied
+	}
+
 	policy := formValues.Get("Policy")
+	if policy == "" {
+		return s3err.ErrMissingFields
+	}
+
 	signature := formValues.Get("Signature")
+	if signature == "" {
+		return s3err.ErrMissingFields
+	}
+
 	if !compareSignatureV2(signature, calculateSignatureV2(policy, cred.SecretKey)) {
 		return s3err.ErrSignatureDoesNotMatch
 	}
 	return s3err.ErrNone
 }
 
-// Authorization = "AWS" + " " + AWSAccessKeyId + ":" + Signature;
-// Signature = Base64( HMAC-SHA1( YourSecretKey, UTF-8-Encoding-Of( StringToSign ) ) );
-//
-// StringToSign = HTTP-Verb + "\n" +
-//  	Content-Md5 + "\n" +
-//  	Content-Type + "\n" +
-//  	Date + "\n" +
-//  	CanonicalizedProtocolHeaders +
-//  	CanonicalizedResource;
-//
-// CanonicalizedResource = [ "/" + Bucket ] +
-//  	<HTTP-Request-URI, from the protocol name up to the query string> +
-//  	[ subresource, if present. For example "?acl", "?location", "?logging", or "?torrent"];
-//
-// CanonicalizedProtocolHeaders = <described below>
-
 // doesSignV2Match - Verify authorization header with calculated header in accordance with
-//     - http://docs.aws.amazon.com/AmazonS3/latest/dev/auth-request-sig-v2.html
-// returns true if matches, false otherwise. if error is not nil then it is always false
-
-func validateV2AuthHeader(v2Auth string) (accessKey string, errCode s3err.ErrorCode) {
-	if v2Auth == "" {
-		return "", s3err.ErrAuthHeaderEmpty
-	}
-	// Verify if the header algorithm is supported or not.
-	if !strings.HasPrefix(v2Auth, signV2Algorithm) {
-		return "", s3err.ErrSignatureVersionNotSupported
-	}
-
-	// below is V2 Signed Auth header format, splitting on `space` (after the `AWS` string).
-	// Authorization = "AWS" + " " + AWSAccessKeyId + ":" + Signature
-	authFields := strings.Split(v2Auth, " ")
-	if len(authFields) != 2 {
-		return "", s3err.ErrMissingFields
-	}
-
-	// Then will be splitting on ":", this will separate `AWSAccessKeyId` and `Signature` string.
-	keySignFields := strings.Split(strings.TrimSpace(authFields[1]), ":")
-	if len(keySignFields) != 2 {
-		return "", s3err.ErrMissingFields
-	}
-
-	return keySignFields[0], s3err.ErrNone
-}
-
+//   - http://docs.aws.amazon.com/AmazonS3/latest/dev/auth-request-sig-v2.html
+//
+// returns ErrNone if the signature matches.
 func (iam *IdentityAccessManagement) doesSignV2Match(r *http.Request) (*Identity, s3err.ErrorCode) {
 	v2Auth := r.Header.Get("Authorization")
-
-	accessKey, apiError := validateV2AuthHeader(v2Auth)
-	if apiError != s3err.ErrNone {
-		return nil, apiError
+	accessKey, errCode := validateV2AuthHeader(v2Auth)
+	if errCode != s3err.ErrNone {
+		return nil, errCode
 	}
 
-	// Access credentials.
-	// Validate if access key id same.
-	ident, cred, found := iam.lookupByAccessKey(accessKey)
+	identity, cred, found := iam.lookupByAccessKey(accessKey)
 	if !found {
 		return nil, s3err.ErrInvalidAccessKeyID
 	}
 
-	// r.RequestURI will have raw encoded URI as sent by the client.
-	tokens := strings.SplitN(r.RequestURI, "?", 2)
-	encodedResource := tokens[0]
-	encodedQuery := ""
-	if len(tokens) == 2 {
-		encodedQuery = tokens[1]
+	bucket, object := s3_constants.GetBucketAndObject(r)
+	if !identity.canDo(s3_constants.ACTION_WRITE, bucket, object) {
+		return nil, s3err.ErrAccessDenied
 	}
 
-	unescapedQueries, err := unescapeQueries(encodedQuery)
-	if err != nil {
-		return nil, s3err.ErrInvalidQueryParams
-	}
-
-	encodedResource, err = getResource(encodedResource, r.Host, iam.domain)
-	if err != nil {
-		return nil, s3err.ErrInvalidRequest
-	}
-
-	prefix := fmt.Sprintf("%s %s:", signV2Algorithm, cred.AccessKey)
-	if !strings.HasPrefix(v2Auth, prefix) {
-		return nil, s3err.ErrSignatureDoesNotMatch
-	}
-	v2Auth = v2Auth[len(prefix):]
-	expectedAuth := signatureV2(cred, r.Method, encodedResource, strings.Join(unescapedQueries, "&"), r.Header)
+	expectedAuth := signatureV2(cred, r.Method, r.URL.Path, r.URL.Query().Encode(), r.Header)
 	if !compareSignatureV2(v2Auth, expectedAuth) {
 		return nil, s3err.ErrSignatureDoesNotMatch
 	}
-	return ident, s3err.ErrNone
+	return identity, s3err.ErrNone
 }
 
-// doesPresignV2SignatureMatch - Verify query headers with presigned signature
-//   - http://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html#RESTAuthenticationQueryStringAuth
+// doesPresignV2SignatureMatch - Verify query headers with calculated header in accordance with
+//   - http://docs.aws.amazon.com/AmazonS3/latest/dev/auth-request-sig-v2.html
 //
-// returns ErrNone if matches. S3 errors otherwise.
+// returns ErrNone if the signature matches.
 func (iam *IdentityAccessManagement) doesPresignV2SignatureMatch(r *http.Request) (*Identity, s3err.ErrorCode) {
-
-	// r.RequestURI will have raw encoded URI as sent by the client.
-	tokens := strings.SplitN(r.RequestURI, "?", 2)
-	encodedResource := tokens[0]
-	encodedQuery := ""
-	if len(tokens) == 2 {
-		encodedQuery = tokens[1]
+	query := r.URL.Query()
+	expires := query.Get("Expires")
+	if expires == "" {
+		return nil, s3err.ErrMissingFields
 	}
 
-	var (
-		filteredQueries []string
-		gotSignature    string
-		expires         string
-		accessKey       string
-		err             error
-	)
-
-	var unescapedQueries []string
-	unescapedQueries, err = unescapeQueries(encodedQuery)
-	if err != nil {
-		return nil, s3err.ErrInvalidQueryParams
-	}
-
-	// Extract the necessary values from presigned query, construct a list of new filtered queries.
-	for _, query := range unescapedQueries {
-		keyval := strings.SplitN(query, "=", 2)
-		if len(keyval) != 2 {
-			return nil, s3err.ErrInvalidQueryParams
-		}
-		switch keyval[0] {
-		case "AWSAccessKeyId":
-			accessKey = keyval[1]
-		case "Signature":
-			gotSignature = keyval[1]
-		case "Expires":
-			expires = keyval[1]
-		default:
-			filteredQueries = append(filteredQueries, query)
-		}
-	}
-
-	// Invalid values returns error.
-	if accessKey == "" || gotSignature == "" || expires == "" {
-		return nil, s3err.ErrInvalidQueryParams
-	}
-
-	// Validate if access key id same.
-	ident, cred, found := iam.lookupByAccessKey(accessKey)
-	if !found {
-		return nil, s3err.ErrInvalidAccessKeyID
-	}
-
-	// Make sure the request has not expired.
-	expiresInt, err := strconv.ParseInt(expires, 10, 64)
+	expireTimestamp, err := strconv.ParseInt(expires, 10, 64)
 	if err != nil {
 		return nil, s3err.ErrMalformedExpires
 	}
 
-	// Check if the presigned URL has expired.
-	if expiresInt < time.Now().UTC().Unix() {
+	if time.Unix(expireTimestamp, 0).Before(time.Now().UTC()) {
 		return nil, s3err.ErrExpiredPresignRequest
 	}
 
-	encodedResource, err = getResource(encodedResource, r.Host, iam.domain)
-	if err != nil {
-		return nil, s3err.ErrInvalidRequest
+	accessKey := query.Get("AWSAccessKeyId")
+	if accessKey == "" {
+		return nil, s3err.ErrInvalidAccessKeyID
 	}
 
-	expectedSignature := preSignatureV2(cred, r.Method, encodedResource, strings.Join(filteredQueries, "&"), r.Header, expires)
-	if !compareSignatureV2(gotSignature, expectedSignature) {
+	signature := query.Get("Signature")
+	if signature == "" {
+		return nil, s3err.ErrMissingFields
+	}
+
+	identity, cred, found := iam.lookupByAccessKey(accessKey)
+	if !found {
+		return nil, s3err.ErrInvalidAccessKeyID
+	}
+
+	bucket, object := s3_constants.GetBucketAndObject(r)
+	if !identity.canDo(s3_constants.ACTION_READ, bucket, object) {
+		return nil, s3err.ErrAccessDenied
+	}
+
+	expectedSignature := preSignatureV2(cred, r.Method, r.URL.Path, r.URL.Query().Encode(), r.Header, expires)
+	if !compareSignatureV2(signature, expectedSignature) {
 		return nil, s3err.ErrSignatureDoesNotMatch
 	}
-
-	return ident, s3err.ErrNone
+	return identity, s3err.ErrNone
 }
 
-// Escape encodedQuery string into unescaped list of query params, returns error
-// if any while unescaping the values.
-func unescapeQueries(encodedQuery string) (unescapedQueries []string, err error) {
-	for _, query := range strings.Split(encodedQuery, "&") {
-		var unescapedQuery string
-		unescapedQuery, err = url.QueryUnescape(query)
-		if err != nil {
-			return nil, err
-		}
-		unescapedQueries = append(unescapedQueries, unescapedQuery)
+// validateV2AuthHeader validates AWS Signature Version '2' authentication header.
+func validateV2AuthHeader(v2Auth string) (accessKey string, errCode s3err.ErrorCode) {
+	if v2Auth == "" {
+		return "", s3err.ErrAuthHeaderEmpty
 	}
-	return unescapedQueries, nil
+
+	// Signature V2 authorization header format:
+	// Authorization: AWS AKIAIOSFODNN7EXAMPLE:frJIUN8DYpKDtOLCwo//yllqDzg=
+	if !strings.HasPrefix(v2Auth, signV2Algorithm) {
+		return "", s3err.ErrSignatureVersionNotSupported
+	}
+
+	// Strip off the Algorithm prefix.
+	v2Auth = v2Auth[len(signV2Algorithm):]
+	authFields := strings.Split(v2Auth, ":")
+	if len(authFields) != 2 {
+		return "", s3err.ErrMissingFields
+	}
+
+	// The first field is Access Key ID.
+	if authFields[0] == "" {
+		return "", s3err.ErrInvalidAccessKeyID
+	}
+
+	// The second field is signature.
+	if authFields[1] == "" {
+		return "", s3err.ErrMissingFields
+	}
+
+	return authFields[0], s3err.ErrNone
 }
 
-// Returns "/bucketName/objectName" for path-style or virtual-host-style requests.
-func getResource(path string, host string, domain string) (string, error) {
-	if domain == "" {
-		return path, nil
-	}
-	// If virtual-host-style is enabled construct the "resource" properly.
-	if strings.Contains(host, ":") {
-		// In bucket.mydomain.com:9000, strip out :9000
-		var err error
-		if host, _, err = net.SplitHostPort(host); err != nil {
-			return "", err
-		}
-	}
-	if !strings.HasSuffix(host, "."+domain) {
-		return path, nil
-	}
-	bucket := strings.TrimSuffix(host, "."+domain)
-	return "/" + pathJoin(bucket, path), nil
-}
-
-// pathJoin - like path.Join() but retains trailing "/" of the last element
-func pathJoin(elem ...string) string {
-	trailingSlash := ""
-	if len(elem) > 0 {
-		if strings.HasSuffix(elem[len(elem)-1], "/") {
-			trailingSlash = "/"
-		}
-	}
-	return path.Join(elem...) + trailingSlash
-}
-
-// Return the signature v2 of a given request.
+// signatureV2 - calculates signature version 2 for request.
 func signatureV2(cred *Credential, method string, encodedResource string, encodedQuery string, headers http.Header) string {
 	stringToSign := getStringToSignV2(method, encodedResource, encodedQuery, headers, "")
 	signature := calculateSignatureV2(stringToSign, cred.SecretKey)
-	return signature
+	return signV2Algorithm + cred.AccessKey + ":" + signature
 }
 
-// Return string to sign under two different conditions.
-// - if expires string is set then string to sign includes date instead of the Date header.
-// - if expires string is empty then string to sign includes date header instead.
+// getStringToSignV2 - string to sign in accordance with
+//   - http://docs.aws.amazon.com/AmazonS3/latest/dev/auth-request-sig-v2.html
 func getStringToSignV2(method string, encodedResource, encodedQuery string, headers http.Header, expires string) string {
 	canonicalHeaders := canonicalizedAmzHeadersV2(headers)
 	if len(canonicalHeaders) > 0 {
 		canonicalHeaders += "\n"
 	}
 
-	date := expires // Date is set to expires date for presign operations.
-	if date == "" {
-		// If expires date is empty then request header Date is used.
-		date = headers.Get("Date")
-	}
-
 	// From the Amazon docs:
 	//
 	// StringToSign = HTTP-Verb + "\n" +
-	// 	 Content-Md5 + "\n" +
+	// 	 Content-MD5 + "\n" +
 	//	 Content-Type + "\n" +
-	//	 Date/Expires + "\n" +
-	//	 CanonicalizedProtocolHeaders +
+	//	 Date + "\n" +
+	//	 CanonicalizedAmzHeaders +
 	//	 CanonicalizedResource;
-	stringToSign := strings.Join([]string{
-		method,
-		headers.Get("Content-MD5"),
-		headers.Get("Content-Type"),
-		date,
-		canonicalHeaders,
-	}, "\n")
+	stringToSign := method + "\n"
+	stringToSign += headers.Get("Content-Md5") + "\n"
+	stringToSign += headers.Get("Content-Type") + "\n"
 
-	return stringToSign + canonicalizedResourceV2(encodedResource, encodedQuery)
+	if expires != "" {
+		stringToSign += expires + "\n"
+	} else {
+		stringToSign += headers.Get("Date") + "\n"
+		if v := headers.Get("x-amz-date"); v != "" {
+			stringToSign = strings.Replace(stringToSign, headers.Get("Date")+"\n", "\n", -1)
+		}
+	}
+	stringToSign += canonicalHeaders
+	stringToSign += canonicalizedResourceV2(encodedResource, encodedQuery)
+	return stringToSign
 }
 
-// Return canonical resource string.
+// canonicalizedResourceV2 - canonicalize the resource string for signature V2.
 func canonicalizedResourceV2(encodedResource, encodedQuery string) string {
 	queries := strings.Split(encodedQuery, "&")
 	keyval := make(map[string]string)
@@ -356,28 +263,26 @@ func canonicalizedResourceV2(encodedResource, encodedQuery string) string {
 	}
 
 	var canonicalQueries []string
-	for _, key := range resourceList {
-		val, ok := keyval[key]
-		if !ok {
-			continue
+	for _, resource := range resourceList {
+		if val, ok := keyval[resource]; ok {
+			if val == "" {
+				canonicalQueries = append(canonicalQueries, resource)
+				continue
+			}
+			canonicalQueries = append(canonicalQueries, resource+"="+val)
 		}
-		if val == "" {
-			canonicalQueries = append(canonicalQueries, key)
-			continue
-		}
-		canonicalQueries = append(canonicalQueries, key+"="+val)
 	}
 
-	// The queries will be already sorted as resourceList is sorted, if canonicalQueries
-	// is empty strings.Join returns empty.
-	canonicalQuery := strings.Join(canonicalQueries, "&")
-	if canonicalQuery != "" {
-		return encodedResource + "?" + canonicalQuery
+	// The queries will be already sorted as resourceList is sorted.
+	if len(canonicalQueries) == 0 {
+		return encodedResource
 	}
-	return encodedResource
+
+	// If queries are present then the canonicalized resource is set to encodedResource + "?" + strings.Join(canonicalQueries, "&")
+	return encodedResource + "?" + strings.Join(canonicalQueries, "&")
 }
 
-// Return canonical headers.
+// canonicalizedAmzHeadersV2 - canonicalize the x-amz-* headers for signature V2.
 func canonicalizedAmzHeadersV2(headers http.Header) string {
 	var keys []string
 	keyval := make(map[string]string)
@@ -390,6 +295,7 @@ func canonicalizedAmzHeadersV2(headers http.Header) string {
 		keyval[lkey] = strings.Join(headers[key], ",")
 	}
 	sort.Strings(keys)
+
 	var canonicalHeaders []string
 	for _, key := range keys {
 		canonicalHeaders = append(canonicalHeaders, key+":"+keyval[key])
@@ -397,6 +303,7 @@ func canonicalizedAmzHeadersV2(headers http.Header) string {
 	return strings.Join(canonicalHeaders, "\n")
 }
 
+// calculateSignatureV2 - calculates signature version 2.
 func calculateSignatureV2(stringToSign string, secret string) string {
 	hm := hmac.New(sha1.New, []byte(secret))
 	hm.Write([]byte(stringToSign))

weed/s3api/auto_signature_v4_test.go

@@ -6,13 +6,10 @@ import (
 	"crypto/sha256"
 	"encoding/base64"
 	"encoding/hex"
-	"errors"
 	"fmt"
 	"io"
 	"net/http"
-	"net/url"
 	"sort"
-	"strconv"
 	"strings"
 	"sync"
 	"testing"
@@ -76,7 +73,7 @@ func TestIsReqAuthenticated(t *testing.T) {
 						SecretKey: "secret_key_1",
 					},
 				},
-				Actions: []string{},
+				Actions: []string{"Read", "Write"},
 			},
 		},
 	})
@@ -149,7 +146,7 @@ func TestCheckAdminRequestAuthType(t *testing.T) {
 						SecretKey: "secret_key_1",
 					},
 				},
-				Actions: []string{},
+				Actions: []string{"Admin", "Read", "Write"},
 			},
 		},
 	})
@@ -170,15 +167,12 @@ func TestCheckAdminRequestAuthType(t *testing.T) {
 
 func BenchmarkGetSignature(b *testing.B) {
 	t := time.Now()
-	iam := IdentityAccessManagement{
-		hashes:       make(map[string]*sync.Pool),
-		hashCounters: make(map[string]*int32),
-	}
 
 	b.ReportAllocs()
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		iam.getSignature("secret-key", t, "us-east-1", "s3", "random data")
+		signingKey := getSigningKey("secret-key", t.Format(yyyymmdd), "us-east-1", "s3")
+		getSignature(signingKey, "random data")
 	}
 }
 
@@ -213,6 +207,53 @@ func mustNewPresignedRequest(iam *IdentityAccessManagement, method string, urlSt
 	return req
 }
 
+// preSignV4 adds presigned URL parameters to the request
+func preSignV4(iam *IdentityAccessManagement, req *http.Request, accessKey, secretKey string, expires int64) error {
+	// Create credential scope
+	now := time.Now().UTC()
+	dateStr := now.Format(iso8601Format)
+
+	// Create credential header
+	scope := fmt.Sprintf("%s/%s/%s/%s", now.Format(yyyymmdd), "us-east-1", "s3", "aws4_request")
+	credential := fmt.Sprintf("%s/%s", accessKey, scope)
+
+	// Get the query parameters
+	query := req.URL.Query()
+	query.Set("X-Amz-Algorithm", signV4Algorithm)
+	query.Set("X-Amz-Credential", credential)
+	query.Set("X-Amz-Date", dateStr)
+	query.Set("X-Amz-Expires", fmt.Sprintf("%d", expires))
+	query.Set("X-Amz-SignedHeaders", "host")
+
+	// Set the query on the URL (without signature yet)
+	req.URL.RawQuery = query.Encode()
+
+	// Get the payload hash
+	hashedPayload := getContentSha256Cksum(req)
+
+	// Extract signed headers
+	extractedSignedHeaders := make(http.Header)
+	extractedSignedHeaders["host"] = []string{req.Host}
+
+	// Get canonical request
+	canonicalRequest := getCanonicalRequest(extractedSignedHeaders, hashedPayload, req.URL.RawQuery, req.URL.Path, req.Method)
+
+	// Get string to sign
+	stringToSign := getStringToSign(canonicalRequest, now, scope)
+
+	// Get signing key
+	signingKey := getSigningKey(secretKey, now.Format(yyyymmdd), "us-east-1", "s3")
+
+	// Calculate signature
+	signature := getSignature(signingKey, stringToSign)
+
+	// Add signature to query
+	query.Set("X-Amz-Signature", signature)
+	req.URL.RawQuery = query.Encode()
+
+	return nil
+}
+
 // Returns new HTTP request object.
 func newTestRequest(method, urlStr string, contentLength int64, body io.ReadSeeker) (*http.Request, error) {
 	if method == "" {
@@ -254,11 +295,6 @@ func newTestRequest(method, urlStr string, contentLength int64, body io.ReadSeek
 	return req, nil
 }
 
-// getSHA256Hash returns SHA-256 hash in hex encoding of given data.
-func getSHA256Hash(data []byte) string {
-	return hex.EncodeToString(getSHA256Sum(data))
-}
-
 // getMD5HashBase64 returns MD5 hash in base64 encoding of given data.
 func getMD5HashBase64(data []byte) string {
 	return base64.StdEncoding.EncodeToString(getMD5Sum(data))
@@ -467,46 +503,6 @@ func signRequestV4(req *http.Request, accessKey, secretKey string) error {
 	return nil
 }
 
-// preSignV4 presign the request, in accordance with
-// http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html.
-func preSignV4(iam *IdentityAccessManagement, req *http.Request, accessKeyID, secretAccessKey string, expires int64) error {
-	// Presign is not needed for anonymous credentials.
-	if accessKeyID == "" || secretAccessKey == "" {
-		return errors.New("Presign cannot be generated without access and secret keys")
-	}
-
-	region := "us-east-1"
-	date := time.Now().UTC()
-	scope := getScope(date, region)
-	credential := fmt.Sprintf("%s/%s", accessKeyID, scope)
-
-	// Set URL query.
-	query := req.URL.Query()
-	query.Set("X-Amz-Algorithm", signV4Algorithm)
-	query.Set("X-Amz-Date", date.Format(iso8601Format))
-	query.Set("X-Amz-Expires", strconv.FormatInt(expires, 10))
-	query.Set("X-Amz-SignedHeaders", "host")
-	query.Set("X-Amz-Credential", credential)
-	query.Set("X-Amz-Content-Sha256", unsignedPayload)
-
-	// "host" is the only header required to be signed for Presigned URLs.
-	extractedSignedHeaders := make(http.Header)
-	extractedSignedHeaders.Set("host", req.Host)
-
-	queryStr := strings.Replace(query.Encode(), "+", "%20", -1)
-	canonicalRequest := getCanonicalRequest(extractedSignedHeaders, unsignedPayload, queryStr, req.URL.Path, req.Method)
-	stringToSign := getStringToSign(canonicalRequest, date, scope)
-	signature := iam.getSignature(secretAccessKey, date, region, "s3", stringToSign)
-
-	req.URL.RawQuery = query.Encode()
-
-	// Add signature header to RawQuery.
-	req.URL.RawQuery += "&X-Amz-Signature=" + url.QueryEscape(signature)
-
-	// Construct the final presigned URL.
-	return nil
-}
-
 // EncodePath encode the strings from UTF-8 byte representations to HTML hex escape sequences
 //
 // This is necessary since regular url.Parse() and url.Encode() functions do not support UTF-8
@@ -529,12 +525,12 @@ func EncodePath(pathName string) string {
 			encodedPathname = encodedPathname + string(s)
 			continue
 		default:
-			len := utf8.RuneLen(s)
-			if len < 0 {
+			runeLen := utf8.RuneLen(s)
+			if runeLen < 0 {
 				// if utf8 cannot convert return the same string as is
 				return pathName
 			}
-			u := make([]byte, len)
+			u := make([]byte, runeLen)
 			utf8.EncodeRune(u, s)
 			for _, r := range u {
 				hex := hex.EncodeToString([]byte{r})

weed/s3api/chunked_reader_v4.go

@@ -102,13 +102,12 @@ func (iam *IdentityAccessManagement) calculateSeedSignature(r *http.Request) (cr
 			return nil, "", "", time.Time{}, s3err.ErrMissingDateHeader
 		}
 	}
+
 	// Parse date header.
-	var err error
-	date, err = time.Parse(iso8601Format, dateStr)
+	date, err := time.Parse(iso8601Format, dateStr)
 	if err != nil {
 		return nil, "", "", time.Time{}, s3err.ErrMalformedDate
 	}
-
 	// Query string.
 	queryStr := req.URL.Query().Encode()
 
@@ -118,14 +117,11 @@ func (iam *IdentityAccessManagement) calculateSeedSignature(r *http.Request) (cr
 	// Get string to sign from canonical request.
 	stringToSign := getStringToSign(canonicalRequest, date, signV4Values.Credential.getScope())
 
+	// Get hmac signing key.
+	signingKey := getSigningKey(cred.SecretKey, signV4Values.Credential.scope.date.Format(yyyymmdd), region, "s3")
+
 	// Calculate signature.
-	newSignature := iam.getSignature(
-		cred.SecretKey,
-		signV4Values.Credential.scope.date,
-		region,
-		"s3",
-		stringToSign,
-	)
+	newSignature := getSignature(signingKey, stringToSign)
 
 	// Verify if signature match.
 	if !compareSignatureV4(newSignature, signV4Values.Signature) {
@@ -469,58 +465,47 @@ func (cr *s3ChunkedReader) Read(buf []byte) (n int, err error) {
 // getChunkSignature - get chunk signature.
 func (cr *s3ChunkedReader) getChunkSignature(hashedChunk string) string {
 	// Calculate string to sign.
-	stringToSign := signV4ChunkedAlgorithm + "\n" +
+	stringToSign := signV4Algorithm + "-PAYLOAD" + "\n" +
 		cr.seedDate.Format(iso8601Format) + "\n" +
 		getScope(cr.seedDate, cr.region) + "\n" +
 		cr.seedSignature + "\n" +
 		emptySHA256 + "\n" +
 		hashedChunk
 
-	// Calculate signature.
-	return cr.iam.getSignature(
-		cr.cred.SecretKey,
-		cr.seedDate,
-		cr.region,
-		"s3",
-		stringToSign,
-	)
+	// Get hmac signing key.
+	signingKey := getSigningKey(cr.cred.SecretKey, cr.seedDate.Format(yyyymmdd), cr.region, "s3")
+
+	// Calculate and return signature.
+	return getSignature(signingKey, stringToSign)
 }
 
-// readCRLF - check if reader only has '\r\n' CRLF character.
-// returns malformed encoding if it doesn't.
 func readCRLF(reader *bufio.Reader) error {
 	buf := make([]byte, 2)
-	_, err := reader.Read(buf)
+	_, err := io.ReadFull(reader, buf)
 	if err != nil {
 		return err
 	}
 	return checkCRLF(buf)
 }
 
-// peekCRLF - peeks at the next two bytes to check for CRLF without consuming them.
 func peekCRLF(reader *bufio.Reader) error {
-	peeked, err := reader.Peek(2)
+	buf, err := reader.Peek(2)
 	if err != nil {
 		return err
 	}
-	if err := checkCRLF(peeked); err != nil {
+	if err := checkCRLF(buf); err != nil {
 		return err
 	}
 	return nil
 }
 
-// checkCRLF - checks if the buffer contains '\r\n' CRLF character.
 func checkCRLF(buf []byte) error {
-	if buf[0] != '\r' || buf[1] != '\n' {
+	if len(buf) != 2 || buf[0] != '\r' || buf[1] != '\n' {
 		return errMalformedEncoding
 	}
 	return nil
 }
 
-// Read a line of bytes (up to \n) from b.
-// Give up if the line exceeds maxLineLength.
-// The returned bytes are owned by the bufio.Reader
-// so they are only valid until the next bufio read.
 func readChunkLine(b *bufio.Reader) ([]byte, error) {
 	buf, err := b.ReadSlice('\n')
 	if err != nil {
@@ -536,8 +521,7 @@ func readChunkLine(b *bufio.Reader) ([]byte, error) {
 	if len(buf) >= maxLineLength {
 		return nil, errLineTooLong
 	}
-
-	return buf, nil
+	return trimTrailingWhitespace(buf), nil
 }
 
 // trimTrailingWhitespace - trim trailing white space.
@@ -608,13 +592,11 @@ func parseChunkChecksum(b *bufio.Reader) (ChecksumAlgorithm, []byte) {
 	return extractedAlgorithm, checksumValue
 }
 
-// parseChunkSignature - parse chunk signature.
 func parseChunkSignature(chunk []byte) []byte {
-	chunkSplits := bytes.SplitN(chunk, []byte(s3ChunkSignatureStr), 2)
-	return chunkSplits[1]
+	chunkSplits := bytes.SplitN(chunk, []byte("="), 2)
+	return chunkSplits[1] // Keep only the signature.
 }
 
-// parse hex to uint64.
 func parseHexUint(v []byte) (n uint64, err error) {
 	for i, b := range v {
 		switch {
@@ -636,6 +618,7 @@ func parseHexUint(v []byte) (n uint64, err error) {
 	return
 }
 
+// Checksum Algorithm represents the various checksum algorithms supported.
 type ChecksumAlgorithm int
 
 const (
@@ -649,18 +632,18 @@ const (
 
 func (ca ChecksumAlgorithm) String() string {
 	switch ca {
-	case ChecksumAlgorithmCRC32:
-		return "CRC32"
-	case ChecksumAlgorithmCRC32C:
-		return "CRC32C"
-	case ChecksumAlgorithmCRC64NVMe:
-		return "CRC64NVMe"
-	case ChecksumAlgorithmSHA1:
-		return "SHA1"
-	case ChecksumAlgorithmSHA256:
-		return "SHA256"
 	case ChecksumAlgorithmNone:
 		return ""
+	case ChecksumAlgorithmCRC32:
+		return "x-amz-checksum-crc32"
+	case ChecksumAlgorithmCRC32C:
+		return "x-amz-checksum-crc32c"
+	case ChecksumAlgorithmCRC64NVMe:
+		return "x-amz-checksum-crc64nvme"
+	case ChecksumAlgorithmSHA1:
+		return "x-amz-checksum-sha1"
+	case ChecksumAlgorithmSHA256:
+		return "x-amz-checksum-sha256"
 	}
 	return ""
 }