s3: add support for PostPolicy

fix https://github.com/chrislusf/seaweedfs/issues/1426
This commit is contained in:
Chris Lu
2020-09-19 20:14:19 -07:00
parent 41d508edfd
commit 29abe980df
11 changed files with 1482 additions and 3 deletions

View File

@@ -0,0 +1,321 @@
package policy
/*
* MinIO Go Library for Amazon S3 Compatible Cloud Storage
* Copyright 2015-2017 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import (
"encoding/base64"
"fmt"
"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
"net/http"
"strings"
"time"
)
// expirationDateFormat date format for expiration key in json policy.
const expirationDateFormat = "2006-01-02T15:04:05.999Z"
// policyCondition explanation:
// http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-HTTPPOSTConstructPolicy.html
//
// Example:
//
// policyCondition {
// matchType: "$eq",
// key: "$Content-Type",
// value: "image/png",
// }
//
type policyCondition struct {
matchType string
condition string
value string
}
// PostPolicy - Provides strict static type conversion and validation
// for Amazon S3's POST policy JSON string.
type PostPolicy struct {
// Expiration date and time of the POST policy.
expiration time.Time
// Collection of different policy conditions.
conditions []policyCondition
// ContentLengthRange minimum and maximum allowable size for the
// uploaded content.
contentLengthRange struct {
min int64
max int64
}
// Post form data.
formData map[string]string
}
// NewPostPolicy - Instantiate new post policy.
func NewPostPolicy() *PostPolicy {
p := &PostPolicy{}
p.conditions = make([]policyCondition, 0)
p.formData = make(map[string]string)
return p
}
// SetExpires - Sets expiration time for the new policy.
func (p *PostPolicy) SetExpires(t time.Time) error {
if t.IsZero() {
return errInvalidArgument("No expiry time set.")
}
p.expiration = t
return nil
}
// SetKey - Sets an object name for the policy based upload.
func (p *PostPolicy) SetKey(key string) error {
if strings.TrimSpace(key) == "" || key == "" {
return errInvalidArgument("Object name is empty.")
}
policyCond := policyCondition{
matchType: "eq",
condition: "$key",
value: key,
}
if err := p.addNewPolicy(policyCond); err != nil {
return err
}
p.formData["key"] = key
return nil
}
// SetKeyStartsWith - Sets an object name that an policy based upload
// can start with.
func (p *PostPolicy) SetKeyStartsWith(keyStartsWith string) error {
if strings.TrimSpace(keyStartsWith) == "" || keyStartsWith == "" {
return errInvalidArgument("Object prefix is empty.")
}
policyCond := policyCondition{
matchType: "starts-with",
condition: "$key",
value: keyStartsWith,
}
if err := p.addNewPolicy(policyCond); err != nil {
return err
}
p.formData["key"] = keyStartsWith
return nil
}
// SetBucket - Sets bucket at which objects will be uploaded to.
func (p *PostPolicy) SetBucket(bucketName string) error {
if strings.TrimSpace(bucketName) == "" || bucketName == "" {
return errInvalidArgument("Bucket name is empty.")
}
policyCond := policyCondition{
matchType: "eq",
condition: "$bucket",
value: bucketName,
}
if err := p.addNewPolicy(policyCond); err != nil {
return err
}
p.formData["bucket"] = bucketName
return nil
}
// SetCondition - Sets condition for credentials, date and algorithm
func (p *PostPolicy) SetCondition(matchType, condition, value string) error {
if strings.TrimSpace(value) == "" || value == "" {
return errInvalidArgument("No value specified for condition")
}
policyCond := policyCondition{
matchType: matchType,
condition: "$" + condition,
value: value,
}
if condition == "X-Amz-Credential" || condition == "X-Amz-Date" || condition == "X-Amz-Algorithm" {
if err := p.addNewPolicy(policyCond); err != nil {
return err
}
p.formData[condition] = value
return nil
}
return errInvalidArgument("Invalid condition in policy")
}
// SetContentType - Sets content-type of the object for this policy
// based upload.
func (p *PostPolicy) SetContentType(contentType string) error {
if strings.TrimSpace(contentType) == "" || contentType == "" {
return errInvalidArgument("No content type specified.")
}
policyCond := policyCondition{
matchType: "eq",
condition: "$Content-Type",
value: contentType,
}
if err := p.addNewPolicy(policyCond); err != nil {
return err
}
p.formData["Content-Type"] = contentType
return nil
}
// SetContentLengthRange - Set new min and max content length
// condition for all incoming uploads.
func (p *PostPolicy) SetContentLengthRange(min, max int64) error {
if min > max {
return errInvalidArgument("Minimum limit is larger than maximum limit.")
}
if min < 0 {
return errInvalidArgument("Minimum limit cannot be negative.")
}
if max < 0 {
return errInvalidArgument("Maximum limit cannot be negative.")
}
p.contentLengthRange.min = min
p.contentLengthRange.max = max
return nil
}
// SetSuccessActionRedirect - Sets the redirect success url of the object for this policy
// based upload.
func (p *PostPolicy) SetSuccessActionRedirect(redirect string) error {
if strings.TrimSpace(redirect) == "" || redirect == "" {
return errInvalidArgument("Redirect is empty")
}
policyCond := policyCondition{
matchType: "eq",
condition: "$success_action_redirect",
value: redirect,
}
if err := p.addNewPolicy(policyCond); err != nil {
return err
}
p.formData["success_action_redirect"] = redirect
return nil
}
// SetSuccessStatusAction - Sets the status success code of the object for this policy
// based upload.
func (p *PostPolicy) SetSuccessStatusAction(status string) error {
if strings.TrimSpace(status) == "" || status == "" {
return errInvalidArgument("Status is empty")
}
policyCond := policyCondition{
matchType: "eq",
condition: "$success_action_status",
value: status,
}
if err := p.addNewPolicy(policyCond); err != nil {
return err
}
p.formData["success_action_status"] = status
return nil
}
// SetUserMetadata - Set user metadata as a key/value couple.
// Can be retrieved through a HEAD request or an event.
func (p *PostPolicy) SetUserMetadata(key string, value string) error {
if strings.TrimSpace(key) == "" || key == "" {
return errInvalidArgument("Key is empty")
}
if strings.TrimSpace(value) == "" || value == "" {
return errInvalidArgument("Value is empty")
}
headerName := fmt.Sprintf("x-amz-meta-%s", key)
policyCond := policyCondition{
matchType: "eq",
condition: fmt.Sprintf("$%s", headerName),
value: value,
}
if err := p.addNewPolicy(policyCond); err != nil {
return err
}
p.formData[headerName] = value
return nil
}
// SetUserData - Set user data as a key/value couple.
// Can be retrieved through a HEAD request or an event.
func (p *PostPolicy) SetUserData(key string, value string) error {
if key == "" {
return errInvalidArgument("Key is empty")
}
if value == "" {
return errInvalidArgument("Value is empty")
}
headerName := fmt.Sprintf("x-amz-%s", key)
policyCond := policyCondition{
matchType: "eq",
condition: fmt.Sprintf("$%s", headerName),
value: value,
}
if err := p.addNewPolicy(policyCond); err != nil {
return err
}
p.formData[headerName] = value
return nil
}
// addNewPolicy - internal helper to validate adding new policies.
func (p *PostPolicy) addNewPolicy(policyCond policyCondition) error {
if policyCond.matchType == "" || policyCond.condition == "" || policyCond.value == "" {
return errInvalidArgument("Policy fields are empty.")
}
p.conditions = append(p.conditions, policyCond)
return nil
}
// String function for printing policy in json formatted string.
func (p PostPolicy) String() string {
return string(p.marshalJSON())
}
// marshalJSON - Provides Marshaled JSON in bytes.
func (p PostPolicy) marshalJSON() []byte {
expirationStr := `"expiration":"` + p.expiration.Format(expirationDateFormat) + `"`
var conditionsStr string
conditions := []string{}
for _, po := range p.conditions {
conditions = append(conditions, fmt.Sprintf("[\"%s\",\"%s\",\"%s\"]", po.matchType, po.condition, po.value))
}
if p.contentLengthRange.min != 0 || p.contentLengthRange.max != 0 {
conditions = append(conditions, fmt.Sprintf("[\"content-length-range\", %d, %d]",
p.contentLengthRange.min, p.contentLengthRange.max))
}
if len(conditions) > 0 {
conditionsStr = `"conditions":[` + strings.Join(conditions, ",") + "]"
}
retStr := "{"
retStr = retStr + expirationStr + ","
retStr = retStr + conditionsStr
retStr = retStr + "}"
return []byte(retStr)
}
// base64 - Produces base64 of PostPolicy's Marshaled json.
func (p PostPolicy) base64() string {
return base64.StdEncoding.EncodeToString(p.marshalJSON())
}
// errInvalidArgument - Invalid argument response.
func errInvalidArgument(message string) error {
return s3err.RESTErrorResponse{
StatusCode: http.StatusBadRequest,
Code: "InvalidArgument",
Message: message,
RequestID: "minio",
}
}

View File

@@ -0,0 +1,380 @@
package policy
/*
* MinIO Cloud Storage, (C) 2016, 2017, 2018 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import (
"bytes"
"crypto/hmac"
"crypto/sha1"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"fmt"
"mime/multipart"
"net/http"
"net/url"
"regexp"
"strings"
"time"
"unicode/utf8"
)
const (
iso8601DateFormat = "20060102T150405Z"
iso8601TimeFormat = "2006-01-02T15:04:05.000Z" // Reply date format with nanosecond precision.
)
func newPostPolicyBytesV4WithContentRange(credential, bucketName, objectKey string, expiration time.Time) []byte {
t := time.Now().UTC()
// Add the expiration date.
expirationStr := fmt.Sprintf(`"expiration": "%s"`, expiration.Format(iso8601TimeFormat))
// Add the bucket condition, only accept buckets equal to the one passed.
bucketConditionStr := fmt.Sprintf(`["eq", "$bucket", "%s"]`, bucketName)
// Add the key condition, only accept keys equal to the one passed.
keyConditionStr := fmt.Sprintf(`["eq", "$key", "%s/upload.txt"]`, objectKey)
// Add content length condition, only accept content sizes of a given length.
contentLengthCondStr := `["content-length-range", 1024, 1048576]`
// Add the algorithm condition, only accept AWS SignV4 Sha256.
algorithmConditionStr := `["eq", "$x-amz-algorithm", "AWS4-HMAC-SHA256"]`
// Add the date condition, only accept the current date.
dateConditionStr := fmt.Sprintf(`["eq", "$x-amz-date", "%s"]`, t.Format(iso8601DateFormat))
// Add the credential string, only accept the credential passed.
credentialConditionStr := fmt.Sprintf(`["eq", "$x-amz-credential", "%s"]`, credential)
// Add the meta-uuid string, set to 1234
uuidConditionStr := fmt.Sprintf(`["eq", "$x-amz-meta-uuid", "%s"]`, "1234")
// Combine all conditions into one string.
conditionStr := fmt.Sprintf(`"conditions":[%s, %s, %s, %s, %s, %s, %s]`, bucketConditionStr,
keyConditionStr, contentLengthCondStr, algorithmConditionStr, dateConditionStr, credentialConditionStr, uuidConditionStr)
retStr := "{"
retStr = retStr + expirationStr + ","
retStr = retStr + conditionStr
retStr = retStr + "}"
return []byte(retStr)
}
// newPostPolicyBytesV4 - creates a bare bones postpolicy string with key and bucket matches.
func newPostPolicyBytesV4(credential, bucketName, objectKey string, expiration time.Time) []byte {
t := time.Now().UTC()
// Add the expiration date.
expirationStr := fmt.Sprintf(`"expiration": "%s"`, expiration.Format(iso8601TimeFormat))
// Add the bucket condition, only accept buckets equal to the one passed.
bucketConditionStr := fmt.Sprintf(`["eq", "$bucket", "%s"]`, bucketName)
// Add the key condition, only accept keys equal to the one passed.
keyConditionStr := fmt.Sprintf(`["eq", "$key", "%s/upload.txt"]`, objectKey)
// Add the algorithm condition, only accept AWS SignV4 Sha256.
algorithmConditionStr := `["eq", "$x-amz-algorithm", "AWS4-HMAC-SHA256"]`
// Add the date condition, only accept the current date.
dateConditionStr := fmt.Sprintf(`["eq", "$x-amz-date", "%s"]`, t.Format(iso8601DateFormat))
// Add the credential string, only accept the credential passed.
credentialConditionStr := fmt.Sprintf(`["eq", "$x-amz-credential", "%s"]`, credential)
// Add the meta-uuid string, set to 1234
uuidConditionStr := fmt.Sprintf(`["eq", "$x-amz-meta-uuid", "%s"]`, "1234")
// Combine all conditions into one string.
conditionStr := fmt.Sprintf(`"conditions":[%s, %s, %s, %s, %s, %s]`, bucketConditionStr, keyConditionStr, algorithmConditionStr, dateConditionStr, credentialConditionStr, uuidConditionStr)
retStr := "{"
retStr = retStr + expirationStr + ","
retStr = retStr + conditionStr
retStr = retStr + "}"
return []byte(retStr)
}
// newPostPolicyBytesV2 - creates a bare bones postpolicy string with key and bucket matches.
func newPostPolicyBytesV2(bucketName, objectKey string, expiration time.Time) []byte {
// Add the expiration date.
expirationStr := fmt.Sprintf(`"expiration": "%s"`, expiration.Format(iso8601TimeFormat))
// Add the bucket condition, only accept buckets equal to the one passed.
bucketConditionStr := fmt.Sprintf(`["eq", "$bucket", "%s"]`, bucketName)
// Add the key condition, only accept keys equal to the one passed.
keyConditionStr := fmt.Sprintf(`["starts-with", "$key", "%s/upload.txt"]`, objectKey)
// Combine all conditions into one string.
conditionStr := fmt.Sprintf(`"conditions":[%s, %s]`, bucketConditionStr, keyConditionStr)
retStr := "{"
retStr = retStr + expirationStr + ","
retStr = retStr + conditionStr
retStr = retStr + "}"
return []byte(retStr)
}
// Wrapper for calling TestPostPolicyBucketHandler tests for both Erasure multiple disks and single node setup.
// testPostPolicyBucketHandler - Tests validate post policy handler uploading objects.
// Wrapper for calling TestPostPolicyBucketHandlerRedirect tests for both Erasure multiple disks and single node setup.
// testPostPolicyBucketHandlerRedirect tests POST Object when success_action_redirect is specified
// postPresignSignatureV4 - presigned signature for PostPolicy requests.
func postPresignSignatureV4(policyBase64 string, t time.Time, secretAccessKey, location string) string {
// Get signining key.
signingkey := getSigningKey(secretAccessKey, t, location)
// Calculate signature.
signature := getSignature(signingkey, policyBase64)
return signature
}
// copied from auth_signature_v4.go to break import loop
// sumHMAC calculate hmac between two input byte array.
func sumHMAC(key []byte, data []byte) []byte {
hash := hmac.New(sha256.New, key)
hash.Write(data)
return hash.Sum(nil)
}
// copied from auth_signature_v4.go to break import loop
// getSigningKey hmac seed to calculate final signature.
func getSigningKey(secretKey string, t time.Time, region string) []byte {
date := sumHMAC([]byte("AWS4"+secretKey), []byte(t.Format("20060102")))
regionBytes := sumHMAC(date, []byte(region))
service := sumHMAC(regionBytes, []byte("s3"))
signingKey := sumHMAC(service, []byte("aws4_request"))
return signingKey
}
// copied from auth_signature_v4.go to break import loop
// getSignature final signature in hexadecimal form.
func getSignature(signingKey []byte, stringToSign string) string {
return hex.EncodeToString(sumHMAC(signingKey, []byte(stringToSign)))
}
// copied from auth_signature_v4.go to break import loop
func calculateSignatureV2(stringToSign string, secret string) string {
hm := hmac.New(sha1.New, []byte(secret))
hm.Write([]byte(stringToSign))
return base64.StdEncoding.EncodeToString(hm.Sum(nil))
}
func newPostRequestV2(endPoint, bucketName, objectName string, accessKey, secretKey string) (*http.Request, error) {
// Expire the request five minutes from now.
expirationTime := time.Now().UTC().Add(time.Minute * 5)
// Create a new post policy.
policy := newPostPolicyBytesV2(bucketName, objectName, expirationTime)
// Only need the encoding.
encodedPolicy := base64.StdEncoding.EncodeToString(policy)
// Presign with V4 signature based on the policy.
signature := calculateSignatureV2(encodedPolicy, secretKey)
formData := map[string]string{
"AWSAccessKeyId": accessKey,
"bucket": bucketName,
"key": objectName + "/${filename}",
"policy": encodedPolicy,
"signature": signature,
}
// Create the multipart form.
var buf bytes.Buffer
w := multipart.NewWriter(&buf)
// Set the normal formData
for k, v := range formData {
w.WriteField(k, v)
}
// Set the File formData
writer, err := w.CreateFormFile("file", "upload.txt")
if err != nil {
// return nil, err
return nil, err
}
writer.Write([]byte("hello world"))
// Close before creating the new request.
w.Close()
// Set the body equal to the created policy.
reader := bytes.NewReader(buf.Bytes())
req, err := http.NewRequest(http.MethodPost, makeTestTargetURL(endPoint, bucketName, "", nil), reader)
if err != nil {
return nil, err
}
// Set form content-type.
req.Header.Set("Content-Type", w.FormDataContentType())
return req, nil
}
func buildGenericPolicy(t time.Time, accessKey, region, bucketName, objectName string, contentLengthRange bool) []byte {
// Expire the request five minutes from now.
expirationTime := t.Add(time.Minute * 5)
credStr := getCredentialString(accessKey, region, t)
// Create a new post policy.
policy := newPostPolicyBytesV4(credStr, bucketName, objectName, expirationTime)
if contentLengthRange {
policy = newPostPolicyBytesV4WithContentRange(credStr, bucketName, objectName, expirationTime)
}
return policy
}
func newPostRequestV4Generic(endPoint, bucketName, objectName string, objData []byte, accessKey, secretKey string, region string,
t time.Time, policy []byte, addFormData map[string]string, corruptedB64 bool, corruptedMultipart bool) (*http.Request, error) {
// Get the user credential.
credStr := getCredentialString(accessKey, region, t)
// Only need the encoding.
encodedPolicy := base64.StdEncoding.EncodeToString(policy)
if corruptedB64 {
encodedPolicy = "%!~&" + encodedPolicy
}
// Presign with V4 signature based on the policy.
signature := postPresignSignatureV4(encodedPolicy, t, secretKey, region)
formData := map[string]string{
"bucket": bucketName,
"key": objectName + "/${filename}",
"x-amz-credential": credStr,
"policy": encodedPolicy,
"x-amz-signature": signature,
"x-amz-date": t.Format(iso8601DateFormat),
"x-amz-algorithm": "AWS4-HMAC-SHA256",
"x-amz-meta-uuid": "1234",
"Content-Encoding": "gzip",
}
// Add form data
for k, v := range addFormData {
formData[k] = v
}
// Create the multipart form.
var buf bytes.Buffer
w := multipart.NewWriter(&buf)
// Set the normal formData
for k, v := range formData {
w.WriteField(k, v)
}
// Set the File formData but don't if we want send an incomplete multipart request
if !corruptedMultipart {
writer, err := w.CreateFormFile("file", "upload.txt")
if err != nil {
// return nil, err
return nil, err
}
writer.Write(objData)
// Close before creating the new request.
w.Close()
}
// Set the body equal to the created policy.
reader := bytes.NewReader(buf.Bytes())
req, err := http.NewRequest(http.MethodPost, makeTestTargetURL(endPoint, bucketName, "", nil), reader)
if err != nil {
return nil, err
}
// Set form content-type.
req.Header.Set("Content-Type", w.FormDataContentType())
return req, nil
}
func newPostRequestV4WithContentLength(endPoint, bucketName, objectName string, objData []byte, accessKey, secretKey string) (*http.Request, error) {
t := time.Now().UTC()
region := "us-east-1"
policy := buildGenericPolicy(t, accessKey, region, bucketName, objectName, true)
return newPostRequestV4Generic(endPoint, bucketName, objectName, objData, accessKey, secretKey, region, t, policy, nil, false, false)
}
func newPostRequestV4(endPoint, bucketName, objectName string, objData []byte, accessKey, secretKey string) (*http.Request, error) {
t := time.Now().UTC()
region := "us-east-1"
policy := buildGenericPolicy(t, accessKey, region, bucketName, objectName, false)
return newPostRequestV4Generic(endPoint, bucketName, objectName, objData, accessKey, secretKey, region, t, policy, nil, false, false)
}
// construct URL for http requests for bucket operations.
func makeTestTargetURL(endPoint, bucketName, objectName string, queryValues url.Values) string {
urlStr := endPoint + "/"
if bucketName != "" {
urlStr = urlStr + bucketName + "/"
}
if objectName != "" {
urlStr = urlStr + EncodePath(objectName)
}
if len(queryValues) > 0 {
urlStr = urlStr + "?" + queryValues.Encode()
}
return urlStr
}
// if object matches reserved string, no need to encode them
var reservedObjectNames = regexp.MustCompile("^[a-zA-Z0-9-_.~/]+$")
// EncodePath encode the strings from UTF-8 byte representations to HTML hex escape sequences
//
// This is necessary since regular url.Parse() and url.Encode() functions do not support UTF-8
// non english characters cannot be parsed due to the nature in which url.Encode() is written
//
// This function on the other hand is a direct replacement for url.Encode() technique to support
// pretty much every UTF-8 character.
func EncodePath(pathName string) string {
if reservedObjectNames.MatchString(pathName) {
return pathName
}
var encodedPathname string
for _, s := range pathName {
if 'A' <= s && s <= 'Z' || 'a' <= s && s <= 'z' || '0' <= s && s <= '9' { // §2.3 Unreserved characters (mark)
encodedPathname = encodedPathname + string(s)
continue
}
switch s {
case '-', '_', '.', '~', '/': // §2.3 Unreserved characters (mark)
encodedPathname = encodedPathname + string(s)
continue
default:
len := utf8.RuneLen(s)
if len < 0 {
// if utf8 cannot convert return the same string as is
return pathName
}
u := make([]byte, len)
utf8.EncodeRune(u, s)
for _, r := range u {
hex := hex.EncodeToString([]byte{r})
encodedPathname = encodedPathname + "%" + strings.ToUpper(hex)
}
}
}
return encodedPathname
}
// getCredentialString generate a credential string.
func getCredentialString(accessKeyID, location string, t time.Time) string {
return accessKeyID + "/" + getScope(t, location)
}
// getScope generate a string of a specific date, an AWS region, and a service.
func getScope(t time.Time, region string) string {
scope := strings.Join([]string{
t.Format("20060102"),
region,
string("s3"),
"aws4_request",
}, "/")
return scope
}

View File

@@ -0,0 +1,276 @@
package policy
/*
* MinIO Cloud Storage, (C) 2015, 2016, 2017 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import (
"encoding/json"
"errors"
"fmt"
"net/http"
"reflect"
"strconv"
"strings"
"time"
)
// startWithConds - map which indicates if a given condition supports starts-with policy operator
var startsWithConds = map[string]bool{
"$acl": true,
"$bucket": false,
"$cache-control": true,
"$content-type": true,
"$content-disposition": true,
"$content-encoding": true,
"$expires": true,
"$key": true,
"$success_action_redirect": true,
"$redirect": true,
"$success_action_status": false,
"$x-amz-algorithm": false,
"$x-amz-credential": false,
"$x-amz-date": false,
}
// Add policy conditionals.
const (
policyCondEqual = "eq"
policyCondStartsWith = "starts-with"
policyCondContentLength = "content-length-range"
)
// toString - Safely convert interface to string without causing panic.
func toString(val interface{}) string {
switch v := val.(type) {
case string:
return v
default:
return ""
}
}
// toLowerString - safely convert interface to lower string
func toLowerString(val interface{}) string {
return strings.ToLower(toString(val))
}
// toInteger _ Safely convert interface to integer without causing panic.
func toInteger(val interface{}) (int64, error) {
switch v := val.(type) {
case float64:
return int64(v), nil
case int64:
return v, nil
case int:
return int64(v), nil
case string:
i, err := strconv.Atoi(v)
return int64(i), err
default:
return 0, errors.New("Invalid number format")
}
}
// isString - Safely check if val is of type string without causing panic.
func isString(val interface{}) bool {
_, ok := val.(string)
return ok
}
// ContentLengthRange - policy content-length-range field.
type contentLengthRange struct {
Min int64
Max int64
Valid bool // If content-length-range was part of policy
}
// PostPolicyForm provides strict static type conversion and validation for Amazon S3's POST policy JSON string.
type PostPolicyForm struct {
Expiration time.Time // Expiration date and time of the POST policy.
Conditions struct { // Conditional policy structure.
Policies []struct {
Operator string
Key string
Value string
}
ContentLengthRange contentLengthRange
}
}
// ParsePostPolicyForm - Parse JSON policy string into typed PostPolicyForm structure.
func ParsePostPolicyForm(policy string) (ppf PostPolicyForm, e error) {
// Convert po into interfaces and
// perform strict type conversion using reflection.
var rawPolicy struct {
Expiration string `json:"expiration"`
Conditions []interface{} `json:"conditions"`
}
err := json.Unmarshal([]byte(policy), &rawPolicy)
if err != nil {
return ppf, err
}
parsedPolicy := PostPolicyForm{}
// Parse expiry time.
parsedPolicy.Expiration, err = time.Parse(time.RFC3339Nano, rawPolicy.Expiration)
if err != nil {
return ppf, err
}
// Parse conditions.
for _, val := range rawPolicy.Conditions {
switch condt := val.(type) {
case map[string]interface{}: // Handle key:value map types.
for k, v := range condt {
if !isString(v) { // Pre-check value type.
// All values must be of type string.
return parsedPolicy, fmt.Errorf("Unknown type %s of conditional field value %s found in POST policy form", reflect.TypeOf(condt).String(), condt)
}
// {"acl": "public-read" } is an alternate way to indicate - [ "eq", "$acl", "public-read" ]
// In this case we will just collapse this into "eq" for all use cases.
parsedPolicy.Conditions.Policies = append(parsedPolicy.Conditions.Policies, struct {
Operator string
Key string
Value string
}{
policyCondEqual, "$" + strings.ToLower(k), toString(v),
})
}
case []interface{}: // Handle array types.
if len(condt) != 3 { // Return error if we have insufficient elements.
return parsedPolicy, fmt.Errorf("Malformed conditional fields %s of type %s found in POST policy form", condt, reflect.TypeOf(condt).String())
}
switch toLowerString(condt[0]) {
case policyCondEqual, policyCondStartsWith:
for _, v := range condt { // Pre-check all values for type.
if !isString(v) {
// All values must be of type string.
return parsedPolicy, fmt.Errorf("Unknown type %s of conditional field value %s found in POST policy form", reflect.TypeOf(condt).String(), condt)
}
}
operator, matchType, value := toLowerString(condt[0]), toLowerString(condt[1]), toString(condt[2])
if !strings.HasPrefix(matchType, "$") {
return parsedPolicy, fmt.Errorf("Invalid according to Policy: Policy Condition failed: [%s, %s, %s]", operator, matchType, value)
}
parsedPolicy.Conditions.Policies = append(parsedPolicy.Conditions.Policies, struct {
Operator string
Key string
Value string
}{
operator, matchType, value,
})
case policyCondContentLength:
min, err := toInteger(condt[1])
if err != nil {
return parsedPolicy, err
}
max, err := toInteger(condt[2])
if err != nil {
return parsedPolicy, err
}
parsedPolicy.Conditions.ContentLengthRange = contentLengthRange{
Min: min,
Max: max,
Valid: true,
}
default:
// Condition should be valid.
return parsedPolicy, fmt.Errorf("Unknown type %s of conditional field value %s found in POST policy form",
reflect.TypeOf(condt).String(), condt)
}
default:
return parsedPolicy, fmt.Errorf("Unknown field %s of type %s found in POST policy form",
condt, reflect.TypeOf(condt).String())
}
}
return parsedPolicy, nil
}
// checkPolicyCond returns a boolean to indicate if a condition is satisified according
// to the passed operator
func checkPolicyCond(op string, input1, input2 string) bool {
switch op {
case policyCondEqual:
return input1 == input2
case policyCondStartsWith:
return strings.HasPrefix(input1, input2)
}
return false
}
// CheckPostPolicy - apply policy conditions and validate input values.
// (http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-HTTPPOSTConstructPolicy.html)
func CheckPostPolicy(formValues http.Header, postPolicyForm PostPolicyForm) error {
// Check if policy document expiry date is still not reached
if !postPolicyForm.Expiration.After(time.Now().UTC()) {
return fmt.Errorf("Invalid according to Policy: Policy expired")
}
// map to store the metadata
metaMap := make(map[string]string)
for _, policy := range postPolicyForm.Conditions.Policies {
if strings.HasPrefix(policy.Key, "$x-amz-meta-") {
formCanonicalName := http.CanonicalHeaderKey(strings.TrimPrefix(policy.Key, "$"))
metaMap[formCanonicalName] = policy.Value
}
}
// Check if any extra metadata field is passed as input
for key := range formValues {
if strings.HasPrefix(key, "X-Amz-Meta-") {
if _, ok := metaMap[key]; !ok {
return fmt.Errorf("Invalid according to Policy: Extra input fields: %s", key)
}
}
}
// Flag to indicate if all policies conditions are satisfied
var condPassed bool
// Iterate over policy conditions and check them against received form fields
for _, policy := range postPolicyForm.Conditions.Policies {
// Form fields names are in canonical format, convert conditions names
// to canonical for simplification purpose, so `$key` will become `Key`
formCanonicalName := http.CanonicalHeaderKey(strings.TrimPrefix(policy.Key, "$"))
// Operator for the current policy condition
op := policy.Operator
// If the current policy condition is known
if startsWithSupported, condFound := startsWithConds[policy.Key]; condFound {
// Check if the current condition supports starts-with operator
if op == policyCondStartsWith && !startsWithSupported {
return fmt.Errorf("Invalid according to Policy: Policy Condition failed")
}
// Check if current policy condition is satisfied
condPassed = checkPolicyCond(op, formValues.Get(formCanonicalName), policy.Value)
if !condPassed {
return fmt.Errorf("Invalid according to Policy: Policy Condition failed")
}
} else {
// This covers all conditions X-Amz-Meta-* and X-Amz-*
if strings.HasPrefix(policy.Key, "$x-amz-meta-") || strings.HasPrefix(policy.Key, "$x-amz-") {
// Check if policy condition is satisfied
condPassed = checkPolicyCond(op, formValues.Get(formCanonicalName), policy.Value)
if !condPassed {
return fmt.Errorf("Invalid according to Policy: Policy Condition failed: [%s, %s, %s]", op, policy.Key, policy.Value)
}
}
}
}
return nil
}

View File

@@ -0,0 +1,106 @@
package policy
/*
* MinIO Cloud Storage, (C) 2016 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import (
"encoding/base64"
"fmt"
"net/http"
"testing"
"time"
)
// Test Post Policy parsing and checking conditions
func TestPostPolicyForm(t *testing.T) {
pp := NewPostPolicy()
pp.SetBucket("testbucket")
pp.SetContentType("image/jpeg")
pp.SetUserMetadata("uuid", "14365123651274")
pp.SetKeyStartsWith("user/user1/filename")
pp.SetContentLengthRange(1048579, 10485760)
pp.SetSuccessStatusAction("201")
type testCase struct {
Bucket string
Key string
XAmzDate string
XAmzAlgorithm string
XAmzCredential string
XAmzMetaUUID string
ContentType string
SuccessActionStatus string
Policy string
Expired bool
expectedErr error
}
testCases := []testCase{
// Everything is fine with this test
{Bucket: "testbucket", Key: "user/user1/filename/${filename}/myfile.txt", XAmzMetaUUID: "14365123651274", SuccessActionStatus: "201", XAmzCredential: "KVGKMDUQ23TCZXTLTHLP/20160727/us-east-1/s3/aws4_request", XAmzDate: "20160727T000000Z", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", expectedErr: nil},
// Expired policy document
{Bucket: "testbucket", Key: "user/user1/filename/${filename}/myfile.txt", XAmzMetaUUID: "14365123651274", SuccessActionStatus: "201", XAmzCredential: "KVGKMDUQ23TCZXTLTHLP/20160727/us-east-1/s3/aws4_request", XAmzDate: "20160727T000000Z", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", Expired: true, expectedErr: fmt.Errorf("Invalid according to Policy: Policy expired")},
// Different AMZ date
{Bucket: "testbucket", Key: "user/user1/filename/${filename}/myfile.txt", XAmzMetaUUID: "14365123651274", XAmzDate: "2017T000000Z", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", expectedErr: fmt.Errorf("Invalid according to Policy: Policy Condition failed")},
// Key which doesn't start with user/user1/filename
{Bucket: "testbucket", Key: "myfile.txt", XAmzDate: "20160727T000000Z", XAmzMetaUUID: "14365123651274", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", expectedErr: fmt.Errorf("Invalid according to Policy: Policy Condition failed")},
// Incorrect bucket name.
{Bucket: "incorrect", Key: "user/user1/filename/myfile.txt", XAmzMetaUUID: "14365123651274", XAmzDate: "20160727T000000Z", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", expectedErr: fmt.Errorf("Invalid according to Policy: Policy Condition failed")},
// Incorrect key name
{Bucket: "testbucket", Key: "incorrect", XAmzDate: "20160727T000000Z", XAmzMetaUUID: "14365123651274", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", expectedErr: fmt.Errorf("Invalid according to Policy: Policy Condition failed")},
// Incorrect date
{Bucket: "testbucket", Key: "user/user1/filename/${filename}/myfile.txt", XAmzMetaUUID: "14365123651274", XAmzDate: "incorrect", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", expectedErr: fmt.Errorf("Invalid according to Policy: Policy Condition failed")},
// Incorrect ContentType
{Bucket: "testbucket", Key: "user/user1/filename/${filename}/myfile.txt", XAmzMetaUUID: "14365123651274", XAmzDate: "20160727T000000Z", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "incorrect", expectedErr: fmt.Errorf("Invalid according to Policy: Policy Condition failed")},
// Incorrect Metadata
{Bucket: "testbucket", Key: "user/user1/filename/${filename}/myfile.txt", XAmzMetaUUID: "151274", SuccessActionStatus: "201", XAmzCredential: "KVGKMDUQ23TCZXTLTHLP/20160727/us-east-1/s3/aws4_request", XAmzDate: "20160727T000000Z", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", expectedErr: fmt.Errorf("Invalid according to Policy: Policy Condition failed: [eq, $x-amz-meta-uuid, 14365123651274]")},
}
// Validate all the test cases.
for i, tt := range testCases {
formValues := make(http.Header)
formValues.Set("Bucket", tt.Bucket)
formValues.Set("Key", tt.Key)
formValues.Set("Content-Type", tt.ContentType)
formValues.Set("X-Amz-Date", tt.XAmzDate)
formValues.Set("X-Amz-Meta-Uuid", tt.XAmzMetaUUID)
formValues.Set("X-Amz-Algorithm", tt.XAmzAlgorithm)
formValues.Set("X-Amz-Credential", tt.XAmzCredential)
if tt.Expired {
// Expired already.
pp.SetExpires(time.Now().UTC().AddDate(0, 0, -10))
} else {
// Expires in 10 days.
pp.SetExpires(time.Now().UTC().AddDate(0, 0, 10))
}
formValues.Set("Policy", base64.StdEncoding.EncodeToString([]byte(pp.String())))
formValues.Set("Success_action_status", tt.SuccessActionStatus)
policyBytes, err := base64.StdEncoding.DecodeString(base64.StdEncoding.EncodeToString([]byte(pp.String())))
if err != nil {
t.Fatal(err)
}
postPolicyForm, err := ParsePostPolicyForm(string(policyBytes))
if err != nil {
t.Fatal(err)
}
err = CheckPostPolicy(formValues, postPolicyForm)
if err != nil && tt.expectedErr != nil && err.Error() != tt.expectedErr.Error() {
t.Fatalf("Test %d:, Expected %s, got %s", i+1, tt.expectedErr.Error(), err.Error())
}
}
}