fix S3 per-user-directory Policy (#6443)

* fix S3 per-user-directory Policy

* Delete docker/config.json

* add tests

* remove logs

* undo modifications of weed/shell/command_volume_balance.go

* remove modifications of docker-compose

* fix failing test

---------

Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>

weed/command/iam.go

@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"net/http"
 
+	"time"
+
 	"github.com/gorilla/mux"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/iamapi"
@@ -12,7 +14,6 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/security"
 	"github.com/seaweedfs/seaweedfs/weed/util"
-	"time"
 )
 
 var (
@@ -35,7 +36,7 @@ func init() {
 }
 
 var cmdIam = &Command{
-	UsageLine: "iam [-port=8111] [-filer=<ip:port>] [-masters=<ip:port>,<ip:port>]",
+	UsageLine: "iam [-port=8111] [-filer=<ip:port>] [-master=<ip:port>,<ip:port>]",
 	Short:     "start a iam API compatible server",
 	Long:      "start a iam API compatible server.",
 }

weed/iamapi/iamapi_management_handlers.go

@@ -332,26 +332,23 @@ func GetActions(policy *PolicyDocument) ([]string, error) {
 			// Parse "arn:aws:s3:::my-bucket/shared/*"
 			res := strings.Split(resource, ":")
 			if len(res) != 6 || res[0] != "arn" || res[1] != "aws" || res[2] != "s3" {
-				return nil, fmt.Errorf("not a valid resource: '%s'. Expected prefix 'arn:aws:s3'", res)
+				glog.Infof("not a valid resource: %s", res)
+				continue
 			}
 			for _, action := range statement.Action {
 				// Parse "s3:Get*"
 				act := strings.Split(action, ":")
 				if len(act) != 2 || act[0] != "s3" {
-					return nil, fmt.Errorf("not a valid action: '%s'. Expected prefix 's3:'", act)
+					glog.Infof("not a valid action: %s", act)
+					continue
 				}
 				statementAction := MapToStatementAction(act[1])
-				if res[5] == "*" {
+				path := res[5]
+				if path == "*" {
 					actions = append(actions, statementAction)
 					continue
 				}
-				// Parse my-bucket/shared/*
-				path := strings.Split(res[5], "/")
-				if len(path) != 2 || path[1] != "*" {
-					glog.Infof("not match bucket: %s", path)
-					continue
-				}
-				actions = append(actions, fmt.Sprintf("%s:%s", statementAction, path[0]))
+				actions = append(actions, fmt.Sprintf("%s:%s", statementAction, path))
 			}
 		}
 	}

weed/iamapi/iamapi_management_handlers_test.go

@@ -0,0 +1,71 @@
+package iamapi
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetActionsUserPath(t *testing.T) {
+
+	policyDocument := PolicyDocument{
+		Version: "2012-10-17",
+		Statement: []*Statement{
+			{
+				Effect: "Allow",
+				Action: []string{
+					"s3:Put*",
+					"s3:PutBucketAcl",
+					"s3:Get*",
+					"s3:GetBucketAcl",
+					"s3:List*",
+					"s3:Tagging*",
+					"s3:DeleteBucket*",
+				},
+				Resource: []string{
+					"arn:aws:s3:::shared/user-Alice/*",
+				},
+			},
+		},
+	}
+
+	actions, _ := GetActions(&policyDocument)
+
+	expectedActions := []string{
+		"Write:shared/user-Alice/*",
+		"WriteAcp:shared/user-Alice/*",
+		"Read:shared/user-Alice/*",
+		"ReadAcp:shared/user-Alice/*",
+		"List:shared/user-Alice/*",
+		"Tagging:shared/user-Alice/*",
+		"DeleteBucket:shared/user-Alice/*",
+	}
+	assert.Equal(t, expectedActions, actions)
+}
+
+func TestGetActionsWildcardPath(t *testing.T) {
+
+	policyDocument := PolicyDocument{
+		Version: "2012-10-17",
+		Statement: []*Statement{
+			{
+				Effect: "Allow",
+				Action: []string{
+					"s3:Get*",
+					"s3:PutBucketAcl",
+				},
+				Resource: []string{
+					"arn:aws:s3:::*",
+				},
+			},
+		},
+	}
+
+	actions, _ := GetActions(&policyDocument)
+
+	expectedActions := []string{
+		"Read",
+		"WriteAcp",
+	}
+	assert.Equal(t, expectedActions, actions)
+}

weed/s3api/auth_credentials.go

@@ -119,11 +119,14 @@ func NewIdentityAccessManagement(option *S3ApiServerOption) *IdentityAccessManag
 		hashes:       make(map[string]*sync.Pool),
 		hashCounters: make(map[string]*int32),
 	}
+
 	if option.Config != "" {
+		glog.V(3).Infof("loading static config file %s", option.Config)
 		if err := iam.loadS3ApiConfigurationFromFile(option.Config); err != nil {
 			glog.Fatalf("fail to load config file %s: %v", option.Config, err)
 		}
 	} else {
+		glog.V(3).Infof("no static config file specified... loading config from filer %s", option.Filer)
 		if err := iam.loadS3ApiConfigurationFromFiler(option); err != nil {
 			glog.Warningf("fail to load config: %v", err)
 		}
@@ -134,6 +137,7 @@ func NewIdentityAccessManagement(option *S3ApiServerOption) *IdentityAccessManag
 func (iam *IdentityAccessManagement) loadS3ApiConfigurationFromFiler(option *S3ApiServerOption) (err error) {
 	var content []byte
 	err = pb.WithFilerClient(false, 0, option.Filer, option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+		glog.V(3).Infof("loading config %s from filer %s", filer.IamConfigDirectory+"/"+filer.IamIdentityFile, option.Filer)
 		content, err = filer.ReadInsideFiler(client, filer.IamConfigDirectory, filer.IamIdentityFile)
 		return err
 	})
@@ -179,6 +183,7 @@ func (iam *IdentityAccessManagement) loadS3ApiConfiguration(config *iam_pb.S3Api
 	foundAccountAnonymous := false
 
 	for _, account := range config.Accounts {
+		glog.V(3).Infof("loading account  name=%s, id=%s", account.DisplayName, account.Id)
 		switch account.Id {
 		case AccountAdmin.Id:
 			AccountAdmin = Account{
@@ -217,6 +222,7 @@ func (iam *IdentityAccessManagement) loadS3ApiConfiguration(config *iam_pb.S3Api
 		emailAccount[AccountAnonymous.EmailAddress] = &AccountAnonymous
 	}
 	for _, ident := range config.Identities {
+		glog.V(3).Infof("loading identity %s", ident.Name)
 		t := &Identity{
 			Name:        ident.Name,
 			Credentials: nil,
@@ -236,6 +242,7 @@ func (iam *IdentityAccessManagement) loadS3ApiConfiguration(config *iam_pb.S3Api
 				glog.Warningf("identity %s is associated with a non exist account ID, the association is invalid", ident.Name)
 			}
 		}
+
 		for _, action := range ident.Actions {
 			t.Actions = append(t.Actions, Action(action))
 		}
@@ -379,8 +386,14 @@ func (iam *IdentityAccessManagement) authRequest(r *http.Request, action Action)
 	}
 
 	glog.V(3).Infof("user name: %v actions: %v, action: %v", identity.Name, identity.Actions, action)
-
 	bucket, object := s3_constants.GetBucketAndObject(r)
+	prefix := s3_constants.GetPrefix(r)
+
+	if object == "/" && prefix != "" {
+		// Using the aws cli with s3, and s3api, and with boto3, the object is always set to "/"
+		// but the prefix is set to the actual object key
+		object = prefix
+	}
 
 	if !identity.canDo(action, bucket, object) {
 		return identity, s3err.ErrAccessDenied
@@ -447,6 +460,10 @@ func (identity *Identity) canDo(action Action, bucket string, objectKey string)
 		return true
 	}
 	for _, a := range identity.Actions {
+		// Case where the Resource provided is
+		// 	"Resource": [
+		//		"arn:aws:s3:::*"
+		//	]
 		if a == action {
 			return true
 		}
@@ -455,10 +472,12 @@ func (identity *Identity) canDo(action Action, bucket string, objectKey string)
 		glog.V(3).Infof("identity %s is not allowed to perform action %s on %s -- bucket is empty", identity.Name, action, bucket+objectKey)
 		return false
 	}
+	glog.V(3).Infof("checking if %s can perform %s on bucket '%s'", identity.Name, action, bucket+objectKey)
 	target := string(action) + ":" + bucket + objectKey
 	adminTarget := s3_constants.ACTION_ADMIN + ":" + bucket + objectKey
 	limitedByBucket := string(action) + ":" + bucket
 	adminLimitedByBucket := s3_constants.ACTION_ADMIN + ":" + bucket
+
 	for _, a := range identity.Actions {
 		act := string(a)
 		if strings.HasSuffix(act, "*") {

weed/s3api/s3_constants/header.go

@@ -17,9 +17,10 @@
 package s3_constants
 
 import (
-	"github.com/gorilla/mux"
 	"net/http"
 	"strings"
+
+	"github.com/gorilla/mux"
 )
 
 // Standard S3 HTTP request constants
@@ -72,6 +73,16 @@ func GetBucketAndObject(r *http.Request) (bucket, object string) {
 	return
 }
 
+func GetPrefix(r *http.Request) string {
+	query := r.URL.Query()
+	prefix := query.Get("prefix")
+	if !strings.HasPrefix(prefix, "/") {
+		prefix = "/" + prefix
+	}
+
+	return prefix
+}
+
 var PassThroughHeaders = map[string]string{
 	"response-cache-control":       "Cache-Control",
 	"response-content-disposition": "Content-Disposition",