From a77dfb1ddd36d071f90acc01a1399e7ae52865d1 Mon Sep 17 00:00:00 2001 From: chrislu Date: Fri, 21 Nov 2025 15:33:38 -0800 Subject: [PATCH] add debugging for InvalidAccessKeyId --- .../filer_etc/filer_etc_identity.go | 50 +++++++++++++++++-- weed/credential/filer_etc/filer_etc_policy.go | 32 ++++++++++-- weed/s3api/auth_credentials.go | 42 +++++++++++++++- weed/s3api/auth_signature_v2.go | 26 ++++++++++ weed/s3api/auth_signature_v4.go | 23 +++++++++ 5 files changed, 163 insertions(+), 10 deletions(-) diff --git a/weed/credential/filer_etc/filer_etc_identity.go b/weed/credential/filer_etc/filer_etc_identity.go index aa9417577..2b231c549 100644 --- a/weed/credential/filer_etc/filer_etc_identity.go +++ b/weed/credential/filer_etc/filer_etc_identity.go @@ -7,6 +7,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/credential" "github.com/seaweedfs/seaweedfs/weed/filer" + "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/pb/iam_pb" ) @@ -14,24 +15,63 @@ import ( func (store *FilerEtcStore) LoadConfiguration(ctx context.Context) (*iam_pb.S3ApiConfiguration, error) { s3cfg := &iam_pb.S3ApiConfiguration{} + glog.V(1).Infof("Loading IAM configuration from %s/%s (filer: %s)", + filer.IamConfigDirectory, filer.IamIdentityFile, store.filerGrpcAddress) + err := store.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { // Use ReadInsideFiler instead of ReadEntry since identity.json is small // and stored inline. ReadEntry requires a master client for chunked files, // but ReadInsideFiler only reads inline content. content, err := filer.ReadInsideFiler(client, filer.IamConfigDirectory, filer.IamIdentityFile) if err != nil { - if err != filer_pb.ErrNotFound { - return err + if err == filer_pb.ErrNotFound { + glog.V(1).Infof("IAM identity file not found at %s/%s, no credentials loaded", + filer.IamConfigDirectory, filer.IamIdentityFile) + return nil } + glog.Errorf("Failed to read IAM identity file from %s/%s: %v", + filer.IamConfigDirectory, filer.IamIdentityFile, err) + return err + } + + if len(content) == 0 { + glog.V(1).Infof("IAM identity file at %s/%s is empty", + filer.IamConfigDirectory, filer.IamIdentityFile) return nil } - if len(content) > 0 { - return filer.ParseS3ConfigurationFromBytes(content, s3cfg) + + glog.V(2).Infof("Read %d bytes from %s/%s", + len(content), filer.IamConfigDirectory, filer.IamIdentityFile) + + if err := filer.ParseS3ConfigurationFromBytes(content, s3cfg); err != nil { + glog.Errorf("Failed to parse IAM configuration from %s/%s: %v", + filer.IamConfigDirectory, filer.IamIdentityFile, err) + return err } + + glog.V(1).Infof("Successfully parsed IAM configuration with %d identities and %d accounts", + len(s3cfg.Identities), len(s3cfg.Accounts)) return nil }) - return s3cfg, err + if err != nil { + return s3cfg, err + } + + // Log loaded identities for debugging + if glog.V(2) { + for _, identity := range s3cfg.Identities { + credCount := len(identity.Credentials) + actionCount := len(identity.Actions) + glog.V(2).Infof(" Identity: %s (credentials: %d, actions: %d)", + identity.Name, credCount, actionCount) + for _, cred := range identity.Credentials { + glog.V(3).Infof(" Access Key: %s", cred.AccessKey) + } + } + } + + return s3cfg, nil } func (store *FilerEtcStore) SaveConfiguration(ctx context.Context, config *iam_pb.S3ApiConfiguration) error { diff --git a/weed/credential/filer_etc/filer_etc_policy.go b/weed/credential/filer_etc/filer_etc_policy.go index 31ac5b078..775ca97cb 100644 --- a/weed/credential/filer_etc/filer_etc_policy.go +++ b/weed/credential/filer_etc/filer_etc_policy.go @@ -27,6 +27,9 @@ func (store *FilerEtcStore) GetPolicies(ctx context.Context) (map[string]policy_ return policiesCollection.Policies, nil } + glog.V(2).Infof("Loading IAM policies from %s/%s (filer: %s)", + filer.IamConfigDirectory, filer.IamPoliciesFile, store.filerGrpcAddress) + err := store.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { // Use ReadInsideFiler instead of ReadEntry since policies.json is small // and stored inline. ReadEntry requires a master client for chunked files, @@ -34,16 +37,32 @@ func (store *FilerEtcStore) GetPolicies(ctx context.Context) (map[string]policy_ content, err := filer.ReadInsideFiler(client, filer.IamConfigDirectory, filer.IamPoliciesFile) if err != nil { if err == filer_pb.ErrNotFound { - glog.V(1).Infof("Policies file not found at %s/%s, returning empty policies", filer.IamConfigDirectory, filer.IamPoliciesFile) + glog.V(1).Infof("Policies file not found at %s/%s, returning empty policies", + filer.IamConfigDirectory, filer.IamPoliciesFile) // If file doesn't exist, return empty collection return nil } + glog.Errorf("Failed to read IAM policies file from %s/%s: %v", + filer.IamConfigDirectory, filer.IamPoliciesFile, err) return err } - if len(content) > 0 { - return json.Unmarshal(content, policiesCollection) + if len(content) == 0 { + glog.V(2).Infof("IAM policies file at %s/%s is empty", + filer.IamConfigDirectory, filer.IamPoliciesFile) + return nil } + + glog.V(2).Infof("Read %d bytes from %s/%s", + len(content), filer.IamConfigDirectory, filer.IamPoliciesFile) + + if err := json.Unmarshal(content, policiesCollection); err != nil { + glog.Errorf("Failed to parse IAM policies from %s/%s: %v", + filer.IamConfigDirectory, filer.IamPoliciesFile, err) + return err + } + + glog.V(1).Infof("Successfully loaded %d IAM policies", len(policiesCollection.Policies)) return nil }) @@ -51,6 +70,13 @@ func (store *FilerEtcStore) GetPolicies(ctx context.Context) (map[string]policy_ return nil, err } + // Log policy names for debugging + if glog.V(2) && len(policiesCollection.Policies) > 0 { + for policyName := range policiesCollection.Policies { + glog.V(2).Infof(" Policy: %s", policyName) + } + } + return policiesCollection.Policies, nil } diff --git a/weed/s3api/auth_credentials.go b/weed/s3api/auth_credentials.go index 0d99e43eb..cebcd17f5 100644 --- a/weed/s3api/auth_credentials.go +++ b/weed/s3api/auth_credentials.go @@ -349,6 +349,17 @@ func (iam *IdentityAccessManagement) loadS3ApiConfiguration(config *iam_pb.S3Api } iam.m.Unlock() + // Log configuration summary + glog.V(1).Infof("Loaded %d identities, %d accounts, %d access keys. Auth enabled: %v", + len(identities), len(accounts), len(accessKeyIdent), iam.isAuthEnabled) + + if glog.V(2) { + glog.V(2).Infof("Access key to identity mapping:") + for accessKey, identity := range accessKeyIdent { + glog.V(2).Infof(" %s -> %s (actions: %d)", accessKey, identity.Name, len(identity.Actions)) + } + } + return nil } @@ -359,14 +370,29 @@ func (iam *IdentityAccessManagement) isEnabled() bool { func (iam *IdentityAccessManagement) lookupByAccessKey(accessKey string) (identity *Identity, cred *Credential, found bool) { iam.m.RLock() defer iam.m.RUnlock() + + glog.V(3).Infof("Looking up access key: %s (total keys registered: %d)", accessKey, len(iam.accessKeyIdent)) + if ident, ok := iam.accessKeyIdent[accessKey]; ok { for _, credential := range ident.Credentials { if credential.AccessKey == accessKey { + glog.V(2).Infof("Found access key %s for identity %s", accessKey, ident.Name) return ident, credential, true } } } - glog.V(1).Infof("could not find accessKey %s", accessKey) + + glog.V(1).Infof("Could not find access key %s. Available keys: %d, Auth enabled: %v", + accessKey, len(iam.accessKeyIdent), iam.isAuthEnabled) + + // Log all registered access keys at higher verbosity for debugging + if glog.V(3) { + glog.V(3).Infof("Registered access keys:") + for key := range iam.accessKeyIdent { + glog.V(3).Infof(" - %s", key) + } + } + return nil, nil, false } @@ -649,12 +675,24 @@ func (iam *IdentityAccessManagement) GetCredentialManager() *credential.Credenti // LoadS3ApiConfigurationFromCredentialManager loads configuration using the credential manager func (iam *IdentityAccessManagement) LoadS3ApiConfigurationFromCredentialManager() error { + glog.V(1).Infof("Loading S3 API configuration from credential manager") + s3ApiConfiguration, err := iam.credentialManager.LoadConfiguration(context.Background()) if err != nil { + glog.Errorf("Failed to load configuration from credential manager: %v", err) return fmt.Errorf("failed to load configuration from credential manager: %w", err) } - return iam.loadS3ApiConfiguration(s3ApiConfiguration) + glog.V(2).Infof("Credential manager returned %d identities and %d accounts", + len(s3ApiConfiguration.Identities), len(s3ApiConfiguration.Accounts)) + + if err := iam.loadS3ApiConfiguration(s3ApiConfiguration); err != nil { + glog.Errorf("Failed to load S3 API configuration: %v", err) + return err + } + + glog.V(1).Infof("Successfully loaded S3 API configuration from credential manager") + return nil } // initializeKMSFromConfig loads KMS configuration from TOML format diff --git a/weed/s3api/auth_signature_v2.go b/weed/s3api/auth_signature_v2.go index b31c37a27..77d04e1e0 100644 --- a/weed/s3api/auth_signature_v2.go +++ b/weed/s3api/auth_signature_v2.go @@ -28,6 +28,7 @@ import ( "strings" "time" + "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" ) @@ -76,6 +77,14 @@ func (iam *IdentityAccessManagement) doesPolicySignatureV2Match(formValues http. identity, cred, found := iam.lookupByAccessKey(accessKey) if !found { + // Log detailed error information for InvalidAccessKeyId (V2 POST) + iam.m.RLock() + availableKeyCount := len(iam.accessKeyIdent) + iam.m.RUnlock() + + glog.Warningf("InvalidAccessKeyId (V2 POST): attempted key '%s' not found. Available keys: %d, Auth enabled: %v", + accessKey, availableKeyCount, iam.isAuthEnabled) + return s3err.ErrInvalidAccessKeyID } @@ -113,6 +122,14 @@ func (iam *IdentityAccessManagement) doesSignV2Match(r *http.Request) (*Identity identity, cred, found := iam.lookupByAccessKey(accessKey) if !found { + // Log detailed error information for InvalidAccessKeyId (V2 signed) + iam.m.RLock() + availableKeyCount := len(iam.accessKeyIdent) + iam.m.RUnlock() + + glog.Warningf("InvalidAccessKeyId (V2 signed): attempted key '%s' not found. Available keys: %d, Auth enabled: %v", + accessKey, availableKeyCount, iam.isAuthEnabled) + return nil, s3err.ErrInvalidAccessKeyID } @@ -145,6 +162,7 @@ func (iam *IdentityAccessManagement) doesPresignV2SignatureMatch(r *http.Request accessKey := query.Get("AWSAccessKeyId") if accessKey == "" { + glog.Warningf("InvalidAccessKeyId (V2 presigned): empty access key in request") return nil, s3err.ErrInvalidAccessKeyID } @@ -155,6 +173,14 @@ func (iam *IdentityAccessManagement) doesPresignV2SignatureMatch(r *http.Request identity, cred, found := iam.lookupByAccessKey(accessKey) if !found { + // Log detailed error information for InvalidAccessKeyId (V2 presigned) + iam.m.RLock() + availableKeyCount := len(iam.accessKeyIdent) + iam.m.RUnlock() + + glog.Warningf("InvalidAccessKeyId (V2 presigned): attempted key '%s' not found. Available keys: %d, Auth enabled: %v", + accessKey, availableKeyCount, iam.isAuthEnabled) + return nil, s3err.ErrInvalidAccessKeyID } diff --git a/weed/s3api/auth_signature_v4.go b/weed/s3api/auth_signature_v4.go index b77540255..d897894bc 100644 --- a/weed/s3api/auth_signature_v4.go +++ b/weed/s3api/auth_signature_v4.go @@ -207,6 +207,21 @@ func (iam *IdentityAccessManagement) verifyV4Signature(r *http.Request, shouldCh // 2. Lookup user and credentials identity, cred, found := iam.lookupByAccessKey(authInfo.AccessKey) if !found { + // Log detailed error information for InvalidAccessKeyId + iam.m.RLock() + availableKeys := make([]string, 0, len(iam.accessKeyIdent)) + for key := range iam.accessKeyIdent { + availableKeys = append(availableKeys, key) + } + iam.m.RUnlock() + + glog.Warningf("InvalidAccessKeyId: attempted key '%s' not found. Available keys: %d, Auth enabled: %v", + authInfo.AccessKey, len(availableKeys), iam.isAuthEnabled) + + if glog.V(2) && len(availableKeys) > 0 { + glog.V(2).Infof("Available access keys: %v", availableKeys) + } + return nil, nil, "", nil, s3err.ErrInvalidAccessKeyID } @@ -543,6 +558,14 @@ func (iam *IdentityAccessManagement) doesPolicySignatureV4Match(formValues http. identity, cred, found := iam.lookupByAccessKey(credHeader.accessKey) if !found { + // Log detailed error information for InvalidAccessKeyId (POST policy) + iam.m.RLock() + availableKeyCount := len(iam.accessKeyIdent) + iam.m.RUnlock() + + glog.Warningf("InvalidAccessKeyId (POST policy): attempted key '%s' not found. Available keys: %d, Auth enabled: %v", + credHeader.accessKey, availableKeyCount, iam.isAuthEnabled) + return s3err.ErrInvalidAccessKeyID }