S3 Object Lock: ensure x-amz-bucket-object-lock-enabled header (#6990)

* ensure x-amz-bucket-object-lock-enabled header

* fix tests

* combine 2 metadata changes into one

* address comments

* Update s3api_bucket_handlers.go

* Update weed/s3api/s3api_bucket_handlers.go

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update test/s3/retention/object_lock_reproduce_test.go

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update test/s3/retention/object_lock_validation_test.go

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update test/s3/retention/s3_bucket_object_lock_test.go

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update weed/s3api/s3api_bucket_handlers.go

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update weed/s3api/s3api_bucket_handlers.go

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update test/s3/retention/s3_bucket_object_lock_test.go

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update weed/s3api/s3api_bucket_handlers.go

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* package name

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
This commit is contained in:
Chris Lu 2025-07-15 23:21:58 -07:00 committed by GitHub
parent 64c5dde2f3
commit dde1cf63c2
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 458 additions and 2 deletions

3
.gitignore vendored
View File

@ -107,3 +107,6 @@ test/s3/cors/weed-test.log
test/s3/cors/weed-server.pid
/test/s3/cors/test-volume-data
test/s3/cors/cors.test
/test/s3/retention/filerldb2
test/s3/retention/weed-test.log
test/s3/retention/weed-server.pid

View File

@ -0,0 +1,114 @@
package retention
import (
"context"
"fmt"
"testing"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/stretchr/testify/require"
)
// TestReproduceObjectLockIssue reproduces the Object Lock header processing issue step by step
func TestReproduceObjectLockIssue(t *testing.T) {
client := getS3Client(t)
bucketName := fmt.Sprintf("object-lock-test-%d", time.Now().UnixNano())
t.Logf("=== Reproducing Object Lock Header Processing Issue ===")
t.Logf("Bucket name: %s", bucketName)
// Step 1: Create bucket with Object Lock enabled header
t.Logf("\n1. Creating bucket with ObjectLockEnabledForBucket=true")
t.Logf(" This should send x-amz-bucket-object-lock-enabled: true header")
createResp, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{
Bucket: aws.String(bucketName),
ObjectLockEnabledForBucket: true, // This sets the x-amz-bucket-object-lock-enabled header
})
if err != nil {
t.Fatalf("Bucket creation failed: %v", err)
}
t.Logf("✅ Bucket created successfully")
t.Logf(" Response: %+v", createResp)
// Step 2: Check if Object Lock is actually enabled
t.Logf("\n2. Checking Object Lock configuration to verify it was enabled")
objectLockResp, err := client.GetObjectLockConfiguration(context.TODO(), &s3.GetObjectLockConfigurationInput{
Bucket: aws.String(bucketName),
})
if err != nil {
t.Logf("❌ GetObjectLockConfiguration FAILED: %v", err)
t.Logf(" This demonstrates the issue with header processing!")
t.Logf(" S3 clients expect this call to succeed if Object Lock is supported")
t.Logf(" When this fails, clients conclude that Object Lock is not supported")
// This failure demonstrates the bug - the bucket was created but Object Lock wasn't enabled
t.Logf("\n🐛 BUG CONFIRMED:")
t.Logf(" - Bucket creation with ObjectLockEnabledForBucket=true succeeded")
t.Logf(" - But GetObjectLockConfiguration fails")
t.Logf(" - This means the x-amz-bucket-object-lock-enabled header was ignored")
} else {
t.Logf("✅ GetObjectLockConfiguration succeeded!")
t.Logf(" Response: %+v", objectLockResp)
t.Logf(" Object Lock is properly enabled - this is the expected behavior")
}
// Step 3: Check versioning status (required for Object Lock)
t.Logf("\n3. Checking bucket versioning status (required for Object Lock)")
versioningResp, err := client.GetBucketVersioning(context.TODO(), &s3.GetBucketVersioningInput{
Bucket: aws.String(bucketName),
})
require.NoError(t, err)
t.Logf(" Versioning status: %v", versioningResp.Status)
if versioningResp.Status != "Enabled" {
t.Logf(" ⚠️ Versioning should be automatically enabled when Object Lock is enabled")
}
// Cleanup
t.Logf("\n4. Cleaning up test bucket")
_, err = client.DeleteBucket(context.TODO(), &s3.DeleteBucketInput{
Bucket: aws.String(bucketName),
})
if err != nil {
t.Logf(" Warning: Failed to delete bucket: %v", err)
}
t.Logf("\n=== Issue Reproduction Complete ===")
t.Logf("Expected behavior after fix:")
t.Logf(" - CreateBucket with ObjectLockEnabledForBucket=true should enable Object Lock")
t.Logf(" - GetObjectLockConfiguration should return enabled configuration")
t.Logf(" - Versioning should be automatically enabled")
}
// TestNormalBucketCreationStillWorks tests that normal bucket creation still works
func TestNormalBucketCreationStillWorks(t *testing.T) {
client := getS3Client(t)
bucketName := fmt.Sprintf("normal-test-%d", time.Now().UnixNano())
t.Logf("=== Testing Normal Bucket Creation ===")
// Create bucket without Object Lock
_, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{
Bucket: aws.String(bucketName),
})
require.NoError(t, err)
t.Logf("✅ Normal bucket creation works")
// Object Lock should NOT be enabled
_, err = client.GetObjectLockConfiguration(context.TODO(), &s3.GetObjectLockConfigurationInput{
Bucket: aws.String(bucketName),
})
require.Error(t, err, "GetObjectLockConfiguration should fail for bucket without Object Lock")
t.Logf("✅ GetObjectLockConfiguration correctly fails for normal bucket")
// Cleanup
client.DeleteBucket(context.TODO(), &s3.DeleteBucketInput{Bucket: aws.String(bucketName)})
}

View File

@ -0,0 +1,105 @@
package retention
import (
"context"
"fmt"
"strings"
"testing"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/stretchr/testify/require"
)
// TestObjectLockValidation tests that S3 Object Lock functionality works end-to-end
// This test focuses on the complete Object Lock workflow that S3 clients expect
func TestObjectLockValidation(t *testing.T) {
client := getS3Client(t)
bucketName := fmt.Sprintf("object-lock-test-%d", time.Now().UnixNano())
t.Logf("=== Validating S3 Object Lock Functionality ===")
t.Logf("Bucket: %s", bucketName)
// Step 1: Create bucket with Object Lock header
t.Log("\n1. Creating bucket with x-amz-bucket-object-lock-enabled: true")
_, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{
Bucket: aws.String(bucketName),
ObjectLockEnabledForBucket: true, // This sends x-amz-bucket-object-lock-enabled: true
})
require.NoError(t, err, "Bucket creation should succeed")
defer client.DeleteBucket(context.TODO(), &s3.DeleteBucketInput{Bucket: aws.String(bucketName)})
t.Log(" ✅ Bucket created successfully")
// Step 2: Check if Object Lock is supported (standard S3 client behavior)
t.Log("\n2. Testing Object Lock support detection")
_, err = client.GetObjectLockConfiguration(context.TODO(), &s3.GetObjectLockConfigurationInput{
Bucket: aws.String(bucketName),
})
require.NoError(t, err, "GetObjectLockConfiguration should succeed for Object Lock enabled bucket")
t.Log(" ✅ GetObjectLockConfiguration succeeded - Object Lock is properly enabled")
// Step 3: Verify versioning is enabled (required for Object Lock)
t.Log("\n3. Verifying versioning is automatically enabled")
versioningResp, err := client.GetBucketVersioning(context.TODO(), &s3.GetBucketVersioningInput{
Bucket: aws.String(bucketName),
})
require.NoError(t, err)
require.Equal(t, types.BucketVersioningStatusEnabled, versioningResp.Status, "Versioning should be automatically enabled")
t.Log(" ✅ Versioning automatically enabled")
// Step 4: Test actual Object Lock functionality
t.Log("\n4. Testing Object Lock retention functionality")
// Create an object
key := "protected-object.dat"
content := "Important data that needs immutable protection"
putResp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(key),
Body: strings.NewReader(content),
})
require.NoError(t, err)
require.NotNil(t, putResp.VersionId, "Object should have a version ID")
t.Log(" ✅ Object created with versioning")
// Apply Object Lock retention
retentionUntil := time.Now().Add(24 * time.Hour)
_, err = client.PutObjectRetention(context.TODO(), &s3.PutObjectRetentionInput{
Bucket: aws.String(bucketName),
Key: aws.String(key),
Retention: &types.ObjectLockRetention{
Mode: types.ObjectLockRetentionModeCompliance,
RetainUntilDate: aws.Time(retentionUntil),
},
})
require.NoError(t, err, "Setting Object Lock retention should succeed")
t.Log(" ✅ Object Lock retention applied successfully")
// Verify retention is in effect
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(key),
})
require.Error(t, err, "Object should be protected by retention and cannot be deleted")
t.Log(" ✅ Object is properly protected by retention policy")
// Verify we can read the object (should still work)
getResp, err := client.GetObject(context.TODO(), &s3.GetObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(key),
})
require.NoError(t, err, "Reading protected object should still work")
defer getResp.Body.Close()
t.Log(" ✅ Protected object can still be read")
t.Log("\n🎉 S3 OBJECT LOCK VALIDATION SUCCESSFUL!")
t.Log(" - Bucket creation with Object Lock header works")
t.Log(" - Object Lock support detection works (GetObjectLockConfiguration succeeds)")
t.Log(" - Versioning is automatically enabled")
t.Log(" - Object Lock retention functionality works")
t.Log(" - Objects are properly protected from deletion")
t.Log("")
t.Log("✅ S3 clients will now recognize SeaweedFS as supporting Object Lock!")
}

View File

@ -0,0 +1,185 @@
package retention
import (
"context"
"strings"
"testing"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// TestBucketCreationWithObjectLockEnabled tests creating a bucket with the
// x-amz-bucket-object-lock-enabled header, which is required for S3 Object Lock compatibility
func TestBucketCreationWithObjectLockEnabled(t *testing.T) {
// This test verifies that bucket creation with
// x-amz-bucket-object-lock-enabled header should automatically enable Object Lock
client := getS3Client(t)
bucketName := getNewBucketName()
defer func() {
// Best effort cleanup
deleteBucket(t, client, bucketName)
}()
// Test 1: Create bucket with Object Lock enabled header using custom HTTP client
t.Run("CreateBucketWithObjectLockHeader", func(t *testing.T) {
// Create bucket with x-amz-bucket-object-lock-enabled header
// This simulates what S3 clients do when testing Object Lock support
createResp, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{
Bucket: aws.String(bucketName),
ObjectLockEnabledForBucket: true, // This should set x-amz-bucket-object-lock-enabled header
})
require.NoError(t, err)
require.NotNil(t, createResp)
// Verify bucket was created
_, err = client.HeadBucket(context.TODO(), &s3.HeadBucketInput{
Bucket: aws.String(bucketName),
})
require.NoError(t, err)
})
// Test 2: Verify that Object Lock is automatically enabled for the bucket
t.Run("VerifyObjectLockAutoEnabled", func(t *testing.T) {
// Try to get the Object Lock configuration
// If the header was processed correctly, this should return an enabled configuration
configResp, err := client.GetObjectLockConfiguration(context.TODO(), &s3.GetObjectLockConfigurationInput{
Bucket: aws.String(bucketName),
})
require.NoError(t, err, "GetObjectLockConfiguration should not fail if Object Lock is enabled")
require.NotNil(t, configResp.ObjectLockConfiguration, "ObjectLockConfiguration should not be nil")
assert.Equal(t, types.ObjectLockEnabledEnabled, configResp.ObjectLockConfiguration.ObjectLockEnabled, "Object Lock should be enabled")
})
// Test 3: Verify versioning is automatically enabled (required for Object Lock)
t.Run("VerifyVersioningAutoEnabled", func(t *testing.T) {
// Object Lock requires versioning to be enabled
// When Object Lock is enabled via header, versioning should also be enabled automatically
versioningResp, err := client.GetBucketVersioning(context.TODO(), &s3.GetBucketVersioningInput{
Bucket: aws.String(bucketName),
})
require.NoError(t, err)
// Versioning should be automatically enabled for Object Lock
assert.Equal(t, types.BucketVersioningStatusEnabled, versioningResp.Status, "Versioning should be automatically enabled for Object Lock")
})
}
// TestBucketCreationWithoutObjectLockHeader tests normal bucket creation
// to ensure we don't break existing functionality
func TestBucketCreationWithoutObjectLockHeader(t *testing.T) {
client := getS3Client(t)
bucketName := getNewBucketName()
defer deleteBucket(t, client, bucketName)
// Create bucket without Object Lock header
_, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{
Bucket: aws.String(bucketName),
})
require.NoError(t, err)
// Verify bucket was created
_, err = client.HeadBucket(context.TODO(), &s3.HeadBucketInput{
Bucket: aws.String(bucketName),
})
require.NoError(t, err)
// Object Lock should NOT be enabled
_, err = client.GetObjectLockConfiguration(context.TODO(), &s3.GetObjectLockConfigurationInput{
Bucket: aws.String(bucketName),
})
// This should fail since Object Lock is not enabled
require.Error(t, err)
t.Logf("GetObjectLockConfiguration correctly failed for bucket without Object Lock: %v", err)
// Versioning should not be enabled by default
versioningResp, err := client.GetBucketVersioning(context.TODO(), &s3.GetBucketVersioningInput{
Bucket: aws.String(bucketName),
})
require.NoError(t, err)
// Should be either empty/unset or Suspended, but not Enabled
if versioningResp.Status != types.BucketVersioningStatusEnabled {
t.Logf("Versioning correctly not enabled: %v", versioningResp.Status)
} else {
t.Errorf("Versioning should not be enabled for bucket without Object Lock header")
}
}
// TestS3ObjectLockWorkflow tests the complete Object Lock workflow that S3 clients would use
func TestS3ObjectLockWorkflow(t *testing.T) {
client := getS3Client(t)
bucketName := getNewBucketName()
defer deleteBucket(t, client, bucketName)
// Step 1: Client creates bucket with Object Lock enabled
t.Run("ClientCreatesBucket", func(t *testing.T) {
_, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{
Bucket: aws.String(bucketName),
ObjectLockEnabledForBucket: true,
})
require.NoError(t, err)
})
// Step 2: Client checks if Object Lock is supported by getting the configuration
t.Run("ClientChecksObjectLockSupport", func(t *testing.T) {
configResp, err := client.GetObjectLockConfiguration(context.TODO(), &s3.GetObjectLockConfigurationInput{
Bucket: aws.String(bucketName),
})
require.NoError(t, err, "Object Lock configuration check should succeed")
// S3 clients should see Object Lock is enabled
require.NotNil(t, configResp.ObjectLockConfiguration)
assert.Equal(t, types.ObjectLockEnabledEnabled, configResp.ObjectLockConfiguration.ObjectLockEnabled)
t.Log("Object Lock configuration retrieved successfully - S3 clients would see this as supported")
})
// Step 3: Client would then configure retention policies and use Object Lock
t.Run("ClientConfiguresRetention", func(t *testing.T) {
// Verify versioning is automatically enabled (required for Object Lock)
versioningResp, err := client.GetBucketVersioning(context.TODO(), &s3.GetBucketVersioningInput{
Bucket: aws.String(bucketName),
})
require.NoError(t, err)
require.Equal(t, types.BucketVersioningStatusEnabled, versioningResp.Status, "Versioning should be automatically enabled")
// Create an object
key := "protected-backup-object"
content := "Backup data with Object Lock protection"
putResp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(key),
Body: strings.NewReader(content),
})
require.NoError(t, err)
require.NotNil(t, putResp.VersionId)
// Set Object Lock retention (what backup clients do to protect data)
retentionUntil := time.Now().Add(24 * time.Hour)
_, err = client.PutObjectRetention(context.TODO(), &s3.PutObjectRetentionInput{
Bucket: aws.String(bucketName),
Key: aws.String(key),
Retention: &types.ObjectLockRetention{
Mode: types.ObjectLockRetentionModeCompliance,
RetainUntilDate: aws.Time(retentionUntil),
},
})
require.NoError(t, err)
// Verify object is protected
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(key),
})
require.Error(t, err, "Object should be protected by retention policy")
t.Log("Object Lock retention successfully applied - data is immutable")
})
}

View File

@ -1,4 +1,4 @@
package s3api
package retention
import (
"context"

View File

@ -1,4 +1,4 @@
package s3api
package retention
import (
"context"

View File

@ -35,4 +35,8 @@ const (
// Object lock enabled status
ObjectLockEnabled = "Enabled"
// Bucket versioning status
VersioningEnabled = "Enabled"
VersioningSuspended = "Suspended"
)

View File

@ -51,6 +51,9 @@ const (
AmzAclReadAcp = "X-Amz-Grant-Read-Acp"
AmzAclWriteAcp = "X-Amz-Grant-Write-Acp"
// S3 Object Lock headers
AmzBucketObjectLockEnabled = "X-Amz-Bucket-Object-Lock-Enabled"
// S3 conditional copy headers
AmzCopySourceIfMatch = "X-Amz-Copy-Source-If-Match"
AmzCopySourceIfNoneMatch = "X-Amz-Copy-Source-If-None-Match"

View File

@ -136,6 +136,48 @@ func (s3a *S3ApiServer) PutBucketHandler(w http.ResponseWriter, r *http.Request)
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
return
}
// Check for x-amz-bucket-object-lock-enabled header (S3 standard compliance)
if objectLockHeaderValue := r.Header.Get(s3_constants.AmzBucketObjectLockEnabled); strings.EqualFold(objectLockHeaderValue, "true") {
glog.V(3).Infof("PutBucketHandler: enabling Object Lock and Versioning for bucket %s due to x-amz-bucket-object-lock-enabled header", bucket)
// Atomically update the configuration of the specified bucket. See the updateBucketConfig
// function definition for detailed documentation on parameters and behavior.
errCode := s3a.updateBucketConfig(bucket, func(bucketConfig *BucketConfig) error {
// Enable versioning (required for Object Lock)
bucketConfig.Versioning = s3_constants.VersioningEnabled
// Enable Object Lock configuration
if bucketConfig.Entry.Extended == nil {
bucketConfig.Entry.Extended = make(map[string][]byte)
}
// Create basic Object Lock configuration (enabled without default retention)
// The ObjectLockConfiguration struct is defined below in this file.
objectLockConfig := &ObjectLockConfiguration{
ObjectLockEnabled: s3_constants.ObjectLockEnabled,
}
// Store the configuration as XML in extended attributes
configXML, err := xml.Marshal(objectLockConfig)
if err != nil {
return fmt.Errorf("failed to marshal Object Lock configuration to XML: %v", err)
}
bucketConfig.Entry.Extended[s3_constants.ExtObjectLockConfigKey] = configXML
bucketConfig.Entry.Extended[s3_constants.ExtObjectLockEnabledKey] = []byte(s3_constants.ObjectLockEnabled)
return nil
})
if errCode != s3err.ErrNone {
glog.Errorf("PutBucketHandler: failed to enable Object Lock for bucket %s: %v", bucket, errCode)
s3err.WriteErrorResponse(w, r, errCode)
return
}
glog.V(3).Infof("PutBucketHandler: enabled Object Lock and Versioning for bucket %s", bucket)
}
w.Header().Set("Location", "/"+bucket)
writeSuccessResponseEmpty(w, r)
}