mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2025-09-19 06:37:56 +08:00
fix listing object versions (#7006)
* fix listing object versions * Update s3api_object_versioning.go * Update s3_directory_versioning_test.go * check previous skipped tests * fix test_versioning_stack_delete_merkers * address test_bucket_list_return_data_versioning * Update s3_directory_versioning_test.go * fix test_versioning_concurrent_multi_object_delete * fix test_versioning_obj_suspend_versions test * fix empty owner * fix listing versioned objects * default owner * fix path
This commit is contained in:
3
.github/workflows/s3tests.yml
vendored
3
.github/workflows/s3tests.yml
vendored
@@ -399,7 +399,8 @@ jobs:
|
|||||||
echo "S3 connection test failed, retrying... ($i/10)"
|
echo "S3 connection test failed, retrying... ($i/10)"
|
||||||
sleep 2
|
sleep 2
|
||||||
done
|
done
|
||||||
tox -- s3tests_boto3/functional/test_s3.py -k "object_lock or (versioning and not test_versioning_obj_suspend_versions and not test_bucket_list_return_data_versioning and not test_versioning_concurrent_multi_object_delete)" --tb=short
|
# tox -- s3tests_boto3/functional/test_s3.py -k "object_lock or (versioning and not test_versioning_obj_suspend_versions and not test_bucket_list_return_data_versioning and not test_versioning_concurrent_multi_object_delete)" --tb=short
|
||||||
|
tox -- s3tests_boto3/functional/test_s3.py -k "object_lock or versioning" --tb=short
|
||||||
kill -9 $pid || true
|
kill -9 $pid || true
|
||||||
# Clean up data directory
|
# Clean up data directory
|
||||||
rm -rf "$WEED_DATA_DIR" || true
|
rm -rf "$WEED_DATA_DIR" || true
|
||||||
|
776
test/s3/versioning/s3_directory_versioning_test.go
Normal file
776
test/s3/versioning/s3_directory_versioning_test.go
Normal file
@@ -0,0 +1,776 @@
|
|||||||
|
package s3api
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/aws/aws-sdk-go-v2/aws"
|
||||||
|
"github.com/aws/aws-sdk-go-v2/config"
|
||||||
|
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||||
|
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||||
|
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestListObjectVersionsIncludesDirectories tests that directories are included in list-object-versions response
|
||||||
|
// This ensures compatibility with Minio and AWS S3 behavior
|
||||||
|
func TestListObjectVersionsIncludesDirectories(t *testing.T) {
|
||||||
|
bucketName := "test-versioning-directories"
|
||||||
|
|
||||||
|
client := setupS3Client(t)
|
||||||
|
|
||||||
|
// Create bucket
|
||||||
|
_, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{
|
||||||
|
Bucket: aws.String(bucketName),
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
// Clean up
|
||||||
|
defer func() {
|
||||||
|
cleanupBucket(t, client, bucketName)
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Enable versioning
|
||||||
|
_, err = client.PutBucketVersioning(context.TODO(), &s3.PutBucketVersioningInput{
|
||||||
|
Bucket: aws.String(bucketName),
|
||||||
|
VersioningConfiguration: &types.VersioningConfiguration{
|
||||||
|
Status: types.BucketVersioningStatusEnabled,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
// First create explicit directory objects (keys ending with "/")
|
||||||
|
// These are the directories that should appear in list-object-versions
|
||||||
|
explicitDirectories := []string{
|
||||||
|
"Veeam/",
|
||||||
|
"Veeam/Archive/",
|
||||||
|
"Veeam/Archive/vbr/",
|
||||||
|
"Veeam/Backup/",
|
||||||
|
"Veeam/Backup/vbr/",
|
||||||
|
"Veeam/Backup/vbr/Clients/",
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create explicit directory objects
|
||||||
|
for _, dirKey := range explicitDirectories {
|
||||||
|
_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||||
|
Bucket: aws.String(bucketName),
|
||||||
|
Key: aws.String(dirKey),
|
||||||
|
Body: strings.NewReader(""), // Empty content for directories
|
||||||
|
})
|
||||||
|
require.NoError(t, err, "Failed to create directory object %s", dirKey)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Now create some test files
|
||||||
|
testFiles := []string{
|
||||||
|
"Veeam/test-file.txt",
|
||||||
|
"Veeam/Archive/test-file2.txt",
|
||||||
|
"Veeam/Archive/vbr/test-file3.txt",
|
||||||
|
"Veeam/Backup/test-file4.txt",
|
||||||
|
"Veeam/Backup/vbr/test-file5.txt",
|
||||||
|
"Veeam/Backup/vbr/Clients/test-file6.txt",
|
||||||
|
}
|
||||||
|
|
||||||
|
// Upload test files
|
||||||
|
for _, objectKey := range testFiles {
|
||||||
|
_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||||
|
Bucket: aws.String(bucketName),
|
||||||
|
Key: aws.String(objectKey),
|
||||||
|
Body: strings.NewReader("test content"),
|
||||||
|
})
|
||||||
|
require.NoError(t, err, "Failed to create file %s", objectKey)
|
||||||
|
}
|
||||||
|
|
||||||
|
// List object versions
|
||||||
|
listResp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
|
||||||
|
Bucket: aws.String(bucketName),
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
// Extract all keys from versions
|
||||||
|
var allKeys []string
|
||||||
|
for _, version := range listResp.Versions {
|
||||||
|
allKeys = append(allKeys, *version.Key)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Expected directories that should be included (with trailing slash)
|
||||||
|
expectedDirectories := []string{
|
||||||
|
"Veeam/",
|
||||||
|
"Veeam/Archive/",
|
||||||
|
"Veeam/Archive/vbr/",
|
||||||
|
"Veeam/Backup/",
|
||||||
|
"Veeam/Backup/vbr/",
|
||||||
|
"Veeam/Backup/vbr/Clients/",
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify that directories are included in the response
|
||||||
|
t.Logf("Found %d total versions", len(listResp.Versions))
|
||||||
|
t.Logf("All keys: %v", allKeys)
|
||||||
|
|
||||||
|
for _, expectedDir := range expectedDirectories {
|
||||||
|
found := false
|
||||||
|
for _, version := range listResp.Versions {
|
||||||
|
if *version.Key == expectedDir {
|
||||||
|
found = true
|
||||||
|
// Verify directory properties
|
||||||
|
assert.Equal(t, "null", *version.VersionId, "Directory %s should have VersionId 'null'", expectedDir)
|
||||||
|
assert.Equal(t, int64(0), *version.Size, "Directory %s should have size 0", expectedDir)
|
||||||
|
assert.True(t, *version.IsLatest, "Directory %s should be marked as latest", expectedDir)
|
||||||
|
assert.Equal(t, "\"d41d8cd98f00b204e9800998ecf8427e\"", *version.ETag, "Directory %s should have MD5 of empty string as ETag", expectedDir)
|
||||||
|
assert.Equal(t, types.ObjectStorageClassStandard, version.StorageClass, "Directory %s should have STANDARD storage class", expectedDir)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert.True(t, found, "Directory %s should be included in list-object-versions response", expectedDir)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Also verify that actual files are included
|
||||||
|
for _, objectKey := range testFiles {
|
||||||
|
found := false
|
||||||
|
for _, version := range listResp.Versions {
|
||||||
|
if *version.Key == objectKey {
|
||||||
|
found = true
|
||||||
|
assert.NotEqual(t, "null", *version.VersionId, "File %s should have a real version ID", objectKey)
|
||||||
|
assert.Greater(t, *version.Size, int64(0), "File %s should have size > 0", objectKey)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert.True(t, found, "File %s should be included in list-object-versions response", objectKey)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Count directories vs files
|
||||||
|
directoryCount := 0
|
||||||
|
fileCount := 0
|
||||||
|
for _, version := range listResp.Versions {
|
||||||
|
if strings.HasSuffix(*version.Key, "/") && *version.Size == 0 && *version.VersionId == "null" {
|
||||||
|
directoryCount++
|
||||||
|
} else {
|
||||||
|
fileCount++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Logf("Found %d directories and %d files", directoryCount, fileCount)
|
||||||
|
assert.Equal(t, len(expectedDirectories), directoryCount, "Should find exactly %d directories", len(expectedDirectories))
|
||||||
|
assert.Equal(t, len(testFiles), fileCount, "Should find exactly %d files", len(testFiles))
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestListObjectVersionsDeleteMarkers tests that delete markers are properly separated from versions
|
||||||
|
// This test verifies the fix for the issue where delete markers were incorrectly categorized as versions
|
||||||
|
func TestListObjectVersionsDeleteMarkers(t *testing.T) {
|
||||||
|
bucketName := "test-delete-markers"
|
||||||
|
|
||||||
|
client := setupS3Client(t)
|
||||||
|
|
||||||
|
// Create bucket
|
||||||
|
_, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{
|
||||||
|
Bucket: aws.String(bucketName),
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
// Clean up
|
||||||
|
defer func() {
|
||||||
|
cleanupBucket(t, client, bucketName)
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Enable versioning
|
||||||
|
_, err = client.PutBucketVersioning(context.TODO(), &s3.PutBucketVersioningInput{
|
||||||
|
Bucket: aws.String(bucketName),
|
||||||
|
VersioningConfiguration: &types.VersioningConfiguration{
|
||||||
|
Status: types.BucketVersioningStatusEnabled,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
objectKey := "test1/a"
|
||||||
|
|
||||||
|
// 1. Create one version of the file
|
||||||
|
_, err = client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||||
|
Bucket: aws.String(bucketName),
|
||||||
|
Key: aws.String(objectKey),
|
||||||
|
Body: strings.NewReader("test content"),
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
// 2. Delete the object 3 times to create 3 delete markers
|
||||||
|
for i := 0; i < 3; i++ {
|
||||||
|
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||||
|
Bucket: aws.String(bucketName),
|
||||||
|
Key: aws.String(objectKey),
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. List object versions and verify the response structure
|
||||||
|
listResp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
|
||||||
|
Bucket: aws.String(bucketName),
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
// 4. Verify that we have exactly 1 version and 3 delete markers
|
||||||
|
assert.Len(t, listResp.Versions, 1, "Should have exactly 1 file version")
|
||||||
|
assert.Len(t, listResp.DeleteMarkers, 3, "Should have exactly 3 delete markers")
|
||||||
|
|
||||||
|
// 5. Verify the version is for our test file
|
||||||
|
version := listResp.Versions[0]
|
||||||
|
assert.Equal(t, objectKey, *version.Key, "Version should be for our test file")
|
||||||
|
assert.NotEqual(t, "null", *version.VersionId, "File version should have a real version ID")
|
||||||
|
assert.Greater(t, *version.Size, int64(0), "File version should have size > 0")
|
||||||
|
|
||||||
|
// 6. Verify all delete markers are for our test file
|
||||||
|
for i, deleteMarker := range listResp.DeleteMarkers {
|
||||||
|
assert.Equal(t, objectKey, *deleteMarker.Key, "Delete marker %d should be for our test file", i)
|
||||||
|
assert.NotEqual(t, "null", *deleteMarker.VersionId, "Delete marker %d should have a real version ID", i)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Logf("Successfully verified: 1 version + 3 delete markers for object %s", objectKey)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestVersionedObjectAcl tests that ACL operations work correctly on objects in versioned buckets
|
||||||
|
// This test verifies the fix for the NoSuchKey error when getting ACLs for objects in versioned buckets
|
||||||
|
func TestVersionedObjectAcl(t *testing.T) {
|
||||||
|
bucketName := "test-versioned-acl"
|
||||||
|
|
||||||
|
client := setupS3Client(t)
|
||||||
|
|
||||||
|
// Create bucket
|
||||||
|
_, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{
|
||||||
|
Bucket: aws.String(bucketName),
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
// Clean up
|
||||||
|
defer func() {
|
||||||
|
cleanupBucket(t, client, bucketName)
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Enable versioning
|
||||||
|
_, err = client.PutBucketVersioning(context.TODO(), &s3.PutBucketVersioningInput{
|
||||||
|
Bucket: aws.String(bucketName),
|
||||||
|
VersioningConfiguration: &types.VersioningConfiguration{
|
||||||
|
Status: types.BucketVersioningStatusEnabled,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
objectKey := "test-acl-object"
|
||||||
|
|
||||||
|
// Create an object in the versioned bucket
|
||||||
|
putResp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||||
|
Bucket: aws.String(bucketName),
|
||||||
|
Key: aws.String(objectKey),
|
||||||
|
Body: strings.NewReader("test content for ACL"),
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NotNil(t, putResp.VersionId, "Object should have a version ID")
|
||||||
|
|
||||||
|
// Test 1: Get ACL for the object (without specifying version ID - should get latest version)
|
||||||
|
getAclResp, err := client.GetObjectAcl(context.TODO(), &s3.GetObjectAclInput{
|
||||||
|
Bucket: aws.String(bucketName),
|
||||||
|
Key: aws.String(objectKey),
|
||||||
|
})
|
||||||
|
require.NoError(t, err, "Should be able to get ACL for object in versioned bucket")
|
||||||
|
require.NotNil(t, getAclResp.Owner, "ACL response should have owner information")
|
||||||
|
|
||||||
|
// Test 2: Get ACL for specific version ID
|
||||||
|
getAclVersionResp, err := client.GetObjectAcl(context.TODO(), &s3.GetObjectAclInput{
|
||||||
|
Bucket: aws.String(bucketName),
|
||||||
|
Key: aws.String(objectKey),
|
||||||
|
VersionId: putResp.VersionId,
|
||||||
|
})
|
||||||
|
require.NoError(t, err, "Should be able to get ACL for specific version")
|
||||||
|
require.NotNil(t, getAclVersionResp.Owner, "Versioned ACL response should have owner information")
|
||||||
|
|
||||||
|
// Test 3: Verify both ACL responses are the same (same object, same version)
|
||||||
|
assert.Equal(t, getAclResp.Owner.ID, getAclVersionResp.Owner.ID, "Owner ID should match for latest and specific version")
|
||||||
|
|
||||||
|
// Test 4: Create another version of the same object
|
||||||
|
putResp2, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||||
|
Bucket: aws.String(bucketName),
|
||||||
|
Key: aws.String(objectKey),
|
||||||
|
Body: strings.NewReader("updated content for ACL"),
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NotNil(t, putResp2.VersionId, "Second object version should have a version ID")
|
||||||
|
require.NotEqual(t, putResp.VersionId, putResp2.VersionId, "Version IDs should be different")
|
||||||
|
|
||||||
|
// Test 5: Get ACL for latest version (should be the second version)
|
||||||
|
getAclLatestResp, err := client.GetObjectAcl(context.TODO(), &s3.GetObjectAclInput{
|
||||||
|
Bucket: aws.String(bucketName),
|
||||||
|
Key: aws.String(objectKey),
|
||||||
|
})
|
||||||
|
require.NoError(t, err, "Should be able to get ACL for latest version after update")
|
||||||
|
require.NotNil(t, getAclLatestResp.Owner, "Latest ACL response should have owner information")
|
||||||
|
|
||||||
|
// Test 6: Get ACL for the first version specifically
|
||||||
|
getAclFirstResp, err := client.GetObjectAcl(context.TODO(), &s3.GetObjectAclInput{
|
||||||
|
Bucket: aws.String(bucketName),
|
||||||
|
Key: aws.String(objectKey),
|
||||||
|
VersionId: putResp.VersionId,
|
||||||
|
})
|
||||||
|
require.NoError(t, err, "Should be able to get ACL for first version specifically")
|
||||||
|
require.NotNil(t, getAclFirstResp.Owner, "First version ACL response should have owner information")
|
||||||
|
|
||||||
|
// Test 7: Verify we can put ACL on versioned objects
|
||||||
|
_, err = client.PutObjectAcl(context.TODO(), &s3.PutObjectAclInput{
|
||||||
|
Bucket: aws.String(bucketName),
|
||||||
|
Key: aws.String(objectKey),
|
||||||
|
ACL: types.ObjectCannedACLPrivate,
|
||||||
|
})
|
||||||
|
require.NoError(t, err, "Should be able to put ACL on versioned object")
|
||||||
|
|
||||||
|
t.Logf("Successfully verified ACL operations on versioned object %s with versions %s and %s",
|
||||||
|
objectKey, *putResp.VersionId, *putResp2.VersionId)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestConcurrentMultiObjectDelete tests that concurrent delete operations work correctly without race conditions
|
||||||
|
// This test verifies the fix for the race condition in deleteSpecificObjectVersion
|
||||||
|
func TestConcurrentMultiObjectDelete(t *testing.T) {
|
||||||
|
bucketName := "test-concurrent-delete"
|
||||||
|
numObjects := 5
|
||||||
|
numThreads := 5
|
||||||
|
|
||||||
|
client := setupS3Client(t)
|
||||||
|
|
||||||
|
// Create bucket
|
||||||
|
_, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{
|
||||||
|
Bucket: aws.String(bucketName),
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
// Clean up
|
||||||
|
defer func() {
|
||||||
|
cleanupBucket(t, client, bucketName)
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Enable versioning
|
||||||
|
_, err = client.PutBucketVersioning(context.TODO(), &s3.PutBucketVersioningInput{
|
||||||
|
Bucket: aws.String(bucketName),
|
||||||
|
VersioningConfiguration: &types.VersioningConfiguration{
|
||||||
|
Status: types.BucketVersioningStatusEnabled,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
// Create objects
|
||||||
|
var objectKeys []string
|
||||||
|
var versionIds []string
|
||||||
|
|
||||||
|
for i := 0; i < numObjects; i++ {
|
||||||
|
objectKey := fmt.Sprintf("key_%d", i)
|
||||||
|
objectKeys = append(objectKeys, objectKey)
|
||||||
|
|
||||||
|
putResp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||||
|
Bucket: aws.String(bucketName),
|
||||||
|
Key: aws.String(objectKey),
|
||||||
|
Body: strings.NewReader(fmt.Sprintf("content for key_%d", i)),
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NotNil(t, putResp.VersionId)
|
||||||
|
versionIds = append(versionIds, *putResp.VersionId)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify objects were created
|
||||||
|
listResp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
|
||||||
|
Bucket: aws.String(bucketName),
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Len(t, listResp.Versions, numObjects, "Should have created %d objects", numObjects)
|
||||||
|
|
||||||
|
// Create delete objects request
|
||||||
|
var objectsToDelete []types.ObjectIdentifier
|
||||||
|
for i, objectKey := range objectKeys {
|
||||||
|
objectsToDelete = append(objectsToDelete, types.ObjectIdentifier{
|
||||||
|
Key: aws.String(objectKey),
|
||||||
|
VersionId: aws.String(versionIds[i]),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run concurrent delete operations
|
||||||
|
results := make([]*s3.DeleteObjectsOutput, numThreads)
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
|
||||||
|
for i := 0; i < numThreads; i++ {
|
||||||
|
wg.Add(1)
|
||||||
|
go func(threadIdx int) {
|
||||||
|
defer wg.Done()
|
||||||
|
deleteResp, err := client.DeleteObjects(context.TODO(), &s3.DeleteObjectsInput{
|
||||||
|
Bucket: aws.String(bucketName),
|
||||||
|
Delete: &types.Delete{
|
||||||
|
Objects: objectsToDelete,
|
||||||
|
Quiet: aws.Bool(false),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("Thread %d: delete objects failed: %v", threadIdx, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
results[threadIdx] = deleteResp
|
||||||
|
}(i)
|
||||||
|
}
|
||||||
|
|
||||||
|
wg.Wait()
|
||||||
|
|
||||||
|
// Verify results
|
||||||
|
for i, result := range results {
|
||||||
|
require.NotNil(t, result, "Thread %d should have a result", i)
|
||||||
|
assert.Len(t, result.Deleted, numObjects, "Thread %d should have deleted all %d objects", i, numObjects)
|
||||||
|
|
||||||
|
if len(result.Errors) > 0 {
|
||||||
|
for _, deleteError := range result.Errors {
|
||||||
|
t.Errorf("Thread %d delete error: %s - %s (Key: %s, VersionId: %s)",
|
||||||
|
i, *deleteError.Code, *deleteError.Message, *deleteError.Key,
|
||||||
|
func() string {
|
||||||
|
if deleteError.VersionId != nil {
|
||||||
|
return *deleteError.VersionId
|
||||||
|
} else {
|
||||||
|
return "nil"
|
||||||
|
}
|
||||||
|
}())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert.Empty(t, result.Errors, "Thread %d should have no delete errors", i)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify objects are deleted (bucket should be empty)
|
||||||
|
finalListResp, err := client.ListObjects(context.TODO(), &s3.ListObjectsInput{
|
||||||
|
Bucket: aws.String(bucketName),
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Nil(t, finalListResp.Contents, "Bucket should be empty after all deletions")
|
||||||
|
|
||||||
|
t.Logf("Successfully verified concurrent deletion of %d objects from %d threads", numObjects, numThreads)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSuspendedVersioningDeleteBehavior tests that delete operations during suspended versioning
|
||||||
|
// actually delete the "null" version object rather than creating delete markers
|
||||||
|
func TestSuspendedVersioningDeleteBehavior(t *testing.T) {
|
||||||
|
bucketName := "test-suspended-versioning-delete"
|
||||||
|
objectKey := "testobj"
|
||||||
|
|
||||||
|
client := setupS3Client(t)
|
||||||
|
|
||||||
|
// Create bucket
|
||||||
|
_, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{
|
||||||
|
Bucket: aws.String(bucketName),
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
// Clean up
|
||||||
|
defer func() {
|
||||||
|
cleanupBucket(t, client, bucketName)
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Enable versioning and create some versions
|
||||||
|
_, err = client.PutBucketVersioning(context.TODO(), &s3.PutBucketVersioningInput{
|
||||||
|
Bucket: aws.String(bucketName),
|
||||||
|
VersioningConfiguration: &types.VersioningConfiguration{
|
||||||
|
Status: types.BucketVersioningStatusEnabled,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
// Create 3 versions
|
||||||
|
var versionIds []string
|
||||||
|
for i := 0; i < 3; i++ {
|
||||||
|
putResp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||||
|
Bucket: aws.String(bucketName),
|
||||||
|
Key: aws.String(objectKey),
|
||||||
|
Body: strings.NewReader(fmt.Sprintf("content version %d", i+1)),
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NotNil(t, putResp.VersionId)
|
||||||
|
versionIds = append(versionIds, *putResp.VersionId)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify 3 versions exist
|
||||||
|
listResp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
|
||||||
|
Bucket: aws.String(bucketName),
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Len(t, listResp.Versions, 3, "Should have 3 versions initially")
|
||||||
|
|
||||||
|
// Suspend versioning
|
||||||
|
_, err = client.PutBucketVersioning(context.TODO(), &s3.PutBucketVersioningInput{
|
||||||
|
Bucket: aws.String(bucketName),
|
||||||
|
VersioningConfiguration: &types.VersioningConfiguration{
|
||||||
|
Status: types.BucketVersioningStatusSuspended,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
// Create a new object during suspended versioning (this should be a "null" version)
|
||||||
|
_, err = client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||||
|
Bucket: aws.String(bucketName),
|
||||||
|
Key: aws.String(objectKey),
|
||||||
|
Body: strings.NewReader("null version content"),
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
// Verify we still have 3 versions + 1 null version = 4 total
|
||||||
|
listResp, err = client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
|
||||||
|
Bucket: aws.String(bucketName),
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Len(t, listResp.Versions, 4, "Should have 3 versions + 1 null version")
|
||||||
|
|
||||||
|
// Find the null version
|
||||||
|
var nullVersionFound bool
|
||||||
|
for _, version := range listResp.Versions {
|
||||||
|
if *version.VersionId == "null" {
|
||||||
|
nullVersionFound = true
|
||||||
|
assert.True(t, *version.IsLatest, "Null version should be marked as latest during suspended versioning")
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert.True(t, nullVersionFound, "Should have found a null version")
|
||||||
|
|
||||||
|
// Delete the object during suspended versioning (should actually delete the null version)
|
||||||
|
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||||
|
Bucket: aws.String(bucketName),
|
||||||
|
Key: aws.String(objectKey),
|
||||||
|
// No VersionId specified - should delete the "null" version during suspended versioning
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
// Verify the null version was actually deleted (not a delete marker created)
|
||||||
|
listResp, err = client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
|
||||||
|
Bucket: aws.String(bucketName),
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Len(t, listResp.Versions, 3, "Should be back to 3 versions after deleting null version")
|
||||||
|
assert.Empty(t, listResp.DeleteMarkers, "Should have no delete markers during suspended versioning delete")
|
||||||
|
|
||||||
|
// Verify null version is gone
|
||||||
|
nullVersionFound = false
|
||||||
|
for _, version := range listResp.Versions {
|
||||||
|
if *version.VersionId == "null" {
|
||||||
|
nullVersionFound = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert.False(t, nullVersionFound, "Null version should be deleted, not present")
|
||||||
|
|
||||||
|
// Create another null version and delete it multiple times to test idempotency
|
||||||
|
_, err = client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||||
|
Bucket: aws.String(bucketName),
|
||||||
|
Key: aws.String(objectKey),
|
||||||
|
Body: strings.NewReader("another null version"),
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
// Delete it twice to test idempotency
|
||||||
|
for i := 0; i < 2; i++ {
|
||||||
|
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||||
|
Bucket: aws.String(bucketName),
|
||||||
|
Key: aws.String(objectKey),
|
||||||
|
})
|
||||||
|
require.NoError(t, err, "Delete should be idempotent - iteration %d", i+1)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Re-enable versioning
|
||||||
|
_, err = client.PutBucketVersioning(context.TODO(), &s3.PutBucketVersioningInput{
|
||||||
|
Bucket: aws.String(bucketName),
|
||||||
|
VersioningConfiguration: &types.VersioningConfiguration{
|
||||||
|
Status: types.BucketVersioningStatusEnabled,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
// Create a new version with versioning enabled
|
||||||
|
putResp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||||
|
Bucket: aws.String(bucketName),
|
||||||
|
Key: aws.String(objectKey),
|
||||||
|
Body: strings.NewReader("new version after re-enabling"),
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NotNil(t, putResp.VersionId)
|
||||||
|
|
||||||
|
// Now delete without version ID (should create delete marker)
|
||||||
|
deleteResp, err := client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||||
|
Bucket: aws.String(bucketName),
|
||||||
|
Key: aws.String(objectKey),
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, "true", deleteResp.DeleteMarker, "Should create delete marker when versioning is enabled")
|
||||||
|
|
||||||
|
// Verify final state
|
||||||
|
listResp, err = client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
|
||||||
|
Bucket: aws.String(bucketName),
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Len(t, listResp.Versions, 4, "Should have 3 original versions + 1 new version")
|
||||||
|
assert.Len(t, listResp.DeleteMarkers, 1, "Should have 1 delete marker")
|
||||||
|
|
||||||
|
t.Logf("Successfully verified suspended versioning delete behavior")
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestVersionedObjectListBehavior tests that list operations show logical object names for versioned objects
|
||||||
|
// and that owner information is properly extracted from S3 metadata
|
||||||
|
func TestVersionedObjectListBehavior(t *testing.T) {
|
||||||
|
bucketName := "test-versioned-list"
|
||||||
|
objectKey := "testfile"
|
||||||
|
|
||||||
|
client := setupS3Client(t)
|
||||||
|
|
||||||
|
// Create bucket with object lock enabled (which enables versioning)
|
||||||
|
_, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{
|
||||||
|
Bucket: aws.String(bucketName),
|
||||||
|
ObjectLockEnabledForBucket: aws.Bool(true),
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
// Clean up
|
||||||
|
defer func() {
|
||||||
|
cleanupBucket(t, client, bucketName)
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Verify versioning is enabled
|
||||||
|
versioningResp, err := client.GetBucketVersioning(context.TODO(), &s3.GetBucketVersioningInput{
|
||||||
|
Bucket: aws.String(bucketName),
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, types.BucketVersioningStatusEnabled, versioningResp.Status, "Bucket versioning should be enabled")
|
||||||
|
|
||||||
|
// Create a versioned object
|
||||||
|
content := "test content for versioned object"
|
||||||
|
putResp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||||
|
Bucket: aws.String(bucketName),
|
||||||
|
Key: aws.String(objectKey),
|
||||||
|
Body: strings.NewReader(content),
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NotNil(t, putResp.VersionId)
|
||||||
|
|
||||||
|
versionId := *putResp.VersionId
|
||||||
|
t.Logf("Created versioned object with version ID: %s", versionId)
|
||||||
|
|
||||||
|
// Test list-objects operation - should show logical object name, not internal versioned path
|
||||||
|
listResp, err := client.ListObjects(context.TODO(), &s3.ListObjectsInput{
|
||||||
|
Bucket: aws.String(bucketName),
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Len(t, listResp.Contents, 1, "Should list exactly one object")
|
||||||
|
|
||||||
|
listedObject := listResp.Contents[0]
|
||||||
|
|
||||||
|
// Verify the object key is the logical name, not the internal versioned path
|
||||||
|
assert.Equal(t, objectKey, *listedObject.Key, "Should show logical object name, not internal versioned path")
|
||||||
|
assert.NotContains(t, *listedObject.Key, ".versions", "Object key should not contain .versions")
|
||||||
|
assert.NotContains(t, *listedObject.Key, versionId, "Object key should not contain version ID")
|
||||||
|
|
||||||
|
// Verify object properties
|
||||||
|
assert.Equal(t, int64(len(content)), listedObject.Size, "Object size should match")
|
||||||
|
assert.NotNil(t, listedObject.ETag, "Object should have ETag")
|
||||||
|
assert.NotNil(t, listedObject.LastModified, "Object should have LastModified")
|
||||||
|
|
||||||
|
// Verify owner information is present (even if anonymous)
|
||||||
|
require.NotNil(t, listedObject.Owner, "Object should have Owner information")
|
||||||
|
assert.NotEmpty(t, listedObject.Owner.ID, "Owner ID should not be empty")
|
||||||
|
assert.NotEmpty(t, listedObject.Owner.DisplayName, "Owner DisplayName should not be empty")
|
||||||
|
|
||||||
|
t.Logf("Listed object: Key=%s, Size=%d, Owner.ID=%s, Owner.DisplayName=%s",
|
||||||
|
*listedObject.Key, listedObject.Size, *listedObject.Owner.ID, *listedObject.Owner.DisplayName)
|
||||||
|
|
||||||
|
// Test list-objects-v2 operation as well
|
||||||
|
listV2Resp, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
|
||||||
|
Bucket: aws.String(bucketName),
|
||||||
|
FetchOwner: aws.Bool(true), // Explicitly request owner information
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Len(t, listV2Resp.Contents, 1, "ListObjectsV2 should also list exactly one object")
|
||||||
|
|
||||||
|
listedObjectV2 := listV2Resp.Contents[0]
|
||||||
|
assert.Equal(t, objectKey, *listedObjectV2.Key, "ListObjectsV2 should also show logical object name")
|
||||||
|
assert.NotNil(t, listedObjectV2.Owner, "ListObjectsV2 should include owner when FetchOwner=true")
|
||||||
|
|
||||||
|
// Create another version to ensure multiple versions don't appear in regular list
|
||||||
|
_, err = client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||||
|
Bucket: aws.String(bucketName),
|
||||||
|
Key: aws.String(objectKey),
|
||||||
|
Body: strings.NewReader("updated content"),
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
// List again - should still show only one logical object (the latest version)
|
||||||
|
listRespAfterUpdate, err := client.ListObjects(context.TODO(), &s3.ListObjectsInput{
|
||||||
|
Bucket: aws.String(bucketName),
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Len(t, listRespAfterUpdate.Contents, 1, "Should still list exactly one object after creating second version")
|
||||||
|
|
||||||
|
// Compare with list-object-versions which should show both versions
|
||||||
|
versionsResp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
|
||||||
|
Bucket: aws.String(bucketName),
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Len(t, versionsResp.Versions, 2, "list-object-versions should show both versions")
|
||||||
|
|
||||||
|
t.Logf("Successfully verified versioned object list behavior")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Helper function to setup S3 client
|
||||||
|
func setupS3Client(t *testing.T) *s3.Client {
|
||||||
|
// S3TestConfig holds configuration for S3 tests
|
||||||
|
type S3TestConfig struct {
|
||||||
|
Endpoint string
|
||||||
|
AccessKey string
|
||||||
|
SecretKey string
|
||||||
|
Region string
|
||||||
|
BucketPrefix string
|
||||||
|
UseSSL bool
|
||||||
|
SkipVerifySSL bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// Default test configuration - should match s3tests.conf
|
||||||
|
defaultConfig := &S3TestConfig{
|
||||||
|
Endpoint: "http://localhost:8333", // Default SeaweedFS S3 port
|
||||||
|
AccessKey: "some_access_key1",
|
||||||
|
SecretKey: "some_secret_key1",
|
||||||
|
Region: "us-east-1",
|
||||||
|
BucketPrefix: "test-versioning-",
|
||||||
|
UseSSL: false,
|
||||||
|
SkipVerifySSL: true,
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg, err := config.LoadDefaultConfig(context.TODO(),
|
||||||
|
config.WithRegion(defaultConfig.Region),
|
||||||
|
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(
|
||||||
|
defaultConfig.AccessKey,
|
||||||
|
defaultConfig.SecretKey,
|
||||||
|
"",
|
||||||
|
)),
|
||||||
|
config.WithEndpointResolverWithOptions(aws.EndpointResolverWithOptionsFunc(
|
||||||
|
func(service, region string, options ...interface{}) (aws.Endpoint, error) {
|
||||||
|
return aws.Endpoint{
|
||||||
|
URL: defaultConfig.Endpoint,
|
||||||
|
SigningRegion: defaultConfig.Region,
|
||||||
|
HostnameImmutable: true,
|
||||||
|
}, nil
|
||||||
|
})),
|
||||||
|
)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
return s3.NewFromConfig(cfg, func(o *s3.Options) {
|
||||||
|
o.UsePathStyle = true // Important for SeaweedFS
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Helper function to clean up bucket
|
||||||
|
func cleanupBucket(t *testing.T, client *s3.Client, bucketName string) {
|
||||||
|
// First, delete all objects and versions
|
||||||
|
err := deleteAllObjectVersions(t, client, bucketName)
|
||||||
|
if err != nil {
|
||||||
|
t.Logf("Warning: failed to delete all object versions: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Then delete the bucket
|
||||||
|
_, err = client.DeleteBucket(context.TODO(), &s3.DeleteBucketInput{
|
||||||
|
Bucket: aws.String(bucketName),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Logf("Warning: failed to delete bucket %s: %v", bucketName, err)
|
||||||
|
}
|
||||||
|
}
|
@@ -198,9 +198,33 @@ func newListEntry(entry *filer_pb.Entry, key string, dir string, name string, bu
|
|||||||
StorageClass: StorageClass(storageClass),
|
StorageClass: StorageClass(storageClass),
|
||||||
}
|
}
|
||||||
if fetchOwner {
|
if fetchOwner {
|
||||||
|
// Extract owner from S3 metadata (Extended attributes) instead of file system attributes
|
||||||
|
var ownerID, displayName string
|
||||||
|
if entry.Extended != nil {
|
||||||
|
if ownerBytes, exists := entry.Extended[s3_constants.ExtAmzOwnerKey]; exists {
|
||||||
|
ownerID = string(ownerBytes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fallback to anonymous if no S3 owner found
|
||||||
|
if ownerID == "" {
|
||||||
|
ownerID = s3_constants.AccountAnonymousId
|
||||||
|
displayName = "anonymous"
|
||||||
|
} else {
|
||||||
|
// Try to resolve display name from IAM system
|
||||||
|
displayName = "unknown"
|
||||||
|
// Note: IAM resolution would require access to the S3ApiServer instance
|
||||||
|
// For now, use a simple fallback or could be enhanced later
|
||||||
|
}
|
||||||
|
|
||||||
|
// Additional fallback to file system username if available and no display name resolved
|
||||||
|
if displayName == "unknown" && entry.Attributes.UserName != "" {
|
||||||
|
displayName = entry.Attributes.UserName
|
||||||
|
}
|
||||||
|
|
||||||
listEntry.Owner = CanonicalUser{
|
listEntry.Owner = CanonicalUser{
|
||||||
ID: fmt.Sprintf("%x", entry.Attributes.Uid),
|
ID: ownerID,
|
||||||
DisplayName: entry.Attributes.UserName,
|
DisplayName: displayName,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return listEntry
|
return listEntry
|
||||||
|
@@ -24,18 +24,63 @@ func (s3a *S3ApiServer) GetObjectAclHandler(w http.ResponseWriter, r *http.Reque
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check if object exists and get its metadata
|
// Check for specific version ID in query parameters
|
||||||
bucketDir := s3a.option.BucketsPath + "/" + bucket
|
versionId := r.URL.Query().Get("versionId")
|
||||||
entry, err := s3a.getEntry(bucketDir, object)
|
|
||||||
|
// Check if versioning is configured for the bucket (Enabled or Suspended)
|
||||||
|
versioningConfigured, err := s3a.isVersioningConfigured(bucket)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
if err == filer_pb.ErrNotFound {
|
||||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
|
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
glog.Errorf("GetObjectAclHandler: error checking object %s/%s: %v", bucket, object, err)
|
glog.Errorf("GetObjectAclHandler: Error checking versioning status for bucket %s: %v", bucket, err)
|
||||||
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
|
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var entry *filer_pb.Entry
|
||||||
|
|
||||||
|
if versioningConfigured {
|
||||||
|
// Handle versioned object ACL retrieval - use same logic as GetObjectHandler
|
||||||
|
if versionId != "" {
|
||||||
|
// Request for specific version
|
||||||
|
glog.V(2).Infof("GetObjectAclHandler: requesting ACL for specific version %s of %s%s", versionId, bucket, object)
|
||||||
|
entry, err = s3a.getSpecificObjectVersion(bucket, object, versionId)
|
||||||
|
} else {
|
||||||
|
// Request for latest version
|
||||||
|
glog.V(2).Infof("GetObjectAclHandler: requesting ACL for latest version of %s%s", bucket, object)
|
||||||
|
entry, err = s3a.getLatestObjectVersion(bucket, object)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
glog.Errorf("GetObjectAclHandler: Failed to get object version %s for %s%s: %v", versionId, bucket, object, err)
|
||||||
|
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if this is a delete marker
|
||||||
|
if entry.Extended != nil {
|
||||||
|
if deleteMarker, exists := entry.Extended[s3_constants.ExtDeleteMarkerKey]; exists && string(deleteMarker) == "true" {
|
||||||
|
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// Handle regular (non-versioned) object ACL retrieval
|
||||||
|
bucketDir := s3a.option.BucketsPath + "/" + bucket
|
||||||
|
entry, err = s3a.getEntry(bucketDir, object)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||||
|
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
glog.Errorf("GetObjectAclHandler: error checking object %s/%s: %v", bucket, object, err)
|
||||||
|
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if entry == nil {
|
if entry == nil {
|
||||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
|
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
|
||||||
return
|
return
|
||||||
@@ -123,18 +168,63 @@ func (s3a *S3ApiServer) PutObjectAclHandler(w http.ResponseWriter, r *http.Reque
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check if object exists and get its metadata
|
// Check for specific version ID in query parameters
|
||||||
bucketDir := s3a.option.BucketsPath + "/" + bucket
|
versionId := r.URL.Query().Get("versionId")
|
||||||
entry, err := s3a.getEntry(bucketDir, object)
|
|
||||||
|
// Check if versioning is configured for the bucket (Enabled or Suspended)
|
||||||
|
versioningConfigured, err := s3a.isVersioningConfigured(bucket)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
if err == filer_pb.ErrNotFound {
|
||||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
|
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
glog.Errorf("PutObjectAclHandler: error checking object %s/%s: %v", bucket, object, err)
|
glog.Errorf("PutObjectAclHandler: Error checking versioning status for bucket %s: %v", bucket, err)
|
||||||
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
|
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var entry *filer_pb.Entry
|
||||||
|
|
||||||
|
if versioningConfigured {
|
||||||
|
// Handle versioned object ACL modification - use same logic as GetObjectHandler
|
||||||
|
if versionId != "" {
|
||||||
|
// Request for specific version
|
||||||
|
glog.V(2).Infof("PutObjectAclHandler: modifying ACL for specific version %s of %s%s", versionId, bucket, object)
|
||||||
|
entry, err = s3a.getSpecificObjectVersion(bucket, object, versionId)
|
||||||
|
} else {
|
||||||
|
// Request for latest version
|
||||||
|
glog.V(2).Infof("PutObjectAclHandler: modifying ACL for latest version of %s%s", bucket, object)
|
||||||
|
entry, err = s3a.getLatestObjectVersion(bucket, object)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
glog.Errorf("PutObjectAclHandler: Failed to get object version %s for %s%s: %v", versionId, bucket, object, err)
|
||||||
|
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if this is a delete marker
|
||||||
|
if entry.Extended != nil {
|
||||||
|
if deleteMarker, exists := entry.Extended[s3_constants.ExtDeleteMarkerKey]; exists && string(deleteMarker) == "true" {
|
||||||
|
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// Handle regular (non-versioned) object ACL modification
|
||||||
|
bucketDir := s3a.option.BucketsPath + "/" + bucket
|
||||||
|
entry, err = s3a.getEntry(bucketDir, object)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||||
|
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
glog.Errorf("PutObjectAclHandler: error checking object %s/%s: %v", bucket, object, err)
|
||||||
|
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if entry == nil {
|
if entry == nil {
|
||||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
|
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
|
||||||
return
|
return
|
||||||
@@ -208,14 +298,44 @@ func (s3a *S3ApiServer) PutObjectAclHandler(w http.ResponseWriter, r *http.Reque
|
|||||||
// Store ACL in object metadata
|
// Store ACL in object metadata
|
||||||
if errCode := AssembleEntryWithAcp(entry, objectOwner, grants); errCode != s3err.ErrNone {
|
if errCode := AssembleEntryWithAcp(entry, objectOwner, grants); errCode != s3err.ErrNone {
|
||||||
glog.Errorf("PutObjectAclHandler: failed to assemble entry with ACP: %v", errCode)
|
glog.Errorf("PutObjectAclHandler: failed to assemble entry with ACP: %v", errCode)
|
||||||
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
|
s3err.WriteErrorResponse(w, r, errCode)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Calculate the correct directory for ACL update
|
||||||
|
var updateDirectory string
|
||||||
|
|
||||||
|
if versioningConfigured {
|
||||||
|
if versionId != "" && versionId != "null" {
|
||||||
|
// Versioned object - update the specific version file in .versions directory
|
||||||
|
updateDirectory = s3a.option.BucketsPath + "/" + bucket + "/" + object + ".versions"
|
||||||
|
} else {
|
||||||
|
// Latest version in versioned bucket - could be null version or versioned object
|
||||||
|
// Extract version ID from the entry to determine where it's stored
|
||||||
|
var actualVersionId string
|
||||||
|
if entry.Extended != nil {
|
||||||
|
if versionIdBytes, exists := entry.Extended[s3_constants.ExtVersionIdKey]; exists {
|
||||||
|
actualVersionId = string(versionIdBytes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if actualVersionId == "null" || actualVersionId == "" {
|
||||||
|
// Null version (pre-versioning object) - stored as regular file
|
||||||
|
updateDirectory = s3a.option.BucketsPath + "/" + bucket
|
||||||
|
} else {
|
||||||
|
// Versioned object - stored in .versions directory
|
||||||
|
updateDirectory = s3a.option.BucketsPath + "/" + bucket + "/" + object + ".versions"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// Non-versioned object - stored as regular file
|
||||||
|
updateDirectory = s3a.option.BucketsPath + "/" + bucket
|
||||||
|
}
|
||||||
|
|
||||||
// Update the object with new ACL metadata
|
// Update the object with new ACL metadata
|
||||||
err = s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
err = s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||||
request := &filer_pb.UpdateEntryRequest{
|
request := &filer_pb.UpdateEntryRequest{
|
||||||
Directory: bucketDir,
|
Directory: updateDirectory,
|
||||||
Entry: entry,
|
Entry: entry,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@@ -32,8 +32,8 @@ func (s3a *S3ApiServer) DeleteObjectHandler(w http.ResponseWriter, r *http.Reque
|
|||||||
// Check for specific version ID in query parameters
|
// Check for specific version ID in query parameters
|
||||||
versionId := r.URL.Query().Get("versionId")
|
versionId := r.URL.Query().Get("versionId")
|
||||||
|
|
||||||
// Check if versioning is configured for the bucket (Enabled or Suspended)
|
// Get detailed versioning state for proper handling of suspended vs enabled versioning
|
||||||
versioningConfigured, err := s3a.isVersioningConfigured(bucket)
|
versioningState, err := s3a.getVersioningState(bucket)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if err == filer_pb.ErrNotFound {
|
if err == filer_pb.ErrNotFound {
|
||||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
|
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
|
||||||
@@ -44,14 +44,19 @@ func (s3a *S3ApiServer) DeleteObjectHandler(w http.ResponseWriter, r *http.Reque
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
versioningEnabled := (versioningState == s3_constants.VersioningEnabled)
|
||||||
|
versioningSuspended := (versioningState == s3_constants.VersioningSuspended)
|
||||||
|
versioningConfigured := (versioningState != "")
|
||||||
|
|
||||||
var auditLog *s3err.AccessLog
|
var auditLog *s3err.AccessLog
|
||||||
if s3err.Logger != nil {
|
if s3err.Logger != nil {
|
||||||
auditLog = s3err.GetAccessLog(r, http.StatusNoContent, s3err.ErrNone)
|
auditLog = s3err.GetAccessLog(r, http.StatusNoContent, s3err.ErrNone)
|
||||||
}
|
}
|
||||||
|
|
||||||
if versioningConfigured {
|
if versioningConfigured {
|
||||||
// Handle versioned delete
|
// Handle versioned delete based on specific versioning state
|
||||||
if versionId != "" {
|
if versionId != "" {
|
||||||
|
// Delete specific version (same for both enabled and suspended)
|
||||||
// Check object lock permissions before deleting specific version
|
// Check object lock permissions before deleting specific version
|
||||||
governanceBypassAllowed := s3a.evaluateGovernanceBypassRequest(r, bucket, object)
|
governanceBypassAllowed := s3a.evaluateGovernanceBypassRequest(r, bucket, object)
|
||||||
if err := s3a.enforceObjectLockProtections(r, bucket, object, versionId, governanceBypassAllowed); err != nil {
|
if err := s3a.enforceObjectLockProtections(r, bucket, object, versionId, governanceBypassAllowed); err != nil {
|
||||||
@@ -71,19 +76,44 @@ func (s3a *S3ApiServer) DeleteObjectHandler(w http.ResponseWriter, r *http.Reque
|
|||||||
// Set version ID in response header
|
// Set version ID in response header
|
||||||
w.Header().Set("x-amz-version-id", versionId)
|
w.Header().Set("x-amz-version-id", versionId)
|
||||||
} else {
|
} else {
|
||||||
// Create delete marker (logical delete)
|
// Delete without version ID - behavior depends on versioning state
|
||||||
// AWS S3 behavior: Delete marker creation is NOT blocked by object retention
|
if versioningEnabled {
|
||||||
// because it's a logical delete that doesn't actually remove the retained version
|
// Enabled versioning: Create delete marker (logical delete)
|
||||||
deleteMarkerVersionId, err := s3a.createDeleteMarker(bucket, object)
|
// AWS S3 behavior: Delete marker creation is NOT blocked by object retention
|
||||||
if err != nil {
|
// because it's a logical delete that doesn't actually remove the retained version
|
||||||
glog.Errorf("Failed to create delete marker: %v", err)
|
deleteMarkerVersionId, err := s3a.createDeleteMarker(bucket, object)
|
||||||
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
|
if err != nil {
|
||||||
return
|
glog.Errorf("Failed to create delete marker: %v", err)
|
||||||
}
|
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
// Set delete marker version ID in response header
|
// Set delete marker version ID in response header
|
||||||
w.Header().Set("x-amz-version-id", deleteMarkerVersionId)
|
w.Header().Set("x-amz-version-id", deleteMarkerVersionId)
|
||||||
w.Header().Set("x-amz-delete-marker", "true")
|
w.Header().Set("x-amz-delete-marker", "true")
|
||||||
|
} else if versioningSuspended {
|
||||||
|
// Suspended versioning: Actually delete the "null" version object
|
||||||
|
glog.V(2).Infof("DeleteObjectHandler: deleting null version for suspended versioning %s/%s", bucket, object)
|
||||||
|
|
||||||
|
// Check object lock permissions before deleting "null" version
|
||||||
|
governanceBypassAllowed := s3a.evaluateGovernanceBypassRequest(r, bucket, object)
|
||||||
|
if err := s3a.enforceObjectLockProtections(r, bucket, object, "null", governanceBypassAllowed); err != nil {
|
||||||
|
glog.V(2).Infof("DeleteObjectHandler: object lock check failed for %s/%s: %v", bucket, object, err)
|
||||||
|
s3err.WriteErrorResponse(w, r, s3err.ErrAccessDenied)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Delete the "null" version (the regular file)
|
||||||
|
err := s3a.deleteSpecificObjectVersion(bucket, object, "null")
|
||||||
|
if err != nil {
|
||||||
|
glog.Errorf("Failed to delete null version: %v", err)
|
||||||
|
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Note: According to AWS S3 spec, suspended versioning should NOT return version ID headers
|
||||||
|
// The object is deleted but no version information is returned
|
||||||
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// Handle regular delete (non-versioned)
|
// Handle regular delete (non-versioned)
|
||||||
@@ -203,8 +233,8 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h
|
|||||||
auditLog = s3err.GetAccessLog(r, http.StatusNoContent, s3err.ErrNone)
|
auditLog = s3err.GetAccessLog(r, http.StatusNoContent, s3err.ErrNone)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check if versioning is configured for the bucket (needed for object lock checks)
|
// Get detailed versioning state for proper handling of suspended vs enabled versioning
|
||||||
versioningConfigured, err := s3a.isVersioningConfigured(bucket)
|
versioningState, err := s3a.getVersioningState(bucket)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if err == filer_pb.ErrNotFound {
|
if err == filer_pb.ErrNotFound {
|
||||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
|
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
|
||||||
@@ -215,6 +245,10 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
versioningEnabled := (versioningState == s3_constants.VersioningEnabled)
|
||||||
|
versioningSuspended := (versioningState == s3_constants.VersioningSuspended)
|
||||||
|
versioningConfigured := (versioningState != "")
|
||||||
|
|
||||||
s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||||
|
|
||||||
// delete file entries
|
// delete file entries
|
||||||
@@ -243,9 +277,9 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h
|
|||||||
var isDeleteMarker bool
|
var isDeleteMarker bool
|
||||||
|
|
||||||
if versioningConfigured {
|
if versioningConfigured {
|
||||||
// Handle versioned delete
|
// Handle versioned delete based on specific versioning state
|
||||||
if object.VersionId != "" {
|
if object.VersionId != "" {
|
||||||
// Delete specific version
|
// Delete specific version (same for both enabled and suspended)
|
||||||
err := s3a.deleteSpecificObjectVersion(bucket, object.Key, object.VersionId)
|
err := s3a.deleteSpecificObjectVersion(bucket, object.Key, object.VersionId)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
deleteErrors = append(deleteErrors, DeleteError{
|
deleteErrors = append(deleteErrors, DeleteError{
|
||||||
@@ -258,19 +292,39 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h
|
|||||||
}
|
}
|
||||||
deleteVersionId = object.VersionId
|
deleteVersionId = object.VersionId
|
||||||
} else {
|
} else {
|
||||||
// Create delete marker (logical delete)
|
// Delete without version ID - behavior depends on versioning state
|
||||||
deleteMarkerVersionId, err := s3a.createDeleteMarker(bucket, object.Key)
|
if versioningEnabled {
|
||||||
if err != nil {
|
// Enabled versioning: Create delete marker (logical delete)
|
||||||
deleteErrors = append(deleteErrors, DeleteError{
|
deleteMarkerVersionId, err := s3a.createDeleteMarker(bucket, object.Key)
|
||||||
Code: "",
|
if err != nil {
|
||||||
Message: err.Error(),
|
deleteErrors = append(deleteErrors, DeleteError{
|
||||||
Key: object.Key,
|
Code: "",
|
||||||
VersionId: object.VersionId,
|
Message: err.Error(),
|
||||||
})
|
Key: object.Key,
|
||||||
continue
|
VersionId: object.VersionId,
|
||||||
|
})
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
deleteVersionId = deleteMarkerVersionId
|
||||||
|
isDeleteMarker = true
|
||||||
|
} else if versioningSuspended {
|
||||||
|
// Suspended versioning: Actually delete the "null" version object
|
||||||
|
glog.V(2).Infof("DeleteMultipleObjectsHandler: deleting null version for suspended versioning %s/%s", bucket, object.Key)
|
||||||
|
|
||||||
|
err := s3a.deleteSpecificObjectVersion(bucket, object.Key, "null")
|
||||||
|
if err != nil {
|
||||||
|
deleteErrors = append(deleteErrors, DeleteError{
|
||||||
|
Code: "",
|
||||||
|
Message: err.Error(),
|
||||||
|
Key: object.Key,
|
||||||
|
VersionId: "null",
|
||||||
|
})
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
deleteVersionId = "null"
|
||||||
|
// Note: For suspended versioning, we don't set isDeleteMarker=true
|
||||||
|
// because we actually deleted the object, not created a delete marker
|
||||||
}
|
}
|
||||||
deleteVersionId = deleteMarkerVersionId
|
|
||||||
isDeleteMarker = true
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Add to successful deletions with version info
|
// Add to successful deletions with version info
|
||||||
|
@@ -4,16 +4,17 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"encoding/xml"
|
"encoding/xml"
|
||||||
"fmt"
|
"fmt"
|
||||||
"github.com/aws/aws-sdk-go/service/s3"
|
|
||||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
|
||||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
|
||||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
|
||||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
|
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"github.com/aws/aws-sdk-go/service/s3"
|
||||||
|
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||||
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||||
|
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||||
|
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
|
||||||
)
|
)
|
||||||
|
|
||||||
type OptionalString struct {
|
type OptionalString struct {
|
||||||
@@ -356,6 +357,9 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Track .versions directories found in this directory for later processing
|
||||||
|
var versionsDirs []string
|
||||||
|
|
||||||
for {
|
for {
|
||||||
resp, recvErr := stream.Recv()
|
resp, recvErr := stream.Recv()
|
||||||
if recvErr != nil {
|
if recvErr != nil {
|
||||||
@@ -386,6 +390,14 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
|
|||||||
if entry.Name == s3_constants.MultipartUploadsFolder { // FIXME no need to apply to all directories. this extra also affects maxKeys
|
if entry.Name == s3_constants.MultipartUploadsFolder { // FIXME no need to apply to all directories. this extra also affects maxKeys
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Skip .versions directories in regular list operations but track them for logical object creation
|
||||||
|
if strings.HasSuffix(entry.Name, ".versions") {
|
||||||
|
glog.V(4).Infof("Found .versions directory: %s", entry.Name)
|
||||||
|
versionsDirs = append(versionsDirs, entry.Name)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
if delimiter != "/" || cursor.prefixEndsOnDelimiter {
|
if delimiter != "/" || cursor.prefixEndsOnDelimiter {
|
||||||
if cursor.prefixEndsOnDelimiter {
|
if cursor.prefixEndsOnDelimiter {
|
||||||
cursor.prefixEndsOnDelimiter = false
|
cursor.prefixEndsOnDelimiter = false
|
||||||
@@ -425,6 +437,48 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
|
|||||||
cursor.prefixEndsOnDelimiter = false
|
cursor.prefixEndsOnDelimiter = false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// After processing all regular entries, handle versioned objects
|
||||||
|
// Create logical entries for objects that have .versions directories
|
||||||
|
for _, versionsDir := range versionsDirs {
|
||||||
|
if cursor.maxKeys <= 0 {
|
||||||
|
cursor.isTruncated = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
// Extract object name from .versions directory name (remove .versions suffix)
|
||||||
|
baseObjectName := strings.TrimSuffix(versionsDir, ".versions")
|
||||||
|
|
||||||
|
// Construct full object path relative to bucket
|
||||||
|
// dir is something like "/buckets/sea-test-1/Veeam/Backup/vbr/Config"
|
||||||
|
// we need to get the path relative to bucket: "Veeam/Backup/vbr/Config/Owner"
|
||||||
|
bucketPath := strings.TrimPrefix(dir, s3a.option.BucketsPath+"/")
|
||||||
|
bucketName := strings.Split(bucketPath, "/")[0]
|
||||||
|
|
||||||
|
// Remove bucket name from path to get directory within bucket
|
||||||
|
bucketRelativePath := strings.Join(strings.Split(bucketPath, "/")[1:], "/")
|
||||||
|
|
||||||
|
var fullObjectPath string
|
||||||
|
if bucketRelativePath == "" {
|
||||||
|
// Object is at bucket root
|
||||||
|
fullObjectPath = baseObjectName
|
||||||
|
} else {
|
||||||
|
// Object is in subdirectory
|
||||||
|
fullObjectPath = bucketRelativePath + "/" + baseObjectName
|
||||||
|
}
|
||||||
|
|
||||||
|
glog.V(4).Infof("Processing versioned object: baseObjectName=%s, bucketRelativePath=%s, fullObjectPath=%s",
|
||||||
|
baseObjectName, bucketRelativePath, fullObjectPath)
|
||||||
|
|
||||||
|
// Get the latest version information for this object
|
||||||
|
if latestVersionEntry, latestVersionErr := s3a.getLatestVersionEntryForListOperation(bucketName, fullObjectPath); latestVersionErr == nil {
|
||||||
|
glog.V(4).Infof("Creating logical entry for versioned object: %s", fullObjectPath)
|
||||||
|
eachEntryFn(dir, latestVersionEntry)
|
||||||
|
} else {
|
||||||
|
glog.V(4).Infof("Failed to get latest version for %s: %v", fullObjectPath, latestVersionErr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -513,3 +567,32 @@ func (s3a *S3ApiServer) ensureDirectoryAllEmpty(filerClient filer_pb.SeaweedFile
|
|||||||
|
|
||||||
return true, nil
|
return true, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// getLatestVersionEntryForListOperation gets the latest version of an object and creates a logical entry for list operations
|
||||||
|
// This is used to show versioned objects as logical object names in regular list operations
|
||||||
|
func (s3a *S3ApiServer) getLatestVersionEntryForListOperation(bucket, object string) (*filer_pb.Entry, error) {
|
||||||
|
// Get the latest version entry
|
||||||
|
latestVersionEntry, err := s3a.getLatestObjectVersion(bucket, object)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to get latest version: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if this is a delete marker (should not be shown in regular list)
|
||||||
|
if latestVersionEntry.Extended != nil {
|
||||||
|
if deleteMarker, exists := latestVersionEntry.Extended[s3_constants.ExtDeleteMarkerKey]; exists && string(deleteMarker) == "true" {
|
||||||
|
return nil, fmt.Errorf("latest version is a delete marker")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create a logical entry that appears to be stored at the object path (not the versioned path)
|
||||||
|
// This allows the list operation to show the logical object name while preserving all metadata
|
||||||
|
logicalEntry := &filer_pb.Entry{
|
||||||
|
Name: strings.TrimPrefix(object, "/"),
|
||||||
|
IsDirectory: false,
|
||||||
|
Attributes: latestVersionEntry.Attributes,
|
||||||
|
Extended: latestVersionEntry.Extended,
|
||||||
|
Chunks: latestVersionEntry.Chunks,
|
||||||
|
}
|
||||||
|
|
||||||
|
return logicalEntry, nil
|
||||||
|
}
|
||||||
|
@@ -90,6 +90,9 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request)
|
|||||||
entry.Content, _ = io.ReadAll(r.Body)
|
entry.Content, _ = io.ReadAll(r.Body)
|
||||||
}
|
}
|
||||||
entry.Attributes.Mime = objectContentType
|
entry.Attributes.Mime = objectContentType
|
||||||
|
|
||||||
|
// Set object owner for directory objects (same as regular objects)
|
||||||
|
s3a.setObjectOwnerFromRequest(r, entry)
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
|
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
|
||||||
return
|
return
|
||||||
|
@@ -19,18 +19,31 @@ import (
|
|||||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
|
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ObjectVersion represents a version of an S3 object
|
// S3ListObjectVersionsResult - Custom struct for S3 list-object-versions response
|
||||||
type ObjectVersion struct {
|
// This avoids conflicts with the XSD generated ListVersionsResult struct
|
||||||
VersionId string
|
// and ensures proper separation of versions and delete markers into arrays
|
||||||
IsLatest bool
|
type S3ListObjectVersionsResult struct {
|
||||||
IsDeleteMarker bool
|
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListVersionsResult"`
|
||||||
LastModified time.Time
|
|
||||||
ETag string
|
Name string `xml:"Name"`
|
||||||
Size int64
|
Prefix string `xml:"Prefix,omitempty"`
|
||||||
Entry *filer_pb.Entry
|
KeyMarker string `xml:"KeyMarker,omitempty"`
|
||||||
|
VersionIdMarker string `xml:"VersionIdMarker,omitempty"`
|
||||||
|
NextKeyMarker string `xml:"NextKeyMarker,omitempty"`
|
||||||
|
NextVersionIdMarker string `xml:"NextVersionIdMarker,omitempty"`
|
||||||
|
MaxKeys int `xml:"MaxKeys"`
|
||||||
|
Delimiter string `xml:"Delimiter,omitempty"`
|
||||||
|
IsTruncated bool `xml:"IsTruncated"`
|
||||||
|
|
||||||
|
// These are the critical fields - arrays instead of single elements
|
||||||
|
Versions []VersionEntry `xml:"Version,omitempty"` // Array for versions
|
||||||
|
DeleteMarkers []DeleteMarkerEntry `xml:"DeleteMarker,omitempty"` // Array for delete markers
|
||||||
|
|
||||||
|
CommonPrefixes []PrefixEntry `xml:"CommonPrefixes,omitempty"`
|
||||||
|
EncodingType string `xml:"EncodingType,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// ListObjectVersionsResult represents the response for ListObjectVersions
|
// Original struct - keeping for compatibility but will use S3ListObjectVersionsResult for XML response
|
||||||
type ListObjectVersionsResult struct {
|
type ListObjectVersionsResult struct {
|
||||||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListVersionsResult"`
|
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListVersionsResult"`
|
||||||
Name string `xml:"Name"`
|
Name string `xml:"Name"`
|
||||||
@@ -47,6 +60,17 @@ type ListObjectVersionsResult struct {
|
|||||||
CommonPrefixes []PrefixEntry `xml:"CommonPrefixes,omitempty"`
|
CommonPrefixes []PrefixEntry `xml:"CommonPrefixes,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ObjectVersion represents a version of an S3 object
|
||||||
|
type ObjectVersion struct {
|
||||||
|
VersionId string
|
||||||
|
IsLatest bool
|
||||||
|
IsDeleteMarker bool
|
||||||
|
LastModified time.Time
|
||||||
|
ETag string
|
||||||
|
Size int64
|
||||||
|
Entry *filer_pb.Entry
|
||||||
|
}
|
||||||
|
|
||||||
// generateVersionId creates a unique version ID that preserves chronological order
|
// generateVersionId creates a unique version ID that preserves chronological order
|
||||||
func generateVersionId() string {
|
func generateVersionId() string {
|
||||||
// Use nanosecond timestamp to ensure chronological ordering
|
// Use nanosecond timestamp to ensure chronological ordering
|
||||||
@@ -124,7 +148,7 @@ func (s3a *S3ApiServer) createDeleteMarker(bucket, object string) (string, error
|
|||||||
}
|
}
|
||||||
|
|
||||||
// listObjectVersions lists all versions of an object
|
// listObjectVersions lists all versions of an object
|
||||||
func (s3a *S3ApiServer) listObjectVersions(bucket, prefix, keyMarker, versionIdMarker, delimiter string, maxKeys int) (*ListObjectVersionsResult, error) {
|
func (s3a *S3ApiServer) listObjectVersions(bucket, prefix, keyMarker, versionIdMarker, delimiter string, maxKeys int) (*S3ListObjectVersionsResult, error) {
|
||||||
var allVersions []interface{} // Can contain VersionEntry or DeleteMarkerEntry
|
var allVersions []interface{} // Can contain VersionEntry or DeleteMarkerEntry
|
||||||
|
|
||||||
// Track objects that have been processed to avoid duplicates
|
// Track objects that have been processed to avoid duplicates
|
||||||
@@ -184,8 +208,8 @@ func (s3a *S3ApiServer) listObjectVersions(bucket, prefix, keyMarker, versionIdM
|
|||||||
return versionIdI > versionIdJ
|
return versionIdI > versionIdJ
|
||||||
})
|
})
|
||||||
|
|
||||||
// Build result
|
// Build result using S3ListObjectVersionsResult to avoid conflicts with XSD structs
|
||||||
result := &ListObjectVersionsResult{
|
result := &S3ListObjectVersionsResult{
|
||||||
Name: bucket,
|
Name: bucket,
|
||||||
Prefix: prefix,
|
Prefix: prefix,
|
||||||
KeyMarker: keyMarker,
|
KeyMarker: keyMarker,
|
||||||
@@ -296,7 +320,35 @@ func (s3a *S3ApiServer) findVersionsRecursively(currentPath, relativePath string
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// Recursively search subdirectories
|
// This is a regular directory - check if it's an explicit S3 directory object
|
||||||
|
// Only include directories that were explicitly created via S3 API (have FolderMimeType)
|
||||||
|
// This excludes implicit directories created when uploading files like "test1/a"
|
||||||
|
if entry.Attributes.Mime == s3_constants.FolderMimeType {
|
||||||
|
directoryKey := entryPath
|
||||||
|
if !strings.HasSuffix(directoryKey, "/") {
|
||||||
|
directoryKey += "/"
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add directory as a version entry with VersionId "null" (following S3/Minio behavior)
|
||||||
|
glog.V(2).Infof("findVersionsRecursively: found explicit S3 directory %s", directoryKey)
|
||||||
|
|
||||||
|
// Calculate ETag for empty directory
|
||||||
|
directoryETag := "\"d41d8cd98f00b204e9800998ecf8427e\""
|
||||||
|
|
||||||
|
versionEntry := &VersionEntry{
|
||||||
|
Key: directoryKey,
|
||||||
|
VersionId: "null",
|
||||||
|
IsLatest: true,
|
||||||
|
LastModified: time.Unix(entry.Attributes.Mtime, 0),
|
||||||
|
ETag: directoryETag,
|
||||||
|
Size: 0, // Directories have size 0
|
||||||
|
Owner: s3a.getObjectOwnerFromEntry(entry),
|
||||||
|
StorageClass: "STANDARD",
|
||||||
|
}
|
||||||
|
*allVersions = append(*allVersions, versionEntry)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Recursively search subdirectories (regardless of whether they're explicit or implicit)
|
||||||
fullPath := path.Join(currentPath, entry.Name)
|
fullPath := path.Join(currentPath, entry.Name)
|
||||||
err := s3a.findVersionsRecursively(fullPath, entryPath, allVersions, processedObjects, seenVersionIds, bucket, prefix)
|
err := s3a.findVersionsRecursively(fullPath, entryPath, allVersions, processedObjects, seenVersionIds, bucket, prefix)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -529,13 +581,7 @@ func (s3a *S3ApiServer) deleteSpecificObjectVersion(bucket, object, versionId st
|
|||||||
versionsDir := s3a.getVersionedObjectDir(bucket, object)
|
versionsDir := s3a.getVersionedObjectDir(bucket, object)
|
||||||
versionFile := s3a.getVersionFileName(versionId)
|
versionFile := s3a.getVersionFileName(versionId)
|
||||||
|
|
||||||
// Delete the specific version from .versions directory
|
// Check if this is the latest version before attempting deletion (for potential metadata update)
|
||||||
_, err := s3a.getEntry(versionsDir, versionFile)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("version %s not found: %v", versionId, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check if this is the latest version before deleting
|
|
||||||
versionsEntry, dirErr := s3a.getEntry(path.Join(s3a.option.BucketsPath, bucket), object+".versions")
|
versionsEntry, dirErr := s3a.getEntry(path.Join(s3a.option.BucketsPath, bucket), object+".versions")
|
||||||
isLatestVersion := false
|
isLatestVersion := false
|
||||||
if dirErr == nil && versionsEntry.Extended != nil {
|
if dirErr == nil && versionsEntry.Extended != nil {
|
||||||
@@ -544,15 +590,19 @@ func (s3a *S3ApiServer) deleteSpecificObjectVersion(bucket, object, versionId st
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Delete the version file
|
// Attempt to delete the version file
|
||||||
|
// Note: We don't check if the file exists first to avoid race conditions
|
||||||
|
// The deletion operation should be idempotent
|
||||||
deleteErr := s3a.rm(versionsDir, versionFile, true, false)
|
deleteErr := s3a.rm(versionsDir, versionFile, true, false)
|
||||||
if deleteErr != nil {
|
if deleteErr != nil {
|
||||||
// Check if file was already deleted by another process
|
// Check if file was already deleted by another process (race condition handling)
|
||||||
if _, checkErr := s3a.getEntry(versionsDir, versionFile); checkErr != nil {
|
if _, checkErr := s3a.getEntry(versionsDir, versionFile); checkErr != nil {
|
||||||
// File doesn't exist anymore, deletion was successful
|
// File doesn't exist anymore, deletion was successful (another thread deleted it)
|
||||||
} else {
|
glog.V(2).Infof("deleteSpecificObjectVersion: version %s for %s%s already deleted by another process", versionId, bucket, object)
|
||||||
return fmt.Errorf("failed to delete version %s: %v", versionId, deleteErr)
|
return nil
|
||||||
}
|
}
|
||||||
|
// File still exists but deletion failed for another reason
|
||||||
|
return fmt.Errorf("failed to delete version %s: %v", versionId, deleteErr)
|
||||||
}
|
}
|
||||||
|
|
||||||
// If we deleted the latest version, update the .versions directory metadata to point to the new latest
|
// If we deleted the latest version, update the .versions directory metadata to point to the new latest
|
||||||
|
Reference in New Issue
Block a user