mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2025-11-24 08:46:54 +08:00
multipart SSE
This commit is contained in:
chrislu
264
weed/s3api/s3_sse_s3_multipart_test.go
Normal file
264
weed/s3api/s3_sse_s3_multipart_test.go
Normal file
@@ -0,0 +1,264 @@
|
||||
package s3api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/aes"
|
||||
"crypto/cipher"
|
||||
"crypto/rand"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
)
|
||||
|
||||
// TestSSES3MultipartChunkViewDecryption tests that multipart SSE-S3 objects use per-chunk IVs
|
||||
func TestSSES3MultipartChunkViewDecryption(t *testing.T) {
|
||||
// Generate test key and base IV
|
||||
key := make([]byte, 32)
|
||||
rand.Read(key)
|
||||
baseIV := make([]byte, 16)
|
||||
rand.Read(baseIV)
|
||||
|
||||
// Create test plaintext
|
||||
plaintext := []byte("This is test data for SSE-S3 multipart encryption testing")
|
||||
|
||||
// Simulate multipart upload with 2 parts at different offsets
|
||||
testCases := []struct {
|
||||
name string
|
||||
partNumber int
|
||||
partOffset int64
|
||||
data []byte
|
||||
}{
|
||||
{"Part 1", 1, 0, plaintext[:30]},
|
||||
{"Part 2", 2, 5 * 1024 * 1024, plaintext[30:]}, // 5MB offset
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
// Calculate IV with offset (simulating upload encryption)
|
||||
adjustedIV, _ := calculateIVWithOffset(baseIV, tc.partOffset)
|
||||
|
||||
// Encrypt the part data
|
||||
block, err := aes.NewCipher(key)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create cipher: %v", err)
|
||||
}
|
||||
|
||||
ciphertext := make([]byte, len(tc.data))
|
||||
stream := cipher.NewCTR(block, adjustedIV)
|
||||
stream.XORKeyStream(ciphertext, tc.data)
|
||||
|
||||
// SSE-S3 stores the offset-adjusted IV directly in chunk metadata
|
||||
// (unlike SSE-C which stores base IV + PartOffset)
|
||||
chunkIV := adjustedIV
|
||||
|
||||
// Verify the IV is offset-adjusted for non-zero offsets
|
||||
if tc.partOffset == 0 {
|
||||
if !bytes.Equal(chunkIV, baseIV) {
|
||||
t.Error("IV should equal base IV when offset is 0")
|
||||
}
|
||||
} else {
|
||||
if bytes.Equal(chunkIV, baseIV) {
|
||||
t.Error("Chunk IV should be offset-adjusted, not base IV")
|
||||
}
|
||||
}
|
||||
|
||||
// Verify decryption works with the chunk's IV
|
||||
decryptedData := make([]byte, len(ciphertext))
|
||||
decryptBlock, _ := aes.NewCipher(key)
|
||||
decryptStream := cipher.NewCTR(decryptBlock, chunkIV)
|
||||
decryptStream.XORKeyStream(decryptedData, ciphertext)
|
||||
|
||||
if !bytes.Equal(decryptedData, tc.data) {
|
||||
t.Errorf("Decryption failed: expected %q, got %q", tc.data, decryptedData)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestSSES3SinglePartChunkViewDecryption tests single-part SSE-S3 objects use object-level IV
|
||||
func TestSSES3SinglePartChunkViewDecryption(t *testing.T) {
|
||||
// Generate test key and IV
|
||||
key := make([]byte, 32)
|
||||
rand.Read(key)
|
||||
iv := make([]byte, 16)
|
||||
rand.Read(iv)
|
||||
|
||||
// Create test plaintext
|
||||
plaintext := []byte("This is test data for SSE-S3 single-part encryption testing")
|
||||
|
||||
// Encrypt the data
|
||||
block, err := aes.NewCipher(key)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create cipher: %v", err)
|
||||
}
|
||||
|
||||
ciphertext := make([]byte, len(plaintext))
|
||||
stream := cipher.NewCTR(block, iv)
|
||||
stream.XORKeyStream(ciphertext, plaintext)
|
||||
|
||||
// Create a mock file chunk WITHOUT per-chunk metadata (single-part path)
|
||||
fileChunk := &filer_pb.FileChunk{
|
||||
FileId: "test-file-id",
|
||||
Offset: 0,
|
||||
Size: uint64(len(ciphertext)),
|
||||
SseType: filer_pb.SSEType_SSE_S3,
|
||||
SseMetadata: nil, // No per-chunk metadata for single-part
|
||||
}
|
||||
|
||||
// Verify the chunk does NOT have per-chunk metadata
|
||||
if len(fileChunk.GetSseMetadata()) > 0 {
|
||||
t.Error("Single-part chunk should not have per-chunk metadata")
|
||||
}
|
||||
|
||||
// For single-part, the object-level IV is used
|
||||
objectLevelIV := iv
|
||||
|
||||
// Verify decryption works with the object-level IV
|
||||
decryptedData := make([]byte, len(ciphertext))
|
||||
decryptBlock, _ := aes.NewCipher(key)
|
||||
decryptStream := cipher.NewCTR(decryptBlock, objectLevelIV)
|
||||
decryptStream.XORKeyStream(decryptedData, ciphertext)
|
||||
|
||||
if !bytes.Equal(decryptedData, plaintext) {
|
||||
t.Errorf("Decryption failed: expected %q, got %q", plaintext, decryptedData)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSSES3IVOffsetCalculation verifies IV offset calculation for multipart uploads
|
||||
func TestSSES3IVOffsetCalculation(t *testing.T) {
|
||||
baseIV := make([]byte, 16)
|
||||
rand.Read(baseIV)
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
partNumber int
|
||||
partSize int64
|
||||
offset int64
|
||||
}{
|
||||
{"Part 1", 1, 5 * 1024 * 1024, 0},
|
||||
{"Part 2", 2, 5 * 1024 * 1024, 5 * 1024 * 1024},
|
||||
{"Part 3", 3, 5 * 1024 * 1024, 10 * 1024 * 1024},
|
||||
{"Part 10", 10, 5 * 1024 * 1024, 45 * 1024 * 1024},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
// Calculate IV with offset
|
||||
adjustedIV, skip := calculateIVWithOffset(baseIV, tc.offset)
|
||||
|
||||
// Verify IV is different from base (except for offset 0)
|
||||
if tc.offset == 0 {
|
||||
if !bytes.Equal(adjustedIV, baseIV) {
|
||||
t.Error("IV should equal base IV when offset is 0")
|
||||
}
|
||||
if skip != 0 {
|
||||
t.Errorf("Skip should be 0 when offset is 0, got %d", skip)
|
||||
}
|
||||
} else {
|
||||
if bytes.Equal(adjustedIV, baseIV) {
|
||||
t.Error("IV should be different from base IV when offset > 0")
|
||||
}
|
||||
}
|
||||
|
||||
// Verify skip is calculated correctly
|
||||
expectedSkip := int(tc.offset % 16)
|
||||
if skip != expectedSkip {
|
||||
t.Errorf("Skip mismatch: expected %d, got %d", expectedSkip, skip)
|
||||
}
|
||||
|
||||
// Verify IV adjustment is deterministic
|
||||
adjustedIV2, skip2 := calculateIVWithOffset(baseIV, tc.offset)
|
||||
if !bytes.Equal(adjustedIV, adjustedIV2) || skip != skip2 {
|
||||
t.Error("IV calculation is not deterministic")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestSSES3ChunkMetadataDetection tests detection of per-chunk vs object-level metadata
|
||||
func TestSSES3ChunkMetadataDetection(t *testing.T) {
|
||||
// Test data for multipart chunk
|
||||
mockMetadata := []byte("mock-serialized-metadata")
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
chunk *filer_pb.FileChunk
|
||||
expectedMultipart bool
|
||||
}{
|
||||
{
|
||||
name: "Multipart chunk with metadata",
|
||||
chunk: &filer_pb.FileChunk{
|
||||
SseType: filer_pb.SSEType_SSE_S3,
|
||||
SseMetadata: mockMetadata,
|
||||
},
|
||||
expectedMultipart: true,
|
||||
},
|
||||
{
|
||||
name: "Single-part chunk without metadata",
|
||||
chunk: &filer_pb.FileChunk{
|
||||
SseType: filer_pb.SSEType_SSE_S3,
|
||||
SseMetadata: nil,
|
||||
},
|
||||
expectedMultipart: false,
|
||||
},
|
||||
{
|
||||
name: "Non-SSE-S3 chunk",
|
||||
chunk: &filer_pb.FileChunk{
|
||||
SseType: filer_pb.SSEType_NONE,
|
||||
SseMetadata: nil,
|
||||
},
|
||||
expectedMultipart: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
hasPerChunkMetadata := tc.chunk.GetSseType() == filer_pb.SSEType_SSE_S3 && len(tc.chunk.GetSseMetadata()) > 0
|
||||
|
||||
if hasPerChunkMetadata != tc.expectedMultipart {
|
||||
t.Errorf("Expected multipart=%v, got hasPerChunkMetadata=%v", tc.expectedMultipart, hasPerChunkMetadata)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestSSES3EncryptionConsistency verifies encryption/decryption roundtrip
|
||||
func TestSSES3EncryptionConsistency(t *testing.T) {
|
||||
plaintext := []byte("Test data for SSE-S3 encryption consistency verification")
|
||||
|
||||
key := make([]byte, 32)
|
||||
rand.Read(key)
|
||||
iv := make([]byte, 16)
|
||||
rand.Read(iv)
|
||||
|
||||
// Encrypt
|
||||
block, err := aes.NewCipher(key)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create cipher: %v", err)
|
||||
}
|
||||
|
||||
ciphertext := make([]byte, len(plaintext))
|
||||
encryptStream := cipher.NewCTR(block, iv)
|
||||
encryptStream.XORKeyStream(ciphertext, plaintext)
|
||||
|
||||
// Decrypt
|
||||
decrypted := make([]byte, len(ciphertext))
|
||||
decryptBlock, _ := aes.NewCipher(key)
|
||||
decryptStream := cipher.NewCTR(decryptBlock, iv)
|
||||
decryptStream.XORKeyStream(decrypted, ciphertext)
|
||||
|
||||
// Verify
|
||||
if !bytes.Equal(decrypted, plaintext) {
|
||||
t.Errorf("Decryption mismatch: expected %q, got %q", plaintext, decrypted)
|
||||
}
|
||||
|
||||
// Verify idempotency - decrypt again should give garbage
|
||||
decrypted2 := make([]byte, len(ciphertext))
|
||||
decryptStream2 := cipher.NewCTR(decryptBlock, iv)
|
||||
decryptStream2.XORKeyStream(decrypted2, ciphertext)
|
||||
|
||||
if !bytes.Equal(decrypted2, plaintext) {
|
||||
t.Error("Second decryption should also work with fresh stream")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1441,7 +1441,52 @@ func (s3a *S3ApiServer) decryptSSEKMSChunkView(ctx context.Context, fileChunk *f
|
||||
|
||||
// decryptSSES3ChunkView decrypts a specific chunk view with SSE-S3
|
||||
func (s3a *S3ApiServer) decryptSSES3ChunkView(ctx context.Context, fileChunk *filer_pb.FileChunk, chunkView *filer.ChunkView, entry *filer_pb.Entry) (io.Reader, error) {
|
||||
// Get SSE-S3 key from object metadata
|
||||
// For multipart SSE-S3, each chunk has its own IV in chunk.SseMetadata
|
||||
if fileChunk.GetSseType() == filer_pb.SSEType_SSE_S3 && len(fileChunk.GetSseMetadata()) > 0 {
|
||||
keyManager := GetSSES3KeyManager()
|
||||
|
||||
// Deserialize per-chunk SSE-S3 metadata to get chunk-specific IV
|
||||
chunkSSES3Metadata, err := DeserializeSSES3Metadata(fileChunk.GetSseMetadata(), keyManager)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to deserialize chunk SSE-S3 metadata: %w", err)
|
||||
}
|
||||
|
||||
// Fetch FULL encrypted chunk (necessary for proper CTR decryption stream)
|
||||
fullChunkReader, err := s3a.fetchFullChunk(ctx, chunkView.FileId)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to fetch full chunk: %w", err)
|
||||
}
|
||||
|
||||
// Use the chunk's IV directly (already adjusted for part offset during encryption)
|
||||
// Note: SSE-S3 stores the offset-adjusted IV in chunk metadata, unlike SSE-C which stores base IV + PartOffset
|
||||
iv := chunkSSES3Metadata.IV
|
||||
|
||||
glog.V(4).Infof("Decrypting multipart SSE-S3 chunk %s with chunk-specific IV length=%d",
|
||||
chunkView.FileId, len(iv))
|
||||
|
||||
// Decrypt the full chunk
|
||||
decryptedReader, decryptErr := CreateSSES3DecryptedReader(fullChunkReader, chunkSSES3Metadata, iv)
|
||||
if decryptErr != nil {
|
||||
fullChunkReader.Close()
|
||||
return nil, fmt.Errorf("failed to create SSE-S3 decrypted reader: %w", decryptErr)
|
||||
}
|
||||
|
||||
// Skip to position within chunk and limit to ViewSize
|
||||
if chunkView.OffsetInChunk > 0 {
|
||||
_, err = io.CopyN(io.Discard, decryptedReader, chunkView.OffsetInChunk)
|
||||
if err != nil {
|
||||
if closer, ok := decryptedReader.(io.Closer); ok {
|
||||
closer.Close()
|
||||
}
|
||||
return nil, fmt.Errorf("failed to skip to offset %d: %w", chunkView.OffsetInChunk, err)
|
||||
}
|
||||
}
|
||||
|
||||
limitedReader := io.LimitReader(decryptedReader, int64(chunkView.ViewSize))
|
||||
return &rc{Reader: limitedReader, Closer: fullChunkReader}, nil
|
||||
}
|
||||
|
||||
// Single-part SSE-S3: use object-level IV and key (fallback path)
|
||||
keyData := entry.Extended[s3_constants.SeaweedFSSSES3Key]
|
||||
keyManager := GetSSES3KeyManager()
|
||||
sseS3Key, err := DeserializeSSES3Metadata(keyData, keyManager)
|
||||
@@ -1455,13 +1500,16 @@ func (s3a *S3ApiServer) decryptSSES3ChunkView(ctx context.Context, fileChunk *fi
|
||||
return nil, fmt.Errorf("failed to fetch full chunk: %w", err)
|
||||
}
|
||||
|
||||
// Get base IV and use it directly (no offset adjustment for full chunk)
|
||||
// Get base IV for single-part object
|
||||
iv, err := GetSSES3IV(entry, sseS3Key, keyManager)
|
||||
if err != nil {
|
||||
fullChunkReader.Close()
|
||||
return nil, fmt.Errorf("failed to get SSE-S3 IV: %w", err)
|
||||
}
|
||||
|
||||
glog.V(4).Infof("Decrypting single-part SSE-S3 chunk %s with entry-level IV length=%d",
|
||||
chunkView.FileId, len(iv))
|
||||
|
||||
decryptedReader, decryptErr := CreateSSES3DecryptedReader(fullChunkReader, sseS3Key, iv)
|
||||
if decryptErr != nil {
|
||||
fullChunkReader.Close()
|
||||
|
||||
Reference in New Issue
Block a user