diff --git a/.github/workflows/s3-sse-tests.yml b/.github/workflows/s3-sse-tests.yml index ed25bccc6..a6d9f13ea 100644 --- a/.github/workflows/s3-sse-tests.yml +++ b/.github/workflows/s3-sse-tests.yml @@ -73,8 +73,12 @@ jobs: # Quick tests - basic SSE-C and SSE-KMS functionality + Range requests make test-with-server TEST_PATTERN="TestSSECIntegrationBasic|TestSSEKMSIntegrationBasic|TestSimpleSSECIntegration|.*RangeRequestsServerBehavior" else - # Comprehensive tests - SSE-C/KMS functionality, excluding copy operations (pre-existing SSE-C issues) - make test-with-server TEST_PATTERN="TestSSECIntegrationBasic|TestSSECIntegrationVariousDataSizes|TestSSEKMSIntegrationBasic|TestSSEKMSIntegrationVariousDataSizes|.*Multipart.*Integration|TestSimpleSSECIntegration|.*RangeRequestsServerBehavior" + # Comprehensive tests - SSE-C/KMS functionality plus cross-SSE copy. + # The copy-operation tests (`.*ObjectCopyIntegration`, `TestCrossSSECopy`, + # `TestSSEMultipartCopy`) were excluded for a long time as "pre-existing + # SSE-C issues" (#9281); fixed and brought back into CI as part of the + # same change that fixed them. + make test-with-server TEST_PATTERN="TestSSECIntegrationBasic|TestSSECIntegrationVariousDataSizes|TestSSEKMSIntegrationBasic|TestSSEKMSIntegrationVariousDataSizes|.*Multipart.*Integration|TestSimpleSSECIntegration|.*RangeRequestsServerBehavior|.*ObjectCopyIntegration|TestCrossSSECopy|TestSSEMultipartCopy" fi - name: Show server logs on failure diff --git a/weed/s3api/s3_sse_metadata.go b/weed/s3api/s3_sse_metadata.go index 7cb695251..c6f90ddc2 100644 --- a/weed/s3api/s3_sse_metadata.go +++ b/weed/s3api/s3_sse_metadata.go @@ -7,23 +7,41 @@ import ( "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" ) -// StoreSSECIVInMetadata stores the SSE-C IV in entry metadata as base64 encoded string -// Used by SSE-C for storing IV in entry.Extended +// StoreSSECIVInMetadata stores the SSE-C IV in entry metadata as raw bytes. +// +// The SSE-C IV is stored as raw bytes (not base64-encoded) in entry.Extended. +// This matches the storage format used by putToFiler() for the same key, and +// what the GET handler reads back at s3api_object_handlers.go for SSE-C +// decryption. Earlier this helper base64-encoded the IV, while putToFiler +// stored raw bytes — leaving CopyObject and GET in disagreement and causing +// 500s on copy and "invalid IV length: expected 16 bytes, got 24" on read of +// copied SSE-C objects (issue #9281). func StoreSSECIVInMetadata(metadata map[string][]byte, iv []byte) { if len(iv) > 0 { - metadata[s3_constants.SeaweedFSSSEIV] = []byte(base64.StdEncoding.EncodeToString(iv)) + metadata[s3_constants.SeaweedFSSSEIV] = iv } } -// GetSSECIVFromMetadata retrieves the SSE-C IV from entry metadata -// Used by SSE-C for retrieving IV from entry.Extended +// GetSSECIVFromMetadata retrieves the SSE-C IV from entry metadata. +// +// Reads the IV as raw bytes — the format StoreSSECIVInMetadata and putToFiler +// both write. For backward compatibility with any objects whose IV was stored +// in the legacy base64 form (24 bytes, matching base64 of a 16-byte IV), this +// also accepts and decodes that form. func GetSSECIVFromMetadata(metadata map[string][]byte) ([]byte, error) { - if ivBase64, exists := metadata[s3_constants.SeaweedFSSSEIV]; exists { - iv, err := base64.StdEncoding.DecodeString(string(ivBase64)) - if err != nil { - return nil, fmt.Errorf("failed to decode SSE-C IV from metadata: %w", err) - } + stored, exists := metadata[s3_constants.SeaweedFSSSEIV] + if !exists { + return nil, fmt.Errorf("SSE-C IV not found in metadata") + } + // Raw 16-byte IV — the canonical format. + if len(stored) == s3_constants.AESBlockSize { + return stored, nil + } + // Legacy base64-encoded form: 24 bytes matches base64 of a 16-byte IV + // (with two '=' padding chars). Try a base64 decode and validate length. + if iv, err := base64.StdEncoding.DecodeString(string(stored)); err == nil && len(iv) == s3_constants.AESBlockSize { return iv, nil } - return nil, fmt.Errorf("SSE-C IV not found in metadata") + return nil, fmt.Errorf("SSE-C IV in metadata has unexpected length %d (expected %d raw or %d base64)", + len(stored), s3_constants.AESBlockSize, base64.StdEncoding.EncodedLen(s3_constants.AESBlockSize)) } diff --git a/weed/s3api/s3api_object_handlers_copy.go b/weed/s3api/s3api_object_handlers_copy.go index 843e7364e..c7e4808d6 100644 --- a/weed/s3api/s3api_object_handlers_copy.go +++ b/weed/s3api/s3api_object_handlers_copy.go @@ -10,6 +10,7 @@ import ( "io" "net/http" "net/url" + "reflect" "strconv" "strings" "time" @@ -1004,10 +1005,13 @@ func (s3a *S3ApiServer) copyChunks(entry *filer_pb.Entry, dstPath string) ([]*fi return dstChunks, nil } -// copySingleChunk copies a single chunk from source to destination +// copySingleChunk copies a single chunk from source to destination, preserving +// the source's SSE tagging (the same-key copy fast path reuses the source +// ciphertext as-is, so the destination chunk must keep the source's SSE_C / +// SSE_KMS / SSE_S3 metadata or the read path will not decrypt — see #9281). func (s3a *S3ApiServer) copySingleChunk(chunk *filer_pb.FileChunk, dstPath string) (*filer_pb.FileChunk, error) { // Create destination chunk - dstChunk := s3a.createDestinationChunk(chunk, chunk.Offset, chunk.Size) + dstChunk := s3a.createDestinationChunkPreservingSSE(chunk, chunk.Offset, chunk.Size) // Prepare chunk copy (assign new volume and get source URL) fileId := chunk.GetFileIdString() @@ -1253,7 +1257,13 @@ func (s3a *S3ApiServer) validateConditionalCopyHeaders(r *http.Request, entry *f return s3err.ErrNone } -// createDestinationChunk creates a new chunk based on the source chunk with modified properties +// createDestinationChunk creates a new chunk based on the source chunk with modified properties. +// +// SseType and SseMetadata are NOT copied here because most call sites +// re-encrypt the chunk with the destination's keys and then set those fields +// to match the new encryption. The same-key fast path (where the bytes are +// copied as-is and the destination should keep the source's SSE tagging) calls +// createDestinationChunkPreservingSSE instead. func (s3a *S3ApiServer) createDestinationChunk(sourceChunk *filer_pb.FileChunk, offset int64, size uint64) *filer_pb.FileChunk { return &filer_pb.FileChunk{ Offset: offset, @@ -1265,6 +1275,21 @@ func (s3a *S3ApiServer) createDestinationChunk(sourceChunk *filer_pb.FileChunk, } } +// createDestinationChunkPreservingSSE returns a destination chunk that mirrors +// the source's SSE tagging in addition to the usual fields. This is used by the +// same-key copy fast paths where the on-disk bytes are reused as-is and the +// destination must therefore declare the same per-chunk SSE encryption as the +// source (otherwise detectPrimarySSEType returns "None" on read and +// GetObjectHandler serves the still-encrypted bytes raw — issue #9281). +func (s3a *S3ApiServer) createDestinationChunkPreservingSSE(sourceChunk *filer_pb.FileChunk, offset int64, size uint64) *filer_pb.FileChunk { + dst := s3a.createDestinationChunk(sourceChunk, offset, size) + dst.SseType = sourceChunk.SseType + if len(sourceChunk.SseMetadata) > 0 { + dst.SseMetadata = append([]byte(nil), sourceChunk.SseMetadata...) + } + return dst +} + // lookupVolumeUrl looks up the volume URL for a given file ID using the filer's LookupVolume method func (s3a *S3ApiServer) lookupVolumeUrl(fileId string) (string, error) { var srcUrl string @@ -1452,6 +1477,12 @@ func (s3a *S3ApiServer) copyMultipartSSEKMSChunks(entry *filer_pb.Entry, destKey // For multipart SSE-KMS, always use decrypt/reencrypt path to ensure proper metadata handling // The standard copyChunks() doesn't preserve SSE metadata, so we need per-chunk processing + // Deserialize the source's entry-level SSE-KMS key once so it can be used + // as a fallback for legacy multipart chunks that lack per-chunk metadata. + // New multipart SSE-KMS uploads always populate per-chunk metadata, but + // objects written by earlier code may have only the entry-level key. + sourceEntrySSEKey := deserializeEntrySSEKMSKey(entry.Extended) + var dstChunks []*filer_pb.FileChunk for _, chunk := range entry.GetChunks() { @@ -1465,8 +1496,9 @@ func (s3a *S3ApiServer) copyMultipartSSEKMSChunks(entry *filer_pb.Entry, destKey continue } - // SSE-KMS chunk: decrypt with stored per-chunk metadata, re-encrypt with dest key - copiedChunk, err := s3a.copyMultipartSSEKMSChunk(chunk, destKeyID, encryptionContext, bucketKeyEnabled, dstPath, bucket) + // SSE-KMS chunk: decrypt with stored per-chunk metadata (or entry-level + // fallback for legacy data), re-encrypt with dest key. + copiedChunk, err := s3a.copyMultipartSSEKMSChunk(chunk, sourceEntrySSEKey, destKeyID, encryptionContext, bucketKeyEnabled, dstPath, bucket) if err != nil { return nil, nil, fmt.Errorf("failed to copy SSE-KMS chunk %s: %w", chunk.GetFileIdString(), err) } @@ -1474,30 +1506,46 @@ func (s3a *S3ApiServer) copyMultipartSSEKMSChunks(entry *filer_pb.Entry, destKey dstChunks = append(dstChunks, copiedChunk) } - // Create destination metadata for SSE-KMS + // Create destination metadata for SSE-KMS. + // + // Prefer the first dst chunk's full per-chunk key (which carries a real + // EDK + IV minted by copyMultipartSSEKMSChunk's + // CreateSSEKMSEncryptedReaderWithBucketKey call) so single-chunk reads on + // the destination can unwrap the EDK on the GET path. Fall back to a stub + // key (KeyID + context + bucket-key only) for 0-byte objects so they're + // still recognised as SSE-KMS encrypted. dstMetadata := make(map[string][]byte) if destKeyID != "" { - // Store SSE-KMS metadata for single-part compatibility - if encryptionContext == nil { - encryptionContext = BuildEncryptionContext(bucket, dstPath, bucketKeyEnabled) - } - sseKey := &SSEKMSKey{ - KeyID: destKeyID, - EncryptionContext: encryptionContext, - BucketKeyEnabled: bucketKeyEnabled, - } - if kmsMetadata, serErr := SerializeSSEKMSMetadata(sseKey); serErr == nil { - dstMetadata[s3_constants.SeaweedFSSSEKMSKey] = kmsMetadata + if len(dstChunks) > 0 && len(dstChunks[0].GetSseMetadata()) > 0 { + dstMetadata[s3_constants.SeaweedFSSSEKMSKey] = dstChunks[0].GetSseMetadata() } else { - glog.Errorf("Failed to serialize SSE-KMS metadata: %v", serErr) + if encryptionContext == nil { + encryptionContext = BuildEncryptionContext(bucket, dstPath, bucketKeyEnabled) + } + sseKey := &SSEKMSKey{ + KeyID: destKeyID, + EncryptionContext: encryptionContext, + BucketKeyEnabled: bucketKeyEnabled, + } + if kmsMetadata, serErr := SerializeSSEKMSMetadata(sseKey); serErr == nil { + dstMetadata[s3_constants.SeaweedFSSSEKMSKey] = kmsMetadata + } else { + glog.Errorf("Failed to serialize SSE-KMS metadata: %v", serErr) + } } } return dstChunks, dstMetadata, nil } -// copyMultipartSSEKMSChunk copies a single SSE-KMS chunk from a multipart object (unified with SSE-C approach) -func (s3a *S3ApiServer) copyMultipartSSEKMSChunk(chunk *filer_pb.FileChunk, destKeyID string, encryptionContext map[string]string, bucketKeyEnabled bool, dstPath, bucket string) (*filer_pb.FileChunk, error) { +// copyMultipartSSEKMSChunk copies a single SSE-KMS chunk from a multipart object (unified with SSE-C approach). +// +// sourceEntrySSEKey is the source object's entry-level SSE-KMS key (deserialized +// from entry.Extended[SeaweedFSSSEKMSKey] by the caller). It's used as a +// fallback when this chunk has no per-chunk SSE-KMS metadata of its own — +// legacy multipart SSE-KMS objects may have only the entry-level key. Newer +// uploads populate per-chunk metadata, in which case this fallback is unused. +func (s3a *S3ApiServer) copyMultipartSSEKMSChunk(chunk *filer_pb.FileChunk, sourceEntrySSEKey *SSEKMSKey, destKeyID string, encryptionContext map[string]string, bucketKeyEnabled bool, dstPath, bucket string) (*filer_pb.FileChunk, error) { // Create destination chunk dstChunk := s3a.createDestinationChunk(chunk, chunk.Offset, chunk.Size) @@ -1521,15 +1569,13 @@ func (s3a *S3ApiServer) copyMultipartSSEKMSChunk(chunk *filer_pb.FileChunk, dest var finalData []byte - // Decrypt source data using stored SSE-KMS metadata (same pattern as SSE-C) - if len(chunk.GetSseMetadata()) == 0 { - return nil, fmt.Errorf("SSE-KMS chunk missing per-chunk metadata") - } - - // Deserialize the SSE-KMS metadata (reusing unified metadata structure) - sourceSSEKey, err := DeserializeSSEKMSMetadata(chunk.GetSseMetadata()) + // Prefer the chunk's own per-chunk SSE-KMS metadata; fall back to the + // source's entry-level key for legacy multipart objects that don't have + // per-chunk metadata. resolveChunkSSEKMSKey centralizes that selection + // so the same logic is used everywhere a chunk needs decryption. + sourceSSEKey, err := resolveChunkSSEKMSKey(chunk, sourceEntrySSEKey) if err != nil { - return nil, fmt.Errorf("failed to deserialize SSE-KMS metadata: %w", err) + return nil, fmt.Errorf("failed to resolve SSE-KMS metadata: %w", err) } // Decrypt the chunk data using the source metadata @@ -1826,15 +1872,28 @@ func (s3a *S3ApiServer) copyMultipartCrossEncryption(entry *filer_pb.Entry, r *h if destKMSEncryptionContext == nil { destKMSEncryptionContext = BuildEncryptionContext(dstBucket, dstPath, destKMSBucketKeyEnabled) } - sseKey := &SSEKMSKey{ - KeyID: destKMSKeyID, - EncryptionContext: destKMSEncryptionContext, - BucketKeyEnabled: destKMSBucketKeyEnabled, - } - if kmsMetadata, serErr := SerializeSSEKMSMetadata(sseKey); serErr == nil { - dstMetadata[s3_constants.SeaweedFSSSEKMSKey] = kmsMetadata + // Take the first dst chunk's full per-chunk metadata as the canonical + // entry-level key — it includes a real EDK + IV minted by + // copyCrossEncryptionChunk's CreateSSEKMSEncryptedReaderWithBucketKey + // call. Earlier this stored only KeyID/context/bucketKey, leaving the + // EncryptedDataKey empty; single-chunk reads then failed with + // "Invalid ciphertext format" when KMS was asked to unwrap an empty + // EDK (#9281). + if len(dstChunks) > 0 && len(dstChunks[0].GetSseMetadata()) > 0 { + dstMetadata[s3_constants.SeaweedFSSSEKMSKey] = dstChunks[0].GetSseMetadata() } else { - glog.Errorf("Failed to serialize SSE-KMS metadata: %v", serErr) + // 0-byte object or no SSE-KMS chunk: fall back to a stub key + // (sufficient for the entry to be recognised as SSE-KMS). + sseKey := &SSEKMSKey{ + KeyID: destKMSKeyID, + EncryptionContext: destKMSEncryptionContext, + BucketKeyEnabled: destKMSBucketKeyEnabled, + } + if kmsMetadata, serErr := SerializeSSEKMSMetadata(sseKey); serErr == nil { + dstMetadata[s3_constants.SeaweedFSSSEKMSKey] = kmsMetadata + } else { + glog.Errorf("Failed to serialize SSE-KMS metadata: %v", serErr) + } } } else if state.DstSSES3 && destSSES3Key != nil { // For SSE-S3 destination, create object-level metadata @@ -2281,9 +2340,20 @@ func (s3a *S3ApiServer) copyChunkWithReencryption(chunk *filer_pb.FileChunk, cop return nil, fmt.Errorf("re-encrypt chunk data: %w", readErr) } finalData = reencryptedData - - // Update chunk size to include IV dstChunk.Size = uint64(len(finalData)) + + // Tag the destination chunk as SSE-C with per-chunk metadata. Without + // this the chunk's SseType stays NONE, detectPrimarySSEType returns + // "None" on read (it counts SSE-C chunks; an entry whose only chunk + // is NONE shows zero), and GetObjectHandler serves the still-encrypted + // volume bytes raw without decryption — yielding deterministic byte + // corruption on the SSE-C copy path (issue #9281). + ssecMetadata, metaErr := SerializeSSECMetadata(destIV, destKey.KeyMD5, chunk.Offset) + if metaErr != nil { + return nil, fmt.Errorf("serialize SSE-C chunk metadata: %w", metaErr) + } + dstChunk.SseType = filer_pb.SSEType_SSE_C + dstChunk.SseMetadata = ssecMetadata } // Upload the processed data @@ -2338,11 +2408,23 @@ func (s3a *S3ApiServer) copyChunksWithSSEKMS(entry *filer_pb.Entry, r *http.Requ } } - // Determine copy strategy + // Determine copy strategy. + // + // DetermineSSEKMSCopyStrategy returns Direct when source and destination + // share the same KMS key ID, but that's not enough on its own — if the + // destination request changes the encryption context or the BucketKey + // flag, the source ciphertext (and its embedded EDK + context) does not + // satisfy the destination's request. Force the slow re-encrypt path in + // that case so the destination object gets a freshly-wrapped EDK bound + // to the requested context/flag. strategy, err := DetermineSSEKMSCopyStrategy(entry.Extended, destKeyID) if err != nil { return nil, nil, err } + if strategy == SSEKMSCopyStrategyDirect && destKeyID != "" && !srcSSEKMSStateMatchesDest(entry.Extended, encryptionContext, bucketKeyEnabled) { + glog.V(2).Infof("SSE-KMS direct copy rejected — encryption context or bucket-key flag differs; falling back to re-encrypt path for %s", dstPath) + strategy = SSEKMSCopyStrategyDecryptEncrypt + } glog.V(2).Infof("SSE-KMS copy strategy for %s: %v", dstPath, strategy) @@ -2358,16 +2440,28 @@ func (s3a *S3ApiServer) copyChunksWithSSEKMS(entry *filer_pb.Entry, r *http.Requ if encryptionContext == nil { encryptionContext = BuildEncryptionContext(bucket, dstPath, bucketKeyEnabled) } - sseKey := &SSEKMSKey{ - KeyID: destKeyID, - EncryptionContext: encryptionContext, - BucketKeyEnabled: bucketKeyEnabled, - } - if kmsMetadata, serializeErr := SerializeSSEKMSMetadata(sseKey); serializeErr == nil { - dstMetadata[s3_constants.SeaweedFSSSEKMSKey] = kmsMetadata - glog.V(3).Infof("Generated SSE-KMS metadata for direct copy: keyID=%s", destKeyID) + // Direct (same-key) fast path: chunks were copied as-is and now + // carry the source's per-chunk SSE-KMS metadata (preserved by + // createDestinationChunkPreservingSSE in copySingleChunk). Use + // the first chunk's full key as entry-level so single-chunk + // reads can unwrap the EDK on the GET path. Earlier this stored + // only KeyID/context/bucketKey, which made single-chunk reads + // fail at GET with "Invalid ciphertext format" (#9281). + if len(chunks) > 0 && len(chunks[0].GetSseMetadata()) > 0 { + dstMetadata[s3_constants.SeaweedFSSSEKMSKey] = chunks[0].GetSseMetadata() + glog.V(3).Infof("Set entry-level SSE-KMS key from first dst chunk for direct copy: keyID=%s", destKeyID) } else { - glog.Errorf("Failed to serialize SSE-KMS metadata for direct copy: %v", serializeErr) + sseKey := &SSEKMSKey{ + KeyID: destKeyID, + EncryptionContext: encryptionContext, + BucketKeyEnabled: bucketKeyEnabled, + } + if kmsMetadata, serializeErr := SerializeSSEKMSMetadata(sseKey); serializeErr == nil { + dstMetadata[s3_constants.SeaweedFSSSEKMSKey] = kmsMetadata + glog.V(3).Infof("Generated SSE-KMS metadata for direct copy: keyID=%s", destKeyID) + } else { + glog.Errorf("Failed to serialize SSE-KMS metadata for direct copy: %v", serializeErr) + } } } return chunks, dstMetadata, err @@ -2382,19 +2476,83 @@ func (s3a *S3ApiServer) copyChunksWithSSEKMS(entry *filer_pb.Entry, r *http.Requ } } +// deserializeEntrySSEKMSKey returns the SSE-KMS key serialized into +// entry.Extended[SeaweedFSSSEKMSKey], or nil if the entry is not SSE-KMS +// encrypted. Errors are logged and treated as "not present" so the caller +// can fall back to per-chunk metadata or fail safely. +func deserializeEntrySSEKMSKey(entryExtended map[string][]byte) *SSEKMSKey { + keyData, ok := entryExtended[s3_constants.SeaweedFSSSEKMSKey] + if !ok || len(keyData) == 0 { + return nil + } + k, err := DeserializeSSEKMSMetadata(keyData) + if err != nil { + glog.V(2).Infof("deserializeEntrySSEKMSKey: failed to deserialize entry-level SSE-KMS key: %v", err) + return nil + } + return k +} + +// resolveChunkSSEKMSKey picks the right SSE-KMS key to decrypt a chunk with: +// the chunk's own per-chunk metadata if present (the post-#9211 layout for +// new uploads), else the source object's entry-level key (legacy multipart +// objects). Returns nil + an error if neither is available; the caller can +// then surface a clear "missing metadata" error to the client. The selection +// must mirror the encryption side: each chunk is encrypted with the key +// recorded in its per-chunk metadata at write time, and entry-level metadata +// is the legacy fallback for parts that were written before per-chunk keys +// existed. +func resolveChunkSSEKMSKey(chunk *filer_pb.FileChunk, entryFallback *SSEKMSKey) (*SSEKMSKey, error) { + if len(chunk.GetSseMetadata()) > 0 { + return DeserializeSSEKMSMetadata(chunk.GetSseMetadata()) + } + if entryFallback != nil { + glog.V(2).Infof("resolveChunkSSEKMSKey: chunk %s has no per-chunk SSE-KMS metadata; falling back to entry-level key (legacy multipart object)", chunk.GetFileIdString()) + return entryFallback, nil + } + return nil, fmt.Errorf("SSE-KMS chunk %s missing per-chunk metadata and no entry-level key available", chunk.GetFileIdString()) +} + +// srcSSEKMSStateMatchesDest reports whether the source object's stored SSE-KMS +// state (encryption context + bucket-key flag) matches the destination request. +// Used to gate the SSE-KMS direct copy fast path: if either differs the source +// ciphertext can't satisfy the destination's request and we must re-encrypt. +func srcSSEKMSStateMatchesDest(srcMetadata map[string][]byte, dstContext map[string]string, dstBucketKeyEnabled bool) bool { + srcKey := deserializeEntrySSEKMSKey(srcMetadata) + if srcKey == nil { + // Source isn't SSE-KMS encrypted (or its key data is malformed — + // we conservatively let CanDirectCopySSEKMS make the call there). + return true + } + if srcKey.BucketKeyEnabled != dstBucketKeyEnabled { + return false + } + if !encryptionContextEqual(srcKey.EncryptionContext, dstContext) { + return false + } + return true +} + +// encryptionContextEqual treats nil and empty maps as equivalent so a request +// that omits the context header doesn't spuriously diverge from a stored one +// that was serialised as an empty map. reflect.DeepEqual returns false for +// nil-vs-empty, so the empty-case shortcut at the top is required. +func encryptionContextEqual(a, b map[string]string) bool { + if len(a) == 0 && len(b) == 0 { + return true + } + return reflect.DeepEqual(a, b) +} + // copyChunksWithSSEKMSReencryption handles the slow path: decrypt source and re-encrypt for destination // Returns chunks and destination metadata like SSE-C for consistency func (s3a *S3ApiServer) copyChunksWithSSEKMSReencryption(entry *filer_pb.Entry, destKeyID string, encryptionContext map[string]string, bucketKeyEnabled bool, dstPath, bucket string) ([]*filer_pb.FileChunk, map[string][]byte, error) { var dstChunks []*filer_pb.FileChunk - // Extract and deserialize source SSE-KMS metadata - var sourceSSEKey *SSEKMSKey - if keyData, exists := entry.Extended[s3_constants.SeaweedFSSSEKMSKey]; exists { - var err error - sourceSSEKey, err = DeserializeSSEKMSMetadata(keyData) - if err != nil { - return nil, nil, fmt.Errorf("failed to deserialize source SSE-KMS metadata: %w", err) - } + // Deserialize the source's entry-level SSE-KMS key once. Used as the + // per-chunk fallback for legacy multipart objects (see resolveChunkSSEKMSKey). + sourceSSEKey := deserializeEntrySSEKMSKey(entry.Extended) + if sourceSSEKey != nil { glog.V(3).Infof("Extracted source SSE-KMS key: keyID=%s, bucketKey=%t", sourceSSEKey.KeyID, sourceSSEKey.BucketKeyEnabled) } @@ -2407,31 +2565,44 @@ func (s3a *S3ApiServer) copyChunksWithSSEKMSReencryption(entry *filer_pb.Entry, dstChunks = append(dstChunks, dstChunk) } - // Generate destination metadata for SSE-KMS encryption (consistent with SSE-C pattern) + // Generate destination metadata for SSE-KMS encryption. + // + // For multi-chunk objects (isMultipartSSEKMS=true on read), the read path + // uses per-chunk metadata (already set by copyChunkWithSSEKMSReencryption + // after #9281). For single-chunk objects (isMultipartSSEKMS=false), the + // read path falls back to the entry-level SSE-KMS key — so it must be a + // fully-formed key (with EncryptedDataKey + IV), not just KeyID. Earlier + // this stored only KeyID/context/bucketKey, leaving EncryptedDataKey + // empty; reads then failed with "Invalid ciphertext format" when KMS was + // asked to unwrap an empty EDK. + // + // Take the first destination chunk's full per-chunk metadata as the + // canonical entry-level key — it includes a real EDK + IV minted by + // CreateSSEKMSEncryptedReaderWithBucketKey. dstMetadata := make(map[string][]byte) if destKeyID != "" { - // Build encryption context if not provided - if encryptionContext == nil { - encryptionContext = BuildEncryptionContext(bucket, dstPath, bucketKeyEnabled) + if len(dstChunks) > 0 && len(dstChunks[0].GetSseMetadata()) > 0 { + dstMetadata[s3_constants.SeaweedFSSSEKMSKey] = dstChunks[0].GetSseMetadata() + glog.V(3).Infof("Set entry-level SSE-KMS key from first dst chunk: keyID=%s, bucketKey=%t", destKeyID, bucketKeyEnabled) + } else { + // 0-byte (no chunks) or no SSE-KMS chunk to crib metadata from: + // fall back to a stub key so the destination entry is still + // recognised as SSE-KMS encrypted on read. Mirrors the fallback + // in copyChunksWithSSEKMS direct branch and copyMultipartCrossEncryption. + if encryptionContext == nil { + encryptionContext = BuildEncryptionContext(bucket, dstPath, bucketKeyEnabled) + } + sseKey := &SSEKMSKey{ + KeyID: destKeyID, + EncryptionContext: encryptionContext, + BucketKeyEnabled: bucketKeyEnabled, + } + if kmsMetadata, serErr := SerializeSSEKMSMetadata(sseKey); serErr == nil { + dstMetadata[s3_constants.SeaweedFSSSEKMSKey] = kmsMetadata + } else { + glog.Errorf("Failed to serialize SSE-KMS metadata for 0-byte destination: %v", serErr) + } } - - // Create SSE-KMS key structure for destination metadata - sseKey := &SSEKMSKey{ - KeyID: destKeyID, - EncryptionContext: encryptionContext, - BucketKeyEnabled: bucketKeyEnabled, - // Note: EncryptedDataKey will be generated during actual encryption - // IV is also generated per chunk during encryption - } - - // Serialize SSE-KMS metadata for storage - kmsMetadata, err := SerializeSSEKMSMetadata(sseKey) - if err != nil { - return nil, nil, fmt.Errorf("serialize destination SSE-KMS metadata: %w", err) - } - - dstMetadata[s3_constants.SeaweedFSSSEKMSKey] = kmsMetadata - glog.V(3).Infof("Generated destination SSE-KMS metadata: keyID=%s, bucketKey=%t", destKeyID, bucketKeyEnabled) } return dstChunks, dstMetadata, nil @@ -2462,11 +2633,24 @@ func (s3a *S3ApiServer) copyChunkWithSSEKMSReencryption(chunk *filer_pb.FileChun var finalData []byte - // Decrypt source data if it's SSE-KMS encrypted - if sourceSSEKey != nil { - // For SSE-KMS, the encrypted chunk data contains IV + encrypted content - // Use the source SSE key to decrypt the chunk data - decryptedReader, err := CreateSSEKMSDecryptedReader(bytes.NewReader(chunkData), sourceSSEKey) + // Decrypt source data if it's SSE-KMS encrypted. + // Multipart SSE-KMS sources have a different EDK + IV per chunk; the + // per-chunk metadata is the only place those values live, so we MUST use + // the chunk's own metadata for decryption rather than the entry-level + // sourceSSEKey (which only matches single-part objects). Earlier this + // always decrypted with the entry-level key, which produced deterministic + // wrong bytes on a multipart-source COPY (issue #9281). Use the shared + // resolveChunkSSEKMSKey helper which centralises this selection. + var chunkSSEKey *SSEKMSKey + if chunk.GetSseType() == filer_pb.SSEType_SSE_KMS || sourceSSEKey != nil { + var resolveErr error + chunkSSEKey, resolveErr = resolveChunkSSEKMSKey(chunk, sourceSSEKey) + if resolveErr != nil { + return nil, fmt.Errorf("resolve SSE-KMS metadata: %w", resolveErr) + } + } + if chunkSSEKey != nil { + decryptedReader, err := CreateSSEKMSDecryptedReader(bytes.NewReader(chunkData), chunkSSEKey) if err != nil { return nil, fmt.Errorf("create SSE-KMS decrypted reader: %w", err) } @@ -2490,7 +2674,7 @@ func (s3a *S3ApiServer) copyChunkWithSSEKMSReencryption(chunk *filer_pb.FileChun encryptionContext = BuildEncryptionContext(bucket, dstPath, bucketKeyEnabled) } - encryptedReader, _, err := CreateSSEKMSEncryptedReaderWithBucketKey(bytes.NewReader(finalData), destKeyID, encryptionContext, bucketKeyEnabled) + encryptedReader, destSSEKey, err := CreateSSEKMSEncryptedReaderWithBucketKey(bytes.NewReader(finalData), destKeyID, encryptionContext, bucketKeyEnabled) if err != nil { return nil, fmt.Errorf("create SSE-KMS encrypted reader: %w", err) } @@ -2500,13 +2684,32 @@ func (s3a *S3ApiServer) copyChunkWithSSEKMSReencryption(chunk *filer_pb.FileChun return nil, fmt.Errorf("re-encrypt chunk data: %w", err) } - // Store original decrypted data size for logging originalSize := len(finalData) finalData = reencryptedData glog.V(4).Infof("Re-encrypted chunk data: %d bytes → %d bytes", originalSize, len(finalData)) - - // Update chunk size to include IV and encryption overhead dstChunk.Size = uint64(len(finalData)) + + // Tag the destination chunk as SSE-KMS with per-chunk metadata. Without + // this, the chunk's SseType stays NONE, detectPrimarySSEType returns + // "None" on read, and GetObjectHandler serves still-encrypted volume + // bytes raw without decryption — yielding deterministic byte + // corruption on the SSE-KMS copy path (issue #9281). + // + // CreateSSEKMSEncryptedReaderWithBucketKey returns destSSEKey freshly + // populated with KeyID, EncryptionContext, EncryptedDataKey, IV and + // BucketKey state, with the encryption stream initialised at counter 0 + // for THIS chunk's bytes (each chunk gets its own random IV, not a + // base-IV-plus-offset scheme). ChunkOffset must therefore stay 0 on + // read; setting it to chunk.Offset would advance the decryption IV by + // chunk.Offset/16 blocks past the position the encryption was at, + // producing deterministic garbage on chunks whose chunk.Offset > 0. + destSSEKey.ChunkOffset = 0 + kmsMetadata, metaErr := SerializeSSEKMSMetadata(destSSEKey) + if metaErr != nil { + return nil, fmt.Errorf("serialize SSE-KMS chunk metadata: %w", metaErr) + } + dstChunk.SseType = filer_pb.SSEType_SSE_KMS + dstChunk.SseMetadata = kmsMetadata } // Upload the processed data