Implementing checksum validation for single-part and multipart downloads

.cspell.json

@@ -261,6 +261,7 @@
     "ossl",
     "ccrng",
     "KEYWRAP",
+    "HKDF",
     "NVME",
     // EC2
     "IMDS",

src/aws-cpp-sdk-transfer/include/aws/transfer/TransferHandle.h

@@ -13,6 +13,7 @@
 #include <aws/core/client/AWSError.h>
 #include <aws/core/client/AsyncCallerContext.h>
 #include <aws/s3/S3Errors.h>
+#include <aws/s3/model/ChecksumAlgorithm.h>
 #include <iostream>
 #include <atomic>
 #include <mutex>
@@ -372,6 +373,7 @@ namespace Aws
              * Return empty string on success, string with error message on error.
              */
             Aws::String WritePartToDownloadStream(Aws::IOStream* partStream, uint64_t writeOffset);
+            void AddChecksumForPart(Aws:: IOStream* partStream, const PartPointer& shared);
 
             void ApplyDownloadConfiguration(const DownloadConfiguration& downloadConfig);
 
@@ -389,6 +391,9 @@ namespace Aws
             Aws::String GetChecksum() const { return m_checksum; }
             void SetChecksum(const Aws::String& checksum) { this->m_checksum = checksum; }
 
+            Aws::S3::Model::ChecksumAlgorithm GetChecksumAlgorithm() const { std::lock_guard<std::mutex> locker(m_getterSetterLock); return m_checksumAlgorithm; }
+            void SetChecksumAlgorithm (const Aws::S3::Model::ChecksumAlgorithm& checksumAlgorithm) { std::lock_guard<std::mutex> locker(m_getterSetterLock); m_checksumAlgorithm = checksumAlgorithm; }
+
            private:
             void CleanupDownloadStream();
 
@@ -430,6 +435,7 @@ namespace Aws
             mutable std::condition_variable m_waitUntilFinishedSignal;
             mutable std::mutex m_getterSetterLock;
             Aws::String m_checksum;
+            Aws::S3::Model::ChecksumAlgorithm m_checksumAlgorithm;
         };
 
         AWS_TRANSFER_API Aws::OStream& operator << (Aws::OStream& s, TransferStatus status);

src/aws-cpp-sdk-transfer/include/aws/transfer/TransferManager.h

@@ -144,6 +144,13 @@ namespace Aws
              * upload. Defaults to CRC64-NVME.
              */
             Aws::S3::Model::ChecksumAlgorithm checksumAlgorithm = S3::Model::ChecksumAlgorithm::CRC64NVME;
+
+            /**
+             * Enable checksum validation for downloads. When enabled, checksums will be
+             * calculated during download and validated against S3 response headers.
+             * Defaults to true.
+             */
+            bool validateChecksums = true;
         };        
 
         /**

src/aws-cpp-sdk-transfer/source/transfer/TransferHandle.cpp

@@ -6,6 +6,12 @@
 #include <aws/transfer/TransferHandle.h>
 #include <aws/core/utils/logging/LogMacros.h>
 #include <aws/core/utils/memory/stl/SimpleStringStream.h>
+#include <aws/core/utils/crypto/CRC32.h>
+#include <aws/core/utils/crypto/CRC64.h>
+#include <aws/core/utils/crypto/Sha1.h>
+#include <aws/core/utils/crypto/Sha256.h>
+#include <aws/core/utils/crypto/Sha256HMAC.h>
+#include "aws/core/utils/HashingUtils.h"
 
 #include <cassert>
 
@@ -370,6 +376,11 @@ namespace Aws
             AWS_LOGSTREAM_TRACE(CLASS_TAG, "Transfer handle ID [" << GetId() << "] Restarting transfer.");
             m_cancel.store(false);
             m_lastPart.store(false);
+
+            // Clear checksum state for retry
+            std::lock_guard<std::mutex> locker(m_getterSetterLock);
+            m_checksum.clear();
+            m_checksumAlgorithm = Aws::S3::Model::ChecksumAlgorithm::NOT_SET;
         }
 
         bool TransferHandle::ShouldContinue() const
@@ -423,6 +434,25 @@ namespace Aws
             return "";
         }
 
+        void TransferHandle::AddChecksumForPart(Aws::IOStream *partStream, const PartPointer& partState) {
+            partStream->seekg(0);
+            Aws::String checksum = "";
+            if (GetChecksumAlgorithm()==S3::Model::ChecksumAlgorithm::CRC32) {
+                checksum = Aws::Utils::HashingUtils::Base64Encode(Aws::Utils::Crypto::CRC32().Calculate(*partStream).GetResult());
+            } else if (GetChecksumAlgorithm()==S3::Model::ChecksumAlgorithm::CRC32C) {
+                checksum = Aws::Utils::HashingUtils::Base64Encode(Aws::Utils::Crypto::CRC32C().Calculate(*partStream).GetResult());
+            } else if (GetChecksumAlgorithm()==S3::Model::ChecksumAlgorithm::CRC64NVME) {
+                checksum = Aws::Utils::HashingUtils::Base64Encode(Aws::Utils::Crypto::CRC64().Calculate(*partStream).GetResult());
+            } else if (GetChecksumAlgorithm()==S3::Model::ChecksumAlgorithm::SHA1) {
+                checksum = Aws::Utils::HashingUtils::Base64Encode(Aws::Utils::Crypto::Sha1().Calculate(*partStream).GetResult());
+            } else if (GetChecksumAlgorithm()==S3::Model::ChecksumAlgorithm::SHA256) {
+                checksum = Aws::Utils::HashingUtils::Base64Encode(Aws::Utils::Crypto::Sha256().Calculate(*partStream).GetResult());
+            }
+            partState->SetChecksum(checksum);
+            partStream->clear();
+            partStream->seekg(0);
+        }
+
         void TransferHandle::ApplyDownloadConfiguration(const DownloadConfiguration& downloadConfig)
         {
             SetVersionId(downloadConfig.versionId);

src/aws-cpp-sdk-transfer/source/transfer/TransferManager.cpp

@@ -15,6 +15,9 @@
 #include <aws/core/utils/memory/stl/AWSStreamFwd.h>
 #include <aws/core/utils/memory/stl/AWSStringStream.h>
 #include <aws/core/utils/stream/PreallocatedStreamBuf.h>
+#include <aws/common/byte_order.h>
+#include <cstring>
+#include <aws/crt/checksum/CRC.h>
 #include <aws/s3/S3Client.h>
 #include <aws/s3/model/AbortMultipartUploadRequest.h>
 #include <aws/s3/model/CompleteMultipartUploadRequest.h>
@@ -406,12 +409,9 @@ namespace Aws
 
             const auto fullObjectHashCalculator = [](const std::shared_ptr<TransferHandle>& handle, bool isRetry, S3::Model::ChecksumAlgorithm algorithm) -> std::shared_ptr<Aws::Utils::Crypto::Hash> {
                 if (handle->GetChecksum().empty() && !isRetry) {
-                    if (algorithm == S3::Model::ChecksumAlgorithm::CRC32) {
+                    if (algorithm == S3::Model::ChecksumAlgorithm::CRC32 || algorithm == S3::Model::ChecksumAlgorithm::CRC32C) {
                         return Aws::MakeShared<Aws::Utils::Crypto::CRC32>("TransferManager");
                     }
-                    if (algorithm == S3::Model::ChecksumAlgorithm::CRC32C) {
-                        return Aws::MakeShared<Aws::Utils::Crypto::CRC32C>("TransferManager");
-                    }
                     if (algorithm == S3::Model::ChecksumAlgorithm::SHA1) {
                         return Aws::MakeShared<Aws::Utils::Crypto::Sha1>("TransferManager");
                     }
@@ -673,6 +673,10 @@ namespace Aws
                         {
                             return outcome.GetResult().GetChecksumCRC32C();
                         }
+                        else if (m_transferConfig.checksumAlgorithm == S3::Model::ChecksumAlgorithm::CRC64NVME)
+                        {
+                            return outcome.GetResult().GetChecksumCRC64NVME();
+                        }
                         else if (m_transferConfig.checksumAlgorithm == S3::Model::ChecksumAlgorithm::SHA1)
                         {
                             return outcome.GetResult().GetChecksumSHA1();
@@ -965,6 +969,7 @@ namespace Aws
                 headObjectRequest.SetCustomizedAccessLogTag(m_transferConfig.customizedAccessLogTag);
                 headObjectRequest.WithBucket(handle->GetBucketName())
                                  .WithKey(handle->GetKey());
+                headObjectRequest.SetChecksumMode(Aws::S3::Model::ChecksumMode::ENABLED);
 
                 if(!handle->GetVersionId().empty())
                 {
@@ -1000,6 +1005,18 @@ namespace Aws
                 handle->SetContentType(headObjectOutcome.GetResult().GetContentType());
                 handle->SetMetadata(headObjectOutcome.GetResult().GetMetadata());
                 handle->SetEtag(headObjectOutcome.GetResult().GetETag());
+                if (headObjectOutcome.GetResult().GetChecksumType() == Aws::S3::Model::ChecksumType::FULL_OBJECT) {
+                    if (!headObjectOutcome.GetResult().GetChecksumCRC32C().empty()) {
+                        handle->SetChecksum(headObjectOutcome.GetResult().GetChecksumCRC32C());
+                        handle->SetChecksumAlgorithm(S3::Model::ChecksumAlgorithm::CRC32C);
+                    } else if (!headObjectOutcome.GetResult().GetChecksumCRC32().empty()) {
+                        handle->SetChecksum(headObjectOutcome.GetResult().GetChecksumCRC32());
+                        handle->SetChecksumAlgorithm(S3::Model::ChecksumAlgorithm::CRC32);
+                    } else if (!headObjectOutcome.GetResult().GetChecksumCRC64NVME().empty()) {
+                        handle->SetChecksum(headObjectOutcome.GetResult().GetChecksumCRC64NVME());
+                        handle->SetChecksumAlgorithm(S3::Model::ChecksumAlgorithm::CRC64NVME);
+                    }
+                }
                 /* When bucket versioning is suspended, head object will return "null" for unversioned object.
                  * Send following GetObject with "null" as versionId will result in 403 access denied error if your IAM role or policy
                  * doesn't have GetObjectVersion permission.
@@ -1082,6 +1099,7 @@ namespace Aws
                     getObjectRangeRequest.SetRange(FormatRangeSpecifier(rangeStart, rangeEnd));
                     getObjectRangeRequest.SetResponseStreamFactory(responseStreamFunction);
                     getObjectRangeRequest.SetIfMatch(handle->GetEtag());
+                    getObjectRangeRequest.SetChecksumMode(Aws::S3::Model::ChecksumMode::ENABLED);
                     if(handle->GetVersionId().size() > 0)
                     {
                         getObjectRangeRequest.SetVersionId(handle->GetVersionId());
@@ -1204,6 +1222,19 @@ namespace Aws
 
                     Aws::String errMsg{handle->WritePartToDownloadStream(bufferStream, partState->GetRangeBegin())};
                     if (errMsg.empty()) {
+                        if (!outcome.GetResult().GetChecksumCRC32().empty()) {
+                            partState->SetChecksum(outcome.GetResult().GetChecksumCRC32());
+                        } else if (!outcome.GetResult().GetChecksumCRC32C().empty()) {
+                            partState->SetChecksum(outcome.GetResult().GetChecksumCRC32C());
+                        } else if (!outcome.GetResult().GetChecksumCRC64NVME().empty()) {
+                            partState->SetChecksum(outcome.GetResult().GetChecksumCRC64NVME());
+                        } else if (!outcome.GetResult().GetChecksumSHA1().empty()) {
+                            partState->SetChecksum(outcome.GetResult().GetChecksumSHA1());
+                        } else if (!outcome.GetResult().GetChecksumSHA256().empty()) {
+                            partState->SetChecksum(outcome.GetResult().GetChecksumSHA256());
+                        } else {
+                            if (m_transferConfig.validateChecksums) { handle->AddChecksumForPart(bufferStream, partState); }
+                        }
                         handle->ChangePartToCompleted(partState, outcome.GetResult().GetETag());
                     } else {
                         Aws::Client::AWSError<Aws::S3::S3Errors> error(Aws::S3::S3Errors::INTERNAL_FAILURE,
@@ -1239,6 +1270,73 @@ namespace Aws
             {
                 if (failedParts.size() == 0 && handle->GetBytesTransferred() == handle->GetBytesTotalSize())
                 {
+                    if (m_transferConfig.validateChecksums && !handle->GetChecksum().empty() &&
+                        (handle->GetChecksumAlgorithm() == S3::Model::ChecksumAlgorithm::CRC32 ||
+                            handle->GetChecksumAlgorithm() == S3::Model::ChecksumAlgorithm::CRC32C ||
+                            handle->GetChecksumAlgorithm() == S3::Model::ChecksumAlgorithm::CRC64NVME)) {
+                        uint64_t combinedChecksum = 0;
+                        bool first = true;
+                        for (const auto& part: handle->GetCompletedParts()) {
+                            Aws::String checksumStr = part.second->GetChecksum();
+                            uint64_t partSize = part.second->GetSizeInBytes();
+                            if (checksumStr.empty()) { continue; }
+                            auto decoded = Aws::Utils::HashingUtils::Base64Decode(checksumStr);
+                            const auto* raw = decoded.GetUnderlyingData();
+                            if (first) {
+                                if (handle->GetChecksumAlgorithm() == S3::Model::ChecksumAlgorithm::CRC64NVME) {
+                                    uint64_t partCrcBE = 0;
+                                    std::memcpy(&partCrcBE, raw, sizeof(uint64_t));
+                                    const uint64_t partCrc = aws_ntoh64(partCrcBE);
+                                    combinedChecksum = partCrc;
+                                } else {
+                                    uint32_t partCrcBE = 0;
+                                    std::memcpy(&partCrcBE, raw, sizeof(uint32_t));
+                                    const uint32_t partCrc = aws_ntoh32(partCrcBE);
+                                    combinedChecksum = partCrc;
+                                }
+                                first = false;
+                            } else {
+                                if (handle->GetChecksumAlgorithm() == S3::Model::ChecksumAlgorithm::CRC64NVME) {
+                                    uint64_t partCrcBE = 0;
+                                    std::memcpy(&partCrcBE, raw, sizeof(uint64_t));
+                                    const uint64_t partCrc = aws_ntoh64(partCrcBE);
+                                    combinedChecksum = Aws::Crt::Checksum::CombineCRC64NVME(combinedChecksum, partCrc, partSize);
+                                }
+                                else {
+                                    uint32_t partCrcBE = 0;
+                                    std::memcpy(&partCrcBE, raw, sizeof(uint32_t));
+                                    const uint32_t partCrc = aws_ntoh32(partCrcBE);
+                                    if (handle->GetChecksumAlgorithm() == S3::Model::ChecksumAlgorithm::CRC32) {
+                                        combinedChecksum = Aws::Crt::Checksum::CombineCRC32(static_cast<uint32_t>(combinedChecksum), partCrc, partSize);
+                                    } else {
+                                        combinedChecksum = Aws::Crt::Checksum::CombineCRC32C(static_cast<uint32_t>(combinedChecksum), partCrc, partSize);
+                                    }
+                                }
+                            }
+                        }
+                        Aws::Utils::ByteBuffer checksumBuffer(handle->GetChecksumAlgorithm()== S3::Model::ChecksumAlgorithm::CRC64NVME ? 8 : 4);
+                        if (handle->GetChecksumAlgorithm() == S3::Model::ChecksumAlgorithm::CRC64NVME) {
+                            const uint64_t be = aws_hton64(combinedChecksum);
+                            std::memcpy(checksumBuffer.GetUnderlyingData(), &be, sizeof(uint64_t));
+                        } else {
+                            const uint32_t be = aws_hton32(static_cast<uint32_t>(combinedChecksum));
+                            std::memcpy(checksumBuffer.GetUnderlyingData(), &be, sizeof(uint32_t));
+                        }
+                        Aws::String combinedChecksumStr = Aws::Utils::HashingUtils::Base64Encode(checksumBuffer);
+
+                        if (combinedChecksumStr != handle->GetChecksum()) {
+                            AWS_LOGSTREAM_ERROR(CLASS_TAG, "Transfer handle [" << handle->GetId()
+                                    << "] Full-object checksum mismatch. Expected: " << handle->GetChecksum()
+                                    << ", Calculated: " << combinedChecksumStr);
+                            Aws::Client::AWSError<Aws::S3::S3Errors> error(Aws::S3::S3Errors::INTERNAL_FAILURE,
+                                                                           "ChecksumMismatch",
+                                                                           "Full-object checksum validation failed",
+                                                                           false);
+                            handle->SetError(error);
+                            handle->UpdateStatus(TransferStatus::FAILED);
+                            TriggerErrorCallback(handle, error);
+                        }
+                    }
                     outcome.GetResult().GetBody().flush();
                     handle->UpdateStatus(TransferStatus::COMPLETED);
                 }