Skip to content

Commit 93cd00c

Browse files
pulimsrpulimsr
committed
removing checksum check on single part download, and updating the hashmap of multipart download part checksums and updating the check lock for full body checksum
1 parent d688c88 commit 93cd00c

File tree

2 files changed

+63
-133
lines changed

2 files changed

+63
-133
lines changed

‎src/aws-cpp-sdk-transfer/include/aws/transfer/TransferHandle.h‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -393,12 +393,14 @@ namespace Aws
393393
Aws::String GetChecksum() const { return m_checksum; }
394394
void SetChecksum(const Aws::String& checksum) { this->m_checksum = checksum; }
395395

396-
void SetPartChecksum(int partId, std::shared_ptr<Aws::Utils::Crypto::Hash> hash) { m_partChecksums[partId] = hash; }
397-
std::shared_ptr<Aws::Utils::Crypto::Hash> GetPartChecksum(int partId) const {
396+
void SetPartChecksum(int partId, const Aws::String& checksum, uint64_t size) {
397+
m_partChecksums[partId] = std::make_pair(checksum, size);
398+
}
399+
std::pair<Aws::String, uint64_t> GetPartChecksum(int partId) const {
398400
auto it = m_partChecksums.find(partId);
399-
return it != m_partChecksums.end() ? it->second : nullptr;
401+
return it != m_partChecksums.end() ? it->second : std::make_pair("", 0);
400402
}
401-
const Aws::Map<int, std::shared_ptr<Aws::Utils::Crypto::Hash>>& GetPartChecksums() const { return m_partChecksums; }
403+
const Aws::Map<int, std::pair<Aws::String, uint64_t>>& GetPartChecksums() const { return m_partChecksums; }
402404

403405
private:
404406
void CleanupDownloadStream();
@@ -442,7 +444,7 @@ namespace Aws
442444
mutable std::mutex m_getterSetterLock;
443445
Aws::String m_checksum;
444446
// Map of part number to Hash instance for multipart download checksum validation
445-
Aws::Map<int, std::shared_ptr<Aws::Utils::Crypto::Hash>> m_partChecksums;
447+
Aws::Map<int, std::pair<Aws::String, uint64_t>> m_partChecksums;
446448
};
447449

448450
AWS_TRANSFER_API Aws::OStream& operator << (Aws::OStream& s, TransferStatus status);

‎src/aws-cpp-sdk-transfer/source/transfer/TransferManager.cpp‎

Lines changed: 56 additions & 128 deletions
Original file line numberDiff line numberDiff line change
@@ -52,21 +52,7 @@ namespace Aws
5252
}
5353
}
5454

55-
static std::shared_ptr<Utils::Crypto::Hash> CreateHashForAlgorithm(S3::Model::ChecksumAlgorithm algorithm) {
56-
if (algorithm == S3::Model::ChecksumAlgorithm::CRC32) {
57-
return Aws::MakeShared<Utils::Crypto::CRC32>(CLASS_TAG);
58-
}
59-
if (algorithm == S3::Model::ChecksumAlgorithm::CRC32C) {
60-
return Aws::MakeShared<Utils::Crypto::CRC32C>(CLASS_TAG);
61-
}
62-
if (algorithm == S3::Model::ChecksumAlgorithm::SHA1) {
63-
return Aws::MakeShared<Utils::Crypto::Sha1>(CLASS_TAG);
64-
}
65-
if (algorithm == S3::Model::ChecksumAlgorithm::SHA256) {
66-
return Aws::MakeShared<Utils::Crypto::Sha256>(CLASS_TAG);
67-
}
68-
return Aws::MakeShared<Utils::Crypto::CRC64>(CLASS_TAG);
69-
}
55+
7056

7157
template <typename ResultT>
7258
static Aws::String GetChecksumFromResult(const ResultT& result, S3::Model::ChecksumAlgorithm algorithm) {
@@ -957,60 +943,6 @@ namespace Aws
957943
handle->ChangePartToCompleted(partState, getObjectOutcome.GetResult().GetETag());
958944
getObjectOutcome.GetResult().GetBody().flush();
959945

960-
// Validate checksum for single-part download by reading file
961-
if (m_transferConfig.validateChecksums)
962-
{
963-
Aws::String expectedChecksum = GetChecksumFromResult(getObjectOutcome.GetResult(), m_transferConfig.checksumAlgorithm);
964-
965-
if (!expectedChecksum.empty() && !handle->GetTargetFilePath().empty())
966-
{
967-
auto hash = CreateHashForAlgorithm(m_transferConfig.checksumAlgorithm);
968-
Aws::IFStream fileStream(handle->GetTargetFilePath().c_str(), std::ios::binary);
969-
970-
if (fileStream.good())
971-
{
972-
const size_t bufferSize = 8192;
973-
char buffer[bufferSize];
974-
while (fileStream.good())
975-
{
976-
fileStream.read(buffer, bufferSize);
977-
std::streamsize bytesRead = fileStream.gcount();
978-
if (bytesRead > 0)
979-
{
980-
hash->Update(reinterpret_cast<unsigned char*>(buffer), static_cast<size_t>(bytesRead));
981-
}
982-
}
983-
fileStream.close();
984-
985-
auto calculatedResult = hash->GetHash();
986-
if (calculatedResult.IsSuccess())
987-
{
988-
Aws::String calculatedChecksum = Utils::HashingUtils::Base64Encode(calculatedResult.GetResult());
989-
if (calculatedChecksum != expectedChecksum)
990-
{
991-
AWS_LOGSTREAM_ERROR(CLASS_TAG, "Transfer handle [" << handle->GetId()
992-
<< "] Checksum mismatch for single-part download. Expected: "
993-
<< expectedChecksum << ", Calculated: " << calculatedChecksum);
994-
995-
// Delete the corrupted file
996-
Aws::FileSystem::RemoveFileIfExists(handle->GetTargetFilePath().c_str());
997-
998-
handle->ChangePartToFailed(partState);
999-
handle->UpdateStatus(TransferStatus::FAILED);
1000-
Aws::Client::AWSError<Aws::S3::S3Errors> error(Aws::S3::S3Errors::INTERNAL_FAILURE,
1001-
"ChecksumMismatch",
1002-
"Single-part download checksum validation failed",
1003-
false);
1004-
handle->SetError(error);
1005-
TriggerErrorCallback(handle, error);
1006-
TriggerTransferStatusUpdatedCallback(handle);
1007-
return;
1008-
}
1009-
}
1010-
}
1011-
}
1012-
}
1013-
1014946
handle->UpdateStatus(TransferStatus::COMPLETED);
1015947
}
1016948
else
@@ -1150,7 +1082,7 @@ namespace Aws
11501082
// Initialize checksum Hash for this part if validation is enabled
11511083
if (m_transferConfig.validateChecksums)
11521084
{
1153-
handle->SetPartChecksum(partState->GetPartId(), CreateHashForAlgorithm(m_transferConfig.checksumAlgorithm));
1085+
handle->SetPartChecksum(partState->GetPartId(), partState->GetChecksum(), partState->GetSizeInBytes());
11541086
}
11551087

11561088
auto getObjectRangeRequest = m_transferConfig.getObjectTemplate;
@@ -1318,67 +1250,63 @@ namespace Aws
13181250
{
13191251
if (failedParts.size() == 0 && handle->GetBytesTransferred() == handle->GetBytesTotalSize())
13201252
{
1321-
// Combine part checksums and validate full-object checksum
1322-
if (m_transferConfig.validateChecksums)
1323-
{
1324-
Aws::String expectedChecksum = GetChecksumFromResult(outcome.GetResult(), m_transferConfig.checksumAlgorithm);
1325-
if (!expectedChecksum.empty())
1326-
{
1327-
auto combinedChecksum = 0ULL;
1328-
bool isCRC64 = (m_transferConfig.checksumAlgorithm == S3::Model::ChecksumAlgorithm::CRC64NVME);
1329-
1330-
for (auto& partChecksum : handle->GetPartChecksums())
1331-
{
1332-
int partNumber = partChecksum.first;
1333-
auto hash = partChecksum.second;
1334-
1335-
// Get part size from completed parts
1336-
auto partSize = handle->GetCompletedParts()[partNumber]->GetSizeInBytes();
1337-
1338-
auto partResult = hash->GetHash();
1339-
auto partData = partResult.GetResult();
1340-
1341-
if (combinedChecksum == 0) {
1342-
if (isCRC64) {
1343-
combinedChecksum = *reinterpret_cast<const unsigned long long*>(partData.GetUnderlyingData());
1344-
} else {
1345-
combinedChecksum = *reinterpret_cast<const unsigned int*>(partData.GetUnderlyingData());
1253+
if (m_transferConfig.validateChecksums) {
1254+
auto checksumType = outcome.GetResult().GetChecksumType();
1255+
if (checksumType == S3::Model::ChecksumType::FULL_OBJECT) {
1256+
Aws::String expectedChecksum = GetChecksumFromResult(outcome.GetResult(),m_transferConfig.checksumAlgorithm);
1257+
if (!expectedChecksum.empty()) {
1258+
auto combinedChecksum = 0ULL;
1259+
bool isCRC64 = (m_transferConfig.checksumAlgorithm == S3::Model::ChecksumAlgorithm::CRC64NVME);
1260+
1261+
for (auto& partChecksum : handle->GetPartChecksums()) {
1262+
Aws::String checksumStr = partChecksum.second.first;
1263+
uint64_t partSize = partChecksum.second.second;
1264+
1265+
auto decoded = Aws::Utils::HashingUtils::Base64Decode(checksumStr);
1266+
1267+
if (combinedChecksum == 0) {
1268+
if (isCRC64) {
1269+
combinedChecksum = *reinterpret_cast<const uint64_t*>(decoded.GetUnderlyingData());
1270+
}
1271+
else {
1272+
combinedChecksum = *reinterpret_cast<const uint32_t*>(decoded.GetUnderlyingData());
1273+
}
13461274
}
1347-
} else {
1348-
if (m_transferConfig.checksumAlgorithm == S3::Model::ChecksumAlgorithm::CRC32) {
1349-
auto partCrc = *reinterpret_cast<const unsigned int*>(partData.GetUnderlyingData());
1350-
combinedChecksum = Aws::Crt::Checksum::CombineCRC32(static_cast<uint32_t>(combinedChecksum), partCrc, partSize);
1351-
} else if (m_transferConfig.checksumAlgorithm == S3::Model::ChecksumAlgorithm::CRC32C) {
1352-
auto partCrc = *reinterpret_cast<const unsigned int*>(partData.GetUnderlyingData());
1353-
combinedChecksum = Aws::Crt::Checksum::CombineCRC32C(static_cast<uint32_t>(combinedChecksum), partCrc, partSize);
1354-
} else if (isCRC64) {
1355-
auto partCrc = *reinterpret_cast<const unsigned long long*>(partData.GetUnderlyingData());
1356-
combinedChecksum = Aws::Crt::Checksum::CombineCRC64NVME(combinedChecksum, partCrc, partSize);
1275+
else {
1276+
if (m_transferConfig.checksumAlgorithm == S3::Model::ChecksumAlgorithm::CRC32) {
1277+
auto partCrc = *reinterpret_cast<const uint32_t*>(decoded.GetUnderlyingData());
1278+
combinedChecksum = Aws::Crt::Checksum::CombineCRC32(static_cast<uint32_t>(combinedChecksum), partCrc, partSize);
1279+
} else if (m_transferConfig.checksumAlgorithm == S3::Model::ChecksumAlgorithm::CRC32C){
1280+
auto partCrc = *reinterpret_cast<const uint32_t*>(decoded.GetUnderlyingData());
1281+
combinedChecksum = Aws::Crt::Checksum::CombineCRC32C(static_cast<uint32_t>(combinedChecksum), partCrc, partSize);
1282+
} else if (isCRC64) {
1283+
auto partCrc = *reinterpret_cast<const uint64_t*>(decoded.GetUnderlyingData());
1284+
combinedChecksum = Aws::Crt::Checksum::CombineCRC64NVME(combinedChecksum, partCrc, partSize);
1285+
}
13571286
}
13581287
}
1359-
}
1360-
1361-
// Compare with expected checksum
1362-
Aws::Utils::ByteBuffer checksumBuffer(isCRC64 ? 8 : 4);
1363-
if (isCRC64) {
1364-
*reinterpret_cast<unsigned long long*>(checksumBuffer.GetUnderlyingData()) = combinedChecksum;
1365-
} else {
1366-
*reinterpret_cast<unsigned int*>(checksumBuffer.GetUnderlyingData()) = static_cast<unsigned int>(combinedChecksum);
1367-
}
1368-
Aws::String calculatedChecksum = Utils::HashingUtils::Base64Encode(checksumBuffer);
1369-
1370-
if (calculatedChecksum != expectedChecksum) {
1371-
AWS_LOGSTREAM_ERROR(CLASS_TAG, "Transfer handle [" << handle->GetId()
1372-
<< "] Full-object checksum mismatch. Expected: " << expectedChecksum
1373-
<< ", Calculated: " << calculatedChecksum);
1374-
Aws::Client::AWSError<Aws::S3::S3Errors> error(Aws::S3::S3Errors::INTERNAL_FAILURE,
1375-
"ChecksumMismatch",
1376-
"Full-object checksum validation failed",
1377-
false);
1378-
handle->SetError(error);
1379-
handle->UpdateStatus(TransferStatus::FAILED);
1380-
TriggerErrorCallback(handle, error);
1381-
return;
1288+
1289+
Aws::Utils::ByteBuffer checksumBuffer(isCRC64 ? 8 : 4);
1290+
if (isCRC64) {
1291+
*reinterpret_cast<uint64_t*>(checksumBuffer.GetUnderlyingData()) = combinedChecksum;
1292+
} else {
1293+
*reinterpret_cast<uint32_t*>(checksumBuffer.GetUnderlyingData()) = static_cast<uint32_t>(combinedChecksum);
1294+
}
1295+
Aws::String calculatedChecksum = Utils::HashingUtils::Base64Encode(checksumBuffer);
1296+
1297+
if (calculatedChecksum != expectedChecksum) {
1298+
AWS_LOGSTREAM_ERROR(CLASS_TAG, "Transfer handle [" << handle->GetId()
1299+
<< "] Full-object checksum mismatch. Expected: " << expectedChecksum
1300+
<< ", Calculated: " << calculatedChecksum);
1301+
Aws::Client::AWSError<Aws::S3::S3Errors> error(Aws::S3::S3Errors::INTERNAL_FAILURE,
1302+
"ChecksumMismatch",
1303+
"Full-object checksum validation failed",
1304+
false);
1305+
handle->SetError(error);
1306+
handle->UpdateStatus(TransferStatus::FAILED);
1307+
TriggerErrorCallback(handle, error);
1308+
return;
1309+
}
13821310
}
13831311
}
13841312
}

0 commit comments

Comments
 (0)