Skip to content

BP5: 32 Bit Crashes Fixed, 32bit-32bit Round-Trips #5104

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ax3l wants to merge 2 commits into
base: master
Choose a base branch
from
Open

BP5: 32 Bit Crashes Fixed, 32bit-32bit Round-Trips #5104

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9c91604
BP5: 32 Bit Does Round-Trip
ax3l Jun 27, 2026
855a9c2
Add DaosWriter
ax3l Jun 27, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion source/adios2/common/ADIOSTypes.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -263,7 +263,7 @@ constexpr bool end_step = true;
constexpr bool LocalValue = true;
constexpr bool GlobalValue = false;

constexpr size_t UnknownStep = MaxU64;
constexpr size_t UnknownStep = std::numeric_limits<size_t>::max();
constexpr double UnknownTime = std::numeric_limits<double>::infinity();

// Dims, Params, vParams are defined in ADIOSBaseTypes.h
Expand Down
3 changes: 2 additions & 1 deletion source/adios2/core/Variable.tcc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -277,7 +277,8 @@ std::pair<T, T> Variable<T>::DoMinMax(const size_t step) const
}

const bool isValue = ((blocksInfo.front().Shape.size() == 1 &&
blocksInfo.front().Shape.front() == LocalValueDim) ||
blocksInfo.front().Shape.front() ==
static_cast<size_t>(LocalValueDim)) ||
m_ShapeID == ShapeID::GlobalValue)
? true
: false;
Expand Down
22 changes: 14 additions & 8 deletions source/adios2/core/VariableBase.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -527,7 +527,7 @@ void VariableBase::InitShapeType()

if (!m_Shape.empty())
{
if (std::count(m_Shape.begin(), m_Shape.end(), JoinedDim) == 1)
if (std::count(m_Shape.begin(), m_Shape.end(), static_cast<size_t>(JoinedDim)) == 1)
{
if (!m_Start.empty() && static_cast<size_t>(std::count(m_Start.begin(), m_Start.end(),
0)) != m_Start.size())
Expand All @@ -543,7 +543,7 @@ void VariableBase::InitShapeType()
}
else if (m_Start.empty() && m_Count.empty())
{
if (m_Shape.size() == 1 && m_Shape.front() == LocalValueDim)
if (m_Shape.size() == 1 && m_Shape.front() == static_cast<size_t>(LocalValueDim))
{
m_ShapeID = ShapeID::LocalValue;
m_Start.resize(1, 0); // start[0] == 0
Expand Down Expand Up @@ -638,9 +638,12 @@ void VariableBase::CheckDimensionsCommon(const std::string hint) const
{
if (m_ShapeID != ShapeID::LocalValue)
{
if ((!m_Shape.empty() && std::count(m_Shape.begin(), m_Shape.end(), LocalValueDim) > 0) ||
(!m_Start.empty() && std::count(m_Start.begin(), m_Start.end(), LocalValueDim) > 0) ||
(!m_Count.empty() && std::count(m_Count.begin(), m_Count.end(), LocalValueDim) > 0))
if ((!m_Shape.empty() &&
std::count(m_Shape.begin(), m_Shape.end(), static_cast<size_t>(LocalValueDim)) > 0) ||
(!m_Start.empty() &&
std::count(m_Start.begin(), m_Start.end(), static_cast<size_t>(LocalValueDim)) > 0) ||
(!m_Count.empty() &&
std::count(m_Count.begin(), m_Count.end(), static_cast<size_t>(LocalValueDim)) > 0))
{
helper::Throw<std::invalid_argument>("Core", "VariableBase", "CheckDimensionsCommon",
"LocalValueDim parameter is only "
Expand All @@ -649,9 +652,12 @@ void VariableBase::CheckDimensionsCommon(const std::string hint) const
}
}

if ((!m_Shape.empty() && std::count(m_Shape.begin(), m_Shape.end(), JoinedDim) > 1) ||
(!m_Start.empty() && std::count(m_Start.begin(), m_Start.end(), JoinedDim) > 0) ||
(!m_Count.empty() && std::count(m_Count.begin(), m_Count.end(), JoinedDim) > 0))
if ((!m_Shape.empty() &&
std::count(m_Shape.begin(), m_Shape.end(), static_cast<size_t>(JoinedDim)) > 1) ||
(!m_Start.empty() &&
std::count(m_Start.begin(), m_Start.end(), static_cast<size_t>(JoinedDim)) > 0) ||
(!m_Count.empty() &&
std::count(m_Count.begin(), m_Count.end(), static_cast<size_t>(JoinedDim)) > 0))
{
helper::Throw<std::invalid_argument>("Core", "VariableBase", "CheckDimensionsCommon",
"JoinedDim is only allowed once in "
Expand Down
30 changes: 21 additions & 9 deletions source/adios2/engine/bp5/BP5Reader.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@
#include <errno.h>
#include <fstream>
#include <iostream>
#include <limits>
#include <stdexcept>
#include <mutex>
#include <thread>
#include <tuple>
Expand Down Expand Up @@ -461,7 +463,7 @@ std::string BP5Reader::UpdateWithTarInfo(const std::string &path, Params &params
}

double BP5Reader::ReadData(PoolableFile *DataFile, const size_t WriterRank, const size_t Timestep,
const size_t StartOffset, const size_t Length, char *Destination)
const uint64_t StartOffset, const size_t Length, char *Destination)
{
/*
* Warning: this function is called by multiple threads
Expand All @@ -473,31 +475,41 @@ double BP5Reader::ReadData(PoolableFile *DataFile, const size_t WriterRank, cons
as if all the flushes were in a single contiguous block in file.
*/
TP startRead = NOW();
auto lf_ReadAt = [&](const uint64_t base, const uint64_t offset) {
const uint64_t maxOffset = static_cast<uint64_t>(std::numeric_limits<size_t>::max());
if (offset > maxOffset || base > maxOffset - offset)
{
helper::Throw<std::overflow_error>(
"Engine", "BP5Reader", "ReadData",
"file offset exceeds size_t on this platform");
}
DataFile->Read(Destination, Length, static_cast<size_t>(base + offset));
};
size_t InfoStartPos = DataPosPos + (WriterRank * (2 * FlushCount + 1) * sizeof(uint64_t));
size_t SumDataSize = 0; // count in contiguous space
uint64_t SumDataSize = 0; // count in contiguous space
for (size_t flush = 0; flush < FlushCount; flush++)
{
size_t ThisDataPos = helper::ReadValue<uint64_t>(m_MetadataIndex.m_Buffer, InfoStartPos,
uint64_t ThisDataPos = helper::ReadValue<uint64_t>(m_MetadataIndex.m_Buffer, InfoStartPos,
m_Minifooter.IsLittleEndian);
size_t ThisDataSize = helper::ReadValue<uint64_t>(m_MetadataIndex.m_Buffer, InfoStartPos,
uint64_t ThisDataSize = helper::ReadValue<uint64_t>(m_MetadataIndex.m_Buffer, InfoStartPos,
m_Minifooter.IsLittleEndian);

if (StartOffset < SumDataSize + ThisDataSize)
{
// discount offsets of skipped flushes
size_t Offset = StartOffset - SumDataSize;
DataFile->Read(Destination, Length, ThisDataPos + Offset);
uint64_t Offset = StartOffset - SumDataSize;
lf_ReadAt(ThisDataPos, Offset);
TP endRead = NOW();
double timeRead = DURATION(startRead, endRead);
return timeRead;
}
SumDataSize += ThisDataSize;
}

size_t ThisDataPos = helper::ReadValue<uint64_t>(m_MetadataIndex.m_Buffer, InfoStartPos,
uint64_t ThisDataPos = helper::ReadValue<uint64_t>(m_MetadataIndex.m_Buffer, InfoStartPos,
m_Minifooter.IsLittleEndian);
size_t Offset = StartOffset - SumDataSize;
DataFile->Read(Destination, Length, ThisDataPos + Offset);
uint64_t Offset = StartOffset - SumDataSize;
lf_ReadAt(ThisDataPos, Offset);

TP endRead = NOW();
double timeRead = DURATION(startRead, endRead);
Expand Down
2 changes: 1 addition & 1 deletion source/adios2/engine/bp5/BP5Reader.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -277,7 +277,7 @@ class BP5Reader : public BP5Engine, public Engine
void InstallMetadataForTimestep(size_t Step);
void ParallelInstallMetadataForTimestep(size_t Step);
double ReadData(PoolableFile *DataFile, const size_t WriterRank, const size_t Timestep,
const size_t StartOffset, const size_t Length, char *Destination);
const uint64_t StartOffset, const size_t Length, char *Destination);

struct WriterMapStruct
{
Expand Down
4 changes: 2 additions & 2 deletions source/adios2/engine/bp5/BP5Writer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -243,8 +243,8 @@ void BP5Writer::WriteMetaMetadata(

for (auto &b : MetaMetaBlocks)
{
m_MetaMetadataFile->Write((char *)&b.MetaMetaIDLen, sizeof(size_t));
m_MetaMetadataFile->Write((char *)&b.MetaMetaInfoLen, sizeof(size_t));
m_MetaMetadataFile->Write((char *)&b.MetaMetaIDLen, sizeof(uint64_t));
m_MetaMetadataFile->Write((char *)&b.MetaMetaInfoLen, sizeof(uint64_t));
m_MetaMetadataFile->Write((char *)b.MetaMetaID, b.MetaMetaIDLen);
m_MetaMetadataFile->Write((char *)b.MetaMetaInfo, b.MetaMetaInfoLen);
}
Expand Down
11 changes: 6 additions & 5 deletions source/adios2/engine/campaign/CampaignData.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,7 @@ static int sqlcb_dataset(void *p, int argc, char **argv, char **azColName)
size_t keyid = helper::StringToSizeT(std::string(argv[15]), hint_text_to_int);
cdr.hasKey = (keyid); // keyid == 0 means there is no key used
cdr.keyIdx = size_t(keyid - 1);
cdr.size = helper::StringToSizeT(std::string(argv[16]), hint_text_to_int);
cdr.size = helper::StringTo<uint64_t>(std::string(argv[16]), hint_text_to_int);

// file
if (argv[17])
Expand Down Expand Up @@ -213,8 +213,8 @@ static int sqlcb_file(void *p, int argc, char **argv, char **azColName)
cf.name = std::string(argv[1]);
int comp = helper::StringTo<int>(std::string(argv[2]), hint_text_to_int);
cf.compressed = (bool)comp;
cf.lengthOriginal = helper::StringToSizeT(std::string(argv[3]), hint_text_to_int);
cf.lengthCompressed = helper::StringToSizeT(std::string(argv[4]), hint_text_to_int);
cf.lengthOriginal = helper::StringTo<uint64_t>(std::string(argv[3]), hint_text_to_int);
cf.lengthCompressed = helper::StringTo<uint64_t>(std::string(argv[4]), hint_text_to_int);
cf.modtime =
helper::StringTo<int64_t>(std::string(argv[5]), "SQL callback convert modtime to int");
cf.checksum = argv[6];
Expand Down Expand Up @@ -703,7 +703,8 @@ void CampaignData::DumpToFileOrMemory(const size_t fileIdx, std::string &keyHex,
// Verify file size matches expected size to detect truncated cache files
// (e.g. from a process killed mid-write)
struct stat st;
if (stat(path.c_str(), &st) == 0 && static_cast<size_t>(st.st_size) == file.lengthOriginal)
if (stat(path.c_str(), &st) == 0 &&
static_cast<uint64_t>(st.st_size) == file.lengthOriginal)
{
return;
}
Expand Down Expand Up @@ -751,7 +752,7 @@ void CampaignData::DumpToFileOrMemory(const size_t fileIdx, std::string &keyHex,
#ifdef ADIOS2_HAVE_SODIUM
if (!keyHex.empty())
{
size_t decryptedSize = (file.compressed ? file.lengthCompressed : file.lengthOriginal);
uint64_t decryptedSize = (file.compressed ? file.lengthCompressed : file.lengthOriginal);
q = malloc(decryptedSize);
free_q = true;
DecryptData(static_cast<const unsigned char *>(blob), blobsize, decryptedSize, file, keyHex,
Expand Down
6 changes: 3 additions & 3 deletions source/adios2/engine/campaign/CampaignData.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,8 @@ struct CampaignFile
// size_t datasetIdx; // index of grandparent CampaignDataset in the map
// std::vector<size_t> replicaIdxs; // CampaignReplicas using this file in CampaignDataset
bool compressed;
size_t lengthOriginal;
size_t lengthCompressed;
uint64_t lengthOriginal;
uint64_t lengthCompressed;
int64_t modtime;
std::string checksum; // SHA1 checksum of file
};
Expand Down Expand Up @@ -101,7 +101,7 @@ struct CampaignReplica
bool deleted;
bool hasKey;
size_t keyIdx;
size_t size; // replica size on remote location
uint64_t size; // replica size on remote location (may exceed 4 GB on 32-bit)
std::vector<size_t> files; // file indices in CampaignDataset, ordered by fileid
// image replicas have resolution information (ds.format == FileFormat::IMAGE)
size_t x;
Expand Down
3 changes: 2 additions & 1 deletion source/adios2/engine/campaign/CampaignReader.tcc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -262,7 +262,8 @@ CampaignReader::BlocksInfoCommon(const core::Variable<std::string> &variable,
blockInfo.MinMaxs = blockCharacteristics.Statistics.MinMaxs;
blockInfo.SubBlockInfo = blockCharacteristics.Statistics.SubBlockInfo;
}
if (blockInfo.Shape.size() == 1 && blockInfo.Shape.front() == LocalValueDim)
if (blockInfo.Shape.size() == 1 &&
blockInfo.Shape.front() == static_cast<size_t>(LocalValueDim))
{
blockInfo.Shape = Dims{blocksIndexOffsets.size()};
blockInfo.Count = Dims{1};
Expand Down
4 changes: 2 additions & 2 deletions source/adios2/engine/daos/DaosWriter.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,8 +139,8 @@ void DaosWriter::WriteMetaMetadata(
profiling::ProfilerGuard g(m_Profiler, "MD_Posix");
for (auto &b : MetaMetaBlocks)
{
m_MetaMetadataFile->Write((char *)&b.MetaMetaIDLen, sizeof(size_t));
m_MetaMetadataFile->Write((char *)&b.MetaMetaInfoLen, sizeof(size_t));
m_MetaMetadataFile->Write((char *)&b.MetaMetaIDLen, sizeof(uint64_t));
m_MetaMetadataFile->Write((char *)&b.MetaMetaInfoLen, sizeof(uint64_t));
m_MetaMetadataFile->Write((char *)b.MetaMetaID, b.MetaMetaIDLen);
m_MetaMetadataFile->Write((char *)b.MetaMetaInfo, b.MetaMetaInfoLen);
}
Expand Down
2 changes: 1 addition & 1 deletion source/adios2/toolkit/format/bp5/BP5Base.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ int BP5Base::BP5BitfieldTest(struct BP5MetadataInfoStruct *MBase, int Bit) const
{"Count", "integer[DBCount]", sizeof(size_t), FMOffset(BP5Base::MetaArrayRec *, Count)}, \
{"Offset", "integer[DBCount]", sizeof(size_t), \
FMOffset(BP5Base::MetaArrayRec *, Offsets)}, \
{"DataBlockLocation", "integer[BlockCount]", sizeof(size_t), \
{"DataBlockLocation", "integer[BlockCount]", sizeof(uint64_t), \
FMOffset(BP5Base::MetaArrayRec *, DataBlockLocation)},

static FMField MetaArrayRecList[] = {BASE_FIELD_ENTRIES{NULL, NULL, 0, 0}};
Expand Down
8 changes: 5 additions & 3 deletions source/adios2/toolkit/format/bp5/BP5Base.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
#ifndef ADIOS2_TOOLKIT_FORMAT_BP5_BP5BASE_H_
#define ADIOS2_TOOLKIT_FORMAT_BP5_BP5BASE_H_

#include <stdint.h>

#include "adios2/core/Attribute.h"
#include "adios2/core/IO.h"
#include "adios2/toolkit/format/buffer/BufferV.h"
Expand All @@ -31,9 +33,9 @@ class BP5Base
struct MetaMetaInfoBlock
{
char *MetaMetaInfo;
size_t MetaMetaInfoLen;
uint64_t MetaMetaInfoLen;
char *MetaMetaID;
size_t MetaMetaIDLen;
uint64_t MetaMetaIDLen;
};

#define BASE_FIELDS \
Expand All @@ -43,7 +45,7 @@ class BP5Base
size_t *Shape; /* Global dimensionality [Dims] NULL for local */ \
size_t *Count; /* Per-block Counts [DBCount] */ \
size_t *Offsets; /* Per-block Offsets [DBCount] NULL for local */ \
size_t *DataBlockLocation; /* Per-block Offset in PG [BlockCount] */
uint64_t *DataBlockLocation; /* Per-block Offset in PG [BlockCount] */

typedef struct _MetaArrayRec
{
Expand Down
12 changes: 6 additions & 6 deletions source/adios2/toolkit/format/bp5/BP5Deserializer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -828,7 +828,7 @@ void BP5Deserializer::InstallMetadataBuffer(void *BaseData, size_t WriterRank, s
{
for (size_t i = 0; i < meta_base->Dims; i++)
{
if (meta_base->Shape[i] == JoinedDim)
if (meta_base->Shape[i] == static_cast<size_t>(JoinedDim))
{
VarRec->JoinedDimen = i;
}
Expand Down Expand Up @@ -1856,7 +1856,7 @@ BP5Deserializer::GenerateReadRequests(BP5GetContext &ctx, const bool doAllocTemp
RR.Timestep = Req->Step;
RR.WriterRank = WriterRank;
RR.StartOffset = writer_meta_base->DataBlockLocation[NeededBlock];
if (RR.StartOffset == (size_t)-1)
if (RR.StartOffset == static_cast<uint64_t>(-1))
throw std::runtime_error("No data exists for this variable");
if (Req->MemSpace != MemorySpace::Host)
RR.DirectToAppMemory = false;
Expand Down Expand Up @@ -2037,7 +2037,7 @@ BP5Deserializer::GenerateReadRequests(BP5GetContext &ctx, const bool doAllocTemp
1);

if (writer_meta_base_input->DataBlockLocation[Block] ==
(size_t)-1)
static_cast<uint64_t>(-1))
helper::Throw<std::runtime_error>(
"Toolkit", "BP5Deserializer",
"GenerateReadRequests",
Expand Down Expand Up @@ -2149,7 +2149,7 @@ BP5Deserializer::GenerateReadRequests(BP5GetContext &ctx, const bool doAllocTemp
1);

if (writer_meta_base_input->DataBlockLocation[Block] ==
(size_t)-1)
static_cast<uint64_t>(-1))
helper::Throw<std::runtime_error>(
"Toolkit", "BP5Deserializer",
"GenerateReadRequests",
Expand Down Expand Up @@ -2195,7 +2195,7 @@ BP5Deserializer::GenerateReadRequests(BP5GetContext &ctx, const bool doAllocTemp
RR.StartOffset = writer_meta_base->DataBlockLocation[Block];
RR.ReadLength = writer_meta_base->DataBlockSize[Block];
RR.DestinationAddr = nullptr;
if (RR.StartOffset == (size_t)-1)
if (RR.StartOffset == static_cast<uint64_t>(-1))
throw std::runtime_error(
"No data exists for this variable");
if (doAllocTempBuffers)
Expand Down Expand Up @@ -2238,7 +2238,7 @@ BP5Deserializer::GenerateReadRequests(BP5GetContext &ctx, const bool doAllocTemp
RR.WriterRank = WriterRank;
RR.StartOffset = writer_meta_base->DataBlockLocation[Block] +
StartOffsetInBlock;
if (writer_meta_base->DataBlockLocation[Block] == (size_t)-1)
if (writer_meta_base->DataBlockLocation[Block] == static_cast<uint64_t>(-1))
throw std::runtime_error(
"No data exists for this variable");
RR.ReadLength = EndOffsetInBlock - StartOffsetInBlock;
Expand Down
2 changes: 1 addition & 1 deletion source/adios2/toolkit/format/bp5/BP5Deserializer.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ class BP5Deserializer : virtual public BP5Base
{
size_t Timestep;
size_t WriterRank;
size_t StartOffset;
uint64_t StartOffset;
size_t ReadLength;
char *DestinationAddr;
bool DirectToAppMemory;
Expand Down
Loading
Loading