Skip to content

Commit f99b52e

Browse files
committed
BP5Writer: singularize m_DataSubstream and substream names
AggTransportData's m_DataSubstreams (vector), m_SubStreamNames (vector), and m_DrainSubStreamNames (vector) become singular shared_ptr<Transport> and string members. The vectors were size-1 by construction (GetBPSubStreamNames is called with a single base name, and the user multi-transport feature is rejected at Init). Drops the for-each loops over m_DataSubstreams in the write path and across the async writer, where the size-1 invariant made the loop vestigial. AsyncWriteInfo carries a shared_ptr<Transport>* instead of a vector*. CloseFiles and FlushProfiler iterations now null-check m_DataSubstream: an AggTransportData entry may exist with no open transport when the rank hasn't written through that aggregator.
1 parent efb56c3 commit f99b52e

6 files changed

Lines changed: 50 additions & 96 deletions

File tree

source/adios2/engine/bp5/BP5Writer.cpp

Lines changed: 26 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -483,11 +483,8 @@ void BP5Writer::WriteData(format::BufferV *Data)
483483
"is not supported in BP5");
484484
}
485485
AggTransportData &aggData = m_AggregatorSpecifics.at(GetCacheKey(m_Aggregator));
486-
for (auto &t : aggData.m_DataSubstreams)
487-
{
488-
t->Flush();
489-
t->FinalizeSegment();
490-
}
486+
aggData.m_DataSubstream->Flush();
487+
aggData.m_DataSubstream->FinalizeSegment();
491488
delete Data;
492489
}
493490
}
@@ -534,10 +531,8 @@ void BP5Writer::WriteData_EveryoneWrites(format::BufferV *Data, bool SerializedW
534531
m_DataPos += Data->Size();
535532
std::vector<core::iovec> DataVec = Data->DataVec();
536533
AggTransportData &aggData = m_AggregatorSpecifics.at(GetCacheKey(m_Aggregator));
537-
for (auto &t : aggData.m_DataSubstreams)
538-
{
539-
t->WriteV(DataVec.data(), static_cast<int>(DataVec.size()), m_StartDataPos);
540-
}
534+
aggData.m_DataSubstream->WriteV(DataVec.data(), static_cast<int>(DataVec.size()),
535+
m_StartDataPos);
541536

542537
if (SerializedWriters && a->m_Comm.Rank() < a->m_Comm.Size() - 1)
543538
{
@@ -1917,8 +1912,8 @@ void BP5Writer::InitTransports()
19171912
{
19181913
const std::vector<std::string> drainTransportNames =
19191914
transportman::TransportMan::GetFilesBaseNames(m_Name, m_DataTransportsParameters);
1920-
aggData.m_DrainSubStreamNames =
1921-
GetBPSubStreamNames(drainTransportNames, m_Aggregator->m_SubStreamIndex);
1915+
aggData.m_DrainSubStreamName =
1916+
GetBPSubStreamName(drainTransportNames[0], m_Aggregator->m_SubStreamIndex);
19221917
/* start up BB thread */
19231918
// m_FileDrainer.SetVerbose(
19241919
// m_Parameters.BurstBufferVerbose,
@@ -1931,18 +1926,15 @@ void BP5Writer::InitTransports()
19311926
if (m_DrainBB)
19321927
{
19331928
/* Create the directories on target anyway by main thread */
1934-
transport::MkDirsBarrier(m_Comm, aggData.m_DrainSubStreamNames, m_DataTransportsParameters,
1929+
transport::MkDirsBarrier(m_Comm, {aggData.m_DrainSubStreamName}, m_DataTransportsParameters,
19351930
m_Parameters.NodeLocal);
19361931
}
19371932

19381933
if (m_IAmDraining)
19391934
{
19401935
if (m_DrainBB)
19411936
{
1942-
for (const auto &name : aggData.m_DrainSubStreamNames)
1943-
{
1944-
m_FileDrainer.AddOperationOpen(name, m_OpenMode);
1945-
}
1937+
m_FileDrainer.AddOperationOpen(aggData.m_DrainSubStreamName, m_OpenMode);
19461938
}
19471939
}
19481940

@@ -1978,7 +1970,7 @@ void BP5Writer::OpenSubfile(bool useComm, bool forceAppend)
19781970
AggTransportData &aggData = m_AggregatorSpecifics.at(cacheKey);
19791971

19801972
// /path/name.bp.dir/name.bp.rank
1981-
aggData.m_SubStreamNames = GetBPSubStreamNames({m_Name}, m_Aggregator->m_SubStreamIndex);
1973+
aggData.m_SubStreamName = GetBPSubStreamName(m_Name, m_Aggregator->m_SubStreamIndex);
19821974

19831975
helper::Comm openSyncComm;
19841976

@@ -2016,12 +2008,9 @@ void BP5Writer::OpenSubfile(bool useComm, bool forceAppend)
20162008
std::cout << "Rank " << m_Comm.Rank() << " opening data file with Comm"
20172009
<< std::endl;
20182010
}
2019-
for (const auto &name : aggData.m_SubStreamNames)
2020-
{
2021-
aggData.m_DataSubstreams.push_back(transport::OpenFileChained(
2022-
m_Comm, name, mode, m_IO.m_TransportsParameters[0], true,
2023-
*DataWritingComm));
2024-
}
2011+
aggData.m_DataSubstream = transport::OpenFileChained(
2012+
m_Comm, aggData.m_SubStreamName, mode, m_IO.m_TransportsParameters[0], true,
2013+
*DataWritingComm);
20252014
}
20262015
else
20272016
{
@@ -2030,11 +2019,8 @@ void BP5Writer::OpenSubfile(bool useComm, bool forceAppend)
20302019
std::cout << "Rank " << m_Comm.Rank() << " opening data file no Comm"
20312020
<< std::endl;
20322021
}
2033-
for (const auto &name : aggData.m_SubStreamNames)
2034-
{
2035-
aggData.m_DataSubstreams.push_back(transport::OpenFile(
2036-
m_Comm, name, mode, m_IO.m_TransportsParameters[0], true));
2037-
}
2022+
aggData.m_DataSubstream = transport::OpenFile(m_Comm, aggData.m_SubStreamName, mode,
2023+
m_IO.m_TransportsParameters[0], true);
20382024
}
20392025
}
20402026
}
@@ -2211,19 +2197,16 @@ void BP5Writer::InitBPBuffer()
22112197
const size_t off = m_AppendDataPos[m_Aggregator->m_SubStreamIndex];
22122198
if (off < MaxSizeT)
22132199
{
2214-
for (auto &t : aggData.m_DataSubstreams)
2215-
{
2216-
t->Truncate(off);
2217-
// Seek is needed since truncate does not seek.
2218-
// Seek instead of SeekToEnd in case a transport
2219-
// does not support actual truncate.
2220-
t->Seek(off);
2221-
}
2200+
aggData.m_DataSubstream->Truncate(off);
2201+
// Seek is needed since truncate does not seek.
2202+
// Seek instead of SeekToEnd in case a transport
2203+
// does not support actual truncate.
2204+
aggData.m_DataSubstream->Seek(off);
22222205
m_DataPos = off;
22232206
}
22242207
else
22252208
{
2226-
m_DataPos = aggData.m_DataSubstreams[0]->GetSize();
2209+
m_DataPos = aggData.m_DataSubstream->GetSize();
22272210
}
22282211
}
22292212

@@ -2287,10 +2270,7 @@ void BP5Writer::InitBPBuffer()
22872270
// data existed but index was missing
22882271
if (m_Aggregator->m_IsAggregator)
22892272
{
2290-
for (auto &t : aggData.m_DataSubstreams)
2291-
{
2292-
t->Seek(0);
2293-
}
2273+
aggData.m_DataSubstream->Seek(0);
22942274
}
22952275
}
22962276

@@ -2446,9 +2426,9 @@ void BP5Writer::DoClose(const int transportIndex)
24462426
(void)transportIndex;
24472427
for (auto &aggPair : m_AggregatorSpecifics)
24482428
{
2449-
for (auto &t : aggPair.second.m_DataSubstreams)
2429+
if (aggPair.second.m_DataSubstream)
24502430
{
2451-
t->Close();
2431+
aggPair.second.m_DataSubstream->Close();
24522432
}
24532433
}
24542434

@@ -2496,7 +2476,7 @@ void BP5Writer::DoClose(const int transportIndex)
24962476
// cleaned up while those resources are still available.
24972477
for (auto &aggPair : m_AggregatorSpecifics)
24982478
{
2499-
aggPair.second.m_DataSubstreams.clear();
2479+
aggPair.second.m_DataSubstream.reset();
25002480
}
25012481
m_MetadataFile.reset();
25022482
m_MetaMetadataFile.reset();
@@ -2511,8 +2491,9 @@ void BP5Writer::FlushProfiler()
25112491

25122492
for (auto &specs : m_AggregatorSpecifics)
25132493
{
2514-
for (auto &t : specs.second.m_DataSubstreams)
2494+
if (specs.second.m_DataSubstream)
25152495
{
2496+
auto &t = specs.second.m_DataSubstream;
25162497
transportTypes.push_back(t->m_Type);
25172498
transportNames.push_back(t->m_Name);
25182499
transportProfilers.push_back(&t->m_Profiler);

source/adios2/engine/bp5/BP5Writer.h

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -56,16 +56,17 @@ class BP5Writer : public BP5Engine, public core::Engine
5656
private:
5757
struct AggTransportData
5858
{
59-
/** Open data-file transports for this aggregator's substreams. One
60-
* entry per name in m_SubStreamNames. */
61-
std::vector<std::shared_ptr<Transport>> m_DataSubstreams;
59+
/** Open data-file transport for this aggregator's substream. Null
60+
* if not yet opened (no write done by this rank in this aggregator
61+
* context). */
62+
std::shared_ptr<Transport> m_DataSubstream;
6263

63-
/** Name of subfiles to directly write to. Either original target or
64-
* burst buffer if used. */
65-
std::vector<std::string> m_SubStreamNames;
64+
/** Name of the subfile to directly write to. Either original target
65+
* or burst buffer if used. */
66+
std::string m_SubStreamName;
6667

67-
/** Name of subfiles on target if burst buffer is used. */
68-
std::vector<std::string> m_DrainSubStreamNames;
68+
/** Name of the subfile on target if burst buffer is used. */
69+
std::string m_DrainSubStreamName;
6970
};
7071

7172
std::string GetCacheKey(aggregator::MPIAggregator *aggregator);
@@ -359,7 +360,7 @@ class BP5Writer : public BP5Engine, public core::Engine
359360
int nproc_chain;
360361
TimePoint tstart;
361362
adios2::shm::TokenChain<uint64_t> *tokenChain;
362-
std::vector<std::shared_ptr<Transport>> *DataSubstreams;
363+
std::shared_ptr<Transport> *DataSubstream;
363364
adios2::format::BufferV *Data;
364365
uint64_t startPos;
365366
uint64_t totalSize;

source/adios2/engine/bp5/BP5Writer_EveryoneWrites_Async.cpp

Lines changed: 5 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -141,10 +141,7 @@ void BP5Writer::AsyncWriteOwnData(AsyncWriteInfo *info, std::vector<core::iovec>
141141
<< " write the rest of " << totalsize - wrote
142142
<< " bytes at pos " << pos << std::endl;*/
143143

144-
for (auto &t : *info->DataSubstreams)
145-
{
146-
t->WriteV(vec.data(), static_cast<int>(vec.size()), pos);
147-
}
144+
(*info->DataSubstream)->WriteV(vec.data(), static_cast<int>(vec.size()), pos);
148145

149146
break; /* Exit loop after this final write */
150147
}
@@ -167,18 +164,12 @@ void BP5Writer::AsyncWriteOwnData(AsyncWriteInfo *info, std::vector<core::iovec>
167164
const char *buf = (const char *)DataVec[block].iov_base + temp_offset;
168165
if (firstWrite)
169166
{
170-
for (auto &t : *info->DataSubstreams)
171-
{
172-
t->Write(buf, n, info->startPos);
173-
}
167+
(*info->DataSubstream)->Write(buf, n, info->startPos);
174168
firstWrite = false;
175169
}
176170
else
177171
{
178-
for (auto &t : *info->DataSubstreams)
179-
{
180-
t->Write(buf, n);
181-
}
172+
(*info->DataSubstream)->Write(buf, n);
182173
}
183174

184175
/* Have we processed the entire block or staying with it? */
@@ -218,10 +209,7 @@ int BP5Writer::AsyncWriteThread_EveryoneWrites(AsyncWriteInfo *info)
218209
info->tokenChain->RecvToken();
219210
}
220211
}
221-
for (auto &t : *info->DataSubstreams)
222-
{
223-
t->FinalizeSegment();
224-
}
212+
(*info->DataSubstream)->FinalizeSegment();
225213
delete info->Data;
226214
return 1;
227215
};
@@ -286,7 +274,7 @@ void BP5Writer::WriteData_EveryoneWrites_Async(format::BufferV *Data, bool Seria
286274
}
287275
m_AsyncWriteInfo->tstart = m_EngineStart;
288276
AggTransportData *aggData = &(m_AggregatorSpecifics.at(GetCacheKey(m_Aggregator)));
289-
m_AsyncWriteInfo->DataSubstreams = &aggData->m_DataSubstreams;
277+
m_AsyncWriteInfo->DataSubstream = &aggData->m_DataSubstream;
290278
m_AsyncWriteInfo->Data = Data;
291279
m_AsyncWriteInfo->startPos = m_StartDataPos;
292280
m_AsyncWriteInfo->totalSize = Data->Size();

source/adios2/engine/bp5/BP5Writer_TwoLevelShm.cpp

Lines changed: 4 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -176,10 +176,8 @@ void BP5Writer::WriteMyOwnData(format::BufferV *Data)
176176
std::vector<core::iovec> DataVec = Data->DataVec();
177177
m_StartDataPos = m_DataPos;
178178
AggTransportData *aggData = &(m_AggregatorSpecifics.at(GetCacheKey(m_Aggregator)));
179-
for (auto &t : aggData->m_DataSubstreams)
180-
{
181-
t->WriteV(DataVec.data(), static_cast<int>(DataVec.size()), m_StartDataPos);
182-
}
179+
aggData->m_DataSubstream->WriteV(DataVec.data(), static_cast<int>(DataVec.size()),
180+
m_StartDataPos);
183181
m_DataPos += Data->Size();
184182
}
185183

@@ -292,10 +290,7 @@ void BP5Writer::WriteOthersData(size_t TotalSize)
292290
aggregator::MPIShmChain::ShmDataBuffer *b = a->LockConsumerBuffer();
293291

294292
// b->actual_size: how much we need to write
295-
for (auto &t : aggData->m_DataSubstreams)
296-
{
297-
t->Write(b->buf, b->actual_size);
298-
}
293+
aggData->m_DataSubstream->Write(b->buf, b->actual_size);
299294

300295
wrote += b->actual_size;
301296

@@ -327,10 +322,7 @@ void BP5Writer::WriteOthersData_DrainOnError(const size_t TotalSize, std::except
327322
{
328323
try
329324
{
330-
for (auto &t : aggData->m_DataSubstreams)
331-
{
332-
t->Write(b->buf, b->actual_size);
333-
}
325+
aggData->m_DataSubstream->Write(b->buf, b->actual_size);
334326
}
335327
catch (...)
336328
{

source/adios2/engine/bp5/BP5Writer_TwoLevelShm_Async.cpp

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -41,10 +41,7 @@ void BP5Writer::AsyncWriteThread_TwoLevelShm_Aggregator(AsyncWriteInfo *info)
4141
{
4242
std::vector<core::iovec> DataVec = info->Data->DataVec();
4343
const uint64_t mysize = info->Data->Size();
44-
for (auto &t : *info->DataSubstreams)
45-
{
46-
t->Seek(info->startPos);
47-
}
44+
(*info->DataSubstream)->Seek(info->startPos);
4845
AsyncWriteOwnData(info, DataVec, mysize, false);
4946
totalSize -= mysize;
5047
}
@@ -148,10 +145,7 @@ int BP5Writer::AsyncWriteThread_TwoLevelShm(AsyncWriteInfo *info)
148145
info->tokenChain->SendToken(nextWriterPos);
149146
AsyncWriteThread_TwoLevelShm_Aggregator(info);
150147
info->tokenChain->RecvToken();
151-
for (auto &t : *info->DataSubstreams)
152-
{
153-
t->FinalizeSegment();
154-
}
148+
(*info->DataSubstream)->FinalizeSegment();
155149
}
156150
else
157151
{
@@ -263,7 +257,7 @@ void BP5Writer::WriteData_TwoLevelShm_Async(format::BufferV *Data)
263257
m_AsyncWriteInfo->tstart = m_EngineStart;
264258
m_AsyncWriteInfo->tokenChain = new shm::TokenChain<uint64_t>(&a->m_Comm);
265259
AggTransportData *aggData = &(m_AggregatorSpecifics.at(GetCacheKey(m_Aggregator)));
266-
m_AsyncWriteInfo->DataSubstreams = &aggData->m_DataSubstreams;
260+
m_AsyncWriteInfo->DataSubstream = &aggData->m_DataSubstream;
267261
m_AsyncWriteInfo->Data = Data;
268262
m_AsyncWriteInfo->flagRush = &m_flagRush;
269263
m_AsyncWriteInfo->lock = &m_AsyncWriteLock;

source/adios2/engine/bp5/BP5Writer_WithRerouting.cpp

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1003,10 +1003,8 @@ void BP5Writer::WriteData_WithRerouting(format::BufferV *Data)
10031003
std::vector<core::iovec> DataVec = Data->DataVec();
10041004

10051005
AggTransportData &aggData = m_AggregatorSpecifics.at(GetCacheKey(m_Aggregator));
1006-
for (auto &t : aggData.m_DataSubstreams)
1007-
{
1008-
t->WriteV(DataVec.data(), static_cast<int>(DataVec.size()), m_StartDataPos);
1009-
}
1006+
aggData.m_DataSubstream->WriteV(DataVec.data(), static_cast<int>(DataVec.size()),
1007+
m_StartDataPos);
10101008

10111009
m_DataPos += Data->Size();
10121010

0 commit comments

Comments
 (0)