Skip to content

Commit b8d5692

Browse files
vepadulanovepadulano
committed
[ntuple] Split long-running test
Split the `ntuple_extended.cxx` test into multiple test files, separating the three tests that take the longest: largefile1, largefile2, randomaccess. This is done in order to have better parallelization in the test suite.
1 parent 435c1ee commit b8d5692

File tree

5 files changed

+226
-203
lines changed

5 files changed

+226
-203
lines changed

Diff for: ‎tree/ntuple/test/CMakeLists.txt

+5
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,11 @@ ROOT_ADD_GTEST(ntuple_minifile ntuple_minifile.cxx LIBRARIES ROOTNTuple Physics
8181
ROOT_ADD_GTEST(ntuple_show ntuple_show.cxx LIBRARIES ROOTNTuple CustomStruct)
8282
ROOT_ADD_GTEST(ntuple_storage ntuple_storage.cxx LIBRARIES ROOTNTuple MathCore CustomStruct)
8383
ROOT_ADD_GTEST(ntuple_extended ntuple_extended.cxx LIBRARIES ROOTNTuple MathCore CustomStruct)
84+
if(NOT MSVC OR win_broken_tests)
85+
ROOT_ADD_GTEST(ntuple_largefile1 ntuple_largefile1.cxx LIBRARIES ROOTNTuple MathCore CustomStruct)
86+
ROOT_ADD_GTEST(ntuple_largefile2 ntuple_largefile2.cxx LIBRARIES ROOTNTuple MathCore CustomStruct)
87+
endif()
88+
ROOT_ADD_GTEST(ntuple_randomaccess ntuple_randomaccess.cxx LIBRARIES ROOTNTuple MathCore CustomStruct)
8489

8590
ROOT_ADD_GTEST(ntuple_parallel_writer ntuple_parallel_writer.cxx LIBRARIES ROOTNTuple)
8691

Diff for: ‎tree/ntuple/test/ntuple_extended.cxx

-203
Original file line numberDiff line numberDiff line change
@@ -158,209 +158,6 @@ TEST(RNTuple, MultiColumnExpansion)
158158
}
159159
}
160160

161-
// Stress test the asynchronous cluster pool by a deliberately unfavourable read pattern
162-
TEST(RNTuple, RandomAccess)
163-
{
164-
#ifdef R__USE_IMT
165-
ROOT::EnableImplicitMT();
166-
#endif
167-
FileRaii fileGuard("test_ntuple_random_access.root");
168-
169-
auto modelWrite = RNTupleModel::Create();
170-
auto wrValue = modelWrite->MakeField<std::int32_t>("value");
171-
172-
constexpr unsigned int nEvents = 1000000;
173-
{
174-
RNTupleWriteOptions options;
175-
options.SetCompression(0);
176-
options.SetEnablePageChecksums(false);
177-
options.SetApproxZippedClusterSize(nEvents * sizeof(std::int32_t) / 10);
178-
auto ntuple = RNTupleWriter::Recreate(std::move(modelWrite), "myNTuple", fileGuard.GetPath(), options);
179-
for (unsigned int i = 0; i < nEvents; ++i) {
180-
*wrValue = i;
181-
ntuple->Fill();
182-
}
183-
}
184-
185-
RNTupleReadOptions options;
186-
options.SetClusterCache(RNTupleReadOptions::EClusterCache::kOn);
187-
auto ntuple = RNTupleReader::Open("myNTuple", fileGuard.GetPath(), options);
188-
EXPECT_EQ(10, ntuple->GetDescriptor().GetNClusters());
189-
190-
auto viewValue = ntuple->GetView<std::int32_t>("value");
191-
192-
std::int64_t sum = 0;
193-
std::int64_t expected = 0;
194-
constexpr unsigned int nSamples = 50000;
195-
TRandom3 rnd(42);
196-
for (unsigned int i = 0; i < nSamples; ++i) {
197-
auto entryId = floor(rnd.Rndm() * (nEvents - 1));
198-
expected += entryId;
199-
sum += viewValue(entryId);
200-
}
201-
EXPECT_EQ(expected, sum);
202-
}
203-
204-
205-
#if !defined(_MSC_VER) || defined(R__ENABLE_BROKEN_WIN_TESTS)
206-
TEST(RNTuple, LargeFile1)
207-
{
208-
#ifdef R__USE_IMT
209-
ROOT::EnableImplicitMT();
210-
#endif
211-
FileRaii fileGuard("test_large_file1.root");
212-
213-
auto modelWrite = RNTupleModel::Create();
214-
auto& wrEnergy = *modelWrite->MakeField<double>("energy");
215-
216-
TRandom3 rnd(42);
217-
double chksumWrite = 0.0;
218-
{
219-
RNTupleWriteOptions options;
220-
options.SetCompression(0);
221-
auto ntuple = RNTupleWriter::Recreate(std::move(modelWrite), "myNTuple", fileGuard.GetPath(), options);
222-
constexpr std::uint64_t nEvents = 1024 * 1024 * 256; // Exceed 2GB file size
223-
for (std::uint64_t i = 0; i < nEvents; ++i) {
224-
wrEnergy = rnd.Rndm();
225-
chksumWrite += wrEnergy;
226-
ntuple->Fill();
227-
}
228-
}
229-
#ifdef R__SEEK64
230-
FILE *file = fopen64(fileGuard.GetPath().c_str(), "rb");
231-
ASSERT_TRUE(file != nullptr);
232-
EXPECT_EQ(0, fseeko64(file, 0, SEEK_END));
233-
EXPECT_GT(ftello64(file), 2048LL * 1024LL * 1024LL);
234-
#else
235-
FILE *file = fopen(fileGuard.GetPath().c_str(), "rb");
236-
ASSERT_TRUE(file != nullptr);
237-
EXPECT_EQ(0, fseek(file, 0, SEEK_END));
238-
EXPECT_GT(ftell(file), 2048LL * 1024LL * 1024LL);
239-
#endif
240-
fclose(file);
241-
242-
{
243-
auto reader = RNTupleReader::Open("myNTuple", fileGuard.GetPath());
244-
auto rdEnergy = reader->GetView<double>("energy");
245-
246-
double chksumRead = 0.0;
247-
for (auto i : reader->GetEntryRange()) {
248-
chksumRead += rdEnergy(i);
249-
}
250-
EXPECT_EQ(chksumRead, chksumWrite);
251-
}
252-
253-
{
254-
auto f = std::unique_ptr<TFile>(TFile::Open(fileGuard.GetPath().c_str(), "READ"));
255-
EXPECT_TRUE(f);
256-
auto ntuple = std::unique_ptr<ROOT::RNTuple>(f->Get<ROOT::RNTuple>("myNTuple"));
257-
auto reader = RNTupleReader::Open(*ntuple);
258-
auto rdEnergy = reader->GetView<double>("energy");
259-
260-
double chksumRead = 0.0;
261-
for (auto i : reader->GetEntryRange()) {
262-
chksumRead += rdEnergy(i);
263-
}
264-
EXPECT_EQ(chksumRead, chksumWrite);
265-
}
266-
}
267-
268-
269-
TEST(RNTuple, LargeFile2)
270-
{
271-
#ifdef R__USE_IMT
272-
ROOT::EnableImplicitMT();
273-
#endif
274-
FileRaii fileGuard("test_large_file2.root");
275-
276-
// Start out with a mini-file created small file
277-
auto model = RNTupleModel::Create();
278-
*model->MakeField<float>("pt") = 42.0;
279-
auto writer = RNTupleWriter::Recreate(std::move(model), "small", fileGuard.GetPath());
280-
writer->Fill();
281-
writer = nullptr;
282-
283-
// Update the file with another object
284-
auto f = std::unique_ptr<TFile>(TFile::Open(fileGuard.GetPath().c_str(), "UPDATE"));
285-
std::string str = "one";
286-
f->WriteObject(&str, "s1");
287-
288-
// Turn it into a large file
289-
model = RNTupleModel::Create();
290-
auto E = model->MakeField<double>("E");
291-
RNTupleWriteOptions options;
292-
options.SetCompression(0);
293-
writer = RNTupleWriter::Append(std::move(model), "large", *f, options);
294-
295-
TRandom3 rnd(42);
296-
double chksumWrite = 0.0;
297-
constexpr std::uint64_t nEvents = 1024 * 1024 * 256; // Exceed 2GB file size
298-
for (std::uint64_t i = 0; i < nEvents; ++i) {
299-
*E = rnd.Rndm();
300-
chksumWrite += *E;
301-
writer->Fill();
302-
}
303-
304-
// Add one more object before the ntuple writer commits the footer
305-
str = "two";
306-
f->WriteObject(&str, "s2");
307-
writer = nullptr;
308-
f->Close();
309-
f = nullptr;
310-
311-
#ifdef R__SEEK64
312-
FILE *file = fopen64(fileGuard.GetPath().c_str(), "rb");
313-
ASSERT_TRUE(file != nullptr);
314-
EXPECT_EQ(0, fseeko64(file, 0, SEEK_END));
315-
EXPECT_GT(ftello64(file), 2048LL * 1024LL * 1024LL);
316-
#else
317-
FILE *file = fopen(fileGuard.GetPath().c_str(), "rb");
318-
ASSERT_TRUE(file != nullptr);
319-
EXPECT_EQ(0, fseek(file, 0, SEEK_END));
320-
EXPECT_GT(ftell(file), 2048LL * 1024LL * 1024LL);
321-
#endif
322-
fclose(file);
323-
324-
f = std::unique_ptr<TFile>(TFile::Open(fileGuard.GetPath().c_str()));
325-
{
326-
auto reader = RNTupleReader::Open("small", fileGuard.GetPath());
327-
reader->LoadEntry(0);
328-
EXPECT_EQ(42.0f, *reader->GetModel().GetDefaultEntry().GetPtr<float>("pt"));
329-
330-
reader = RNTupleReader::Open("large", fileGuard.GetPath());
331-
auto viewE = reader->GetView<double>("E");
332-
double chksumRead = 0.0;
333-
for (auto i : reader->GetEntryRange()) {
334-
chksumRead += viewE(i);
335-
}
336-
EXPECT_EQ(chksumRead, chksumWrite);
337-
}
338-
339-
{
340-
f = std::unique_ptr<TFile>(TFile::Open(fileGuard.GetPath().c_str(), "READ"));
341-
EXPECT_TRUE(f);
342-
auto s1 = f->Get<std::string>("s1");
343-
EXPECT_EQ("one", *s1);
344-
auto s2 = f->Get<std::string>("s2");
345-
EXPECT_EQ("two", *s2);
346-
347-
auto small = std::unique_ptr<ROOT::RNTuple>(f->Get<ROOT::RNTuple>("small"));
348-
auto reader = RNTupleReader::Open(*small);
349-
reader->LoadEntry(0);
350-
EXPECT_EQ(42.0f, *reader->GetModel().GetDefaultEntry().GetPtr<float>("pt"));
351-
352-
auto large = std::unique_ptr<ROOT::RNTuple>(f->Get<ROOT::RNTuple>("large"));
353-
reader = RNTupleReader::Open(*large);
354-
auto viewE = reader->GetView<double>("E");
355-
double chksumRead = 0.0;
356-
for (auto i : reader->GetEntryRange()) {
357-
chksumRead += viewE(i);
358-
}
359-
EXPECT_EQ(chksumRead, chksumWrite);
360-
}
361-
}
362-
#endif
363-
364161
TEST(RNTuple, LargePages)
365162
{
366163
FileRaii fileGuard("test_ntuple_large_pages.root");

Diff for: ‎tree/ntuple/test/ntuple_largefile1.cxx

+69
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
#include <ROOT/RConfig.hxx>
2+
3+
#include "ntuple_test.hxx"
4+
5+
#include <TRandom3.h>
6+
7+
#include <algorithm>
8+
9+
TEST(RNTuple, LargeFile1)
10+
{
11+
#ifdef R__USE_IMT
12+
IMTRAII _;
13+
#endif
14+
FileRaii fileGuard("test_large_file1.root");
15+
16+
auto modelWrite = RNTupleModel::Create();
17+
auto &wrEnergy = *modelWrite->MakeField<double>("energy");
18+
19+
TRandom3 rnd(42);
20+
double chksumWrite = 0.0;
21+
{
22+
RNTupleWriteOptions options;
23+
options.SetCompression(0);
24+
auto ntuple = RNTupleWriter::Recreate(std::move(modelWrite), "myNTuple", fileGuard.GetPath(), options);
25+
constexpr std::uint64_t nEvents = 1024 * 1024 * 256; // Exceed 2GB file size
26+
for (std::uint64_t i = 0; i < nEvents; ++i) {
27+
wrEnergy = rnd.Rndm();
28+
chksumWrite += wrEnergy;
29+
ntuple->Fill();
30+
}
31+
}
32+
#ifdef R__SEEK64
33+
FILE *file = fopen64(fileGuard.GetPath().c_str(), "rb");
34+
ASSERT_TRUE(file != nullptr);
35+
EXPECT_EQ(0, fseeko64(file, 0, SEEK_END));
36+
EXPECT_GT(ftello64(file), 2048LL * 1024LL * 1024LL);
37+
#else
38+
FILE *file = fopen(fileGuard.GetPath().c_str(), "rb");
39+
ASSERT_TRUE(file != nullptr);
40+
EXPECT_EQ(0, fseek(file, 0, SEEK_END));
41+
EXPECT_GT(ftell(file), 2048LL * 1024LL * 1024LL);
42+
#endif
43+
fclose(file);
44+
45+
{
46+
auto reader = RNTupleReader::Open("myNTuple", fileGuard.GetPath());
47+
auto rdEnergy = reader->GetView<double>("energy");
48+
49+
double chksumRead = 0.0;
50+
for (auto i : reader->GetEntryRange()) {
51+
chksumRead += rdEnergy(i);
52+
}
53+
EXPECT_EQ(chksumRead, chksumWrite);
54+
}
55+
56+
{
57+
auto f = std::unique_ptr<TFile>(TFile::Open(fileGuard.GetPath().c_str(), "READ"));
58+
EXPECT_TRUE(f);
59+
auto ntuple = std::unique_ptr<ROOT::RNTuple>(f->Get<ROOT::RNTuple>("myNTuple"));
60+
auto reader = RNTupleReader::Open(*ntuple);
61+
auto rdEnergy = reader->GetView<double>("energy");
62+
63+
double chksumRead = 0.0;
64+
for (auto i : reader->GetEntryRange()) {
65+
chksumRead += rdEnergy(i);
66+
}
67+
EXPECT_EQ(chksumRead, chksumWrite);
68+
}
69+
}

Diff for: ‎tree/ntuple/test/ntuple_largefile2.cxx

+102
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
#include <ROOT/RConfig.hxx>
2+
3+
#include "ntuple_test.hxx"
4+
5+
#include <TRandom3.h>
6+
7+
#include <algorithm>
8+
#include <random>
9+
10+
TEST(RNTuple, LargeFile2)
11+
{
12+
#ifdef R__USE_IMT
13+
IMTRAII _;
14+
#endif
15+
FileRaii fileGuard("test_large_file2.root");
16+
17+
// Start out with a mini-file created small file
18+
auto model = RNTupleModel::Create();
19+
*model->MakeField<float>("pt") = 42.0;
20+
auto writer = RNTupleWriter::Recreate(std::move(model), "small", fileGuard.GetPath());
21+
writer->Fill();
22+
writer = nullptr;
23+
24+
// Update the file with another object
25+
auto f = std::unique_ptr<TFile>(TFile::Open(fileGuard.GetPath().c_str(), "UPDATE"));
26+
std::string str = "one";
27+
f->WriteObject(&str, "s1");
28+
29+
// Turn it into a large file
30+
model = RNTupleModel::Create();
31+
auto E = model->MakeField<double>("E");
32+
RNTupleWriteOptions options;
33+
options.SetCompression(0);
34+
writer = RNTupleWriter::Append(std::move(model), "large", *f, options);
35+
36+
TRandom3 rnd(42);
37+
double chksumWrite = 0.0;
38+
constexpr std::uint64_t nEvents = 1024 * 1024 * 256; // Exceed 2GB file size
39+
for (std::uint64_t i = 0; i < nEvents; ++i) {
40+
*E = rnd.Rndm();
41+
chksumWrite += *E;
42+
writer->Fill();
43+
}
44+
45+
// Add one more object before the ntuple writer commits the footer
46+
str = "two";
47+
f->WriteObject(&str, "s2");
48+
writer = nullptr;
49+
f->Close();
50+
f = nullptr;
51+
52+
#ifdef R__SEEK64
53+
FILE *file = fopen64(fileGuard.GetPath().c_str(), "rb");
54+
ASSERT_TRUE(file != nullptr);
55+
EXPECT_EQ(0, fseeko64(file, 0, SEEK_END));
56+
EXPECT_GT(ftello64(file), 2048LL * 1024LL * 1024LL);
57+
#else
58+
FILE *file = fopen(fileGuard.GetPath().c_str(), "rb");
59+
ASSERT_TRUE(file != nullptr);
60+
EXPECT_EQ(0, fseek(file, 0, SEEK_END));
61+
EXPECT_GT(ftell(file), 2048LL * 1024LL * 1024LL);
62+
#endif
63+
fclose(file);
64+
65+
f = std::unique_ptr<TFile>(TFile::Open(fileGuard.GetPath().c_str()));
66+
{
67+
auto reader = RNTupleReader::Open("small", fileGuard.GetPath());
68+
reader->LoadEntry(0);
69+
EXPECT_EQ(42.0f, *reader->GetModel().GetDefaultEntry().GetPtr<float>("pt"));
70+
71+
reader = RNTupleReader::Open("large", fileGuard.GetPath());
72+
auto viewE = reader->GetView<double>("E");
73+
double chksumRead = 0.0;
74+
for (auto i : reader->GetEntryRange()) {
75+
chksumRead += viewE(i);
76+
}
77+
EXPECT_EQ(chksumRead, chksumWrite);
78+
}
79+
80+
{
81+
f = std::unique_ptr<TFile>(TFile::Open(fileGuard.GetPath().c_str(), "READ"));
82+
EXPECT_TRUE(f);
83+
auto s1 = f->Get<std::string>("s1");
84+
EXPECT_EQ("one", *s1);
85+
auto s2 = f->Get<std::string>("s2");
86+
EXPECT_EQ("two", *s2);
87+
88+
auto small = std::unique_ptr<ROOT::RNTuple>(f->Get<ROOT::RNTuple>("small"));
89+
auto reader = RNTupleReader::Open(*small);
90+
reader->LoadEntry(0);
91+
EXPECT_EQ(42.0f, *reader->GetModel().GetDefaultEntry().GetPtr<float>("pt"));
92+
93+
auto large = std::unique_ptr<ROOT::RNTuple>(f->Get<ROOT::RNTuple>("large"));
94+
reader = RNTupleReader::Open(*large);
95+
auto viewE = reader->GetView<double>("E");
96+
double chksumRead = 0.0;
97+
for (auto i : reader->GetEntryRange()) {
98+
chksumRead += viewE(i);
99+
}
100+
EXPECT_EQ(chksumRead, chksumWrite);
101+
}
102+
}

0 commit comments

Comments
 (0)