Skip to content

Commit ec3699d

Browse files
vepadulanovepadulano
committed
[ntuple] Split long-running test
1 parent 800af77 commit ec3699d

8 files changed

+445
-0
lines changed

Diff for: ‎tree/ntuple/v7/test/CMakeLists.txt

+10
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,16 @@ ROOT_ADD_GTEST(ntuple_show ntuple_show.cxx LIBRARIES ROOTNTuple CustomStruct)
8282
ROOT_ADD_GTEST(ntuple_storage ntuple_storage.cxx LIBRARIES ROOTNTuple MathCore CustomStruct)
8383
ROOT_ADD_GTEST(ntuple_extended ntuple_extended.cxx LIBRARIES ROOTNTuple MathCore CustomStruct)
8484

85+
ROOT_ADD_GTEST(ntuple_double32imt ntuple_double32imt.cxx LIBRARIES ROOTNTuple MathCore CustomStruct)
86+
if(NOT MSVC OR win_broken_tests)
87+
ROOT_ADD_GTEST(ntuple_largefile1 ntuple_largefile1.cxx LIBRARIES ROOTNTuple MathCore CustomStruct)
88+
ROOT_ADD_GTEST(ntuple_largefile2 ntuple_largefile2.cxx LIBRARIES ROOTNTuple MathCore CustomStruct)
89+
endif()
90+
ROOT_ADD_GTEST(ntuple_largepages ntuple_largepages.cxx LIBRARIES ROOTNTuple MathCore CustomStruct)
91+
ROOT_ADD_GTEST(ntuple_multicolumnexpansion ntuple_multicolumnexpansion.cxx LIBRARIES ROOTNTuple MathCore CustomStruct)
92+
ROOT_ADD_GTEST(ntuple_randomaccess ntuple_randomaccess.cxx LIBRARIES ROOTNTuple MathCore CustomStruct)
93+
ROOT_ADD_GTEST(ntuple_realworld1 ntuple_realworld1.cxx LIBRARIES ROOTNTuple MathCore CustomStruct)
94+
8595
ROOT_ADD_GTEST(ntuple_parallel_writer ntuple_parallel_writer.cxx LIBRARIES ROOTNTuple)
8696

8797
ROOT_ADD_GTEST(ntuple_limits ntuple_limits.cxx LIBRARIES ROOTNTuple)

Diff for: ‎tree/ntuple/v7/test/ntuple_double32imt.cxx

+35
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
#include <ROOT/RConfig.hxx>
2+
3+
#include "ntuple_test.hxx"
4+
5+
#include <algorithm>
6+
7+
TEST(RNTuple, Double32IMT)
8+
{
9+
// Tests if parallel decompression correctly compresses the on-disk float to an in-memory double
10+
#ifdef R__USE_IMT
11+
IMTRAII _;
12+
#endif
13+
FileRaii fileGuard("test_ntuple_double32_imt.root");
14+
15+
constexpr int kNEvents = 10;
16+
17+
{
18+
auto model = RNTupleModel::Create();
19+
model->AddField(RFieldBase::Create("pt", "Double32_t").Unwrap());
20+
auto writer = RNTupleWriter::Recreate(std::move(model), "ntpl", fileGuard.GetPath());
21+
22+
auto ptrPt = writer->GetModel().GetDefaultEntry().GetPtr<double>("pt");
23+
24+
for (int i = 0; i < kNEvents; ++i) {
25+
*ptrPt = i;
26+
writer->Fill();
27+
}
28+
}
29+
30+
auto reader = RNTupleReader::Open("ntpl", fileGuard.GetPath());
31+
auto viewPt = reader->GetView<double>("pt");
32+
for (int i = 0; i < kNEvents; ++i) {
33+
EXPECT_DOUBLE_EQ(i, viewPt(i));
34+
}
35+
}

Diff for: ‎tree/ntuple/v7/test/ntuple_largefile1.cxx

+69
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
#include <ROOT/RConfig.hxx>
2+
3+
#include "ntuple_test.hxx"
4+
5+
#include <TRandom3.h>
6+
7+
#include <algorithm>
8+
9+
TEST(RNTuple, LargeFile1)
10+
{
11+
#ifdef R__USE_IMT
12+
IMTRAII _;
13+
#endif
14+
FileRaii fileGuard("test_large_file1.root");
15+
16+
auto modelWrite = RNTupleModel::Create();
17+
auto &wrEnergy = *modelWrite->MakeField<double>("energy");
18+
19+
TRandom3 rnd(42);
20+
double chksumWrite = 0.0;
21+
{
22+
RNTupleWriteOptions options;
23+
options.SetCompression(0);
24+
auto ntuple = RNTupleWriter::Recreate(std::move(modelWrite), "myNTuple", fileGuard.GetPath(), options);
25+
constexpr std::uint64_t nEvents = 1024 * 1024 * 256; // Exceed 2GB file size
26+
for (std::uint64_t i = 0; i < nEvents; ++i) {
27+
wrEnergy = rnd.Rndm();
28+
chksumWrite += wrEnergy;
29+
ntuple->Fill();
30+
}
31+
}
32+
#ifdef R__SEEK64
33+
FILE *file = fopen64(fileGuard.GetPath().c_str(), "rb");
34+
ASSERT_TRUE(file != nullptr);
35+
EXPECT_EQ(0, fseeko64(file, 0, SEEK_END));
36+
EXPECT_GT(ftello64(file), 2048LL * 1024LL * 1024LL);
37+
#else
38+
FILE *file = fopen(fileGuard.GetPath().c_str(), "rb");
39+
ASSERT_TRUE(file != nullptr);
40+
EXPECT_EQ(0, fseek(file, 0, SEEK_END));
41+
EXPECT_GT(ftell(file), 2048LL * 1024LL * 1024LL);
42+
#endif
43+
fclose(file);
44+
45+
{
46+
auto reader = RNTupleReader::Open("myNTuple", fileGuard.GetPath());
47+
auto rdEnergy = reader->GetView<double>("energy");
48+
49+
double chksumRead = 0.0;
50+
for (auto i : reader->GetEntryRange()) {
51+
chksumRead += rdEnergy(i);
52+
}
53+
EXPECT_EQ(chksumRead, chksumWrite);
54+
}
55+
56+
{
57+
auto f = std::unique_ptr<TFile>(TFile::Open(fileGuard.GetPath().c_str(), "READ"));
58+
EXPECT_TRUE(f);
59+
auto ntuple = std::unique_ptr<ROOT::RNTuple>(f->Get<ROOT::RNTuple>("myNTuple"));
60+
auto reader = RNTupleReader::Open(*ntuple);
61+
auto rdEnergy = reader->GetView<double>("energy");
62+
63+
double chksumRead = 0.0;
64+
for (auto i : reader->GetEntryRange()) {
65+
chksumRead += rdEnergy(i);
66+
}
67+
EXPECT_EQ(chksumRead, chksumWrite);
68+
}
69+
}

Diff for: ‎tree/ntuple/v7/test/ntuple_largefile2.cxx

+102
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
#include <ROOT/RConfig.hxx>
2+
3+
#include "ntuple_test.hxx"
4+
5+
#include <TRandom3.h>
6+
7+
#include <algorithm>
8+
#include <random>
9+
10+
TEST(RNTuple, LargeFile2)
11+
{
12+
#ifdef R__USE_IMT
13+
IMTRAII _;
14+
#endif
15+
FileRaii fileGuard("test_large_file2.root");
16+
17+
// Start out with a mini-file created small file
18+
auto model = RNTupleModel::Create();
19+
*model->MakeField<float>("pt") = 42.0;
20+
auto writer = RNTupleWriter::Recreate(std::move(model), "small", fileGuard.GetPath());
21+
writer->Fill();
22+
writer = nullptr;
23+
24+
// Update the file with another object
25+
auto f = std::unique_ptr<TFile>(TFile::Open(fileGuard.GetPath().c_str(), "UPDATE"));
26+
std::string str = "one";
27+
f->WriteObject(&str, "s1");
28+
29+
// Turn it into a large file
30+
model = RNTupleModel::Create();
31+
auto E = model->MakeField<double>("E");
32+
RNTupleWriteOptions options;
33+
options.SetCompression(0);
34+
writer = RNTupleWriter::Append(std::move(model), "large", *f, options);
35+
36+
TRandom3 rnd(42);
37+
double chksumWrite = 0.0;
38+
constexpr std::uint64_t nEvents = 1024 * 1024 * 256; // Exceed 2GB file size
39+
for (std::uint64_t i = 0; i < nEvents; ++i) {
40+
*E = rnd.Rndm();
41+
chksumWrite += *E;
42+
writer->Fill();
43+
}
44+
45+
// Add one more object before the ntuple writer commits the footer
46+
str = "two";
47+
f->WriteObject(&str, "s2");
48+
writer = nullptr;
49+
f->Close();
50+
f = nullptr;
51+
52+
#ifdef R__SEEK64
53+
FILE *file = fopen64(fileGuard.GetPath().c_str(), "rb");
54+
ASSERT_TRUE(file != nullptr);
55+
EXPECT_EQ(0, fseeko64(file, 0, SEEK_END));
56+
EXPECT_GT(ftello64(file), 2048LL * 1024LL * 1024LL);
57+
#else
58+
FILE *file = fopen(fileGuard.GetPath().c_str(), "rb");
59+
ASSERT_TRUE(file != nullptr);
60+
EXPECT_EQ(0, fseek(file, 0, SEEK_END));
61+
EXPECT_GT(ftell(file), 2048LL * 1024LL * 1024LL);
62+
#endif
63+
fclose(file);
64+
65+
f = std::unique_ptr<TFile>(TFile::Open(fileGuard.GetPath().c_str()));
66+
{
67+
auto reader = RNTupleReader::Open("small", fileGuard.GetPath());
68+
reader->LoadEntry(0);
69+
EXPECT_EQ(42.0f, *reader->GetModel().GetDefaultEntry().GetPtr<float>("pt"));
70+
71+
reader = RNTupleReader::Open("large", fileGuard.GetPath());
72+
auto viewE = reader->GetView<double>("E");
73+
double chksumRead = 0.0;
74+
for (auto i : reader->GetEntryRange()) {
75+
chksumRead += viewE(i);
76+
}
77+
EXPECT_EQ(chksumRead, chksumWrite);
78+
}
79+
80+
{
81+
f = std::unique_ptr<TFile>(TFile::Open(fileGuard.GetPath().c_str(), "READ"));
82+
EXPECT_TRUE(f);
83+
auto s1 = f->Get<std::string>("s1");
84+
EXPECT_EQ("one", *s1);
85+
auto s2 = f->Get<std::string>("s2");
86+
EXPECT_EQ("two", *s2);
87+
88+
auto small = std::unique_ptr<ROOT::RNTuple>(f->Get<ROOT::RNTuple>("small"));
89+
auto reader = RNTupleReader::Open(*small);
90+
reader->LoadEntry(0);
91+
EXPECT_EQ(42.0f, *reader->GetModel().GetDefaultEntry().GetPtr<float>("pt"));
92+
93+
auto large = std::unique_ptr<ROOT::RNTuple>(f->Get<ROOT::RNTuple>("large"));
94+
reader = RNTupleReader::Open(*large);
95+
auto viewE = reader->GetView<double>("E");
96+
double chksumRead = 0.0;
97+
for (auto i : reader->GetEntryRange()) {
98+
chksumRead += viewE(i);
99+
}
100+
EXPECT_EQ(chksumRead, chksumWrite);
101+
}
102+
}

Diff for: ‎tree/ntuple/v7/test/ntuple_largepages.cxx

+46
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
#include <ROOT/RConfig.hxx>
2+
3+
#include "ntuple_test.hxx"
4+
5+
#include <TROOT.h>
6+
7+
#include <algorithm>
8+
#include <random>
9+
10+
TEST(RNTuple, LargePages)
11+
{
12+
FileRaii fileGuard("test_ntuple_large_pages.root");
13+
14+
for (const auto useBufferedWrite : {true, false}) {
15+
{
16+
auto model = RNTupleModel::Create();
17+
auto fldRnd = model->MakeField<std::uint32_t>("rnd");
18+
RNTupleWriteOptions options;
19+
// Larger than the 16MB compression block limit
20+
options.SetMaxUnzippedPageSize(32 * 1024 * 1024);
21+
options.SetUseBufferedWrite(useBufferedWrite);
22+
auto writer = RNTupleWriter::Recreate(std::move(model), "ntpl", fileGuard.GetPath(), options);
23+
24+
std::mt19937 gen;
25+
std::uniform_int_distribution<std::uint32_t> distrib;
26+
for (int i = 0; i < 25 * 1000 * 1000; ++i) { // 100 MB of int data
27+
*fldRnd = distrib(gen);
28+
writer->Fill();
29+
}
30+
writer.reset();
31+
}
32+
33+
auto reader = RNTupleReader::Open("ntpl", fileGuard.GetPath());
34+
const auto &desc = reader->GetDescriptor();
35+
const auto rndColId = desc.FindPhysicalColumnId(desc.FindFieldId("rnd"), 0, 0);
36+
const auto &clusterDesc = desc.GetClusterDescriptor(desc.FindClusterId(rndColId, 0));
37+
EXPECT_GT(clusterDesc.GetPageRange(rndColId).Find(0).GetLocator().GetNBytesOnStorage(), kMAXZIPBUF);
38+
39+
auto viewRnd = reader->GetView<std::uint32_t>("rnd");
40+
std::mt19937 gen;
41+
std::uniform_int_distribution<std::uint32_t> distrib;
42+
for (const auto i : reader->GetEntryRange()) {
43+
EXPECT_EQ(distrib(gen), viewRnd(i));
44+
}
45+
}
46+
}

Diff for: ‎tree/ntuple/v7/test/ntuple_multicolumnexpansion.cxx

+58
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
#include <ROOT/RConfig.hxx>
2+
3+
#include "ntuple_test.hxx"
4+
5+
#include <algorithm>
6+
#include <random>
7+
8+
TEST(RNTuple, MultiColumnExpansion)
9+
{
10+
// Tests if on-disk columns that expand to multiple in-memory types are correctly handled
11+
#ifdef R__USE_IMT
12+
IMTRAII _;
13+
#endif
14+
FileRaii fileGuard("test_ntuple_multi_column_expansion.root");
15+
16+
constexpr int kNEvents = 1000;
17+
18+
{
19+
auto model = RNTupleModel::Create();
20+
model->AddField(RFieldBase::Create("pt", "Double32_t").Unwrap());
21+
RNTupleWriteOptions options;
22+
options.SetInitialUnzippedPageSize(8);
23+
options.SetMaxUnzippedPageSize(32);
24+
auto writer = RNTupleWriter::Recreate(std::move(model), "ntpl", fileGuard.GetPath(), options);
25+
26+
auto ptrPt = writer->GetModel().GetDefaultEntry().GetPtr<double>("pt");
27+
28+
for (int i = 0; i < kNEvents; ++i) {
29+
*ptrPt = i;
30+
writer->Fill();
31+
if (i % 50 == 0)
32+
writer->CommitCluster();
33+
}
34+
}
35+
36+
auto reader = RNTupleReader::Open("ntpl", fileGuard.GetPath());
37+
auto viewPt = reader->GetView<double>("pt");
38+
auto viewPtAsFloat = reader->GetView<float>("pt");
39+
40+
std::random_device rd;
41+
std::mt19937 gen(rd());
42+
std::vector<unsigned int> indexes;
43+
indexes.reserve(kNEvents);
44+
for (unsigned int i = 0; i < kNEvents; ++i)
45+
indexes.emplace_back(i);
46+
std::shuffle(indexes.begin(), indexes.end(), gen);
47+
48+
std::bernoulli_distribution dist(0.5);
49+
for (auto idx : indexes) {
50+
if (dist(gen)) {
51+
EXPECT_DOUBLE_EQ(idx, viewPt(idx));
52+
EXPECT_DOUBLE_EQ(idx, viewPtAsFloat(idx));
53+
} else {
54+
EXPECT_DOUBLE_EQ(idx, viewPtAsFloat(idx));
55+
EXPECT_DOUBLE_EQ(idx, viewPt(idx));
56+
}
57+
}
58+
}

Diff for: ‎tree/ntuple/v7/test/ntuple_randomaccess.cxx

+50
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
#include <ROOT/RConfig.hxx>
2+
3+
#include "ntuple_test.hxx"
4+
5+
#include <TRandom3.h>
6+
7+
#include <algorithm>
8+
9+
// Stress test the asynchronous cluster pool by a deliberately unfavourable read pattern
10+
TEST(RNTuple, RandomAccess)
11+
{
12+
#ifdef R__USE_IMT
13+
IMTRAII _;
14+
#endif
15+
FileRaii fileGuard("test_ntuple_random_access.root");
16+
17+
auto modelWrite = RNTupleModel::Create();
18+
auto wrValue = modelWrite->MakeField<std::int32_t>("value");
19+
20+
constexpr unsigned int nEvents = 1000000;
21+
{
22+
RNTupleWriteOptions options;
23+
options.SetCompression(0);
24+
options.SetEnablePageChecksums(false);
25+
options.SetApproxZippedClusterSize(nEvents * sizeof(std::int32_t) / 10);
26+
auto ntuple = RNTupleWriter::Recreate(std::move(modelWrite), "myNTuple", fileGuard.GetPath(), options);
27+
for (unsigned int i = 0; i < nEvents; ++i) {
28+
*wrValue = i;
29+
ntuple->Fill();
30+
}
31+
}
32+
33+
RNTupleReadOptions options;
34+
options.SetClusterCache(RNTupleReadOptions::EClusterCache::kOn);
35+
auto ntuple = RNTupleReader::Open("myNTuple", fileGuard.GetPath(), options);
36+
EXPECT_EQ(10, ntuple->GetDescriptor().GetNClusters());
37+
38+
auto viewValue = ntuple->GetView<std::int32_t>("value");
39+
40+
std::int64_t sum = 0;
41+
std::int64_t expected = 0;
42+
constexpr unsigned int nSamples = 50000;
43+
TRandom3 rnd(42);
44+
for (unsigned int i = 0; i < nSamples; ++i) {
45+
auto entryId = floor(rnd.Rndm() * (nEvents - 1));
46+
expected += entryId;
47+
sum += viewValue(entryId);
48+
}
49+
EXPECT_EQ(expected, sum);
50+
}

0 commit comments

Comments
 (0)