Skip to content

Commit 08d828e

Browse files
makortelmakortel
committed
Add a new test for reduced ProcessHistory using mocked CMSSW release version
The existing test relies on old (4_2_X-time) files.
1 parent 5cbf6c4 commit 08d828e

File tree

7 files changed

+215
-10
lines changed

7 files changed

+215
-10
lines changed

‎FWCore/Framework/src/ScheduleItems.cc

+9-2
Original file line numberDiff line numberDiff line change
@@ -134,8 +134,15 @@ namespace edm {
134134

135135
act_table_ = std::make_unique<ExceptionToActionTable>(parameterSet);
136136
std::string processName = parameterSet.getParameter<std::string>("@process_name");
137-
processConfiguration_ = std::make_shared<ProcessConfiguration>(
138-
processName, getReleaseVersion(), getPassID()); // propagate_const<T> has no reset() function
137+
std::string releaseVersion;
138+
if (parameterSet.existsAs<std::string>("@special_override_release_version_only_for_testing", false)) {
139+
releaseVersion =
140+
parameterSet.getUntrackedParameter<std::string>("@special_override_release_version_only_for_testing");
141+
} else {
142+
releaseVersion = getReleaseVersion();
143+
}
144+
// propagate_const<T> has no reset() function
145+
processConfiguration_ = std::make_shared<ProcessConfiguration>(processName, releaseVersion, getPassID());
139146
auto common = std::make_shared<CommonParams>(
140147
parameterSet.getUntrackedParameterSet("maxEvents").getUntrackedParameter<int>("input"),
141148
parameterSet.getUntrackedParameterSet("maxLuminosityBlocks").getUntrackedParameter<int>("input"),

‎FWCore/ParameterSet/python/Config.py

+19
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,7 @@ def __init__(self,name: str, *Mods):
149149
self.__isStrict = False
150150
self.__dict__['_Process__modifiers'] = Mods
151151
self.__dict__['_Process__accelerators'] = {}
152+
self.__dict__['_Process__specialOverrideReleaseVersionOnlyForTesting'] = None
152153
self.__injectValidValue('options', Process.defaultOptions_())
153154
self.__injectValidValue('maxEvents', Process.defaultMaxEvents_())
154155
self.maxLuminosityBlocks = Process.defaultMaxLuminosityBlocks_()
@@ -813,6 +814,10 @@ def extend(self,other,items=()):
813814

814815
self.__dict__['_Process__InExtendCall'] = False
815816

817+
def _specialOverrideReleaseVersionOnlyForTesting(self, version):
818+
"This function is intended only for specific framework tests. Do not use for anything else."
819+
self.__specialOverrideReleaseVersionOnlyForTesting = version
820+
816821
def _dumpConfigNamedList(self,items,typeName:str,options:PrintOptions) -> str:
817822
returnValue = ''
818823
for name,item in items:
@@ -1458,6 +1463,8 @@ def __extractPSet(self,pset):
14581463

14591464
self.validate()
14601465
processPSet.addString(True, "@process_name", self.name_())
1466+
if self.__specialOverrideReleaseVersionOnlyForTesting is not None:
1467+
processPSet.addString(False, "@special_override_release_version_only_for_testing", self.__specialOverrideReleaseVersionOnlyForTesting)
14611468
self.handleProcessAccelerators(processPSet)
14621469
all_modules = self.producers_().copy()
14631470
all_modules.update(self.filters_())
@@ -4944,4 +4951,16 @@ def testProcessAccelerator(self):
49444951
self.assertEqual((False, "sp@test1"), p.values["sp"][1].values["@chosen_case"])
49454952
self.assertEqual(["cpu", "test1"], p.values["@available_accelerators"][1])
49464953

4954+
def testProcessSpecialOverrideReleaseVersion(self):
4955+
proc = Process("TEST")
4956+
p = TestMakePSet()
4957+
proc.fillProcessDesc(p)
4958+
self.assertFalse("@special_override_release_version_only_for_testing" in p.values)
4959+
4960+
proc = Process("TEST")
4961+
proc._specialOverrideReleaseVersionOnlyForTesting("CMSSW_15_0_0")
4962+
p = TestMakePSet()
4963+
proc.fillProcessDesc(p)
4964+
self.assertEqual((False, "CMSSW_15_0_0"), p.values["@special_override_release_version_only_for_testing"])
4965+
49474966
unittest.main()

‎IOPool/Input/test/BuildFile.xml

+1
Original file line numberDiff line numberDiff line change
@@ -38,5 +38,6 @@
3838
<lib name="SchemaEvolutionTest"/>
3939
</test>
4040
<test name="TestIOPoolInputRefProductIDMetadataConsistency" command="testRefProductIDMetadataConsistencyRoot.sh"/>
41+
<test name="TestIOPoolInputReducedProcessHistory" command="testReducedProcessHistory.sh"/>
4142

4243
</environment>

‎IOPool/Input/test/testReducedProcessHistory.sh

+33
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
#!/bin/bash
2+
3+
function die { echo $1: status $2 ; exit $2; }
4+
5+
VERSION_ARR=(${CMSSW_VERSION//_/ })
6+
VERSION1="${VERSION_ARR[0]}_${VERSION_ARR[1]}_${VERSION_ARR[2]}_0"
7+
VERSION2="${VERSION_ARR[0]}_${VERSION_ARR[1]}_${VERSION_ARR[2]}_1"
8+
VERSION3="${VERSION_ARR[0]}_${VERSION_ARR[1]}_$((${VERSION_ARR[2]}+1))_0"
9+
10+
# Check that changing the patch version does not lead to new run
11+
cmsRun ${SCRAM_TEST_PATH}/testReducedProcessHistoryCreate_cfg.py --version ${VERSION1} --lumi 1 --output version1.root || die "testReducedProcessHistoryCreate_cfg.py --version ${VERSION1} failed" $?
12+
cmsRun ${SCRAM_TEST_PATH}/testReducedProcessHistoryCreate_cfg.py --version ${VERSION2} --lumi 2 --output version2.root || die "testReducedProcessHistoryCreate_cfg.py --version ${VERSION2} failed" $?
13+
14+
edmProvDump version1.root | grep -q "PROD.*'${VERSION1}'" || die "Did not find ${VERSION1} from version.root provenance" $?
15+
edmProvDump version2.root | grep -q "PROD.*'${VERSION2}'" || die "Did not find ${VERSION2} from version.root provenance" $?
16+
17+
cmsRun ${SCRAM_TEST_PATH}/test_merge_two_files.py version1.root version2.root || die "Merge version1.root version2.root failed" $?
18+
19+
cmsRun ${SCRAM_TEST_PATH}/testReducedProcessHistory_cfg.py --input merged_files.root || die "testReducedProcessHistory_cfg.py merged_files.root failed" $?
20+
21+
22+
# Check that changing the minor version leads to new run
23+
cmsRun ${SCRAM_TEST_PATH}/testReducedProcessHistoryCreate_cfg.py --version ${VERSION3} --lumi 3 --output version3.root || die "testReducedProcessHistoryCreate_cfg.py --version ${VERSION3} failed" $?
24+
25+
edmProvDump version3.root | grep -q "PROD.*'${VERSION3}'" || die "Did not find ${VERSION3} from version.root provenance" $?
26+
27+
cmsRun ${SCRAM_TEST_PATH}/test_merge_two_files.py version1.root version3.root --output merged_files3.root --bypassVersionCheck|| die "Merge version1.root version3.root failed" $?
28+
29+
cmsRun ${SCRAM_TEST_PATH}/testReducedProcessHistory_cfg.py --input merged_files3.root --bypassVersionCheck && die "testReducedProcessHistory_cfg.py merged_files3.root did not fail" 1
30+
31+
cmsRun ${SCRAM_TEST_PATH}/testReducedProcessHistory_cfg.py --input merged_files3.root --bypassVersionCheck --expectNewRun || die "testReducedProcessHistory_cfg.py merged_files3.root --expectNewRun failed" $?
32+
33+
exit 0

‎IOPool/Input/test/testReducedProcessHistoryCreate_cfg.py

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
import FWCore.ParameterSet.Config as cms
2+
import argparse
3+
4+
parser = argparse.ArgumentParser(description='Create files for reduced ProcessHistory test')
5+
parser.add_argument("--version", type=str, help="CMSSW version to be used in the ProcessHistory")
6+
parser.add_argument("--lumi", type=int, help="LuminosityBlock number")
7+
parser.add_argument("--output", type=str, help="Output file name")
8+
9+
args = parser.parse_args()
10+
11+
process = cms.Process("PROD")
12+
process._specialOverrideReleaseVersionOnlyForTesting(args.version)
13+
14+
process.maxEvents.input = 10
15+
16+
from FWCore.Modules.modules import EmptySource
17+
process.source = EmptySource(
18+
firstLuminosityBlock = args.lumi
19+
)
20+
21+
from IOPool.Output.modules import PoolOutputModule
22+
process.out = PoolOutputModule(
23+
fileName = args.output
24+
)
25+
26+
from FWCore.Framework.modules import IntProducer
27+
process.intProducer = IntProducer(ivalue = 42)
28+
29+
from FWCore.Integration.modules import ThingWithMergeProducer
30+
process.thingWithMergeProducer = ThingWithMergeProducer()
31+
32+
process.t = cms.Task(
33+
process.intProducer,
34+
process.thingWithMergeProducer,
35+
)
36+
process.p = cms.Path(process.t)
37+
process.ep = cms.EndPath(process.out)

‎IOPool/Input/test/testReducedProcessHistory_cfg.py

+98
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
import FWCore.ParameterSet.Config as cms
2+
import argparse
3+
4+
parser = argparse.ArgumentParser(description='Test reduced ProcessHistory')
5+
parser.add_argument("--input", type=str, help="Input file")
6+
parser.add_argument("--bypassVersionCheck", action="store_true", help="Bypass version check")
7+
parser.add_argument("--expectNewRun", action="store_true", help="Set this if a new run is expected between the original files")
8+
9+
args = parser.parse_args()
10+
11+
process = cms.Process("READ")
12+
13+
from IOPool.Input.modules import PoolSource
14+
process.source = PoolSource(
15+
fileNames = [f"file:{args.input}"],
16+
bypassVersionCheck = args.bypassVersionCheck,
17+
)
18+
19+
from FWCore.Framework.modules import TestMergeResults, RunLumiEventAnalyzer
20+
process.testmerge = TestMergeResults(
21+
expectedBeginRunProd = [10001, 20004, 10003], # should be only one run
22+
expectedEndRunProd = [100001, 200004, 100003],
23+
expectedBeginLumiProd = [101, 102, 103,
24+
101, 102, 103],
25+
expectedEndLumiProd = [1001, 1002, 1003,
26+
1001, 1002, 1003],
27+
28+
expectedBeginRunNew = [10001, 10002, 10003], # should be only one run
29+
expectedEndRunNew = [100001, 100002, 100003],
30+
expectedBeginLumiNew = [101, 102, 103,
31+
101, 102, 103],
32+
expectedEndLumiNew = [1001, 1002, 1003,
33+
1001, 1002, 1003],
34+
expectedProcessHistoryInRuns = [
35+
'PROD',
36+
'MERGETWOFILES',
37+
'READ'
38+
]
39+
)
40+
if args.expectNewRun:
41+
process.testmerge.expectedBeginRunProd[1] = 10002
42+
process.testmerge.expectedEndRunProd[1] = 100002
43+
44+
process.test = RunLumiEventAnalyzer(
45+
expectedRunLumiEvents = [
46+
1, 0, 0, # beginRun
47+
1, 1, 0, # beginLumi
48+
1, 1, 1,
49+
1, 1, 2,
50+
1, 1, 3,
51+
1, 1, 4,
52+
1, 1, 5,
53+
1, 1, 6,
54+
1, 1, 7,
55+
1, 1, 8,
56+
1, 1, 9,
57+
1, 1, 10,
58+
1, 1, 0, # endLumi
59+
1, 2, 0, # beginLumi
60+
1, 2, 1,
61+
1, 2, 2,
62+
1, 2, 3,
63+
1, 2, 4,
64+
1, 2, 5,
65+
1, 2, 6,
66+
1, 2, 7,
67+
1, 2, 8,
68+
1, 2, 9,
69+
1, 2, 10,
70+
1, 2, 0, # endLumi
71+
1, 0, 0, # endRun
72+
]
73+
)
74+
if args.expectNewRun:
75+
# Insert endRun+beginRun to the expected transition sequence
76+
endLumiIndex = 3*(10+3)
77+
process.test.expectedRunLumiEvents = process.test.expectedRunLumiEvents[:endLumiIndex] + [
78+
1, 0, 0, # endRun
79+
1, 0, 0, # beginRun
80+
1, 3, 0, # beginLumi
81+
1, 3, 1,
82+
1, 3, 2,
83+
1, 3, 3,
84+
1, 3, 4,
85+
1, 3, 5,
86+
1, 3, 6,
87+
1, 3, 7,
88+
1, 3, 8,
89+
1, 3, 9,
90+
1, 3, 10,
91+
1, 3, 0, # endLumi
92+
1, 0, 0, # endRun
93+
]
94+
95+
process.p = cms.Path(
96+
process.testmerge +
97+
process.test
98+
)

‎IOPool/Input/test/test_merge_two_files.py

+18-8
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,28 @@
11
import FWCore.ParameterSet.Config as cms
2-
import sys
2+
import argparse
3+
4+
parser = argparse.ArgumentParser(description='Create files for reduced ProcessHistory test')
5+
parser.add_argument("file1", type=str, help="First file to merge")
6+
parser.add_argument("file2", type=str, help="Second file to merge")
7+
parser.add_argument("--output", default="merged_files.root", help="Output file name")
8+
parser.add_argument("--bypassVersionCheck", action="store_true", help="Bypass version check")
9+
10+
args = parser.parse_args()
311

412
process = cms.Process("MERGETWOFILES")
513

6-
process.source = cms.Source("PoolSource",
7-
fileNames = cms.untracked.vstring("file:"+sys.argv[1],
8-
"file:"+sys.argv[2]),
9-
duplicateCheckMode = cms.untracked.string("noDuplicateCheck")
14+
from IOPool.Input.modules import PoolSource
15+
process.source = PoolSource(
16+
fileNames = ["file:"+args.file1,"file:"+args.file2],
17+
duplicateCheckMode = "noDuplicateCheck",
18+
bypassVersionCheck = args.bypassVersionCheck,
1019
)
1120

12-
process.thingWithMergeProducer = cms.EDProducer("ThingWithMergeProducer")
21+
from FWCore.Integration.modules import ThingWithMergeProducer
22+
process.thingWithMergeProducer = ThingWithMergeProducer()
1323

14-
process.out = cms.OutputModule("PoolOutputModule",
15-
fileName = cms.untracked.string("merged_files.root"))
24+
from IOPool.Output.modules import PoolOutputModule
25+
process.out = PoolOutputModule(fileName = args.output)
1626

1727
process.p = cms.Path(process.thingWithMergeProducer)
1828

0 commit comments

Comments
 (0)