Skip to content

Commit 3363930

Browse files
guitargeekanigamova
guitargeek
authored and
anigamova
committed
Automatically create script to recreate the reference files
1 parent 9832dc7 commit 3363930

File tree

1 file changed

+127
-78
lines changed

1 file changed

+127
-78
lines changed

‎test/CMakeLists.txt‎

Lines changed: 127 additions & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -26,106 +26,130 @@ function(COMBINE_ADD_GTEST test_suite)
2626
)
2727
endfunction()
2828

29+
30+
# Arguments:
31+
# TEST_BASENAME Base name of the test (e.g. "parametric_analysis" or "cmshistsum")
32+
# COPY_TO_BUILDDIR List of required files to copy (e.g. datacard and ROOT inputs)
33+
# T2W_COMMAND
34+
# COMBINE_COMMANDS
35+
function(ADD_COMBINE_TEST TEST_BASENAME)
36+
cmake_parse_arguments(ARG "" "" "COPY_TO_BUILDDIR;T2W_COMMAND;COMBINE_COMMANDS" ${ARGN})
37+
38+
# --- text2workspace test ---
39+
ROOT_ADD_TEST(${TEST_BASENAME}-text2workspace
40+
COMMAND ${CMAKE_BINARY_DIR}/bin/${ARG_T2W_COMMAND}
41+
COPY_TO_BUILDDIR ${ARG_COPY_TO_BUILDDIR}
42+
FIXTURES_SETUP ${TEST_BASENAME}
43+
ENVIRONMENT
44+
LD_LIBRARY_PATH=${CMAKE_BINARY_DIR}/lib:$ENV{LD_LIBRARY_PATH}
45+
PYTHONPATH=${CMAKE_BINARY_DIR}/python:$ENV{PYTHONPATH}
46+
)
47+
48+
# Combine multiple commands into a single shell command chain
49+
set(_combined_command "")
50+
foreach(cmd IN LISTS ARG_COMBINE_COMMANDS)
51+
string(APPEND _combined_command "${cmd} && ")
52+
set_property(GLOBAL APPEND PROPERTY ALL_COMBINE_COMMANDS "${cmd} >> references/${TEST_BASENAME}.out")
53+
endforeach()
54+
string(REGEX REPLACE " && $" "" _combined_command "${_combined_command}") # remove trailing &&
55+
56+
# --- combine test ---
57+
ROOT_ADD_TEST(${TEST_BASENAME}
58+
COMMAND bash -c "${_combined_command}"
59+
FIXTURES_REQUIRED ${TEST_BASENAME} # requires corresponding text2workspace run
60+
# We compare the output to reference files to validate the best-fit parameter values
61+
WORKING_DIR ${CMAKE_CURRENT_BINARY_DIR}
62+
CHECKOUT OUTPUT ${TEST_BASENAME}.out OUTREF ${CMAKE_CURRENT_SOURCE_DIR}/references/${TEST_BASENAME}.out
63+
# For this test, we are not interested in the standard error, but you could
64+
# compare this too:
65+
# CHECKERR
66+
# ERROR ${TEST_BASENAME}.err
67+
# ERRREF ${CMAKE_CURRENT_SOURCE_DIR}/references/${TEST_BASENAME}.err
68+
ENVIRONMENT PATH=${CMAKE_BINARY_DIR}/bin:$ENV{PATH}
69+
)
70+
endfunction()
71+
2972
# Set the location of text2workspace.py, assuming installation to a virtual
3073
# environment inside the source directory.
3174
set(t2w ${CMAKE_BINARY_DIR}/bin/text2workspace.py)
32-
set(combineTool ${CMAKE_BINARY_DIR}/bin/combineTool.py)
33-
34-
35-
# Counting datacard - text2workspace
36-
ROOT_ADD_TEST(counting_datacard-text2workspace
37-
COMMAND ${t2w} toy-hgg-125.txt -m 125 -P HiggsAnalysis.CombinedLimit.PhysicsModel:floatingXSHiggs --PO modes=ggH,qqH
38-
COPY_TO_BUILDDIR ${CMAKE_SOURCE_DIR}/data/tutorials/multiDim/toy-hgg-125.txt
39-
FIXTURES_SETUP counting_datacard
40-
# Append the build directory to the library Path to we can use Combine classes
41-
ENVIRONMENT LD_LIBRARY_PATH=${CMAKE_BINARY_DIR}/lib:$ENV{LD_LIBRARY_PATH}
42-
PYTHONPATH=${CMAKE_BINARY_DIR}/python:$ENV{PYTHONPATH}
43-
)
4475

45-
# Counting datacard - combine
46-
ROOT_ADD_TEST(counting_datacard
47-
COMMAND combine -M MultiDimFit toy-hgg-125.root --setParameterRanges r=-1,1
48-
FIXTURES_REQUIRED counting_datacard # requires corresponding text2workspace run
49-
# We compare the output to reference files to validate the best-fit parameter values
50-
WORKING_DIR ${CMAKE_CURRENT_BINARY_DIR}
51-
CHECKOUT
52-
OUTPUT counting_datacard.out
53-
OUTREF ${CMAKE_CURRENT_SOURCE_DIR}/references/counting_datacard.out
54-
# For this test, we are not interested in the standard error, but you could
55-
# compare this too:
56-
# CHECKERR
57-
# ERROR counting_datacard.err
58-
# ERRREF ${CMAKE_CURRENT_SOURCE_DIR}/references/counting_datacard.err
76+
# Counting datacard
77+
ADD_COMBINE_TEST(counting_datacard
78+
COPY_TO_BUILDDIR ${CMAKE_SOURCE_DIR}/data/tutorials/multiDim/toy-hgg-125.txt
79+
T2W_COMMAND
80+
text2workspace.py toy-hgg-125.txt -m 125 -P HiggsAnalysis.CombinedLimit.PhysicsModel:floatingXSHiggs --PO modes=ggH,qqH
81+
COMBINE_COMMANDS
82+
"combine -M MultiDimFit toy-hgg-125.root --setParameterRanges r=-1,1"
5983
)
6084

6185
# Counting datacard Fixed Point from csv - combineTool.py
6286
ROOT_ADD_TEST(counting_datacard_from_csv
63-
COMMAND ${combineTool} -M MultiDimFit toy-hgg-125.root --fromfile fixed.csv
87+
COMMAND ${CMAKE_BINARY_DIR}/bin/combineTool.py -M MultiDimFit toy-hgg-125.root --fromfile fixed.csv
6488
COPY_TO_BUILDDIR ${CMAKE_SOURCE_DIR}/data/tutorials/multiDim/fixed.csv
6589
FIXTURES_REQUIRED counting_datacard
6690
WORKING_DIR ${CMAKE_CURRENT_BINARY_DIR}
6791
CHECKOUT OUTPUT counting_datacard_from_csv.out OUTREF ${CMAKE_CURRENT_SOURCE_DIR}/references/counting_datacard_from_csv.out
6892
ENVIRONMENT PATH=${CMAKE_BINARY_DIR}/bin:$ENV{PATH} # So the combineTool.py finds the combine executable
6993
PYTHONPATH=${CMAKE_BINARY_DIR}/python:$ENV{PYTHONPATH}
7094
)
71-
72-
# Parametric analysis - text2workspace
73-
ROOT_ADD_TEST(parametric_analysis-text2workspace
74-
COMMAND ${t2w} datacard-3-parametric-analysis.txt --mass 125
75-
COPY_TO_BUILDDIR ${CMAKE_SOURCE_DIR}/data/tutorials/CAT23001/datacard-3-parametric-analysis.txt
76-
COPY_TO_BUILDDIR ${CMAKE_SOURCE_DIR}/data/tutorials/CAT23001/parametric-analysis-datacard-input.root
77-
FIXTURES_SETUP parametric_analysis
78-
ENVIRONMENT LD_LIBRARY_PATH=${CMAKE_BINARY_DIR}/lib:$ENV{LD_LIBRARY_PATH}
79-
PYTHONPATH=${CMAKE_BINARY_DIR}/python:$ENV{PYTHONPATH}
95+
set_property(GLOBAL APPEND PROPERTY ALL_COMBINE_COMMANDS "combineTool.py -M MultiDimFit toy-hgg-125.root --fromfile fixed.csv >> references/counting_datacard_from_csv.out")
96+
97+
# Parametric analysis
98+
ADD_COMBINE_TEST(parametric_analysis
99+
COPY_TO_BUILDDIR ${CMAKE_SOURCE_DIR}/data/tutorials/CAT23001/datacard-3-parametric-analysis.txt
100+
${CMAKE_SOURCE_DIR}/data/tutorials/CAT23001/parametric-analysis-datacard-input.root
101+
T2W_COMMAND
102+
text2workspace.py datacard-3-parametric-analysis.txt --mass 125
103+
COMBINE_COMMANDS
104+
"combine -M MultiDimFit datacard-3-parametric-analysis.root --algo singles --setParameterRanges r=-2,1"
80105
)
81106

82-
# Parametric analysis - combine
83-
ROOT_ADD_TEST(parametric_analysis
84-
COMMAND combine -M MultiDimFit datacard-3-parametric-analysis.root --algo singles --setParameterRanges r=-2,1
85-
FIXTURES_REQUIRED parametric_analysis
86-
WORKING_DIR ${CMAKE_CURRENT_BINARY_DIR}
87-
CHECKOUT OUTPUT parametric_analysis.out OUTREF ${CMAKE_CURRENT_SOURCE_DIR}/references/parametric_analysis.out
107+
# TODO: make these tests work too
108+
#
109+
# # Template analysis CMSHistFunc
110+
# ADD_COMBINE_TEST(cmshistfunc
111+
# COPY_TO_BUILDDIR ${CMAKE_SOURCE_DIR}/data/ci/template-analysis_shapeInterp.txt
112+
# ${CMAKE_SOURCE_DIR}/data/ci/htt_input.root
113+
# T2W_COMMAND
114+
# text2workspace.py template-analysis_shapeInterp.txt -o ws_template-analysis.root --mass 200
115+
# COMBINE_COMMANDS
116+
# "combine -M MultiDimFit ws_template-analysis.root --algo singles --setParameterRanges r=-1,1"
117+
# "combine -M FitDiagnostics ws_template-analysis.root -t -1 --setParameters r=1 --plots --setParameterRanges r=-1,1"
118+
# )
119+
#
120+
# # Template analysis CMSHistFunc shapeN
121+
# ADD_COMBINE_TEST(cmshistfunc_shapeN
122+
# COPY_TO_BUILDDIR ${CMAKE_SOURCE_DIR}/data/ci/template-analysis_shapeNInterp.txt
123+
# ${CMAKE_SOURCE_DIR}/data/ci/htt_input.root
124+
# T2W_COMMAND
125+
# text2workspace.py template-analysis_shapeNInterp.txt -o ws_template-analysis-shapeN.root --mass 200
126+
# COMBINE_COMMANDS
127+
# "combine -M MultiDimFit ws_template-analysis-shapeN.root --algo singles --setParameterRanges r=-1,1"
128+
# "combine -M FitDiagnostics ws_template-analysis-shapeN.root -t -1 --setParameters r=1 --plots --setParameterRanges r=-1,1"
129+
# )
130+
131+
# Template analysis CMSHistSum
132+
ADD_COMBINE_TEST(cmshistsum
133+
COPY_TO_BUILDDIR ${CMAKE_SOURCE_DIR}/data/ci/template-analysis_shapeInterp.txt
134+
${CMAKE_SOURCE_DIR}/data/ci/htt_input.root
135+
T2W_COMMAND
136+
text2workspace.py template-analysis_shapeInterp.txt --mass 200 --for-fits --no-wrappers --use-histsum
137+
COMBINE_COMMANDS
138+
"combine -M MultiDimFit template-analysis_shapeInterp.root --algo singles --setParameterRanges r=-1,1 --X-rtd FAST_VERTICAL_MORPH"
88139
)
89140

90-
91-
# Template analysis CMSHistSum - text2workspace
92-
ROOT_ADD_TEST(cmshistsum-text2workspace
93-
COMMAND ${t2w} template-analysis_shapeInterp.txt --mass 200 --for-fits --no-wrappers --use-histsum
94-
COPY_TO_BUILDDIR ${CMAKE_SOURCE_DIR}/data/ci/template-analysis_shapeInterp.txt
95-
COPY_TO_BUILDDIR ${CMAKE_SOURCE_DIR}/data/ci/htt_input.root
96-
FIXTURES_SETUP cmshistsum
97-
ENVIRONMENT LD_LIBRARY_PATH=${CMAKE_BINARY_DIR}/lib:$ENV{LD_LIBRARY_PATH}
98-
PYTHONPATH=${CMAKE_BINARY_DIR}/python:$ENV{PYTHONPATH}
99-
)
100-
101-
# Template analysis CMSHistSum - combine
102-
ROOT_ADD_TEST(cmshistsum
103-
COMMAND combine -M MultiDimFit template-analysis_shapeInterp.root --algo singles --setParameterRanges r=-1,1 --X-rtd FAST_VERTICAL_MORPH
104-
FIXTURES_REQUIRED cmshistsum
105-
WORKING_DIR ${CMAKE_CURRENT_BINARY_DIR}
106-
CHECKOUT OUTPUT cmshistsum.out OUTREF ${CMAKE_CURRENT_SOURCE_DIR}/references/cmshistsum.out
107-
)
108-
109-
# Template analysis CMSHistSum with shapeN - text2workspace
110-
ROOT_ADD_TEST(cmshistsum_shapeN-text2workspace
111-
COMMAND ${t2w} template-analysis_shapeNInterp.txt --mass 200 --for-fits --no-wrappers --use-histsum
112-
COPY_TO_BUILDDIR ${CMAKE_SOURCE_DIR}/data/ci/template-analysis_shapeNInterp.txt
113-
COPY_TO_BUILDDIR ${CMAKE_SOURCE_DIR}/data/ci/htt_input.root
114-
FIXTURES_SETUP cmshistsum_shapeN
115-
ENVIRONMENT LD_LIBRARY_PATH=${CMAKE_BINARY_DIR}/lib:$ENV{LD_LIBRARY_PATH}
116-
PYTHONPATH=${CMAKE_BINARY_DIR}/python:$ENV{PYTHONPATH}
117-
)
118-
119-
# Template analysis CMSHistSum with shapeN - combine
120-
ROOT_ADD_TEST(cmshistsum_shapeN
121-
COMMAND combine -M MultiDimFit template-analysis_shapeNInterp.root --algo singles --setParameterRanges r=-1,1 --X-rtd FAST_VERTICAL_MORPH
122-
FIXTURES_REQUIRED cmshistsum_shapeN
123-
WORKING_DIR ${CMAKE_CURRENT_BINARY_DIR}
124-
CHECKOUT OUTPUT cmshistsum_shapeN.out OUTREF ${CMAKE_CURRENT_SOURCE_DIR}/references/cmshistsum_shapeN.out
141+
# Template analysis CMSHistSum with shapeN
142+
ADD_COMBINE_TEST(cmshistsum_shapeN
143+
COPY_TO_BUILDDIR ${CMAKE_SOURCE_DIR}/data/ci/template-analysis_shapeNInterp.txt
144+
${CMAKE_SOURCE_DIR}/data/ci/htt_input.root
145+
T2W_COMMAND
146+
text2workspace.py template-analysis_shapeNInterp.txt --mass 200 --for-fits --no-wrappers --use-histsum
147+
COMBINE_COMMANDS
148+
"combine -M MultiDimFit template-analysis_shapeNInterp.root --algo singles --setParameterRanges r=-1,1 --X-rtd FAST_VERTICAL_MORPH"
125149
)
126150

127151
# Template-analysis datacard -> text2workspace
128-
ROOT_ADD_TEST(template_analysis-t2w
152+
ROOT_ADD_TEST(template_analysis-text2workspace
129153
COMMAND ${t2w} template-analysis_shape_autoMCStats.txt -o template-analysis_shape_autoMCStats.root
130154
COPY_TO_BUILDDIR
131155
${CMAKE_SOURCE_DIR}/data/ci/template-analysis_shape_autoMCStats.txt
@@ -143,3 +167,28 @@ COMBINE_ADD_GTEST(template_analysis-testCreateNLL
143167
set_property(TEST gtest-template-analysis-testCreateNLL
144168
PROPERTY FIXTURES_REQUIRED template_analysis_workspace
145169
)
170+
171+
172+
# Write the script to produce the reference files. After building combine, you
173+
# can produce the reference files as follows:
174+
#
175+
# cd build
176+
# ctest -j8 .
177+
# cd test
178+
# sh create_reference_files.sh
179+
#
180+
# You have now a new reference files directory in build/test/references that
181+
# you can use to replace test/references in the repository.
182+
183+
get_property(all_combine_commands GLOBAL PROPERTY ALL_COMBINE_COMMANDS)
184+
message(STATUS "All combine commands:")
185+
set(_outfile "${CMAKE_BINARY_DIR}/test/create_reference_files.sh")
186+
file(WRITE "${_outfile}" "#!/usr/bin/env bash\n\n")
187+
file(APPEND "${_outfile}" "mkdir references\n\n")
188+
file(APPEND "${_outfile}" "PATH=${CMAKE_BINARY_DIR}/bin:$PATH\n")
189+
file(APPEND "${_outfile}" "LD_LIBRARY_PATH=${CMAKE_BINARY_DIR}/lib:$LD_LIBRARY_PATH\n")
190+
file(APPEND "${_outfile}" "PYTHONPATH=${CMAKE_BINARY_DIR}/python:$PYTHONPATH\n\n")
191+
foreach(cmd IN LISTS all_combine_commands)
192+
file(APPEND "${_outfile}" "${cmd}\n")
193+
endforeach()
194+
message(STATUS "Wrote script to produce test reference files ${_outfile}")

0 commit comments

Comments
 (0)