Skip to content

provide a fillDescriptions method for CondDBESSource #47630

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Mar 23, 2025
Merged

provide a fillDescriptions method for CondDBESSource #47630

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4cf7332
provide a fillDescriptions for CondDBESSource
mmusich Mar 19, 2025
e8b046b
add customization for HLT menu for removal of unsupported parameters
mmusich Mar 19, 2025
cbe91c6
unit tests for loading conditions (and miscellaneous fixes to make th…
mmusich Mar 22, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
123 changes: 80 additions & 43 deletions CondCore/ESSources/plugins/CondDBESSource.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
#include <exception>

#include <iomanip>
#include <limits>

#include <nlohmann/json.hpp>

Expand Down Expand Up @@ -123,8 +124,9 @@ namespace {
CondDBESSource::CondDBESSource(const edm::ParameterSet& iConfig)
: m_jsonDumpFilename(iConfig.getUntrackedParameter<std::string>("JsonDumpFileName", "")),
m_connection(),
m_connectionString(""),
m_frontierKey(""),
m_connectionString(iConfig.getParameter<std::string>("connect")),
m_globalTag(iConfig.getParameter<std::string>("globaltag")),
m_frontierKey(iConfig.getUntrackedParameter<std::string>("frontierKey", "")),
m_lastRun(0), // for the stat
m_lastLumi(0), // for the stat
m_policy(NOREFRESH),
Expand All @@ -147,69 +149,65 @@ CondDBESSource::CondDBESSource(const edm::ParameterSet& iConfig)

/*parameter set parsing
*/
std::string globaltag("");
if (iConfig.exists("globaltag")) {
globaltag = iConfig.getParameter<std::string>("globaltag");
// the global tag _requires_ a connection string
m_connectionString = iConfig.getParameter<std::string>("connect");

if (!globaltag.empty()) {
edm::Service<edm::SiteLocalConfig> siteLocalConfig;
if (siteLocalConfig.isAvailable()) {
if (siteLocalConfig->useLocalConnectString()) {
std::string const& localConnectPrefix = siteLocalConfig->localConnectPrefix();
std::string const& localConnectSuffix = siteLocalConfig->localConnectSuffix();
m_connectionString = localConnectPrefix + globaltag + localConnectSuffix;
}
if (!m_globalTag.empty()) {
edm::Service<edm::SiteLocalConfig> siteLocalConfig;
if (siteLocalConfig.isAvailable()) {
if (siteLocalConfig->useLocalConnectString()) {
std::string const& localConnectPrefix = siteLocalConfig->localConnectPrefix();
std::string const& localConnectSuffix = siteLocalConfig->localConnectSuffix();
m_connectionString = localConnectPrefix + m_globalTag + localConnectSuffix;
}
}
} else if (iConfig.exists("connect")) // default connection string
m_connectionString = iConfig.getParameter<std::string>("connect");

// frontier key
m_frontierKey = iConfig.getUntrackedParameter<std::string>("frontierKey", "");
}

// snapshot
boost::posix_time::ptime snapshotTime;
if (iConfig.exists("snapshotTime")) {
std::string snapshotTimeString = iConfig.getParameter<std::string>("snapshotTime");
if (!snapshotTimeString.empty())
snapshotTime = boost::posix_time::time_from_string(snapshotTimeString);
std::string snapshotTimeString = iConfig.getParameter<std::string>("snapshotTime");
if (!snapshotTimeString.empty()) {
snapshotTime = boost::posix_time::time_from_string(snapshotTimeString);
}

// connection configuration
if (iConfig.exists("DBParameters")) {
edm::ParameterSet connectionPset = iConfig.getParameter<edm::ParameterSet>("DBParameters");
m_connection.setParameters(connectionPset);
}
edm::ParameterSet connectionPset = iConfig.getParameter<edm::ParameterSet>("DBParameters");
m_connection.setParameters(connectionPset);
m_connection.configure();

// load specific record/tag info - it will overwrite the global tag ( if any )
std::map<std::string, cond::GTEntry_t> replacements;
std::map<std::string, boost::posix_time::ptime> specialSnapshots;
if (iConfig.exists("toGet")) {
typedef std::vector<edm::ParameterSet> Parameters;
Parameters toGet = iConfig.getParameter<Parameters>("toGet");

typedef std::vector<edm::ParameterSet> Parameters;
Parameters toGet = iConfig.getParameter<Parameters>("toGet");
if (!toGet.empty()) {
for (Parameters::iterator itToGet = toGet.begin(); itToGet != toGet.end(); ++itToGet) {
std::string recordname = itToGet->getParameter<std::string>("record");
if (recordname.empty())
throw cond::Exception("ESSource: The record name has not been provided in a \"toGet\" entry.");

std::string labelname = itToGet->getUntrackedParameter<std::string>("label", "");
std::string pfn("");
if (m_connectionString.empty() || itToGet->exists("connect"))
pfn = itToGet->getParameter<std::string>("connect");
std::string tag("");
const auto& recordConnection = itToGet->getParameter<std::string>("connect");
if (m_connectionString.empty() || !recordConnection.empty()) {
pfn = recordConnection;
}
std::string tag = itToGet->getParameter<std::string>("tag");
std::string fqTag("");
if (itToGet->exists("tag")) {
tag = itToGet->getParameter<std::string>("tag");

if (!tag.empty()) {
fqTag = cond::persistency::fullyQualifiedTag(tag, pfn);
}

boost::posix_time::ptime tagSnapshotTime =
boost::posix_time::time_from_string(std::string(cond::time::MAX_TIMESTAMP));
if (itToGet->exists("snapshotTime"))
tagSnapshotTime = boost::posix_time::time_from_string(itToGet->getParameter<std::string>("snapshotTime"));
if (itToGet->exists("refreshTime")) {
cond::Time_t refreshTime = itToGet->getParameter<unsigned long long>("refreshTime");

const auto& snapshotTimeTagString = itToGet->getParameter<std::string>("snapshotTime");
if (!snapshotTimeTagString.empty()) {
tagSnapshotTime = boost::posix_time::time_from_string(snapshotTimeTagString);
}

const auto& refreshTimeTag = itToGet->getParameter<unsigned long long>("refreshTime");
if (refreshTimeTag != std::numeric_limits<unsigned long long>::max()) {
cond::Time_t refreshTime = refreshTimeTag;
m_refreshTimeForRecord.insert(std::make_pair(recordname, std::make_pair(refreshTime, true)));
}

Expand All @@ -225,10 +223,10 @@ CondDBESSource::CondDBESSource(const edm::ParameterSet& iConfig)
std::vector<std::string> connectList;
std::vector<std::string> pfnPrefixList;
std::vector<std::string> pfnPostfixList;
if (!globaltag.empty()) {
if (!m_globalTag.empty()) {
std::string pfnPrefix(iConfig.getUntrackedParameter<std::string>("pfnPrefix", ""));
std::string pfnPostfix(iConfig.getUntrackedParameter<std::string>("pfnPostfix", ""));
boost::split(globaltagList, globaltag, boost::is_any_of("|"), boost::token_compress_off);
boost::split(globaltagList, m_globalTag, boost::is_any_of("|"), boost::token_compress_off);
fillList(m_connectionString, connectList, globaltagList.size(), "connection");
fillList(pfnPrefix, pfnPrefixList, globaltagList.size(), "pfnPrefix");
fillList(pfnPostfix, pfnPostfixList, globaltagList.size(), "pfnPostfix");
Expand Down Expand Up @@ -741,6 +739,45 @@ void CondDBESSource::fillTagCollectionFromDB(const std::vector<std::string>& con
}
}

void CondDBESSource::fillDescriptions(edm::ConfigurationDescriptions& descriptions) {
edm::ParameterSetDescription desc;

edm::ParameterSetDescription dbParams;
dbParams.addUntracked<std::string>("authenticationPath", "");
dbParams.addUntracked<int>("authenticationSystem", 0);
dbParams.addUntracked<std::string>("security", "");
dbParams.addUntracked<int>("messageLevel", 0);
dbParams.addUntracked<int>("connectionTimeout", 0);
desc.add("DBParameters", dbParams);

desc.add<std::string>("connect", std::string("frontier://FrontierProd/CMS_CONDITIONS"));
desc.add<std::string>("globaltag", "");
desc.add<std::string>("snapshotTime", "");
desc.addUntracked<std::string>("frontierKey", "");

edm::ParameterSetDescription toGetDesc;
toGetDesc.add<std::string>("record", "");
toGetDesc.add<std::string>("tag", "");
toGetDesc.add<std::string>("snapshotTime", "");
toGetDesc.add<std::string>("connect", "");
toGetDesc.add<unsigned long long>("refreshTime", std::numeric_limits<unsigned long long>::max());
toGetDesc.addUntracked<std::string>("label", "");

std::vector<edm::ParameterSet> default_toGet(1);
desc.addVPSet("toGet", toGetDesc, default_toGet);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why one empty as the default value?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

to validate the layout of the possible PSet, but not putting anything inside.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The framework will do that without injecting a dummy empty one.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The framework will do that without injecting a dummy empty one.

How do you actually propose to modify the code to do that?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I asume the above means an empty toGet vector supplied by the user (which is valid/should be valid overall for no record to get) will get an empty PSet inserted by fillDescriptions, whihc in turn will make the plugin fail. This is not the expected behaviour.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I asume the above means an empty toGet vector supplied by the user (which is valid/should be valid overall for no record to get) will get an empty PSet inserted by fillDescriptions, whihc in turn will make the plugin fail. This is not the expected behaviour.

No, simply no! Please read #47630 (comment)

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This line indeed leads to toGet = cms.VPSet(cms.PSet()) being generated by default (and injected in the configuration validation if the toGet parameter is missing). For example

$ cmsrel CMSSW_15_1_X_2025-03-23-2300
$ cd CMSSW_15_1_X_2025-03-23-2300/src
$ cmsenv
$ python3 -i -m CondCore.ESSources.default_CondDBESource_cfi
>>> default_CondDBESource.toGet
cms.VPSet(cms.PSet(

),
template = cms.PSetTemplate(
    connect = cms.string(''),
    label = cms.untracked.string(''),
    record = cms.string(''),
    refreshTime = cms.uint64(18446744073709551615),
    snapshotTime = cms.string(''),
    tag = cms.string('')
))

To avoid an empty cms.PSet() being injected by default, these lines should be e.g.

  std::vector<edm::ParameterSet> default_toGet;
  desc.addVPSet("toGet", toGetDesc, default_toGet);

or

  desc.addVPSet("toGet", toGetDesc, std::vector<edm::ParameterSet>{});

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OK, I followed up at #47688.


desc.addUntracked<std::string>("JsonDumpFileName", "");
desc.addUntracked<bool>("DumpStat", false);
desc.addUntracked<bool>("ReconnectEachRun", false);
desc.addUntracked<bool>("RefreshAlways", false);
desc.addUntracked<bool>("RefreshEachRun", false);
desc.addUntracked<bool>("RefreshOpenIOVs", false);
desc.addUntracked<std::string>("pfnPostfix", "");
desc.addUntracked<std::string>("pfnPrefix", "");

descriptions.add("default_CondDBESource", desc);
}

// backward compatibility for configuration files
class PoolDBESSource : public CondDBESSource {
public:
Expand Down
4 changes: 4 additions & 0 deletions CondCore/ESSources/plugins/CondDBESSource.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@
#include "FWCore/Framework/interface/ESProductResolverProvider.h"
#include "FWCore/Framework/interface/EventSetupRecordIntervalFinder.h"
#include "FWCore/Concurrency/interface/SerialTaskQueue.h"
#include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h"

namespace edm {
class ParameterSet;
Expand All @@ -98,6 +99,8 @@ class CondDBESSource : public edm::eventsetup::ESProductResolverProvider, public
explicit CondDBESSource(const edm::ParameterSet&);
~CondDBESSource() override;

static void fillDescriptions(edm::ConfigurationDescriptions& descriptions);

protected:
void setIntervalFor(const EventSetupRecordKey&, const edm::IOVSyncValue&, edm::ValidityInterval&) override;

Expand All @@ -112,6 +115,7 @@ class CondDBESSource : public edm::eventsetup::ESProductResolverProvider, public

cond::persistency::ConnectionPool m_connection;
std::string m_connectionString;
std::string m_globalTag;
std::string m_frontierKey;

// Container of ProductResolver, implemented as multi-map keyed by records
Expand Down
30 changes: 17 additions & 13 deletions CondCore/ESSources/python/CondDBESSource_cfi.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,16 +5,20 @@
from CondCore.CondDB.CondDB_cfi import *

CondDBConnection = CondDB.clone( connect = cms.string( 'frontier://FrontierProd/CMS_CONDITIONS' ) )
GlobalTag = cms.ESSource( "PoolDBESSource",
CondDBConnection,
globaltag = cms.string( '' ),
snapshotTime = cms.string( '' ),
toGet = cms.VPSet(), # hook to override or add single payloads
DumpStat = cms.untracked.bool( False ),
ReconnectEachRun = cms.untracked.bool( False ),
RefreshAlways = cms.untracked.bool( False ),
RefreshEachRun = cms.untracked.bool( False ),
RefreshOpenIOVs = cms.untracked.bool( False ),
pfnPostfix = cms.untracked.string( '' ),
pfnPrefix = cms.untracked.string( '' ),
)
from CondCore.ESSources.default_CondDBESource_cfi import PoolDBESSource as _PoolDBESSource

GlobalTag = _PoolDBESSource(
CondDBConnection,
globaltag = '',
snapshotTime = '',
frontierKey = '',
toGet = [], # hook to override or add single payloads
JsonDumpFileName = '',
DumpStat = False,
ReconnectEachRun = False,
RefreshAlways = False,
RefreshEachRun = False,
RefreshOpenIOVs = False,
pfnPostfix = '',
pfnPrefix = '' ,
)
2 changes: 1 addition & 1 deletion CondCore/ESSources/test/BuildFile.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,4 +38,4 @@
<flags EDM_PLUGIN="1"/>
</library>


<test name="UnitTestLoadConditions" command="runConditionsLoadTests.sh"/>
8 changes: 4 additions & 4 deletions CondCore/ESSources/test/python/load_from_globaltag_cfg.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,10 @@
process.maxEvents = cms.untracked.PSet( input = cms.untracked.int32(100) )

process.source = cms.Source("EmptyIOVSource",
lastValue = cms.uint64(3),
timetype = cms.string('runnumber'),
firstValue = cms.uint64(1),
interval = cms.uint64(1)
lastValue = cms.uint64(3),
timetype = cms.string('runnumber'),
firstValue = cms.uint64(1),
interval = cms.uint64(1)
)

from CondCore.ESSources.GlobalTag import GlobalTag
Expand Down
2 changes: 1 addition & 1 deletion CondCore/ESSources/test/python/load_record_empty_source_cfg.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@

process.source = cms.Source( "EmptySource",
firstRun = cms.untracked.uint32( options.runNumber ),
firstTime = cms.untracked.uint64( ( long( time.time() ) - 24 * 3600 ) << 32 ), #24 hours ago in nanoseconds
firstTime = cms.untracked.uint64((int(time.time()) - 24 * 3600) << 32), # 24 hours ago in nanoseconds
numberEventsInRun = cms.untracked.uint32( options.eventsPerLumi * options.numberOfLumis ), # options.numberOfLumis lumi sections per run
numberEventsInLuminosityBlock = cms.untracked.uint32( options.eventsPerLumi )
)
Expand Down
6 changes: 3 additions & 3 deletions CondCore/ESSources/test/python/load_records_cfg.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,15 +11,15 @@
tag = cms.string('mytest')
), cms.PSet(
record = cms.string('anotherPedestalsRcd'),
tag = cms.string('anothermytest')
tag = cms.string('mytest_1')
)),
connect = cms.string('sqlite_file:test.db')
connect = cms.string('frontier://FrontierPrep/CMS_CONDITIONS')
)

process.source = cms.Source("EmptyIOVSource",
lastValue = cms.uint64(3),
timetype = cms.string('runnumber'),
firstValue = cms.uint64(1),
firstValue = cms.uint64(200000),
interval = cms.uint64(1)
)

Expand Down
28 changes: 12 additions & 16 deletions CondCore/ESSources/test/python/load_tagcollection_cfg.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,28 +1,24 @@
import FWCore.ParameterSet.Config as cms

process = cms.Process("TEST")
process.load("CondCore.DBCommon.CondDBCommon_cfi")
process.CondDBCommon.connect = cms.string("")
process.CondDBCommon.DBParameters.messageLevel = 0

CondDBSetup = cms.PSet(DBParameters = cms.PSet(messageLevel = cms.untracked.int32(1)))

process.PoolDBESSource = cms.ESSource("PoolDBESSource",
connect = cms.string(''),
DBParameters = cms.PSet(
messageLevel = cms.untracked.int32(0),
authenticationPath = cms.untracked.string('/afs/cern.ch/cms/DB/conddb')
),
CondDBSetup,
connect = cms.string(''),
toGet = cms.VPSet(
cms.PSet(
connect = cms.untracked.string('oracle://cms_orcoff_prod/CMS_COND_20X_DT'),
record = cms.string('DTTtrigRcd'),
tag = cms.string('tTrig_GRUMM_080313_hlt'),
label = cms.untracked.string('t2')
connect = cms.string('frontier://FrontierProd/CMS_CONDITIONS'),
record = cms.string('DTTtrigRcd'),
tag = cms.string('DTTtrig_STARTUP_V01_mc'),
label = cms.untracked.string('t2')
),
cms.PSet(
connect = cms.untracked.string('sqlite_file:orconGRUMM_200p9.db'),
record = cms.string('DTTtrigRcd'),
tag = cms.string('tTrig_GRUMM_080313'),
label = cms.untracked.string('t1')
connect = cms.string('frontier://FrontierPrep/CMS_CONDITIONS'),
record = cms.string('DTTtrigRcd'),
tag = cms.string('DTTtrig_IDEAL_V02_mc'),
label = cms.untracked.string('t1')
)
)
)
Expand Down
26 changes: 26 additions & 0 deletions CondCore/ESSources/test/runConditionsLoadTests.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
#!/bin/sh

function die { echo $1: status $2 ; exit $2; }

echo " testing CondCore/ESSources/test/python/load* "

# List of successful configuration files
configs=(
"load_records_cfg.py"
"load_modifiedglobaltag_cfg.py"
"loadall_from_prodglobaltag_cfg.py"
"load_record_empty_source_cfg.py"
"loadall_from_one_record_empty_source_cfg.py"
"load_from_multiplesources_cfg.py"
"loadall_from_gt_empty_source_cfg.py"
"load_tagcollection_cfg.py"
"loadall_from_gt_cfg.py"
"load_from_globaltag_cfg.py"
)

# Loop through each successful configuration file and run cmsRun
for config in "${configs[@]}";
do
echo "===== Test \"cmsRun $config \" ===="
(cmsRun "${SCRAM_TEST_PATH}/python/"$config) || die "Failure using cmsRun $config" $?
done
Loading