Skip to content

Commit c881612

Browse files
taeppertaepper
committed
feat(silo)!: support multiple lineage systems
BREAKING CHANGE: Multiple breaking changes to lineage configuration: - PreprocessingConfig: `lineageDefinitionsFilename` renamed to `lineageDefinitionFilenames` and changed from string to array - DatabaseConfig: `generateLineageIndex` changed from boolean to string (expects lineage filename with automatic prefix/suffix detection)
1 parent 6baeaec commit c881612

30 files changed

Lines changed: 502 additions & 129 deletions

‎src/config/source/yaml_file.cpp‎

Lines changed: 29 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,20 @@ using silo::config::ConfigKeyPath;
1717

1818
namespace {
1919

20+
bool isSequenceOfScalars(const YAML::Node& node) {
21+
if (!node.IsSequence()) {
22+
SPDLOG_TRACE("isSequenceOfScalars = false, node is not a sequence");
23+
return false;
24+
}
25+
for (auto element : node) {
26+
if (!element.IsScalar()) {
27+
SPDLOG_TRACE("isSequenceOfScalars = false, list element is not a string");
28+
return false;
29+
}
30+
}
31+
return true;
32+
}
33+
2034
bool isProperSingularValue(const YAML::Node& node) {
2135
if (node.IsMap()) {
2236
SPDLOG_TRACE("isProperSingularValue = false, node is a map");
@@ -26,8 +40,8 @@ bool isProperSingularValue(const YAML::Node& node) {
2640
SPDLOG_TRACE("isProperSingularValue = false, node is not defined");
2741
return false;
2842
}
29-
if (!node.IsScalar()) {
30-
SPDLOG_TRACE("isProperSingularValue = false, node is not a scalar");
43+
if (!node.IsScalar() && !isSequenceOfScalars(node)) {
44+
SPDLOG_TRACE("isProperSingularValue = false, node is not a scalar or sequence of strings");
3145
return false;
3246
}
3347
return true;
@@ -200,7 +214,19 @@ ConfigValue yamlNodeToConfigValue(
200214
try {
201215
switch (attribute_spec.type) {
202216
case ConfigValueType::STRING:
203-
return ConfigValue::fromString(yaml.as<std::string>());
217+
if (yaml.IsSequence()) {
218+
std::vector<std::string> scalars_as_string;
219+
std::ranges::transform(
220+
yaml.begin(),
221+
yaml.end(),
222+
std::back_inserter(scalars_as_string),
223+
[](const YAML::Node& element) { return element.as<std::string>(); }
224+
);
225+
auto joined_scalars = boost::join(scalars_as_string, ",");
226+
return ConfigValue::fromString(joined_scalars);
227+
} else {
228+
return ConfigValue::fromString(yaml.as<std::string>());
229+
}
204230
case ConfigValueType::PATH:
205231
return ConfigValue::fromPath({std::filesystem::path{yaml.as<std::string>()}});
206232
case ConfigValueType::INT32:

‎src/config/source/yaml_file.test.cpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ TEST(YamlFile, containsCorrectFieldsFromFlatYAML) {
5757
inputDirectory: "./testBaseData/exampleDataset/"
5858
outputDirectory: "./output/"
5959
ndjsonInputFilename: "input_file.ndjson"
60-
lineageDefinitionsFilename: "lineage_definitions.yaml"
60+
lineageDefinitionFilenames: "lineage_definition.yaml"
6161
phyloTreeFilename: "phylogenetic_tree.yaml"
6262
referenceGenomeFilename: "reference_genomes.json"
6363
)"
@@ -69,8 +69,8 @@ referenceGenomeFilename: "reference_genomes.json"
6969
YAML::Node{"./testBaseData/exampleDataset/"}},
7070
{YamlFile::stringToConfigKeyPath("outputDirectory"), YAML::Node{"./output/"}},
7171
{YamlFile::stringToConfigKeyPath("ndjsonInputFilename"), YAML::Node{"input_file.ndjson"}},
72-
{YamlFile::stringToConfigKeyPath("lineageDefinitionsFilename"),
73-
YAML::Node{"lineage_definitions.yaml"}},
72+
{YamlFile::stringToConfigKeyPath("lineageDefinitionFilenames"),
73+
YAML::Node{"lineage_definition.yaml"}},
7474
{YamlFile::stringToConfigKeyPath("phyloTreeFilename"), YAML::Node{"phylogenetic_tree.yaml"}},
7575
{YamlFile::stringToConfigKeyPath("referenceGenomeFilename"),
7676
YAML::Node{"reference_genomes.json"}},

‎src/main.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ int runPreprocessor(const silo::config::PreprocessingConfig& preprocessing_confi
4747
database.saveDatabaseState(preprocessing_config.output_directory);
4848
return 0;
4949
} catch (const silo::preprocessing::PreprocessingException& preprocessing_exception) {
50-
SPDLOG_ERROR("initialize - error: {}", preprocessing_exception.what());
50+
SPDLOG_ERROR("preprocessing - error: {}", preprocessing_exception.what());
5151
return 1;
5252
}
5353
}

‎src/silo/common/lineage_tree.test.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -254,7 +254,7 @@ TEST(LineageDefinitionFile, errorOnLineageAsAlias) {
254254

255255
TEST(containsCycle, doesNotFindCycleInPangoLineageTree) {
256256
ASSERT_NO_THROW(LineageTreeAndIdMap::fromLineageDefinitionFilePath(
257-
"testBaseData/exampleDataset/lineage_definitions.yaml"
257+
"testBaseData/exampleDataset/lineage_definition.yaml"
258258
));
259259
}
260260

‎src/silo/config/database_config.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -138,9 +138,9 @@ bool YAML::convert<silo::config::DatabaseMetadata>::decode(
138138
metadata.generate_index = false;
139139
}
140140
if (node["generateLineageIndex"].IsDefined()) {
141-
metadata.generate_lineage_index = node["generateLineageIndex"].as<bool>();
141+
metadata.generate_lineage_index = node["generateLineageIndex"].as<std::string>();
142142
} else {
143-
metadata.generate_lineage_index = false;
143+
metadata.generate_lineage_index = std::nullopt;
144144
}
145145
if (node["isPhyloTreeField"].IsDefined()) {
146146
metadata.phylo_tree_node_identifier = node["isPhyloTreeField"].as<bool>();

‎src/silo/config/database_config.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ class DatabaseMetadata {
2323
std::string name;
2424
ValueType type;
2525
bool generate_index;
26-
bool generate_lineage_index;
26+
std::optional<std::string> generate_lineage_index;
2727
bool phylo_tree_node_identifier;
2828

2929
[[nodiscard]] schema::ColumnType getColumnType() const;

‎src/silo/config/database_config.test.cpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,7 @@ TEST(DatabaseConfig, shouldReadConfigWithCorrectParameters) {
115115
ASSERT_EQ(config.schema.metadata[0].name, "gisaid_epi_isl");
116116
ASSERT_EQ(config.schema.metadata[0].type, ValueType::STRING);
117117
ASSERT_EQ(config.schema.metadata[0].generate_index, false);
118-
ASSERT_EQ(config.schema.metadata[0].generate_lineage_index, false);
118+
ASSERT_EQ(config.schema.metadata[0].generate_lineage_index, std::nullopt);
119119
ASSERT_EQ(config.schema.metadata[0].phylo_tree_node_identifier, true);
120120
ASSERT_EQ(config.schema.metadata[0].generate_index, false);
121121
ASSERT_EQ(config.schema.metadata[1].name, "date");
@@ -133,11 +133,11 @@ TEST(DatabaseConfig, shouldReadConfigWithCorrectParameters) {
133133
ASSERT_EQ(config.schema.metadata[5].name, "pango_lineage");
134134
ASSERT_EQ(config.schema.metadata[5].type, ValueType::STRING);
135135
ASSERT_EQ(config.schema.metadata[5].generate_index, true);
136-
ASSERT_EQ(config.schema.metadata[5].generate_lineage_index, true);
136+
ASSERT_EQ(config.schema.metadata[5].generate_lineage_index, "some_test_value");
137137
ASSERT_EQ(config.schema.metadata[6].name, "division");
138138
ASSERT_EQ(config.schema.metadata[6].type, ValueType::STRING);
139139
ASSERT_EQ(config.schema.metadata[6].generate_index, true);
140-
ASSERT_EQ(config.schema.metadata[6].generate_lineage_index, false);
140+
ASSERT_EQ(config.schema.metadata[6].generate_lineage_index, std::nullopt);
141141
ASSERT_EQ(config.schema.metadata[7].name, "age");
142142
ASSERT_EQ(config.schema.metadata[7].type, ValueType::INT);
143143
ASSERT_EQ(config.schema.metadata[7].generate_index, false);

‎src/silo/config/initialize_config.cpp‎

Lines changed: 24 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,8 @@ ConfigKeyPath inputDirectoryOptionKey() {
2626
ConfigKeyPath outputDirectoryOptionKey() {
2727
return YamlFile::stringToConfigKeyPath("outputDirectory");
2828
}
29-
ConfigKeyPath lineageDefinitionsFilenameOptionKey() {
30-
return YamlFile::stringToConfigKeyPath("lineageDefinitionsFilename");
29+
ConfigKeyPath lineageDefinitionFilenamesOptionKey() {
30+
return YamlFile::stringToConfigKeyPath("lineageDefinitionFilenames");
3131
}
3232
ConfigKeyPath phyloTreeFilenameOptionKey() {
3333
return YamlFile::stringToConfigKeyPath("phyloTreeFilename");
@@ -67,9 +67,10 @@ ConfigSpecification InitializeConfig::getConfigSpecification() {
6767
"The path to the directory to hold the output files."
6868
),
6969
ConfigAttributeSpecification::createWithoutDefault(
70-
lineageDefinitionsFilenameOptionKey(),
71-
ConfigValueType::PATH,
72-
"File name of the file holding the lineage definitions. Relative from inputDirectory."
70+
lineageDefinitionFilenamesOptionKey(),
71+
ConfigValueType::STRING,
72+
"Comma separated list of file names holding the lineage definitions. This can also be "
73+
"a Sequence in YAML. Relative from inputDirectory."
7374
),
7475
ConfigAttributeSpecification::createWithoutDefault(
7576
phyloTreeFilenameOptionKey(),
@@ -108,10 +109,22 @@ std::filesystem::path InitializationFiles::getDatabaseConfigFilename() const {
108109
return directory / database_config_file;
109110
}
110111

111-
std::optional<std::filesystem::path> InitializationFiles::getLineageDefinitionsFilename() const {
112-
return lineage_definitions_file.has_value()
113-
? std::optional(directory / lineage_definitions_file.value())
114-
: std::nullopt;
112+
std::vector<std::filesystem::path> InitializationFiles::getLineageDefinitionFilenames() const {
113+
if (not lineage_definition_files.has_value()) {
114+
return {};
115+
}
116+
auto comma_separated_string = lineage_definition_files.value();
117+
std::vector<std::filesystem::path> filenames;
118+
std::stringstream ss(comma_separated_string);
119+
std::string item;
120+
121+
while (std::getline(ss, item, ',')) {
122+
if (!item.empty()) {
123+
filenames.emplace_back(directory / item);
124+
}
125+
}
126+
127+
return filenames;
115128
}
116129

117130
std::optional<std::filesystem::path> InitializationFiles::getPhyloTreeFilename() const {
@@ -128,8 +141,8 @@ void InitializeConfig::overwriteFrom(const VerifiedConfigAttributes& config_sour
128141
if (auto var = config_source.getPath(inputDirectoryOptionKey())) {
129142
initialization_files.directory = var.value();
130143
}
131-
if (auto var = config_source.getPath(lineageDefinitionsFilenameOptionKey())) {
132-
initialization_files.lineage_definitions_file = var.value();
144+
if (auto var = config_source.getString(lineageDefinitionFilenamesOptionKey())) {
145+
initialization_files.lineage_definition_files = var.value();
133146
}
134147
if (auto var = config_source.getPath(phyloTreeFilenameOptionKey())) {
135148
initialization_files.phylogenetic_tree_file = var.value();

‎src/silo/config/initialize_config.h‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ class PreprocessingConfig;
2020

2121
class InitializationFiles {
2222
public:
23-
std::optional<std::filesystem::path> lineage_definitions_file;
23+
std::optional<std::string> lineage_definition_files;
2424
std::optional<std::filesystem::path> phylogenetic_tree_file;
2525
std::filesystem::path database_config_file;
2626
std::filesystem::path reference_genome_file;
@@ -30,7 +30,7 @@ class InitializationFiles {
3030

3131
[[nodiscard]] std::filesystem::path getDatabaseConfigFilename() const;
3232

33-
[[nodiscard]] std::optional<std::filesystem::path> getLineageDefinitionsFilename() const;
33+
[[nodiscard]] std::vector<std::filesystem::path> getLineageDefinitionFilenames() const;
3434

3535
[[nodiscard]] std::optional<std::filesystem::path> getPhyloTreeFilename() const;
3636

@@ -39,7 +39,7 @@ class InitializationFiles {
3939
NLOHMANN_DEFINE_TYPE_INTRUSIVE(
4040
InitializationFiles,
4141
directory,
42-
lineage_definitions_file,
42+
lineage_definition_files,
4343
phylogenetic_tree_file,
4444
database_config_file,
4545
reference_genome_file

‎src/silo/config/preprocessing_config.cpp‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,8 @@ ConfigKeyPath inputDirectoryOptionKey() {
2929
ConfigKeyPath outputDirectoryOptionKey() {
3030
return YamlFile::stringToConfigKeyPath("outputDirectory");
3131
}
32-
ConfigKeyPath lineageDefinitionsFilenameOptionKey() {
33-
return YamlFile::stringToConfigKeyPath("lineageDefinitionsFilename");
32+
ConfigKeyPath lineageDefinitionFilenamesOptionKey() {
33+
return YamlFile::stringToConfigKeyPath("lineageDefinitionFilenames");
3434
}
3535
ConfigKeyPath phyloTreeFilenameOptionKey() {
3636
return YamlFile::stringToConfigKeyPath("phyloTreeFilename");
@@ -90,9 +90,10 @@ ConfigSpecification PreprocessingConfig::getConfigSpecification() {
9090
"The path to the directory to hold the output files."
9191
),
9292
ConfigAttributeSpecification::createWithoutDefault(
93-
lineageDefinitionsFilenameOptionKey(),
94-
ConfigValueType::PATH,
95-
"File name of the file holding the lineage definitions. Relative from inputDirectory."
93+
lineageDefinitionFilenamesOptionKey(),
94+
ConfigValueType::STRING,
95+
"Comma separated list of file names holding the lineage definitions. This can also be "
96+
"a Sequence in YAML. Relative from inputDirectory."
9697
),
9798
ConfigAttributeSpecification::createWithoutDefault(
9899
phyloTreeFilenameOptionKey(),
@@ -152,8 +153,8 @@ void PreprocessingConfig::overwriteFrom(const VerifiedConfigAttributes& config_s
152153
if (auto var = config_source.getPath(inputDirectoryOptionKey())) {
153154
initialization_files.directory = var.value();
154155
}
155-
if (auto var = config_source.getPath(lineageDefinitionsFilenameOptionKey())) {
156-
initialization_files.lineage_definitions_file = var.value();
156+
if (auto var = config_source.getString(lineageDefinitionFilenamesOptionKey())) {
157+
initialization_files.lineage_definition_files = var.value();
157158
}
158159
if (auto var = config_source.getPath(phyloTreeFilenameOptionKey())) {
159160
initialization_files.phylogenetic_tree_file = var.value();

0 commit comments

Comments
 (0)