From 390839571160822fb728b4122fb20a9b324dc173 Mon Sep 17 00:00:00 2001 From: Alexander Taepper Date: Thu, 25 Sep 2025 14:31:12 +0200 Subject: [PATCH] feat(silo)!: support multiple lineage systems BREAKING CHANGE: Multiple breaking changes to lineage configuration: - PreprocessingConfig: `lineageDefinitionsFilename` renamed to `lineageDefinitionFilenames` and changed from string to list - DatabaseConfig: `generateLineageIndex` changed from boolean to string (expects lineage definition filename) --- src/config/config_specification.cpp | 2 + src/config/config_value.cpp | 6 + src/config/config_value.h | 27 +++- src/config/source/yaml_file.cpp | 13 +- src/config/source/yaml_file.test.cpp | 6 +- src/config/verified_config_attributes.cpp | 6 + src/config/verified_config_attributes.h | 4 + src/main.cpp | 2 +- src/silo/common/lineage_tree.test.cpp | 2 +- src/silo/config/database_config.cpp | 4 +- src/silo/config/database_config.h | 2 +- src/silo/config/database_config.test.cpp | 10 +- src/silo/config/initialize_config.cpp | 24 ++-- src/silo/config/initialize_config.h | 6 +- src/silo/config/preprocessing_config.cpp | 14 +- src/silo/config/preprocessing_config.test.cpp | 23 ++-- src/silo/database.test.cpp | 12 +- src/silo/initialize/initialize_exception.h | 4 + src/silo/initialize/initializer.cpp | 87 ++++++++----- src/silo/initialize/initializer.h | 7 +- src/silo/initialize/initializer.test.cpp | 122 ++++++++++++++++-- src/silo/preprocessing/preprocessing.cpp | 22 ++-- src/silo/preprocessing/preprocessing.test.cpp | 98 +++++++++++++- .../expressions/lineage_filter.test.cpp | 4 +- .../column/indexed_string_column.test.cpp | 22 ++-- src/silo/test/default_sequence.test.cpp | 2 +- src/silo/test/query_fixture.test.h | 4 +- .../exampleDataset/database_config.yaml | 2 +- ...finitions.yaml => lineage_definition.yaml} | 0 .../exampleDataset/preprocessing_config.yaml | 3 +- testBaseData/test_database_config.yaml | 2 +- ..._database_config_without_partition_by.yaml | 2 +- testBaseData/test_preprocessing_config.yaml | 3 +- .../unitTestDummyDataset/database_config.yaml | 2 +- .../lineage_definitions.yaml | 1 - .../preprocessing_config.yaml | 3 +- .../test_lineage_definition.yaml | 1 + 37 files changed, 416 insertions(+), 138 deletions(-) rename testBaseData/exampleDataset/{lineage_definitions.yaml => lineage_definition.yaml} (100%) delete mode 120000 testBaseData/unitTestDummyDataset/lineage_definitions.yaml create mode 120000 testBaseData/unitTestDummyDataset/test_lineage_definition.yaml diff --git a/src/config/config_specification.cpp b/src/config/config_specification.cpp index 4992c8968..49d296388 100644 --- a/src/config/config_specification.cpp +++ b/src/config/config_specification.cpp @@ -152,6 +152,8 @@ ConfigValue ConfigAttributeSpecification::parseValueFromString(std::string value throw ConfigException( fmt::format("'{}' is not a valid string for a boolean", value_string) ); + case ConfigValueType::LIST: + throw ConfigException("List values can currently no be specified as strings."); } SILO_UNREACHABLE(); } catch (boost::bad_lexical_cast&) { diff --git a/src/config/config_value.cpp b/src/config/config_value.cpp index afa65bef0..950b25ca3 100644 --- a/src/config/config_value.cpp +++ b/src/config/config_value.cpp @@ -1,5 +1,6 @@ #include "config/config_value.h" +#include #include #include @@ -28,6 +29,9 @@ ConfigValueType ConfigValue::getValueType() const { if (std::holds_alternative(value)) { return ConfigValueType::BOOL; } + if (std::holds_alternative>(value)) { + return ConfigValueType::LIST; + } SILO_UNREACHABLE(); } @@ -39,6 +43,8 @@ std::string ConfigValue::toString() const { return fmt::format("'{}'", value); } else if constexpr (std::is_same_v) { return fmt::format("'{}'", value.string()); + } else if constexpr (std::is_same_v>) { + return fmt::format("{}", fmt::join(value, ",")); } else { return fmt::format("{}", value); } diff --git a/src/config/config_value.h b/src/config/config_value.h index 02ab587c8..29b0e4325 100644 --- a/src/config/config_value.h +++ b/src/config/config_value.h @@ -12,7 +12,7 @@ namespace silo::config { -enum class ConfigValueType { STRING, PATH, INT32, UINT32, UINT16, BOOL }; +enum class ConfigValueType { STRING, PATH, INT32, UINT32, UINT16, BOOL, LIST }; constexpr std::string_view configValueTypeToString(ConfigValueType type) { switch (type) { @@ -28,18 +28,33 @@ constexpr std::string_view configValueTypeToString(ConfigValueType type) { return "u16"; case ConfigValueType::BOOL: return "bool"; + case ConfigValueType::LIST: + return "list"; } SILO_UNREACHABLE(); } class ConfigValue { - explicit ConfigValue( - std::variant value - ) + explicit ConfigValue(std::variant< + std::string, + std::filesystem::path, + int32_t, + uint32_t, + uint16_t, + bool, + std::vector> value) : value(std::move(value)) {} public: - std::variant value; + std::variant< + std::string, + std::filesystem::path, + int32_t, + uint32_t, + uint16_t, + bool, + std::vector> + value; static ConfigValue fromString(const std::string& value) { return ConfigValue{value}; } @@ -57,6 +72,8 @@ class ConfigValue { static ConfigValue fromBool(bool value) { return ConfigValue{value}; } + static ConfigValue fromList(const std::vector& value) { return ConfigValue{value}; } + [[nodiscard]] ConfigValueType getValueType() const; [[nodiscard]] std::string toString() const; diff --git a/src/config/source/yaml_file.cpp b/src/config/source/yaml_file.cpp index b8be8a509..599092183 100644 --- a/src/config/source/yaml_file.cpp +++ b/src/config/source/yaml_file.cpp @@ -17,6 +17,13 @@ using silo::config::ConfigKeyPath; namespace { +bool isSequenceOfScalars(const YAML::Node& node) { + if (!node.IsSequence()) { + return false; + } + return std::ranges::all_of(node, [](const auto& element) { return element.IsScalar(); }); +} + bool isProperSingularValue(const YAML::Node& node) { if (node.IsMap()) { SPDLOG_TRACE("isProperSingularValue = false, node is a map"); @@ -26,8 +33,8 @@ bool isProperSingularValue(const YAML::Node& node) { SPDLOG_TRACE("isProperSingularValue = false, node is not defined"); return false; } - if (!node.IsScalar()) { - SPDLOG_TRACE("isProperSingularValue = false, node is not a scalar"); + if (!node.IsScalar() && !isSequenceOfScalars(node)) { + SPDLOG_TRACE("isProperSingularValue = false, node is not a scalar or sequence of strings"); return false; } return true; @@ -211,6 +218,8 @@ ConfigValue yamlNodeToConfigValue( return ConfigValue::fromUint16(yaml.as()); case ConfigValueType::BOOL: return ConfigValue::fromBool(yaml.as()); + case ConfigValueType::LIST: + return ConfigValue::fromList(yaml.as>()); } SILO_UNREACHABLE(); } catch (YAML::BadConversion& error) { diff --git a/src/config/source/yaml_file.test.cpp b/src/config/source/yaml_file.test.cpp index 5c6f20f5a..13c8c1fd4 100644 --- a/src/config/source/yaml_file.test.cpp +++ b/src/config/source/yaml_file.test.cpp @@ -57,7 +57,7 @@ TEST(YamlFile, containsCorrectFieldsFromFlatYAML) { inputDirectory: "./testBaseData/exampleDataset/" outputDirectory: "./output/" ndjsonInputFilename: "input_file.ndjson" -lineageDefinitionsFilename: "lineage_definitions.yaml" +lineageDefinitionFilenames: "lineage_definition.yaml" phyloTreeFilename: "phylogenetic_tree.yaml" referenceGenomeFilename: "reference_genomes.json" )" @@ -69,8 +69,8 @@ referenceGenomeFilename: "reference_genomes.json" YAML::Node{"./testBaseData/exampleDataset/"}}, {YamlFile::stringToConfigKeyPath("outputDirectory"), YAML::Node{"./output/"}}, {YamlFile::stringToConfigKeyPath("ndjsonInputFilename"), YAML::Node{"input_file.ndjson"}}, - {YamlFile::stringToConfigKeyPath("lineageDefinitionsFilename"), - YAML::Node{"lineage_definitions.yaml"}}, + {YamlFile::stringToConfigKeyPath("lineageDefinitionFilenames"), + YAML::Node{"lineage_definition.yaml"}}, {YamlFile::stringToConfigKeyPath("phyloTreeFilename"), YAML::Node{"phylogenetic_tree.yaml"}}, {YamlFile::stringToConfigKeyPath("referenceGenomeFilename"), YAML::Node{"reference_genomes.json"}}, diff --git a/src/config/verified_config_attributes.cpp b/src/config/verified_config_attributes.cpp index 14f28663f..8825e6dd7 100644 --- a/src/config/verified_config_attributes.cpp +++ b/src/config/verified_config_attributes.cpp @@ -88,6 +88,12 @@ std::optional VerifiedConfigAttributes::getBool(const ConfigKeyPath& confi return std::nullopt; } +std::optional> VerifiedConfigAttributes::getList( + const ConfigKeyPath& config_key_path +) const { + return getValue, ConfigValueType::LIST>(config_key_path, config_values); +} + VerifiedCommandLineArguments VerifiedCommandLineArguments::askingForHelp() { VerifiedCommandLineArguments result; result.asks_for_help = true; diff --git a/src/config/verified_config_attributes.h b/src/config/verified_config_attributes.h index 5d5086455..93f44e19f 100644 --- a/src/config/verified_config_attributes.h +++ b/src/config/verified_config_attributes.h @@ -33,6 +33,10 @@ class VerifiedConfigAttributes { [[nodiscard]] std::optional getUint16(const ConfigKeyPath& config_key_path) const; [[nodiscard]] std::optional getBool(const ConfigKeyPath& config_key_path) const; + + [[nodiscard]] std::optional> getList( + const ConfigKeyPath& config_key_path + ) const; }; class VerifiedCommandLineArguments : public VerifiedConfigAttributes { diff --git a/src/main.cpp b/src/main.cpp index 7b7443295..69732e1ba 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -47,7 +47,7 @@ int runPreprocessor(const silo::config::PreprocessingConfig& preprocessing_confi database.saveDatabaseState(preprocessing_config.output_directory); return 0; } catch (const silo::preprocessing::PreprocessingException& preprocessing_exception) { - SPDLOG_ERROR("initialize - error: {}", preprocessing_exception.what()); + SPDLOG_ERROR("preprocessing - error: {}", preprocessing_exception.what()); return 1; } } diff --git a/src/silo/common/lineage_tree.test.cpp b/src/silo/common/lineage_tree.test.cpp index c07bfc504..8205d211b 100644 --- a/src/silo/common/lineage_tree.test.cpp +++ b/src/silo/common/lineage_tree.test.cpp @@ -254,7 +254,7 @@ TEST(LineageDefinitionFile, errorOnLineageAsAlias) { TEST(containsCycle, doesNotFindCycleInPangoLineageTree) { ASSERT_NO_THROW(LineageTreeAndIdMap::fromLineageDefinitionFilePath( - "testBaseData/exampleDataset/lineage_definitions.yaml" + "testBaseData/exampleDataset/lineage_definition.yaml" )); } diff --git a/src/silo/config/database_config.cpp b/src/silo/config/database_config.cpp index cdc3195ca..d57b0f7b3 100644 --- a/src/silo/config/database_config.cpp +++ b/src/silo/config/database_config.cpp @@ -138,9 +138,9 @@ bool YAML::convert::decode( metadata.generate_index = false; } if (node["generateLineageIndex"].IsDefined()) { - metadata.generate_lineage_index = node["generateLineageIndex"].as(); + metadata.generate_lineage_index = node["generateLineageIndex"].as(); } else { - metadata.generate_lineage_index = false; + metadata.generate_lineage_index = std::nullopt; } if (node["isPhyloTreeField"].IsDefined()) { metadata.phylo_tree_node_identifier = node["isPhyloTreeField"].as(); diff --git a/src/silo/config/database_config.h b/src/silo/config/database_config.h index ed59c40e5..03253cc2b 100644 --- a/src/silo/config/database_config.h +++ b/src/silo/config/database_config.h @@ -23,7 +23,7 @@ class DatabaseMetadata { std::string name; ValueType type; bool generate_index; - bool generate_lineage_index; + std::optional generate_lineage_index; bool phylo_tree_node_identifier; [[nodiscard]] schema::ColumnType getColumnType() const; diff --git a/src/silo/config/database_config.test.cpp b/src/silo/config/database_config.test.cpp index e16a4f189..116508702 100644 --- a/src/silo/config/database_config.test.cpp +++ b/src/silo/config/database_config.test.cpp @@ -115,7 +115,7 @@ TEST(DatabaseConfig, shouldReadConfigWithCorrectParameters) { ASSERT_EQ(config.schema.metadata[0].name, "gisaid_epi_isl"); ASSERT_EQ(config.schema.metadata[0].type, ValueType::STRING); ASSERT_EQ(config.schema.metadata[0].generate_index, false); - ASSERT_EQ(config.schema.metadata[0].generate_lineage_index, false); + ASSERT_EQ(config.schema.metadata[0].generate_lineage_index, std::nullopt); ASSERT_EQ(config.schema.metadata[0].phylo_tree_node_identifier, true); ASSERT_EQ(config.schema.metadata[0].generate_index, false); ASSERT_EQ(config.schema.metadata[1].name, "date"); @@ -133,11 +133,11 @@ TEST(DatabaseConfig, shouldReadConfigWithCorrectParameters) { ASSERT_EQ(config.schema.metadata[5].name, "pango_lineage"); ASSERT_EQ(config.schema.metadata[5].type, ValueType::STRING); ASSERT_EQ(config.schema.metadata[5].generate_index, true); - ASSERT_EQ(config.schema.metadata[5].generate_lineage_index, true); + ASSERT_EQ(config.schema.metadata[5].generate_lineage_index, "some_test_value"); ASSERT_EQ(config.schema.metadata[6].name, "division"); ASSERT_EQ(config.schema.metadata[6].type, ValueType::STRING); ASSERT_EQ(config.schema.metadata[6].generate_index, true); - ASSERT_EQ(config.schema.metadata[6].generate_lineage_index, false); + ASSERT_EQ(config.schema.metadata[6].generate_lineage_index, std::nullopt); ASSERT_EQ(config.schema.metadata[7].name, "age"); ASSERT_EQ(config.schema.metadata[7].type, ValueType::INT); ASSERT_EQ(config.schema.metadata[7].generate_index, false); @@ -250,7 +250,7 @@ defaultNucleotideSequence: "main" - name: "metadata1" type: "string" generateIndex: true - generateLineageIndex: true + generateLineageIndex: lineage - name: "metadata2" type: "date" - name: "metadata3" @@ -331,7 +331,7 @@ defaultNucleotideSequence: "main" type: "string" - name: "some lineage" type: "string" - generateLineageIndex: true + generateLineageIndex: lineage primaryKey: "testPrimaryKey" )"; diff --git a/src/silo/config/initialize_config.cpp b/src/silo/config/initialize_config.cpp index 8294ee3a3..883b74bf2 100644 --- a/src/silo/config/initialize_config.cpp +++ b/src/silo/config/initialize_config.cpp @@ -26,8 +26,8 @@ ConfigKeyPath inputDirectoryOptionKey() { ConfigKeyPath outputDirectoryOptionKey() { return YamlFile::stringToConfigKeyPath("outputDirectory"); } -ConfigKeyPath lineageDefinitionsFilenameOptionKey() { - return YamlFile::stringToConfigKeyPath("lineageDefinitionsFilename"); +ConfigKeyPath lineageDefinitionFilenamesOptionKey() { + return YamlFile::stringToConfigKeyPath("lineageDefinitionFilenames"); } ConfigKeyPath phyloTreeFilenameOptionKey() { return YamlFile::stringToConfigKeyPath("phyloTreeFilename"); @@ -67,9 +67,9 @@ ConfigSpecification InitializeConfig::getConfigSpecification() { "The path to the directory to hold the output files." ), ConfigAttributeSpecification::createWithoutDefault( - lineageDefinitionsFilenameOptionKey(), - ConfigValueType::PATH, - "File name of the file holding the lineage definitions. Relative from inputDirectory." + lineageDefinitionFilenamesOptionKey(), + ConfigValueType::LIST, + "List of file names holding the lineage definitions. Relative from inputDirectory." ), ConfigAttributeSpecification::createWithoutDefault( phyloTreeFilenameOptionKey(), @@ -108,10 +108,12 @@ std::filesystem::path InitializationFiles::getDatabaseConfigFilename() const { return directory / database_config_file; } -std::optional InitializationFiles::getLineageDefinitionsFilename() const { - return lineage_definitions_file.has_value() - ? std::optional(directory / lineage_definitions_file.value()) - : std::nullopt; +std::vector InitializationFiles::getLineageDefinitionFilenames() const { + std::vector paths; + for (const auto& file_name : lineage_definition_files) { + paths.push_back(directory / file_name); + } + return paths; } std::optional InitializationFiles::getPhyloTreeFilename() const { @@ -128,8 +130,8 @@ void InitializeConfig::overwriteFrom(const VerifiedConfigAttributes& config_sour if (auto var = config_source.getPath(inputDirectoryOptionKey())) { initialization_files.directory = var.value(); } - if (auto var = config_source.getPath(lineageDefinitionsFilenameOptionKey())) { - initialization_files.lineage_definitions_file = var.value(); + if (auto var = config_source.getList(lineageDefinitionFilenamesOptionKey())) { + initialization_files.lineage_definition_files = var.value(); } if (auto var = config_source.getPath(phyloTreeFilenameOptionKey())) { initialization_files.phylogenetic_tree_file = var.value(); diff --git a/src/silo/config/initialize_config.h b/src/silo/config/initialize_config.h index b132b211e..0243a2c8f 100644 --- a/src/silo/config/initialize_config.h +++ b/src/silo/config/initialize_config.h @@ -20,7 +20,7 @@ class PreprocessingConfig; class InitializationFiles { public: - std::optional lineage_definitions_file; + std::vector lineage_definition_files; std::optional phylogenetic_tree_file; std::filesystem::path database_config_file; std::filesystem::path reference_genome_file; @@ -30,7 +30,7 @@ class InitializationFiles { [[nodiscard]] std::filesystem::path getDatabaseConfigFilename() const; - [[nodiscard]] std::optional getLineageDefinitionsFilename() const; + [[nodiscard]] std::vector getLineageDefinitionFilenames() const; [[nodiscard]] std::optional getPhyloTreeFilename() const; @@ -39,7 +39,7 @@ class InitializationFiles { NLOHMANN_DEFINE_TYPE_INTRUSIVE( InitializationFiles, directory, - lineage_definitions_file, + lineage_definition_files, phylogenetic_tree_file, database_config_file, reference_genome_file diff --git a/src/silo/config/preprocessing_config.cpp b/src/silo/config/preprocessing_config.cpp index 20bbf4652..26d06b5d1 100644 --- a/src/silo/config/preprocessing_config.cpp +++ b/src/silo/config/preprocessing_config.cpp @@ -29,8 +29,8 @@ ConfigKeyPath inputDirectoryOptionKey() { ConfigKeyPath outputDirectoryOptionKey() { return YamlFile::stringToConfigKeyPath("outputDirectory"); } -ConfigKeyPath lineageDefinitionsFilenameOptionKey() { - return YamlFile::stringToConfigKeyPath("lineageDefinitionsFilename"); +ConfigKeyPath lineageDefinitionFilenamesOptionKey() { + return YamlFile::stringToConfigKeyPath("lineageDefinitionFilenames"); } ConfigKeyPath phyloTreeFilenameOptionKey() { return YamlFile::stringToConfigKeyPath("phyloTreeFilename"); @@ -90,9 +90,9 @@ ConfigSpecification PreprocessingConfig::getConfigSpecification() { "The path to the directory to hold the output files." ), ConfigAttributeSpecification::createWithoutDefault( - lineageDefinitionsFilenameOptionKey(), - ConfigValueType::PATH, - "File name of the file holding the lineage definitions. Relative from inputDirectory." + lineageDefinitionFilenamesOptionKey(), + ConfigValueType::LIST, + "List of file names holding the lineage definitions. Relative from inputDirectory." ), ConfigAttributeSpecification::createWithoutDefault( phyloTreeFilenameOptionKey(), @@ -152,8 +152,8 @@ void PreprocessingConfig::overwriteFrom(const VerifiedConfigAttributes& config_s if (auto var = config_source.getPath(inputDirectoryOptionKey())) { initialization_files.directory = var.value(); } - if (auto var = config_source.getPath(lineageDefinitionsFilenameOptionKey())) { - initialization_files.lineage_definitions_file = var.value(); + if (auto var = config_source.getList(lineageDefinitionFilenamesOptionKey())) { + initialization_files.lineage_definition_files = var.value(); } if (auto var = config_source.getPath(phyloTreeFilenameOptionKey())) { initialization_files.phylogenetic_tree_file = var.value(); diff --git a/src/silo/config/preprocessing_config.test.cpp b/src/silo/config/preprocessing_config.test.cpp index 2e9ca40d3..a6daf3638 100644 --- a/src/silo/config/preprocessing_config.test.cpp +++ b/src/silo/config/preprocessing_config.test.cpp @@ -15,15 +15,15 @@ TEST(PreprocessingConfig, shouldReadConfigWithCorrectParametersAndDefaults) { config.overwriteFrom(YamlFile::readFile("./testBaseData/test_preprocessing_config.yaml") .verify(PreprocessingConfig::getConfigSpecification())); - const std::string input_directory = "./testBaseData/exampleDataset/"; + const std::filesystem::path input_directory = "./testBaseData/exampleDataset/"; ASSERT_TRUE(config.input_file.has_value()); - ASSERT_EQ(config.getInputFilePath(), input_directory + "input_file.ndjson"); + ASSERT_EQ(config.getInputFilePath(), input_directory / "input_file.ndjson"); ASSERT_EQ( - config.initialization_files.getLineageDefinitionsFilename(), - input_directory + "lineage_definitions.yaml" + config.initialization_files.getLineageDefinitionFilenames(), + (std::vector{input_directory / "lineage_definition.yaml"}) ); ASSERT_EQ( - config.initialization_files.getPhyloTreeFilename(), input_directory + "phylogenetic_tree.nwk" + config.initialization_files.getPhyloTreeFilename(), input_directory / "phylogenetic_tree.nwk" ); } @@ -35,22 +35,23 @@ inputDirectory: "./testBaseData/exampleDataset/" outputDirectory: "./output/custom/" intermediateResultsDirectory: "./output/overriddenTemp/" ndjsonInputFilename: "input_file.ndjson" -lineageDefinitionsFilename: "lineage_definitions.yaml" +lineageDefinitionFilenames: + - "lineage_definition.yaml" phyloTreeFilename: "phylogenetic_tree.yaml" referenceGenomeFilename: "reference_genomes.json" preprocessingDatabaseLocation: "preprocessing.duckdb" duckdbMemoryLimitInG: 8)") .verify(PreprocessingConfig::getConfigSpecification())); - const std::string input_directory = "./testBaseData/exampleDataset/"; + const std::filesystem::path input_directory = "./testBaseData/exampleDataset/"; ASSERT_TRUE(config.input_file.has_value()); - ASSERT_EQ(config.getInputFilePath(), input_directory + "input_file.ndjson"); + ASSERT_EQ(config.getInputFilePath(), input_directory / "input_file.ndjson"); ASSERT_EQ( - config.initialization_files.getLineageDefinitionsFilename(), - input_directory + "lineage_definitions.yaml" + config.initialization_files.getLineageDefinitionFilenames(), + std::vector{input_directory / "lineage_definition.yaml"} ); ASSERT_EQ( - config.initialization_files.getPhyloTreeFilename(), input_directory + "phylogenetic_tree.yaml" + config.initialization_files.getPhyloTreeFilename(), input_directory / "phylogenetic_tree.yaml" ); ASSERT_EQ(config.output_directory, "./output/custom/"); diff --git a/src/silo/database.test.cpp b/src/silo/database.test.cpp index 30f787006..b32946f98 100644 --- a/src/silo/database.test.cpp +++ b/src/silo/database.test.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include @@ -36,11 +37,10 @@ std::shared_ptr buildTestDatabase() { silo::ReferenceGenomes::readFromFile(config.initialization_files.getReferenceGenomeFilename() ); - silo::common::LineageTreeAndIdMap lineage_tree; - if (config.initialization_files.getLineageDefinitionsFilename().has_value()) { - lineage_tree = silo::common::LineageTreeAndIdMap::fromLineageDefinitionFilePath( - config.initialization_files.getLineageDefinitionsFilename().value() - ); + std::map lineage_trees; + for (auto filename : config.initialization_files.getLineageDefinitionFilenames()) { + lineage_trees[filename] = + silo::common::LineageTreeAndIdMap::fromLineageDefinitionFilePath(filename); } silo::common::PhyloTree phylo_tree_file; @@ -53,7 +53,7 @@ std::shared_ptr buildTestDatabase() { silo::Database{silo::initialize::Initializer::createSchemaFromConfigFiles( std::move(database_config), std::move(reference_genomes), - std::move(lineage_tree), + std::move(lineage_trees), std::move(phylo_tree_file), /*without_unaligned_columns=*/false )} diff --git a/src/silo/initialize/initialize_exception.h b/src/silo/initialize/initialize_exception.h index c55cd0e73..ff029e586 100644 --- a/src/silo/initialize/initialize_exception.h +++ b/src/silo/initialize/initialize_exception.h @@ -9,6 +9,10 @@ class InitializeException : public std::runtime_error { public: explicit InitializeException(const std::string& error_message) : std::runtime_error(error_message) {} + + template + explicit InitializeException(fmt::format_string fmt_str, Args&&... args) + : std::runtime_error(fmt::format(fmt_str, std::forward(args)...)) {} }; } // namespace silo::initialize diff --git a/src/silo/initialize/initializer.cpp b/src/silo/initialize/initializer.cpp index f76ed3f40..79d38e055 100644 --- a/src/silo/initialize/initializer.cpp +++ b/src/silo/initialize/initializer.cpp @@ -1,5 +1,6 @@ #include "silo/initialize/initializer.h" +#include #include #include @@ -17,11 +18,10 @@ namespace silo::initialize { Database Initializer::initializeDatabase(const config::InitializationFiles& initialization_files) { EVOBENCH_SCOPE("Initializer", "initializeDatabase"); - common::LineageTreeAndIdMap lineage_tree; - if (initialization_files.getLineageDefinitionsFilename().has_value()) { - lineage_tree = common::LineageTreeAndIdMap::fromLineageDefinitionFilePath( - initialization_files.getLineageDefinitionsFilename().value() - ); + std::map lineage_trees; + for (auto filename : initialization_files.getLineageDefinitionFilenames()) { + lineage_trees[filename] = + common::LineageTreeAndIdMap::fromLineageDefinitionFilePath(filename); } common::PhyloTree phylo_tree_file; @@ -29,12 +29,15 @@ Database Initializer::initializeDatabase(const config::InitializationFiles& init if (opt_path.has_value()) { phylo_tree_file = common::PhyloTree::fromFile(opt_path.value()); } + + auto validated_config = config::DatabaseConfig::getValidatedConfigFromFile( + initialization_files.getDatabaseConfigFilename() + ); + silo::schema::DatabaseSchema schema = createSchemaFromConfigFiles( - config::DatabaseConfig::getValidatedConfigFromFile( - initialization_files.getDatabaseConfigFilename() - ), + validated_config, ReferenceGenomes::readFromFile(initialization_files.getReferenceGenomeFilename()), - std::move(lineage_tree), + std::move(lineage_trees), std::move(phylo_tree_file), initialization_files.without_unaligned_sequences ); @@ -47,7 +50,7 @@ struct ColumnMetadataInitializer { std::shared_ptr& metadata, const config::DatabaseMetadata& config_metadata, const ReferenceGenomes& reference_genomes, - const common::LineageTreeAndIdMap& lineage_tree, + const std::map& lineage_trees, const common::PhyloTree& phylo_tree_file ); }; @@ -56,13 +59,27 @@ template <> void ColumnMetadataInitializer::operator()( std::shared_ptr& metadata, const config::DatabaseMetadata& config_metadata, - const ReferenceGenomes& reference_genomes, - const common::LineageTreeAndIdMap& lineage_tree, - const common::PhyloTree& phylo_tree_file + const ReferenceGenomes& /*reference_genomes*/, + const std::map& lineage_trees, + const common::PhyloTree& /*phylo_tree_file*/ ) { - if (config_metadata.generate_lineage_index) { + if (config_metadata.generate_lineage_index.has_value()) { + auto lineage_tree_name = config_metadata.generate_lineage_index.value(); + auto lineage_tree = Initializer::findLineageTreeForName(lineage_trees, lineage_tree_name); + if (not lineage_tree.has_value()) { + auto keys = + lineage_trees | std::views::keys | + std::views::transform([](const std::filesystem::path& p) { return p.string(); }); + throw silo::initialize::InitializeException( + "Column '{}' has lineage tree '{}' configured, but did not find corresponding lineage " + "tree in the provided lineageDefinitionFilenames: {}", + config_metadata.name, + config_metadata.generate_lineage_index.value(), + fmt::join(keys, ",") + ); + } metadata = std::make_shared( - config_metadata.name, lineage_tree + config_metadata.name, lineage_tree.value() ); } else { metadata = std::make_shared( @@ -75,8 +92,8 @@ template <> void ColumnMetadataInitializer::operator()( std::shared_ptr& metadata, const config::DatabaseMetadata& config_metadata, - const ReferenceGenomes& reference_genomes, - const common::LineageTreeAndIdMap& lineage_tree, + const ReferenceGenomes& /*reference_genomes*/, + const std::map& /*lineage_trees*/, const common::PhyloTree& phylo_tree_file ) { if (config_metadata.phylo_tree_node_identifier) { @@ -93,9 +110,9 @@ template <> void ColumnMetadataInitializer::operator()( std::shared_ptr& metadata, const config::DatabaseMetadata& config_metadata, - const ReferenceGenomes& reference_genomes, - const common::LineageTreeAndIdMap& lineage_tree, - const common::PhyloTree& phylo_tree_file + const ReferenceGenomes& /*reference_genomes*/, + const std::map& /*lineage_trees*/, + const common::PhyloTree& /*phylo_tree_file*/ ) { SILO_PANIC("unaligned nucleotide sequences cannot be in config::DatabaseMetadata"); } @@ -104,9 +121,9 @@ template <> void ColumnMetadataInitializer::operator()>( std::shared_ptr& metadata, const config::DatabaseMetadata& config_metadata, - const ReferenceGenomes& reference_genomes, - const common::LineageTreeAndIdMap& lineage_tree, - const common::PhyloTree& phylo_tree_file + const ReferenceGenomes& /*reference_genomes*/, + const std::map& /*lineage_trees*/, + const common::PhyloTree& /*phylo_tree_file*/ ) { SILO_PANIC("nucleotides cannot be in config::DatabaseMetadata"); } @@ -115,9 +132,9 @@ template <> void ColumnMetadataInitializer::operator()>( std::shared_ptr& metadata, const config::DatabaseMetadata& config_metadata, - const ReferenceGenomes& reference_genomes, - const common::LineageTreeAndIdMap& lineage_tree, - const common::PhyloTree& phylo_tree_file + const ReferenceGenomes& /*reference_genomes*/, + const std::map& /*lineage_trees*/, + const common::PhyloTree& /*phylo_tree_file*/ ) { SILO_PANIC("amino acid cannot be in config::DatabaseMetadata"); } @@ -127,7 +144,7 @@ void ColumnMetadataInitializer::operator()( std::shared_ptr& metadata, const config::DatabaseMetadata& config_metadata, const ReferenceGenomes& /*reference_genomes*/, - const common::LineageTreeAndIdMap& /*lineage_tree*/, + const std::map& /*lineage_trees*/, const common::PhyloTree& /*phylo_tree_file*/ ) { metadata = std::make_shared(config_metadata.name); @@ -214,7 +231,7 @@ static const std::string UNALIGNED_NUCLEOTIDE_SEQUENCE_PREFIX = "unaligned_"; silo::schema::DatabaseSchema Initializer::createSchemaFromConfigFiles( config::DatabaseConfig database_config, ReferenceGenomes reference_genomes, - common::LineageTreeAndIdMap lineage_tree, + std::map lineage_trees, common::PhyloTree phylo_tree_file, bool without_unaligned_sequences ) { @@ -240,7 +257,7 @@ silo::schema::DatabaseSchema Initializer::createSchemaFromConfigFiles( metadata, config_metadata, reference_genomes, - lineage_tree, + lineage_trees, phylo_tree_file ); column_metadata.emplace(column_identifier, metadata); @@ -300,4 +317,16 @@ silo::schema::DatabaseSchema Initializer::createSchemaFromConfigFiles( return silo::schema::DatabaseSchema{.tables = {{schema::TableName{"default"}, table_schema}}}; } +std::optional Initializer::findLineageTreeForName( + const std::map& lineage_trees, + std::string lineage_tree_name +) { + for (const auto& [path, lineage_tree] : lineage_trees) { + if (path.filename() == lineage_tree_name || path.filename() == lineage_tree_name + ".yaml") { + return lineage_tree; + } + } + return std::nullopt; +} + } // namespace silo::initialize diff --git a/src/silo/initialize/initializer.h b/src/silo/initialize/initializer.h index 78c62463a..6548bb9da 100644 --- a/src/silo/initialize/initializer.h +++ b/src/silo/initialize/initializer.h @@ -16,9 +16,14 @@ class Initializer { static silo::schema::DatabaseSchema createSchemaFromConfigFiles( config::DatabaseConfig database_config, ReferenceGenomes reference_genomes, - common::LineageTreeAndIdMap lineage_tree, + std::map lineage_trees, common::PhyloTree phylo_tree_file, bool without_unaligned_columns ); + + static std::optional findLineageTreeForName( + const std::map& lineage_trees, + std::string lineage_tree_name + ); }; } // namespace silo::initialize diff --git a/src/silo/initialize/initializer.test.cpp b/src/silo/initialize/initializer.test.cpp index 51fa9469b..11c7726f1 100644 --- a/src/silo/initialize/initializer.test.cpp +++ b/src/silo/initialize/initializer.test.cpp @@ -5,19 +5,24 @@ #include "silo/storage/column/sequence_column.h" #include "silo/storage/column/zstd_compressed_string_column.h" +using silo::ReferenceGenomes; +using silo::common::LineageTreeAndIdMap; +using silo::common::PhyloTree; +using silo::initialize::Initializer; + TEST(Initializer, correctlyCreatesSchemaFromInitializationFiles) { silo::config::DatabaseConfig database_config = silo::config::DatabaseConfig::getValidatedConfigFromFile( "testBaseData/unitTestDummyDataset/database_config.yaml" ); - silo::ReferenceGenomes reference_genomes = silo::ReferenceGenomes::readFromFile( - "testBaseData/unitTestDummyDataset/reference_genomes.json" - ); - silo::common::PhyloTree phylo_tree_file = silo::common::PhyloTree::fromNewickFile( - "testBaseData/unitTestDummyDataset/phylogenetic_tree.nwk" - ); - auto lineage_tree = silo::common::LineageTreeAndIdMap::fromLineageDefinitionFile( - silo::preprocessing::LineageDefinitionFile::fromYAMLString(R"( + ReferenceGenomes reference_genomes = + ReferenceGenomes::readFromFile("testBaseData/unitTestDummyDataset/reference_genomes.json"); + PhyloTree phylo_tree_file = + PhyloTree::fromNewickFile("testBaseData/unitTestDummyDataset/phylogenetic_tree.nwk"); + std::map lineage_trees{ + {"test_lineage_definition.yaml", + LineageTreeAndIdMap::fromLineageDefinitionFile( + silo::preprocessing::LineageDefinitionFile::fromYAMLString(R"( A: aliases: - X @@ -31,11 +36,12 @@ A.11: parents: - A )") - ); - auto schema = silo::initialize::Initializer::createSchemaFromConfigFiles( + )} + }; + auto schema = Initializer::createSchemaFromConfigFiles( database_config, reference_genomes, - lineage_tree, + lineage_trees, phylo_tree_file, /*without_unaligned_columns=*/false ); @@ -202,3 +208,97 @@ A.11: ASSERT_EQ(table_schema.primary_key.name, "primaryKey"); ASSERT_EQ(table_schema.primary_key.type, ColumnType::STRING); } + +class findLineageTreeForName : public ::testing::Test { + protected: + void SetUp() override { + // Set up test data + test_lineage_tree1 = {}; + test_lineage_tree2 = LineageTreeAndIdMap::fromLineageDefinitionFile( + silo::preprocessing::LineageDefinitionFile::fromYAMLString(R"( +some_lineage: + parents: + - some_parent +some_parent: ~)") + ); + test_lineage_tree3 = LineageTreeAndIdMap::fromLineageDefinitionFile( + silo::preprocessing::LineageDefinitionFile::fromYAMLString(R"( +some_other_lineage: + parents: + - some_parent +some_parent: ~)") + ); + } + + LineageTreeAndIdMap test_lineage_tree1; + LineageTreeAndIdMap test_lineage_tree2; + LineageTreeAndIdMap test_lineage_tree3; +}; + +// Test finding with exact match (no prefix/suffix) +TEST_F(findLineageTreeForName, FindLineageTree_ExactMatch) { + std::map lineage_trees; + lineage_trees["test_tree"] = test_lineage_tree1; + + auto result = Initializer::findLineageTreeForName(lineage_trees, "test_tree"); + + ASSERT_TRUE(result.has_value()); + EXPECT_EQ(result.value().file, test_lineage_tree1.file); +} + +// Test finding with various suffixes +TEST_F(findLineageTreeForName, FindLineageTree_WithFileEnding) { + std::map lineage_trees; + + lineage_trees["test.yaml"] = test_lineage_tree1; + + auto result1 = Initializer::findLineageTreeForName(lineage_trees, "test"); + ASSERT_TRUE(result1.has_value()); + EXPECT_EQ(result1.value().file, test_lineage_tree1.file); +} + +// Test not found scenario +TEST_F(findLineageTreeForName, FindLineageTree_NotFound) { + std::map lineage_trees; + + lineage_trees["completely_different_name"] = test_lineage_tree1; + lineage_trees["another_name.yaml"] = test_lineage_tree2; + + auto result = Initializer::findLineageTreeForName(lineage_trees, "test"); + EXPECT_FALSE(result.has_value()); +} + +// Test empty map +TEST_F(findLineageTreeForName, FindLineageTree_EmptyMap) { + std::map lineage_trees; + + auto result = Initializer::findLineageTreeForName(lineage_trees, "test"); + EXPECT_FALSE(result.has_value()); +} + +// Test case sensitivity +TEST_F(findLineageTreeForName, FindLineageTree_CaseSensitive) { + std::map lineage_trees; + + lineage_trees["Test"] = test_lineage_tree1; + lineage_trees["TEST"] = test_lineage_tree2; + + auto result = Initializer::findLineageTreeForName(lineage_trees, "test"); + EXPECT_FALSE(result.has_value()); // Should not find due to case mismatch + + auto result_correct_case = Initializer::findLineageTreeForName(lineage_trees, "Test"); + ASSERT_TRUE(result_correct_case.has_value()); + EXPECT_EQ(result_correct_case.value().file, test_lineage_tree1.file); +} + +// Test with special characters in name +TEST_F(findLineageTreeForName, FindLineageTree_SpecialCharacters) { + std::map lineage_trees; + + lineage_trees["test-name_123"] = test_lineage_tree1; + lineage_trees["lineage_definition_test-name_123.yaml"] = test_lineage_tree2; + + auto result = Initializer::findLineageTreeForName(lineage_trees, "test-name_123"); + ASSERT_TRUE(result.has_value()); + EXPECT_EQ(result.value().file, test_lineage_tree1.file); +} diff --git a/src/silo/preprocessing/preprocessing.cpp b/src/silo/preprocessing/preprocessing.cpp index 20bd8d5d8..2f52d47e0 100644 --- a/src/silo/preprocessing/preprocessing.cpp +++ b/src/silo/preprocessing/preprocessing.cpp @@ -10,25 +10,29 @@ namespace silo::preprocessing { Database preprocessing(const config::PreprocessingConfig& preprocessing_config) { - SPDLOG_INFO("preprocessing - initializing Database"); - auto database = - initialize::Initializer::initializeDatabase(preprocessing_config.initialization_files); + try { + SPDLOG_INFO("preprocessing - initializing Database"); + auto database = + initialize::Initializer::initializeDatabase(preprocessing_config.initialization_files); - SPDLOG_INFO("preprocessing - successfully initialized Database, now opening input"); - auto input = InputStreamWrapper::openFileOrStdIn(preprocessing_config.getInputFilePath()); + SPDLOG_INFO("preprocessing - successfully initialized Database, now opening input"); + auto input = InputStreamWrapper::openFileOrStdIn(preprocessing_config.getInputFilePath()); - try { SPDLOG_INFO("preprocessing - appending data to Database"); auto input_data = append::NdjsonLineReader{input.getInputStream()}; append::appendDataToDatabase(database, input_data); + + SPDLOG_INFO("preprocessing - finished preprocessing"); + return database; + } catch (const initialize::InitializeException& exception) { + throw preprocessing::PreprocessingException( + "preprocessing - exception when initializing database: {}", exception.what() + ); } catch (const append::AppendException& exception) { throw preprocessing::PreprocessingException( "preprocessing - exception when appending data: {}", exception.what() ); } - - SPDLOG_INFO("preprocessing - finished preprocessing"); - return database; } } // namespace silo::preprocessing diff --git a/src/silo/preprocessing/preprocessing.test.cpp b/src/silo/preprocessing/preprocessing.test.cpp index 273d8b0b0..2a9aa84a5 100644 --- a/src/silo/preprocessing/preprocessing.test.cpp +++ b/src/silo/preprocessing/preprocessing.test.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include #include @@ -40,7 +41,7 @@ struct Scenario { std::function()> input_data; std::string database_config; std::string reference_genomes; - std::string lineage_tree; + std::map lineage_trees; Assertion assertion; }; @@ -63,13 +64,23 @@ silo::config::PreprocessingConfig prepareInputDirAndPreprocessorForScenario( reference_genomes_file << scenario.reference_genomes; reference_genomes_file.close(); - std::ofstream lineage_definitions_file(input_directory / "lineage_definitions.yaml"); - lineage_definitions_file << scenario.lineage_tree; - lineage_definitions_file.close(); + for (const auto& [filename, lineage_tree] : scenario.lineage_trees) { + // Assert that 'filename' is a filename and not a path + SILO_ASSERT_EQ(filename.filename(), filename); + std::ofstream lineage_definition_file(input_directory / filename); + lineage_definition_file << lineage_tree; + lineage_definition_file.close(); + } auto config_with_input_dir = PreprocessingConfig::withDefaults(); config_with_input_dir.initialization_files.directory = input_directory; config_with_input_dir.input_file = "input.json"; + if (not scenario.lineage_trees.empty()) { + auto keys = scenario.lineage_trees | std::views::keys | + std::views::transform([](const std::filesystem::path& p) { return p.string(); }); + config_with_input_dir.initialization_files.lineage_definition_files = + std::vector(keys.begin(), keys.end()); + } config_with_input_dir.validate(); std::ofstream file(config_with_input_dir.getInputFilePath().value()); @@ -350,7 +361,7 @@ const Scenario EMPTY_INPUT_NDJSON = { - name: "2" type: "string" generateIndex: true - generateLineageIndex: true + generateLineageIndex: test_lineage_definition.yaml primaryKey: "accessionVersion" )", .reference_genomes = R"( @@ -376,6 +387,7 @@ const Scenario EMPTY_INPUT_NDJSON = { } ] })", + .lineage_trees = {{"test_lineage_definition.yaml", "main: ~\n"}}, .assertion{ .expected_sequence_count = 0, .query = R"( @@ -781,6 +793,79 @@ const Scenario PREVENT_LATE_AUTO_CASTING = { } }; +const Scenario TWO_LINEAGE_SYSTEMS = { + .test_name = "TWO_LINEAGE_SYSTEMS", + .input_data = + []() { + std::vector result; + result.push_back(nlohmann::json::parse(R"({ +"accessionVersion": "0", "lineage_1": "root_1", "lineage_2": "root_2" +})")); + result.push_back(nlohmann::json::parse(R"({ +"accessionVersion": "1", "lineage_1": "child_1", "lineage_2": null +})")); + result.push_back(nlohmann::json::parse(R"({ +"accessionVersion": "2", "lineage_1": null, "lineage_2": "child_2" +})")); + return result; + }, + .database_config = + R"( +schema: + instanceName: "Test" + metadata: + - name: "accessionVersion" + type: "string" + - name: "lineage_1" + type: "string" + generateIndex: true + generateLineageIndex: lineage_definition_1 + - name: "lineage_2" + type: "string" + generateIndex: true + generateLineageIndex: lineage_definition_2 + primaryKey: "accessionVersion" +)", + .reference_genomes = R"( +{ + "nucleotideSequences": [], + "genes": [] +})", + .lineage_trees = + {{"lineage_definition_1.yaml", R"( +root_1: ~ +child_1: + parents: + - root_1 + )"}, + {"lineage_definition_2.yaml", R"( +root_2: ~ +child_2: + parents: + - root_2)"}}, + .assertion{ + .expected_sequence_count = 3, + .query = R"( + { + "action": { + "type": "Details", + "orderByFields": ["accessionVersion"] + }, + "filterExpression": { + "type": "Lineage", + "column": "lineage_1", + "value": "root_1", + "includeSublineages": true + } + } + )", + .expected_query_result = nlohmann::json::parse(R"([ +{"accessionVersion":"0","lineage_1":"root_1","lineage_2":"root_2"}, +{"accessionVersion":"1","lineage_1":"child_1","lineage_2":null} +])") + } +}; + class PreprocessorTestFixture : public ::testing::TestWithParam> {}; const auto testCases = ::testing::Values( @@ -793,7 +878,8 @@ const auto testCases = ::testing::Values( NO_NUCLEOTIDE_SEQUENCES, NO_SEQUENCES, DIVERSE_SEQUENCE_NAMES_NDJSON, - PREVENT_LATE_AUTO_CASTING + PREVENT_LATE_AUTO_CASTING, + TWO_LINEAGE_SYSTEMS ); INSTANTIATE_TEST_SUITE_P(PreprocessorTest, PreprocessorTestFixture, testCases, printTestName); diff --git a/src/silo/query_engine/filter/expressions/lineage_filter.test.cpp b/src/silo/query_engine/filter/expressions/lineage_filter.test.cpp index e0ff08d18..ca317e55f 100644 --- a/src/silo/query_engine/filter/expressions/lineage_filter.test.cpp +++ b/src/silo/query_engine/filter/expressions/lineage_filter.test.cpp @@ -58,7 +58,7 @@ defaultNucleotideSequence: "segment1" - name: "pango_lineage" type: "string" generateIndex: true - generateLineageIndex: true + generateLineageIndex: test_lineage_index primaryKey: "primaryKey" )"; @@ -87,7 +87,7 @@ const QueryTestData TEST_DATA{ .ndjson_input_data = DATA, .database_config = DATABASE_CONFIG, .reference_genomes = REFERENCE_GENOMES, - .lineage_tree = LINEAGE_TREE + .lineage_trees = {{"test_lineage_index", LINEAGE_TREE}} }; nlohmann::json createLineageQuery(const nlohmann::json value, bool include_sublineages) { diff --git a/src/silo/storage/column/indexed_string_column.test.cpp b/src/silo/storage/column/indexed_string_column.test.cpp index fa1448c27..837ce97c9 100644 --- a/src/silo/storage/column/indexed_string_column.test.cpp +++ b/src/silo/storage/column/indexed_string_column.test.cpp @@ -55,10 +55,10 @@ TEST(IndexedStringColumnPartition, insertValuesToPartition) { } TEST(IndexedStringColumnPartition, addingLineageAndThenSublineageFiltersCorrectly) { - auto lineage_definitions = LineageTreeAndIdMap::fromLineageDefinitionFilePath( - "testBaseData/exampleDataset/lineage_definitions.yaml" + auto lineage_definition = LineageTreeAndIdMap::fromLineageDefinitionFilePath( + "testBaseData/exampleDataset/lineage_definition.yaml" ); - IndexedStringColumnMetadata column_metadata("some_column", lineage_definitions); + IndexedStringColumnMetadata column_metadata("some_column", lineage_definition); IndexedStringColumnPartition under_test{&column_metadata}; under_test.insert({"BA.1.1"}); @@ -89,10 +89,10 @@ TEST(IndexedStringColumnPartition, addingLineageAndThenSublineageFiltersCorrectl } TEST(IndexedStringColumnPartition, addingSublineageAndThenLineageFiltersCorrectly) { - auto lineage_definitions = LineageTreeAndIdMap::fromLineageDefinitionFilePath( - "testBaseData/exampleDataset/lineage_definitions.yaml" + auto lineage_definition = LineageTreeAndIdMap::fromLineageDefinitionFilePath( + "testBaseData/exampleDataset/lineage_definition.yaml" ); - IndexedStringColumnMetadata column_metadata("some_column", lineage_definitions); + IndexedStringColumnMetadata column_metadata("some_column", lineage_definition); IndexedStringColumnPartition under_test{&column_metadata}; under_test.insert({"BA.1.1.1"}); @@ -137,10 +137,10 @@ TEST(IndexedStringColumnPartition, addingSublineageAndThenLineageFiltersCorrectl } TEST(IndexedStringColumnPartition, queryParentLineageThatWasNeverInserted) { - auto lineage_definitions = LineageTreeAndIdMap::fromLineageDefinitionFilePath( - "testBaseData/exampleDataset/lineage_definitions.yaml" + auto lineage_definition = LineageTreeAndIdMap::fromLineageDefinitionFilePath( + "testBaseData/exampleDataset/lineage_definition.yaml" ); - IndexedStringColumnMetadata column_metadata("some_column", lineage_definitions); + IndexedStringColumnMetadata column_metadata("some_column", lineage_definition); IndexedStringColumnPartition under_test{&column_metadata}; under_test.insert({"BA.1.1.1"}); @@ -164,13 +164,13 @@ TEST(IndexedStringColumnPartition, queryParentLineageThatWasNeverInserted) { } TEST(IndexedStringColumnPartition, errorWhenInsertingIncorrectLineages) { - auto lineage_definitions = + auto lineage_definition = LineageTreeAndIdMap::fromLineageDefinitionFile(LineageDefinitionFile::fromYAMLString(R"( A: {} A.1: parents: ["A"] )")); - IndexedStringColumnMetadata column_metadata("some_column", lineage_definitions); + IndexedStringColumnMetadata column_metadata("some_column", lineage_definition); IndexedStringColumnPartition under_test{&column_metadata}; under_test.insert({"A"}); EXPECT_THAT( diff --git a/src/silo/test/default_sequence.test.cpp b/src/silo/test/default_sequence.test.cpp index e884f4634..499b3a184 100644 --- a/src/silo/test/default_sequence.test.cpp +++ b/src/silo/test/default_sequence.test.cpp @@ -55,7 +55,7 @@ const QueryTestData TEST_DATA{ .ndjson_input_data = {DATA_DIFFERENT_FROM_REFERENCE, DATA_EQUALS_TO_REFERENCE}, .database_config = DATABASE_CONFIG, .reference_genomes = REFERENCE_GENOMES, - .lineage_tree = silo::common::LineageTreeAndIdMap() + .lineage_trees = {{"test", silo::common::LineageTreeAndIdMap()}} }; nlohmann::json createQueryWithFilter(const nlohmann::json filter) { diff --git a/src/silo/test/query_fixture.test.h b/src/silo/test/query_fixture.test.h index c0fd9fc48..0ac8a462d 100644 --- a/src/silo/test/query_fixture.test.h +++ b/src/silo/test/query_fixture.test.h @@ -60,7 +60,7 @@ struct QueryTestData { const std::vector ndjson_input_data; const std::string database_config; const silo::ReferenceGenomes reference_genomes; - const silo::common::LineageTreeAndIdMap lineage_tree; + const std::map lineage_trees; const silo::common::PhyloTree phylo_tree_file; const bool without_unaligned_sequences = false; }; @@ -88,7 +88,7 @@ class QueryTestFixture : public ::testing::TestWithParam { Database{silo::initialize::Initializer::createSchemaFromConfigFiles( silo::config::DatabaseConfig::getValidatedConfig(test_data.database_config), std::move(test_data.reference_genomes), - std::move(test_data.lineage_tree), + std::move(test_data.lineage_trees), std::move(test_data.phylo_tree_file), test_data.without_unaligned_sequences )} diff --git a/testBaseData/exampleDataset/database_config.yaml b/testBaseData/exampleDataset/database_config.yaml index 73b56b5fa..df26749cc 100644 --- a/testBaseData/exampleDataset/database_config.yaml +++ b/testBaseData/exampleDataset/database_config.yaml @@ -18,7 +18,7 @@ schema: - name: pango_lineage type: string generateIndex: true - generateLineageIndex: true + generateLineageIndex: lineage_definition.yaml - name: division type: string generateIndex: true diff --git a/testBaseData/exampleDataset/lineage_definitions.yaml b/testBaseData/exampleDataset/lineage_definition.yaml similarity index 100% rename from testBaseData/exampleDataset/lineage_definitions.yaml rename to testBaseData/exampleDataset/lineage_definition.yaml diff --git a/testBaseData/exampleDataset/preprocessing_config.yaml b/testBaseData/exampleDataset/preprocessing_config.yaml index af7b10a9a..9920c51df 100644 --- a/testBaseData/exampleDataset/preprocessing_config.yaml +++ b/testBaseData/exampleDataset/preprocessing_config.yaml @@ -1,4 +1,5 @@ ndjsonInputFilename: "input_file.ndjson" -lineageDefinitionsFilename: "lineage_definitions.yaml" +lineageDefinitionFilenames: + - "lineage_definition.yaml" referenceGenomeFilename: "reference_genomes.json" phyloTreeFilename: "phylogenetic_tree.nwk" \ No newline at end of file diff --git a/testBaseData/test_database_config.yaml b/testBaseData/test_database_config.yaml index 206fdeb94..850a69805 100644 --- a/testBaseData/test_database_config.yaml +++ b/testBaseData/test_database_config.yaml @@ -17,7 +17,7 @@ schema: - name: pango_lineage type: string generateIndex: true - generateLineageIndex: true + generateLineageIndex: some_test_value - name: division type: string generateIndex: true diff --git a/testBaseData/test_database_config_without_partition_by.yaml b/testBaseData/test_database_config_without_partition_by.yaml index 9bd62d54e..9ea95de46 100644 --- a/testBaseData/test_database_config_without_partition_by.yaml +++ b/testBaseData/test_database_config_without_partition_by.yaml @@ -17,7 +17,7 @@ schema: - name: pango_lineage type: string generateIndex: true - generateLineageIndex: true + generateLineageIndex: lineage - name: division type: string generateIndex: true diff --git a/testBaseData/test_preprocessing_config.yaml b/testBaseData/test_preprocessing_config.yaml index 2060888eb..b62e0c7a6 100644 --- a/testBaseData/test_preprocessing_config.yaml +++ b/testBaseData/test_preprocessing_config.yaml @@ -1,6 +1,7 @@ inputDirectory: "./testBaseData/exampleDataset/" outputDirectory: "./output/" ndjsonInputFilename: "input_file.ndjson" -lineageDefinitionsFilename: "lineage_definitions.yaml" +lineageDefinitionFilenames: + - "lineage_definition.yaml" phyloTreeFilename: "phylogenetic_tree.nwk" referenceGenomeFilename: "reference_genomes.json" diff --git a/testBaseData/unitTestDummyDataset/database_config.yaml b/testBaseData/unitTestDummyDataset/database_config.yaml index b7b5b011e..e9c5bb0f3 100644 --- a/testBaseData/unitTestDummyDataset/database_config.yaml +++ b/testBaseData/unitTestDummyDataset/database_config.yaml @@ -17,7 +17,7 @@ schema: - name: pango_lineage type: string generateIndex: true - generateLineageIndex: true + generateLineageIndex: "test_lineage_definition" - name: division type: string generateIndex: true diff --git a/testBaseData/unitTestDummyDataset/lineage_definitions.yaml b/testBaseData/unitTestDummyDataset/lineage_definitions.yaml deleted file mode 120000 index 158904910..000000000 --- a/testBaseData/unitTestDummyDataset/lineage_definitions.yaml +++ /dev/null @@ -1 +0,0 @@ -../exampleDataset/lineage_definitions.yaml \ No newline at end of file diff --git a/testBaseData/unitTestDummyDataset/preprocessing_config.yaml b/testBaseData/unitTestDummyDataset/preprocessing_config.yaml index 2a50adb84..f4d98b9cd 100644 --- a/testBaseData/unitTestDummyDataset/preprocessing_config.yaml +++ b/testBaseData/unitTestDummyDataset/preprocessing_config.yaml @@ -1,6 +1,7 @@ inputDirectory: "./testBaseData/unitTestDummyDataset/" outputDirectory: "./output/" ndjsonInputFilename: "input.ndjson" -lineageDefinitionsFilename: "lineage_definitions.yaml" +lineageDefinitionFilenames: + - "test_lineage_definition.yaml" phyloTreeFilename: "phylogenetic_tree.nwk" referenceGenomeFilename: "reference_genomes.json" diff --git a/testBaseData/unitTestDummyDataset/test_lineage_definition.yaml b/testBaseData/unitTestDummyDataset/test_lineage_definition.yaml new file mode 120000 index 000000000..76aefff65 --- /dev/null +++ b/testBaseData/unitTestDummyDataset/test_lineage_definition.yaml @@ -0,0 +1 @@ +../exampleDataset/lineage_definition.yaml \ No newline at end of file