feat(silo)!: support multiple lineage systems

src/config/config_specification.cpp

@@ -152,6 +152,8 @@ ConfigValue ConfigAttributeSpecification::parseValueFromString(std::string value
             throw ConfigException(
                fmt::format("'{}' is not a valid string for a boolean", value_string)
             );
+         case ConfigValueType::LIST:
+            throw ConfigException("List values can currently no be specified as strings.");
       }
       SILO_UNREACHABLE();
    } catch (boost::bad_lexical_cast&) {

src/config/config_value.cpp

@@ -1,5 +1,6 @@
 #include "config/config_value.h"
 
+#include <fmt/ranges.h>
 #include <boost/functional/hash.hpp>
 #include <boost/lexical_cast.hpp>
 
@@ -28,6 +29,9 @@ ConfigValueType ConfigValue::getValueType() const {
    if (std::holds_alternative<bool>(value)) {
       return ConfigValueType::BOOL;
    }
+   if (std::holds_alternative<std::vector<std::string>>(value)) {
+      return ConfigValueType::LIST;
+   }
    SILO_UNREACHABLE();
 }
 
@@ -39,6 +43,8 @@ std::string ConfigValue::toString() const {
             return fmt::format("'{}'", value);
          } else if constexpr (std::is_same_v<T, std::filesystem::path>) {
             return fmt::format("'{}'", value.string());
+         } else if constexpr (std::is_same_v<T, std::vector<std::string>>) {
+            return fmt::format("{}", fmt::join(value, ","));
          } else {
             return fmt::format("{}", value);
          }

src/config/config_value.h

@@ -12,7 +12,7 @@
 
 namespace silo::config {
 
-enum class ConfigValueType { STRING, PATH, INT32, UINT32, UINT16, BOOL };
+enum class ConfigValueType { STRING, PATH, INT32, UINT32, UINT16, BOOL, LIST };
 
 constexpr std::string_view configValueTypeToString(ConfigValueType type) {
    switch (type) {
@@ -28,18 +28,33 @@ constexpr std::string_view configValueTypeToString(ConfigValueType type) {
          return "u16";
       case ConfigValueType::BOOL:
          return "bool";
+      case ConfigValueType::LIST:
+         return "list";
    }
    SILO_UNREACHABLE();
 }
 
 class ConfigValue {
-   explicit ConfigValue(
-      std::variant<std::string, std::filesystem::path, int32_t, uint32_t, uint16_t, bool> value
-   )
+   explicit ConfigValue(std::variant<
+                        std::string,
+                        std::filesystem::path,
+                        int32_t,
+                        uint32_t,
+                        uint16_t,
+                        bool,
+                        std::vector<std::string>> value)
        : value(std::move(value)) {}
 
   public:
-   std::variant<std::string, std::filesystem::path, int32_t, uint32_t, uint16_t, bool> value;
+   std::variant<
+      std::string,
+      std::filesystem::path,
+      int32_t,
+      uint32_t,
+      uint16_t,
+      bool,
+      std::vector<std::string>>
+      value;
 
    static ConfigValue fromString(const std::string& value) { return ConfigValue{value}; }
 
@@ -57,6 +72,8 @@ class ConfigValue {
 
    static ConfigValue fromBool(bool value) { return ConfigValue{value}; }
 
+   static ConfigValue fromList(const std::vector<std::string>& value) { return ConfigValue{value}; }
+
    [[nodiscard]] ConfigValueType getValueType() const;
 
    [[nodiscard]] std::string toString() const;

src/config/source/yaml_file.cpp

@@ -17,6 +17,13 @@ using silo::config::ConfigKeyPath;
 
 namespace {
 
+bool isSequenceOfScalars(const YAML::Node& node) {
+   if (!node.IsSequence()) {
+      return false;
+   }
+   return std::ranges::all_of(node, [](const auto& element) { return element.IsScalar(); });
+}
+
 bool isProperSingularValue(const YAML::Node& node) {
    if (node.IsMap()) {
       SPDLOG_TRACE("isProperSingularValue = false, node is a map");
@@ -26,8 +33,8 @@ bool isProperSingularValue(const YAML::Node& node) {
       SPDLOG_TRACE("isProperSingularValue = false, node is not defined");
       return false;
    }
-   if (!node.IsScalar()) {
-      SPDLOG_TRACE("isProperSingularValue = false, node is not a scalar");
+   if (!node.IsScalar() && !isSequenceOfScalars(node)) {
+      SPDLOG_TRACE("isProperSingularValue = false, node is not a scalar or sequence of strings");
       return false;
    }
    return true;
@@ -211,6 +218,8 @@ ConfigValue yamlNodeToConfigValue(
             return ConfigValue::fromUint16(yaml.as<uint16_t>());
          case ConfigValueType::BOOL:
             return ConfigValue::fromBool(yaml.as<bool>());
+         case ConfigValueType::LIST:
+            return ConfigValue::fromList(yaml.as<std::vector<std::string>>());
       }
       SILO_UNREACHABLE();
    } catch (YAML::BadConversion& error) {

src/config/source/yaml_file.test.cpp

@@ -57,7 +57,7 @@ TEST(YamlFile, containsCorrectFieldsFromFlatYAML) {
 inputDirectory: "./testBaseData/exampleDataset/"
 outputDirectory: "./output/"
 ndjsonInputFilename: "input_file.ndjson"
-lineageDefinitionsFilename: "lineage_definitions.yaml"
+lineageDefinitionFilenames: "lineage_definition.yaml"
 phyloTreeFilename: "phylogenetic_tree.yaml"
 referenceGenomeFilename: "reference_genomes.json"
 )"
@@ -69,8 +69,8 @@ referenceGenomeFilename: "reference_genomes.json"
        YAML::Node{"./testBaseData/exampleDataset/"}},
       {YamlFile::stringToConfigKeyPath("outputDirectory"), YAML::Node{"./output/"}},
       {YamlFile::stringToConfigKeyPath("ndjsonInputFilename"), YAML::Node{"input_file.ndjson"}},
-      {YamlFile::stringToConfigKeyPath("lineageDefinitionsFilename"),
-       YAML::Node{"lineage_definitions.yaml"}},
+      {YamlFile::stringToConfigKeyPath("lineageDefinitionFilenames"),
+       YAML::Node{"lineage_definition.yaml"}},
       {YamlFile::stringToConfigKeyPath("phyloTreeFilename"), YAML::Node{"phylogenetic_tree.yaml"}},
       {YamlFile::stringToConfigKeyPath("referenceGenomeFilename"),
        YAML::Node{"reference_genomes.json"}},

src/config/verified_config_attributes.cpp

@@ -88,6 +88,12 @@ std::optional<bool> VerifiedConfigAttributes::getBool(const ConfigKeyPath& confi
    return std::nullopt;
 }
 
+std::optional<std::vector<std::string>> VerifiedConfigAttributes::getList(
+   const ConfigKeyPath& config_key_path
+) const {
+   return getValue<std::vector<std::string>, ConfigValueType::LIST>(config_key_path, config_values);
+}
+
 VerifiedCommandLineArguments VerifiedCommandLineArguments::askingForHelp() {
    VerifiedCommandLineArguments result;
    result.asks_for_help = true;

src/config/verified_config_attributes.h

@@ -33,6 +33,10 @@ class VerifiedConfigAttributes {
    [[nodiscard]] std::optional<uint16_t> getUint16(const ConfigKeyPath& config_key_path) const;
 
    [[nodiscard]] std::optional<bool> getBool(const ConfigKeyPath& config_key_path) const;
+
+   [[nodiscard]] std::optional<std::vector<std::string>> getList(
+      const ConfigKeyPath& config_key_path
+   ) const;
 };
 
 class VerifiedCommandLineArguments : public VerifiedConfigAttributes {

src/main.cpp

@@ -47,7 +47,7 @@ int runPreprocessor(const silo::config::PreprocessingConfig& preprocessing_confi
       database.saveDatabaseState(preprocessing_config.output_directory);
       return 0;
    } catch (const silo::preprocessing::PreprocessingException& preprocessing_exception) {
-      SPDLOG_ERROR("initialize - error: {}", preprocessing_exception.what());
+      SPDLOG_ERROR("preprocessing - error: {}", preprocessing_exception.what());
       return 1;
    }
 }

src/silo/common/lineage_tree.test.cpp

@@ -254,7 +254,7 @@ TEST(LineageDefinitionFile, errorOnLineageAsAlias) {
 
 TEST(containsCycle, doesNotFindCycleInPangoLineageTree) {
    ASSERT_NO_THROW(LineageTreeAndIdMap::fromLineageDefinitionFilePath(
-      "testBaseData/exampleDataset/lineage_definitions.yaml"
+      "testBaseData/exampleDataset/lineage_definition.yaml"
    ));
 }
 

src/silo/config/database_config.cpp

@@ -138,9 +138,9 @@ bool YAML::convert<silo::config::DatabaseMetadata>::decode(
       metadata.generate_index = false;
    }
    if (node["generateLineageIndex"].IsDefined()) {
-      metadata.generate_lineage_index = node["generateLineageIndex"].as<bool>();
+      metadata.generate_lineage_index = node["generateLineageIndex"].as<std::string>();
    } else {
-      metadata.generate_lineage_index = false;
+      metadata.generate_lineage_index = std::nullopt;
    }
    if (node["isPhyloTreeField"].IsDefined()) {
       metadata.phylo_tree_node_identifier = node["isPhyloTreeField"].as<bool>();

src/silo/config/database_config.h

@@ -23,7 +23,7 @@ class DatabaseMetadata {
    std::string name;
    ValueType type;
    bool generate_index;
-   bool generate_lineage_index;
+   std::optional<std::string> generate_lineage_index;
    bool phylo_tree_node_identifier;
 
    [[nodiscard]] schema::ColumnType getColumnType() const;

src/silo/config/database_config.test.cpp

@@ -115,7 +115,7 @@ TEST(DatabaseConfig, shouldReadConfigWithCorrectParameters) {
    ASSERT_EQ(config.schema.metadata[0].name, "gisaid_epi_isl");
    ASSERT_EQ(config.schema.metadata[0].type, ValueType::STRING);
    ASSERT_EQ(config.schema.metadata[0].generate_index, false);
-   ASSERT_EQ(config.schema.metadata[0].generate_lineage_index, false);
+   ASSERT_EQ(config.schema.metadata[0].generate_lineage_index, std::nullopt);
    ASSERT_EQ(config.schema.metadata[0].phylo_tree_node_identifier, true);
    ASSERT_EQ(config.schema.metadata[0].generate_index, false);
    ASSERT_EQ(config.schema.metadata[1].name, "date");
@@ -133,11 +133,11 @@ TEST(DatabaseConfig, shouldReadConfigWithCorrectParameters) {
    ASSERT_EQ(config.schema.metadata[5].name, "pango_lineage");
    ASSERT_EQ(config.schema.metadata[5].type, ValueType::STRING);
    ASSERT_EQ(config.schema.metadata[5].generate_index, true);
-   ASSERT_EQ(config.schema.metadata[5].generate_lineage_index, true);
+   ASSERT_EQ(config.schema.metadata[5].generate_lineage_index, "some_test_value");
    ASSERT_EQ(config.schema.metadata[6].name, "division");
    ASSERT_EQ(config.schema.metadata[6].type, ValueType::STRING);
    ASSERT_EQ(config.schema.metadata[6].generate_index, true);
-   ASSERT_EQ(config.schema.metadata[6].generate_lineage_index, false);
+   ASSERT_EQ(config.schema.metadata[6].generate_lineage_index, std::nullopt);
    ASSERT_EQ(config.schema.metadata[7].name, "age");
    ASSERT_EQ(config.schema.metadata[7].type, ValueType::INT);
    ASSERT_EQ(config.schema.metadata[7].generate_index, false);
@@ -250,7 +250,7 @@ defaultNucleotideSequence: "main"
     - name: "metadata1"
       type: "string"
       generateIndex: true
-      generateLineageIndex: true
+      generateLineageIndex: lineage
     - name: "metadata2"
       type: "date"
     - name: "metadata3"
@@ -331,7 +331,7 @@ defaultNucleotideSequence: "main"
       type: "string"
     - name: "some lineage"
       type: "string"
-      generateLineageIndex: true
+      generateLineageIndex: lineage
   primaryKey: "testPrimaryKey"
 )";
 

src/silo/config/initialize_config.cpp

@@ -26,8 +26,8 @@ ConfigKeyPath inputDirectoryOptionKey() {
 ConfigKeyPath outputDirectoryOptionKey() {
    return YamlFile::stringToConfigKeyPath("outputDirectory");
 }
-ConfigKeyPath lineageDefinitionsFilenameOptionKey() {
-   return YamlFile::stringToConfigKeyPath("lineageDefinitionsFilename");
+ConfigKeyPath lineageDefinitionFilenamesOptionKey() {
+   return YamlFile::stringToConfigKeyPath("lineageDefinitionFilenames");
 }
 ConfigKeyPath phyloTreeFilenameOptionKey() {
    return YamlFile::stringToConfigKeyPath("phyloTreeFilename");
@@ -67,9 +67,9 @@ ConfigSpecification InitializeConfig::getConfigSpecification() {
             "The path to the directory to hold the output files."
          ),
          ConfigAttributeSpecification::createWithoutDefault(
-            lineageDefinitionsFilenameOptionKey(),
-            ConfigValueType::PATH,
-            "File name of the file holding the lineage definitions. Relative from inputDirectory."
+            lineageDefinitionFilenamesOptionKey(),
+            ConfigValueType::LIST,
+            "List of file names holding the lineage definitions. Relative from inputDirectory."
          ),
          ConfigAttributeSpecification::createWithoutDefault(
             phyloTreeFilenameOptionKey(),
@@ -108,10 +108,12 @@ std::filesystem::path InitializationFiles::getDatabaseConfigFilename() const {
    return directory / database_config_file;
 }
 
-std::optional<std::filesystem::path> InitializationFiles::getLineageDefinitionsFilename() const {
-   return lineage_definitions_file.has_value()
-             ? std::optional(directory / lineage_definitions_file.value())
-             : std::nullopt;
+std::vector<std::filesystem::path> InitializationFiles::getLineageDefinitionFilenames() const {
+   std::vector<std::filesystem::path> paths;
+   for (const auto& file_name : lineage_definition_files) {
+      paths.push_back(directory / file_name);
+   }
+   return paths;
 }
 
 std::optional<std::filesystem::path> InitializationFiles::getPhyloTreeFilename() const {
@@ -128,8 +130,8 @@ void InitializeConfig::overwriteFrom(const VerifiedConfigAttributes& config_sour
    if (auto var = config_source.getPath(inputDirectoryOptionKey())) {
       initialization_files.directory = var.value();
    }
-   if (auto var = config_source.getPath(lineageDefinitionsFilenameOptionKey())) {
-      initialization_files.lineage_definitions_file = var.value();
+   if (auto var = config_source.getList(lineageDefinitionFilenamesOptionKey())) {
+      initialization_files.lineage_definition_files = var.value();
    }
    if (auto var = config_source.getPath(phyloTreeFilenameOptionKey())) {
       initialization_files.phylogenetic_tree_file = var.value();

src/silo/config/initialize_config.h

@@ -20,7 +20,7 @@ class PreprocessingConfig;
 
 class InitializationFiles {
   public:
-   std::optional<std::filesystem::path> lineage_definitions_file;
+   std::vector<std::string> lineage_definition_files;
    std::optional<std::filesystem::path> phylogenetic_tree_file;
    std::filesystem::path database_config_file;
    std::filesystem::path reference_genome_file;
@@ -30,7 +30,7 @@ class InitializationFiles {
 
    [[nodiscard]] std::filesystem::path getDatabaseConfigFilename() const;
 
-   [[nodiscard]] std::optional<std::filesystem::path> getLineageDefinitionsFilename() const;
+   [[nodiscard]] std::vector<std::filesystem::path> getLineageDefinitionFilenames() const;
 
    [[nodiscard]] std::optional<std::filesystem::path> getPhyloTreeFilename() const;
 
@@ -39,7 +39,7 @@ class InitializationFiles {
    NLOHMANN_DEFINE_TYPE_INTRUSIVE(
       InitializationFiles,
       directory,
-      lineage_definitions_file,
+      lineage_definition_files,
       phylogenetic_tree_file,
       database_config_file,
       reference_genome_file

src/silo/config/preprocessing_config.cpp

@@ -29,8 +29,8 @@ ConfigKeyPath inputDirectoryOptionKey() {
 ConfigKeyPath outputDirectoryOptionKey() {
    return YamlFile::stringToConfigKeyPath("outputDirectory");
 }
-ConfigKeyPath lineageDefinitionsFilenameOptionKey() {
-   return YamlFile::stringToConfigKeyPath("lineageDefinitionsFilename");
+ConfigKeyPath lineageDefinitionFilenamesOptionKey() {
+   return YamlFile::stringToConfigKeyPath("lineageDefinitionFilenames");
 }
 ConfigKeyPath phyloTreeFilenameOptionKey() {
    return YamlFile::stringToConfigKeyPath("phyloTreeFilename");
@@ -90,9 +90,9 @@ ConfigSpecification PreprocessingConfig::getConfigSpecification() {
             "The path to the directory to hold the output files."
          ),
          ConfigAttributeSpecification::createWithoutDefault(
-            lineageDefinitionsFilenameOptionKey(),
-            ConfigValueType::PATH,
-            "File name of the file holding the lineage definitions. Relative from inputDirectory."
+            lineageDefinitionFilenamesOptionKey(),
+            ConfigValueType::LIST,
+            "List of file names holding the lineage definitions. Relative from inputDirectory."
          ),
          ConfigAttributeSpecification::createWithoutDefault(
             phyloTreeFilenameOptionKey(),
@@ -152,8 +152,8 @@ void PreprocessingConfig::overwriteFrom(const VerifiedConfigAttributes& config_s
    if (auto var = config_source.getPath(inputDirectoryOptionKey())) {
       initialization_files.directory = var.value();
    }
-   if (auto var = config_source.getPath(lineageDefinitionsFilenameOptionKey())) {
-      initialization_files.lineage_definitions_file = var.value();
+   if (auto var = config_source.getList(lineageDefinitionFilenamesOptionKey())) {
+      initialization_files.lineage_definition_files = var.value();
    }
    if (auto var = config_source.getPath(phyloTreeFilenameOptionKey())) {
       initialization_files.phylogenetic_tree_file = var.value();