feat(silo)!: support multiple lineage systems

BREAKING CHANGE: Multiple breaking changes to lineage configuration:
- PreprocessingConfig: `lineageDefinitionsFilename` renamed to
  `lineageDefinitionFilenames` and changed from string to array
- DatabaseConfig: `generateLineageIndex` changed from boolean to string
  (expects lineage filename with automatic prefix/suffix detection)

Author: taepper

src/config/config_specification.cpp

@@ -152,6 +152,8 @@ ConfigValue ConfigAttributeSpecification::parseValueFromString(std::string value
             throw ConfigException(
                fmt::format("'{}' is not a valid string for a boolean", value_string)
             );
+         case ConfigValueType::LIST:
+            throw ConfigException("List values can currently no be specified as strings.");
       }
       SILO_UNREACHABLE();
    } catch (boost::bad_lexical_cast&) {

src/config/config_value.cpp

@@ -1,5 +1,6 @@
 #include "config/config_value.h"
 
+#include <fmt/ranges.h>
 #include <boost/functional/hash.hpp>
 #include <boost/lexical_cast.hpp>
 
@@ -28,6 +29,9 @@ ConfigValueType ConfigValue::getValueType() const {
    if (std::holds_alternative<bool>(value)) {
       return ConfigValueType::BOOL;
    }
+   if (std::holds_alternative<std::vector<std::string>>(value)) {
+      return ConfigValueType::LIST;
+   }
    SILO_UNREACHABLE();
 }
 
@@ -39,6 +43,8 @@ std::string ConfigValue::toString() const {
             return fmt::format("'{}'", value);
          } else if constexpr (std::is_same_v<T, std::filesystem::path>) {
             return fmt::format("'{}'", value.string());
+         } else if constexpr (std::is_same_v<T, std::vector<std::string>>) {
+            return fmt::format("{}", fmt::join(value, ","));
          } else {
             return fmt::format("{}", value);
          }

src/config/config_value.h

@@ -12,7 +12,7 @@
 
 namespace silo::config {
 
-enum class ConfigValueType { STRING, PATH, INT32, UINT32, UINT16, BOOL };
+enum class ConfigValueType { STRING, PATH, INT32, UINT32, UINT16, BOOL, LIST };
 
 constexpr std::string_view configValueTypeToString(ConfigValueType type) {
    switch (type) {
@@ -28,18 +28,33 @@ constexpr std::string_view configValueTypeToString(ConfigValueType type) {
          return "u16";
       case ConfigValueType::BOOL:
          return "bool";
+      case ConfigValueType::LIST:
+         return "list";
    }
    SILO_UNREACHABLE();
 }
 
 class ConfigValue {
-   explicit ConfigValue(
-      std::variant<std::string, std::filesystem::path, int32_t, uint32_t, uint16_t, bool> value
-   )
+   explicit ConfigValue(std::variant<
+                        std::string,
+                        std::filesystem::path,
+                        int32_t,
+                        uint32_t,
+                        uint16_t,
+                        bool,
+                        std::vector<std::string>> value)
        : value(std::move(value)) {}
 
   public:
-   std::variant<std::string, std::filesystem::path, int32_t, uint32_t, uint16_t, bool> value;
+   std::variant<
+      std::string,
+      std::filesystem::path,
+      int32_t,
+      uint32_t,
+      uint16_t,
+      bool,
+      std::vector<std::string>>
+      value;
 
    static ConfigValue fromString(const std::string& value) { return ConfigValue{value}; }
 
@@ -57,6 +72,8 @@ class ConfigValue {
 
    static ConfigValue fromBool(bool value) { return ConfigValue{value}; }
 
+   static ConfigValue fromList(const std::vector<std::string>& value) { return ConfigValue{value}; }
+
    [[nodiscard]] ConfigValueType getValueType() const;
 
    [[nodiscard]] std::string toString() const;

src/config/source/yaml_file.cpp

@@ -17,6 +17,13 @@ using silo::config::ConfigKeyPath;
 
 namespace {
 
+bool isSequenceOfScalars(const YAML::Node& node) {
+   if (!node.IsSequence()) {
+      return false;
+   }
+   return std::ranges::all_of(node, [](const auto& element) { return element.IsScalar(); });
+}
+
 bool isProperSingularValue(const YAML::Node& node) {
    if (node.IsMap()) {
       SPDLOG_TRACE("isProperSingularValue = false, node is a map");
@@ -26,8 +33,8 @@ bool isProperSingularValue(const YAML::Node& node) {
       SPDLOG_TRACE("isProperSingularValue = false, node is not defined");
       return false;
    }
-   if (!node.IsScalar()) {
-      SPDLOG_TRACE("isProperSingularValue = false, node is not a scalar");
+   if (!node.IsScalar() && !isSequenceOfScalars(node)) {
+      SPDLOG_TRACE("isProperSingularValue = false, node is not a scalar or sequence of strings");
       return false;
    }
    return true;
@@ -211,6 +218,8 @@ ConfigValue yamlNodeToConfigValue(
             return ConfigValue::fromUint16(yaml.as<uint16_t>());
          case ConfigValueType::BOOL:
             return ConfigValue::fromBool(yaml.as<bool>());
+         case ConfigValueType::LIST:
+            return ConfigValue::fromList(yaml.as<std::vector<std::string>>());
       }
       SILO_UNREACHABLE();
    } catch (YAML::BadConversion& error) {

src/config/source/yaml_file.test.cpp

@@ -57,7 +57,7 @@ TEST(YamlFile, containsCorrectFieldsFromFlatYAML) {
 inputDirectory: "./testBaseData/exampleDataset/"
 outputDirectory: "./output/"
 ndjsonInputFilename: "input_file.ndjson"
-lineageDefinitionsFilename: "lineage_definitions.yaml"
+lineageDefinitionFilenames: "lineage_definition.yaml"
 phyloTreeFilename: "phylogenetic_tree.yaml"
 referenceGenomeFilename: "reference_genomes.json"
 )"
@@ -69,8 +69,8 @@ referenceGenomeFilename: "reference_genomes.json"
        YAML::Node{"./testBaseData/exampleDataset/"}},
       {YamlFile::stringToConfigKeyPath("outputDirectory"), YAML::Node{"./output/"}},
       {YamlFile::stringToConfigKeyPath("ndjsonInputFilename"), YAML::Node{"input_file.ndjson"}},
-      {YamlFile::stringToConfigKeyPath("lineageDefinitionsFilename"),
-       YAML::Node{"lineage_definitions.yaml"}},
+      {YamlFile::stringToConfigKeyPath("lineageDefinitionFilenames"),
+       YAML::Node{"lineage_definition.yaml"}},
       {YamlFile::stringToConfigKeyPath("phyloTreeFilename"), YAML::Node{"phylogenetic_tree.yaml"}},
       {YamlFile::stringToConfigKeyPath("referenceGenomeFilename"),
        YAML::Node{"reference_genomes.json"}},

src/config/verified_config_attributes.cpp

@@ -88,6 +88,12 @@ std::optional<bool> VerifiedConfigAttributes::getBool(const ConfigKeyPath& confi
    return std::nullopt;
 }
 
+std::optional<std::vector<std::string>> VerifiedConfigAttributes::getList(
+   const ConfigKeyPath& config_key_path
+) const {
+   return getValue<std::vector<std::string>, ConfigValueType::LIST>(config_key_path, config_values);
+}
+
 VerifiedCommandLineArguments VerifiedCommandLineArguments::askingForHelp() {
    VerifiedCommandLineArguments result;
    result.asks_for_help = true;

src/config/verified_config_attributes.h

@@ -33,6 +33,10 @@ class VerifiedConfigAttributes {
    [[nodiscard]] std::optional<uint16_t> getUint16(const ConfigKeyPath& config_key_path) const;
 
    [[nodiscard]] std::optional<bool> getBool(const ConfigKeyPath& config_key_path) const;
+
+   [[nodiscard]] std::optional<std::vector<std::string>> getList(
+      const ConfigKeyPath& config_key_path
+   ) const;
 };
 
 class VerifiedCommandLineArguments : public VerifiedConfigAttributes {

src/main.cpp

@@ -47,7 +47,7 @@ int runPreprocessor(const silo::config::PreprocessingConfig& preprocessing_confi
       database.saveDatabaseState(preprocessing_config.output_directory);
       return 0;
    } catch (const silo::preprocessing::PreprocessingException& preprocessing_exception) {
-      SPDLOG_ERROR("initialize - error: {}", preprocessing_exception.what());
+      SPDLOG_ERROR("preprocessing - error: {}", preprocessing_exception.what());
       return 1;
    }
 }

src/silo/common/lineage_tree.test.cpp

@@ -254,7 +254,7 @@ TEST(LineageDefinitionFile, errorOnLineageAsAlias) {
 
 TEST(containsCycle, doesNotFindCycleInPangoLineageTree) {
    ASSERT_NO_THROW(LineageTreeAndIdMap::fromLineageDefinitionFilePath(
-      "testBaseData/exampleDataset/lineage_definitions.yaml"
+      "testBaseData/exampleDataset/lineage_definition.yaml"
    ));
 }
 

src/silo/config/database_config.cpp

@@ -138,9 +138,9 @@ bool YAML::convert<silo::config::DatabaseMetadata>::decode(
       metadata.generate_index = false;
    }
    if (node["generateLineageIndex"].IsDefined()) {
-      metadata.generate_lineage_index = node["generateLineageIndex"].as<bool>();
+      metadata.generate_lineage_index = node["generateLineageIndex"].as<std::string>();
    } else {
-      metadata.generate_lineage_index = false;
+      metadata.generate_lineage_index = std::nullopt;
    }
    if (node["isPhyloTreeField"].IsDefined()) {
       metadata.phylo_tree_node_identifier = node["isPhyloTreeField"].as<bool>();