fix: do not write out column metadata as yaml, which might mess up utf-8

Author: taepper

endToEndTests/test/lineageDefinition.test.js

@@ -7,10 +7,10 @@ describe('The /lineageDefinition endpoint', () => {
     let searchString = `A: \{\}
 A.1:
   parents:
-    - A
+  - A
 A.11:
   parents:
-    - A`;
+  - A`;
     await server
       .get('/lineageDefinition/pango_lineage')
       .expect(200)

src/silo/api/request_handler_factory.test.cpp

@@ -15,15 +15,15 @@ namespace {
 
 std::unique_ptr<SiloRequestHandlerFactory> createRequestHandlerWithInitializedDatabase() {
    auto handle = std::make_shared<silo::api::ActiveDatabase>();
-   handle->setActiveDatabase(silo::Database{silo::schema::DatabaseSchema::fromYAML(YAML::Load(R"(
-default:
-  primaryKey: primary_key
-  columns:
-    - name: primary_key
-      type: string
-      metadata:
-         dictionary: []
-)"))});
+   silo::schema::TableSchema table_schema;
+   table_schema.primary_key = {"primary_key", silo::schema::ColumnType::STRING};
+   table_schema.column_metadata.emplace(
+      silo::schema::ColumnIdentifier{"primary_key", silo::schema::ColumnType::STRING},
+      std::make_shared<silo::storage::column::StringColumnMetadata>("primary_key")
+   );
+   silo::schema::DatabaseSchema schema;
+   schema.tables.emplace(silo::schema::TableName::getDefault(), table_schema);
+   handle->setActiveDatabase(silo::Database(schema));
    auto request_handler = std::make_unique<SiloRequestHandlerFactory>(
       silo::config::RuntimeConfig::withDefaults(), handle
    );

src/silo/common/bidirectional_map.h

@@ -63,20 +63,10 @@ class BidirectionalMap {
       return identifier;
    }
 
-   static BidirectionalMap fromYAML(const YAML::Node& yaml_node) {
-      BidirectionalMap result;
-      for (const auto& value : yaml_node) {
-         (void)result.getOrCreateId(value.as<V>());
-      }
-      return result;
-   }
-
-   YAML::Node toYAML() const {
-      YAML::Node yaml_node{YAML::NodeType::Sequence};
-      for (const auto& value : id_to_value) {
-         yaml_node.push_back(YAML::Node{value});
-      }
-      return yaml_node;
+   template <class Archive>
+   void serialize(Archive& archive, const uint32_t /* version */) {
+      archive & id_to_value;
+      archive & value_to_id;
    }
 };
 

src/silo/common/data_version.h

@@ -15,7 +15,7 @@ class DataVersion {
       uint32_t value;
    };
 
-   static constexpr SerializationVersion CURRENT_SILO_SERIALIZATION_VERSION{8};
+   static constexpr SerializationVersion CURRENT_SILO_SERIALIZATION_VERSION{9};
 
    class Timestamp {
      public:

src/silo/common/lineage_tree.h

@@ -70,6 +70,16 @@ class LineageTreeAndIdMap {
       BidirectionalMap<std::string>&& lineage_id_lookup_map,
       std::string&& file
    );
+
+   friend class boost::serialization::access;
+   template <class Archive>
+   [[maybe_unused]] void serialize(Archive& archive, const uint32_t /* version */) {
+      // clang-format off
+      archive & lineage_tree;
+      archive & lineage_id_lookup_map;
+      archive & file;
+      // clang-format on
+   }
 };
 
 std::optional<std::vector<Idx>> containsCycle(

src/silo/database.cpp

@@ -22,15 +22,15 @@
 #include "silo/common/data_version.h"
 #include "silo/common/file_to_string.h"
 #include "silo/common/format_number.h"
+#include "silo/common/lineage_tree.h"
 #include "silo/common/nucleotide_symbols.h"
 #include "silo/common/panic.h"
 #include "silo/common/silo_directory.h"
 #include "silo/common/version.h"
 #include "silo/database_info.h"
 #include "silo/persistence/exception.h"
 #include "silo/roaring/roaring_serialize.h"
-#include "silo/storage/column/sequence_column.h"
-#include "silo/storage/serialize_optional.h"
+#include "silo/schema/database_schema.h"
 #include "silo/storage/table_partition.h"
 
 namespace silo {
@@ -69,6 +69,9 @@ DatabaseInfo Database::getDatabaseInfo() const {
    };
 }
 
+const std::string DATABASE_SCHEMA_FILENAME = "database_schema.silo";
+const std::string DATA_VERSION_FILENAME = "data_version.silo";
+
 void Database::saveDatabaseState(const std::filesystem::path& save_directory) {
    if (getDataVersionTimestamp().value.empty()) {
       throw persistence::SaveDatabaseException(
@@ -105,20 +108,15 @@ void Database::saveDatabaseState(const std::filesystem::path& save_directory) {
 
    SPDLOG_INFO("Saving database schema");
 
-   auto yaml_string =
-      YAML::Dump(schema::DatabaseSchema{{{schema::TableName::getDefault(), table->schema}}}.toYAML()
-      );
-   const auto database_schema_filename = versioned_save_directory / "database_schema.yaml";
-   std::ofstream database_schema_file{database_schema_filename};
-   database_schema_file << yaml_string;
-   database_schema_file.close();
+   const auto database_schema_path = versioned_save_directory / DATABASE_SCHEMA_FILENAME;
+   schema.saveToFile(database_schema_path);
 
    std::string table_name = schema.tables.begin()->first.getName();
    SPDLOG_DEBUG("Saving table data");
    std::filesystem::create_directory(versioned_save_directory / table_name);
    table->saveData(versioned_save_directory / table_name);
 
-   data_version_.saveToFile(versioned_save_directory / "data_version.silo");
+   data_version_.saveToFile(versioned_save_directory / DATA_VERSION_FILENAME);
 }
 
 namespace {
@@ -143,26 +141,19 @@ Database Database::loadDatabaseState(const silo::SiloDataSource& silo_data_sourc
    SPDLOG_INFO("Loading database from data source: {}", silo_data_source.toDebugString());
    const auto save_directory = silo_data_source.path;
 
-   const auto database_schema_filename = save_directory / "database_schema.yaml";
-   auto yaml_string = common::fileToString(database_schema_filename);
-   if (yaml_string == std::nullopt) {
-      throw silo::persistence::LoadDatabaseException(
-         fmt::format("Could not load DatabaseSchema from {}", database_schema_filename)
-      );
-   }
-   YAML::Node yaml_schema = YAML::Load(yaml_string.value());
-   schema::DatabaseSchema schema = schema::DatabaseSchema::fromYAML(yaml_schema);
+   const auto database_schema_path = save_directory / DATABASE_SCHEMA_FILENAME;
+   auto schema = schema::DatabaseSchema::loadFromFile(database_schema_path);
 
    Database database{schema};
 
    std::string table_name = schema.tables.begin()->first.getName();
    SPDLOG_DEBUG("Loading data for table ");
    database.table->loadData(save_directory / table_name);
 
-   database.data_version_ = loadDataVersion(save_directory / "data_version.silo");
+   database.data_version_ = loadDataVersion(save_directory / DATA_VERSION_FILENAME);
 
    SPDLOG_INFO(
-      "Finished loading data_version from {}", (save_directory / "data_version.silo").string()
+      "Finished loading data_version from {}", (save_directory / DATA_VERSION_FILENAME).string()
    );
    SPDLOG_INFO("Database info after loading: {}", database.getDatabaseInfo());