Internal game data imports

Author: Su5eD

modules/Deps.cmake

@@ -100,4 +100,12 @@ CPMAddPackage(
 
 if (sentry_ADDED)
     set_target_properties(sentry PROPERTIES C_STANDARD 11 C_EXTENSIONS ON)
-endif()
+endif()
+
+CPMAddPackage(
+        NAME pugixml
+        GITHUB_REPOSITORY zeux/pugixml
+        GIT_TAG v1.15
+        OPTIONS
+        "BUILD_SHARED_LIBS OFF"
+)

src/CMakeLists.txt

@@ -51,6 +51,7 @@ target_link_libraries(${PROJECT_NAME} PRIVATE
         libzip::zip
         sentry::sentry
         yaml-cpp::yaml-cpp
+        pugixml::pugixml
 )
 
 # ##############################################################################

src/api/v1/system.cc

@@ -5,7 +5,6 @@
 #include <models/Project.h>
 #include <service/auth.h>
 #include <service/serializers.h>
-#include <service/system/data_import.h>
 #include <service/util.h>
 #include <version.h>
 
@@ -14,6 +13,7 @@
 #include <service/system/access_keys.h>
 #include <service/system/lang.h>
 #include "base.h"
+#include "system/game_data.h"
 
 using namespace std;
 using namespace drogon;
@@ -33,8 +33,7 @@ namespace api::v1 {
         NLOHMANN_DEFINE_TYPE_INTRUSIVE(DataMigration, id, title, desc)
     };
 
-    std::vector<DataMigration> dataMigrations = {
-        {"deployments", "Project deployments", "Run initial project deployments", sys::runInitialDeployments}};
+    std::vector<DataMigration> dataMigrations = {};
 
     Task<bool> authAccessKey(const HttpRequestPtr req) {
         const auto authHeader = req->getHeader("Authorization");
@@ -89,12 +88,11 @@ namespace api::v1 {
             co_await global::auth->ensurePrivilegedAccess(req);
         }
 
-        const auto json(BaseProjectController::jsonBody(req));
+        const auto body(BaseProjectController::jsonBody(req));
+        const auto updateLoader = body.contains("update_loader") ? body["update_loader"].get<bool>() : false;
+        const auto result = co_await global::gameData->importGameData(updateLoader);
 
-        constexpr sys::SystemDataImporter importer;
-        const auto result = co_await importer.importData(json);
-
-        callback(statusResponse(result == Error::Ok ? k200OK : k500InternalServerError));
+        callback(statusResponse(result ? k200OK : k500InternalServerError));
     }
 
     Task<> SystemController::getAvailableMigrations(const HttpRequestPtr req,

src/api/v1/system.h

@@ -10,7 +10,7 @@ namespace api::v1 {
         METHOD_LIST_BEGIN
         // Public
         ADD_METHOD_TO(SystemController::getLocales, "/api/v1/system/locales", drogon::Get, "AuthFilter");
-        ADD_METHOD_TO(SystemController::importData, "/api/v1/system/import", drogon::Post);
+        ADD_METHOD_TO(SystemController::importData, "/api/v1/system/import", drogon::Post, "AuthFilter");
         // Internal
         ADD_METHOD_TO(SystemController::getSystemInformation, "/api/v1/system/info", drogon::Get, "AuthFilter");
         ADD_METHOD_TO(SystemController::getDataImports, "/api/v1/system/imports", drogon::Get, "AuthFilter");

src/main.cc

@@ -16,20 +16,21 @@
 #include <log/log.h>
 
 #include <service/database/database.h>
-#include <service/external/github.h>
-#include <service/storage/realtime.h>
-#include <service/system/lang.h>
 #include <service/external/crowdin.h>
 #include <service/external/frontend.h>
+#include <service/external/github.h>
+#include <service/project/virtual/virtual.h>
+#include <service/storage/ingestor/recipe/recipe_builtin.h>
+#include <service/storage/realtime.h>
 #include <service/system/access_keys.h>
 #include <service/system/game_data.h>
-#include <service/storage/ingestor/recipe/recipe_builtin.h>
-#include <service/project/virtual/virtual.h>
+#include <service/system/lang.h>
 #include <service/system/startup.h>
 
 using namespace drogon;
 using namespace logging;
 using namespace service;
+namespace fs = std::filesystem;
 
 namespace service {
     trantor::EventLoopThreadPool cacheAwaiterThreadPool{10};
@@ -65,11 +66,11 @@ void globalExceptionHandler(const std::exception &e, const HttpRequestPtr &req,
     callback(statusResponse(k500InternalServerError));
 }
 
-Task<> runStartupTaks() {
-    global::virtualProject = co_await createVirtualProject();
+Task<> runStartupTaks(const fs::path gameFilesPath) {
+    global::virtualProject = co_await createVirtualProject(gameFilesPath);
 
     co_await cleanupLoadingDeployments();
-    co_await global::gameData->setupGameData();
+    co_await global::gameData->importGameData(false);
     co_await global::crowdin->getAvailableLocales();
 }
 
@@ -82,13 +83,16 @@ int main() {
         app().setLogLevel(level).addListener("0.0.0.0", port).setThreadNum(16);
         configureLoggingLevel();
 
-        const auto [authConfig, githubAppConfig, mrApp, crowdinConfig, sentryConfig, appUrl, curseForgeKey, storagePath,
-                    salt, local] = config::configure();
+        const auto [authConfig, githubAppConfig, mrApp, crowdinConfig, sentryConfig, appUrl, curseForgeKey, storagePath, salt, local] =
+            config::configure();
 
         if (!sentryConfig.dsn.empty()) {
             monitor::initSentry(sentryConfig.dsn);
         }
 
+        const auto gameFilesPath = fs::path(storagePath) / ".game";
+        fs::create_directories(gameFilesPath);
+
         global::database = std::make_shared<Database>();
         global::cache = std::make_shared<MemoryCache>();
         global::github = std::make_shared<GitHub>();
@@ -97,7 +101,7 @@ int main() {
         global::auth = std::make_shared<Auth>(appUrl, OAuthApp{githubAppConfig.clientId, githubAppConfig.clientSecret},
                                               OAuthApp{mrApp.clientId, mrApp.clientSecret});
         global::lang = std::make_shared<LangService>();
-        global::gameData = std::make_shared<GameDataService>(storagePath);
+        global::gameData = std::make_shared<GameDataService>();
         global::crowdin = std::make_shared<Crowdin>(crowdinConfig);
         global::accessKeys = std::make_shared<AccessKeys>(salt);
         global::frontend = std::make_shared<FrontendService>(authConfig.frontendUrl, authConfig.frontendApiKey);
@@ -131,7 +135,7 @@ int main() {
 
         content::loadBuiltinRecipeTypes();
 
-        app().getLoop()->queueInLoop(async_func([]() -> Task<> { co_await runStartupTaks(); }));
+        app().getLoop()->queueInLoop(async_func([=]() -> Task<> { co_await runStartupTaks(gameFilesPath); }));
 
         app().run();
 

src/schemas/definition/game-recipe.schema.json

@@ -377,7 +377,8 @@
           "description": "An item ID.",
           "type": "string"
         }
-      }
+      },
+      "required": ["id"]
     }
   },
   "description": "Configuration file defining a recipe for a data pack for Minecraft.",

src/service/CMakeLists.txt

@@ -24,6 +24,7 @@ add_library(service STATIC
         external/frontend.cc
         external/github.cc
 
+        project/virtual/format.cc
         project/virtual/virtual.cc
         project/cached/cached.cc
         project/content.cc
@@ -50,7 +51,6 @@ add_library(service STATIC
         storage/realtime.cc
 
         system/access_keys.cc
-        system/data_import.cc
         system/game_data.cc
         system/lang.cc
         system/startup.cc
@@ -73,6 +73,7 @@ target_link_libraries(service PRIVATE
         OpenSSL::Crypto
         libzip::zip
         yaml-cpp::yaml-cpp
+        pugixml::pugixml
 )
 
 target_include_directories(service PUBLIC

src/service/database/database.cc

@@ -303,11 +303,7 @@ namespace service {
         co_return res.value_or({});
     }
 
-    Task<TaskResult<DataImport>> Database::addDataImportRecord(const DataImport data) const {
-        co_return co_await handleDatabaseOperation([&data](const DbClientPtr &client) -> Task<DataImport> {
-            CoroMapper<DataImport> mapper(client);
-
-            co_return co_await mapper.insert(data);
-        });
+    Task<TaskResult<DataImport>> Database::getDataImport(std::string gameVersion) const {
+        co_return co_await findOne<DataImport>(Criteria(DataImport::Cols::_game_version, CompareOperator::EQ, gameVersion));
     }
 }

src/service/database/database.h

@@ -9,7 +9,6 @@
 #include <models/Deployment.h>
 #include <models/ProjectIssue.h>
 #include <models/ProjectVersion.h>
-#include <models/Recipe.h>
 #include <models/Report.h>
 #include <nlohmann/json.hpp>
 #include <service/error.h>
@@ -97,16 +96,11 @@ namespace service {
 
         // Content
         drogon::Task<TaskResult<>> refreshFlatTagItemView() const;
-        // TODO Remove after data imports are reworked
-        drogon::Task<int64_t> addRecipe(std::string id, std::string type) const;
-        drogon::Task<TaskResult<>> addRecipeIngredientItem(int64_t recipe_id, std::string item, int slot, int count, bool input) const;
-        drogon::Task<TaskResult<>> addRecipeIngredientTag(int64_t recipe_id, std::string tag, int slot, int count, bool input) const;
-
         drogon::Task<std::vector<std::string>> getItemSourceProjects(int64_t item) const;
         drogon::Task<std::vector<GlobalItem>> getGlobalTagItems(int64_t tagId) const;
 
         // System data
-        drogon::Task<TaskResult<DataImport>> addDataImportRecord(DataImport data) const;
+        drogon::Task<TaskResult<DataImport>> getDataImport(std::string gameVersion) const;
         drogon::Task<PaginatedData<DataImport>> getDataImports(std::string searchQuery, int page) const;
 
         // Deployments

src/service/database/database_base.h

@@ -1,6 +1,5 @@
 #pragma once
 
-#include <drogon/HttpAppFramework.h>
 #include <drogon/orm/CoroMapper.h>
 #include <drogon/utils/coroutine.h>
 #include <log/log.h>