DBA migrations and postgres

.devcontainer/docker-compose.yml → .devcontainer/docker-compose.mysql.yml

@@ -9,6 +9,7 @@ services:
         - CONTAINER_USER_CMD_PRE
         - CONTAINER_USER_CMD_POST
     environment:
+      HASHTOPOLIS_DB_TYPE: mysql
       HASHTOPOLIS_DB_USER: hashtopolis
       HASHTOPOLIS_DB_PASS: hashtopolis
       HASHTOPOLIS_DB_HOST: hashtopolis-db-dev

.devcontainer/docker-compose.postgres.yml

@@ -0,0 +1,51 @@
+version: "3.7"
+services:
+  hashtopolis-server-dev:
+    container_name: hashtopolis-server-dev
+    build:
+      context: ..
+      target: hashtopolis-server-dev
+      args:
+        - CONTAINER_USER_CMD_PRE
+        - CONTAINER_USER_CMD_POST
+    environment:
+      HASHTOPOLIS_DB_TYPE: postgres
+      HASHTOPOLIS_DB_USER: hashtopolis
+      HASHTOPOLIS_DB_PASS: hashtopolis
+      HASHTOPOLIS_DB_HOST: hashtopolis-db-dev
+      HASHTOPOLIS_DB_DATABASE: hashtopolis
+      HASHTOPOLIS_APIV2_ENABLE: 1
+    depends_on:
+      - hashtopolis-db-dev
+    ports:
+      - "8080:80"
+    volumes:
+      # This is where VS Code should expect to find your project's source code
+      # and the value of "workspaceFolder" in .devcontainer/devcontainer.json
+      - ..:/var/www/html
+      - hashtopolis-server-dev:/usr/local/share/hashtopolis:Z
+    networks:
+      - hashtopolis_dev
+  hashtopolis-db-dev:
+    container_name: hashtopolis-db-dev
+    image: postgres:13
+    restart: always
+    ports:
+      - "5432:5432"
+    volumes:
+      - hashtopolis-db-dev:/var/lib/postgresql/data
+    environment:
+      POSTGRES_DB: hashtopolis
+      POSTGRES_USER: hashtopolis
+      POSTGRES_PASSWORD: hashtopolis
+    networks:
+      - hashtopolis_dev
+
+volumes:
+  hashtopolis-db-dev:
+  hashtopolis-server-dev:
+
+networks:
+  hashtopolis_dev:
+    # This network will also be used by the python-agent
+    name: hashtopolis_dev

.github/actions/start-hashtopolis/action.yml

@@ -1,16 +1,23 @@
 name: Start Hashtopolis server
 description: Starts application containers and waits for Hashtopolis to be ready.
 
+inputs:
+  db_system:
+    description: "Used to set which DB system should be used"
+    required: true
+    default: "mysql"
+
 runs:
   using: "composite"
   steps:
     - name: Start application containers
-      working-directory: .devcontainer
-      run: docker compose up -d
-      shell: bash
-    - name: Install composer dependencies packages
-      run: docker exec hashtopolis-server-dev composer install --working-dir=/var/www/html/
+      working-directory: .github
+      run: docker compose -f docker-compose.${{ inputs.db_system }}.yml up -d
       shell: bash
+# should not be needed anymore as it is installed during build
+#    - name: Install composer dependencies packages
+#      run: docker exec hashtopolis-server-dev composer install --working-dir=/var/www/html/
+#      shell: bash
     - name: Wait until entrypoint is finished and Hashtopolis is started
       run: bash .github/scripts/await-hashtopolis-startup.sh
       shell: bash

.github/docker-compose.mysql.yml

@@ -0,0 +1,47 @@
+version: "3.7"
+services:
+  hashtopolis-server-dev:
+    container_name: hashtopolis-server-dev
+    build:
+      context: ..
+      target: hashtopolis-server-dev
+      args:
+        - CONTAINER_USER_CMD_PRE
+        - CONTAINER_USER_CMD_POST
+    environment:
+      HASHTOPOLIS_DB_TYPE: mysql
+      HASHTOPOLIS_DB_USER: hashtopolis
+      HASHTOPOLIS_DB_PASS: hashtopolis
+      HASHTOPOLIS_DB_HOST: hashtopolis-db-dev
+      HASHTOPOLIS_DB_DATABASE: hashtopolis
+      HASHTOPOLIS_APIV2_ENABLE: 1
+    depends_on:
+      - hashtopolis-db-dev
+    ports:
+      - "8080:80"
+    volumes:
+      - hashtopolis-server-dev:/usr/local/share/hashtopolis:Z
+    networks:
+      - hashtopolis_dev
+  hashtopolis-db-dev:
+    container_name: hashtopolis-db-dev
+    image: mysql:8.0
+    restart: always
+    ports:
+      - "3306:3306"
+    volumes:
+      - hashtopolis-db-dev:/var/lib/mysql
+    environment:
+      MYSQL_ROOT_PASSWORD: hashtopolis
+      MYSQL_DATABASE: hashtopolis
+      MYSQL_USER: hashtopolis
+      MYSQL_PASSWORD: hashtopolis
+    networks:
+      - hashtopolis_dev
+volumes:
+  hashtopolis-db-dev:
+  hashtopolis-server-dev:
+
+networks:
+  hashtopolis_dev:
+    name: hashtopolis_dev

.github/docker-compose.postgres.yml

@@ -0,0 +1,47 @@
+version: "3.7"
+services:
+  hashtopolis-server-dev:
+    container_name: hashtopolis-server-dev
+    build:
+      context: ..
+      target: hashtopolis-server-dev
+      args:
+        - CONTAINER_USER_CMD_PRE
+        - CONTAINER_USER_CMD_POST
+    environment:
+      HASHTOPOLIS_DB_TYPE: postgres
+      HASHTOPOLIS_DB_USER: hashtopolis
+      HASHTOPOLIS_DB_PASS: hashtopolis
+      HASHTOPOLIS_DB_HOST: hashtopolis-db-dev
+      HASHTOPOLIS_DB_DATABASE: hashtopolis
+      HASHTOPOLIS_APIV2_ENABLE: 1
+    depends_on:
+      - hashtopolis-db-dev
+    ports:
+      - "8080:80"
+    volumes:
+      - hashtopolis-server-dev:/usr/local/share/hashtopolis:Z
+    networks:
+      - hashtopolis_dev
+  hashtopolis-db-dev:
+    container_name: hashtopolis-db-dev
+    image: postgres:13
+    restart: always
+    ports:
+      - "5432:5432"
+    volumes:
+      - hashtopolis-db-dev:/var/lib/postgresql/data
+    environment:
+      POSTGRES_DB: hashtopolis
+      POSTGRES_USER: hashtopolis
+      POSTGRES_PASSWORD: hashtopolis
+    networks:
+      - hashtopolis_dev
+
+volumes:
+  hashtopolis-db-dev:
+  hashtopolis-server-dev:
+
+networks:
+  hashtopolis_dev:
+    name: hashtopolis_dev

.github/workflows/ci.yml

@@ -13,15 +13,23 @@ on:
 jobs:
   build:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - db_system: mysql
+          - db_system: postgres
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
       - name: Start Hashtopolis server
         uses: ./.github/actions/start-hashtopolis
+        with:
+          db_system: ${{ matrix.db_system }}
       - name: Give Apache permissions on necessary directories # for the tests, only src/files and src/inc/utils/locks seem necessary
         run: docker exec -u root hashtopolis-server-dev bash -c "chown -R www-data:www-data /var/www/html/src && chmod -R g+w /var/www/html/src"
-      - name: Run test suite
-        run: docker exec hashtopolis-server-dev php /var/www/html/ci/run.php -vmaster
+#      - name: Run test suite
+#        if: ${{ matrix.db_system == 'mysql' }} # the legacy is only supposed to work with mysql
+#        run: docker exec hashtopolis-server-dev php /var/www/html/ci/run.php -vmaster
       - name: Test with pytest
         run: docker exec hashtopolis-server-dev pytest /var/www/html/ci/apiv2
       - name: Test if pytest is removing all test objects

.github/workflows/docs-build.yml

@@ -29,6 +29,8 @@ jobs:
         sudo apt-get install npm
     - name: Start Hashtopolis server
       uses: ./.github/actions/start-hashtopolis
+      with:
+        db_system: "mysql"
     - name: Download newest apiv2 spec
       run: |
         wget http://localhost:8080/api/v2/openapi.json -P /tmp/

.github/workflows/docs.yml

@@ -26,9 +26,11 @@ jobs:
         sudo apt-get install php
         sudo apt-get install -y lftp
         sudo apt-get install nodejs
-        sudo apt-get install npm 
+        sudo apt-get install npm
     - name: Start Hashtopolis server
       uses: ./.github/actions/start-hashtopolis
+      with:
+        db_system: "mysql"
     - name: Download newest apiv2 spec
       run: |
         wget http://localhost:8080/api/v2/openapi.json -P /tmp/

Dockerfile

@@ -1,3 +1,7 @@
+FROM rust:1.91-trixie AS prebuild
+
+RUN cargo install sqlx-cli --no-default-features --features native-tls,mysql,postgres
+
 FROM alpine/git AS preprocess
 
 COPY .gi[t] /.git
@@ -37,12 +41,12 @@ RUN apt-get update \
     #
     # Install git, procps, lsb-release (useful for CLI installs)
     && apt-get -y install git iproute2 procps lsb-release \
-    && apt-get -y install mariadb-client \
+    && apt-get -y install mariadb-client postgresql-client libpq-dev \
     && apt-get -y install libpng-dev \
     && apt-get -y install ssmtp \
     \
     # Install extensions (optional)
-    && docker-php-ext-install pdo_mysql gd \
+    && docker-php-ext-install pdo_mysql pgsql pdo_pgsql gd \
     \
     # Install Composer
     && curl -sS https://getcomposer.org/installer | php \
@@ -82,6 +86,8 @@ RUN mkdir -p ${HASHTOPOLIS_DOCUMENT_ROOT} \
     && chown www-data:www-data ${HASHTOPOLIS_BINARIES_PATH} \
     && chmod g+w ${HASHTOPOLIS_BINARIES_PATH}
 
+COPY --from=prebuild /usr/local/cargo/bin/sqlx /usr/bin/
+
 COPY --from=preprocess /HEA[D] ${HASHTOPOLIS_DOCUMENT_ROOT}/../.git/
 
 # Install composer

ci/HashtopolisTest.class.php

@@ -19,7 +19,7 @@ abstract class HashtopolisTest {
   protected $user;
   protected $apiKey;
 
-  const USER_PASS = "HG78Ghdfs87gh";
+  const USER_PASS = "hashtopolis";
 
   const RUN_FULL = 0;
   const RUN_FAST = 1;
@@ -70,28 +70,27 @@ public function init($version) {
     global $PEPPER, $VERSION;
 
     // drop old data and create empty DB
-    Factory::getAgentFactory()->getDB()->query("DROP DATABASE IF EXISTS hashtopolis");
+    /*Factory::getAgentFactory()->getDB()->query("DROP DATABASE IF EXISTS hashtopolis");
     Factory::getAgentFactory()->getDB()->query("CREATE DATABASE hashtopolis");
     Factory::getAgentFactory()->getDB()->query("USE hashtopolis");
 
     // load DB
     if ($version == "master") {
-      Factory::getAgentFactory()->getDB()->query(file_get_contents(dirname(__FILE__) ."/../src/install/hashtopolis.sql"));
+      Factory::getAgentFactory()->getDB()->query(file_get_contents(dirname(__FILE__) ."/../src/migrations/mysql/20251127000000_initial.sql"));
     }
     else {
       Factory::getAgentFactory()->getDB()->query(file_get_contents(dirname(__FILE__) . "/files/db_" . $version . ".sql"));
     }
 
-    sleep(1);
+    sleep(1);*/
 
     // insert user and api key
-    $salt = Util::randomString(30);
+    /*$salt = Util::randomString(30);
     $hash = Encryption::passwordHash(HashtopolisTest::USER_PASS, $salt);
     $this->user = new User(null, 'testuser', '', $hash, $salt, 1, 0, 0, 0, 3600, AccessUtils::getOrCreateDefaultAccessGroup()->getId(), 0, '', '', '', '');
-    $this->user = Factory::getUserFactory()->save($this->user);
-    $accessGroup = new AccessGroupUser(null, 1, $this->user->getId());
-    Factory::getAccessGroupUserFactory()->save($accessGroup);
-    $this->apiKey = new ApiKey(null, 0, time() + 3600, 'mykey', 0, $this->user->getId(), 1);
+    $this->user = Factory::getUserFactory()->save($this->user);*/
+    AccessUtils::getOrCreateDefaultAccessGroup();
+    $this->apiKey = new ApiKey(null, 0, time() + 3600, 'mykey', 0, 1, 1);
     $this->apiKey = Factory::getApiKeyFactory()->save($this->apiKey);
     // $versionStore = new StoredValue("version", ($version == 'master') ? explode("+", $VERSION)[0] : $version);
     // Factory::getStoredValueFactory()->save($versionStore);

ci/server/setup.php

@@ -30,7 +30,7 @@
 try {
   $db->query("CREATE DATABASE IF NOT EXISTS hashtopolis;");
   $db->query("USE hashtopolis;");
-  $db->query(file_get_contents($envPath . "src/install/hashtopolis.sql"));
+  $db->query(file_get_contents($envPath . "src/migrations/mysql/20251127000000_initial.sql"));
 }
 catch (PDOException $e) {
   fwrite(STDERR, "Failed to initialize database: " . $e->getMessage());

docker-compose.yml → docker-compose.mysql.yml

@@ -8,13 +8,14 @@ services:
       - hashtopolis:/usr/local/share/hashtopolis:Z
       # - ./ssmtp.conf:/etc/ssmtp/ssmtp.conf
     environment:
+      HASHTOPOLIS_DB_TYPE: mysql
       HASHTOPOLIS_DB_USER: $MYSQL_USER
       HASHTOPOLIS_DB_PASS: $MYSQL_PASSWORD
       HASHTOPOLIS_DB_HOST: $HASHTOPOLIS_DB_HOST
       HASHTOPOLIS_DB_DATABASE: $MYSQL_DATABASE
       HASHTOPOLIS_ADMIN_USER: $HASHTOPOLIS_ADMIN_USER
       HASHTOPOLIS_ADMIN_PASSWORD: $HASHTOPOLIS_ADMIN_PASSWORD
-      HASHTOPOLIS_APIV2_ENABLE: $HASHTOPOLIS_APIV2_ENABLE 
+      HASHTOPOLIS_APIV2_ENABLE: $HASHTOPOLIS_APIV2_ENABLE
       HASHTOPOLIS_FRONTEND_URLS: $HASHTOPOLIS_FRONTEND_URLS
     depends_on:
       - db
@@ -37,7 +38,7 @@ services:
     environment:
       HASHTOPOLIS_BACKEND_URL: $HASHTOPOLIS_BACKEND_URL
     restart: always
-    depends_on: 
+    depends_on:
       - hashtopolis-backend
     ports:
       - 4200:80

docker-compose.postgres.yml

@@ -0,0 +1,45 @@
+version: '3.7'
+services:
+  hashtopolis-backend:
+    container_name: hashtopolis-backend
+    image: hashtopolis/backend:latest
+    restart: always
+    volumes:
+      - hashtopolis:/usr/local/share/hashtopolis:Z
+      # - ./ssmtp.conf:/etc/ssmtp/ssmtp.conf
+    environment:
+      HASHTOPOLIS_DB_TYPE: postgres
+      HASHTOPOLIS_DB_USER: $POSTGRES_USER
+      HASHTOPOLIS_DB_PASS: $POSTGRES_PASSWORD
+      HASHTOPOLIS_DB_HOST: $HASHTOPOLIS_DB_HOST
+      HASHTOPOLIS_DB_DATABASE: $POSTGRES_DATABASE
+      HASHTOPOLIS_ADMIN_USER: $HASHTOPOLIS_ADMIN_USER
+      HASHTOPOLIS_ADMIN_PASSWORD: $HASHTOPOLIS_ADMIN_PASSWORD
+      HASHTOPOLIS_APIV2_ENABLE: $HASHTOPOLIS_APIV2_ENABLE
+    depends_on:
+      - db
+    ports:
+      - 8080:80
+  db:
+    container_name: db
+    image: postgres:13
+    restart: always
+    volumes:
+      - db:/var/lib/postgresql/data
+    environment:
+      POSTGRES_DB: $POSTGRES_DATABASE
+      POSTGRES_USER: $POSTGRES_USER
+      POSTGRES_PASSWORD: $POSTGRES_PASSWORD
+  hashtopolis-frontend:
+    container_name: hashtopolis-frontend
+    image: hashtopolis/frontend:latest
+    environment:
+      HASHTOPOLIS_BACKEND_URL: $HASHTOPOLIS_BACKEND_URL
+    restart: always
+    depends_on:
+      - hashtopolis-backend
+    ports:
+      - 4200:80
+volumes:
+  db:
+  hashtopolis: