Migrate data from mariadb to postgres

klass-api/src/main/resources/application-postgres-local.properties

@@ -0,0 +1,20 @@
+# custom properties used when "postgres-local" Profile is used
+
+spring.datasource.url=jdbc:postgresql://${POSTGRES_INSTANCE}:5432/klass
+spring.datasource.username=klass
+spring.datasource.password=${POSTGRES_PASSWORD}
+spring.datasource.driverclassName=org.postgresql.Driver
+
+spring.jpa.hibernate.ddl-auto=none
+spring.flyway.enabled=false
+
+#spring.flyway.locations=classpath:db/migration/{vendor}
+
+spring.datasource.hikari.connection-test-query=SELECT 1
+spring.datasource.hikari.validation-timeout=3000
+
+spring.jpa.show-sql=true
+spring.jpa.properties.hibernate.format_sql=true
+logging.level.org.hibernate.SQL=DEBUG
+logging.level.org.hibernate.type.descriptor.sql=TRACE
+spring.jpa.properties.hibernate.dialect=org.hibernate.dialect.PostgreSQLDialect

klass-api/src/main/resources/application.properties

@@ -17,6 +17,7 @@ info.build.version=@project.version@
 #   h2                  = use h2 database (stored on your filesystem)   
 #   h2-inmemory         = use h2 database but keep everything in memory only.(tests must use h2-inmemory)
 #   embedded-solr       = run an instance of solr as part of the application (no need to start the solr application)
+#   postgres-local      = use postgresql database (local container)
 #----------------------
 
 #-----------------------
@@ -26,6 +27,9 @@ info.build.version=@project.version@
 # Dev. self contained
 spring.profiles.active=api, h2-inmemory , small-import, skip-indexing, embedded-solr
 
+# Dev. data migrated from sql file
+#spring.profiles.active=api, postgres-local, skip-indexing, embedded-solr
+
 # Dev. with external dependencies (db and solr)
 #spring.profiles.active=api,mariadb,remote-solr,skip-indexing
 
@@ -83,3 +87,4 @@ spring.mvc.contentnegotiation.media-types.csv=text/csv
 # Required when running behind a load balancer or reverse proxy server
 server.forward-headers-strategy=NATIVE
 
+

klass-shared/alter_timestamp.sql

@@ -0,0 +1,44 @@
+ALTER TABLE changelog
+    ALTER COLUMN last_modified TYPE timestamp(0) USING last_modified,
+    ALTER COLUMN change_occured TYPE timestamp(0) USING change_occured;
+
+ALTER TABLE classification_access_counter
+    ALTER COLUMN time_stamp TYPE timestamp(0) USING time_stamp;
+
+ALTER TABLE classification_family
+    ALTER COLUMN last_modified TYPE timestamp(0) USING last_modified;
+
+ALTER TABLE classification_series
+    ALTER COLUMN last_modified TYPE timestamp(0) USING last_modified;
+
+ALTER TABLE correspondence_map
+    ALTER COLUMN last_modified TYPE timestamp(0) USING last_modified;
+
+ALTER TABLE correspondence_table
+    ALTER COLUMN last_modified TYPE timestamp(0) USING last_modified;
+
+ALTER TABLE level
+    ALTER COLUMN last_modified TYPE timestamp(0) USING last_modified;
+
+ALTER TABLE search_words
+    ALTER COLUMN time_stamp TYPE timestamp(0) USING time_stamp;
+
+ALTER TABLE statistical_classification
+    ALTER COLUMN last_modified TYPE timestamp(0) USING last_modified;
+
+ALTER TABLE statistical_unit
+    ALTER COLUMN last_modified TYPE timestamp(0) USING last_modified;
+
+ALTER TABLE subscriber
+    ALTER COLUMN last_modified TYPE timestamp(0) USING last_modified;
+
+ALTER TABLE subscription
+    ALTER COLUMN last_modified TYPE timestamp(0) USING last_modified,
+    ALTER COLUMN expiry_date TYPE timestamp(0) USING expiry_date;
+
+ALTER TABLE classification_item
+    ALTER COLUMN last_modified TYPE timestamp(0) USING last_modified;
+
+
+ALTER TABLE "user"
+    ALTER COLUMN last_modified TYPE timestamp(0) USING last_modified;

klass-shared/docker-compose.yaml

@@ -0,0 +1,31 @@
+services:
+  mariadb:
+    image: mariadb:11.7
+    container_name: klass_mariadb
+    ports:
+      - "3306:3306"
+    environment:
+      MARIADB_ROOT_PASSWORD: ${MARIADB_ROOT_PASSWORD}
+      MARIADB_USER: "klass"
+      MARIADB_PASSWORD: ${MARIADB_LOCAL_PASSWORD}
+      MARIADB_LOCAL_FILEPATH: ${MARIADB_LOCAL_FILEPATH}
+      MARIADB_DATABASE: "klass"
+  postgresql:
+    image: postgres:17.4
+    container_name: klass_postgres
+    ports:
+      - "5432:5432"
+    environment:
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+      POSTGRES_USER: "klass"
+  klass-api:
+    build:
+      context: ../klass-api
+      dockerfile: ../klass-api/Dockerfile
+    container_name: klass-api
+    ports:
+      - "8080:8080"
+    environment:
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+      POSTGRES_USER: "klass"
+      POSTGRES_INSTANCE: ${POSTGRES_INSTANCE}

klass-shared/init-klass-api.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+# This script depends on 'init-migration.sh'.
+# Services can be stopped, but not removed
+
+# Load variables from .env
+set -a
+source .env
+set +a
+
+# Check if postgresql container is running
+DB_CONTAINER_NAME="postgresql"
+
+DB_STATUS=$(docker compose ps --services --filter "status=running" | grep -w "$DB_CONTAINER_NAME")
+
+if [ -z "$DB_STATUS" ]; then
+  echo "Database container '$DB_CONTAINER_NAME' is not running. Restarting it..."
+  docker compose restart "$DB_CONTAINER_NAME"
+else
+  echo "Database container '$DB_CONTAINER_NAME' is running."
+fi
+
+echo "Build klass-api image"
+docker compose build
+
+echo "Restart klass api"
+docker compose restart klass-api

klass-shared/init-migration.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+# Load variables from .env
+set -a
+source .env
+set +a
+
+# Start containers
+docker compose up -d
+
+echo "Wait for mariadb to be ready"
+sleep 10
+
+echo "Load data from file to mariadb database"
+docker compose exec -T mariadb mariadb -u root -p"$MARIADB_ROOT_PASSWORD" --host=127.0.0.1 klass < "$MARIADB_LOCAL_FILEPATH"
+
+echo "Wait for load from file to finish"
+sleep 10
+
+echo "Migrate data from mariadb database to postgres database"
+migrate_data/pgloader/load.sh
+
+echo "Wait for migration to finish"
+sleep 10
+
+# Run necessary postgresql alteration on type timestamp to remove microseconds
+echo "Alter columns of type timestamp"
+docker cp alter_timestamp.sql "$(docker compose ps -q postgresql)":/tmp/alter_timestamp.sql
+docker compose exec -T postgresql psql -U klass -d klass -f /tmp/alter_timestamp.sql
+
+echo "Dump data from postgres database to file"
+pg_dump -U klass -h localhost -p 5432 klass > "$POSTGRES_LOCAL_FILEPATH"/"$POSTGRES_DUMP_FILENAME"

klass-shared/migrate_data/pgloader/load.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# Resolve directory
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+
+# Loop over all template load files
+for template in "$SCRIPT_DIR"/load*.load; do
+  echo "Processing $template..."
+
+  # Expand environment variables and run pgloader
+  envsubst < "$template" | pgloader /dev/stdin
+
+done

klass-shared/migrate_data/pgloader/load_classifications.load

@@ -0,0 +1,11 @@
+LOAD DATABASE
+      FROM mysql://root:${MARIADB_ROOT_PASSWORD}@localhost:3306/klass
+      INTO postgresql://klass:${POSTGRES_PASSWORD}@localhost:5432/klass
+
+WITH prefetch rows = 3000
+
+CAST type datetime to timestamp
+
+INCLUDING ONLY TABLE NAMES MATCHING 'classification_family',
+                                    'classification_series',
+                                    'classification_item';

klass-shared/migrate_data/pgloader/load_correspondance.load

@@ -0,0 +1,9 @@
+LOAD DATABASE
+    FROM mysql://root:${MARIADB_ROOT_PASSWORD}@localhost:3306/klass
+    INTO postgresql://klass:${POSTGRES_PASSWORD}@localhost:5432/klass
+
+WITH prefetch rows = 1000
+
+CAST type datetime to timestamp
+
+INCLUDING ONLY TABLE NAMES MATCHING 'correspondence_map', 'correspondence_table', 'correspondencetable_changelog';

klass-shared/migrate_data/pgloader/load_statistical.load

@@ -0,0 +1,13 @@
+LOAD DATABASE
+     FROM mysql://root:${MARIADB_ROOT_PASSWORD}@localhost:3306/klass
+     INTO postgresql://klass:${POSTGRES_PASSWORD}@localhost:5432/klass
+
+WITH prefetch rows = 2000
+
+CAST type datetime to timestamp
+
+INCLUDING ONLY TABLE NAMES MATCHING 'classification_series_statistical_units',
+                                    'statistical_classification',
+                                    'statistical_unit',
+                                    'classification_series_statistical_units',
+                                    'statisticalclassification_changelog';

klass-shared/migrate_data/pgloader/load_subscription.load

@@ -0,0 +1,9 @@
+LOAD DATABASE
+     FROM mysql://root:${MARIADB_ROOT_PASSWORD}@localhost:3306/klass
+     INTO postgresql://klass:${POSTGRES_PASSWORD}@localhost:5432/klass
+
+WITH prefetch rows = 1000
+
+CAST type datetime to timestamp
+
+INCLUDING ONLY TABLE NAMES MATCHING 'subscriber', 'subscription';

klass-shared/migrate_data/pgloader/load_support.load

@@ -0,0 +1,12 @@
+LOAD DATABASE
+     FROM mysql://root:${MARIADB_ROOT_PASSWORD}@localhost:3306/klass
+     INTO postgresql://klass:${POSTGRES_PASSWORD}@localhost:5432/klass
+
+WITH prefetch rows = 2000
+
+CAST type datetime to timestamp
+
+INCLUDING ONLY TABLE NAMES MATCHING 'changelog',
+                                    'classification_access_counter',
+                                    'search_words',
+                                    'level';

klass-shared/migrate_data/pgloader/load_user.load

@@ -0,0 +1,9 @@
+LOAD DATABASE
+     FROM mysql://root:${MARIADB_ROOT_PASSWORD}@localhost:3306/klass
+     INTO postgresql://klass:${POSTGRES_PASSWORD}@localhost:5432/klass
+
+WITH prefetch rows = 1000
+
+CAST type datetime to timestamp
+
+INCLUDING ONLY TABLE NAMES MATCHING 'user', 'user_favorites';

klass-shared/src/main/java/no/ssb/klass/core/model/ClassificationSeries.java

@@ -60,7 +60,6 @@ public class ClassificationSeries extends BaseEntity implements ClassificationEn
     private String nameNn;
     @Column(name = "name_en")
     private String nameEn;
-    @Lob
     @Column(columnDefinition = "text", nullable = false)
     @Convert(converter = TranslatablePersistenceConverter.class)
     private Translatable description;

klass-shared/src/main/java/no/ssb/klass/core/model/CorrespondenceTable.java

@@ -32,7 +32,6 @@
 @Table(indexes = { @Index(columnList = "source_id", name = "ct_source_idx"),
         @Index(columnList = "target_id", name = "ct_target_idx") })
 public class CorrespondenceTable extends BaseEntity implements ClassificationEntityOperations, Publishable, Draftable {
-    @Lob
     @Column(columnDefinition = "text", nullable = false)
     @Convert(converter = TranslatablePersistenceConverter.class)
     private Translatable description;

klass-shared/src/main/java/no/ssb/klass/core/model/StatisticalClassification.java

@@ -33,7 +33,6 @@
 public abstract class StatisticalClassification extends BaseEntity implements ClassificationEntityOperations,
         Publishable, Draftable {
     public static final int FIRST_LEVEL_NUMBER = 1;
-    @Lob
     @Column(columnDefinition = "text", nullable = false)
     @Convert(converter = TranslatablePersistenceConverter.class)
     protected Translatable introduction;

klass-shared/src/main/java/no/ssb/klass/core/model/User.java

@@ -12,7 +12,9 @@
 import javax.persistence.Table;
 
 @Entity
-@Table(indexes = {
+@Table(
+        name = "user",
+        indexes = {
         @Index(columnList = "username", name = "user_username_idx", unique = true),
         @Index(columnList = "fullname", name = "user_fullname_idx")
 })