Stop place import script

.gitignore

@@ -45,4 +45,7 @@ digiroad_stops*.csv
 
 #possible jore3 backup file
 jore3dump/*
-!jore3dump/put-bak-file-here.txt
+!jore3dump/put-bak-file-here.txt
+
+#stop place import script environment file
+.env

README.md

@@ -531,3 +531,35 @@ If a test case fails because the `com.microsoft.sqlserver.jdbc.SQLServerExceptio
 the error message says that it cannot find a database object, the problem is that the script which
 creates the source MSSQL database (_docker/mssql_init/populate.sql_) was changed. You can solve this problem by running the command:
 `./development.sh recreate` at command prompt.
+
+---
+
+## Jore 3 stop import script
+
+### Requirements
+
+You need to have Docker installed on your system to run the script
+
+### How to use
+
+To run the stop registry importer script you need to have a Jore 3 database, a populated Jore 4 routes database and the `jore4-hasura` and `jore4-tiamat` microservices running.
+
+By default the script runs from the local Jore 3 `mssqltestdb` database and uses the base local `jore4-hasura` service as the target.
+You can change the source database and target Hasura instance by creating a `.env` file in the same directory as the script.
+
+Set the values for variables you want to set:
+
+```
+GRAPHQL_URL=
+GRAPHQL_SECRET=
+JORE3_USERNAME=
+JORE3_PASSWORD=
+JORE3_DATABASE_URL=
+JORE3_DATABASE_NAME=
+```
+
+You should have run the base Jore 3 importer first which ensures the Jore 4 database has the required scheduled stop points. Then run the stop registry import script which will match scheduled stop points in the Jore 4 routes database with stops in the Jore 3 database and generate GraphQL mutations to Hasura/Tiamat according to the data. The script will also link the generated stop registry stops with the scheduled stop points by their NeTEx ID using Hasura for the mutation.
+
+Running the script will produce multiple errors as there are many stops in the Jore 3 database with overlapping validity. These can be ingored as only the most recent one is the one which ends up being imported.
+
+To run the script simply run `run-stop-registry-importer.sh` in the stop-registry-importer directory, this will create and run a Docker container with the environment set up.

development.sh

@@ -135,7 +135,7 @@ download_docker_compose_bundle() {
 }
 
 start_all() {
-  $DOCKER_COMPOSE_CMD up --build -d importer-jooq-database importer-test-database jore4-mssqltestdb jore4-hasura jore4-testdb jore4-jore3importer jore4-mapmatchingdb jore4-mapmatching
+  $DOCKER_COMPOSE_CMD up --build -d importer-jooq-database importer-test-database jore4-mssqltestdb jore4-hasura jore4-testdb jore4-jore3importer jore4-mapmatchingdb jore4-mapmatching jore4-tiamat
 }
 
 start_deps() {
@@ -145,7 +145,7 @@ start_deps() {
   # jore4-mssqltestdb - The Jore 3 MSSQL database which contains the source data which is read by the importer
   # jore4-hasura - Hasura. We have to start Hasura because it ensures that db migrations are run to the Jore 4 database.
   # jore4-testdb - Jore 4 database. This is the destination database of the import process.
-  $DOCKER_COMPOSE_CMD up --build -d importer-jooq-database importer-test-database jore4-mssqltestdb jore4-hasura jore4-testdb jore4-mapmatchingdb jore4-mapmatching
+  $DOCKER_COMPOSE_CMD up --build -d importer-jooq-database importer-test-database jore4-mssqltestdb jore4-hasura jore4-testdb jore4-mapmatchingdb jore4-mapmatching jore4-tiamat
 }
 
 stop() {
@@ -178,6 +178,12 @@ generate_jooq() {
   mvn clean generate-sources -Pci
 }
 
+upload_zones() {
+  echo "Uploading municipality and fare zones to Tiamat"
+
+  curl --silent --output /dev/null --show-error --fail -X POST -H"Content-Type: application/xml" -d @netex/hsl-zones-netex.xml localhost:3010/services/stop_places/netex
+}
+
 ### Control flow
 
 COMMAND=${1:-}
@@ -191,11 +197,13 @@ case $COMMAND in
   start)
     download_docker_compose_bundle
     start_all
+    upload_zones
     ;;
 
   start:deps)
     download_docker_compose_bundle
     start_deps
+    upload_zones
     ;;
 
   generate:jooq)
@@ -214,6 +222,7 @@ case $COMMAND in
   recreate)
     remove
     start_deps
+    upload_zones
     ;;
 
   list)

docker/docker-compose.custom.yml

@@ -38,8 +38,17 @@ services:
     volumes:
       - ../jore3dump:/mnt/jore3dump
 
+  jore4-tiamat:
+    # Pin tiamat to a compatible version.
+    image: "hsldevcom/jore4-tiamat:main--20250317-1136ff34a5b6030b0e9c85b0373f5ddc02946267"
+
   jore4-hasura:
     # pin compatible version of jore4 data model
-    image: "hsldevcom/jore4-hasura:hsl-main--20231218-8a28ef5f4b22b6351312788be4e674144f2bb0a2"
+    image: "hsldevcom/jore4-hasura:hsl-main--20250317-60eaf1217447890591ca3bbe262c96f2cc68ba57"
+    depends_on:
+      jore4-testdb:
+        condition: service_healthy
+      jore4-tiamat:
+        condition: service_healthy
 networks:
-    jore4:
+  jore4:

stop-registry-importer/Dockerfile

@@ -0,0 +1,11 @@
+FROM python:3.12
+
+WORKDIR /app
+COPY requirements.txt .
+
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY importer.py .
+COPY .env .
+
+CMD ["python", "importer.py"]