Transkribus fixes

.github/workflows/push.yml

@@ -255,6 +255,15 @@ jobs:
       tags: ${{ needs.tags.outputs.tags }}
     secrets: inherit
     needs: [tags, java]
+  transkribus:
+    uses: ./.github/workflows/build.yml
+    with:
+      contexts: ${{ needs.imagemagick.outputs.context }}
+      image: transkribus
+      repository: ${{ vars.repository }}
+      tags: ${{ needs.tags.outputs.tags }}
+    secrets: inherit
+    needs: [tags, base, imagemagick]
   #############################################################################
   ## Tests.
   #############################################################################
@@ -482,6 +491,13 @@ jobs:
       image: tomcat
     secrets: inherit
     needs: [tomcat]
+  test-transkribus:
+    uses: ./.github/workflows/test.yml
+    with:
+      digests: ${{ needs.transkribus.outputs.digest }}
+      image: transkribus
+    secrets: inherit
+    needs: [transkribus]
   #############################################################################
   ## Grype.
   #############################################################################
@@ -654,3 +670,10 @@ jobs:
       image: tomcat
     secrets: inherit
     needs: [tomcat]
+  grype-transkribus:
+    uses: ./.github/workflows/grype.yml
+    with:
+      digest: ${{ needs.transkribus.outputs.digest }}
+      image: transkribus
+    secrets: inherit
+    needs: [transkribus]

README.md

@@ -974,6 +974,7 @@ adding the following, and restarting `Docker`:
 [solr]: ./solr/README.md
 [test]: ./test/README.md
 [tomcat]: ./tomcat/README.md
+[transkribus]: /transkribus/README.md
 
 [Alpine Docker Image]: https://hub.docker.com/_/alpine
 [Ansible]: https://docs.ansible.com/ansible/latest/user_guide/index.html#getting-started

alpaca/Dockerfile

@@ -50,7 +50,13 @@ ENV \
     ALPACA_DERIVATIVE_OCR_MAX_CONSUMERS=-1 \
     ALPACA_DERIVATIVE_OCR_QUEUE=queue:islandora-connector-ocr \
     ALPACA_DERIVATIVE_OCR_URL=http://hypercube:8000/ \
-    ALPACA_DERIVATIVE_SYSTEMS=fits,homarus,houdini,ocr \
+    ALPACA_DERIVATIVE_SYSTEMS=fits,homarus,houdini,ocr,transkribus \
+    ALPACA_DERIVATIVE_TRANSKRIBUS_ASYNC_CONSUMER=true \
+    ALPACA_DERIVATIVE_TRANSKRIBUS_CONSUMERS=-1 \
+    ALPACA_DERIVATIVE_TRANSKRIBUS_ENABLED=false \
+    ALPACA_DERIVATIVE_TRANSKRIBUS_MAX_CONSUMERS=-1 \
+    ALPACA_DERIVATIVE_TRANSKRIBUS_QUEUE=queue:islandora-connector-transkribus \
+    ALPACA_DERIVATIVE_TRANSKRIBUS_URL=http://transkribus:5000/ \
     ALPACA_FCREPO_INDEXER_ASYNC_CONSUMER=true \
     ALPACA_FCREPO_INDEXER_CONSUMERS=-1 \
     ALPACA_FCREPO_INDEXER_ENABLED=true \

alpaca/rootfs/etc/confd/templates/alpaca.properties.tmpl

@@ -75,3 +75,10 @@ derivative.ocr.service.url={{ getenv "ALPACA_DERIVATIVE_OCR_URL" }}
 derivative.ocr.concurrent-consumers={{ getenv "ALPACA_DERIVATIVE_OCR_CONSUMERS" }}
 derivative.ocr.max-concurrent-consumers={{ getenv "ALPACA_DERIVATIVE_OCR_MAX_CONSUMERS" }}
 derivative.ocr.async-consumer={{ getenv "ALPACA_DERIVATIVE_OCR_ASYNC_CONSUMER" }}
+
+derivative.transkribus.enabled={{ getenv "ALPACA_DERIVATIVE_TRANSKRIBUS_ENABLED" }}
+derivative.transkribus.in.stream={{ getenv "ALPACA_DERIVATIVE_TRANSKRIBUS_QUEUE" }}
+derivative.transkribus.service.url={{ getenv "ALPACA_DERIVATIVE_TRANSKRIBUS_URL" }}
+derivative.transkribus.concurrent-consumers={{ getenv "ALPACA_DERIVATIVE_TRANSKRIBUS_CONSUMERS" }}
+derivative.transkribus.max-concurrent-consumers={{ getenv "ALPACA_DERIVATIVE_TRANSKRIBUS_MAX_CONSUMERS" }}
+derivative.transkribus.async-consumer={{ getenv "ALPACA_DERIVATIVE_TRANSKRIBUS_ASYNC_CONSUMER" }}

ci/update-sha.sh

@@ -153,6 +153,12 @@ elif [ "$DEP" = "jdbc-postgres" ]; then
   ARG=POSTGRES_DRIVER_FILE_SHA256
   DOCKERFILES=("handle/Dockerfile")
 
+elif [ "$DEP" = "transkribus-process" ]; then
+  URL="https://github.com/ulsdevteam/transkribus-process/archive/refs/tags/v${NEW_VERSION}.tar.gz"
+  ARG="TRANSKRIBUS_PROCESS_SHA256"
+  DOCKERFILES=("transkribus/Dockerfile")
+  README="transkribus/README.md"
+
 elif [ "$DEP" = "s6-overlay" ]; then
   BASE_URL="https://github.com/just-containers/s6-overlay/releases/download/v${NEW_VERSION}"
   declare -A URLS_AND_ARGS=(

docker-bake.hcl

@@ -30,6 +30,7 @@ IMAGES = [
   "solr",
   "test",
   "tomcat",
+  "transkribus",
 ]
 
 DEPENDENCIES = {
@@ -56,6 +57,7 @@ DEPENDENCIES = {
   solr = ["java"]
   test = ["drupal"]
   tomcat = ["java"]
+  transkribus = ["base", "imagemagick"]
 }
 
 ###############################################################################
@@ -322,6 +324,11 @@ target "tomcat-common" {
   context = "tomcat"
 }
 
+target "transkribus-common" {
+  inherits = ["common"]
+  context = "transkribus"
+}
+
 ###############################################################################
 # Default Image targets for local builds.
 ###############################################################################
@@ -504,6 +511,13 @@ target "tomcat" {
   tags = tags("tomcat", "")
 }
 
+target "transkribus" {
+  inherits = ["transkribus-common"]
+  contexts = dependencies("transkribus", "")
+  cache-from = cacheFrom("transkribus", hostArch())
+  tags = tags("transkribus", "")
+}
+
 ###############################################################################
 # linux/amd64 targets.
 ###############################################################################
@@ -686,6 +700,13 @@ target "tomcat-amd64" {
   tags = tags("tomcat", "amd64")
 }
 
+target "transkribus-amd64" {
+  inherits = ["transkribus-common", "amd64-common"]
+  contexts = dependencies("transkribus", "amd64")
+  cache-from = cacheFrom("transkribus", "amd64")
+  tags = tags("transkribus", "amd64")
+}
+
 ###############################################################################
 # linux/arm64 targets.
 ###############################################################################
@@ -867,3 +888,10 @@ target "tomcat-arm64" {
   cache-from = cacheFrom("tomcat", "arm64")
   tags = tags("tomcat", "arm64")
 }
+
+target "transkribus-arm64" {
+  inherits = ["transkribus-common", "arm64-common"]
+  contexts = dependencies("transkribus", "arm64")
+  cache-from = cacheFrom("transkribus", "arm64")
+  tags = tags("transkribus", "arm64")
+}

docker-compose.yml

@@ -65,6 +65,9 @@ services:
     milliner:
         <<: *common
         image: ${REPOSITORY:-local}/milliner:${TAG:-latest}
+    transkribus:
+        <<: *common
+        image: ${REPOSITORY:-local}/transkribus:${TAG:-latest}
     activemq:
         <<: *common
         image: ${REPOSITORY:-local}/activemq:${TAG:-latest}

renovate.json

@@ -101,6 +101,7 @@
         "islandora-starter-site",
         "s6-overlay",
         "solr-ocrhighlighting",
+        "transkribus-process",
         "vscode"
       ],
       "postUpgradeTasks": {

transkribus/Dockerfile

@@ -0,0 +1,77 @@
+# syntax=docker/dockerfile:1.5.1
+FROM base AS build
+
+ARG TARGETARCH
+
+# renovate: datasource=github-releases depName=transkribus-process packageName=ulsdevteam/transkribus-process
+ARG TRANSKRIBUS_PROCESS_VERSION=1.1.0
+ARG TRANSKRIBUS_PROCESS_FILE=v${TRANSKRIBUS_PROCESS_VERSION}.tar.gz
+ARG TRANSKRIBUS_PROCESS_URL=https://github.com/ulsdevteam/transkribus-process/archive/refs/tags/${TRANSKRIBUS_PROCESS_FILE}
+ARG TRANSKRIBUS_PROCESS_SHA256="8a88c135fed7668d3bdbfaefa0ca75f0cbb8daa406158784a05aea53bf0eb9f6"
+
+# renovate: datasource=repology depName=alpine_3_20/dotnet8-sdk
+ARG DOTNET_SDK_VERSION=8.0.111-r0
+
+RUN --mount=type=cache,id=transkribus-downloads-${TARGETARCH},sharing=locked,target=/opt/downloads \
+    download.sh \
+        --url "${TRANSKRIBUS_PROCESS_URL}" \
+        --sha256 "${TRANSKRIBUS_PROCESS_SHA256}" \
+        --strip \
+        --dest "/source" \
+    && \
+    cleanup.sh
+
+RUN --mount=type=cache,id=transkribus-apk-${TARGETARCH},sharing=locked,target=/var/cache/apk \
+    apk add \
+        dotnet8-sdk=="${DOTNET_SDK_VERSION}" \
+    && \
+    cd source && \
+    dotnet publish -c Release -o /opt/transkribus && \
+    cleanup.sh
+
+FROM imagemagick
+FROM base AS runtime
+
+ARG TARGETARCH
+
+ARG \
+    # renovate: datasource=repology depName=alpine_3_20/aspnetcore8-runtime
+    ASPNETCORE_VERSION=8.0.11-r0 \
+    # renovate: datasource=repology depName=alpine_3_20/nodejs
+    NODEJS_VERSION=20.15.1-r0 \
+    # renovate: datasource=repology depName=alpine_3_20/npm
+    NPM_VERSION=10.9.1-r0 \
+    # renovate: datasource=npm depName=xslt3
+    XSLT3_VERSION=2.7.0
+
+
+# Platform specific does require arch specific identifier.
+RUN --mount=type=cache,id=transkribus-apk-${TARGETARCH},sharing=locked,target=/var/cache/apk \
+    --mount=type=bind,from=imagemagick,source=/packages,target=/packages \
+    --mount=type=bind,from=imagemagick,source=/etc/apk/keys,target=/etc/apk/keys \
+    apk add \
+        /packages/imagemagick-*.apk \
+        aspnetcore8-runtime=="${ASPNETCORE_VERSION}" \
+        nodejs=="${NODEJS_VERSION}" \
+        npm=="${NPM_VERSION}" \
+    && \
+    npm install -g "xslt3@${XSLT3_VERSION}" && \
+    cleanup.sh
+
+RUN create-service-user.sh \
+    --name transkribus \
+    --group jwt \
+    /data \
+    && \
+    cleanup.sh
+
+ENV \
+    TRANSKRIBUS_ALTO_TO_HOCR_SEF_PATH=alto_to_hocr.sef.json \
+    TRANSKRIBUS_ASPNETCORE_URLS=http://transkribus:5000/ \
+    TRANSKRIBUS_ASPNETCORE_ENVIRONMENT=Production \
+    TRANSKRIBUS_CONNECTION_STRING="Filename=/data/transkribus-process.db" \
+    TRANSKRIBUS_USE_JWT_AUTHENTICATION=true
+
+COPY --from=build /opt/transkribus /opt/transkribus
+
+COPY --link rootfs /

transkribus/README.md

@@ -0,0 +1,42 @@
+# Transkribus
+
+Docker image for version 1.1.0 of the [transkribus-process microservice]
+
+Built from [Islandora-DevOps/isle-buildkit transkribus](https://github.com/Islandora-DevOps/isle-buildkit/tree/main/transkribus)
+
+Please refer to the [Transkribus Documentation] for more in-depth information.
+
+## Dependencies
+
+Requires `islandora/base` docker image to build. Please refer to the
+[Base Image README](../base/README.md) for additional information.
+
+## Ports
+
+| Port | Description                                                           |
+| :--- | :-------------------------------------------------------------------- |
+| 5000 | This can be changed using the `ASPNETCORE_URLS` environment variable. |
+
+## Volumes
+
+| Path  | Description           |
+| :---- | :-------------------- |
+| /data | For SQLite Database files and ASP.NET Core data. |
+
+## Settings
+
+JWT authentication is enabled by default, and can be configured by setting `USE_JWT_AUTHENTICATION` to true or false.
+
+| Environment Variable               | Default                                 | Description                                                                                        |
+| :--------------------------------- | :-------------------------------------- | :------------------------------------------------------------------------------------------------- |
+| TRANSKRIBUS_ALTO_TO_HOCR_SEF_PATH  | alto_to_hocr.sef.json                   | Path to an `xslt` that has been compiled into an sef.json by the `xslt3` utility                   |
+| TRANSKRIBUS_ASPNETCORE_URLS        | http://transkribus:5000/                | See [ASPNET Server URLs] for more information                                                      |
+| TRANSKRIBUS_CONNECTION_STRING      | "Filename=/data/transkribus-process.db" | Connection string for a SQLite or MySQL database                                                   |
+| TRANSKRIBUS_SERVICE_PASSWORD       |                                         | Password, used to connect to [Transkribus]                                                         |
+| TRANSKRIBUS_SERVICE_USERNAME       |                                         | Username, used to connect to [Transkribus]                                                         |
+| TRANSKRIBUS_USE_JWT_AUTHENTICATION | true                                    | Connect using `JWT`, see `JWT_PUBLIC_KEY` in the `base` images documentation, for more information |
+
+[transkribus-process microservice]: https://github.com/ulsdevteam/transkribus-process
+[Transkribus Documentation]: https://help.transkribus.org/
+[ASPNET Server URLs]: https://learn.microsoft.com/en-us/aspnet/core/fundamentals/host/web-host?view=aspnetcore-8.0#server-urls
+[Transkribus]: https://www.transkribus.org/

transkribus/rootfs/etc/s6-overlay/s6-rc.d/transkribus/finish

@@ -0,0 +1,2 @@
+#!/command/execlineb -S2
+/usr/local/share/s6/finish ${1} ${2}

transkribus/rootfs/etc/s6-overlay/s6-rc.d/transkribus/run

@@ -0,0 +1,22 @@
+#!/command/with-contenv bash
+# shellcheck shell=bash
+set -e
+
+# When bind mounting we need to ensure that we
+# actually can write to the folder.
+chown transkribus:transkribus /data
+
+# Set the HOME directory to the data directory.
+# This is so that Data Protection keys are stored in the data directory.
+export \
+    HOME="/data" \
+    ALTO_TO_HOCR_SEF_PATH="${TRANSKRIBUS_ALTO_TO_HOCR_SEF_PATH}" \
+    ASPNETCORE_CONTENTROOT="/opt/transkribus" \
+    ASPNETCORE_URLS="${TRANSKRIBUS_ASPNETCORE_URLS}" \
+    CONNECTION_STRING="${TRANSKRIBUS_CONNECTION_STRING}" \
+    TRANSKRIBUS_PASSWORD="${TRANSKRIBUS_SERVICE_PASSWORD}" \
+    TRANSKRIBUS_USERNAME="${TRANSKRIBUS_SERVICE_USERNAME}" \
+    USE_JWT_AUTHENTICATION="${TRANSKRIBUS_USE_JWT_AUTHENTICATION}"
+
+cd /opt/transkribus
+exec s6-setuidgid transkribus dotnet transkribus-process.dll msvc

transkribus/rootfs/etc/s6-overlay/s6-rc.d/transkribus/type

@@ -0,0 +1 @@
+longrun

transkribus/tests/ServiceStartsWithDefaults/build.gradle.kts

@@ -0,0 +1,5 @@
+import plugins.TestsPlugin.DockerComposeUp
+
+tasks.named<DockerComposeUp>("test") {
+    expectExitCodes("transkribus", 0)
+}

transkribus/tests/ServiceStartsWithDefaults/docker-compose.yml

@@ -0,0 +1,15 @@
+---
+
+# Common to all services
+x-common: &common
+  restart: "no"
+
+name: transkribus-servicestartswithdefaults
+services:
+  transkribus:
+    <<: *common
+    image: ${TRANSKRIBUS:-islandora/transkribus:local}
+    volumes:
+      - ./test.sh:/test.sh # Test to run.
+    command:
+      - /test.sh # Run test and exit.

transkribus/tests/ServiceStartsWithDefaults/test.sh

@@ -0,0 +1,15 @@
+#!/command/with-contenv bash
+# shellcheck shell=bash
+
+# shellcheck disable=SC1091
+source /usr/local/share/isle/utilities.sh
+
+# Wait for service to start.
+# We don't check if it is functioning as it requires a Transkribus account.
+echo "Waiting for reponse on http://localhost:5000/"
+while status=$(curl -s -w "%{http_code}" http://localhost:5000/ 2>/dev/null || echo "000") && [ "$status" != "400" ]; do
+  sleep 5
+done
+
+# Service must start for us to get to this point.
+exit 0