chore: make e2e workflow a bit fast

.github/workflows/build-and-test.yml

@@ -6,11 +6,13 @@ on:
 jobs:
   build-and-test:
     runs-on: ubuntu-latest
+    timeout-minutes: 30
 
     steps:
-      - uses: actions/checkout@v5
+      - name: Checkout code
+        uses: actions/checkout@v4
 
-      - name: Set up JDK
+      - name: Set up JDK 21
         uses: actions/setup-java@v5
         with:
           distribution: "temurin"
@@ -25,50 +27,39 @@ jobs:
             services/mq-interceptor/go.sum
             services/cega-mock/go.sum
 
-      - name: Cache gradle packages
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.gradle/cache
-            ~/.gradle/wrapper
-          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
-          restore-keys: |
-            ${{ runner.os }}-gradle-
-
-      - name: Build with Gradle
-        run: ./gradlew assemble
-
-      - name: Run tests
-        run: ./gradlew :lib:crypt4gh:test :lib:clearinghouse:test :lib:tsd-file-api-client:test :services:tsd-api-mock:test :services:mq-interceptor:test :services:localega-tsd-proxy:test :services:cega-mock:test :cli:lega-commander:test
-
+      - name: Build builder image
+        run: docker build -f builder.Dockerfile -t temp-builder:pr-${{ github.run_id }} .
 
-      - name: Run e2e tests
+      - name: Start E2E environment
         run: |
-          ./gradlew clean
-          ./gradlew start-docker-containers
+          chmod +x ./e2eTests/scripts/bootstrap.sh
+          ./e2eTests/scripts/bootstrap.sh cleanup_workspace || return 1
+          ./e2eTests/scripts/bootstrap.sh apply_configs || return 1
+          ./e2eTests/scripts/bootstrap.sh check_requirements || return 1
+          export BUILDER_IMAGE=temp-builder:pr-${GITHUB_RUN_ID}
+          cd e2eTests/ && docker compose -f docker-compose.yml -f docker-compose.override.yml up --pull always --build -d
 
-          # Wait for the e2e-tests container to complete
-          echo "Waiting for e2e-tests container to complete..."
-          while [ "$(docker inspect --format='{{.State.Running}}' e2e-tests 2>/dev/null)" == "true" ]; do
-            echo "e2e-tests still running, waiting for 5 seconds..."
-            sleep 5
-          done
-
-          # Show the logs from the e2e-tests container
-          echo "=============== e2e-tests Container Logs ==============="
+      - name: Wait for E2E tests to complete
+        run: |
+          echo "Waiting for e2e-tests container..."
+          timeout 600 bash -c "while [ \"\$(docker inspect --format='{{.State.Running}}' e2e-tests 2>/dev/null)\" == \"true\" ]; do sleep 5; done"
+
+          EXIT_CODE=$(docker inspect --format='{{.State.ExitCode}}' e2e-tests 2>/dev/null || echo "255")
+
+          echo "=============== E2E Test Logs ==============="
           docker logs e2e-tests
-          echo "========================================================"
-
-          # Get the exit code once the container has finished
-          EXIT_CODE=$(docker inspect --format='{{.State.ExitCode}}' e2e-tests)
-          if [ $EXIT_CODE -ne 0 ]; then
-            echo "e2e tests failed with exit code $EXIT_CODE"
+          echo "============================================="
+
+          if [ "$EXIT_CODE" -ne 0 ]; then
+            echo "E2E tests failed with exit code $EXIT_CODE"
             exit 1
-          else
-            echo "e2e tests completed successfully"
           fi
 
-      - name: Containers status
+      - name: Show diagnostics on failure
         if: failure()
         run: |
+          echo "=============== Container Status ==============="
           docker ps -a
+          echo ""
+          echo "=============== All Logs ==============="
+          docker compose -f e2eTests/docker-compose.yml logs || true

builder.Dockerfile

@@ -0,0 +1,26 @@
+# Start with the base image for Gradle and Java
+FROM gradle:8-jdk21
+
+WORKDIR /app
+
+ARG GO_VERSION=1.25.2
+
+# Install wget/tar, download, extract, and clean up
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    wget \
+    tar \
+    && wget -q https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz -O go.tar.gz \
+    && tar -C /usr/local -xzf go.tar.gz \
+    && rm go.tar.gz \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+# Set up the Go environment variables
+ENV PATH="/usr/local/go/bin:${PATH}"
+
+RUN go version
+
+COPY . .
+
+# Run your original Gradle command
+RUN gradle clean build test --no-daemon --stacktrace

dev.sh

@@ -2,6 +2,8 @@
 
 set -Eeuo pipefail
 
+export TERM=${TERM:-xterm}
+
 source ./e2eTests/env.sh
 
 # Cross-platform compatibility checks
@@ -78,23 +80,44 @@ function show_header() {
 
 # Service management functions
 
-function start() {
+function test() {
     show_header
-    log_step "Starting development environment"
+    log_step "Running unit tests"
+    if ./gradlew \
+      :lib:crypt4gh:test \
+      :lib:clearinghouse:test \
+      :lib:tsd-file-api-client:test \
+      :services:tsd-api-mock:test \
+      :services:mq-interceptor:test \
+      :services:localega-tsd-proxy:test \
+      --parallel; then
+        log_success "Tests passed successfully!"
+    else
+        log_error "Tests failed"
+        return 1
+    fi
+}
 
-    if ./gradlew clean && bash -c "./gradlew start-docker-containers"; then
-        log_success "Development environment started successfully!"
+function start() {
+    show_header
+    log_step "Preparing E2E test environment"
+    chmod +x ./e2eTests/scripts/bootstrap.sh
+    ./e2eTests/scripts/bootstrap.sh cleanup_workspace || return 1
+    ./e2eTests/scripts/bootstrap.sh apply_configs || return 1
+    ./e2eTests/scripts/bootstrap.sh check_requirements || return 1
+    log_step "Starting Docker containers"
+    if cd e2eTests/ && docker compose up --pull always --build -d; then
+        log_success "E2E test environment started successfully!"
     else
-        log_error "Failed to start development environment"
+        log_error "Failed to start Docker containers"
         return 1
     fi
 }
 
 function stop() {
     show_header
     log_step "Stopping development environment"
-
-    if ./gradlew stop-docker-containers; then
+    if docker compose -f e2eTests/docker-compose.yml down --rmi local -v; then
         log_success "Development environment stopped successfully!"
     else
         log_error "Failed to stop development environment"

e2eTests/build.gradle.kts

@@ -27,69 +27,6 @@ dependencies {
     testAnnotationProcessor("org.projectlombok:lombok:1.18.42")
 }
 
-// Start setup scripts.
-
-tasks.register<Exec>("make-executable") {
-    commandLine("chmod", "+x", "./scripts/bootstrap.sh")
-}
-
-tasks.register<Exec>("cleanup") {
-    dependsOn("make-executable")
-    commandLine("sh", "-c", "./scripts/bootstrap.sh cleanup_workspace")
-}
-
-tasks.register<Exec>("assemble-binaries") {
-    dependsOn("cleanup")
-    commandLine(
-        "../gradlew",
-        ":e2eTests:jar",
-        ":cli:lega-commander:build",
-        ":lib:crypt4gh:build",
-        ":lib:clearinghouse:build",
-        ":lib:tsd-file-api-client:build",
-        ":services:cega-mock:build",
-        ":services:tsd-api-mock:build",
-        ":services:mq-interceptor:build",
-        ":services:localega-tsd-proxy:build",
-        "-x",
-        "test",
-        "--parallel"
-    )
-}
-
-tasks.register<Exec>("check-requirements") {
-    dependsOn("assemble-binaries")
-    commandLine("sh", "-c", "./scripts/bootstrap.sh apply_configs")
-}
-
-tasks.register<Exec>("apply-configs") {
-    dependsOn("check-requirements")
-    commandLine("sh", "-c", "./scripts/bootstrap.sh check_requirements")
-}
-
-tasks.register<Exec>("start-docker-containers") {
-    dependsOn("apply-configs")
-    commandLine("docker", "compose", "up", "--pull", "always", "--build", "-d")
-}
-
-tasks.register<Exec>("stop-docker-containers") {
-    commandLine("docker", "compose", "down", "--rmi", "local", "-v")
-}
-
-tasks.test {
-    useJUnitPlatform()
-    // test tasks are completed
-    mustRunAfter(
-        ":lib:crypt4gh:test",
-        ":lib:clearinghouse:test",
-        ":lib:tsd-file-api-client:test",
-        ":services:tsd-api-mock:test",
-        ":services:mq-interceptor:test",
-        ":services:localega-tsd-proxy:test"
-    )
-    testLogging.showStandardStreams = true
-}
-
 tasks.jar {
     duplicatesStrategy = DuplicatesStrategy.EXCLUDE // This will exclude duplicate files
     manifest {

e2eTests/docker-compose.override.yml

@@ -0,0 +1,20 @@
+name: fega-norway
+
+services:
+  tsd:
+    build:
+      target: runtime
+      args:
+        BUILDER_IMAGE: ${BUILDER_IMAGE}
+
+  proxy:
+    build:
+      target: runtime
+      args:
+        BUILDER_IMAGE: ${BUILDER_IMAGE}
+
+  e2e-tests:
+    build:
+      target: runtime
+      args:
+        BUILDER_IMAGE: ${BUILDER_IMAGE}

e2eTests/docker-compose.template.yml

@@ -644,8 +644,8 @@ services:
     container_name: e2e-tests
     hostname: e2e_tests
     build:
-      context: .
-      dockerfile: e2e-tests.Dockerfile
+      context: ..
+      dockerfile: e2eTests/e2e-tests.Dockerfile
     depends_on:
       proxy:
         condition: service_healthy

e2eTests/e2e-tests.Dockerfile

@@ -1,19 +1,33 @@
-# Use Temurin 21 as the base image for Java 21
-FROM eclipse-temurin:21-jdk-alpine
+ARG BUILDER_IMAGE=gradle:8-jdk21
+
+FROM ${BUILDER_IMAGE} AS builder
+
+WORKDIR /app
+
+COPY build.gradle.kts settings.gradle.kts ./
+COPY buildSrc ./buildSrc
+COPY lib/crypt4gh ./lib/crypt4gh
+COPY e2eTests/ ./e2eTests
+
+RUN gradle clean :lib:crypt4gh:assemble :e2eTests:jar --no-daemon
+
+# Use JDK instead of JRE because the entrypoint requires
+# 'keytool' to import certificates at runtime
+FROM eclipse-temurin:21-jdk-alpine AS runtime
 
 # Install bash
-RUN apk add --no-cache bash
+RUN apk update && apk add --no-cache bash tree
 
 # Set the working directory in the container
 WORKDIR /fega-norway
 
-# Copy the application JAR and scripts
-COPY /build/libs/e2eTests.jar /fega-norway/e2eTests.jar
-COPY env.sh /fega-norway/env.sh
-COPY entrypoint.sh /entrypoint.sh
+# Copy the application JAR, env, and scripts
+COPY --from=builder /app/e2eTests/build/libs/e2eTests.jar /fega-norway/e2eTests.jar
+COPY --from=builder /app/e2eTests/env.sh /fega-norway/env.sh
+COPY --from=builder /app/e2eTests/scripts/e2e-tests-entrypoint.sh /fega-norway/entrypoint.sh
 
 # Make entrypoint executable
-RUN chmod +x /entrypoint.sh
+RUN chmod +x /fega-norway/entrypoint.sh
 
 # Run the entrypoint using bash
-CMD ["/bin/bash", "/entrypoint.sh"]
+CMD ["/bin/bash", "/fega-norway/entrypoint.sh"]

e2eTests/env.sh

@@ -235,3 +235,5 @@ export E2E_TRUSTSTORE_PASSWORD=$KEYTOOL_TRUSTSTORE_PASSWORD
 export E2E_PROXY_TOKEN_AUDIENCE=$PROXY_TOKEN_AUDIENCE
 export E2E_PROXY_ADMIN_USERNAME=$PROXY_ADMIN_USER
 export E2E_PROXY_ADMIN_PASSWORD=$PROXY_ADMIN_PASSWORD
+
+export BUILDER_IMAGE=gradle:8-jdk21

e2eTests/file-orchestrator.Dockerfile

@@ -24,13 +24,19 @@ RUN mkdir -p "confs"
 RUN mkdir -p "certs"
 
 COPY confs confs
-COPY scripts/* .
+
+COPY scripts/generate_certs.sh .
+COPY scripts/copy_certificates_to_dest.sh .
+COPY scripts/copy_confs_to_dest.sh .
+COPY scripts/replace_template_variables.sh .
+COPY scripts/change_ownerships.sh .
+COPY scripts/file-orchestrator-entrypoint.sh .
 
 COPY "env.sh" "env.sh"
 
 RUN chmod +x *.sh
 
-ENTRYPOINT [ "./entrypoint.sh" ]
+ENTRYPOINT [ "./file-orchestrator-entrypoint.sh" ]
 
 # Add a HEALTHCHECK to verify readiness
 HEALTHCHECK --interval=5s --timeout=3s --retries=5 CMD [ "/bin/sh", "-c", "[ -f /storage/ready ] || exit 1" ]

e2eTests/scripts/bootstrap.sh

@@ -1,5 +1,10 @@
 #!/bin/bash
 
+# Get the directory where this script resides
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# Change to the parent directory e2eTests/
+cd "$SCRIPT_DIR/.."
+
 source env.sh
 
 # Utility functions --
@@ -107,8 +112,7 @@ function check_requirements() {
 }
 
 function cleanup_workspace() {
-    rm -f *.raw *.raw.enc &&
-      ../gradlew clean
+    rm -f *.raw *.raw.enc
 }
 
 # Entrypoint --

e2eTests/entrypoint.sh → e2eTests/scripts/e2e-tests-entrypoint.sh

@@ -1,7 +1,16 @@
-#!/bin/sh
+#!/bin/bash
 
 set -e
 
+# Wait until all key subdirectories in /storage are non-empty
+for dir in certs confs; do
+  echo "Checking $dir..."
+  while [ -z "$(ls -A /storage/$dir 2>/dev/null)" ]; do
+    echo "Waiting for /storage/$dir to have content..."
+    sleep 2
+  done
+done
+
 # Import the root certificate if it exists
 if [ -f "/storage/certs/rootCA.pem" ]; then
     keytool -importcert -file /storage/certs/rootCA.pem \