TIKA-4820 -- refactor ci to improve wall clock and environment

TIKA-4820 -- refactor ci to improve wall clock and environment

Author: tballison

.github/workflows/main-jdk17-build.yml

@@ -27,6 +27,26 @@ on:
     paths-ignore:
       - 'docs/**'
 
+# Superseded PR runs are dead work: cancel them. Never cancel a main build --
+# that run is the record for a commit that is already merged.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+env:
+  # Single source of truth for the build/integration-tests shard split: the
+  # `integration-tests` job runs exactly these modules and the `build` job
+  # excludes exactly these modules, both derived from this one list.
+  #
+  # Do NOT split this into two hand-maintained lists. The dangerous direction is
+  # silent: a module excluded from `build` but missing from the shard runs
+  # nowhere, and CI stays green while the tests stop existing.
+  #
+  # These six are ~15 of the build's 37 minutes -- five are testcontainers-backed
+  # (Elasticsearch/OpenSearch/Solr/Kafka/MinIO) and tika-pipes-integration-tests
+  # forks JVMs. Adding a slow IT module here is the intended way to rebalance.
+  IT_MODULES: ':tika-pipes-integration-tests,:tika-pipes-es-integration-tests,:tika-pipes-opensearch-integration-tests,:tika-pipes-solr-integration-tests,:tika-pipes-kafka-integration-tests,:tika-pipes-s3-integration-tests'
+
 jobs:
   build:
     runs-on: ubuntu-latest
@@ -45,14 +65,54 @@ jobs:
           cache: 'maven'
       - name: Install external tools
         run: sudo apt-get update && sudo apt-get install -y ffmpeg libimage-exiftool-perl
+      # Everything except $IT_MODULES, which the integration-tests job owns.
+      # sed turns each ':artifactId' into the '!:artifactId' exclusion form.
       - name: Build with Maven
-        run: mvn clean apache-rat:check test install javadoc:aggregate -Pci -B "-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn"
+        run: |
+          mvn clean apache-rat:check test install javadoc:aggregate -Pci \
+            -pl "$(echo "$IT_MODULES" | sed 's/:/!:/g')" \
+            -B "-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn"
+
+  integration-tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    # No `needs: build`, same reasoning as e2e-tests below: this job installs the
+    # reactor itself, so gating on build would only serialize it.
+    strategy:
+      matrix:
+        java: [ '17' ]
+
+    steps:
+      - uses: actions/checkout@v6
+      - name: Set up JDK ${{ matrix.java }}
+        uses: actions/setup-java@v5
+        with:
+          distribution: 'temurin'
+          java-version: ${{ matrix.java }}
+          cache: 'maven'
+      - name: Install external tools
+        run: sudo apt-get update && sudo apt-get install -y ffmpeg libimage-exiftool-perl
+      # Full reactor for the same reason as the e2e job: these modules depend on
+      # the tika-pipes plugin zips, and a zip-type dependency never matches a
+      # jar-packaging reactor module, so -am cannot supply them. -Pfast keeps it
+      # to ~3 min; the tests themselves run in the next step.
+      - name: Install all modules (produces the plugin zips)
+        run: mvn clean install -Pfast -B "-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn"
+      # apache-rat:check here too -- the build job no longer sees these modules,
+      # so without this they would drop out of license checking entirely.
+      - name: Run integration tests
+        run: |
+          mvn clean apache-rat:check test -Pci \
+            -pl "$IT_MODULES" \
+            -B "-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn"
 
   e2e-tests:
     runs-on: ubuntu-latest
     # full-reactor install added; 30 was tight for it
     timeout-minutes: 45
-    needs: build
+    # No `needs: build` on purpose: this job checks out and installs the reactor
+    # itself, so it consumes nothing from `build`. Gating on it only serialized
+    # ~5 min onto the critical path.
     strategy:
       matrix:
         java: [ '17' ]
@@ -69,9 +129,10 @@ jobs:
       # zip-type dependency never matches a jar-packaging reactor module, so -am treats every
       # plugin zip as external and skips building those modules entirely. install (not verify)
       # because the assembly sets attach=false (TIKA-4723) and an install-phase install-file is
-      # what puts each zip in the local repo. Tests are skipped here; the build job owns them.
+      # what puts each zip in the local repo. -Pfast (skipTests + rat/checkstyle/spotless
+      # off) because the build job owns all of that; this job only needs the artifacts.
       - name: Install all modules (produces the plugin zips)
-        run: mvn clean install -DskipTests -B "-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn"
+        run: mvn clean install -Pfast -B "-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn"
       # -pl must name the leaf modules: tika-e2e-tests is an aggregator pom, and Maven does
       # not pull in a selected aggregator's children, so this job built two pom-only modules
       # and ran zero tests while reporting green.

.github/workflows/main-jdk17-locale-build.yml

@@ -0,0 +1,65 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# tr_TR is the highest-value locale to test in Java: dotless-i means
+# "TIFF".toLowerCase() is not "tiff" unless the call passes Locale.ROOT.
+#
+# Linux, not Windows: locale bugs are JVM-level, so the 2x-cost Windows runner
+# buys nothing here -- main-jdk17-windows-build covers the OS-specific surface.
+#
+# Locale via -Duser.language/-Duser.country, not LANG/LC_ALL: the JVM silently
+# falls back to en_US when the named locale is not generated on the runner, so
+# the env-var form can pass while testing nothing.
+#
+# push-only, like the jdk21/jdk25 builds: locale regressions are rare and not
+# usually PR-specific, so a full reactor build per PR is not worth the cost.
+name: main jdk17 locale build (tr_TR)
+
+on:
+  push:
+    branches: [ main ]
+    paths-ignore:
+      - 'docs/**'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    strategy:
+      matrix:
+        java: [ '17' ]
+
+    steps:
+      - uses: actions/checkout@v6
+      - name: Set up JDK ${{ matrix.java }}
+        uses: actions/setup-java@v5
+        with:
+          distribution: 'temurin'
+          java-version: ${{ matrix.java }}
+          cache: 'maven'
+      - name: Install external tools
+        run: sudo apt-get update && sudo apt-get install -y ffmpeg libimage-exiftool-perl
+      # The Docker-backed integration tests spin up Elasticsearch/OpenSearch/Solr/Kafka/MinIO
+      # for ~7.5 min and carry no locale signal, so they are excluded here; the main jdk17
+      # build runs them. If a new testcontainers module appears, add it to this list --
+      # forgetting only makes this job slower, it does not weaken it.
+      - name: Build with Maven (tr_TR locale)
+        run: |
+          mvn clean test install -Pci \
+            -pl '!:tika-pipes-es-integration-tests,!:tika-pipes-kafka-integration-tests,!:tika-pipes-opensearch-integration-tests,!:tika-pipes-s3-integration-tests,!:tika-pipes-solr-integration-tests' \
+            -Duser.language=tr -Duser.country=TR \
+            -B "-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn"

.github/workflows/main-jdk17-windows-build-multi-locale.yml

@@ -15,13 +15,23 @@
 # limitations under the License.
 #
 
-name: main jdk17 windows build
+# The one Windows job. Two things it uniquely covers:
+#   - path handling: the checkout dir below deliberately contains a space
+#   - an alternate (non-en_US) locale
+#
+# Locale is set with -Duser.language/-Duser.country, NOT LANG/LC_ALL. Those env
+# vars are POSIX-only: the Windows JVM reads the OS locale via Win32 and ignores
+# them, and even on Linux the JVM silently falls back to en_US when the named
+# locale is not generated on the box. -D always applies.
+#
+# push-only, like the jdk21/jdk25 and tr_TR builds. This is the most expensive
+# job in CI -- a full reactor on a 2x-cost runner, ~48 min -- and it gated PR
+# wall clock all by itself while every Linux job finished in ~22. Windows-only
+# regressions are real but rare, so they are caught on main within the hour
+# rather than paid for on every PR push.
+name: main jdk17 windows build (de_DE)
 
 on:
-  pull_request:
-    branches: [ main ]
-    paths-ignore:
-      - 'docs/**'
   push:
     branches: [ main ]
     paths-ignore:
@@ -36,6 +46,7 @@ jobs:
         java: [ '17' ]
 
     steps:
+      # The space in this path is deliberate -- it is what catches path-quoting bugs.
       - uses: actions/checkout@v6
         with:
           path: 'tika build dir'
@@ -45,6 +56,6 @@ jobs:
           distribution: 'temurin'
           java-version: ${{ matrix.java }}
           cache: 'maven'
-      - name: Build with Maven
+      - name: Build with Maven (de_DE locale)
         working-directory: 'tika build dir'
-        run: mvn clean test install javadoc:aggregate -Pci -Pe2e -B "-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn"
+        run: mvn clean test install javadoc:aggregate -Pci -Pe2e -Duser.language=de -Duser.country=DE -B "-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn"

.github/workflows/main-jdk17-windows-build.yml

@@ -23,6 +23,12 @@ on:
   push:
     branches: [ main ]
 
+# Superseded PR runs are dead work: cancel them. Never cancel a main build --
+# that run is the record for a commit that is already merged.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 jobs:
   check-split-packages:
     runs-on: ubuntu-latest