[skip-release] have renovate check downloads.apache for updates

Author: joecorall

.github/workflows/push.yml

@@ -1,6 +1,8 @@
 name: Build and Push Docker Images
 on:
   push:
+    paths-ignore:
+      - "renovate.json"
   workflow_dispatch:
     inputs:
       tag:

README.md

@@ -525,6 +525,9 @@ Many dependencies in this repo are automatically updated using [renovate](https:
   - these dependencies are automatically merged when our CI tests pass using [automerge](https://docs.renovatebot.com/key-concepts/automerge/#automerging-and-scheduling)
 - [github-releases](https://docs.renovatebot.com/modules/datasource/github-releases/) and [github-tags](https://docs.renovatebot.com/modules/datasource/github-tags/) for software we install manually
 - [git-refs](https://docs.renovatebot.com/modules/datasource/git-refs/) when we pin to a specific commit on a branch
+- [apache-downloads](https://docs.renovatebot.com/modules/datasource/custom) a custom data source that parses the HTML list of releases ([e.g. solr/solr](https://downloads.apache.org/solr/solr/)). This was done since using apache's GitHub repos is not a good indicator when apache cuts a release for their software. Apache's GitHub repos often have tags before apache officially releases their software, resulting in preemptive PRs from renovate before the release is available.
+
+##### Why isle-buildkit can't use Renovate's GitHub app to track updates
 
 Since renovate does not natively support the ability to extract a sha256 from a file, we need [a custom shell script](./ci/update-sha.sh) in the [postUpgradeTasks](https://docs.renovatebot.com/configuration-options/#postupgradetasks) to calculate the sha256 of our files and update our Dockerfile accordingly.
 

activemq/Dockerfile

@@ -2,7 +2,7 @@
 FROM java
 
 ARG TARGETARCH
-# renovate: datasource=github-tags depName=apache-activemq packageName=apache/activemq
+# renovate: datasource=apache-downloads depName=apache-activemq packageName=activemq
 ARG ACTIVEMQ_VERSION=5.19.0
 ARG ACTIVEMQ_FILE="apache-activemq-${ACTIVEMQ_VERSION}-bin.tar.gz"
 ARG ACTIVEMQ_URL="https://archive.apache.org/dist/activemq/${ACTIVEMQ_VERSION}/${ACTIVEMQ_FILE}"

blazegraph/Dockerfile

@@ -7,7 +7,7 @@ ARG BLAZEGRAPH_FILE="blazegraph.war"
 ARG BLAZEGRAPH_URL="https://github.com/blazegraph/database/releases/download/BLAZEGRAPH_RELEASE_${BLAZEGRAPH_VERSION}/${BLAZEGRAPH_FILE}"
 ARG BLAZEGRAPH_SHA256="b22f1a1aa8e536443db9a57da63720813374ef59e4021cfa9ad0e98f9a420e85"
 
-# renovate: datasource=github-releases depName=apache-log4j packageName=apache/logging-log4j2
+# renovate: datasource=apache-downloads depName=apache-log4j packageName=logging/log4j
 ARG LOG4J_VERSION=2.24.3
 ARG LOG4J_FILE="apache-log4j-${LOG4J_VERSION}-bin.zip"
 ARG LOG4J_URL="https://archive.apache.org/dist/logging/log4j/${LOG4J_VERSION}/${LOG4J_FILE}"

fits/Dockerfile

@@ -15,7 +15,7 @@ ARG FITS_FILE="fits-${FITS_VERSION}.zip"
 ARG FITS_URL="https://github.com/harvard-lts/fits/releases/download/${FITS_VERSION}/${FITS_FILE}"
 ARG FITS_SHA256="32e436effe7251c5b067ec3f02321d5baf4944b3f0d1010fb8ec42039d9e3b73"
 
-# renovate: datasource=github-releases depName=apache-log4j packageName=apache/logging-log4j2
+# renovate: datasource=apache-downloads depName=apache-log4j packageName=logging/log4j
 ARG LOG4J_VERSION=2.24.3
 ARG LOG4J_FILE="apache-log4j-${LOG4J_VERSION}-bin.zip"
 ARG LOG4J_URL="https://archive.apache.org/dist/logging/log4j/${LOG4J_VERSION}/${LOG4J_FILE}"
@@ -57,7 +57,7 @@ ARG \
     PIP_VERSION=24.0-r2	 \
     # renovate: datasource=repology depName=alpine_3_20/python3
     PYTHON_VERSION=3.12.9-r0
-  
+
 # Replace linux shared libraries with ones that target muslibc and are platform specific.
 # Also add perl for exiftool, and platform specific jna so native libs can be loaded.
 #

renovate.json

@@ -39,20 +39,12 @@
     },
     {
       "matchPackageNames": [
-        "apache/activemq",
-        "apache/logging-log4j2",
         "fcrepo/fcrepo",
         "fcrepo-exts/fcrepo-import-export",
         "fcrepo-exts/fcrepo-upgrade-utils",
         "pgjdbc/pgjdbc"
       ],
-      "extractVersion": "^(REL|fcrepo|activemq|rel|fcrepo-import-export|fcrepo-upgrade-utils)(/|-)?(?<version>\\d+\\.\\d+\\.\\d+)$"
-    },
-    {
-      "extractVersion": "^releases/solr/(?<version>\\d+\\.\\d+\\.\\d+)$",
-      "matchPackageNames": [
-        "/^apache/solr$/"
-      ]
+      "extractVersion": "^(REL|fcrepo|rel|fcrepo-import-export|fcrepo-upgrade-utils)(/|-)?(?<version>\\d+\\.\\d+\\.\\d+)$"
     },
     {
       "extractVersion": "^v(?<version>\\d+\\.\\d+\\.\\d+\\.\\d+)$",
@@ -139,6 +131,13 @@
     }
   ],
   "customDatasources": {
+    "apache-downloads": {
+      "defaultRegistryUrlTemplate": "https://downloads.apache.org/{{ packageName }}/",
+      "format": "html",
+      "transformTemplates": [
+        "{ \"releases\":[ $map($filter(releases.version,function($v,$i,$a){ $match($v, /^(v)?\\d+\\.\\d+\\.\\d+(\\/)?$/) }),function($v,$i,$a){ { \"version\": $match($v, /\\d+\\.\\d+\\.\\d+/).match } })] }"
+       ]
+    },
     "handle": {
       "defaultRegistryUrlTemplate": "https://handle.net/hnr-source/",
       "format": "html",

solr/Dockerfile

@@ -2,7 +2,7 @@
 FROM java
 
 ARG TARGETARCH
-# renovate: datasource=github-tags depName=apache-solr packageName=apache/solr
+# renovate: datasource=apache-downloads depName=apache-solr packageName=solr/solr
 ARG SOLR_VERSION=9.8.1
 ARG SOLR_FILE=solr-${SOLR_VERSION}.tgz
 ARG SOLR_URL=https://archive.apache.org/dist/solr/solr/${SOLR_VERSION}/solr-${SOLR_VERSION}.tgz

tomcat/Dockerfile

@@ -2,7 +2,7 @@
 FROM java
 
 ARG TARGETARCH
-# renovate: datasource=github-tags depName=apache-tomcat packageName=apache/tomcat
+# renovate: datasource=apache-downloads depName=apache-tomcat packageName=tomcat/tomcat-9
 ARG TOMCAT_VERSION=9.0.102
 ARG TOMCAT_FILE_SHA256="23e0a3b16d53a121c8c86b0ea4e6967d1a4ab689ccf6490ce09eae57ac394787"