verify that all sources of libraries are included in gradle/verification-metadata.xml

Author: daniellehrner

.github/workflows/pre-review.yml

@@ -36,6 +36,25 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.head.sha || github.ref }}
       - uses: gradle/actions/wrapper-validation@39e147cb9de83bb9910b8ef8bd7fff0ee20fcd6f # v6.0.1
+  verify-source-checksums:
+    name: "Verify Source Checksums"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          ref: ${{ github.event.pull_request.head.sha || github.ref }}
+      - name: Set up Java
+        uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0
+        with:
+          distribution: temurin
+          java-version: 21
+      - name: Setup Gradle
+        uses: gradle/actions/setup-gradle@39e147cb9de83bb9910b8ef8bd7fff0ee20fcd6f # v6.0.1
+        with:
+          cache-disabled: true
+      - name: Check source checksums are present in verification-metadata.xml
+        run: ./gradlew checkSourceVerification
   spotless-checkLicense:
     name: "Spotless & Check License"
     runs-on: ubuntu-latest
@@ -126,7 +145,7 @@ jobs:
   unittests-passed:
     name: "unittests-passed"
     runs-on: ubuntu-latest
-    needs: [compile, unitTests]
+    needs: [compile, unitTests, verify-source-checksums]
     permissions:
       checks: write
       statuses: write

build.gradle

@@ -18,6 +18,7 @@ import groovy.transform.CompileStatic
 import groovy.transform.Memoized
 import net.ltgt.gradle.errorprone.CheckSeverity
 
+import groovy.xml.XmlSlurper
 import java.text.SimpleDateFormat
 import java.util.regex.Pattern
 
@@ -474,73 +475,119 @@ configure(allprojects - project(':platform')) {
   }
 }
 
-task resolveSourceArtifacts {
-  group = 'verification'
-  description = 'Resolves source artifacts for all configurations so they can be included in dependency verification metadata'
-  doLast {
-    def componentIds = new HashSet()
-    def unresolvedConfigs = 0
-
-    def collectComponentIds = { configs, String label ->
-      configs.matching { it.canBeResolved }.each { config ->
-        try {
-          config.incoming.resolutionResult.allComponents.each { component ->
-            componentIds.add(component.id)
-          }
-        } catch (Exception e) {
-          unresolvedConfigs++
-          logger.info("Could not resolve ${label} '${config.name}': ${e.message}")
+// Shared helpers for source artifact verification tasks
+ext.collectExternalComponentIds = { allprojects, logger ->
+  def componentIds = new HashSet()
+  def unresolvedConfigs = 0
+  def collectFromConfigs = { configs, String label ->
+    configs.matching { it.canBeResolved }.each { config ->
+      try {
+        config.incoming.resolutionResult.allComponents.each { component ->
+          componentIds.add(component.id)
         }
+      } catch (Exception e) {
+        unresolvedConfigs++
+        logger.info("Could not resolve ${label} '${config.name}': ${e.message}")
       }
     }
+  }
+  allprojects.each { proj ->
+    collectFromConfigs(proj.configurations, "${proj.path} config")
+    collectFromConfigs(proj.buildscript.configurations, "${proj.path} buildscript config")
+  }
+  if (unresolvedConfigs > 0) {
+    logger.warn("${unresolvedConfigs} configurations could not be resolved (run with --info for details)")
+  }
+  return componentIds.findAll { it instanceof org.gradle.api.artifacts.component.ModuleComponentIdentifier }
+}
 
-    allprojects.each { proj ->
-      collectComponentIds(proj.configurations, "${proj.path} config")
-      collectComponentIds(proj.buildscript.configurations, "${proj.path} buildscript config")
+ext.resolveSourceComponents = { depHandler, buildscriptDepHandler, externalIds, logger ->
+  def resolvedComponentIds = new HashSet()
+  def doResolve = { handler, idsToResolve, String label ->
+    def sizeBefore = resolvedComponentIds.size()
+    def result = handler.createArtifactResolutionQuery()
+      .forComponents(idsToResolve)
+      .withArtifacts(JvmLibrary, SourcesArtifact)
+      .execute()
+    result.resolvedComponents.each { component ->
+      component.getArtifacts(SourcesArtifact).each { source ->
+        if (source instanceof ResolvedArtifactResult) {
+          source.file // access .file to trigger download and register with verification metadata
+          resolvedComponentIds.add(component.id)
+        } else {
+          logger.warn("Could not download sources for ${component.id}: ${source}")
+        }
+      }
     }
+    def count = resolvedComponentIds.size() - sizeBefore
+    logger.lifecycle("Resolved sources for ${count} components from ${label}")
+  }
+  doResolve(depHandler, externalIds, "project repositories")
+  def unresolvedIds = externalIds.findAll { !resolvedComponentIds.contains(it) }
+  if (!unresolvedIds.isEmpty()) {
+    doResolve(buildscriptDepHandler, unresolvedIds, "buildscript repositories")
+  }
+  return resolvedComponentIds
+}
 
-    if (unresolvedConfigs > 0) {
-      logger.warn("${unresolvedConfigs} configurations could not be resolved (run with --info for details)")
+task resolveSourceArtifacts {
+  group = 'verification'
+  description = 'Resolves source artifacts for all configurations so they can be included in dependency verification metadata'
+  doLast {
+    def externalIds = collectExternalComponentIds(allprojects, logger)
+    logger.lifecycle("Resolving sources for ${externalIds.size()} external dependencies...")
+
+    def resolved = resolveSourceComponents(dependencies, buildscript.dependencies, externalIds, logger)
+
+    def finalUnresolved = externalIds.size() - resolved.size()
+    if (finalUnresolved > 0) {
+      logger.warn("${finalUnresolved} dependencies have no published source artifacts")
     }
+    logger.lifecycle("Total: sources resolved for ${resolved.size()}/${externalIds.size()} components")
+  }
+}
 
-    def externalIds = componentIds.findAll { it instanceof org.gradle.api.artifacts.component.ModuleComponentIdentifier }
-    logger.lifecycle("Resolving sources for ${externalIds.size()} external dependencies...")
+task checkSourceVerification {
+  group = 'verification'
+  description = 'Checks that all resolvable source artifacts have checksums in verification-metadata.xml'
+  doLast {
+    def verificationFile = file("gradle/verification-metadata.xml")
+    if (!verificationFile.exists()) {
+      throw new GradleException("gradle/verification-metadata.xml not found")
+    }
 
-    // Populated by resolveSources across both project and buildscript repository calls
-    def resolvedComponentIds = new HashSet()
-    def resolveSources = { depHandler, componentIdsToResolve, String label ->
-      def sizeBefore = resolvedComponentIds.size()
-      def result = depHandler.createArtifactResolutionQuery()
-        .forComponents(componentIdsToResolve)
-        .withArtifacts(JvmLibrary, SourcesArtifact)
-        .execute()
-      result.resolvedComponents.each { component ->
-        component.getArtifacts(SourcesArtifact).each { source ->
-          if (source instanceof ResolvedArtifactResult) {
-            source.file // access .file to trigger download and register with verification metadata
-            resolvedComponentIds.add(component.id)
-          } else {
-            logger.warn("Could not download sources for ${component.id}: ${source}")
-          }
+    def xml = new XmlSlurper().parse(verificationFile)
+    def knownSources = new HashSet()
+    xml.components.component.each { component ->
+      def key = "${component.@group}:${component.@name}:${component.@version}"
+      component.artifact.each { artifact ->
+        if (artifact.@name.toString().contains('-sources.jar')) {
+          knownSources.add(key)
         }
       }
-      def count = resolvedComponentIds.size() - sizeBefore
-      logger.lifecycle("Resolved sources for ${count} components from ${label}")
     }
 
-    resolveSources(dependencies, externalIds, "project repositories")
+    def externalIds = collectExternalComponentIds(allprojects, logger)
+    def resolved = resolveSourceComponents(dependencies, buildscript.dependencies, externalIds, logger)
 
-    // Retry unresolved sources using buildscript repositories (includes Gradle Plugin Portal)
-    def unresolvedIds = externalIds.findAll { !resolvedComponentIds.contains(it) }
-    if (!unresolvedIds.isEmpty()) {
-      resolveSources(buildscript.dependencies, unresolvedIds, "buildscript repositories")
+    def missing = []
+    resolved.each { id ->
+      def key = "${id.group}:${id.module}:${id.version}"
+      if (!knownSources.contains(key)) {
+        missing.add(key)
+      }
     }
 
-    def finalUnresolved = externalIds.size() - resolvedComponentIds.size()
-    if (finalUnresolved > 0) {
-      logger.warn("${finalUnresolved} dependencies have no published source artifacts")
+    if (!missing.isEmpty()) {
+      missing.sort()
+      logger.error("The following dependencies have published sources but are missing source checksums in verification-metadata.xml:")
+      missing.each { logger.error("  - ${it}") }
+      throw new GradleException(
+      "${missing.size()} source checksum(s) missing from verification-metadata.xml. " +
+      "Run './gradlew --write-verification-metadata sha256 resolveSourceArtifacts' and commit the updated file."
+      )
     }
-    logger.lifecycle("Total: sources resolved for ${resolvedComponentIds.size()}/${externalIds.size()} components")
+    logger.lifecycle("All ${resolved.size()} resolvable source artifacts have checksums in verification-metadata.xml")
   }
 }