CI: Kill leaf jobs when aborting stale PR dispatchers

The OSS dispatcher already aborts the prior dispatcher build for the same
PR, but the 8 leaf jobs it spawned via `build job: ..., wait: true` keep
running until they finish naturally. That ties up build executors every
time someone pushes a new commit to a PR.

Port the pattern NIXL uses: scrape each stale dispatcher's console log
for `Starting building: <name> #<num>` lines to find its children, kill
the leaves first so the dispatcher's wait unblocks, then kill the
dispatcher. Use `doKill()` instead of `doStop()` for a hard stop, and
retry up to 3 times with a 5s gap so builds caught mid-startup still get
torn down.

Failures inside the abort block are still swallowed and logged - we
never want stale cleanup to break a fresh build.

Author: Alexey-Rivkin

.ci/jenkins/proj_jjb_oss.yaml

@@ -53,23 +53,48 @@
         githubHelper = GithubHelper.getInstance("${{GIT_PASSWORD}}", VARIABLE_FROM_POST)
       }}
 
-      // Abort previous dispatcher runs for the same PR
+      // Abort previous dispatcher runs for the same PR and their leaf jobs
       def currentPrNumber = githubHelper.getPRNumber()?.toString()
       if (currentPrNumber) {{
         def slurper = new groovy.json.JsonSlurper()
         try {{
-          Jenkins.instance.getItemByFullName(env.JOB_NAME).builds.findAll {{
+          def staleBuilds = Jenkins.instance.getItemByFullName(env.JOB_NAME).builds.findAll {{
             it.isBuilding() && it.number < currentBuild.number
-          }}.each {{ b ->
+          }}.findAll {{ b ->
             def gd = b.getAction(hudson.model.ParametersAction)?.getParameters()?.find {{ it.name == 'VARIABLE_FROM_POST' }}?.value
             def otherPr = gd ? slurper.parseText(gd)?.issue?.number?.toString() : null
-            if (otherPr == currentPrNumber) {{
-              echo "Aborting dispatcher #${{b.number}} for same PR #${{currentPrNumber}}"
-              b.doStop()
+            otherPr == currentPrNumber
+          }}
+
+          if (staleBuilds) {{
+            echo "PR #${{currentPrNumber}}: found ${{staleBuilds.size()}} stale dispatcher(s) to abort"
+          }}
+
+          // Collect leaf builds from each stale dispatcher's console log
+          def leafBuilds = []
+          staleBuilds.each {{ b ->
+            b.getLog(1000).each {{ line ->
+              def m = (line =~ /Starting building: (\S+) #(\d+)/)
+              if (m) {{
+                def child = Jenkins.instance.getItemByFullName(m[0][1])?.getBuildByNumber(m[0][2] as int)
+                if (child) leafBuilds.add(child)
+              }}
+            }}
+          }}
+
+          // Kill leaf jobs first (unblocks dispatcher), then dispatchers
+          def allTargets = leafBuilds + staleBuilds
+          for (int attempt = 1; attempt <= 3; attempt++) {{
+            def alive = allTargets.findAll {{ it.isBuilding() }}
+            if (!alive) break
+            alive.each {{ target ->
+              echo "${{attempt > 1 ? 'Retry ' + attempt + ': ' : ''}}Killing ${{target.fullDisplayName}}"
+              target.doKill()
             }}
+            if (attempt < 3) sleep 5
           }}
         }} catch(Exception e) {{
-          echo "Could not abort previous dispatchers: ${{e.message}}"
+          echo "Could not abort previous builds: ${{e.message}}"
         }}
       }}