Update nf-azure: Add Azure Compute Gallery image and verification options

docs/azure.mdx

@@ -597,6 +597,43 @@ azure {
 }
 ```
 
+**Custom images from an Azure Compute Gallery**
+
+You can provision pool nodes from a custom VM image published in an [Azure Compute Gallery](https://learn.microsoft.com/en-us/azure/virtual-machines/azure-compute-gallery) by setting `virtualMachineImageId` to the image version resource ID. When set, `publisher` and `offer` are ignored, and `sku` must be set to the Batch node agent SKU id that matches the image operating system.
+
+```groovy
+azure {
+    batch {
+        pools {
+            <pool-name> {
+                virtualMachineImageId = '/subscriptions/<subscription>/resourceGroups/<group>/providers/Microsoft.Compute/galleries/<gallery>/images/<definition>/versions/<version>'
+                sku = 'batch.node.ubuntu 24.04'
+            }
+        }
+    }
+}
+```
+
+:::warning
+Custom images require Microsoft Entra authentication (service principal or managed identity). The **Azure Batch account identity** must have read access to the gallery image, and the pool typically requires `virtualNetwork` to be set.
+:::
+
+**Image verification**
+
+By default, Nextflow only selects images that Azure Batch has formally verified. Set `verification` to `unverified` to also allow images that Azure Batch lists but has not verified, or to `any` to accept both. This setting is ignored when `virtualMachineImageId` is set.
+
+```groovy
+azure {
+    batch {
+        pools {
+            <pool-name> {
+                verification = 'unverified'  // 'verified' (default), 'unverified' or 'any'
+            }
+        }
+    }
+}
+```
+
 ### Advanced features
 
 **Virtual networks**

docs/reference/config.mdx

@@ -577,6 +577,14 @@ Enable the `startTask` to run with elevated access (default`false`).
 
 The `startTask` that is executed as the node joins the Azure Batch node pool.
 
+##### `azure.batch.pools.<name>.verification`
+
+The image verification type to match when resolving the VM image from the Batch supported-images list. Can be `verified`, `unverified`, or `any` (default`verified`). Ignored when `virtualMachineImageId` is set.
+
+##### `azure.batch.pools.<name>.virtualMachineImageId`
+
+The resource ID of a custom VM image from an Azure Compute Gallery to use for the pool nodes (e.g. `/subscriptions/<id>/resourceGroups/<group>/providers/Microsoft.Compute/galleries/<gallery>/images/<definition>/versions/<version>`). When set, `publisher` and `offer` are ignored, and `sku` must be set to the Batch node agent SKU id that matches the image OS (e.g. `batch.node.ubuntu 24.04`).
+
 ##### `azure.batch.pools.<name>.virtualNetwork`
 
 <AddedInVersion version="23.04" />

plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzBatchService.groovy

@@ -51,6 +51,8 @@ import com.azure.compute.batch.models.BatchContainerConfiguration
 import com.azure.compute.batch.models.ContainerRegistryReference
 import com.azure.compute.batch.models.ContainerType
 import com.azure.compute.batch.models.ElevationLevel
+import com.azure.compute.batch.models.ImageReference
+import com.azure.compute.batch.models.MetadataItem
 import com.azure.compute.batch.models.BatchMetadataItem
 import com.azure.compute.batch.models.MountConfiguration
 import com.azure.compute.batch.models.NetworkConfiguration
@@ -704,16 +706,22 @@ class AzBatchService implements Closeable {
                 continue
             if( it.osType != opts.osType )
                 continue
-            if( it.verificationType != opts.verification )
+            if( opts.verification != null && it.verificationType != opts.verification )
                 continue
             if( !it.imageReference.publisher.equalsIgnoreCase(opts.publisher) )
                 continue
             if( it.imageReference.offer.equalsIgnoreCase(opts.offer) )
                 return it
         }
 
-        log.debug "[AZURE BATCH] No VM image matching sku=$opts.sku; publisher=$opts.publisher; offer=$opts.offer; OS type=$opts.osType; verification type=$opts.verification - supported images: $available"
-        throw new IllegalStateException("Cannot find a matching VM image with publisher=$opts.publisher; offer=$opts.offer; OS type=$opts.osType; verification type=$opts.verification")
+        log.debug "[AZURE BATCH] No VM image matching sku=$opts.sku; publisher=$opts.publisher; offer=$opts.offer; OS type=$opts.osType; verification type=${opts.verification ?: 'any'} - supported images: $available"
+        throw new IllegalStateException("Cannot find a matching VM image with publisher=$opts.publisher; offer=$opts.offer; OS type=$opts.osType; verification type=${opts.verification ?: 'any'}")
+    }
+
+    protected ImageReference customImageReference(AzPoolOpts opts) {
+        if( !opts.sku )
+            throw new IllegalArgumentException("Azure Batch pool option 'sku' is required when 'virtualMachineImageId' is set - it must be a valid node agent SKU id (e.g. 'batch.node.ubuntu 24.04')")
+        return new ImageReference().setVirtualMachineImageId(opts.virtualMachineImageId)
     }
 
     protected AzVmPoolSpec specFromPoolConfig(String poolId) {
@@ -883,9 +891,20 @@ class AzBatchService implements Closeable {
             log.debug "[AZURE BATCH] Connecting Azure Batch pool to Container Registry '$registryOpts.server'"
         }
 
-        final image = getImage(opts)
+        final ImageReference imageRef
+        final String nodeAgentSkuId
+        if( opts.virtualMachineImageId ) {
+            imageRef = customImageReference(opts)
+            nodeAgentSkuId = opts.sku
+            log.debug "[AZURE BATCH] Using custom VM image from Compute Gallery: $opts.virtualMachineImageId (node agent SKU: $nodeAgentSkuId)"
+        }
+        else {
+            final image = getImage(opts)
+            imageRef = image.imageReference
+            nodeAgentSkuId = image.nodeAgentSkuId
+        }
 
-        new VirtualMachineConfiguration(image.imageReference, image.nodeAgentSkuId)
+        new VirtualMachineConfiguration(imageRef, nodeAgentSkuId)
                 .setContainerConfiguration(containerConfig)
     }
 

plugins/nf-azure/src/main/nextflow/cloud/azure/config/AzPoolOpts.groovy

@@ -144,8 +144,21 @@ class AzPoolOpts implements CacheFunnel, ConfigScope {
     """)
     final String vmType
 
+    @ConfigOption
+    @Description("""
+        The resource ID of a custom VM image from an Azure Compute Gallery to use for the pool nodes
+        (e.g. `/subscriptions/<id>/resourceGroups/<group>/providers/Microsoft.Compute/galleries/<gallery>/images/<definition>/versions/<version>`).
+        When set, `publisher` and `offer` are ignored, and `sku` must be set to the Batch node agent SKU id that matches the image OS (e.g. `batch.node.ubuntu 24.04`).
+    """)
+    final String virtualMachineImageId
+
+    @ConfigOption
+    @Description("""
+        The image verification type to match when resolving the VM image from the Batch supported-images list. Can be `verified`, `unverified`, or `any` (default: `verified`). Ignored when `virtualMachineImageId` is set.
+    """)
+    final ImageVerificationType verification
+
     OSType osType = DEFAULT_OS_TYPE
-    ImageVerificationType verification = ImageVerificationType.VERIFIED
 
     String registry
     String userName
@@ -175,6 +188,20 @@ class AzPoolOpts implements CacheFunnel, ConfigScope {
         this.password = opts.password
         this.virtualNetwork = opts.virtualNetwork
         this.lowPriority = opts.lowPriority as boolean
+        this.virtualMachineImageId = opts.virtualMachineImageId ?: null
+        this.verification = parseVerification(opts.verification)
+    }
+
+    protected static ImageVerificationType parseVerification(value) {
+        if( value == null )
+            return ImageVerificationType.VERIFIED
+        if( value instanceof ImageVerificationType )
+            return value
+        final str = value.toString().toLowerCase()
+        if( str == 'verified' ) return ImageVerificationType.VERIFIED
+        if( str == 'unverified' ) return ImageVerificationType.UNVERIFIED
+        if( str == 'any' ) return null
+        throw new IllegalArgumentException("Invalid azure.batch.pools.<name>.verification value: '$value' - expected 'verified', 'unverified' or 'any'")
     }
 
     @Override
@@ -195,6 +222,8 @@ class AzPoolOpts implements CacheFunnel, ConfigScope {
         hasher.putUnencodedChars(schedulePolicy ?: '')
         hasher.putUnencodedChars(virtualNetwork ?: '')
         hasher.putBoolean(lowPriority)
+        hasher.putUnencodedChars(virtualMachineImageId ?: '')
+        hasher.putUnencodedChars(verification?.toString() ?: 'any')
         hasher.putUnencodedChars(startTask.script ?: '')
         hasher.putBoolean(startTask.privileged)
         return hasher

plugins/nf-azure/src/test/nextflow/cloud/azure/batch/AzBatchServiceTest.groovy

@@ -520,7 +520,7 @@ class AzBatchServiceTest extends Specification {
         then:
         1 * svc.guessBestVm(LOC, CPUS, MEM, null, TYPE) >> VM
         and:
-        spec.poolId == 'nf-pool-42f3635f3fb8b71160900efa959f7809-Standard_X1'
+        spec.poolId == 'nf-pool-7483c5b1874eb7b96e8cb13e0b3781da-Standard_X1'
         spec.metadata == [foo: 'bar']
 
     }

plugins/nf-azure/src/test/nextflow/cloud/azure/config/AzPoolOptsTest.groovy

@@ -16,6 +16,9 @@
 
 package nextflow.cloud.azure.config
 
+import com.azure.compute.batch.models.ImageVerificationType
+import com.google.common.hash.Hashing
+import nextflow.util.CacheHelper
 import nextflow.util.Duration
 import spock.lang.Specification
 /**
@@ -48,6 +51,54 @@ class AzPoolOptsTest extends Specification {
         !opts.lowPriority
         !opts.startTask.script
         !opts.startTask.privileged
+        !opts.virtualMachineImageId
+        opts.verification == ImageVerificationType.VERIFIED
+    }
+
+    def 'should configure a custom compute gallery image' () {
+        when:
+        def opts = new AzPoolOpts([
+            virtualMachineImageId: '/subscriptions/abc/resourceGroups/rg/providers/Microsoft.Compute/galleries/g/images/d/versions/1.0.0',
+            sku: 'batch.node.ubuntu 24.04',
+            verification: 'unverified',
+        ])
+        then:
+        opts.virtualMachineImageId == '/subscriptions/abc/resourceGroups/rg/providers/Microsoft.Compute/galleries/g/images/d/versions/1.0.0'
+        opts.sku == 'batch.node.ubuntu 24.04'
+        opts.verification == ImageVerificationType.UNVERIFIED
+    }
+
+    def 'should parse the verification value' () {
+        expect:
+        new AzPoolOpts([verification: VALUE]).verification == EXPECTED
+        where:
+        VALUE        | EXPECTED
+        null         | ImageVerificationType.VERIFIED
+        'verified'   | ImageVerificationType.VERIFIED
+        'unverified' | ImageVerificationType.UNVERIFIED
+        'any'        | null
+    }
+
+    def 'should reject an invalid verification value' () {
+        when:
+        new AzPoolOpts([verification: 'bogus'])
+        then:
+        def e = thrown(IllegalArgumentException)
+        e.message.contains('verification')
+    }
+
+    private static String hash(AzPoolOpts opts) {
+        opts.funnel(Hashing.murmur3_128().newHasher(), CacheHelper.HashMode.STANDARD).hash().toString()
+    }
+
+    def 'pool hash should differ when image config differs' () {
+        given:
+        def base = new AzPoolOpts()
+        def gallery = new AzPoolOpts([virtualMachineImageId: '/subscriptions/x/resourceGroups/rg/providers/Microsoft.Compute/galleries/g/images/d/versions/1'])
+        def unverified = new AzPoolOpts([verification: 'unverified'])
+        expect:
+        hash(base) != hash(gallery)
+        hash(base) != hash(unverified)
     }
 
     def 'should create pool with custom options' () {