Update nf-azure: Add Azure Compute Gallery image and verification options

docs/azure.mdx

@@ -597,8 +597,45 @@ azure {
 }
 ```
 
+**Custom images from an Azure Compute Gallery**
+
+To provision pool nodes from a custom VM image published in an [Azure Compute Gallery](https://learn.microsoft.com/en-us/azure/virtual-machines/azure-compute-gallery), set `virtualMachineImageId` to the image version resource ID. When set, `publisher` and `offer` are ignored, and `sku` must be set to the Batch node agent SKU ID that matches the image operating system.
+
+```groovy
+azure {
+    batch {
+        pools {
+             {
+                virtualMachineImageId = '/subscriptions//resourceGroups//providers/Microsoft.Compute/galleries//images//versions/'
+                sku = 'batch.node.ubuntu 24.04'
+            }
+        }
+    }
+}
+```
+
+:::warning
+Custom images require Microsoft Entra authentication (service principal or managed identity). The **Azure Batch account identity** must have read access to the gallery image. The pool typically requires `virtualNetwork` to be set.
+:::
+
 ### Advanced features
 
+**Image verification**
+
+By default, Nextflow selects only images that Azure Batch has verified. Set `allowUnverifiedImages` to `true` to also allow images that Azure Batch lists but has not verified. Ignored when `virtualMachineImageId` is set.
+
+```groovy
+azure {
+    batch {
+        pools {
+            <pool-name> {
+                allowUnverifiedImages = true
+            }
+        }
+    }
+}
+```
+
 **Virtual networks**
 
 Pools can be configured to use virtual networks to connect to your existing network infrastructure.

docs/reference/config.mdx

@@ -509,6 +509,10 @@ The name of the batch service region, e.g. `westeurope` or `eastus2`. Not needed
 
 The client ID for an Azure [managed identity](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/overview) that is available on all Azure Batch node pools. This identity is used by Fusion to authenticate to Azure storage. If set to `'auto'`, Fusion will use the first available managed identity.
 
+##### `azure.batch.pools.<name>.allowUnverifiedImages`
+
+Allow the use of unverified VM images when resolving the image from the Batch supported-images list (default: `false`). Ignored when `virtualMachineImageId` is set.
+
 ##### `azure.batch.pools.<name>.autoScale`
 
 Enable autoscaling feature for the pool identified with `<name>`.
@@ -577,6 +581,11 @@ Enable the `startTask` to run with elevated access (default`false`).
 
 The `startTask` that is executed as the node joins the Azure Batch node pool.
 
+##### `azure.batch.pools.<name>.virtualMachineImageId`
+
+The resource ID of a custom VM image from an Azure Compute Gallery for the pool nodes (e.g., `/subscriptions/<id>/resourceGroups/<group>/providers/Microsoft.Compute/galleries/<gallery>/images/<definition>/versions/<version>`). When set, `publisher` and `offer` are ignored, and `sku` must be the Batch node agent SKU id matching the image OS (for example, `batch.node.ubuntu 24.04`).
+
+
 ##### `azure.batch.pools.<name>.virtualNetwork`
 
 <AddedInVersion version="23.04" />

plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzBatchService.groovy

@@ -51,6 +51,8 @@ import com.azure.compute.batch.models.BatchContainerConfiguration
 import com.azure.compute.batch.models.ContainerRegistryReference
 import com.azure.compute.batch.models.ContainerType
 import com.azure.compute.batch.models.ElevationLevel
+import com.azure.compute.batch.models.ImageVerificationType
+import com.azure.compute.batch.models.BatchVmImageReference
 import com.azure.compute.batch.models.BatchMetadataItem
 import com.azure.compute.batch.models.MountConfiguration
 import com.azure.compute.batch.models.NetworkConfiguration
@@ -704,16 +706,20 @@ class AzBatchService implements Closeable {
                 continue
             if( it.osType != opts.osType )
                 continue
-            if( it.verificationType != opts.verification )
+            if( !opts.allowUnverifiedImages && it.verificationType != ImageVerificationType.VERIFIED )
                 continue
             if( !it.imageReference.publisher.equalsIgnoreCase(opts.publisher) )
                 continue
             if( it.imageReference.offer.equalsIgnoreCase(opts.offer) )
                 return it
         }
 
-        log.debug "[AZURE BATCH] No VM image matching sku=$opts.sku; publisher=$opts.publisher; offer=$opts.offer; OS type=$opts.osType; verification type=$opts.verification - supported images: $available"
-        throw new IllegalStateException("Cannot find a matching VM image with publisher=$opts.publisher; offer=$opts.offer; OS type=$opts.osType; verification type=$opts.verification")
+        log.debug "[AZURE BATCH] No VM image matching sku=$opts.sku; publisher=$opts.publisher; offer=$opts.offer; OS type=$opts.osType; allow unverified images=${opts.allowUnverifiedImages} - supported images: $available"
+        throw new IllegalStateException("Cannot find a matching VM image with publisher=$opts.publisher; offer=$opts.offer; OS type=$opts.osType; allow unverified images=${opts.allowUnverifiedImages}")
+    }
+
+    protected BatchVmImageReference customImageReference(AzPoolOpts opts) {
+        return new BatchVmImageReference().setVirtualMachineImageId(opts.virtualMachineImageId)
     }
 
     protected AzVmPoolSpec specFromPoolConfig(String poolId) {
@@ -883,9 +889,20 @@ class AzBatchService implements Closeable {
             log.debug "[AZURE BATCH] Connecting Azure Batch pool to Container Registry '$registryOpts.server'"
         }
 
-        final image = getImage(opts)
+        final BatchVmImageReference imageRef
+        final String nodeAgentSkuId
+        if( opts.virtualMachineImageId ) {
+            imageRef = customImageReference(opts)
+            nodeAgentSkuId = opts.sku
+            log.debug "[AZURE BATCH] Using custom VM image from Compute Gallery: $opts.virtualMachineImageId (node agent SKU: $nodeAgentSkuId)"
+        }
+        else {
+            final image = getImage(opts)
+            imageRef = image.imageReference
+            nodeAgentSkuId = image.nodeAgentSkuId
+        }
 
-        new VirtualMachineConfiguration(image.imageReference, image.nodeAgentSkuId)
+        new VirtualMachineConfiguration(imageRef, nodeAgentSkuId)
                 .setContainerConfiguration(containerConfig)
     }
 

plugins/nf-azure/src/main/nextflow/cloud/azure/config/AzPoolOpts.groovy

@@ -16,7 +16,6 @@
 
 package nextflow.cloud.azure.config
 
-import com.azure.compute.batch.models.ImageVerificationType
 import com.azure.compute.batch.models.OSType
 import com.google.common.hash.Hasher
 import groovy.transform.CompileStatic
@@ -144,8 +143,21 @@ class AzPoolOpts implements CacheFunnel, ConfigScope {
     """)
     final String vmType
 
+    @ConfigOption
+    @Description("""
+        The resource ID of a custom VM image from an Azure Compute Gallery to use for the pool nodes
+        (e.g. `/subscriptions/<id>/resourceGroups/<group>/providers/Microsoft.Compute/galleries/<gallery>/images/<definition>/versions/<version>`).
+        When set, `publisher` and `offer` are ignored, and `sku` must be set to the Batch node agent SKU id that matches the image OS (e.g. `batch.node.ubuntu 24.04`).
+    """)
+    final String virtualMachineImageId
+
+    @ConfigOption
+    @Description("""
+        Allow the use of unverified VM images when resolving the image from the Batch supported-images list (default: `false`). Ignored when `virtualMachineImageId` is set.
+    """)
+    final boolean allowUnverifiedImages
+
     OSType osType = DEFAULT_OS_TYPE
-    ImageVerificationType verification = ImageVerificationType.VERIFIED
 
     String registry
     String userName
@@ -160,6 +172,10 @@ class AzPoolOpts implements CacheFunnel, ConfigScope {
         this.privileged = opts.privileged ?: false
         this.publisher = opts.publisher ?: DEFAULT_PUBLISHER
         this.offer = opts.offer ?: DEFAULT_OFFER
+        this.virtualMachineImageId = opts.virtualMachineImageId ?: null
+        this.allowUnverifiedImages = opts.allowUnverifiedImages as boolean
+        if( this.virtualMachineImageId && !opts.sku )
+            throw new IllegalArgumentException("Azure Batch pool option 'sku' is required when 'virtualMachineImageId' is set - it must be set to the Batch node agent SKU id that matches the image OS (e.g. 'batch.node.ubuntu 24.04')")
         this.sku = opts.sku ?: DEFAULT_SKU
         this.vmType = opts.vmType ?: DEFAULT_VM_TYPE
         this.fileShareRootPath = opts.fileShareRootPath ?: buildFileShareRootPath()
@@ -195,6 +211,10 @@ class AzPoolOpts implements CacheFunnel, ConfigScope {
         hasher.putUnencodedChars(schedulePolicy ?: '')
         hasher.putUnencodedChars(virtualNetwork ?: '')
         hasher.putBoolean(lowPriority)
+        hasher.putUnencodedChars(virtualMachineImageId ?: '')
+        // 'allowUnverifiedImages' only affects marketplace image resolution; it's ignored when a gallery image is set
+        if( !virtualMachineImageId && allowUnverifiedImages )
+            hasher.putBoolean(allowUnverifiedImages)
         hasher.putUnencodedChars(startTask.script ?: '')
         hasher.putBoolean(startTask.privileged)
         return hasher

plugins/nf-azure/src/test/nextflow/cloud/azure/config/AzPoolOptsTest.groovy

@@ -16,6 +16,8 @@
 
 package nextflow.cloud.azure.config
 
+import com.google.common.hash.Hashing
+import nextflow.util.CacheHelper
 import nextflow.util.Duration
 import spock.lang.Specification
 /**
@@ -48,6 +50,52 @@ class AzPoolOptsTest extends Specification {
         !opts.lowPriority
         !opts.startTask.script
         !opts.startTask.privileged
+        !opts.virtualMachineImageId
+        !opts.allowUnverifiedImages
+    }
+
+    def 'should configure a custom compute gallery image' () {
+        when:
+        def opts = new AzPoolOpts([
+            virtualMachineImageId: '/subscriptions/abc/resourceGroups/rg/providers/Microsoft.Compute/galleries/g/images/d/versions/1.0.0',
+            sku: 'batch.node.ubuntu 24.04',
+            allowUnverifiedImages: true,
+        ])
+        then:
+        opts.virtualMachineImageId == '/subscriptions/abc/resourceGroups/rg/providers/Microsoft.Compute/galleries/g/images/d/versions/1.0.0'
+        opts.sku == 'batch.node.ubuntu 24.04'
+        opts.allowUnverifiedImages
+    }
+
+    def 'should require sku for a compute gallery image' () {
+        when:
+        new AzPoolOpts([virtualMachineImageId: '/subscriptions/abc/resourceGroups/rg/providers/Microsoft.Compute/galleries/g/images/d/versions/1.0.0'])
+        then:
+        def e = thrown(IllegalArgumentException)
+        e.message.contains('sku')
+    }
+
+    private static String hash(AzPoolOpts opts) {
+        opts.funnel(Hashing.murmur3_128().newHasher(), CacheHelper.HashMode.STANDARD).hash().toString()
+    }
+
+    def 'pool hash should differ when image config differs' () {
+        given:
+        def base = new AzPoolOpts()
+        def gallery = new AzPoolOpts([virtualMachineImageId: '/subscriptions/x/resourceGroups/rg/providers/Microsoft.Compute/galleries/g/images/d/versions/1', sku: 'batch.node.ubuntu 24.04'])
+        def unverified = new AzPoolOpts([allowUnverifiedImages: true])
+        expect:
+        hash(base) != hash(gallery)
+        hash(base) != hash(unverified)
+    }
+
+    def 'pool hash should ignore allowUnverifiedImages for a gallery image' () {
+        given:
+        def opts = [virtualMachineImageId: '/subscriptions/x/resourceGroups/rg/providers/Microsoft.Compute/galleries/g/images/d/versions/1', sku: 'batch.node.ubuntu 24.04']
+        def a = new AzPoolOpts(opts)
+        def b = new AzPoolOpts(opts + [allowUnverifiedImages: true])
+        expect:
+        hash(a) == hash(b)
     }
 
     def 'should create pool with custom options' () {