diff --git a/docs/azure.mdx b/docs/azure.mdx index d931b16205..a2f627fa53 100644 --- a/docs/azure.mdx +++ b/docs/azure.mdx @@ -597,8 +597,45 @@ azure { } ``` +**Custom images from an Azure Compute Gallery** + +To provision pool nodes from a custom VM image published in an [Azure Compute Gallery](https://learn.microsoft.com/en-us/azure/virtual-machines/azure-compute-gallery), set `virtualMachineImageId` to the image version resource ID. When set, `publisher` and `offer` are ignored, and `sku` must be set to the Batch node agent SKU ID that matches the image operating system. + +```groovy +azure { + batch { + pools { + { + virtualMachineImageId = '/subscriptions//resourceGroups//providers/Microsoft.Compute/galleries//images//versions/' + sku = 'batch.node.ubuntu 24.04' + } + } + } +} +``` + +:::warning +Custom images require Microsoft Entra authentication (service principal or managed identity). The **Azure Batch account identity** must have read access to the gallery image. +::: + ### Advanced features +**Image verification** + +By default, Nextflow selects only images that Azure Batch has verified. Set `allowUnverifiedImages` to `true` to also allow images that Azure Batch lists but has not verified. Ignored when `virtualMachineImageId` is set. + +```groovy +azure { + batch { + pools { + { + allowUnverifiedImages = true + } + } + } +} +``` + **Virtual networks** Pools can be configured to use virtual networks to connect to your existing network infrastructure. diff --git a/docs/reference/config.mdx b/docs/reference/config.mdx index 5beece4846..e3362555c5 100644 --- a/docs/reference/config.mdx +++ b/docs/reference/config.mdx @@ -515,6 +515,12 @@ The name of the batch service region, e.g. `westeurope` or `eastus2`. Not needed The client ID for an Azure [managed identity](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/overview) that is available on all Azure Batch node pools. This identity is used by Fusion to authenticate to Azure storage. If set to `'auto'`, Fusion will use the first available managed identity. +##### `azure.batch.pools..allowUnverifiedImages` + + + +Allow the use of unverified VM images when resolving the image from the Batch supported-images list (default: `false`). Ignored when `virtualMachineImageId` is set. + ##### `azure.batch.pools..autoScale` Enable autoscaling feature for the pool identified with ``. @@ -583,6 +589,13 @@ Enable the `startTask` to run with elevated access (default`false`). The `startTask` that is executed as the node joins the Azure Batch node pool. +##### `azure.batch.pools..virtualMachineImageId` + + + +The resource ID of a custom VM image from an Azure Compute Gallery for the pool nodes (e.g., `/subscriptions//resourceGroups//providers/Microsoft.Compute/galleries//images//versions/`). When set, `publisher` and `offer` are ignored, and `sku` must be the Batch node agent SKU id matching the image OS (for example, `batch.node.ubuntu 24.04`). + + ##### `azure.batch.pools..virtualNetwork` diff --git a/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzBatchService.groovy b/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzBatchService.groovy index 5e40bb38dd..554b9f1f3a 100644 --- a/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzBatchService.groovy +++ b/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzBatchService.groovy @@ -51,6 +51,8 @@ import com.azure.compute.batch.models.BatchContainerConfiguration import com.azure.compute.batch.models.ContainerRegistryReference import com.azure.compute.batch.models.ContainerType import com.azure.compute.batch.models.ElevationLevel +import com.azure.compute.batch.models.ImageVerificationType +import com.azure.compute.batch.models.BatchVmImageReference import com.azure.compute.batch.models.BatchMetadataItem import com.azure.compute.batch.models.MountConfiguration import com.azure.compute.batch.models.NetworkConfiguration @@ -704,7 +706,7 @@ class AzBatchService implements Closeable { continue if( it.osType != opts.osType ) continue - if( it.verificationType != opts.verification ) + if( !opts.allowUnverifiedImages && it.verificationType != ImageVerificationType.VERIFIED ) continue if( !it.imageReference.publisher.equalsIgnoreCase(opts.publisher) ) continue @@ -712,8 +714,12 @@ class AzBatchService implements Closeable { return it } - log.debug "[AZURE BATCH] No VM image matching sku=$opts.sku; publisher=$opts.publisher; offer=$opts.offer; OS type=$opts.osType; verification type=$opts.verification - supported images: $available" - throw new IllegalStateException("Cannot find a matching VM image with publisher=$opts.publisher; offer=$opts.offer; OS type=$opts.osType; verification type=$opts.verification") + log.debug "[AZURE BATCH] No VM image matching sku=$opts.sku; publisher=$opts.publisher; offer=$opts.offer; OS type=$opts.osType; allow unverified images=${opts.allowUnverifiedImages} - supported images: $available" + throw new IllegalStateException("Cannot find a matching VM image with publisher=$opts.publisher; offer=$opts.offer; OS type=$opts.osType; allow unverified images=${opts.allowUnverifiedImages}") + } + + protected BatchVmImageReference customImageReference(AzPoolOpts opts) { + return new BatchVmImageReference().setVirtualMachineImageId(opts.virtualMachineImageId) } protected AzVmPoolSpec specFromPoolConfig(String poolId) { @@ -883,9 +889,20 @@ class AzBatchService implements Closeable { log.debug "[AZURE BATCH] Connecting Azure Batch pool to Container Registry '$registryOpts.server'" } - final image = getImage(opts) + final BatchVmImageReference imageRef + final String nodeAgentSkuId + if( opts.virtualMachineImageId ) { + imageRef = customImageReference(opts) + nodeAgentSkuId = opts.sku + log.debug "[AZURE BATCH] Using custom VM image from Compute Gallery: $opts.virtualMachineImageId (node agent SKU: $nodeAgentSkuId)" + } + else { + final image = getImage(opts) + imageRef = image.imageReference + nodeAgentSkuId = image.nodeAgentSkuId + } - new VirtualMachineConfiguration(image.imageReference, image.nodeAgentSkuId) + new VirtualMachineConfiguration(imageRef, nodeAgentSkuId) .setContainerConfiguration(containerConfig) } diff --git a/plugins/nf-azure/src/main/nextflow/cloud/azure/config/AzPoolOpts.groovy b/plugins/nf-azure/src/main/nextflow/cloud/azure/config/AzPoolOpts.groovy index 9714374481..3b21a69e2b 100644 --- a/plugins/nf-azure/src/main/nextflow/cloud/azure/config/AzPoolOpts.groovy +++ b/plugins/nf-azure/src/main/nextflow/cloud/azure/config/AzPoolOpts.groovy @@ -16,7 +16,6 @@ package nextflow.cloud.azure.config -import com.azure.compute.batch.models.ImageVerificationType import com.azure.compute.batch.models.OSType import com.google.common.hash.Hasher import groovy.transform.CompileStatic @@ -144,8 +143,21 @@ class AzPoolOpts implements CacheFunnel, ConfigScope { """) final String vmType + @ConfigOption + @Description(""" + The resource ID of a custom VM image from an Azure Compute Gallery to use for the pool nodes + (e.g. `/subscriptions//resourceGroups//providers/Microsoft.Compute/galleries//images//versions/`). + When set, `publisher` and `offer` are ignored, and `sku` must be set to the Batch node agent SKU id that matches the image OS (e.g. `batch.node.ubuntu 24.04`). + """) + final String virtualMachineImageId + + @ConfigOption + @Description(""" + Allow the use of unverified VM images when resolving the image from the Batch supported-images list (default: `false`). Ignored when `virtualMachineImageId` is set. + """) + final boolean allowUnverifiedImages + OSType osType = DEFAULT_OS_TYPE - ImageVerificationType verification = ImageVerificationType.VERIFIED String registry String userName @@ -160,6 +172,10 @@ class AzPoolOpts implements CacheFunnel, ConfigScope { this.privileged = opts.privileged ?: false this.publisher = opts.publisher ?: DEFAULT_PUBLISHER this.offer = opts.offer ?: DEFAULT_OFFER + this.virtualMachineImageId = opts.virtualMachineImageId ?: null + this.allowUnverifiedImages = opts.allowUnverifiedImages as boolean + if( this.virtualMachineImageId && !opts.sku ) + throw new IllegalArgumentException("Azure Batch pool option 'sku' is required when 'virtualMachineImageId' is set - it must be set to the Batch node agent SKU id that matches the image OS (e.g. 'batch.node.ubuntu 24.04')") this.sku = opts.sku ?: DEFAULT_SKU this.vmType = opts.vmType ?: DEFAULT_VM_TYPE this.fileShareRootPath = opts.fileShareRootPath ?: buildFileShareRootPath() @@ -195,6 +211,12 @@ class AzPoolOpts implements CacheFunnel, ConfigScope { hasher.putUnencodedChars(schedulePolicy ?: '') hasher.putUnencodedChars(virtualNetwork ?: '') hasher.putBoolean(lowPriority) + hasher.putUnencodedChars(virtualMachineImageId ?: '') + // 'allowUnverifiedImages' only affects marketplace image resolution; it's ignored when a gallery image is set. + // NOTE: only hashed when set, so that default configs keep the same hash as previous Nextflow versions + // (no auto-pool-id churn on upgrade) - do not change to an unconditional putBoolean + if( !virtualMachineImageId && allowUnverifiedImages ) + hasher.putBoolean(allowUnverifiedImages) hasher.putUnencodedChars(startTask.script ?: '') hasher.putBoolean(startTask.privileged) return hasher diff --git a/plugins/nf-azure/src/test/nextflow/cloud/azure/config/AzPoolOptsTest.groovy b/plugins/nf-azure/src/test/nextflow/cloud/azure/config/AzPoolOptsTest.groovy index f47d8d673a..6abec7eabd 100644 --- a/plugins/nf-azure/src/test/nextflow/cloud/azure/config/AzPoolOptsTest.groovy +++ b/plugins/nf-azure/src/test/nextflow/cloud/azure/config/AzPoolOptsTest.groovy @@ -16,6 +16,8 @@ package nextflow.cloud.azure.config +import com.google.common.hash.Hashing +import nextflow.util.CacheHelper import nextflow.util.Duration import spock.lang.Specification /** @@ -48,6 +50,52 @@ class AzPoolOptsTest extends Specification { !opts.lowPriority !opts.startTask.script !opts.startTask.privileged + !opts.virtualMachineImageId + !opts.allowUnverifiedImages + } + + def 'should configure a custom compute gallery image' () { + when: + def opts = new AzPoolOpts([ + virtualMachineImageId: '/subscriptions/abc/resourceGroups/rg/providers/Microsoft.Compute/galleries/g/images/d/versions/1.0.0', + sku: 'batch.node.ubuntu 24.04', + allowUnverifiedImages: true, + ]) + then: + opts.virtualMachineImageId == '/subscriptions/abc/resourceGroups/rg/providers/Microsoft.Compute/galleries/g/images/d/versions/1.0.0' + opts.sku == 'batch.node.ubuntu 24.04' + opts.allowUnverifiedImages + } + + def 'should require sku for a compute gallery image' () { + when: + new AzPoolOpts([virtualMachineImageId: '/subscriptions/abc/resourceGroups/rg/providers/Microsoft.Compute/galleries/g/images/d/versions/1.0.0']) + then: + def e = thrown(IllegalArgumentException) + e.message.contains('sku') + } + + private static String hash(AzPoolOpts opts) { + opts.funnel(Hashing.murmur3_128().newHasher(), CacheHelper.HashMode.STANDARD).hash().toString() + } + + def 'pool hash should differ when image config differs' () { + given: + def base = new AzPoolOpts() + def gallery = new AzPoolOpts([virtualMachineImageId: '/subscriptions/x/resourceGroups/rg/providers/Microsoft.Compute/galleries/g/images/d/versions/1', sku: 'batch.node.ubuntu 24.04']) + def unverified = new AzPoolOpts([allowUnverifiedImages: true]) + expect: + hash(base) != hash(gallery) + hash(base) != hash(unverified) + } + + def 'pool hash should ignore allowUnverifiedImages for a gallery image' () { + given: + def opts = [virtualMachineImageId: '/subscriptions/x/resourceGroups/rg/providers/Microsoft.Compute/galleries/g/images/d/versions/1', sku: 'batch.node.ubuntu 24.04'] + def a = new AzPoolOpts(opts) + def b = new AzPoolOpts(opts + [allowUnverifiedImages: true]) + expect: + hash(a) == hash(b) } def 'should create pool with custom options' () {