Skip to content

Commit a6093f7

Browse files
pditommasoclaude
pditommaso
and
claude
committed
Add hints process directive for executor-specific scheduling hints
Introduce a new map-type `hints` process directive that provides a structured, extensible way for pipeline authors to pass executor-specific scheduling hints. Keys use `[executor/][scope.]hintName` format. Core features: - Multiple `hints` calls within a process body accumulate (merge) - Config overrides via withName:/withLabel: replace the entire map - Values support String, Integer, and Closure types - Two-tier validation: warnings for unknown unprefixed keys (global registry), errors for unknown executor-prefixed keys - Initial global catalog contains only `consumableResource` Seqera Platform integration: - `seqera/machineRequirement.*` hints map to MachineRequirementOpts fields (arch, provisioning, maxSpotAttempts, machineTypes, diskType, diskThroughputMiBps, diskIops, diskEncrypted, diskAllocation, diskMountPath, diskSize, capacityMode) - Hints override Seqera config scope values at the task level - Unknown `seqera/` keys produce an error Ref: #5917, #6960 Signed-off-by: Paolo Di Tommaso <paolo.ditommaso@gmail.com> Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> Signed-off-by: Paolo Di Tommaso <paolo.ditommaso@gmail.com>
1 parent 87181e3 commit a6093f7

11 files changed

Lines changed: 656 additions & 1 deletion

File tree

‎modules/nextflow/src/main/groovy/nextflow/processor/HintDefs.groovy‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
/*
2+
* Copyright 2013-2026, Seqera Labs
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package nextflow.processor
18+
19+
import groovy.transform.CompileStatic
20+
import groovy.util.logging.Slf4j
21+
22+
/**
23+
* Defines the global hint key registry and provides validation
24+
* for the {@code hints} process directive.
25+
*
26+
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
27+
*/
28+
@Slf4j
29+
@CompileStatic
30+
class HintDefs {
31+
32+
/**
33+
* Global registry of known unprefixed hint keys and their expected value types.
34+
* Executor-prefixed keys (e.g. {@code seqera/machineRequirement.arch}) are
35+
* validated by the target executor, not this registry.
36+
*/
37+
static final Map<String, Class> KNOWN_HINTS = [
38+
'consumableResource': String
39+
]
40+
41+
/**
42+
* Validates the given hints map against the global registry.
43+
* <p>
44+
* Unprefixed keys are checked against {@link #KNOWN_HINTS}:
45+
* <ul>
46+
* <li>Unknown keys produce a warning with a "did you mean?" suggestion if a close match exists</li>
47+
* <li>Value types are validated (must resolve to String or Integer)</li>
48+
* </ul>
49+
* Executor-prefixed keys (containing {@code /}) are skipped — they are validated by the target executor.
50+
*
51+
* @param hints the resolved hints map
52+
*/
53+
static void validateHints(Map<String, Object> hints) {
54+
if( !hints )
55+
return
56+
57+
for( Map.Entry<String, Object> entry : hints.entrySet() ) {
58+
final key = entry.key
59+
final value = entry.value
60+
61+
// skip executor-prefixed keys — validated by the executor
62+
if( key.contains('/') )
63+
continue
64+
65+
// validate value type
66+
if( value != null && !(value instanceof String) && !(value instanceof Integer) ) {
67+
throw new IllegalArgumentException("Invalid hint value type for key '${key}': expected String or Integer, got ${value.getClass().getName()}")
68+
}
69+
70+
// validate key against registry
71+
if( !KNOWN_HINTS.containsKey(key) ) {
72+
final suggestions = KNOWN_HINTS.keySet().toList().closest(key)
73+
if( suggestions ) {
74+
log.warn "Unknown process hint: '${key}' — did you mean '${suggestions.first()}'?"
75+
}
76+
else {
77+
log.warn "Unknown process hint: '${key}'"
78+
}
79+
}
80+
}
81+
}
82+
83+
}

‎modules/nextflow/src/main/groovy/nextflow/processor/TaskConfig.groovy‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -529,6 +529,10 @@ class TaskConfig extends LazyMap implements Cloneable {
529529
return CmdLineOptionMap.emptyOption()
530530
}
531531

532+
Map<String, Object> getHints() {
533+
return get('hints') as Map<String, Object> ?: Collections.<String,Object>emptyMap()
534+
}
535+
532536
Map<String, String> getResourceLabels() {
533537
return get('resourceLabels') as Map<String, String> ?: Collections.<String,String>emptyMap()
534538
}

‎modules/nextflow/src/main/groovy/nextflow/script/ProcessConfig.groovy‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -173,6 +173,10 @@ class ProcessConfig implements Map<String,Object>, Cloneable {
173173
HashMode.of(configProperties.cache) ?: HashMode.DEFAULT()
174174
}
175175

176+
Map<String,Object> getHints() {
177+
(configProperties.get('hints') ?: Collections.emptyMap()) as Map<String, Object>
178+
}
179+
176180
Map<String,Object> getResourceLabels() {
177181
(configProperties.get('resourceLabels') ?: Collections.emptyMap()) as Map<String, Object>
178182
}

‎modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessBuilder.groovy‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ class ProcessBuilder {
5959
'executor',
6060
'ext',
6161
'fair',
62+
'hints',
6263
'label',
6364
'machineType',
6465
'maxErrors',
@@ -377,6 +378,25 @@ class ProcessBuilder {
377378
config.put('resourceLabels', allLabels)
378379
}
379380

381+
/**
382+
* Implements the {@code hints} directive.
383+
*
384+
* This directive can be specified (invoked) multiple times in
385+
* the process definition. Multiple calls accumulate entries.
386+
*
387+
* @param map
388+
*/
389+
void hints(Map<String, Object> map) {
390+
if( !map ) return
391+
392+
def allHints = (Map)config.get('hints')
393+
if( !allHints ) {
394+
allHints = [:]
395+
}
396+
allHints += map
397+
config.put('hints', allHints)
398+
}
399+
380400
private static final List<String> VALID_RESOURCE_LIMITS = List.of('cpus', 'memory', 'disk', 'time')
381401

382402
/**

‎modules/nextflow/src/test/groovy/nextflow/processor/HintDefsTest.groovy‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
/*
2+
* Copyright 2013-2026, Seqera Labs
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package nextflow.processor
18+
19+
import spock.lang.Specification
20+
21+
/**
22+
* Tests for {@link HintDefs}
23+
*
24+
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
25+
*/
26+
class HintDefsTest extends Specification {
27+
28+
def 'should accept known hint key'() {
29+
when:
30+
HintDefs.validateHints([consumableResource: 'my-license'])
31+
then:
32+
noExceptionThrown()
33+
}
34+
35+
def 'should warn on unknown key with close match'() {
36+
// consumableResourc is close to consumableResource
37+
when:
38+
HintDefs.validateHints([consumableResourc: 'my-license'])
39+
then:
40+
noExceptionThrown()
41+
// warning is logged — verified by log output in integration tests
42+
}
43+
44+
def 'should warn on unknown key with no close match'() {
45+
when:
46+
HintDefs.validateHints([somethingRandom: 'value'])
47+
then:
48+
noExceptionThrown()
49+
}
50+
51+
def 'should reject invalid value type'() {
52+
when:
53+
HintDefs.validateHints([consumableResource: ['a', 'b']])
54+
then:
55+
def e = thrown(IllegalArgumentException)
56+
e.message.contains('Invalid hint value type')
57+
e.message.contains('consumableResource')
58+
}
59+
60+
def 'should accept string and integer values'() {
61+
when:
62+
HintDefs.validateHints([consumableResource: 'my-license', 'scheduling.priority': 10])
63+
then:
64+
noExceptionThrown()
65+
}
66+
67+
def 'should skip executor-prefixed keys'() {
68+
when:
69+
HintDefs.validateHints(['seqera/machineRequirement.arch': 'arm64', 'seqera/unknownKey': 'value'])
70+
then:
71+
noExceptionThrown()
72+
}
73+
74+
def 'should handle null and empty maps'() {
75+
when:
76+
HintDefs.validateHints(null)
77+
then:
78+
noExceptionThrown()
79+
80+
when:
81+
HintDefs.validateHints([:])
82+
then:
83+
noExceptionThrown()
84+
}
85+
86+
def 'should accept null hint value'() {
87+
when:
88+
HintDefs.validateHints([consumableResource: null])
89+
then:
90+
noExceptionThrown()
91+
}
92+
93+
}

‎modules/nextflow/src/test/groovy/nextflow/processor/TaskConfigTest.groovy‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -622,6 +622,49 @@ class TaskConfigTest extends Specification {
622622
config.getResourceLabelsAsString() == 'region=eu-west-1,organization=A,user=this,team=that'
623623
}
624624

625+
def 'should configure hints options'() {
626+
given:
627+
def script = Mock(BaseScript)
628+
629+
when:
630+
def process = new ProcessConfig(script)
631+
def dsl = new ProcessBuilder(process)
632+
dsl.hints( 'seqera/machineRequirement.arch': 'arm64', consumableResource: 'my-license' )
633+
634+
then:
635+
process.get('hints') == ['seqera/machineRequirement.arch': 'arm64', consumableResource: 'my-license']
636+
637+
when:
638+
def config = process.createTaskConfig()
639+
then:
640+
config.getHints() == ['seqera/machineRequirement.arch': 'arm64', consumableResource: 'my-license']
641+
}
642+
643+
def 'should return empty map when no hints set'() {
644+
when:
645+
def config = new TaskConfig([:])
646+
then:
647+
config.getHints() == [:]
648+
}
649+
650+
def 'should replace hints via config override'() {
651+
given:
652+
def script = Mock(BaseScript)
653+
654+
when: 'set hints in process definition'
655+
def process = new ProcessConfig(script)
656+
def dsl = new ProcessBuilder(process)
657+
dsl.hints( 'seqera/machineRequirement.arch': 'arm64', consumableResource: 'my-license' )
658+
then:
659+
process.getHints() == ['seqera/machineRequirement.arch': 'arm64', consumableResource: 'my-license']
660+
661+
when: 'config override replaces the entire map'
662+
def config = process.createTaskConfig()
663+
config.put('hints', ['scheduling.priority': 5])
664+
then:
665+
config.getHints() == ['scheduling.priority': 5]
666+
}
667+
625668
def 'should report error on negative cpus' () {
626669
when:
627670
def config = new TaskConfig([cpus:-1])

‎modules/nextflow/src/test/groovy/nextflow/script/dsl/ProcessBuilderTest.groovy‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -171,6 +171,41 @@ class ProcessBuilderTest extends Specification {
171171

172172
}
173173

174+
def 'should apply hints config' () {
175+
given:
176+
def builder = createBuilder()
177+
def config = builder.getConfig()
178+
expect:
179+
config.getHints() == [:]
180+
181+
when:
182+
builder.hints 'seqera/machineRequirement.arch': 'arm64'
183+
then:
184+
config.getHints() == ['seqera/machineRequirement.arch': 'arm64']
185+
186+
when:
187+
builder.hints 'seqera/machineRequirement.provisioning': 'spot', 'seqera/machineRequirement.maxSpotAttempts': 3
188+
then:
189+
config.getHints() == ['seqera/machineRequirement.arch': 'arm64', 'seqera/machineRequirement.provisioning': 'spot', 'seqera/machineRequirement.maxSpotAttempts': 3]
190+
191+
when: 'duplicate key overwrites'
192+
builder.hints 'seqera/machineRequirement.arch': 'x86_64'
193+
then:
194+
config.getHints() == ['seqera/machineRequirement.arch': 'x86_64', 'seqera/machineRequirement.provisioning': 'spot', 'seqera/machineRequirement.maxSpotAttempts': 3]
195+
}
196+
197+
def 'should store closure values in hints' () {
198+
given:
199+
def builder = createBuilder()
200+
def config = builder.getConfig()
201+
202+
when:
203+
def closure = { 'spot' }
204+
builder.hints 'seqera/machineRequirement.provisioning': closure
205+
then:
206+
config.getHints()['seqera/machineRequirement.provisioning'].is(closure)
207+
}
208+
174209
def 'should check a valid label' () {
175210

176211
expect:

‎modules/nf-lang/src/main/java/nextflow/script/types/TaskConfig.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -320,6 +320,14 @@ public interface TaskConfig {
320320
""")
321321
String getQueue();
322322

323+
@Constant("hints")
324+
@Description("""
325+
The `hints` directive allows you to specify executor-specific scheduling hints as key-value pairs.
326+
327+
[Read more](https://nextflow.io/docs/latest/reference/process.html#hints)
328+
""")
329+
Map<String,Object> getHints();
330+
323331
@Constant("resourceLabels")
324332
@Description("""
325333
The `resourceLabels` directive allows you to specify custom name-value pairs which are applied to the compute resources used for the process execution.

‎plugins/nf-seqera/src/main/io/seqera/executor/SeqeraTaskHandler.groovy‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import io.seqera.sched.api.schema.v1a1.Task
3030
import io.seqera.sched.api.schema.v1a1.TaskState as SchedTaskState
3131
import io.seqera.sched.api.schema.v1a1.TaskStatus as SchedTaskStatus
3232
import io.seqera.sched.client.SchedClient
33+
import io.seqera.util.HintHelper
3334
import io.seqera.util.SchemaMapperUtil
3435
import nextflow.cloud.types.CloudMachineInfo
3536
import nextflow.exception.ProcessException
@@ -113,8 +114,13 @@ class SeqeraTaskHandler extends TaskHandler implements FusionAwareTask {
113114
resourceReq.acceleratorName(accelerator.type)
114115
}
115116
// build machine requirement merging config settings with task arch, disk, and snapshot settings
116-
final machineReq = SchemaMapperUtil.toMachineRequirement(
117+
// overlay any seqera/machineRequirement.* hints on top of config-scope values (hints win)
118+
final baseMachineOpts = HintHelper.overlayHints(
117119
executor.getSeqeraConfig().machineRequirement,
120+
task.config.getHints()
121+
)
122+
final machineReq = SchemaMapperUtil.toMachineRequirement(
123+
baseMachineOpts,
118124
task.getContainerPlatform(),
119125
task.config.getDisk(),
120126
fusionConfig().snapshotsEnabled()

0 commit comments

Comments
 (0)