diff --git a/docs/modules/module-registry.md b/docs/modules/module-registry.md index 5cdefdd6c8..b8a2d5bbb3 100644 --- a/docs/modules/module-registry.md +++ b/docs/modules/module-registry.md @@ -130,6 +130,9 @@ See {ref}`module publish ` for the full command reference. By default, Nextflow uses the public registry at `https://registry.nextflow.io`. Configure alternative or additional registries using the `registry` scope in your Nextflow configuration. +See the {ref}`registry config scope ` for the full reference. + +The registries defined in the `registry` scope are authoritative: when one or more are configured, Nextflow queries exactly those, in the order they are listed, and the public registry is *not* added implicitly. To keep using the public registry alongside your own, include its URL in the list explicitly. ### Use a private registry @@ -142,16 +145,17 @@ registry { } ``` -Nextflow uses this registry for all module operations (search, install, and publish). +In this example, Nextflow uses `https://registry.myorg.com` for all module operations (search, install, and publish), and does not query the public registry. ### Use multiple registries Specify multiple registries as a list. -Nextflow queries them in order when searching for or installing a module: +Nextflow queries them in the order listed: ```groovy registry { url = [ + 'https://registry.nextflow.io', 'https://registry.myorg.com', 'https://private.registry.nextflow.io' ] @@ -159,7 +163,7 @@ registry { } ``` -In this example, Nextflow searches the private registry first and falls back to the public registry. +In this example, Nextflow queries `https://registry.nextflow.io` first, then `https://registry.myorg.com`, and finally `https://private.registry.nextflow.io`. The public registry is queried only because it is listed explicitly. :::{note} The `apiKey` authenticates with the primary (first) registry. diff --git a/docs/plugins/plugin-registry.md b/docs/plugins/plugin-registry.md index ef1c691e0d..3903eca5ed 100644 --- a/docs/plugins/plugin-registry.md +++ b/docs/plugins/plugin-registry.md @@ -63,3 +63,33 @@ To create an API access token: 6. Copy and past token somewhere safe, you won't be able to see it again. Once you have your token, see {ref}`gradle-plugin-publish` for instructions on how to use it. + +(plugin-registry-config)= + +## Configuring plugin registries + +:::{versionadded} 26.04.0 +::: + +By default, Nextflow resolves plugins from the public registry at `https://registry.nextflow.io`. +Configure the plugin registries using the `registry` scope in your Nextflow configuration: + +```groovy +registry { + url = 'https://registry.myorg.com' +} +``` + +The registries defined in `registry.url` are authoritative: when one or more are configured, Nextflow resolves plugins from exactly those, in the order they are listed, and the public registry is *not* added implicitly. To keep resolving plugins from the public registry as well, include its URL (`https://registry.nextflow.io/api`) in the list explicitly. + +See the {ref}`registry config scope ` for the full reference. + +Alternatively, to override the default registry without using the `registry` scope, set the `NXF_PLUGINS_REGISTRY_URL` environment variable. + +:::{note} +Both `registry.url` and `NXF_PLUGINS_REGISTRY_URL` override the default plugin registry. When both are set, `registry.url` takes precedence. Unlike `registry.url` — which configures the registries for both modules and plugins — `NXF_PLUGINS_REGISTRY_URL` applies to plugin resolution only. +::: + +:::{warning} +The `registry` scope is only applied once the configuration has been resolved. Plugins required *before* that point — for example a filesystem provider for a remote pipeline script or an `includeConfig` location (such as `s3://`), or an SCM provider — are resolved against the default plugin registry (or `NXF_PLUGINS_REGISTRY_URL`). As a result, the default registry may still be fetched even when it is not listed in `registry.url`. Plugins needed at this early stage must be available in the default registry, set via `NXF_PLUGINS_REGISTRY_URL`, or pre-installed. +::: diff --git a/docs/reference/config.md b/docs/reference/config.md index 9e91bc7f6a..395c5e69ed 100644 --- a/docs/reference/config.md +++ b/docs/reference/config.md @@ -1401,6 +1401,25 @@ The following settings are available: `report.overwrite` : Overwrite any existing report file with the same name (default: `false`). +(config-registry)= + +## `registry` + +:::{versionadded} 26.04.0 +::: + +The `registry` scope configures the registries used to resolve {ref}`modules ` and {ref}`plugins `. + +The registries defined in this scope are authoritative: when one or more are configured, Nextflow queries exactly those, in the order they are listed, and the public Nextflow registry (`https://registry.nextflow.io/api`) is *not* added implicitly. When the scope is not configured, the public registry is used. To keep using the public registry alongside your own, include its URL in `registry.url` explicitly. + +The following settings are available: + +`registry.apiKey` +: The API key used to authenticate with the registry. When not set, the `NXF_REGISTRY_TOKEN` environment variable is used. + +`registry.url` +: A registry URL, or a list of registry URLs queried in the order they are listed. When set, these replace the default public registry. + (config-sarus)= ## `sarus` diff --git a/modules/nextflow/src/main/groovy/nextflow/module/RegistryClientFactory.groovy b/modules/nextflow/src/main/groovy/nextflow/module/RegistryClientFactory.groovy index 581cdfce21..4d2d39fd90 100644 --- a/modules/nextflow/src/main/groovy/nextflow/module/RegistryClientFactory.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/module/RegistryClientFactory.groovy @@ -36,6 +36,8 @@ class RegistryClientFactory { static RegistryClient forConfig(RegistryConfig config) { final cfg = config ?: new RegistryConfig() + // the configured registries are authoritative: use exactly the URLs declared in the + // `registry` scope (which falls back to the default registry when none are configured) return new RegistryClient( cfg.allUrls as List, cfg.apiKey, diff --git a/modules/nextflow/src/test/groovy/nextflow/module/RegistryClientFactoryTest.groovy b/modules/nextflow/src/test/groovy/nextflow/module/RegistryClientFactoryTest.groovy index 2a6d00e7cb..6b49c34730 100644 --- a/modules/nextflow/src/test/groovy/nextflow/module/RegistryClientFactoryTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/module/RegistryClientFactoryTest.groovy @@ -94,6 +94,72 @@ class RegistryClientFactoryTest extends Specification { verify(getRequestedFor(urlEqualTo(MODULES_API_PATH + '/nf-core%2Ffastqc'))) } + def 'should query the configured registries in the order listed'() { + given: + def moduleResponse = [ + module: [ + name: 'nf-core/fastqc', + latest: [ + version: '1.0.0', + createdAt: '2024-02-01T00:00:00Z' + ] + ] + ] + and: 'two registries are configured; the first misses and the second serves the module' + String first = "http://localhost:${wireMock.port()}/first" + String second = "http://localhost:${wireMock.port()}/second" + stubFor(get(urlEqualTo('/first/v1/modules/nf-core%2Ffastqc')) + .willReturn(aResponse().withStatus(404).withBody('Module not found'))) + stubFor(get(urlEqualTo('/second/v1/modules/nf-core%2Ffastqc')) + .willReturn(aResponse() + .withStatus(200) + .withHeader('Content-Type', 'application/json') + .withBody(JsonOutput.toJson(moduleResponse)))) + and: + def config = new RegistryConfig([url: [first, second]]) + def client = RegistryClientFactory.forConfig(config) + + when: + def result = client.getModule('nf-core/fastqc') + + then: 'the module is resolved from the second registry after the first misses' + result != null + result.name == 'nf-core/fastqc' + + and: 'only the configured registries are queried, in order' + verify(getRequestedFor(urlEqualTo('/first/v1/modules/nf-core%2Ffastqc'))) + verify(getRequestedFor(urlEqualTo('/second/v1/modules/nf-core%2Ffastqc'))) + } + + def 'should query only the configured registry, without a default fallback'() { + given: + def moduleResponse = [ + module: [ + name: 'nf-core/fastqc', + latest: [ + version: '1.0.0', + createdAt: '2024-02-01T00:00:00Z' + ] + ] + ] + stubFor(get(urlEqualTo(MODULES_API_PATH + '/nf-core%2Ffastqc')) + .willReturn(aResponse() + .withStatus(200) + .withHeader('Content-Type', 'application/json') + .withBody(JsonOutput.toJson(moduleResponse)))) + and: + def config = new RegistryConfig([url: url]) + def client = RegistryClientFactory.forConfig(config) + + when: + def result = client.getModule('nf-core/fastqc') + + then: + result.name == 'nf-core/fastqc' + and: 'the registry is queried exactly once (no default fallback appended)' + verify(1, getRequestedFor(urlEqualTo(MODULES_API_PATH + '/nf-core%2Ffastqc'))) + } + def 'should search modules in registry'() { given: def searchResponse = [ diff --git a/modules/nf-commons/src/main/nextflow/plugin/HttpPluginRepository.groovy b/modules/nf-commons/src/main/nextflow/plugin/HttpPluginRepository.groovy index fd21cbd06b..6234d7676f 100644 --- a/modules/nf-commons/src/main/nextflow/plugin/HttpPluginRepository.groovy +++ b/modules/nf-commons/src/main/nextflow/plugin/HttpPluginRepository.groovy @@ -120,6 +120,10 @@ class HttpPluginRepository implements PrefetchUpdateRepository { @Override void refresh() { + // nothing to refresh when no metadata was prefetched (null or empty): avoid a + // pointless registry round-trip. In Groovy an empty or null map is falsy. + if( !plugins ) + return plugins = fetchMetadataByIds(plugins.keySet()) } diff --git a/modules/nf-commons/src/main/nextflow/plugin/PluginUpdater.groovy b/modules/nf-commons/src/main/nextflow/plugin/PluginUpdater.groovy index e261563ed1..9c6efeccfc 100644 --- a/modules/nf-commons/src/main/nextflow/plugin/PluginUpdater.groovy +++ b/modules/nf-commons/src/main/nextflow/plugin/PluginUpdater.groovy @@ -32,6 +32,7 @@ import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import nextflow.BuildInfo import nextflow.SysEnv +import nextflow.config.RegistryConfig import nextflow.extension.FilesEx import nextflow.file.FileHelper import nextflow.file.FileMutex @@ -100,6 +101,44 @@ class PluginUpdater extends UpdateManager { return result } + /** Ids of the default registry repositories created by {@link #wrap} */ + private static final List DEFAULT_REGISTRY_REPO_IDS = ['registry', 'nextflow.io'] + + /** + * Apply the plugin registry endpoints declared in the {@code registry} config scope. + * + * The configured registries are authoritative: the URLs provided by {@link RegistryConfig} + * fully replace the default registry repository. Users that want to keep resolving plugins + * from the public registry must include its URL explicitly in {@code registry.url}. + * + * Safe to call after construction and before {@link #prefetchMetadata}, which initialises + * each {@link HttpPluginRepository} with the metadata it actually needs. + */ + void addRegistryRepos(RegistryConfig registryConfig) { + if( offline || !registryConfig ) + return + final urls = registryConfig.getAllUrls() + if( !urls ) + return + // the configured registries take over: drop the default registry repository so that + // only the user-provided endpoints are queried + for( String id : DEFAULT_REGISTRY_REPO_IDS ) + removeRepository(id) + final existingIds = new HashSet() + for( UpdateRepository repo : this.@repositories ) + existingIds.add(repo.id) + int counter = 0 + for( String url : urls ) { + String repoId = "registry-${counter++}" + while( repoId in existingIds ) + repoId = "registry-${counter++}" + existingIds.add(repoId) + final repo = new HttpPluginRepository(repoId, URI.create(url)) + log.debug "Adding plugin repository: ${repo.getClass().getSimpleName()} [${repo.id}]; url=${url}" + addRepository(repo) + } + } + static private List customRepos() { final repos = SysEnv.get('NXF_PLUGINS_TEST_REPOSITORY') if( !repos ) diff --git a/modules/nf-commons/src/main/nextflow/plugin/PluginsFacade.groovy b/modules/nf-commons/src/main/nextflow/plugin/PluginsFacade.groovy index 2dfa96e089..a68beaa479 100644 --- a/modules/nf-commons/src/main/nextflow/plugin/PluginsFacade.groovy +++ b/modules/nf-commons/src/main/nextflow/plugin/PluginsFacade.groovy @@ -24,6 +24,7 @@ import groovy.transform.Memoized import groovy.transform.PackageScope import groovy.util.logging.Slf4j import nextflow.SysEnv +import nextflow.config.RegistryConfig import nextflow.exception.AbortOperationException import nextflow.extension.Bolts import nextflow.extension.FilesEx @@ -57,6 +58,7 @@ class PluginsFacade implements PluginStateListener { private CustomPluginManager manager private DefaultPlugins defaultPlugins = DefaultPlugins.INSTANCE private String indexUrl + private RegistryConfig registryConfig private boolean embedded PluginsFacade() { @@ -305,9 +307,20 @@ class PluginsFacade implements PluginStateListener { void load(Map config) { if( !manager ) throw new IllegalArgumentException("Plugin system has not been initialized") + applyRegistryConfig(config) start(pluginsRequirement(config)) } + protected void applyRegistryConfig(Map config) { + final registryMap = Bolts.navigate(config, 'registry') as Map + // only override the plugin repositories when one or more registry URLs are explicitly + // configured; the configured registries are authoritative and replace the default one + if( !registryMap?.url ) + return + this.registryConfig = new RegistryConfig(registryMap) + updater?.addRegistryRepos(registryConfig) + } + synchronized void stop() { if( manager ) { manager.stopPlugins() diff --git a/modules/nf-commons/src/test/nextflow/plugin/PluginUpdaterTest.groovy b/modules/nf-commons/src/test/nextflow/plugin/PluginUpdaterTest.groovy index bb69e588f1..b45c940578 100644 --- a/modules/nf-commons/src/test/nextflow/plugin/PluginUpdaterTest.groovy +++ b/modules/nf-commons/src/test/nextflow/plugin/PluginUpdaterTest.groovy @@ -27,6 +27,7 @@ import java.util.zip.ZipEntry import java.util.zip.ZipOutputStream import nextflow.BuildInfo +import nextflow.config.RegistryConfig import com.github.zafarkhaja.semver.Version import org.pf4j.Plugin import org.pf4j.PluginDescriptor @@ -469,6 +470,99 @@ class PluginUpdaterTest extends Specification { 'xpack-google-1.0.0-beta.3-meta.json' | true | 'xpack-google' } + def 'should replace the default registry with the configured registries' () { + given: + def folder = Files.createTempDirectory('test') + def remote = remoteRepository(folder.resolve('repo'), ['1.0.0']) + def local = localCache(folder.resolve('plugins'), []) + def manager = new LocalPluginManager(local) + def updater = new PluginUpdater(manager, local, remote, false) + and: + def cfg = new RegistryConfig([url: ['https://reg-a.example/api', 'https://reg-b.example/api']]) + + when: + updater.addRegistryRepos(cfg) + def repos = updater.getRepositories() + + then: 'the default registry repo is dropped; only the configured registries remain, in order' + repos.size() == 2 + repos[0] instanceof HttpPluginRepository + repos[0].id == 'registry-0' + repos[0].url.toString().startsWith('https://reg-a.example/api') + and: + repos[1] instanceof HttpPluginRepository + repos[1].id == 'registry-1' + repos[1].url.toString().startsWith('https://reg-b.example/api') + + cleanup: + folder?.deleteDir() + } + + def 'should replace the default registry even when a configured url matches it' () { + given: + def folder = Files.createTempDirectory('test') + def local = localCache(folder.resolve('plugins'), []) + def manager = new LocalPluginManager(local) + def remote = new URL('http://primary.example/api') + def updater = new PluginUpdater(manager, local, remote, false) + and: + def cfg = new RegistryConfig([url: ['http://primary.example/api', 'http://other.example/api']]) + + when: + updater.addRegistryRepos(cfg) + def repos = updater.getRepositories() + + then: 'the default registry is replaced and all configured registries are added, in order' + repos.size() == 2 + repos*.id == ['registry-0', 'registry-1'] + repos[0].url.toString().startsWith('http://primary.example/api') + repos[1].url.toString().startsWith('http://other.example/api') + + cleanup: + folder?.deleteDir() + } + + def 'should not add registry repos when offline' () { + given: + def folder = Files.createTempDirectory('test') + def remote = remoteRepository(folder.resolve('repo'), ['1.0.0']) + def local = localCache(folder.resolve('plugins'), []) + def manager = new LocalPluginManager(local) + def updater = new PluginUpdater(manager, local, remote, true) + and: + def cfg = new RegistryConfig([url: ['https://reg.example/api']]) + + when: + updater.addRegistryRepos(cfg) + def repos = updater.getRepositories() + + then: + repos.size() == 1 + repos[0].id == 'downloaded' + + cleanup: + folder?.deleteDir() + } + + def 'should be no-op when registry config is null' () { + given: + def folder = Files.createTempDirectory('test') + def remote = remoteRepository(folder.resolve('repo'), ['1.0.0']) + def local = localCache(folder.resolve('plugins'), []) + def manager = new LocalPluginManager(local) + def updater = new PluginUpdater(manager, local, remote, false) + def before = updater.getRepositories().size() + + when: + updater.addRegistryRepos(null) + + then: + updater.getRepositories().size() == before + + cleanup: + folder?.deleteDir() + } + // ------------------------------------------------------------------------------------- // setup helpers