Use outdir param to place metadata files inside output subdirectory

When the Nextflow outdir param points deeper than the Quilt package root
(e.g. s3://bucket/ns/pkg/run-name/), README_NF_QUILT.md, quilt_summarize.json,
and nf-quilt/*.json metadata files now land inside the run subdirectory rather
than at the package root. This fixes the reported issue where these files
appeared one level above the pipeline output.

The observer extracts `outdir` from session params and passes it through to
QuiltProduct, which computes the prefix by stripping the bucket/namespace/name
components from the outdir path.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: drernie

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## [0.9.2] 2025-05-XX
 
+- Use outdir param to determine package-relative paths instead of positional S3 path guessing
 - Use "package" as default prefix
 - Modernize main*.nf files
 

plugins/nf-quilt/src/main/nextflow/quilt/QuiltObserver.groovy

@@ -36,6 +36,7 @@ class QuiltObserver implements TraceObserver {
 
     private Session session
     private String workDir
+    private String outdir
 
     final private Lock lock = new ReentrantLock() // Need this because of threads
     // Is this overkill? Do we ever have more than one output URI per run?
@@ -68,6 +69,10 @@ class QuiltObserver implements TraceObserver {
         log.info("`onFlowCreate` $session")
         this.session = session
         this.workDir = session.config.workDir
+        this.outdir = session.getParams()?.get('outdir')?.toString()
+        if (this.outdir) {
+            log.info("onFlowCreate.outdir: ${this.outdir}")
+        }
     }
 
     @Override
@@ -96,7 +101,7 @@ class QuiltObserver implements TraceObserver {
         // create a QuiltProduct for each unique package key
         publishedPaths.each { key, path ->
             log.debug("onFlowComplete: $key -> $path")
-            new QuiltProduct(path, session)
+            new QuiltProduct(path, session, outdir)
         }
     }
 

plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy

@@ -147,6 +147,7 @@ ${nextflow}
     protected final QuiltPackage pkg
     protected final Session session
     protected final Map<String, Map<String,Object>> config
+    protected final String outdirPrefix
 
     protected final Map metadata
     protected final Expando flags = new Expando([
@@ -159,12 +160,13 @@ ${nextflow}
         workflow: false,
     ])
 
-    QuiltProduct(QuiltPathify pathify, Session session) {
+    QuiltProduct(QuiltPathify pathify, Session session, String outdir = null) {
         log.debug("Creating QuiltProduct: ${pathify}")
         this.session = session
         this.config = session.config ?: [:]
         this.path = pathify.path
         this.pkg = pathify.pkg
+        this.outdirPrefix = computeOutdirPrefix(outdir)
         this.metadata = collectMetadata()
         if (session.isSuccess() || flags.getProperty(QuiltParser.P_FORCE) == true) {
             publish()
@@ -173,6 +175,41 @@ ${nextflow}
         }
     }
 
+    /**
+     * Compute the sub-path prefix within the package that corresponds to the outdir.
+     *
+     * Given outdir 's3://bucket/ns/pkg/run-name/' and the package being 'ns/pkg',
+     * the outdir prefix is 'run-name' — the portion of the outdir path that lives
+     * inside the package.  README and summarize files are written here instead of
+     * the package root.
+     *
+     * Returns empty string when outdir cannot be parsed or matches the package root exactly.
+     */
+    String computeOutdirPrefix(String outdir) {
+        if (!outdir) {
+            return ''
+        }
+        try {
+            // Strip scheme (s3://) to get bare path components
+            String barePath = outdir
+                .replaceFirst('^s3://', '')
+                .replaceFirst('^quilt\\+s3://', '')
+                .replaceAll('/+$', '')  // trim trailing slashes
+
+            String[] parts = barePath.split('/')
+            // parts[0] = bucket, parts[1] = prefix (namespace), parts[2] = suffix (name)
+            // parts[3+] = sub-path within the package
+            if (parts.length > 3) {
+                String prefix = parts[3..-1].join('/')
+                log.debug("computeOutdirPrefix: '${prefix}' from outdir '${outdir}'")
+                return prefix
+            }
+        } catch (Exception e) {
+            log.warn("computeOutdirPrefix: failed to parse outdir '${outdir}': ${e.message}")
+        }
+        return ''
+    }
+
     Map collectMetadata() {
         if (shouldSkip(KEY_META)) {
             log.debug("SKIP: metadata for ${pkg}")
@@ -262,8 +299,16 @@ ${nextflow}
         flags.setProperty(QuiltParser.P_PKG, pkgName)
     }
 
+    /**
+     * Prepend the outdirPrefix to a filename so that generated files
+     * land inside the outdir sub-directory of the package (when known).
+     */
+    String prefixedPath(String filename) {
+        return outdirPrefix ? "${outdirPrefix}/${filename}" : filename
+    }
+
     String writeMapToPackage(Map map, String prefix) {
-        String filename = "nf-quilt/${prefix}.json"
+        String filename = prefixedPath("nf-quilt/${prefix}.json")
         log.debug("writeMapToPackage[$prefix]: ${filename}")
         try {
             writeString(toJson(map), pkg, filename)
@@ -353,7 +398,7 @@ ${nextflow}
         }
         if (text != null && text.length() > 0) {
             log.debug("writeReadme: ${text.length()} bytes")
-            writeString(text, pkg, README_FILE)
+            writeString(text, pkg, prefixedPath(README_FILE))
         }
         return text
     }
@@ -407,7 +452,7 @@ ${nextflow}
 
         try {
             String qs_json = toJson(quilt_summarize)
-            writeString(qs_json, pkg, SUMMARY_FILE)
+            writeString(qs_json, pkg, prefixedPath(SUMMARY_FILE))
         }
         catch (Exception e) {
             log.error("writeSummarize.toJson failed: ${e.getMessage()}\n{$e}", SUMMARY_FILE)

plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF

@@ -1,7 +1,7 @@
 Manifest-Version: 1.0
 Plugin-Class: nextflow.quilt.QuiltPlugin
 Plugin-Id: nf-quilt
-Plugin-Version: 0.9.1
+Plugin-Version: 0.9.2
 Plugin-Provider: Quilt Data
 Plugin-Requires: >=24.10.0
 

plugins/nf-quilt/src/test/nextflow/quilt/QuiltObserverTest.groovy

@@ -35,7 +35,7 @@ class QuiltObserverTest extends QuiltSpecification {
     Session mockSession(boolean success = false) {
         String quilt_uri = 'quilt+s3://bucket#package=prefix%2fsuffix'
         return GroovyMock(Session) {
-            getParams() >> [pubNot: 'foo', pubBad: 'foo:bar', outdir: SpecURI(), pubDir: testURI, inDir: quilt_uri]
+            getParams() >> [pubNot: 'foo', pubBad: 'foo:bar', outdir: 's3://udp-spec/nf-quilt/source', pubDir: testURI, inDir: quilt_uri]
             isSuccess() >> success
             config >> [quilt: [metadata: [key: 'value']]]
             workDir >> Paths.get('./work')

plugins/nf-quilt/src/test/nextflow/quilt/QuiltProductTest.groovy

@@ -40,19 +40,19 @@ import spock.lang.Unroll
 @CompileDynamic
 class QuiltProductTest extends QuiltSpecification {
 
-    QuiltProduct makeProductFromUrl(String url, boolean success = false) {
+    QuiltProduct makeProductFromUrl(String url, boolean success = false, String outdir = null) {
         WorkflowMetadata wf_meta = GroovyMock(WorkflowMetadata) {
             toMap() >> [start:'2022-01-01', complete:'2022-01-02']
         }
         QuiltPath path = QuiltPathFactory.parse(url)
         QuiltPathify pathify = new QuiltPathify(path)
         Session session = GroovyMock(Session) {
             getWorkflowMetadata() >> wf_meta
-            getParams() >> [outdir: url]
+            getParams() >> [outdir: outdir ?: url]
             isSuccess() >> success
             config >> [quilt: [meta: [cf_key: 'cf_val']]]
         }
-        return new QuiltProduct(pathify, session)
+        return new QuiltProduct(pathify, session, outdir)
     }
 
     QuiltProduct makeProduct(String query=null, boolean success = false) {
@@ -372,4 +372,69 @@ class QuiltProductTest extends QuiltSpecification {
         true
     }
 
+    void 'computeOutdirPrefix extracts sub-path from outdir'() {
+        given:
+        QuiltProduct product = makeProduct()
+
+        expect:
+        product.computeOutdirPrefix(outdir) == expected
+
+        where:
+        outdir                                      | expected
+        null                                        | ''
+        ''                                          | ''
+        's3://bucket/ns/pkg'                        | ''
+        's3://bucket/ns/pkg/'                       | ''
+        's3://bucket/ns/pkg/run-name'               | 'run-name'
+        's3://bucket/ns/pkg/run-name/'              | 'run-name'
+        's3://bucket/ns/pkg/run-name/sub'           | 'run-name/sub'
+        's3://bucket/ns/pkg/run-name/sub/'          | 'run-name/sub'
+    }
+
+    void 'prefixedPath uses outdirPrefix when set'() {
+        when:
+        QuiltProduct withPrefix = makeProductFromUrl(testURI, false, 's3://bkt/pre/suf/my-run')
+
+        then:
+        withPrefix.outdirPrefix == 'my-run'
+        withPrefix.prefixedPath('README.md') == 'my-run/README.md'
+        withPrefix.prefixedPath('nf-quilt/params.json') == 'my-run/nf-quilt/params.json'
+
+        when:
+        QuiltProduct noPrefix = makeProductFromUrl(testURI, false, 's3://bkt/pre/suf')
+
+        then:
+        noPrefix.outdirPrefix == ''
+        noPrefix.prefixedPath('README.md') == 'README.md'
+    }
+
+    void 'writeReadme places file under outdirPrefix'() {
+        given:
+        QuiltProduct product = makeProductFromUrl(testURI, false, 's3://bkt/pre/suf/my-run')
+        product.pkg.reset()
+
+        when:
+        product.writeReadme('test message')
+
+        then:
+        product.match("my-run/${QuiltProduct.README_FILE}").size() == 1
+        product.match(QuiltProduct.README_FILE).size() == 0
+    }
+
+    void 'writeSummarize places file under outdirPrefix'() {
+        given:
+        QuiltProduct product = makeProductFromUrl(testURI, false, 's3://bkt/pre/suf/my-run')
+        product.pkg.reset()
+
+        when:
+        // Write a file in the prefixed subdir so summarize has something to find
+        String prefixedFile = "my-run/test.md"
+        QuiltProduct.writeString('# Test', product.pkg, prefixedFile)
+        product.writeSummarize()
+
+        then:
+        product.match("my-run/${QuiltProduct.SUMMARY_FILE}").size() == 1
+        product.match(QuiltProduct.SUMMARY_FILE).size() == 0
+    }
+
 }