Merge pull request #92 from seqeralabs/dev

Dev -> Main for release 0.7.0

Author: maxulysse

.devcontainer/devcontainer.json

@@ -1,20 +1,32 @@
+//
+// MAIN PRODUCTION DEVCONTAINER CONFIG
+// Uses image that is pre-built and pushed to GitHub
+// See .github/.devcontainer/devcontainer.json for build
+//
 {
-    "name": "nfcore",
-    "image": "nfcore/gitpod:latest",
-    "remoteUser": "gitpod",
-    "runArgs": ["--privileged"],
-
-    // Configure tool-specific properties.
+    "name": "nf-aggregate",
+    "image": "ghcr.io/nextflow-io/training:latest",
+    "workspaceFolder": "/workspaces/nf-aggregate",
+    "remoteUser": "root",
+    "remoteEnv": {
+        // Nextflow installation version
+        "NXF_HOME": "/workspaces/.nextflow",
+        "NXF_EDGE": "0",
+        "NXF_VER": "24.10.4",
+        // Other env vars
+        "HOST_PROJECT_PATH": "/workspaces/nf-aggregate",
+        "SHELL": "/bin/bash" // Ush bash
+    },
+    "onCreateCommand": "bash .devcontainer/setup.sh",
     "customizations": {
-        // Configure properties specific to VS Code.
         "vscode": {
-            // Set *default* container specific settings.json values on container create.
+            "extensions": ["nf-core.nf-core-extensionpack"],
+            // Use Python from conda
             "settings": {
                 "python.defaultInterpreterPath": "/opt/conda/bin/python"
             },
-
-            // Add the IDs of extensions you want installed when the container is created.
-            "extensions": ["ms-python.python", "ms-python.vscode-pylance", "nf-core.nf-core-extensionpack"]
+            // Use bash
+            "terminal.integrated.defaultProfile.linux": "bash"
         }
     }
 }

.devcontainer/setup.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+# Fix for Java options
+printf 'unset JAVA_TOOL_OPTIONS\n' >> $HOME/.bashrc
+unset JAVA_TOOL_OPTIONS
+
+# Customise the terminal command prompt
+printf "export PS1='\\[\\e[3;36m\\]\${PWD#/workspaces/} ->\\[\\e[0m\\] '\n" >> $HOME/.bashrc
+export PS1='\[\e[3;36m\]${PWD#/workspaces/} ->\[\e[0m\] '
+
+# Force Java environment variables to use conda installation
+printf 'export JAVA_HOME=/opt/conda\nexport JAVA_CMD=/opt/conda/bin/java\n' >> $HOME/.bashrc
+export JAVA_HOME=/opt/conda
+export JAVA_CMD=/opt/conda/bin/java
+
+# Update Nextflow
+nextflow self-update
+nextflow -version
+
+cat /usr/local/etc/vscode-dev-containers/first-run-notice.txt

.nf-core.yml

@@ -58,4 +58,4 @@ template:
     - ci
     - igenomes
     - nf_core_configs
-  version: 0.6.0
+  version: 0.7.0

.vscode/settings.json

@@ -1,3 +1,4 @@
 {
-    "markdown.styles": ["public/vscode_markdown.css"]
+    "markdown.styles": ["public/vscode_markdown.css"],
+    "nextflow.telemetry.enabled": false
 }

CHANGELOG.md

@@ -3,7 +3,36 @@
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## 0.6.0
+## [[0.7.0](https://github.com/seqeralabs/nf-aggregate/releases/tag/0.7.0)] - 2025-05-05
+
+Special thanks to the following for their contributions to the release:
+
+- [Edmund Miller](https://github.com/edmundmiller)
+- [Florian Wuennemann](https://github.com/FloWuenne)
+- [Maxime Garcia](https://github.com/maxulysse)
+
+Thank you to everyone else that has contributed by reporting bugs, enhancements or in any other way, shape or form.
+
+### Enhancements & fixes
+
+- [PR #88](https://github.com/seqeralabs/nf-aggregate/pull/88) - Update tw cli container version to 0.11.2 and allow .nextflow.log to be missing from tw call
+- [PR #89](https://github.com/seqeralabs/nf-aggregate/pull/89) - Enable usage of external run dumps with nf-aggregate & update devcontainer specifications
+- [PR #91](https://github.com/seqeralabs/nf-aggregate/pull/91) - Update benchmark report image to include a fix causing large memory footprint for reshaping large AWS cost report files
+
+### Software dependencies
+
+| Dependency        | Old version | New version |
+| ----------------- | ----------- | ----------- |
+| `tower-cli`       | 0.9.2       | 0.11.2      |
+| BENCHMARK_REPORTS | sha-48cfed7 | sha-a6d15e8 |
+
+> **NB:** Dependency has been **updated** if both old and new version information is present.
+>
+> **NB:** Dependency has been **added** if just the new version information is present.
+>
+> **NB:** Dependency has been **removed** if new version information isn't present.
+
+## [[0.6.0](https://github.com/seqeralabs/nf-aggregate/releases/tag/0.6.0)] - 2025-03-31
 
 ### Credits
 

README.md

@@ -69,6 +69,21 @@ id,workspace,group
 4VLRs7nuqbAhDy,community/showcase,group2
 ```
 
+## Use logs from an external Seqera Platform deployment
+
+Sometimes we want to compile benchmark reports from runs from two different Seqera platform deployments, for example a dev and a production environment to compare performance. External logs in nf-aggregate can be used by specifying the workspace as `external` and providing some additional optional columns that point to the log folder and specify whether these external logs contain fusion logs (did you export them with the `--add-fusion-logs` flag in your `tw run dumps`. If they do contain fusion logs, you can generate a gannt plot for them, as for runs supplied only via id.)
+
+Here is an example of using a mix of run ids for which we want to extract logs from our platform deployment and some run logs from another deployment we want to compare. In the example below, `1JI5B1avuj3o58` is a run that contains fusion logs, while `1vsww7GjKBsVNa` does not contain fusion logs.
+
+```
+id,workspace,group,logs,fusion
+3VcLMAI8wyy0Ld,community/showcase,group1,
+1JI5B1avuj3o58,external,group2,/path/to/my/run_dumps_tarball.tar.gz,true
+1vsww7GjKBsVNa,external,group2,/path/to/my/run_dumps_folder,false
+```
+
+## Incorporate AWS split cost allocation data
+
 To incorporate AWS cost data into the benchmark report, use the `benchmark_aws_cur_report` parameter. This should point to a valid AWS Cost and Usage Report (CUR) file in Parquet format, currently only supporting CUR 1.0. The file can be stored locally or in a cloud bucket.
 To run nf-aggregate and generate benchmark reports, you can use the following command:
 

assets/schema_input.json

@@ -15,15 +15,26 @@
             },
             "workspace": {
                 "type": "string",
-                "pattern": "^[a-zA-Z0-9](?:[a-zA-Z0-9]|[-_](?=[a-zA-Z0-9])){1,38}/[a-zA-Z0-9](?:[a-zA-Z0-9]|[-_](?=[a-zA-Z0-9])){1,38}$",
-                "errorMessage": "Please provide a valid Seqera Platform Workspace name",
+                "pattern": "^(?:external|[a-zA-Z0-9][a-zA-Z0-9-_]{0,38}/[a-zA-Z0-9][a-zA-Z0-9-_]{0,38})$",
+                "errorMessage": "Please provide a valid Seqera Platform Workspace name or 'external'",
                 "meta": ["workspace"]
             },
             "group": {
                 "type": "string",
                 "pattern": "^[a-zA-Z0-9][-a-zA-Z0-9_ ]{0,37}$",
                 "errorMessage": "Please provide a valid group name",
                 "meta": ["group"]
+            },
+            "logs": {
+                "type": "string",
+                "format": "file-path",
+                "pattern": "^\\S+$",
+                "errorMessage": "Please provide a valid file path to your Seqera Platform logs.",
+                "meta": ["logs"]
+            },
+            "fusion": {
+                "type": "boolean",
+                "meta": ["fusion"]
             }
         },
         "required": ["id", "workspace"]

modules/local/benchmark_report/main.nf

@@ -1,6 +1,6 @@
 process BENCHMARK_REPORT {
 
-    container 'cr.seqera.io/scidev/benchmark-reports:sha-48cfed7'
+    container 'cr.seqera.io/scidev/benchmark-reports:sha-a6d15e8'
 
     input:
     path run_dumps

modules/local/benchmark_report/nextflow.config

@@ -2,7 +2,7 @@ process {
     withName: 'BENCHMARK_REPORT' {
         publishDir = [
             path: { "${params.outdir}/${metaOut?.projectName?.replace("/", "_") ?: ""}/benchmark_report" },
-            mode: params.publish_dir_mode,
+            mode: 'copy',
             saveAs: { filename -> filename.equals('versions.yml') || filename.endsWith('.json') ? null : filename }
         ]
         containerOptions = "--user root"

modules/local/plot_run_gantt/nextflow.config

@@ -2,7 +2,7 @@ process {
     withName: 'PLOT_RUN_GANTT' {
         publishDir = [
             path: { "${params.outdir}/${meta.projectName?.replace("/", "_") ?: ""}/gantt" },
-            mode: params.publish_dir_mode,
+            mode: 'copy',
             saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
         ]
     }