Restructure all the configs

.github/workflows/fix-linting.yml

@@ -45,7 +45,7 @@ jobs:
         continue-on-error: true
 
       # indication that the linting has finished
-      - name: react if linting finished succesfully
+      - name: react if linting finished successfully
         if: steps.pre-commit.outcome == 'success'
         uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4
         with:
@@ -79,8 +79,8 @@ jobs:
           comment-id: ${{ github.event.comment.id }}
           reactions: confused
 
-      - name: react if linting errors were not fixed
-        if: steps.commit-and-push.outcome  == 'failure'
+      - name: comment if linting errors were not fixed
+        if: steps.commit-and-push.outcome == 'failure'
         uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4
         with:
           issue-number: ${{ github.event.issue.number }}

.github/workflows/update-pipeline-configs.yml

@@ -22,8 +22,8 @@ jobs:
           pipeline=$(jq -r '.pipeline[]' pipeline_names.json)
           for pipeline in $pipeline; do
             printf "Checking pipeline: %s\n" $pipeline
-            if [ ! -f "pipeline/${pipeline}.config" ]; then
-              cat << EOF > "pipeline/${pipeline}.config"
+            if [ ! -f "conf/pipeline/${pipeline}.config" ]; then
+              cat << EOF > "conf/pipeline/${pipeline}.config"
           /*
           * -------------------------------------------------
           *  nfcore/${pipeline} custom profile Nextflow config file

CLAUDE.md

@@ -0,0 +1,104 @@
+# How this repo is used currently
+
+In each nf-core config or pipeline, these lines are included. So basically, nf-core custom.config and the pipeline rnaseq.config will get pulled in.
+
+```nextflow
+// Load nf-core custom profiles from different institutions
+
+// If params.custom_config_base is set AND either the NXF_OFFLINE environment variable is not set or params.custom_config_base is a local path, the nfcore_custom.config file from the specified base path is included.
+// Load nf-core/rnaseq custom profiles from different institutions.
+includeConfig params.custom_config_base && (!System.getenv('NXF_OFFLINE') || !params.custom_config_base.startsWith('http')) ? "${params.custom_config_base}/nfcore_custom.config" : "/dev/null"
+
+// Load nf-core/rnaseq custom profiles from different institutions.
+includeConfig params.custom_config_base && (!System.getenv('NXF_OFFLINE') || !params.custom_config_base.startsWith('http')) ? "${params.custom_config_base}/pipeline/rnaseq.config" : "/dev/null"
+```
+
+# New Directory Structure (2024)
+
+We've reorganized cluster configurations to improve maintainability and co-locate related files. The new structure follows this pattern:
+
+## Structure Overview
+
+```
+conf/<cluster>/
+├── nextflow.config                    # Main cluster configuration
+├── README.md                         # Cluster documentation and usage
+└── pipelines/                        # Pipeline-specific configurations
+    ├── <pipeline1>/
+    │   ├── nextflow.config           # Pipeline-specific config
+    │   └── README.md                 # Pipeline-specific docs
+    └── <pipeline2>/
+        ├── nextflow.config
+        └── README.md
+```
+
+## Examples
+
+### UPPMAX (complex cluster with pipeline-specific configs)
+```
+conf/uppmax/
+├── nextflow.config                    # Main UPPMAX configuration
+├── README.md                         # UPPMAX usage guide
+└── pipelines/
+    ├── ampliseq/
+    │   ├── nextflow.config           # AmplicSeq-specific UPPMAX config
+    │   └── README.md                 # AmplicSeq on UPPMAX docs
+    └── sarek/
+        ├── nextflow.config           # Sarek-specific UPPMAX config
+        └── README.md                 # Sarek on UPPMAX docs
+```
+
+### Sanger (simple cluster with just main config)
+```
+conf/sanger/
+├── nextflow.config                    # Main Sanger configuration
+└── README.md                         # Sanger usage guide (if exists)
+```
+
+## Migration Process
+
+### Automated Migration
+Use the provided automation script to reorganize existing clusters:
+
+```bash
+./bin/reorganize-cluster.sh <cluster_name>
+```
+
+This script:
+1. Creates the cluster directory if it doesn't exist
+2. Moves `conf/cluster.config` → `conf/cluster/nextflow.config`
+3. Renames `pipeline/` → `pipelines/` (if exists)
+4. Moves pipeline configs from `conf/pipeline/*/cluster.config`
+5. Moves pipeline docs from `docs/pipeline/*/cluster.md`
+6. Updates `nfcore_custom.config` with the new path
+7. Cleans up duplicate files
+
+### Manual Steps (if needed)
+1. **Create directory**: `mkdir conf/<cluster>`
+2. **Move main config**: `mv conf/<cluster>.config conf/<cluster>/nextflow.config`
+3. **Move pipeline configs**: `mv conf/pipeline/<pipeline>/<cluster>.config conf/<cluster>/pipelines/<pipeline>/nextflow.config`
+4. **Move documentation**: `mv docs/pipeline/<pipeline>/<cluster>.md conf/<cluster>/pipelines/<pipeline>/README.md`
+5. **Update registry**: Edit `nfcore_custom.config` to point to `conf/<cluster>/nextflow.config`
+
+## Benefits
+
+- **Co-location**: All cluster-related files are in one place
+- **Consistent naming**: `nextflow.config` and `README.md` everywhere
+- **Better discoverability**: Easier for maintainers to find cluster-specific files
+- **Reduced duplication**: Single source of truth for each cluster configuration
+- **Improved maintainability**: Clear separation between general and pipeline-specific configs
+
+## Registry Update Required
+
+When moving cluster configs, always update `nfcore_custom.config`:
+
+```diff
+profiles {
+    cluster_name {
+-       includeConfig "${params.custom_config_base}/conf/cluster_name.config"
++       includeConfig "${params.custom_config_base}/conf/cluster_name/nextflow.config"
+    }
+}
+```
+
+This is automatically handled by the reorganization script.

bin/migrate-all-clusters.sh

@@ -0,0 +1,196 @@
+#!/bin/bash
+
+# migrate-all-clusters.sh
+# Mass migration script to reorganize all cluster configurations
+# to the new directory structure using the existing reorganize-cluster.sh script
+
+set -euo pipefail
+
+# Script configuration
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(dirname "$SCRIPT_DIR")"
+REORGANIZE_SCRIPT="$SCRIPT_DIR/reorganize-cluster.sh"
+
+# Counters
+TOTAL_CLUSTERS=0
+SUCCESSFUL_MIGRATIONS=0
+FAILED_MIGRATIONS=0
+SKIPPED_CLUSTERS=0
+
+# Arrays to track results
+SUCCESSFUL_CLUSTERS=()
+FAILED_CLUSTERS=()
+SKIPPED_CLUSTERS_LIST=()
+
+# Usage function
+usage() {
+    echo "Usage: $0 [OPTIONS]"
+    echo ""
+    echo "Mass migration tool to reorganize all cluster configurations."
+    echo ""
+    echo "Options:"
+    echo "  --dry-run     Show what would be migrated without making changes"
+    echo "  --test-only   Only migrate first 3 clusters for testing"
+    echo "  --help        Show this help message"
+    echo ""
+    echo "Examples:"
+    echo "  $0 --dry-run              # Preview what would be migrated"
+    echo "  $0 --test-only            # Test migration on first 3 clusters"
+    echo "  $0                        # Migrate all remaining clusters"
+    echo ""
+    exit 1
+}
+
+# Parse command line arguments
+DRY_RUN=false
+TEST_ONLY=false
+
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        --dry-run)
+            DRY_RUN=true
+            shift
+            ;;
+        --test-only)
+            TEST_ONLY=true
+            shift
+            ;;
+        --help)
+            usage
+            ;;
+        *)
+            echo "Unknown option: $1"
+            usage
+            ;;
+    esac
+done
+
+# Change to repository root
+cd "$REPO_ROOT"
+
+# Validate reorganize script exists
+if [[ ! -x "$REORGANIZE_SCRIPT" ]]; then
+    echo "❌ Error: reorganize-cluster.sh script not found or not executable at $REORGANIZE_SCRIPT"
+    exit 1
+fi
+
+echo "🔄 Mass Cluster Configuration Migration"
+echo "======================================="
+
+# Find all remaining .config files that need migration
+echo "🔍 Discovering clusters to migrate..."
+CLUSTER_CONFIGS=($(find conf/ -maxdepth 1 -name "*.config" -type f | sort))
+TOTAL_CLUSTERS=${#CLUSTER_CONFIGS[@]}
+
+if [[ $TOTAL_CLUSTERS -eq 0 ]]; then
+    echo "✅ No clusters found that need migration!"
+    exit 0
+fi
+
+# Apply test-only limit if requested
+if [[ "$TEST_ONLY" == true ]]; then
+    if [[ $TOTAL_CLUSTERS -gt 3 ]]; then
+        CLUSTER_CONFIGS=("${CLUSTER_CONFIGS[@]:0:3}")
+        echo "🧪 TEST MODE: Only processing first 3 clusters"
+    fi
+fi
+
+echo "📊 Found ${#CLUSTER_CONFIGS[@]} clusters to process (${TOTAL_CLUSTERS} total remaining)"
+
+# Show what would be migrated in dry-run mode
+if [[ "$DRY_RUN" == true ]]; then
+    echo ""
+    echo "🔍 DRY RUN - Clusters that would be migrated:"
+    echo "=============================================="
+    for config_file in "${CLUSTER_CONFIGS[@]}"; do
+        cluster_name=$(basename "$config_file" .config)
+        echo "  • $cluster_name (from $config_file)"
+    done
+    echo ""
+    echo "Total clusters to migrate: ${#CLUSTER_CONFIGS[@]}"
+    echo ""
+    echo "To perform the actual migration, run without --dry-run"
+    exit 0
+fi
+
+echo ""
+echo "🚀 Starting migration process..."
+echo ""
+
+# Process each cluster
+for config_file in "${CLUSTER_CONFIGS[@]}"; do
+    cluster_name=$(basename "$config_file" .config)
+    echo "📁 Processing cluster: $cluster_name"
+
+    # Check if cluster already has a directory (shouldn't happen but be safe)
+    if [[ -d "conf/$cluster_name" ]]; then
+        echo "  ⚠️  Cluster directory already exists, skipping..."
+        SKIPPED_CLUSTERS_LIST+=("$cluster_name")
+        ((SKIPPED_CLUSTERS++))
+        echo ""
+        continue
+    fi
+
+    # Run the reorganization script
+    if "$REORGANIZE_SCRIPT" "$cluster_name" >/dev/null 2>&1; then
+        echo "  ✅ Migration successful"
+        SUCCESSFUL_CLUSTERS+=("$cluster_name")
+        ((SUCCESSFUL_MIGRATIONS++))
+    else
+        echo "  ❌ Migration failed"
+        FAILED_CLUSTERS+=("$cluster_name")
+        ((FAILED_MIGRATIONS++))
+    fi
+
+    echo ""
+done
+
+# Generate summary report
+echo "🎉 Migration Summary"
+echo "==================="
+echo "Total clusters processed: ${#CLUSTER_CONFIGS[@]}"
+echo "✅ Successful migrations: $SUCCESSFUL_MIGRATIONS"
+echo "❌ Failed migrations: $FAILED_MIGRATIONS"
+echo "⚠️  Skipped clusters: $SKIPPED_CLUSTERS"
+echo ""
+
+# Show detailed results
+if [[ $SUCCESSFUL_MIGRATIONS -gt 0 ]]; then
+    echo "✅ Successfully migrated clusters:"
+    printf '   • %s\n' "${SUCCESSFUL_CLUSTERS[@]}"
+    echo ""
+fi
+
+if [[ $FAILED_MIGRATIONS -gt 0 ]]; then
+    echo "❌ Failed migrations (may need manual attention):"
+    printf '   • %s\n' "${FAILED_CLUSTERS[@]}"
+    echo ""
+fi
+
+if [[ $SKIPPED_CLUSTERS -gt 0 ]]; then
+    echo "⚠️  Skipped clusters (already migrated):"
+    printf '   • %s\n' "${SKIPPED_CLUSTERS_LIST[@]}"
+    echo ""
+fi
+
+# Show git status
+if [[ $SUCCESSFUL_MIGRATIONS -gt 0 ]]; then
+    echo "📝 Git Status:"
+    echo "=============="
+    if git status --porcelain | head -5; then
+        echo ""
+        echo "📋 Next steps:"
+        echo "1. Review the changes: git status"
+        echo "2. Test a few profiles: nextflow config -show-profiles ."
+        echo "3. Commit the changes: git add . && git commit -m 'refactor: migrate all clusters to new directory structure'"
+        echo ""
+    fi
+fi
+
+# Exit with error code if any migrations failed
+if [[ $FAILED_MIGRATIONS -gt 0 ]]; then
+    echo "⚠️  Some migrations failed. Please check the failed clusters manually."
+    exit 1
+fi
+
+echo "🎊 All migrations completed successfully!"