run_pipeline.py

#!/usr/bin/env python3
"""
run_pipeline.py — BODHI master pipeline
-----------------------------------------
Runs the full pipeline for both bodhi-s and bodhi-m in sequence:

  Step 1  — Neo4j ingest bodhi-s    bodhi-s/neo4j/ingest.py
  Step 2  — Neo4j ingest bodhi-m    bodhi-m/neo4j/ingest.py
  Step 3  — bodhi-s exports         bodhi-s/exports/scripts/export_csv.py
                                    bodhi-s/exports/scripts/export_jsonl.py
                                    bodhi-s/exports/scripts/export_pyg.py
                                    bodhi-s/exports/scripts/export_rdf.py
                                    bodhi-s/exports/scripts/export_neo4j_dump.py
  Step 4  — bodhi-m exports         bodhi-m/exports/scripts/export_csv.py
                                    bodhi-m/exports/scripts/export_jsonl.py
                                    bodhi-m/exports/scripts/export_pyg.py
                                    bodhi-m/exports/scripts/export_rdf.py
                                    bodhi-m/exports/scripts/export_neo4j_dump.py
  Step 5  — Browser JSON bodhi-s    bodhi-s/exports/generate_browser_json.py
  Step 6  — Browser JSON bodhi-m    bodhi-m/exports/generate_browser_json.py
  Step 7  — Stats                   stats.py

Usage:
    python run_pipeline.py [--wipe]
                           [--skip-ingest-s] [--skip-ingest-m]
                           [--skip-exports-s] [--skip-exports-m]
                           [--skip-browser-json]
                           [--skip-stats]

Options:
    --wipe            Pass --wipe to both ingest steps (deletes all graph data first).
    --skip-ingest-s   Skip bodhi-s Neo4j ingest.
    --skip-ingest-m   Skip bodhi-m Neo4j ingest.
    --skip-exports-s  Skip bodhi-s export scripts.
    --skip-exports-m  Skip bodhi-m export scripts.
    --skip-browser-json  Skip browser JSON generation for both networks.
    --skip-stats      Skip stats calculation.
"""

import argparse
import subprocess
import sys
from pathlib import Path

# Always use the same Python interpreter that launched this script
# so venv packages are available to all subprocesses.
PYTHON = sys.executable

ROOT = Path(__file__).resolve().parent

INGEST_S       = ROOT / "bodhi-s" / "neo4j" / "ingest.py"
INGEST_M       = ROOT / "bodhi-m" / "neo4j" / "ingest.py"
BROWSER_JSON_S = ROOT / "bodhi-s" / "exports" / "generate_browser_json.py"
BROWSER_JSON_M = ROOT / "bodhi-m" / "exports" / "generate_browser_json.py"
STATS_SCRIPT   = ROOT / "stats.py"

EXPORT_SCRIPTS_S = [
    ROOT / "bodhi-s" / "exports" / "scripts" / "export_csv.py",
    ROOT / "bodhi-s" / "exports" / "scripts" / "export_jsonl.py",
    ROOT / "bodhi-s" / "exports" / "scripts" / "export_pyg.py",
    ROOT / "bodhi-s" / "exports" / "scripts" / "export_rdf.py",
    ROOT / "bodhi-s" / "exports" / "scripts" / "export_neo4j_dump.py",
]

EXPORT_SCRIPTS_M = [
    ROOT / "bodhi-m" / "exports" / "scripts" / "export_csv.py",
    ROOT / "bodhi-m" / "exports" / "scripts" / "export_jsonl.py",
    ROOT / "bodhi-m" / "exports" / "scripts" / "export_pyg.py",
    ROOT / "bodhi-m" / "exports" / "scripts" / "export_rdf.py",
    ROOT / "bodhi-m" / "exports" / "scripts" / "export_neo4j_dump.py",
]


def run(label: str, cmd: list[str]) -> None:
    print(f"\n{'='*60}")
    print(f"  {label}")
    print(f"  {' '.join(str(c) for c in cmd)}")
    print(f"{'='*60}")
    result = subprocess.run(cmd)
    if result.returncode != 0:
        print(f"\n[pipeline] FAILED at step: {label}", file=sys.stderr)
        sys.exit(result.returncode)


def main():
    parser = argparse.ArgumentParser(description="Run the full BODHI pipeline (bodhi-s + bodhi-m).")
    parser.add_argument("--wipe",             action="store_true", help="Wipe graph before each ingest")
    parser.add_argument("--skip-ingest-s",    action="store_true", help="Skip bodhi-s Neo4j ingest")
    parser.add_argument("--skip-ingest-m",    action="store_true", help="Skip bodhi-m Neo4j ingest")
    parser.add_argument("--skip-exports-s",   action="store_true", help="Skip bodhi-s export scripts")
    parser.add_argument("--skip-exports-m",   action="store_true", help="Skip bodhi-m export scripts")
    parser.add_argument("--skip-browser-json",action="store_true", help="Skip browser JSON generation")
    parser.add_argument("--skip-stats",       action="store_true", help="Skip stats calculation")
    args = parser.parse_args()

    py = PYTHON

    # ------------------------------------------------------------------
    # Step 1 — Neo4j ingest bodhi-s
    # ------------------------------------------------------------------
    if not args.skip_ingest_s:
        cmd = [py, str(INGEST_S)]
        if args.wipe:
            cmd.append("--wipe")
        run("Step 1 — Neo4j ingest (bodhi-s)", cmd)
    else:
        print("\n[pipeline] Skipping Step 1 (bodhi-s ingest)")

    # ------------------------------------------------------------------
    # Step 2 — Neo4j ingest bodhi-m
    # ------------------------------------------------------------------
    if not args.skip_ingest_m:
        cmd = [py, str(INGEST_M)]
        if args.wipe:
            cmd.append("--wipe")
        run("Step 2 — Neo4j ingest (bodhi-m)", cmd)
    else:
        print("\n[pipeline] Skipping Step 2 (bodhi-m ingest)")

    # ------------------------------------------------------------------
    # Step 3 — bodhi-s exports
    # ------------------------------------------------------------------
    if not args.skip_exports_s:
        for script in EXPORT_SCRIPTS_S:
            run(f"Step 3 — bodhi-s export: {script.name}", [py, str(script)])
    else:
        print("\n[pipeline] Skipping Step 3 (bodhi-s exports)")

    # ------------------------------------------------------------------
    # Step 4 — bodhi-m exports
    # ------------------------------------------------------------------
    if not args.skip_exports_m:
        for script in EXPORT_SCRIPTS_M:
            run(f"Step 4 — bodhi-m export: {script.name}", [py, str(script)])
    else:
        print("\n[pipeline] Skipping Step 4 (bodhi-m exports)")

    # ------------------------------------------------------------------
    # Step 5 — Browser JSON bodhi-s
    # ------------------------------------------------------------------
    if not args.skip_browser_json:
        run("Step 5 — Browser JSON (bodhi-s)", [py, str(BROWSER_JSON_S)])
        run("Step 6 — Browser JSON (bodhi-m)", [py, str(BROWSER_JSON_M)])
    else:
        print("\n[pipeline] Skipping Steps 5-6 (browser JSON)")

    # ------------------------------------------------------------------
    # Step 7 — Stats
    # ------------------------------------------------------------------
    if not args.skip_stats:
        run("Step 7 — Stats (bodhi-s + bodhi-m)", [py, str(STATS_SCRIPT)])
    else:
        print("\n[pipeline] Skipping Step 7 (stats)")

    print(f"\n{'='*60}")
    print("  Pipeline complete.")
    print(f"{'='*60}")


if __name__ == "__main__":
    main()