Add functionality to check unused store paths

Author: pinkwah

src/deploy/__main__.py

@@ -5,6 +5,7 @@
 
 import click
 from pathlib import Path
+from dataclasses import dataclass
 
 from deploy.build import Build
 from deploy.config import load_config
@@ -13,35 +14,45 @@
 from deploy.sync import do_sync
 
 
-USE_SYSTEM: bool = False
+@dataclass
+class _Args:
+    config_dir: Path = Path()
+    use_system: bool = False
+
+
+Args = _Args()
 
 
 @click.group()
+@click.option(
+    "--config-dir",
+    "-C",
+    help="Directory of config.yaml [default=.]",
+    default=".",
+)
 @click.option(
     "--system",
     "-s",
     is_flag=True,
     help="Install to /prog/pflotran instead of ~/cirrus",
     default=False,
 )
-def cli(system: bool) -> None:
-    global USE_SYSTEM
-
-    USE_SYSTEM = system
+def cli(config_dir: str, system: bool) -> None:
+    Args.config_dir = Path(config_dir).resolve()
+    Args.use_system = system
 
 
 @cli.command(help="Check locations")
 def check() -> None:
-    configpath = Path.cwd()
-    config = load_config(configpath)
+    config = load_config(Path(Args.config_dir))
     do_check(config)
 
 
 @cli.command(help="Synchronise all locations")
 def sync() -> None:
     configpath = Path.cwd()
     config = load_config(configpath)
-    do_sync(config, system=USE_SYSTEM)
+    do_sync(config, system=Args.use_system)
 
 
 @cli.command(help="Build Cirrus and dependencies")
@@ -58,15 +69,15 @@ def build(force: bool) -> None:
 
     configpath = Path.cwd()
     config = load_config(configpath)
-    builder = Build(configpath, config, force=force, system=USE_SYSTEM)
+    builder = Build(configpath, config, force=force, system=Args.use_system)
     builder.build()
 
 
 @cli.command(help="Generate symlinks from ./symlinks.json")
 def links() -> None:
     configpath = Path.cwd()
     config = load_config(configpath)
-    make_links(config, system=USE_SYSTEM)
+    make_links(config, system=Args.use_system)
 
 
 @cli.command(help="Run tests in ./deploy_tests using pytest")
@@ -76,7 +87,7 @@ def test(args: tuple[str, ...]) -> None:
 
     configpath = Path.cwd()
     config = load_config(configpath)
-    builder = Build(configpath, config, system=USE_SYSTEM)
+    builder = Build(configpath, config, system=Args.use_system)
 
     testpath = Path("./deploy_tests")
     if not testpath.is_dir():

src/deploy/_check.py

@@ -0,0 +1,48 @@
+#!/usr/bin/python3.6
+"""This script is meant to run standalone on the cluster which uses RHEL8 and
+only has Python 3.6"""
+
+import sys
+from typing import List, Sequence, Dict, Any
+from pathlib import Path
+import json
+import argparse
+
+
+def get_unused_store_paths(store: Path, versions: Sequence[Path]) -> List[str]:
+    used = {
+        path.replace("/prog/replace", "/prog/cirrus")
+        for vpath in versions
+        for manifest in vpath.glob("*/manifest")
+        for path in manifest.read_text().splitlines()
+    }
+
+    available = {str(p) for p in store.glob("*")}
+    return list(sorted(available - used))
+
+
+def details(store: Path, versions: Sequence[Path]) -> Dict[str, Any]:
+    return {
+        "unused_store_paths": get_unused_store_paths(store, versions),
+    }
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--store", help="Location of the store", required=True)
+    ap.add_argument(
+        "--versions",
+        nargs="+",
+        help="Locations of 'versions' directories",
+        required=True,
+    )
+    args = ap.parse_args()
+
+    store = Path(args.store)
+    versions = [Path(p) for p in args.versions]
+
+    json.dump(details(store, versions), sys.stdout)
+
+
+if __name__ == "__main__":
+    main()

src/deploy/check.py

@@ -1,89 +1,41 @@
 from __future__ import annotations
+import sys
 import asyncio
-from typing import Literal
+from pathlib import Path
 
 import yaml
 from pydantic import BaseModel
 
 from deploy.config import Config, AreaConfig
 
-COLLECTOR = b"""\
-#!/usr/bin/env python3
-import os
-import json
-import re
-
-
-def gather_versions() -> None:
-    print("versions:")
-    base = "/prog/pflotran/versions"
-    for name in os.listdir(base):
-        if name[0] == ".":
-            continue
-        path = os.path.join(base, name)
-        if os.path.islink(path):
-            target = os.readlink(path)
-            print("- type: link")
-            print(f"  name: !!str {name}")
-            print(f"  target: !!str {target}")
-        elif os.path.isdir(path):
-            print(f"- type: dir")
-            print(f"  name: !!str {name}")
-        else:
-            print(f"- type: other")
-            print(f"  name: !!str {name}")
-            
-            
-def gather_store() -> None:
-    print("store:")
-    base = "/prog/pflotran/versions/.builds"
-    for name in os.listdir(base):
-        print(f"- !!str {name}")
-
-
-def main() -> None:
-    gather_versions()
-    gather_store()
-
-
-if __name__ == "__main__":
-    main()
-"""
-
-
-class VersionsLinkType(BaseModel):
-    type: Literal["link"]
-    name: str
-    target: str
-
-
-class VersionsDirType(BaseModel):
-    type: Literal["dir"]
-    name: str
 
+class Collect(BaseModel):
+    unused_store_paths: set[str]
 
-class VersionsOtherType(BaseModel):
-    type: Literal["other"]
-    name: str
 
+SCRIPT: bytes = (Path(__file__).parent / "_check.py").read_bytes()
 
-class Collect(BaseModel):
-    versions: list[VersionsLinkType | VersionsDirType | VersionsOtherType] | None = None
-    store: list[str] | None = None
 
+async def collect(config: Config, area: AreaConfig) -> Collect:
+    base = Path(config.paths.system_base)
+    store = base / config.paths.store
+    versions = [base / env.dest for env in config.envs]
 
-async def collect(area: AreaConfig) -> Collect:
     proc = await asyncio.create_subprocess_exec(
         "ssh",
         "-T",
         area.host,
-        "/usr/bin/env",
-        "python3",
+        "/usr/bin/python3.6",
+        "-",
+        "--store",
+        store,
+        "--versions",
+        *versions,
         stdin=asyncio.subprocess.PIPE,
         stdout=asyncio.subprocess.PIPE,
         stderr=asyncio.subprocess.PIPE,
     )
-    stdout, stderr = await proc.communicate(COLLECTOR)
+    stdout, stderr = await proc.communicate(SCRIPT)
 
     print(f"Finished area {area.name}")
 
@@ -93,22 +45,34 @@ async def collect(area: AreaConfig) -> Collect:
 async def _check(config: Config) -> None:
     tasks: list[asyncio.Task[Collect]] = []
 
+    if not config.areas:
+        sys.exit("No areas specified in config.yaml")
+
     for area in config.areas:
-        task = asyncio.create_task(collect(area))
+        task = asyncio.create_task(collect(config, area))
         tasks.append(task)
 
+    collected: dict[str, Collect] = {}
+
     for area, info in zip(
         config.areas, await asyncio.gather(*tasks, return_exceptions=True)
     ):
-        print(f"--- {area.name} ---")
+        print(f"Processed {area.name}")
         if isinstance(info, BaseException):
             raise info
 
-        for d in info.versions or []:
-            if isinstance(d, VersionsDirType):
-                print(f"  # {d.name}")
-            elif isinstance(d, VersionsLinkType):
-                print(f"  - {d.name} -> {d.target}")
+        collected[area.name] = info
+
+    all_areas = set(collected)
+    all_unused_store_paths = set()
+    for c in collected.values():
+        all_unused_store_paths.update(c.unused_store_paths)
+
+    print("Unused store paths:")
+    for path in sorted(all_unused_store_paths):
+        which = {k for k, v in collected.items() if path in v.unused_store_paths}
+        which_str = "" if which == all_areas else ", ".join(which)
+        print(f"{path:<120} {which_str}")
 
 
 def do_check(config: Config) -> None: