diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ef4ab2..61dc2c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Hevelius Runner Changelog +0.3.0 (unreleased) + +- Exclude patterns - specified files or directories can be excluded from the volumes handling. +- Mass rename implemented (`hevelius-runner volumes rename "SUBSTRING1" "SUBSTRING2"`) + 0.2.0 (2026-04-28) - YAML configuration with `run`, `config`, and `doctor` subcommands; redacted `config` output diff --git a/bin/hevelius-runner.cmd b/bin/hevelius-runner.cmd new file mode 100644 index 0000000..833be9a --- /dev/null +++ b/bin/hevelius-runner.cmd @@ -0,0 +1,6 @@ +@echo off +setlocal +set "SCRIPT_DIR=%~dp0" +set "PROJECT_ROOT=%SCRIPT_DIR%.." +set PYTHONUTF8=1 +"%PROJECT_ROOT%\venv\Scripts\python.exe" "%PROJECT_ROOT%\src\hevelius-runner.py" %* diff --git a/bin/install-hevelius.ps1 b/bin/install-hevelius.ps1 new file mode 100644 index 0000000..032dfec --- /dev/null +++ b/bin/install-hevelius.ps1 @@ -0,0 +1,37 @@ +param( + [string]$ShortcutName = "Hevelius Runner" +) + +$ErrorActionPreference = "Stop" + +$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path +$binDir = Join-Path $repoRoot "bin" + +$runnerCmdPath = Join-Path $binDir "hevelius-runner.cmd" +$openShellPath = Join-Path $binDir "open-hevelius.ps1" + +$missing = @() +if (-not (Test-Path $runnerCmdPath)) { $missing += $runnerCmdPath } +if (-not (Test-Path $openShellPath)) { $missing += $openShellPath } +if ($missing.Count -gt 0) { + Write-Error "Required launcher file(s) missing from the repository:`n $($missing -join "`n ")`nRun 'git restore bin/' to restore them, then re-run this script." + exit 1 +} + +$startMenuPrograms = [Environment]::GetFolderPath("Programs") +$shortcutPath = Join-Path $startMenuPrograms "$ShortcutName.lnk" +$powershellExe = "powershell.exe" +$shortcutArgs = "-NoExit -ExecutionPolicy Bypass -File `"$openShellPath`"" + +$wshShell = New-Object -ComObject WScript.Shell +$shortcut = $wshShell.CreateShortcut($shortcutPath) +$shortcut.TargetPath = $powershellExe +$shortcut.Arguments = $shortcutArgs +$shortcut.WorkingDirectory = $repoRoot +$shortcut.IconLocation = "$env:SystemRoot\System32\WindowsPowerShell\v1.0\powershell.exe,0" +$shortcut.Save() + +Write-Host "Start Menu shortcut created:" -ForegroundColor Green +Write-Host " $shortcutPath" +Write-Host "" +Write-Host "Open Start Menu, find '$ShortcutName', and pin it if desired." diff --git a/bin/open-hevelius.ps1 b/bin/open-hevelius.ps1 new file mode 100644 index 0000000..acf63b3 --- /dev/null +++ b/bin/open-hevelius.ps1 @@ -0,0 +1,15 @@ +$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path +Set-Location $repoRoot + +$activateScript = Join-Path $repoRoot "venv\Scripts\Activate.ps1" +if (Test-Path $activateScript) { + . $activateScript +} +else { + Write-Warning "Virtual environment activation script not found at: $activateScript" +} + +$runnerCmd = Join-Path $repoRoot "bin\hevelius-runner.cmd" +Set-Alias -Name hevelius-runner -Value $runnerCmd -Scope Global + +Write-Host "Hevelius runner shell ready. Use: hevelius-runner " -ForegroundColor Green diff --git a/config/config.yaml.example b/config/config.yaml.example index 2b6ef6f..69f31cc 100644 --- a/config/config.yaml.example +++ b/config/config.yaml.example @@ -12,6 +12,11 @@ paths: repo-path: data/repo template_dir: templates output_dir: output + # Optional full-path exclusion patterns for volumes/list/file scans. + # If a file's full path matches any pattern, it is ignored completely. + exclude_patterns: + - '*FLAT*' + - '*shit*' # One or more FITS watch locations; each volume can define an optional nickname. volumes: - path: 'C:\astro\fits-live' diff --git a/doc/install.md b/doc/install.md index f9f1079..23965ee 100644 --- a/doc/install.md +++ b/doc/install.md @@ -14,6 +14,8 @@ This software requires Python. Once python is installed, get the sources into a To check if it works, you can run `python src/hevelius-runner.py doctor`. +Optionally, you can run bin/install-hevelius.ps1. This should create a hevelius runner +shortcut in the start menu. # Running hevelius-runner @@ -23,3 +25,7 @@ To check if it works, you can run `python src/hevelius-runner.py doctor`. 3. venv\Scripts\activate 4. python src/hevelius-runner.py ``` + +or + +click on the hevelius runner if you ran the install-hevelius script. diff --git a/doc/usage.md b/doc/usage.md index dc5654f..77a2983 100644 --- a/doc/usage.md +++ b/doc/usage.md @@ -76,6 +76,20 @@ task will be created. Files can be assigned to one of known projects and then th project's statistics will be updated, if `--projects` option is specified. Finally, there's `--orphans` option to list all files that were not able to assign to any project. +## Excluding files from volumes scan + +`hevelius-runner volumes` can skip files based on full-path patterns from +`config/config.yaml`: + +```yaml +paths: + exclude_patterns: + - '*FLAT*' + - '*shit*' +``` + +If a discovered file path matches any pattern, that file is ignored completely +and is not processed (no header read, no task/project updates). ## Run @@ -88,9 +102,6 @@ Run the automation loop: python src/hevelius-runner.py run ``` - - - The `run` command will: 1. Load configuration 2. Execute startup scripts diff --git a/requirements.txt b/requirements.txt index 976575a..7435f9c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ PyYAML +colorama>=0.4.6 requests psycopg2-binary watchdog diff --git a/src/cmd_doctor.py b/src/cmd_doctor.py index 73b8c91..169c243 100644 --- a/src/cmd_doctor.py +++ b/src/cmd_doctor.py @@ -1,76 +1,118 @@ +import logging +import os import sys from pathlib import Path from typing import Any, Callable + from api_client import APIClient from config_manager import ConfigManager +from console_color import GREEN, RED, color_segment +from cmd_volumes import _monitor_volumes_from_config + +logger = logging.getLogger(__name__) + + +def _ok() -> str: + return color_segment(GREEN, "OK", sys.stdout) + + +def _fail() -> str: + return color_segment(RED, "FAILED", sys.stdout) + def cmd_doctor(cm: ConfigManager, api_client_factory: Callable[[dict], Any] = APIClient) -> int: # The api_client_factory is used in tests, to override the actual APIClient class with a mock. + exit_code = 0 api_cfg = cm.get_api_config() required_api = ("base_url", "timeout", "username", "password") missing = [k for k in required_api if not str(api_cfg.get(k, "")).strip()] if missing: - print( - f"API configuration is incomplete (missing: {', '.join(missing)}).", - file=sys.stderr, - ) + logger.error(f"API config — missing: {', '.join(missing)} {_fail()}") return 1 client = api_client_factory(api_cfg) + version_ok = False try: version = client.get_version() - print(f"API reachable at {api_cfg['base_url'].rstrip('/')}/ - backend version: {version}") + logger.info(f"API reachable — {api_cfg['base_url'].rstrip('/')}/ (version {version}) {_ok()}") + version_ok = True except Exception as e: - print(f"API check failed (version endpoint): {e}", file=sys.stderr) - return 1 + exit_code = 1 + logger.error(f"API reachable — unreachable: {e} {_fail()}") - try: - login = client.login() - if not login.status or not login.token: - print(f"Login failed: {login.msg or 'no JWT token in response'}", file=sys.stderr) - return 1 - print(f"Authenticated as {api_cfg['username']!r} (user id {login.user_id}).") - except Exception as e: - print(f"API check failed (login): {e}", file=sys.stderr) - return 1 + login_ok = False + if version_ok: + try: + login = client.login() + if not login.status or not login.token: + exit_code = 1 + logger.error(f"Login — {login.msg or 'no JWT token in response'} {_fail()}") + else: + login_ok = True + logger.info(f"Login — authenticated as {api_cfg['username']!r} (user id {login.user_id}) {_ok()}") + except Exception as e: + exit_code = 1 + logger.error(f"Login — {e} {_fail()}") + else: + logger.info("Login — skipped (API not reached)") - try: - scopes = client.list_telescopes() - print(f"Telescopes available: {len(scopes)}") - sid = api_cfg.get("scope_id") - if sid is not None and str(sid).strip() != "": - want = int(sid) - ids = {int(t.get("scope_id")) for t in scopes if t.get("scope_id") is not None} - if want not in ids: - print( - f"Configured api.scope_id={want} is not in the telescope list from the API.", - file=sys.stderr, - ) - return 1 - names = [t.get("name") for t in scopes if int(t.get("scope_id", -1)) == want] - label = names[0] if names else "?" - print(f"Configured scope_id {want} OK ({label!r}).") - else: - print( - "Note: api.scope_id is not set; use `telescope list` and `telescope set`.", - file=sys.stderr, - ) - except Exception as e: - print(f"API check failed (telescope list): {e}", file=sys.stderr) - return 1 + if version_ok and login_ok: + try: + scopes = client.list_telescopes() + sid = api_cfg.get("scope_id") + if sid is None or not str(sid).strip(): + logger.info("Telescope — api.scope_id not set; use `telescope list` and `telescope set`") + else: + want = int(sid) + ids = {int(t.get("scope_id")) for t in scopes if t.get("scope_id") is not None} + if want not in ids: + exit_code = 1 + logger.error( + f"Telescope — scope_id={want} not found in API list ({len(scopes)} telescope(s)) {_fail()}" + ) + else: + names = [t.get("name") for t in scopes if int(t.get("scope_id", -1)) == want] + label = names[0] if names else "?" + logger.info( + f"Telescope — scope_id={want} ({label!r}), {len(scopes)} telescope(s) available {_ok()}" + ) + except Exception as e: + exit_code = 1 + logger.error(f"Telescope — {e} {_fail()}") + elif version_ok: + logger.info("Telescope — skipped (not authenticated)") + else: + logger.info("Telescope — skipped (API not reached)") nina_cfg = cm.get_nina_config() exe = nina_cfg.get("executable_path", "") nina_path = Path(str(exe)) if exe else None if not nina_path or not nina_path.is_file(): - print( - f"NINA executable not found or not configured: {exe!r}", - file=sys.stderr, - ) - return 1 - print(f"NINA executable found: {nina_path}") + exit_code = 1 + logger.error(f"NINA — executable not found: {exe!r} {_fail()}") + else: + logger.info(f"NINA — {nina_path} {_ok()}") + + volumes = _monitor_volumes_from_config(cm) + if not volumes: + logger.info("Volumes — none configured") + else: + for path, nickname in volumes: + if not os.path.isdir(path): + exit_code = 1 + logger.error(f"Volume '{nickname}' — {path} {_fail()} (directory not found)") + else: + try: + os.listdir(path) + logger.info(f"Volume '{nickname}' — {path} {_ok()}") + except OSError as e: + exit_code = 1 + logger.error(f"Volume '{nickname}' — {path} {_fail()} ({e})") - print("Configuration and connectivity check passed.") - return 0 + if exit_code == 0: + logger.info(f"All checks passed {_ok()}") + else: + logger.error(f"Doctor — one or more checks failed (see above) {_fail()}") + return exit_code diff --git a/src/cmd_volumes.py b/src/cmd_volumes.py index f60f47f..3879891 100644 --- a/src/cmd_volumes.py +++ b/src/cmd_volumes.py @@ -5,6 +5,7 @@ import logging import os import sys +import fnmatch from typing import Any, Dict, List, Optional, Tuple from fits import ( @@ -15,6 +16,11 @@ from api_client import APIClient from cmd_projects import _get_projects_from_endpoint from config_manager import ConfigManager +from console_color import GREEN, ORANGE, RED, YELLOW, color_segment + + +def _color(code: str, text: str) -> str: + return color_segment(code, text, sys.stdout) def get_tasks_files_list(client: APIClient) -> List[Tuple[int, Optional[str]]]: @@ -137,6 +143,70 @@ def _monitor_volumes_from_config(cm: ConfigManager) -> List[Tuple[str, str]]: return [] +def _exclude_patterns_from_config(cm: ConfigManager) -> List[str]: + """ + Return path exclusion patterns from config. + + Reads ``paths.exclude_patterns`` and supports both a single string and list + of strings. Patterns are matched against normalized full file paths. + """ + paths = cm.get_paths_config() + raw = paths.get("exclude_patterns") + if raw is None: + return [] + if isinstance(raw, str): + pattern = raw.strip() + return [pattern] if pattern else [] + if isinstance(raw, list): + out: List[str] = [] + for idx, item in enumerate(raw): + if item is None: + continue + if not isinstance(item, str): + print( + f"paths.exclude_patterns[{idx}] must be a string pattern.", + file=sys.stderr, + ) + continue + p = item.strip() + if p: + out.append(p) + return out + print("paths.exclude_patterns must be a string or list of strings.", file=sys.stderr) + return [] + + +def _normalize_full_path(path: str) -> str: + return os.path.normcase(os.path.normpath(os.path.abspath(path))) + + +def _resolve_path(path: str) -> str: + """Absolute path for filesystem I/O (preserves filename case on Windows).""" + return os.path.normpath(os.path.abspath(path)) + + +def _is_excluded(full_path: str, exclude_patterns: List[str]) -> bool: + """ + True if the normalized full path matches any exclusion pattern. + + Matching rules: + - fnmatch against normalized full path + - plain substring fallback against normalized full path + """ + if not exclude_patterns: + return False + normalized = _normalize_full_path(full_path) + for pattern in exclude_patterns: + candidate = os.path.normcase(pattern.strip()) + if not candidate: + continue + if fnmatch.fnmatch(normalized, candidate): + return True + if candidate in normalized: + return True + return False + + def _scope_id_from_config(cm: ConfigManager) -> Optional[int]: raw = cm.get_api_config().get("scope_id") if raw is None or not str(raw).strip(): @@ -213,6 +283,9 @@ def _print_project_stats( files_without_project: int, files_without_project_names: Optional[List[str]] = None, ) -> None: + """Print the orphan-files header. Per-bucket details (with color and the + add/update/skip action) are printed by :func:`_sync_project_stats_to_server`, + which is the only code path that knows the action that was actually taken.""" print() print("=== PROJECT STATISTICS ===") print(f"Files without project match: {files_without_project}") @@ -221,22 +294,6 @@ def _print_project_stats( print(f" {name}") if not project_stats: print("No project subframes were matched.") - return - for project_id in sorted(project_stats.keys()): - entry = project_stats[project_id] - project_name = entry.get("name") or f"project-{project_id}" - print(f"Project: {project_name}") - buckets = entry.get("buckets", {}) - total = sum(buckets.values()) - print(f" Total matched files: {total}") - for (filter_name, exposure), count in sorted( - buckets.items(), - key=lambda item: ( - "" if item[0][0] is None else str(item[0][0]), - -1.0 if item[0][1] is None else float(item[0][1]), - ), - ): - print(f" filter={filter_name} exposure={exposure} -> {count}") def _subframe_filter_name(subframe: Dict[str, Any]) -> Optional[str]: @@ -270,22 +327,158 @@ def _match_subframe( return None +def _fetch_project_fresh(client: APIClient, project_id: int) -> Optional[Dict[str, Any]]: + """GET /api/projects/{id} just before sync so we compare against current + server state (count/goal_count may have moved while we were scanning). + + Returns the project dict on success, or None when the call fails (the caller + falls back to the cached project data from the initial list fetch).""" + try: + url = f"{client.base_url.rstrip('/')}/projects/{int(project_id)}" + resp = client.session.get( + url, + timeout=client.timeout, + headers=client._get_auth_headers(), + ) + resp.raise_for_status() + data = resp.json() + if isinstance(data, dict): + project = data.get("project") + if isinstance(project, dict): + return project + except Exception: + return None + return None + + +def _bucket_action_color(action: str, count: int, goal_count: Optional[int]) -> str: + """Pick a color for a per-bucket sync result line. + + skipped (no change) -> orange + added/updated, count < goal -> yellow + added/updated, count >= goal -> green + failed -> red + """ + if action == "skipped": + return ORANGE + if action == "failed": + return RED + if goal_count is None or goal_count <= 0: + return GREEN + return GREEN if count >= goal_count else YELLOW + + +def _sync_one_bucket( + client: APIClient, + project_id: int, + project_name: str, + matched: Optional[Dict[str, Any]], + filter_name: str, + exposure_f: float, + count: int, +) -> Tuple[str, Optional[int], Optional[str]]: + """Sync a single (filter, exposure, count) bucket. + + Returns ``(action, goal_count, error)``: + - action: ``added``, ``updated``, ``skipped``, or ``failed`` + - goal_count: server-side goal_count when known (used for color logic) + - error: optional error message when action is ``failed`` + """ + if matched is None: + url = f"{client.base_url.rstrip('/')}/projects/{project_id}/subframes" + payload = { + "filter": str(filter_name), + "exposure_time": exposure_f, + "count": int(count), + } + try: + resp = client.session.post( + url, json=payload, timeout=client.timeout, + headers=client._get_auth_headers(), + ) + resp.raise_for_status() + except Exception as e: + return "failed", None, f"creating subframe failed: {e}" + return "added", None, None + + subframe_id = matched.get("id") + if subframe_id is None: + return "failed", None, "matched subframe has no id" + + matched_count = matched.get("count") + goal_count = matched.get("goal_count") + try: + goal_count_int = int(goal_count) if goal_count is not None else None + except (TypeError, ValueError): + goal_count_int = None + + # Skip when the captured count on the server already matches what we'd + # send. This avoids spurious last_updated bumps and keeps logs clean for + # incremental rescans of the same volume. + try: + if matched_count is not None and int(matched_count) == int(count): + return "skipped", goal_count_int, None + except (TypeError, ValueError): + pass + + url = f"{client.base_url.rstrip('/')}/projects/{project_id}/subframes/{subframe_id}" + payload = {"count": int(count)} + try: + resp = client.session.patch( + url, json=payload, timeout=client.timeout, + headers=client._get_auth_headers(), + ) + resp.raise_for_status() + except Exception as e: + return "failed", goal_count_int, f"updating subframe {subframe_id} failed: {e}" + return "updated", goal_count_int, None + + def _sync_project_stats_to_server(client: APIClient, project_stats: Dict[int, Dict[str, Any]]) -> bool: + """Sync collected per-project subframe counts to the server. + + For each project, the runner first re-fetches the project (so comparisons + against ``count``/``goal_count`` use fresh data), then for every bucket: + + * creates a new subframe if no matching (filter, exposure_time) row exists; + * sends a PATCH with only ``count`` if the server count differs; + * skips the call entirely when the server count already matches. + + Each per-bucket result is printed on a single color-coded line. + """ ok = True + if not project_stats: + return ok + for project_id in sorted(project_stats.keys()): entry = project_stats[project_id] project_name = entry.get("name") or f"project-{project_id}" - project = entry.get("project") or {} - subframes = project.get("subframes") if isinstance(project, dict) else None + + # Fetch fresh server state for this project; fall back to the cached + # copy from the initial listing if the GET fails. + fresh = _fetch_project_fresh(client, project_id) + if fresh is None: + fresh = entry.get("project") or {} + subframes = fresh.get("subframes") if isinstance(fresh, dict) else None if not isinstance(subframes, list): subframes = [] - for (filter_name, exposure), count in entry.get("buckets", {}).items(): + print(f"Project: {project_name}") + buckets = entry.get("buckets", {}) + total_files = sum(buckets.values()) + print(f" Total matched files: {total_files}") + + for (filter_name, exposure), count in sorted( + buckets.items(), + key=lambda item: ( + "" if item[0][0] is None else str(item[0][0]), + -1.0 if item[0][1] is None else float(item[0][1]), + ), + ): if filter_name is None or exposure is None: print( - f"Skipping subframe sync for project {project_name!r}: " - f"incomplete bucket filter={filter_name} exposure={exposure}", - file=sys.stderr, + f" {_color(RED, '[failed ]')} incomplete bucket " + f"filter={filter_name} exposure={exposure} count={count}" ) ok = False continue @@ -293,72 +486,31 @@ def _sync_project_stats_to_server(client: APIClient, project_stats: Dict[int, Di exposure_f = float(exposure) except (TypeError, ValueError): print( - f"Skipping subframe sync for project {project_name!r}: " - f"invalid exposure {exposure!r}", - file=sys.stderr, + f" {_color(RED, '[failed ]')} invalid exposure " + f"filter={filter_name} exposure={exposure!r} count={count}" ) ok = False continue matched = _match_subframe(subframes, str(filter_name), exposure_f) - if matched is None: - url = f"{client.base_url.rstrip('/')}/projects/{project_id}/subframes" - payload = { - "filter": str(filter_name), - "exposure_time": exposure_f, - "count": int(count), - "goal_count": int(count), - } - try: - resp = client.session.post( - url, - json=payload, - timeout=client.timeout, - headers=client._get_auth_headers(), - ) - resp.raise_for_status() - print( - f"Synced project {project_name!r}: created subframe " - f"filter={filter_name} exposure={exposure_f} count={count} goal_count={count}" - ) - except Exception as e: - print( - f"Failed creating subframe for project {project_name!r} " - f"(filter={filter_name}, exposure={exposure_f}): {e}", - file=sys.stderr, - ) - ok = False - continue - - subframe_id = matched.get("id") - if subframe_id is None: - print( - f"Failed updating subframe for project {project_name!r}: matched subframe has no id.", - file=sys.stderr, - ) + action, goal_count, error = _sync_one_bucket( + client, project_id, project_name, matched, + str(filter_name), exposure_f, int(count), + ) + if action == "failed": ok = False - continue - url = f"{client.base_url.rstrip('/')}/projects/{project_id}/subframes/{subframe_id}" - payload = {"count": int(count)} - try: - resp = client.session.patch( - url, - json=payload, - timeout=client.timeout, - headers=client._get_auth_headers(), - ) - resp.raise_for_status() - print( - f"Synced project {project_name!r}: updated subframe_id={subframe_id} " - f"filter={filter_name} exposure={exposure_f} count={count}" - ) - except Exception as e: - print( - f"Failed updating subframe {subframe_id} for project {project_name!r}: {e}", - file=sys.stderr, - ) - ok = False + color = _bucket_action_color(action, int(count), goal_count) + tag = f"[{action:<9}]" + line = ( + f" {tag} filter={filter_name} exposure={exposure_f} " + f"count={count}" + ) + if goal_count is not None: + line += f" goal_count={goal_count}" + if error: + line += f" ({error})" + print(_color(color, line)) return ok def process_fits_list( @@ -369,6 +521,7 @@ def process_fits_list( projects: Optional[List[Dict[str, Any]]] = None, project_stats: Optional[Dict[int, Dict[str, Any]]] = None, project_tracker: Optional[Dict[str, int]] = None, + exclude_patterns: Optional[List[str]] = None, ) -> None: """ Processes all FITS files listed in a specified text file. @@ -380,29 +533,188 @@ def process_fits_list( with open(fname, encoding="utf-8") as f: lines = f.readlines() - total = len(lines) + file_lines = [l for l in lines if l.strip() and not l.strip().startswith("#")] + total = len(file_lines) print(f"Found {total} filename(s) in file {fname}") cnt = 1 for line in lines: line = line.strip() if len(line) == 0 or line[0] == "#": - # Skip empty and commented out lines + # Skip empty and commented out lines (do not bump the counter so + # the user-visible "n of total" tracks file rows, not blank ones). + continue + + full_path = _normalize_full_path(line) + if _is_excluded(full_path, exclude_patterns or []): + progress = _format_progress(cnt, total) + print(f"{progress}{_format_status_tag('skipped')} {full_path} (excluded)") + cnt += 1 continue - print(f"Processing file {cnt} of {total}: {line}") process_fits_file( client, - line, + full_path, show_hdr=show_hdr, update_task=update_task, projects=projects, project_stats=project_stats, project_tracker=project_tracker, + idx=cnt, + total=total, ) cnt += 1 +def _fits_files_in_dir(dir_path: str, *, resolve: bool = False) -> List[str]: + """Return full paths of all ``*.fit`` / ``*.fits`` under ``dir_path``.""" + to_path = _resolve_path if resolve else _normalize_full_path + base = os.path.normpath(dir_path) + pattern_fit = os.path.join(base, "**", "*.fit") + pattern_fits = os.path.join(base, "**", "*.fits") + found_fit = glob.glob(pattern_fit, recursive=True) + found_fits = glob.glob(pattern_fits, recursive=True) + return sorted({to_path(str(f)) for f in set(found_fit) | set(found_fits)}) + + +def _paths_from_list_file(list_path: str, *, resolve: bool = False) -> List[str]: + """Return full paths from a text file (one path per line).""" + to_path = _resolve_path if resolve else _normalize_full_path + paths: List[str] = [] + with open(list_path, encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line or line[0] == "#": + continue + paths.append(to_path(line)) + return paths + + +def _collect_target_paths( + cm: ConfigManager, + args, + *, + resolve: bool = False, +) -> Tuple[int, List[str]]: + """ + Resolve FITS file paths from ``-f`` / ``-l`` / ``-d`` or all configured volumes. + + Mirrors the path-selection rules used by :func:`sanity_files`. + When ``resolve`` is True, paths keep filesystem casing (needed for rename on Windows). + """ + to_path = _resolve_path if resolve else _normalize_full_path + exclude_patterns = _exclude_patterns_from_config(cm) + paths: List[str] = [] + + if args.file: + paths.append(to_path(args.file)) + elif args.list: + paths.extend(_paths_from_list_file(args.list, resolve=resolve)) + elif args.dir: + paths.extend(_fits_files_in_dir(args.dir, resolve=resolve)) + else: + volumes = _monitor_volumes_from_config(cm) + if not volumes: + rp = _repo_path_from_config(cm) + if rp is None: + print( + "No source paths configured. Configure paths.volumes " + "(or legacy paths.fits_monitor_dir), or use -f / -l / -d.", + file=sys.stderr, + ) + return 1, [] + volumes = [(rp, "repo-path")] + for path, nickname in volumes: + print(f"Volume '{nickname}': {path}") + paths.extend(_fits_files_in_dir(path, resolve=resolve)) + + if exclude_patterns: + paths = [p for p in paths if not _is_excluded(p, exclude_patterns)] + return 0, paths + + +def _rename_basename(basename: str, old: str, new: str) -> Optional[str]: + """Return a new basename with all occurrences of ``old`` replaced by ``new``, + or ``None`` if ``old`` does not appear. Matching is case-insensitive on Windows.""" + if os.name == "nt": + lower_base = basename.lower() + lower_old = old.lower() + if lower_old not in lower_base: + return None + result = [] + i = 0 + while i < len(basename): + if lower_base[i:i + len(lower_old)] == lower_old: + result.append(new) + i += len(lower_old) + else: + result.append(basename[i]) + i += 1 + return "".join(result) + if old not in basename: + return None + return basename.replace(old, new) + + +def rename_files(cm: ConfigManager, args) -> int: + """Replace ``old_string`` with ``new_string`` in each selected file's basename.""" + code = _require_loaded(cm) + if code != 0: + return code + + old = str(getattr(args, "old_string", "") or "") + new = str(getattr(args, "new_string", "") or "") + if not old: + print("rename requires a non-empty search string.", file=sys.stderr) + return 1 + + code, paths = _collect_target_paths(cm, args, resolve=True) + if code != 0: + return code + if not paths: + print("No files to rename.") + return 0 + + print(f"Renaming basename substring {old!r} -> {new!r} in {len(paths)} file(s).") + renamed = 0 + skipped = 0 + failed = 0 + + for src in paths: + if not os.path.isfile(src): + print(f" skip (not a file): {src}", file=sys.stderr) + skipped += 1 + continue + + base = os.path.basename(src) + new_base = _rename_basename(base, old, new) + if new_base is None: + skipped += 1 + continue + if new_base == base: + skipped += 1 + continue + + dst = os.path.join(os.path.dirname(src), new_base) + if os.path.exists(dst): + print(f" failed (target exists): {src} -> {dst}", file=sys.stderr) + failed += 1 + continue + + try: + os.rename(src, dst) + except OSError as e: + print(f" failed: {src} -> {dst}: {e}", file=sys.stderr) + failed += 1 + continue + + print(f" {base} -> {new_base}") + renamed += 1 + + print(f"Done: {renamed} renamed, {skipped} unchanged, {failed} failed.") + return 1 if failed else 0 + + def process_fits_dir( client: APIClient, dir: str, @@ -411,6 +723,7 @@ def process_fits_dir( projects: Optional[List[Dict[str, Any]]] = None, project_stats: Optional[Dict[int, Dict[str, Any]]] = None, project_tracker: Optional[Dict[str, int]] = None, + exclude_patterns: Optional[List[str]] = None, ) -> None: """ Processes all FITS files in specified directory. @@ -419,13 +732,7 @@ def process_fits_dir( :param show_hdr: bool governing whether FITS headers will be printed or not """ - base = os.path.normpath(dir) - pattern_fit = os.path.join(base, "**", "*.fit") - pattern_fits = os.path.join(base, "**", "*.fits") - print(f"patterns={pattern_fit!r}, {pattern_fits!r}") - found_fit = glob.glob(pattern_fit, recursive=True) - found_fits = glob.glob(pattern_fits, recursive=True) - files = sorted(set(found_fit) | set(found_fits)) + files = _fits_files_in_dir(dir) print(f"Found {len(files)} files(s) in directory {dir}") @@ -433,25 +740,80 @@ def process_fits_dir( total = len(files) for f in files: - print(f"Processing file {cnt} of {total}: {f}") + full_path = _normalize_full_path(str(f)) + if _is_excluded(full_path, exclude_patterns or []): + progress = _format_progress(cnt, total) + print(f"{progress}{_format_status_tag('skipped')} {full_path} (excluded)") + cnt += 1 + continue + process_fits_file( client, - str(f), + full_path, show_hdr, update_task=update_task, projects=projects, project_stats=project_stats, project_tracker=project_tracker, + idx=cnt, + total=total, ) cnt += 1 +def _format_progress(idx: Optional[int], total: Optional[int]) -> str: + """Render the per-file progress prefix (e.g. ``[ 12/345]``) or empty string.""" + if idx is None or total is None or total <= 0: + return "" + width = max(1, len(str(total))) + return f"[{idx:>{width}}/{total}] " + + +def _format_status_tag(status: str) -> str: + """ + Render a fixed-width, color-coded status tag for compact per-file logging. + + matched -> green (file matched a project) + unmatched -> red (no project matched) + skipped -> yellow (file was excluded or otherwise not processed) + """ + label = f"{status:<9}" + if status == "matched": + return f"[{_color(GREEN, label)}]" + if status == "unmatched": + return f"[{_color(RED, label)}]" + if status == "skipped": + return f"[{_color(YELLOW, label)}]" + return f"[{label}]" + + +def _format_file_params(filter_name: Optional[str], exposure: Optional[float], + object_name: Optional[str]) -> str: + """Render ``filter=… exposure=… object=…`` skipping fields that are None.""" + parts = [] + if filter_name is not None: + parts.append(f"filter={filter_name}") + if exposure is not None: + parts.append(f"exposure={exposure}") + if object_name is not None: + parts.append(f"object={object_name}") + return " ".join(parts) + + def process_fits_file(client: APIClient, fname, show_hdr: bool, verbose: bool = False, update_task: bool = False, projects: Optional[List[Dict[str, Any]]] = None, project_stats: Optional[Dict[int, Dict[str, Any]]] = None, - project_tracker: Optional[Dict[str, Any]] = None): - """Processes a FITS file: optional header dump and task lookup via the API.""" + project_tracker: Optional[Dict[str, Any]] = None, + idx: Optional[int] = None, + total: Optional[int] = None): + """Processes a FITS file: optional header dump and task lookup via the API. + + Emits a single status line per file with a color-coded tag: + ``matched`` (green), ``unmatched`` (red), or ``skipped`` (yellow). The + ``idx``/``total`` parameters are optional and only used to render the + progress prefix when called from a batch processor. + """ key = os.path.basename(fname) @@ -461,44 +823,59 @@ def process_fits_file(client: APIClient, fname, show_hdr: bool, verbose: bool = object = _h_str(h, "OBJECT") exposure = _h_float(h, "EXPTIME") + task = None if update_task: task = get_task_by_filename(client, key) - if task: - tid, imagename = task - print(f" Task found: task_id={tid} imagename={imagename!r} (matched suffix {key!r}), filter={filter}, object={object}") - else: - print(f" No task found for filename suffix {key!r}, filter={filter}, object={object}") + project = None if projects is not None: project = _find_project_for_filename(key, projects) if project is not None: - project_name = _project_name(project) - print(f" Project found: {project_name!r}") if project_stats is not None: _update_project_stats(project_stats, project, filter, exposure) else: - print(" Project not found") if project_tracker is not None: project_tracker["files_without_project"] = project_tracker.get("files_without_project", 0) + 1 files_without_project_names = project_tracker.get("files_without_project_names") if isinstance(files_without_project_names, list): files_without_project_names.append(key) + # Build the compact one-line summary for this file. + if projects is not None: + if project is not None: + status = "matched" + else: + status = "unmatched" + else: + # Without --projects, fall back to the task status if available, + # otherwise mark as matched (we still have a file on disk). + status = "matched" if (not update_task or task) else "unmatched" + + extras: List[str] = [] + if project is not None: + extras.append(f"project={_project_name(project)!r}") + if update_task: + if task: + extras.append(f"task_id={task[0]}") + else: + extras.append("task=none") + + params = _format_file_params(filter, exposure, object) + progress = _format_progress(idx, total) + tag = _format_status_tag(status) + extras_s = (" " + " ".join(extras)) if extras else "" + params_s = (" " + params) if params else "" + print(f"{progress}{tag} {fname}{params_s}{extras_s}") + if show_hdr: for k in h.keys(): print(f" {k}: {h[k]}") - # OK, so we have a file on disk and there might or might not be a task for it. - # TODO: add ability to insert new or update existing task. - if update_task: if task: task_update(client, fname, task[0], verbose=verbose) else: task_add(client, fname, verbose=verbose) - else: - print(" Task DB update skipped (pass --task to enable API upsert).") - return def _h_str(h, key: str) -> Optional[str]: @@ -629,6 +1006,9 @@ def sanity_files(cm: ConfigManager, args) -> int: update_task = bool(getattr(args, "tasks", False)) use_projects = bool(getattr(args, "projects", False)) collect_orphans = bool(getattr(args, "orphans", False)) + exclude_patterns = _exclude_patterns_from_config(cm) + if exclude_patterns: + print(f"Exclude patterns enabled ({len(exclude_patterns)}): {exclude_patterns}") if collect_orphans and not use_projects: print("--orphans requires --projects.", file=sys.stderr) return 1 @@ -669,11 +1049,15 @@ def _project_kwargs() -> Dict[str, Any]: } if args.file: - print(f"Processing single file: {args.file}") + full_path = _normalize_full_path(args.file) + if _is_excluded(full_path, exclude_patterns): + print(f"Ignoring single file (excluded): {full_path}") + return 0 + print(f"Processing single file: {full_path}") if update_task: - process_fits_file(client, args.file, show_hdr=args.show_header, update_task=True, **_project_kwargs()) + process_fits_file(client, full_path, show_hdr=args.show_header, update_task=True, **_project_kwargs()) else: - process_fits_file(client, args.file, show_hdr=args.show_header, **_project_kwargs()) + process_fits_file(client, full_path, show_hdr=args.show_header, **_project_kwargs()) if use_projects: _print_project_stats( project_stats, @@ -687,9 +1071,22 @@ def _project_kwargs() -> Dict[str, Any]: if args.list: print(f"Processing list of files stored in {args.list}") if update_task: - process_fits_list(client, args.list, show_hdr=args.show_header, update_task=True, **_project_kwargs()) + process_fits_list( + client, + args.list, + show_hdr=args.show_header, + update_task=True, + exclude_patterns=exclude_patterns, + **_project_kwargs(), + ) else: - process_fits_list(client, args.list, show_hdr=args.show_header, **_project_kwargs()) + process_fits_list( + client, + args.list, + show_hdr=args.show_header, + exclude_patterns=exclude_patterns, + **_project_kwargs(), + ) if use_projects: _print_project_stats( project_stats, @@ -704,9 +1101,22 @@ def _project_kwargs() -> Dict[str, Any]: path = args.dir print(f"Processing all files in dir: {path}") if update_task: - process_fits_dir(client, path, show_hdr=args.show_header, update_task=True, **_project_kwargs()) + process_fits_dir( + client, + path, + show_hdr=args.show_header, + update_task=True, + exclude_patterns=exclude_patterns, + **_project_kwargs(), + ) else: - process_fits_dir(client, path, show_hdr=args.show_header, **_project_kwargs()) + process_fits_dir( + client, + path, + show_hdr=args.show_header, + exclude_patterns=exclude_patterns, + **_project_kwargs(), + ) if use_projects: _print_project_stats( project_stats, @@ -733,9 +1143,22 @@ def _project_kwargs() -> Dict[str, Any]: for path, nickname in volumes: print(f"Volume '{nickname}': {path}") if update_task: - process_fits_dir(client, path, show_hdr=args.show_header, update_task=True, **_project_kwargs()) + process_fits_dir( + client, + path, + show_hdr=args.show_header, + update_task=True, + exclude_patterns=exclude_patterns, + **_project_kwargs(), + ) else: - process_fits_dir(client, path, show_hdr=args.show_header, **_project_kwargs()) + process_fits_dir( + client, + path, + show_hdr=args.show_header, + exclude_patterns=exclude_patterns, + **_project_kwargs(), + ) if use_projects: _print_project_stats( project_stats, @@ -838,6 +1261,9 @@ def sanity_db(cm: ConfigManager, args) -> int: def cmd_volumes(cm: ConfigManager, args) -> int: """Manages the on disk images repository.""" + if getattr(args, "volumes_cmd", None) == "rename": + return rename_files(cm, args) + if args.file or args.list or args.dir or args.all_files: return sanity_files(cm, args) diff --git a/src/console_color.py b/src/console_color.py new file mode 100644 index 0000000..c581bce --- /dev/null +++ b/src/console_color.py @@ -0,0 +1,77 @@ +""" +TTY-aware ANSI color segments for CLI output and logging. + +Uses colorama's ``just_fix_windows_console()`` once at process start so +PowerShell/conhost interpret escapes on both stdout and stderr (Windows). + +For partial coloring inside a line (e.g. a status token in a log message), +build the message with :func:`color_segment`:: + + msg = f"task {tid} {color_segment(GREEN, 'ok', sys.stdout)} done" + logger.info(msg) + +``NO_COLOR`` disables all escapes. Non-TTY streams return plain text. +""" + +from __future__ import annotations + +import os +import sys +from typing import TextIO + +# SGR sequences (work with just_fix_windows_console on Windows). +_GREEN = "\033[92m" +_YELLOW = "\033[93m" +_RED = "\033[91m" +_ORANGE = "\033[38;5;208m" +_DIM = "\033[2m" +_RESET = "\033[0m" + +# Public aliases for call sites that prefer short names matching prior cmd_volumes. +GREEN = _GREEN +YELLOW = _YELLOW +RED = _RED +ORANGE = _ORANGE +DIM = _DIM +RESET = _RESET + +_win32_console_inited = False + + +def init_windows_console() -> None: + """Enable VT processing and UTF-8 on Windows stdout/stderr; no-op elsewhere.""" + global _win32_console_inited + if _win32_console_inited: + return + _win32_console_inited = True + if sys.platform != "win32": + return + try: + from colorama import just_fix_windows_console + + just_fix_windows_console() + except Exception: + pass + # Reconfigure to UTF-8 so emoji and other non-ASCII characters don't raise + # UnicodeEncodeError on consoles that default to a narrow encoding (e.g. cp1252). + # errors="replace" is a safety net; UTF-8 itself can encode all Unicode code points. + for _stream in (sys.stdout, sys.stderr): + try: + if hasattr(_stream, "reconfigure"): + _stream.reconfigure(encoding="utf-8", errors="replace") + except Exception: + pass + + +def stream_color_enabled(stream: TextIO) -> bool: + if os.environ.get("NO_COLOR"): + return False + return bool(hasattr(stream, "isatty") and stream.isatty()) + + +def color_segment(code: str, text: str, stream: TextIO) -> str: + """Wrap ``text`` with ``code`` and reset when ``stream`` is a color-capable TTY.""" + init_windows_console() + if not code or not stream_color_enabled(stream): + return text + return f"{code}{text}{_RESET}" diff --git a/src/hevelius-runner.py b/src/hevelius-runner.py index 67be360..ece57ee 100644 --- a/src/hevelius-runner.py +++ b/src/hevelius-runner.py @@ -26,6 +26,7 @@ # 4. Handles status updates import argparse +import copy import logging import sys import time @@ -36,6 +37,7 @@ import yaml +from console_color import init_windows_console, color_segment, RED, YELLOW from config_manager import ConfigManager from api_client import APIClient, resolve_scope_id_from_identifier from task_manager import TaskManager @@ -48,24 +50,62 @@ from cmd_doctor import cmd_doctor from cmd_projects import cmd_projects +_LEVEL_ABBREV = { + "DEBUG": "DEBG", + "INFO": "INFO", + "WARNING": "WARN", + "ERROR": "ERRO", + "CRITICAL": "CRIT", +} + +_LEVEL_COLOR = { + "WARNING": YELLOW, + "ERROR": RED, + "CRITICAL": RED, +} + + +class _ShortLevelFormatter(logging.Formatter): + """Formats the log level as a fixed 4-letter abbreviation.""" + + def format(self, record: logging.LogRecord) -> str: + record = copy.copy(record) + record.levelname = _LEVEL_ABBREV.get(record.levelname, record.levelname[:4].upper()) + return super().format(record) + + +class _ColoredShortLevelFormatter(logging.Formatter): + """Like _ShortLevelFormatter but colorizes WARN/ERRO/CRIT on color-capable streams.""" + + def __init__(self, fmt: str, stream) -> None: + super().__init__(fmt) + self._stream = stream + + def format(self, record: logging.LogRecord) -> str: + record = copy.copy(record) + color = _LEVEL_COLOR.get(record.levelname) + abbrev = _LEVEL_ABBREV.get(record.levelname, record.levelname[:4].upper()) + record.levelname = color_segment(color, abbrev, self._stream) if color else abbrev + return super().format(record) + + def setup_logging(): """Configure logging for the application.""" log_dir = Path("logs") log_dir.mkdir(exist_ok=True) - # Full logging: format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' - logging.basicConfig( - level=logging.INFO, - format='%(levelname)s - %(message)s', - handlers=[ - RotatingFileHandler( - log_dir / 'observatory.log', - maxBytes=1024*1024, - backupCount=5 - ), - logging.StreamHandler(sys.stdout) - ] + fmt = "%(levelname)s %(message)s" + stdout_handler = logging.StreamHandler(sys.stdout) + stdout_handler.setFormatter(_ColoredShortLevelFormatter(fmt, sys.stdout)) + + file_handler = RotatingFileHandler( + log_dir / 'observatory.log', + maxBytes=1024*1024, + backupCount=5, ) + file_handler.setFormatter(_ShortLevelFormatter(fmt)) + + logging.basicConfig(level=logging.INFO, handlers=[file_handler, stdout_handler]) class ObservatoryAutomation: @@ -316,6 +356,34 @@ def cmd_version(args: argparse.Namespace, cm: ConfigManager) -> int: return 0 +def _add_volumes_path_args(parser: argparse.ArgumentParser) -> None: + """File-selection switches shared by ``volumes rename`` and sanity scan.""" + parser.add_argument( + "-f", + "--file", + help="Rename one specific file (any extension).", + type=str, + ) + parser.add_argument( + "-l", + "--list", + help="Rename files listed in a text file (one path per line).", + type=str, + ) + parser.add_argument( + "-d", + "--dir", + help="Rename all FITS files recursively under this directory.", + type=str, + ) + parser.add_argument( + "-a", + "--all-files", + help="Rename FITS files in all configured paths.volumes (default when no -f/-l/-d).", + action="store_true", + ) + + def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( description="Hevelius observatory runner - NINA integration and API client.", @@ -368,6 +436,20 @@ def build_parser() -> argparse.ArgumentParser: p_view.add_argument("--project-id", type=int, help="Numeric project_id.") repo_parser = sub.add_parser('volumes', help="Manages files repository (volumes)on local storage.") + volumes_sub = repo_parser.add_subparsers(dest="volumes_cmd", metavar="SUBCOMMAND") + rename_parser = volumes_sub.add_parser( + "rename", + help="Replace a substring in filenames (basename only) for selected FITS files.", + ) + rename_parser.add_argument( + "old_string", + help="Substring to find in each file basename (literal match, not a regex).", + ) + rename_parser.add_argument( + "new_string", + help="Replacement text for each match in the basename.", + ) + _add_volumes_path_args(rename_parser) repo_parser.add_argument('-f', "--file", help="Check one specific FITS file", type=str) repo_parser.add_argument("-l", "--list", help="Check FITS files listed in a text file (one path per line)", type=str) repo_parser.add_argument("-d", "--dir", help="Check all FITS files recursively in this specific directory", type=str) @@ -384,7 +466,7 @@ def build_parser() -> argparse.ArgumentParser: return parser def main(argv: Optional[List[str]] = None) -> int: - + init_windows_console() #logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s") setup_logging() diff --git a/tests/test_cli.py b/tests/test_cli.py index 4e1d98e..fbd5de1 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,3 +1,4 @@ +import logging import textwrap from unittest.mock import MagicMock, patch @@ -63,7 +64,7 @@ def test_config_command_prints_yaml_without_password(tmp_path, capsys, runner_mo assert str(cfg.resolve()) in out -def test_doctor_command_success(tmp_path, capsys, runner_mod): +def test_doctor_command_success(tmp_path, caplog, runner_mod): cfg = tmp_path / "c.yaml" fake_nina = tmp_path / "nina_fake.exe" fake_nina.write_bytes(b"") @@ -102,11 +103,52 @@ def test_doctor_command_success(tmp_path, capsys, runner_mod): client.list_telescopes.return_value = [{"scope_id": 1, "name": "Test"}] MockClient.return_value = client - ret = runner_mod.main(["-c", str(cfg), "doctor"]) + with caplog.at_level(logging.INFO): + ret = runner_mod.main(["-c", str(cfg), "doctor"]) assert ret == 0 - out = capsys.readouterr().out - assert "1.0.0" in out - assert "Connectivity check passed" in out or "check passed" in out + assert "1.0.0" in caplog.text + assert "All checks passed" in caplog.text + + +def test_doctor_continues_when_api_unreachable(tmp_path, caplog, runner_mod): + cfg = tmp_path / "c.yaml" + fake_nina = tmp_path / "nina_fake.exe" + fake_nina.write_bytes(b"") + + cfg.write_text( + textwrap.dedent( + f""" + api: + base_url: https://example.test/api/ + timeout: 5 + username: u + password: p + verify_ssl: false + scope_id: 1 + paths: + template_dir: t + output_dir: o + nina: + executable_path: {fake_nina} + scripts: {{}} + """ + ).strip(), + encoding="utf-8", + ) + + with patch.object(runner_mod, "APIClient") as MockClient: + client = MagicMock() + client.get_version.side_effect = ConnectionError("refused") + MockClient.return_value = client + + with caplog.at_level(logging.INFO): + ret = runner_mod.main(["-c", str(cfg), "doctor"]) + assert ret == 1 + assert "unreachable" in caplog.text + assert "skipped" in caplog.text + assert "NINA" in caplog.text + client.login.assert_not_called() + client.list_telescopes.assert_not_called() def test_version_prints_runner_version(capsys, runner_mod): diff --git a/tests/test_cmd_repo_sanity_files.py b/tests/test_cmd_repo_sanity_files.py index a49a5f5..5fcd0a3 100644 --- a/tests/test_cmd_repo_sanity_files.py +++ b/tests/test_cmd_repo_sanity_files.py @@ -1,4 +1,5 @@ from argparse import Namespace +import builtins import cmd_volumes @@ -58,8 +59,8 @@ def test_sanity_files_list_file(monkeypatch): monkeypatch.setattr( cmd_volumes, "process_fits_list", - lambda client, fname, show_hdr=False: calls.append( - (client, fname, show_hdr) + lambda client, fname, show_hdr=False, **kwargs: calls.append( + (client, fname, show_hdr, kwargs) ), ) @@ -77,8 +78,8 @@ def test_sanity_files_specific_directory(monkeypatch): monkeypatch.setattr( cmd_volumes, "process_fits_dir", - lambda client, path, show_hdr=False: calls.append( - (client, path, show_hdr) + lambda client, path, show_hdr=False, **kwargs: calls.append( + (client, path, show_hdr, kwargs) ), ) @@ -99,7 +100,7 @@ def test_sanity_files_defaults_to_all_configured_volumes(monkeypatch): monkeypatch.setattr( cmd_volumes, "process_fits_dir", - lambda client, path, show_hdr=False: calls.append(path), + lambda client, path, show_hdr=False, **kwargs: calls.append(path), ) ret = cmd_volumes.sanity_files(cm, _base_args()) @@ -107,6 +108,47 @@ def test_sanity_files_defaults_to_all_configured_volumes(monkeypatch): assert calls == [r"c:\astro\live", r"d:\astro\archive"] +def test_process_fits_list_skips_excluded_full_path(monkeypatch): + calls = [] + + monkeypatch.setattr( + cmd_volumes, + "process_fits_file", + lambda client, fname, show_hdr=False, update_task=False, **kwargs: calls.append(fname), + ) + + listed = [ + r"C:\astro\night\good_001.fits", + r"C:\astro\night\FLAT_001.fits", + r"C:\astro\night\also_shit_001.fits", + ] + + def _fake_open(*args, **kwargs): + class _F: + def __enter__(self_inner): + return self_inner + + def __exit__(self_inner, exc_type, exc, tb): + return False + + def readlines(self_inner): + return [f"{line}\n" for line in listed] + + return _F() + + monkeypatch.setattr(builtins, "open", _fake_open) + + cmd_volumes.process_fits_list( + client=object(), + fname="ignored.txt", + show_hdr=False, + exclude_patterns=["*FLAT*", "*shit*"], + ) + + assert len(calls) == 1 + assert calls[0].lower().endswith(r"good_001.fits") + + def test_sanity_files_with_project_prefetches_and_prints_stats(monkeypatch, capsys): cm = _DummyConfigManager() calls = [] @@ -143,8 +185,10 @@ def _fake_process_dir(client, path, show_hdr=False, **kwargs): assert calls[0][1]["projects"] == [{"project_id": 42, "name": "M42", "subframes": []}] out = capsys.readouterr().out assert "PROJECT STATISTICS" in out - assert "Project: M42" in out - assert "filter=Ha exposure=300.0 -> 2" in out + assert "Files without project match: 1" in out + # Per-project bucket lines are emitted by _sync_project_stats_to_server + # which is mocked away in this test, so we only assert the orphan-files + # header section here. def test_process_fits_file_project_found_updates_stats(monkeypatch, capsys): @@ -162,7 +206,9 @@ def test_process_fits_file_project_found_updates_stats(monkeypatch, capsys): project_tracker=tracker, ) out = capsys.readouterr().out - assert "Project found: 'M42'" in out + assert "[matched" in out + assert "project='M42'" in out + assert "M42_001.fits" in out assert project_stats[42]["buckets"][("Ha", 300.0)] == 1 assert tracker["files_without_project"] == 0 @@ -182,20 +228,35 @@ def test_process_fits_file_project_not_found_tracks_counter(monkeypatch, capsys) project_tracker=tracker, ) out = capsys.readouterr().out - assert "Project not found" in out + assert "[unmatched" in out + assert "unknown_001.fits" in out assert project_stats == {} assert tracker["files_without_project"] == 1 class _FakeResponse: + def __init__(self, payload=None): + self._payload = payload or {} + def raise_for_status(self): return None + def json(self): + return self._payload + class _FakeSession: - def __init__(self): + """Test double for ``requests.Session`` used by the runner. + + Records POST/PATCH/GET calls and lets each test inject a stable response + payload for the GET /api/projects/{id} fresh-fetch the runner now performs. + """ + + def __init__(self, get_payload=None): self.post_calls = [] self.patch_calls = [] + self.get_calls = [] + self._get_payload = get_payload def post(self, url, json=None, timeout=None, headers=None): self.post_calls.append((url, json, timeout, headers)) @@ -205,10 +266,13 @@ def patch(self, url, json=None, timeout=None, headers=None): self.patch_calls.append((url, json, timeout, headers)) return _FakeResponse() + def get(self, url, timeout=None, headers=None): + self.get_calls.append((url, timeout, headers)) + return _FakeResponse(self._get_payload) -def test_sync_project_stats_creates_missing_subframe(): - session = _FakeSession() - client = type( + +def _make_client(session): + return type( "C", (), { @@ -218,6 +282,11 @@ def test_sync_project_stats_creates_missing_subframe(): "_get_auth_headers": lambda self: {"Authorization": "Bearer x"}, }, )() + + +def test_sync_project_stats_creates_missing_subframe(): + session = _FakeSession(get_payload={"status": True, "project": {"project_id": 42, "name": "M42", "subframes": []}}) + client = _make_client(session) stats = { 42: { "name": "M42", @@ -231,22 +300,20 @@ def test_sync_project_stats_creates_missing_subframe(): assert session.post_calls[0][0].endswith("/projects/42/subframes") assert session.post_calls[0][1]["filter"] == "Ha" assert session.post_calls[0][1]["count"] == 3 - assert session.post_calls[0][1]["goal_count"] == 3 + assert "goal_count" not in session.post_calls[0][1] + assert "active" not in session.post_calls[0][1] assert session.patch_calls == [] def test_sync_project_stats_updates_existing_subframe_count_only(): - session = _FakeSession() - client = type( - "C", - (), - { - "base_url": "https://example.test/api/", - "timeout": 5, - "session": session, - "_get_auth_headers": lambda self: {"Authorization": "Bearer x"}, - }, - )() + fresh = { + "project_id": 42, "name": "M42", + "subframes": [ + {"id": 77, "filter": {"short_name": "Ha"}, "exposure_time": 300.0, "count": 1, "goal_count": 10} + ], + } + session = _FakeSession(get_payload={"status": True, "project": fresh}) + client = _make_client(session) stats = { 42: { "name": "M42", @@ -266,3 +333,86 @@ def test_sync_project_stats_updates_existing_subframe_count_only(): assert len(session.patch_calls) == 1 assert session.patch_calls[0][0].endswith("/projects/42/subframes/77") assert session.patch_calls[0][1] == {"count": 5} + # Runner re-fetches the project right before sync so it compares against + # fresh server state instead of a possibly stale cached copy. + assert len(session.get_calls) == 1 + assert session.get_calls[0][0].endswith("/projects/42") + + +def test_sync_project_stats_skips_when_count_unchanged(capsys): + """No PATCH is issued when server count already matches the bucket count.""" + fresh = { + "project_id": 42, "name": "M42", + "subframes": [ + {"id": 77, "filter": {"short_name": "Ha"}, "exposure_time": 300.0, "count": 5, "goal_count": 10} + ], + } + session = _FakeSession(get_payload={"status": True, "project": fresh}) + client = _make_client(session) + stats = { + 42: { + "name": "M42", + "project": {"project_id": 42, "name": "M42", "subframes": fresh["subframes"]}, + "buckets": {("Ha", 300.0): 5}, + } + } + ok = cmd_volumes._sync_project_stats_to_server(client, stats) + assert ok is True + assert session.patch_calls == [] + assert session.post_calls == [] + out = capsys.readouterr().out + assert "skipped" in out + assert "goal_count=10" in out + + +def test_sync_project_stats_uses_fresh_data_when_cache_is_stale(capsys): + """The fresh GET trumps the stale cached project so we don't double-PATCH.""" + fresh = { + "project_id": 42, "name": "M42", + "subframes": [ + {"id": 77, "filter": {"short_name": "Ha"}, "exposure_time": 300.0, "count": 7, "goal_count": 10} + ], + } + session = _FakeSession(get_payload={"status": True, "project": fresh}) + client = _make_client(session) + stats = { + 42: { + "name": "M42", + # Cached count is 1 (stale) but server now reports 7 == bucket → skip. + "project": { + "project_id": 42, "name": "M42", + "subframes": [{"id": 77, "filter": {"short_name": "Ha"}, "exposure_time": 300.0, "count": 1, "goal_count": 10}], + }, + "buckets": {("Ha", 300.0): 7}, + } + } + ok = cmd_volumes._sync_project_stats_to_server(client, stats) + assert ok is True + assert session.patch_calls == [] + out = capsys.readouterr().out + assert "skipped" in out + + +def test_sync_project_stats_color_codes_at_or_above_goal(capsys): + """When count >= goal_count the line uses the green ANSI sequence.""" + fresh = { + "project_id": 42, "name": "M42", + "subframes": [ + {"id": 77, "filter": {"short_name": "Ha"}, "exposure_time": 300.0, "count": 1, "goal_count": 10} + ], + } + session = _FakeSession(get_payload={"status": True, "project": fresh}) + client = _make_client(session) + stats = { + 42: { + "name": "M42", + "project": {"project_id": 42, "name": "M42", "subframes": fresh["subframes"]}, + "buckets": {("Ha", 300.0): 12}, # exceeds goal_count=10 → green + } + } + cmd_volumes._sync_project_stats_to_server(client, stats) + assert len(session.patch_calls) == 1 + out = capsys.readouterr().out + assert "updated" in out + assert "count=12" in out + assert "goal_count=10" in out diff --git a/tests/test_cmd_volumes_rename.py b/tests/test_cmd_volumes_rename.py new file mode 100644 index 0000000..cfaeaec --- /dev/null +++ b/tests/test_cmd_volumes_rename.py @@ -0,0 +1,103 @@ +from argparse import Namespace +import os + +import cmd_volumes +import pytest + + +class _DummyConfigManager: + loaded = True + + def get_paths_config(self): + return {} + + +def _rename_args(**overrides): + args = Namespace( + old_string="Caldwell 75", + new_string="RCW 38", + file=None, + list=None, + dir=None, + all_files=False, + ) + for key, value in overrides.items(): + setattr(args, key, value) + return args + + +def test_rename_basename_literal_with_spaces_and_parens(): + old = "Caldwell 75" + new = "RCW 38" + base = "2026-04-18_22-59-27_Caldwell 75_L_180.00s_LIGHT_0004.fits" + assert cmd_volumes._rename_basename(base, old, new) == ( + "2026-04-18_22-59-27_RCW 38_L_180.00s_LIGHT_0004.fits" + ) + + paren_base = "2026-04-18_M42 (neb)_L_120.00s_LIGHT_0001.fits" + assert cmd_volumes._rename_basename(paren_base, "M42 (neb)", "NGC 1976") == ( + "2026-04-18_NGC 1976_L_120.00s_LIGHT_0001.fits" + ) + + +def test_rename_basename_returns_none_when_no_match(): + assert cmd_volumes._rename_basename("plain.fits", "missing", "x") is None + + +def test_rename_single_file(tmp_path, capsys): + cm = _DummyConfigManager() + src = tmp_path / "2026-04-18_22-59-27_Caldwell 75_L_180.00s_LIGHT_0004.fits" + src.write_bytes(b"fits") + dst = tmp_path / "2026-04-18_22-59-27_RCW 38_L_180.00s_LIGHT_0004.fits" + + ret = cmd_volumes.rename_files(cm, _rename_args(file=str(src))) + assert ret == 0 + assert not src.exists() + assert dst.is_file() + out = capsys.readouterr().out + assert "Caldwell 75" in out + assert "RCW 38" in out + assert "1 renamed" in out + + +def test_rename_dir_skips_unchanged_and_renames_matches(tmp_path, capsys): + cm = _DummyConfigManager() + match = tmp_path / "night_Caldwell 75_001.fits" + other = tmp_path / "night_M42_002.fits" + match.write_bytes(b"a") + other.write_bytes(b"b") + + ret = cmd_volumes.rename_files(cm, _rename_args(dir=str(tmp_path))) + assert ret == 0 + assert not match.exists() + assert (tmp_path / "night_RCW 38_001.fits").is_file() + assert other.is_file() + out = capsys.readouterr().out + assert "1 renamed" in out + assert "1 unchanged" in out + + +def test_rename_cli_parses_subcommand(runner_mod): + args = runner_mod.build_parser().parse_args( + ["volumes", "rename", "Caldwell 75", "RCW 38", "-a"] + ) + assert args.command == "volumes" + assert args.volumes_cmd == "rename" + assert args.old_string == "Caldwell 75" + assert args.new_string == "RCW 38" + assert args.all_files is True + + +def test_rename_fails_when_target_exists(tmp_path, capsys): + cm = _DummyConfigManager() + src = tmp_path / "frame_Caldwell 75.fits" + dst = tmp_path / "frame_RCW 38.fits" + src.write_bytes(b"a") + dst.write_bytes(b"b") + + ret = cmd_volumes.rename_files(cm, _rename_args(file=str(src))) + assert ret == 1 + assert src.is_file() + captured = capsys.readouterr() + combined = (captured.out + captured.err).lower() + assert "failed" in combined