From 5a3c9968eeadf4abfbd0735d1b95cc2082f1856b Mon Sep 17 00:00:00 2001 From: Tomek Mrugalski Date: Wed, 6 May 2026 21:32:25 +0200 Subject: [PATCH 01/11] Basic installation script --- bin/hevelius-runner.cmd | 5 +++ bin/install-hevelius.ps1 | 66 ++++++++++++++++++++++++++++++++++++++++ bin/open-hevelius.ps1 | 15 +++++++++ doc/install.md | 6 ++++ 4 files changed, 92 insertions(+) create mode 100644 bin/hevelius-runner.cmd create mode 100644 bin/install-hevelius.ps1 create mode 100644 bin/open-hevelius.ps1 diff --git a/bin/hevelius-runner.cmd b/bin/hevelius-runner.cmd new file mode 100644 index 0000000..2eff128 --- /dev/null +++ b/bin/hevelius-runner.cmd @@ -0,0 +1,5 @@ +@echo off +setlocal +set "SCRIPT_DIR=%~dp0" +set "PROJECT_ROOT=%SCRIPT_DIR%.." +"%PROJECT_ROOT%\venv\Scripts\python.exe" "%PROJECT_ROOT%\src\hevelius-runner.py" %* diff --git a/bin/install-hevelius.ps1 b/bin/install-hevelius.ps1 new file mode 100644 index 0000000..55a93f6 --- /dev/null +++ b/bin/install-hevelius.ps1 @@ -0,0 +1,66 @@ +param( + [string]$ShortcutName = "Hevelius Runner" +) + +$ErrorActionPreference = "Stop" + +$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path +$binDir = Join-Path $repoRoot "bin" + +if (-not (Test-Path $binDir)) { + New-Item -Path $binDir -ItemType Directory | Out-Null +} + +$runnerCmdPath = Join-Path $binDir "hevelius-runner.cmd" +$openShellPath = Join-Path $binDir "open-hevelius.ps1" + +$runnerCmdContent = @' +@echo off +setlocal +set "SCRIPT_DIR=%~dp0" +set "PROJECT_ROOT=%SCRIPT_DIR%.." +"%PROJECT_ROOT%\venv\Scripts\python.exe" "%PROJECT_ROOT%\src\hevelius-runner.py" %* +'@ + +$openShellContent = @' +$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path +Set-Location $repoRoot + +$activateScript = Join-Path $repoRoot "venv\Scripts\Activate.ps1" +if (Test-Path $activateScript) { + . $activateScript +} +else { + Write-Warning "Virtual environment activation script not found at: $activateScript" +} + +$runnerCmd = Join-Path $repoRoot "bin\hevelius-runner.cmd" +Set-Alias -Name hevelius-runner -Value $runnerCmd -Scope Global + +Write-Host "Hevelius runner shell ready. Use: hevelius-runner " -ForegroundColor Green +'@ + +Set-Content -Path $runnerCmdPath -Value $runnerCmdContent -Encoding ascii +Set-Content -Path $openShellPath -Value $openShellContent -Encoding utf8 + +$startMenuPrograms = [Environment]::GetFolderPath("Programs") +$shortcutPath = Join-Path $startMenuPrograms "$ShortcutName.lnk" +$powershellExe = "powershell.exe" +$shortcutArgs = "-NoExit -ExecutionPolicy Bypass -File `"$openShellPath`"" + +$wshShell = New-Object -ComObject WScript.Shell +$shortcut = $wshShell.CreateShortcut($shortcutPath) +$shortcut.TargetPath = $powershellExe +$shortcut.Arguments = $shortcutArgs +$shortcut.WorkingDirectory = $repoRoot +$shortcut.IconLocation = "$env:SystemRoot\System32\WindowsPowerShell\v1.0\powershell.exe,0" +$shortcut.Save() + +Write-Host "Installed launcher scripts:" -ForegroundColor Green +Write-Host " $runnerCmdPath" +Write-Host " $openShellPath" +Write-Host "" +Write-Host "Start Menu shortcut created:" -ForegroundColor Green +Write-Host " $shortcutPath" +Write-Host "" +Write-Host "Open Start Menu, find '$ShortcutName', and pin it if desired." diff --git a/bin/open-hevelius.ps1 b/bin/open-hevelius.ps1 new file mode 100644 index 0000000..acf63b3 --- /dev/null +++ b/bin/open-hevelius.ps1 @@ -0,0 +1,15 @@ +$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path +Set-Location $repoRoot + +$activateScript = Join-Path $repoRoot "venv\Scripts\Activate.ps1" +if (Test-Path $activateScript) { + . $activateScript +} +else { + Write-Warning "Virtual environment activation script not found at: $activateScript" +} + +$runnerCmd = Join-Path $repoRoot "bin\hevelius-runner.cmd" +Set-Alias -Name hevelius-runner -Value $runnerCmd -Scope Global + +Write-Host "Hevelius runner shell ready. Use: hevelius-runner " -ForegroundColor Green diff --git a/doc/install.md b/doc/install.md index f9f1079..857722f 100644 --- a/doc/install.md +++ b/doc/install.md @@ -14,6 +14,8 @@ This software requires Python. Once python is installed, get the sources into a To check if it works, you can run `python src/hevelius-runner.py doctor`. +Optionally, you can run bin/install-hevelius.ps1. This should create a hevelius runner +shortcut in the start menu. # Running hevelius-runner @@ -23,3 +25,7 @@ To check if it works, you can run `python src/hevelius-runner.py doctor`. 3. venv\Scripts\activate 4. python src/hevelius-runner.py ``` + +or + +click on the hevelius runner if you ran the install-hevelius script. \ No newline at end of file From a2a5a548c64448d16687cbfded0d167d5caaa357 Mon Sep 17 00:00:00 2001 From: Tomek Mrugalski Date: Wed, 6 May 2026 22:19:00 +0200 Subject: [PATCH 02/11] Excludes for directories --- CHANGELOG.md | 4 + config/config.yaml.example | 5 + doc/install.md | 17 +++- src/cmd_volumes.py | 146 +++++++++++++++++++++++++--- tests/test_cmd_repo_sanity_files.py | 52 +++++++++- 5 files changed, 205 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ef4ab2..79832dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Hevelius Runner Changelog +0.3.0 (unreleased) + +- Exclude patterns - specified files or directories can be excluded from the volumes handling. + 0.2.0 (2026-04-28) - YAML configuration with `run`, `config`, and `doctor` subcommands; redacted `config` output diff --git a/config/config.yaml.example b/config/config.yaml.example index 2b6ef6f..69f31cc 100644 --- a/config/config.yaml.example +++ b/config/config.yaml.example @@ -12,6 +12,11 @@ paths: repo-path: data/repo template_dir: templates output_dir: output + # Optional full-path exclusion patterns for volumes/list/file scans. + # If a file's full path matches any pattern, it is ignored completely. + exclude_patterns: + - '*FLAT*' + - '*shit*' # One or more FITS watch locations; each volume can define an optional nickname. volumes: - path: 'C:\astro\fits-live' diff --git a/doc/install.md b/doc/install.md index 857722f..488260d 100644 --- a/doc/install.md +++ b/doc/install.md @@ -28,4 +28,19 @@ shortcut in the start menu. or -click on the hevelius runner if you ran the install-hevelius script. \ No newline at end of file +click on the hevelius runner if you ran the install-hevelius script. + +# Excluding files from volumes scan + +`hevelius-runner volumes` can skip files based on full-path patterns from +`config/config.yaml`: + +```yaml +paths: + exclude_patterns: + - '*FLAT*' + - '*shit*' +``` + +If a discovered file path matches any pattern, that file is ignored completely +and is not processed (no header read, no task/project updates). \ No newline at end of file diff --git a/src/cmd_volumes.py b/src/cmd_volumes.py index f60f47f..90414a9 100644 --- a/src/cmd_volumes.py +++ b/src/cmd_volumes.py @@ -5,6 +5,7 @@ import logging import os import sys +import fnmatch from typing import Any, Dict, List, Optional, Tuple from fits import ( @@ -137,6 +138,65 @@ def _monitor_volumes_from_config(cm: ConfigManager) -> List[Tuple[str, str]]: return [] +def _exclude_patterns_from_config(cm: ConfigManager) -> List[str]: + """ + Return path exclusion patterns from config. + + Reads ``paths.exclude_patterns`` and supports both a single string and list + of strings. Patterns are matched against normalized full file paths. + """ + paths = cm.get_paths_config() + raw = paths.get("exclude_patterns") + if raw is None: + return [] + if isinstance(raw, str): + pattern = raw.strip() + return [pattern] if pattern else [] + if isinstance(raw, list): + out: List[str] = [] + for idx, item in enumerate(raw): + if item is None: + continue + if not isinstance(item, str): + print( + f"paths.exclude_patterns[{idx}] must be a string pattern.", + file=sys.stderr, + ) + continue + p = item.strip() + if p: + out.append(p) + return out + print("paths.exclude_patterns must be a string or list of strings.", file=sys.stderr) + return [] + + +def _normalize_full_path(path: str) -> str: + return os.path.normcase(os.path.normpath(os.path.abspath(path))) + + +def _is_excluded(full_path: str, exclude_patterns: List[str]) -> bool: + """ + True if the normalized full path matches any exclusion pattern. + + Matching rules: + - fnmatch against normalized full path + - plain substring fallback against normalized full path + """ + if not exclude_patterns: + return False + normalized = _normalize_full_path(full_path) + for pattern in exclude_patterns: + candidate = os.path.normcase(pattern.strip()) + if not candidate: + continue + if fnmatch.fnmatch(normalized, candidate): + return True + if candidate in normalized: + return True + return False + + def _scope_id_from_config(cm: ConfigManager) -> Optional[int]: raw = cm.get_api_config().get("scope_id") if raw is None or not str(raw).strip(): @@ -369,6 +429,7 @@ def process_fits_list( projects: Optional[List[Dict[str, Any]]] = None, project_stats: Optional[Dict[int, Dict[str, Any]]] = None, project_tracker: Optional[Dict[str, int]] = None, + exclude_patterns: Optional[List[str]] = None, ) -> None: """ Processes all FITS files listed in a specified text file. @@ -390,10 +451,16 @@ def process_fits_list( # Skip empty and commented out lines continue - print(f"Processing file {cnt} of {total}: {line}") + full_path = _normalize_full_path(line) + if _is_excluded(full_path, exclude_patterns or []): + print(f"Ignoring file {cnt} of {total} (excluded): {full_path}") + cnt += 1 + continue + + print(f"Processing file {cnt} of {total}: {full_path}") process_fits_file( client, - line, + full_path, show_hdr=show_hdr, update_task=update_task, projects=projects, @@ -411,6 +478,7 @@ def process_fits_dir( projects: Optional[List[Dict[str, Any]]] = None, project_stats: Optional[Dict[int, Dict[str, Any]]] = None, project_tracker: Optional[Dict[str, int]] = None, + exclude_patterns: Optional[List[str]] = None, ) -> None: """ Processes all FITS files in specified directory. @@ -433,10 +501,16 @@ def process_fits_dir( total = len(files) for f in files: - print(f"Processing file {cnt} of {total}: {f}") + full_path = _normalize_full_path(str(f)) + if _is_excluded(full_path, exclude_patterns or []): + print(f"Ignoring file {cnt} of {total} (excluded): {full_path}") + cnt += 1 + continue + + print(f"Processing file {cnt} of {total}: {full_path}") process_fits_file( client, - str(f), + full_path, show_hdr, update_task=update_task, projects=projects, @@ -629,6 +703,9 @@ def sanity_files(cm: ConfigManager, args) -> int: update_task = bool(getattr(args, "tasks", False)) use_projects = bool(getattr(args, "projects", False)) collect_orphans = bool(getattr(args, "orphans", False)) + exclude_patterns = _exclude_patterns_from_config(cm) + if exclude_patterns: + print(f"Exclude patterns enabled ({len(exclude_patterns)}): {exclude_patterns}") if collect_orphans and not use_projects: print("--orphans requires --projects.", file=sys.stderr) return 1 @@ -669,11 +746,15 @@ def _project_kwargs() -> Dict[str, Any]: } if args.file: - print(f"Processing single file: {args.file}") + full_path = _normalize_full_path(args.file) + if _is_excluded(full_path, exclude_patterns): + print(f"Ignoring single file (excluded): {full_path}") + return 0 + print(f"Processing single file: {full_path}") if update_task: - process_fits_file(client, args.file, show_hdr=args.show_header, update_task=True, **_project_kwargs()) + process_fits_file(client, full_path, show_hdr=args.show_header, update_task=True, **_project_kwargs()) else: - process_fits_file(client, args.file, show_hdr=args.show_header, **_project_kwargs()) + process_fits_file(client, full_path, show_hdr=args.show_header, **_project_kwargs()) if use_projects: _print_project_stats( project_stats, @@ -687,9 +768,22 @@ def _project_kwargs() -> Dict[str, Any]: if args.list: print(f"Processing list of files stored in {args.list}") if update_task: - process_fits_list(client, args.list, show_hdr=args.show_header, update_task=True, **_project_kwargs()) + process_fits_list( + client, + args.list, + show_hdr=args.show_header, + update_task=True, + exclude_patterns=exclude_patterns, + **_project_kwargs(), + ) else: - process_fits_list(client, args.list, show_hdr=args.show_header, **_project_kwargs()) + process_fits_list( + client, + args.list, + show_hdr=args.show_header, + exclude_patterns=exclude_patterns, + **_project_kwargs(), + ) if use_projects: _print_project_stats( project_stats, @@ -704,9 +798,22 @@ def _project_kwargs() -> Dict[str, Any]: path = args.dir print(f"Processing all files in dir: {path}") if update_task: - process_fits_dir(client, path, show_hdr=args.show_header, update_task=True, **_project_kwargs()) + process_fits_dir( + client, + path, + show_hdr=args.show_header, + update_task=True, + exclude_patterns=exclude_patterns, + **_project_kwargs(), + ) else: - process_fits_dir(client, path, show_hdr=args.show_header, **_project_kwargs()) + process_fits_dir( + client, + path, + show_hdr=args.show_header, + exclude_patterns=exclude_patterns, + **_project_kwargs(), + ) if use_projects: _print_project_stats( project_stats, @@ -733,9 +840,22 @@ def _project_kwargs() -> Dict[str, Any]: for path, nickname in volumes: print(f"Volume '{nickname}': {path}") if update_task: - process_fits_dir(client, path, show_hdr=args.show_header, update_task=True, **_project_kwargs()) + process_fits_dir( + client, + path, + show_hdr=args.show_header, + update_task=True, + exclude_patterns=exclude_patterns, + **_project_kwargs(), + ) else: - process_fits_dir(client, path, show_hdr=args.show_header, **_project_kwargs()) + process_fits_dir( + client, + path, + show_hdr=args.show_header, + exclude_patterns=exclude_patterns, + **_project_kwargs(), + ) if use_projects: _print_project_stats( project_stats, diff --git a/tests/test_cmd_repo_sanity_files.py b/tests/test_cmd_repo_sanity_files.py index a49a5f5..6183f80 100644 --- a/tests/test_cmd_repo_sanity_files.py +++ b/tests/test_cmd_repo_sanity_files.py @@ -1,4 +1,5 @@ from argparse import Namespace +import builtins import cmd_volumes @@ -58,8 +59,8 @@ def test_sanity_files_list_file(monkeypatch): monkeypatch.setattr( cmd_volumes, "process_fits_list", - lambda client, fname, show_hdr=False: calls.append( - (client, fname, show_hdr) + lambda client, fname, show_hdr=False, **kwargs: calls.append( + (client, fname, show_hdr, kwargs) ), ) @@ -77,8 +78,8 @@ def test_sanity_files_specific_directory(monkeypatch): monkeypatch.setattr( cmd_volumes, "process_fits_dir", - lambda client, path, show_hdr=False: calls.append( - (client, path, show_hdr) + lambda client, path, show_hdr=False, **kwargs: calls.append( + (client, path, show_hdr, kwargs) ), ) @@ -99,7 +100,7 @@ def test_sanity_files_defaults_to_all_configured_volumes(monkeypatch): monkeypatch.setattr( cmd_volumes, "process_fits_dir", - lambda client, path, show_hdr=False: calls.append(path), + lambda client, path, show_hdr=False, **kwargs: calls.append(path), ) ret = cmd_volumes.sanity_files(cm, _base_args()) @@ -107,6 +108,47 @@ def test_sanity_files_defaults_to_all_configured_volumes(monkeypatch): assert calls == [r"c:\astro\live", r"d:\astro\archive"] +def test_process_fits_list_skips_excluded_full_path(monkeypatch): + calls = [] + + monkeypatch.setattr( + cmd_volumes, + "process_fits_file", + lambda client, fname, show_hdr=False, update_task=False, **kwargs: calls.append(fname), + ) + + listed = [ + r"C:\astro\night\good_001.fits", + r"C:\astro\night\FLAT_001.fits", + r"C:\astro\night\also_shit_001.fits", + ] + + def _fake_open(*args, **kwargs): + class _F: + def __enter__(self_inner): + return self_inner + + def __exit__(self_inner, exc_type, exc, tb): + return False + + def readlines(self_inner): + return [f"{line}\n" for line in listed] + + return _F() + + monkeypatch.setattr(builtins, "open", _fake_open) + + cmd_volumes.process_fits_list( + client=object(), + fname="ignored.txt", + show_hdr=False, + exclude_patterns=["*FLAT*", "*shit*"], + ) + + assert len(calls) == 1 + assert calls[0].lower().endswith(r"good_001.fits") + + def test_sanity_files_with_project_prefetches_and_prints_stats(monkeypatch, capsys): cm = _DummyConfigManager() calls = [] From 4d0f93e1a4bbf0ea3d7b9ba6c6f375ba0fc67b0a Mon Sep 17 00:00:00 2001 From: Tomek Mrugalski Date: Thu, 7 May 2026 00:25:38 +0200 Subject: [PATCH 03/11] Avoid overwriting goalcount in subframes --- src/cmd_volumes.py | 7 ++++--- tests/test_cmd_repo_sanity_files.py | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/cmd_volumes.py b/src/cmd_volumes.py index 90414a9..11123e1 100644 --- a/src/cmd_volumes.py +++ b/src/cmd_volumes.py @@ -367,7 +367,6 @@ def _sync_project_stats_to_server(client: APIClient, project_stats: Dict[int, Di "filter": str(filter_name), "exposure_time": exposure_f, "count": int(count), - "goal_count": int(count), } try: resp = client.session.post( @@ -379,12 +378,14 @@ def _sync_project_stats_to_server(client: APIClient, project_stats: Dict[int, Di resp.raise_for_status() print( f"Synced project {project_name!r}: created subframe " - f"filter={filter_name} exposure={exposure_f} count={count} goal_count={count}" + f"filter={filter_name} exposure={exposure_f} count={count}" ) except Exception as e: print( f"Failed creating subframe for project {project_name!r} " - f"(filter={filter_name}, exposure={exposure_f}): {e}", + f"(filter={filter_name}, exposure={exposure_f}): {e}. " + "Backend note: allow creating project subframes with " + "filter, exposure_time, and count only (without goal_count/active).", file=sys.stderr, ) ok = False diff --git a/tests/test_cmd_repo_sanity_files.py b/tests/test_cmd_repo_sanity_files.py index 6183f80..2b504ac 100644 --- a/tests/test_cmd_repo_sanity_files.py +++ b/tests/test_cmd_repo_sanity_files.py @@ -273,7 +273,8 @@ def test_sync_project_stats_creates_missing_subframe(): assert session.post_calls[0][0].endswith("/projects/42/subframes") assert session.post_calls[0][1]["filter"] == "Ha" assert session.post_calls[0][1]["count"] == 3 - assert session.post_calls[0][1]["goal_count"] == 3 + assert "goal_count" not in session.post_calls[0][1] + assert "active" not in session.post_calls[0][1] assert session.patch_calls == [] From 176f22493f3a3b70833ef54a4cfce6d4ab1c9f29 Mon Sep 17 00:00:00 2001 From: Tomek Mrugalski Date: Fri, 8 May 2026 00:13:25 +0200 Subject: [PATCH 04/11] Conditional subframes update --- src/cmd_volumes.py | 389 ++++++++++++++++++++-------- tests/test_cmd_repo_sanity_files.py | 145 +++++++++-- 2 files changed, 411 insertions(+), 123 deletions(-) diff --git a/src/cmd_volumes.py b/src/cmd_volumes.py index 11123e1..e480507 100644 --- a/src/cmd_volumes.py +++ b/src/cmd_volumes.py @@ -18,6 +18,40 @@ from config_manager import ConfigManager +# ---- Color helpers -------------------------------------------------------- +# ANSI escape codes are emitted only when stdout is a real TTY; when output is +# piped to a file or a non-tty terminal, colors collapse to empty strings to +# avoid littering logs with control sequences. + +_GREEN = "\033[92m" +_YELLOW = "\033[93m" +_RED = "\033[91m" +# Orange is a 256-color extension; falls back gracefully on terminals that do +# not understand the sequence (most modern terminals do). +_ORANGE = "\033[38;5;208m" +_DIM = "\033[2m" +_RESET = "\033[0m" + + +def _supports_color() -> bool: + """True if stdout looks like an interactive terminal that supports ANSI. + + Honors the NO_COLOR convention (https://no-color.org/). Re-evaluated on + every call so tests that capture stdout (which is not a TTY) see plain + ASCII output without depending on import-time state. + """ + if os.environ.get("NO_COLOR"): + return False + stream = sys.stdout + return hasattr(stream, "isatty") and stream.isatty() + + +def _color(code: str, text: str) -> str: + if not code or not _supports_color(): + return text + return f"{code}{text}{_RESET}" + + def get_tasks_files_list(client: APIClient) -> List[Tuple[int, Optional[str]]]: """ Return every task as ``(task_id, imagename)`` from the API (JWT required). @@ -273,6 +307,9 @@ def _print_project_stats( files_without_project: int, files_without_project_names: Optional[List[str]] = None, ) -> None: + """Print the orphan-files header. Per-bucket details (with color and the + add/update/skip action) are printed by :func:`_sync_project_stats_to_server`, + which is the only code path that knows the action that was actually taken.""" print() print("=== PROJECT STATISTICS ===") print(f"Files without project match: {files_without_project}") @@ -281,22 +318,6 @@ def _print_project_stats( print(f" {name}") if not project_stats: print("No project subframes were matched.") - return - for project_id in sorted(project_stats.keys()): - entry = project_stats[project_id] - project_name = entry.get("name") or f"project-{project_id}" - print(f"Project: {project_name}") - buckets = entry.get("buckets", {}) - total = sum(buckets.values()) - print(f" Total matched files: {total}") - for (filter_name, exposure), count in sorted( - buckets.items(), - key=lambda item: ( - "" if item[0][0] is None else str(item[0][0]), - -1.0 if item[0][1] is None else float(item[0][1]), - ), - ): - print(f" filter={filter_name} exposure={exposure} -> {count}") def _subframe_filter_name(subframe: Dict[str, Any]) -> Optional[str]: @@ -330,22 +351,158 @@ def _match_subframe( return None +def _fetch_project_fresh(client: APIClient, project_id: int) -> Optional[Dict[str, Any]]: + """GET /api/projects/{id} just before sync so we compare against current + server state (count/goal_count may have moved while we were scanning). + + Returns the project dict on success, or None when the call fails (the caller + falls back to the cached project data from the initial list fetch).""" + try: + url = f"{client.base_url.rstrip('/')}/projects/{int(project_id)}" + resp = client.session.get( + url, + timeout=client.timeout, + headers=client._get_auth_headers(), + ) + resp.raise_for_status() + data = resp.json() + if isinstance(data, dict): + project = data.get("project") + if isinstance(project, dict): + return project + except Exception: + return None + return None + + +def _bucket_action_color(action: str, count: int, goal_count: Optional[int]) -> str: + """Pick a color for a per-bucket sync result line. + + skipped (no change) -> orange + added/updated, count < goal -> yellow + added/updated, count >= goal -> green + failed -> red + """ + if action == "skipped": + return _ORANGE + if action == "failed": + return _RED + if goal_count is None or goal_count <= 0: + return _GREEN + return _GREEN if count >= goal_count else _YELLOW + + +def _sync_one_bucket( + client: APIClient, + project_id: int, + project_name: str, + matched: Optional[Dict[str, Any]], + filter_name: str, + exposure_f: float, + count: int, +) -> Tuple[str, Optional[int], Optional[str]]: + """Sync a single (filter, exposure, count) bucket. + + Returns ``(action, goal_count, error)``: + - action: ``added``, ``updated``, ``skipped``, or ``failed`` + - goal_count: server-side goal_count when known (used for color logic) + - error: optional error message when action is ``failed`` + """ + if matched is None: + url = f"{client.base_url.rstrip('/')}/projects/{project_id}/subframes" + payload = { + "filter": str(filter_name), + "exposure_time": exposure_f, + "count": int(count), + } + try: + resp = client.session.post( + url, json=payload, timeout=client.timeout, + headers=client._get_auth_headers(), + ) + resp.raise_for_status() + except Exception as e: + return "failed", None, f"creating subframe failed: {e}" + return "added", None, None + + subframe_id = matched.get("id") + if subframe_id is None: + return "failed", None, "matched subframe has no id" + + matched_count = matched.get("count") + goal_count = matched.get("goal_count") + try: + goal_count_int = int(goal_count) if goal_count is not None else None + except (TypeError, ValueError): + goal_count_int = None + + # Skip when the captured count on the server already matches what we'd + # send. This avoids spurious last_updated bumps and keeps logs clean for + # incremental rescans of the same volume. + try: + if matched_count is not None and int(matched_count) == int(count): + return "skipped", goal_count_int, None + except (TypeError, ValueError): + pass + + url = f"{client.base_url.rstrip('/')}/projects/{project_id}/subframes/{subframe_id}" + payload = {"count": int(count)} + try: + resp = client.session.patch( + url, json=payload, timeout=client.timeout, + headers=client._get_auth_headers(), + ) + resp.raise_for_status() + except Exception as e: + return "failed", goal_count_int, f"updating subframe {subframe_id} failed: {e}" + return "updated", goal_count_int, None + + def _sync_project_stats_to_server(client: APIClient, project_stats: Dict[int, Dict[str, Any]]) -> bool: + """Sync collected per-project subframe counts to the server. + + For each project, the runner first re-fetches the project (so comparisons + against ``count``/``goal_count`` use fresh data), then for every bucket: + + * creates a new subframe if no matching (filter, exposure_time) row exists; + * sends a PATCH with only ``count`` if the server count differs; + * skips the call entirely when the server count already matches. + + Each per-bucket result is printed on a single color-coded line. + """ ok = True + if not project_stats: + return ok + for project_id in sorted(project_stats.keys()): entry = project_stats[project_id] project_name = entry.get("name") or f"project-{project_id}" - project = entry.get("project") or {} - subframes = project.get("subframes") if isinstance(project, dict) else None + + # Fetch fresh server state for this project; fall back to the cached + # copy from the initial listing if the GET fails. + fresh = _fetch_project_fresh(client, project_id) + if fresh is None: + fresh = entry.get("project") or {} + subframes = fresh.get("subframes") if isinstance(fresh, dict) else None if not isinstance(subframes, list): subframes = [] - for (filter_name, exposure), count in entry.get("buckets", {}).items(): + print(f"Project: {project_name}") + buckets = entry.get("buckets", {}) + total_files = sum(buckets.values()) + print(f" Total matched files: {total_files}") + + for (filter_name, exposure), count in sorted( + buckets.items(), + key=lambda item: ( + "" if item[0][0] is None else str(item[0][0]), + -1.0 if item[0][1] is None else float(item[0][1]), + ), + ): if filter_name is None or exposure is None: print( - f"Skipping subframe sync for project {project_name!r}: " - f"incomplete bucket filter={filter_name} exposure={exposure}", - file=sys.stderr, + f" {_color(_RED, '[failed ]')} incomplete bucket " + f"filter={filter_name} exposure={exposure} count={count}" ) ok = False continue @@ -353,73 +510,31 @@ def _sync_project_stats_to_server(client: APIClient, project_stats: Dict[int, Di exposure_f = float(exposure) except (TypeError, ValueError): print( - f"Skipping subframe sync for project {project_name!r}: " - f"invalid exposure {exposure!r}", - file=sys.stderr, + f" {_color(_RED, '[failed ]')} invalid exposure " + f"filter={filter_name} exposure={exposure!r} count={count}" ) ok = False continue matched = _match_subframe(subframes, str(filter_name), exposure_f) - if matched is None: - url = f"{client.base_url.rstrip('/')}/projects/{project_id}/subframes" - payload = { - "filter": str(filter_name), - "exposure_time": exposure_f, - "count": int(count), - } - try: - resp = client.session.post( - url, - json=payload, - timeout=client.timeout, - headers=client._get_auth_headers(), - ) - resp.raise_for_status() - print( - f"Synced project {project_name!r}: created subframe " - f"filter={filter_name} exposure={exposure_f} count={count}" - ) - except Exception as e: - print( - f"Failed creating subframe for project {project_name!r} " - f"(filter={filter_name}, exposure={exposure_f}): {e}. " - "Backend note: allow creating project subframes with " - "filter, exposure_time, and count only (without goal_count/active).", - file=sys.stderr, - ) - ok = False - continue - - subframe_id = matched.get("id") - if subframe_id is None: - print( - f"Failed updating subframe for project {project_name!r}: matched subframe has no id.", - file=sys.stderr, - ) + action, goal_count, error = _sync_one_bucket( + client, project_id, project_name, matched, + str(filter_name), exposure_f, int(count), + ) + if action == "failed": ok = False - continue - url = f"{client.base_url.rstrip('/')}/projects/{project_id}/subframes/{subframe_id}" - payload = {"count": int(count)} - try: - resp = client.session.patch( - url, - json=payload, - timeout=client.timeout, - headers=client._get_auth_headers(), - ) - resp.raise_for_status() - print( - f"Synced project {project_name!r}: updated subframe_id={subframe_id} " - f"filter={filter_name} exposure={exposure_f} count={count}" - ) - except Exception as e: - print( - f"Failed updating subframe {subframe_id} for project {project_name!r}: {e}", - file=sys.stderr, - ) - ok = False + color = _bucket_action_color(action, int(count), goal_count) + tag = f"[{action:<9}]" + line = ( + f" {tag} filter={filter_name} exposure={exposure_f} " + f"count={count}" + ) + if goal_count is not None: + line += f" goal_count={goal_count}" + if error: + line += f" ({error})" + print(_color(color, line)) return ok def process_fits_list( @@ -449,16 +564,17 @@ def process_fits_list( for line in lines: line = line.strip() if len(line) == 0 or line[0] == "#": - # Skip empty and commented out lines + # Skip empty and commented out lines (do not bump the counter so + # the user-visible "n of total" tracks file rows, not blank ones). continue full_path = _normalize_full_path(line) if _is_excluded(full_path, exclude_patterns or []): - print(f"Ignoring file {cnt} of {total} (excluded): {full_path}") + progress = _format_progress(cnt, total) + print(f"{progress}{_format_status_tag('skipped')} {full_path} (excluded)") cnt += 1 continue - print(f"Processing file {cnt} of {total}: {full_path}") process_fits_file( client, full_path, @@ -467,6 +583,8 @@ def process_fits_list( projects=projects, project_stats=project_stats, project_tracker=project_tracker, + idx=cnt, + total=total, ) cnt += 1 @@ -491,7 +609,6 @@ def process_fits_dir( base = os.path.normpath(dir) pattern_fit = os.path.join(base, "**", "*.fit") pattern_fits = os.path.join(base, "**", "*.fits") - print(f"patterns={pattern_fit!r}, {pattern_fits!r}") found_fit = glob.glob(pattern_fit, recursive=True) found_fits = glob.glob(pattern_fits, recursive=True) files = sorted(set(found_fit) | set(found_fits)) @@ -504,11 +621,11 @@ def process_fits_dir( for f in files: full_path = _normalize_full_path(str(f)) if _is_excluded(full_path, exclude_patterns or []): - print(f"Ignoring file {cnt} of {total} (excluded): {full_path}") + progress = _format_progress(cnt, total) + print(f"{progress}{_format_status_tag('skipped')} {full_path} (excluded)") cnt += 1 continue - print(f"Processing file {cnt} of {total}: {full_path}") process_fits_file( client, full_path, @@ -517,16 +634,65 @@ def process_fits_dir( projects=projects, project_stats=project_stats, project_tracker=project_tracker, + idx=cnt, + total=total, ) cnt += 1 +def _format_progress(idx: Optional[int], total: Optional[int]) -> str: + """Render the per-file progress prefix (e.g. ``[ 12/345]``) or empty string.""" + if idx is None or total is None or total <= 0: + return "" + width = max(1, len(str(total))) + return f"[{idx:>{width}}/{total}] " + + +def _format_status_tag(status: str) -> str: + """ + Render a fixed-width, color-coded status tag for compact per-file logging. + + matched -> green (file matched a project) + unmatched -> red (no project matched) + skipped -> yellow (file was excluded or otherwise not processed) + """ + label = f"{status:<9}" + if status == "matched": + return f"[{_color(_GREEN, label)}]" + if status == "unmatched": + return f"[{_color(_RED, label)}]" + if status == "skipped": + return f"[{_color(_YELLOW, label)}]" + return f"[{label}]" + + +def _format_file_params(filter_name: Optional[str], exposure: Optional[float], + object_name: Optional[str]) -> str: + """Render ``filter=… exposure=… object=…`` skipping fields that are None.""" + parts = [] + if filter_name is not None: + parts.append(f"filter={filter_name}") + if exposure is not None: + parts.append(f"exposure={exposure}") + if object_name is not None: + parts.append(f"object={object_name}") + return " ".join(parts) + + def process_fits_file(client: APIClient, fname, show_hdr: bool, verbose: bool = False, update_task: bool = False, projects: Optional[List[Dict[str, Any]]] = None, project_stats: Optional[Dict[int, Dict[str, Any]]] = None, - project_tracker: Optional[Dict[str, Any]] = None): - """Processes a FITS file: optional header dump and task lookup via the API.""" + project_tracker: Optional[Dict[str, Any]] = None, + idx: Optional[int] = None, + total: Optional[int] = None): + """Processes a FITS file: optional header dump and task lookup via the API. + + Emits a single status line per file with a color-coded tag: + ``matched`` (green), ``unmatched`` (red), or ``skipped`` (yellow). The + ``idx``/``total`` parameters are optional and only used to render the + progress prefix when called from a batch processor. + """ key = os.path.basename(fname) @@ -536,44 +702,59 @@ def process_fits_file(client: APIClient, fname, show_hdr: bool, verbose: bool = object = _h_str(h, "OBJECT") exposure = _h_float(h, "EXPTIME") + task = None if update_task: task = get_task_by_filename(client, key) - if task: - tid, imagename = task - print(f" Task found: task_id={tid} imagename={imagename!r} (matched suffix {key!r}), filter={filter}, object={object}") - else: - print(f" No task found for filename suffix {key!r}, filter={filter}, object={object}") + project = None if projects is not None: project = _find_project_for_filename(key, projects) if project is not None: - project_name = _project_name(project) - print(f" Project found: {project_name!r}") if project_stats is not None: _update_project_stats(project_stats, project, filter, exposure) else: - print(" Project not found") if project_tracker is not None: project_tracker["files_without_project"] = project_tracker.get("files_without_project", 0) + 1 files_without_project_names = project_tracker.get("files_without_project_names") if isinstance(files_without_project_names, list): files_without_project_names.append(key) + # Build the compact one-line summary for this file. + if projects is not None: + if project is not None: + status = "matched" + else: + status = "unmatched" + else: + # Without --projects, fall back to the task status if available, + # otherwise mark as matched (we still have a file on disk). + status = "matched" if (not update_task or task) else "unmatched" + + extras: List[str] = [] + if project is not None: + extras.append(f"project={_project_name(project)!r}") + if update_task: + if task: + extras.append(f"task_id={task[0]}") + else: + extras.append("task=none") + + params = _format_file_params(filter, exposure, object) + progress = _format_progress(idx, total) + tag = _format_status_tag(status) + extras_s = (" " + " ".join(extras)) if extras else "" + params_s = (" " + params) if params else "" + print(f"{progress}{tag} {fname}{params_s}{extras_s}") + if show_hdr: for k in h.keys(): print(f" {k}: {h[k]}") - # OK, so we have a file on disk and there might or might not be a task for it. - # TODO: add ability to insert new or update existing task. - if update_task: if task: task_update(client, fname, task[0], verbose=verbose) else: task_add(client, fname, verbose=verbose) - else: - print(" Task DB update skipped (pass --task to enable API upsert).") - return def _h_str(h, key: str) -> Optional[str]: diff --git a/tests/test_cmd_repo_sanity_files.py b/tests/test_cmd_repo_sanity_files.py index 2b504ac..5fcd0a3 100644 --- a/tests/test_cmd_repo_sanity_files.py +++ b/tests/test_cmd_repo_sanity_files.py @@ -185,8 +185,10 @@ def _fake_process_dir(client, path, show_hdr=False, **kwargs): assert calls[0][1]["projects"] == [{"project_id": 42, "name": "M42", "subframes": []}] out = capsys.readouterr().out assert "PROJECT STATISTICS" in out - assert "Project: M42" in out - assert "filter=Ha exposure=300.0 -> 2" in out + assert "Files without project match: 1" in out + # Per-project bucket lines are emitted by _sync_project_stats_to_server + # which is mocked away in this test, so we only assert the orphan-files + # header section here. def test_process_fits_file_project_found_updates_stats(monkeypatch, capsys): @@ -204,7 +206,9 @@ def test_process_fits_file_project_found_updates_stats(monkeypatch, capsys): project_tracker=tracker, ) out = capsys.readouterr().out - assert "Project found: 'M42'" in out + assert "[matched" in out + assert "project='M42'" in out + assert "M42_001.fits" in out assert project_stats[42]["buckets"][("Ha", 300.0)] == 1 assert tracker["files_without_project"] == 0 @@ -224,20 +228,35 @@ def test_process_fits_file_project_not_found_tracks_counter(monkeypatch, capsys) project_tracker=tracker, ) out = capsys.readouterr().out - assert "Project not found" in out + assert "[unmatched" in out + assert "unknown_001.fits" in out assert project_stats == {} assert tracker["files_without_project"] == 1 class _FakeResponse: + def __init__(self, payload=None): + self._payload = payload or {} + def raise_for_status(self): return None + def json(self): + return self._payload + class _FakeSession: - def __init__(self): + """Test double for ``requests.Session`` used by the runner. + + Records POST/PATCH/GET calls and lets each test inject a stable response + payload for the GET /api/projects/{id} fresh-fetch the runner now performs. + """ + + def __init__(self, get_payload=None): self.post_calls = [] self.patch_calls = [] + self.get_calls = [] + self._get_payload = get_payload def post(self, url, json=None, timeout=None, headers=None): self.post_calls.append((url, json, timeout, headers)) @@ -247,10 +266,13 @@ def patch(self, url, json=None, timeout=None, headers=None): self.patch_calls.append((url, json, timeout, headers)) return _FakeResponse() + def get(self, url, timeout=None, headers=None): + self.get_calls.append((url, timeout, headers)) + return _FakeResponse(self._get_payload) -def test_sync_project_stats_creates_missing_subframe(): - session = _FakeSession() - client = type( + +def _make_client(session): + return type( "C", (), { @@ -260,6 +282,11 @@ def test_sync_project_stats_creates_missing_subframe(): "_get_auth_headers": lambda self: {"Authorization": "Bearer x"}, }, )() + + +def test_sync_project_stats_creates_missing_subframe(): + session = _FakeSession(get_payload={"status": True, "project": {"project_id": 42, "name": "M42", "subframes": []}}) + client = _make_client(session) stats = { 42: { "name": "M42", @@ -279,17 +306,14 @@ def test_sync_project_stats_creates_missing_subframe(): def test_sync_project_stats_updates_existing_subframe_count_only(): - session = _FakeSession() - client = type( - "C", - (), - { - "base_url": "https://example.test/api/", - "timeout": 5, - "session": session, - "_get_auth_headers": lambda self: {"Authorization": "Bearer x"}, - }, - )() + fresh = { + "project_id": 42, "name": "M42", + "subframes": [ + {"id": 77, "filter": {"short_name": "Ha"}, "exposure_time": 300.0, "count": 1, "goal_count": 10} + ], + } + session = _FakeSession(get_payload={"status": True, "project": fresh}) + client = _make_client(session) stats = { 42: { "name": "M42", @@ -309,3 +333,86 @@ def test_sync_project_stats_updates_existing_subframe_count_only(): assert len(session.patch_calls) == 1 assert session.patch_calls[0][0].endswith("/projects/42/subframes/77") assert session.patch_calls[0][1] == {"count": 5} + # Runner re-fetches the project right before sync so it compares against + # fresh server state instead of a possibly stale cached copy. + assert len(session.get_calls) == 1 + assert session.get_calls[0][0].endswith("/projects/42") + + +def test_sync_project_stats_skips_when_count_unchanged(capsys): + """No PATCH is issued when server count already matches the bucket count.""" + fresh = { + "project_id": 42, "name": "M42", + "subframes": [ + {"id": 77, "filter": {"short_name": "Ha"}, "exposure_time": 300.0, "count": 5, "goal_count": 10} + ], + } + session = _FakeSession(get_payload={"status": True, "project": fresh}) + client = _make_client(session) + stats = { + 42: { + "name": "M42", + "project": {"project_id": 42, "name": "M42", "subframes": fresh["subframes"]}, + "buckets": {("Ha", 300.0): 5}, + } + } + ok = cmd_volumes._sync_project_stats_to_server(client, stats) + assert ok is True + assert session.patch_calls == [] + assert session.post_calls == [] + out = capsys.readouterr().out + assert "skipped" in out + assert "goal_count=10" in out + + +def test_sync_project_stats_uses_fresh_data_when_cache_is_stale(capsys): + """The fresh GET trumps the stale cached project so we don't double-PATCH.""" + fresh = { + "project_id": 42, "name": "M42", + "subframes": [ + {"id": 77, "filter": {"short_name": "Ha"}, "exposure_time": 300.0, "count": 7, "goal_count": 10} + ], + } + session = _FakeSession(get_payload={"status": True, "project": fresh}) + client = _make_client(session) + stats = { + 42: { + "name": "M42", + # Cached count is 1 (stale) but server now reports 7 == bucket → skip. + "project": { + "project_id": 42, "name": "M42", + "subframes": [{"id": 77, "filter": {"short_name": "Ha"}, "exposure_time": 300.0, "count": 1, "goal_count": 10}], + }, + "buckets": {("Ha", 300.0): 7}, + } + } + ok = cmd_volumes._sync_project_stats_to_server(client, stats) + assert ok is True + assert session.patch_calls == [] + out = capsys.readouterr().out + assert "skipped" in out + + +def test_sync_project_stats_color_codes_at_or_above_goal(capsys): + """When count >= goal_count the line uses the green ANSI sequence.""" + fresh = { + "project_id": 42, "name": "M42", + "subframes": [ + {"id": 77, "filter": {"short_name": "Ha"}, "exposure_time": 300.0, "count": 1, "goal_count": 10} + ], + } + session = _FakeSession(get_payload={"status": True, "project": fresh}) + client = _make_client(session) + stats = { + 42: { + "name": "M42", + "project": {"project_id": 42, "name": "M42", "subframes": fresh["subframes"]}, + "buckets": {("Ha", 300.0): 12}, # exceeds goal_count=10 → green + } + } + cmd_volumes._sync_project_stats_to_server(client, stats) + assert len(session.patch_calls) == 1 + out = capsys.readouterr().out + assert "updated" in out + assert "count=12" in out + assert "goal_count=10" in out From 2fcb9f4dbf22c206457efb7194aa85870c6009f6 Mon Sep 17 00:00:00 2001 From: Tomek Mrugalski Date: Thu, 14 May 2026 09:15:16 +0200 Subject: [PATCH 05/11] doctor doesn't abort on API check fail --- src/cmd_doctor.py | 166 +++++++++++++++++++++++++++++++++++----------- tests/test_cli.py | 41 ++++++++++++ 2 files changed, 170 insertions(+), 37 deletions(-) diff --git a/src/cmd_doctor.py b/src/cmd_doctor.py index 73b8c91..5e06119 100644 --- a/src/cmd_doctor.py +++ b/src/cmd_doctor.py @@ -1,13 +1,80 @@ +import os import sys from pathlib import Path -from typing import Any, Callable +from typing import Any, Callable, Optional + from api_client import APIClient from config_manager import ConfigManager +_RED = "\033[91m" +_RESET = "\033[0m" + +# Windows: stderr may be a TTY but ANSI is ignored until VT processing is enabled +# on that console handle (classic PowerShell / conhost). +_win32_stderr_vt: Optional[bool] = None + + +def _try_enable_win32_stderr_vt() -> bool: + """Enable ANSI on the stderr console handle. Return True if escapes will work.""" + global _win32_stderr_vt + if sys.platform != "win32": + return True + if _win32_stderr_vt is not None: + return _win32_stderr_vt + err = sys.stderr + if not (hasattr(err, "isatty") and err.isatty()): + _win32_stderr_vt = False + return False + try: + import ctypes + from ctypes import wintypes + + kernel32 = ctypes.windll.kernel32 + STD_ERROR_HANDLE = -12 + ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x0004 + + handle = kernel32.GetStdHandle(STD_ERROR_HANDLE) + if handle in (-1, 0): + _win32_stderr_vt = False + return False + mode = wintypes.DWORD() + if not kernel32.GetConsoleMode(handle, ctypes.byref(mode)): + _win32_stderr_vt = False + return False + if mode.value & ENABLE_VIRTUAL_TERMINAL_PROCESSING: + _win32_stderr_vt = True + return True + if kernel32.SetConsoleMode(handle, mode.value | ENABLE_VIRTUAL_TERMINAL_PROCESSING): + _win32_stderr_vt = True + return True + except Exception: + pass + _win32_stderr_vt = False + return False + + +def _stderr_color_enabled() -> bool: + if os.environ.get("NO_COLOR"): + return False + err = sys.stderr + if not (hasattr(err, "isatty") and err.isatty()): + return False + if sys.platform == "win32" and not _try_enable_win32_stderr_vt(): + return False + return True + + +def _red(text: str) -> str: + if not _stderr_color_enabled(): + return text + return f"{_RED}{text}{_RESET}" + + def cmd_doctor(cm: ConfigManager, api_client_factory: Callable[[dict], Any] = APIClient) -> int: # The api_client_factory is used in tests, to override the actual APIClient class with a mock. + exit_code = 0 api_cfg = cm.get_api_config() required_api = ("base_url", "timeout", "username", "password") missing = [k for k in required_api if not str(api_cfg.get(k, "")).strip()] @@ -19,58 +86,83 @@ def cmd_doctor(cm: ConfigManager, api_client_factory: Callable[[dict], Any] = AP return 1 client = api_client_factory(api_cfg) + version_ok = False try: version = client.get_version() print(f"API reachable at {api_cfg['base_url'].rstrip('/')}/ - backend version: {version}") + version_ok = True except Exception as e: - print(f"API check failed (version endpoint): {e}", file=sys.stderr) - return 1 + exit_code = 1 + print( + f"Backend API: {_red('unreachable')} ({e})", + file=sys.stderr, + ) + print( + "Skipping API login and telescope checks (backend not reached).", + file=sys.stderr, + ) - try: - login = client.login() - if not login.status or not login.token: - print(f"Login failed: {login.msg or 'no JWT token in response'}", file=sys.stderr) - return 1 - print(f"Authenticated as {api_cfg['username']!r} (user id {login.user_id}).") - except Exception as e: - print(f"API check failed (login): {e}", file=sys.stderr) - return 1 + login_ok = False + if version_ok: + try: + login = client.login() + if not login.status or not login.token: + exit_code = 1 + print(f"Login failed: {login.msg or 'no JWT token in response'}", file=sys.stderr) + else: + login_ok = True + print(f"Authenticated as {api_cfg['username']!r} (user id {login.user_id}).") + except Exception as e: + exit_code = 1 + print(f"API check failed (login): {e}", file=sys.stderr) - try: - scopes = client.list_telescopes() - print(f"Telescopes available: {len(scopes)}") - sid = api_cfg.get("scope_id") - if sid is not None and str(sid).strip() != "": - want = int(sid) - ids = {int(t.get("scope_id")) for t in scopes if t.get("scope_id") is not None} - if want not in ids: + if version_ok and not login_ok: + print("Skipping telescope list (not authenticated).", file=sys.stderr) + + if version_ok and login_ok: + try: + scopes = client.list_telescopes() + print(f"Telescopes available: {len(scopes)}") + sid = api_cfg.get("scope_id") + if sid is not None and str(sid).strip() != "": + want = int(sid) + ids = {int(t.get("scope_id")) for t in scopes if t.get("scope_id") is not None} + if want not in ids: + exit_code = 1 + print( + f"Configured api.scope_id={want} is not in the telescope list from the API.", + file=sys.stderr, + ) + else: + names = [t.get("name") for t in scopes if int(t.get("scope_id", -1)) == want] + label = names[0] if names else "?" + print(f"Configured scope_id {want} OK ({label!r}).") + else: print( - f"Configured api.scope_id={want} is not in the telescope list from the API.", + "Note: api.scope_id is not set; use `telescope list` and `telescope set`.", file=sys.stderr, ) - return 1 - names = [t.get("name") for t in scopes if int(t.get("scope_id", -1)) == want] - label = names[0] if names else "?" - print(f"Configured scope_id {want} OK ({label!r}).") - else: - print( - "Note: api.scope_id is not set; use `telescope list` and `telescope set`.", - file=sys.stderr, - ) - except Exception as e: - print(f"API check failed (telescope list): {e}", file=sys.stderr) - return 1 + except Exception as e: + exit_code = 1 + print(f"API check failed (telescope list): {e}", file=sys.stderr) nina_cfg = cm.get_nina_config() exe = nina_cfg.get("executable_path", "") nina_path = Path(str(exe)) if exe else None if not nina_path or not nina_path.is_file(): + exit_code = 1 print( f"NINA executable not found or not configured: {exe!r}", file=sys.stderr, ) - return 1 - print(f"NINA executable found: {nina_path}") + else: + print(f"NINA executable found: {nina_path}") - print("Configuration and connectivity check passed.") - return 0 + if exit_code == 0: + print("Configuration and connectivity check passed.") + else: + print( + "Configuration check finished with one or more issues (see messages above).", + file=sys.stderr, + ) + return exit_code diff --git a/tests/test_cli.py b/tests/test_cli.py index 4e1d98e..678d8fe 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -109,6 +109,47 @@ def test_doctor_command_success(tmp_path, capsys, runner_mod): assert "Connectivity check passed" in out or "check passed" in out +def test_doctor_continues_when_api_unreachable(tmp_path, capsys, runner_mod): + cfg = tmp_path / "c.yaml" + fake_nina = tmp_path / "nina_fake.exe" + fake_nina.write_bytes(b"") + + cfg.write_text( + textwrap.dedent( + f""" + api: + base_url: https://example.test/api/ + timeout: 5 + username: u + password: p + verify_ssl: false + scope_id: 1 + paths: + template_dir: t + output_dir: o + nina: + executable_path: {fake_nina} + scripts: {{}} + """ + ).strip(), + encoding="utf-8", + ) + + with patch.object(runner_mod, "APIClient") as MockClient: + client = MagicMock() + client.get_version.side_effect = ConnectionError("refused") + MockClient.return_value = client + + ret = runner_mod.main(["-c", str(cfg), "doctor"]) + assert ret == 1 + captured = capsys.readouterr() + assert "unreachable" in captured.err + assert "Skipping API login" in captured.err + assert "NINA executable found" in captured.out + client.login.assert_not_called() + client.list_telescopes.assert_not_called() + + def test_version_prints_runner_version(capsys, runner_mod): ret = runner_mod.main(["version"]) assert ret == 0 From d561df8c6c8207c6bb5457679c94fa38b2d708f5 Mon Sep 17 00:00:00 2001 From: Tomek Mrugalski Date: Thu, 14 May 2026 09:15:28 +0200 Subject: [PATCH 06/11] doc cleanup --- doc/install.md | 15 --------------- doc/usage.md | 17 ++++++++++++++--- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/doc/install.md b/doc/install.md index 488260d..23965ee 100644 --- a/doc/install.md +++ b/doc/install.md @@ -29,18 +29,3 @@ shortcut in the start menu. or click on the hevelius runner if you ran the install-hevelius script. - -# Excluding files from volumes scan - -`hevelius-runner volumes` can skip files based on full-path patterns from -`config/config.yaml`: - -```yaml -paths: - exclude_patterns: - - '*FLAT*' - - '*shit*' -``` - -If a discovered file path matches any pattern, that file is ignored completely -and is not processed (no header read, no task/project updates). \ No newline at end of file diff --git a/doc/usage.md b/doc/usage.md index dc5654f..77a2983 100644 --- a/doc/usage.md +++ b/doc/usage.md @@ -76,6 +76,20 @@ task will be created. Files can be assigned to one of known projects and then th project's statistics will be updated, if `--projects` option is specified. Finally, there's `--orphans` option to list all files that were not able to assign to any project. +## Excluding files from volumes scan + +`hevelius-runner volumes` can skip files based on full-path patterns from +`config/config.yaml`: + +```yaml +paths: + exclude_patterns: + - '*FLAT*' + - '*shit*' +``` + +If a discovered file path matches any pattern, that file is ignored completely +and is not processed (no header read, no task/project updates). ## Run @@ -88,9 +102,6 @@ Run the automation loop: python src/hevelius-runner.py run ``` - - - The `run` command will: 1. Load configuration 2. Execute startup scripts From e9987e922fda178ae2f8f900de507fe3d589d875 Mon Sep 17 00:00:00 2001 From: Tomek Mrugalski Date: Fri, 15 May 2026 21:00:03 +0200 Subject: [PATCH 07/11] Print colors using colorama --- requirements.txt | 1 + src/cmd_doctor.py | 65 ++-------------------------------------- src/cmd_volumes.py | 51 +++++++------------------------ src/console_color.py | 68 ++++++++++++++++++++++++++++++++++++++++++ src/hevelius-runner.py | 3 +- 5 files changed, 85 insertions(+), 103 deletions(-) create mode 100644 src/console_color.py diff --git a/requirements.txt b/requirements.txt index 976575a..7435f9c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ PyYAML +colorama>=0.4.6 requests psycopg2-binary watchdog diff --git a/src/cmd_doctor.py b/src/cmd_doctor.py index 5e06119..7691e4a 100644 --- a/src/cmd_doctor.py +++ b/src/cmd_doctor.py @@ -1,73 +1,14 @@ -import os import sys from pathlib import Path -from typing import Any, Callable, Optional +from typing import Any, Callable from api_client import APIClient from config_manager import ConfigManager - -_RED = "\033[91m" -_RESET = "\033[0m" - -# Windows: stderr may be a TTY but ANSI is ignored until VT processing is enabled -# on that console handle (classic PowerShell / conhost). -_win32_stderr_vt: Optional[bool] = None - - -def _try_enable_win32_stderr_vt() -> bool: - """Enable ANSI on the stderr console handle. Return True if escapes will work.""" - global _win32_stderr_vt - if sys.platform != "win32": - return True - if _win32_stderr_vt is not None: - return _win32_stderr_vt - err = sys.stderr - if not (hasattr(err, "isatty") and err.isatty()): - _win32_stderr_vt = False - return False - try: - import ctypes - from ctypes import wintypes - - kernel32 = ctypes.windll.kernel32 - STD_ERROR_HANDLE = -12 - ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x0004 - - handle = kernel32.GetStdHandle(STD_ERROR_HANDLE) - if handle in (-1, 0): - _win32_stderr_vt = False - return False - mode = wintypes.DWORD() - if not kernel32.GetConsoleMode(handle, ctypes.byref(mode)): - _win32_stderr_vt = False - return False - if mode.value & ENABLE_VIRTUAL_TERMINAL_PROCESSING: - _win32_stderr_vt = True - return True - if kernel32.SetConsoleMode(handle, mode.value | ENABLE_VIRTUAL_TERMINAL_PROCESSING): - _win32_stderr_vt = True - return True - except Exception: - pass - _win32_stderr_vt = False - return False - - -def _stderr_color_enabled() -> bool: - if os.environ.get("NO_COLOR"): - return False - err = sys.stderr - if not (hasattr(err, "isatty") and err.isatty()): - return False - if sys.platform == "win32" and not _try_enable_win32_stderr_vt(): - return False - return True +from console_color import RED, color_segment def _red(text: str) -> str: - if not _stderr_color_enabled(): - return text - return f"{_RED}{text}{_RESET}" + return color_segment(RED, text, sys.stderr) def cmd_doctor(cm: ConfigManager, api_client_factory: Callable[[dict], Any] = APIClient) -> int: diff --git a/src/cmd_volumes.py b/src/cmd_volumes.py index e480507..64975b6 100644 --- a/src/cmd_volumes.py +++ b/src/cmd_volumes.py @@ -16,40 +16,11 @@ from api_client import APIClient from cmd_projects import _get_projects_from_endpoint from config_manager import ConfigManager - - -# ---- Color helpers -------------------------------------------------------- -# ANSI escape codes are emitted only when stdout is a real TTY; when output is -# piped to a file or a non-tty terminal, colors collapse to empty strings to -# avoid littering logs with control sequences. - -_GREEN = "\033[92m" -_YELLOW = "\033[93m" -_RED = "\033[91m" -# Orange is a 256-color extension; falls back gracefully on terminals that do -# not understand the sequence (most modern terminals do). -_ORANGE = "\033[38;5;208m" -_DIM = "\033[2m" -_RESET = "\033[0m" - - -def _supports_color() -> bool: - """True if stdout looks like an interactive terminal that supports ANSI. - - Honors the NO_COLOR convention (https://no-color.org/). Re-evaluated on - every call so tests that capture stdout (which is not a TTY) see plain - ASCII output without depending on import-time state. - """ - if os.environ.get("NO_COLOR"): - return False - stream = sys.stdout - return hasattr(stream, "isatty") and stream.isatty() +from console_color import GREEN, ORANGE, RED, YELLOW, color_segment def _color(code: str, text: str) -> str: - if not code or not _supports_color(): - return text - return f"{code}{text}{_RESET}" + return color_segment(code, text, sys.stdout) def get_tasks_files_list(client: APIClient) -> List[Tuple[int, Optional[str]]]: @@ -384,12 +355,12 @@ def _bucket_action_color(action: str, count: int, goal_count: Optional[int]) -> failed -> red """ if action == "skipped": - return _ORANGE + return ORANGE if action == "failed": - return _RED + return RED if goal_count is None or goal_count <= 0: - return _GREEN - return _GREEN if count >= goal_count else _YELLOW + return GREEN + return GREEN if count >= goal_count else YELLOW def _sync_one_bucket( @@ -501,7 +472,7 @@ def _sync_project_stats_to_server(client: APIClient, project_stats: Dict[int, Di ): if filter_name is None or exposure is None: print( - f" {_color(_RED, '[failed ]')} incomplete bucket " + f" {_color(RED, '[failed ]')} incomplete bucket " f"filter={filter_name} exposure={exposure} count={count}" ) ok = False @@ -510,7 +481,7 @@ def _sync_project_stats_to_server(client: APIClient, project_stats: Dict[int, Di exposure_f = float(exposure) except (TypeError, ValueError): print( - f" {_color(_RED, '[failed ]')} invalid exposure " + f" {_color(RED, '[failed ]')} invalid exposure " f"filter={filter_name} exposure={exposure!r} count={count}" ) ok = False @@ -658,11 +629,11 @@ def _format_status_tag(status: str) -> str: """ label = f"{status:<9}" if status == "matched": - return f"[{_color(_GREEN, label)}]" + return f"[{_color(GREEN, label)}]" if status == "unmatched": - return f"[{_color(_RED, label)}]" + return f"[{_color(RED, label)}]" if status == "skipped": - return f"[{_color(_YELLOW, label)}]" + return f"[{_color(YELLOW, label)}]" return f"[{label}]" diff --git a/src/console_color.py b/src/console_color.py new file mode 100644 index 0000000..8a88502 --- /dev/null +++ b/src/console_color.py @@ -0,0 +1,68 @@ +""" +TTY-aware ANSI color segments for CLI output and logging. + +Uses colorama's ``just_fix_windows_console()`` once at process start so +PowerShell/conhost interpret escapes on both stdout and stderr (Windows). + +For partial coloring inside a line (e.g. a status token in a log message), +build the message with :func:`color_segment`:: + + msg = f"task {tid} {color_segment(GREEN, 'ok', sys.stdout)} done" + logger.info(msg) + +``NO_COLOR`` disables all escapes. Non-TTY streams return plain text. +""" + +from __future__ import annotations + +import os +import sys +from typing import TextIO + +# SGR sequences (work with just_fix_windows_console on Windows). +_GREEN = "\033[92m" +_YELLOW = "\033[93m" +_RED = "\033[91m" +_ORANGE = "\033[38;5;208m" +_DIM = "\033[2m" +_RESET = "\033[0m" + +# Public aliases for call sites that prefer short names matching prior cmd_volumes. +GREEN = _GREEN +YELLOW = _YELLOW +RED = _RED +ORANGE = _ORANGE +DIM = _DIM +RESET = _RESET + +_win32_console_inited = False + + +def init_windows_console() -> None: + """Enable VT processing on Windows stdout/stderr; no-op elsewhere or if unavailable.""" + global _win32_console_inited + if _win32_console_inited: + return + _win32_console_inited = True + if sys.platform != "win32": + return + try: + from colorama import just_fix_windows_console + + just_fix_windows_console() + except Exception: + pass + + +def stream_color_enabled(stream: TextIO) -> bool: + if os.environ.get("NO_COLOR"): + return False + return bool(hasattr(stream, "isatty") and stream.isatty()) + + +def color_segment(code: str, text: str, stream: TextIO) -> str: + """Wrap ``text`` with ``code`` and reset when ``stream`` is a color-capable TTY.""" + init_windows_console() + if not code or not stream_color_enabled(stream): + return text + return f"{code}{text}{_RESET}" diff --git a/src/hevelius-runner.py b/src/hevelius-runner.py index 67be360..5bd1b54 100644 --- a/src/hevelius-runner.py +++ b/src/hevelius-runner.py @@ -36,6 +36,7 @@ import yaml +from console_color import init_windows_console from config_manager import ConfigManager from api_client import APIClient, resolve_scope_id_from_identifier from task_manager import TaskManager @@ -384,7 +385,7 @@ def build_parser() -> argparse.ArgumentParser: return parser def main(argv: Optional[List[str]] = None) -> int: - + init_windows_console() #logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s") setup_logging() From ed0f48d79f9bc76d8d3d3c393a71df51fd9a1922 Mon Sep 17 00:00:00 2001 From: Tomek Mrugalski Date: Fri, 15 May 2026 22:49:38 +0200 Subject: [PATCH 08/11] Mass rename implemented --- CHANGELOG.md | 1 + src/cmd_volumes.py | 155 +++++++++++++++++++++++++++++-- src/hevelius-runner.py | 42 +++++++++ tests/test_cmd_volumes_rename.py | 103 ++++++++++++++++++++ 4 files changed, 295 insertions(+), 6 deletions(-) create mode 100644 tests/test_cmd_volumes_rename.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 79832dc..61dc2c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ 0.3.0 (unreleased) - Exclude patterns - specified files or directories can be excluded from the volumes handling. +- Mass rename implemented (`hevelius-runner volumes rename "SUBSTRING1" "SUBSTRING2"`) 0.2.0 (2026-04-28) diff --git a/src/cmd_volumes.py b/src/cmd_volumes.py index 64975b6..e4be21a 100644 --- a/src/cmd_volumes.py +++ b/src/cmd_volumes.py @@ -180,6 +180,11 @@ def _normalize_full_path(path: str) -> str: return os.path.normcase(os.path.normpath(os.path.abspath(path))) +def _resolve_path(path: str) -> str: + """Absolute path for filesystem I/O (preserves filename case on Windows).""" + return os.path.normpath(os.path.abspath(path)) + + def _is_excluded(full_path: str, exclude_patterns: List[str]) -> bool: """ True if the normalized full path matches any exclusion pattern. @@ -560,6 +565,146 @@ def process_fits_list( cnt += 1 +def _fits_files_in_dir(dir_path: str, *, resolve: bool = False) -> List[str]: + """Return full paths of all ``*.fit`` / ``*.fits`` under ``dir_path``.""" + to_path = _resolve_path if resolve else _normalize_full_path + base = os.path.normpath(dir_path) + pattern_fit = os.path.join(base, "**", "*.fit") + pattern_fits = os.path.join(base, "**", "*.fits") + found_fit = glob.glob(pattern_fit, recursive=True) + found_fits = glob.glob(pattern_fits, recursive=True) + return sorted({to_path(str(f)) for f in set(found_fit) | set(found_fits)}) + + +def _paths_from_list_file(list_path: str, *, resolve: bool = False) -> List[str]: + """Return full paths from a text file (one path per line).""" + to_path = _resolve_path if resolve else _normalize_full_path + paths: List[str] = [] + with open(list_path, encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line or line[0] == "#": + continue + paths.append(to_path(line)) + return paths + + +def _collect_target_paths( + cm: ConfigManager, + args, + *, + resolve: bool = False, +) -> Tuple[int, List[str]]: + """ + Resolve FITS file paths from ``-f`` / ``-l`` / ``-d`` or all configured volumes. + + Mirrors the path-selection rules used by :func:`sanity_files`. + When ``resolve`` is True, paths keep filesystem casing (needed for rename on Windows). + """ + to_path = _resolve_path if resolve else _normalize_full_path + exclude_patterns = _exclude_patterns_from_config(cm) + paths: List[str] = [] + + if args.file: + paths.append(to_path(args.file)) + elif args.list: + paths.extend(_paths_from_list_file(args.list, resolve=resolve)) + elif args.dir: + paths.extend(_fits_files_in_dir(args.dir, resolve=resolve)) + else: + volumes = _monitor_volumes_from_config(cm) + if not volumes: + rp = _repo_path_from_config(cm) + if rp is None: + print( + "No source paths configured. Configure paths.volumes " + "(or legacy paths.fits_monitor_dir), or use -f / -l / -d.", + file=sys.stderr, + ) + return 1, [] + volumes = [(rp, "repo-path")] + for path, nickname in volumes: + print(f"Volume '{nickname}': {path}") + paths.extend(_fits_files_in_dir(path, resolve=resolve)) + + if exclude_patterns: + paths = [p for p in paths if not _is_excluded(p, exclude_patterns)] + return 0, paths + + +def _rename_basename(basename: str, old: str, new: str) -> Optional[str]: + """Return a new basename when ``old`` appears in ``basename``, else ``None``.""" + if os.name == "nt": + lower_base = basename.lower() + lower_old = old.lower() + idx = lower_base.find(lower_old) + if idx < 0: + return None + return basename[:idx] + new + basename[idx + len(old) :] + if old not in basename: + return None + return basename.replace(old, new) + + +def rename_files(cm: ConfigManager, args) -> int: + """Replace ``old_string`` with ``new_string`` in each selected file's basename.""" + code = _require_loaded(cm) + if code != 0: + return code + + old = str(getattr(args, "old_string", "") or "") + new = str(getattr(args, "new_string", "") or "") + if not old: + print("rename requires a non-empty search string.", file=sys.stderr) + return 1 + + code, paths = _collect_target_paths(cm, args, resolve=True) + if code != 0: + return code + if not paths: + print("No files to rename.") + return 0 + + print(f"Renaming basename substring {old!r} -> {new!r} in {len(paths)} file(s).") + renamed = 0 + skipped = 0 + failed = 0 + + for src in paths: + if not os.path.isfile(src): + print(f" skip (not a file): {src}", file=sys.stderr) + failed += 1 + continue + + base = os.path.basename(src) + new_base = _rename_basename(base, old, new) + if new_base is None: + skipped += 1 + continue + if new_base == base: + skipped += 1 + continue + + dst = os.path.join(os.path.dirname(src), new_base) + if os.path.exists(dst): + print(f" failed (target exists): {src} -> {dst}", file=sys.stderr) + failed += 1 + continue + + try: + os.rename(src, dst) + except OSError as e: + print(f" failed: {src} -> {dst}: {e}", file=sys.stderr) + failed += 1 + continue + + print(f" {base} -> {new_base}") + renamed += 1 + + print(f"Done: {renamed} renamed, {skipped} unchanged, {failed} failed.") + return 1 if failed else 0 + + def process_fits_dir( client: APIClient, dir: str, @@ -577,12 +722,7 @@ def process_fits_dir( :param show_hdr: bool governing whether FITS headers will be printed or not """ - base = os.path.normpath(dir) - pattern_fit = os.path.join(base, "**", "*.fit") - pattern_fits = os.path.join(base, "**", "*.fits") - found_fit = glob.glob(pattern_fit, recursive=True) - found_fits = glob.glob(pattern_fits, recursive=True) - files = sorted(set(found_fit) | set(found_fits)) + files = _fits_files_in_dir(dir) print(f"Found {len(files)} files(s) in directory {dir}") @@ -1111,6 +1251,9 @@ def sanity_db(cm: ConfigManager, args) -> int: def cmd_volumes(cm: ConfigManager, args) -> int: """Manages the on disk images repository.""" + if getattr(args, "volumes_cmd", None) == "rename": + return rename_files(cm, args) + if args.file or args.list or args.dir or args.all_files: return sanity_files(cm, args) diff --git a/src/hevelius-runner.py b/src/hevelius-runner.py index 5bd1b54..01ba3bb 100644 --- a/src/hevelius-runner.py +++ b/src/hevelius-runner.py @@ -317,6 +317,34 @@ def cmd_version(args: argparse.Namespace, cm: ConfigManager) -> int: return 0 +def _add_volumes_path_args(parser: argparse.ArgumentParser) -> None: + """File-selection switches shared by ``volumes rename`` and sanity scan.""" + parser.add_argument( + "-f", + "--file", + help="Rename one specific file (any extension).", + type=str, + ) + parser.add_argument( + "-l", + "--list", + help="Rename files listed in a text file (one path per line).", + type=str, + ) + parser.add_argument( + "-d", + "--dir", + help="Rename all FITS files recursively under this directory.", + type=str, + ) + parser.add_argument( + "-a", + "--all-files", + help="Rename FITS files in all configured paths.volumes (default when no -f/-l/-d).", + action="store_true", + ) + + def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( description="Hevelius observatory runner - NINA integration and API client.", @@ -369,6 +397,20 @@ def build_parser() -> argparse.ArgumentParser: p_view.add_argument("--project-id", type=int, help="Numeric project_id.") repo_parser = sub.add_parser('volumes', help="Manages files repository (volumes)on local storage.") + volumes_sub = repo_parser.add_subparsers(dest="volumes_cmd", metavar="SUBCOMMAND") + rename_parser = volumes_sub.add_parser( + "rename", + help="Replace a substring in filenames (basename only) for selected FITS files.", + ) + rename_parser.add_argument( + "old_string", + help="Substring to find in each file basename (literal match, not a regex).", + ) + rename_parser.add_argument( + "new_string", + help="Replacement text for each match in the basename.", + ) + _add_volumes_path_args(rename_parser) repo_parser.add_argument('-f', "--file", help="Check one specific FITS file", type=str) repo_parser.add_argument("-l", "--list", help="Check FITS files listed in a text file (one path per line)", type=str) repo_parser.add_argument("-d", "--dir", help="Check all FITS files recursively in this specific directory", type=str) diff --git a/tests/test_cmd_volumes_rename.py b/tests/test_cmd_volumes_rename.py new file mode 100644 index 0000000..cfaeaec --- /dev/null +++ b/tests/test_cmd_volumes_rename.py @@ -0,0 +1,103 @@ +from argparse import Namespace +import os + +import cmd_volumes +import pytest + + +class _DummyConfigManager: + loaded = True + + def get_paths_config(self): + return {} + + +def _rename_args(**overrides): + args = Namespace( + old_string="Caldwell 75", + new_string="RCW 38", + file=None, + list=None, + dir=None, + all_files=False, + ) + for key, value in overrides.items(): + setattr(args, key, value) + return args + + +def test_rename_basename_literal_with_spaces_and_parens(): + old = "Caldwell 75" + new = "RCW 38" + base = "2026-04-18_22-59-27_Caldwell 75_L_180.00s_LIGHT_0004.fits" + assert cmd_volumes._rename_basename(base, old, new) == ( + "2026-04-18_22-59-27_RCW 38_L_180.00s_LIGHT_0004.fits" + ) + + paren_base = "2026-04-18_M42 (neb)_L_120.00s_LIGHT_0001.fits" + assert cmd_volumes._rename_basename(paren_base, "M42 (neb)", "NGC 1976") == ( + "2026-04-18_NGC 1976_L_120.00s_LIGHT_0001.fits" + ) + + +def test_rename_basename_returns_none_when_no_match(): + assert cmd_volumes._rename_basename("plain.fits", "missing", "x") is None + + +def test_rename_single_file(tmp_path, capsys): + cm = _DummyConfigManager() + src = tmp_path / "2026-04-18_22-59-27_Caldwell 75_L_180.00s_LIGHT_0004.fits" + src.write_bytes(b"fits") + dst = tmp_path / "2026-04-18_22-59-27_RCW 38_L_180.00s_LIGHT_0004.fits" + + ret = cmd_volumes.rename_files(cm, _rename_args(file=str(src))) + assert ret == 0 + assert not src.exists() + assert dst.is_file() + out = capsys.readouterr().out + assert "Caldwell 75" in out + assert "RCW 38" in out + assert "1 renamed" in out + + +def test_rename_dir_skips_unchanged_and_renames_matches(tmp_path, capsys): + cm = _DummyConfigManager() + match = tmp_path / "night_Caldwell 75_001.fits" + other = tmp_path / "night_M42_002.fits" + match.write_bytes(b"a") + other.write_bytes(b"b") + + ret = cmd_volumes.rename_files(cm, _rename_args(dir=str(tmp_path))) + assert ret == 0 + assert not match.exists() + assert (tmp_path / "night_RCW 38_001.fits").is_file() + assert other.is_file() + out = capsys.readouterr().out + assert "1 renamed" in out + assert "1 unchanged" in out + + +def test_rename_cli_parses_subcommand(runner_mod): + args = runner_mod.build_parser().parse_args( + ["volumes", "rename", "Caldwell 75", "RCW 38", "-a"] + ) + assert args.command == "volumes" + assert args.volumes_cmd == "rename" + assert args.old_string == "Caldwell 75" + assert args.new_string == "RCW 38" + assert args.all_files is True + + +def test_rename_fails_when_target_exists(tmp_path, capsys): + cm = _DummyConfigManager() + src = tmp_path / "frame_Caldwell 75.fits" + dst = tmp_path / "frame_RCW 38.fits" + src.write_bytes(b"a") + dst.write_bytes(b"b") + + ret = cmd_volumes.rename_files(cm, _rename_args(file=str(src))) + assert ret == 1 + assert src.is_file() + captured = capsys.readouterr() + combined = (captured.out + captured.err).lower() + assert "failed" in combined From c6c56ffc0f782f9fc0d659b7b66d46451cde2538 Mon Sep 17 00:00:00 2001 From: Tomek Mrugalski Date: Wed, 20 May 2026 22:39:11 +0200 Subject: [PATCH 09/11] Fix three bugs from PR review - _rename_basename: replace all occurrences on Windows (was first-only), making behavior consistent with non-Windows - process_fits_list: count only real file lines in total (not blanks/comments), so progress display is accurate - rename_files: count missing-file paths as skipped, not failed - install-hevelius.ps1: remove embedded copies of committed bin/ files; script now verifies they exist and creates only the Start Menu shortcut Co-Authored-By: Claude Sonnet 4.6 --- bin/install-hevelius.ps1 | 41 ++++++---------------------------------- src/cmd_volumes.py | 22 +++++++++++++++------ 2 files changed, 22 insertions(+), 41 deletions(-) diff --git a/bin/install-hevelius.ps1 b/bin/install-hevelius.ps1 index 55a93f6..032dfec 100644 --- a/bin/install-hevelius.ps1 +++ b/bin/install-hevelius.ps1 @@ -7,41 +7,16 @@ $ErrorActionPreference = "Stop" $repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path $binDir = Join-Path $repoRoot "bin" -if (-not (Test-Path $binDir)) { - New-Item -Path $binDir -ItemType Directory | Out-Null -} - $runnerCmdPath = Join-Path $binDir "hevelius-runner.cmd" $openShellPath = Join-Path $binDir "open-hevelius.ps1" -$runnerCmdContent = @' -@echo off -setlocal -set "SCRIPT_DIR=%~dp0" -set "PROJECT_ROOT=%SCRIPT_DIR%.." -"%PROJECT_ROOT%\venv\Scripts\python.exe" "%PROJECT_ROOT%\src\hevelius-runner.py" %* -'@ - -$openShellContent = @' -$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path -Set-Location $repoRoot - -$activateScript = Join-Path $repoRoot "venv\Scripts\Activate.ps1" -if (Test-Path $activateScript) { - . $activateScript +$missing = @() +if (-not (Test-Path $runnerCmdPath)) { $missing += $runnerCmdPath } +if (-not (Test-Path $openShellPath)) { $missing += $openShellPath } +if ($missing.Count -gt 0) { + Write-Error "Required launcher file(s) missing from the repository:`n $($missing -join "`n ")`nRun 'git restore bin/' to restore them, then re-run this script." + exit 1 } -else { - Write-Warning "Virtual environment activation script not found at: $activateScript" -} - -$runnerCmd = Join-Path $repoRoot "bin\hevelius-runner.cmd" -Set-Alias -Name hevelius-runner -Value $runnerCmd -Scope Global - -Write-Host "Hevelius runner shell ready. Use: hevelius-runner " -ForegroundColor Green -'@ - -Set-Content -Path $runnerCmdPath -Value $runnerCmdContent -Encoding ascii -Set-Content -Path $openShellPath -Value $openShellContent -Encoding utf8 $startMenuPrograms = [Environment]::GetFolderPath("Programs") $shortcutPath = Join-Path $startMenuPrograms "$ShortcutName.lnk" @@ -56,10 +31,6 @@ $shortcut.WorkingDirectory = $repoRoot $shortcut.IconLocation = "$env:SystemRoot\System32\WindowsPowerShell\v1.0\powershell.exe,0" $shortcut.Save() -Write-Host "Installed launcher scripts:" -ForegroundColor Green -Write-Host " $runnerCmdPath" -Write-Host " $openShellPath" -Write-Host "" Write-Host "Start Menu shortcut created:" -ForegroundColor Green Write-Host " $shortcutPath" Write-Host "" diff --git a/src/cmd_volumes.py b/src/cmd_volumes.py index e4be21a..3879891 100644 --- a/src/cmd_volumes.py +++ b/src/cmd_volumes.py @@ -533,7 +533,8 @@ def process_fits_list( with open(fname, encoding="utf-8") as f: lines = f.readlines() - total = len(lines) + file_lines = [l for l in lines if l.strip() and not l.strip().startswith("#")] + total = len(file_lines) print(f"Found {total} filename(s) in file {fname}") cnt = 1 @@ -633,14 +634,23 @@ def _collect_target_paths( def _rename_basename(basename: str, old: str, new: str) -> Optional[str]: - """Return a new basename when ``old`` appears in ``basename``, else ``None``.""" + """Return a new basename with all occurrences of ``old`` replaced by ``new``, + or ``None`` if ``old`` does not appear. Matching is case-insensitive on Windows.""" if os.name == "nt": lower_base = basename.lower() lower_old = old.lower() - idx = lower_base.find(lower_old) - if idx < 0: + if lower_old not in lower_base: return None - return basename[:idx] + new + basename[idx + len(old) :] + result = [] + i = 0 + while i < len(basename): + if lower_base[i:i + len(lower_old)] == lower_old: + result.append(new) + i += len(lower_old) + else: + result.append(basename[i]) + i += 1 + return "".join(result) if old not in basename: return None return basename.replace(old, new) @@ -673,7 +683,7 @@ def rename_files(cm: ConfigManager, args) -> int: for src in paths: if not os.path.isfile(src): print(f" skip (not a file): {src}", file=sys.stderr) - failed += 1 + skipped += 1 continue base = os.path.basename(src) From e0213be803554dc7c6ed9b3603a507581594196e Mon Sep 17 00:00:00 2001 From: Tomek Mrugalski Date: Wed, 20 May 2026 23:12:38 +0200 Subject: [PATCH 10/11] Doctor: structured logger output with color status per check - All output via logging.getLogger; one line per check with green OK / red FAILED appended at the end of each line (no emoji) - Volume checks added: one line per configured volume, FAILED if the directory is missing or cannot be listed (e.g. permission error) - console_color: reconfigure stdout/stderr to UTF-8 in init_windows_console - hevelius-runner.cmd: set PYTHONUTF8=1 for reliable Unicode on Windows - tests: switch doctor assertions from capsys to caplog Co-Authored-By: Claude Sonnet 4.6 --- bin/hevelius-runner.cmd | 1 + src/cmd_doctor.py | 97 ++++++++++++++++++++++------------------- src/console_color.py | 11 ++++- tests/test_cli.py | 23 +++++----- 4 files changed, 76 insertions(+), 56 deletions(-) diff --git a/bin/hevelius-runner.cmd b/bin/hevelius-runner.cmd index 2eff128..833be9a 100644 --- a/bin/hevelius-runner.cmd +++ b/bin/hevelius-runner.cmd @@ -2,4 +2,5 @@ setlocal set "SCRIPT_DIR=%~dp0" set "PROJECT_ROOT=%SCRIPT_DIR%.." +set PYTHONUTF8=1 "%PROJECT_ROOT%\venv\Scripts\python.exe" "%PROJECT_ROOT%\src\hevelius-runner.py" %* diff --git a/src/cmd_doctor.py b/src/cmd_doctor.py index 7691e4a..169c243 100644 --- a/src/cmd_doctor.py +++ b/src/cmd_doctor.py @@ -1,14 +1,23 @@ +import logging +import os import sys from pathlib import Path from typing import Any, Callable from api_client import APIClient from config_manager import ConfigManager -from console_color import RED, color_segment +from console_color import GREEN, RED, color_segment +from cmd_volumes import _monitor_volumes_from_config +logger = logging.getLogger(__name__) -def _red(text: str) -> str: - return color_segment(RED, text, sys.stderr) + +def _ok() -> str: + return color_segment(GREEN, "OK", sys.stdout) + + +def _fail() -> str: + return color_segment(RED, "FAILED", sys.stdout) def cmd_doctor(cm: ConfigManager, api_client_factory: Callable[[dict], Any] = APIClient) -> int: @@ -20,28 +29,18 @@ def cmd_doctor(cm: ConfigManager, api_client_factory: Callable[[dict], Any] = AP required_api = ("base_url", "timeout", "username", "password") missing = [k for k in required_api if not str(api_cfg.get(k, "")).strip()] if missing: - print( - f"API configuration is incomplete (missing: {', '.join(missing)}).", - file=sys.stderr, - ) + logger.error(f"API config — missing: {', '.join(missing)} {_fail()}") return 1 client = api_client_factory(api_cfg) version_ok = False try: version = client.get_version() - print(f"API reachable at {api_cfg['base_url'].rstrip('/')}/ - backend version: {version}") + logger.info(f"API reachable — {api_cfg['base_url'].rstrip('/')}/ (version {version}) {_ok()}") version_ok = True except Exception as e: exit_code = 1 - print( - f"Backend API: {_red('unreachable')} ({e})", - file=sys.stderr, - ) - print( - "Skipping API login and telescope checks (backend not reached).", - file=sys.stderr, - ) + logger.error(f"API reachable — unreachable: {e} {_fail()}") login_ok = False if version_ok: @@ -49,61 +48,71 @@ def cmd_doctor(cm: ConfigManager, api_client_factory: Callable[[dict], Any] = AP login = client.login() if not login.status or not login.token: exit_code = 1 - print(f"Login failed: {login.msg or 'no JWT token in response'}", file=sys.stderr) + logger.error(f"Login — {login.msg or 'no JWT token in response'} {_fail()}") else: login_ok = True - print(f"Authenticated as {api_cfg['username']!r} (user id {login.user_id}).") + logger.info(f"Login — authenticated as {api_cfg['username']!r} (user id {login.user_id}) {_ok()}") except Exception as e: exit_code = 1 - print(f"API check failed (login): {e}", file=sys.stderr) - - if version_ok and not login_ok: - print("Skipping telescope list (not authenticated).", file=sys.stderr) + logger.error(f"Login — {e} {_fail()}") + else: + logger.info("Login — skipped (API not reached)") if version_ok and login_ok: try: scopes = client.list_telescopes() - print(f"Telescopes available: {len(scopes)}") sid = api_cfg.get("scope_id") - if sid is not None and str(sid).strip() != "": + if sid is None or not str(sid).strip(): + logger.info("Telescope — api.scope_id not set; use `telescope list` and `telescope set`") + else: want = int(sid) ids = {int(t.get("scope_id")) for t in scopes if t.get("scope_id") is not None} if want not in ids: exit_code = 1 - print( - f"Configured api.scope_id={want} is not in the telescope list from the API.", - file=sys.stderr, + logger.error( + f"Telescope — scope_id={want} not found in API list ({len(scopes)} telescope(s)) {_fail()}" ) else: names = [t.get("name") for t in scopes if int(t.get("scope_id", -1)) == want] label = names[0] if names else "?" - print(f"Configured scope_id {want} OK ({label!r}).") - else: - print( - "Note: api.scope_id is not set; use `telescope list` and `telescope set`.", - file=sys.stderr, - ) + logger.info( + f"Telescope — scope_id={want} ({label!r}), {len(scopes)} telescope(s) available {_ok()}" + ) except Exception as e: exit_code = 1 - print(f"API check failed (telescope list): {e}", file=sys.stderr) + logger.error(f"Telescope — {e} {_fail()}") + elif version_ok: + logger.info("Telescope — skipped (not authenticated)") + else: + logger.info("Telescope — skipped (API not reached)") nina_cfg = cm.get_nina_config() exe = nina_cfg.get("executable_path", "") nina_path = Path(str(exe)) if exe else None if not nina_path or not nina_path.is_file(): exit_code = 1 - print( - f"NINA executable not found or not configured: {exe!r}", - file=sys.stderr, - ) + logger.error(f"NINA — executable not found: {exe!r} {_fail()}") + else: + logger.info(f"NINA — {nina_path} {_ok()}") + + volumes = _monitor_volumes_from_config(cm) + if not volumes: + logger.info("Volumes — none configured") else: - print(f"NINA executable found: {nina_path}") + for path, nickname in volumes: + if not os.path.isdir(path): + exit_code = 1 + logger.error(f"Volume '{nickname}' — {path} {_fail()} (directory not found)") + else: + try: + os.listdir(path) + logger.info(f"Volume '{nickname}' — {path} {_ok()}") + except OSError as e: + exit_code = 1 + logger.error(f"Volume '{nickname}' — {path} {_fail()} ({e})") if exit_code == 0: - print("Configuration and connectivity check passed.") + logger.info(f"All checks passed {_ok()}") else: - print( - "Configuration check finished with one or more issues (see messages above).", - file=sys.stderr, - ) + logger.error(f"Doctor — one or more checks failed (see above) {_fail()}") return exit_code diff --git a/src/console_color.py b/src/console_color.py index 8a88502..c581bce 100644 --- a/src/console_color.py +++ b/src/console_color.py @@ -39,7 +39,7 @@ def init_windows_console() -> None: - """Enable VT processing on Windows stdout/stderr; no-op elsewhere or if unavailable.""" + """Enable VT processing and UTF-8 on Windows stdout/stderr; no-op elsewhere.""" global _win32_console_inited if _win32_console_inited: return @@ -52,6 +52,15 @@ def init_windows_console() -> None: just_fix_windows_console() except Exception: pass + # Reconfigure to UTF-8 so emoji and other non-ASCII characters don't raise + # UnicodeEncodeError on consoles that default to a narrow encoding (e.g. cp1252). + # errors="replace" is a safety net; UTF-8 itself can encode all Unicode code points. + for _stream in (sys.stdout, sys.stderr): + try: + if hasattr(_stream, "reconfigure"): + _stream.reconfigure(encoding="utf-8", errors="replace") + except Exception: + pass def stream_color_enabled(stream: TextIO) -> bool: diff --git a/tests/test_cli.py b/tests/test_cli.py index 678d8fe..fbd5de1 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,3 +1,4 @@ +import logging import textwrap from unittest.mock import MagicMock, patch @@ -63,7 +64,7 @@ def test_config_command_prints_yaml_without_password(tmp_path, capsys, runner_mo assert str(cfg.resolve()) in out -def test_doctor_command_success(tmp_path, capsys, runner_mod): +def test_doctor_command_success(tmp_path, caplog, runner_mod): cfg = tmp_path / "c.yaml" fake_nina = tmp_path / "nina_fake.exe" fake_nina.write_bytes(b"") @@ -102,14 +103,14 @@ def test_doctor_command_success(tmp_path, capsys, runner_mod): client.list_telescopes.return_value = [{"scope_id": 1, "name": "Test"}] MockClient.return_value = client - ret = runner_mod.main(["-c", str(cfg), "doctor"]) + with caplog.at_level(logging.INFO): + ret = runner_mod.main(["-c", str(cfg), "doctor"]) assert ret == 0 - out = capsys.readouterr().out - assert "1.0.0" in out - assert "Connectivity check passed" in out or "check passed" in out + assert "1.0.0" in caplog.text + assert "All checks passed" in caplog.text -def test_doctor_continues_when_api_unreachable(tmp_path, capsys, runner_mod): +def test_doctor_continues_when_api_unreachable(tmp_path, caplog, runner_mod): cfg = tmp_path / "c.yaml" fake_nina = tmp_path / "nina_fake.exe" fake_nina.write_bytes(b"") @@ -140,12 +141,12 @@ def test_doctor_continues_when_api_unreachable(tmp_path, capsys, runner_mod): client.get_version.side_effect = ConnectionError("refused") MockClient.return_value = client - ret = runner_mod.main(["-c", str(cfg), "doctor"]) + with caplog.at_level(logging.INFO): + ret = runner_mod.main(["-c", str(cfg), "doctor"]) assert ret == 1 - captured = capsys.readouterr() - assert "unreachable" in captured.err - assert "Skipping API login" in captured.err - assert "NINA executable found" in captured.out + assert "unreachable" in caplog.text + assert "skipped" in caplog.text + assert "NINA" in caplog.text client.login.assert_not_called() client.list_telescopes.assert_not_called() From dc96b58c704b08c49a23267b6bc8aa7a6d4d6961 Mon Sep 17 00:00:00 2001 From: Tomek Mrugalski Date: Wed, 20 May 2026 23:18:00 +0200 Subject: [PATCH 11/11] Logging: 4-letter level abbreviations with color on console WARN prints in yellow, ERRO and CRIT in red (abbreviation only, not the full message). File handler uses plain abbreviations without ANSI codes. Co-Authored-By: Claude Sonnet 4.6 --- src/hevelius-runner.py | 65 +++++++++++++++++++++++++++++++++--------- 1 file changed, 52 insertions(+), 13 deletions(-) diff --git a/src/hevelius-runner.py b/src/hevelius-runner.py index 01ba3bb..ece57ee 100644 --- a/src/hevelius-runner.py +++ b/src/hevelius-runner.py @@ -26,6 +26,7 @@ # 4. Handles status updates import argparse +import copy import logging import sys import time @@ -36,7 +37,7 @@ import yaml -from console_color import init_windows_console +from console_color import init_windows_console, color_segment, RED, YELLOW from config_manager import ConfigManager from api_client import APIClient, resolve_scope_id_from_identifier from task_manager import TaskManager @@ -49,24 +50,62 @@ from cmd_doctor import cmd_doctor from cmd_projects import cmd_projects +_LEVEL_ABBREV = { + "DEBUG": "DEBG", + "INFO": "INFO", + "WARNING": "WARN", + "ERROR": "ERRO", + "CRITICAL": "CRIT", +} + +_LEVEL_COLOR = { + "WARNING": YELLOW, + "ERROR": RED, + "CRITICAL": RED, +} + + +class _ShortLevelFormatter(logging.Formatter): + """Formats the log level as a fixed 4-letter abbreviation.""" + + def format(self, record: logging.LogRecord) -> str: + record = copy.copy(record) + record.levelname = _LEVEL_ABBREV.get(record.levelname, record.levelname[:4].upper()) + return super().format(record) + + +class _ColoredShortLevelFormatter(logging.Formatter): + """Like _ShortLevelFormatter but colorizes WARN/ERRO/CRIT on color-capable streams.""" + + def __init__(self, fmt: str, stream) -> None: + super().__init__(fmt) + self._stream = stream + + def format(self, record: logging.LogRecord) -> str: + record = copy.copy(record) + color = _LEVEL_COLOR.get(record.levelname) + abbrev = _LEVEL_ABBREV.get(record.levelname, record.levelname[:4].upper()) + record.levelname = color_segment(color, abbrev, self._stream) if color else abbrev + return super().format(record) + + def setup_logging(): """Configure logging for the application.""" log_dir = Path("logs") log_dir.mkdir(exist_ok=True) - # Full logging: format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' - logging.basicConfig( - level=logging.INFO, - format='%(levelname)s - %(message)s', - handlers=[ - RotatingFileHandler( - log_dir / 'observatory.log', - maxBytes=1024*1024, - backupCount=5 - ), - logging.StreamHandler(sys.stdout) - ] + fmt = "%(levelname)s %(message)s" + stdout_handler = logging.StreamHandler(sys.stdout) + stdout_handler.setFormatter(_ColoredShortLevelFormatter(fmt, sys.stdout)) + + file_handler = RotatingFileHandler( + log_dir / 'observatory.log', + maxBytes=1024*1024, + backupCount=5, ) + file_handler.setFormatter(_ShortLevelFormatter(fmt)) + + logging.basicConfig(level=logging.INFO, handlers=[file_handler, stdout_handler]) class ObservatoryAutomation: