From 837851f603b1610792b74bbb85c30c5d5eb49e5c Mon Sep 17 00:00:00 2001 From: Frank Kong Date: Fri, 6 Feb 2026 11:58:47 -0500 Subject: [PATCH 01/35] chore: add source repo cleaning mechanism for git clone Signed-off-by: Frank Kong --- CONTRIBUTING.md | 17 ++++++----------- README.md | 1 + src/rhdh_dynamic_plugin_factory/cli.py | 8 +++++++- src/rhdh_dynamic_plugin_factory/config.py | 21 ++++++++++++++++++--- src/rhdh_dynamic_plugin_factory/utils.py | 13 +++++++++++++ 5 files changed, 45 insertions(+), 15 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 76f051e..857d8f1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -133,7 +133,9 @@ Once you have the local environment set up, you can run the factory directly wit python -m src.rhdh_dynamic_plugin_factory \ --config-dir ./config \ --workspace-path workspaces/todo \ - --output-dir ./outputs + --repo-path ./source \ + --output-dir ./outputs \ + --clean ``` #### Build and push to registry @@ -153,7 +155,9 @@ Then run: python -m src.rhdh_dynamic_plugin_factory \ --config-dir ./config \ --workspace-path workspaces/announcements \ - --push-images + --repo-path ./source/ \ + --push-images \ + --clean ``` The factory will automatically read the registry credentials from `./config/.env`. @@ -256,15 +260,6 @@ pytest tests/ --cov=src/rhdh_dynamic_plugin_factory --cov-report=term-missing This will show which lines of code are not covered by tests. -### Run Tests in Watch Mode - -For active development, you can use pytest-watch: - -```bash -pip install pytest-watch -ptw tests/ -``` - ### Writing Tests When adding new features: diff --git a/README.md b/README.md index 3696012..5d0e539 100644 --- a/README.md +++ b/README.md @@ -299,6 +299,7 @@ See the [TODO plugin example config](./examples/example-config-todo/README.md) a | `--use-local` | `false` | Use local repository instead of cloning from source.json | | `--log-level` | `INFO` | Logging level: `DEBUG`, `INFO`, `WARNING`, `ERROR`, `CRITICAL` | | `--verbose` | `false` | Show verbose output with file and line numbers | +| `--clean` | `false` | Automatically removes content of `--repo-path` directory when cloning from `source.json`. Ignored if `--use-local` is used. | ### Understanding Volume Mounts diff --git a/src/rhdh_dynamic_plugin_factory/cli.py b/src/rhdh_dynamic_plugin_factory/cli.py index 1cdef34..b497553 100644 --- a/src/rhdh_dynamic_plugin_factory/cli.py +++ b/src/rhdh_dynamic_plugin_factory/cli.py @@ -86,6 +86,12 @@ def create_parser() -> argparse.ArgumentParser: action="store_true", help="Show verbose output (show file and line number)" ) + parser.add_argument( + "--clean", + action="store_true", + default=False, + help="Clean the source directory before cloning source repository. WARNING: This will all the contents of the source directory." + ) return parser def install_dependencies(workspace_path: Path) -> bool: @@ -143,7 +149,7 @@ def main(): if source_config and not config.use_local: logger.info("[bold blue]Repository Setup[/bold blue]") - if not source_config.clone_to_path(config.repo_path): + if not source_config.clone_to_path(config.repo_path, clean=args.clean): logger.error("Failed to clone repository") sys.exit(1) elif config.use_local or not source_config: diff --git a/src/rhdh_dynamic_plugin_factory/config.py b/src/rhdh_dynamic_plugin_factory/config.py index 3dcd3ec..f65cb4e 100644 --- a/src/rhdh_dynamic_plugin_factory/config.py +++ b/src/rhdh_dynamic_plugin_factory/config.py @@ -14,7 +14,7 @@ import subprocess from .logger import get_logger -from .utils import run_command_with_streaming, display_export_results +from .utils import clean_directory, run_command_with_streaming, display_export_results @dataclass class PluginFactoryConfig: @@ -420,18 +420,33 @@ def from_file(cls, source_file: Path) -> "SourceConfig": return config - def clone_to_path(self, repo_path: Path) -> bool: + def clone_to_path(self, repo_path: Path, clean: bool = False) -> bool: """Clone the source repository to the specified path.""" logger = get_logger("cli") if not repo_path.exists(): self.logger.error(f"[red]Destination directory does not exist: {repo_path}[/red]") return True - + self.logger.info("[bold blue]Cloning repository[/bold blue]") self.logger.info(f"Repository: {self.repo}") self.logger.info(f"Reference: {self.repo_ref}") self.logger.info(f"Destination directory: {repo_path}") + + if any(repo_path.iterdir()): + self.logger.warning(f"[yellow]Source directory {repo_path} is not empty[/yellow]") + if clean: + self.logger.warning(f"[yellow]`--clean` argument set, automatically cleaning {repo_path}[/yellow]") + clean_directory(repo_path) + else: + self.logger.warning(f"[yellow]WARNING: Are you sure you want to remove the contents of {repo_path}/? \\[y/N][/yellow]") + confirm = input() + if confirm != "y": + self.logger.warning("[yellow]Aborted[/yellow]") + return False + else: + self.logger.warning(f"[yellow]`y` selected. Cleaning {repo_path}. Note: you can use the `--clean` argument to automatically clean the directory and skip this prompt next time.[/yellow]") + clean_directory(repo_path) try: cmd = ["git", "clone", self.repo, str(repo_path)] diff --git a/src/rhdh_dynamic_plugin_factory/utils.py b/src/rhdh_dynamic_plugin_factory/utils.py index 05adb03..0e824fa 100644 --- a/src/rhdh_dynamic_plugin_factory/utils.py +++ b/src/rhdh_dynamic_plugin_factory/utils.py @@ -2,6 +2,7 @@ Utility functions for RHDH Plugin Factory. """ +import shutil import subprocess import threading from pathlib import Path @@ -111,3 +112,15 @@ def display_export_results(workspace_path: Path, logger) -> bool: return has_failures +def clean_directory(directory: Path) -> None: + """Clean the directory by removing all contents but keeping the directory itself. + This is to handle cleaning volume mounted directories. + + Args: + directory: Path to the directory to clean. + """ + for item in directory.iterdir(): + if item.is_dir(): + shutil.rmtree(item) + else: + item.unlink() \ No newline at end of file From b9c7fc59418c75a8762dad75432825e338ac6895 Mon Sep 17 00:00:00 2001 From: Frank Kong Date: Fri, 6 Feb 2026 12:41:21 -0500 Subject: [PATCH 02/35] chore: add unit tests for --clean logic Assisted-By: Cursor Signed-off-by: Frank Kong --- tests/test_cli.py | 62 ++++++++++ tests/test_source_config.py | 227 ++++++++++++++++++++++++++++++++++++ 2 files changed, 289 insertions(+) create mode 100644 tests/test_cli.py diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..bfcad64 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,62 @@ +""" +Unit tests for CLI argument parsing. + +Tests the argument parser to ensure all arguments are correctly defined and parsed. +""" + +from src.rhdh_dynamic_plugin_factory.cli import create_parser + + +class TestCreateParserCleanArgument: + """Tests for the --clean CLI argument.""" + + def test_clean_flag_default_is_false(self): + """Test that --clean defaults to False when not provided.""" + parser = create_parser() + args = parser.parse_args([ + "--workspace-path", "workspaces/todo", + ]) + + assert args.clean is False + + def test_clean_flag_set_to_true(self): + """Test that --clean sets the flag to True.""" + parser = create_parser() + args = parser.parse_args([ + "--workspace-path", "workspaces/todo", + "--clean", + ]) + + assert args.clean is True + + def test_clean_flag_is_store_true_action(self): + """Test that --clean is a boolean flag (not requiring a value).""" + parser = create_parser() + + # Should work without a value after --clean + args = parser.parse_args([ + "--workspace-path", "workspaces/todo", + "--clean", + "--log-level", "DEBUG", + ]) + + assert args.clean is True + assert args.log_level == "DEBUG" + + def test_clean_flag_combined_with_other_args(self): + """Test that --clean works correctly alongside other arguments.""" + parser = create_parser() + args = parser.parse_args([ + "--workspace-path", "workspaces/todo", + "--config-dir", "/custom/config", + "--repo-path", "/custom/source", + "--clean", + "--use-local", + "--log-level", "WARNING", + ]) + + assert args.clean is True + assert args.use_local is True + assert str(args.config_dir) == "/custom/config" + assert str(args.repo_path) == "/custom/source" + assert args.log_level == "WARNING" diff --git a/tests/test_source_config.py b/tests/test_source_config.py index b08523d..28cd9f4 100644 --- a/tests/test_source_config.py +++ b/tests/test_source_config.py @@ -194,3 +194,230 @@ def test_clone_to_path_exception(self, tmp_path): assert result is False + +class TestSourceConfigCloneToPathClean: + """Tests for SourceConfig.clone_to_path clean argument and user prompt behavior.""" + + def test_clean_flag_auto_cleans_nonempty_directory(self, tmp_path): + """Test that clean=True automatically cleans a non-empty directory without prompting.""" + config = SourceConfig( + repo="https://github.com/testowner/testrepo", + repo_ref="main" + ) + + repo_path = tmp_path / "repo" + repo_path.mkdir() + (repo_path / "existing_file.txt").write_text("existing content") + (repo_path / "subdir").mkdir() + (repo_path / "subdir" / "nested.txt").write_text("nested content") + + with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run, \ + patch("src.rhdh_dynamic_plugin_factory.config.clean_directory") as mock_clean: + mock_run.return_value = 0 + + result = config.clone_to_path(repo_path, clean=True) + + assert result is True + mock_clean.assert_called_once_with(repo_path) + + def test_clean_flag_does_not_prompt_user(self, tmp_path): + """Test that clean=True does not call input() for user confirmation.""" + config = SourceConfig( + repo="https://github.com/testowner/testrepo", + repo_ref="main" + ) + + repo_path = tmp_path / "repo" + repo_path.mkdir() + (repo_path / "existing_file.txt").write_text("existing content") + + with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run, \ + patch("src.rhdh_dynamic_plugin_factory.config.clean_directory"), \ + patch("builtins.input") as mock_input: + mock_run.return_value = 0 + + config.clone_to_path(repo_path, clean=True) + + mock_input.assert_not_called() + + def test_no_clean_flag_prompts_user_confirm_yes(self, tmp_path): + """Test that clean=False prompts user and proceeds when user enters 'y'.""" + config = SourceConfig( + repo="https://github.com/testowner/testrepo", + repo_ref="main" + ) + + repo_path = tmp_path / "repo" + repo_path.mkdir() + (repo_path / "existing_file.txt").write_text("existing content") + + with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run, \ + patch("src.rhdh_dynamic_plugin_factory.config.clean_directory") as mock_clean, \ + patch("builtins.input", return_value="y"): + mock_run.return_value = 0 + + result = config.clone_to_path(repo_path, clean=False) + + assert result is True + mock_clean.assert_called_once_with(repo_path) + + def test_no_clean_flag_prompts_user_confirm_no(self, tmp_path): + """Test that clean=False prompts user and aborts when user enters anything other than 'y'.""" + config = SourceConfig( + repo="https://github.com/testowner/testrepo", + repo_ref="main" + ) + + repo_path = tmp_path / "repo" + repo_path.mkdir() + (repo_path / "existing_file.txt").write_text("existing content") + + with patch("builtins.input", return_value="n") as mock_input: + result = config.clone_to_path(repo_path, clean=False) + + assert result is False + mock_input.assert_called_once() + + def test_no_clean_flag_prompts_user_empty_input_aborts(self, tmp_path): + """Test that clean=False aborts when user presses Enter without typing anything.""" + config = SourceConfig( + repo="https://github.com/testowner/testrepo", + repo_ref="main" + ) + + repo_path = tmp_path / "repo" + repo_path.mkdir() + (repo_path / "existing_file.txt").write_text("existing content") + + with patch("builtins.input", return_value=""): + result = config.clone_to_path(repo_path, clean=False) + + assert result is False + + def test_empty_directory_skips_clean_and_prompt(self, tmp_path): + """Test that an empty directory skips both clean and prompt logic.""" + config = SourceConfig( + repo="https://github.com/testowner/testrepo", + repo_ref="main" + ) + + repo_path = tmp_path / "repo" + repo_path.mkdir() + # Directory is empty + + with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run, \ + patch("src.rhdh_dynamic_plugin_factory.config.clean_directory") as mock_clean, \ + patch("builtins.input") as mock_input: + mock_run.return_value = 0 + + result = config.clone_to_path(repo_path, clean=True) + + assert result is True + mock_clean.assert_not_called() + mock_input.assert_not_called() + + def test_empty_directory_no_clean_flag_skips_prompt(self, tmp_path): + """Test that an empty directory with clean=False does not prompt user.""" + config = SourceConfig( + repo="https://github.com/testowner/testrepo", + repo_ref="main" + ) + + repo_path = tmp_path / "repo" + repo_path.mkdir() + # Directory is empty + + with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run, \ + patch("builtins.input") as mock_input: + mock_run.return_value = 0 + + result = config.clone_to_path(repo_path, clean=False) + + assert result is True + mock_input.assert_not_called() + + def test_clean_flag_default_is_false(self, tmp_path): + """Test that the clean parameter defaults to False.""" + config = SourceConfig( + repo="https://github.com/testowner/testrepo", + repo_ref="main" + ) + + repo_path = tmp_path / "repo" + repo_path.mkdir() + (repo_path / "existing_file.txt").write_text("existing content") + + with patch("builtins.input", return_value="n"): + # Call without clean argument - should prompt (default clean=False) + result = config.clone_to_path(repo_path) + + assert result is False + + def test_clean_proceeds_with_clone_after_cleaning(self, tmp_path): + """Test that after cleaning, git clone and checkout are executed.""" + config = SourceConfig( + repo="https://github.com/testowner/testrepo", + repo_ref="v1.0.0" + ) + + repo_path = tmp_path / "repo" + repo_path.mkdir() + (repo_path / "old_file.txt").write_text("old content") + + with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run, \ + patch("src.rhdh_dynamic_plugin_factory.config.clean_directory"): + mock_run.return_value = 0 + + result = config.clone_to_path(repo_path, clean=True) + + assert result is True + assert mock_run.call_count == 2 # clone + checkout + + # Verify clone command + clone_call = mock_run.call_args_list[0] + assert clone_call[0][0] == ["git", "clone", "https://github.com/testowner/testrepo", str(repo_path)] + + # Verify checkout command + checkout_call = mock_run.call_args_list[1] + assert checkout_call[0][0] == ["git", "checkout", "v1.0.0"] + + def test_prompt_confirm_yes_proceeds_with_clone(self, tmp_path): + """Test that after user confirms 'y', git clone and checkout are executed.""" + config = SourceConfig( + repo="https://github.com/testowner/testrepo", + repo_ref="main" + ) + + repo_path = tmp_path / "repo" + repo_path.mkdir() + (repo_path / "old_file.txt").write_text("old content") + + with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run, \ + patch("src.rhdh_dynamic_plugin_factory.config.clean_directory"), \ + patch("builtins.input", return_value="y"): + mock_run.return_value = 0 + + result = config.clone_to_path(repo_path, clean=False) + + assert result is True + assert mock_run.call_count == 2 # clone + checkout + + def test_prompt_confirm_no_does_not_clone(self, tmp_path): + """Test that when user declines, git clone is not executed.""" + config = SourceConfig( + repo="https://github.com/testowner/testrepo", + repo_ref="main" + ) + + repo_path = tmp_path / "repo" + repo_path.mkdir() + (repo_path / "existing_file.txt").write_text("existing content") + + with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run, \ + patch("builtins.input", return_value="n"): + + result = config.clone_to_path(repo_path, clean=False) + + assert result is False + mock_run.assert_not_called() + From 95cbac66da4ad023e11f7186e5dc79522ca5a01b Mon Sep 17 00:00:00 2001 From: Frank Kong Date: Fri, 6 Feb 2026 12:51:35 -0500 Subject: [PATCH 03/35] chore: split existing unit test files into smaller files for maintainability Assisted-By: Cursor Signed-off-by: Frank Kong --- .cursor/rules/pytest-unit-tests.mdc | 10 +- tests/test_config.py | 817 ---------------------- tests/test_config_export_plugins.py | 274 ++++++++ tests/test_config_load_from_env.py | 291 ++++++++ tests/test_config_patches_and_overlays.py | 92 +++ tests/test_config_registry.py | 192 +++++ 6 files changed, 856 insertions(+), 820 deletions(-) delete mode 100644 tests/test_config.py create mode 100644 tests/test_config_export_plugins.py create mode 100644 tests/test_config_load_from_env.py create mode 100644 tests/test_config_patches_and_overlays.py create mode 100644 tests/test_config_registry.py diff --git a/.cursor/rules/pytest-unit-tests.mdc b/.cursor/rules/pytest-unit-tests.mdc index 062fc04..dc4e9f3 100644 --- a/.cursor/rules/pytest-unit-tests.mdc +++ b/.cursor/rules/pytest-unit-tests.mdc @@ -7,7 +7,7 @@ globs: tests/test_*.py ## General Principles - All unit tests must reside in the `tests/` directory. - Use **class-based grouping** for tests (e.g., `class TestClassName:`). -- One test file per module (e.g., `test_config.py` for `config.py`). +- One test file per class or method group. For small modules, use `test_.py`. When tests exceed ~300-400 lines, split into `test__.py`. - Ensure **100% isolation**: No network calls, no real file system changes outside `tmp_path`, no system command execution. ## Test Structure @@ -81,11 +81,15 @@ def test_with_registry_config(self, make_config): ``` ## Naming Conventions -- Files: `test_.py` +- Files: `test_.py` for small modules, or `test__.py` when a module's tests exceed ~300-400 lines - Classes: `Test` (optional MethodName if grouping by method) - Functions: `test_` ## Current Test Files -- `test_config.py`: Tests for `PluginFactoryConfig` class (load_from_env, load_registry_config, apply_patches_and_overlays, export_plugins) +- `test_cli.py`: Tests for CLI argument parsing (`create_parser`) +- `test_config_load_from_env.py`: Tests for `PluginFactoryConfig.load_from_env` method +- `test_config_registry.py`: Tests for `PluginFactoryConfig.load_registry_config` method +- `test_config_patches_and_overlays.py`: Tests for `PluginFactoryConfig.apply_patches_and_overlays` method +- `test_config_export_plugins.py`: Tests for `PluginFactoryConfig.export_plugins` method - `test_source_config.py`: Tests for `SourceConfig` class (from_file, clone_to_path) - `test_plugin_list_config.py`: Tests for `PluginListConfig` class (from_file, get_plugins, add_plugin, remove_plugin) diff --git a/tests/test_config.py b/tests/test_config.py deleted file mode 100644 index 54ae8fc..0000000 --- a/tests/test_config.py +++ /dev/null @@ -1,817 +0,0 @@ -""" -Unit tests for PluginFactoryConfig class. - -Tests the configuration loading, validation, and setup functionality -without executing shell scripts. -""" - -import os -from pathlib import Path -from unittest.mock import patch, MagicMock -import pytest -import subprocess - -from src.rhdh_dynamic_plugin_factory.config import PluginFactoryConfig - - -class TestPluginFactoryConfigLoadFromEnv: - """Tests for PluginFactoryConfig.load_from_env method.""" - - def test_load_from_env_valid_configuration(self, mock_args, setup_test_env, monkeypatch): - """Test loading configuration with all required fields present.""" - # Update mock_args to use the setup_test_env paths - mock_args.config_dir = setup_test_env["config_dir"] - mock_args.repo_path = setup_test_env["source_dir"] - mock_args.workspace_path = "." - - # Ensure environment variables are set - monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") - monkeypatch.setenv("WORKSPACE_PATH", ".") - - # Load configuration - config = PluginFactoryConfig.load_from_env(mock_args) - - # Verify all required fields are set - assert config.rhdh_cli_version == "1.7.2" - assert config.log_level == "INFO" - assert config.config_dir == setup_test_env["config_dir"] - assert config.repo_path == setup_test_env["source_dir"] - assert config.workspace_path == "." - assert config.use_local is False - - # Verify directories exist - assert os.path.exists(config.config_dir) - assert os.path.exists(config.repo_path) - - # Verify path types are strings - assert isinstance(config.config_dir, str) - assert isinstance(config.repo_path, str) - assert isinstance(config.workspace_path, str) - - def test_load_from_env_missing_rhdh_cli_version(self, mock_args, setup_test_env, clean_env): - """Test that missing RHDH_CLI_VERSION raises ValueError.""" - # Don't set RHDH_CLI_VERSION - clean_env.setenv("WORKSPACE_PATH", ".") - - mock_args.config_dir = setup_test_env["config_dir"] - mock_args.repo_path = setup_test_env["source_dir"] - mock_args.workspace_path = "." - - # Patch to prevent loading from default.env - with patch.object(Path, 'exists', return_value=False): - with pytest.raises(ValueError, match="RHDH_CLI_VERSION must be set"): - PluginFactoryConfig.load_from_env(mock_args) - - def test_load_from_env_invalid_log_level(self, mock_args, setup_test_env, monkeypatch): - """Test that invalid log level raises ValueError.""" - # Set required environment variables - monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") - monkeypatch.setenv("WORKSPACE_PATH", ".") - monkeypatch.setenv("LOG_LEVEL", "INVALID_LEVEL") - - mock_args.config_dir = setup_test_env["config_dir"] - mock_args.repo_path = setup_test_env["source_dir"] - mock_args.workspace_path = "." - mock_args.log_level = "INVALID_LEVEL" - - with pytest.raises(ValueError, match="Invalid log level"): - PluginFactoryConfig.load_from_env(mock_args) - - @pytest.mark.parametrize("log_level", ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]) - def test_load_from_env_valid_log_levels(self, mock_args, setup_test_env, monkeypatch, log_level): - """Test that all valid log levels are accepted.""" - monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") - monkeypatch.setenv("WORKSPACE_PATH", ".") - monkeypatch.setenv("LOG_LEVEL", log_level) - - mock_args.config_dir = setup_test_env["config_dir"] - mock_args.repo_path = setup_test_env["source_dir"] - mock_args.workspace_path = "." - mock_args.log_level = log_level - - config = PluginFactoryConfig.load_from_env(mock_args) - assert config.log_level == log_level - - def test_load_from_env_environment_variable_precedence(self, mock_args, setup_test_env, clean_env, tmp_path): - """Test that custom .env file with override=True overrides environment variables.""" - # Set initial environment variables - clean_env.setenv("RHDH_CLI_VERSION", "1.7.2") - clean_env.setenv("WORKSPACE_PATH", ".") - - # Create a custom .env file with different values - # Since load_dotenv is called with override=True, these values should win - custom_env_file = tmp_path / "custom.env" - custom_env_file.write_text("RHDH_CLI_VERSION=1.5.0\n") - - mock_args.config_dir = setup_test_env["config_dir"] - mock_args.repo_path = setup_test_env["source_dir"] - mock_args.workspace_path = "." - - # Load the config - the custom env file is loaded with override=True - # So its values will override the environment variables - config = PluginFactoryConfig.load_from_env(mock_args, env_file=custom_env_file) - - # Custom .env file values should override the initial env vars - assert config.rhdh_cli_version == "1.5.0" - - def test_load_from_env_additional_env_file_loading(self, mock_args, setup_test_env, tmp_path, monkeypatch): - """Test that additional .env file merges with defaults.""" - # Create a custom .env file with additional configuration - custom_env_file = tmp_path / "custom.env" - custom_env_file.write_text( - "RHDH_CLI_VERSION=1.6.0\n" - "REGISTRY_URL=quay.io\n" - "REGISTRY_NAMESPACE=test-namespace\n" - ) - - # Set required variables - monkeypatch.setenv("WORKSPACE_PATH", ".") - - mock_args.config_dir = setup_test_env["config_dir"] - mock_args.repo_path = setup_test_env["source_dir"] - mock_args.workspace_path = "." - - # Load with custom env file (should be loaded and override defaults) - with patch("src.rhdh_dynamic_plugin_factory.config.load_dotenv") as mock_load_dotenv: - # Let the actual load_dotenv run but track it - from dotenv import load_dotenv as real_load_dotenv - mock_load_dotenv.side_effect = real_load_dotenv - - # Actually set the env vars for the test - monkeypatch.setenv("RHDH_CLI_VERSION", "1.6.0") - monkeypatch.setenv("REGISTRY_URL", "quay.io") - monkeypatch.setenv("REGISTRY_NAMESPACE", "test-namespace") - - config = PluginFactoryConfig.load_from_env(mock_args, env_file=custom_env_file) - - # Verify custom env file was loaded - assert mock_load_dotenv.call_count >= 1 - - # Verify values from custom env file - assert config.rhdh_cli_version == "1.6.0" - assert config.registry_url == "quay.io" - assert config.registry_namespace == "test-namespace" - - def test_load_from_env_missing_workspace_path(self, mock_args, setup_test_env, monkeypatch): - """Test that missing workspace_path raises ValueError.""" - monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") - # Don't set WORKSPACE_PATH env var to test fallback - monkeypatch.delenv("WORKSPACE_PATH", raising=False) - - mock_args.config_dir = setup_test_env["config_dir"] - mock_args.repo_path = setup_test_env["source_dir"] - mock_args.workspace_path = None # Missing workspace_path - - # When workspace_path is None and WORKSPACE_PATH env var is not set, - # validation should raise ValueError - with pytest.raises(ValueError, match="WORKSPACE_PATH must be set"): - PluginFactoryConfig.load_from_env(mock_args) - - def test_load_from_env_directory_creation(self, mock_args, tmp_path, monkeypatch): - """Test that config_dir and repo_path directories are created.""" - monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") - monkeypatch.setenv("WORKSPACE_PATH", ".") - - # Use non-existent directories - new_config_dir = tmp_path / "new_config" - new_repo_path = tmp_path / "new_workspace" - - # Create minimal required files in config_dir for validation - new_config_dir.mkdir(parents=True, exist_ok=True) - new_repo_path.mkdir(parents=True, exist_ok=True) - - # Create dummy file in repo_path to satisfy validation - (new_repo_path / "dummy.txt").write_text("test") - - mock_args.config_dir = str(new_config_dir) - mock_args.repo_path = str(new_repo_path) - mock_args.workspace_path = "." - - config = PluginFactoryConfig.load_from_env(mock_args) - - # Verify directories exist - assert os.path.exists(config.config_dir) - assert os.path.exists(config.repo_path) - assert new_config_dir.exists() - assert new_repo_path.exists() - - def test_load_from_env_registry_config_from_environment(self, mock_args, setup_test_env, monkeypatch): - """Test that registry configuration is loaded from environment variables.""" - monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") - monkeypatch.setenv("WORKSPACE_PATH", ".") - monkeypatch.setenv("REGISTRY_URL", "quay.io") - monkeypatch.setenv("REGISTRY_USERNAME", "test_user") - monkeypatch.setenv("REGISTRY_PASSWORD", "test_pass") - monkeypatch.setenv("REGISTRY_NAMESPACE", "test_namespace") - monkeypatch.setenv("REGISTRY_INSECURE", "true") - - mock_args.config_dir = setup_test_env["config_dir"] - mock_args.repo_path = setup_test_env["source_dir"] - mock_args.workspace_path = "." - - config = PluginFactoryConfig.load_from_env(mock_args) - - # Verify registry configuration - assert config.registry_url == "quay.io" - assert config.registry_username == "test_user" - assert config.registry_password == "test_pass" - assert config.registry_namespace == "test_namespace" - assert config.registry_insecure is True - - def test_load_from_env_registry_insecure_false(self, mock_args, setup_test_env, monkeypatch): - """Test that REGISTRY_INSECURE defaults to False.""" - monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") - monkeypatch.setenv("WORKSPACE_PATH", ".") - monkeypatch.setenv("REGISTRY_INSECURE", "false") - - mock_args.config_dir = setup_test_env["config_dir"] - mock_args.repo_path = setup_test_env["source_dir"] - mock_args.workspace_path = "." - - config = PluginFactoryConfig.load_from_env(mock_args) - - assert config.registry_insecure is False - - def test_load_from_env_use_local_flag(self, mock_args, setup_test_env, monkeypatch): - """Test that use_local flag is loaded from args.""" - monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") - monkeypatch.setenv("WORKSPACE_PATH", ".") - - mock_args.config_dir = setup_test_env["config_dir"] - mock_args.repo_path = setup_test_env["source_dir"] - mock_args.workspace_path = "." - mock_args.use_local = True - - config = PluginFactoryConfig.load_from_env(mock_args) - - assert config.use_local is True - - def test_load_from_env_source_json_missing_repo_path_empty(self, mock_args, tmp_path, monkeypatch): - """Test that missing source.json with empty repo_path raises ValueError.""" - monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") - monkeypatch.setenv("WORKSPACE_PATH", ".") - - # Create empty directories - config_dir = tmp_path / "config" - config_dir.mkdir(parents=True, exist_ok=True) - - repo_path = tmp_path / "workspace" - repo_path.mkdir(parents=True, exist_ok=True) - # repo_path is empty (no files) - - mock_args.config_dir = str(config_dir) - mock_args.repo_path = str(repo_path) - mock_args.workspace_path = "." - - with pytest.raises(ValueError, match="source.json not found"): - PluginFactoryConfig.load_from_env(mock_args) - - def test_load_from_env_source_json_missing_repo_path_has_content(self, mock_args, tmp_path, monkeypatch): - """Test that missing source.json with non-empty repo_path logs warning but passes.""" - monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") - monkeypatch.setenv("WORKSPACE_PATH", ".") - - # Create config dir without source.json - config_dir = tmp_path / "config" - config_dir.mkdir(parents=True, exist_ok=True) - - # Create repo_path with some content - repo_path = tmp_path / "workspace" - repo_path.mkdir(parents=True, exist_ok=True) - (repo_path / "some_file.txt").write_text("content") - - mock_args.config_dir = str(config_dir) - mock_args.repo_path = str(repo_path) - mock_args.workspace_path = "." - - # Should not raise, just log warning - config = PluginFactoryConfig.load_from_env(mock_args) - - assert config is not None - assert config.config_dir == str(config_dir) - assert config.repo_path == str(repo_path) - - -class TestLoadRegistryConfig: - """Tests for PluginFactoryConfig.load_registry_config method.""" - - def test_skip_when_push_images_false(self, make_config): - """Test that registry configuration is skipped when push_images is False.""" - config = make_config() - - with patch.object(config, 'logger') as mock_logger: - config.load_registry_config(push_images=False) - mock_logger.info.assert_called_once_with( - "Skipping registry configuration (not pushing images)" - ) - - def test_missing_registry_url(self, make_config): - """Test that missing REGISTRY_URL raises ValueError when push_images is True.""" - config = make_config(registry_url=None, registry_namespace="test-namespace") - - with pytest.raises(ValueError, match="REGISTRY_URL environment variable is required"): - config.load_registry_config(push_images=True) - - def test_missing_registry_namespace(self, make_config): - """Test that missing REGISTRY_NAMESPACE raises ValueError when push_images is True.""" - config = make_config(registry_url="quay.io", registry_namespace=None) - - with pytest.raises(ValueError, match="REGISTRY_NAMESPACE environment variable is required"): - config.load_registry_config(push_images=True) - - def test_successful_buildah_login(self, make_config): - """Test successful buildah login with valid credentials.""" - config = make_config( - registry_url="quay.io", - registry_namespace="test-namespace", - registry_username="test-user", - registry_password="test-password", - registry_insecure=False - ) - - with patch('subprocess.run') as mock_run: - mock_run.return_value = MagicMock(returncode=0) - - with patch.object(config, 'logger') as mock_logger: - config.load_registry_config(push_images=True) - - mock_run.assert_called_once() - call_args = mock_run.call_args - - expected_cmd = [ - "buildah", "login", - "--username", "test-user", - "--password", "test-password", - "quay.io" - ] - assert call_args[0][0] == expected_cmd - assert call_args[1]['check'] is True - assert call_args[1]['stdout'] == subprocess.PIPE - assert call_args[1]['stderr'] == subprocess.PIPE - - mock_logger.info.assert_called_with( - "Logged in to registry quay.io with buildah." - ) - - def test_failed_buildah_login(self, make_config): - """Test that failed buildah login logs warning but doesn't raise.""" - config = make_config( - registry_url="quay.io", - registry_namespace="test-namespace", - registry_username="test-user", - registry_password="wrong-password", - registry_insecure=False - ) - - with patch('subprocess.run') as mock_run: - mock_error = subprocess.CalledProcessError( - returncode=1, - cmd=['buildah', 'login'], - stderr=b"Authentication failed" - ) - mock_run.side_effect = mock_error - - with patch.object(config, 'logger') as mock_logger: - config.load_registry_config(push_images=True) - - mock_logger.warning.assert_called_once() - warning_call = mock_logger.warning.call_args[0][0] - assert "Failed to login to registry quay.io" in warning_call - assert "Authentication failed" in warning_call - - def test_missing_registry_credentials(self, make_config): - """Test that missing credentials raise ValueError when push_images is True.""" - config = make_config( - registry_url="quay.io", - registry_namespace="test-namespace", - registry_username=None, - registry_password=None, - registry_insecure=False - ) - - with pytest.raises(ValueError, match="REGISTRY_USERNAME and REGISTRY_PASSWORD environment variables are required"): - config.load_registry_config(push_images=True) - - def test_missing_registry_username(self, make_config): - """Test that missing username raises ValueError when push_images is True.""" - config = make_config( - registry_url="quay.io", - registry_namespace="test-namespace", - registry_username=None, - registry_password="test-password", - registry_insecure=False - ) - - with pytest.raises(ValueError, match="REGISTRY_USERNAME and REGISTRY_PASSWORD environment variables are required"): - config.load_registry_config(push_images=True) - - def test_missing_registry_password(self, make_config): - """Test that missing password raises ValueError when push_images is True.""" - config = make_config( - registry_url="quay.io", - registry_namespace="test-namespace", - registry_username="test-user", - registry_password=None, - registry_insecure=False - ) - - with pytest.raises(ValueError, match="REGISTRY_USERNAME and REGISTRY_PASSWORD environment variables are required"): - config.load_registry_config(push_images=True) - - def test_insecure_registry_flag(self, make_config): - """Test that insecure flag is added to buildah command when registry_insecure is True.""" - config = make_config( - registry_url="localhost:5000", - registry_namespace="test-namespace", - registry_username="test-user", - registry_password="test-password", - registry_insecure=True - ) - - with patch('subprocess.run') as mock_run: - mock_run.return_value = MagicMock(returncode=0) - - config.load_registry_config(push_images=True) - - mock_run.assert_called_once() - call_args = mock_run.call_args - - expected_cmd = [ - "buildah", "login", - "--username", "test-user", - "--password", "test-password", - "--tls-verify=false", - "localhost:5000" - ] - assert call_args[0][0] == expected_cmd - - def test_secure_registry_default(self, make_config): - """Test that insecure flag is NOT added when registry_insecure is False.""" - config = make_config( - registry_url="quay.io", - registry_namespace="test-namespace", - registry_username="test-user", - registry_password="test-password", - registry_insecure=False - ) - - with patch('subprocess.run') as mock_run: - mock_run.return_value = MagicMock(returncode=0) - - config.load_registry_config(push_images=True) - - mock_run.assert_called_once() - call_args = mock_run.call_args - - expected_cmd = [ - "buildah", "login", - "--username", "test-user", - "--password", "test-password", - "quay.io" - ] - assert call_args[0][0] == expected_cmd - assert "--tls-verify=false" not in call_args[0][0] - - -class TestApplyPatchesAndOverlays: - """Tests for PluginFactoryConfig.apply_patches_and_overlays method.""" - - def test_apply_patches_and_overlays_success(self, make_config): - """Test successful execution of apply_patches_and_overlays.""" - config = make_config() - - script_dir = Path(__file__).parent.parent / "scripts" - script_path = script_dir / "override-sources.sh" - - with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run_cmd: - with patch.object(Path, "exists", return_value=True): - mock_run_cmd.return_value = 0 - - result = config.apply_patches_and_overlays() - - assert result is True - - mock_run_cmd.assert_called_once() - call_args = mock_run_cmd.call_args - - cmd = call_args[0][0] - assert len(cmd) == 3 - assert cmd[0] == str(script_path.absolute()) - assert cmd[1] == os.path.abspath(config.config_dir) - expected_workspace = os.path.abspath(os.path.join(config.repo_path, config.workspace_path)) - assert cmd[2] == expected_workspace - - assert call_args[0][1] == config.logger - assert call_args[1]["cwd"] == Path(expected_workspace) - assert call_args[1]["stderr_log_func"] == config.logger.error - - def test_apply_patches_and_overlays_script_not_found(self, make_config): - """Test that apply_patches_and_overlays returns False when script doesn't exist.""" - config = make_config() - - with patch.object(Path, "exists", return_value=False): - with patch.object(config, "logger") as mock_logger: - result = config.apply_patches_and_overlays() - - assert result is False - - mock_logger.error.assert_called_once() - error_msg = mock_logger.error.call_args[0][0] - assert "Script not found" in error_msg - - def test_apply_patches_and_overlays_script_fails(self, make_config): - """Test that apply_patches_and_overlays returns False when script returns non-zero exit code.""" - config = make_config() - - with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run_cmd: - with patch.object(Path, "exists", return_value=True): - with patch.object(config, "logger") as mock_logger: - mock_run_cmd.return_value = 1 - - result = config.apply_patches_and_overlays() - - assert result is False - - error_calls = [call[0][0] for call in mock_logger.error.call_args_list] - assert any("exit code 1" in str(call) for call in error_calls) - - def test_apply_patches_and_overlays_exception(self, make_config): - """Test that apply_patches_and_overlays handles exceptions gracefully.""" - config = make_config() - - with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run_cmd: - with patch.object(Path, "exists", return_value=True): - with patch.object(config, "logger") as mock_logger: - mock_run_cmd.side_effect = Exception("Test exception") - - result = config.apply_patches_and_overlays() - - assert result is False - - mock_logger.error.assert_called() - error_msg = mock_logger.error.call_args[0][0] - assert "Failed to run patch script" in error_msg - assert "Test exception" in error_msg - - -class TestExportPlugins: - """Tests for PluginFactoryConfig.export_plugins method.""" - - def test_export_plugins_success(self, make_config, setup_test_env): - """Test successful execution of export_plugins.""" - config = make_config( - registry_url="quay.io", - registry_namespace="test-namespace" - ) - - output_dir = str(setup_test_env["tmp_path"] / "output") - - with patch.object(Path, "exists", return_value=True): - with patch("os.path.exists", return_value=True): - with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run_cmd: - with patch("src.rhdh_dynamic_plugin_factory.config.display_export_results") as mock_display: - with patch("src.rhdh_dynamic_plugin_factory.config.load_dotenv"): - mock_run_cmd.return_value = 0 - mock_display.return_value = False - - result = config.export_plugins(output_dir, push_images=False) - - assert result is True - - mock_run_cmd.assert_called_once() - call_args = mock_run_cmd.call_args - - cmd = call_args[0][0] - assert len(cmd) == 1 - assert "export-workspace.sh" in cmd[0] - - assert call_args[0][1] == config.logger - - expected_workspace = os.path.abspath(os.path.join(config.repo_path, config.workspace_path)) - assert call_args[1]["cwd"] == Path(expected_workspace) - - env = call_args[1]["env"] - assert "INPUTS_DESTINATION" in env - assert "INPUTS_PLUGINS_FILE" in env - assert "INPUTS_PUSH_CONTAINER_IMAGE" in env - - def test_export_plugins_environment_variables_no_push(self, make_config, setup_test_env): - """Test that environment variables are correctly set when push_images is False.""" - config = make_config( - registry_url="quay.io", - registry_namespace="test-namespace" - ) - - tmp_path = setup_test_env["tmp_path"] - output_dir = str(tmp_path / "output") - - with patch.object(Path, "exists", return_value=True): - with patch("os.path.exists", return_value=True): - with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run_cmd: - with patch("src.rhdh_dynamic_plugin_factory.config.display_export_results") as mock_display: - with patch("src.rhdh_dynamic_plugin_factory.config.load_dotenv"): - mock_run_cmd.return_value = 0 - mock_display.return_value = False - - _result = config.export_plugins(output_dir, push_images=False) - - env = mock_run_cmd.call_args[1]["env"] - - assert env["INPUTS_SCALPRUM_CONFIG_FILE_NAME"] == "scalprum-config.json" - assert env["INPUTS_SOURCE_OVERLAY_FOLDER_NAME"] == "overlay" - assert env["INPUTS_SOURCE_PATCH_FILE_NAME"] == "patch" - assert env["INPUTS_APP_CONFIG_FILE_NAME"] == "app-config.dynamic.yaml" - assert env["INPUTS_CLI_PACKAGE"] == "@red-hat-developer-hub/cli" - assert env["INPUTS_PUSH_CONTAINER_IMAGE"] == "false" - assert env["INPUTS_JANUS_CLI_VERSION"] == "1.7.2" - assert env["INPUTS_IMAGE_REPOSITORY_PREFIX"] == "quay.io/test-namespace" - assert env["INPUTS_CONTAINER_BUILD_TOOL"] == "buildah" - assert str((tmp_path / "output").absolute()) in env["INPUTS_DESTINATION"] - assert "plugins-list.yaml" in env["INPUTS_PLUGINS_FILE"] - - def test_export_plugins_environment_variables_with_push(self, make_config, setup_test_env): - """Test that environment variables are correctly set when push_images is True.""" - config = make_config( - registry_url="quay.io", - registry_namespace="test-namespace" - ) - - output_dir = str(setup_test_env["tmp_path"] / "output") - - with patch.object(Path, "exists", return_value=True): - with patch("os.path.exists", return_value=True): - with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run_cmd: - with patch("src.rhdh_dynamic_plugin_factory.config.display_export_results") as mock_display: - with patch("src.rhdh_dynamic_plugin_factory.config.load_dotenv"): - mock_run_cmd.return_value = 0 - mock_display.return_value = False - - _result = config.export_plugins(output_dir, push_images=True) - - env = mock_run_cmd.call_args[1]["env"] - assert env["INPUTS_PUSH_CONTAINER_IMAGE"] == "true" - - def test_export_plugins_default_registry_values(self, make_config, setup_test_env): - """Test that default values are used when registry_url or registry_namespace are None.""" - config = make_config( - registry_url=None, - registry_namespace=None - ) - - output_dir = str(setup_test_env["tmp_path"] / "output") - - with patch.object(Path, "exists", return_value=True): - with patch("os.path.exists", return_value=True): - with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run_cmd: - with patch("src.rhdh_dynamic_plugin_factory.config.display_export_results") as mock_display: - with patch("src.rhdh_dynamic_plugin_factory.config.load_dotenv"): - mock_run_cmd.return_value = 0 - mock_display.return_value = False - - _result = config.export_plugins(output_dir, push_images=False) - - env = mock_run_cmd.call_args[1]["env"] - assert env["INPUTS_IMAGE_REPOSITORY_PREFIX"] == "localhost/default" - - def test_export_plugins_script_not_found(self, make_config, setup_test_env): - """Test that export_plugins returns False when script doesn't exist.""" - config = make_config() - - output_dir = str(setup_test_env["tmp_path"] / "output") - - with patch.object(Path, "exists", return_value=False): - with patch.object(config, "logger") as mock_logger: - result = config.export_plugins(output_dir, push_images=False) - - assert result is False - - mock_logger.error.assert_called_once() - error_msg = mock_logger.error.call_args[0][0] - assert "Script not found" in error_msg - - def test_export_plugins_no_plugins_list(self, make_config, setup_test_env): - """Test that export_plugins returns False when plugins-list.yaml doesn't exist.""" - config = make_config() - - output_dir = str(setup_test_env["tmp_path"] / "output") - - original_path_exists = Path.exists - - def path_exists_side_effect(path_obj): - if "export-workspace.sh" in str(path_obj): - return True - return original_path_exists(path_obj) - - def os_exists_side_effect(path_str): - if "plugins-list.yaml" in str(path_str): - return False - return True - - with patch.object(Path, "exists", new=path_exists_side_effect): - with patch("os.path.exists", side_effect=os_exists_side_effect): - with patch.object(config, "logger") as mock_logger: - result = config.export_plugins(output_dir, push_images=False) - - assert result is False - - error_calls = [call[0][0] for call in mock_logger.error.call_args_list] - assert any("No plugins file found" in str(call) for call in error_calls) - - def test_export_plugins_script_fails(self, make_config, setup_test_env): - """Test that export_plugins returns False when script returns non-zero exit code.""" - config = make_config( - registry_url="quay.io", - registry_namespace="test-namespace" - ) - - output_dir = str(setup_test_env["tmp_path"] / "output") - - with patch.object(Path, "exists", return_value=True): - with patch("os.path.exists", return_value=True): - with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run_cmd: - with patch("src.rhdh_dynamic_plugin_factory.config.load_dotenv"): - with patch.object(config, "logger") as mock_logger: - mock_run_cmd.return_value = 1 - - result = config.export_plugins(output_dir, push_images=False) - - assert result is False - - error_calls = [call[0][0] for call in mock_logger.error.call_args_list] - assert any("exit code 1" in str(call) for call in error_calls) - - def test_export_plugins_has_failures(self, make_config, setup_test_env): - """Test that export_plugins returns False when display_export_results indicates failures.""" - config = make_config( - registry_url="quay.io", - registry_namespace="test-namespace" - ) - - output_dir = str(setup_test_env["tmp_path"] / "output") - - with patch.object(Path, "exists", return_value=True): - with patch("os.path.exists", return_value=True): - with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run_cmd: - with patch("src.rhdh_dynamic_plugin_factory.config.display_export_results") as mock_display: - with patch("src.rhdh_dynamic_plugin_factory.config.load_dotenv"): - with patch.object(config, "logger") as mock_logger: - mock_run_cmd.return_value = 0 - mock_display.return_value = True - - result = config.export_plugins(output_dir, push_images=False) - - assert result is False - - error_calls = [call[0][0] for call in mock_logger.error.call_args_list] - assert any("completed with failures" in str(call) for call in error_calls) - - def test_export_plugins_exception(self, make_config, setup_test_env): - """Test that export_plugins handles exceptions gracefully.""" - config = make_config() - - output_dir = str(setup_test_env["tmp_path"] / "output") - - with patch.object(Path, "exists", return_value=True): - with patch("os.path.exists", return_value=True): - with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run_cmd: - with patch("src.rhdh_dynamic_plugin_factory.config.load_dotenv"): - with patch.object(config, "logger") as mock_logger: - mock_run_cmd.side_effect = Exception("Test exception") - - result = config.export_plugins(output_dir, push_images=False) - - assert result is False - - mock_logger.error.assert_called() - error_msg = mock_logger.error.call_args[0][0] - assert "Failed to run export script" in error_msg - assert "Test exception" in error_msg - - def test_export_plugins_custom_env_file(self, make_config, setup_test_env): - """Test that export_plugins loads custom .env file from config directory.""" - config = make_config( - registry_url="quay.io", - registry_namespace="test-namespace" - ) - - custom_env = Path(setup_test_env["config_dir"]) / ".env" - custom_env.write_text("CUSTOM_VAR=custom_value\n") - - output_dir = str(setup_test_env["tmp_path"] / "output") - - with patch.object(Path, "exists", return_value=True): - with patch("os.path.exists", return_value=True): - with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run_cmd: - with patch("src.rhdh_dynamic_plugin_factory.config.display_export_results") as mock_display: - with patch("src.rhdh_dynamic_plugin_factory.config.load_dotenv") as mock_load_dotenv: - with patch.object(config, "logger") as mock_logger: - mock_run_cmd.return_value = 0 - mock_display.return_value = False - - _result = config.export_plugins(output_dir, push_images=False) - - assert mock_load_dotenv.call_count >= 1 - - debug_calls = [call[0][0] for call in mock_logger.debug.call_args_list] - assert any(".env" in str(call) for call in debug_calls) - diff --git a/tests/test_config_export_plugins.py b/tests/test_config_export_plugins.py new file mode 100644 index 0000000..bece9f5 --- /dev/null +++ b/tests/test_config_export_plugins.py @@ -0,0 +1,274 @@ +""" +Unit tests for PluginFactoryConfig.export_plugins method. + +Tests the plugin export functionality including environment variable setup +and script execution. +""" + +import os +from pathlib import Path +from unittest.mock import patch + +from src.rhdh_dynamic_plugin_factory.config import PluginFactoryConfig + + +class TestExportPlugins: + """Tests for PluginFactoryConfig.export_plugins method.""" + + def test_export_plugins_success(self, make_config, setup_test_env): + """Test successful execution of export_plugins.""" + config = make_config( + registry_url="quay.io", + registry_namespace="test-namespace" + ) + + output_dir = str(setup_test_env["tmp_path"] / "output") + + with patch.object(Path, "exists", return_value=True): + with patch("os.path.exists", return_value=True): + with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run_cmd: + with patch("src.rhdh_dynamic_plugin_factory.config.display_export_results") as mock_display: + with patch("src.rhdh_dynamic_plugin_factory.config.load_dotenv"): + mock_run_cmd.return_value = 0 + mock_display.return_value = False + + result = config.export_plugins(output_dir, push_images=False) + + assert result is True + + mock_run_cmd.assert_called_once() + call_args = mock_run_cmd.call_args + + cmd = call_args[0][0] + assert len(cmd) == 1 + assert "export-workspace.sh" in cmd[0] + + assert call_args[0][1] == config.logger + + expected_workspace = os.path.abspath(os.path.join(config.repo_path, config.workspace_path)) + assert call_args[1]["cwd"] == Path(expected_workspace) + + env = call_args[1]["env"] + assert "INPUTS_DESTINATION" in env + assert "INPUTS_PLUGINS_FILE" in env + assert "INPUTS_PUSH_CONTAINER_IMAGE" in env + + def test_export_plugins_environment_variables_no_push(self, make_config, setup_test_env): + """Test that environment variables are correctly set when push_images is False.""" + config = make_config( + registry_url="quay.io", + registry_namespace="test-namespace" + ) + + tmp_path = setup_test_env["tmp_path"] + output_dir = str(tmp_path / "output") + + with patch.object(Path, "exists", return_value=True): + with patch("os.path.exists", return_value=True): + with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run_cmd: + with patch("src.rhdh_dynamic_plugin_factory.config.display_export_results") as mock_display: + with patch("src.rhdh_dynamic_plugin_factory.config.load_dotenv"): + mock_run_cmd.return_value = 0 + mock_display.return_value = False + + _result = config.export_plugins(output_dir, push_images=False) + + env = mock_run_cmd.call_args[1]["env"] + + assert env["INPUTS_SCALPRUM_CONFIG_FILE_NAME"] == "scalprum-config.json" + assert env["INPUTS_SOURCE_OVERLAY_FOLDER_NAME"] == "overlay" + assert env["INPUTS_SOURCE_PATCH_FILE_NAME"] == "patch" + assert env["INPUTS_APP_CONFIG_FILE_NAME"] == "app-config.dynamic.yaml" + assert env["INPUTS_CLI_PACKAGE"] == "@red-hat-developer-hub/cli" + assert env["INPUTS_PUSH_CONTAINER_IMAGE"] == "false" + assert env["INPUTS_JANUS_CLI_VERSION"] == "1.7.2" + assert env["INPUTS_IMAGE_REPOSITORY_PREFIX"] == "quay.io/test-namespace" + assert env["INPUTS_CONTAINER_BUILD_TOOL"] == "buildah" + assert str((tmp_path / "output").absolute()) in env["INPUTS_DESTINATION"] + assert "plugins-list.yaml" in env["INPUTS_PLUGINS_FILE"] + + def test_export_plugins_environment_variables_with_push(self, make_config, setup_test_env): + """Test that environment variables are correctly set when push_images is True.""" + config = make_config( + registry_url="quay.io", + registry_namespace="test-namespace" + ) + + output_dir = str(setup_test_env["tmp_path"] / "output") + + with patch.object(Path, "exists", return_value=True): + with patch("os.path.exists", return_value=True): + with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run_cmd: + with patch("src.rhdh_dynamic_plugin_factory.config.display_export_results") as mock_display: + with patch("src.rhdh_dynamic_plugin_factory.config.load_dotenv"): + mock_run_cmd.return_value = 0 + mock_display.return_value = False + + _result = config.export_plugins(output_dir, push_images=True) + + env = mock_run_cmd.call_args[1]["env"] + assert env["INPUTS_PUSH_CONTAINER_IMAGE"] == "true" + + def test_export_plugins_default_registry_values(self, make_config, setup_test_env): + """Test that default values are used when registry_url or registry_namespace are None.""" + config = make_config( + registry_url=None, + registry_namespace=None + ) + + output_dir = str(setup_test_env["tmp_path"] / "output") + + with patch.object(Path, "exists", return_value=True): + with patch("os.path.exists", return_value=True): + with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run_cmd: + with patch("src.rhdh_dynamic_plugin_factory.config.display_export_results") as mock_display: + with patch("src.rhdh_dynamic_plugin_factory.config.load_dotenv"): + mock_run_cmd.return_value = 0 + mock_display.return_value = False + + _result = config.export_plugins(output_dir, push_images=False) + + env = mock_run_cmd.call_args[1]["env"] + assert env["INPUTS_IMAGE_REPOSITORY_PREFIX"] == "localhost/default" + + def test_export_plugins_script_not_found(self, make_config, setup_test_env): + """Test that export_plugins returns False when script doesn't exist.""" + config = make_config() + + output_dir = str(setup_test_env["tmp_path"] / "output") + + with patch.object(Path, "exists", return_value=False): + with patch.object(config, "logger") as mock_logger: + result = config.export_plugins(output_dir, push_images=False) + + assert result is False + + mock_logger.error.assert_called_once() + error_msg = mock_logger.error.call_args[0][0] + assert "Script not found" in error_msg + + def test_export_plugins_no_plugins_list(self, make_config, setup_test_env): + """Test that export_plugins returns False when plugins-list.yaml doesn't exist.""" + config = make_config() + + output_dir = str(setup_test_env["tmp_path"] / "output") + + original_path_exists = Path.exists + + def path_exists_side_effect(path_obj): + if "export-workspace.sh" in str(path_obj): + return True + return original_path_exists(path_obj) + + def os_exists_side_effect(path_str): + if "plugins-list.yaml" in str(path_str): + return False + return True + + with patch.object(Path, "exists", new=path_exists_side_effect): + with patch("os.path.exists", side_effect=os_exists_side_effect): + with patch.object(config, "logger") as mock_logger: + result = config.export_plugins(output_dir, push_images=False) + + assert result is False + + error_calls = [call[0][0] for call in mock_logger.error.call_args_list] + assert any("No plugins file found" in str(call) for call in error_calls) + + def test_export_plugins_script_fails(self, make_config, setup_test_env): + """Test that export_plugins returns False when script returns non-zero exit code.""" + config = make_config( + registry_url="quay.io", + registry_namespace="test-namespace" + ) + + output_dir = str(setup_test_env["tmp_path"] / "output") + + with patch.object(Path, "exists", return_value=True): + with patch("os.path.exists", return_value=True): + with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run_cmd: + with patch("src.rhdh_dynamic_plugin_factory.config.load_dotenv"): + with patch.object(config, "logger") as mock_logger: + mock_run_cmd.return_value = 1 + + result = config.export_plugins(output_dir, push_images=False) + + assert result is False + + error_calls = [call[0][0] for call in mock_logger.error.call_args_list] + assert any("exit code 1" in str(call) for call in error_calls) + + def test_export_plugins_has_failures(self, make_config, setup_test_env): + """Test that export_plugins returns False when display_export_results indicates failures.""" + config = make_config( + registry_url="quay.io", + registry_namespace="test-namespace" + ) + + output_dir = str(setup_test_env["tmp_path"] / "output") + + with patch.object(Path, "exists", return_value=True): + with patch("os.path.exists", return_value=True): + with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run_cmd: + with patch("src.rhdh_dynamic_plugin_factory.config.display_export_results") as mock_display: + with patch("src.rhdh_dynamic_plugin_factory.config.load_dotenv"): + with patch.object(config, "logger") as mock_logger: + mock_run_cmd.return_value = 0 + mock_display.return_value = True + + result = config.export_plugins(output_dir, push_images=False) + + assert result is False + + error_calls = [call[0][0] for call in mock_logger.error.call_args_list] + assert any("completed with failures" in str(call) for call in error_calls) + + def test_export_plugins_exception(self, make_config, setup_test_env): + """Test that export_plugins handles exceptions gracefully.""" + config = make_config() + + output_dir = str(setup_test_env["tmp_path"] / "output") + + with patch.object(Path, "exists", return_value=True): + with patch("os.path.exists", return_value=True): + with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run_cmd: + with patch("src.rhdh_dynamic_plugin_factory.config.load_dotenv"): + with patch.object(config, "logger") as mock_logger: + mock_run_cmd.side_effect = Exception("Test exception") + + result = config.export_plugins(output_dir, push_images=False) + + assert result is False + + mock_logger.error.assert_called() + error_msg = mock_logger.error.call_args[0][0] + assert "Failed to run export script" in error_msg + assert "Test exception" in error_msg + + def test_export_plugins_custom_env_file(self, make_config, setup_test_env): + """Test that export_plugins loads custom .env file from config directory.""" + config = make_config( + registry_url="quay.io", + registry_namespace="test-namespace" + ) + + custom_env = Path(setup_test_env["config_dir"]) / ".env" + custom_env.write_text("CUSTOM_VAR=custom_value\n") + + output_dir = str(setup_test_env["tmp_path"] / "output") + + with patch.object(Path, "exists", return_value=True): + with patch("os.path.exists", return_value=True): + with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run_cmd: + with patch("src.rhdh_dynamic_plugin_factory.config.display_export_results") as mock_display: + with patch("src.rhdh_dynamic_plugin_factory.config.load_dotenv") as mock_load_dotenv: + with patch.object(config, "logger") as mock_logger: + mock_run_cmd.return_value = 0 + mock_display.return_value = False + + _result = config.export_plugins(output_dir, push_images=False) + + assert mock_load_dotenv.call_count >= 1 + + debug_calls = [call[0][0] for call in mock_logger.debug.call_args_list] + assert any(".env" in str(call) for call in debug_calls) diff --git a/tests/test_config_load_from_env.py b/tests/test_config_load_from_env.py new file mode 100644 index 0000000..4896f22 --- /dev/null +++ b/tests/test_config_load_from_env.py @@ -0,0 +1,291 @@ +""" +Unit tests for PluginFactoryConfig.load_from_env method. + +Tests the configuration loading and validation from environment variables +and .env files. +""" + +import os +from pathlib import Path +from unittest.mock import patch +import pytest + +from src.rhdh_dynamic_plugin_factory.config import PluginFactoryConfig + + +class TestPluginFactoryConfigLoadFromEnv: + """Tests for PluginFactoryConfig.load_from_env method.""" + + def test_load_from_env_valid_configuration(self, mock_args, setup_test_env, monkeypatch): + """Test loading configuration with all required fields present.""" + # Update mock_args to use the setup_test_env paths + mock_args.config_dir = setup_test_env["config_dir"] + mock_args.repo_path = setup_test_env["source_dir"] + mock_args.workspace_path = "." + + # Ensure environment variables are set + monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") + monkeypatch.setenv("WORKSPACE_PATH", ".") + + # Load configuration + config = PluginFactoryConfig.load_from_env(mock_args) + + # Verify all required fields are set + assert config.rhdh_cli_version == "1.7.2" + assert config.log_level == "INFO" + assert config.config_dir == setup_test_env["config_dir"] + assert config.repo_path == setup_test_env["source_dir"] + assert config.workspace_path == "." + assert config.use_local is False + + # Verify directories exist + assert os.path.exists(config.config_dir) + assert os.path.exists(config.repo_path) + + # Verify path types are strings + assert isinstance(config.config_dir, str) + assert isinstance(config.repo_path, str) + assert isinstance(config.workspace_path, str) + + def test_load_from_env_missing_rhdh_cli_version(self, mock_args, setup_test_env, clean_env): + """Test that missing RHDH_CLI_VERSION raises ValueError.""" + # Don't set RHDH_CLI_VERSION + clean_env.setenv("WORKSPACE_PATH", ".") + + mock_args.config_dir = setup_test_env["config_dir"] + mock_args.repo_path = setup_test_env["source_dir"] + mock_args.workspace_path = "." + + # Patch to prevent loading from default.env + with patch.object(Path, 'exists', return_value=False): + with pytest.raises(ValueError, match="RHDH_CLI_VERSION must be set"): + PluginFactoryConfig.load_from_env(mock_args) + + def test_load_from_env_invalid_log_level(self, mock_args, setup_test_env, monkeypatch): + """Test that invalid log level raises ValueError.""" + # Set required environment variables + monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") + monkeypatch.setenv("WORKSPACE_PATH", ".") + monkeypatch.setenv("LOG_LEVEL", "INVALID_LEVEL") + + mock_args.config_dir = setup_test_env["config_dir"] + mock_args.repo_path = setup_test_env["source_dir"] + mock_args.workspace_path = "." + mock_args.log_level = "INVALID_LEVEL" + + with pytest.raises(ValueError, match="Invalid log level"): + PluginFactoryConfig.load_from_env(mock_args) + + @pytest.mark.parametrize("log_level", ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]) + def test_load_from_env_valid_log_levels(self, mock_args, setup_test_env, monkeypatch, log_level): + """Test that all valid log levels are accepted.""" + monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") + monkeypatch.setenv("WORKSPACE_PATH", ".") + monkeypatch.setenv("LOG_LEVEL", log_level) + + mock_args.config_dir = setup_test_env["config_dir"] + mock_args.repo_path = setup_test_env["source_dir"] + mock_args.workspace_path = "." + mock_args.log_level = log_level + + config = PluginFactoryConfig.load_from_env(mock_args) + assert config.log_level == log_level + + def test_load_from_env_environment_variable_precedence(self, mock_args, setup_test_env, clean_env, tmp_path): + """Test that custom .env file with override=True overrides environment variables.""" + # Set initial environment variables + clean_env.setenv("RHDH_CLI_VERSION", "1.7.2") + clean_env.setenv("WORKSPACE_PATH", ".") + + # Create a custom .env file with different values + # Since load_dotenv is called with override=True, these values should win + custom_env_file = tmp_path / "custom.env" + custom_env_file.write_text("RHDH_CLI_VERSION=1.5.0\n") + + mock_args.config_dir = setup_test_env["config_dir"] + mock_args.repo_path = setup_test_env["source_dir"] + mock_args.workspace_path = "." + + # Load the config - the custom env file is loaded with override=True + # So its values will override the environment variables + config = PluginFactoryConfig.load_from_env(mock_args, env_file=custom_env_file) + + # Custom .env file values should override the initial env vars + assert config.rhdh_cli_version == "1.5.0" + + def test_load_from_env_additional_env_file_loading(self, mock_args, setup_test_env, tmp_path, monkeypatch): + """Test that additional .env file merges with defaults.""" + # Create a custom .env file with additional configuration + custom_env_file = tmp_path / "custom.env" + custom_env_file.write_text( + "RHDH_CLI_VERSION=1.6.0\n" + "REGISTRY_URL=quay.io\n" + "REGISTRY_NAMESPACE=test-namespace\n" + ) + + # Set required variables + monkeypatch.setenv("WORKSPACE_PATH", ".") + + mock_args.config_dir = setup_test_env["config_dir"] + mock_args.repo_path = setup_test_env["source_dir"] + mock_args.workspace_path = "." + + # Load with custom env file (should be loaded and override defaults) + with patch("src.rhdh_dynamic_plugin_factory.config.load_dotenv") as mock_load_dotenv: + # Let the actual load_dotenv run but track it + from dotenv import load_dotenv as real_load_dotenv + mock_load_dotenv.side_effect = real_load_dotenv + + # Actually set the env vars for the test + monkeypatch.setenv("RHDH_CLI_VERSION", "1.6.0") + monkeypatch.setenv("REGISTRY_URL", "quay.io") + monkeypatch.setenv("REGISTRY_NAMESPACE", "test-namespace") + + config = PluginFactoryConfig.load_from_env(mock_args, env_file=custom_env_file) + + # Verify custom env file was loaded + assert mock_load_dotenv.call_count >= 1 + + # Verify values from custom env file + assert config.rhdh_cli_version == "1.6.0" + assert config.registry_url == "quay.io" + assert config.registry_namespace == "test-namespace" + + def test_load_from_env_missing_workspace_path(self, mock_args, setup_test_env, monkeypatch): + """Test that missing workspace_path raises ValueError.""" + monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") + # Don't set WORKSPACE_PATH env var to test fallback + monkeypatch.delenv("WORKSPACE_PATH", raising=False) + + mock_args.config_dir = setup_test_env["config_dir"] + mock_args.repo_path = setup_test_env["source_dir"] + mock_args.workspace_path = None # Missing workspace_path + + # When workspace_path is None and WORKSPACE_PATH env var is not set, + # validation should raise ValueError + with pytest.raises(ValueError, match="WORKSPACE_PATH must be set"): + PluginFactoryConfig.load_from_env(mock_args) + + def test_load_from_env_directory_creation(self, mock_args, tmp_path, monkeypatch): + """Test that config_dir and repo_path directories are created.""" + monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") + monkeypatch.setenv("WORKSPACE_PATH", ".") + + # Use non-existent directories + new_config_dir = tmp_path / "new_config" + new_repo_path = tmp_path / "new_workspace" + + # Create minimal required files in config_dir for validation + new_config_dir.mkdir(parents=True, exist_ok=True) + new_repo_path.mkdir(parents=True, exist_ok=True) + + # Create dummy file in repo_path to satisfy validation + (new_repo_path / "dummy.txt").write_text("test") + + mock_args.config_dir = str(new_config_dir) + mock_args.repo_path = str(new_repo_path) + mock_args.workspace_path = "." + + config = PluginFactoryConfig.load_from_env(mock_args) + + # Verify directories exist + assert os.path.exists(config.config_dir) + assert os.path.exists(config.repo_path) + assert new_config_dir.exists() + assert new_repo_path.exists() + + def test_load_from_env_registry_config_from_environment(self, mock_args, setup_test_env, monkeypatch): + """Test that registry configuration is loaded from environment variables.""" + monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") + monkeypatch.setenv("WORKSPACE_PATH", ".") + monkeypatch.setenv("REGISTRY_URL", "quay.io") + monkeypatch.setenv("REGISTRY_USERNAME", "test_user") + monkeypatch.setenv("REGISTRY_PASSWORD", "test_pass") + monkeypatch.setenv("REGISTRY_NAMESPACE", "test_namespace") + monkeypatch.setenv("REGISTRY_INSECURE", "true") + + mock_args.config_dir = setup_test_env["config_dir"] + mock_args.repo_path = setup_test_env["source_dir"] + mock_args.workspace_path = "." + + config = PluginFactoryConfig.load_from_env(mock_args) + + # Verify registry configuration + assert config.registry_url == "quay.io" + assert config.registry_username == "test_user" + assert config.registry_password == "test_pass" + assert config.registry_namespace == "test_namespace" + assert config.registry_insecure is True + + def test_load_from_env_registry_insecure_false(self, mock_args, setup_test_env, monkeypatch): + """Test that REGISTRY_INSECURE defaults to False.""" + monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") + monkeypatch.setenv("WORKSPACE_PATH", ".") + monkeypatch.setenv("REGISTRY_INSECURE", "false") + + mock_args.config_dir = setup_test_env["config_dir"] + mock_args.repo_path = setup_test_env["source_dir"] + mock_args.workspace_path = "." + + config = PluginFactoryConfig.load_from_env(mock_args) + + assert config.registry_insecure is False + + def test_load_from_env_use_local_flag(self, mock_args, setup_test_env, monkeypatch): + """Test that use_local flag is loaded from args.""" + monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") + monkeypatch.setenv("WORKSPACE_PATH", ".") + + mock_args.config_dir = setup_test_env["config_dir"] + mock_args.repo_path = setup_test_env["source_dir"] + mock_args.workspace_path = "." + mock_args.use_local = True + + config = PluginFactoryConfig.load_from_env(mock_args) + + assert config.use_local is True + + def test_load_from_env_source_json_missing_repo_path_empty(self, mock_args, tmp_path, monkeypatch): + """Test that missing source.json with empty repo_path raises ValueError.""" + monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") + monkeypatch.setenv("WORKSPACE_PATH", ".") + + # Create empty directories + config_dir = tmp_path / "config" + config_dir.mkdir(parents=True, exist_ok=True) + + repo_path = tmp_path / "workspace" + repo_path.mkdir(parents=True, exist_ok=True) + # repo_path is empty (no files) + + mock_args.config_dir = str(config_dir) + mock_args.repo_path = str(repo_path) + mock_args.workspace_path = "." + + with pytest.raises(ValueError, match="source.json not found"): + PluginFactoryConfig.load_from_env(mock_args) + + def test_load_from_env_source_json_missing_repo_path_has_content(self, mock_args, tmp_path, monkeypatch): + """Test that missing source.json with non-empty repo_path logs warning but passes.""" + monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") + monkeypatch.setenv("WORKSPACE_PATH", ".") + + # Create config dir without source.json + config_dir = tmp_path / "config" + config_dir.mkdir(parents=True, exist_ok=True) + + # Create repo_path with some content + repo_path = tmp_path / "workspace" + repo_path.mkdir(parents=True, exist_ok=True) + (repo_path / "some_file.txt").write_text("content") + + mock_args.config_dir = str(config_dir) + mock_args.repo_path = str(repo_path) + mock_args.workspace_path = "." + + # Should not raise, just log warning + config = PluginFactoryConfig.load_from_env(mock_args) + + assert config is not None + assert config.config_dir == str(config_dir) + assert config.repo_path == str(repo_path) diff --git a/tests/test_config_patches_and_overlays.py b/tests/test_config_patches_and_overlays.py new file mode 100644 index 0000000..0f6a768 --- /dev/null +++ b/tests/test_config_patches_and_overlays.py @@ -0,0 +1,92 @@ +""" +Unit tests for PluginFactoryConfig.apply_patches_and_overlays method. + +Tests the patch and overlay application functionality. +""" + +import os +from pathlib import Path +from unittest.mock import patch + +from src.rhdh_dynamic_plugin_factory.config import PluginFactoryConfig + + +class TestApplyPatchesAndOverlays: + """Tests for PluginFactoryConfig.apply_patches_and_overlays method.""" + + def test_apply_patches_and_overlays_success(self, make_config): + """Test successful execution of apply_patches_and_overlays.""" + config = make_config() + + script_dir = Path(__file__).parent.parent / "scripts" + script_path = script_dir / "override-sources.sh" + + with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run_cmd: + with patch.object(Path, "exists", return_value=True): + mock_run_cmd.return_value = 0 + + result = config.apply_patches_and_overlays() + + assert result is True + + mock_run_cmd.assert_called_once() + call_args = mock_run_cmd.call_args + + cmd = call_args[0][0] + assert len(cmd) == 3 + assert cmd[0] == str(script_path.absolute()) + assert cmd[1] == os.path.abspath(config.config_dir) + expected_workspace = os.path.abspath(os.path.join(config.repo_path, config.workspace_path)) + assert cmd[2] == expected_workspace + + assert call_args[0][1] == config.logger + assert call_args[1]["cwd"] == Path(expected_workspace) + assert call_args[1]["stderr_log_func"] == config.logger.error + + def test_apply_patches_and_overlays_script_not_found(self, make_config): + """Test that apply_patches_and_overlays returns False when script doesn't exist.""" + config = make_config() + + with patch.object(Path, "exists", return_value=False): + with patch.object(config, "logger") as mock_logger: + result = config.apply_patches_and_overlays() + + assert result is False + + mock_logger.error.assert_called_once() + error_msg = mock_logger.error.call_args[0][0] + assert "Script not found" in error_msg + + def test_apply_patches_and_overlays_script_fails(self, make_config): + """Test that apply_patches_and_overlays returns False when script returns non-zero exit code.""" + config = make_config() + + with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run_cmd: + with patch.object(Path, "exists", return_value=True): + with patch.object(config, "logger") as mock_logger: + mock_run_cmd.return_value = 1 + + result = config.apply_patches_and_overlays() + + assert result is False + + error_calls = [call[0][0] for call in mock_logger.error.call_args_list] + assert any("exit code 1" in str(call) for call in error_calls) + + def test_apply_patches_and_overlays_exception(self, make_config): + """Test that apply_patches_and_overlays handles exceptions gracefully.""" + config = make_config() + + with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run_cmd: + with patch.object(Path, "exists", return_value=True): + with patch.object(config, "logger") as mock_logger: + mock_run_cmd.side_effect = Exception("Test exception") + + result = config.apply_patches_and_overlays() + + assert result is False + + mock_logger.error.assert_called() + error_msg = mock_logger.error.call_args[0][0] + assert "Failed to run patch script" in error_msg + assert "Test exception" in error_msg diff --git a/tests/test_config_registry.py b/tests/test_config_registry.py new file mode 100644 index 0000000..a68a34b --- /dev/null +++ b/tests/test_config_registry.py @@ -0,0 +1,192 @@ +""" +Unit tests for PluginFactoryConfig.load_registry_config method. + +Tests the registry configuration loading and buildah login functionality. +""" + +import subprocess +from unittest.mock import patch, MagicMock +import pytest + +from src.rhdh_dynamic_plugin_factory.config import PluginFactoryConfig + + +class TestLoadRegistryConfig: + """Tests for PluginFactoryConfig.load_registry_config method.""" + + def test_skip_when_push_images_false(self, make_config): + """Test that registry configuration is skipped when push_images is False.""" + config = make_config() + + with patch.object(config, 'logger') as mock_logger: + config.load_registry_config(push_images=False) + mock_logger.info.assert_called_once_with( + "Skipping registry configuration (not pushing images)" + ) + + def test_missing_registry_url(self, make_config): + """Test that missing REGISTRY_URL raises ValueError when push_images is True.""" + config = make_config(registry_url=None, registry_namespace="test-namespace") + + with pytest.raises(ValueError, match="REGISTRY_URL environment variable is required"): + config.load_registry_config(push_images=True) + + def test_missing_registry_namespace(self, make_config): + """Test that missing REGISTRY_NAMESPACE raises ValueError when push_images is True.""" + config = make_config(registry_url="quay.io", registry_namespace=None) + + with pytest.raises(ValueError, match="REGISTRY_NAMESPACE environment variable is required"): + config.load_registry_config(push_images=True) + + def test_successful_buildah_login(self, make_config): + """Test successful buildah login with valid credentials.""" + config = make_config( + registry_url="quay.io", + registry_namespace="test-namespace", + registry_username="test-user", + registry_password="test-password", + registry_insecure=False + ) + + with patch('subprocess.run') as mock_run: + mock_run.return_value = MagicMock(returncode=0) + + with patch.object(config, 'logger') as mock_logger: + config.load_registry_config(push_images=True) + + mock_run.assert_called_once() + call_args = mock_run.call_args + + expected_cmd = [ + "buildah", "login", + "--username", "test-user", + "--password", "test-password", + "quay.io" + ] + assert call_args[0][0] == expected_cmd + assert call_args[1]['check'] is True + assert call_args[1]['stdout'] == subprocess.PIPE + assert call_args[1]['stderr'] == subprocess.PIPE + + mock_logger.info.assert_called_with( + "Logged in to registry quay.io with buildah." + ) + + def test_failed_buildah_login(self, make_config): + """Test that failed buildah login logs warning but doesn't raise.""" + config = make_config( + registry_url="quay.io", + registry_namespace="test-namespace", + registry_username="test-user", + registry_password="wrong-password", + registry_insecure=False + ) + + with patch('subprocess.run') as mock_run: + mock_error = subprocess.CalledProcessError( + returncode=1, + cmd=['buildah', 'login'], + stderr=b"Authentication failed" + ) + mock_run.side_effect = mock_error + + with patch.object(config, 'logger') as mock_logger: + config.load_registry_config(push_images=True) + + mock_logger.warning.assert_called_once() + warning_call = mock_logger.warning.call_args[0][0] + assert "Failed to login to registry quay.io" in warning_call + assert "Authentication failed" in warning_call + + def test_missing_registry_credentials(self, make_config): + """Test that missing credentials raise ValueError when push_images is True.""" + config = make_config( + registry_url="quay.io", + registry_namespace="test-namespace", + registry_username=None, + registry_password=None, + registry_insecure=False + ) + + with pytest.raises(ValueError, match="REGISTRY_USERNAME and REGISTRY_PASSWORD environment variables are required"): + config.load_registry_config(push_images=True) + + def test_missing_registry_username(self, make_config): + """Test that missing username raises ValueError when push_images is True.""" + config = make_config( + registry_url="quay.io", + registry_namespace="test-namespace", + registry_username=None, + registry_password="test-password", + registry_insecure=False + ) + + with pytest.raises(ValueError, match="REGISTRY_USERNAME and REGISTRY_PASSWORD environment variables are required"): + config.load_registry_config(push_images=True) + + def test_missing_registry_password(self, make_config): + """Test that missing password raises ValueError when push_images is True.""" + config = make_config( + registry_url="quay.io", + registry_namespace="test-namespace", + registry_username="test-user", + registry_password=None, + registry_insecure=False + ) + + with pytest.raises(ValueError, match="REGISTRY_USERNAME and REGISTRY_PASSWORD environment variables are required"): + config.load_registry_config(push_images=True) + + def test_insecure_registry_flag(self, make_config): + """Test that insecure flag is added to buildah command when registry_insecure is True.""" + config = make_config( + registry_url="localhost:5000", + registry_namespace="test-namespace", + registry_username="test-user", + registry_password="test-password", + registry_insecure=True + ) + + with patch('subprocess.run') as mock_run: + mock_run.return_value = MagicMock(returncode=0) + + config.load_registry_config(push_images=True) + + mock_run.assert_called_once() + call_args = mock_run.call_args + + expected_cmd = [ + "buildah", "login", + "--username", "test-user", + "--password", "test-password", + "--tls-verify=false", + "localhost:5000" + ] + assert call_args[0][0] == expected_cmd + + def test_secure_registry_default(self, make_config): + """Test that insecure flag is NOT added when registry_insecure is False.""" + config = make_config( + registry_url="quay.io", + registry_namespace="test-namespace", + registry_username="test-user", + registry_password="test-password", + registry_insecure=False + ) + + with patch('subprocess.run') as mock_run: + mock_run.return_value = MagicMock(returncode=0) + + config.load_registry_config(push_images=True) + + mock_run.assert_called_once() + call_args = mock_run.call_args + + expected_cmd = [ + "buildah", "login", + "--username", "test-user", + "--password", "test-password", + "quay.io" + ] + assert call_args[0][0] == expected_cmd + assert "--tls-verify=false" not in call_args[0][0] From 0030e28dcdab4151d8330bdb842568e7934bab64 Mon Sep 17 00:00:00 2001 From: Frank Kong Date: Mon, 9 Feb 2026 14:00:25 -0500 Subject: [PATCH 04/35] chore: refactor error handling Signed-off-by: Frank Kong --- src/rhdh_dynamic_plugin_factory/__init__.py | 13 +- src/rhdh_dynamic_plugin_factory/cli.py | 105 +++++---- src/rhdh_dynamic_plugin_factory/config.py | 209 +++++++++++------- src/rhdh_dynamic_plugin_factory/exceptions.py | 47 ++++ tests/test_config_export_plugins.py | 80 +++---- tests/test_config_load_from_env.py | 19 +- tests/test_config_patches_and_overlays.py | 50 ++--- tests/test_config_registry.py | 21 +- tests/test_source_config.py | 101 ++++----- 9 files changed, 357 insertions(+), 288 deletions(-) create mode 100644 src/rhdh_dynamic_plugin_factory/exceptions.py diff --git a/src/rhdh_dynamic_plugin_factory/__init__.py b/src/rhdh_dynamic_plugin_factory/__init__.py index 049a429..e125e82 100644 --- a/src/rhdh_dynamic_plugin_factory/__init__.py +++ b/src/rhdh_dynamic_plugin_factory/__init__.py @@ -13,6 +13,11 @@ SourceConfig, PluginListConfig, ) +from .exceptions import ( + PluginFactoryError, + ConfigurationError, + ExecutionError, +) from .logger import ( setup_logging, get_logger, @@ -21,6 +26,7 @@ from .utils import ( run_command_with_streaming, display_export_results, + clean_directory, ) __all__ = [ @@ -33,6 +39,11 @@ "SourceConfig", "PluginListConfig", + # Exceptions + "PluginFactoryError", + "ConfigurationError", + "ExecutionError", + # Logging "setup_logging", "get_logger", @@ -41,7 +52,7 @@ # Utilities "run_command_with_streaming", "display_export_results", - + "clean_directory", # Version "__version__", ] diff --git a/src/rhdh_dynamic_plugin_factory/cli.py b/src/rhdh_dynamic_plugin_factory/cli.py index b497553..8dbb328 100644 --- a/src/rhdh_dynamic_plugin_factory/cli.py +++ b/src/rhdh_dynamic_plugin_factory/cli.py @@ -13,6 +13,7 @@ from .logger import setup_logging, get_logger from .config import PluginFactoryConfig from .utils import run_command_with_streaming + from .exceptions import PluginFactoryError, ConfigurationError, ExecutionError except ImportError: # For direct script execution, add parent directory to path sys.path.insert(0, str(Path(__file__).parent.parent)) @@ -20,6 +21,7 @@ from rhdh_dynamic_plugin_factory.logger import setup_logging, get_logger from rhdh_dynamic_plugin_factory.config import PluginFactoryConfig from rhdh_dynamic_plugin_factory.utils import run_command_with_streaming + from rhdh_dynamic_plugin_factory.exceptions import PluginFactoryError, ConfigurationError, ExecutionError logger = get_logger("cli") @@ -94,10 +96,15 @@ def create_parser() -> argparse.ArgumentParser: ) return parser -def install_dependencies(workspace_path: Path) -> bool: - """Install dependencies in the workspace using yarn install with corepack.""" - logger.info("[bold blue]Installing workspace dependencies[/bold blue]") +def install_dependencies(workspace_path: Path) -> None: + """Install dependencies in the workspace using yarn install with corepack. + Raises: + ExecutionError: If any dependency installation step fails. + """ + logger.info("[bold blue]Installing workspace dependencies[/bold blue]") + STEP_NAME = "install dependencies" + commands = [ (["pwd"], "Checking workspace path"), (["corepack", "enable"], "Enabling corepack"), @@ -121,69 +128,83 @@ def install_dependencies(workspace_path: Path) -> bool: ) if returncode != 0: - logger.error(f"{description} failed with exit code {returncode}") - return False + raise ExecutionError( + f"{description} failed with exit code {returncode}", + step=STEP_NAME, + returncode=returncode + ) logger.info(f"[green]✓ {description} completed[/green]") - - return True + except ExecutionError: + raise except Exception as e: - logger.error(f"Failed to install dependencies: {e}") - return False + raise ExecutionError( + f"Failed to install dependencies: {e}", + step=STEP_NAME, + ) from e -def main(): - """Main entry point for the RHDH Dynamic Plugin Factory.""" - parser = create_parser() - args = parser.parse_args() - setup_logging(level=args.log_level, verbose=args.verbose) +def _run(args: argparse.Namespace) -> None: + """Execute the main plugin factory workflow. + + All steps either succeed silently or raise a PluginFactoryError subclass, + which is caught by the centralized handler in main(). + """ logger.info("[bold blue]Setting up configuration directory[/bold blue]") config = PluginFactoryConfig.load_from_env(args=args, env_file=args.config_dir / ".env") - try: - config.load_registry_config(push_images=args.push_images) - source_config = config.setup_config_directory() - except Exception as e: - config.logger.error(f"[red]Configuration error: {e}[/red]") - sys.exit(1) - + config.load_registry_config(push_images=args.push_images) + source_config = config.setup_config_directory() + if source_config and not config.use_local: logger.info("[bold blue]Repository Setup[/bold blue]") - if not source_config.clone_to_path(config.repo_path, clean=args.clean): - logger.error("Failed to clone repository") - sys.exit(1) + source_config.clone_to_path(config.repo_path, clean=args.clean) elif config.use_local or not source_config: if config.use_local: logger.info("[bold blue]--use-local flag is set, using local repository[/bold blue]") else: logger.info("[bold blue]No source configuration found, using local repository[/bold blue]") if not config.repo_path.exists(): - logger.error(f"Local repository does not exist at: {config.repo_path}") - logger.error("Either provide source.json to clone the repository, or ensure workspace exists at directory specified by --repo-path") - sys.exit(1) + raise ConfigurationError( + f"Local repository does not exist at: {config.repo_path}. " + "Either provide source.json to clone the repository, " + "or ensure workspace exists at directory specified by --repo-path" + ) logger.info(f"Using local repository at: {config.repo_path}") - + # Auto-generate plugins-list.yaml if needed (after repository is available) - if not config.auto_generate_plugins_list(): - logger.error("Failed to generate plugins list") - sys.exit(1) + config.auto_generate_plugins_list() + logger.info("[bold blue]Applying Patches and Overlays[/bold blue]") - if not config.apply_patches_and_overlays(): - logger.error("Failed to apply patches and overlays") - sys.exit(1) + config.apply_patches_and_overlays() logger.info("[bold blue]Installing Dependencies[/bold blue]") - workspace_path = config.repo_path.joinpath(config.workspace_path).absolute() - if not install_dependencies(workspace_path): - logger.error("Failed to install dependencies") - sys.exit(1) - + install_dependencies(workspace_path) + logger.info("[bold blue]Exporting plugins using RHDH CLI[/bold blue]") - if not config.export_plugins(args.output_dir, args.push_images): - logger.error("Plugin export failed") + config.export_plugins(args.output_dir, args.push_images) + + +def main(): + """Main entry point for the RHDH Dynamic Plugin Factory.""" + parser = create_parser() + args = parser.parse_args() + setup_logging(level=args.log_level, verbose=args.verbose) + + try: + _run(args) + except ConfigurationError as e: + logger.error(f"[red]Configuration error: {e}[/red]") sys.exit(1) - + except ExecutionError as e: + step_info = f"{e.step}: " if e.step else "" + logger.error(f"[red]{step_info}{e}[/red]") + sys.exit(e.returncode or 1) + except PluginFactoryError as e: + logger.error(f"[red]{e}[/red]") + sys.exit(1) + logger.info("[green]✓ All operations completed successfully[/green]") diff --git a/src/rhdh_dynamic_plugin_factory/config.py b/src/rhdh_dynamic_plugin_factory/config.py index f65cb4e..866b9da 100644 --- a/src/rhdh_dynamic_plugin_factory/config.py +++ b/src/rhdh_dynamic_plugin_factory/config.py @@ -5,7 +5,6 @@ import argparse import os from pathlib import Path -import sys from typing import Dict, Optional from dataclasses import dataclass, field from dotenv import load_dotenv @@ -13,6 +12,7 @@ import json import subprocess +from .exceptions import PluginFactoryError, ConfigurationError, ExecutionError from .logger import get_logger from .utils import clean_directory, run_command_with_streaming, display_export_results @@ -86,14 +86,14 @@ def load_from_env(cls, args: argparse.Namespace, env_file: Optional[Path] = None os.makedirs(dir_path, exist_ok=True) if not config.rhdh_cli_version: - raise ValueError("RHDH_CLI_VERSION must be set (usually loaded from default.env)") + raise ConfigurationError("RHDH_CLI_VERSION must be set (usually loaded from default.env)") if not config.workspace_path: - raise ValueError("WORKSPACE_PATH must be set via environment variable or --workspace-path argument") + raise ConfigurationError("WORKSPACE_PATH must be set via environment variable or --workspace-path argument") valid_log_levels = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] if config.log_level.upper() not in valid_log_levels: - raise ValueError(f"Invalid log level: {config.log_level}") + raise ConfigurationError(f"Invalid log level: {config.log_level}") config._validate_source_json() config._validate_plugins_list() @@ -111,13 +111,13 @@ def load_registry_config(self, push_images: bool = False) -> None: return if not self.registry_url: - raise ValueError("REGISTRY_URL environment variable is required when --push-images is enabled") + raise ConfigurationError("REGISTRY_URL environment variable is required when --push-images is enabled") if not self.registry_namespace: - raise ValueError("REGISTRY_NAMESPACE environment variable is required when --push-images is enabled") + raise ConfigurationError("REGISTRY_NAMESPACE environment variable is required when --push-images is enabled") if not self.registry_username or not self.registry_password: - raise ValueError("REGISTRY_USERNAME and REGISTRY_PASSWORD environment variables are required when --push-images is enabled") + raise ConfigurationError("REGISTRY_USERNAME and REGISTRY_PASSWORD environment variables are required when --push-images is enabled") ## TODO: Add support for token logins for ghcr.io registry as well try: @@ -150,7 +150,7 @@ def _validate_source_json(self) -> None: if not os.path.exists(source_file): if not os.path.exists(self.repo_path) or not os.listdir(self.repo_path): - raise ValueError( + raise ConfigurationError( f"source.json not found at {source_file} and {self.repo_path} is empty. " "Please provide source.json to clone a repository or use --use-local with a locally mounted repository." ) @@ -172,31 +172,34 @@ def _validate_plugins_list(self) -> None: else: self.logger.debug(f"Using plugins-list.yaml from: {plugins_file}") - def auto_generate_plugins_list(self) -> bool: + def auto_generate_plugins_list(self) -> None: """ - Auto-generate plugins-list.yaml + Auto-generate plugins-list.yaml if it doesn't already exist. + Assumes the following: - The repository is cloned to the `repo_path` - The plugins are located in the plugins/* directory - `workspace_path` is the path to the workspace from the root of the repository + + Raises: + PluginFactoryError: If auto-generation fails. """ plugins_file = os.path.join(self.config_dir, "plugins-list.yaml") if os.path.exists(plugins_file): self.logger.debug(f"[green]✓ plugins-list.yaml already exists at {plugins_file}. Skipping auto-generation.[/green]") - return True + return self.logger.info("[bold blue]Auto-generating plugins-list.yaml[/bold blue]") + if not os.path.exists(self.repo_path): + raise PluginFactoryError(f"Repository does not exist at {self.repo_path}") + + workspace_full_path = os.path.abspath(os.path.join(self.repo_path, self.workspace_path)) + if not os.path.exists(workspace_full_path): + raise PluginFactoryError(f"Workspace does not exist at {workspace_full_path}") + try: - if not os.path.exists(self.repo_path): - self.logger.error(f"[red]Repository does not exist at {self.repo_path}[/red]") - return False - workspace_full_path = os.path.abspath(os.path.join(self.repo_path, self.workspace_path)) - if not os.path.exists(workspace_full_path): - self.logger.error(f"[red]Workspace does not exist at {workspace_full_path}[/red]") - return False - # TODO: Implement PluginListConfig.create_default function plugin_cfg = PluginListConfig.create_default(workspace_path=Path(workspace_full_path)) plugin_cfg.to_file(Path(plugins_file)) @@ -208,33 +211,34 @@ def auto_generate_plugins_list(self) -> bool: self.logger.info(f" - {plugin_path}: {build_args}") else: self.logger.warning("No plugins found in workspace") - - return True - + except PluginFactoryError: + raise except Exception as e: - self.logger.error(f"Failed to auto-generate plugins list: {e}") - return False + raise PluginFactoryError(f"Failed to auto-generate plugins list: {e}") from e def discover_source_config(self) -> Optional["SourceConfig"]: - """Discovers and loads source configuration from config_dir/source.json.""" + """Discovers and loads source configuration from config_dir/source.json. + + Returns: + SourceConfig if source.json exists and is valid, None if falling back to local repo. + + Raises: + ConfigurationError: If source.json is invalid or no valid source is available. + """ source_file = os.path.join(self.config_dir, "source.json") if os.path.exists(source_file) and not self.use_local: - try: - source_config = SourceConfig.from_file(Path(source_file)) - self.logger.debug(f"Using source config from: {source_config}") - return source_config - except Exception as e: - self.logger.error(f"[red]Failed to load {source_file}: {e}[/red]") - sys.exit(1) + # SourceConfig.from_file() raises ConfigurationError on failure, so let it propagate to cli.py + source_config = SourceConfig.from_file(Path(source_file)) + self.logger.debug(f"Using source config from: {source_config}") + return source_config elif self.repo_path and os.path.exists(self.repo_path): self.logger.warning("Source configuration not found, will attempt to use locally stored plugin source code") else: - self.logger.error( - f"[red]No valid source configuration found and {self.repo_path} is empty or does not exist[/red]" + raise ConfigurationError( + f"No valid source configuration found and {self.repo_path} is empty or does not exist. " f"Either provide a valid {source_file} or ensure locally stored plugin source code exists at {self.repo_path}" ) - sys.exit(1) return None @@ -267,14 +271,21 @@ def setup_config_directory(self) -> Optional["SourceConfig"]: self.logger.warning(f"{plugins_list_file} not found, will auto-generate after repository is available") return source_config - def apply_patches_and_overlays(self) -> bool: - """Apply patches and overlays using override-sources.sh script.""" + def apply_patches_and_overlays(self) -> None: + """Apply patches and overlays using override-sources.sh script. + + Raises: + ExecutionError: If the patch script is not found or fails. + """ script_dir = Path(__file__).parent.parent.parent / "scripts" script_path = script_dir / "override-sources.sh" + STEP_NAME = "apply patches and overlays" if not script_path.exists(): - self.logger.error(f"[red]Script not found: {script_path}[/red]") - return False + raise ExecutionError( + f"Script not found: {script_path}", + step=STEP_NAME + ) workspace_full_path = os.path.abspath(os.path.join(self.repo_path, self.workspace_path)) self.logger.debug(f"Applying patches and overlays to workspace: {workspace_full_path}") @@ -294,29 +305,41 @@ def apply_patches_and_overlays(self) -> bool: if returncode == 0: self.logger.info("[green]Patches and overlays applied successfully[/green]") - return True else: - self.logger.error(f"[red]Patches/overlays failed with exit code {returncode}[/red]") - return False - + raise ExecutionError( + f"Patches/overlays failed with exit code {returncode}", + step=STEP_NAME, + returncode=returncode + ) + except ExecutionError: + raise except Exception as e: - self.logger.error(f"[red]Failed to run patch script: {e}[/red]") - return False + raise ExecutionError( + f"Failed to run patch script: {e}", + step=STEP_NAME + ) from e - def export_plugins(self, output_dir: str, push_images: bool) -> bool: - """Export plugins using export-workspace.sh script.""" + def export_plugins(self, output_dir: str, push_images: bool) -> None: + """Export plugins using export-workspace.sh script. + + Raises: + ExecutionError: If the export script is not found or fails. + ConfigurationError: If no plugins list file is found. + """ script_dir = Path(__file__).parent.parent.parent / "scripts" script_path = script_dir / "export-workspace.sh" + STEP_NAME = "export plugins" if not script_path.exists(): - self.logger.error(f"[red]Script not found: {script_path}[/red]") - return False + raise ExecutionError( + f"Script not found: {script_path}", + step=STEP_NAME + ) plugins_list_file = os.path.join(self.config_dir, "plugins-list.yaml") if not os.path.exists(plugins_list_file): - self.logger.error("[red]No plugins file found[/red]") - return False + raise ConfigurationError("No plugins file found") config_env_file = os.path.join(self.config_dir, ".env") default_env_file = Path(__file__).parent.parent.parent / "default.env" @@ -364,21 +387,29 @@ def conditional_stderr_log(line: str) -> None: ) if returncode != 0: - self.logger.error(f"[red]Plugin export script failed with exit code {returncode}[/red]") - return False + raise ExecutionError( + f"Plugin export script failed with exit code {returncode}", + step=STEP_NAME, + returncode=returncode + ) has_failures = display_export_results(Path(workspace_full_path), self.logger) if has_failures: - self.logger.error("[red]Plugin export completed with failures[/red]") - return False - else: - self.logger.info("[green]Plugin export completed successfully[/green]") - return True + raise ExecutionError( + "Plugin export completed with failures", + step=STEP_NAME, + ) + + self.logger.info("[green]Plugin export completed successfully[/green]") + except ExecutionError: + raise except Exception as e: - self.logger.error(f"[red]Failed to run export script: {e}[/red]") - return False + raise ExecutionError( + f"Failed to run export script: {e}", + step=STEP_NAME, + ) from e @dataclass @@ -391,22 +422,26 @@ class SourceConfig: @classmethod def from_file(cls, source_file: Path) -> "SourceConfig": - """Load source configuration from JSON file.""" + """Load source configuration from JSON file. + + Raises: + ConfigurationError: If the file is missing, malformed, or has invalid data. + """ try: with open(source_file, 'r') as f: data = json.load(f) except FileNotFoundError: - raise ValueError(f"Source configuration file not found: {source_file}") + raise ConfigurationError(f"Source configuration file not found: {source_file}") except json.JSONDecodeError as e: - raise ValueError(f"Invalid JSON in {source_file}: {e}") + raise ConfigurationError(f"Invalid JSON in {source_file}: {e}") except Exception as e: - raise ValueError(f"Failed to read {source_file}: {e}") + raise ConfigurationError(f"Failed to read {source_file}: {e}") try: repo = data["repo"] repo_ref = data.get("repo-ref") except KeyError as e: - raise ValueError(f"Missing required field {e} in {source_file}") + raise ConfigurationError(f"Missing required field {e} in {source_file}") config = cls( repo=repo, @@ -414,19 +449,24 @@ def from_file(cls, source_file: Path) -> "SourceConfig": ) if not config.repo: - raise ValueError("repo is required") + raise ConfigurationError("repo is required") if not config.repo_ref: - raise ValueError("repo_ref is required") + raise ConfigurationError("repo_ref is required") return config - def clone_to_path(self, repo_path: Path, clean: bool = False) -> bool: - """Clone the source repository to the specified path.""" + def clone_to_path(self, repo_path: Path, clean: bool = False) -> None: + """Clone the source repository to the specified path. + + Raises: + ConfigurationError: If the destination directory does not exist. + PluginFactoryError: If the user aborts the clone. + ExecutionError: If git clone or checkout fails. + """ logger = get_logger("cli") if not repo_path.exists(): - self.logger.error(f"[red]Destination directory does not exist: {repo_path}[/red]") - return True + raise ConfigurationError(f"Destination directory does not exist: {repo_path}") self.logger.info("[bold blue]Cloning repository[/bold blue]") self.logger.info(f"Repository: {self.repo}") @@ -443,7 +483,7 @@ def clone_to_path(self, repo_path: Path, clean: bool = False) -> bool: confirm = input() if confirm != "y": self.logger.warning("[yellow]Aborted[/yellow]") - return False + raise PluginFactoryError("Repository clone aborted by user") else: self.logger.warning(f"[yellow]`y` selected. Cleaning {repo_path}. Note: you can use the `--clean` argument to automatically clean the directory and skip this prompt next time.[/yellow]") clean_directory(repo_path) @@ -458,8 +498,11 @@ def clone_to_path(self, repo_path: Path, clean: bool = False) -> bool: ) if returncode != 0: - logger.error(f"Failed to clone repository (exit code {returncode})") - return False + raise ExecutionError( + f"Failed to clone repository (exit code {returncode})", + step="git clone", + returncode=returncode + ) if self.repo_ref: cmd = ["git", "checkout", self.repo_ref] @@ -473,15 +516,21 @@ def clone_to_path(self, repo_path: Path, clean: bool = False) -> bool: ) if returncode != 0: - logger.error(f"Failed to checkout ref {self.repo_ref} (exit code {returncode})") - return False + raise ExecutionError( + f"Failed to checkout ref {self.repo_ref} (exit code {returncode})", + step="git checkout", + returncode=returncode + ) logger.info("[green]✓ Repository cloned successfully[/green]") - return True - + + except PluginFactoryError: + raise except Exception as e: - logger.error(f"Failed to clone repository: {e}") - return False + raise ExecutionError( + f"Failed during repository clone/checkout: {e}", + step="git clone/checkout" + ) from e class PluginListConfig: """Configuration for plugin list (YAML format).""" diff --git a/src/rhdh_dynamic_plugin_factory/exceptions.py b/src/rhdh_dynamic_plugin_factory/exceptions.py new file mode 100644 index 0000000..82d9bd7 --- /dev/null +++ b/src/rhdh_dynamic_plugin_factory/exceptions.py @@ -0,0 +1,47 @@ +""" +Custom exceptions for RHDH Plugin Factory. +""" + +class PluginFactoryError(Exception): + """Base exception for all plugin factory errors. + + Used directly for internal factory logic failures such as + auto-generating configuration files, plugin list management, + or other non-external operations that go wrong. + + Attributes: + reason: A human-readable description of what went wrong. + """ + + def __init__(self, reason: str = ""): + super().__init__(reason) + self.reason = reason + +class ConfigurationError(PluginFactoryError): + """User-facing configuration validation errors. + + Raised when configuration is invalid or missing, such as + missing environment variables, bad source.json, invalid log levels, + or missing registry credentials. + + Attributes: + reason: A human-readable description of the configuration problem. + """ + +class ExecutionError(PluginFactoryError): + """External command or script execution failure. + + Raised when an external tool (git, buildah, yarn, shell scripts) + fails during execution. + + Attributes: + reason: A human-readable description of the failure. + step: A short description of what was being attempted + (e.g., "git clone", "export plugins"). + returncode: The exit code of the failed process, if available. + """ + + def __init__(self, reason: str = "", step: str = "", returncode: int | None = None): + super().__init__(reason) + self.step = step + self.returncode = returncode diff --git a/tests/test_config_export_plugins.py b/tests/test_config_export_plugins.py index bece9f5..4649efe 100644 --- a/tests/test_config_export_plugins.py +++ b/tests/test_config_export_plugins.py @@ -8,9 +8,9 @@ import os from pathlib import Path from unittest.mock import patch +import pytest -from src.rhdh_dynamic_plugin_factory.config import PluginFactoryConfig - +from src.rhdh_dynamic_plugin_factory.exceptions import ExecutionError, PluginFactoryError class TestExportPlugins: """Tests for PluginFactoryConfig.export_plugins method.""" @@ -32,9 +32,7 @@ def test_export_plugins_success(self, make_config, setup_test_env): mock_run_cmd.return_value = 0 mock_display.return_value = False - result = config.export_plugins(output_dir, push_images=False) - - assert result is True + config.export_plugins(output_dir, push_images=False) # Should not raise any exceptions mock_run_cmd.assert_called_once() call_args = mock_run_cmd.call_args @@ -132,23 +130,17 @@ def test_export_plugins_default_registry_values(self, make_config, setup_test_en assert env["INPUTS_IMAGE_REPOSITORY_PREFIX"] == "localhost/default" def test_export_plugins_script_not_found(self, make_config, setup_test_env): - """Test that export_plugins returns False when script doesn't exist.""" + """Test that export_plugins raises ExecutionError when script doesn't exist.""" config = make_config() output_dir = str(setup_test_env["tmp_path"] / "output") with patch.object(Path, "exists", return_value=False): - with patch.object(config, "logger") as mock_logger: - result = config.export_plugins(output_dir, push_images=False) - - assert result is False - - mock_logger.error.assert_called_once() - error_msg = mock_logger.error.call_args[0][0] - assert "Script not found" in error_msg + with pytest.raises(ExecutionError, match="Script not found"): + config.export_plugins(output_dir, push_images=False) def test_export_plugins_no_plugins_list(self, make_config, setup_test_env): - """Test that export_plugins returns False when plugins-list.yaml doesn't exist.""" + """Test that export_plugins raises PluginFactoryError when plugins-list.yaml doesn't exist.""" config = make_config() output_dir = str(setup_test_env["tmp_path"] / "output") @@ -167,16 +159,11 @@ def os_exists_side_effect(path_str): with patch.object(Path, "exists", new=path_exists_side_effect): with patch("os.path.exists", side_effect=os_exists_side_effect): - with patch.object(config, "logger") as mock_logger: - result = config.export_plugins(output_dir, push_images=False) - - assert result is False - - error_calls = [call[0][0] for call in mock_logger.error.call_args_list] - assert any("No plugins file found" in str(call) for call in error_calls) + with pytest.raises(PluginFactoryError, match="No plugins file found"): + config.export_plugins(output_dir, push_images=False) def test_export_plugins_script_fails(self, make_config, setup_test_env): - """Test that export_plugins returns False when script returns non-zero exit code.""" + """Test that export_plugins raises ExecutionError when script returns non-zero exit code.""" config = make_config( registry_url="quay.io", registry_namespace="test-namespace" @@ -188,18 +175,13 @@ def test_export_plugins_script_fails(self, make_config, setup_test_env): with patch("os.path.exists", return_value=True): with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run_cmd: with patch("src.rhdh_dynamic_plugin_factory.config.load_dotenv"): - with patch.object(config, "logger") as mock_logger: - mock_run_cmd.return_value = 1 - - result = config.export_plugins(output_dir, push_images=False) - - assert result is False - - error_calls = [call[0][0] for call in mock_logger.error.call_args_list] - assert any("exit code 1" in str(call) for call in error_calls) + mock_run_cmd.return_value = 1 + + with pytest.raises(ExecutionError, match="exit code 1"): + config.export_plugins(output_dir, push_images=False) def test_export_plugins_has_failures(self, make_config, setup_test_env): - """Test that export_plugins returns False when display_export_results indicates failures.""" + """Test that export_plugins raises ExecutionError when display_export_results indicates failures.""" config = make_config( registry_url="quay.io", registry_namespace="test-namespace" @@ -212,19 +194,14 @@ def test_export_plugins_has_failures(self, make_config, setup_test_env): with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run_cmd: with patch("src.rhdh_dynamic_plugin_factory.config.display_export_results") as mock_display: with patch("src.rhdh_dynamic_plugin_factory.config.load_dotenv"): - with patch.object(config, "logger") as mock_logger: - mock_run_cmd.return_value = 0 - mock_display.return_value = True - - result = config.export_plugins(output_dir, push_images=False) - - assert result is False - - error_calls = [call[0][0] for call in mock_logger.error.call_args_list] - assert any("completed with failures" in str(call) for call in error_calls) + mock_run_cmd.return_value = 0 + mock_display.return_value = True + + with pytest.raises(ExecutionError, match="completed with failures"): + config.export_plugins(output_dir, push_images=False) def test_export_plugins_exception(self, make_config, setup_test_env): - """Test that export_plugins handles exceptions gracefully.""" + """Test that export_plugins wraps exceptions in ExecutionError.""" config = make_config() output_dir = str(setup_test_env["tmp_path"] / "output") @@ -233,17 +210,10 @@ def test_export_plugins_exception(self, make_config, setup_test_env): with patch("os.path.exists", return_value=True): with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run_cmd: with patch("src.rhdh_dynamic_plugin_factory.config.load_dotenv"): - with patch.object(config, "logger") as mock_logger: - mock_run_cmd.side_effect = Exception("Test exception") - - result = config.export_plugins(output_dir, push_images=False) - - assert result is False - - mock_logger.error.assert_called() - error_msg = mock_logger.error.call_args[0][0] - assert "Failed to run export script" in error_msg - assert "Test exception" in error_msg + mock_run_cmd.side_effect = Exception("Test exception") + + with pytest.raises(ExecutionError, match="Failed to run export script.*Test exception"): + config.export_plugins(output_dir, push_images=False) def test_export_plugins_custom_env_file(self, make_config, setup_test_env): """Test that export_plugins loads custom .env file from config directory.""" diff --git a/tests/test_config_load_from_env.py b/tests/test_config_load_from_env.py index 4896f22..d44c092 100644 --- a/tests/test_config_load_from_env.py +++ b/tests/test_config_load_from_env.py @@ -11,6 +11,7 @@ import pytest from src.rhdh_dynamic_plugin_factory.config import PluginFactoryConfig +from src.rhdh_dynamic_plugin_factory.exceptions import ConfigurationError class TestPluginFactoryConfigLoadFromEnv: @@ -48,7 +49,7 @@ def test_load_from_env_valid_configuration(self, mock_args, setup_test_env, monk assert isinstance(config.workspace_path, str) def test_load_from_env_missing_rhdh_cli_version(self, mock_args, setup_test_env, clean_env): - """Test that missing RHDH_CLI_VERSION raises ValueError.""" + """Test that missing RHDH_CLI_VERSION raises ConfigurationError.""" # Don't set RHDH_CLI_VERSION clean_env.setenv("WORKSPACE_PATH", ".") @@ -58,11 +59,11 @@ def test_load_from_env_missing_rhdh_cli_version(self, mock_args, setup_test_env, # Patch to prevent loading from default.env with patch.object(Path, 'exists', return_value=False): - with pytest.raises(ValueError, match="RHDH_CLI_VERSION must be set"): + with pytest.raises(ConfigurationError, match="RHDH_CLI_VERSION must be set"): PluginFactoryConfig.load_from_env(mock_args) def test_load_from_env_invalid_log_level(self, mock_args, setup_test_env, monkeypatch): - """Test that invalid log level raises ValueError.""" + """Test that invalid log level raises ConfigurationError.""" # Set required environment variables monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") monkeypatch.setenv("WORKSPACE_PATH", ".") @@ -73,7 +74,7 @@ def test_load_from_env_invalid_log_level(self, mock_args, setup_test_env, monkey mock_args.workspace_path = "." mock_args.log_level = "INVALID_LEVEL" - with pytest.raises(ValueError, match="Invalid log level"): + with pytest.raises(ConfigurationError, match="Invalid log level"): PluginFactoryConfig.load_from_env(mock_args) @pytest.mark.parametrize("log_level", ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]) @@ -152,7 +153,7 @@ def test_load_from_env_additional_env_file_loading(self, mock_args, setup_test_e assert config.registry_namespace == "test-namespace" def test_load_from_env_missing_workspace_path(self, mock_args, setup_test_env, monkeypatch): - """Test that missing workspace_path raises ValueError.""" + """Test that missing workspace_path raises ConfigurationError.""" monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") # Don't set WORKSPACE_PATH env var to test fallback monkeypatch.delenv("WORKSPACE_PATH", raising=False) @@ -162,8 +163,8 @@ def test_load_from_env_missing_workspace_path(self, mock_args, setup_test_env, m mock_args.workspace_path = None # Missing workspace_path # When workspace_path is None and WORKSPACE_PATH env var is not set, - # validation should raise ValueError - with pytest.raises(ValueError, match="WORKSPACE_PATH must be set"): + # validation should raise ConfigurationError + with pytest.raises(ConfigurationError, match="WORKSPACE_PATH must be set"): PluginFactoryConfig.load_from_env(mock_args) def test_load_from_env_directory_creation(self, mock_args, tmp_path, monkeypatch): @@ -246,7 +247,7 @@ def test_load_from_env_use_local_flag(self, mock_args, setup_test_env, monkeypat assert config.use_local is True def test_load_from_env_source_json_missing_repo_path_empty(self, mock_args, tmp_path, monkeypatch): - """Test that missing source.json with empty repo_path raises ValueError.""" + """Test that missing source.json with empty repo_path raises ConfigurationError.""" monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") monkeypatch.setenv("WORKSPACE_PATH", ".") @@ -262,7 +263,7 @@ def test_load_from_env_source_json_missing_repo_path_empty(self, mock_args, tmp_ mock_args.repo_path = str(repo_path) mock_args.workspace_path = "." - with pytest.raises(ValueError, match="source.json not found"): + with pytest.raises(ConfigurationError, match="source.json not found"): PluginFactoryConfig.load_from_env(mock_args) def test_load_from_env_source_json_missing_repo_path_has_content(self, mock_args, tmp_path, monkeypatch): diff --git a/tests/test_config_patches_and_overlays.py b/tests/test_config_patches_and_overlays.py index 0f6a768..6a7ea13 100644 --- a/tests/test_config_patches_and_overlays.py +++ b/tests/test_config_patches_and_overlays.py @@ -7,8 +7,10 @@ import os from pathlib import Path from unittest.mock import patch +import pytest from src.rhdh_dynamic_plugin_factory.config import PluginFactoryConfig +from src.rhdh_dynamic_plugin_factory.exceptions import ExecutionError class TestApplyPatchesAndOverlays: @@ -25,9 +27,7 @@ def test_apply_patches_and_overlays_success(self, make_config): with patch.object(Path, "exists", return_value=True): mock_run_cmd.return_value = 0 - result = config.apply_patches_and_overlays() - - assert result is True + config.apply_patches_and_overlays() # Should not raise any exceptions mock_run_cmd.assert_called_once() call_args = mock_run_cmd.call_args @@ -44,49 +44,31 @@ def test_apply_patches_and_overlays_success(self, make_config): assert call_args[1]["stderr_log_func"] == config.logger.error def test_apply_patches_and_overlays_script_not_found(self, make_config): - """Test that apply_patches_and_overlays returns False when script doesn't exist.""" + """Test that apply_patches_and_overlays raises ExecutionError when script doesn't exist.""" config = make_config() with patch.object(Path, "exists", return_value=False): - with patch.object(config, "logger") as mock_logger: - result = config.apply_patches_and_overlays() - - assert result is False - - mock_logger.error.assert_called_once() - error_msg = mock_logger.error.call_args[0][0] - assert "Script not found" in error_msg + with pytest.raises(ExecutionError, match="Script not found"): + config.apply_patches_and_overlays() def test_apply_patches_and_overlays_script_fails(self, make_config): - """Test that apply_patches_and_overlays returns False when script returns non-zero exit code.""" + """Test that apply_patches_and_overlays raises ExecutionError when script returns non-zero exit code.""" config = make_config() with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run_cmd: with patch.object(Path, "exists", return_value=True): - with patch.object(config, "logger") as mock_logger: - mock_run_cmd.return_value = 1 - - result = config.apply_patches_and_overlays() - - assert result is False - - error_calls = [call[0][0] for call in mock_logger.error.call_args_list] - assert any("exit code 1" in str(call) for call in error_calls) + mock_run_cmd.return_value = 1 + + with pytest.raises(ExecutionError, match="exit code 1"): + config.apply_patches_and_overlays() def test_apply_patches_and_overlays_exception(self, make_config): - """Test that apply_patches_and_overlays handles exceptions gracefully.""" + """Test that apply_patches_and_overlays wraps exceptions in ExecutionError.""" config = make_config() with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run_cmd: with patch.object(Path, "exists", return_value=True): - with patch.object(config, "logger") as mock_logger: - mock_run_cmd.side_effect = Exception("Test exception") - - result = config.apply_patches_and_overlays() - - assert result is False - - mock_logger.error.assert_called() - error_msg = mock_logger.error.call_args[0][0] - assert "Failed to run patch script" in error_msg - assert "Test exception" in error_msg + mock_run_cmd.side_effect = Exception("Test exception") + + with pytest.raises(ExecutionError, match="Failed to run patch script.*Test exception"): + config.apply_patches_and_overlays() diff --git a/tests/test_config_registry.py b/tests/test_config_registry.py index a68a34b..1fedcad 100644 --- a/tests/test_config_registry.py +++ b/tests/test_config_registry.py @@ -9,6 +9,7 @@ import pytest from src.rhdh_dynamic_plugin_factory.config import PluginFactoryConfig +from src.rhdh_dynamic_plugin_factory.exceptions import ConfigurationError class TestLoadRegistryConfig: @@ -25,17 +26,17 @@ def test_skip_when_push_images_false(self, make_config): ) def test_missing_registry_url(self, make_config): - """Test that missing REGISTRY_URL raises ValueError when push_images is True.""" + """Test that missing REGISTRY_URL raises ConfigurationError when push_images is True.""" config = make_config(registry_url=None, registry_namespace="test-namespace") - with pytest.raises(ValueError, match="REGISTRY_URL environment variable is required"): + with pytest.raises(ConfigurationError, match="REGISTRY_URL environment variable is required"): config.load_registry_config(push_images=True) def test_missing_registry_namespace(self, make_config): - """Test that missing REGISTRY_NAMESPACE raises ValueError when push_images is True.""" + """Test that missing REGISTRY_NAMESPACE raises ConfigurationError when push_images is True.""" config = make_config(registry_url="quay.io", registry_namespace=None) - with pytest.raises(ValueError, match="REGISTRY_NAMESPACE environment variable is required"): + with pytest.raises(ConfigurationError, match="REGISTRY_NAMESPACE environment variable is required"): config.load_registry_config(push_images=True) def test_successful_buildah_login(self, make_config): @@ -99,7 +100,7 @@ def test_failed_buildah_login(self, make_config): assert "Authentication failed" in warning_call def test_missing_registry_credentials(self, make_config): - """Test that missing credentials raise ValueError when push_images is True.""" + """Test that missing credentials raise ConfigurationError when push_images is True.""" config = make_config( registry_url="quay.io", registry_namespace="test-namespace", @@ -108,11 +109,11 @@ def test_missing_registry_credentials(self, make_config): registry_insecure=False ) - with pytest.raises(ValueError, match="REGISTRY_USERNAME and REGISTRY_PASSWORD environment variables are required"): + with pytest.raises(ConfigurationError, match="REGISTRY_USERNAME and REGISTRY_PASSWORD environment variables are required"): config.load_registry_config(push_images=True) def test_missing_registry_username(self, make_config): - """Test that missing username raises ValueError when push_images is True.""" + """Test that missing username raises ConfigurationError when push_images is True.""" config = make_config( registry_url="quay.io", registry_namespace="test-namespace", @@ -121,11 +122,11 @@ def test_missing_registry_username(self, make_config): registry_insecure=False ) - with pytest.raises(ValueError, match="REGISTRY_USERNAME and REGISTRY_PASSWORD environment variables are required"): + with pytest.raises(ConfigurationError, match="REGISTRY_USERNAME and REGISTRY_PASSWORD environment variables are required"): config.load_registry_config(push_images=True) def test_missing_registry_password(self, make_config): - """Test that missing password raises ValueError when push_images is True.""" + """Test that missing password raises ConfigurationError when push_images is True.""" config = make_config( registry_url="quay.io", registry_namespace="test-namespace", @@ -134,7 +135,7 @@ def test_missing_registry_password(self, make_config): registry_insecure=False ) - with pytest.raises(ValueError, match="REGISTRY_USERNAME and REGISTRY_PASSWORD environment variables are required"): + with pytest.raises(ConfigurationError, match="REGISTRY_USERNAME and REGISTRY_PASSWORD environment variables are required"): config.load_registry_config(push_images=True) def test_insecure_registry_flag(self, make_config): diff --git a/tests/test_source_config.py b/tests/test_source_config.py index 28cd9f4..ab69ff3 100644 --- a/tests/test_source_config.py +++ b/tests/test_source_config.py @@ -9,6 +9,7 @@ import pytest from src.rhdh_dynamic_plugin_factory.config import SourceConfig +from src.rhdh_dynamic_plugin_factory.exceptions import ConfigurationError, ExecutionError, PluginFactoryError class TestSourceConfigFromFile: """Tests for SourceConfig.from_file method.""" @@ -29,7 +30,7 @@ def test_from_file_valid_source_json(self, tmp_path): assert config.repo_ref == "78df9399a81cfd95265cab53815f54210b1d7f50" def test_from_file_missing_repo(self, tmp_path): - """Test that missing repo field raises ValueError with descriptive message.""" + """Test that missing repo field raises ConfigurationError with descriptive message.""" source_data = { "repo-ref": "main" } @@ -37,11 +38,11 @@ def test_from_file_missing_repo(self, tmp_path): source_file = tmp_path / "source.json" source_file.write_text(json.dumps(source_data)) - with pytest.raises(ValueError, match="Missing required field"): + with pytest.raises(ConfigurationError, match="Missing required field"): SourceConfig.from_file(source_file) def test_from_file_empty_repo(self, tmp_path): - """Test that empty repo field raises ValueError.""" + """Test that empty repo field raises ConfigurationError.""" source_data = { "repo": "", "repo-ref": "main" @@ -50,11 +51,11 @@ def test_from_file_empty_repo(self, tmp_path): source_file = tmp_path / "source.json" source_file.write_text(json.dumps(source_data)) - with pytest.raises(ValueError, match="repo is required"): + with pytest.raises(ConfigurationError, match="repo is required"): SourceConfig.from_file(source_file) def test_from_file_empty_repo_ref(self, tmp_path): - """Test that empty repo_ref field raises ValueError.""" + """Test that empty repo_ref field raises ConfigurationError.""" source_data = { "repo": "https://github.com/test/repo", "repo-ref": "" @@ -63,11 +64,11 @@ def test_from_file_empty_repo_ref(self, tmp_path): source_file = tmp_path / "source.json" source_file.write_text(json.dumps(source_data)) - with pytest.raises(ValueError, match="repo_ref is required"): + with pytest.raises(ConfigurationError, match="repo_ref is required"): SourceConfig.from_file(source_file) def test_from_file_missing_repo_ref(self, tmp_path): - """Test that missing repo_ref field raises ValueError.""" + """Test that missing repo_ref field raises ConfigurationError.""" source_data = { "repo": "https://github.com/test/repo" } @@ -75,23 +76,23 @@ def test_from_file_missing_repo_ref(self, tmp_path): source_file = tmp_path / "source.json" source_file.write_text(json.dumps(source_data)) - # repo_ref is now required and will raise ValueError if missing/None - with pytest.raises(ValueError, match="repo_ref is required"): + # repo_ref is now required and will raise ConfigurationError if missing/None + with pytest.raises(ConfigurationError, match="repo_ref is required"): SourceConfig.from_file(source_file) def test_from_file_malformed_json(self, tmp_path): - """Test that malformed JSON raises ValueError with descriptive message.""" + """Test that malformed JSON raises ConfigurationError with descriptive message.""" source_file = tmp_path / "source.json" source_file.write_text("{ invalid json }") - with pytest.raises(ValueError, match="Invalid JSON"): + with pytest.raises(ConfigurationError, match="Invalid JSON"): SourceConfig.from_file(source_file) def test_from_file_nonexistent_file(self, tmp_path): - """Test that nonexistent file raises ValueError with descriptive message.""" + """Test that nonexistent file raises ConfigurationError with descriptive message.""" source_file = tmp_path / "nonexistent.json" - with pytest.raises(ValueError, match="Source configuration file not found"): + with pytest.raises(ConfigurationError, match="Source configuration file not found"): SourceConfig.from_file(source_file) @@ -111,9 +112,8 @@ def test_clone_to_path_success(self, tmp_path): with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run: mock_run.return_value = 0 - result = config.clone_to_path(repo_path) + config.clone_to_path(repo_path) # Should not raise any exceptions - assert result is True assert mock_run.call_count == 2 # clone + checkout # Verify clone command @@ -129,7 +129,7 @@ def test_clone_to_path_success(self, tmp_path): assert checkout_call[0][0][2] == "main" def test_clone_to_path_repo_path_does_not_exist(self, tmp_path): - """Test that non-existent repo_path returns True early.""" + """Test that non-existent repo_path raises ConfigurationError.""" config = SourceConfig( repo="https://github.com/testowner/testrepo", repo_ref="main" @@ -137,13 +137,11 @@ def test_clone_to_path_repo_path_does_not_exist(self, tmp_path): repo_path = tmp_path / "nonexistent" - result = config.clone_to_path(repo_path) - - # According to the code, it returns True if repo_path doesn't exist - assert result is True + with pytest.raises(ConfigurationError, match="Destination directory does not exist"): + config.clone_to_path(repo_path) def test_clone_to_path_clone_fails(self, tmp_path): - """Test that clone failure returns False.""" + """Test that clone failure raises ExecutionError.""" config = SourceConfig( repo="https://github.com/testowner/testrepo", repo_ref="main" @@ -155,12 +153,11 @@ def test_clone_to_path_clone_fails(self, tmp_path): with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run: mock_run.return_value = 1 # Failed - result = config.clone_to_path(repo_path) - - assert result is False + with pytest.raises(ExecutionError, match="Failed to clone repository"): + config.clone_to_path(repo_path) def test_clone_to_path_checkout_fails(self, tmp_path): - """Test that checkout failure returns False.""" + """Test that checkout failure raises ExecutionError.""" config = SourceConfig( repo="https://github.com/testowner/testrepo", repo_ref="main" @@ -173,12 +170,11 @@ def test_clone_to_path_checkout_fails(self, tmp_path): # First call (clone) succeeds, second call (checkout) fails mock_run.side_effect = [0, 1] - result = config.clone_to_path(repo_path) - - assert result is False + with pytest.raises(ExecutionError, match="Failed to checkout ref"): + config.clone_to_path(repo_path) def test_clone_to_path_exception(self, tmp_path): - """Test that exceptions return False.""" + """Test that exceptions are wrapped in ExecutionError.""" config = SourceConfig( repo="https://github.com/testowner/testrepo", repo_ref="main" @@ -190,9 +186,8 @@ def test_clone_to_path_exception(self, tmp_path): with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run: mock_run.side_effect = Exception("Test exception") - result = config.clone_to_path(repo_path) - - assert result is False + with pytest.raises(ExecutionError, match="Failed during repository clone/checkout"): + config.clone_to_path(repo_path) class TestSourceConfigCloneToPathClean: @@ -215,9 +210,8 @@ def test_clean_flag_auto_cleans_nonempty_directory(self, tmp_path): patch("src.rhdh_dynamic_plugin_factory.config.clean_directory") as mock_clean: mock_run.return_value = 0 - result = config.clone_to_path(repo_path, clean=True) + config.clone_to_path(repo_path, clean=True) # Should not raise any exceptions - assert result is True mock_clean.assert_called_once_with(repo_path) def test_clean_flag_does_not_prompt_user(self, tmp_path): @@ -256,13 +250,12 @@ def test_no_clean_flag_prompts_user_confirm_yes(self, tmp_path): patch("builtins.input", return_value="y"): mock_run.return_value = 0 - result = config.clone_to_path(repo_path, clean=False) + config.clone_to_path(repo_path, clean=False) # Should not raise any exceptions - assert result is True mock_clean.assert_called_once_with(repo_path) def test_no_clean_flag_prompts_user_confirm_no(self, tmp_path): - """Test that clean=False prompts user and aborts when user enters anything other than 'y'.""" + """Test that clean=False prompts user and raises PluginFactoryError when user declines.""" config = SourceConfig( repo="https://github.com/testowner/testrepo", repo_ref="main" @@ -273,13 +266,13 @@ def test_no_clean_flag_prompts_user_confirm_no(self, tmp_path): (repo_path / "existing_file.txt").write_text("existing content") with patch("builtins.input", return_value="n") as mock_input: - result = config.clone_to_path(repo_path, clean=False) + with pytest.raises(PluginFactoryError, match="aborted by user"): + config.clone_to_path(repo_path, clean=False) - assert result is False mock_input.assert_called_once() def test_no_clean_flag_prompts_user_empty_input_aborts(self, tmp_path): - """Test that clean=False aborts when user presses Enter without typing anything.""" + """Test that clean=False raises PluginFactoryError when user presses Enter without typing anything.""" config = SourceConfig( repo="https://github.com/testowner/testrepo", repo_ref="main" @@ -290,9 +283,8 @@ def test_no_clean_flag_prompts_user_empty_input_aborts(self, tmp_path): (repo_path / "existing_file.txt").write_text("existing content") with patch("builtins.input", return_value=""): - result = config.clone_to_path(repo_path, clean=False) - - assert result is False + with pytest.raises(PluginFactoryError, match="aborted by user"): + config.clone_to_path(repo_path, clean=False) def test_empty_directory_skips_clean_and_prompt(self, tmp_path): """Test that an empty directory skips both clean and prompt logic.""" @@ -310,9 +302,8 @@ def test_empty_directory_skips_clean_and_prompt(self, tmp_path): patch("builtins.input") as mock_input: mock_run.return_value = 0 - result = config.clone_to_path(repo_path, clean=True) + config.clone_to_path(repo_path, clean=True) # Should not raise - assert result is True mock_clean.assert_not_called() mock_input.assert_not_called() @@ -331,9 +322,8 @@ def test_empty_directory_no_clean_flag_skips_prompt(self, tmp_path): patch("builtins.input") as mock_input: mock_run.return_value = 0 - result = config.clone_to_path(repo_path, clean=False) + config.clone_to_path(repo_path, clean=False) # Should not raise any exceptions] - assert result is True mock_input.assert_not_called() def test_clean_flag_default_is_false(self, tmp_path): @@ -348,10 +338,9 @@ def test_clean_flag_default_is_false(self, tmp_path): (repo_path / "existing_file.txt").write_text("existing content") with patch("builtins.input", return_value="n"): - # Call without clean argument - should prompt (default clean=False) - result = config.clone_to_path(repo_path) - - assert result is False + # Call without clean argument - should prompt and raise PluginFactoryError when user declines + with pytest.raises(PluginFactoryError, match="aborted by user"): + config.clone_to_path(repo_path) def test_clean_proceeds_with_clone_after_cleaning(self, tmp_path): """Test that after cleaning, git clone and checkout are executed.""" @@ -368,9 +357,8 @@ def test_clean_proceeds_with_clone_after_cleaning(self, tmp_path): patch("src.rhdh_dynamic_plugin_factory.config.clean_directory"): mock_run.return_value = 0 - result = config.clone_to_path(repo_path, clean=True) + config.clone_to_path(repo_path, clean=True) # Should not raise any exceptions - assert result is True assert mock_run.call_count == 2 # clone + checkout # Verify clone command @@ -397,9 +385,8 @@ def test_prompt_confirm_yes_proceeds_with_clone(self, tmp_path): patch("builtins.input", return_value="y"): mock_run.return_value = 0 - result = config.clone_to_path(repo_path, clean=False) + config.clone_to_path(repo_path, clean=False) # Should not raise any exceptions - assert result is True assert mock_run.call_count == 2 # clone + checkout def test_prompt_confirm_no_does_not_clone(self, tmp_path): @@ -416,8 +403,8 @@ def test_prompt_confirm_no_does_not_clone(self, tmp_path): with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run, \ patch("builtins.input", return_value="n"): - result = config.clone_to_path(repo_path, clean=False) + with pytest.raises(PluginFactoryError, match="aborted by user"): + config.clone_to_path(repo_path, clean=False) - assert result is False mock_run.assert_not_called() From 84901b39c277d28ee05727445c2d512d36bd2f23 Mon Sep 17 00:00:00 2001 From: Frank Kong Date: Mon, 9 Feb 2026 17:57:58 -0500 Subject: [PATCH 05/35] chore: refactor to move validation into __post_init__ functions Signed-off-by: Frank Kong --- src/rhdh_dynamic_plugin_factory/cli.py | 7 +- src/rhdh_dynamic_plugin_factory/config.py | 146 ++++++++++--------- tests/conftest.py | 16 +-- tests/test_config_export_plugins.py | 25 ++-- tests/test_config_registry.py | 163 ++++++++++++---------- tests/test_source_config.py | 60 +++++--- 6 files changed, 235 insertions(+), 182 deletions(-) diff --git a/src/rhdh_dynamic_plugin_factory/cli.py b/src/rhdh_dynamic_plugin_factory/cli.py index 8dbb328..2bd1b98 100644 --- a/src/rhdh_dynamic_plugin_factory/cli.py +++ b/src/rhdh_dynamic_plugin_factory/cli.py @@ -63,7 +63,7 @@ def create_parser() -> argparse.ArgumentParser: parser.add_argument( "--workspace-path", type=Path, - help="Path to the workspace from root of the repository" + help="Path to the workspace from root of the repository. Required if `source.json` is not provided." ) parser.add_argument( "--push-images", @@ -151,9 +151,8 @@ def _run(args: argparse.Namespace) -> None: """ logger.info("[bold blue]Setting up configuration directory[/bold blue]") - config = PluginFactoryConfig.load_from_env(args=args, env_file=args.config_dir / ".env") + config = PluginFactoryConfig.load_from_env(args=args, env_file=args.config_dir / ".env", push_images=args.push_images) - config.load_registry_config(push_images=args.push_images) source_config = config.setup_config_directory() if source_config and not config.use_local: @@ -183,7 +182,7 @@ def _run(args: argparse.Namespace) -> None: install_dependencies(workspace_path) logger.info("[bold blue]Exporting plugins using RHDH CLI[/bold blue]") - config.export_plugins(args.output_dir, args.push_images) + config.export_plugins(args.output_dir) def main(): diff --git a/src/rhdh_dynamic_plugin_factory/config.py b/src/rhdh_dynamic_plugin_factory/config.py index 866b9da..50ea20b 100644 --- a/src/rhdh_dynamic_plugin_factory/config.py +++ b/src/rhdh_dynamic_plugin_factory/config.py @@ -3,9 +3,10 @@ """ import argparse +from logging import Logger import os from pathlib import Path -from typing import Dict, Optional +from typing import Dict, Optional, ClassVar from dataclasses import dataclass, field from dotenv import load_dotenv import yaml @@ -36,90 +37,95 @@ class PluginFactoryConfig: log_level: str = field(default="INFO") use_local: bool = field(default=False) + push_images: bool = field(default=False) + + logger: ClassVar[Logger] = get_logger("config") + + def __post_init__(self) -> None: + """Validate configuration fields after initialization.""" + if not self.rhdh_cli_version: + raise ConfigurationError("RHDH_CLI_VERSION must be set (usually loaded from default.env)") + + if not self.workspace_path: + raise ConfigurationError("WORKSPACE_PATH must be set via environment variable or --workspace-path argument") + + valid_log_levels = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] + if self.log_level.upper() not in valid_log_levels: + raise ConfigurationError(f"Invalid log level: {self.log_level}") + + if self.push_images: + if not self.registry_url: + raise ConfigurationError("REGISTRY_URL environment variable is required when --push-images is enabled") + if not self.registry_namespace: + raise ConfigurationError("REGISTRY_NAMESPACE environment variable is required when --push-images is enabled") + if not self.registry_username or not self.registry_password: + raise ConfigurationError("REGISTRY_USERNAME and REGISTRY_PASSWORD environment variables are required when --push-images is enabled") - logger = get_logger("config") @classmethod - def load_from_env(cls, args: argparse.Namespace, env_file: Optional[Path] = None) -> "PluginFactoryConfig": + def load_from_env(cls, args: argparse.Namespace, env_file: Optional[Path] = None, push_images: bool = False) -> "PluginFactoryConfig": """Load configuration from environment variables and .env files. Loads default.env first, then optionally loads additional env file to override defaults or provide additional values. Environment variables take precedence over .env file values. Args: - env_file: Optional additional .env file to merge with defaults + args: Parsed CLI arguments. + env_file: Optional additional .env file to merge with defaults. + push_images: Whether to push images to a registry (triggers registry validation and login). """ - - config = cls() - default_env_path = Path(__file__).parent.parent.parent / "default.env" - config.logger.debug(f'[bold blue]Loading environment variables from {default_env_path}[/bold blue]') + cls.logger.debug(f'[bold blue]Loading environment variables from {default_env_path}[/bold blue]') if default_env_path.exists(): load_dotenv(default_env_path) - config.logger.debug(f'[green]✓ Loaded {default_env_path}[/green]') + cls.logger.debug(f'[green]✓ Loaded {default_env_path}[/green]') if env_file and env_file.exists(): load_dotenv(env_file, override=True) - config.logger.debug(f'[green]✓ Loaded {env_file}[/green]') + cls.logger.debug(f'[green]✓ Loaded {env_file}[/green]') - config.logger.debug('[bold blue]Loading configuration from environment variables and CLI arguments[/bold blue]') + cls.logger.debug('[bold blue]Loading configuration from environment variables and CLI arguments[/bold blue]') - config.workspace_path = os.getenv("WORKSPACE_PATH", args.workspace_path) - config.config_dir = args.config_dir - config.repo_path = args.repo_path + config_dir = args.config_dir + repo_path = args.repo_path - config.rhdh_cli_version = os.getenv("RHDH_CLI_VERSION", "") - - config.registry_url = os.getenv("REGISTRY_URL") - config.registry_username = os.getenv("REGISTRY_USERNAME") - config.registry_password = os.getenv("REGISTRY_PASSWORD") - config.registry_namespace = os.getenv("REGISTRY_NAMESPACE") - config.registry_insecure = os.getenv("REGISTRY_INSECURE", "false").lower() == "true" - - config.log_level = os.getenv("LOG_LEVEL", args.log_level) - - config.use_local = args.use_local - - dirs_to_create = [config.config_dir, config.repo_path] - for dir_path in dirs_to_create: + # Ensure required directories exist before constructing config + for dir_path in [config_dir, repo_path]: os.makedirs(dir_path, exist_ok=True) - if not config.rhdh_cli_version: - raise ConfigurationError("RHDH_CLI_VERSION must be set (usually loaded from default.env)") - - if not config.workspace_path: - raise ConfigurationError("WORKSPACE_PATH must be set via environment variable or --workspace-path argument") - - valid_log_levels = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] - if config.log_level.upper() not in valid_log_levels: - raise ConfigurationError(f"Invalid log level: {config.log_level}") + config = cls( + rhdh_cli_version=os.getenv("RHDH_CLI_VERSION", ""), + repo_path=repo_path, + config_dir=config_dir, + workspace_path=os.getenv("WORKSPACE_PATH", args.workspace_path), + registry_url=os.getenv("REGISTRY_URL"), + registry_username=os.getenv("REGISTRY_USERNAME"), + registry_password=os.getenv("REGISTRY_PASSWORD"), + registry_namespace=os.getenv("REGISTRY_NAMESPACE"), + registry_insecure=os.getenv("REGISTRY_INSECURE", "false").lower() == "true", + log_level=os.getenv("LOG_LEVEL", args.log_level), + use_local=args.use_local, + push_images=push_images, + ) config._validate_source_json() config._validate_plugins_list() + if push_images: + config._buildah_login() + return config - def load_registry_config(self, push_images: bool = False) -> None: - """ - Load registry configuration from environment variables and attempt buildah login. - Only validates required registry fields if `push_images` is True. - """ - # Only validate registry configuration if we're pushing images - if not push_images: - self.logger.info("Skipping registry configuration (not pushing images)") - return - - if not self.registry_url: - raise ConfigurationError("REGISTRY_URL environment variable is required when --push-images is enabled") + def _buildah_login(self) -> None: + """Login to the container registry using buildah. - if not self.registry_namespace: - raise ConfigurationError("REGISTRY_NAMESPACE environment variable is required when --push-images is enabled") + Assumes registry fields have already been validated by __post_init__. - if not self.registry_username or not self.registry_password: - raise ConfigurationError("REGISTRY_USERNAME and REGISTRY_PASSWORD environment variables are required when --push-images is enabled") + Raises: + ExecutionError: If the buildah login command fails. + """ ## TODO: Add support for token logins for ghcr.io registry as well - try: cmd = [ "buildah", "login", @@ -140,9 +146,11 @@ def load_registry_config(self, push_images: bool = False) -> None: ) self.logger.info(f"Logged in to registry {self.registry_url} with buildah.") except subprocess.CalledProcessError as e: - self.logger.warning( - f"Failed to login to registry {self.registry_url} with buildah: {e.stderr.decode().strip()}" - ) + raise ExecutionError( + f"Failed to login to registry {self.registry_url} with buildah: {e.stderr.decode().strip()}", + step="buildah login", + returncode=e.returncode, + ) from e def _validate_source_json(self) -> None: """Validate source.json file existence and repo_path state.""" @@ -319,7 +327,7 @@ def apply_patches_and_overlays(self) -> None: step=STEP_NAME ) from e - def export_plugins(self, output_dir: str, push_images: bool) -> None: + def export_plugins(self, output_dir: str) -> None: """Export plugins using export-workspace.sh script. Raises: @@ -361,7 +369,7 @@ def export_plugins(self, output_dir: str, push_images: bool) -> None: "INPUTS_APP_CONFIG_FILE_NAME": "app-config.dynamic.yaml", "INPUTS_PLUGINS_FILE": os.path.abspath(plugins_list_file), "INPUTS_CLI_PACKAGE": "@red-hat-developer-hub/cli", - "INPUTS_PUSH_CONTAINER_IMAGE": "true" if push_images else "false", + "INPUTS_PUSH_CONTAINER_IMAGE": "true" if self.push_images else "false", "INPUTS_JANUS_CLI_VERSION": self.rhdh_cli_version, "INPUTS_IMAGE_REPOSITORY_PREFIX": f"{self.registry_url or 'localhost'}/{self.registry_namespace or 'default'}", "INPUTS_DESTINATION": os.path.abspath(output_dir), @@ -417,8 +425,17 @@ class SourceConfig: """Configuration for plugin source repository.""" repo: str repo_ref: str - - logger = get_logger("source_config") + workspace_path: str + logger: ClassVar[Logger] = get_logger("source_config") + + def __post_init__(self) -> None: + if not self.workspace_path: + raise ConfigurationError("workspace-path is required") + if not self.repo: + raise ConfigurationError("repo is required") + if not self.repo_ref: + raise ConfigurationError("repo_ref is required") + @classmethod def from_file(cls, source_file: Path) -> "SourceConfig": @@ -440,19 +457,16 @@ def from_file(cls, source_file: Path) -> "SourceConfig": try: repo = data["repo"] repo_ref = data.get("repo-ref") + workspace_path = data.get("workspace-path") except KeyError as e: raise ConfigurationError(f"Missing required field {e} in {source_file}") config = cls( repo=repo, repo_ref=repo_ref, + workspace_path=workspace_path, ) - if not config.repo: - raise ConfigurationError("repo is required") - if not config.repo_ref: - raise ConfigurationError("repo_ref is required") - return config def clone_to_path(self, repo_path: Path, clean: bool = False) -> None: diff --git a/tests/conftest.py b/tests/conftest.py index 18c5782..6a42891 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -186,13 +186,13 @@ def make_config(setup_test_env): config = make_config(registry_url=None) # Explicitly set to None """ def _make_config(**overrides): - config = PluginFactoryConfig() - config.config_dir = setup_test_env["config_dir"] - config.repo_path = setup_test_env["source_dir"] - config.rhdh_cli_version = "1.7.2" - config.workspace_path = "." - for key, value in overrides.items(): - setattr(config, key, value) - return config + defaults = { + "config_dir": setup_test_env["config_dir"], + "repo_path": setup_test_env["source_dir"], + "rhdh_cli_version": "1.7.2", + "workspace_path": ".", + } + defaults.update(overrides) + return PluginFactoryConfig(**defaults) return _make_config diff --git a/tests/test_config_export_plugins.py b/tests/test_config_export_plugins.py index 4649efe..35fbcaf 100644 --- a/tests/test_config_export_plugins.py +++ b/tests/test_config_export_plugins.py @@ -32,7 +32,7 @@ def test_export_plugins_success(self, make_config, setup_test_env): mock_run_cmd.return_value = 0 mock_display.return_value = False - config.export_plugins(output_dir, push_images=False) # Should not raise any exceptions + config.export_plugins(output_dir) # Should not raise any exceptions mock_run_cmd.assert_called_once() call_args = mock_run_cmd.call_args @@ -69,7 +69,7 @@ def test_export_plugins_environment_variables_no_push(self, make_config, setup_t mock_run_cmd.return_value = 0 mock_display.return_value = False - _result = config.export_plugins(output_dir, push_images=False) + _result = config.export_plugins(output_dir) env = mock_run_cmd.call_args[1]["env"] @@ -88,8 +88,11 @@ def test_export_plugins_environment_variables_no_push(self, make_config, setup_t def test_export_plugins_environment_variables_with_push(self, make_config, setup_test_env): """Test that environment variables are correctly set when push_images is True.""" config = make_config( + push_images=True, registry_url="quay.io", - registry_namespace="test-namespace" + registry_namespace="test-namespace", + registry_username="test-user", + registry_password="test-password", ) output_dir = str(setup_test_env["tmp_path"] / "output") @@ -102,7 +105,7 @@ def test_export_plugins_environment_variables_with_push(self, make_config, setup mock_run_cmd.return_value = 0 mock_display.return_value = False - _result = config.export_plugins(output_dir, push_images=True) + _result = config.export_plugins(output_dir) env = mock_run_cmd.call_args[1]["env"] assert env["INPUTS_PUSH_CONTAINER_IMAGE"] == "true" @@ -124,7 +127,7 @@ def test_export_plugins_default_registry_values(self, make_config, setup_test_en mock_run_cmd.return_value = 0 mock_display.return_value = False - _result = config.export_plugins(output_dir, push_images=False) + _result = config.export_plugins(output_dir) env = mock_run_cmd.call_args[1]["env"] assert env["INPUTS_IMAGE_REPOSITORY_PREFIX"] == "localhost/default" @@ -137,7 +140,7 @@ def test_export_plugins_script_not_found(self, make_config, setup_test_env): with patch.object(Path, "exists", return_value=False): with pytest.raises(ExecutionError, match="Script not found"): - config.export_plugins(output_dir, push_images=False) + config.export_plugins(output_dir) def test_export_plugins_no_plugins_list(self, make_config, setup_test_env): """Test that export_plugins raises PluginFactoryError when plugins-list.yaml doesn't exist.""" @@ -160,7 +163,7 @@ def os_exists_side_effect(path_str): with patch.object(Path, "exists", new=path_exists_side_effect): with patch("os.path.exists", side_effect=os_exists_side_effect): with pytest.raises(PluginFactoryError, match="No plugins file found"): - config.export_plugins(output_dir, push_images=False) + config.export_plugins(output_dir) def test_export_plugins_script_fails(self, make_config, setup_test_env): """Test that export_plugins raises ExecutionError when script returns non-zero exit code.""" @@ -178,7 +181,7 @@ def test_export_plugins_script_fails(self, make_config, setup_test_env): mock_run_cmd.return_value = 1 with pytest.raises(ExecutionError, match="exit code 1"): - config.export_plugins(output_dir, push_images=False) + config.export_plugins(output_dir) def test_export_plugins_has_failures(self, make_config, setup_test_env): """Test that export_plugins raises ExecutionError when display_export_results indicates failures.""" @@ -198,7 +201,7 @@ def test_export_plugins_has_failures(self, make_config, setup_test_env): mock_display.return_value = True with pytest.raises(ExecutionError, match="completed with failures"): - config.export_plugins(output_dir, push_images=False) + config.export_plugins(output_dir) def test_export_plugins_exception(self, make_config, setup_test_env): """Test that export_plugins wraps exceptions in ExecutionError.""" @@ -213,7 +216,7 @@ def test_export_plugins_exception(self, make_config, setup_test_env): mock_run_cmd.side_effect = Exception("Test exception") with pytest.raises(ExecutionError, match="Failed to run export script.*Test exception"): - config.export_plugins(output_dir, push_images=False) + config.export_plugins(output_dir) def test_export_plugins_custom_env_file(self, make_config, setup_test_env): """Test that export_plugins loads custom .env file from config directory.""" @@ -236,7 +239,7 @@ def test_export_plugins_custom_env_file(self, make_config, setup_test_env): mock_run_cmd.return_value = 0 mock_display.return_value = False - _result = config.export_plugins(output_dir, push_images=False) + _result = config.export_plugins(output_dir) assert mock_load_dotenv.call_count >= 1 diff --git a/tests/test_config_registry.py b/tests/test_config_registry.py index 1fedcad..4436697 100644 --- a/tests/test_config_registry.py +++ b/tests/test_config_registry.py @@ -1,7 +1,8 @@ """ -Unit tests for PluginFactoryConfig.load_registry_config method. +Unit tests for PluginFactoryConfig registry validation and buildah login. -Tests the registry configuration loading and buildah login functionality. +Tests the registry configuration validation in __post_init__ and +the _buildah_login functionality. """ import subprocess @@ -9,51 +10,105 @@ import pytest from src.rhdh_dynamic_plugin_factory.config import PluginFactoryConfig -from src.rhdh_dynamic_plugin_factory.exceptions import ConfigurationError +from src.rhdh_dynamic_plugin_factory.exceptions import ConfigurationError, ExecutionError -class TestLoadRegistryConfig: - """Tests for PluginFactoryConfig.load_registry_config method.""" +class TestRegistryValidation: + """Tests for registry validation in __post_init__ when push_images is True.""" - def test_skip_when_push_images_false(self, make_config): - """Test that registry configuration is skipped when push_images is False.""" + def test_no_validation_when_push_images_false(self, make_config): + """Test that registry fields are not validated when push_images is False.""" + # Should not raise exceptions even with no registry fields set config = make_config() - - with patch.object(config, 'logger') as mock_logger: - config.load_registry_config(push_images=False) - mock_logger.info.assert_called_once_with( - "Skipping registry configuration (not pushing images)" - ) + assert config.push_images is False def test_missing_registry_url(self, make_config): """Test that missing REGISTRY_URL raises ConfigurationError when push_images is True.""" - config = make_config(registry_url=None, registry_namespace="test-namespace") - with pytest.raises(ConfigurationError, match="REGISTRY_URL environment variable is required"): - config.load_registry_config(push_images=True) + make_config( + push_images=True, + registry_url=None, + registry_namespace="test-namespace", + registry_username="test-user", + registry_password="test-password", + ) def test_missing_registry_namespace(self, make_config): """Test that missing REGISTRY_NAMESPACE raises ConfigurationError when push_images is True.""" - config = make_config(registry_url="quay.io", registry_namespace=None) - with pytest.raises(ConfigurationError, match="REGISTRY_NAMESPACE environment variable is required"): - config.load_registry_config(push_images=True) + make_config( + push_images=True, + registry_url="quay.io", + registry_namespace=None, + registry_username="test-user", + registry_password="test-password", + ) + + def test_missing_registry_credentials(self, make_config): + """Test that missing credentials raise ConfigurationError when push_images is True.""" + with pytest.raises(ConfigurationError, match="REGISTRY_USERNAME and REGISTRY_PASSWORD environment variables are required"): + make_config( + push_images=True, + registry_url="quay.io", + registry_namespace="test-namespace", + registry_username=None, + registry_password=None, + ) + + def test_missing_registry_username(self, make_config): + """Test that missing username raises ConfigurationError when push_images is True.""" + with pytest.raises(ConfigurationError, match="REGISTRY_USERNAME and REGISTRY_PASSWORD environment variables are required"): + make_config( + push_images=True, + registry_url="quay.io", + registry_namespace="test-namespace", + registry_username=None, + registry_password="test-password", + ) + + def test_missing_registry_password(self, make_config): + """Test that missing password raises ConfigurationError when push_images is True.""" + with pytest.raises(ConfigurationError, match="REGISTRY_USERNAME and REGISTRY_PASSWORD environment variables are required"): + make_config( + push_images=True, + registry_url="quay.io", + registry_namespace="test-namespace", + registry_username="test-user", + registry_password=None, + ) + + def test_valid_registry_config(self, make_config): + """Test that valid registry configuration passes validation.""" + config = make_config( + push_images=True, + registry_url="quay.io", + registry_namespace="test-namespace", + registry_username="test-user", + registry_password="test-password", + ) + assert config.push_images is True + assert config.registry_url == "quay.io" + + +class TestBuildahLogin: + """Tests for PluginFactoryConfig._buildah_login method.""" def test_successful_buildah_login(self, make_config): """Test successful buildah login with valid credentials.""" config = make_config( + push_images=True, registry_url="quay.io", registry_namespace="test-namespace", registry_username="test-user", registry_password="test-password", - registry_insecure=False + registry_insecure=False, ) with patch('subprocess.run') as mock_run: mock_run.return_value = MagicMock(returncode=0) with patch.object(config, 'logger') as mock_logger: - config.load_registry_config(push_images=True) + config._buildah_login() mock_run.assert_called_once() call_args = mock_run.call_args @@ -74,13 +129,14 @@ def test_successful_buildah_login(self, make_config): ) def test_failed_buildah_login(self, make_config): - """Test that failed buildah login logs warning but doesn't raise.""" + """Test that failed buildah login raises ExecutionError.""" config = make_config( + push_images=True, registry_url="quay.io", registry_namespace="test-namespace", registry_username="test-user", registry_password="wrong-password", - registry_insecure=False + registry_insecure=False, ) with patch('subprocess.run') as mock_run: @@ -91,67 +147,27 @@ def test_failed_buildah_login(self, make_config): ) mock_run.side_effect = mock_error - with patch.object(config, 'logger') as mock_logger: - config.load_registry_config(push_images=True) - - mock_logger.warning.assert_called_once() - warning_call = mock_logger.warning.call_args[0][0] - assert "Failed to login to registry quay.io" in warning_call - assert "Authentication failed" in warning_call - - def test_missing_registry_credentials(self, make_config): - """Test that missing credentials raise ConfigurationError when push_images is True.""" - config = make_config( - registry_url="quay.io", - registry_namespace="test-namespace", - registry_username=None, - registry_password=None, - registry_insecure=False - ) - - with pytest.raises(ConfigurationError, match="REGISTRY_USERNAME and REGISTRY_PASSWORD environment variables are required"): - config.load_registry_config(push_images=True) - - def test_missing_registry_username(self, make_config): - """Test that missing username raises ConfigurationError when push_images is True.""" - config = make_config( - registry_url="quay.io", - registry_namespace="test-namespace", - registry_username=None, - registry_password="test-password", - registry_insecure=False - ) - - with pytest.raises(ConfigurationError, match="REGISTRY_USERNAME and REGISTRY_PASSWORD environment variables are required"): - config.load_registry_config(push_images=True) - - def test_missing_registry_password(self, make_config): - """Test that missing password raises ConfigurationError when push_images is True.""" - config = make_config( - registry_url="quay.io", - registry_namespace="test-namespace", - registry_username="test-user", - registry_password=None, - registry_insecure=False - ) - - with pytest.raises(ConfigurationError, match="REGISTRY_USERNAME and REGISTRY_PASSWORD environment variables are required"): - config.load_registry_config(push_images=True) + with pytest.raises(ExecutionError, match="Failed to login to registry quay.io") as exc_info: + config._buildah_login() + + assert exc_info.value.step == "buildah login" + assert exc_info.value.returncode == 1 def test_insecure_registry_flag(self, make_config): """Test that insecure flag is added to buildah command when registry_insecure is True.""" config = make_config( + push_images=True, registry_url="localhost:5000", registry_namespace="test-namespace", registry_username="test-user", registry_password="test-password", - registry_insecure=True + registry_insecure=True, ) with patch('subprocess.run') as mock_run: mock_run.return_value = MagicMock(returncode=0) - config.load_registry_config(push_images=True) + config._buildah_login() mock_run.assert_called_once() call_args = mock_run.call_args @@ -168,17 +184,18 @@ def test_insecure_registry_flag(self, make_config): def test_secure_registry_default(self, make_config): """Test that insecure flag is NOT added when registry_insecure is False.""" config = make_config( + push_images=True, registry_url="quay.io", registry_namespace="test-namespace", registry_username="test-user", registry_password="test-password", - registry_insecure=False + registry_insecure=False, ) with patch('subprocess.run') as mock_run: mock_run.return_value = MagicMock(returncode=0) - config.load_registry_config(push_images=True) + config._buildah_login() mock_run.assert_called_once() call_args = mock_run.call_args diff --git a/tests/test_source_config.py b/tests/test_source_config.py index ab69ff3..26a274a 100644 --- a/tests/test_source_config.py +++ b/tests/test_source_config.py @@ -18,7 +18,8 @@ def test_from_file_valid_source_json(self, tmp_path): """Test loading valid source.json with all required fields.""" source_data = { "repo": "https://github.com/awslabs/backstage-plugins-for-aws", - "repo-ref": "78df9399a81cfd95265cab53815f54210b1d7f50" + "repo-ref": "78df9399a81cfd95265cab53815f54210b1d7f50", + "workspace-path": "." } source_file = tmp_path / "source.json" @@ -45,7 +46,8 @@ def test_from_file_empty_repo(self, tmp_path): """Test that empty repo field raises ConfigurationError.""" source_data = { "repo": "", - "repo-ref": "main" + "repo-ref": "main", + "workspace-path": "." } source_file = tmp_path / "source.json" @@ -58,7 +60,8 @@ def test_from_file_empty_repo_ref(self, tmp_path): """Test that empty repo_ref field raises ConfigurationError.""" source_data = { "repo": "https://github.com/test/repo", - "repo-ref": "" + "repo-ref": "", + "workspace-path": "." } source_file = tmp_path / "source.json" @@ -70,7 +73,8 @@ def test_from_file_empty_repo_ref(self, tmp_path): def test_from_file_missing_repo_ref(self, tmp_path): """Test that missing repo_ref field raises ConfigurationError.""" source_data = { - "repo": "https://github.com/test/repo" + "repo": "https://github.com/test/repo", + "workspace-path": "." } source_file = tmp_path / "source.json" @@ -103,7 +107,8 @@ def test_clone_to_path_success(self, tmp_path): """Test successful clone with mock git commands.""" config = SourceConfig( repo="https://github.com/testowner/testrepo", - repo_ref="main" + repo_ref="main", + workspace_path="." ) repo_path = tmp_path / "repo" @@ -132,7 +137,8 @@ def test_clone_to_path_repo_path_does_not_exist(self, tmp_path): """Test that non-existent repo_path raises ConfigurationError.""" config = SourceConfig( repo="https://github.com/testowner/testrepo", - repo_ref="main" + repo_ref="main", + workspace_path="." ) repo_path = tmp_path / "nonexistent" @@ -144,7 +150,8 @@ def test_clone_to_path_clone_fails(self, tmp_path): """Test that clone failure raises ExecutionError.""" config = SourceConfig( repo="https://github.com/testowner/testrepo", - repo_ref="main" + repo_ref="main", + workspace_path="." ) repo_path = tmp_path / "repo" @@ -160,7 +167,8 @@ def test_clone_to_path_checkout_fails(self, tmp_path): """Test that checkout failure raises ExecutionError.""" config = SourceConfig( repo="https://github.com/testowner/testrepo", - repo_ref="main" + repo_ref="main", + workspace_path="." ) repo_path = tmp_path / "repo" @@ -177,7 +185,8 @@ def test_clone_to_path_exception(self, tmp_path): """Test that exceptions are wrapped in ExecutionError.""" config = SourceConfig( repo="https://github.com/testowner/testrepo", - repo_ref="main" + repo_ref="main", + workspace_path="." ) repo_path = tmp_path / "repo" @@ -197,7 +206,8 @@ def test_clean_flag_auto_cleans_nonempty_directory(self, tmp_path): """Test that clean=True automatically cleans a non-empty directory without prompting.""" config = SourceConfig( repo="https://github.com/testowner/testrepo", - repo_ref="main" + repo_ref="main", + workspace_path="." ) repo_path = tmp_path / "repo" @@ -218,7 +228,8 @@ def test_clean_flag_does_not_prompt_user(self, tmp_path): """Test that clean=True does not call input() for user confirmation.""" config = SourceConfig( repo="https://github.com/testowner/testrepo", - repo_ref="main" + repo_ref="main", + workspace_path="." ) repo_path = tmp_path / "repo" @@ -238,7 +249,8 @@ def test_no_clean_flag_prompts_user_confirm_yes(self, tmp_path): """Test that clean=False prompts user and proceeds when user enters 'y'.""" config = SourceConfig( repo="https://github.com/testowner/testrepo", - repo_ref="main" + repo_ref="main", + workspace_path="." ) repo_path = tmp_path / "repo" @@ -258,7 +270,8 @@ def test_no_clean_flag_prompts_user_confirm_no(self, tmp_path): """Test that clean=False prompts user and raises PluginFactoryError when user declines.""" config = SourceConfig( repo="https://github.com/testowner/testrepo", - repo_ref="main" + repo_ref="main", + workspace_path="." ) repo_path = tmp_path / "repo" @@ -275,7 +288,8 @@ def test_no_clean_flag_prompts_user_empty_input_aborts(self, tmp_path): """Test that clean=False raises PluginFactoryError when user presses Enter without typing anything.""" config = SourceConfig( repo="https://github.com/testowner/testrepo", - repo_ref="main" + repo_ref="main", + workspace_path="." ) repo_path = tmp_path / "repo" @@ -290,7 +304,8 @@ def test_empty_directory_skips_clean_and_prompt(self, tmp_path): """Test that an empty directory skips both clean and prompt logic.""" config = SourceConfig( repo="https://github.com/testowner/testrepo", - repo_ref="main" + repo_ref="main", + workspace_path="." ) repo_path = tmp_path / "repo" @@ -311,7 +326,8 @@ def test_empty_directory_no_clean_flag_skips_prompt(self, tmp_path): """Test that an empty directory with clean=False does not prompt user.""" config = SourceConfig( repo="https://github.com/testowner/testrepo", - repo_ref="main" + repo_ref="main", + workspace_path="." ) repo_path = tmp_path / "repo" @@ -330,7 +346,8 @@ def test_clean_flag_default_is_false(self, tmp_path): """Test that the clean parameter defaults to False.""" config = SourceConfig( repo="https://github.com/testowner/testrepo", - repo_ref="main" + repo_ref="main", + workspace_path="." ) repo_path = tmp_path / "repo" @@ -346,7 +363,8 @@ def test_clean_proceeds_with_clone_after_cleaning(self, tmp_path): """Test that after cleaning, git clone and checkout are executed.""" config = SourceConfig( repo="https://github.com/testowner/testrepo", - repo_ref="v1.0.0" + repo_ref="v1.0.0", + workspace_path="." ) repo_path = tmp_path / "repo" @@ -373,7 +391,8 @@ def test_prompt_confirm_yes_proceeds_with_clone(self, tmp_path): """Test that after user confirms 'y', git clone and checkout are executed.""" config = SourceConfig( repo="https://github.com/testowner/testrepo", - repo_ref="main" + repo_ref="main", + workspace_path="." ) repo_path = tmp_path / "repo" @@ -393,7 +412,8 @@ def test_prompt_confirm_no_does_not_clone(self, tmp_path): """Test that when user declines, git clone is not executed.""" config = SourceConfig( repo="https://github.com/testowner/testrepo", - repo_ref="main" + repo_ref="main", + workspace_path="." ) repo_path = tmp_path / "repo" From 1dcbbbb74cd7c2cd0aa1ca87b46a67d2d54c1294 Mon Sep 17 00:00:00 2001 From: Frank Kong Date: Tue, 17 Feb 2026 17:45:16 -0500 Subject: [PATCH 06/35] feat: add CLI option to define source repo and ref Signed-off-by: Frank Kong --- README.md | 66 ++++- examples/example-config-aws-ecs/README.md | 16 +- .../example-config-aws-ecs/backstage.json | 2 +- examples/example-config-aws-ecs/source.json | 2 +- examples/example-config-gitlab/README.md | 19 +- examples/example-config-gitlab/source.json | 2 +- examples/example-config-todo/README.md | 17 +- examples/example-config-todo/source.json | 2 +- src/rhdh_dynamic_plugin_factory/cli.py | 31 ++- src/rhdh_dynamic_plugin_factory/config.py | 189 +++++++++++--- tests/conftest.py | 15 +- tests/test_cli.py | 82 +++++- tests/test_config_load_from_env.py | 18 +- tests/test_source_config.py | 244 +++++++++++++++--- 14 files changed, 604 insertions(+), 101 deletions(-) diff --git a/README.md b/README.md index 5d0e539..337e41b 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,8 @@ A comprehensive tool for building and exporting dynamic plugins for Red Hat Deve - [Command-Line Options](#command-line-options) - [Understanding Volume Mounts](#understanding-volume-mounts) - [Container Usage Examples](#container-usage-examples) - - [Minimal example (no local files saved)](#minimal-example-no-local-files-saved) + - [Minimal example using `source.json`](#minimal-example-using-sourcejson) + - [Minimal example using CLI args (no `source.json` needed)](#minimal-example-using-cli-args-no-sourcejson-needed) - [Build plugins and save outputs locally](#build-plugins-and-save-outputs-locally) - [Build and push to registry](#build-and-push-to-registry) - [Using a local repository (skip cloning)](#using-a-local-repository-skip-cloning) @@ -138,6 +139,19 @@ podman run --rm -it \ --workspace-path ``` +Or, without a `source.json`, specify the repository directly: + +```bash +podman run --rm -it \ + --device /dev/fuse \ + -v ./config:/config:z \ + -v ./source:/source:z \ + -v ./outputs:/outputs:z \ + quay.io/rhdh-community/dynamic-plugins-factory:latest \ + --source-repo \ + --workspace-path +``` + **Note:** The `--config-dir`, `--repo-path`, and `--output-dir` options use default values of `/config`, `/source`, and `/outputs` respectively, which map to your local directories through volume mounts. ### Running Locally Without Containers @@ -175,7 +189,7 @@ This file contains required version settings and defaults for RHDH CLI: RHDH_CLI_VERSION="1.8.0" ``` -#### 2. `config/source.json` (Required for remote repositories) +#### 2. `config/source.json` (Required for remote repositories, unless using `--source-repo`) Defines the source repository to clone: @@ -183,13 +197,17 @@ Defines the source repository to clone: { "repo": "https://github.com/backstage/community-plugins", "repo-ref": "main", + "workspace-path": "workspaces/todo" } ``` **Fields:** - `repo`: Repository URL (HTTPS or SSH) -- `repo-ref`: Git reference (branch, tag, or commit SHA) +- `repo-ref` *(optional)*: Git reference (branch, tag, or commit SHA). When omitted, the repository's default branch is used. +- `workspace-path` *(optional)*: Path to the workspace from the repository root. Can be used instead of the `--workspace-path` CLI argument. The CLI argument takes precedence if both are provided. + +> **Note:** `source.json` is not needed when using the `--source-repo` CLI argument, which provides an alternative way to specify the repository directly from the command line. See [Command-Line Options](#command-line-options) for details. #### 3. `config/plugins-list.yaml` (Required) @@ -293,7 +311,9 @@ See the [TODO plugin example config](./examples/example-config-todo/README.md) a |--------|---------|-------------| | `--config-dir` | `/config` | Configuration directory containing `source.json`, `plugins-list.yaml`, patches, and overlays | | `--repo-path` | `/source` | Path where plugin source code will be cloned/stored | -| `--workspace-path` | (required) | Path to the workspace from repository root (e.g., `workspaces/todo`) | +| `--workspace-path` | *(see below)* | Path to the workspace from repository root (e.g., `workspaces/todo`). Can also be set via `source.json`'s `workspace-path` field. | +| `--source-repo` | `None` | Git repository URL. When provided, `source.json` is not required and the repository is cloned from this URL. | +| `--source-ref` | `None` | Git ref (branch/tag/commit) to check out. Defaults to the repository's default branch. Requires `--source-repo`. | | `--output-dir` | `/outputs` | Directory for build artifacts (`.tgz` files and container images) | | `--push-images` / `--no-push-images` | `--no-push-images` | Whether to push container images to registry. Defaults to not pushing if no argument is provided | | `--use-local` | `false` | Use local repository instead of cloning from source.json | @@ -301,6 +321,10 @@ See the [TODO plugin example config](./examples/example-config-todo/README.md) a | `--verbose` | `false` | Show verbose output with file and line numbers | | `--clean` | `false` | Automatically removes content of `--repo-path` directory when cloning from `source.json`. Ignored if `--use-local` is used. | +**Workspace path resolution:** The workspace path can be provided via the `--workspace-path` CLI argument, the `WORKSPACE_PATH` environment variable, or the `workspace-path` field in `source.json`. The CLI argument takes highest precedence, followed by the environment variable, then `source.json`. + +**Using `--source-repo` instead of `source.json`:** For single-workspace use cases, you can skip creating a `source.json` file entirely by using `--source-repo` (and optionally `--source-ref`) on the command line. When `--source-repo` is provided, `source.json` is ignored even if present. If `--source-ref` is omitted, the repository's default branch is used. + ### Understanding Volume Mounts When using the container, you can mount directories based on your needs: @@ -319,9 +343,9 @@ When using the container, you can mount directories based on your needs: The following examples demonstrate common use cases with the container image. All examples assume you have the necessary configuration files (`source.json`, `plugins-list.yaml`, and optionally patches/overlays) in your configuration directory. See the [Configuration](#configuration) section for details. -#### Minimal example (no local files saved) +#### Minimal example using `source.json` -This minimal example builds the TODO plugins without saving the workspace or output files locally: +This minimal example builds the TODO plugins without saving the workspace or output files locally. The repository, ref, and workspace path are all defined in the example's `source.json`: ```bash podman run --rm -it \ @@ -333,6 +357,22 @@ podman run --rm -it \ This will clone the repository, build the plugins, and NOT push the result to a remote repository. +#### Minimal example using CLI args (no `source.json` needed) + +You can skip `source.json` entirely by specifying the repository via CLI arguments: + +```bash +podman run --rm -it \ + --device /dev/fuse \ + -v ./config:/config:z \ + quay.io/rhdh-community/dynamic-plugins-factory:latest \ + --source-repo https://github.com/backstage/community-plugins \ + --source-ref main \ + --workspace-path workspaces/todo +``` + +If `--source-ref` is omitted, the repository's default branch is used automatically. + #### Build plugins and save outputs locally This example builds plugins and saves the `.tgz` files to your local `./outputs/` directory: @@ -432,13 +472,25 @@ The `examples` directory contains ready-to-use configuration examples demonstrat ### Quick Example: TODO Workspace -Build the TODO plugin from Backstage community plugins: +Build the TODO plugin from Backstage community plugins using the example config: + +```bash +podman run --rm -it \ + --device /dev/fuse \ + -v ./examples/example-config-todo:/config:z \ + quay.io/rhdh-community/dynamic-plugins-factory:latest \ + --workspace-path workspaces/todo \ + --no-push-images +``` + +Or build the same plugin using only CLI arguments (only a `plugins-list.yaml` in the config directory is needed): ```bash podman run --rm -it \ --device /dev/fuse \ -v ./examples/example-config-todo:/config:z \ quay.io/rhdh-community/dynamic-plugins-factory:latest \ + --source-repo https://github.com/backstage/community-plugins \ --workspace-path workspaces/todo \ --no-push-images ``` diff --git a/examples/example-config-aws-ecs/README.md b/examples/example-config-aws-ecs/README.md index 05d26a2..69f5313 100644 --- a/examples/example-config-aws-ecs/README.md +++ b/examples/example-config-aws-ecs/README.md @@ -41,7 +41,7 @@ example-config-aws-ecs/ ### Configuration Files -- **`source.json`**: Specifies the AWS plugins repository and git reference to clone from +- **`source.json`**: Specifies the AWS plugins repository and git reference to clone from. The `repo-ref` field is optional; when omitted, the repository's default branch is used. The `workspace-path` field can also be set here instead of using `--workspace-path`. - **`plugins-list.yaml`**: Lists plugins with `--embed-package` arguments for shared dependencies - **`backstage.json`**: Backstage configuration file - **`patches/`**: Contains patch files @@ -109,10 +109,9 @@ podman run --rm -it \ -v ./examples/example-config-aws-ecs:/config:z \ -v ./outputs:/outputs:z \ quay.io/rhdh-community/dynamic-plugins-factory:latest \ - --workspace-path . ``` -Note: `--workspace-path .` is used because this repository does not follow the backstage community plugins (BCP) repository structure where there are multiple yarn workspaces. Instead the plugins are stored in the main workspace, so root of the workspace is also the root of the repository in this example. +Note: `workspace-path` is set to `.` in `source.json` because this repository does not follow the backstage community plugins (BCP) repository structure where there are multiple yarn workspaces. Instead the plugins are stored in the main workspace, so root of the workspace is also the root of the repository in this example. ### Local Development @@ -120,6 +119,17 @@ From the repository root, run: ```bash python -m src.rhdh_dynamic_plugin_factory \ + --config-dir ./examples/example-config-aws-ecs \ + --repo-path ./source \ + --output-dir ./outputs +``` + +Or using CLI args instead of `source.json`: + +```bash +python -m src.rhdh_dynamic_plugin_factory \ + --source-repo https://github.com/awslabs/backstage-plugins-for-aws \ + --source-ref 78df9399a81cfd95265cab53815f54210b1d7f50 \ --config-dir ./examples/example-config-aws-ecs \ --workspace-path . \ --repo-path ./source \ diff --git a/examples/example-config-aws-ecs/backstage.json b/examples/example-config-aws-ecs/backstage.json index 6c12971..655d5ff 100644 --- a/examples/example-config-aws-ecs/backstage.json +++ b/examples/example-config-aws-ecs/backstage.json @@ -1,3 +1,3 @@ { - "version": "1.42.5" + "version": "1.45.3" } diff --git a/examples/example-config-aws-ecs/source.json b/examples/example-config-aws-ecs/source.json index a112a99..077cf6e 100644 --- a/examples/example-config-aws-ecs/source.json +++ b/examples/example-config-aws-ecs/source.json @@ -1 +1 @@ -{"repo":"https://github.com/awslabs/backstage-plugins-for-aws","repo-ref":"78df9399a81cfd95265cab53815f54210b1d7f50","repo-flat":true,"repo-backstage-version":"1.42.5"} +{"repo":"https://github.com/awslabs/backstage-plugins-for-aws","repo-ref":"3a5efbf212a029ecbe6990b1b5b7ea1709dd7c2d","workspace-path":"."} diff --git a/examples/example-config-gitlab/README.md b/examples/example-config-gitlab/README.md index 3f9d972..c2c11f9 100644 --- a/examples/example-config-gitlab/README.md +++ b/examples/example-config-gitlab/README.md @@ -38,7 +38,7 @@ example-config-gitlab/ ### Configuration Files -- **`source.json`**: Specifies the GitLab plugin repository and git eference to clone from +- **`source.json`**: Specifies the GitLab plugin repository and git reference to clone from. The `repo-ref` field is optional; when omitted, the repository's default branch is used. The `workspace-path` field can also be set here instead of using `--workspace-path`. - **`plugins-list.yaml`**: List of path to the GitLab frontend and backend packages to build - **`packages/gitlab-backend/overlay/`**: Contains replacement source files @@ -68,12 +68,11 @@ podman run --rm -it \ --device /dev/fuse \ -v ./examples/example-config-gitlab:/config:z \ -v ./outputs:/outputs:z \ - -f ./source:/source:z \ + -v ./source:/source:z \ quay.io/rhdh-community/dynamic-plugins-factory:latest \ - --workspace-path . ``` -Note: `--workspace-path .` is used because this repository does not follow the backstage community plugins (BCP) repository structure where there are multiple yarn workspaces. Instead the plugins are stored in the main workspace, so root of the workspace is also the root of the repository in this example. +Note: `workspace-path` is set to `.` in the `source.json` because this repository does not follow the backstage community plugins (BCP) repository structure where there are multiple yarn workspaces. Instead the plugins are stored in the main workspace, so root of the workspace is also the root of the repository in this example. ### Local Development @@ -87,6 +86,18 @@ python -m src.rhdh_dynamic_plugin_factory \ --output-dir ./outputs ``` +Or using CLI args instead of `source.json`: + +```bash +python -m src.rhdh_dynamic_plugin_factory \ + --source-repo https://github.com/immobiliare/backstage-plugin-gitlab \ + --source-ref v6.13.0 \ + --config-dir ./examples/example-config-gitlab \ + --workspace-path . \ + --repo-path ./source \ + --output-dir ./outputs +``` + This will do the following: 1. The factory clones the GitLab plugin repository to `./source` diff --git a/examples/example-config-gitlab/source.json b/examples/example-config-gitlab/source.json index f7df1aa..028ae10 100644 --- a/examples/example-config-gitlab/source.json +++ b/examples/example-config-gitlab/source.json @@ -1 +1 @@ -{"repo":"https://github.com/immobiliare/backstage-plugin-gitlab","repo-ref":"v6.13.0","repo-flat":"true","repo-backstage-version":"1.42.5"} +{"repo":"https://github.com/immobiliare/backstage-plugin-gitlab","repo-ref":"v6.13.0","workspace-path":"."} diff --git a/examples/example-config-todo/README.md b/examples/example-config-todo/README.md index ecfa441..48166b9 100644 --- a/examples/example-config-todo/README.md +++ b/examples/example-config-todo/README.md @@ -41,7 +41,7 @@ example-config-todo/ ### Configuration Files -- **`source.json`**: Specifies the Backstage Community Plugins repository and git reference to clone from +- **`source.json`**: Specifies the Backstage Community Plugins repository and git reference to clone from. The `repo-ref` field is optional; when omitted, the repository's default branch is used. The `workspace-path` field can also be set here instead of using `--workspace-path`. - **`plugins-list.yaml`**: Lists the path to the TODO frontend and backend plugins to build with respect to the workspace path - **`plugins/todo/scalprum-config.json`**: Custom Scalprum configuration that will be overlaid on top of the plugin source directory @@ -57,9 +57,10 @@ podman run --rm -it \ -v ./examples/example-config-todo:/config:z \ -v ./outputs:/outputs:z \ quay.io/rhdh-community/dynamic-plugins-factory:latest \ - --workspace-path workspaces/todo ``` +When `--source-ref` is omitted, the repository's default branch is used automatically. + ### Local Development From the repository root, run: @@ -72,6 +73,18 @@ python -m src.rhdh_dynamic_plugin_factory \ --output-dir ./outputs ``` +Or using CLI args instead of `source.json`: + +```bash +python -m src.rhdh_dynamic_plugin_factory \ + --source-repo https://github.com/backstage/community-plugins \ + --source-ref main \ + --config-dir ./examples/example-config-todo \ + --repo-path ./source \ + --workspace-path workspaces/todo \ + --output-dir ./outputs +``` + This will do the following: 1. The factory clones the Backstage Community Plugins repository to `./source` diff --git a/examples/example-config-todo/source.json b/examples/example-config-todo/source.json index 1d11f35..120e53e 100644 --- a/examples/example-config-todo/source.json +++ b/examples/example-config-todo/source.json @@ -1 +1 @@ -{"repo":"https://github.com/backstage/community-plugins","repo-ref":"b554cdfb0d5d75ee69a9e4e4becf84d31d824513","repo-flat":false,"repo-backstage-version":"1.42.3"} \ No newline at end of file +{"repo":"https://github.com/backstage/community-plugins","repo-ref":"3fadf6b0595d1d1804f5c7f2690f4db7b81275f1","workspace-path":"workspaces/todo"} \ No newline at end of file diff --git a/src/rhdh_dynamic_plugin_factory/cli.py b/src/rhdh_dynamic_plugin_factory/cli.py index 2bd1b98..0b1cf00 100644 --- a/src/rhdh_dynamic_plugin_factory/cli.py +++ b/src/rhdh_dynamic_plugin_factory/cli.py @@ -33,8 +33,11 @@ def create_parser() -> argparse.ArgumentParser: epilog=""" Examples: # Build plugins for the todo workspace in backstage/community-plugins w/o pushing to a registry - # This assumes that ./config is populated with the required source.json and plugins-list.yaml files + # This assumes that ./config is populated with the source.json and plugins-list.yaml files python src/rhdh_dynamic_plugin_factory --config-dir ./config --repo-path ./source --workspace-path workspaces/todo --log-level DEBUG --output-dir ./outputs + + # Build plugins using CLI args instead of source.json + python src/rhdh_dynamic_plugin_factory --source-repo https://github.com/backstage/community-plugins --source-ref main --workspace-path workspaces/todo --config-dir ./config --repo-path ./source --output-dir ./outputs """ ) parser.add_argument( @@ -63,7 +66,19 @@ def create_parser() -> argparse.ArgumentParser: parser.add_argument( "--workspace-path", type=Path, - help="Path to the workspace from root of the repository. Required if `source.json` is not provided." + help="Path to the workspace from root of the repository. Can also be provided via the source.json workspace-path field." + ) + parser.add_argument( + "--source-repo", + type=str, + default=None, + help="Git repository URL. When provided, source.json is ignored and the repository is cloned from this URL." + ) + parser.add_argument( + "--source-ref", + type=str, + default=None, + help="Git ref (branch/tag/commit) to check out. Optional: defaults to the repository's default branch. Requires --source-repo." ) parser.add_argument( "--push-images", @@ -155,6 +170,17 @@ def _run(args: argparse.Namespace) -> None: source_config = config.setup_config_directory() + # Resolve workspace_path from source_config if not set via CLI/env + if not config.workspace_path and source_config: + config.workspace_path = source_config.workspace_path + + # Validate workspace_path is set (may come from CLI, env var, or source.json) + if not config.workspace_path: + raise ConfigurationError( + "WORKSPACE_PATH must be set via --workspace-path argument, " + "WORKSPACE_PATH environment variable, or source.json workspace-path field" + ) + if source_config and not config.use_local: logger.info("[bold blue]Repository Setup[/bold blue]") source_config.clone_to_path(config.repo_path, clean=args.clean) @@ -167,6 +193,7 @@ def _run(args: argparse.Namespace) -> None: raise ConfigurationError( f"Local repository does not exist at: {config.repo_path}. " "Either provide source.json to clone the repository, " + "use --source-repo to specify a repository via CLI, " "or ensure workspace exists at directory specified by --repo-path" ) logger.info(f"Using local repository at: {config.repo_path}") diff --git a/src/rhdh_dynamic_plugin_factory/config.py b/src/rhdh_dynamic_plugin_factory/config.py index 50ea20b..cafe07e 100644 --- a/src/rhdh_dynamic_plugin_factory/config.py +++ b/src/rhdh_dynamic_plugin_factory/config.py @@ -28,6 +28,11 @@ class PluginFactoryConfig: config_dir: str = field(default="/config") workspace_path: str = field(default="") # Relative path from repo_path to the workspace + # Source repository CLI overrides (take precedence over source.json) + # Used for single workspace case + source_repo: Optional[str] = field(default=None) + source_ref: Optional[str] = field(default=None) + # Registry configuration (loaded from environment variables, only required for push operations) registry_url: Optional[str] = field(default=None) registry_username: Optional[str] = field(default=None) @@ -42,12 +47,18 @@ class PluginFactoryConfig: logger: ClassVar[Logger] = get_logger("config") def __post_init__(self) -> None: - """Validate configuration fields after initialization.""" + """Validate configuration fields after initialization. + + Note: workspace_path is NOT validated here because it may be resolved + later from source.json. Validation happens in cli._run() after source + configuration discovery. + """ if not self.rhdh_cli_version: raise ConfigurationError("RHDH_CLI_VERSION must be set (usually loaded from default.env)") - if not self.workspace_path: - raise ConfigurationError("WORKSPACE_PATH must be set via environment variable or --workspace-path argument") + # Validate source arg constraints: --source-ref requires --source-repo + if self.source_ref and not self.source_repo: + raise ConfigurationError("--source-ref requires --source-repo to be provided") valid_log_levels = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] if self.log_level.upper() not in valid_log_levels: @@ -94,11 +105,21 @@ def load_from_env(cls, args: argparse.Namespace, env_file: Optional[Path] = None for dir_path in [config_dir, repo_path]: os.makedirs(dir_path, exist_ok=True) + # Resolve workspace_path: env var takes precedence, then CLI arg, then empty (potentially resolved later from source.json) + workspace_path = os.getenv("WORKSPACE_PATH") or "" + if not workspace_path and args.workspace_path: + workspace_path = str(args.workspace_path) + + source_repo = getattr(args, 'source_repo', None) + source_ref = getattr(args, 'source_ref', None) + config = cls( rhdh_cli_version=os.getenv("RHDH_CLI_VERSION", ""), repo_path=repo_path, config_dir=config_dir, - workspace_path=os.getenv("WORKSPACE_PATH", args.workspace_path), + workspace_path=workspace_path, + source_repo=source_repo, + source_ref=source_ref, registry_url=os.getenv("REGISTRY_URL"), registry_username=os.getenv("REGISTRY_USERNAME"), registry_password=os.getenv("REGISTRY_PASSWORD"), @@ -153,14 +174,23 @@ def _buildah_login(self) -> None: ) from e def _validate_source_json(self) -> None: - """Validate source.json file existence and repo_path state.""" + """Validate source.json file existence and repo_path state. + + Skips validation when --source-repo CLI arg is provided, since + CLI args fully replace source.json. + """ + if self.source_repo: + self.logger.debug("Using --source-repo CLI argument, skipping source.json validation") + return + source_file = os.path.join(self.config_dir, "source.json") if not os.path.exists(source_file): if not os.path.exists(self.repo_path) or not os.listdir(self.repo_path): raise ConfigurationError( f"source.json not found at {source_file} and {self.repo_path} is empty. " - "Please provide source.json to clone a repository or use --use-local with a locally mounted repository." + "Please provide source.json to clone a repository, use --source-repo to specify a repository via CLI, " + "or use --use-local with a locally mounted repository." ) else: self.logger.warning( @@ -225,14 +255,28 @@ def auto_generate_plugins_list(self) -> None: raise PluginFactoryError(f"Failed to auto-generate plugins list: {e}") from e def discover_source_config(self) -> Optional["SourceConfig"]: - """Discovers and loads source configuration from config_dir/source.json. + """Discovers and loads source configuration. + + CLI args (--source-repo/--source-ref) take precedence over source.json. + Falls back to local repo if no source configuration is available. Returns: - SourceConfig if source.json exists and is valid, None if falling back to local repo. + SourceConfig if source is configured, None if falling back to local repo. Raises: - ConfigurationError: If source.json is invalid or no valid source is available. + ConfigurationError: If source configuration is invalid or no valid source is available. """ + # CLI args take precedence over source.json + if self.source_repo and not self.use_local: + self.logger.info("Using source configuration from CLI arguments") + source_config = SourceConfig.from_cli_args( + repo=self.source_repo, + repo_ref=self.source_ref, + workspace_path=self.workspace_path, + ) + self.logger.debug(f"Using source config from CLI: {source_config}") + return source_config + source_file = os.path.join(self.config_dir, "source.json") if os.path.exists(source_file) and not self.use_local: @@ -245,7 +289,8 @@ def discover_source_config(self) -> Optional["SourceConfig"]: else: raise ConfigurationError( f"No valid source configuration found and {self.repo_path} is empty or does not exist. " - f"Either provide a valid {source_file} or ensure locally stored plugin source code exists at {self.repo_path}" + f"Either provide a valid {source_file}, use --source-repo to specify a repository via CLI, " + f"or ensure locally stored plugin source code exists at {self.repo_path}" ) return None @@ -424,25 +469,30 @@ def conditional_stderr_log(line: str) -> None: class SourceConfig: """Configuration for plugin source repository.""" repo: str - repo_ref: str + repo_ref: Optional[str] # None triggers default branch resolution in __post_init__ workspace_path: str logger: ClassVar[Logger] = get_logger("source_config") def __post_init__(self) -> None: - if not self.workspace_path: - raise ConfigurationError("workspace-path is required") if not self.repo: raise ConfigurationError("repo is required") - if not self.repo_ref: - raise ConfigurationError("repo_ref is required") + if not self.workspace_path: + raise ConfigurationError("workspace-path is required") + # Resolve default branch at creation time if no ref was provided + if not self.repo_ref: + self.repo_ref = self.resolve_default_ref(self.repo) @classmethod def from_file(cls, source_file: Path) -> "SourceConfig": """Load source configuration from JSON file. + repo-ref is optional. When omitted, the default branch is resolved + automatically during construction via resolve_default_ref(). + Raises: ConfigurationError: If the file is missing, malformed, or has invalid data. + ExecutionError: If default branch resolution fails (when repo-ref is omitted). """ try: with open(source_file, 'r') as f: @@ -456,7 +506,7 @@ def from_file(cls, source_file: Path) -> "SourceConfig": try: repo = data["repo"] - repo_ref = data.get("repo-ref") + repo_ref = data.get("repo-ref") or None # Treat empty string as None workspace_path = data.get("workspace-path") except KeyError as e: raise ConfigurationError(f"Missing required field {e} in {source_file}") @@ -469,6 +519,82 @@ def from_file(cls, source_file: Path) -> "SourceConfig": return config + @classmethod + def from_cli_args(cls, repo: str, repo_ref: Optional[str], workspace_path: str) -> "SourceConfig": + """Create source configuration from CLI arguments. + + Args: + repo: Git repository URL (--source-repo). + repo_ref: Git ref to check out (--source-ref). None means default branch + (resolved automatically during construction via resolve_default_ref()). + workspace_path: Path to workspace within the repository (--workspace-path). + + Returns: + SourceConfig instance with repo_ref always resolved. + + Raises: + ConfigurationError: If required fields are missing. + ExecutionError: If default branch resolution fails (when repo_ref is None). + """ + return cls( + repo=repo, + repo_ref=repo_ref, + workspace_path=workspace_path, + ) + + @staticmethod + def resolve_default_ref(repo: str) -> str: + """Resolve the default branch ref for a repository using git ls-remote since repository is not cloned yet + + Args: + repo: Git repository URL. + + Returns: + The default branch ref (e.g., 'refs/heads/main'). + + Raises: + ExecutionError: If git ls-remote fails or the default branch cannot be determined. + """ + logger = get_logger("source_config") + logger.info(f"[cyan]Resolving default branch for {repo}...[/cyan]") + + try: + result = subprocess.run( + ["git", "ls-remote", "--symref", repo, "HEAD"], + capture_output=True, + text=True, + check=True, + ) + + # Output format: + # ref: refs/heads/main\tHEAD + # \tHEAD + for line in result.stdout.splitlines(): + if line.startswith("ref:"): + # Ex: Extract "refs/heads/main" from "ref: refs/heads/main\tHEAD" + ref_part = line.split("\t")[0].replace("ref: ", "").strip() + logger.info(f"[green]Resolved default branch: {ref_part}[/green]") + return ref_part + + # Fallback: if no symbolic ref, use the HEAD SHA directly + for line in result.stdout.splitlines(): + parts = line.split("\t") + if len(parts) == 2 and parts[1].strip() == "HEAD": + sha = parts[0].strip() + logger.info(f"[green]Resolved default ref (SHA): {sha}[/green]") + return sha + + raise ExecutionError( + "Could not determine default branch from git ls-remote output", + step="resolve default ref", + ) + except subprocess.CalledProcessError as e: + raise ExecutionError( + f"Failed to resolve default branch for {repo}: {e.stderr.strip()}", + step="resolve default ref", + returncode=e.returncode, + ) from e + def clone_to_path(self, repo_path: Path, clean: bool = False) -> None: """Clone the source repository to the specified path. @@ -518,23 +644,22 @@ def clone_to_path(self, repo_path: Path, clean: bool = False) -> None: returncode=returncode ) - if self.repo_ref: - cmd = ["git", "checkout", self.repo_ref] - logger.info(f"[cyan]Checking out ref: {self.repo_ref}[/cyan]") - # Git writes informational messages to stderr - returncode = run_command_with_streaming( - cmd, - logger, - cwd=repo_path, - stderr_log_func=logger.info + cmd = ["git", "checkout", self.repo_ref] + logger.info(f"[cyan]Checking out ref: {self.repo_ref}[/cyan]") + # Git writes informational messages to stderr + returncode = run_command_with_streaming( + cmd, + logger, + cwd=repo_path, + stderr_log_func=logger.info + ) + + if returncode != 0: + raise ExecutionError( + f"Failed to checkout ref {self.repo_ref} (exit code {returncode})", + step="git checkout", + returncode=returncode ) - - if returncode != 0: - raise ExecutionError( - f"Failed to checkout ref {self.repo_ref} (exit code {returncode})", - step="git checkout", - returncode=returncode - ) logger.info("[green]✓ Repository cloned successfully[/green]") diff --git a/tests/conftest.py b/tests/conftest.py index 6a42891..945d616 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -21,16 +21,21 @@ def mock_logger(): @pytest.fixture def mock_args(tmp_path): - """Create mock argparse.Namespace with default valid arguments.""" + """Create mock argparse.Namespace with default valid arguments. + + Uses Path objects for config_dir and repo_path to match argparse type=Path behavior. + """ args = argparse.Namespace( workspace_path=".", - config_dir=str(tmp_path / "config"), - repo_path=str(tmp_path / "workspace"), + config_dir=tmp_path / "config", + repo_path=tmp_path / "workspace", log_level="INFO", use_local=False, push_images=False, output_dir=str(tmp_path / "outputs"), - verbose=False + verbose=False, + source_repo=None, + source_ref=None, ) return args @@ -59,6 +64,7 @@ def valid_source_json(tmp_path: Path): source_data = { "repo": "https://github.com/awslabs/backstage-plugins-for-aws", "repo-ref": "78df9399a81cfd95265cab53815f54210b1d7f50", + "workspace-path": ".", "repo-flat": True, "repo-backstage-version": "1.42.5" } @@ -133,6 +139,7 @@ def setup_test_env(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): source_data = { "repo": "https://github.com/awslabs/backstage-plugins-for-aws", "repo-ref": "78df9399a81cfd95265cab53815f54210b1d7f50", + "workspace-path": ".", "repo-flat": True, "repo-backstage-version": "1.42.5" } diff --git a/tests/test_cli.py b/tests/test_cli.py index bfcad64..17814db 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -4,7 +4,10 @@ Tests the argument parser to ensure all arguments are correctly defined and parsed. """ -from src.rhdh_dynamic_plugin_factory.cli import create_parser +import pytest + +from src.rhdh_dynamic_plugin_factory.cli import create_parser, _run +from src.rhdh_dynamic_plugin_factory.exceptions import ConfigurationError class TestCreateParserCleanArgument: @@ -13,9 +16,7 @@ class TestCreateParserCleanArgument: def test_clean_flag_default_is_false(self): """Test that --clean defaults to False when not provided.""" parser = create_parser() - args = parser.parse_args([ - "--workspace-path", "workspaces/todo", - ]) + args = parser.parse_args([]) assert args.clean is False @@ -23,7 +24,6 @@ def test_clean_flag_set_to_true(self): """Test that --clean sets the flag to True.""" parser = create_parser() args = parser.parse_args([ - "--workspace-path", "workspaces/todo", "--clean", ]) @@ -35,7 +35,6 @@ def test_clean_flag_is_store_true_action(self): # Should work without a value after --clean args = parser.parse_args([ - "--workspace-path", "workspaces/todo", "--clean", "--log-level", "DEBUG", ]) @@ -60,3 +59,74 @@ def test_clean_flag_combined_with_other_args(self): assert str(args.config_dir) == "/custom/config" assert str(args.repo_path) == "/custom/source" assert args.log_level == "WARNING" + + +class TestCreateParserSourceRepoArgument: + """Tests for the --source-repo and --source-ref CLI arguments.""" + + def test_source_repo_default_is_none(self): + """Test that --source-repo defaults to None when not provided.""" + parser = create_parser() + args = parser.parse_args([]) + + assert args.source_repo is None + + def test_source_ref_default_is_none(self): + """Test that --source-ref defaults to None when not provided.""" + parser = create_parser() + args = parser.parse_args([]) + + assert args.source_ref is None + + def test_source_repo_set(self): + """Test that --source-repo is correctly parsed.""" + parser = create_parser() + args = parser.parse_args([ + "--source-repo", "https://github.com/backstage/community-plugins", + ]) + + assert args.source_repo == "https://github.com/backstage/community-plugins" + assert args.source_ref is None + + def test_source_ref_set(self): + """Test that --source-ref is correctly parsed alongside --source-repo.""" + parser = create_parser() + args = parser.parse_args([ + "--source-repo", "https://github.com/backstage/community-plugins", + "--source-ref", "abc123", + ]) + + assert args.source_repo == "https://github.com/backstage/community-plugins" + assert args.source_ref == "abc123" + + def test_source_args_combined_with_workspace_path(self): + """Test that source args work alongside --workspace-path.""" + parser = create_parser() + args = parser.parse_args([ + "--source-repo", "https://github.com/backstage/community-plugins", + "--source-ref", "main", + "--workspace-path", "workspaces/todo", + ]) + + assert args.source_repo == "https://github.com/backstage/community-plugins" + assert args.source_ref == "main" + assert str(args.workspace_path) == "workspaces/todo" + + +class TestRunSourceArgValidation: + """Tests for --source-repo/--source-ref validation in _run(). + + The validation now occurs in PluginFactoryConfig.__post_init__, + which is called during load_from_env() inside _run(). + """ + + def test_source_ref_without_source_repo_raises_error(self, mock_args, monkeypatch): + """Test that --source-ref without --source-repo raises ConfigurationError.""" + # RHDH_CLI_VERSION must be set so __post_init__ reaches the source arg check + monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") + + mock_args.source_ref = "main" + mock_args.source_repo = None + + with pytest.raises(ConfigurationError, match="--source-ref requires --source-repo"): + _run(mock_args) diff --git a/tests/test_config_load_from_env.py b/tests/test_config_load_from_env.py index d44c092..c6a792f 100644 --- a/tests/test_config_load_from_env.py +++ b/tests/test_config_load_from_env.py @@ -152,20 +152,24 @@ def test_load_from_env_additional_env_file_loading(self, mock_args, setup_test_e assert config.registry_url == "quay.io" assert config.registry_namespace == "test-namespace" - def test_load_from_env_missing_workspace_path(self, mock_args, setup_test_env, monkeypatch): - """Test that missing workspace_path raises ConfigurationError.""" + def test_load_from_env_missing_workspace_path_deferred(self, mock_args, setup_test_env, monkeypatch): + """Test that missing workspace_path is allowed during load_from_env. + + workspace_path validation is deferred to cli._run() because it may be + resolved later from source.json's workspace-path field. + """ monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") - # Don't set WORKSPACE_PATH env var to test fallback + # Don't set WORKSPACE_PATH env var monkeypatch.delenv("WORKSPACE_PATH", raising=False) mock_args.config_dir = setup_test_env["config_dir"] mock_args.repo_path = setup_test_env["source_dir"] mock_args.workspace_path = None # Missing workspace_path - # When workspace_path is None and WORKSPACE_PATH env var is not set, - # validation should raise ConfigurationError - with pytest.raises(ConfigurationError, match="WORKSPACE_PATH must be set"): - PluginFactoryConfig.load_from_env(mock_args) + # Should not raise -- workspace_path validation is deferred + config = PluginFactoryConfig.load_from_env(mock_args) + + assert config.workspace_path == "" # Empty, to be resolved later def test_load_from_env_directory_creation(self, mock_args, tmp_path, monkeypatch): """Test that config_dir and repo_path directories are created.""" diff --git a/tests/test_source_config.py b/tests/test_source_config.py index 26a274a..3eca34c 100644 --- a/tests/test_source_config.py +++ b/tests/test_source_config.py @@ -1,14 +1,15 @@ """ Unit tests for SourceConfig class. -Tests the source configuration loading and repository URL parsing. +Tests the source configuration loading, CLI arg construction, and repository cloning. """ import json -from unittest.mock import patch +import subprocess +from unittest.mock import patch, MagicMock import pytest -from src.rhdh_dynamic_plugin_factory.config import SourceConfig +from src.rhdh_dynamic_plugin_factory.config import SourceConfig, PluginFactoryConfig from src.rhdh_dynamic_plugin_factory.exceptions import ConfigurationError, ExecutionError, PluginFactoryError class TestSourceConfigFromFile: @@ -56,33 +57,7 @@ def test_from_file_empty_repo(self, tmp_path): with pytest.raises(ConfigurationError, match="repo is required"): SourceConfig.from_file(source_file) - def test_from_file_empty_repo_ref(self, tmp_path): - """Test that empty repo_ref field raises ConfigurationError.""" - source_data = { - "repo": "https://github.com/test/repo", - "repo-ref": "", - "workspace-path": "." - } - - source_file = tmp_path / "source.json" - source_file.write_text(json.dumps(source_data)) - - with pytest.raises(ConfigurationError, match="repo_ref is required"): - SourceConfig.from_file(source_file) - - def test_from_file_missing_repo_ref(self, tmp_path): - """Test that missing repo_ref field raises ConfigurationError.""" - source_data = { - "repo": "https://github.com/test/repo", - "workspace-path": "." - } - - source_file = tmp_path / "source.json" - source_file.write_text(json.dumps(source_data)) - - # repo_ref is now required and will raise ConfigurationError if missing/None - with pytest.raises(ConfigurationError, match="repo_ref is required"): - SourceConfig.from_file(source_file) + def test_from_file_malformed_json(self, tmp_path): """Test that malformed JSON raises ConfigurationError with descriptive message.""" @@ -99,6 +74,98 @@ def test_from_file_nonexistent_file(self, tmp_path): with pytest.raises(ConfigurationError, match="Source configuration file not found"): SourceConfig.from_file(source_file) +class TestSourceConfigFromCliArgs: + """Tests for SourceConfig.from_cli_args classmethod.""" + + def test_from_cli_args_with_all_fields(self): + """Test creating SourceConfig from CLI args with all fields.""" + config = SourceConfig.from_cli_args( + repo="https://github.com/backstage/community-plugins", + repo_ref="abc123", + workspace_path="workspaces/todo", + ) + + assert config.repo == "https://github.com/backstage/community-plugins" + assert config.repo_ref == "abc123" + assert config.workspace_path == "workspaces/todo" + + def test_from_cli_args_with_none_repo_ref_resolves_default(self): + """Test creating SourceConfig from CLI args with None repo_ref triggers resolution.""" + with patch.object(SourceConfig, "resolve_default_ref", return_value="refs/heads/main"): + config = SourceConfig.from_cli_args( + repo="https://github.com/backstage/community-plugins", + repo_ref=None, + workspace_path="workspaces/todo", + ) + + assert config.repo == "https://github.com/backstage/community-plugins" + assert config.repo_ref == "refs/heads/main" + assert config.workspace_path == "workspaces/todo" + + def test_from_cli_args_missing_repo_raises_error(self): + """Test that missing repo raises ConfigurationError.""" + with pytest.raises(ConfigurationError, match="repo is required"): + SourceConfig.from_cli_args( + repo="", + repo_ref="main", + workspace_path="workspaces/todo", + ) + + def test_from_cli_args_missing_workspace_path_raises_error(self): + """Test that missing workspace_path raises ConfigurationError.""" + with pytest.raises(ConfigurationError, match="workspace-path is required"): + SourceConfig.from_cli_args( + repo="https://github.com/backstage/community-plugins", + repo_ref="main", + workspace_path="", + ) + + +class TestResolveDefaultRef: + """Tests for SourceConfig.resolve_default_ref static method. + + Happy-path tests use real git ls-remote calls to verify parsing + against actual git output. Error cases use mocks since they can't + be reliably triggered against real repositories. + """ + + def test_resolve_default_ref_real_repo(self): + """Test resolving default branch against a real public repository.""" + ref = SourceConfig.resolve_default_ref("https://github.com/git/git.git") + + # Should return the master branch + assert (ref == "refs/heads/master") + # Branch name should be non-empty + branch_name = ref.removeprefix("refs/heads/") + assert branch_name == "master" + + def test_resolve_default_ref_git_failure(self): + """Test that git ls-remote failure raises ExecutionError.""" + error = subprocess.CalledProcessError(128, "git", stderr="fatal: repository not found") + + with patch("subprocess.run", side_effect=error): + with pytest.raises(ExecutionError, match="Failed to resolve default branch"): + SourceConfig.resolve_default_ref("https://github.com/test/nonexistent") + + def test_resolve_default_ref_empty_output(self): + """Test that empty git ls-remote output raises ExecutionError.""" + mock_result = MagicMock() + mock_result.stdout = "" + + with patch("subprocess.run", return_value=mock_result): + with pytest.raises(ExecutionError, match="Could not determine default branch"): + SourceConfig.resolve_default_ref("https://github.com/test/repo") + + def test_resolve_default_ref_sha_fallback(self): + """Test falling back to SHA when no symbolic ref line is present.""" + mock_result = MagicMock() + mock_result.stdout = "abc123def456\tHEAD\n" + + with patch("subprocess.run", return_value=mock_result): + ref = SourceConfig.resolve_default_ref("https://github.com/test/repo") + + assert ref == "abc123def456" + class TestSourceConfigCloneToPath: """Tests for SourceConfig.clone_to_path method.""" @@ -133,6 +200,37 @@ def test_clone_to_path_success(self, tmp_path): assert checkout_call[0][0][1] == "checkout" assert checkout_call[0][0][2] == "main" + def test_clone_to_path_resolved_default_ref(self, tmp_path): + """Test that clone works correctly when repo_ref was resolved from default branch.""" + with patch.object(SourceConfig, "resolve_default_ref", return_value="refs/heads/main"): + config = SourceConfig( + repo="https://github.com/testowner/testrepo", + repo_ref=None, + workspace_path="." + ) + + # repo_ref should already be resolved at creation time + assert config.repo_ref == "refs/heads/main" + + repo_path = tmp_path / "repo" + repo_path.mkdir() + + with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run: + mock_run.return_value = 0 + + config.clone_to_path(repo_path) + + # Both clone and checkout should happen + assert mock_run.call_count == 2 + + # Verify clone command + clone_call = mock_run.call_args_list[0] + assert clone_call[0][0] == ["git", "clone", "https://github.com/testowner/testrepo", str(repo_path)] + + # Verify checkout command with resolved ref + checkout_call = mock_run.call_args_list[1] + assert checkout_call[0][0] == ["git", "checkout", "refs/heads/main"] + def test_clone_to_path_repo_path_does_not_exist(self, tmp_path): """Test that non-existent repo_path raises ConfigurationError.""" config = SourceConfig( @@ -428,3 +526,89 @@ def test_prompt_confirm_no_does_not_clone(self, tmp_path): mock_run.assert_not_called() + +class TestDiscoverSourceConfigCliArgs: + """Tests for PluginFactoryConfig.discover_source_config with CLI args.""" + + def test_cli_args_take_precedence_over_source_json(self, make_config, setup_test_env): + """Test that --source-repo takes precedence over source.json.""" + config = make_config( + source_repo="https://github.com/cli/override-repo", + source_ref="v2.0.0", + ) + + source_config = config.discover_source_config() + + assert source_config is not None + assert source_config.repo == "https://github.com/cli/override-repo" + assert source_config.repo_ref == "v2.0.0" + + def test_cli_args_with_none_repo_ref_resolves_default(self, make_config, setup_test_env): + """Test that --source-repo without --source-ref resolves default branch.""" + config = make_config( + source_repo="https://github.com/cli/override-repo", + source_ref=None, + ) + + with patch.object(SourceConfig, "resolve_default_ref", return_value="refs/heads/main"): + source_config = config.discover_source_config() + + assert source_config is not None + assert source_config.repo == "https://github.com/cli/override-repo" + assert source_config.repo_ref == "refs/heads/main" + + def test_cli_args_skipped_when_use_local(self, make_config, setup_test_env): + """Test that CLI args are ignored when --use-local is set.""" + config = make_config( + source_repo="https://github.com/cli/override-repo", + source_ref="v2.0.0", + use_local=True, + ) + + source_config = config.discover_source_config() + + # Should return None (using local), not the CLI args + assert source_config is None + + def test_falls_back_to_source_json_when_no_cli_args(self, make_config, setup_test_env): + """Test that source.json is used when no CLI args are provided.""" + config = make_config( + source_repo=None, + source_ref=None, + ) + + source_config = config.discover_source_config() + + # Should use the source.json from setup_test_env fixture + assert source_config is not None + assert source_config.repo == "https://github.com/awslabs/backstage-plugins-for-aws" + assert source_config.repo_ref == "78df9399a81cfd95265cab53815f54210b1d7f50" + + def test_workspace_path_from_source_json(self, tmp_path, monkeypatch): + """Test that workspace_path is resolved from source.json when not provided via CLI.""" + monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") + + config_dir = tmp_path / "config" + config_dir.mkdir(parents=True, exist_ok=True) + source_dir = tmp_path / "source" + source_dir.mkdir(parents=True, exist_ok=True) + + # Create source.json with workspace-path + source_data = { + "repo": "https://github.com/test/repo", + "repo-ref": "main", + "workspace-path": "workspaces/todo" + } + (config_dir / "source.json").write_text(json.dumps(source_data)) + + config = PluginFactoryConfig( + rhdh_cli_version="1.7.2", + config_dir=str(config_dir), + repo_path=str(source_dir), + workspace_path="", # Not provided + ) + + source_config = config.discover_source_config() + + assert source_config is not None + assert source_config.workspace_path == "workspaces/todo" From 814980888fcca694008e918e5f26c463c5706f3b Mon Sep 17 00:00:00 2001 From: Frank Kong Date: Tue, 10 Mar 2026 18:20:00 -0400 Subject: [PATCH 07/35] feat: add multi-workspace support Assisted-By: Cursor Signed-off-by: Frank Kong --- .gitignore | 5 +- README.md | 137 ++- default.env | 3 - .../example-config-multi-workspace/README.md | 97 ++ .../aws-ecs/backstage.json | 3 + .../patches/1-avoid-double-wildcards.patch | 11 + .../aws-ecs/plugins-list.yaml | 2 + .../aws-ecs/source.json | 1 + .../todo/plugins-list.yaml | 2 + .../plugins/todo/overlay/scalprum-config.json | 7 + .../todo/source.json | 1 + examples/example-config-todo/README.md | 1 - src/rhdh_dynamic_plugin_factory/__init__.py | 10 + src/rhdh_dynamic_plugin_factory/cli.py | 247 +++- src/rhdh_dynamic_plugin_factory/config.py | 351 ++++-- src/rhdh_dynamic_plugin_factory/utils.py | 63 +- tests/conftest.py | 54 +- tests/test_config_load_from_env.py | 62 - tests/test_config_patches_and_overlays.py | 5 +- tests/test_multi_workspace.py | 1073 +++++++++++++++++ tests/test_source_config.py | 273 ++--- 21 files changed, 2035 insertions(+), 373 deletions(-) create mode 100644 examples/example-config-multi-workspace/README.md create mode 100644 examples/example-config-multi-workspace/aws-ecs/backstage.json create mode 100644 examples/example-config-multi-workspace/aws-ecs/patches/1-avoid-double-wildcards.patch create mode 100644 examples/example-config-multi-workspace/aws-ecs/plugins-list.yaml create mode 100644 examples/example-config-multi-workspace/aws-ecs/source.json create mode 100644 examples/example-config-multi-workspace/todo/plugins-list.yaml create mode 100644 examples/example-config-multi-workspace/todo/plugins/todo/overlay/scalprum-config.json create mode 100644 examples/example-config-multi-workspace/todo/source.json create mode 100644 tests/test_multi_workspace.py diff --git a/.gitignore b/.gitignore index 33f1ff4..e2066f9 100644 --- a/.gitignore +++ b/.gitignore @@ -16,4 +16,7 @@ outputs/ **/.coverage.* # Logs -**/*.log \ No newline at end of file +**/*.log + +# Local .env files +**/.env \ No newline at end of file diff --git a/README.md b/README.md index 337e41b..fa55fb9 100644 --- a/README.md +++ b/README.md @@ -17,9 +17,11 @@ A comprehensive tool for building and exporting dynamic plugins for Red Hat Deve - [Running Locally Without Containers](#running-locally-without-containers) - [Configuration](#configuration) - [Directory Structure](#directory-structure) + - [Single-Workspace Layout](#single-workspace-layout) + - [Multi-Workspace Layout](#multi-workspace-layout) - [Configuration Files](#configuration-files) - [1. `default.env` (Provided)](#1-defaultenv-provided) - - [2. `config/source.json` (Required for remote repositories)](#2-configsourcejson-required-for-remote-repositories) + - [2. `config/source.json` (Required for remote repositories, unless using `--source-repo`)](#2-configsourcejson-required-for-remote-repositories-unless-using---source-repo) - [3. `config/plugins-list.yaml` (Required)](#3-configplugins-listyaml-required) - [4. `config/.env` (Optional)](#4-configenv-optional) - [Patches and Overlays](#patches-and-overlays) @@ -34,8 +36,14 @@ A comprehensive tool for building and exporting dynamic plugins for Red Hat Deve - [Build plugins and save outputs locally](#build-plugins-and-save-outputs-locally) - [Build and push to registry](#build-and-push-to-registry) - [Using a local repository (skip cloning)](#using-a-local-repository-skip-cloning) + - [Multi-Workspace Mode](#multi-workspace-mode) + - [How It Works](#how-it-works) + - [Multi-Workspace CLI Restrictions](#multi-workspace-cli-restrictions) + - [Multi-Workspace Example](#multi-workspace-example) - [Output](#output) - [Build Artifacts](#build-artifacts) + - [Single Workspace Mode Outputs](#single-workspace-mode-outputs) + - [Multi-Workspace Mode Outputs](#multi-workspace-mode-outputs) - [Container Images](#container-images) - [Examples](#examples) - [Quick Example: TODO Workspace](#quick-example-todo-workspace) @@ -52,6 +60,7 @@ A comprehensive tool for building and exporting dynamic plugins for Red Hat Deve The RHDH Plugin Factory automates the process of converting Backstage plugins into RHDH dynamic plugins. It provides: - **Source Repository Management**: Clone and checkout plugin source repositories +- **Multi-Workspace Support**: Export plugins from multiple workspaces across different repositories in a single run - **Patch & Overlay System**: Apply custom modifications to plugin source code before exporting - **Dependency Management**: Automated yarn installation with TypeScript compilation - **Dynamic Plugin Packaging**: Build, export and package plugins using the RHDH CLI @@ -162,18 +171,48 @@ For local execution without containers, see [CONTRIBUTING.md](./CONTRIBUTING.md) ### Directory Structure -The factory expects the following directory structure: +The factory supports two directory layouts depending on whether you are building plugins from a single workspace or multiple workspaces. + +#### Single-Workspace Layout ```bash ./ -├── config/ # Configuration directory (Can be set with --config-dir) -│ ├── .env # Optional (if not pushing): Override environment variables + provide registry credentials -│ ├── source.json # Source repository configuration +├── config/ # Configuration directory (--config-dir) +│ ├── .env # Optional: Override environment variables + registry credentials +│ ├── source.json # Source repository configuration (not needed with --source-repo) │ ├── plugins-list.yaml # List of plugins to build │ ├── patches/ # Optional: Patch files to apply │ └── /overlays/ # Optional: Files to overlay on plugin source -├── source/ # Source code location (Can be set with --repo-path) -└── outputs/ # Build output directory (Can be set with --output-dir) +├── source/ # Source code location (--repo-path) +└── outputs/ # Build output directory (--output-dir) +``` + +#### Multi-Workspace Layout + +When the config directory contains subdirectories with `source.json` files, the factory enters multi-workspace mode. Each subdirectory represents an independent workspace with similar directory layout as a single workspace directory: + +```bash +./ +├── config/ # Configuration directory (--config-dir) +│ ├── .env # Optional: Root env, inherited by all workspaces +│ ├── todo/ # Workspace "todo" +│ │ ├── source.json # Required: repo, repo-ref, workspace-path +│ │ ├── plugins-list.yaml # Plugins to build for this workspace +│ │ ├── .env # Optional: Workspace-specific env overrides +│ │ └── patches/ # Optional: Patches for this workspace +│ └── aws-ecs/ # Workspace "aws-ecs" +│ ├── source.json +│ ├── plugins-list.yaml +│ └── patches/ +├── source/ # Source code location (--repo-path) +│ ├── .clones/ # Bare clones (one per unique repo URL) +│ │ ├── backstage-plugins-for-aws/ +│ │ └── community-plugins/ +│ ├── todo/ # Worktree for "todo" workspace +│ └── aws-ecs/ # Worktree for "aws-ecs" workspace +└── outputs/ # Build output directory (--output-dir) + ├── todo/ # Outputs for "todo" workspace + └── aws-ecs/ # Outputs for "aws-ecs" workspace ``` Note: `source/` in this case refers to the default source code location if not provided by `--repo-path` and is not to be mistaken with the workspace containing the plugins to export. Refer to [Key Terminology](#--workspace-path-vs---repo-path) for more details. @@ -243,15 +282,8 @@ REGISTRY_PASSWORD=your_password REGISTRY_NAMESPACE=your_namespace REGISTRY_INSECURE=false -# Logging -LOG_LEVEL=DEBUG -WORKSPACE_PATH= ``` -`LOG_LEVEL` can be set to one of `DEBUG`, `INFO` (default), `WARN`, `ERROR`, or `CRITICAL` - -`WORKSPACE_PATH` can be set in lieu of the `--workspace-path` argument - Alternatively, you can pass the `.env` file directly through podman using the `--env-file` argument instead of placing a `.env` file in the config directory: ```bash @@ -321,9 +353,9 @@ See the [TODO plugin example config](./examples/example-config-todo/README.md) a | `--verbose` | `false` | Show verbose output with file and line numbers | | `--clean` | `false` | Automatically removes content of `--repo-path` directory when cloning from `source.json`. Ignored if `--use-local` is used. | -**Workspace path resolution:** The workspace path can be provided via the `--workspace-path` CLI argument, the `WORKSPACE_PATH` environment variable, or the `workspace-path` field in `source.json`. The CLI argument takes highest precedence, followed by the environment variable, then `source.json`. +**Workspace path resolution:** In single-workspace use cases, the workspace path can be provided via the `--workspace-path` CLI argument, or the `workspace-path` field in `source.json`. The CLI argument takes highest precedence, followed by the `source.json`. For the multi-workspace case, only the `workspace-path` field in `source.json` is supported. -**Using `--source-repo` instead of `source.json`:** For single-workspace use cases, you can skip creating a `source.json` file entirely by using `--source-repo` (and optionally `--source-ref`) on the command line. When `--source-repo` is provided, `source.json` is ignored even if present. If `--source-ref` is omitted, the repository's default branch is used. +**Using `--source-repo` instead of `source.json`:** For single-workspace use cases only, you can skip creating a `source.json` file entirely by using `--source-repo` (and optionally `--source-ref`) on the command line. When `--source-repo` is provided, `source.json` is ignored even if present. If `--source-ref` is omitted, the repository's default branch is used. ### Understanding Volume Mounts @@ -437,10 +469,58 @@ podman run --rm -it \ **Note:** When using `--use-local`, patches and overlays will still be applied to your local repository. Make sure you have backups or are using version control. +### Multi-Workspace Mode + +When the config directory contains subdirectories with `source.json` files, the factory automatically enters multi-workspace mode. This allows you to build plugins from multiple workspaces across different (or the same) repositories in a single run. Each workspace will have the same directory layout as a normal single workspace config directory. + +#### How It Works + +1. **Workspace Discovery**: The factory scans `--config-dir` for subdirectories containing `source.json`. Directories without `source.json` are ignored. +2. **Repository Cloning**: Workspaces sharing the same repository URL share a single bare clone. Each workspace gets its own isolated [git worktree](https://git-scm.com/docs/git-worktree) at its specified ref which are stored in the `--repo-path` directory. +3. **Environment Isolation**: Each workspace's `.env` file is loaded independently. The order in which environmental variables are loaded is as follows: `default.env` -> root `config/.env` -> workspace `.env`, with full env isolation between workspaces. +4. **Error Collection**: A failure in one workspace does not stop processing of other workspaces. Errors are collected and reported in a summary at the end. + +#### Multi-Workspace CLI Restrictions + +The following CLI arguments are **not allowed** in multi-workspace mode since each workspace defines its own source configuration: + +- `--source-repo` +- `--source-ref` +- `--workspace-path` + +#### Multi-Workspace Example + +Create workspace subdirectories under your config directory: + +```bash +config/ +├── .env # Shared env (e.g., registry credentials) +├── todo/ +│ ├── source.json # {"repo": "https://github.com/backstage/community-plugins", "repo-ref": "main", "workspace-path": "workspaces/todo"} +│ └── plugins-list.yaml +└── gitlab/ + ├── source.json # {"repo": "https://github.com/immobiliare/backstage-plugin-gitlab", "repo-ref": "main", "workspace-path": "."} + ├── plugins-list.yaml + └── .env # Optional workspace level environmental variable overrides +``` + +```bash +podman run --rm -it \ + --device /dev/fuse \ + -v ./config:/config:z \ + -v ./source:/source:z \ + -v ./outputs:/outputs:z \ + quay.io/rhdh-community/dynamic-plugins-factory:latest +``` + +The factory will process each workspace sequentially, creating worktrees under `/source/` and outputs under `/outputs/`. + ## Output ### Build Artifacts +#### Single Workspace Mode Outputs + The factory also produces the following outputs in the directory specified by `--output-dir`: ```bash @@ -450,6 +530,24 @@ outputs/ └── ... ``` +#### Multi-Workspace Mode Outputs + +In multi-workspace mode, the `--output-dir` directory will be partitioned into separate subdirectories, one for each workspace: + +```bash +outputs +├── aws-ecs +│ ├── aws-amazon-ecs-plugin-for-backstage-backend-dynamic-0.9.0.tgz +│ ├── aws-amazon-ecs-plugin-for-backstage-backend-dynamic-0.9.0.tgz.integrity +│ ├── aws-amazon-ecs-plugin-for-backstage-dynamic-0.6.2.tgz +│ └── aws-amazon-ecs-plugin-for-backstage-dynamic-0.6.2.tgz.integrity +└── todo + ├── backstage-community-plugin-todo-backend-dynamic-0.15.0.tgz + ├── backstage-community-plugin-todo-backend-dynamic-0.15.0.tgz.integrity + ├── backstage-community-plugin-todo-dynamic-0.14.0.tgz + └── backstage-community-plugin-todo-dynamic-0.14.0.tgz.integrity +``` + ### Container Images When `--push-images` is enabled, images are tagged as: @@ -466,9 +564,10 @@ The `examples` directory contains ready-to-use configuration examples demonstrat | Example | Description | Details | |---------|-------------|---------| -| **TODO** | Basic workspace with custom scalprum-config | [View README](./examples/example-config-todo/) | -| **GitLab** | Overlays for non Backstage Community Plugins workspace format | [View README](./examples/example-config-gitlab/) | -| **AWS ECS** | Patches and embed packages in plugins-list.yaml | [View README](./examples/example-config-aws-ecs/) | +| **TODO** | Basic single-workspace with custom scalprum-config | [View README](./examples/example-config-todo/README.md) | +| **GitLab** | Overlays for non Backstage Community Plugins workspace format | [View README](./examples/example-config-gitlab/README.md) | +| **AWS ECS** | Patches and embed packages in plugins-list.yaml | [View README](./examples/example-config-aws-ecs/README.md) | +| **Multi-Workspace** | Multiple workspaces from different repos in a single run | [View README](./examples/example-config-multi-workspace/README.md) | ### Quick Example: TODO Workspace diff --git a/default.env b/default.env index e33100e..b37e208 100644 --- a/default.env +++ b/default.env @@ -4,9 +4,6 @@ # Tooling versions RHDH_CLI_VERSION="1.10.0" -# Logging -LOG_LEVEL="INFO" - # Registry configuration (required for push operations) # Set these in your .env file or as environment variables # REGISTRY_URL="quay.io" diff --git a/examples/example-config-multi-workspace/README.md b/examples/example-config-multi-workspace/README.md new file mode 100644 index 0000000..4800dc3 --- /dev/null +++ b/examples/example-config-multi-workspace/README.md @@ -0,0 +1,97 @@ +# Multi-Workspace Example + +This directory contains various example plugins to export in multi-workspace mode, where multiple workspaces are built in a single run. + +## Directory Structure + +```bash +example-oconfig-multi-workspace/ +├── todo/ # Workspace: TODO plugins from community-plugins +│ ├── source.json # repo, repo-ref, workspace-path for this workspace +│ └── plugins-list.yaml # Plugins to build from the TODO workspace +└── aws-ecs/ # Workspace: AWS ECS plugins + ├── source.json # repo, repo-ref, workspace-path for this workspace + └── plugins-list.yaml # Plugins to build from the AWS ECS workspace +``` + +Each subdirectory containing a `source.json` file is treated as an independent workspace. Directories without `source.json` are ignored. + +## How It Works + +1. The factory discovers workspace subdirectories by scanning for `source.json` files. +2. Workspaces sharing the same repository URL share a single bare clone (via `git worktree`). +3. Each workspace is processed sequentially -- failures in one workspace do not stop others. +4. Outputs are written to per-workspace subdirectories under `--output-dir`. + +## Quick Start + +### Using Containers + +```bash +podman run --rm -it \ + --device /dev/fuse \ + -v ./examples/example-multi-workspace:/config:z \ + -v ./source:/source:z \ + -v ./outputs:/outputs:z \ + quay.io/rhdh-community/dynamic-plugins-factory:latest +``` + +Note: `--workspace-path`, `--source-repo`, and `--source-ref` cannot be used. Instead each workspace's `source.json` provides these values. + +### Running Locally + +```bash +python src/rhdh_dynamic_plugin_factory \ + --config-dir ./examples/example-multi-workspace \ + --repo-path ./source \ + --output-dir ./outputs +``` + +## Source Repository Structure + +When cloning the remote repositories into the directory defined by `--repo-path`, it will be partitioned into separate git worktrees for each workspace alongside a `.clones` repository containing a bare clone of the source repositories of each workspace. + +```bash +source/ +├── .clones/ +│ ├── backstage-plugins-for-aws +│ ├── community-plugins +├── todo/ +│ └── ... +└── aws-ecs/ + └── ... +``` + +## Output Structure + +After running, the output directory will contain per-workspace subdirectories: + +```bash +outputs/ +├── todo/ +│ ├── backstage-community-plugin-todo-dynamic-X.Y.Z.tgz +│ ├── backstage-community-plugin-todo-backend-dynamic-X.Y.Z.tgz +│ └── ... +└── aws-ecs/ + ├── aws-ecs-plugin-frontend-dynamic-X.Y.Z.tgz + ├── aws-ecs-plugin-backend-dynamic-X.Y.Z.tgz + └── ... +``` + +## Environment Variables + +You can add a root `.env` file to configure shared settings (e.g., registry credentials) inherited by all workspaces. Each workspace can also have its own `.env` to override specific values: + +```bash +example-config-multi-workspace/ +├── .env # Root: shared by all workspaces +├── todo/ +│ ├── .env # Optional: overrides for todo workspace +│ ├── source.json +│ └── plugins-list.yaml +└── aws-ecs/ + ├── source.json + └── plugins-list.yaml +``` + +The environmental variable priority is defined as follows from lowest priority to highest: `default.env` -> root `.env` -> workspace `.env`. diff --git a/examples/example-config-multi-workspace/aws-ecs/backstage.json b/examples/example-config-multi-workspace/aws-ecs/backstage.json new file mode 100644 index 0000000..655d5ff --- /dev/null +++ b/examples/example-config-multi-workspace/aws-ecs/backstage.json @@ -0,0 +1,3 @@ +{ + "version": "1.45.3" +} diff --git a/examples/example-config-multi-workspace/aws-ecs/patches/1-avoid-double-wildcards.patch b/examples/example-config-multi-workspace/aws-ecs/patches/1-avoid-double-wildcards.patch new file mode 100644 index 0000000..9e76ada --- /dev/null +++ b/examples/example-config-multi-workspace/aws-ecs/patches/1-avoid-double-wildcards.patch @@ -0,0 +1,11 @@ +--- a/package.json ++++ b/package.json +@@ -26,7 +26,7 @@ + "workspaces": { + "packages": [ + "packages/*", +- "plugins/**" ++ "plugins/*/*" + ] + }, + "devDependencies": { diff --git a/examples/example-config-multi-workspace/aws-ecs/plugins-list.yaml b/examples/example-config-multi-workspace/aws-ecs/plugins-list.yaml new file mode 100644 index 0000000..75bfa23 --- /dev/null +++ b/examples/example-config-multi-workspace/aws-ecs/plugins-list.yaml @@ -0,0 +1,2 @@ +plugins/ecs/frontend: +plugins/ecs/backend: --embed-package @aws/aws-core-plugin-for-backstage-common --embed-package @aws/aws-core-plugin-for-backstage-node diff --git a/examples/example-config-multi-workspace/aws-ecs/source.json b/examples/example-config-multi-workspace/aws-ecs/source.json new file mode 100644 index 0000000..077cf6e --- /dev/null +++ b/examples/example-config-multi-workspace/aws-ecs/source.json @@ -0,0 +1 @@ +{"repo":"https://github.com/awslabs/backstage-plugins-for-aws","repo-ref":"3a5efbf212a029ecbe6990b1b5b7ea1709dd7c2d","workspace-path":"."} diff --git a/examples/example-config-multi-workspace/todo/plugins-list.yaml b/examples/example-config-multi-workspace/todo/plugins-list.yaml new file mode 100644 index 0000000..918c0d2 --- /dev/null +++ b/examples/example-config-multi-workspace/todo/plugins-list.yaml @@ -0,0 +1,2 @@ +plugins/todo: +plugins/todo-backend: \ No newline at end of file diff --git a/examples/example-config-multi-workspace/todo/plugins/todo/overlay/scalprum-config.json b/examples/example-config-multi-workspace/todo/plugins/todo/overlay/scalprum-config.json new file mode 100644 index 0000000..ec343a0 --- /dev/null +++ b/examples/example-config-multi-workspace/todo/plugins/todo/overlay/scalprum-config.json @@ -0,0 +1,7 @@ +{ + "name": "backstage-community.plugin-todo", + "exposedModules": { + "PluginRoot": "./src/index.ts", + "alpha": "./src/alpha.ts" + } +} diff --git a/examples/example-config-multi-workspace/todo/source.json b/examples/example-config-multi-workspace/todo/source.json new file mode 100644 index 0000000..120e53e --- /dev/null +++ b/examples/example-config-multi-workspace/todo/source.json @@ -0,0 +1 @@ +{"repo":"https://github.com/backstage/community-plugins","repo-ref":"3fadf6b0595d1d1804f5c7f2690f4db7b81275f1","workspace-path":"workspaces/todo"} \ No newline at end of file diff --git a/examples/example-config-todo/README.md b/examples/example-config-todo/README.md index 48166b9..82c689f 100644 --- a/examples/example-config-todo/README.md +++ b/examples/example-config-todo/README.md @@ -69,7 +69,6 @@ From the repository root, run: python -m src.rhdh_dynamic_plugin_factory \ --config-dir ./examples/example-config-todo \ --repo-path ./source \ - --workspace-path workspaces/todo \ --output-dir ./outputs ``` diff --git a/src/rhdh_dynamic_plugin_factory/__init__.py b/src/rhdh_dynamic_plugin_factory/__init__.py index e125e82..6749450 100644 --- a/src/rhdh_dynamic_plugin_factory/__init__.py +++ b/src/rhdh_dynamic_plugin_factory/__init__.py @@ -11,7 +11,10 @@ from .config import ( PluginFactoryConfig, SourceConfig, + WorkspaceInfo, PluginListConfig, + discover_workspaces, + clone_workspaces_with_worktrees, ) from .exceptions import ( PluginFactoryError, @@ -27,6 +30,8 @@ run_command_with_streaming, display_export_results, clean_directory, + prompt_or_clean_directory, + repo_dir_name, ) __all__ = [ @@ -37,7 +42,10 @@ # Configuration "PluginFactoryConfig", "SourceConfig", + "WorkspaceInfo", "PluginListConfig", + "discover_workspaces", + "clone_workspaces_with_worktrees", # Exceptions "PluginFactoryError", @@ -53,6 +61,8 @@ "run_command_with_streaming", "display_export_results", "clean_directory", + "prompt_or_clean_directory", + "repo_dir_name", # Version "__version__", ] diff --git a/src/rhdh_dynamic_plugin_factory/cli.py b/src/rhdh_dynamic_plugin_factory/cli.py index 0b1cf00..c47a8d2 100644 --- a/src/rhdh_dynamic_plugin_factory/cli.py +++ b/src/rhdh_dynamic_plugin_factory/cli.py @@ -11,16 +11,16 @@ try: from .__version__ import __version__ from .logger import setup_logging, get_logger - from .config import PluginFactoryConfig - from .utils import run_command_with_streaming + from .config import PluginFactoryConfig, WorkspaceInfo, discover_workspaces, clone_workspaces_with_worktrees + from .utils import run_command_with_streaming, prompt_or_clean_directory from .exceptions import PluginFactoryError, ConfigurationError, ExecutionError except ImportError: # For direct script execution, add parent directory to path sys.path.insert(0, str(Path(__file__).parent.parent)) from rhdh_dynamic_plugin_factory.__version__ import __version__ from rhdh_dynamic_plugin_factory.logger import setup_logging, get_logger - from rhdh_dynamic_plugin_factory.config import PluginFactoryConfig - from rhdh_dynamic_plugin_factory.utils import run_command_with_streaming + from rhdh_dynamic_plugin_factory.config import PluginFactoryConfig, WorkspaceInfo, discover_workspaces, clone_workspaces_with_worktrees + from rhdh_dynamic_plugin_factory.utils import run_command_with_streaming, prompt_or_clean_directory from rhdh_dynamic_plugin_factory.exceptions import PluginFactoryError, ConfigurationError, ExecutionError logger = get_logger("cli") @@ -34,9 +34,9 @@ def create_parser() -> argparse.ArgumentParser: Examples: # Build plugins for the todo workspace in backstage/community-plugins w/o pushing to a registry # This assumes that ./config is populated with the source.json and plugins-list.yaml files - python src/rhdh_dynamic_plugin_factory --config-dir ./config --repo-path ./source --workspace-path workspaces/todo --log-level DEBUG --output-dir ./outputs + python src/rhdh_dynamic_plugin_factory --config-dir ./config --repo-path ./source --log-level DEBUG --output-dir ./outputs - # Build plugins using CLI args instead of source.json + # Build a single workspace of plugins using CLI args instead of source.json python src/rhdh_dynamic_plugin_factory --source-repo https://github.com/backstage/community-plugins --source-ref main --workspace-path workspaces/todo --config-dir ./config --repo-path ./source --output-dir ./outputs """ ) @@ -133,7 +133,7 @@ def install_dependencies(workspace_path: Path) -> None: env['COREPACK_ENABLE_DOWNLOAD_PROMPT'] = '0' # Disable download prompts for cmd, description in commands: - logger.info(f"[cyan]{description}...[/cyan]") + logger.info(f"[cyan]{description}[/cyan]") returncode = run_command_with_streaming( cmd, @@ -149,7 +149,7 @@ def install_dependencies(workspace_path: Path) -> None: returncode=returncode ) - logger.info(f"[green]✓ {description} completed[/green]") + logger.info(f"[green]{description} completed successfully[/green]") except ExecutionError: raise except Exception as e: @@ -158,15 +158,213 @@ def install_dependencies(workspace_path: Path) -> None: step=STEP_NAME, ) from e +def _process_workspace( + config: PluginFactoryConfig, + workspace_config_dir: str, + repo_path: str, + workspace_path: str, + output_dir: str, +) -> None: + """Execute the plugin factory pipeline for a single workspace. + + Args: + config: Global factory configuration. + workspace_config_dir: Config directory for this workspace (patches, overlays, plugins-list). + repo_path: Path to the repository checkout for this workspace. + workspace_path: Relative path from repo_path to the workspace. + output_dir: Output directory for build artifacts. + """ + config.auto_generate_plugins_list( + config_dir=workspace_config_dir, + repo_path=repo_path, + workspace_path=workspace_path, + ) + + logger.info("[bold blue]Applying Patches and Overlays[/bold blue]") + config.apply_patches_and_overlays( + config_dir=workspace_config_dir, + repo_path=repo_path, + workspace_path=workspace_path, + ) + + logger.info("[bold blue]Installing Dependencies[/bold blue]") + full_workspace_path = Path(repo_path).joinpath(workspace_path).absolute() + install_dependencies(full_workspace_path) + + logger.info("[bold blue]Exporting plugins using RHDH CLI[/bold blue]") + config.export_plugins( + output_dir=output_dir, + config_dir=workspace_config_dir, + repo_path=repo_path, + workspace_path=workspace_path, + ) + + +def _load_env_for_workspace( + base_env: dict[str, str], + workspace_env_path: Path, +) -> None: + """Apply workspace-specific .env overrides on top of the base environment. + + Precedence (highest to lowest): + workspace .env > root .env > Podman/system env vars > default.env + + base_env already contains Podman + default.env + root .env (captured once + after load_from_env in _run_multi_workspace). This function restores that + baseline and layers only the workspace-specific .env on top. + """ + from dotenv import load_dotenv + + os.environ.clear() + os.environ.update(base_env) + + if workspace_env_path.exists(): + load_dotenv(workspace_env_path, override=True) + + def _run(args: argparse.Namespace) -> None: """Execute the main plugin factory workflow. - All steps either succeed silently or raise a PluginFactoryError subclass, + Detects multi-workspace vs single-workspace mode and dispatches accordingly. + All steps either succeed silently or raise an exception, which is caught by the centralized handler in main(). """ - logger.info("[bold blue]Setting up configuration directory[/bold blue]") + config_dir = Path(str(args.config_dir)) + + workspaces = discover_workspaces(config_dir) + + if workspaces: + _run_multi_workspace(args, workspaces) + else: + _run_single_workspace(args) - config = PluginFactoryConfig.load_from_env(args=args, env_file=args.config_dir / ".env", push_images=args.push_images) + +def _run_multi_workspace(args: argparse.Namespace, workspaces: list[WorkspaceInfo]) -> None: + """Execute multi-workspace mode.""" + # Reject single-workspace-only CLI args + if getattr(args, 'source_repo', None): + raise ConfigurationError( + "--source-repo cannot be used in multi-workspace mode. " + "Each workspace must define its source in its own source.json." + ) + if getattr(args, 'source_ref', None): + raise ConfigurationError( + "--source-ref cannot be used in multi-workspace mode. " + "Each workspace must define its source in its own source.json." + ) + if getattr(args, 'workspace_path', None): + raise ConfigurationError( + "--workspace-path cannot be used in multi-workspace mode. " + "Each workspace defines its workspace-path in its own source.json." + ) + + config_dir = Path(str(args.config_dir)) + base_repo_path = Path(str(args.repo_path)) + base_output_dir = Path(str(args.output_dir)) + + logger.info(f"[bold blue]Multi-workspace mode: discovered {len(workspaces)} workspace(s)[/bold blue]") + + # Warn about any root-level content that is not a workspace or the root .env + workspace_names = {ws.name for ws in workspaces} + ignored_items: list[str] = [] + for entry in sorted(config_dir.iterdir()): + if entry.name == ".env": + continue + if entry.name in workspace_names: + continue + suffix = "directory — not a workspace, missing source.json" if entry.is_dir() else "file" + ignored_items.append(f" - {entry.name}{'/' if entry.is_dir() else ''} ({suffix})") + if ignored_items: + items_str = "\n".join(ignored_items) + logger.warning( + f"[yellow]The following items in the config directory will be ignored in multi-workspace mode\n" + f"and should be moved into a workspace subdirectory or removed:\n" + f"{items_str}[/yellow]" + ) + + logger.info("[bold blue]Workspaces to be processed:[/bold blue]") + for ws in workspaces: + logger.info(f" - {ws.name}: {ws.source_config.repo} @ {ws.source_config.repo_ref}") + # Resolve per-workspace source and output paths + ws.resolve_paths(base_repo_path, base_output_dir) + + # Load global config (uses root .env for global settings like registry credentials) + config = PluginFactoryConfig.load_from_env( + args=args, + env_file=config_dir / ".env", + push_images=args.push_images, + multi_workspace=True, + ) + + # base_env now contains Podman + default.env + root .env — everything that is + # constant across workspaces. Per-workspace loop only layers workspace .env on top. + base_env = dict(os.environ) + + # Clone repositories / create worktrees (unless --use-local) + if not config.use_local: + logger.info("[bold blue]Setting up repositories with git worktrees[/bold blue]") + prompt_or_clean_directory(base_repo_path, args.clean, logger) + clone_workspaces_with_worktrees(workspaces, base_repo_path) + else: + logger.info("[bold blue]--use-local flag is set, expecting repositories pre-placed[/bold blue]") + for ws in workspaces: + if ws.repo_path and not ws.repo_path.exists(): + raise ConfigurationError( + f"Local repository for workspace '{ws.name}' not found at {ws.repo_path}. " + f"When using --use-local in multi-workspace mode, place repos at //." + ) + + errors: list[tuple[str, Exception]] = [] + successes: list[str] = [] + + for ws in workspaces: + logger.info(f"\n[bold blue]{'=' * 60}[/bold blue]") + logger.info(f"[bold blue]Processing workspace: {ws.name}[/bold blue]") + logger.info(f"[bold blue]{'=' * 60}[/bold blue]") + + # Restore base env and layer workspace-specific .env on top + _load_env_for_workspace(base_env, ws.config_dir / ".env") + config.refresh_registry_config() + + try: + _process_workspace( + config=config, + workspace_config_dir=str(ws.config_dir), + repo_path=str(ws.repo_path), + workspace_path=ws.source_config.workspace_path, + output_dir=str(ws.output_dir), + ) + successes.append(ws.name) + logger.info(f"[green]Workspace '{ws.name}' export completed successfully[/green]") + except PluginFactoryError as e: + errors.append((ws.name, e)) + logger.error(f"[red]Workspace '{ws.name}' export failed: {e}[/red]") + + # Report summary + logger.info(f"\n[bold blue]{'=' * 60}[/bold blue]") + logger.info("[bold blue]Multi-workspace Summary[/bold blue]") + logger.info(f"[bold blue]{'=' * 60}[/bold blue]") + logger.info(f" Total: {len(workspaces)} | Succeeded: {len(successes)} | Failed: {len(errors)}") + + for name in successes: + logger.info(f" [green]{name} completed successfully[/green]") + for name, error in errors: + logger.error(f" [red]{name} failed: {error}[/red]") + + if errors: + raise ExecutionError( + f"{len(errors)} of {len(workspaces)} workspace(s) failed", + step="multi-workspace processing", + ) + + +def _run_single_workspace(args: argparse.Namespace) -> None: + """Execute single-workspace mode""" + config = PluginFactoryConfig.load_from_env( + args=args, + env_file=args.config_dir / ".env", + push_images=args.push_images, + ) source_config = config.setup_config_directory() @@ -177,8 +375,7 @@ def _run(args: argparse.Namespace) -> None: # Validate workspace_path is set (may come from CLI, env var, or source.json) if not config.workspace_path: raise ConfigurationError( - "WORKSPACE_PATH must be set via --workspace-path argument, " - "WORKSPACE_PATH environment variable, or source.json workspace-path field" + "workspace-path must be set via --workspace-path argument or source.json workspace-path field" ) if source_config and not config.use_local: @@ -189,7 +386,8 @@ def _run(args: argparse.Namespace) -> None: logger.info("[bold blue]--use-local flag is set, using local repository[/bold blue]") else: logger.info("[bold blue]No source configuration found, using local repository[/bold blue]") - if not config.repo_path.exists(): + repo_path = Path(str(config.repo_path)) + if not repo_path.exists(): raise ConfigurationError( f"Local repository does not exist at: {config.repo_path}. " "Either provide source.json to clone the repository, " @@ -198,18 +396,13 @@ def _run(args: argparse.Namespace) -> None: ) logger.info(f"Using local repository at: {config.repo_path}") - # Auto-generate plugins-list.yaml if needed (after repository is available) - config.auto_generate_plugins_list() - - logger.info("[bold blue]Applying Patches and Overlays[/bold blue]") - config.apply_patches_and_overlays() - - logger.info("[bold blue]Installing Dependencies[/bold blue]") - workspace_path = config.repo_path.joinpath(config.workspace_path).absolute() - install_dependencies(workspace_path) - - logger.info("[bold blue]Exporting plugins using RHDH CLI[/bold blue]") - config.export_plugins(args.output_dir) + _process_workspace( + config=config, + workspace_config_dir=str(config.config_dir), + repo_path=str(config.repo_path), + workspace_path=str(config.workspace_path), + output_dir=str(args.output_dir), + ) def main(): @@ -231,7 +424,7 @@ def main(): logger.error(f"[red]{e}[/red]") sys.exit(1) - logger.info("[green]✓ All operations completed successfully[/green]") + logger.info("[green]All operations completed successfully[/green]") if __name__ == "__main__": diff --git a/src/rhdh_dynamic_plugin_factory/config.py b/src/rhdh_dynamic_plugin_factory/config.py index cafe07e..6cb653b 100644 --- a/src/rhdh_dynamic_plugin_factory/config.py +++ b/src/rhdh_dynamic_plugin_factory/config.py @@ -15,7 +15,7 @@ from .exceptions import PluginFactoryError, ConfigurationError, ExecutionError from .logger import get_logger -from .utils import clean_directory, run_command_with_streaming, display_export_results +from .utils import run_command_with_streaming, display_export_results, prompt_or_clean_directory, repo_dir_name @dataclass class PluginFactoryConfig: @@ -40,7 +40,6 @@ class PluginFactoryConfig: registry_namespace: Optional[str] = field(default=None) registry_insecure: bool = field(default=False) - log_level: str = field(default="INFO") use_local: bool = field(default=False) push_images: bool = field(default=False) @@ -60,20 +59,58 @@ def __post_init__(self) -> None: if self.source_ref and not self.source_repo: raise ConfigurationError("--source-ref requires --source-repo to be provided") - valid_log_levels = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] - if self.log_level.upper() not in valid_log_levels: - raise ConfigurationError(f"Invalid log level: {self.log_level}") - if self.push_images: - if not self.registry_url: - raise ConfigurationError("REGISTRY_URL environment variable is required when --push-images is enabled") - if not self.registry_namespace: - raise ConfigurationError("REGISTRY_NAMESPACE environment variable is required when --push-images is enabled") - if not self.registry_username or not self.registry_password: - raise ConfigurationError("REGISTRY_USERNAME and REGISTRY_PASSWORD environment variables are required when --push-images is enabled") + self._validate_registry_fields() + + def _validate_registry_fields(self) -> None: + """Validate that all required registry fields are present. + + Raises: + ConfigurationError: If any required registry field is missing. + """ + if not self.registry_url: + raise ConfigurationError("REGISTRY_URL environment variable is required when --push-images is enabled") + if not self.registry_namespace: + raise ConfigurationError("REGISTRY_NAMESPACE environment variable is required when --push-images is enabled") + if not self.registry_username or not self.registry_password: + raise ConfigurationError("REGISTRY_USERNAME and REGISTRY_PASSWORD environment variables are required when --push-images is enabled") + + def refresh_registry_config(self) -> None: + """Re-read registry fields from os.environ and re-login if credentials changed. + + Called per workspace in multi-workspace mode after loading workspace-specific + .env files so that each workspace can target a different registry. + + Raises: + ConfigurationError: If push_images is enabled and required registry fields are missing. + ExecutionError: If buildah login fails after credential change. + """ + new_url = os.getenv("REGISTRY_URL") + new_username = os.getenv("REGISTRY_USERNAME") + new_password = os.getenv("REGISTRY_PASSWORD") + new_namespace = os.getenv("REGISTRY_NAMESPACE") + new_insecure = os.getenv("REGISTRY_INSECURE", "false").lower() == "true" + + creds_changed = ( + new_url != self.registry_url + or new_username != self.registry_username + or new_password != self.registry_password + or new_insecure != self.registry_insecure + ) + + self.registry_url = new_url + self.registry_username = new_username + self.registry_password = new_password + self.registry_namespace = new_namespace + self.registry_insecure = new_insecure + + if self.push_images and creds_changed: + self._validate_registry_fields() + self._buildah_login() @classmethod - def load_from_env(cls, args: argparse.Namespace, env_file: Optional[Path] = None, push_images: bool = False) -> "PluginFactoryConfig": + def load_from_env(cls, args: argparse.Namespace, env_file: Optional[Path] = None, + push_images: bool = False, multi_workspace: bool = False) -> "PluginFactoryConfig": """Load configuration from environment variables and .env files. Loads default.env first, then optionally loads additional env file to override defaults or provide additional values. @@ -83,6 +120,8 @@ def load_from_env(cls, args: argparse.Namespace, env_file: Optional[Path] = None args: Parsed CLI arguments. env_file: Optional additional .env file to merge with defaults. push_images: Whether to push images to a registry (triggers registry validation and login). + multi_workspace: If True, skip root-level source.json and plugins-list.yaml + validation since each workspace manages its own. """ default_env_path = Path(__file__).parent.parent.parent / "default.env" @@ -90,11 +129,11 @@ def load_from_env(cls, args: argparse.Namespace, env_file: Optional[Path] = None if default_env_path.exists(): load_dotenv(default_env_path) - cls.logger.debug(f'[green]✓ Loaded {default_env_path}[/green]') + cls.logger.debug(f'[green]Loaded {default_env_path}[/green]') if env_file and env_file.exists(): load_dotenv(env_file, override=True) - cls.logger.debug(f'[green]✓ Loaded {env_file}[/green]') + cls.logger.debug(f'[green]Loaded {env_file}[/green]') cls.logger.debug('[bold blue]Loading configuration from environment variables and CLI arguments[/bold blue]') @@ -105,10 +144,7 @@ def load_from_env(cls, args: argparse.Namespace, env_file: Optional[Path] = None for dir_path in [config_dir, repo_path]: os.makedirs(dir_path, exist_ok=True) - # Resolve workspace_path: env var takes precedence, then CLI arg, then empty (potentially resolved later from source.json) - workspace_path = os.getenv("WORKSPACE_PATH") or "" - if not workspace_path and args.workspace_path: - workspace_path = str(args.workspace_path) + workspace_path = getattr(args, 'workspace_path', None) source_repo = getattr(args, 'source_repo', None) source_ref = getattr(args, 'source_ref', None) @@ -117,7 +153,7 @@ def load_from_env(cls, args: argparse.Namespace, env_file: Optional[Path] = None rhdh_cli_version=os.getenv("RHDH_CLI_VERSION", ""), repo_path=repo_path, config_dir=config_dir, - workspace_path=workspace_path, + workspace_path=workspace_path or "", source_repo=source_repo, source_ref=source_ref, registry_url=os.getenv("REGISTRY_URL"), @@ -125,13 +161,13 @@ def load_from_env(cls, args: argparse.Namespace, env_file: Optional[Path] = None registry_password=os.getenv("REGISTRY_PASSWORD"), registry_namespace=os.getenv("REGISTRY_NAMESPACE"), registry_insecure=os.getenv("REGISTRY_INSECURE", "false").lower() == "true", - log_level=os.getenv("LOG_LEVEL", args.log_level), use_local=args.use_local, push_images=push_images, ) - config._validate_source_json() - config._validate_plugins_list() + if not multi_workspace: + config._validate_source_json() + config._validate_plugins_list() if push_images: config._buildah_login() @@ -210,32 +246,37 @@ def _validate_plugins_list(self) -> None: else: self.logger.debug(f"Using plugins-list.yaml from: {plugins_file}") - def auto_generate_plugins_list(self) -> None: - """ - Auto-generate plugins-list.yaml if it doesn't already exist. + def auto_generate_plugins_list(self, config_dir: Optional[str] = None, + repo_path: Optional[str] = None, + workspace_path: Optional[str] = None) -> None: + """Auto-generate plugins-list.yaml if it doesn't already exist. - Assumes the following: - - The repository is cloned to the `repo_path` - - The plugins are located in the plugins/* directory - - `workspace_path` is the path to the workspace from the root of the repository + Args: + config_dir: Config directory containing plugins-list.yaml. Defaults to self.config_dir. + repo_path: Repository path. Defaults to self.repo_path. + workspace_path: Workspace path relative to repo. Defaults to self.workspace_path. Raises: PluginFactoryError: If auto-generation fails. """ - plugins_file = os.path.join(self.config_dir, "plugins-list.yaml") + config_dir = config_dir or self.config_dir + repo_path = repo_path or self.repo_path + workspace_path = workspace_path or self.workspace_path + + plugins_file = os.path.join(config_dir, "plugins-list.yaml") if os.path.exists(plugins_file): - self.logger.debug(f"[green]✓ plugins-list.yaml already exists at {plugins_file}. Skipping auto-generation.[/green]") + self.logger.debug(f"[green]plugins-list.yaml already exists at {plugins_file}. Skipping auto-generation.[/green]") return self.logger.info("[bold blue]Auto-generating plugins-list.yaml[/bold blue]") - if not os.path.exists(self.repo_path): - raise PluginFactoryError(f"Repository does not exist at {self.repo_path}") + if not os.path.exists(repo_path): + raise PluginFactoryError(f"Source code repository does not exist at {repo_path}") - workspace_full_path = os.path.abspath(os.path.join(self.repo_path, self.workspace_path)) + workspace_full_path = os.path.abspath(os.path.join(repo_path, workspace_path)) if not os.path.exists(workspace_full_path): - raise PluginFactoryError(f"Workspace does not exist at {workspace_full_path}") + raise PluginFactoryError(f"Plugin workspace does not exist at {workspace_full_path}") try: # TODO: Implement PluginListConfig.create_default function @@ -324,38 +365,51 @@ def setup_config_directory(self) -> Optional["SourceConfig"]: self.logger.warning(f"{plugins_list_file} not found, will auto-generate after repository is available") return source_config - def apply_patches_and_overlays(self) -> None: + def apply_patches_and_overlays(self, config_dir: Optional[str] = None, + repo_path: Optional[str] = None, + workspace_path: Optional[str] = None) -> None: """Apply patches and overlays using override-sources.sh script. + Args: + config_dir: Config directory containing patches/ and overlays. Defaults to self.config_dir. + repo_path: Repository root path (worktree root in multi-workspace mode). Defaults to self.repo_path. + workspace_path: Workspace path relative to repo_path. Defaults to self.workspace_path. + Raises: ExecutionError: If the patch script is not found or fails. """ + config_dir = config_dir or self.config_dir + repo_path = repo_path or self.repo_path + workspace_path = workspace_path or self.workspace_path + script_dir = Path(__file__).parent.parent.parent / "scripts" script_path = script_dir / "override-sources.sh" STEP_NAME = "apply patches and overlays" - + if not script_path.exists(): raise ExecutionError( f"Script not found: {script_path}", step=STEP_NAME ) - - workspace_full_path = os.path.abspath(os.path.join(self.repo_path, self.workspace_path)) - self.logger.debug(f"Applying patches and overlays to workspace: {workspace_full_path}") + + repo_root = os.path.abspath(repo_path) + workspace_full_path = os.path.abspath(os.path.join(repo_path, workspace_path)) + self.logger.debug(f"Applying patches at repo root: {repo_root}") + self.logger.debug(f"Applying overlays to workspace: {workspace_full_path}") cmd = [ str(script_path.absolute()), - os.path.abspath(self.config_dir), # Overlay root directory - workspace_full_path, # Target directory + os.path.abspath(config_dir), + workspace_full_path, ] try: returncode = run_command_with_streaming( cmd, self.logger, - cwd=Path(workspace_full_path), + cwd=Path(repo_root), stderr_log_func=self.logger.error ) - + if returncode == 0: self.logger.info("[green]Patches and overlays applied successfully[/green]") else: @@ -372,13 +426,25 @@ def apply_patches_and_overlays(self) -> None: step=STEP_NAME ) from e - def export_plugins(self, output_dir: str) -> None: + def export_plugins(self, output_dir: str, config_dir: Optional[str] = None, + repo_path: Optional[str] = None, + workspace_path: Optional[str] = None) -> None: """Export plugins using export-workspace.sh script. + Args: + output_dir: Directory for build artifacts. + config_dir: Config directory containing plugins-list.yaml and .env. Defaults to self.config_dir. + repo_path: Repository path. Defaults to self.repo_path. + workspace_path: Workspace path relative to repo. Defaults to self.workspace_path. + Raises: ExecutionError: If the export script is not found or fails. ConfigurationError: If no plugins list file is found. """ + config_dir = config_dir or self.config_dir + repo_path = repo_path or self.repo_path + workspace_path = workspace_path or self.workspace_path + script_dir = Path(__file__).parent.parent.parent / "scripts" script_path = script_dir / "export-workspace.sh" STEP_NAME = "export plugins" @@ -389,12 +455,12 @@ def export_plugins(self, output_dir: str) -> None: step=STEP_NAME ) - plugins_list_file = os.path.join(self.config_dir, "plugins-list.yaml") + plugins_list_file = os.path.join(config_dir, "plugins-list.yaml") if not os.path.exists(plugins_list_file): raise ConfigurationError("No plugins file found") - config_env_file = os.path.join(self.config_dir, ".env") + config_env_file = os.path.join(config_dir, ".env") default_env_file = Path(__file__).parent.parent.parent / "default.env" load_dotenv(default_env_file) env = dict(os.environ) @@ -402,7 +468,6 @@ def export_plugins(self, output_dir: str) -> None: if os.path.exists(config_env_file): self.logger.debug(f"Loading script configuration from: {config_env_file}") load_dotenv(config_env_file, override=True) - # Reload env after loading .env env = dict(os.environ) os.makedirs(output_dir, exist_ok=True) @@ -421,7 +486,7 @@ def export_plugins(self, output_dir: str) -> None: "INPUTS_CONTAINER_BUILD_TOOL": "buildah", }) - workspace_full_path = os.path.abspath(os.path.join(self.repo_path, self.workspace_path)) + workspace_full_path = os.path.abspath(os.path.join(repo_path, workspace_path)) try: def conditional_stderr_log(line: str) -> None: if "Error" in line: @@ -556,7 +621,7 @@ def resolve_default_ref(repo: str) -> str: ExecutionError: If git ls-remote fails or the default branch cannot be determined. """ logger = get_logger("source_config") - logger.info(f"[cyan]Resolving default branch for {repo}...[/cyan]") + logger.info(f"[cyan]Resolving default branch for {repo}cyan]") try: result = subprocess.run( @@ -566,27 +631,17 @@ def resolve_default_ref(repo: str) -> str: check=True, ) - # Output format: - # ref: refs/heads/main\tHEAD - # \tHEAD for line in result.stdout.splitlines(): if line.startswith("ref:"): # Ex: Extract "refs/heads/main" from "ref: refs/heads/main\tHEAD" ref_part = line.split("\t")[0].replace("ref: ", "").strip() - logger.info(f"[green]Resolved default branch: {ref_part}[/green]") + logger.info(f"[green]Resolved default branch: {ref_part} for {repo}[/green]") return ref_part - # Fallback: if no symbolic ref, use the HEAD SHA directly - for line in result.stdout.splitlines(): - parts = line.split("\t") - if len(parts) == 2 and parts[1].strip() == "HEAD": - sha = parts[0].strip() - logger.info(f"[green]Resolved default ref (SHA): {sha}[/green]") - return sha - - raise ExecutionError( - "Could not determine default branch from git ls-remote output", - step="resolve default ref", + raise ConfigurationError( + f"Could not resolve the default branch for '{repo}'. " + "Please specify a branch or ref explicitly via 'repo-ref' in source.json " + "or the --source-ref CLI argument." ) except subprocess.CalledProcessError as e: raise ExecutionError( @@ -613,20 +668,7 @@ def clone_to_path(self, repo_path: Path, clean: bool = False) -> None: self.logger.info(f"Reference: {self.repo_ref}") self.logger.info(f"Destination directory: {repo_path}") - if any(repo_path.iterdir()): - self.logger.warning(f"[yellow]Source directory {repo_path} is not empty[/yellow]") - if clean: - self.logger.warning(f"[yellow]`--clean` argument set, automatically cleaning {repo_path}[/yellow]") - clean_directory(repo_path) - else: - self.logger.warning(f"[yellow]WARNING: Are you sure you want to remove the contents of {repo_path}/? \\[y/N][/yellow]") - confirm = input() - if confirm != "y": - self.logger.warning("[yellow]Aborted[/yellow]") - raise PluginFactoryError("Repository clone aborted by user") - else: - self.logger.warning(f"[yellow]`y` selected. Cleaning {repo_path}. Note: you can use the `--clean` argument to automatically clean the directory and skip this prompt next time.[/yellow]") - clean_directory(repo_path) + prompt_or_clean_directory(repo_path, clean, self.logger) try: cmd = ["git", "clone", self.repo, str(repo_path)] @@ -671,6 +713,155 @@ def clone_to_path(self, repo_path: Path, clean: bool = False) -> None: step="git clone/checkout" ) from e +@dataclass +class WorkspaceInfo: + """Per-workspace configuration for multi-workspace mode. + + Represents a single workspace discovered from a config subdirectory. + """ + name: str + config_dir: Path + source_config: SourceConfig + repo_path: Optional[Path] = None + output_dir: Optional[Path] = None + + def resolve_paths(self, base_repo_path: Path, base_output_dir: Path) -> None: + """Set per-workspace source code repo and output paths from base directories.""" + self.repo_path = base_repo_path / self.name + self.output_dir = base_output_dir / self.name + + +def discover_workspaces(config_dir: Path) -> list["WorkspaceInfo"]: + """Scan config directory for workspace subdirectories. + + A subdirectory is considered a workspace if it contains a source.json file. + Non-workspace entries are skipped silently; the caller is responsible for + warning the user about ignored content. + + Args: + config_dir: Root configuration directory to scan. + + Returns: + List of WorkspaceInfo instances sorted by repo URL (primary) then name (secondary), + making downstream groupby(repo) trivial. + + Raises: + ConfigurationError: If a workspace's source.json is invalid. + """ + logger = get_logger("config") + workspaces: list[WorkspaceInfo] = [] + + if not config_dir.is_dir(): + return workspaces + + for entry in sorted(config_dir.iterdir()): + if not entry.is_dir(): + continue + + source_file = entry / "source.json" + if not source_file.exists(): + logger.debug(f"Skipping {entry.name}/ — no source.json") + continue + + workspace_name = entry.name + logger.debug(f"Discovered workspace: {workspace_name}") + + source_config = SourceConfig.from_file(source_file) + + workspaces.append(WorkspaceInfo( + name=workspace_name, + config_dir=entry, + source_config=source_config, + )) + + # Sort by repo URL (primary) then workspace name (secondary) + workspaces.sort(key=lambda w: (w.source_config.repo, w.name)) + + return workspaces + + +def clone_workspaces_with_worktrees( + workspaces: list["WorkspaceInfo"], + base_repo_path: Path, +) -> None: + """Clone repositories and create git worktrees for multi-workspace mode. + + Groups workspaces by repo URL, clones each unique repo once into + /.clones//, then creates a worktree per + workspace at //. + + The caller is responsible for cleaning base_repo_path before calling + this function (e.g. via prompt_or_clean_directory). This function + assumes it can write freely into base_repo_path. + + Args: + workspaces: List of WorkspaceInfo (must already have repo_path set via resolve_paths). + base_repo_path: Base directory for repo clones and worktrees. + + Raises: + PluginFactoryError: If a workspace has no repo_path resolved (internal error). + ExecutionError: If any git operation fails. + """ + from itertools import groupby + + logger = get_logger("config") + clones_dir = base_repo_path / ".clones" + os.makedirs(clones_dir, exist_ok=True) + + for repo_url, group in groupby(workspaces, key=lambda w: w.source_config.repo): + workspace_list = list(group) + repo_name = repo_dir_name(repo_url) + clone_path = clones_dir / repo_name + + logger.info(f"[bold blue]\nCloning base repository: {repo_url}[/bold blue]") + logger.info(f" Destination: {clone_path}") + + cmd = ["git", "clone", "--bare", repo_url, str(clone_path)] + returncode = run_command_with_streaming( + cmd, logger, stderr_log_func=logger.info + ) + + if returncode != 0: + raise ExecutionError( + f"Failed to clone repository '{repo_url}' (exit code {returncode}). " + f"Please verify the 'repo' URL in the source.json for workspaces using this repository. " + f"Ensure the URL is correct and accessible from your environment.", + step="git clone (bare)", + returncode=returncode, + ) + + logger.info(f"[green]Cloned {repo_url} to {clone_path}[/green]") + + for ws in workspace_list: + worktree_path = ws.repo_path + if worktree_path is None: + raise PluginFactoryError( + f"Internal error: workspace '{ws.name}' has no resolved repository path. " + f"This is a bug in the plugin factory. Please report this issue." + ) + + ref = ws.source_config.repo_ref + logger.info(f"[cyan]\nCreating git worktree for '{ws.name}' at ref {ref}[/cyan]") + + # Must use absolute path: git worktree add runs with cwd=clone_path, + # so a relative worktree_path would resolve against the clone directory. + cmd = ["git", "worktree", "add", "--detach", str(worktree_path.resolve()), ref] + returncode = run_command_with_streaming( + cmd, logger, cwd=clone_path, stderr_log_func=logger.info + ) + + if returncode != 0: + raise ExecutionError( + f"Failed to create worktree for workspace '{ws.name}' at ref '{ref}' (exit code {returncode}). " + f"Please verify the 'repo-ref' value in {ws.config_dir / 'source.json'}. " + f"Ensure the branch, tag, or commit exists in the repository.", + step="git worktree add", + returncode=returncode, + ) + + logger.info(f"[green] Worktree created for '{ws.name}' at {worktree_path}[/green]") + + class PluginListConfig: """Configuration for plugin list (YAML format).""" diff --git a/src/rhdh_dynamic_plugin_factory/utils.py b/src/rhdh_dynamic_plugin_factory/utils.py index 0e824fa..8465811 100644 --- a/src/rhdh_dynamic_plugin_factory/utils.py +++ b/src/rhdh_dynamic_plugin_factory/utils.py @@ -8,6 +8,8 @@ from pathlib import Path from typing import Optional, Callable +from .exceptions import ExecutionError, PluginFactoryError + def _stream_output(pipe, log_func: Callable[[str], None]) -> None: """ @@ -118,9 +120,62 @@ def clean_directory(directory: Path) -> None: Args: directory: Path to the directory to clean. + Raises: + ExecutionError: If the directory or files cannot be removed. """ - for item in directory.iterdir(): - if item.is_dir(): - shutil.rmtree(item) + try: + for item in directory.iterdir(): + if item.is_dir(): + shutil.rmtree(item) + else: + item.unlink() + except Exception as e: + raise ExecutionError( + f"Failed to clean directory '{directory}': {e}", + step="clean directory", + returncode=1, + ) from e + +def prompt_or_clean_directory(path: Path, clean: bool, logger) -> None: + """Clears contents of a non-empty directory by automatically or by prompting the user. + + If the directory is empty or does not exist, this is a no-op. + + Args: + path: Directory to clean + clean: If True, auto-clean without prompting. + logger: Logger instance. + + Raises: + PluginFactoryError: If the user declines to clean. + ExecutionError: If the directory or files cannot be removed. (Thrown from clean_directory) + """ + if not path.exists() or not any(path.iterdir()): + return + + logger.warning(f"[yellow]Source directory {path} is not empty[/yellow]") + if clean: + logger.warning(f"[yellow]`--clean` argument set, automatically cleaning {path}[/yellow]") + clean_directory(path) + else: + logger.warning(f"[yellow]WARNING: Are you sure you want to remove the contents of {path}/? \\[y/N][/yellow]") + confirm = input() + if confirm.lower() != "y": + logger.warning("[yellow]Aborted[/yellow]") + raise PluginFactoryError("Directory clean aborted by user") else: - item.unlink() \ No newline at end of file + logger.warning(f"[yellow]`y` selected. Cleaning {path}. Note: you can use the `--clean` argument to automatically clean the directory and skip this prompt next time.[/yellow]") + clean_directory(path) + + +def repo_dir_name(repo_url: str) -> str: + """Derive a directory name from a git repository URL. + + Examples: + https://github.com/backstage/community-plugins.git -> community-plugins + https://github.com/awslabs/backstage-plugins-for-aws -> backstage-plugins-for-aws + """ + name = repo_url.rstrip("/").rsplit("/", 1)[-1] + if name.endswith(".git"): + name = name[:-4] + return name \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index 945d616..dfed6c2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,6 +12,23 @@ from src.rhdh_dynamic_plugin_factory.config import PluginFactoryConfig +def _write_source_json(directory: Path, repo: str, repo_ref: str, workspace_path: str = ".") -> None: + """Write a source.json file into the given directory, creating it if needed.""" + directory.mkdir(parents=True, exist_ok=True) + data = {"repo": repo, "repo-ref": repo_ref, "workspace-path": workspace_path} + (directory / "source.json").write_text(json.dumps(data)) + + +@pytest.fixture +def write_source_json(): + """Factory fixture that returns a helper to write source.json files. + + Usage: + write_source_json(directory, repo, repo_ref, workspace_path=".") + """ + return _write_source_json + + @pytest.fixture def mock_logger(): """Create a mocked logger to avoid output during tests.""" @@ -29,7 +46,6 @@ def mock_args(tmp_path): workspace_path=".", config_dir=tmp_path / "config", repo_path=tmp_path / "workspace", - log_level="INFO", use_local=False, push_images=False, output_dir=str(tmp_path / "outputs"), @@ -61,21 +77,13 @@ def valid_default_env(monkeypatch): @pytest.fixture def valid_source_json(tmp_path: Path): """Create a valid source.json file.""" - source_data = { - "repo": "https://github.com/awslabs/backstage-plugins-for-aws", - "repo-ref": "78df9399a81cfd95265cab53815f54210b1d7f50", - "workspace-path": ".", - "repo-flat": True, - "repo-backstage-version": "1.42.5" - } - config_dir = tmp_path / "config" - config_dir.mkdir(parents=True, exist_ok=True) - - source_file = config_dir / "source.json" - source_file.write_text(json.dumps(source_data, indent=2)) - - return source_file + _write_source_json( + config_dir, + "https://github.com/awslabs/backstage-plugins-for-aws", + "78df9399a81cfd95265cab53815f54210b1d7f50", + ) + return config_dir / "source.json" @pytest.fixture @@ -136,14 +144,11 @@ def setup_test_env(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): source_dir.mkdir(parents=True, exist_ok=True) # Create source.json - source_data = { - "repo": "https://github.com/awslabs/backstage-plugins-for-aws", - "repo-ref": "78df9399a81cfd95265cab53815f54210b1d7f50", - "workspace-path": ".", - "repo-flat": True, - "repo-backstage-version": "1.42.5" - } - (config_dir / "source.json").write_text(json.dumps(source_data, indent=2)) + _write_source_json( + config_dir, + "https://github.com/awslabs/backstage-plugins-for-aws", + "78df9399a81cfd95265cab53815f54210b1d7f50", + ) # Create plugins-list.yaml plugins_content = """plugins/ecs/frontend: @@ -153,7 +158,6 @@ def setup_test_env(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): # Set common environment variables monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") - monkeypatch.setenv("WORKSPACE_PATH", ".") return { "config_dir": str(config_dir), @@ -168,8 +172,6 @@ def clean_env(monkeypatch: pytest.MonkeyPatch): # Remove all relevant environment variables env_vars = [ "RHDH_CLI_VERSION", - "LOG_LEVEL", - "WORKSPACE_PATH", "REGISTRY_URL", "REGISTRY_USERNAME", "REGISTRY_PASSWORD", diff --git a/tests/test_config_load_from_env.py b/tests/test_config_load_from_env.py index c6a792f..a11175a 100644 --- a/tests/test_config_load_from_env.py +++ b/tests/test_config_load_from_env.py @@ -26,14 +26,12 @@ def test_load_from_env_valid_configuration(self, mock_args, setup_test_env, monk # Ensure environment variables are set monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") - monkeypatch.setenv("WORKSPACE_PATH", ".") # Load configuration config = PluginFactoryConfig.load_from_env(mock_args) # Verify all required fields are set assert config.rhdh_cli_version == "1.7.2" - assert config.log_level == "INFO" assert config.config_dir == setup_test_env["config_dir"] assert config.repo_path == setup_test_env["source_dir"] assert config.workspace_path == "." @@ -51,7 +49,6 @@ def test_load_from_env_valid_configuration(self, mock_args, setup_test_env, monk def test_load_from_env_missing_rhdh_cli_version(self, mock_args, setup_test_env, clean_env): """Test that missing RHDH_CLI_VERSION raises ConfigurationError.""" # Don't set RHDH_CLI_VERSION - clean_env.setenv("WORKSPACE_PATH", ".") mock_args.config_dir = setup_test_env["config_dir"] mock_args.repo_path = setup_test_env["source_dir"] @@ -62,41 +59,10 @@ def test_load_from_env_missing_rhdh_cli_version(self, mock_args, setup_test_env, with pytest.raises(ConfigurationError, match="RHDH_CLI_VERSION must be set"): PluginFactoryConfig.load_from_env(mock_args) - def test_load_from_env_invalid_log_level(self, mock_args, setup_test_env, monkeypatch): - """Test that invalid log level raises ConfigurationError.""" - # Set required environment variables - monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") - monkeypatch.setenv("WORKSPACE_PATH", ".") - monkeypatch.setenv("LOG_LEVEL", "INVALID_LEVEL") - - mock_args.config_dir = setup_test_env["config_dir"] - mock_args.repo_path = setup_test_env["source_dir"] - mock_args.workspace_path = "." - mock_args.log_level = "INVALID_LEVEL" - - with pytest.raises(ConfigurationError, match="Invalid log level"): - PluginFactoryConfig.load_from_env(mock_args) - - @pytest.mark.parametrize("log_level", ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]) - def test_load_from_env_valid_log_levels(self, mock_args, setup_test_env, monkeypatch, log_level): - """Test that all valid log levels are accepted.""" - monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") - monkeypatch.setenv("WORKSPACE_PATH", ".") - monkeypatch.setenv("LOG_LEVEL", log_level) - - mock_args.config_dir = setup_test_env["config_dir"] - mock_args.repo_path = setup_test_env["source_dir"] - mock_args.workspace_path = "." - mock_args.log_level = log_level - - config = PluginFactoryConfig.load_from_env(mock_args) - assert config.log_level == log_level - def test_load_from_env_environment_variable_precedence(self, mock_args, setup_test_env, clean_env, tmp_path): """Test that custom .env file with override=True overrides environment variables.""" # Set initial environment variables clean_env.setenv("RHDH_CLI_VERSION", "1.7.2") - clean_env.setenv("WORKSPACE_PATH", ".") # Create a custom .env file with different values # Since load_dotenv is called with override=True, these values should win @@ -124,9 +90,6 @@ def test_load_from_env_additional_env_file_loading(self, mock_args, setup_test_e "REGISTRY_NAMESPACE=test-namespace\n" ) - # Set required variables - monkeypatch.setenv("WORKSPACE_PATH", ".") - mock_args.config_dir = setup_test_env["config_dir"] mock_args.repo_path = setup_test_env["source_dir"] mock_args.workspace_path = "." @@ -152,29 +115,9 @@ def test_load_from_env_additional_env_file_loading(self, mock_args, setup_test_e assert config.registry_url == "quay.io" assert config.registry_namespace == "test-namespace" - def test_load_from_env_missing_workspace_path_deferred(self, mock_args, setup_test_env, monkeypatch): - """Test that missing workspace_path is allowed during load_from_env. - - workspace_path validation is deferred to cli._run() because it may be - resolved later from source.json's workspace-path field. - """ - monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") - # Don't set WORKSPACE_PATH env var - monkeypatch.delenv("WORKSPACE_PATH", raising=False) - - mock_args.config_dir = setup_test_env["config_dir"] - mock_args.repo_path = setup_test_env["source_dir"] - mock_args.workspace_path = None # Missing workspace_path - - # Should not raise -- workspace_path validation is deferred - config = PluginFactoryConfig.load_from_env(mock_args) - - assert config.workspace_path == "" # Empty, to be resolved later - def test_load_from_env_directory_creation(self, mock_args, tmp_path, monkeypatch): """Test that config_dir and repo_path directories are created.""" monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") - monkeypatch.setenv("WORKSPACE_PATH", ".") # Use non-existent directories new_config_dir = tmp_path / "new_config" @@ -202,7 +145,6 @@ def test_load_from_env_directory_creation(self, mock_args, tmp_path, monkeypatch def test_load_from_env_registry_config_from_environment(self, mock_args, setup_test_env, monkeypatch): """Test that registry configuration is loaded from environment variables.""" monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") - monkeypatch.setenv("WORKSPACE_PATH", ".") monkeypatch.setenv("REGISTRY_URL", "quay.io") monkeypatch.setenv("REGISTRY_USERNAME", "test_user") monkeypatch.setenv("REGISTRY_PASSWORD", "test_pass") @@ -225,7 +167,6 @@ def test_load_from_env_registry_config_from_environment(self, mock_args, setup_t def test_load_from_env_registry_insecure_false(self, mock_args, setup_test_env, monkeypatch): """Test that REGISTRY_INSECURE defaults to False.""" monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") - monkeypatch.setenv("WORKSPACE_PATH", ".") monkeypatch.setenv("REGISTRY_INSECURE", "false") mock_args.config_dir = setup_test_env["config_dir"] @@ -239,7 +180,6 @@ def test_load_from_env_registry_insecure_false(self, mock_args, setup_test_env, def test_load_from_env_use_local_flag(self, mock_args, setup_test_env, monkeypatch): """Test that use_local flag is loaded from args.""" monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") - monkeypatch.setenv("WORKSPACE_PATH", ".") mock_args.config_dir = setup_test_env["config_dir"] mock_args.repo_path = setup_test_env["source_dir"] @@ -253,7 +193,6 @@ def test_load_from_env_use_local_flag(self, mock_args, setup_test_env, monkeypat def test_load_from_env_source_json_missing_repo_path_empty(self, mock_args, tmp_path, monkeypatch): """Test that missing source.json with empty repo_path raises ConfigurationError.""" monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") - monkeypatch.setenv("WORKSPACE_PATH", ".") # Create empty directories config_dir = tmp_path / "config" @@ -273,7 +212,6 @@ def test_load_from_env_source_json_missing_repo_path_empty(self, mock_args, tmp_ def test_load_from_env_source_json_missing_repo_path_has_content(self, mock_args, tmp_path, monkeypatch): """Test that missing source.json with non-empty repo_path logs warning but passes.""" monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") - monkeypatch.setenv("WORKSPACE_PATH", ".") # Create config dir without source.json config_dir = tmp_path / "config" diff --git a/tests/test_config_patches_and_overlays.py b/tests/test_config_patches_and_overlays.py index 6a7ea13..f053efa 100644 --- a/tests/test_config_patches_and_overlays.py +++ b/tests/test_config_patches_and_overlays.py @@ -33,14 +33,15 @@ def test_apply_patches_and_overlays_success(self, make_config): call_args = mock_run_cmd.call_args cmd = call_args[0][0] + expected_repo_root = os.path.abspath(config.repo_path) + expected_workspace = os.path.abspath(os.path.join(config.repo_path, config.workspace_path)) assert len(cmd) == 3 assert cmd[0] == str(script_path.absolute()) assert cmd[1] == os.path.abspath(config.config_dir) - expected_workspace = os.path.abspath(os.path.join(config.repo_path, config.workspace_path)) assert cmd[2] == expected_workspace assert call_args[0][1] == config.logger - assert call_args[1]["cwd"] == Path(expected_workspace) + assert call_args[1]["cwd"] == Path(expected_repo_root) assert call_args[1]["stderr_log_func"] == config.logger.error def test_apply_patches_and_overlays_script_not_found(self, make_config): diff --git a/tests/test_multi_workspace.py b/tests/test_multi_workspace.py new file mode 100644 index 0000000..8de8505 --- /dev/null +++ b/tests/test_multi_workspace.py @@ -0,0 +1,1073 @@ +""" +Unit tests for multi-workspace support. + +Tests workspace discovery, mode detection, WorkspaceInfo, .env inheritance, +git worktree cloning, and per-workspace path management. +""" + +import os +import subprocess +from pathlib import Path +from unittest.mock import patch, MagicMock +import pytest + +from src.rhdh_dynamic_plugin_factory.config import ( + PluginFactoryConfig, + SourceConfig, + WorkspaceInfo, + discover_workspaces, + clone_workspaces_with_worktrees, +) +from src.rhdh_dynamic_plugin_factory.utils import repo_dir_name +from src.rhdh_dynamic_plugin_factory.cli import ( + _run, + _run_multi_workspace, + _load_env_for_workspace, +) +from src.rhdh_dynamic_plugin_factory.exceptions import PluginFactoryError, ConfigurationError, ExecutionError + + +class TestWorkspaceInfo: + """Tests for WorkspaceInfo dataclass.""" + + def test_resolve_paths(self, tmp_path): + """Test that resolve_paths sets repo_path and output_dir correctly.""" + ws = WorkspaceInfo( + name="todo", + config_dir=tmp_path / "config" / "todo", + source_config=MagicMock(), + ) + + base_repo = tmp_path / "source" + base_output = tmp_path / "outputs" + ws.resolve_paths(base_repo, base_output) + + assert ws.repo_path == base_repo / "todo" + assert ws.output_dir == base_output / "todo" + + def test_paths_default_to_none(self): + """Test that repo_path and output_dir default to None.""" + ws = WorkspaceInfo( + name="test", + config_dir=Path("/tmp/test"), + source_config=MagicMock(), + ) + + assert ws.repo_path is None + assert ws.output_dir is None + + +class TestDiscoverWorkspaces: + """Tests for discover_workspaces function.""" + + def test_discovers_multiple_workspaces(self, tmp_path, write_source_json): + """Test discovering multiple workspaces with source.json files.""" + write_source_json(tmp_path / "todo", "https://github.com/backstage/community-plugins", "main", "workspaces/todo") + write_source_json(tmp_path / "aws-ecs", "https://github.com/awslabs/backstage-plugins-for-aws", "abc123", ".") + + with patch.object(SourceConfig, "resolve_default_ref", return_value="refs/heads/main"): + workspaces = discover_workspaces(tmp_path) + + assert len(workspaces) == 2 + names = [ws.name for ws in workspaces] + assert "todo" in names + assert "aws-ecs" in names + + def test_sorted_by_repo_then_name(self, tmp_path, write_source_json): + """Test that workspaces are sorted by repo URL then name.""" + write_source_json(tmp_path / "zz-ws", "https://github.com/aaa/repo", "main", ".") + write_source_json(tmp_path / "aa-ws", "https://github.com/zzz/repo", "main", ".") + write_source_json(tmp_path / "bb-ws", "https://github.com/aaa/repo", "v1.0", ".") + + with patch.object(SourceConfig, "resolve_default_ref", return_value="refs/heads/main"): + workspaces = discover_workspaces(tmp_path) + + # aaa/repo workspaces first (sorted by name), then zzz/repo + assert workspaces[0].name == "bb-ws" + assert workspaces[1].name == "zz-ws" + assert workspaces[2].name == "aa-ws" + + def test_ignores_directories_without_source_json(self, tmp_path, write_source_json): + """Test that directories without source.json are ignored.""" + write_source_json(tmp_path / "valid", "https://github.com/test/repo", "main", ".") + (tmp_path / "patches").mkdir() + (tmp_path / "no-source").mkdir() + (tmp_path / ".env").write_text("KEY=VALUE") + + with patch.object(SourceConfig, "resolve_default_ref", return_value="refs/heads/main"): + workspaces = discover_workspaces(tmp_path) + + assert len(workspaces) == 1 + assert workspaces[0].name == "valid" + + def test_empty_config_dir(self, tmp_path): + """Test that empty config dir returns empty list.""" + workspaces = discover_workspaces(tmp_path) + assert workspaces == [] + + def test_nonexistent_config_dir(self, tmp_path): + """Test that nonexistent config dir returns empty list.""" + workspaces = discover_workspaces(tmp_path / "nonexistent") + assert workspaces == [] + + def test_files_at_root_ignored(self, tmp_path): + """Test that files (not directories) at config root are ignored.""" + (tmp_path / "source.json").write_text('{"repo": "test", "repo-ref": "main"}') + (tmp_path / "plugins-list.yaml").write_text("plugins/foo:") + + workspaces = discover_workspaces(tmp_path) + assert workspaces == [] + + def test_invalid_source_json_raises_error(self, tmp_path): + """Test that invalid source.json in a workspace raises ConfigurationError.""" + ws_dir = tmp_path / "bad-workspace" + ws_dir.mkdir() + (ws_dir / "source.json").write_text("{ invalid json }") + + with pytest.raises(ConfigurationError, match="Invalid JSON"): + discover_workspaces(tmp_path) + + def test_single_workspace_subdirectory(self, tmp_path, write_source_json): + """Test that even a single subdirectory with source.json is detected.""" + write_source_json(tmp_path / "only-one", "https://github.com/test/repo", "v1.0", ".") + + workspaces = discover_workspaces(tmp_path) + + assert len(workspaces) == 1 + assert workspaces[0].name == "only-one" + assert workspaces[0].source_config.repo == "https://github.com/test/repo" + + +class TestModeDetection: + """Tests for mode detection in _run().""" + + def test_multi_workspace_rejects_source_repo(self, tmp_path, mock_args, monkeypatch, write_source_json): + """Test that --source-repo is rejected in multi-workspace mode.""" + monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") + config_dir = tmp_path / "config" + write_source_json(config_dir / "ws1", "https://github.com/test/repo", "main", ".") + + mock_args.config_dir = config_dir + mock_args.source_repo = "https://github.com/other/repo" + + with pytest.raises(ConfigurationError, match="--source-repo cannot be used in multi-workspace mode"): + _run(mock_args) + + def test_multi_workspace_rejects_source_ref(self, tmp_path, mock_args, monkeypatch, write_source_json): + """Test that --source-ref is rejected in multi-workspace mode.""" + monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") + config_dir = tmp_path / "config" + write_source_json(config_dir / "ws1", "https://github.com/test/repo", "main", ".") + + mock_args.config_dir = config_dir + mock_args.source_ref = "v1.0" + + with pytest.raises(ConfigurationError, match="--source-ref cannot be used in multi-workspace mode"): + _run(mock_args) + + def test_multi_workspace_rejects_workspace_path(self, tmp_path, mock_args, monkeypatch, write_source_json): + """Test that --workspace-path is rejected in multi-workspace mode.""" + monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") + config_dir = tmp_path / "config" + write_source_json(config_dir / "ws1", "https://github.com/test/repo", "main", ".") + + mock_args.config_dir = config_dir + mock_args.workspace_path = "workspaces/todo" + + with pytest.raises(ConfigurationError, match="--workspace-path cannot be used in multi-workspace mode"): + _run(mock_args) + + def test_no_workspaces_uses_single_mode(self, tmp_path, mock_args, monkeypatch): + """Test that empty config dir uses single-workspace mode.""" + monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") + config_dir = tmp_path / "config" + config_dir.mkdir(parents=True) + + mock_args.config_dir = config_dir + mock_args.workspace_path = "." + + # Should call _run_single_workspace path (will fail at source validation, but mode is correct) + with patch("src.rhdh_dynamic_plugin_factory.cli._run_single_workspace") as mock_single: + _run(mock_args) + mock_single.assert_called_once_with(mock_args) + + def test_multi_workspace_skips_root_source_and_plugins_validation( + self, tmp_path, mock_args, monkeypatch, write_source_json + ): + """Test that multi-workspace mode does NOT emit root-level source.json / plugins-list.yaml warnings.""" + monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") + + config_dir = tmp_path / "config" + write_source_json(config_dir / "ws1", "https://github.com/test/repo", "main", ".") + + mock_args.config_dir = config_dir + mock_args.repo_path = tmp_path / "source" + mock_args.output_dir = tmp_path / "outputs" + mock_args.clean = True + mock_args.workspace_path = None + + with patch("src.rhdh_dynamic_plugin_factory.cli.clone_workspaces_with_worktrees"), \ + patch("src.rhdh_dynamic_plugin_factory.cli._process_workspace"), \ + patch("src.rhdh_dynamic_plugin_factory.config.PluginFactoryConfig._validate_source_json") as mock_src, \ + patch("src.rhdh_dynamic_plugin_factory.config.PluginFactoryConfig._validate_plugins_list") as mock_pl: + try: + _run_multi_workspace(mock_args, discover_workspaces(config_dir)) + except Exception: + pass + + mock_src.assert_not_called() + mock_pl.assert_not_called() + + +class TestRepoDirName: + """Tests for repo_dir_name helper.""" + + def test_https_url(self): + assert repo_dir_name("https://github.com/backstage/community-plugins") == "community-plugins" + + def test_https_url_with_git_suffix(self): + assert repo_dir_name("https://github.com/git/git.git") == "git" + + def test_trailing_slash(self): + assert repo_dir_name("https://github.com/backstage/community-plugins/") == "community-plugins" + + def test_ssh_url(self): + assert repo_dir_name("git@github.com:backstage/community-plugins.git") == "community-plugins" + + +class TestEnvInheritance: + """Tests for .env inheritance and isolation between workspaces.""" + + def test_load_env_for_workspace_layered(self, tmp_path, monkeypatch): + """Test that workspace .env overrides base_env values.""" + ws_env = tmp_path / "ws.env" + ws_env.write_text("LEVEL=workspace\nWS_ONLY=yes\n") + + snapshot = dict(os.environ) + + # base_env simulates Podman + default.env + root .env already loaded + base_env = dict(snapshot) + base_env["LEVEL"] = "root" + base_env["DEFAULT_ONLY"] = "yes" + base_env["ROOT_ONLY"] = "yes" + + _load_env_for_workspace(base_env, ws_env) + + # Workspace .env should win for LEVEL + assert os.environ.get("LEVEL") == "workspace" + assert os.environ.get("DEFAULT_ONLY") == "yes" + assert os.environ.get("ROOT_ONLY") == "yes" + assert os.environ.get("WS_ONLY") == "yes" + + # Restore + os.environ.clear() + os.environ.update(snapshot) + + def test_load_env_isolates_between_workspaces(self, tmp_path, monkeypatch): + """Test that env from one workspace doesn't leak into the next.""" + ws1_env = tmp_path / "ws1.env" + ws1_env.write_text("WS1_SECRET=secret1\n") + + ws2_env = tmp_path / "ws2.env" + ws2_env.write_text("WS2_VALUE=value2\n") + + base_env = dict(os.environ) + + # Load workspace 1 + _load_env_for_workspace(base_env, ws1_env) + assert os.environ.get("WS1_SECRET") == "secret1" + + # Load workspace 2 -- ws1 vars should be gone + _load_env_for_workspace(base_env, ws2_env) + assert os.environ.get("WS1_SECRET") is None + assert os.environ.get("WS2_VALUE") == "value2" + + # Restore + os.environ.clear() + os.environ.update(base_env) + + def test_load_env_missing_files_no_error(self, tmp_path): + """Test that a missing workspace .env is silently skipped.""" + base_env = dict(os.environ) + + _load_env_for_workspace(base_env, tmp_path / "nonexistent.env") + + # Restore + os.environ.clear() + os.environ.update(base_env) + + def test_podman_env_var_precedence(self, tmp_path): + """Test precedence: workspace .env > root .env > Podman/system env vars > default.env. + + base_env simulates the state after load_from_env has already applied + default.env (no override) and root .env (override) on top of Podman/system vars. + """ + ws_env = tmp_path / "ws.env" + ws_env.write_text("ROOT_VAR=from_workspace\n") + + original_snapshot = dict(os.environ) + + # base_env simulates: Podman vars + default.env (no override) + root .env (override) + # Podman had PODMAN_VAR, SHARED_VAR, DEFAULT_VAR + # default.env tried DEFAULT_VAR=from_default, SHARED_VAR=from_default (no override -> Podman wins) + # root .env set SHARED_VAR=from_root (override -> root wins), ROOT_VAR=from_root + base_env = dict(original_snapshot) + base_env["PODMAN_VAR"] = "from_podman" + base_env["DEFAULT_VAR"] = "from_podman" + base_env["SHARED_VAR"] = "from_root" + base_env["ROOT_VAR"] = "from_root" + + _load_env_for_workspace(base_env, ws_env) + + # Podman-only var survives (not overridden by any .env file) + assert os.environ.get("PODMAN_VAR") == "from_podman" + # default.env did NOT override Podman var (already baked into base_env) + assert os.environ.get("DEFAULT_VAR") == "from_podman" + # root .env DID override Podman var (already baked into base_env) + assert os.environ.get("SHARED_VAR") == "from_root" + # workspace .env overrides root .env + assert os.environ.get("ROOT_VAR") == "from_workspace" + + # Restore + os.environ.clear() + os.environ.update(original_snapshot) + + +class TestCloneWorkspacesWithWorktrees: + """Tests for clone_workspaces_with_worktrees function.""" + + def test_groups_by_repo_and_clones_once(self, tmp_path): + """Test that the same repo is only cloned once for multiple workspaces.""" + base_repo = tmp_path / "source" + base_repo.mkdir() + + ws1 = WorkspaceInfo( + name="ws1", + config_dir=tmp_path / "config" / "ws1", + source_config=MagicMock(repo="https://github.com/test/repo", repo_ref="main"), + repo_path=base_repo / "ws1", + output_dir=tmp_path / "out" / "ws1", + ) + ws2 = WorkspaceInfo( + name="ws2", + config_dir=tmp_path / "config" / "ws2", + source_config=MagicMock(repo="https://github.com/test/repo", repo_ref="v1.0"), + repo_path=base_repo / "ws2", + output_dir=tmp_path / "out" / "ws2", + ) + + with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming", return_value=0) as mock_stream: + clone_workspaces_with_worktrees([ws1, ws2], base_repo) + + cmds = [c[0][0] for c in mock_stream.call_args_list] + clone_cmds = [c for c in cmds if "clone" in c] + worktree_cmds = [c for c in cmds if "worktree" in c] + + assert len(clone_cmds) == 1 + assert len(worktree_cmds) == 2 + + def test_raises_if_repo_path_not_set(self, tmp_path): + """Test that missing repo_path raises PluginFactoryError.""" + base_repo = tmp_path / "source" + base_repo.mkdir() + + ws = WorkspaceInfo( + name="test", + config_dir=tmp_path, + source_config=MagicMock(repo="https://github.com/test/repo", repo_ref="main"), + repo_path=None, + ) + + with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming", return_value=0): + with pytest.raises(PluginFactoryError, match="no resolved repository path"): + clone_workspaces_with_worktrees([ws], base_repo) + + def test_clone_failure_raises_error(self, tmp_path): + """Test that git clone failure raises ExecutionError.""" + base_repo = tmp_path / "source" + base_repo.mkdir() + + ws = WorkspaceInfo( + name="ws1", + config_dir=tmp_path / "config" / "ws1", + source_config=MagicMock(repo="https://github.com/test/nonexistent", repo_ref="main"), + repo_path=base_repo / "ws1", + output_dir=tmp_path / "out" / "ws1", + ) + + with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming", return_value=128): + with pytest.raises(ExecutionError, match="Failed to clone repository"): + clone_workspaces_with_worktrees([ws], base_repo) + + def test_multiple_repos_each_cloned_once(self, tmp_path): + """Test that different repos are each cloned once.""" + base_repo = tmp_path / "source" + base_repo.mkdir() + + ws1 = WorkspaceInfo( + name="ws1", + config_dir=tmp_path / "config" / "ws1", + source_config=MagicMock(repo="https://github.com/org/repo-a", repo_ref="main"), + repo_path=base_repo / "ws1", + output_dir=tmp_path / "out" / "ws1", + ) + ws2 = WorkspaceInfo( + name="ws2", + config_dir=tmp_path / "config" / "ws2", + source_config=MagicMock(repo="https://github.com/org/repo-b", repo_ref="v1.0"), + repo_path=base_repo / "ws2", + output_dir=tmp_path / "out" / "ws2", + ) + + with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming", return_value=0) as mock_stream: + clone_workspaces_with_worktrees([ws1, ws2], base_repo) + + cmds = [c[0][0] for c in mock_stream.call_args_list] + clone_cmds = [c for c in cmds if "clone" in c] + worktree_cmds = [c for c in cmds if "worktree" in c] + + assert len(clone_cmds) == 2 + assert len(worktree_cmds) == 2 + + +class TestCloneWorkspacesWithWorktreesIntegration: + """Integration tests for clone_workspaces_with_worktrees using real git operations. + + Verifies that worktrees are created at the correct paths with the expected + file contents, not just that git commands are invoked. + """ + + @staticmethod + def _create_test_repo(tmp_path: Path, name: str = "test-repo", + files: dict[str, str] | None = None) -> tuple[Path, str]: + """Create a real local git repository with known files. + + Returns: + Tuple of (repo_path, commit_sha). + """ + repo = tmp_path / name + repo.mkdir() + + files = files or {"package.json": '{"name": "test"}'} + for relpath, content in files.items(): + fpath = repo / relpath + fpath.parent.mkdir(parents=True, exist_ok=True) + fpath.write_text(content) + + subprocess.run(["git", "init"], cwd=repo, capture_output=True, check=True) + subprocess.run(["git", "add", "."], cwd=repo, capture_output=True, check=True) + subprocess.run( + ["git", "commit", "-m", "init", "--author", "test "], + cwd=repo, capture_output=True, check=True, + env={**os.environ, "GIT_COMMITTER_NAME": "test", "GIT_COMMITTER_EMAIL": "test@test.com"}, + ) + sha = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=repo, capture_output=True, text=True, check=True + ).stdout.strip() + return repo, sha + + def test_worktree_contains_expected_files(self, tmp_path): + """Test that worktree directories contain actual checked-out files.""" + origin, sha = self._create_test_repo(tmp_path, files={ + "package.json": '{"name": "root"}', + "src/index.ts": "export {}", + "plugins/ecs/README.md": "# ECS Plugin", + }) + + base_repo = tmp_path / "source" + base_repo.mkdir() + + ws = WorkspaceInfo( + name="ws1", + config_dir=tmp_path / "config" / "ws1", + source_config=SourceConfig(repo=str(origin), repo_ref=sha, workspace_path="."), + repo_path=base_repo / "ws1", + output_dir=tmp_path / "out" / "ws1", + ) + + clone_workspaces_with_worktrees([ws], base_repo) + + assert (base_repo / "ws1" / "package.json").exists() + assert (base_repo / "ws1" / "package.json").read_text() == '{"name": "root"}' + assert (base_repo / "ws1" / "src" / "index.ts").exists() + assert (base_repo / "ws1" / "plugins" / "ecs" / "README.md").exists() + + def test_worktree_with_nested_workspace_path(self, tmp_path): + """Test that a workspace_path like 'workspaces/todo' exists inside the worktree.""" + origin, sha = self._create_test_repo(tmp_path, files={ + "package.json": '{"name": "monorepo"}', + "workspaces/todo/package.json": '{"name": "todo"}', + "workspaces/todo/src/index.ts": "export const TODO = true;", + }) + + base_repo = tmp_path / "source" + base_repo.mkdir() + + ws = WorkspaceInfo( + name="todo", + config_dir=tmp_path / "config" / "todo", + source_config=SourceConfig(repo=str(origin), repo_ref=sha, workspace_path="workspaces/todo"), + repo_path=base_repo / "todo", + output_dir=tmp_path / "out" / "todo", + ) + + clone_workspaces_with_worktrees([ws], base_repo) + + workspace_full = base_repo / "todo" / "workspaces" / "todo" + assert workspace_full.exists(), "Nested workspace path must exist in worktree" + assert (workspace_full / "package.json").read_text() == '{"name": "todo"}' + assert (workspace_full / "src" / "index.ts").exists() + + def test_multiple_worktrees_from_same_repo(self, tmp_path): + """Test that two workspaces from the same repo get independent worktrees.""" + origin, sha = self._create_test_repo(tmp_path, files={ + "package.json": '{"name": "shared"}', + "workspaces/a/file.txt": "workspace-a", + "workspaces/b/file.txt": "workspace-b", + }) + + base_repo = tmp_path / "source" + base_repo.mkdir() + + ws_a = WorkspaceInfo( + name="ws-a", + config_dir=tmp_path / "config" / "ws-a", + source_config=SourceConfig(repo=str(origin), repo_ref=sha, workspace_path="workspaces/a"), + repo_path=base_repo / "ws-a", + output_dir=tmp_path / "out" / "ws-a", + ) + ws_b = WorkspaceInfo( + name="ws-b", + config_dir=tmp_path / "config" / "ws-b", + source_config=SourceConfig(repo=str(origin), repo_ref=sha, workspace_path="workspaces/b"), + repo_path=base_repo / "ws-b", + output_dir=tmp_path / "out" / "ws-b", + ) + + clone_workspaces_with_worktrees([ws_a, ws_b], base_repo) + + assert (base_repo / "ws-a" / "workspaces" / "a" / "file.txt").read_text() == "workspace-a" + assert (base_repo / "ws-b" / "workspaces" / "b" / "file.txt").read_text() == "workspace-b" + # Both worktrees have the full repo content + assert (base_repo / "ws-a" / "package.json").exists() + assert (base_repo / "ws-b" / "package.json").exists() + + def test_worktrees_from_different_repos(self, tmp_path): + """Test that workspaces from different repos each get correct content.""" + origin_a, sha_a = self._create_test_repo(tmp_path, name="repo-a", files={ + "package.json": '{"name": "repo-a"}', + }) + origin_b, sha_b = self._create_test_repo(tmp_path, name="repo-b", files={ + "package.json": '{"name": "repo-b"}', + }) + + base_repo = tmp_path / "source" + base_repo.mkdir() + + ws_a = WorkspaceInfo( + name="ws-a", + config_dir=tmp_path / "config" / "ws-a", + source_config=SourceConfig(repo=str(origin_a), repo_ref=sha_a, workspace_path="."), + repo_path=base_repo / "ws-a", + output_dir=tmp_path / "out" / "ws-a", + ) + ws_b = WorkspaceInfo( + name="ws-b", + config_dir=tmp_path / "config" / "ws-b", + source_config=SourceConfig(repo=str(origin_b), repo_ref=sha_b, workspace_path="."), + repo_path=base_repo / "ws-b", + output_dir=tmp_path / "out" / "ws-b", + ) + + clone_workspaces_with_worktrees([ws_a, ws_b], base_repo) + + assert (base_repo / "ws-a" / "package.json").read_text() == '{"name": "repo-a"}' + assert (base_repo / "ws-b" / "package.json").read_text() == '{"name": "repo-b"}' + + def test_git_apply_works_in_worktree(self, tmp_path): + """Test that git apply works correctly inside a worktree (patch paths resolve properly).""" + origin, sha = self._create_test_repo(tmp_path, files={ + "package.json": '{"version": "1.0.0"}\n', + }) + + base_repo = tmp_path / "source" + base_repo.mkdir() + + ws = WorkspaceInfo( + name="ws1", + config_dir=tmp_path / "config" / "ws1", + source_config=SourceConfig(repo=str(origin), repo_ref=sha, workspace_path="."), + repo_path=base_repo / "ws1", + output_dir=tmp_path / "out" / "ws1", + ) + + clone_workspaces_with_worktrees([ws], base_repo) + + patch_content = ( + '--- a/package.json\n' + '+++ b/package.json\n' + '@@ -1 +1 @@\n' + '-{"version": "1.0.0"}\n' + '+{"version": "2.0.0"}\n' + ) + patch_file = tmp_path / "test.patch" + patch_file.write_text(patch_content) + + worktree = base_repo / "ws1" + result = subprocess.run( + ["git", "apply", str(patch_file)], + cwd=worktree, capture_output=True, text=True, + ) + assert result.returncode == 0, f"git apply failed: {result.stderr}" + assert (worktree / "package.json").read_text() == '{"version": "2.0.0"}\n' + +class TestApplyPatchesAndOverlaysIntegration: + """Integration tests for apply_patches_and_overlays with real git worktrees. + + Verifies that overlays are applied in the workspace subdirectory and that + patches work correctly when workspace_path is ``"."``. + """ + + @staticmethod + def _create_test_repo(tmp_path: Path, files: dict[str, str]) -> tuple[Path, str]: + repo = tmp_path / "origin" + repo.mkdir() + for relpath, content in files.items(): + fpath = repo / relpath + fpath.parent.mkdir(parents=True, exist_ok=True) + fpath.write_text(content) + subprocess.run(["git", "init"], cwd=repo, capture_output=True, check=True) + subprocess.run(["git", "add", "."], cwd=repo, capture_output=True, check=True) + subprocess.run( + ["git", "commit", "-m", "init", "--author", "test "], + cwd=repo, capture_output=True, check=True, + env={**os.environ, "GIT_COMMITTER_NAME": "test", "GIT_COMMITTER_EMAIL": "test@test.com"}, + ) + sha = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=repo, capture_output=True, text=True, check=True + ).stdout.strip() + return repo, sha + + def test_overlays_applied_in_workspace_subdir(self, tmp_path, monkeypatch): + """Overlays are copied into the workspace subdirectory, not the repo root.""" + monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") + + origin, sha = self._create_test_repo(tmp_path, files={ + "package.json": '{"name": "monorepo"}\n', + "workspaces/todo/plugins/todo/index.ts": "export {};\n", + }) + + base_repo = tmp_path / "source" + base_repo.mkdir() + + ws = WorkspaceInfo( + name="todo", + config_dir=tmp_path / "config" / "todo", + source_config=SourceConfig(repo=str(origin), repo_ref=sha, workspace_path="workspaces/todo"), + repo_path=base_repo / "todo", + output_dir=tmp_path / "out" / "todo", + ) + clone_workspaces_with_worktrees([ws], base_repo) + + # Set up config dir with plugins-list.yaml and an overlay + config_dir = tmp_path / "config" / "todo" + config_dir.mkdir(parents=True, exist_ok=True) + (config_dir / "plugins-list.yaml").write_text("plugins/todo:\n") + + overlay_dir = config_dir / "plugins" / "todo" / "overlay" + overlay_dir.mkdir(parents=True) + (overlay_dir / "scalprum-config.json").write_text('{"overlay": true}\n') + + config = PluginFactoryConfig( + rhdh_cli_version="1.7.2", + repo_path=str(base_repo / "todo"), + config_dir=str(config_dir), + workspace_path="workspaces/todo", + ) + + config.apply_patches_and_overlays( + config_dir=str(config_dir), + repo_path=str(base_repo / "todo"), + workspace_path="workspaces/todo", + ) + + workspace_dir = base_repo / "todo" / "workspaces" / "todo" + assert (workspace_dir / "plugins" / "todo" / "scalprum-config.json").read_text() == '{"overlay": true}\n' + + def test_workspace_path_dot_applies_both_at_same_dir(self, tmp_path, monkeypatch): + """When workspace_path is '.', patches and overlays target the same directory.""" + monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") + + origin, sha = self._create_test_repo(tmp_path, files={ + "package.json": '{"version": "1.0.0"}\n', + "plugins/ecs/index.ts": "export {};\n", + }) + + base_repo = tmp_path / "source" + base_repo.mkdir() + + ws = WorkspaceInfo( + name="ecs", + config_dir=tmp_path / "config" / "ecs", + source_config=SourceConfig(repo=str(origin), repo_ref=sha, workspace_path="."), + repo_path=base_repo / "ecs", + output_dir=tmp_path / "out" / "ecs", + ) + clone_workspaces_with_worktrees([ws], base_repo) + + config_dir = tmp_path / "config" / "ecs" + config_dir.mkdir(parents=True, exist_ok=True) + + patches_dir = config_dir / "patches" + patches_dir.mkdir() + (patches_dir / "1-bump.patch").write_text( + '--- a/package.json\n' + '+++ b/package.json\n' + '@@ -1 +1 @@\n' + '-{"version": "1.0.0"}\n' + '+{"version": "2.0.0"}\n' + ) + + (config_dir / "plugins-list.yaml").write_text("plugins/ecs:\n") + overlay_dir = config_dir / "plugins" / "ecs" / "overlay" + overlay_dir.mkdir(parents=True) + (overlay_dir / "config.json").write_text('{"cfg": true}\n') + + config = PluginFactoryConfig( + rhdh_cli_version="1.7.2", + repo_path=str(base_repo / "ecs"), + config_dir=str(config_dir), + workspace_path=".", + ) + + config.apply_patches_and_overlays( + config_dir=str(config_dir), + repo_path=str(base_repo / "ecs"), + workspace_path=".", + ) + + worktree = base_repo / "ecs" + assert (worktree / "package.json").read_text() == '{"version": "2.0.0"}\n' + assert (worktree / "plugins" / "ecs" / "config.json").read_text() == '{"cfg": true}\n' + + +class TestUpfrontCleanInMultiWorkspace: + """Tests for upfront source directory clean/prompt in _run_multi_workspace. + + In multi-workspace mode, the entire base_repo_path is cleaned once upfront + before clone_workspaces_with_worktrees is called, mirroring single-workspace + behavior where clone_to_path handles existing content. + """ + + def test_clean_flag_cleans_source_dir_before_cloning(self, tmp_path, mock_args, monkeypatch, write_source_json): + """Test that --clean auto-cleans base_repo_path before worktree setup.""" + monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") + + config_dir = tmp_path / "config" + write_source_json(config_dir / "ws1", "https://github.com/test/repo", "main", ".") + + base_repo = tmp_path / "source" + base_repo.mkdir() + (base_repo / "stale-content").write_text("old data") + + mock_args.config_dir = config_dir + mock_args.repo_path = base_repo + mock_args.output_dir = tmp_path / "outputs" + mock_args.clean = True + mock_args.workspace_path = None + + with patch("src.rhdh_dynamic_plugin_factory.cli.clone_workspaces_with_worktrees") as mock_clone, \ + patch("src.rhdh_dynamic_plugin_factory.cli.PluginFactoryConfig.load_from_env") as mock_load: + mock_config = MagicMock() + mock_config.use_local = False + mock_load.return_value = mock_config + + try: + _run_multi_workspace(mock_args, discover_workspaces(config_dir)) + except Exception: + pass + + # base_repo_path should have been cleaned (stale-content removed) + assert not (base_repo / "stale-content").exists() + + # clone_workspaces_with_worktrees should have been called + mock_clone.assert_called_once() + + def test_no_clean_prompts_user_for_nonempty_source_dir(self, tmp_path, mock_args, monkeypatch, write_source_json): + """Test that non-empty base_repo_path without --clean prompts user.""" + monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") + + config_dir = tmp_path / "config" + write_source_json(config_dir / "ws1", "https://github.com/test/repo", "main", ".") + + base_repo = tmp_path / "source" + base_repo.mkdir() + (base_repo / "stale-content").write_text("old data") + + mock_args.config_dir = config_dir + mock_args.repo_path = base_repo + mock_args.output_dir = tmp_path / "outputs" + mock_args.clean = False + mock_args.workspace_path = None + + with patch("src.rhdh_dynamic_plugin_factory.cli.PluginFactoryConfig.load_from_env") as mock_load, \ + patch("builtins.input", return_value="n"): + mock_config = MagicMock() + mock_config.use_local = False + mock_load.return_value = mock_config + + with pytest.raises(PluginFactoryError, match="aborted by user"): + _run_multi_workspace(mock_args, discover_workspaces(config_dir)) + + def test_empty_source_dir_skips_prompt(self, tmp_path, mock_args, monkeypatch, write_source_json): + """Test that an empty base_repo_path proceeds without prompting.""" + monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") + + config_dir = tmp_path / "config" + write_source_json(config_dir / "ws1", "https://github.com/test/repo", "main", ".") + + base_repo = tmp_path / "source" + base_repo.mkdir() + + mock_args.config_dir = config_dir + mock_args.repo_path = base_repo + mock_args.output_dir = tmp_path / "outputs" + mock_args.clean = False + mock_args.workspace_path = None + + with patch("src.rhdh_dynamic_plugin_factory.cli.clone_workspaces_with_worktrees") as mock_clone, \ + patch("src.rhdh_dynamic_plugin_factory.cli.PluginFactoryConfig.load_from_env") as mock_load, \ + patch("builtins.input") as mock_input: + mock_config = MagicMock() + mock_config.use_local = False + mock_load.return_value = mock_config + + try: + _run_multi_workspace(mock_args, discover_workspaces(config_dir)) + except Exception: + pass + + mock_input.assert_not_called() + mock_clone.assert_called_once() + + +class TestIgnoredContentWarnings: + """Tests that _run_multi_workspace warns about all non-workspace root-level content.""" + + def test_warns_about_all_ignored_content(self, tmp_path, mock_args, monkeypatch, write_source_json): + """Loose files and non-workspace dirs produce a single grouped warning.""" + monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") + + config_dir = tmp_path / "config" + write_source_json(config_dir / "ws1", "https://github.com/test/repo", "main", ".") + + # Intentionally ignored root-level content + (config_dir / "source.json").write_text('{"repo":"x"}') + (config_dir / "plugins-list.yaml").write_text("plugins/foo:") + (config_dir / "notes.txt").write_text("scratch") + (config_dir / "patches").mkdir() + (config_dir / ".env").write_text("ROOT=1") + + mock_args.config_dir = config_dir + mock_args.repo_path = tmp_path / "source" + mock_args.output_dir = tmp_path / "outputs" + mock_args.clean = True + mock_args.workspace_path = None + + with patch("src.rhdh_dynamic_plugin_factory.cli.clone_workspaces_with_worktrees"), \ + patch("src.rhdh_dynamic_plugin_factory.cli.PluginFactoryConfig.load_from_env") as mock_load, \ + patch("src.rhdh_dynamic_plugin_factory.cli.logger") as mock_logger: + mock_config = MagicMock() + mock_config.use_local = False + mock_load.return_value = mock_config + + try: + _run_multi_workspace(mock_args, discover_workspaces(config_dir)) + except Exception: + pass + + warning_calls = [ + c for c in mock_logger.warning.call_args_list + if "will be ignored" in str(c) + ] + assert len(warning_calls) == 1 + msg = warning_calls[0][0][0] + assert "notes.txt" in msg + assert "source.json" in msg + assert "plugins-list.yaml" in msg + assert "patches/" in msg + assert ".env" not in msg + + def test_no_warning_when_only_workspaces_and_env(self, tmp_path, mock_args, monkeypatch, write_source_json): + """No ignored-content warning when root only has workspaces and .env.""" + monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") + + config_dir = tmp_path / "config" + write_source_json(config_dir / "ws1", "https://github.com/test/repo", "main", ".") + (config_dir / ".env").write_text("ROOT=1") + + mock_args.config_dir = config_dir + mock_args.repo_path = tmp_path / "source" + mock_args.output_dir = tmp_path / "outputs" + mock_args.clean = True + mock_args.workspace_path = None + + with patch("src.rhdh_dynamic_plugin_factory.cli.clone_workspaces_with_worktrees"), \ + patch("src.rhdh_dynamic_plugin_factory.cli.PluginFactoryConfig.load_from_env") as mock_load, \ + patch("src.rhdh_dynamic_plugin_factory.cli.logger") as mock_logger: + mock_config = MagicMock() + mock_config.use_local = False + mock_load.return_value = mock_config + + try: + _run_multi_workspace(mock_args, discover_workspaces(config_dir)) + except Exception: + pass + + warning_calls = [ + c for c in mock_logger.warning.call_args_list + if "will be ignored" in str(c) + ] + assert len(warning_calls) == 0 + + def test_distinguishes_files_from_directories(self, tmp_path, mock_args, monkeypatch, write_source_json): + """Warning labels files as '(file)' and dirs as '(directory ...)'.""" + monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") + + config_dir = tmp_path / "config" + write_source_json(config_dir / "ws1", "https://github.com/test/repo", "main", ".") + (config_dir / "stray.txt").write_text("data") + (config_dir / "leftover").mkdir() + + mock_args.config_dir = config_dir + mock_args.repo_path = tmp_path / "source" + mock_args.output_dir = tmp_path / "outputs" + mock_args.clean = True + mock_args.workspace_path = None + + with patch("src.rhdh_dynamic_plugin_factory.cli.clone_workspaces_with_worktrees"), \ + patch("src.rhdh_dynamic_plugin_factory.cli.PluginFactoryConfig.load_from_env") as mock_load, \ + patch("src.rhdh_dynamic_plugin_factory.cli.logger") as mock_logger: + mock_config = MagicMock() + mock_config.use_local = False + mock_load.return_value = mock_config + + try: + _run_multi_workspace(mock_args, discover_workspaces(config_dir)) + except Exception: + pass + + warning_calls = [ + c for c in mock_logger.warning.call_args_list + if "will be ignored" in str(c) + ] + assert len(warning_calls) == 1 + msg = warning_calls[0][0][0] + assert "stray.txt (file)" in msg + assert "leftover/ (directory" in msg + + +class TestRegistryRefresh: + """Tests for PluginFactoryConfig.refresh_registry_config().""" + + def _make_config(self, monkeypatch, tmp_path, push_images=False, **registry_overrides): + """Helper to build a PluginFactoryConfig with registry fields.""" + monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") + + defaults = { + "REGISTRY_URL": "quay.io", + "REGISTRY_USERNAME": "user", + "REGISTRY_PASSWORD": "pass", + "REGISTRY_NAMESPACE": "ns", + "REGISTRY_INSECURE": "false", + } + defaults.update(registry_overrides) + for k, v in defaults.items(): + monkeypatch.setenv(k, v) + + config_dir = tmp_path / "config" + config_dir.mkdir(parents=True, exist_ok=True) + repo_path = tmp_path / "source" + repo_path.mkdir(parents=True, exist_ok=True) + (repo_path / "placeholder").write_text("") + + import argparse + args = argparse.Namespace( + config_dir=config_dir, + repo_path=repo_path, + workspace_path=".", + use_local=False, + push_images=push_images, + log_level="INFO", + source_repo=None, + source_ref=None, + ) + + with patch("src.rhdh_dynamic_plugin_factory.config.PluginFactoryConfig._buildah_login"): + return PluginFactoryConfig.load_from_env(args, push_images=push_images) + + def test_updates_fields_from_environ(self, tmp_path, monkeypatch): + """Test that refresh reads new values from os.environ.""" + config = self._make_config(monkeypatch, tmp_path) + + assert config.registry_url == "quay.io" + assert config.registry_namespace == "ns" + + monkeypatch.setenv("REGISTRY_URL", "ghcr.io") + monkeypatch.setenv("REGISTRY_NAMESPACE", "new-ns") + + config.refresh_registry_config() + + assert config.registry_url == "ghcr.io" + assert config.registry_namespace == "new-ns" + + def test_relogin_triggered_when_creds_change(self, tmp_path, monkeypatch): + """Test that buildah login is re-run when registry credentials change.""" + config = self._make_config(monkeypatch, tmp_path, push_images=True) + + monkeypatch.setenv("REGISTRY_URL", "ghcr.io") + monkeypatch.setenv("REGISTRY_USERNAME", "new-user") + monkeypatch.setenv("REGISTRY_PASSWORD", "new-pass") + + with patch.object(config, "_buildah_login") as mock_login: + config.refresh_registry_config() + mock_login.assert_called_once() + + def test_relogin_skipped_when_creds_unchanged(self, tmp_path, monkeypatch): + """Test that buildah login is NOT re-run when credentials haven't changed.""" + config = self._make_config(monkeypatch, tmp_path, push_images=True) + + with patch.object(config, "_buildah_login") as mock_login: + config.refresh_registry_config() + mock_login.assert_not_called() + + def test_relogin_skipped_when_push_images_disabled(self, tmp_path, monkeypatch): + """Test that buildah login is skipped when push_images is False.""" + config = self._make_config(monkeypatch, tmp_path, push_images=False) + + monkeypatch.setenv("REGISTRY_URL", "ghcr.io") + + with patch.object(config, "_buildah_login") as mock_login: + config.refresh_registry_config() + mock_login.assert_not_called() + + def test_validation_error_on_missing_fields_after_refresh(self, tmp_path, monkeypatch): + """Test that missing required fields after refresh raise ConfigurationError.""" + config = self._make_config(monkeypatch, tmp_path, push_images=True) + + monkeypatch.delenv("REGISTRY_URL") + monkeypatch.setenv("REGISTRY_USERNAME", "changed") + + with pytest.raises(ConfigurationError, match="REGISTRY_URL"): + config.refresh_registry_config() + + def test_namespace_change_without_cred_change_no_relogin(self, tmp_path, monkeypatch): + """Test that changing only namespace updates the field but skips re-login.""" + config = self._make_config(monkeypatch, tmp_path, push_images=True) + + monkeypatch.setenv("REGISTRY_NAMESPACE", "different-ns") + + with patch.object(config, "_buildah_login") as mock_login: + config.refresh_registry_config() + assert config.registry_namespace == "different-ns" + mock_login.assert_not_called() diff --git a/tests/test_source_config.py b/tests/test_source_config.py index 3eca34c..cde56f8 100644 --- a/tests/test_source_config.py +++ b/tests/test_source_config.py @@ -6,6 +6,7 @@ import json import subprocess +from pathlib import Path from unittest.mock import patch, MagicMock import pytest @@ -57,7 +58,38 @@ def test_from_file_empty_repo(self, tmp_path): with pytest.raises(ConfigurationError, match="repo is required"): SourceConfig.from_file(source_file) - + def test_from_file_empty_repo_ref_resolves_default(self, tmp_path): + """Test that empty repo-ref triggers default branch resolution.""" + source_data = { + "repo": "https://github.com/test/repo", + "repo-ref": "", + "workspace-path": "." + } + + source_file = tmp_path / "source.json" + source_file.write_text(json.dumps(source_data)) + + with patch.object(SourceConfig, "resolve_default_ref", return_value="refs/heads/main"): + config = SourceConfig.from_file(source_file) + assert config.repo == "https://github.com/test/repo" + assert config.repo_ref == "refs/heads/main" + assert config.workspace_path == "." + + def test_from_file_missing_repo_ref_resolves_default(self, tmp_path): + """Test that omitted repo-ref triggers default branch resolution.""" + source_data = { + "repo": "https://github.com/test/repo", + "workspace-path": "." + } + + source_file = tmp_path / "source.json" + source_file.write_text(json.dumps(source_data)) + + with patch.object(SourceConfig, "resolve_default_ref", return_value="refs/heads/main"): + config = SourceConfig.from_file(source_file) + assert config.repo == "https://github.com/test/repo" + assert config.repo_ref == "refs/heads/main" + assert config.workspace_path == "." def test_from_file_malformed_json(self, tmp_path): """Test that malformed JSON raises ConfigurationError with descriptive message.""" @@ -147,24 +179,23 @@ def test_resolve_default_ref_git_failure(self): with pytest.raises(ExecutionError, match="Failed to resolve default branch"): SourceConfig.resolve_default_ref("https://github.com/test/nonexistent") - def test_resolve_default_ref_empty_output(self): - """Test that empty git ls-remote output raises ExecutionError.""" + def test_resolve_default_ref_no_symbolic_ref(self): + """Test that missing symbolic ref raises ConfigurationError with actionable message.""" mock_result = MagicMock() - mock_result.stdout = "" + mock_result.stdout = "abc123def456\tHEAD\n" with patch("subprocess.run", return_value=mock_result): - with pytest.raises(ExecutionError, match="Could not determine default branch"): + with pytest.raises(ConfigurationError, match="Could not resolve the default branch"): SourceConfig.resolve_default_ref("https://github.com/test/repo") - def test_resolve_default_ref_sha_fallback(self): - """Test falling back to SHA when no symbolic ref line is present.""" + def test_resolve_default_ref_empty_output(self): + """Test that empty git ls-remote output raises ConfigurationError.""" mock_result = MagicMock() - mock_result.stdout = "abc123def456\tHEAD\n" + mock_result.stdout = "" with patch("subprocess.run", return_value=mock_result): - ref = SourceConfig.resolve_default_ref("https://github.com/test/repo") - - assert ref == "abc123def456" + with pytest.raises(ConfigurationError, match="Could not resolve the default branch"): + SourceConfig.resolve_default_ref("https://github.com/test/repo") class TestSourceConfigCloneToPath: @@ -298,234 +329,187 @@ def test_clone_to_path_exception(self, tmp_path): class TestSourceConfigCloneToPathClean: - """Tests for SourceConfig.clone_to_path clean argument and user prompt behavior.""" - - def test_clean_flag_auto_cleans_nonempty_directory(self, tmp_path): - """Test that clean=True automatically cleans a non-empty directory without prompting.""" - config = SourceConfig( - repo="https://github.com/testowner/testrepo", - repo_ref="main", - workspace_path="." - ) + """Tests for SourceConfig.clone_to_path clean argument and user prompt behavior. + + Uses real filesystem operations to verify that nested directory contents are + actually removed or preserved as expected. + """ - repo_path = tmp_path / "repo" - repo_path.mkdir() + @staticmethod + def _make_nested_repo(repo_path: Path) -> None: + """Create a realistic nested directory structure for testing.""" + repo_path.mkdir(exist_ok=True) (repo_path / "existing_file.txt").write_text("existing content") - (repo_path / "subdir").mkdir() - (repo_path / "subdir" / "nested.txt").write_text("nested content") - - with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run, \ - patch("src.rhdh_dynamic_plugin_factory.config.clean_directory") as mock_clean: - mock_run.return_value = 0 - - config.clone_to_path(repo_path, clean=True) # Should not raise any exceptions - - mock_clean.assert_called_once_with(repo_path) - - def test_clean_flag_does_not_prompt_user(self, tmp_path): - """Test that clean=True does not call input() for user confirmation.""" - config = SourceConfig( + (repo_path / ".hidden_config").write_text("hidden") + src = repo_path / "src" + src.mkdir() + (src / "index.ts").write_text("export {}") + components = src / "components" + components.mkdir() + (components / "App.tsx").write_text("") + modules = repo_path / "node_modules" / "package" + modules.mkdir(parents=True) + (modules / "index.js").write_text("module.exports = {}") + + @staticmethod + def _make_config(repo_ref: str = "main") -> SourceConfig: + return SourceConfig( repo="https://github.com/testowner/testrepo", - repo_ref="main", + repo_ref=repo_ref, workspace_path="." ) + def test_clean_flag_auto_cleans_nested_contents(self, tmp_path): + """Test that clean=True removes all nested contents without prompting.""" + config = self._make_config() repo_path = tmp_path / "repo" - repo_path.mkdir() - (repo_path / "existing_file.txt").write_text("existing content") + self._make_nested_repo(repo_path) with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run, \ - patch("src.rhdh_dynamic_plugin_factory.config.clean_directory"), \ patch("builtins.input") as mock_input: mock_run.return_value = 0 config.clone_to_path(repo_path, clean=True) mock_input.assert_not_called() + assert repo_path.exists(), "Directory itself should still exist" + assert list(repo_path.iterdir()) == [], "All nested contents should be removed" def test_no_clean_flag_prompts_user_confirm_yes(self, tmp_path): - """Test that clean=False prompts user and proceeds when user enters 'y'.""" - config = SourceConfig( - repo="https://github.com/testowner/testrepo", - repo_ref="main", - workspace_path="." - ) - + """Test that clean=False prompts user and cleans nested contents when user enters 'y'.""" + config = self._make_config() repo_path = tmp_path / "repo" - repo_path.mkdir() - (repo_path / "existing_file.txt").write_text("existing content") + self._make_nested_repo(repo_path) with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run, \ - patch("src.rhdh_dynamic_plugin_factory.config.clean_directory") as mock_clean, \ patch("builtins.input", return_value="y"): mock_run.return_value = 0 - config.clone_to_path(repo_path, clean=False) # Should not raise any exceptions + config.clone_to_path(repo_path, clean=False) - mock_clean.assert_called_once_with(repo_path) - - def test_no_clean_flag_prompts_user_confirm_no(self, tmp_path): - """Test that clean=False prompts user and raises PluginFactoryError when user declines.""" - config = SourceConfig( - repo="https://github.com/testowner/testrepo", - repo_ref="main", - workspace_path="." - ) + assert repo_path.exists(), "Directory itself should still exist" + assert list(repo_path.iterdir()) == [], "All nested contents should be removed" + def test_no_clean_flag_prompts_user_confirm_no_preserves_contents(self, tmp_path): + """Test that declining the prompt preserves all nested contents.""" + config = self._make_config() repo_path = tmp_path / "repo" - repo_path.mkdir() - (repo_path / "existing_file.txt").write_text("existing content") + self._make_nested_repo(repo_path) - with patch("builtins.input", return_value="n") as mock_input: + original_contents = {p.name for p in repo_path.rglob("*")} + + with patch("builtins.input", return_value="n"): with pytest.raises(PluginFactoryError, match="aborted by user"): config.clone_to_path(repo_path, clean=False) - mock_input.assert_called_once() - - def test_no_clean_flag_prompts_user_empty_input_aborts(self, tmp_path): - """Test that clean=False raises PluginFactoryError when user presses Enter without typing anything.""" - config = SourceConfig( - repo="https://github.com/testowner/testrepo", - repo_ref="main", - workspace_path="." - ) + remaining_contents = {p.name for p in repo_path.rglob("*")} + assert remaining_contents == original_contents, "No files should have been removed" + def test_no_clean_flag_empty_input_preserves_contents(self, tmp_path): + """Test that pressing Enter without input preserves all nested contents.""" + config = self._make_config() repo_path = tmp_path / "repo" - repo_path.mkdir() - (repo_path / "existing_file.txt").write_text("existing content") + self._make_nested_repo(repo_path) + + original_contents = {p.name for p in repo_path.rglob("*")} with patch("builtins.input", return_value=""): with pytest.raises(PluginFactoryError, match="aborted by user"): config.clone_to_path(repo_path, clean=False) - def test_empty_directory_skips_clean_and_prompt(self, tmp_path): - """Test that an empty directory skips both clean and prompt logic.""" - config = SourceConfig( - repo="https://github.com/testowner/testrepo", - repo_ref="main", - workspace_path="." - ) + remaining_contents = {p.name for p in repo_path.rglob("*")} + assert remaining_contents == original_contents, "No files should have been removed" + def test_empty_directory_skips_clean_and_prompt(self, tmp_path): + """Test that an empty directory skips both clean and user prompt.""" + config = self._make_config() repo_path = tmp_path / "repo" repo_path.mkdir() - # Directory is empty with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run, \ - patch("src.rhdh_dynamic_plugin_factory.config.clean_directory") as mock_clean, \ patch("builtins.input") as mock_input: mock_run.return_value = 0 - config.clone_to_path(repo_path, clean=True) # Should not raise + config.clone_to_path(repo_path, clean=True) - mock_clean.assert_not_called() mock_input.assert_not_called() def test_empty_directory_no_clean_flag_skips_prompt(self, tmp_path): """Test that an empty directory with clean=False does not prompt user.""" - config = SourceConfig( - repo="https://github.com/testowner/testrepo", - repo_ref="main", - workspace_path="." - ) - + config = self._make_config() repo_path = tmp_path / "repo" repo_path.mkdir() - # Directory is empty with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run, \ patch("builtins.input") as mock_input: mock_run.return_value = 0 - config.clone_to_path(repo_path, clean=False) # Should not raise any exceptions] + config.clone_to_path(repo_path, clean=False) mock_input.assert_not_called() def test_clean_flag_default_is_false(self, tmp_path): """Test that the clean parameter defaults to False.""" - config = SourceConfig( - repo="https://github.com/testowner/testrepo", - repo_ref="main", - workspace_path="." - ) - + config = self._make_config() repo_path = tmp_path / "repo" - repo_path.mkdir() - (repo_path / "existing_file.txt").write_text("existing content") + self._make_nested_repo(repo_path) with patch("builtins.input", return_value="n"): - # Call without clean argument - should prompt and raise PluginFactoryError when user declines with pytest.raises(PluginFactoryError, match="aborted by user"): config.clone_to_path(repo_path) def test_clean_proceeds_with_clone_after_cleaning(self, tmp_path): - """Test that after cleaning, git clone and checkout are executed.""" - config = SourceConfig( - repo="https://github.com/testowner/testrepo", - repo_ref="v1.0.0", - workspace_path="." - ) - + """Test that after cleaning nested contents, git clone and checkout are executed.""" + config = self._make_config(repo_ref="v1.0.0") repo_path = tmp_path / "repo" - repo_path.mkdir() - (repo_path / "old_file.txt").write_text("old content") + self._make_nested_repo(repo_path) - with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run, \ - patch("src.rhdh_dynamic_plugin_factory.config.clean_directory"): + with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run: mock_run.return_value = 0 - config.clone_to_path(repo_path, clean=True) # Should not raise any exceptions + config.clone_to_path(repo_path, clean=True) - assert mock_run.call_count == 2 # clone + checkout + assert list(repo_path.iterdir()) == [], "Directory should be empty before clone runs" + assert mock_run.call_count == 2 - # Verify clone command clone_call = mock_run.call_args_list[0] assert clone_call[0][0] == ["git", "clone", "https://github.com/testowner/testrepo", str(repo_path)] - # Verify checkout command checkout_call = mock_run.call_args_list[1] assert checkout_call[0][0] == ["git", "checkout", "v1.0.0"] def test_prompt_confirm_yes_proceeds_with_clone(self, tmp_path): - """Test that after user confirms 'y', git clone and checkout are executed.""" - config = SourceConfig( - repo="https://github.com/testowner/testrepo", - repo_ref="main", - workspace_path="." - ) - + """Test that after user confirms 'y', nested contents are cleaned and clone/checkout run.""" + config = self._make_config() repo_path = tmp_path / "repo" - repo_path.mkdir() - (repo_path / "old_file.txt").write_text("old content") + self._make_nested_repo(repo_path) with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run, \ - patch("src.rhdh_dynamic_plugin_factory.config.clean_directory"), \ patch("builtins.input", return_value="y"): mock_run.return_value = 0 - config.clone_to_path(repo_path, clean=False) # Should not raise any exceptions + config.clone_to_path(repo_path, clean=False) - assert mock_run.call_count == 2 # clone + checkout + assert list(repo_path.iterdir()) == [], "Directory should be empty before clone runs" + assert mock_run.call_count == 2 def test_prompt_confirm_no_does_not_clone(self, tmp_path): - """Test that when user declines, git clone is not executed.""" - config = SourceConfig( - repo="https://github.com/testowner/testrepo", - repo_ref="main", - workspace_path="." - ) - + """Test that when user declines, no nested contents are removed and clone does not run.""" + config = self._make_config() repo_path = tmp_path / "repo" - repo_path.mkdir() - (repo_path / "existing_file.txt").write_text("existing content") + self._make_nested_repo(repo_path) + + original_contents = {p.name for p in repo_path.rglob("*")} with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run, \ patch("builtins.input", return_value="n"): - with pytest.raises(PluginFactoryError, match="aborted by user"): config.clone_to_path(repo_path, clean=False) mock_run.assert_not_called() + remaining_contents = {p.name for p in repo_path.rglob("*")} + assert remaining_contents == original_contents, "No files should have been removed" class TestDiscoverSourceConfigCliArgs: """Tests for PluginFactoryConfig.discover_source_config with CLI args.""" @@ -584,22 +568,15 @@ def test_falls_back_to_source_json_when_no_cli_args(self, make_config, setup_tes assert source_config.repo == "https://github.com/awslabs/backstage-plugins-for-aws" assert source_config.repo_ref == "78df9399a81cfd95265cab53815f54210b1d7f50" - def test_workspace_path_from_source_json(self, tmp_path, monkeypatch): + def test_workspace_path_from_source_json(self, tmp_path, monkeypatch, write_source_json): """Test that workspace_path is resolved from source.json when not provided via CLI.""" monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") config_dir = tmp_path / "config" - config_dir.mkdir(parents=True, exist_ok=True) source_dir = tmp_path / "source" source_dir.mkdir(parents=True, exist_ok=True) - # Create source.json with workspace-path - source_data = { - "repo": "https://github.com/test/repo", - "repo-ref": "main", - "workspace-path": "workspaces/todo" - } - (config_dir / "source.json").write_text(json.dumps(source_data)) + write_source_json(config_dir, "https://github.com/test/repo", "main", "workspaces/todo") config = PluginFactoryConfig( rhdh_cli_version="1.7.2", From ffc6519ced49f683fabaca962fe055370ab11f7b Mon Sep 17 00:00:00 2001 From: Frank Kong Date: Tue, 10 Mar 2026 18:20:09 -0400 Subject: [PATCH 08/35] feat: add initial basic plugins-list.yaml generation Assisted-By: Cursor Signed-off-by: Frank Kong --- .cursor/rules/development-workflow.mdc | 7 + src/rhdh_dynamic_plugin_factory/cli.py | 14 +- src/rhdh_dynamic_plugin_factory/config.py | 144 +++++++++++--- tests/test_plugin_list_config.py | 218 ++++++++++++++++++++++ 4 files changed, 346 insertions(+), 37 deletions(-) diff --git a/.cursor/rules/development-workflow.mdc b/.cursor/rules/development-workflow.mdc index adeb1df..86d66fb 100644 --- a/.cursor/rules/development-workflow.mdc +++ b/.cursor/rules/development-workflow.mdc @@ -3,3 +3,10 @@ description: Development workflow commands and processes for the rhdh-dynamic-pl globs: alwaysApply: true --- + +## Dependency Management + +When adding new Python dependencies: +1. Install using the `.venv` virtual environment: `.venv/bin/pip install ` +2. Update `requirements.txt` (runtime deps) or `requirements.dev.txt` (dev/test deps) to reflect the change +3. Verify the installed version matches what is recorded in the requirements file diff --git a/src/rhdh_dynamic_plugin_factory/cli.py b/src/rhdh_dynamic_plugin_factory/cli.py index c47a8d2..3eabdd6 100644 --- a/src/rhdh_dynamic_plugin_factory/cli.py +++ b/src/rhdh_dynamic_plugin_factory/cli.py @@ -174,12 +174,6 @@ def _process_workspace( workspace_path: Relative path from repo_path to the workspace. output_dir: Output directory for build artifacts. """ - config.auto_generate_plugins_list( - config_dir=workspace_config_dir, - repo_path=repo_path, - workspace_path=workspace_path, - ) - logger.info("[bold blue]Applying Patches and Overlays[/bold blue]") config.apply_patches_and_overlays( config_dir=workspace_config_dir, @@ -191,6 +185,12 @@ def _process_workspace( full_workspace_path = Path(repo_path).joinpath(workspace_path).absolute() install_dependencies(full_workspace_path) + config.auto_generate_plugins_list( + config_dir=workspace_config_dir, + repo_path=repo_path, + workspace_path=workspace_path, + ) + logger.info("[bold blue]Exporting plugins using RHDH CLI[/bold blue]") config.export_plugins( output_dir=output_dir, @@ -285,7 +285,7 @@ def _run_multi_workspace(args: argparse.Namespace, workspaces: list[WorkspaceInf logger.info("[bold blue]Workspaces to be processed:[/bold blue]") for ws in workspaces: logger.info(f" - {ws.name}: {ws.source_config.repo} @ {ws.source_config.repo_ref}") - # Resolve per-workspace source and output paths + # Resolve per-workspace source and output paths to avoid conflicts between workspaces ws.resolve_paths(base_repo_path, base_output_dir) # Load global config (uses root .env for global settings like registry credentials) diff --git a/src/rhdh_dynamic_plugin_factory/config.py b/src/rhdh_dynamic_plugin_factory/config.py index 6cb653b..819fc66 100644 --- a/src/rhdh_dynamic_plugin_factory/config.py +++ b/src/rhdh_dynamic_plugin_factory/config.py @@ -20,6 +20,8 @@ @dataclass class PluginFactoryConfig: """Main configuration for the plugin factory.""" + PLUGIN_LIST_FILE: ClassVar[str] = "plugins-list.yaml" + SOURCE_CONFIG_FILE: ClassVar[str] = "source.json" # Required fields loaded from default.env file (can be overridden by environment variables) rhdh_cli_version: str = field(default="") @@ -216,35 +218,35 @@ def _validate_source_json(self) -> None: CLI args fully replace source.json. """ if self.source_repo: - self.logger.debug("Using --source-repo CLI argument, skipping source.json validation") + self.logger.debug(f"Using --source-repo CLI argument, skipping {self.SOURCE_CONFIG_FILE} validation") return - source_file = os.path.join(self.config_dir, "source.json") + source_file = os.path.join(self.config_dir, self.SOURCE_CONFIG_FILE) if not os.path.exists(source_file): if not os.path.exists(self.repo_path) or not os.listdir(self.repo_path): raise ConfigurationError( - f"source.json not found at {source_file} and {self.repo_path} is empty. " - "Please provide source.json to clone a repository, use --source-repo to specify a repository via CLI, " + f"{self.SOURCE_CONFIG_FILE} not found at {source_file} and {self.repo_path} is empty. " + "Please provide {self.SOURCE_CONFIG_FILE} to clone a repository, use --source-repo to specify a repository via CLI, " "or use --use-local with a locally mounted repository." ) else: self.logger.warning( - f"source.json not found at {source_file}. Will attempt to use local repository content at {self.repo_path}" + f"{self.SOURCE_CONFIG_FILE} not found at {source_file}. Will attempt to use local repository content at {self.repo_path}" ) else: self.logger.debug(f"Using source configuration from: {source_file}") def _validate_plugins_list(self) -> None: """Validate plugins-list.yaml file existence.""" - plugins_file = os.path.join(self.config_dir, "plugins-list.yaml") + plugins_file = os.path.join(self.config_dir, self.PLUGIN_LIST_FILE) if not os.path.exists(plugins_file): self.logger.warning( - f"plugins-list.yaml not found at {plugins_file}. Will attempt to auto-generate after repository is available." + f"{self.PLUGIN_LIST_FILE} not found at {plugins_file}. Will attempt to auto-generate after repository is available." ) else: - self.logger.debug(f"Using plugins-list.yaml from: {plugins_file}") + self.logger.debug(f"Using {self.PLUGIN_LIST_FILE} from: {plugins_file}") def auto_generate_plugins_list(self, config_dir: Optional[str] = None, repo_path: Optional[str] = None, @@ -263,13 +265,13 @@ def auto_generate_plugins_list(self, config_dir: Optional[str] = None, repo_path = repo_path or self.repo_path workspace_path = workspace_path or self.workspace_path - plugins_file = os.path.join(config_dir, "plugins-list.yaml") + plugins_file = os.path.join(config_dir, self.PLUGIN_LIST_FILE) if os.path.exists(plugins_file): - self.logger.debug(f"[green]plugins-list.yaml already exists at {plugins_file}. Skipping auto-generation.[/green]") + self.logger.debug(f"[green]{self.PLUGIN_LIST_FILE} already exists at {plugins_file}. Skipping auto-generation.[/green]") return - self.logger.info("[bold blue]Auto-generating plugins-list.yaml[/bold blue]") + self.logger.info(f"[bold blue]Auto-generating {self.PLUGIN_LIST_FILE}[/bold blue]") if not os.path.exists(repo_path): raise PluginFactoryError(f"Source code repository does not exist at {repo_path}") @@ -279,15 +281,17 @@ def auto_generate_plugins_list(self, config_dir: Optional[str] = None, raise PluginFactoryError(f"Plugin workspace does not exist at {workspace_full_path}") try: - # TODO: Implement PluginListConfig.create_default function plugin_cfg = PluginListConfig.create_default(workspace_path=Path(workspace_full_path)) plugin_cfg.to_file(Path(plugins_file)) plugins = plugin_cfg.get_plugins() if plugins: - self.logger.info(f"Generated plugins-list.yaml with {len(plugins)} plugins") + self.logger.info(f"Generated {self.PLUGIN_LIST_FILE} with {len(plugins)} plugin(s)") for plugin_path, build_args in plugins.items(): - self.logger.info(f" - {plugin_path}: {build_args}") + if build_args: + self.logger.info(f" - {plugin_path}: {build_args}") + else: + self.logger.info(f" - {plugin_path}") else: self.logger.warning("No plugins found in workspace") except PluginFactoryError: @@ -318,7 +322,7 @@ def discover_source_config(self) -> Optional["SourceConfig"]: self.logger.debug(f"Using source config from CLI: {source_config}") return source_config - source_file = os.path.join(self.config_dir, "source.json") + source_file = os.path.join(self.config_dir, self.SOURCE_CONFIG_FILE) if os.path.exists(source_file) and not self.use_local: # SourceConfig.from_file() raises ConfigurationError on failure, so let it propagate to cli.py @@ -353,7 +357,7 @@ def setup_config_directory(self) -> Optional["SourceConfig"]: self.logger.info(f" Repository: {source_config.repo}") self.logger.info(f" Reference: {source_config.repo_ref}") - plugins_list_file = os.path.join(self.config_dir, "plugins-list.yaml") + plugins_list_file = os.path.join(self.config_dir, self.PLUGIN_LIST_FILE) if os.path.exists(plugins_list_file): self.logger.info(f"Using plugin list file: {plugins_list_file}") @@ -455,7 +459,7 @@ def export_plugins(self, output_dir: str, config_dir: Optional[str] = None, step=STEP_NAME ) - plugins_list_file = os.path.join(config_dir, "plugins-list.yaml") + plugins_list_file = os.path.join(config_dir, self.PLUGIN_LIST_FILE) if not os.path.exists(plugins_list_file): raise ConfigurationError("No plugins file found") @@ -640,7 +644,7 @@ def resolve_default_ref(repo: str) -> str: raise ConfigurationError( f"Could not resolve the default branch for '{repo}'. " - "Please specify a branch or ref explicitly via 'repo-ref' in source.json " + f"Please specify a branch or ref explicitly via 'repo-ref' in {PluginFactoryConfig.SOURCE_CONFIG_FILE} " "or the --source-ref CLI argument." ) except subprocess.CalledProcessError as e: @@ -758,9 +762,9 @@ def discover_workspaces(config_dir: Path) -> list["WorkspaceInfo"]: if not entry.is_dir(): continue - source_file = entry / "source.json" + source_file = entry / PluginFactoryConfig.SOURCE_CONFIG_FILE if not source_file.exists(): - logger.debug(f"Skipping {entry.name}/ — no source.json") + logger.debug(f"Skipping {entry.name}/ — no {PluginFactoryConfig.SOURCE_CONFIG_FILE}") continue workspace_name = entry.name @@ -824,7 +828,7 @@ def clone_workspaces_with_worktrees( if returncode != 0: raise ExecutionError( f"Failed to clone repository '{repo_url}' (exit code {returncode}). " - f"Please verify the 'repo' URL in the source.json for workspaces using this repository. " + f"Please verify the 'repo' URL in the {PluginFactoryConfig.SOURCE_CONFIG_FILE} for workspaces using this repository. " f"Ensure the URL is correct and accessible from your environment.", step="git clone (bare)", returncode=returncode, @@ -853,7 +857,7 @@ def clone_workspaces_with_worktrees( if returncode != 0: raise ExecutionError( f"Failed to create worktree for workspace '{ws.name}' at ref '{ref}' (exit code {returncode}). " - f"Please verify the 'repo-ref' value in {ws.config_dir / 'source.json'}. " + f"Please verify the 'repo-ref' value in {ws.config_dir / PluginFactoryConfig.SOURCE_CONFIG_FILE}. " f"Ensure the branch, tag, or commit exists in the repository.", step="git worktree add", returncode=returncode, @@ -864,7 +868,22 @@ def clone_workspaces_with_worktrees( class PluginListConfig: """Configuration for plugin list (YAML format).""" - + VALID_BACKSTAGE_PLUGIN_ROLES: ClassVar[set[str]] = { + "frontend-plugin", + "backend-plugin", + "frontend-plugin-module", + "backend-plugin-module", + } + + SKIP_DIRS: ClassVar[set[str]] = { + "node_modules", + "dist", + "dist-dynamic", + ".git", + } + + logger: ClassVar[Logger] = get_logger("plugin_list") + def __init__(self, plugins: Dict[str, str]): """ Initialize plugin list configuration. @@ -891,9 +910,20 @@ def from_file(cls, plugin_list_file: Path) -> "PluginListConfig": return cls(plugins) def to_file(self, plugin_list_file: Path) -> None: - """Save plugin list to YAML file.""" - # TODO: Implement this function - raise NotImplementedError("TODO: Saving plugin list to file is not supported yet") + """Save plugin list to YAML file. + + Writes manually rather than via yaml.dump so that entries with no + build args appear as ``key:`` (YAML null) instead of ``key: ''``. + + Args: + plugin_list_file: Destination path for the YAML file. + """ + with open(plugin_list_file, 'w') as f: + for path, args in self.plugins.items(): + if args: + f.write(f"{path}: {args}\n") + else: + f.write(f"{path}:\n") def get_plugins(self) -> Dict[str, str]: return self.plugins.copy() @@ -903,10 +933,64 @@ def add_plugin(self, plugin_path: str, build_args: str = "") -> None: def remove_plugin(self, plugin_path: str) -> None: self.plugins.pop(plugin_path, None) - + @classmethod def create_default(cls, workspace_path: Path) -> "PluginListConfig": - """Create a default plugin list by scanning workspace.""" - # TODO: Implement this function - raise NotImplementedError("TODO: default plugin list creation is not supported yet") + """Create a default plugin list by scanning workspace for Backstage plugins. + + Recursively walks ``workspace_path`` to find ``package.json`` files whose + ``backstage.role`` field matches one of :pyattr:`VALID_BACKSTAGE_ROLES`, + and returns a :class:`PluginListConfig` keyed by plugin directory paths relative to ``workspace_path``. + + Args: + workspace_path: Absolute path to the workspace root. + + Returns: + A :class:`PluginListConfig` with discovered plugins (no build args). + """ + plugins: Dict[str, str] = {} + + for pkg_json_path in cls._find_package_jsons(workspace_path): + role = cls._read_backstage_role(pkg_json_path) + if role and role in cls.VALID_BACKSTAGE_PLUGIN_ROLES: + plugin_path = pkg_json_path.parent.relative_to(workspace_path).as_posix() + plugins[plugin_path] = "" + + sorted_plugins = dict(sorted(plugins.items())) + cls.logger.debug(f"Discovered {len(sorted_plugins)} plugin(s) in {workspace_path}") + return cls(sorted_plugins) + + @classmethod + def _find_package_jsons(cls, root: Path) -> list[Path]: + """Recursively find package.json files, skipping non-plugin directories.""" + results: list[Path] = [] + + for entry in sorted(root.iterdir()): + if not entry.is_dir(): + continue + if entry.name in cls.SKIP_DIRS or entry.name.startswith("."): + continue + + pkg_json = entry / "package.json" + if pkg_json.is_file(): + results.append(pkg_json) + + results.extend(cls._find_package_jsons(entry)) + + return results + + @classmethod + def _read_backstage_role(cls, pkg_json_path: Path) -> Optional[str]: + """Read the ``backstage.role`` field from a package.json file. + + Returns: + The role string, or *None* if the file cannot be parsed or has no role. + """ + try: + data = json.loads(pkg_json_path.read_text(encoding="utf-8")) + cls.logger.debug(f"Read backstage role from {pkg_json_path}: {data.get('backstage', {}).get('role')}") + return data.get("backstage", {}).get("role") + except (json.JSONDecodeError, OSError) as e: + cls.logger.warning(f"Failed to read {pkg_json_path}: {e}") + return None diff --git a/tests/test_plugin_list_config.py b/tests/test_plugin_list_config.py index 360aa5a..e318245 100644 --- a/tests/test_plugin_list_config.py +++ b/tests/test_plugin_list_config.py @@ -4,6 +4,7 @@ Tests the plugin list configuration loading and management. """ +import json import yaml import pytest @@ -155,3 +156,220 @@ def test_remove_plugin_nonexistent(self): # Original plugin should still be there assert "plugins/test1" in config.plugins + + +class TestPluginListConfigToFile: + """Tests for PluginListConfig.to_file method.""" + + def test_to_file_with_args(self, tmp_path): + """Test writing plugins with build args.""" + config = PluginListConfig({ + "plugins/ecs/frontend": "", + "plugins/ecs/backend": "--embed-package @aws/aws-core-plugin-for-backstage-common", + }) + out = tmp_path / "plugins-list.yaml" + config.to_file(out) + + lines = out.read_text().splitlines() + assert lines[0] == "plugins/ecs/frontend:" + assert lines[1] == "plugins/ecs/backend: --embed-package @aws/aws-core-plugin-for-backstage-common" + + def test_to_file_empty_plugins(self, tmp_path): + """Test writing empty plugins dict produces empty file.""" + config = PluginListConfig({}) + out = tmp_path / "plugins-list.yaml" + config.to_file(out) + + assert out.read_text() == "" + + def test_to_file_roundtrip(self, tmp_path): + """Test that to_file output can be read back by from_file.""" + original = PluginListConfig({ + "plugins/todo": "", + "plugins/todo-backend": "", + "plugins/ecs/backend": "--embed-package @aws/common --embed-package @aws/node", + }) + out = tmp_path / "plugins-list.yaml" + original.to_file(out) + + loaded = PluginListConfig.from_file(out) + assert loaded.get_plugins() == original.get_plugins() + + def test_to_file_null_values_format(self, tmp_path): + """Test that empty args produce 'key:' (YAML null) not 'key: \"\"'.""" + config = PluginListConfig({"plugins/test": ""}) + out = tmp_path / "plugins-list.yaml" + config.to_file(out) + + content = out.read_text() + assert "plugins/test:" in content + assert "''" not in content + assert '""' not in content + + +def _make_plugin_dir(base, rel_path, name, role): + """Helper to create a plugin directory with a package.json.""" + plugin_dir = base / rel_path + plugin_dir.mkdir(parents=True, exist_ok=True) + pkg = {"name": name, "version": "1.0.0", "backstage": {"role": role}} + (plugin_dir / "package.json").write_text(json.dumps(pkg)) + return plugin_dir + + +class TestPluginListConfigCreateDefault: + """Tests for PluginListConfig.create_default method.""" + + def test_discovers_backend_plugin(self, tmp_path): + """Test discovering a backend plugin.""" + _make_plugin_dir(tmp_path, "plugins/todo-backend", "@backstage/plugin-todo-backend", "backend-plugin") + + config = PluginListConfig.create_default(tmp_path) + plugins = config.get_plugins() + + assert "plugins/todo-backend" in plugins + assert plugins["plugins/todo-backend"] == "" + + def test_discovers_frontend_plugin(self, tmp_path): + """Test discovering a frontend plugin.""" + _make_plugin_dir(tmp_path, "plugins/todo", "@backstage/plugin-todo", "frontend-plugin") + + config = PluginListConfig.create_default(tmp_path) + + assert "plugins/todo" in config.get_plugins() + + def test_discovers_plugin_modules(self, tmp_path): + """Test discovering frontend and backend plugin modules.""" + _make_plugin_dir(tmp_path, "plugins/auth-backend-module-github", "@backstage/plugin-auth-backend-module-github", "backend-plugin-module") + _make_plugin_dir(tmp_path, "plugins/catalog-react-module", "@backstage/plugin-catalog-react-module", "frontend-plugin-module") + + config = PluginListConfig.create_default(tmp_path) + plugins = config.get_plugins() + + assert "plugins/auth-backend-module-github" in plugins + assert "plugins/catalog-react-module" in plugins + + def test_ignores_non_plugin_roles(self, tmp_path): + """Test that packages with roles like 'node-library' or 'common-library' are skipped.""" + _make_plugin_dir(tmp_path, "packages/backend-defaults", "@backstage/backend-defaults", "node-library") + _make_plugin_dir(tmp_path, "packages/catalog-common", "@backstage/catalog-common", "common-library") + + config = PluginListConfig.create_default(tmp_path) + + assert config.get_plugins() == {} + + def test_ignores_package_without_backstage_field(self, tmp_path): + """Test that package.json without backstage field is skipped.""" + pkg_dir = tmp_path / "packages" / "some-lib" + pkg_dir.mkdir(parents=True) + (pkg_dir / "package.json").write_text(json.dumps({"name": "some-lib", "version": "1.0.0"})) + + config = PluginListConfig.create_default(tmp_path) + + assert config.get_plugins() == {} + + def test_skips_node_modules(self, tmp_path): + """Test that plugins inside node_modules are not discovered.""" + nm = tmp_path / "node_modules" / "@backstage" / "plugin-todo" + nm.mkdir(parents=True) + (nm / "package.json").write_text(json.dumps({ + "name": "@backstage/plugin-todo", + "backstage": {"role": "frontend-plugin"}, + })) + + config = PluginListConfig.create_default(tmp_path) + + assert config.get_plugins() == {} + + def test_skips_hidden_directories(self, tmp_path): + """Test that plugins inside hidden directories are not discovered.""" + hidden = tmp_path / ".hidden" / "plugin-todo" + hidden.mkdir(parents=True) + (hidden / "package.json").write_text(json.dumps({ + "name": "@test/plugin-todo", + "backstage": {"role": "frontend-plugin"}, + })) + + config = PluginListConfig.create_default(tmp_path) + + assert config.get_plugins() == {} + + def test_skips_dist_directories(self, tmp_path): + """Test that dist and dist-dynamic directories are skipped.""" + for d in ["dist", "dist-dynamic"]: + dist = tmp_path / d / "plugin-todo" + dist.mkdir(parents=True) + (dist / "package.json").write_text(json.dumps({ + "name": "@test/plugin-todo", + "backstage": {"role": "frontend-plugin"}, + })) + + config = PluginListConfig.create_default(tmp_path) + + assert config.get_plugins() == {} + + def test_discovers_nested_plugins(self, tmp_path): + """Test discovering plugins in nested directory structures like plugins/ecs/backend.""" + _make_plugin_dir(tmp_path, "plugins/ecs/frontend", "@aws/ecs-frontend", "frontend-plugin") + _make_plugin_dir(tmp_path, "plugins/ecs/backend", "@aws/ecs-backend", "backend-plugin") + + config = PluginListConfig.create_default(tmp_path) + plugins = config.get_plugins() + + assert len(plugins) == 2 + assert "plugins/ecs/frontend" in plugins + assert "plugins/ecs/backend" in plugins + + def test_results_sorted_alphabetically(self, tmp_path): + """Test that discovered plugins are sorted by path.""" + _make_plugin_dir(tmp_path, "plugins/z-plugin", "@test/z-plugin", "frontend-plugin") + _make_plugin_dir(tmp_path, "plugins/a-plugin", "@test/a-plugin", "backend-plugin") + _make_plugin_dir(tmp_path, "plugins/m-plugin", "@test/m-plugin", "frontend-plugin") + + config = PluginListConfig.create_default(tmp_path) + paths = list(config.get_plugins().keys()) + + assert paths == ["plugins/a-plugin", "plugins/m-plugin", "plugins/z-plugin"] + + def test_empty_workspace(self, tmp_path): + """Test scanning an empty workspace returns no plugins.""" + config = PluginListConfig.create_default(tmp_path) + + assert config.get_plugins() == {} + + def test_malformed_package_json_skipped(self, tmp_path): + """Test that a malformed package.json does not crash discovery.""" + bad_dir = tmp_path / "plugins" / "broken" + bad_dir.mkdir(parents=True) + (bad_dir / "package.json").write_text("{ not valid json") + + _make_plugin_dir(tmp_path, "plugins/good", "@test/good", "backend-plugin") + + config = PluginListConfig.create_default(tmp_path) + plugins = config.get_plugins() + + assert len(plugins) == 1 + assert "plugins/good" in plugins + + def test_todo_workspace_structure(self, tmp_path): + """Test a workspace matching the community-plugins todo example.""" + _make_plugin_dir(tmp_path, "plugins/todo", "@backstage-community/plugin-todo", "frontend-plugin") + _make_plugin_dir(tmp_path, "plugins/todo-backend", "@backstage-community/plugin-todo-backend", "backend-plugin") + + config = PluginListConfig.create_default(tmp_path) + plugins = config.get_plugins() + + assert plugins == {"plugins/todo": "", "plugins/todo-backend": ""} + + def test_aws_ecs_workspace_structure(self, tmp_path): + """Test a workspace matching the AWS ECS plugins example (nested dirs).""" + _make_plugin_dir(tmp_path, "plugins/ecs/frontend", "@aws/amazon-ecs-plugin-for-backstage", "frontend-plugin") + _make_plugin_dir(tmp_path, "plugins/ecs/backend", "@aws/amazon-ecs-plugin-for-backstage-backend", "backend-plugin") + # Also has common/node packages that should be ignored (not plugin roles) + _make_plugin_dir(tmp_path, "plugins/ecs/common", "@aws/aws-core-plugin-for-backstage-common", "common-library") + + config = PluginListConfig.create_default(tmp_path) + plugins = config.get_plugins() + + assert len(plugins) == 2 + assert "plugins/ecs/frontend" in plugins + assert "plugins/ecs/backend" in plugins From 4ba59bfed8f26b3820835e781e497494e1b72fee Mon Sep 17 00:00:00 2001 From: Frank Kong Date: Fri, 6 Mar 2026 18:01:58 -0500 Subject: [PATCH 09/35] chore: update version argument to return script and yarn lock commit hashes Signed-off-by: Frank Kong --- src/rhdh_dynamic_plugin_factory/cli.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/rhdh_dynamic_plugin_factory/cli.py b/src/rhdh_dynamic_plugin_factory/cli.py index 3eabdd6..01f714d 100644 --- a/src/rhdh_dynamic_plugin_factory/cli.py +++ b/src/rhdh_dynamic_plugin_factory/cli.py @@ -2,6 +2,7 @@ Command-line interface for RHDH Plugin Factory - Setup and orchestration tool. """ +import json import sys import os import argparse @@ -25,6 +26,22 @@ logger = get_logger("cli") +_PROJECT_ROOT = Path(__file__).parent.parent.parent + + +def _build_version_string() -> str: + """Build version string including external resource metadata.""" + lines = [f"rhdh-dynamic-plugin-factory: {__version__}"] + try: + metadata_path = _PROJECT_ROOT / "resources" / "metadata.json" + metadata = json.loads(metadata_path.read_text()) + lines.append(f"RHDH commit: {metadata['rhdh-hash']}") + lines.append(f"export-util script commit: {metadata['export-util-script-hash']}") + except (FileNotFoundError, KeyError, json.JSONDecodeError): + pass + return "\n".join(lines) + + def create_parser() -> argparse.ArgumentParser: """Create the argument parser for the CLI.""" parser = argparse.ArgumentParser( @@ -43,7 +60,7 @@ def create_parser() -> argparse.ArgumentParser: parser.add_argument( "-v", "--version", action="version", - version=__version__ + version=_build_version_string() ) parser.add_argument( "--log-level", From 86fc972c0f2acae472351b7e1e75b971590bb331 Mon Sep 17 00:00:00 2001 From: Frank Kong Date: Tue, 10 Mar 2026 14:10:41 -0400 Subject: [PATCH 10/35] feat: add initial plugins-list.yaml auto generate Assisted-By: Cursor Signed-off-by: Frank Kong --- src/rhdh_dynamic_plugin_factory/config.py | 303 +++++++++++++++++++++- 1 file changed, 289 insertions(+), 14 deletions(-) diff --git a/src/rhdh_dynamic_plugin_factory/config.py b/src/rhdh_dynamic_plugin_factory/config.py index 819fc66..15039a4 100644 --- a/src/rhdh_dynamic_plugin_factory/config.py +++ b/src/rhdh_dynamic_plugin_factory/config.py @@ -6,6 +6,7 @@ from logging import Logger import os from pathlib import Path +import re from typing import Dict, Optional, ClassVar from dataclasses import dataclass, field from dotenv import load_dotenv @@ -284,7 +285,7 @@ def auto_generate_plugins_list(self, config_dir: Optional[str] = None, plugin_cfg = PluginListConfig.create_default(workspace_path=Path(workspace_full_path)) plugin_cfg.to_file(Path(plugins_file)) - plugins = plugin_cfg.get_plugins() + plugins: Dict[str, str] = plugin_cfg.get_plugins() if plugins: self.logger.info(f"Generated {self.PLUGIN_LIST_FILE} with {len(plugins)} plugin(s)") for plugin_path, build_args in plugins.items(): @@ -351,7 +352,7 @@ def setup_config_directory(self) -> Optional["SourceConfig"]: load_dotenv(env_file, override=True) self.logger.debug(f"Loaded .env file: {env_file}") - source_config = self.discover_source_config() + source_config: Optional["SourceConfig"] = self.discover_source_config() if source_config: self.logger.info("Found source configuration") self.logger.info(f" Repository: {source_config.repo}") @@ -467,12 +468,12 @@ def export_plugins(self, output_dir: str, config_dir: Optional[str] = None, config_env_file = os.path.join(config_dir, ".env") default_env_file = Path(__file__).parent.parent.parent / "default.env" load_dotenv(default_env_file) - env = dict(os.environ) + env = dict[str, str](os.environ) if os.path.exists(config_env_file): self.logger.debug(f"Loading script configuration from: {config_env_file}") load_dotenv(config_env_file, override=True) - env = dict(os.environ) + env = dict[str, str](os.environ) os.makedirs(output_dir, exist_ok=True) env["INPUTS_DESTINATION"] = output_dir @@ -814,8 +815,8 @@ def clone_workspaces_with_worktrees( for repo_url, group in groupby(workspaces, key=lambda w: w.source_config.repo): workspace_list = list(group) - repo_name = repo_dir_name(repo_url) - clone_path = clones_dir / repo_name + repo_name: str = repo_dir_name(repo_url) + clone_path: Path = clones_dir / repo_name logger.info(f"[bold blue]\nCloning base repository: {repo_url}[/bold blue]") logger.info(f" Destination: {clone_path}") @@ -875,13 +876,33 @@ class PluginListConfig: "backend-plugin-module", } + BACKEND_ROLES: ClassVar[set[str]] = { + "backend-plugin", + "backend-plugin-module", + } + SKIP_DIRS: ClassVar[set[str]] = { "node_modules", "dist", "dist-dynamic", ".git", + "__fixtures__", } + _PKG_JSON: ClassVar[str] = "package.json" + + HOST_LOCKFILE: ClassVar[Path] = ( + Path(__file__).parent.parent.parent / "resources" / "rhdh" / "yarn.lock" + ) + + _LOCKFILE_BACKSTAGE_RE: ClassVar[re.Pattern] = re.compile( + r'"(@backstage/[\w.-]+)@npm:' + ) + + _NATIVE_DEP_MARKERS: ClassVar[frozenset[str]] = frozenset[str]({ + "bindings", "prebuild", "nan", "node-pre-gyp", "node-gyp-build", + }) + logger: ClassVar[Logger] = get_logger("plugin_list") def __init__(self, plugins: Dict[str, str]): @@ -938,25 +959,36 @@ def remove_plugin(self, plugin_path: str) -> None: def create_default(cls, workspace_path: Path) -> "PluginListConfig": """Create a default plugin list by scanning workspace for Backstage plugins. - Recursively walks ``workspace_path`` to find ``package.json`` files whose - ``backstage.role`` field matches one of :pyattr:`VALID_BACKSTAGE_ROLES`, - and returns a :class:`PluginListConfig` keyed by plugin directory paths relative to ``workspace_path``. + Recursively walks *workspace_path* to find ``package.json`` files whose + ``backstage.role`` matches one of :pyattr:`VALID_BACKSTAGE_PLUGIN_ROLES`. + + For backend plugins, dependency analysis is performed against the + bundled RHDH host lockfile to determine ``--embed-package`` and + ``--shared-package`` arguments. Args: workspace_path: Absolute path to the workspace root. Returns: - A :class:`PluginListConfig` with discovered plugins (no build args). + A :class:`PluginListConfig` with discovered plugins and build arg(s) (if any). """ plugins: Dict[str, str] = {} + host_packages = cls._parse_host_backstage_packages(cls.HOST_LOCKFILE) for pkg_json_path in cls._find_package_jsons(workspace_path): role = cls._read_backstage_role(pkg_json_path) if role and role in cls.VALID_BACKSTAGE_PLUGIN_ROLES: - plugin_path = pkg_json_path.parent.relative_to(workspace_path).as_posix() - plugins[plugin_path] = "" + plugin_dir = pkg_json_path.parent.relative_to(workspace_path).as_posix() - sorted_plugins = dict(sorted(plugins.items())) + if role in cls.BACKEND_ROLES: + build_args = cls._compute_backend_build_args( + workspace_path, plugin_dir, pkg_json_path, host_packages, + ) + plugins[plugin_dir] = build_args + else: + plugins[plugin_dir] = "" + + sorted_plugins = dict[str, str](sorted(plugins.items())) cls.logger.debug(f"Discovered {len(sorted_plugins)} plugin(s) in {workspace_path}") return cls(sorted_plugins) @@ -971,7 +1003,7 @@ def _find_package_jsons(cls, root: Path) -> list[Path]: if entry.name in cls.SKIP_DIRS or entry.name.startswith("."): continue - pkg_json = entry / "package.json" + pkg_json = entry / cls._PKG_JSON if pkg_json.is_file(): results.append(pkg_json) @@ -994,3 +1026,246 @@ def _read_backstage_role(cls, pkg_json_path: Path) -> Optional[str]: cls.logger.warning(f"Failed to read {pkg_json_path}: {e}") return None + @classmethod + def _parse_host_backstage_packages(cls, lockfile_path: Path) -> set[str]: + """Extract ``@backstage/*`` package names from a Yarn Berry lockfile (Yarn 2+). + + Scans top-level key lines (e.g. + ``"@backstage/catalog-model@npm:^1.7.2, …":``) and collects distinct + package names. + + Args: + lockfile_path: Path to the host ``yarn.lock`` file. + + Returns: + Set of ``@backstage/*`` package names found in the lockfile, + or an empty set if the file does not exist. + """ + if not lockfile_path.is_file(): + cls.logger.warning(f"Host lockfile not found at {lockfile_path}") + return set[str]() + + packages: set[str] = set[str]() + for line in lockfile_path.read_text(encoding="utf-8").splitlines(): + if not line.startswith('"@backstage/'): + continue + for match in cls._LOCKFILE_BACKSTAGE_RE.finditer(line): + packages.add(match.group(1)) + + cls.logger.debug(f"Parsed {len(packages)} @backstage/* packages from host lockfile") + return packages + + @staticmethod + def _get_sibling_names(plugin_name: str, role: str) -> set[str]: + """Derive sibling package names that the RHDH CLI auto-embeds. + + Replicates the rhdh-cli logic: for backend plugins the CLI + automatically embeds the ``-common`` and ``-node`` siblings. + + Args: + plugin_name: The npm package name (e.g. ``@scope/my-plugin-backend``). + role: The ``backstage.role`` value. + + Returns: + Set of sibling package names, empty for non-backend roles. + """ + if role == "backend-plugin": + base = re.sub(r"-backend$", "", plugin_name) + elif role == "backend-plugin-module": + base = re.sub(r"-backend-module-.+$", "", plugin_name) + else: + return set[str]() + + if base == plugin_name: + return set[str]() + + return {f"{base}-common", f"{base}-node"} + + @classmethod + def _resolve_node_module_package_json( + cls, workspace_path: Path, dep_name: str + ) -> Optional[Path]: + """Locate a dependency's ``package.json`` in the workspace root ``node_modules``. + + Yarn workspaces hoist all packages to the workspace root, so only that + location is checked. + + Args: + workspace_path: Absolute path to the workspace root. + dep_name: npm package name (may be scoped, e.g. ``@aws/foo``). + + Returns: + Path to the dependency's ``package.json``, or *None* if not found. + """ + candidate = workspace_path / "node_modules" / dep_name / cls._PKG_JSON + if candidate.is_file(): + return candidate + return None + + @staticmethod + def _is_native_module(pkg_data: dict) -> bool: + """Check whether a ``package.json`` describes a native Node.js module. + + Replicates the logic of the ``is-native-module`` npm package used by RHDH CLI. + """ + deps = pkg_data.get("dependencies", {}) + if any(marker in deps for marker in PluginListConfig._NATIVE_DEP_MARKERS): + return True + if pkg_data.get("gypfile"): + return True + if pkg_data.get("binary"): + return True + return False + + @classmethod + def _gather_native_modules( + cls, + workspace_path: Path, + private_dep_names: set[str], + ) -> set[str]: + """Find native modules in the transitive dependency tree of private deps. + + Recursively walks each dep's dependencies via ``node_modules``, + checking :meth:`_is_native_module` on every package encountered. + Tracks visited packages to avoid cycles. + + Args: + workspace_path: Absolute workspace root. + private_dep_names: Direct dep names to start the walk from. + + Returns: + Set of native package names found. + """ + native: set[str] = set[str]() + visited: set[str] = set[str]() + + def _walk(dep_name: str) -> None: + if dep_name in visited: + return + visited.add(dep_name) + + pkg_json = cls._resolve_node_module_package_json(workspace_path, dep_name) + if pkg_json is None: + return + + try: + data = json.loads(pkg_json.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError): + return + + if cls._is_native_module(data): + native.add(dep_name) + + for field in ("dependencies", "optionalDependencies"): + for sub_dep in data.get(field, {}): + _walk(sub_dep) + + for dep in private_dep_names: + _walk(dep) + + return native + + @classmethod + def _check_third_party_dep( + cls, + workspace_path: Path, + dep_name: str, + host_packages: set[str], + embed_packages: set[str], + unshare_packages: set[str], + ) -> None: + """Check a non-``@backstage/*`` dep for transitive shared-package usage. + + If the dependency has any ``@backstage/*`` dependencies it is + marked for embedding. Dependencies absent from *host_packages* are + additionally marked for unsharing. + + Results are collected directly into *embed_packages* / *unshare_packages*. + """ + dep_pkg_json = cls._resolve_node_module_package_json(workspace_path, dep_name) + if dep_pkg_json is None: + cls.logger.debug(f"Could not resolve {dep_name} in node_modules") + return + + try: + dep_data = json.loads(dep_pkg_json.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError) as e: + cls.logger.debug(f"Failed to read {dep_pkg_json}: {e}") + return + + dep_deps = dep_data.get("dependencies", {}) + backstage_deps = [d for d in dep_deps if d.startswith("@backstage/")] + + if backstage_deps: + embed_packages.add(dep_name) + for dep in backstage_deps: + if dep not in host_packages: + unshare_packages.add(dep) + + @classmethod + def _compute_backend_build_args( + cls, + workspace_path: Path, + plugin_dir: str, + pkg_json_path: Path, + host_packages: set[str], + ) -> str: + """Compute ``--embed-package`` / ``--shared-package`` args for a backend plugin. + + Analyses the plugin's direct dependencies: + + * ``@backstage/*`` deps missing from *host_packages* are unshared + **and** embedded (the host won't provide them at runtime). + * Non-``@backstage/*``, non-sibling deps whose own dependencies + include ``@backstage/*`` packages are embedded. Any of those + sub-deps missing from *host_packages* are additionally unshared. + + Args: + workspace_path: Absolute workspace root. + plugin_dir: Plugin directory relative to *workspace_path*. + pkg_json_path: Path to the plugin's ``package.json``. + host_packages: ``@backstage/*`` names present in the host lockfile. + + Returns: + CLI argument string, or ``""`` if no extra args are needed. + """ + try: + pkg_data = json.loads(pkg_json_path.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError): + return "" + + plugin_name: str = pkg_data.get("name", "") + role: str = pkg_data.get("backstage", {}).get("role", "") + dependencies: dict = pkg_data.get("dependencies", {}) + + siblings = cls._get_sibling_names(plugin_name, role) + + embed_packages: set[str] = set[str]() + unshare_packages: set[str] = set[str]() + private_deps: set[str] = set[str]() + + for dep_name in dependencies: + if dep_name in siblings: + continue + + if dep_name.startswith("@backstage/"): + if dep_name not in host_packages: + embed_packages.add(dep_name) + unshare_packages.add(dep_name) + continue + + private_deps.add(dep_name) + cls._check_third_party_dep( + workspace_path, dep_name, + host_packages, embed_packages, unshare_packages, + ) + + suppress_native = cls._gather_native_modules( + workspace_path, private_deps | embed_packages | siblings, + ) + + parts = [f"--embed-package {pkg}" for pkg in sorted(embed_packages)] + parts += [f"--shared-package !{pkg}" for pkg in sorted(unshare_packages)] + parts += [f"--suppress-native-package {pkg}" for pkg in sorted(suppress_native)] + return " ".join(parts) + From ac0eaa71bb415a004f2d17013a87723bc0c75a7d Mon Sep 17 00:00:00 2001 From: Frank Kong Date: Tue, 10 Mar 2026 15:35:28 -0400 Subject: [PATCH 11/35] chore: add granular build arg generation Assisted-By: Cursor Signed-off-by: Frank Kong --- src/rhdh_dynamic_plugin_factory/cli.py | 13 ++ src/rhdh_dynamic_plugin_factory/config.py | 170 ++++++++++++++++++++-- 2 files changed, 171 insertions(+), 12 deletions(-) diff --git a/src/rhdh_dynamic_plugin_factory/cli.py b/src/rhdh_dynamic_plugin_factory/cli.py index 01f714d..1360daa 100644 --- a/src/rhdh_dynamic_plugin_factory/cli.py +++ b/src/rhdh_dynamic_plugin_factory/cli.py @@ -126,6 +126,14 @@ def create_parser() -> argparse.ArgumentParser: default=False, help="Clean the source directory before cloning source repository. WARNING: This will all the contents of the source directory." ) + parser.add_argument( + "--generate-build-args", + action="store_true", + default=False, + help="When plugins-list.yaml exists, (re)compute build arguments for all " + "listed plugins using dependency analysis. WARNING: This overwrites " + "your plugins-list.yaml with updated build args." + ) return parser def install_dependencies(workspace_path: Path) -> None: @@ -181,6 +189,7 @@ def _process_workspace( repo_path: str, workspace_path: str, output_dir: str, + generate_build_args: bool = False, ) -> None: """Execute the plugin factory pipeline for a single workspace. @@ -190,6 +199,7 @@ def _process_workspace( repo_path: Path to the repository checkout for this workspace. workspace_path: Relative path from repo_path to the workspace. output_dir: Output directory for build artifacts. + generate_build_args: If True, (re)compute build args for an existing plugins-list.yaml. """ logger.info("[bold blue]Applying Patches and Overlays[/bold blue]") config.apply_patches_and_overlays( @@ -206,6 +216,7 @@ def _process_workspace( config_dir=workspace_config_dir, repo_path=repo_path, workspace_path=workspace_path, + generate_build_args=generate_build_args, ) logger.info("[bold blue]Exporting plugins using RHDH CLI[/bold blue]") @@ -350,6 +361,7 @@ def _run_multi_workspace(args: argparse.Namespace, workspaces: list[WorkspaceInf repo_path=str(ws.repo_path), workspace_path=ws.source_config.workspace_path, output_dir=str(ws.output_dir), + generate_build_args=args.generate_build_args, ) successes.append(ws.name) logger.info(f"[green]Workspace '{ws.name}' export completed successfully[/green]") @@ -419,6 +431,7 @@ def _run_single_workspace(args: argparse.Namespace) -> None: repo_path=str(config.repo_path), workspace_path=str(config.workspace_path), output_dir=str(args.output_dir), + generate_build_args=args.generate_build_args, ) diff --git a/src/rhdh_dynamic_plugin_factory/config.py b/src/rhdh_dynamic_plugin_factory/config.py index 15039a4..eb2e691 100644 --- a/src/rhdh_dynamic_plugin_factory/config.py +++ b/src/rhdh_dynamic_plugin_factory/config.py @@ -251,16 +251,24 @@ def _validate_plugins_list(self) -> None: def auto_generate_plugins_list(self, config_dir: Optional[str] = None, repo_path: Optional[str] = None, - workspace_path: Optional[str] = None) -> None: - """Auto-generate plugins-list.yaml if it doesn't already exist. + workspace_path: Optional[str] = None, + generate_build_args: bool = False) -> None: + """Auto-generate plugins-list.yaml, or populate build args for an existing one. + + When the file does not exist, a full scan is performed (all plugins + in the workspace are discovered). + + When the file already exists and ``generate_build_args``*`` is ``True``, + build arguments are (re)computed for every plugin listed in the file. Args: config_dir: Config directory containing plugins-list.yaml. Defaults to self.config_dir. repo_path: Repository path. Defaults to self.repo_path. workspace_path: Workspace path relative to repo. Defaults to self.workspace_path. + generate_build_args: If True, recompute build args for an existing plugins-list.yaml. Raises: - PluginFactoryError: If auto-generation fails. + PluginFactoryError: If auto-generation or build-arg population fails. """ config_dir = config_dir or self.config_dir repo_path = repo_path or self.repo_path @@ -269,7 +277,10 @@ def auto_generate_plugins_list(self, config_dir: Optional[str] = None, plugins_file = os.path.join(config_dir, self.PLUGIN_LIST_FILE) if os.path.exists(plugins_file): - self.logger.debug(f"[green]{self.PLUGIN_LIST_FILE} already exists at {plugins_file}. Skipping auto-generation.[/green]") + if generate_build_args: + self._populate_build_args_for_existing(plugins_file, repo_path, workspace_path) + else: + self.logger.debug(f"[green]{self.PLUGIN_LIST_FILE} already exists at {plugins_file}. Skipping auto-generation.[/green]") return self.logger.info(f"[bold blue]Auto-generating {self.PLUGIN_LIST_FILE}[/bold blue]") @@ -299,6 +310,39 @@ def auto_generate_plugins_list(self, config_dir: Optional[str] = None, raise except Exception as e: raise PluginFactoryError(f"Failed to auto-generate plugins list: {e}") from e + + def _populate_build_args_for_existing( + self, plugins_file: str, repo_path: str, workspace_path: str, + ) -> None: + """Load an existing plugins-list.yaml, recompute build args, and write it back. + + Args: + plugins_file: Absolute path to the plugins-list.yaml file. + repo_path: Repository root path. + workspace_path: Workspace path relative to repo_path. + + Raises: + PluginFactoryError: If the workspace cannot be found or population fails. + """ + self.logger.warning( + f"[yellow]--generate-build-args: Modifying existing {self.PLUGIN_LIST_FILE} " + f"to (re)compute build arguments. Your file will be overwritten.[/yellow]" + ) + + workspace_full_path = os.path.abspath(os.path.join(repo_path, workspace_path)) + if not os.path.exists(workspace_full_path): + raise PluginFactoryError(f"Plugin workspace does not exist at {workspace_full_path}") + + try: + plugin_cfg = PluginListConfig.from_file(Path(plugins_file)) + plugin_cfg.populate_build_args(Path(workspace_full_path)) + plugin_cfg.to_file(Path(plugins_file)) + except PluginFactoryError: + raise + except Exception as e: + raise PluginFactoryError( + f"Failed to populate build args for {self.PLUGIN_LIST_FILE}: {e}" + ) from e def discover_source_config(self) -> Optional["SourceConfig"]: """Discovers and loads source configuration. @@ -955,6 +999,113 @@ def add_plugin(self, plugin_path: str, build_args: str = "") -> None: def remove_plugin(self, plugin_path: str) -> None: self.plugins.pop(plugin_path, None) + def populate_build_args(self, workspace_path: Path) -> "PluginListConfig": + """(Re)compute build arguments for every plugin in the list. + + Uses the same dependency analysis as :meth:`create_default` but only + for the plugins already present in ``self.plugins``. All existing + build args are overwritten with freshly computed values. + + A before/after diff is logged for each plugin whose args changed so + the user has a record to revert from if needed. + + Args: + workspace_path: Absolute path to the workspace root + (must already have ``node_modules`` installed). + + Returns: + ``self``, mutated in place. + """ + original = self.plugins.copy() + host_packages = self._parse_host_backstage_packages(self.HOST_LOCKFILE) + + for plugin_dir in self.plugins: + pkg_json_path = workspace_path / plugin_dir / self._PKG_JSON + if not pkg_json_path.is_file(): + self.logger.warning( + f"Plugin package.json not found in workspace: {plugin_dir} " + f"(expected at {pkg_json_path})" + ) + self.plugins[plugin_dir] = "" + continue + + role = self._read_backstage_role(pkg_json_path) + if not role or role not in self.VALID_BACKSTAGE_PLUGIN_ROLES: + self.logger.warning( + f"Plugin {plugin_dir} has no valid backstage.role — skipping build-arg computation. " + f"Found role: {role}. Valid roles are: {', '.join(self.VALID_BACKSTAGE_PLUGIN_ROLES)}" + ) + self.plugins[plugin_dir] = "" + continue + + self.plugins[plugin_dir] = self._compute_plugin_build_args( + workspace_path, plugin_dir, pkg_json_path, host_packages, + ) + + self._log_build_args_diff(original, self.plugins) + return self + + @classmethod + def _log_build_args_diff( + cls, before: Dict[str, str], after: Dict[str, str], + ) -> None: + """Log a before/after comparison for plugins whose build args changed.""" + changed: list[str] = [] + unchanged: list[str] = [] + + for plugin_dir in after: + old = before.get(plugin_dir, "") + new = after[plugin_dir] + if old != new: + changed.append(plugin_dir) + else: + unchanged.append(plugin_dir) + + if changed: + cls.logger.info( + f"Build args updated for {len(changed)} of " + f"{len(after)} plugin(s):" + ) + for plugin_dir in changed: + old = before.get(plugin_dir, "") + new = after[plugin_dir] + cls.logger.info(f" {plugin_dir}:") + cls.logger.info(f" before: {old or '(empty)'}") + cls.logger.info(f" after: {new or '(empty)'}") + + if unchanged: + cls.logger.info( + f"Build args unchanged for {len(unchanged)} plugin(s):" + ) + for plugin_dir in unchanged: + cls.logger.info( + f" {plugin_dir}: {after[plugin_dir] or '(empty)'}" + ) + + @classmethod + def _compute_plugin_build_args( + cls, + workspace_path: Path, + plugin_dir: str, + pkg_json_path: Path, + host_packages: set[str], + ) -> str: + """Compute build args for a single plugin based on its backstage role. + + Returns the CLI argument string for backend plugins, or empty + string for frontend plugins. Returns empty string if the role + is not a valid plugin role. + """ + role = cls._read_backstage_role(pkg_json_path) + if not role or role not in cls.VALID_BACKSTAGE_PLUGIN_ROLES: + return "" + + if role in cls.BACKEND_ROLES: + return cls._compute_backend_build_args( + workspace_path, plugin_dir, pkg_json_path, host_packages, + ) + return "" + @classmethod def create_default(cls, workspace_path: Path) -> "PluginListConfig": """Create a default plugin list by scanning workspace for Backstage plugins. @@ -979,14 +1130,9 @@ def create_default(cls, workspace_path: Path) -> "PluginListConfig": role = cls._read_backstage_role(pkg_json_path) if role and role in cls.VALID_BACKSTAGE_PLUGIN_ROLES: plugin_dir = pkg_json_path.parent.relative_to(workspace_path).as_posix() - - if role in cls.BACKEND_ROLES: - build_args = cls._compute_backend_build_args( - workspace_path, plugin_dir, pkg_json_path, host_packages, - ) - plugins[plugin_dir] = build_args - else: - plugins[plugin_dir] = "" + plugins[plugin_dir] = cls._compute_plugin_build_args( + workspace_path, plugin_dir, pkg_json_path, host_packages, + ) sorted_plugins = dict[str, str](sorted(plugins.items())) cls.logger.debug(f"Discovered {len(sorted_plugins)} plugin(s) in {workspace_path}") From c7c837d71ec23f236cc24c3fe4347a1ffb28d0d3 Mon Sep 17 00:00:00 2001 From: Frank Kong Date: Tue, 10 Mar 2026 16:30:13 -0400 Subject: [PATCH 12/35] chore: split config file and refactor constants Assisted-By: Cursor Signed-off-by: Frank Kong --- src/rhdh_dynamic_plugin_factory/__init__.py | 6 +- src/rhdh_dynamic_plugin_factory/cli.py | 6 +- src/rhdh_dynamic_plugin_factory/config.py | 883 +----------------- src/rhdh_dynamic_plugin_factory/constants.py | 42 + .../plugin_list_config.py | 487 ++++++++++ .../source_config.py | 344 +++++++ 6 files changed, 902 insertions(+), 866 deletions(-) create mode 100644 src/rhdh_dynamic_plugin_factory/constants.py create mode 100644 src/rhdh_dynamic_plugin_factory/plugin_list_config.py create mode 100644 src/rhdh_dynamic_plugin_factory/source_config.py diff --git a/src/rhdh_dynamic_plugin_factory/__init__.py b/src/rhdh_dynamic_plugin_factory/__init__.py index 6749450..08d13c8 100644 --- a/src/rhdh_dynamic_plugin_factory/__init__.py +++ b/src/rhdh_dynamic_plugin_factory/__init__.py @@ -8,14 +8,14 @@ from .__version__ import __version__ from .cli import main, create_parser -from .config import ( - PluginFactoryConfig, +from .config import PluginFactoryConfig +from .source_config import ( SourceConfig, WorkspaceInfo, - PluginListConfig, discover_workspaces, clone_workspaces_with_worktrees, ) +from .plugin_list_config import PluginListConfig from .exceptions import ( PluginFactoryError, ConfigurationError, diff --git a/src/rhdh_dynamic_plugin_factory/cli.py b/src/rhdh_dynamic_plugin_factory/cli.py index 1360daa..f893d57 100644 --- a/src/rhdh_dynamic_plugin_factory/cli.py +++ b/src/rhdh_dynamic_plugin_factory/cli.py @@ -12,7 +12,8 @@ try: from .__version__ import __version__ from .logger import setup_logging, get_logger - from .config import PluginFactoryConfig, WorkspaceInfo, discover_workspaces, clone_workspaces_with_worktrees + from .config import PluginFactoryConfig + from .source_config import WorkspaceInfo, discover_workspaces, clone_workspaces_with_worktrees from .utils import run_command_with_streaming, prompt_or_clean_directory from .exceptions import PluginFactoryError, ConfigurationError, ExecutionError except ImportError: @@ -20,7 +21,8 @@ sys.path.insert(0, str(Path(__file__).parent.parent)) from rhdh_dynamic_plugin_factory.__version__ import __version__ from rhdh_dynamic_plugin_factory.logger import setup_logging, get_logger - from rhdh_dynamic_plugin_factory.config import PluginFactoryConfig, WorkspaceInfo, discover_workspaces, clone_workspaces_with_worktrees + from rhdh_dynamic_plugin_factory.config import PluginFactoryConfig + from rhdh_dynamic_plugin_factory.source_config import WorkspaceInfo, discover_workspaces, clone_workspaces_with_worktrees from rhdh_dynamic_plugin_factory.utils import run_command_with_streaming, prompt_or_clean_directory from rhdh_dynamic_plugin_factory.exceptions import PluginFactoryError, ConfigurationError, ExecutionError diff --git a/src/rhdh_dynamic_plugin_factory/config.py b/src/rhdh_dynamic_plugin_factory/config.py index eb2e691..a7c524f 100644 --- a/src/rhdh_dynamic_plugin_factory/config.py +++ b/src/rhdh_dynamic_plugin_factory/config.py @@ -6,23 +6,23 @@ from logging import Logger import os from pathlib import Path -import re from typing import Dict, Optional, ClassVar from dataclasses import dataclass, field from dotenv import load_dotenv import yaml -import json import subprocess +from .constants import PLUGIN_LIST_FILE, SOURCE_CONFIG_FILE from .exceptions import PluginFactoryError, ConfigurationError, ExecutionError from .logger import get_logger -from .utils import run_command_with_streaming, display_export_results, prompt_or_clean_directory, repo_dir_name +from .utils import run_command_with_streaming, display_export_results + +from .source_config import SourceConfig +from .plugin_list_config import PluginListConfig @dataclass class PluginFactoryConfig: """Main configuration for the plugin factory.""" - PLUGIN_LIST_FILE: ClassVar[str] = "plugins-list.yaml" - SOURCE_CONFIG_FILE: ClassVar[str] = "source.json" # Required fields loaded from default.env file (can be overridden by environment variables) rhdh_cli_version: str = field(default="") @@ -219,35 +219,35 @@ def _validate_source_json(self) -> None: CLI args fully replace source.json. """ if self.source_repo: - self.logger.debug(f"Using --source-repo CLI argument, skipping {self.SOURCE_CONFIG_FILE} validation") + self.logger.debug(f"Using --source-repo CLI argument, skipping {SOURCE_CONFIG_FILE} validation") return - source_file = os.path.join(self.config_dir, self.SOURCE_CONFIG_FILE) + source_file = os.path.join(self.config_dir, SOURCE_CONFIG_FILE) if not os.path.exists(source_file): if not os.path.exists(self.repo_path) or not os.listdir(self.repo_path): raise ConfigurationError( - f"{self.SOURCE_CONFIG_FILE} not found at {source_file} and {self.repo_path} is empty. " - "Please provide {self.SOURCE_CONFIG_FILE} to clone a repository, use --source-repo to specify a repository via CLI, " + f"{SOURCE_CONFIG_FILE} not found at {source_file} and {self.repo_path} is empty. " + "Please provide {SOURCE_CONFIG_FILE} to clone a repository, use --source-repo to specify a repository via CLI, " "or use --use-local with a locally mounted repository." ) else: self.logger.warning( - f"{self.SOURCE_CONFIG_FILE} not found at {source_file}. Will attempt to use local repository content at {self.repo_path}" + f"{SOURCE_CONFIG_FILE} not found at {source_file}. Will attempt to use local repository content at {self.repo_path}" ) else: self.logger.debug(f"Using source configuration from: {source_file}") def _validate_plugins_list(self) -> None: """Validate plugins-list.yaml file existence.""" - plugins_file = os.path.join(self.config_dir, self.PLUGIN_LIST_FILE) + plugins_file = os.path.join(self.config_dir, PLUGIN_LIST_FILE) if not os.path.exists(plugins_file): self.logger.warning( - f"{self.PLUGIN_LIST_FILE} not found at {plugins_file}. Will attempt to auto-generate after repository is available." + f"{PLUGIN_LIST_FILE} not found at {plugins_file}. Will attempt to auto-generate after repository is available." ) else: - self.logger.debug(f"Using {self.PLUGIN_LIST_FILE} from: {plugins_file}") + self.logger.debug(f"Using {PLUGIN_LIST_FILE} from: {plugins_file}") def auto_generate_plugins_list(self, config_dir: Optional[str] = None, repo_path: Optional[str] = None, @@ -274,16 +274,16 @@ def auto_generate_plugins_list(self, config_dir: Optional[str] = None, repo_path = repo_path or self.repo_path workspace_path = workspace_path or self.workspace_path - plugins_file = os.path.join(config_dir, self.PLUGIN_LIST_FILE) + plugins_file = os.path.join(config_dir, PLUGIN_LIST_FILE) if os.path.exists(plugins_file): if generate_build_args: self._populate_build_args_for_existing(plugins_file, repo_path, workspace_path) else: - self.logger.debug(f"[green]{self.PLUGIN_LIST_FILE} already exists at {plugins_file}. Skipping auto-generation.[/green]") + self.logger.debug(f"[green]{PLUGIN_LIST_FILE} already exists at {plugins_file}. Skipping auto-generation.[/green]") return - self.logger.info(f"[bold blue]Auto-generating {self.PLUGIN_LIST_FILE}[/bold blue]") + self.logger.info(f"[bold blue]Auto-generating {PLUGIN_LIST_FILE}[/bold blue]") if not os.path.exists(repo_path): raise PluginFactoryError(f"Source code repository does not exist at {repo_path}") @@ -298,7 +298,7 @@ def auto_generate_plugins_list(self, config_dir: Optional[str] = None, plugins: Dict[str, str] = plugin_cfg.get_plugins() if plugins: - self.logger.info(f"Generated {self.PLUGIN_LIST_FILE} with {len(plugins)} plugin(s)") + self.logger.info(f"Generated {PLUGIN_LIST_FILE} with {len(plugins)} plugin(s)") for plugin_path, build_args in plugins.items(): if build_args: self.logger.info(f" - {plugin_path}: {build_args}") @@ -325,7 +325,7 @@ def _populate_build_args_for_existing( PluginFactoryError: If the workspace cannot be found or population fails. """ self.logger.warning( - f"[yellow]--generate-build-args: Modifying existing {self.PLUGIN_LIST_FILE} " + f"[yellow]--generate-build-args: Modifying existing {PLUGIN_LIST_FILE} " f"to (re)compute build arguments. Your file will be overwritten.[/yellow]" ) @@ -341,7 +341,7 @@ def _populate_build_args_for_existing( raise except Exception as e: raise PluginFactoryError( - f"Failed to populate build args for {self.PLUGIN_LIST_FILE}: {e}" + f"Failed to populate build args for {PLUGIN_LIST_FILE}: {e}" ) from e def discover_source_config(self) -> Optional["SourceConfig"]: @@ -367,10 +367,9 @@ def discover_source_config(self) -> Optional["SourceConfig"]: self.logger.debug(f"Using source config from CLI: {source_config}") return source_config - source_file = os.path.join(self.config_dir, self.SOURCE_CONFIG_FILE) + source_file = os.path.join(self.config_dir, SOURCE_CONFIG_FILE) if os.path.exists(source_file) and not self.use_local: - # SourceConfig.from_file() raises ConfigurationError on failure, so let it propagate to cli.py source_config = SourceConfig.from_file(Path(source_file)) self.logger.debug(f"Using source config from: {source_config}") return source_config @@ -402,7 +401,7 @@ def setup_config_directory(self) -> Optional["SourceConfig"]: self.logger.info(f" Repository: {source_config.repo}") self.logger.info(f" Reference: {source_config.repo_ref}") - plugins_list_file = os.path.join(self.config_dir, self.PLUGIN_LIST_FILE) + plugins_list_file = os.path.join(self.config_dir, PLUGIN_LIST_FILE) if os.path.exists(plugins_list_file): self.logger.info(f"Using plugin list file: {plugins_list_file}") @@ -504,7 +503,7 @@ def export_plugins(self, output_dir: str, config_dir: Optional[str] = None, step=STEP_NAME ) - plugins_list_file = os.path.join(config_dir, self.PLUGIN_LIST_FILE) + plugins_list_file = os.path.join(config_dir, PLUGIN_LIST_FILE) if not os.path.exists(plugins_list_file): raise ConfigurationError("No plugins file found") @@ -577,841 +576,3 @@ def conditional_stderr_log(line: str) -> None: f"Failed to run export script: {e}", step=STEP_NAME, ) from e - - -@dataclass -class SourceConfig: - """Configuration for plugin source repository.""" - repo: str - repo_ref: Optional[str] # None triggers default branch resolution in __post_init__ - workspace_path: str - logger: ClassVar[Logger] = get_logger("source_config") - - def __post_init__(self) -> None: - if not self.repo: - raise ConfigurationError("repo is required") - if not self.workspace_path: - raise ConfigurationError("workspace-path is required") - - # Resolve default branch at creation time if no ref was provided - if not self.repo_ref: - self.repo_ref = self.resolve_default_ref(self.repo) - - @classmethod - def from_file(cls, source_file: Path) -> "SourceConfig": - """Load source configuration from JSON file. - - repo-ref is optional. When omitted, the default branch is resolved - automatically during construction via resolve_default_ref(). - - Raises: - ConfigurationError: If the file is missing, malformed, or has invalid data. - ExecutionError: If default branch resolution fails (when repo-ref is omitted). - """ - try: - with open(source_file, 'r') as f: - data = json.load(f) - except FileNotFoundError: - raise ConfigurationError(f"Source configuration file not found: {source_file}") - except json.JSONDecodeError as e: - raise ConfigurationError(f"Invalid JSON in {source_file}: {e}") - except Exception as e: - raise ConfigurationError(f"Failed to read {source_file}: {e}") - - try: - repo = data["repo"] - repo_ref = data.get("repo-ref") or None # Treat empty string as None - workspace_path = data.get("workspace-path") - except KeyError as e: - raise ConfigurationError(f"Missing required field {e} in {source_file}") - - config = cls( - repo=repo, - repo_ref=repo_ref, - workspace_path=workspace_path, - ) - - return config - - @classmethod - def from_cli_args(cls, repo: str, repo_ref: Optional[str], workspace_path: str) -> "SourceConfig": - """Create source configuration from CLI arguments. - - Args: - repo: Git repository URL (--source-repo). - repo_ref: Git ref to check out (--source-ref). None means default branch - (resolved automatically during construction via resolve_default_ref()). - workspace_path: Path to workspace within the repository (--workspace-path). - - Returns: - SourceConfig instance with repo_ref always resolved. - - Raises: - ConfigurationError: If required fields are missing. - ExecutionError: If default branch resolution fails (when repo_ref is None). - """ - return cls( - repo=repo, - repo_ref=repo_ref, - workspace_path=workspace_path, - ) - - @staticmethod - def resolve_default_ref(repo: str) -> str: - """Resolve the default branch ref for a repository using git ls-remote since repository is not cloned yet - - Args: - repo: Git repository URL. - - Returns: - The default branch ref (e.g., 'refs/heads/main'). - - Raises: - ExecutionError: If git ls-remote fails or the default branch cannot be determined. - """ - logger = get_logger("source_config") - logger.info(f"[cyan]Resolving default branch for {repo}cyan]") - - try: - result = subprocess.run( - ["git", "ls-remote", "--symref", repo, "HEAD"], - capture_output=True, - text=True, - check=True, - ) - - for line in result.stdout.splitlines(): - if line.startswith("ref:"): - # Ex: Extract "refs/heads/main" from "ref: refs/heads/main\tHEAD" - ref_part = line.split("\t")[0].replace("ref: ", "").strip() - logger.info(f"[green]Resolved default branch: {ref_part} for {repo}[/green]") - return ref_part - - raise ConfigurationError( - f"Could not resolve the default branch for '{repo}'. " - f"Please specify a branch or ref explicitly via 'repo-ref' in {PluginFactoryConfig.SOURCE_CONFIG_FILE} " - "or the --source-ref CLI argument." - ) - except subprocess.CalledProcessError as e: - raise ExecutionError( - f"Failed to resolve default branch for {repo}: {e.stderr.strip()}", - step="resolve default ref", - returncode=e.returncode, - ) from e - - def clone_to_path(self, repo_path: Path, clean: bool = False) -> None: - """Clone the source repository to the specified path. - - Raises: - ConfigurationError: If the destination directory does not exist. - PluginFactoryError: If the user aborts the clone. - ExecutionError: If git clone or checkout fails. - """ - logger = get_logger("cli") - - if not repo_path.exists(): - raise ConfigurationError(f"Destination directory does not exist: {repo_path}") - - self.logger.info("[bold blue]Cloning repository[/bold blue]") - self.logger.info(f"Repository: {self.repo}") - self.logger.info(f"Reference: {self.repo_ref}") - self.logger.info(f"Destination directory: {repo_path}") - - prompt_or_clean_directory(repo_path, clean, self.logger) - - try: - cmd = ["git", "clone", self.repo, str(repo_path)] - # Git writes progress to stderr, so log it as info instead of error - returncode = run_command_with_streaming( - cmd, - logger, - stderr_log_func=logger.info - ) - - if returncode != 0: - raise ExecutionError( - f"Failed to clone repository (exit code {returncode})", - step="git clone", - returncode=returncode - ) - - cmd = ["git", "checkout", self.repo_ref] - logger.info(f"[cyan]Checking out ref: {self.repo_ref}[/cyan]") - # Git writes informational messages to stderr - returncode = run_command_with_streaming( - cmd, - logger, - cwd=repo_path, - stderr_log_func=logger.info - ) - - if returncode != 0: - raise ExecutionError( - f"Failed to checkout ref {self.repo_ref} (exit code {returncode})", - step="git checkout", - returncode=returncode - ) - - logger.info("[green]✓ Repository cloned successfully[/green]") - - except PluginFactoryError: - raise - except Exception as e: - raise ExecutionError( - f"Failed during repository clone/checkout: {e}", - step="git clone/checkout" - ) from e - -@dataclass -class WorkspaceInfo: - """Per-workspace configuration for multi-workspace mode. - - Represents a single workspace discovered from a config subdirectory. - """ - name: str - config_dir: Path - source_config: SourceConfig - repo_path: Optional[Path] = None - output_dir: Optional[Path] = None - - def resolve_paths(self, base_repo_path: Path, base_output_dir: Path) -> None: - """Set per-workspace source code repo and output paths from base directories.""" - self.repo_path = base_repo_path / self.name - self.output_dir = base_output_dir / self.name - - -def discover_workspaces(config_dir: Path) -> list["WorkspaceInfo"]: - """Scan config directory for workspace subdirectories. - - A subdirectory is considered a workspace if it contains a source.json file. - Non-workspace entries are skipped silently; the caller is responsible for - warning the user about ignored content. - - Args: - config_dir: Root configuration directory to scan. - - Returns: - List of WorkspaceInfo instances sorted by repo URL (primary) then name (secondary), - making downstream groupby(repo) trivial. - - Raises: - ConfigurationError: If a workspace's source.json is invalid. - """ - logger = get_logger("config") - workspaces: list[WorkspaceInfo] = [] - - if not config_dir.is_dir(): - return workspaces - - for entry in sorted(config_dir.iterdir()): - if not entry.is_dir(): - continue - - source_file = entry / PluginFactoryConfig.SOURCE_CONFIG_FILE - if not source_file.exists(): - logger.debug(f"Skipping {entry.name}/ — no {PluginFactoryConfig.SOURCE_CONFIG_FILE}") - continue - - workspace_name = entry.name - logger.debug(f"Discovered workspace: {workspace_name}") - - source_config = SourceConfig.from_file(source_file) - - workspaces.append(WorkspaceInfo( - name=workspace_name, - config_dir=entry, - source_config=source_config, - )) - - # Sort by repo URL (primary) then workspace name (secondary) - workspaces.sort(key=lambda w: (w.source_config.repo, w.name)) - - return workspaces - - -def clone_workspaces_with_worktrees( - workspaces: list["WorkspaceInfo"], - base_repo_path: Path, -) -> None: - """Clone repositories and create git worktrees for multi-workspace mode. - - Groups workspaces by repo URL, clones each unique repo once into - /.clones//, then creates a worktree per - workspace at //. - - The caller is responsible for cleaning base_repo_path before calling - this function (e.g. via prompt_or_clean_directory). This function - assumes it can write freely into base_repo_path. - - Args: - workspaces: List of WorkspaceInfo (must already have repo_path set via resolve_paths). - base_repo_path: Base directory for repo clones and worktrees. - - Raises: - PluginFactoryError: If a workspace has no repo_path resolved (internal error). - ExecutionError: If any git operation fails. - """ - from itertools import groupby - - logger = get_logger("config") - clones_dir = base_repo_path / ".clones" - os.makedirs(clones_dir, exist_ok=True) - - for repo_url, group in groupby(workspaces, key=lambda w: w.source_config.repo): - workspace_list = list(group) - repo_name: str = repo_dir_name(repo_url) - clone_path: Path = clones_dir / repo_name - - logger.info(f"[bold blue]\nCloning base repository: {repo_url}[/bold blue]") - logger.info(f" Destination: {clone_path}") - - cmd = ["git", "clone", "--bare", repo_url, str(clone_path)] - returncode = run_command_with_streaming( - cmd, logger, stderr_log_func=logger.info - ) - - if returncode != 0: - raise ExecutionError( - f"Failed to clone repository '{repo_url}' (exit code {returncode}). " - f"Please verify the 'repo' URL in the {PluginFactoryConfig.SOURCE_CONFIG_FILE} for workspaces using this repository. " - f"Ensure the URL is correct and accessible from your environment.", - step="git clone (bare)", - returncode=returncode, - ) - - logger.info(f"[green]Cloned {repo_url} to {clone_path}[/green]") - - for ws in workspace_list: - worktree_path = ws.repo_path - if worktree_path is None: - raise PluginFactoryError( - f"Internal error: workspace '{ws.name}' has no resolved repository path. " - f"This is a bug in the plugin factory. Please report this issue." - ) - - ref = ws.source_config.repo_ref - logger.info(f"[cyan]\nCreating git worktree for '{ws.name}' at ref {ref}[/cyan]") - - # Must use absolute path: git worktree add runs with cwd=clone_path, - # so a relative worktree_path would resolve against the clone directory. - cmd = ["git", "worktree", "add", "--detach", str(worktree_path.resolve()), ref] - returncode = run_command_with_streaming( - cmd, logger, cwd=clone_path, stderr_log_func=logger.info - ) - - if returncode != 0: - raise ExecutionError( - f"Failed to create worktree for workspace '{ws.name}' at ref '{ref}' (exit code {returncode}). " - f"Please verify the 'repo-ref' value in {ws.config_dir / PluginFactoryConfig.SOURCE_CONFIG_FILE}. " - f"Ensure the branch, tag, or commit exists in the repository.", - step="git worktree add", - returncode=returncode, - ) - - logger.info(f"[green] Worktree created for '{ws.name}' at {worktree_path}[/green]") - - -class PluginListConfig: - """Configuration for plugin list (YAML format).""" - VALID_BACKSTAGE_PLUGIN_ROLES: ClassVar[set[str]] = { - "frontend-plugin", - "backend-plugin", - "frontend-plugin-module", - "backend-plugin-module", - } - - BACKEND_ROLES: ClassVar[set[str]] = { - "backend-plugin", - "backend-plugin-module", - } - - SKIP_DIRS: ClassVar[set[str]] = { - "node_modules", - "dist", - "dist-dynamic", - ".git", - "__fixtures__", - } - - _PKG_JSON: ClassVar[str] = "package.json" - - HOST_LOCKFILE: ClassVar[Path] = ( - Path(__file__).parent.parent.parent / "resources" / "rhdh" / "yarn.lock" - ) - - _LOCKFILE_BACKSTAGE_RE: ClassVar[re.Pattern] = re.compile( - r'"(@backstage/[\w.-]+)@npm:' - ) - - _NATIVE_DEP_MARKERS: ClassVar[frozenset[str]] = frozenset[str]({ - "bindings", "prebuild", "nan", "node-pre-gyp", "node-gyp-build", - }) - - logger: ClassVar[Logger] = get_logger("plugin_list") - - def __init__(self, plugins: Dict[str, str]): - """ - Initialize plugin list configuration. - - Args: - plugins: Dictionary mapping plugin paths to build arguments - """ - self.plugins = plugins - - @classmethod - def from_file(cls, plugin_list_file: Path) -> "PluginListConfig": - """Load plugin list from YAML file.""" - - with open(plugin_list_file, 'r') as f: - data = yaml.safe_load(f) or {} - - plugins = {} - for key, value in data.items(): - if value is None: - plugins[key] = "" - else: - plugins[key] = str(value) - - return cls(plugins) - - def to_file(self, plugin_list_file: Path) -> None: - """Save plugin list to YAML file. - - Writes manually rather than via yaml.dump so that entries with no - build args appear as ``key:`` (YAML null) instead of ``key: ''``. - - Args: - plugin_list_file: Destination path for the YAML file. - """ - with open(plugin_list_file, 'w') as f: - for path, args in self.plugins.items(): - if args: - f.write(f"{path}: {args}\n") - else: - f.write(f"{path}:\n") - - def get_plugins(self) -> Dict[str, str]: - return self.plugins.copy() - - def add_plugin(self, plugin_path: str, build_args: str = "") -> None: - self.plugins[plugin_path] = build_args - - def remove_plugin(self, plugin_path: str) -> None: - self.plugins.pop(plugin_path, None) - - def populate_build_args(self, workspace_path: Path) -> "PluginListConfig": - """(Re)compute build arguments for every plugin in the list. - - Uses the same dependency analysis as :meth:`create_default` but only - for the plugins already present in ``self.plugins``. All existing - build args are overwritten with freshly computed values. - - A before/after diff is logged for each plugin whose args changed so - the user has a record to revert from if needed. - - Args: - workspace_path: Absolute path to the workspace root - (must already have ``node_modules`` installed). - - Returns: - ``self``, mutated in place. - """ - original = self.plugins.copy() - host_packages = self._parse_host_backstage_packages(self.HOST_LOCKFILE) - - for plugin_dir in self.plugins: - pkg_json_path = workspace_path / plugin_dir / self._PKG_JSON - if not pkg_json_path.is_file(): - self.logger.warning( - f"Plugin package.json not found in workspace: {plugin_dir} " - f"(expected at {pkg_json_path})" - ) - self.plugins[plugin_dir] = "" - continue - - role = self._read_backstage_role(pkg_json_path) - if not role or role not in self.VALID_BACKSTAGE_PLUGIN_ROLES: - self.logger.warning( - f"Plugin {plugin_dir} has no valid backstage.role — skipping build-arg computation. " - f"Found role: {role}. Valid roles are: {', '.join(self.VALID_BACKSTAGE_PLUGIN_ROLES)}" - ) - self.plugins[plugin_dir] = "" - continue - - self.plugins[plugin_dir] = self._compute_plugin_build_args( - workspace_path, plugin_dir, pkg_json_path, host_packages, - ) - - self._log_build_args_diff(original, self.plugins) - return self - - @classmethod - def _log_build_args_diff( - cls, before: Dict[str, str], after: Dict[str, str], - ) -> None: - """Log a before/after comparison for plugins whose build args changed.""" - changed: list[str] = [] - unchanged: list[str] = [] - - for plugin_dir in after: - old = before.get(plugin_dir, "") - new = after[plugin_dir] - if old != new: - changed.append(plugin_dir) - else: - unchanged.append(plugin_dir) - - if changed: - cls.logger.info( - f"Build args updated for {len(changed)} of " - f"{len(after)} plugin(s):" - ) - for plugin_dir in changed: - old = before.get(plugin_dir, "") - new = after[plugin_dir] - cls.logger.info(f" {plugin_dir}:") - cls.logger.info(f" before: {old or '(empty)'}") - cls.logger.info(f" after: {new or '(empty)'}") - - if unchanged: - cls.logger.info( - f"Build args unchanged for {len(unchanged)} plugin(s):" - ) - for plugin_dir in unchanged: - cls.logger.info( - f" {plugin_dir}: {after[plugin_dir] or '(empty)'}" - ) - - @classmethod - def _compute_plugin_build_args( - cls, - workspace_path: Path, - plugin_dir: str, - pkg_json_path: Path, - host_packages: set[str], - ) -> str: - """Compute build args for a single plugin based on its backstage role. - - Returns the CLI argument string for backend plugins, or empty - string for frontend plugins. Returns empty string if the role - is not a valid plugin role. - """ - role = cls._read_backstage_role(pkg_json_path) - if not role or role not in cls.VALID_BACKSTAGE_PLUGIN_ROLES: - return "" - - if role in cls.BACKEND_ROLES: - return cls._compute_backend_build_args( - workspace_path, plugin_dir, pkg_json_path, host_packages, - ) - return "" - - @classmethod - def create_default(cls, workspace_path: Path) -> "PluginListConfig": - """Create a default plugin list by scanning workspace for Backstage plugins. - - Recursively walks *workspace_path* to find ``package.json`` files whose - ``backstage.role`` matches one of :pyattr:`VALID_BACKSTAGE_PLUGIN_ROLES`. - - For backend plugins, dependency analysis is performed against the - bundled RHDH host lockfile to determine ``--embed-package`` and - ``--shared-package`` arguments. - - Args: - workspace_path: Absolute path to the workspace root. - - Returns: - A :class:`PluginListConfig` with discovered plugins and build arg(s) (if any). - """ - plugins: Dict[str, str] = {} - host_packages = cls._parse_host_backstage_packages(cls.HOST_LOCKFILE) - - for pkg_json_path in cls._find_package_jsons(workspace_path): - role = cls._read_backstage_role(pkg_json_path) - if role and role in cls.VALID_BACKSTAGE_PLUGIN_ROLES: - plugin_dir = pkg_json_path.parent.relative_to(workspace_path).as_posix() - plugins[plugin_dir] = cls._compute_plugin_build_args( - workspace_path, plugin_dir, pkg_json_path, host_packages, - ) - - sorted_plugins = dict[str, str](sorted(plugins.items())) - cls.logger.debug(f"Discovered {len(sorted_plugins)} plugin(s) in {workspace_path}") - return cls(sorted_plugins) - - @classmethod - def _find_package_jsons(cls, root: Path) -> list[Path]: - """Recursively find package.json files, skipping non-plugin directories.""" - results: list[Path] = [] - - for entry in sorted(root.iterdir()): - if not entry.is_dir(): - continue - if entry.name in cls.SKIP_DIRS or entry.name.startswith("."): - continue - - pkg_json = entry / cls._PKG_JSON - if pkg_json.is_file(): - results.append(pkg_json) - - results.extend(cls._find_package_jsons(entry)) - - return results - - @classmethod - def _read_backstage_role(cls, pkg_json_path: Path) -> Optional[str]: - """Read the ``backstage.role`` field from a package.json file. - - Returns: - The role string, or *None* if the file cannot be parsed or has no role. - """ - try: - data = json.loads(pkg_json_path.read_text(encoding="utf-8")) - cls.logger.debug(f"Read backstage role from {pkg_json_path}: {data.get('backstage', {}).get('role')}") - return data.get("backstage", {}).get("role") - except (json.JSONDecodeError, OSError) as e: - cls.logger.warning(f"Failed to read {pkg_json_path}: {e}") - return None - - @classmethod - def _parse_host_backstage_packages(cls, lockfile_path: Path) -> set[str]: - """Extract ``@backstage/*`` package names from a Yarn Berry lockfile (Yarn 2+). - - Scans top-level key lines (e.g. - ``"@backstage/catalog-model@npm:^1.7.2, …":``) and collects distinct - package names. - - Args: - lockfile_path: Path to the host ``yarn.lock`` file. - - Returns: - Set of ``@backstage/*`` package names found in the lockfile, - or an empty set if the file does not exist. - """ - if not lockfile_path.is_file(): - cls.logger.warning(f"Host lockfile not found at {lockfile_path}") - return set[str]() - - packages: set[str] = set[str]() - for line in lockfile_path.read_text(encoding="utf-8").splitlines(): - if not line.startswith('"@backstage/'): - continue - for match in cls._LOCKFILE_BACKSTAGE_RE.finditer(line): - packages.add(match.group(1)) - - cls.logger.debug(f"Parsed {len(packages)} @backstage/* packages from host lockfile") - return packages - - @staticmethod - def _get_sibling_names(plugin_name: str, role: str) -> set[str]: - """Derive sibling package names that the RHDH CLI auto-embeds. - - Replicates the rhdh-cli logic: for backend plugins the CLI - automatically embeds the ``-common`` and ``-node`` siblings. - - Args: - plugin_name: The npm package name (e.g. ``@scope/my-plugin-backend``). - role: The ``backstage.role`` value. - - Returns: - Set of sibling package names, empty for non-backend roles. - """ - if role == "backend-plugin": - base = re.sub(r"-backend$", "", plugin_name) - elif role == "backend-plugin-module": - base = re.sub(r"-backend-module-.+$", "", plugin_name) - else: - return set[str]() - - if base == plugin_name: - return set[str]() - - return {f"{base}-common", f"{base}-node"} - - @classmethod - def _resolve_node_module_package_json( - cls, workspace_path: Path, dep_name: str - ) -> Optional[Path]: - """Locate a dependency's ``package.json`` in the workspace root ``node_modules``. - - Yarn workspaces hoist all packages to the workspace root, so only that - location is checked. - - Args: - workspace_path: Absolute path to the workspace root. - dep_name: npm package name (may be scoped, e.g. ``@aws/foo``). - - Returns: - Path to the dependency's ``package.json``, or *None* if not found. - """ - candidate = workspace_path / "node_modules" / dep_name / cls._PKG_JSON - if candidate.is_file(): - return candidate - return None - - @staticmethod - def _is_native_module(pkg_data: dict) -> bool: - """Check whether a ``package.json`` describes a native Node.js module. - - Replicates the logic of the ``is-native-module`` npm package used by RHDH CLI. - """ - deps = pkg_data.get("dependencies", {}) - if any(marker in deps for marker in PluginListConfig._NATIVE_DEP_MARKERS): - return True - if pkg_data.get("gypfile"): - return True - if pkg_data.get("binary"): - return True - return False - - @classmethod - def _gather_native_modules( - cls, - workspace_path: Path, - private_dep_names: set[str], - ) -> set[str]: - """Find native modules in the transitive dependency tree of private deps. - - Recursively walks each dep's dependencies via ``node_modules``, - checking :meth:`_is_native_module` on every package encountered. - Tracks visited packages to avoid cycles. - - Args: - workspace_path: Absolute workspace root. - private_dep_names: Direct dep names to start the walk from. - - Returns: - Set of native package names found. - """ - native: set[str] = set[str]() - visited: set[str] = set[str]() - - def _walk(dep_name: str) -> None: - if dep_name in visited: - return - visited.add(dep_name) - - pkg_json = cls._resolve_node_module_package_json(workspace_path, dep_name) - if pkg_json is None: - return - - try: - data = json.loads(pkg_json.read_text(encoding="utf-8")) - except (json.JSONDecodeError, OSError): - return - - if cls._is_native_module(data): - native.add(dep_name) - - for field in ("dependencies", "optionalDependencies"): - for sub_dep in data.get(field, {}): - _walk(sub_dep) - - for dep in private_dep_names: - _walk(dep) - - return native - - @classmethod - def _check_third_party_dep( - cls, - workspace_path: Path, - dep_name: str, - host_packages: set[str], - embed_packages: set[str], - unshare_packages: set[str], - ) -> None: - """Check a non-``@backstage/*`` dep for transitive shared-package usage. - - If the dependency has any ``@backstage/*`` dependencies it is - marked for embedding. Dependencies absent from *host_packages* are - additionally marked for unsharing. - - Results are collected directly into *embed_packages* / *unshare_packages*. - """ - dep_pkg_json = cls._resolve_node_module_package_json(workspace_path, dep_name) - if dep_pkg_json is None: - cls.logger.debug(f"Could not resolve {dep_name} in node_modules") - return - - try: - dep_data = json.loads(dep_pkg_json.read_text(encoding="utf-8")) - except (json.JSONDecodeError, OSError) as e: - cls.logger.debug(f"Failed to read {dep_pkg_json}: {e}") - return - - dep_deps = dep_data.get("dependencies", {}) - backstage_deps = [d for d in dep_deps if d.startswith("@backstage/")] - - if backstage_deps: - embed_packages.add(dep_name) - for dep in backstage_deps: - if dep not in host_packages: - unshare_packages.add(dep) - - @classmethod - def _compute_backend_build_args( - cls, - workspace_path: Path, - plugin_dir: str, - pkg_json_path: Path, - host_packages: set[str], - ) -> str: - """Compute ``--embed-package`` / ``--shared-package`` args for a backend plugin. - - Analyses the plugin's direct dependencies: - - * ``@backstage/*`` deps missing from *host_packages* are unshared - **and** embedded (the host won't provide them at runtime). - * Non-``@backstage/*``, non-sibling deps whose own dependencies - include ``@backstage/*`` packages are embedded. Any of those - sub-deps missing from *host_packages* are additionally unshared. - - Args: - workspace_path: Absolute workspace root. - plugin_dir: Plugin directory relative to *workspace_path*. - pkg_json_path: Path to the plugin's ``package.json``. - host_packages: ``@backstage/*`` names present in the host lockfile. - - Returns: - CLI argument string, or ``""`` if no extra args are needed. - """ - try: - pkg_data = json.loads(pkg_json_path.read_text(encoding="utf-8")) - except (json.JSONDecodeError, OSError): - return "" - - plugin_name: str = pkg_data.get("name", "") - role: str = pkg_data.get("backstage", {}).get("role", "") - dependencies: dict = pkg_data.get("dependencies", {}) - - siblings = cls._get_sibling_names(plugin_name, role) - - embed_packages: set[str] = set[str]() - unshare_packages: set[str] = set[str]() - private_deps: set[str] = set[str]() - - for dep_name in dependencies: - if dep_name in siblings: - continue - - if dep_name.startswith("@backstage/"): - if dep_name not in host_packages: - embed_packages.add(dep_name) - unshare_packages.add(dep_name) - continue - - private_deps.add(dep_name) - cls._check_third_party_dep( - workspace_path, dep_name, - host_packages, embed_packages, unshare_packages, - ) - - suppress_native = cls._gather_native_modules( - workspace_path, private_deps | embed_packages | siblings, - ) - - parts = [f"--embed-package {pkg}" for pkg in sorted(embed_packages)] - parts += [f"--shared-package !{pkg}" for pkg in sorted(unshare_packages)] - parts += [f"--suppress-native-package {pkg}" for pkg in sorted(suppress_native)] - return " ".join(parts) - diff --git a/src/rhdh_dynamic_plugin_factory/constants.py b/src/rhdh_dynamic_plugin_factory/constants.py new file mode 100644 index 0000000..d202521 --- /dev/null +++ b/src/rhdh_dynamic_plugin_factory/constants.py @@ -0,0 +1,42 @@ +""" +Shared constants for RHDH Plugin Factory. +""" + +import re +from pathlib import Path + +PLUGIN_LIST_FILE: str = "plugins-list.yaml" +SOURCE_CONFIG_FILE: str = "source.json" +PKG_JSON: str = "package.json" + +VALID_BACKSTAGE_PLUGIN_ROLES: set[str] = { + "frontend-plugin", + "backend-plugin", + "frontend-plugin-module", + "backend-plugin-module", +} + +BACKEND_ROLES: set[str] = { + "backend-plugin", + "backend-plugin-module", +} + +SKIP_DIRS: set[str] = { + "node_modules", + "dist", + "dist-dynamic", + ".git", + "__fixtures__", +} + +HOST_LOCKFILE: Path = ( + Path(__file__).parent.parent.parent / "resources" / "rhdh" / "yarn.lock" +) + +LOCKFILE_BACKSTAGE_RE: re.Pattern = re.compile( + r'"(@backstage/[\w.-]+)@npm:' +) + +NATIVE_DEP_MARKERS: frozenset[str] = frozenset[str]({ + "bindings", "prebuild", "nan", "node-pre-gyp", "node-gyp-build", +}) diff --git a/src/rhdh_dynamic_plugin_factory/plugin_list_config.py b/src/rhdh_dynamic_plugin_factory/plugin_list_config.py new file mode 100644 index 0000000..d2108d9 --- /dev/null +++ b/src/rhdh_dynamic_plugin_factory/plugin_list_config.py @@ -0,0 +1,487 @@ +""" +Plugin list configuration for RHDH Plugin Factory. + +Handles loading, saving, and build-argument computation for plugins-list.yaml files. +""" + +from logging import Logger +from pathlib import Path +import re +from typing import Dict, Optional, ClassVar +import yaml +import json + +from . import constants +from .logger import get_logger + + +class PluginListConfig: + """Configuration for plugin list (YAML format).""" + + logger: ClassVar[Logger] = get_logger("plugin_list") + + def __init__(self, plugins: Dict[str, str]): + """ + Initialize plugin list configuration. + + Args: + plugins: Dictionary mapping plugin paths to build arguments + """ + self.plugins = plugins + + @classmethod + def from_file(cls, plugin_list_file: Path) -> "PluginListConfig": + """Load plugin list from YAML file.""" + + with open(plugin_list_file, 'r') as f: + data = yaml.safe_load(f) or {} + + plugins = {} + for key, value in data.items(): + if value is None: + plugins[key] = "" + else: + plugins[key] = str(value) + + return cls(plugins) + + def to_file(self, plugin_list_file: Path) -> None: + """Save plugin list to YAML file. + + Writes manually rather than via yaml.dump so that entries with no + build args appear as ``key:`` (YAML null) instead of ``key: ''``. + + Args: + plugin_list_file: Destination path for the YAML file. + """ + with open(plugin_list_file, 'w') as f: + for path, args in self.plugins.items(): + if args: + f.write(f"{path}: {args}\n") + else: + f.write(f"{path}:\n") + + def get_plugins(self) -> Dict[str, str]: + return self.plugins.copy() + + def add_plugin(self, plugin_path: str, build_args: str = "") -> None: + self.plugins[plugin_path] = build_args + + def remove_plugin(self, plugin_path: str) -> None: + self.plugins.pop(plugin_path, None) + + def populate_build_args(self, workspace_path: Path) -> "PluginListConfig": + """(Re)compute build arguments for every plugin in the list. + + Uses the same dependency analysis as :meth:`create_default` but only + for the plugins already present in ``self.plugins``. All existing + build args are overwritten with freshly computed values. + + A before/after diff is logged for each plugin whose args changed so + the user has a record to revert from if needed. + + Args: + workspace_path: Absolute path to the workspace root + (must already have ``node_modules`` installed). + + Returns: + ``self``, mutated in place. + """ + original = self.plugins.copy() + host_packages = self._parse_host_backstage_packages(constants.HOST_LOCKFILE) + + for plugin_dir in self.plugins: + pkg_json_path = workspace_path / plugin_dir / constants.PKG_JSON + if not pkg_json_path.is_file(): + self.logger.warning( + f"Plugin package.json not found in workspace: {plugin_dir} " + f"(expected at {pkg_json_path})" + ) + self.plugins[plugin_dir] = "" + continue + + role = self._read_backstage_role(pkg_json_path) + if not role or role not in constants.VALID_BACKSTAGE_PLUGIN_ROLES: + self.logger.warning( + f"Plugin {plugin_dir} has no valid backstage.role — skipping build-arg computation. " + f"Found role: {role}. Valid roles are: {', '.join(constants.VALID_BACKSTAGE_PLUGIN_ROLES)}" + ) + self.plugins[plugin_dir] = "" + continue + + self.plugins[plugin_dir] = self._compute_plugin_build_args( + workspace_path, plugin_dir, pkg_json_path, host_packages, + ) + + self._log_build_args_diff(original, self.plugins) + return self + + @classmethod + def _log_build_args_diff( + cls, before: Dict[str, str], after: Dict[str, str], + ) -> None: + """Log a before/after comparison for plugins whose build args changed.""" + changed: list[str] = [] + unchanged: list[str] = [] + + for plugin_dir in after: + old = before.get(plugin_dir, "") + new = after[plugin_dir] + if old != new: + changed.append(plugin_dir) + else: + unchanged.append(plugin_dir) + + if changed: + cls.logger.info( + f"Build args updated for {len(changed)} of " + f"{len(after)} plugin(s):" + ) + for plugin_dir in changed: + old = before.get(plugin_dir, "") + new = after[plugin_dir] + cls.logger.info(f" {plugin_dir}:") + cls.logger.info(f" before: {old or '(empty)'}") + cls.logger.info(f" after: {new or '(empty)'}") + + if unchanged: + cls.logger.info( + f"Build args unchanged for {len(unchanged)} plugin(s):" + ) + for plugin_dir in unchanged: + cls.logger.info( + f" {plugin_dir}: {after[plugin_dir] or '(empty)'}" + ) + + @classmethod + def _compute_plugin_build_args( + cls, + workspace_path: Path, + plugin_dir: str, + pkg_json_path: Path, + host_packages: set[str], + ) -> str: + """Compute build args for a single plugin based on its backstage role. + + Returns the CLI argument string for backend plugins, or empty + string for frontend plugins. Returns empty string if the role + is not a valid plugin role. + """ + role = cls._read_backstage_role(pkg_json_path) + if not role or role not in constants.VALID_BACKSTAGE_PLUGIN_ROLES: + return "" + + if role in constants.BACKEND_ROLES: + return cls._compute_backend_build_args( + workspace_path, plugin_dir, pkg_json_path, host_packages, + ) + return "" + + @classmethod + def create_default(cls, workspace_path: Path) -> "PluginListConfig": + """Create a default plugin list by scanning workspace for Backstage plugins. + + Recursively walks *workspace_path* to find ``package.json`` files whose + ``backstage.role`` matches one of :pyattr:`VALID_BACKSTAGE_PLUGIN_ROLES`. + + For backend plugins, dependency analysis is performed against the + bundled RHDH host lockfile to determine ``--embed-package`` and + ``--shared-package`` arguments. + + Args: + workspace_path: Absolute path to the workspace root. + + Returns: + A :class:`PluginListConfig` with discovered plugins and build arg(s) (if any). + """ + plugins: Dict[str, str] = {} + host_packages = cls._parse_host_backstage_packages(constants.HOST_LOCKFILE) + + for pkg_json_path in cls._find_package_jsons(workspace_path): + role = cls._read_backstage_role(pkg_json_path) + if role and role in constants.VALID_BACKSTAGE_PLUGIN_ROLES: + plugin_dir = pkg_json_path.parent.relative_to(workspace_path).as_posix() + plugins[plugin_dir] = cls._compute_plugin_build_args( + workspace_path, plugin_dir, pkg_json_path, host_packages, + ) + + sorted_plugins = dict[str, str](sorted(plugins.items())) + cls.logger.debug(f"Discovered {len(sorted_plugins)} plugin(s) in {workspace_path}") + return cls(sorted_plugins) + + @classmethod + def _find_package_jsons(cls, root: Path) -> list[Path]: + """Recursively find package.json files, skipping non-plugin directories.""" + results: list[Path] = [] + + for entry in sorted(root.iterdir()): + if not entry.is_dir(): + continue + if entry.name in constants.SKIP_DIRS or entry.name.startswith("."): + continue + + pkg_json = entry / constants.PKG_JSON + if pkg_json.is_file(): + results.append(pkg_json) + + results.extend(cls._find_package_jsons(entry)) + + return results + + @classmethod + def _read_backstage_role(cls, pkg_json_path: Path) -> Optional[str]: + """Read the ``backstage.role`` field from a package.json file. + + Returns: + The role string, or *None* if the file cannot be parsed or has no role. + """ + try: + data = json.loads(pkg_json_path.read_text(encoding="utf-8")) + cls.logger.debug(f"Read backstage role from {pkg_json_path}: {data.get('backstage', {}).get('role')}") + return data.get("backstage", {}).get("role") + except (json.JSONDecodeError, OSError) as e: + cls.logger.warning(f"Failed to read {pkg_json_path}: {e}") + return None + + @classmethod + def _parse_host_backstage_packages(cls, lockfile_path: Path) -> set[str]: + """Extract ``@backstage/*`` package names from a Yarn Berry lockfile (Yarn 2+). + + Scans top-level key lines (e.g. + ``"@backstage/catalog-model@npm:^1.7.2, …":``) and collects distinct + package names. + + Args: + lockfile_path: Path to the host ``yarn.lock`` file. + + Returns: + Set of ``@backstage/*`` package names found in the lockfile, + or an empty set if the file does not exist. + """ + if not lockfile_path.is_file(): + cls.logger.warning(f"Host lockfile not found at {lockfile_path}") + return set[str]() + + packages: set[str] = set[str]() + for line in lockfile_path.read_text(encoding="utf-8").splitlines(): + if not line.startswith('"@backstage/'): + continue + for match in constants.LOCKFILE_BACKSTAGE_RE.finditer(line): + packages.add(match.group(1)) + + cls.logger.debug(f"Parsed {len(packages)} @backstage/* packages from host lockfile") + return packages + + @staticmethod + def _get_sibling_names(plugin_name: str, role: str) -> set[str]: + """Derive sibling package names that the RHDH CLI auto-embeds. + + Replicates the rhdh-cli logic: for backend plugins the CLI + automatically embeds the ``-common`` and ``-node`` siblings. + + Args: + plugin_name: The npm package name (e.g. ``@scope/my-plugin-backend``). + role: The ``backstage.role`` value. + + Returns: + Set of sibling package names, empty for non-backend roles. + """ + if role == "backend-plugin": + base = re.sub(r"-backend$", "", plugin_name) + elif role == "backend-plugin-module": + base = re.sub(r"-backend-module-.+$", "", plugin_name) + else: + return set[str]() + + if base == plugin_name: + return set[str]() + + return {f"{base}-common", f"{base}-node"} + + @classmethod + def _resolve_node_module_package_json( + cls, workspace_path: Path, dep_name: str + ) -> Optional[Path]: + """Locate a dependency's ``package.json`` in the workspace root ``node_modules``. + + Yarn workspaces hoist all packages to the workspace root, so only that + location is checked. + + Args: + workspace_path: Absolute path to the workspace root. + dep_name: npm package name (may be scoped, e.g. ``@aws/foo``). + + Returns: + Path to the dependency's ``package.json``, or *None* if not found. + """ + candidate = workspace_path / "node_modules" / dep_name / constants.PKG_JSON + if candidate.is_file(): + return candidate + return None + + @staticmethod + def _is_native_module(pkg_data: dict) -> bool: + """Check whether a ``package.json`` describes a native Node.js module. + + Replicates the logic of the ``is-native-module`` npm package used by RHDH CLI. + """ + deps = pkg_data.get("dependencies", {}) + if any(marker in deps for marker in constants.NATIVE_DEP_MARKERS): + return True + if pkg_data.get("gypfile"): + return True + if pkg_data.get("binary"): + return True + return False + + @classmethod + def _gather_native_modules( + cls, + workspace_path: Path, + private_dep_names: set[str], + ) -> set[str]: + """Find native modules in the transitive dependency tree of private deps. + + Recursively walks each dep's dependencies via ``node_modules``, + checking :meth:`_is_native_module` on every package encountered. + Tracks visited packages to avoid cycles. + + Args: + workspace_path: Absolute workspace root. + private_dep_names: Direct dep names to start the walk from. + + Returns: + Set of native package names found. + """ + native: set[str] = set[str]() + visited: set[str] = set[str]() + + def _walk(dep_name: str) -> None: + if dep_name in visited: + return + visited.add(dep_name) + + pkg_json = cls._resolve_node_module_package_json(workspace_path, dep_name) + if pkg_json is None: + return + + try: + data = json.loads(pkg_json.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError): + return + + if cls._is_native_module(data): + native.add(dep_name) + + for field in ("dependencies", "optionalDependencies"): + for sub_dep in data.get(field, {}): + _walk(sub_dep) + + for dep in private_dep_names: + _walk(dep) + + return native + + @classmethod + def _check_third_party_dep( + cls, + workspace_path: Path, + dep_name: str, + host_packages: set[str], + embed_packages: set[str], + unshare_packages: set[str], + ) -> None: + """Check a non-``@backstage/*`` dep for transitive shared-package usage. + + If the dependency has any ``@backstage/*`` dependencies it is + marked for embedding. Dependencies absent from *host_packages* are + additionally marked for unsharing. + + Results are collected directly into *embed_packages* / *unshare_packages*. + """ + dep_pkg_json = cls._resolve_node_module_package_json(workspace_path, dep_name) + if dep_pkg_json is None: + cls.logger.debug(f"Could not resolve {dep_name} in node_modules") + return + + try: + dep_data = json.loads(dep_pkg_json.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError) as e: + cls.logger.debug(f"Failed to read {dep_pkg_json}: {e}") + return + + dep_deps = dep_data.get("dependencies", {}) + backstage_deps = [d for d in dep_deps if d.startswith("@backstage/")] + + if backstage_deps: + embed_packages.add(dep_name) + for dep in backstage_deps: + if dep not in host_packages: + unshare_packages.add(dep) + + @classmethod + def _compute_backend_build_args( + cls, + workspace_path: Path, + plugin_dir: str, + pkg_json_path: Path, + host_packages: set[str], + ) -> str: + """Compute ``--embed-package`` / ``--shared-package`` args for a backend plugin. + + Analyses the plugin's direct dependencies: + + * ``@backstage/*`` deps missing from *host_packages* are unshared + **and** embedded (the host won't provide them at runtime). + * Non-``@backstage/*``, non-sibling deps whose own dependencies + include ``@backstage/*`` packages are embedded. Any of those + sub-deps missing from *host_packages* are additionally unshared. + + Args: + workspace_path: Absolute workspace root. + plugin_dir: Plugin directory relative to *workspace_path*. + pkg_json_path: Path to the plugin's ``package.json``. + host_packages: ``@backstage/*`` names present in the host lockfile. + + Returns: + CLI argument string, or ``""`` if no extra args are needed. + """ + try: + pkg_data = json.loads(pkg_json_path.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError): + return "" + + plugin_name: str = pkg_data.get("name", "") + role: str = pkg_data.get("backstage", {}).get("role", "") + dependencies: dict = pkg_data.get("dependencies", {}) + + siblings = cls._get_sibling_names(plugin_name, role) + + embed_packages: set[str] = set[str]() + unshare_packages: set[str] = set[str]() + private_deps: set[str] = set[str]() + + for dep_name in dependencies: + if dep_name in siblings: + continue + + if dep_name.startswith("@backstage/"): + if dep_name not in host_packages: + embed_packages.add(dep_name) + unshare_packages.add(dep_name) + continue + + private_deps.add(dep_name) + cls._check_third_party_dep( + workspace_path, dep_name, + host_packages, embed_packages, unshare_packages, + ) + + suppress_native = cls._gather_native_modules( + workspace_path, private_deps | embed_packages | siblings, + ) + + parts = [f"--embed-package {pkg}" for pkg in sorted(embed_packages)] + parts += [f"--shared-package !{pkg}" for pkg in sorted(unshare_packages)] + parts += [f"--suppress-native-package {pkg}" for pkg in sorted(suppress_native)] + return " ".join(parts) diff --git a/src/rhdh_dynamic_plugin_factory/source_config.py b/src/rhdh_dynamic_plugin_factory/source_config.py new file mode 100644 index 0000000..300a10f --- /dev/null +++ b/src/rhdh_dynamic_plugin_factory/source_config.py @@ -0,0 +1,344 @@ +""" +Source repository and workspace configuration for RHDH Plugin Factory. + +Handles git repository cloning, workspace discovery, and worktree management. +""" + +from logging import Logger +import os +from pathlib import Path +from typing import Optional, ClassVar +from dataclasses import dataclass +import json +import subprocess + +from .constants import SOURCE_CONFIG_FILE +from .exceptions import PluginFactoryError, ConfigurationError, ExecutionError +from .logger import get_logger +from .utils import run_command_with_streaming, prompt_or_clean_directory, repo_dir_name + + +@dataclass +class SourceConfig: + """Configuration for plugin source repository.""" + repo: str + repo_ref: Optional[str] # None triggers default branch resolution in __post_init__ + workspace_path: str + logger: ClassVar[Logger] = get_logger("source_config") + + def __post_init__(self) -> None: + if not self.repo: + raise ConfigurationError("repo is required") + if not self.workspace_path: + raise ConfigurationError("workspace-path is required") + + if not self.repo_ref: + self.repo_ref = self.resolve_default_ref(self.repo) + + @classmethod + def from_file(cls, source_file: Path) -> "SourceConfig": + """Load source configuration from JSON file. + + repo-ref is optional. When omitted, the default branch is resolved + automatically during construction via resolve_default_ref(). + + Raises: + ConfigurationError: If the file is missing, malformed, or has invalid data. + ExecutionError: If default branch resolution fails (when repo-ref is omitted). + """ + try: + with open(source_file, 'r') as f: + data = json.load(f) + except FileNotFoundError: + raise ConfigurationError(f"Source configuration file not found: {source_file}") + except json.JSONDecodeError as e: + raise ConfigurationError(f"Invalid JSON in {source_file}: {e}") + except Exception as e: + raise ConfigurationError(f"Failed to read {source_file}: {e}") + + try: + repo = data["repo"] + repo_ref = data.get("repo-ref") or None # Treat empty string as None + workspace_path = data.get("workspace-path") + except KeyError as e: + raise ConfigurationError(f"Missing required field {e} in {source_file}") + + config = cls( + repo=repo, + repo_ref=repo_ref, + workspace_path=workspace_path, + ) + + return config + + @classmethod + def from_cli_args(cls, repo: str, repo_ref: Optional[str], workspace_path: str) -> "SourceConfig": + """Create source configuration from CLI arguments. + + Args: + repo: Git repository URL (--source-repo). + repo_ref: Git ref to check out (--source-ref). None means default branch + (resolved automatically during construction via resolve_default_ref()). + workspace_path: Path to workspace within the repository (--workspace-path). + + Returns: + SourceConfig instance with repo_ref always resolved. + + Raises: + ConfigurationError: If required fields are missing. + ExecutionError: If default branch resolution fails (when repo_ref is None). + """ + return cls( + repo=repo, + repo_ref=repo_ref, + workspace_path=workspace_path, + ) + + @staticmethod + def resolve_default_ref(repo: str) -> str: + """Resolve the default branch ref for a repository using git ls-remote since repository is not cloned yet + + Args: + repo: Git repository URL. + + Returns: + The default branch ref (e.g., 'refs/heads/main'). + + Raises: + ExecutionError: If git ls-remote fails or the default branch cannot be determined. + """ + logger = get_logger("source_config") + logger.info(f"[cyan]Resolving default branch for {repo}cyan]") + + try: + result = subprocess.run( + ["git", "ls-remote", "--symref", repo, "HEAD"], + capture_output=True, + text=True, + check=True, + ) + + for line in result.stdout.splitlines(): + if line.startswith("ref:"): + # Ex: Extract "refs/heads/main" from "ref: refs/heads/main\tHEAD" + ref_part = line.split("\t")[0].replace("ref: ", "").strip() + logger.info(f"[green]Resolved default branch: {ref_part} for {repo}[/green]") + return ref_part + + raise ConfigurationError( + f"Could not resolve the default branch for '{repo}'. " + f"Please specify a branch or ref explicitly via 'repo-ref' in {SOURCE_CONFIG_FILE} " + "or the --source-ref CLI argument." + ) + except subprocess.CalledProcessError as e: + raise ExecutionError( + f"Failed to resolve default branch for {repo}: {e.stderr.strip()}", + step="resolve default ref", + returncode=e.returncode, + ) from e + + def clone_to_path(self, repo_path: Path, clean: bool = False) -> None: + """Clone the source repository to the specified path. + + Raises: + ConfigurationError: If the destination directory does not exist. + PluginFactoryError: If the user aborts the clone. + ExecutionError: If git clone or checkout fails. + """ + logger = get_logger("cli") + + if not repo_path.exists(): + raise ConfigurationError(f"Destination directory does not exist: {repo_path}") + + self.logger.info("[bold blue]Cloning repository[/bold blue]") + self.logger.info(f"Repository: {self.repo}") + self.logger.info(f"Reference: {self.repo_ref}") + self.logger.info(f"Destination directory: {repo_path}") + + prompt_or_clean_directory(repo_path, clean, self.logger) + + try: + cmd = ["git", "clone", self.repo, str(repo_path)] + returncode = run_command_with_streaming( + cmd, + logger, + stderr_log_func=logger.info + ) + + if returncode != 0: + raise ExecutionError( + f"Failed to clone repository (exit code {returncode})", + step="git clone", + returncode=returncode + ) + + cmd = ["git", "checkout", self.repo_ref] + logger.info(f"[cyan]Checking out ref: {self.repo_ref}[/cyan]") + returncode = run_command_with_streaming( + cmd, + logger, + cwd=repo_path, + stderr_log_func=logger.info + ) + + if returncode != 0: + raise ExecutionError( + f"Failed to checkout ref {self.repo_ref} (exit code {returncode})", + step="git checkout", + returncode=returncode + ) + + logger.info("[green]Repository cloned successfully[/green]") + + except PluginFactoryError: + raise + except Exception as e: + raise ExecutionError( + f"Failed during repository clone/checkout: {e}", + step="git clone/checkout" + ) from e + +@dataclass +class WorkspaceInfo: + """Per-workspace configuration for multi-workspace mode. + + Represents a single workspace discovered from a config subdirectory. + """ + name: str + config_dir: Path + source_config: SourceConfig + repo_path: Optional[Path] = None + output_dir: Optional[Path] = None + + def resolve_paths(self, base_repo_path: Path, base_output_dir: Path) -> None: + """Set per-workspace source code repo and output paths from base directories.""" + self.repo_path = base_repo_path / self.name + self.output_dir = base_output_dir / self.name + + +def discover_workspaces(config_dir: Path) -> list["WorkspaceInfo"]: + """Scan config directory for workspace subdirectories. + + A subdirectory is considered a workspace if it contains a source.json file. + Non-workspace entries are skipped silently; the caller is responsible for + warning the user about ignored content. + + Args: + config_dir: Root configuration directory to scan. + + Returns: + List of WorkspaceInfo instances sorted by repo URL (primary) then name (secondary), + making downstream groupby(repo) trivial. + + Raises: + ConfigurationError: If a workspace's source.json is invalid. + """ + logger = get_logger("config") + workspaces: list[WorkspaceInfo] = [] + + if not config_dir.is_dir(): + return workspaces + + for entry in sorted(config_dir.iterdir()): + if not entry.is_dir(): + continue + + source_file = entry / SOURCE_CONFIG_FILE + if not source_file.exists(): + logger.debug(f"Skipping {entry.name}/ — no {SOURCE_CONFIG_FILE}") + continue + + workspace_name = entry.name + logger.debug(f"Discovered workspace: {workspace_name}") + + source_config = SourceConfig.from_file(source_file) + + workspaces.append(WorkspaceInfo( + name=workspace_name, + config_dir=entry, + source_config=source_config, + )) + + workspaces.sort(key=lambda w: (w.source_config.repo, w.name)) + + return workspaces + + +def clone_workspaces_with_worktrees( + workspaces: list["WorkspaceInfo"], + base_repo_path: Path, +) -> None: + """Clone repositories and create git worktrees for multi-workspace mode. + + Groups workspaces by repo URL, clones each unique repo once into + /.clones//, then creates a worktree per + workspace at //. + + The caller is responsible for cleaning base_repo_path before calling + this function (e.g. via prompt_or_clean_directory). This function + assumes it can write freely into base_repo_path. + + Args: + workspaces: List of WorkspaceInfo (must already have repo_path set via resolve_paths). + base_repo_path: Base directory for repo clones and worktrees. + + Raises: + PluginFactoryError: If a workspace has no repo_path resolved (internal error). + ExecutionError: If any git operation fails. + """ + from itertools import groupby + + logger = get_logger("config") + clones_dir = base_repo_path / ".clones" + os.makedirs(clones_dir, exist_ok=True) + + for repo_url, group in groupby(workspaces, key=lambda w: w.source_config.repo): + workspace_list = list(group) + repo_name: str = repo_dir_name(repo_url) + clone_path: Path = clones_dir / repo_name + + logger.info(f"[bold blue]\nCloning base repository: {repo_url}[/bold blue]") + logger.info(f" Destination: {clone_path}") + + cmd = ["git", "clone", "--bare", repo_url, str(clone_path)] + returncode = run_command_with_streaming( + cmd, logger, stderr_log_func=logger.info + ) + + if returncode != 0: + raise ExecutionError( + f"Failed to clone repository '{repo_url}' (exit code {returncode}). " + f"Please verify the 'repo' URL in the {SOURCE_CONFIG_FILE} for workspaces using this repository. " + f"Ensure the URL is correct and accessible from your environment.", + step="git clone (bare)", + returncode=returncode, + ) + + logger.info(f"[green]Cloned {repo_url} to {clone_path}[/green]") + + for ws in workspace_list: + worktree_path = ws.repo_path + if worktree_path is None: + raise PluginFactoryError( + f"Internal error: workspace '{ws.name}' has no resolved repository path. " + f"This is a bug in the plugin factory. Please report this issue." + ) + + ref = ws.source_config.repo_ref + logger.info(f"[cyan]\nCreating git worktree for '{ws.name}' at ref {ref}[/cyan]") + + cmd = ["git", "worktree", "add", "--detach", str(worktree_path.resolve()), ref] + returncode = run_command_with_streaming( + cmd, logger, cwd=clone_path, stderr_log_func=logger.info + ) + + if returncode != 0: + raise ExecutionError( + f"Failed to create worktree for workspace '{ws.name}' at ref '{ref}' (exit code {returncode}). " + f"Please verify the 'repo-ref' value in {ws.config_dir / SOURCE_CONFIG_FILE}. " + f"Ensure the branch, tag, or commit exists in the repository.", + step="git worktree add", + returncode=returncode, + ) + + logger.info(f"[green] Worktree created for '{ws.name}' at {worktree_path}[/green]") From 6ad748466fbc909f3751cc049847ab7850643b2c Mon Sep 17 00:00:00 2001 From: Frank Kong Date: Tue, 10 Mar 2026 16:57:49 -0400 Subject: [PATCH 13/35] chore: add plugin build arg generation unit tests Assisted-By: Cursor Signed-off-by: Frank Kong --- tests/test_multi_workspace.py | 12 +- tests/test_plugin_list_config.py | 1046 +++++++++++++++++++++++++++++- tests/test_source_config.py | 27 +- 3 files changed, 1063 insertions(+), 22 deletions(-) diff --git a/tests/test_multi_workspace.py b/tests/test_multi_workspace.py index 8de8505..f77770d 100644 --- a/tests/test_multi_workspace.py +++ b/tests/test_multi_workspace.py @@ -11,8 +11,8 @@ from unittest.mock import patch, MagicMock import pytest -from src.rhdh_dynamic_plugin_factory.config import ( - PluginFactoryConfig, +from src.rhdh_dynamic_plugin_factory.config import PluginFactoryConfig +from src.rhdh_dynamic_plugin_factory.source_config import ( SourceConfig, WorkspaceInfo, discover_workspaces, @@ -356,7 +356,7 @@ def test_groups_by_repo_and_clones_once(self, tmp_path): output_dir=tmp_path / "out" / "ws2", ) - with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming", return_value=0) as mock_stream: + with patch("src.rhdh_dynamic_plugin_factory.source_config.run_command_with_streaming", return_value=0) as mock_stream: clone_workspaces_with_worktrees([ws1, ws2], base_repo) cmds = [c[0][0] for c in mock_stream.call_args_list] @@ -378,7 +378,7 @@ def test_raises_if_repo_path_not_set(self, tmp_path): repo_path=None, ) - with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming", return_value=0): + with patch("src.rhdh_dynamic_plugin_factory.source_config.run_command_with_streaming", return_value=0): with pytest.raises(PluginFactoryError, match="no resolved repository path"): clone_workspaces_with_worktrees([ws], base_repo) @@ -395,7 +395,7 @@ def test_clone_failure_raises_error(self, tmp_path): output_dir=tmp_path / "out" / "ws1", ) - with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming", return_value=128): + with patch("src.rhdh_dynamic_plugin_factory.source_config.run_command_with_streaming", return_value=128): with pytest.raises(ExecutionError, match="Failed to clone repository"): clone_workspaces_with_worktrees([ws], base_repo) @@ -419,7 +419,7 @@ def test_multiple_repos_each_cloned_once(self, tmp_path): output_dir=tmp_path / "out" / "ws2", ) - with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming", return_value=0) as mock_stream: + with patch("src.rhdh_dynamic_plugin_factory.source_config.run_command_with_streaming", return_value=0) as mock_stream: clone_workspaces_with_worktrees([ws1, ws2], base_repo) cmds = [c[0][0] for c in mock_stream.call_args_list] diff --git a/tests/test_plugin_list_config.py b/tests/test_plugin_list_config.py index e318245..11abda9 100644 --- a/tests/test_plugin_list_config.py +++ b/tests/test_plugin_list_config.py @@ -8,7 +8,8 @@ import yaml import pytest -from src.rhdh_dynamic_plugin_factory.config import PluginListConfig +from src.rhdh_dynamic_plugin_factory.plugin_list_config import PluginListConfig +from src.rhdh_dynamic_plugin_factory import constants class TestPluginListConfigFromFile: @@ -207,15 +208,31 @@ def test_to_file_null_values_format(self, tmp_path): assert '""' not in content -def _make_plugin_dir(base, rel_path, name, role): +def _make_plugin_dir(base, rel_path, name, role, dependencies=None): """Helper to create a plugin directory with a package.json.""" plugin_dir = base / rel_path plugin_dir.mkdir(parents=True, exist_ok=True) pkg = {"name": name, "version": "1.0.0", "backstage": {"role": role}} + if dependencies: + pkg["dependencies"] = dependencies (plugin_dir / "package.json").write_text(json.dumps(pkg)) return plugin_dir +def _make_node_module(base, dep_name, dependencies=None, + optional_dependencies=None): + """Helper to create a workspace-root node_modules//package.json entry.""" + nm_dir = base / "node_modules" / dep_name + nm_dir.mkdir(parents=True, exist_ok=True) + pkg = {"name": dep_name, "version": "1.0.0"} + if dependencies: + pkg["dependencies"] = dependencies + if optional_dependencies: + pkg["optionalDependencies"] = optional_dependencies + (nm_dir / "package.json").write_text(json.dumps(pkg)) + return nm_dir + + class TestPluginListConfigCreateDefault: """Tests for PluginListConfig.create_default method.""" @@ -364,7 +381,6 @@ def test_aws_ecs_workspace_structure(self, tmp_path): """Test a workspace matching the AWS ECS plugins example (nested dirs).""" _make_plugin_dir(tmp_path, "plugins/ecs/frontend", "@aws/amazon-ecs-plugin-for-backstage", "frontend-plugin") _make_plugin_dir(tmp_path, "plugins/ecs/backend", "@aws/amazon-ecs-plugin-for-backstage-backend", "backend-plugin") - # Also has common/node packages that should be ignored (not plugin roles) _make_plugin_dir(tmp_path, "plugins/ecs/common", "@aws/aws-core-plugin-for-backstage-common", "common-library") config = PluginListConfig.create_default(tmp_path) @@ -373,3 +389,1027 @@ def test_aws_ecs_workspace_structure(self, tmp_path): assert len(plugins) == 2 assert "plugins/ecs/frontend" in plugins assert "plugins/ecs/backend" in plugins + + +class TestParseHostBackstagePackages: + """Tests for PluginListConfig._parse_host_backstage_packages.""" + + def test_single_entry(self, tmp_path): + lockfile = tmp_path / "yarn.lock" + lockfile.write_text( + '"@backstage/catalog-model@npm:^1.7.2":\n version: 1.7.2\n' + ) + result = PluginListConfig._parse_host_backstage_packages(lockfile) + assert result == {"@backstage/catalog-model"} + + def test_multi_version_entry(self, tmp_path): + lockfile = tmp_path / "yarn.lock" + lockfile.write_text( + '"@backstage/catalog-model@npm:^1.7.2, @backstage/catalog-model@npm:^1.7.3":\n' + " version: 1.9.0\n" + ) + result = PluginListConfig._parse_host_backstage_packages(lockfile) + assert result == {"@backstage/catalog-model"} + + def test_multiple_packages(self, tmp_path): + lockfile = tmp_path / "yarn.lock" + lockfile.write_text( + '"@backstage/catalog-model@npm:^1.7.2":\n version: 1.7.2\n\n' + '"@backstage/errors@npm:^1.2.7":\n version: 1.2.7\n' + ) + result = PluginListConfig._parse_host_backstage_packages(lockfile) + assert result == {"@backstage/catalog-model", "@backstage/errors"} + + def test_non_backstage_entries_ignored(self, tmp_path): + lockfile = tmp_path / "yarn.lock" + lockfile.write_text( + '"@aws/sdk@npm:^1.0.0":\n version: 1.0.0\n\n' + '"@backstage/errors@npm:^1.2.7":\n version: 1.2.7\n' + ) + result = PluginListConfig._parse_host_backstage_packages(lockfile) + assert result == {"@backstage/errors"} + + def test_missing_file_returns_empty(self, tmp_path): + lockfile = tmp_path / "nonexistent.lock" + result = PluginListConfig._parse_host_backstage_packages(lockfile) + assert result == set() + + def test_backstage_in_dependency_lines_ignored(self, tmp_path): + """Only top-level key lines (starting with quotes) are parsed, not indented dep lines.""" + lockfile = tmp_path / "yarn.lock" + lockfile.write_text( + '"@some/package@npm:^1.0.0":\n' + ' version: 1.0.0\n' + ' dependencies:\n' + ' "@backstage/types": "npm:^1.2.1"\n' + ) + result = PluginListConfig._parse_host_backstage_packages(lockfile) + assert result == set() + + +class TestGetSiblingNames: + """Tests for PluginListConfig._get_sibling_names.""" + + def test_backend_plugin(self): + result = PluginListConfig._get_sibling_names( + "@scope/my-plugin-backend", "backend-plugin" + ) + assert result == {"@scope/my-plugin-common", "@scope/my-plugin-node"} + + def test_backend_plugin_module(self): + result = PluginListConfig._get_sibling_names( + "@scope/my-plugin-backend-module-github", "backend-plugin-module" + ) + assert result == {"@scope/my-plugin-common", "@scope/my-plugin-node"} + + def test_frontend_plugin_returns_empty(self): + result = PluginListConfig._get_sibling_names( + "@scope/my-plugin", "frontend-plugin" + ) + assert result == set() + + def test_frontend_plugin_module_returns_empty(self): + result = PluginListConfig._get_sibling_names( + "@scope/my-plugin-module", "frontend-plugin-module" + ) + assert result == set() + + def test_name_without_matching_suffix(self): + """If the name doesn't end with -backend, no siblings are derived.""" + result = PluginListConfig._get_sibling_names( + "@scope/weird-name", "backend-plugin" + ) + assert result == set() + + def test_scoped_package_preserves_scope(self): + result = PluginListConfig._get_sibling_names( + "@red-hat-developer-hub/backstage-plugin-bulk-import-backend", + "backend-plugin", + ) + assert result == { + "@red-hat-developer-hub/backstage-plugin-bulk-import-common", + "@red-hat-developer-hub/backstage-plugin-bulk-import-node", + } + + +class TestResolveNodeModule: + """Tests for PluginListConfig._resolve_node_module_package_json.""" + + def test_finds_package_in_root_node_modules(self, tmp_path): + _make_node_module(tmp_path, "@aws/common") + result = PluginListConfig._resolve_node_module_package_json(tmp_path, "@aws/common") + assert result is not None + assert result.is_file() + + def test_not_found_returns_none(self, tmp_path): + result = PluginListConfig._resolve_node_module_package_json(tmp_path, "@aws/missing") + assert result is None + + def test_scoped_package(self, tmp_path): + _make_node_module(tmp_path, "@aws/aws-core-plugin-for-backstage-common") + result = PluginListConfig._resolve_node_module_package_json( + tmp_path, "@aws/aws-core-plugin-for-backstage-common", + ) + assert result is not None + + +class TestComputeBackendBuildArgs: + """Tests for PluginListConfig._compute_backend_build_args.""" + + def test_dep_with_backstage_sub_deps_in_host(self, tmp_path): + """Third-party dep has @backstage/* sub-deps in host -> embed only, no unshare.""" + _make_plugin_dir( + tmp_path, "plugins/backend", "@test/my-backend", "backend-plugin", + dependencies={"@aws/common": "^1.0.0", "@backstage/core": "^1.0.0"}, + ) + _make_node_module( + tmp_path, "@aws/common", + dependencies={"@backstage/catalog-model": "^1.7.0"}, + ) + + host = {"@backstage/catalog-model", "@backstage/core"} + result = PluginListConfig._compute_backend_build_args( + tmp_path, "plugins/backend", + tmp_path / "plugins/backend/package.json", host, + ) + assert "--embed-package @aws/common" in result + assert "--shared-package" not in result + + def test_dep_with_backstage_sub_deps_not_in_host(self, tmp_path): + """Third-party dep has @backstage/* sub-deps NOT in host -> embed + unshare.""" + _make_plugin_dir( + tmp_path, "plugins/backend", "@test/my-backend", "backend-plugin", + dependencies={"@custom/lib": "^1.0.0"}, + ) + _make_node_module( + tmp_path, "@custom/lib", + dependencies={"@backstage/new-pkg": "^1.0.0"}, + ) + + result = PluginListConfig._compute_backend_build_args( + tmp_path, "plugins/backend", + tmp_path / "plugins/backend/package.json", {"@backstage/core"}, + ) + assert "--embed-package @custom/lib" in result + assert "--shared-package !@backstage/new-pkg" in result + + def test_no_backstage_sub_deps(self, tmp_path): + """Third-party dep with no @backstage/* sub-deps -> no args.""" + _make_plugin_dir( + tmp_path, "plugins/backend", "@test/my-backend", "backend-plugin", + dependencies={"lodash": "^4.0.0"}, + ) + _make_node_module( + tmp_path, "lodash", + dependencies={"underscore": "^1.0.0"}, + ) + + result = PluginListConfig._compute_backend_build_args( + tmp_path, "plugins/backend", + tmp_path / "plugins/backend/package.json", set(), + ) + assert result == "" + + def test_sibling_deps_skipped(self, tmp_path): + """Sibling deps (-common, -node) are not embedded even if they have @backstage/* sub-deps.""" + _make_plugin_dir( + tmp_path, "plugins/backend", "@test/my-plugin-backend", "backend-plugin", + dependencies={ + "@test/my-plugin-common": "^1.0.0", + "@test/my-plugin-node": "^1.0.0", + }, + ) + _make_node_module( + tmp_path, "@test/my-plugin-common", + dependencies={"@backstage/catalog-model": "^1.0.0"}, + ) + + result = PluginListConfig._compute_backend_build_args( + tmp_path, "plugins/backend", + tmp_path / "plugins/backend/package.json", set(), + ) + assert result == "" + + def test_backstage_direct_dep_missing_from_host(self, tmp_path): + """@backstage/* direct dep NOT in host -> embed + unshare.""" + _make_plugin_dir( + tmp_path, "plugins/backend", "@test/my-backend", "backend-plugin", + dependencies={"@backstage/new-experimental": "^0.1.0"}, + ) + + result = PluginListConfig._compute_backend_build_args( + tmp_path, "plugins/backend", + tmp_path / "plugins/backend/package.json", + {"@backstage/core", "@backstage/errors"}, + ) + assert "--embed-package @backstage/new-experimental" in result + assert "--shared-package !@backstage/new-experimental" in result + + def test_backstage_direct_dep_present_in_host(self, tmp_path): + """@backstage/* direct dep in host -> no action (it stays shared).""" + _make_plugin_dir( + tmp_path, "plugins/backend", "@test/my-backend", "backend-plugin", + dependencies={"@backstage/catalog-model": "^1.7.0"}, + ) + + result = PluginListConfig._compute_backend_build_args( + tmp_path, "plugins/backend", + tmp_path / "plugins/backend/package.json", + {"@backstage/catalog-model"}, + ) + assert result == "" + + def test_mixed_scenario(self, tmp_path): + """Multiple deps with different outcomes combined correctly.""" + _make_plugin_dir( + tmp_path, "plugins/backend", "@test/my-plugin-backend", "backend-plugin", + dependencies={ + "@backstage/catalog-model": "^1.7.0", + "@backstage/new-pkg": "^0.1.0", + "@test/my-plugin-common": "^1.0.0", + "@custom/lib": "^1.0.0", + "lodash": "^4.0.0", + }, + ) + _make_node_module( + tmp_path, "@custom/lib", + dependencies={"@backstage/errors": "^1.2.0", "@backstage/missing": "^0.1.0"}, + ) + _make_node_module(tmp_path, "lodash") + + host = {"@backstage/catalog-model", "@backstage/errors"} + result = PluginListConfig._compute_backend_build_args( + tmp_path, "plugins/backend", + tmp_path / "plugins/backend/package.json", host, + ) + assert "--embed-package @backstage/new-pkg" in result + assert "--embed-package @custom/lib" in result + assert "--shared-package !@backstage/new-pkg" in result + assert "--shared-package !@backstage/missing" in result + assert "lodash" not in result + assert "@test/my-plugin-common" not in result + + def test_unresolvable_dep_skipped(self, tmp_path): + """Dep not found in node_modules is silently skipped.""" + _make_plugin_dir( + tmp_path, "plugins/backend", "@test/my-backend", "backend-plugin", + dependencies={"@missing/package": "^1.0.0"}, + ) + + result = PluginListConfig._compute_backend_build_args( + tmp_path, "plugins/backend", + tmp_path / "plugins/backend/package.json", set(), + ) + assert result == "" + + def test_malformed_dep_package_json_skipped(self, tmp_path): + """Dep with invalid package.json in node_modules is skipped.""" + _make_plugin_dir( + tmp_path, "plugins/backend", "@test/my-backend", "backend-plugin", + dependencies={"bad-dep": "^1.0.0"}, + ) + nm_dir = tmp_path / "node_modules/bad-dep" + nm_dir.mkdir(parents=True) + (nm_dir / "package.json").write_text("{ broken json") + + result = PluginListConfig._compute_backend_build_args( + tmp_path, "plugins/backend", + tmp_path / "plugins/backend/package.json", set(), + ) + assert result == "" + + +class TestCreateDefaultWithEmbedArgs: + """End-to-end tests for create_default with Phase 2 embed/unshare detection.""" + + def test_aws_ecs_like_workspace(self, tmp_path, monkeypatch): + """Backend plugin with third-party dep that has @backstage/* sub-deps.""" + lockfile = tmp_path / "host-yarn.lock" + lockfile.write_text( + '"@backstage/catalog-model@npm:^1.7.2":\n version: 1.9.0\n\n' + '"@backstage/errors@npm:^1.2.7":\n version: 1.2.7\n' + ) + monkeypatch.setattr(constants, "HOST_LOCKFILE", lockfile) + + workspace = tmp_path / "workspace" + workspace.mkdir() + + _make_plugin_dir( + workspace, "plugins/ecs/frontend", + "@aws/amazon-ecs-plugin-for-backstage", "frontend-plugin", + ) + _make_plugin_dir( + workspace, "plugins/ecs/backend", + "@aws/amazon-ecs-plugin-for-backstage-backend", "backend-plugin", + dependencies={ + "@aws/aws-core-plugin-for-backstage-common": "^0.2.0", + "@backstage/catalog-model": "^1.7.0", + }, + ) + _make_node_module( + workspace, "@aws/aws-core-plugin-for-backstage-common", + dependencies={"@backstage/catalog-model": "^1.7.0"}, + ) + + config = PluginListConfig.create_default(workspace) + plugins = config.get_plugins() + + assert plugins["plugins/ecs/frontend"] == "" + assert "--embed-package @aws/aws-core-plugin-for-backstage-common" in plugins["plugins/ecs/backend"] + assert "--shared-package" not in plugins["plugins/ecs/backend"] + + def test_frontend_plugins_get_no_args(self, tmp_path, monkeypatch): + lockfile = tmp_path / "host-yarn.lock" + lockfile.write_text("") + monkeypatch.setattr(constants, "HOST_LOCKFILE", lockfile) + + workspace = tmp_path / "workspace" + workspace.mkdir() + _make_plugin_dir( + workspace, "plugins/todo", + "@backstage-community/plugin-todo", "frontend-plugin", + ) + + config = PluginListConfig.create_default(workspace) + assert config.get_plugins()["plugins/todo"] == "" + + def test_backend_no_deps_no_args(self, tmp_path, monkeypatch): + lockfile = tmp_path / "host-yarn.lock" + lockfile.write_text('"@backstage/core@npm:^1.0.0":\n version: 1.0.0\n') + monkeypatch.setattr(constants, "HOST_LOCKFILE", lockfile) + + workspace = tmp_path / "workspace" + workspace.mkdir() + _make_plugin_dir( + workspace, "plugins/todo-backend", + "@backstage-community/plugin-todo-backend", "backend-plugin", + ) + + config = PluginListConfig.create_default(workspace) + assert config.get_plugins()["plugins/todo-backend"] == "" + + def test_backstage_dep_not_in_host_unshared_and_embedded(self, tmp_path, monkeypatch): + """@backstage/* direct dep missing from host gets both embed and unshare.""" + lockfile = tmp_path / "host-yarn.lock" + lockfile.write_text('"@backstage/core@npm:^1.0.0":\n version: 1.0.0\n') + monkeypatch.setattr(constants, "HOST_LOCKFILE", lockfile) + + workspace = tmp_path / "workspace" + workspace.mkdir() + _make_plugin_dir( + workspace, "plugins/my-backend", + "@test/my-backend", "backend-plugin", + dependencies={"@backstage/new-experimental": "^0.1.0"}, + ) + + config = PluginListConfig.create_default(workspace) + args = config.get_plugins()["plugins/my-backend"] + assert "--embed-package @backstage/new-experimental" in args + assert "--shared-package !@backstage/new-experimental" in args + + def test_missing_host_lockfile_still_works(self, tmp_path, monkeypatch): + """When host lockfile is missing, all @backstage/* deps are treated as absent.""" + monkeypatch.setattr( + constants, "HOST_LOCKFILE", tmp_path / "nonexistent.lock" + ) + + workspace = tmp_path / "workspace" + workspace.mkdir() + _make_plugin_dir( + workspace, "plugins/backend", + "@test/my-backend", "backend-plugin", + dependencies={"@backstage/catalog-model": "^1.7.0"}, + ) + + config = PluginListConfig.create_default(workspace) + args = config.get_plugins()["plugins/backend"] + assert "--embed-package @backstage/catalog-model" in args + assert "--shared-package !@backstage/catalog-model" in args + + +class TestIsNativeModule: + """Tests for PluginListConfig._is_native_module.""" + + def test_bindings_dependency(self): + assert PluginListConfig._is_native_module( + {"dependencies": {"bindings": "^1.5.0"}} + ) + + def test_prebuild_dependency(self): + assert PluginListConfig._is_native_module( + {"dependencies": {"prebuild": "^1.0.0"}} + ) + + def test_nan_dependency(self): + assert PluginListConfig._is_native_module( + {"dependencies": {"nan": "^2.0.0"}} + ) + + def test_node_pre_gyp_dependency(self): + assert PluginListConfig._is_native_module( + {"dependencies": {"node-pre-gyp": "^0.15.0"}} + ) + + def test_node_gyp_build_dependency(self): + assert PluginListConfig._is_native_module( + {"dependencies": {"node-gyp-build": "^4.0.0"}} + ) + + def test_gypfile_field(self): + assert PluginListConfig._is_native_module({"gypfile": True}) + + def test_binary_field(self): + assert PluginListConfig._is_native_module( + {"binary": {"module_name": "addon"}} + ) + + def test_non_native_package(self): + assert not PluginListConfig._is_native_module( + {"dependencies": {"lodash": "^4.0.0"}} + ) + + def test_empty_package(self): + assert not PluginListConfig._is_native_module({}) + + def test_no_dependencies_key(self): + assert not PluginListConfig._is_native_module({"name": "foo", "version": "1.0.0"}) + + +class TestGatherNativeModules: + """Tests for PluginListConfig._gather_native_modules.""" + + def test_finds_native_transitive_dep(self, tmp_path): + """A private dep depends on a native module -> that module is found.""" + _make_node_module( + tmp_path, "ssh2", + dependencies={"cpu-features": "^0.0.9"}, + ) + _make_node_module( + tmp_path, "cpu-features", + dependencies={"node-gyp-build": "^4.0.0"}, + ) + + result = PluginListConfig._gather_native_modules(tmp_path, {"ssh2"}) + assert result == {"cpu-features"} + + def test_no_native_deps(self, tmp_path): + _make_node_module( + tmp_path, "lodash", + dependencies={"underscore": "^1.0.0"}, + ) + _make_node_module(tmp_path, "underscore") + + result = PluginListConfig._gather_native_modules(tmp_path, {"lodash"}) + assert result == set() + + def test_cycle_avoidance(self, tmp_path): + """Circular dependencies don't cause infinite recursion.""" + _make_node_module( + tmp_path, "a", + dependencies={"b": "^1.0.0"}, + ) + _make_node_module( + tmp_path, "b", + dependencies={"a": "^1.0.0"}, + ) + + result = PluginListConfig._gather_native_modules(tmp_path, {"a"}) + assert result == set() + + def test_unresolvable_dep_skipped(self, tmp_path): + result = PluginListConfig._gather_native_modules(tmp_path, {"nonexistent"}) + assert result == set() + + def test_direct_dep_is_native(self, tmp_path): + """The private dep itself is native.""" + _make_node_module( + tmp_path, "cpu-features", + dependencies={"node-gyp-build": "^4.0.0"}, + ) + + result = PluginListConfig._gather_native_modules(tmp_path, {"cpu-features"}) + assert result == {"cpu-features"} + + def test_multiple_native_deps(self, tmp_path): + """Multiple native modules found across different branches.""" + _make_node_module( + tmp_path, "parent", + dependencies={"native-a": "^1.0.0", "native-b": "^1.0.0"}, + ) + _make_node_module( + tmp_path, "native-a", + dependencies={"nan": "^2.0.0"}, + ) + _make_node_module( + tmp_path, "native-b", + dependencies={"bindings": "^1.5.0"}, + ) + + result = PluginListConfig._gather_native_modules(tmp_path, {"parent"}) + assert result == {"native-a", "native-b"} + + def test_native_in_optional_dependencies(self, tmp_path): + """Native modules reachable via optionalDependencies are found.""" + _make_node_module( + tmp_path, "ssh2", + optional_dependencies={"cpu-features": "~0.0.10"}, + ) + _make_node_module( + tmp_path, "cpu-features", + dependencies={"nan": "^2.19.0"}, + ) + + result = PluginListConfig._gather_native_modules(tmp_path, {"ssh2"}) + assert result == {"cpu-features"} + + def test_mixed_deps_and_optional_deps(self, tmp_path): + """Walker follows both dependencies and optionalDependencies.""" + _make_node_module( + tmp_path, "docker-modem", + dependencies={"readable-stream": "^3.0.0"}, + optional_dependencies={"ssh2": "^1.15.0"}, + ) + _make_node_module(tmp_path, "readable-stream") + _make_node_module( + tmp_path, "ssh2", + optional_dependencies={"cpu-features": "~0.0.10"}, + ) + _make_node_module( + tmp_path, "cpu-features", + dependencies={"nan": "^2.19.0"}, + ) + + result = PluginListConfig._gather_native_modules(tmp_path, {"docker-modem"}) + assert result == {"cpu-features"} + + +class TestComputeBackendBuildArgsWithNative: + """Tests for --suppress-native-package in _compute_backend_build_args.""" + + def test_private_dep_with_native_transitive(self, tmp_path): + """Private dep has a native transitive dep -> suppress it.""" + _make_plugin_dir( + tmp_path, "plugins/backend", "@test/my-backend", "backend-plugin", + dependencies={"ssh2": "^1.0.0"}, + ) + _make_node_module( + tmp_path, "ssh2", + dependencies={"cpu-features": "^0.0.9"}, + ) + _make_node_module( + tmp_path, "cpu-features", + dependencies={"node-gyp-build": "^4.0.0"}, + ) + + result = PluginListConfig._compute_backend_build_args( + tmp_path, "plugins/backend", + tmp_path / "plugins/backend/package.json", set(), + ) + assert "--suppress-native-package cpu-features" in result + + def test_no_native_deps_no_suppress(self, tmp_path): + """Private dep with no native transitive deps -> no suppress flags.""" + _make_plugin_dir( + tmp_path, "plugins/backend", "@test/my-backend", "backend-plugin", + dependencies={"lodash": "^4.0.0"}, + ) + _make_node_module(tmp_path, "lodash") + + result = PluginListConfig._compute_backend_build_args( + tmp_path, "plugins/backend", + tmp_path / "plugins/backend/package.json", set(), + ) + assert "--suppress-native-package" not in result + + def test_combined_embed_and_suppress(self, tmp_path): + """Embed, unshare, AND suppress flags all present.""" + _make_plugin_dir( + tmp_path, "plugins/backend", "@test/my-backend", "backend-plugin", + dependencies={ + "@custom/lib": "^1.0.0", + "ssh2": "^1.0.0", + }, + ) + _make_node_module( + tmp_path, "@custom/lib", + dependencies={"@backstage/catalog-model": "^1.7.0"}, + ) + _make_node_module( + tmp_path, "ssh2", + dependencies={"cpu-features": "^0.0.9"}, + ) + _make_node_module( + tmp_path, "cpu-features", + dependencies={"node-gyp-build": "^4.0.0"}, + ) + + host = {"@backstage/catalog-model"} + result = PluginListConfig._compute_backend_build_args( + tmp_path, "plugins/backend", + tmp_path / "plugins/backend/package.json", host, + ) + assert "--embed-package @custom/lib" in result + assert "--suppress-native-package cpu-features" in result + + def test_args_ordering(self, tmp_path): + """Flags are ordered: --embed-package, --shared-package, --suppress-native-package.""" + _make_plugin_dir( + tmp_path, "plugins/backend", "@test/my-backend", "backend-plugin", + dependencies={ + "@backstage/new-pkg": "^0.1.0", + "@custom/lib": "^1.0.0", + "ssh2": "^1.0.0", + }, + ) + _make_node_module( + tmp_path, "@custom/lib", + dependencies={"@backstage/missing": "^0.1.0"}, + ) + _make_node_module( + tmp_path, "ssh2", + dependencies={"cpu-features": "^0.0.9"}, + ) + _make_node_module( + tmp_path, "cpu-features", + dependencies={"node-gyp-build": "^4.0.0"}, + ) + + result = PluginListConfig._compute_backend_build_args( + tmp_path, "plugins/backend", + tmp_path / "plugins/backend/package.json", set(), + ) + embed_idx = result.index("--embed-package") + shared_idx = result.index("--shared-package") + suppress_idx = result.index("--suppress-native-package") + assert embed_idx < shared_idx < suppress_idx + + +class TestComputeBackendBuildArgsEmbeddedBackstageNative: + """Native modules in embedded @backstage/* deps should be suppressed.""" + + def test_embedded_backstage_dep_with_native_transitive(self, tmp_path): + """An @backstage/* dep not in host gets embedded; its transitive native dep is suppressed. + + Mirrors the real chain: backend-common -> dockerode -> docker-modem + -> ssh2 -[optional]-> cpu-features (native). + """ + _make_plugin_dir( + tmp_path, "plugins/backend", "@test/my-backend", "backend-plugin", + dependencies={"@backstage/backend-common": "^1.0.0"}, + ) + _make_node_module( + tmp_path, "@backstage/backend-common", + dependencies={"dockerode": "^4.0.0"}, + ) + _make_node_module( + tmp_path, "dockerode", + dependencies={"docker-modem": "^3.0.0"}, + ) + _make_node_module( + tmp_path, "docker-modem", + optional_dependencies={"ssh2": "^1.15.0"}, + ) + _make_node_module( + tmp_path, "ssh2", + optional_dependencies={"cpu-features": "~0.0.10"}, + ) + _make_node_module( + tmp_path, "cpu-features", + dependencies={"nan": "^2.19.0"}, + ) + + result = PluginListConfig._compute_backend_build_args( + tmp_path, "plugins/backend", + tmp_path / "plugins/backend/package.json", set(), + ) + assert "--embed-package @backstage/backend-common" in result + assert "--shared-package !@backstage/backend-common" in result + assert "--suppress-native-package cpu-features" in result + + +class TestComputeBackendBuildArgsSiblingNative: + """Native modules in auto-embedded sibling deps should be suppressed.""" + + def test_sibling_node_dep_with_native_transitive(self, tmp_path): + """Mirrors the real techdocs-backend chain: + sibling techdocs-node -> dockerode -> docker-modem -> ssh2 -> cpu-features. + """ + _make_plugin_dir( + tmp_path, "plugins/techdocs-backend", + "@backstage/plugin-techdocs-backend", "backend-plugin", + dependencies={ + "@backstage/plugin-techdocs-node": "workspace:^", + "express": "^4.22.0", + }, + ) + _make_node_module( + tmp_path, "@backstage/plugin-techdocs-node", + dependencies={"dockerode": "^4.0.0"}, + ) + _make_node_module( + tmp_path, "dockerode", + dependencies={"docker-modem": "^3.0.0"}, + ) + _make_node_module( + tmp_path, "docker-modem", + optional_dependencies={"ssh2": "^1.15.0"}, + ) + _make_node_module( + tmp_path, "ssh2", + optional_dependencies={"cpu-features": "~0.0.10"}, + ) + _make_node_module( + tmp_path, "cpu-features", + dependencies={"nan": "^2.19.0"}, + ) + _make_node_module(tmp_path, "express") + + result = PluginListConfig._compute_backend_build_args( + tmp_path, "plugins/techdocs-backend", + tmp_path / "plugins/techdocs-backend/package.json", set(), + ) + assert "--suppress-native-package cpu-features" in result + assert "--embed-package" not in result + + def test_sibling_without_native_deps_no_suppress(self, tmp_path): + """Sibling with no native transitive deps produces no suppress flags.""" + _make_plugin_dir( + tmp_path, "plugins/todo-backend", + "@test/plugin-todo-backend", "backend-plugin", + dependencies={ + "@test/plugin-todo-common": "^1.0.0", + "@test/plugin-todo-node": "^1.0.0", + }, + ) + _make_node_module( + tmp_path, "@test/plugin-todo-common", + dependencies={"lodash": "^4.0.0"}, + ) + _make_node_module( + tmp_path, "@test/plugin-todo-node", + dependencies={"express": "^4.0.0"}, + ) + _make_node_module(tmp_path, "lodash") + _make_node_module(tmp_path, "express") + + result = PluginListConfig._compute_backend_build_args( + tmp_path, "plugins/todo-backend", + tmp_path / "plugins/todo-backend/package.json", set(), + ) + assert result == "" + + def test_sibling_not_in_node_modules_skipped(self, tmp_path): + """Unresolvable sibling (not in node_modules) is silently skipped.""" + _make_plugin_dir( + tmp_path, "plugins/my-backend", + "@test/my-plugin-backend", "backend-plugin", + dependencies={"lodash": "^4.0.0"}, + ) + _make_node_module(tmp_path, "lodash") + + result = PluginListConfig._compute_backend_build_args( + tmp_path, "plugins/my-backend", + tmp_path / "plugins/my-backend/package.json", set(), + ) + assert result == "" + + +class TestCreateDefaultWithNativeSuppression: + """End-to-end tests for create_default with native module suppression.""" + + def test_native_dep_detected_end_to_end(self, tmp_path, monkeypatch): + lockfile = tmp_path / "host-yarn.lock" + lockfile.write_text('"@backstage/core@npm:^1.0.0":\n version: 1.0.0\n') + monkeypatch.setattr(constants, "HOST_LOCKFILE", lockfile) + + workspace = tmp_path / "workspace" + workspace.mkdir() + _make_plugin_dir( + workspace, "plugins/backend", + "@test/my-backend", "backend-plugin", + dependencies={"ssh2": "^1.0.0"}, + ) + _make_node_module( + workspace, "ssh2", + dependencies={"cpu-features": "^0.0.9"}, + ) + _make_node_module( + workspace, "cpu-features", + dependencies={"node-gyp-build": "^4.0.0"}, + ) + + config = PluginListConfig.create_default(workspace) + args = config.get_plugins()["plugins/backend"] + assert "--suppress-native-package cpu-features" in args + + +class TestPluginListConfigPopulateBuildArgs: + """Tests for PluginListConfig.populate_build_args method.""" + + def test_backend_plugin_gets_args_computed(self, tmp_path, monkeypatch): + """Backend plugin with empty args gets build args populated.""" + lockfile = tmp_path / "host-yarn.lock" + lockfile.write_text('"@backstage/core@npm:^1.0.0":\n version: 1.0.0\n') + monkeypatch.setattr(constants, "HOST_LOCKFILE", lockfile) + + workspace = tmp_path / "workspace" + workspace.mkdir() + _make_plugin_dir( + workspace, "plugins/backend", + "@test/my-backend", "backend-plugin", + dependencies={"@backstage/new-experimental": "^0.1.0"}, + ) + + cfg = PluginListConfig({"plugins/backend": ""}) + cfg.populate_build_args(workspace) + + args = cfg.get_plugins()["plugins/backend"] + assert "--embed-package @backstage/new-experimental" in args + assert "--shared-package !@backstage/new-experimental" in args + + def test_frontend_plugin_stays_empty(self, tmp_path, monkeypatch): + """Frontend plugin with empty args stays empty after population.""" + lockfile = tmp_path / "host-yarn.lock" + lockfile.write_text("") + monkeypatch.setattr(constants, "HOST_LOCKFILE", lockfile) + + workspace = tmp_path / "workspace" + workspace.mkdir() + _make_plugin_dir( + workspace, "plugins/todo", + "@test/plugin-todo", "frontend-plugin", + ) + + cfg = PluginListConfig({"plugins/todo": ""}) + cfg.populate_build_args(workspace) + + assert cfg.get_plugins()["plugins/todo"] == "" + + def test_existing_args_overwritten(self, tmp_path, monkeypatch): + """Stale build args are overwritten with freshly computed ones.""" + lockfile = tmp_path / "host-yarn.lock" + lockfile.write_text('"@backstage/core@npm:^1.0.0":\n version: 1.0.0\n') + monkeypatch.setattr(constants, "HOST_LOCKFILE", lockfile) + + workspace = tmp_path / "workspace" + workspace.mkdir() + _make_plugin_dir( + workspace, "plugins/backend", + "@test/my-backend", "backend-plugin", + dependencies={"@backstage/new-pkg": "^0.1.0"}, + ) + + cfg = PluginListConfig({"plugins/backend": "--embed-package @old/stale-dep"}) + cfg.populate_build_args(workspace) + + args = cfg.get_plugins()["plugins/backend"] + assert "@old/stale-dep" not in args + assert "--embed-package @backstage/new-pkg" in args + + def test_nonexistent_plugin_warns_and_keeps_empty(self, tmp_path, monkeypatch): + """Plugin path not found in workspace logs a warning and stays with empty args.""" + lockfile = tmp_path / "host-yarn.lock" + lockfile.write_text("") + monkeypatch.setattr(constants, "HOST_LOCKFILE", lockfile) + + workspace = tmp_path / "workspace" + workspace.mkdir() + + cfg = PluginListConfig({"plugins/nonexistent": "--old-arg"}) + cfg.populate_build_args(workspace) + + assert cfg.get_plugins()["plugins/nonexistent"] == "" + + def test_plugin_without_backstage_role_stays_empty(self, tmp_path, monkeypatch): + """Plugin without backstage.role field stays with empty args.""" + lockfile = tmp_path / "host-yarn.lock" + lockfile.write_text("") + monkeypatch.setattr(constants, "HOST_LOCKFILE", lockfile) + + workspace = tmp_path / "workspace" + workspace.mkdir() + pkg_dir = workspace / "plugins" / "no-role" + pkg_dir.mkdir(parents=True) + (pkg_dir / "package.json").write_text( + json.dumps({"name": "@test/no-role", "version": "1.0.0"}) + ) + + cfg = PluginListConfig({"plugins/no-role": ""}) + cfg.populate_build_args(workspace) + + assert cfg.get_plugins()["plugins/no-role"] == "" + + def test_mixed_scenario(self, tmp_path, monkeypatch): + """Backend, frontend, and invalid paths handled together.""" + lockfile = tmp_path / "host-yarn.lock" + lockfile.write_text('"@backstage/core@npm:^1.0.0":\n version: 1.0.0\n') + monkeypatch.setattr(constants, "HOST_LOCKFILE", lockfile) + + workspace = tmp_path / "workspace" + workspace.mkdir() + _make_plugin_dir( + workspace, "plugins/backend", + "@test/my-backend", "backend-plugin", + dependencies={"@backstage/new-pkg": "^0.1.0"}, + ) + _make_plugin_dir( + workspace, "plugins/frontend", + "@test/my-frontend", "frontend-plugin", + ) + + cfg = PluginListConfig({ + "plugins/backend": "", + "plugins/frontend": "", + "plugins/missing": "--old", + }) + cfg.populate_build_args(workspace) + plugins = cfg.get_plugins() + + assert "--embed-package @backstage/new-pkg" in plugins["plugins/backend"] + assert plugins["plugins/frontend"] == "" + assert plugins["plugins/missing"] == "" + + def test_roundtrip_from_file(self, tmp_path, monkeypatch): + """from_file -> populate_build_args -> to_file roundtrip.""" + lockfile = tmp_path / "host-yarn.lock" + lockfile.write_text('"@backstage/core@npm:^1.0.0":\n version: 1.0.0\n') + monkeypatch.setattr(constants, "HOST_LOCKFILE", lockfile) + + workspace = tmp_path / "workspace" + workspace.mkdir() + _make_plugin_dir( + workspace, "plugins/todo", + "@test/plugin-todo", "frontend-plugin", + ) + _make_plugin_dir( + workspace, "plugins/todo-backend", + "@test/plugin-todo-backend", "backend-plugin", + dependencies={"@backstage/new-experimental": "^0.1.0"}, + ) + + input_file = tmp_path / "plugins-list.yaml" + input_file.write_text("plugins/todo:\nplugins/todo-backend:\n") + + cfg = PluginListConfig.from_file(input_file) + cfg.populate_build_args(workspace) + output_file = tmp_path / "plugins-list-out.yaml" + cfg.to_file(output_file) + + reloaded = PluginListConfig.from_file(output_file) + plugins = reloaded.get_plugins() + assert plugins["plugins/todo"] == "" + assert "--embed-package @backstage/new-experimental" in plugins["plugins/todo-backend"] + + def test_returns_self(self, tmp_path, monkeypatch): + """populate_build_args returns self for method chaining.""" + lockfile = tmp_path / "host-yarn.lock" + lockfile.write_text("") + monkeypatch.setattr(constants, "HOST_LOCKFILE", lockfile) + + workspace = tmp_path / "workspace" + workspace.mkdir() + _make_plugin_dir( + workspace, "plugins/todo", + "@test/plugin-todo", "frontend-plugin", + ) + + cfg = PluginListConfig({"plugins/todo": ""}) + result = cfg.populate_build_args(workspace) + assert result is cfg + + def test_backend_no_deps_stays_empty(self, tmp_path, monkeypatch): + """Backend plugin with no deps that need embedding stays with empty args.""" + lockfile = tmp_path / "host-yarn.lock" + lockfile.write_text('"@backstage/core@npm:^1.0.0":\n version: 1.0.0\n') + monkeypatch.setattr(constants, "HOST_LOCKFILE", lockfile) + + workspace = tmp_path / "workspace" + workspace.mkdir() + _make_plugin_dir( + workspace, "plugins/simple-backend", + "@test/simple-backend", "backend-plugin", + ) + + cfg = PluginListConfig({"plugins/simple-backend": ""}) + cfg.populate_build_args(workspace) + + assert cfg.get_plugins()["plugins/simple-backend"] == "" + + +class TestLogBuildArgsDiff: + """Tests for PluginListConfig._log_build_args_diff.""" + + def test_changed_plugins_logged(self): + before = {"plugins/a": "", "plugins/b": "--old"} + after = {"plugins/a": "--new", "plugins/b": "--new"} + PluginListConfig._log_build_args_diff(before, after) + + def test_unchanged_plugins_logged(self): + before = {"plugins/a": "--same"} + after = {"plugins/a": "--same"} + PluginListConfig._log_build_args_diff(before, after) + + def test_mixed_changed_and_unchanged(self): + before = {"plugins/a": "", "plugins/b": "--keep"} + after = {"plugins/a": "--new", "plugins/b": "--keep"} + PluginListConfig._log_build_args_diff(before, after) diff --git a/tests/test_source_config.py b/tests/test_source_config.py index cde56f8..b20ef86 100644 --- a/tests/test_source_config.py +++ b/tests/test_source_config.py @@ -10,7 +10,8 @@ from unittest.mock import patch, MagicMock import pytest -from src.rhdh_dynamic_plugin_factory.config import SourceConfig, PluginFactoryConfig +from src.rhdh_dynamic_plugin_factory.config import PluginFactoryConfig +from src.rhdh_dynamic_plugin_factory.source_config import SourceConfig from src.rhdh_dynamic_plugin_factory.exceptions import ConfigurationError, ExecutionError, PluginFactoryError class TestSourceConfigFromFile: @@ -212,7 +213,7 @@ def test_clone_to_path_success(self, tmp_path): repo_path = tmp_path / "repo" repo_path.mkdir() - with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run: + with patch("src.rhdh_dynamic_plugin_factory.source_config.run_command_with_streaming") as mock_run: mock_run.return_value = 0 config.clone_to_path(repo_path) # Should not raise any exceptions @@ -246,7 +247,7 @@ def test_clone_to_path_resolved_default_ref(self, tmp_path): repo_path = tmp_path / "repo" repo_path.mkdir() - with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run: + with patch("src.rhdh_dynamic_plugin_factory.source_config.run_command_with_streaming") as mock_run: mock_run.return_value = 0 config.clone_to_path(repo_path) @@ -286,7 +287,7 @@ def test_clone_to_path_clone_fails(self, tmp_path): repo_path = tmp_path / "repo" repo_path.mkdir() - with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run: + with patch("src.rhdh_dynamic_plugin_factory.source_config.run_command_with_streaming") as mock_run: mock_run.return_value = 1 # Failed with pytest.raises(ExecutionError, match="Failed to clone repository"): @@ -303,7 +304,7 @@ def test_clone_to_path_checkout_fails(self, tmp_path): repo_path = tmp_path / "repo" repo_path.mkdir() - with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run: + with patch("src.rhdh_dynamic_plugin_factory.source_config.run_command_with_streaming") as mock_run: # First call (clone) succeeds, second call (checkout) fails mock_run.side_effect = [0, 1] @@ -321,7 +322,7 @@ def test_clone_to_path_exception(self, tmp_path): repo_path = tmp_path / "repo" repo_path.mkdir() - with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run: + with patch("src.rhdh_dynamic_plugin_factory.source_config.run_command_with_streaming") as mock_run: mock_run.side_effect = Exception("Test exception") with pytest.raises(ExecutionError, match="Failed during repository clone/checkout"): @@ -365,7 +366,7 @@ def test_clean_flag_auto_cleans_nested_contents(self, tmp_path): repo_path = tmp_path / "repo" self._make_nested_repo(repo_path) - with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run, \ + with patch("src.rhdh_dynamic_plugin_factory.source_config.run_command_with_streaming") as mock_run, \ patch("builtins.input") as mock_input: mock_run.return_value = 0 @@ -381,7 +382,7 @@ def test_no_clean_flag_prompts_user_confirm_yes(self, tmp_path): repo_path = tmp_path / "repo" self._make_nested_repo(repo_path) - with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run, \ + with patch("src.rhdh_dynamic_plugin_factory.source_config.run_command_with_streaming") as mock_run, \ patch("builtins.input", return_value="y"): mock_run.return_value = 0 @@ -426,7 +427,7 @@ def test_empty_directory_skips_clean_and_prompt(self, tmp_path): repo_path = tmp_path / "repo" repo_path.mkdir() - with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run, \ + with patch("src.rhdh_dynamic_plugin_factory.source_config.run_command_with_streaming") as mock_run, \ patch("builtins.input") as mock_input: mock_run.return_value = 0 @@ -440,7 +441,7 @@ def test_empty_directory_no_clean_flag_skips_prompt(self, tmp_path): repo_path = tmp_path / "repo" repo_path.mkdir() - with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run, \ + with patch("src.rhdh_dynamic_plugin_factory.source_config.run_command_with_streaming") as mock_run, \ patch("builtins.input") as mock_input: mock_run.return_value = 0 @@ -464,7 +465,7 @@ def test_clean_proceeds_with_clone_after_cleaning(self, tmp_path): repo_path = tmp_path / "repo" self._make_nested_repo(repo_path) - with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run: + with patch("src.rhdh_dynamic_plugin_factory.source_config.run_command_with_streaming") as mock_run: mock_run.return_value = 0 config.clone_to_path(repo_path, clean=True) @@ -484,7 +485,7 @@ def test_prompt_confirm_yes_proceeds_with_clone(self, tmp_path): repo_path = tmp_path / "repo" self._make_nested_repo(repo_path) - with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run, \ + with patch("src.rhdh_dynamic_plugin_factory.source_config.run_command_with_streaming") as mock_run, \ patch("builtins.input", return_value="y"): mock_run.return_value = 0 @@ -501,7 +502,7 @@ def test_prompt_confirm_no_does_not_clone(self, tmp_path): original_contents = {p.name for p in repo_path.rglob("*")} - with patch("src.rhdh_dynamic_plugin_factory.config.run_command_with_streaming") as mock_run, \ + with patch("src.rhdh_dynamic_plugin_factory.source_config.run_command_with_streaming") as mock_run, \ patch("builtins.input", return_value="n"): with pytest.raises(PluginFactoryError, match="aborted by user"): config.clone_to_path(repo_path, clean=False) From 2f31ad75e9cfa435473a3fec51296218fce990b4 Mon Sep 17 00:00:00 2001 From: Frank Kong Date: Tue, 10 Mar 2026 18:04:07 -0400 Subject: [PATCH 14/35] chore: scan more than one layer of dep for backstage deps Assisted-By: Cursor Signed-off-by: Frank Kong --- .../plugin_list_config.py | 70 +++--- tests/test_plugin_list_config.py | 201 ++++++++++++++++++ 2 files changed, 241 insertions(+), 30 deletions(-) diff --git a/src/rhdh_dynamic_plugin_factory/plugin_list_config.py b/src/rhdh_dynamic_plugin_factory/plugin_list_config.py index d2108d9..f2fec35 100644 --- a/src/rhdh_dynamic_plugin_factory/plugin_list_config.py +++ b/src/rhdh_dynamic_plugin_factory/plugin_list_config.py @@ -383,41 +383,49 @@ def _walk(dep_name: str) -> None: return native @classmethod - def _check_third_party_dep( + def _gather_backstage_deps( cls, workspace_path: Path, dep_name: str, - host_packages: set[str], - embed_packages: set[str], - unshare_packages: set[str], - ) -> None: - """Check a non-``@backstage/*`` dep for transitive shared-package usage. + ) -> set[str]: + """Find ``@backstage/*`` packages in the transitive dependency tree. - If the dependency has any ``@backstage/*`` dependencies it is - marked for embedding. Dependencies absent from *host_packages* are - additionally marked for unsharing. + Recursively walks *dep_name*'s ``dependencies`` and + ``optionalDependencies`` via ``node_modules``. ``@backstage/*`` + packages are collected but not recursed into. Tracks visited + packages to avoid cycles. - Results are collected directly into *embed_packages* / *unshare_packages*. + Returns: + Set of ``@backstage/*`` package names found. """ - dep_pkg_json = cls._resolve_node_module_package_json(workspace_path, dep_name) - if dep_pkg_json is None: - cls.logger.debug(f"Could not resolve {dep_name} in node_modules") - return + found: set[str] = set() + visited: set[str] = set() - try: - dep_data = json.loads(dep_pkg_json.read_text(encoding="utf-8")) - except (json.JSONDecodeError, OSError) as e: - cls.logger.debug(f"Failed to read {dep_pkg_json}: {e}") - return + def _walk(pkg_name: str) -> None: + if pkg_name in visited: + return + visited.add(pkg_name) - dep_deps = dep_data.get("dependencies", {}) - backstage_deps = [d for d in dep_deps if d.startswith("@backstage/")] + if pkg_name.startswith("@backstage/"): + found.add(pkg_name) + return - if backstage_deps: - embed_packages.add(dep_name) - for dep in backstage_deps: - if dep not in host_packages: - unshare_packages.add(dep) + pkg_json = cls._resolve_node_module_package_json(workspace_path, pkg_name) + if pkg_json is None: + return + + try: + data = json.loads(pkg_json.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError) as e: + cls.logger.warning(f"Failed to read {pkg_json}: {e}") + return + + for field in ("dependencies", "optionalDependencies"): + for sub_dep in data.get(field, {}): + _walk(sub_dep) + + _walk(dep_name) + return found @classmethod def _compute_backend_build_args( @@ -472,10 +480,12 @@ def _compute_backend_build_args( continue private_deps.add(dep_name) - cls._check_third_party_dep( - workspace_path, dep_name, - host_packages, embed_packages, unshare_packages, - ) + backstage_deps = cls._gather_backstage_deps(workspace_path, dep_name) + if backstage_deps: + embed_packages.add(dep_name) + unshare_packages.update( + bs for bs in backstage_deps if bs not in host_packages + ) suppress_native = cls._gather_native_modules( workspace_path, private_deps | embed_packages | siblings, diff --git a/tests/test_plugin_list_config.py b/tests/test_plugin_list_config.py index 11abda9..c88c61b 100644 --- a/tests/test_plugin_list_config.py +++ b/tests/test_plugin_list_config.py @@ -679,6 +679,207 @@ def test_malformed_dep_package_json_skipped(self, tmp_path): assert result == "" +class TestGatherBackstageDeps: + """Tests for PluginListConfig._gather_backstage_deps.""" + + def test_direct_backstage_dep(self, tmp_path): + """A dep that directly depends on @backstage/* finds it.""" + _make_node_module( + tmp_path, "some-lib", + dependencies={"@backstage/catalog-model": "^1.7.0"}, + ) + + result = PluginListConfig._gather_backstage_deps(tmp_path, "some-lib") + assert result == {"@backstage/catalog-model"} + + def test_deep_transitive_backstage_dep(self, tmp_path): + """@backstage/* found two levels deep is still detected.""" + _make_node_module( + tmp_path, "dep-a", + dependencies={"dep-b": "^1.0.0"}, + ) + _make_node_module( + tmp_path, "dep-b", + dependencies={"@backstage/catalog-model": "^1.7.0"}, + ) + + result = PluginListConfig._gather_backstage_deps(tmp_path, "dep-a") + assert result == {"@backstage/catalog-model"} + + def test_three_levels_deep(self, tmp_path): + """@backstage/* found three levels deep is detected.""" + _make_node_module( + tmp_path, "dep-a", + dependencies={"dep-b": "^1.0.0"}, + ) + _make_node_module( + tmp_path, "dep-b", + dependencies={"dep-c": "^1.0.0"}, + ) + _make_node_module( + tmp_path, "dep-c", + dependencies={"@backstage/errors": "^1.2.0"}, + ) + + result = PluginListConfig._gather_backstage_deps(tmp_path, "dep-a") + assert result == {"@backstage/errors"} + + def test_multiple_backstage_at_different_depths(self, tmp_path): + """@backstage/* packages at multiple depths are all collected.""" + _make_node_module( + tmp_path, "dep-a", + dependencies={ + "@backstage/catalog-model": "^1.7.0", + "dep-b": "^1.0.0", + }, + ) + _make_node_module( + tmp_path, "dep-b", + dependencies={"@backstage/errors": "^1.2.0"}, + ) + + result = PluginListConfig._gather_backstage_deps(tmp_path, "dep-a") + assert result == {"@backstage/catalog-model", "@backstage/errors"} + + def test_no_backstage_deps(self, tmp_path): + """Dep tree with no @backstage/* returns empty set.""" + _make_node_module( + tmp_path, "dep-a", + dependencies={"dep-b": "^1.0.0"}, + ) + _make_node_module(tmp_path, "dep-b") + + result = PluginListConfig._gather_backstage_deps(tmp_path, "dep-a") + assert result == set() + + def test_does_not_recurse_into_backstage(self, tmp_path): + """Walk stops at @backstage/* nodes -- does not read their deps.""" + _make_node_module( + tmp_path, "dep-a", + dependencies={"@backstage/catalog-model": "^1.7.0"}, + ) + _make_node_module( + tmp_path, "@backstage/catalog-model", + dependencies={"@backstage/errors": "^1.2.0"}, + ) + + result = PluginListConfig._gather_backstage_deps(tmp_path, "dep-a") + assert result == {"@backstage/catalog-model"} + assert "@backstage/errors" not in result + + def test_cycle_avoidance(self, tmp_path): + """Circular deps don't cause infinite recursion.""" + _make_node_module( + tmp_path, "dep-a", + dependencies={"dep-b": "^1.0.0"}, + ) + _make_node_module( + tmp_path, "dep-b", + dependencies={ + "dep-a": "^1.0.0", + "@backstage/config": "^1.0.0", + }, + ) + + result = PluginListConfig._gather_backstage_deps(tmp_path, "dep-a") + assert result == {"@backstage/config"} + + def test_unresolvable_dep_skipped(self, tmp_path): + """Missing packages in node_modules are silently skipped.""" + _make_node_module( + tmp_path, "dep-a", + dependencies={"nonexistent": "^1.0.0"}, + ) + + result = PluginListConfig._gather_backstage_deps(tmp_path, "dep-a") + assert result == set() + + def test_optional_dep_with_backstage(self, tmp_path): + """@backstage/* found via optionalDependencies is detected.""" + _make_node_module( + tmp_path, "dep-a", + optional_dependencies={"dep-b": "^1.0.0"}, + ) + _make_node_module( + tmp_path, "dep-b", + dependencies={"@backstage/types": "^1.0.0"}, + ) + + result = PluginListConfig._gather_backstage_deps(tmp_path, "dep-a") + assert result == {"@backstage/types"} + + +class TestComputeBackendBuildArgsDeepTransitive: + """Tests for deep transitive @backstage/* detection in build args.""" + + def test_deep_transitive_triggers_embed(self, tmp_path): + """dep-a -> dep-b -> @backstage/catalog-model triggers embed for dep-a.""" + _make_plugin_dir( + tmp_path, "plugins/backend", "@test/my-backend", "backend-plugin", + dependencies={"dep-a": "^1.0.0"}, + ) + _make_node_module( + tmp_path, "dep-a", + dependencies={"dep-b": "^1.0.0"}, + ) + _make_node_module( + tmp_path, "dep-b", + dependencies={"@backstage/catalog-model": "^1.7.0"}, + ) + + host = {"@backstage/catalog-model"} + result = PluginListConfig._compute_backend_build_args( + tmp_path, "plugins/backend", + tmp_path / "plugins/backend/package.json", host, + ) + assert "--embed-package dep-a" in result + assert "--shared-package" not in result + + def test_deep_transitive_missing_from_host(self, tmp_path): + """Deep @backstage/* dep not in host triggers both embed and unshare.""" + _make_plugin_dir( + tmp_path, "plugins/backend", "@test/my-backend", "backend-plugin", + dependencies={"dep-a": "^1.0.0"}, + ) + _make_node_module( + tmp_path, "dep-a", + dependencies={"dep-b": "^1.0.0"}, + ) + _make_node_module( + tmp_path, "dep-b", + dependencies={"@backstage/new-experimental": "^0.1.0"}, + ) + + result = PluginListConfig._compute_backend_build_args( + tmp_path, "plugins/backend", + tmp_path / "plugins/backend/package.json", set(), + ) + assert "--embed-package dep-a" in result + assert "--shared-package !@backstage/new-experimental" in result + + def test_no_deep_backstage_no_embed(self, tmp_path): + """Deep deps without @backstage/* produce no embed args.""" + _make_plugin_dir( + tmp_path, "plugins/backend", "@test/my-backend", "backend-plugin", + dependencies={"dep-a": "^1.0.0"}, + ) + _make_node_module( + tmp_path, "dep-a", + dependencies={"dep-b": "^1.0.0"}, + ) + _make_node_module( + tmp_path, "dep-b", + dependencies={"lodash": "^4.0.0"}, + ) + _make_node_module(tmp_path, "lodash") + + result = PluginListConfig._compute_backend_build_args( + tmp_path, "plugins/backend", + tmp_path / "plugins/backend/package.json", set(), + ) + assert result == "" + + class TestCreateDefaultWithEmbedArgs: """End-to-end tests for create_default with Phase 2 embed/unshare detection.""" From 81e0fe3245330ffde2be1f325b729329d5571938 Mon Sep 17 00:00:00 2001 From: Frank Kong Date: Tue, 10 Mar 2026 23:48:39 -0400 Subject: [PATCH 15/35] docs: add plugins-list.yaml auto-gen docs Signed-off-by: Frank Kong --- CONTRIBUTING.md | 28 ++++++++--- README.md | 122 ++++++++++++++++++++++++++++++++++-------------- 2 files changed, 108 insertions(+), 42 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 857d8f1..d57a757 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -178,25 +178,35 @@ python -m src.rhdh_dynamic_plugin_factory \ ```bash rhdh-dynamic-plugin-factory/ ├── src/rhdh_dynamic_plugin_factory/ -│ ├── __init__.py # Package initialization +│ ├── __init__.py # Package initialization and public API │ ├── __main__.py # Package entry point │ ├── cli.py # CLI implementation and argument parsing │ ├── config.py # Configuration classes and validation +│ ├── constants.py # Shared constants and configuration values +│ ├── exceptions.py # Custom exception hierarchy │ ├── logger.py # Logging setup and utilities +│ ├── plugin_list_config.py # Plugin list YAML handling and build-arg computation +│ ├── source_config.py # Source repository and multi-workspace configuration │ └── utils.py # Utility functions ├── scripts/ │ ├── export-workspace.sh # Plugin export script (called by CLI) │ └── override-sources.sh # Patch/overlay application script ├── tests/ │ ├── __init__.py -│ ├── conftest.py # Pytest fixtures and configuration -│ ├── test_config.py # Configuration tests -│ ├── test_plugin_list_config.py # Plugin list parsing tests -│ └── test_source_config.py # Source configuration tests +│ ├── conftest.py # Pytest fixtures and configuration +│ ├── test_cli.py # CLI argument parsing tests +│ ├── test_config_export_plugins.py # Plugin export tests +│ ├── test_config_load_from_env.py # Environment loading tests +│ ├── test_config_patches_and_overlays.py # Patch/overlay tests +│ ├── test_config_registry.py # Registry configuration tests +│ ├── test_multi_workspace.py # Multi-workspace mode tests +│ ├── test_plugin_list_config.py # Plugin list and build-arg tests +│ └── test_source_config.py # Source configuration tests ├── examples/ # Example configuration sets │ ├── example-config-todo/ │ ├── example-config-gitlab/ -│ └── example-config-aws-ecs/ +│ ├── example-config-aws-ecs/ +│ └── example-config-multi-workspace/ ├── .cursor/rules/ # Development guidelines │ ├── commit-standards.mdc │ ├── documentation-standards.mdc @@ -213,9 +223,13 @@ rhdh-dynamic-plugin-factory/ ### Key Components -- **`cli.py`**: Handles command-line arguments, orchestrates the build process +- **`cli.py`**: Handles command-line arguments, orchestrates the build process (single and multi-workspace) - **`config.py`**: Loads and validates configuration from files and environment +- **`constants.py`**: Shared constants (plugin roles, skip directories, native module markers) +- **`exceptions.py`**: Custom exception hierarchy (`PluginFactoryError`, `ConfigurationError`, `ExecutionError`) - **`logger.py`**: Configures structured logging with color output +- **`plugin_list_config.py`**: Plugin list YAML loading/saving and build-argument computation logic +- **`source_config.py`**: Source repository configuration, git cloning, and multi-workspace discovery - **`utils.py`**: Helper functions for file operations, subprocess execution - **`export-workspace.sh`**: Shell script that calls the RHDH CLI to export plugins - **`override-sources.sh`**: Applies patches and overlays to source code diff --git a/README.md b/README.md index fa55fb9..dc48035 100644 --- a/README.md +++ b/README.md @@ -22,8 +22,12 @@ A comprehensive tool for building and exporting dynamic plugins for Red Hat Deve - [Configuration Files](#configuration-files) - [1. `default.env` (Provided)](#1-defaultenv-provided) - [2. `config/source.json` (Required for remote repositories, unless using `--source-repo`)](#2-configsourcejson-required-for-remote-repositories-unless-using---source-repo) - - [3. `config/plugins-list.yaml` (Required)](#3-configplugins-listyaml-required) + - [3. `config/plugins-list.yaml` (Optional -- auto-generated if absent)](#3-configplugins-listyaml-optional----auto-generated-if-absent) - [4. `config/.env` (Optional)](#4-configenv-optional) + - [Plugin List Auto-Generation](#plugin-list-auto-generation) + - [How Auto-Generation Works](#how-auto-generation-works) + - [Build Argument Types](#build-argument-types) + - [Using `--generate-build-args`](#using---generate-build-args) - [Patches and Overlays](#patches-and-overlays) - [Patches Directory (`config/patches/`)](#patches-directory-configpatches) - [Overlays Directory (`config//overlays/`)](#overlays-directory-configpath-to-plugin-root-with-respect-to-workspaceoverlays) @@ -225,7 +229,7 @@ This file contains required version settings and defaults for RHDH CLI: ```bash # Tooling versions -RHDH_CLI_VERSION="1.8.0" +RHDH_CLI_VERSION="1.10.0" ``` #### 2. `config/source.json` (Required for remote repositories, unless using `--source-repo`) @@ -248,9 +252,9 @@ Defines the source repository to clone: > **Note:** `source.json` is not needed when using the `--source-repo` CLI argument, which provides an alternative way to specify the repository directly from the command line. See [Command-Line Options](#command-line-options) for details. -#### 3. `config/plugins-list.yaml` (Required) +#### 3. `config/plugins-list.yaml` (Optional -- auto-generated if absent) -A list of plugin paths (with respect to root of workspace) to plugins to build along with optional build arguments: +A YAML map of plugin paths (relative to the workspace root) to build, along with optional build arguments: ```yaml # Simple plugins (no additional arguments) @@ -259,16 +263,11 @@ plugins/todo-backend: ``` ```yaml -# Plugins with embed packages +# Plugins with build arguments plugins/scaffolder-backend: --embed-package @backstage/plugin-scaffolder-backend-module-github ``` -```yaml -# Multiple embed packages -plugins/search-backend: | - --embed-package @backstage/plugin-search-backend-module-catalog - --embed-package @backstage/plugin-search-backend-module-techdocs -``` +If this file is not provided, the factory will auto-generate it by scanning the **entire workspace** and attempting to export **all** discovered frontend/backend plugins. See [Plugin List Auto-Generation](#plugin-list-auto-generation) for details on how discovery and build-arg computation work, and how to use `--generate-build-args` to auto-compute build arguments for only specific plugins. #### 4. `config/.env` (Optional) @@ -298,6 +297,58 @@ podman run --rm -it \ This approach keeps your credentials separate from the config directory and can be useful for CI/CD pipelines or when you want to reuse the same environment file across different configurations. +### Plugin List Auto-Generation + +If no `plugins-list.yaml` file is provided for a workspace, the factory will scan the **entire** workspace, discover **all** frontend/backend plugins, compute build arguments for backend plugins and generate a `plugins-list.yaml` with the required build arguments. + +If you only want to take advantage of the build argument auto-generation for specific plugin(s), you can provide a barebones `plugins-list.yaml` containing your desired plugin(s) and the [`--generate-build-args` argument](#using---generate-build-args). + +#### How Auto-Generation Works + +When `plugins-list.yaml` is absent, the factory recursively scans the workspace for `package.json` files. A package is included if it has a `backstage.role` field set to one of: + +- `frontend-plugin` +- `backend-plugin` +- `frontend-plugin-module` +- `backend-plugin-module` + +For **frontend** plugins (and frontend plugin modules), no build arguments are needed. + +For **backend** plugins (and backend plugin modules), the factory performs dependency analysis against the bundled RHDH host lockfile (`yarn.lock`) to determine which dependencies need additional build arguments during export. + +#### Build Argument Types + +The following build arguments may be automatically computed for backend plugins: + +| Argument | Purpose | +| --------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `--embed-package ` | Bundles a dependency into the dynamic plugin. Applied to `@backstage/`* packages not provided by the RHDH host, and to non-`@backstage` packages that have transitive `@backstage/*` dependencies. | +| `--shared-package !` | Marks an embedded `@backstage/*` package as unshared so the plugin uses its own bundled copy instead of the host's version. (Optional Argument) | +| `--suppress-native-package ` | Suppresses native Node.js modules that cannot be bundled. Detected by the presence of markers such as `bindings`, `prebuild`, `nan`, `node-gyp-build` in the package's dependencies, or `gypfile`/`binary` fields in its `package.json`. | + +#### Using `--generate-build-args` + +For larger workspaces that contain multiple plugins, using the auto generation feature will result in many unnecessary plugins being included in the `plugins-list.yaml`. To recompute build arguments for specific plugin(s), you will need to provide a barebones `plugins-list.yaml` with your desired plugin(s), and use the `--generate-build-args` argument. + +```yaml +# Barebones plugins-list.yaml -- list only the plugins you want to export +plugins/todo: +plugins/todo-backend: +``` + +```bash +podman run --rm -it \ + --device /dev/fuse \ + -v ./config:/config:z \ + quay.io/rhdh-community/dynamic-plugins-factory:latest \ + --workspace-path workspaces/todo \ + --generate-build-args +``` + +If the `--generate-build-args` argument is not provided when a `plugins-list.yaml` already exists, the factory will use it as-is and **will not** rescan or modify it. + +> **Warning:** `--generate-build-args` overwrites the build arguments in your existing `plugins-list.yaml`. Make a backup if you have manually tuned values you want to preserve. + ### Patches and Overlays > WARNING: This is a destructive operation @@ -339,19 +390,20 @@ See the [TODO plugin example config](./examples/example-config-todo/README.md) a ### Command-Line Options -| Option | Default | Description | -|--------|---------|-------------| -| `--config-dir` | `/config` | Configuration directory containing `source.json`, `plugins-list.yaml`, patches, and overlays | -| `--repo-path` | `/source` | Path where plugin source code will be cloned/stored | -| `--workspace-path` | *(see below)* | Path to the workspace from repository root (e.g., `workspaces/todo`). Can also be set via `source.json`'s `workspace-path` field. | -| `--source-repo` | `None` | Git repository URL. When provided, `source.json` is not required and the repository is cloned from this URL. | -| `--source-ref` | `None` | Git ref (branch/tag/commit) to check out. Defaults to the repository's default branch. Requires `--source-repo`. | -| `--output-dir` | `/outputs` | Directory for build artifacts (`.tgz` files and container images) | -| `--push-images` / `--no-push-images` | `--no-push-images` | Whether to push container images to registry. Defaults to not pushing if no argument is provided | -| `--use-local` | `false` | Use local repository instead of cloning from source.json | -| `--log-level` | `INFO` | Logging level: `DEBUG`, `INFO`, `WARNING`, `ERROR`, `CRITICAL` | -| `--verbose` | `false` | Show verbose output with file and line numbers | -| `--clean` | `false` | Automatically removes content of `--repo-path` directory when cloning from `source.json`. Ignored if `--use-local` is used. | +| Option | Default | Description | +| ------------------------------------ | ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `--config-dir` | `/config` | Configuration directory containing `source.json`, `plugins-list.yaml`, patches, and overlays | +| `--repo-path` | `/source` | Path where plugin source code will be cloned/stored | +| `--workspace-path` | *(see below)* | Path to the workspace from repository root (e.g., `workspaces/todo`). Can also be set via `source.json`'s `workspace-path` field. | +| `--source-repo` | `None` | Git repository URL. When provided, `source.json` is not required and the repository is cloned from this URL. | +| `--source-ref` | `None` | Git ref (branch/tag/commit) to check out. Defaults to the repository's default branch. Requires `--source-repo`. | +| `--output-dir` | `/outputs` | Directory for build artifacts (`.tgz` files and integrity hash files) | +| `--push-images` / `--no-push-images` | `--no-push-images` | Whether to push container images to registry. Defaults to not pushing if no argument is provided | +| `--use-local` | `false` | Use local repository instead of cloning from source.json | +| `--log-level` | `INFO` | Logging level: `DEBUG`, `INFO`, `WARNING`, `ERROR`, `CRITICAL` | +| `--verbose` | `false` | Show verbose output with file and line numbers | +| `--clean` | `false` | Automatically removes content of `--repo-path` directory when cloning from `source.json`. Ignored if `--use-local` is used. | +| `--generate-build-args` | `false` | When `plugins-list.yaml` exists, recompute build arguments for all listed plugins using dependency analysis. See [Plugin List Auto-Generation](#plugin-list-auto-generation). **WARNING: This overwrites your `plugins-list.yaml` with updated build args.** | **Workspace path resolution:** In single-workspace use cases, the workspace path can be provided via the `--workspace-path` CLI argument, or the `workspace-path` field in `source.json`. The CLI argument takes highest precedence, followed by the `source.json`. For the multi-workspace case, only the `workspace-path` field in `source.json` is supported. @@ -361,11 +413,11 @@ See the [TODO plugin example config](./examples/example-config-todo/README.md) a When using the container, you can mount directories based on your needs: -| Volume Mount | Required? | Purpose | When to Use | -|--------------|-----------|---------|-------------| -| `-v ./config:/config:z` | **Required** | Configuration files | Always - contains your `plugins-list.yaml`, `source.json`, patches, and overlays | -| `-v ./source:/source:z` | Optional | Source code location | Only if using `--use-local` OR if you want to preserve/inspect the cloned/patched remote repository | -| `-v ./outputs:/outputs:z` | Optional | Stores bBuild artifacts | Only if you want the output `.tgz` files saved locally (otherwise they stay in the container) | +| Volume Mount | Required? | Purpose | When to Use | +| ------------------------- | ------------ | ----------------------- | --------------------------------------------------------------------------------------------------- | +| `-v ./config:/config:z` | **Required** | Configuration files | Always - contains your `plugins-list.yaml`, `source.json`, patches, and overlays | +| `-v ./source:/source:z` | Optional | Source code location | Only if using `--use-local` OR if you want to preserve/inspect the cloned/patched remote repository | +| `-v ./outputs:/outputs:z` | Optional | Stores build artifacts | Only if you want the output `.tgz` files saved locally (otherwise they stay in the container) | **Important**: These volume mount paths (`/config`, `/source`, `/outputs`) correspond to the default values of `--config-dir`, `--repo-path`, and `--output-dir`. If you override these arguments with custom paths, adjust your volume mounts accordingly. @@ -562,12 +614,12 @@ NOTE: If the repository name (ex: plugin-name-dynamic) in the namespace specifie The `examples` directory contains ready-to-use configuration examples demonstrating different use cases and features. -| Example | Description | Details | -|---------|-------------|---------| -| **TODO** | Basic single-workspace with custom scalprum-config | [View README](./examples/example-config-todo/README.md) | -| **GitLab** | Overlays for non Backstage Community Plugins workspace format | [View README](./examples/example-config-gitlab/README.md) | -| **AWS ECS** | Patches and embed packages in plugins-list.yaml | [View README](./examples/example-config-aws-ecs/README.md) | -| **Multi-Workspace** | Multiple workspaces from different repos in a single run | [View README](./examples/example-config-multi-workspace/README.md) | +| Example | Description | Details | +| ------------------- | ------------------------------------------------------------- | ------------------------------------------------------------------ | +| **TODO** | Basic single-workspace with custom scalprum-config | [View README](./examples/example-config-todo/README.md) | +| **GitLab** | Overlays for non Backstage Community Plugins workspace format | [View README](./examples/example-config-gitlab/README.md) | +| **AWS ECS** | Patches and embed packages in plugins-list.yaml | [View README](./examples/example-config-aws-ecs/README.md) | +| **Multi-Workspace** | Multiple workspaces from different repos in a single run | [View README](./examples/example-config-multi-workspace/README.md) | ### Quick Example: TODO Workspace From c4081402a673dc357f383d9e8b984f3b77167643 Mon Sep 17 00:00:00 2001 From: Frank Kong Date: Wed, 11 Mar 2026 18:22:39 -0400 Subject: [PATCH 16/35] chore: fix mypy errors Signed-off-by: Frank Kong --- src/rhdh_dynamic_plugin_factory/cli.py | 2 +- src/rhdh_dynamic_plugin_factory/source_config.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/rhdh_dynamic_plugin_factory/cli.py b/src/rhdh_dynamic_plugin_factory/cli.py index f893d57..2db2084 100644 --- a/src/rhdh_dynamic_plugin_factory/cli.py +++ b/src/rhdh_dynamic_plugin_factory/cli.py @@ -411,7 +411,7 @@ def _run_single_workspace(args: argparse.Namespace) -> None: if source_config and not config.use_local: logger.info("[bold blue]Repository Setup[/bold blue]") - source_config.clone_to_path(config.repo_path, clean=args.clean) + source_config.clone_to_path(Path(config.repo_path), clean=args.clean) elif config.use_local or not source_config: if config.use_local: logger.info("[bold blue]--use-local flag is set, using local repository[/bold blue]") diff --git a/src/rhdh_dynamic_plugin_factory/source_config.py b/src/rhdh_dynamic_plugin_factory/source_config.py index 300a10f..f04454a 100644 --- a/src/rhdh_dynamic_plugin_factory/source_config.py +++ b/src/rhdh_dynamic_plugin_factory/source_config.py @@ -172,7 +172,7 @@ def clone_to_path(self, repo_path: Path, clean: bool = False) -> None: returncode=returncode ) - cmd = ["git", "checkout", self.repo_ref] + cmd = ["git", "checkout", str(self.repo_ref)] logger.info(f"[cyan]Checking out ref: {self.repo_ref}[/cyan]") returncode = run_command_with_streaming( cmd, From 00922267799598920df395069dcf8a1db5762c46 Mon Sep 17 00:00:00 2001 From: Frank Kong Date: Wed, 11 Mar 2026 18:23:10 -0400 Subject: [PATCH 17/35] chore: cache host yarn.lock dep load Assisted-By: Cursor Signed-off-by: Frank Kong --- src/rhdh_dynamic_plugin_factory/constants.py | 4 +++ .../plugin_list_config.py | 33 +++++++++++++------ 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/src/rhdh_dynamic_plugin_factory/constants.py b/src/rhdh_dynamic_plugin_factory/constants.py index d202521..3eb1641 100644 --- a/src/rhdh_dynamic_plugin_factory/constants.py +++ b/src/rhdh_dynamic_plugin_factory/constants.py @@ -37,6 +37,10 @@ r'"(@backstage/[\w.-]+)@npm:' ) +LOCKFILE_PACKAGE_RE: re.Pattern = re.compile( + r'"((?:@[\w.-]+/)?[\w.-]+)@npm:' +) + NATIVE_DEP_MARKERS: frozenset[str] = frozenset[str]({ "bindings", "prebuild", "nan", "node-pre-gyp", "node-gyp-build", }) diff --git a/src/rhdh_dynamic_plugin_factory/plugin_list_config.py b/src/rhdh_dynamic_plugin_factory/plugin_list_config.py index f2fec35..cd8e6f0 100644 --- a/src/rhdh_dynamic_plugin_factory/plugin_list_config.py +++ b/src/rhdh_dynamic_plugin_factory/plugin_list_config.py @@ -19,6 +19,7 @@ class PluginListConfig: """Configuration for plugin list (YAML format).""" logger: ClassVar[Logger] = get_logger("plugin_list") + _host_packages_cache: ClassVar[set[str] | None] = None def __init__(self, plugins: Dict[str, str]): """ @@ -88,7 +89,7 @@ def populate_build_args(self, workspace_path: Path) -> "PluginListConfig": ``self``, mutated in place. """ original = self.plugins.copy() - host_packages = self._parse_host_backstage_packages(constants.HOST_LOCKFILE) + host_packages = self._get_host_packages() for plugin_dir in self.plugins: pkg_json_path = workspace_path / plugin_dir / constants.PKG_JSON @@ -195,7 +196,7 @@ def create_default(cls, workspace_path: Path) -> "PluginListConfig": A :class:`PluginListConfig` with discovered plugins and build arg(s) (if any). """ plugins: Dict[str, str] = {} - host_packages = cls._parse_host_backstage_packages(constants.HOST_LOCKFILE) + host_packages = cls._get_host_packages() for pkg_json_path in cls._find_package_jsons(workspace_path): role = cls._read_backstage_role(pkg_json_path) @@ -244,18 +245,27 @@ def _read_backstage_role(cls, pkg_json_path: Path) -> Optional[str]: return None @classmethod - def _parse_host_backstage_packages(cls, lockfile_path: Path) -> set[str]: - """Extract ``@backstage/*`` package names from a Yarn Berry lockfile (Yarn 2+). + def _get_host_packages(cls) -> set[str]: + """Return cached host packages, parsing the lockfile on first call.""" + if cls._host_packages_cache is None: + cls._host_packages_cache = cls._parse_host_packages(constants.HOST_LOCKFILE) + return cls._host_packages_cache + + @classmethod + def _parse_host_packages(cls, lockfile_path: Path) -> set[str]: + """Extract all package names from a Yarn Berry lockfile (Yarn 2+). Scans top-level key lines (e.g. - ``"@backstage/catalog-model@npm:^1.7.2, …":``) and collects distinct - package names. + ``"@backstage/catalog-model@npm:^1.7.2, …":`` or + ``"better-sqlite3@npm:^12.0.0":``) and collects distinct + package names. The returned set includes ``@backstage/*`` + packages as well as every other scoped or unscoped package. Args: lockfile_path: Path to the host ``yarn.lock`` file. Returns: - Set of ``@backstage/*`` package names found in the lockfile, + Set of package names found in the lockfile, or an empty set if the file does not exist. """ if not lockfile_path.is_file(): @@ -264,9 +274,10 @@ def _parse_host_backstage_packages(cls, lockfile_path: Path) -> set[str]: packages: set[str] = set[str]() for line in lockfile_path.read_text(encoding="utf-8").splitlines(): - if not line.startswith('"@backstage/'): + # skip non-package lines + if not line.startswith('"'): continue - for match in constants.LOCKFILE_BACKSTAGE_RE.finditer(line): + for match in constants.LOCKFILE_PACKAGE_RE.finditer(line): packages.add(match.group(1)) cls.logger.debug(f"Parsed {len(packages)} @backstage/* packages from host lockfile") @@ -444,12 +455,14 @@ def _compute_backend_build_args( * Non-``@backstage/*``, non-sibling deps whose own dependencies include ``@backstage/*`` packages are embedded. Any of those sub-deps missing from *host_packages* are additionally unshared. + * Native modules are unconditionally suppressed (removed from the + bundle) since dynamic plugins do not support them. Args: workspace_path: Absolute workspace root. plugin_dir: Plugin directory relative to *workspace_path*. pkg_json_path: Path to the plugin's ``package.json``. - host_packages: ``@backstage/*`` names present in the host lockfile. + host_packages: All package names present in the host lockfile. Returns: CLI argument string, or ``""`` if no extra args are needed. From c5125c9241040da856db780b0a9812cc9906aa4c Mon Sep 17 00:00:00 2001 From: Frank Kong Date: Wed, 11 Mar 2026 18:24:43 -0400 Subject: [PATCH 18/35] chore: update unit tests Assisted-By: Cursor Signed-off-by: Frank Kong --- tests/conftest.py | 6 +++ tests/test_plugin_list_config.py | 89 ++++++++++++++++++++++---------- 2 files changed, 67 insertions(+), 28 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index dfed6c2..09feba4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,6 +10,12 @@ from dotenv import dotenv_values from src.rhdh_dynamic_plugin_factory.config import PluginFactoryConfig +from src.rhdh_dynamic_plugin_factory.plugin_list_config import PluginListConfig + + +@pytest.fixture(autouse=True) +def _clear_host_packages_cache(): + PluginListConfig._host_packages_cache = None def _write_source_json(directory: Path, repo: str, repo_ref: str, workspace_path: str = ".") -> None: diff --git a/tests/test_plugin_list_config.py b/tests/test_plugin_list_config.py index c88c61b..0fe6cc3 100644 --- a/tests/test_plugin_list_config.py +++ b/tests/test_plugin_list_config.py @@ -391,16 +391,16 @@ def test_aws_ecs_workspace_structure(self, tmp_path): assert "plugins/ecs/backend" in plugins -class TestParseHostBackstagePackages: - """Tests for PluginListConfig._parse_host_backstage_packages.""" +class TestParseHostPackages: + """Tests for PluginListConfig._parse_host_packages.""" - def test_single_entry(self, tmp_path): + def test_single_backstage_entry(self, tmp_path): lockfile = tmp_path / "yarn.lock" lockfile.write_text( '"@backstage/catalog-model@npm:^1.7.2":\n version: 1.7.2\n' ) - result = PluginListConfig._parse_host_backstage_packages(lockfile) - assert result == {"@backstage/catalog-model"} + result = PluginListConfig._parse_host_packages(lockfile) + assert "@backstage/catalog-model" in result def test_multi_version_entry(self, tmp_path): lockfile = tmp_path / "yarn.lock" @@ -408,7 +408,7 @@ def test_multi_version_entry(self, tmp_path): '"@backstage/catalog-model@npm:^1.7.2, @backstage/catalog-model@npm:^1.7.3":\n' " version: 1.9.0\n" ) - result = PluginListConfig._parse_host_backstage_packages(lockfile) + result = PluginListConfig._parse_host_packages(lockfile) assert result == {"@backstage/catalog-model"} def test_multiple_packages(self, tmp_path): @@ -417,24 +417,35 @@ def test_multiple_packages(self, tmp_path): '"@backstage/catalog-model@npm:^1.7.2":\n version: 1.7.2\n\n' '"@backstage/errors@npm:^1.2.7":\n version: 1.2.7\n' ) - result = PluginListConfig._parse_host_backstage_packages(lockfile) - assert result == {"@backstage/catalog-model", "@backstage/errors"} + result = PluginListConfig._parse_host_packages(lockfile) + assert {"@backstage/catalog-model", "@backstage/errors"} <= result - def test_non_backstage_entries_ignored(self, tmp_path): + def test_non_backstage_scoped_packages_included(self, tmp_path): lockfile = tmp_path / "yarn.lock" lockfile.write_text( '"@aws/sdk@npm:^1.0.0":\n version: 1.0.0\n\n' '"@backstage/errors@npm:^1.2.7":\n version: 1.2.7\n' ) - result = PluginListConfig._parse_host_backstage_packages(lockfile) - assert result == {"@backstage/errors"} + result = PluginListConfig._parse_host_packages(lockfile) + assert result == {"@aws/sdk", "@backstage/errors"} + + def test_unscoped_packages_included(self, tmp_path): + """Unscoped packages like better-sqlite3 are parsed.""" + lockfile = tmp_path / "yarn.lock" + lockfile.write_text( + '"better-sqlite3@npm:^12.0.0":\n version: 12.6.2\n\n' + '"cpu-features@npm:~0.0.10":\n version: 0.0.10\n\n' + '"@backstage/core@npm:^1.0.0":\n version: 1.0.0\n' + ) + result = PluginListConfig._parse_host_packages(lockfile) + assert result == {"better-sqlite3", "cpu-features", "@backstage/core"} def test_missing_file_returns_empty(self, tmp_path): lockfile = tmp_path / "nonexistent.lock" - result = PluginListConfig._parse_host_backstage_packages(lockfile) + result = PluginListConfig._parse_host_packages(lockfile) assert result == set() - def test_backstage_in_dependency_lines_ignored(self, tmp_path): + def test_indented_dependency_lines_ignored(self, tmp_path): """Only top-level key lines (starting with quotes) are parsed, not indented dep lines.""" lockfile = tmp_path / "yarn.lock" lockfile.write_text( @@ -442,9 +453,10 @@ def test_backstage_in_dependency_lines_ignored(self, tmp_path): ' version: 1.0.0\n' ' dependencies:\n' ' "@backstage/types": "npm:^1.2.1"\n' + ' better-sqlite3: "npm:^12.0.0"\n' ) - result = PluginListConfig._parse_host_backstage_packages(lockfile) - assert result == set() + result = PluginListConfig._parse_host_packages(lockfile) + assert result == {"@some/package"} class TestGetSiblingNames: @@ -1145,10 +1157,10 @@ def test_mixed_deps_and_optional_deps(self, tmp_path): class TestComputeBackendBuildArgsWithNative: - """Tests for --suppress-native-package in _compute_backend_build_args.""" + """Tests for native module handling in _compute_backend_build_args.""" - def test_private_dep_with_native_transitive(self, tmp_path): - """Private dep has a native transitive dep -> suppress it.""" + def test_native_not_in_host_suppressed(self, tmp_path): + """Native dep NOT in host -> suppress it.""" _make_plugin_dir( tmp_path, "plugins/backend", "@test/my-backend", "backend-plugin", dependencies={"ssh2": "^1.0.0"}, @@ -1168,8 +1180,30 @@ def test_private_dep_with_native_transitive(self, tmp_path): ) assert "--suppress-native-package cpu-features" in result - def test_no_native_deps_no_suppress(self, tmp_path): - """Private dep with no native transitive deps -> no suppress flags.""" + def test_native_in_host_still_suppressed(self, tmp_path): + """Native dep IN host -> still suppressed unconditionally.""" + _make_plugin_dir( + tmp_path, "plugins/backend", "@test/my-backend", "backend-plugin", + dependencies={"ssh2": "^1.0.0"}, + ) + _make_node_module( + tmp_path, "ssh2", + dependencies={"cpu-features": "^0.0.9"}, + ) + _make_node_module( + tmp_path, "cpu-features", + dependencies={"node-gyp-build": "^4.0.0"}, + ) + + host = {"cpu-features"} + result = PluginListConfig._compute_backend_build_args( + tmp_path, "plugins/backend", + tmp_path / "plugins/backend/package.json", host, + ) + assert "--suppress-native-package cpu-features" in result + + def test_no_native_deps_no_flags(self, tmp_path): + """Private dep with no native transitive deps -> no suppress or share flags.""" _make_plugin_dir( tmp_path, "plugins/backend", "@test/my-backend", "backend-plugin", dependencies={"lodash": "^4.0.0"}, @@ -1181,6 +1215,7 @@ def test_no_native_deps_no_suppress(self, tmp_path): tmp_path / "plugins/backend/package.json", set(), ) assert "--suppress-native-package" not in result + assert "--shared-package" not in result def test_combined_embed_and_suppress(self, tmp_path): """Embed, unshare, AND suppress flags all present.""" @@ -1212,8 +1247,8 @@ def test_combined_embed_and_suppress(self, tmp_path): assert "--embed-package @custom/lib" in result assert "--suppress-native-package cpu-features" in result - def test_args_ordering(self, tmp_path): - """Flags are ordered: --embed-package, --shared-package, --suppress-native-package.""" + def test_args_ordering_with_suppress(self, tmp_path): + """Flags ordered: --embed, --shared-package !, --suppress-native-package.""" _make_plugin_dir( tmp_path, "plugins/backend", "@test/my-backend", "backend-plugin", dependencies={ @@ -1240,11 +1275,10 @@ def test_args_ordering(self, tmp_path): tmp_path / "plugins/backend/package.json", set(), ) embed_idx = result.index("--embed-package") - shared_idx = result.index("--shared-package") + shared_idx = result.index("--shared-package !") suppress_idx = result.index("--suppress-native-package") assert embed_idx < shared_idx < suppress_idx - class TestComputeBackendBuildArgsEmbeddedBackstageNative: """Native modules in embedded @backstage/* deps should be suppressed.""" @@ -1375,10 +1409,10 @@ def test_sibling_not_in_node_modules_skipped(self, tmp_path): assert result == "" -class TestCreateDefaultWithNativeSuppression: - """End-to-end tests for create_default with native module suppression.""" +class TestCreateDefaultWithNativeHandling: + """End-to-end tests for create_default with native module handling.""" - def test_native_dep_detected_end_to_end(self, tmp_path, monkeypatch): + def test_native_not_in_host_suppressed(self, tmp_path, monkeypatch): lockfile = tmp_path / "host-yarn.lock" lockfile.write_text('"@backstage/core@npm:^1.0.0":\n version: 1.0.0\n') monkeypatch.setattr(constants, "HOST_LOCKFILE", lockfile) @@ -1403,7 +1437,6 @@ def test_native_dep_detected_end_to_end(self, tmp_path, monkeypatch): args = config.get_plugins()["plugins/backend"] assert "--suppress-native-package cpu-features" in args - class TestPluginListConfigPopulateBuildArgs: """Tests for PluginListConfig.populate_build_args method.""" From e77b6e68aed15feb9fd776a4c58e1c64f3b356a4 Mon Sep 17 00:00:00 2001 From: Frank Kong Date: Wed, 11 Mar 2026 18:25:13 -0400 Subject: [PATCH 19/35] chore: update troubleshooting docs with special edge case for native deps Signed-off-by: Frank Kong --- README.md | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 52 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index dc48035..0fa6310 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,7 @@ A comprehensive tool for building and exporting dynamic plugins for Red Hat Deve - [Backend Module Not Loading (Missing Dependencies)](#backend-module-not-loading-missing-dependencies) - [Skopeo Fails During Plugin Installation](#skopeo-fails-during-plugin-installation) - [Quay.io Repository Publishing Issues](#quayio-repository-publishing-issues) + - [Plugin Export Fails Entry Point Validation Check](#plugin-export-fails-entry-point-validation-check) - [Local Development \& Contributing](#local-development--contributing) - [Resources](#resources) @@ -318,13 +319,15 @@ For **backend** plugins (and backend plugin modules), the factory performs depen #### Build Argument Types -The following build arguments may be automatically computed for backend plugins: +The following build arguments can be automatically computed for backend plugins or manually defined: | Argument | Purpose | | --------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `--embed-package ` | Bundles a dependency into the dynamic plugin. Applied to `@backstage/`* packages not provided by the RHDH host, and to non-`@backstage` packages that have transitive `@backstage/*` dependencies. | -| `--shared-package !` | Marks an embedded `@backstage/*` package as unshared so the plugin uses its own bundled copy instead of the host's version. (Optional Argument) | +| `--shared-package !` | Marks an embedded `@backstage/*` package as unshared so the plugin uses its own bundled copy instead of the host's version. | +| `--shared-package ` | Marks a dependency to be exported as a peerDependency to use the package already present in the RHDH host. | | `--suppress-native-package ` | Suppresses native Node.js modules that cannot be bundled. Detected by the presence of markers such as `bindings`, `prebuild`, `nan`, `node-gyp-build` in the package's dependencies, or `gypfile`/`binary` fields in its `package.json`. | +| `--allow-native-package ` | Experimental argument to allow bundling of specified native module. | #### Using `--generate-build-args` @@ -397,7 +400,7 @@ See the [TODO plugin example config](./examples/example-config-todo/README.md) a | `--workspace-path` | *(see below)* | Path to the workspace from repository root (e.g., `workspaces/todo`). Can also be set via `source.json`'s `workspace-path` field. | | `--source-repo` | `None` | Git repository URL. When provided, `source.json` is not required and the repository is cloned from this URL. | | `--source-ref` | `None` | Git ref (branch/tag/commit) to check out. Defaults to the repository's default branch. Requires `--source-repo`. | -| `--output-dir` | `/outputs` | Directory for build artifacts (`.tgz` files and integrity hash files) | +| `--output-dir` | `/outputs` | Directory for build artifacts (`.tgz` files and integrity hash files) | | `--push-images` / `--no-push-images` | `--no-push-images` | Whether to push container images to registry. Defaults to not pushing if no argument is provided | | `--use-local` | `false` | Use local repository instead of cloning from source.json | | `--log-level` | `INFO` | Logging level: `DEBUG`, `INFO`, `WARNING`, `ERROR`, `CRITICAL` | @@ -721,6 +724,52 @@ This may be due to Quay.io silently failing to publish images since your account To mitigate this, you may need to pre-create the repositories on `quay.io` before publishing to avoid having the factory attempt to create the repositories. Alternatively, you can upgrade your `quay.io` plan to increase the private repository allocation. +### Plugin Export Fails Entry Point Validation Check + +The build argument auto-generation handles native modules as follows: + +In some cases a plugin may fail the entry point validation check because the RHDH CLI attempts to load the plugin and a required native module has been suppressed. If the failure is due to a native module removed via `--suppress-native-package`, you can that argument with `--shared-package` for that specific module in your `plugins-list.yaml` since the native module most likely already exists in the `rhdh` container. + +If the export still fails due to a dependency depending on this native module, you will need to embed it via `--embed-packages`. The error logs will indicate which dependency should be embedded. + +Example Error Log: + +```bash +Error: Following shared package(s) should not be part of the plugin private dependencies: +- better-sqlite3 + +Either unshare them with the --shared-package ! option, or use the --embed-package to embed the following packages which use shared dependencies: +- @langchain/langgraph-checkpoint-sqlite +``` + +If the plugin fails to startup properly after installation due to the native module not being installed in the `rhdh` container, you will need to use experimental `--allow-native-package` arg instead to package the native module with the plugin instead. + +Note: Be sure to re-run the factory in a clean `--repo-path` environment since this can result in `yarn install --immutable` failing due to `yarn.lock` files present from previous factory runs. + +Example Entry Point Validation Error: + +Auto-generated `plugins-list.yaml` entry that will fail: + +```yaml +plugins/scaffolder-backend: --suppress-native-package isolated-vm --suppress-native-package napi-build-utils +``` + +```bash +Validating plugin entry points + adding typescript extension support to enable entry point validation + +Error: Unable to validate plugin entry points: Error: The package "isolated-vm" has been marked as + a native module and removed from this dynamic plugin package + "@backstage/plugin-scaffolder-backend-dynamic", as native modules are not currently supported by + dynamic plugins +``` + +Fixed `plugins-list.yaml` entry: + +```yaml +plugins/scaffolder-backend: --allow-native-package isolated-vm --suppress-native-package napi-build-utils +``` + ## Local Development & Contributing For users who want to run the factory locally without containers or contribute to the project, see [CONTRIBUTING.md](./CONTRIBUTING.md). From d1e3ae374a848b9fe7c29a2d3af3fd6255d52cf9 Mon Sep 17 00:00:00 2001 From: Frank Kong Date: Thu, 12 Mar 2026 14:34:38 -0400 Subject: [PATCH 20/35] chore: update troubleshooting docs to share package instead of allowing native Signed-off-by: Frank Kong --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0fa6310..1318554 100644 --- a/README.md +++ b/README.md @@ -767,7 +767,7 @@ Error: Unable to validate plugin entry points: Error: The package "isolated-vm" Fixed `plugins-list.yaml` entry: ```yaml -plugins/scaffolder-backend: --allow-native-package isolated-vm --suppress-native-package napi-build-utils +plugins/scaffolder-backend: --shared-package isolated-vm --suppress-native-package napi-build-utils ``` ## Local Development & Contributing From 30aa937cfb7c2d74c9bad4b470a139ff06b50c24 Mon Sep 17 00:00:00 2001 From: Frank Kong Date: Fri, 13 Mar 2026 15:55:42 -0400 Subject: [PATCH 21/35] chore: handle single directory workspace corner case Signed-off-by: Frank Kong --- pytest.ini | 2 + .../plugin_list_config.py | 9 ++ tests/test_plugin_list_config.py | 99 +++++++++++++++++++ 3 files changed, 110 insertions(+) diff --git a/pytest.ini b/pytest.ini index a1dd982..b0b692d 100644 --- a/pytest.ini +++ b/pytest.ini @@ -15,6 +15,7 @@ addopts = --tb=short --strict-markers --disable-warnings + -m "not e2e" # Coverage options (when using pytest-cov) # Uncomment to enable coverage reporting @@ -29,6 +30,7 @@ markers = unit: Unit tests integration: Integration tests slow: Tests that take a long time to run + e2e: End-to-end integration tests (require container runtime and E2E_IMAGE) # Logging log_cli = false diff --git a/src/rhdh_dynamic_plugin_factory/plugin_list_config.py b/src/rhdh_dynamic_plugin_factory/plugin_list_config.py index cd8e6f0..8bcb2ff 100644 --- a/src/rhdh_dynamic_plugin_factory/plugin_list_config.py +++ b/src/rhdh_dynamic_plugin_factory/plugin_list_config.py @@ -198,6 +198,15 @@ def create_default(cls, workspace_path: Path) -> "PluginListConfig": plugins: Dict[str, str] = {} host_packages = cls._get_host_packages() + # Corner case: if the workspace root has a valid backstage role, add it as a plugin + root_pkg_json = workspace_path / constants.PKG_JSON + if root_pkg_json.is_file(): + role = cls._read_backstage_role(root_pkg_json) + if role and role in constants.VALID_BACKSTAGE_PLUGIN_ROLES: + plugins["."] = cls._compute_plugin_build_args( + workspace_path, ".", root_pkg_json, host_packages, + ) + for pkg_json_path in cls._find_package_jsons(workspace_path): role = cls._read_backstage_role(pkg_json_path) if role and role in constants.VALID_BACKSTAGE_PLUGIN_ROLES: diff --git a/tests/test_plugin_list_config.py b/tests/test_plugin_list_config.py index 0fe6cc3..fec779c 100644 --- a/tests/test_plugin_list_config.py +++ b/tests/test_plugin_list_config.py @@ -207,6 +207,26 @@ def test_to_file_null_values_format(self, tmp_path): assert "''" not in content assert '""' not in content + def test_to_file_roundtrip_dot_key(self, tmp_path): + """Test that '.' as a plugin key survives a to_file -> from_file roundtrip.""" + original = PluginListConfig({ + ".": "--embed-package @backstage/new-pkg", + }) + out = tmp_path / "plugins-list.yaml" + original.to_file(out) + + loaded = PluginListConfig.from_file(out) + assert loaded.get_plugins() == original.get_plugins() + + def test_to_file_dot_key_no_args(self, tmp_path): + """Test that '.' key with no args produces '.:' in the output.""" + config = PluginListConfig({".": ""}) + out = tmp_path / "plugins-list.yaml" + config.to_file(out) + + content = out.read_text() + assert ".:" in content + def _make_plugin_dir(base, rel_path, name, role, dependencies=None): """Helper to create a plugin directory with a package.json.""" @@ -390,6 +410,62 @@ def test_aws_ecs_workspace_structure(self, tmp_path): assert "plugins/ecs/frontend" in plugins assert "plugins/ecs/backend" in plugins + def test_discovers_root_package_json_as_plugin(self, tmp_path): + """Test that a root package.json with a valid backstage role is discovered with key '.'.""" + pkg = { + "name": "@parfuemerie-douglas/scaffolder-backend-module-azure-pipelines", + "version": "1.3.0", + "backstage": {"role": "backend-plugin-module"}, + } + (tmp_path / "package.json").write_text(json.dumps(pkg)) + + config = PluginListConfig.create_default(tmp_path) + plugins = config.get_plugins() + + assert len(plugins) == 1 + assert "." in plugins + + def test_root_package_json_without_backstage_role_ignored(self, tmp_path): + """Test that a root package.json without backstage.role is ignored (normal monorepo root).""" + pkg = {"name": "my-workspace", "version": "1.0.0", "private": True} + (tmp_path / "package.json").write_text(json.dumps(pkg)) + _make_plugin_dir(tmp_path, "plugins/todo", "@test/plugin-todo", "frontend-plugin") + + config = PluginListConfig.create_default(tmp_path) + plugins = config.get_plugins() + + assert "." not in plugins + assert "plugins/todo" in plugins + + def test_root_package_json_with_non_plugin_role_ignored(self, tmp_path): + """Test that a root package.json with a non-plugin role (e.g. common-library) is ignored.""" + pkg = { + "name": "@test/my-common", + "version": "1.0.0", + "backstage": {"role": "common-library"}, + } + (tmp_path / "package.json").write_text(json.dumps(pkg)) + + config = PluginListConfig.create_default(tmp_path) + assert config.get_plugins() == {} + + def test_root_plugin_coexists_with_subdirectory_plugins(self, tmp_path): + """Test that root plugin '.' and subdirectory plugins are both discovered.""" + pkg = { + "name": "@test/root-backend", + "version": "1.0.0", + "backstage": {"role": "backend-plugin"}, + } + (tmp_path / "package.json").write_text(json.dumps(pkg)) + _make_plugin_dir(tmp_path, "plugins/todo", "@test/plugin-todo", "frontend-plugin") + + config = PluginListConfig.create_default(tmp_path) + plugins = config.get_plugins() + + assert len(plugins) == 2 + assert "." in plugins + assert "plugins/todo" in plugins + class TestParseHostPackages: """Tests for PluginListConfig._parse_host_packages.""" @@ -1629,6 +1705,29 @@ def test_backend_no_deps_stays_empty(self, tmp_path, monkeypatch): assert cfg.get_plugins()["plugins/simple-backend"] == "" + def test_dot_key_resolves_to_root_package_json(self, tmp_path, monkeypatch): + """Test that '.' as a plugin key resolves to the root package.json.""" + lockfile = tmp_path / "host-yarn.lock" + lockfile.write_text('"@backstage/core@npm:^1.0.0":\n version: 1.0.0\n') + monkeypatch.setattr(constants, "HOST_LOCKFILE", lockfile) + + workspace = tmp_path / "workspace" + workspace.mkdir() + pkg = { + "name": "@test/scaffolder-backend-module-azure", + "version": "1.0.0", + "backstage": {"role": "backend-plugin-module"}, + "dependencies": {"@backstage/new-experimental": "^0.1.0"}, + } + (workspace / "package.json").write_text(json.dumps(pkg)) + + cfg = PluginListConfig({".": ""}) + cfg.populate_build_args(workspace) + + args = cfg.get_plugins()["."] + assert "--embed-package @backstage/new-experimental" in args + assert "--shared-package !@backstage/new-experimental" in args + class TestLogBuildArgsDiff: """Tests for PluginListConfig._log_build_args_diff.""" From ab84357021271b21c0a72c32ce23d60cc105b722 Mon Sep 17 00:00:00 2001 From: Frank Kong Date: Fri, 13 Mar 2026 16:20:25 -0400 Subject: [PATCH 22/35] chore: add initial e2e integration tests Assisted-By: Cursor Signed-off-by: Frank Kong --- .cursor/rules/integration-tests.mdc | 2 +- .github/workflows/e2e-test.yaml | 312 ++++++++++++++++++++++++ tests/e2e/__init__.py | 0 tests/e2e/conftest.py | 229 +++++++++++++++++ tests/e2e/logs/.gitignore | 2 + tests/e2e/test_todo_single_workspace.py | 98 ++++++++ 6 files changed, 642 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/e2e-test.yaml create mode 100644 tests/e2e/__init__.py create mode 100644 tests/e2e/conftest.py create mode 100644 tests/e2e/logs/.gitignore create mode 100644 tests/e2e/test_todo_single_workspace.py diff --git a/.cursor/rules/integration-tests.mdc b/.cursor/rules/integration-tests.mdc index a2f3c98..be7d175 100644 --- a/.cursor/rules/integration-tests.mdc +++ b/.cursor/rules/integration-tests.mdc @@ -44,7 +44,7 @@ def test_full_config_flow(self, make_config): ## Container E2E Tests -> **Note:** E2E tests are not yet implemented. The `tests/e2e/` directory does not exist. The following guidance is for future implementation. +> E2E tests are implemented in `tests/e2e/`. The `E2E_IMAGE` env var is **required** (no default fallback). - **Location**: `tests/e2e/`. - **Goal**: Validate that the container image works correctly with the provided `examples/`. diff --git a/.github/workflows/e2e-test.yaml b/.github/workflows/e2e-test.yaml new file mode 100644 index 0000000..363caa9 --- /dev/null +++ b/.github/workflows/e2e-test.yaml @@ -0,0 +1,312 @@ +# Run e2e integration tests against the PR container image after PR Publish completes. +# Pulls the published PR image, runs the factory with test fixtures, and validates outputs. +name: E2E Tests + +on: + workflow_run: + workflows: ["PR Publish"] + types: [completed] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.workflow_run.id }} + cancel-in-progress: true + +permissions: + contents: read + +env: + REGISTRY: quay.io + REGISTRY_IMAGE: rhdh-community/dynamic-plugins-factory + +jobs: + e2e-test: + runs-on: ubuntu-latest + if: github.event.workflow_run.conclusion == 'success' + permissions: + contents: read + pull-requests: write + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + # Resolve PR number using gh cli — works for both same-repo and fork PRs. + # github.event.workflow_run.pull_requests is empty for fork PRs. + # See https://github.com/orgs/community/discussions/25220 + - name: Resolve PR number + id: pr-info + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_BRANCH: |- + ${{ + (github.event.workflow_run.head_repository.owner.login != github.event.workflow_run.repository.owner.login) + && format('{0}:{1}', github.event.workflow_run.head_repository.owner.login, github.event.workflow_run.head_branch) + || github.event.workflow_run.head_branch + }} + PR_TARGET_REPO: ${{ github.repository }} + run: | + echo "Resolving PR for branch: ${PR_BRANCH}" + PR_NUMBER=$(gh pr view --repo "${PR_TARGET_REPO}" "${PR_BRANCH}" --json number --jq '.number') + + if ! [[ "$PR_NUMBER" =~ ^[0-9]{1,7}$ ]]; then + echo "Error: Could not resolve a valid PR number from branch '${PR_BRANCH}'" + echo "Resolved value: '${PR_NUMBER}'" + exit 1 + fi + + IMAGE="${REGISTRY}/${REGISTRY_IMAGE}:pr-${PR_NUMBER}" + echo "pr_number=${PR_NUMBER}" >> $GITHUB_OUTPUT + echo "image=${IMAGE}" >> $GITHUB_OUTPUT + echo "Resolved PR #${PR_NUMBER}, image: ${IMAGE}" + + - name: Set up Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.14' + cache: 'pip' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install -r requirements.dev.txt + + - name: Pull container image + env: + IMAGE: ${{ steps.pr-info.outputs.image }} + run: | + echo "Pulling image: ${IMAGE}" + if ! podman pull "${IMAGE}"; then + echo "Error: Failed to pull image '${IMAGE}'" + echo "This indicates the PR build/publish pipeline failed or the image has expired." + exit 1 + fi + echo "Successfully pulled ${IMAGE}" + podman inspect "${IMAGE}" --format '{{.Id}}' + + - name: Run E2E tests + id: e2e-tests + continue-on-error: true + env: + E2E_IMAGE: ${{ steps.pr-info.outputs.image }} + E2E_CONTAINER_RUNTIME: podman + run: | + pytest tests/e2e/ -v --tb=long --junitxml=e2e-results.xml -m e2e + + - name: Upload E2E test logs + if: always() && steps.e2e-tests.outcome != 'skipped' + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: e2e-test-logs-pr-${{ steps.pr-info.outputs.pr_number }} + path: tests/e2e/logs/ + retention-days: 14 + if-no-files-found: ignore + + - name: Generate test summary + if: always() && steps.e2e-tests.outcome != 'skipped' + env: + IMAGE: ${{ steps.pr-info.outputs.image }} + PR_NUMBER: ${{ steps.pr-info.outputs.pr_number }} + PUBLISH_RUN_ID: ${{ github.event.workflow_run.id }} + E2E_RUN_ID: ${{ github.run_id }} + SERVER_URL: ${{ github.server_url }} + REPOSITORY: ${{ github.repository }} + run: | + python3 << 'PYTHON_SCRIPT' + import xml.etree.ElementTree as ET + import os + + image = os.environ["IMAGE"] + pr_number = os.environ["PR_NUMBER"] + publish_run_id = os.environ["PUBLISH_RUN_ID"] + e2e_run_id = os.environ["E2E_RUN_ID"] + server_url = os.environ["SERVER_URL"] + repository = os.environ["REPOSITORY"] + summary_file = os.environ.get("GITHUB_STEP_SUMMARY", "/dev/stdout") + + with open(summary_file, "a") as f: + try: + tree = ET.parse("e2e-results.xml") + root = tree.getroot() + suite = root.find("testsuite") if root.tag != "testsuite" else root + + tests = int(suite.get("tests", "0")) + failures = int(suite.get("failures", "0")) + errors = int(suite.get("errors", "0")) + skipped = int(suite.get("skipped", "0")) + time_taken = float(suite.get("time", "0")) + passed = tests - failures - errors - skipped + + all_passed = (failures + errors) == 0 + status_emoji = "✅" if all_passed else "❌" + status_text = "Passed" if all_passed else "Failed" + + f.write(f"## {status_emoji} E2E Test Results — {status_text}\n\n") + f.write(f"**Image:** `{image}`\n") + f.write(f"**PR:** #{pr_number}\n") + f.write(f"**Duration:** {time_taken:.1f}s\n\n") + + f.write("| Status | Count |\n") + f.write("|--------|-------|\n") + f.write(f"| ✅ Passed | {passed} |\n") + f.write(f"| ❌ Failed | {failures} |\n") + f.write(f"| 💥 Errors | {errors} |\n") + f.write(f"| ⏭️ Skipped | {skipped} |\n") + f.write(f"| **Total** | **{tests}** |\n\n") + + f.write("### Test Details\n\n") + f.write("| Test | Status | Duration |\n") + f.write("|------|--------|----------|\n") + + for tc in suite.iter("testcase"): + name = tc.get("name", "unknown") + tc_time = float(tc.get("time", "0")) + + failure = tc.find("failure") + error = tc.find("error") + skip = tc.find("skipped") + + if failure is not None: + tc_status = "❌ Failed" + elif error is not None: + tc_status = "💥 Error" + elif skip is not None: + tc_status = "⏭️ Skipped" + else: + tc_status = "✅ Passed" + + f.write(f"| `{name}` | {tc_status} | {tc_time:.1f}s |\n") + + f.write("\n") + + has_failures = False + for tc in suite.iter("testcase"): + failure = tc.find("failure") + error = tc.find("error") + detail = failure if failure is not None else error + + if detail is not None: + if not has_failures: + f.write("### Failure Details\n\n") + has_failures = True + + name = tc.get("name", "unknown") + message = detail.get("message", "") + text = detail.text or "" + if len(text) > 3000: + text = text[:3000] + "\n... (truncated)" + + f.write(f"
\n{name}: {message[:200]}\n\n") + f.write(f"```\n{text}\n```\n\n") + f.write("
\n\n") + + except FileNotFoundError: + f.write("## ❌ E2E Test Results\n\n") + f.write(f"**Image:** `{image}`\n") + f.write(f"**PR:** #{pr_number}\n\n") + f.write("No test results file found. Tests may have failed to start.\n\n") + + f.write("### Traceability\n\n") + f.write(f"- **Publish:** [{server_url}/{repository}/actions/runs/{publish_run_id}]") + f.write(f"({server_url}/{repository}/actions/runs/{publish_run_id})\n") + f.write(f"- **E2E:** [{server_url}/{repository}/actions/runs/{e2e_run_id}]") + f.write(f"({server_url}/{repository}/actions/runs/{e2e_run_id})\n") + PYTHON_SCRIPT + + - name: Comment on PR + if: always() && steps.pr-info.outcome == 'success' + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + env: + PR_NUMBER: ${{ steps.pr-info.outputs.pr_number }} + IMAGE: ${{ steps.pr-info.outputs.image }} + E2E_RUN_ID: ${{ github.run_id }} + PUBLISH_RUN_ID: ${{ github.event.workflow_run.id }} + TEST_OUTCOME: ${{ steps.e2e-tests.outcome }} + with: + script: | + const prNumber = parseInt(process.env.PR_NUMBER); + const image = process.env.IMAGE; + const e2eRunId = process.env.E2E_RUN_ID; + const publishRunId = process.env.PUBLISH_RUN_ID; + const testOutcome = process.env.TEST_OUTCOME; + + const e2eUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${e2eRunId}`; + const publishUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${publishRunId}`; + + let testSummary = ''; + const fs = require('fs'); + try { + const xml = fs.readFileSync('e2e-results.xml', 'utf8'); + const testsMatch = xml.match(/tests="(\d+)"/); + const failuresMatch = xml.match(/failures="(\d+)"/); + const errorsMatch = xml.match(/errors="(\d+)"/); + const timeMatch = xml.match(/time="([\d.]+)"/); + + const total = testsMatch ? parseInt(testsMatch[1]) : 0; + const failures = failuresMatch ? parseInt(failuresMatch[1]) : 0; + const errors = errorsMatch ? parseInt(errorsMatch[1]) : 0; + const time = timeMatch ? parseFloat(timeMatch[1]).toFixed(1) : '?'; + const passed = total - failures - errors; + + testSummary = [ + '| Tests | Passed | Failed | Duration |', + '|-------|--------|--------|----------|', + `| ${total} | ${passed} | ${failures + errors} | ${time}s |`, + ].join('\n'); + } catch (e) { + testSummary = '_No detailed results available._'; + } + + const statusEmoji = testOutcome === 'success' ? '✅' : '❌'; + const statusText = testOutcome === 'success' ? 'Passed' : 'Failed'; + const statusDetail = testOutcome === 'success' + ? 'All end-to-end integration tests passed for the container image.' + : 'End-to-end integration tests failed for the container image. See the workflow run for details.'; + + const body = [ + `## ${statusEmoji} E2E Tests ${statusText}`, + '', + statusDetail, + '', + `**Image:** \`${image}\``, + '', + testSummary, + '', + '### Traceability', + '', + `- **Publish:** [PR Publish #${publishRunId}](${publishUrl})`, + `- **E2E:** [E2E Tests #${e2eRunId}](${e2eUrl})`, + ].join('\n'); + + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + body: body + }); + + - name: Fail workflow if tests failed + if: steps.e2e-tests.outcome != 'success' + run: | + echo "E2E tests failed. Failing workflow." + exit 1 + + # Handle failed upstream workflow (PR Publish failed) + notify-publish-failure: + runs-on: ubuntu-latest + if: github.event.workflow_run.conclusion == 'failure' + permissions: + pull-requests: write + steps: + - name: E2E Skipped Summary + env: + PUBLISH_RUN_ID: ${{ github.event.workflow_run.id }} + SERVER_URL: ${{ github.server_url }} + REPOSITORY: ${{ github.repository }} + run: | + echo "## ⏭️ E2E Tests Skipped" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "The upstream PR Publish workflow failed. E2E tests were not run." >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "- **PR Publish:** [Run #${PUBLISH_RUN_ID}](${SERVER_URL}/${REPOSITORY}/actions/runs/${PUBLISH_RUN_ID}) (failed)" >> $GITHUB_STEP_SUMMARY diff --git a/tests/e2e/__init__.py b/tests/e2e/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py new file mode 100644 index 0000000..a521ece --- /dev/null +++ b/tests/e2e/conftest.py @@ -0,0 +1,229 @@ +""" +E2E test fixtures for RHDH Dynamic Plugin Factory container tests. + +These tests run the actual container image against known config fixtures +and validate that builds succeed without errors and produce expected outputs. + +Required environment variables: + E2E_IMAGE: Container image to test (e.g. quay.io/.../dynamic-plugins-factory:pr-42) + +Optional environment variables: + E2E_CONTAINER_RUNTIME: Container runtime binary (default: podman) + E2E_LOG_DIR: Directory to save container logs (default: tests/e2e/logs/) +""" + +import os +import re +import shutil +import subprocess +from dataclasses import dataclass +from pathlib import Path +from typing import Optional + +import pytest +import yaml + +FIXTURES_DIR = Path(__file__).parent / "fixtures" +DEFAULT_LOG_DIR = Path(__file__).parent / "logs" +CONTAINER_TIMEOUT = 900 # 15 minutes + + +@dataclass +class ContainerResult: + """Structured result from running the factory container. + + ``output`` contains the merged stdout+stderr stream in chronological order. + ``log_file`` is the path to the persisted log file on disk. + """ + + returncode: int + output: str + output_dir: Path + log_file: Path + + +# --------------------------------------------------------------------------- +# Helper functions +# --------------------------------------------------------------------------- + +_FAILED_EXPORTS_MARKER = "Plugins with failed exports:" + +_ERROR_PATTERNS: list[str] = [ + "Error running CLI", + "Error pushing container", + "Error building container", +] + + +def parse_plugins_from_config(config_dir: Path) -> list[str]: + """Read plugins-list.yaml from *config_dir* and return plugin paths. + + Handles both valued and empty YAML keys, comments, and blank lines. + """ + plugins_file = config_dir / "plugins-list.yaml" + data = yaml.safe_load(plugins_file.read_text()) + if not data or not isinstance(data, dict): + return [] + return list(data.keys()) + + +def plugin_path_to_output_pattern(plugin_path: str) -> re.Pattern: + """Build a regex that matches the npm-pack tarball for *plugin_path*. + + For ``plugins/todo`` the last path component is ``todo`` and the expected + tarball looks like ``backstage-community-plugin-todo-0.12.0.tgz`` (or with + an optional ``-dynamic`` suffix before the version). The pattern ensures + we do not accidentally match ``todo-backend`` by requiring a digit (the + start of the semver version) immediately after the plugin name. + """ + plugin_name = Path(plugin_path).name + return re.compile(rf"-{re.escape(plugin_name)}(-dynamic)?-\d.*\.tgz$") + + +def get_output_tgz_files(output_dir: Path) -> list[Path]: + """Return all ``.tgz`` files in ``output_dir`` (sorted).""" + return sorted(output_dir.glob("*.tgz")) + + +def get_output_integrity_files(output_dir: Path) -> list[Path]: + """Return all ``.tgz.integrity`` files in ``output_dir`` (sorted).""" + return sorted(output_dir.glob("*.tgz.integrity")) + + +def find_outputs_for_plugin( + plugin_path: str, + tgz_files: list[Path], +) -> list[Path]: + """Return tgz files from *tgz_files* that match *plugin_path*.""" + pattern = plugin_path_to_output_pattern(plugin_path) + return [f for f in tgz_files if pattern.search(f.name)] + + +def _collect_log_errors(output: str) -> list[str]: + """Scan combined container output for known error indicators.""" + errors: list[str] = [] + + for pattern in _ERROR_PATTERNS: + if pattern in output: + errors.append(f"Found error pattern: '{pattern}'") + + if _FAILED_EXPORTS_MARKER not in output: + return errors + + for line in output.splitlines(): + if _FAILED_EXPORTS_MARKER not in line: + continue + exports_part = line.split(_FAILED_EXPORTS_MARKER)[-1].strip() + if exports_part: + errors.append(f"Failed plugin exports: {exports_part}") + break + + return errors + + +def assert_no_errors_in_logs(result: ContainerResult) -> None: + """Fail the test if the container logs contain error indicators.""" + errors = _collect_log_errors(result.output) + if not errors: + return + + max_tail = 3000 + tail = result.output[-max_tail:] + pytest.fail( + "Errors detected in container logs:\n" + + "\n".join(f" - {e}" for e in errors) + + f"\n\nFull log: {result.log_file}" + + f"\n\n--- container output (last {max_tail} chars) ---\n{tail}" + ) + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture(scope="session") +def container_image() -> str: + """Container image to test — read from ``E2E_IMAGE`` (required).""" + image = os.environ.get("E2E_IMAGE") + if not image: + pytest.fail( + "E2E_IMAGE environment variable is required but not set.\n" + "Set it to the PR container image to test, e.g.:\n" + " E2E_IMAGE=quay.io/rhdh-community/dynamic-plugins-factory:pr-42 " + "pytest tests/e2e/ -m e2e" + ) + return image + + +@pytest.fixture(scope="session") +def container_runtime() -> str: + """Container runtime binary — read from ``E2E_CONTAINER_RUNTIME`` (default: podman).""" + runtime = os.environ.get("E2E_CONTAINER_RUNTIME", "podman") + if not shutil.which(runtime): + pytest.fail( + f"Container runtime '{runtime}' not found on $PATH.\n" + f"Install {runtime} or set E2E_CONTAINER_RUNTIME to an available runtime." + ) + return runtime + + +@pytest.fixture(scope="session") +def e2e_log_dir() -> Path: + """Directory where container logs are persisted.""" + log_dir = Path(os.environ.get("E2E_LOG_DIR", str(DEFAULT_LOG_DIR))) + log_dir.mkdir(parents=True, exist_ok=True) + return log_dir + + +@pytest.fixture(scope="session") +def run_factory_container( + container_image: str, + container_runtime: str, + tmp_path_factory: pytest.TempPathFactory, + e2e_log_dir: Path, +): + """Factory fixture: call the returned function to run the container. + + Each call creates a fresh output directory, persists the container log + to ``E2E_LOG_DIR``, and returns a :class:`ContainerResult`. + """ + + def _run( + config_dir: Path, + extra_args: Optional[list[str]] = None, + timeout: int = CONTAINER_TIMEOUT, + ) -> ContainerResult: + output_dir = tmp_path_factory.mktemp("outputs") + log_name = config_dir.parent.name if config_dir.name == "config" else config_dir.name + log_file = e2e_log_dir / f"{log_name}.log" + + cmd = [ + container_runtime, + "run", + "--rm", + "--device", "/dev/fuse", + "-v", f"{config_dir}:/config:z", + "-v", f"{output_dir}:/outputs:z", + container_image, + ] + if extra_args: + cmd.extend(extra_args) + + completed = subprocess.run( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + timeout=timeout, + ) + log_file.write_text(completed.stdout) + + return ContainerResult( + returncode=completed.returncode, + output=completed.stdout, + output_dir=output_dir, + log_file=log_file, + ) + + return _run diff --git a/tests/e2e/logs/.gitignore b/tests/e2e/logs/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/tests/e2e/logs/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/tests/e2e/test_todo_single_workspace.py b/tests/e2e/test_todo_single_workspace.py new file mode 100644 index 0000000..30bdd7d --- /dev/null +++ b/tests/e2e/test_todo_single_workspace.py @@ -0,0 +1,98 @@ +""" +E2E tests for the TODO single-workspace example. + +Runs the factory container with the todo fixture config and validates that +both frontend and backend plugins are built successfully. +""" + +import pytest + +from .conftest import ( + FIXTURES_DIR, + ContainerResult, + assert_no_errors_in_logs, + find_outputs_for_plugin, + get_output_integrity_files, + get_output_tgz_files, + parse_plugins_from_config, +) + +TODO_CONFIG_DIR = FIXTURES_DIR / "todo" / "config" + + +@pytest.fixture(scope="class") +def todo_result(run_factory_container) -> ContainerResult: + """Run the factory container once for the todo fixture; shared by the class.""" + return run_factory_container(TODO_CONFIG_DIR) + + +@pytest.fixture(scope="class") +def expected_plugins() -> list[str]: + """Plugin paths parsed from the todo fixture's plugins-list.yaml.""" + return parse_plugins_from_config(TODO_CONFIG_DIR) + + +@pytest.mark.e2e +class TestTodoSingleWorkspace: + """Validate a full container run of the TODO single-workspace example.""" + + def test_container_exits_successfully(self, todo_result: ContainerResult) -> None: + assert todo_result.returncode == 0, ( + f"Container exited with code {todo_result.returncode}\n" + f"Full log: {todo_result.log_file}\n" + f"output:\n{todo_result.output[-3000:]}" + ) + + def test_no_errors_in_logs(self, todo_result: ContainerResult) -> None: + assert_no_errors_in_logs(todo_result) + + def test_all_plugins_produce_tgz( + self, + todo_result: ContainerResult, + expected_plugins: list[str], + ) -> None: + tgz_files = get_output_tgz_files(todo_result.output_dir) + + for plugin_path in expected_plugins: + matches = find_outputs_for_plugin(plugin_path, tgz_files) + assert matches, ( + f"No .tgz output found for plugin '{plugin_path}'\n" + f"Available tgz files: {[f.name for f in tgz_files]}" + ) + + def test_all_plugins_produce_integrity( + self, + todo_result: ContainerResult, + expected_plugins: list[str], + ) -> None: + integrity_files = get_output_integrity_files(todo_result.output_dir) + + for plugin_path in expected_plugins: + plugin_name = plugin_path.split("/")[-1] + matches = [f for f in integrity_files if plugin_name in f.name] + assert matches, ( + f"No .tgz.integrity output found for plugin '{plugin_path}'\n" + f"Available integrity files: {[f.name for f in integrity_files]}" + ) + + def test_output_tarballs_are_nonzero( + self, todo_result: ContainerResult + ) -> None: + tgz_files = get_output_tgz_files(todo_result.output_dir) + assert tgz_files, "No .tgz files found in output directory" + + for tgz in tgz_files: + assert tgz.stat().st_size > 0, f"Tarball is empty: {tgz.name}" + + def test_output_count_matches_plugins( + self, + todo_result: ContainerResult, + expected_plugins: list[str], + ) -> None: + tgz_files = get_output_tgz_files(todo_result.output_dir) + assert len(tgz_files) >= len(expected_plugins), ( + f"Expected at least {len(expected_plugins)} tgz outputs " + f"(one per plugin), got {len(tgz_files)}.\n" + f"Plugins: {expected_plugins}\n" + f"Outputs: {[f.name for f in tgz_files]}" + ) From 3625ac9310e50433c5ac62cdded1759f82001b1f Mon Sep 17 00:00:00 2001 From: Frank Kong Date: Fri, 13 Mar 2026 17:30:31 -0400 Subject: [PATCH 23/35] chore: update skip action to be incremental and build for e2e tests Signed-off-by: Frank Kong --- .github/actions/should-skip-build/action.yaml | 60 ++++++++++++++++--- .github/workflows/pr-build.yaml | 3 + 2 files changed, 55 insertions(+), 8 deletions(-) diff --git a/.github/actions/should-skip-build/action.yaml b/.github/actions/should-skip-build/action.yaml index f61fcc6..effc13f 100644 --- a/.github/actions/should-skip-build/action.yaml +++ b/.github/actions/should-skip-build/action.yaml @@ -18,6 +18,10 @@ inputs: description: 'Regex patterns to match files for skip logic (space, comma, or newline-separated)' required: false default: '\.md$' + pr-number: + description: 'PR number for first-build protection. When set, skips only if a pr-{number} image already exists in the registry.' + required: false + default: '' outputs: should_skip: description: 'Whether the build should be skipped (true/false)' @@ -29,7 +33,7 @@ runs: using: 'composite' steps: - name: Install skopeo - if: inputs.check-image == 'true' + if: inputs.check-image == 'true' || inputs.pr-number != '' shell: bash run: | if ! command -v skopeo &> /dev/null; then @@ -93,13 +97,26 @@ runs: ;; pull_request|pull_request_target) - echo "Pull request event detected" - if [ -n "${{ github.event.pull_request.base.sha }}" ]; then - # Find the merge base between PR base and HEAD + echo "Pull request event detected (action: ${{ github.event.action }})" + + # For synchronize events, use incremental diff (since last push) + # to avoid rebuilding for non-build affecting commits in a PR. + # Fall back to full PR diff for opened/reopened or if incremental fails. + if [ "${{ github.event.action }}" = "synchronize" ] && [ -n "${{ github.event.before }}" ]; then + BEFORE_SHA="${{ github.event.before }}" + echo "Synchronize event — attempting incremental diff against previous HEAD: ${BEFORE_SHA}" + if CHANGED_FILES=$(git diff --name-only "${BEFORE_SHA}" HEAD 2>/dev/null); then + echo "Using incremental diff (since last push)" + else + echo "Warning: incremental diff failed (before SHA unreachable), falling back to full PR diff" + BASE_COMMIT=$(git merge-base ${{ github.event.pull_request.base.sha }} HEAD) + echo "Base commit (merge-base): $BASE_COMMIT" + CHANGED_FILES=$(git diff --name-only "$BASE_COMMIT" HEAD) + fi + elif [ -n "${{ github.event.pull_request.base.sha }}" ]; then BASE_COMMIT=$(git merge-base ${{ github.event.pull_request.base.sha }} HEAD) echo "Base commit (merge-base): $BASE_COMMIT" CHANGED_FILES=$(git diff --name-only "$BASE_COMMIT" HEAD) - else echo "Warning: No PR base SHA available, skipping file check" echo "should_skip=${SHOULD_SKIP}" >> $GITHUB_OUTPUT @@ -177,9 +194,36 @@ runs: done <<< "$CHANGED_FILES" if [ "$ALL_FILES_MATCH_PATTERN" = "true" ]; then - SHOULD_SKIP="true" - SKIP_REASON="All changed files match skip patterns: $FILE_PATTERNS" - echo "✓ All changed files match skip patterns - build will be skipped" + echo "✓ All changed files match skip patterns" + + # First-build protection: when pr-number is set, only skip if a PR + # image already exists. This ensures the first build always runs so + # that e2e tests have an image to test against. + PR_NUMBER="${{ inputs.pr-number }}" + if [ -n "$PR_NUMBER" ]; then + echo "" + echo "::group::First-build protection — checking for existing PR image" + PR_IMAGE_TAG="docker://${{ inputs.registry }}/${{ inputs.image }}:pr-${PR_NUMBER}" + echo "Checking for PR image: ${PR_IMAGE_TAG}" + + skopeo inspect "${PR_IMAGE_TAG}" &>/dev/null + PR_IMAGE_EXISTS=$? + + if [ $PR_IMAGE_EXISTS -eq 0 ]; then + SHOULD_SKIP="true" + SKIP_REASON="All changed files match skip patterns and PR image pr-${PR_NUMBER} already exists" + echo "PR image exists - safe to skip build" + else + SHOULD_SKIP="false" + SKIP_REASON="All changed files match skip patterns but no PR image exists yet — first build needed for e2e tests" + echo "No PR image found - first build required" + fi + echo "::endgroup::" + else + SHOULD_SKIP="true" + SKIP_REASON="All changed files match skip patterns: $FILE_PATTERNS" + echo "Build will be skipped" + fi else echo "✗ Found ${#NON_MATCHING_FILES[@]} file(s) not matching skip patterns:" printf ' - %s\n' "${NON_MATCHING_FILES[@]}" diff --git a/.github/workflows/pr-build.yaml b/.github/workflows/pr-build.yaml index d7f78ae..217dd10 100644 --- a/.github/workflows/pr-build.yaml +++ b/.github/workflows/pr-build.yaml @@ -33,6 +33,9 @@ jobs: with: commit-sha: ${{ github.event.pull_request.head.sha }} check-image: 'false' + registry: ${{ env.REGISTRY }} + image: ${{ env.REGISTRY_IMAGE }} + pr-number: ${{ github.event.number }} file-patterns: | \.md$ ^renovate\.json$ From d0a4d31f6951e158976ab7eb32c6ff1d412cbe1f Mon Sep 17 00:00:00 2001 From: Frank Kong Date: Fri, 13 Mar 2026 17:55:58 -0400 Subject: [PATCH 24/35] chore: fix pr resolution in e2e-test workflow Signed-off-by: Frank Kong --- .github/workflows/e2e-test.yaml | 45 ++++++++++++++++--------------- .github/workflows/pr-publish.yaml | 8 ++++++ 2 files changed, 32 insertions(+), 21 deletions(-) diff --git a/.github/workflows/e2e-test.yaml b/.github/workflows/e2e-test.yaml index 363caa9..f9f6809 100644 --- a/.github/workflows/e2e-test.yaml +++ b/.github/workflows/e2e-test.yaml @@ -26,39 +26,42 @@ jobs: contents: read pull-requests: write steps: - - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + # Download the PR metadata artifact uploaded by PR Publish since we can't get PR info from the github event + - name: Download PR metadata + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: - persist-credentials: false + name: e2e-pr-metadata + path: /tmp/e2e-pr-metadata + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ secrets.GITHUB_TOKEN }} - # Resolve PR number using gh cli — works for both same-repo and fork PRs. - # github.event.workflow_run.pull_requests is empty for fork PRs. - # See https://github.com/orgs/community/discussions/25220 - - name: Resolve PR number + - name: Read PR metadata id: pr-info - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - PR_BRANCH: |- - ${{ - (github.event.workflow_run.head_repository.owner.login != github.event.workflow_run.repository.owner.login) - && format('{0}:{1}', github.event.workflow_run.head_repository.owner.login, github.event.workflow_run.head_branch) - || github.event.workflow_run.head_branch - }} - PR_TARGET_REPO: ${{ github.repository }} run: | - echo "Resolving PR for branch: ${PR_BRANCH}" - PR_NUMBER=$(gh pr view --repo "${PR_TARGET_REPO}" "${PR_BRANCH}" --json number --jq '.number') + if [[ ! -f "/tmp/e2e-pr-metadata/pr-info.json" ]]; then + echo "Error: PR metadata file not found" + exit 1 + fi + + PR_NUMBER=$(jq -r '.pr_number' /tmp/e2e-pr-metadata/pr-info.json) + COMMIT_SHA=$(jq -r '.commit_sha' /tmp/e2e-pr-metadata/pr-info.json) + SHORT_SHA=$(jq -r '.short_sha' /tmp/e2e-pr-metadata/pr-info.json) if ! [[ "$PR_NUMBER" =~ ^[0-9]{1,7}$ ]]; then - echo "Error: Could not resolve a valid PR number from branch '${PR_BRANCH}'" - echo "Resolved value: '${PR_NUMBER}'" + echo "Error: Invalid PR number from metadata: '${PR_NUMBER}'" exit 1 fi IMAGE="${REGISTRY}/${REGISTRY_IMAGE}:pr-${PR_NUMBER}" echo "pr_number=${PR_NUMBER}" >> $GITHUB_OUTPUT echo "image=${IMAGE}" >> $GITHUB_OUTPUT - echo "Resolved PR #${PR_NUMBER}, image: ${IMAGE}" + echo "short_sha=${SHORT_SHA}" >> $GITHUB_OUTPUT + echo "Resolved PR #${PR_NUMBER}, commit: ${SHORT_SHA}, image: ${IMAGE}" + + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - name: Set up Python uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 diff --git a/.github/workflows/pr-publish.yaml b/.github/workflows/pr-publish.yaml index 3715058..4eb43f9 100644 --- a/.github/workflows/pr-publish.yaml +++ b/.github/workflows/pr-publish.yaml @@ -90,6 +90,14 @@ jobs: echo "Commit SHA: $COMMIT_SHA" echo "Primary Tag: $PRIMARY_TAG" + - name: Upload PR metadata for E2E tests + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: e2e-pr-metadata + path: /tmp/pr-metadata/pr-info.json + retention-days: 7 + if-no-files-found: error + - name: Download amd64 container artifact uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: From 5662ead485f272286cc91edc437e640e3834e629 Mon Sep 17 00:00:00 2001 From: Frank Kong Date: Mon, 16 Mar 2026 11:42:01 -0400 Subject: [PATCH 25/35] chore: add more traceability for the e2e test workflow Signed-off-by: Frank Kong --- .github/workflows/e2e-test.yaml | 320 ++++++++++++++++++++------------ 1 file changed, 204 insertions(+), 116 deletions(-) diff --git a/.github/workflows/e2e-test.yaml b/.github/workflows/e2e-test.yaml index f9f6809..1e24d1b 100644 --- a/.github/workflows/e2e-test.yaml +++ b/.github/workflows/e2e-test.yaml @@ -76,19 +76,21 @@ jobs: pip install -r requirements.dev.txt - name: Pull container image + id: pull-image + continue-on-error: true env: IMAGE: ${{ steps.pr-info.outputs.image }} run: | echo "Pulling image: ${IMAGE}" if ! podman pull "${IMAGE}"; then - echo "Error: Failed to pull image '${IMAGE}'" - echo "This indicates the PR build/publish pipeline failed or the image has expired." + echo "::warning::Failed to pull image '${IMAGE}'. The PR build may have been skipped or the image has expired." exit 1 fi echo "Successfully pulled ${IMAGE}" podman inspect "${IMAGE}" --format '{{.Id}}' - name: Run E2E tests + if: steps.pull-image.outcome == 'success' id: e2e-tests continue-on-error: true env: @@ -107,7 +109,7 @@ jobs: if-no-files-found: ignore - name: Generate test summary - if: always() && steps.e2e-tests.outcome != 'skipped' + if: always() && steps.pr-info.outcome == 'success' env: IMAGE: ${{ steps.pr-info.outputs.image }} PR_NUMBER: ${{ steps.pr-info.outputs.pr_number }} @@ -115,6 +117,7 @@ jobs: E2E_RUN_ID: ${{ github.run_id }} SERVER_URL: ${{ github.server_url }} REPOSITORY: ${{ github.repository }} + PULL_OUTCOME: ${{ steps.pull-image.outcome }} run: | python3 << 'PYTHON_SCRIPT' import xml.etree.ElementTree as ET @@ -126,89 +129,99 @@ jobs: e2e_run_id = os.environ["E2E_RUN_ID"] server_url = os.environ["SERVER_URL"] repository = os.environ["REPOSITORY"] + pull_outcome = os.environ["PULL_OUTCOME"] summary_file = os.environ.get("GITHUB_STEP_SUMMARY", "/dev/stdout") with open(summary_file, "a") as f: - try: - tree = ET.parse("e2e-results.xml") - root = tree.getroot() - suite = root.find("testsuite") if root.tag != "testsuite" else root - - tests = int(suite.get("tests", "0")) - failures = int(suite.get("failures", "0")) - errors = int(suite.get("errors", "0")) - skipped = int(suite.get("skipped", "0")) - time_taken = float(suite.get("time", "0")) - passed = tests - failures - errors - skipped - - all_passed = (failures + errors) == 0 - status_emoji = "✅" if all_passed else "❌" - status_text = "Passed" if all_passed else "Failed" - - f.write(f"## {status_emoji} E2E Test Results — {status_text}\n\n") - f.write(f"**Image:** `{image}`\n") - f.write(f"**PR:** #{pr_number}\n") - f.write(f"**Duration:** {time_taken:.1f}s\n\n") - - f.write("| Status | Count |\n") - f.write("|--------|-------|\n") - f.write(f"| ✅ Passed | {passed} |\n") - f.write(f"| ❌ Failed | {failures} |\n") - f.write(f"| 💥 Errors | {errors} |\n") - f.write(f"| ⏭️ Skipped | {skipped} |\n") - f.write(f"| **Total** | **{tests}** |\n\n") - - f.write("### Test Details\n\n") - f.write("| Test | Status | Duration |\n") - f.write("|------|--------|----------|\n") - - for tc in suite.iter("testcase"): - name = tc.get("name", "unknown") - tc_time = float(tc.get("time", "0")) - - failure = tc.find("failure") - error = tc.find("error") - skip = tc.find("skipped") - - if failure is not None: - tc_status = "❌ Failed" - elif error is not None: - tc_status = "💥 Error" - elif skip is not None: - tc_status = "⏭️ Skipped" - else: - tc_status = "✅ Passed" - - f.write(f"| `{name}` | {tc_status} | {tc_time:.1f}s |\n") - - f.write("\n") - - has_failures = False - for tc in suite.iter("testcase"): - failure = tc.find("failure") - error = tc.find("error") - detail = failure if failure is not None else error - - if detail is not None: - if not has_failures: - f.write("### Failure Details\n\n") - has_failures = True - - name = tc.get("name", "unknown") - message = detail.get("message", "") - text = detail.text or "" - if len(text) > 3000: - text = text[:3000] + "\n... (truncated)" - - f.write(f"
\n{name}: {message[:200]}\n\n") - f.write(f"```\n{text}\n```\n\n") - f.write("
\n\n") - - except FileNotFoundError: - f.write("## ❌ E2E Test Results\n\n") + if pull_outcome != "success": + f.write("## ⚠️ E2E Tests — Image Not Available\n\n") f.write(f"**Image:** `{image}`\n") f.write(f"**PR:** #{pr_number}\n\n") - f.write("No test results file found. Tests may have failed to start.\n\n") + f.write("The PR container image could not be pulled. " + "This typically means the PR build was skipped " + "(e.g., only non-build files were changed) or the image has expired.\n\n") + f.write("E2E tests were **not executed**.\n\n") + else: + try: + tree = ET.parse("e2e-results.xml") + root = tree.getroot() + suite = root.find("testsuite") if root.tag != "testsuite" else root + + tests = int(suite.get("tests", "0")) + failures = int(suite.get("failures", "0")) + errors = int(suite.get("errors", "0")) + skipped = int(suite.get("skipped", "0")) + time_taken = float(suite.get("time", "0")) + passed = tests - failures - errors - skipped + + all_passed = (failures + errors) == 0 + status_emoji = "✅" if all_passed else "❌" + status_text = "Passed" if all_passed else "Failed" + + f.write(f"## {status_emoji} E2E Test Results — {status_text}\n\n") + f.write(f"**Image:** `{image}`\n") + f.write(f"**PR:** #{pr_number}\n") + f.write(f"**Duration:** {time_taken:.1f}s\n\n") + + f.write("| Status | Count |\n") + f.write("|--------|-------|\n") + f.write(f"| ✅ Passed | {passed} |\n") + f.write(f"| ❌ Failed | {failures} |\n") + f.write(f"| 💥 Errors | {errors} |\n") + f.write(f"| ⏭️ Skipped | {skipped} |\n") + f.write(f"| **Total** | **{tests}** |\n\n") + + f.write("### Test Details\n\n") + f.write("| Test | Status | Duration |\n") + f.write("|------|--------|----------|\n") + + for tc in suite.iter("testcase"): + name = tc.get("name", "unknown") + tc_time = float(tc.get("time", "0")) + + failure = tc.find("failure") + error = tc.find("error") + skip = tc.find("skipped") + + if failure is not None: + tc_status = "❌ Failed" + elif error is not None: + tc_status = "💥 Error" + elif skip is not None: + tc_status = "⏭️ Skipped" + else: + tc_status = "✅ Passed" + + f.write(f"| `{name}` | {tc_status} | {tc_time:.1f}s |\n") + + f.write("\n") + + has_failures = False + for tc in suite.iter("testcase"): + failure = tc.find("failure") + error = tc.find("error") + detail = failure if failure is not None else error + + if detail is not None: + if not has_failures: + f.write("### Failure Details\n\n") + has_failures = True + + name = tc.get("name", "unknown") + message = detail.get("message", "") + text = detail.text or "" + if len(text) > 3000: + text = text[:3000] + "\n... (truncated)" + + f.write(f"
\n{name}: {message[:200]}\n\n") + f.write(f"```\n{text}\n```\n\n") + f.write("
\n\n") + + except FileNotFoundError: + f.write("## ❌ E2E Test Results\n\n") + f.write(f"**Image:** `{image}`\n") + f.write(f"**PR:** #{pr_number}\n\n") + f.write("No test results file found. Tests may have failed to start.\n\n") f.write("### Traceability\n\n") f.write(f"- **Publish:** [{server_url}/{repository}/actions/runs/{publish_run_id}]") @@ -226,6 +239,7 @@ jobs: E2E_RUN_ID: ${{ github.run_id }} PUBLISH_RUN_ID: ${{ github.event.workflow_run.id }} TEST_OUTCOME: ${{ steps.e2e-tests.outcome }} + PULL_OUTCOME: ${{ steps.pull-image.outcome }} with: script: | const prNumber = parseInt(process.env.PR_NUMBER); @@ -233,64 +247,85 @@ jobs: const e2eRunId = process.env.E2E_RUN_ID; const publishRunId = process.env.PUBLISH_RUN_ID; const testOutcome = process.env.TEST_OUTCOME; + const pullOutcome = process.env.PULL_OUTCOME; const e2eUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${e2eRunId}`; const publishUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${publishRunId}`; - let testSummary = ''; - const fs = require('fs'); - try { - const xml = fs.readFileSync('e2e-results.xml', 'utf8'); - const testsMatch = xml.match(/tests="(\d+)"/); - const failuresMatch = xml.match(/failures="(\d+)"/); - const errorsMatch = xml.match(/errors="(\d+)"/); - const timeMatch = xml.match(/time="([\d.]+)"/); - - const total = testsMatch ? parseInt(testsMatch[1]) : 0; - const failures = failuresMatch ? parseInt(failuresMatch[1]) : 0; - const errors = errorsMatch ? parseInt(errorsMatch[1]) : 0; - const time = timeMatch ? parseFloat(timeMatch[1]).toFixed(1) : '?'; - const passed = total - failures - errors; - - testSummary = [ - '| Tests | Passed | Failed | Duration |', - '|-------|--------|--------|----------|', - `| ${total} | ${passed} | ${failures + errors} | ${time}s |`, + let statusEmoji, statusText, statusDetail, testSummary; + + if (pullOutcome !== 'success') { + statusEmoji = '⚠️'; + statusText = 'Skipped — Image Not Available'; + statusDetail = [ + 'The PR container image could not be pulled. This typically means the PR build', + 'was skipped (e.g., only non-build files were changed) or the image has expired.', + '', + 'E2E tests were **not executed**.', ].join('\n'); - } catch (e) { - testSummary = '_No detailed results available._'; + testSummary = ''; + } else { + statusEmoji = testOutcome === 'success' ? '✅' : '❌'; + statusText = testOutcome === 'success' ? 'Passed' : 'Failed'; + statusDetail = testOutcome === 'success' + ? 'All end-to-end integration tests passed for the container image.' + : 'End-to-end integration tests failed for the container image. See the workflow run for details.'; + + testSummary = ''; + const fs = require('fs'); + try { + const xml = fs.readFileSync('e2e-results.xml', 'utf8'); + const testsMatch = xml.match(/tests="(\d+)"/); + const failuresMatch = xml.match(/failures="(\d+)"/); + const errorsMatch = xml.match(/errors="(\d+)"/); + const timeMatch = xml.match(/time="([\d.]+)"/); + + const total = testsMatch ? parseInt(testsMatch[1]) : 0; + const failures = failuresMatch ? parseInt(failuresMatch[1]) : 0; + const errors = errorsMatch ? parseInt(errorsMatch[1]) : 0; + const time = timeMatch ? parseFloat(timeMatch[1]).toFixed(1) : '?'; + const passed = total - failures - errors; + + testSummary = [ + '| Tests | Passed | Failed | Duration |', + '|-------|--------|--------|----------|', + `| ${total} | ${passed} | ${failures + errors} | ${time}s |`, + ].join('\n'); + } catch (e) { + testSummary = '_No detailed results available._'; + } } - const statusEmoji = testOutcome === 'success' ? '✅' : '❌'; - const statusText = testOutcome === 'success' ? 'Passed' : 'Failed'; - const statusDetail = testOutcome === 'success' - ? 'All end-to-end integration tests passed for the container image.' - : 'End-to-end integration tests failed for the container image. See the workflow run for details.'; - - const body = [ + const bodyParts = [ `## ${statusEmoji} E2E Tests ${statusText}`, '', statusDetail, '', `**Image:** \`${image}\``, - '', - testSummary, + ]; + + if (testSummary) { + bodyParts.push('', testSummary); + } + + bodyParts.push( '', '### Traceability', '', `- **Publish:** [PR Publish #${publishRunId}](${publishUrl})`, `- **E2E:** [E2E Tests #${e2eRunId}](${e2eUrl})`, - ].join('\n'); + ); + + const body = bodyParts.join('\n'); await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, + ...context.repo, issue_number: prNumber, - body: body + body: body, }); - name: Fail workflow if tests failed - if: steps.e2e-tests.outcome != 'success' + if: always() && steps.e2e-tests.outcome == 'failure' run: | echo "E2E tests failed. Failing workflow." exit 1 @@ -300,8 +335,28 @@ jobs: runs-on: ubuntu-latest if: github.event.workflow_run.conclusion == 'failure' permissions: + actions: read pull-requests: write steps: + - name: Download PR metadata + id: download-metadata + continue-on-error: true + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + with: + name: e2e-pr-metadata + path: /tmp/e2e-pr-metadata + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Read PR metadata + id: pr-info + if: steps.download-metadata.outcome == 'success' + run: | + PR_NUMBER=$(jq -r '.pr_number' /tmp/e2e-pr-metadata/pr-info.json) + if [[ "$PR_NUMBER" =~ ^[0-9]{1,7}$ ]]; then + echo "pr_number=${PR_NUMBER}" >> $GITHUB_OUTPUT + fi + - name: E2E Skipped Summary env: PUBLISH_RUN_ID: ${{ github.event.workflow_run.id }} @@ -313,3 +368,36 @@ jobs: echo "The upstream PR Publish workflow failed. E2E tests were not run." >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "- **PR Publish:** [Run #${PUBLISH_RUN_ID}](${SERVER_URL}/${REPOSITORY}/actions/runs/${PUBLISH_RUN_ID}) (failed)" >> $GITHUB_STEP_SUMMARY + + - name: Comment on PR + if: steps.pr-info.outputs.pr_number != '' + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + env: + PR_NUMBER: ${{ steps.pr-info.outputs.pr_number }} + E2E_RUN_ID: ${{ github.run_id }} + PUBLISH_RUN_ID: ${{ github.event.workflow_run.id }} + with: + script: | + const prNumber = parseInt(process.env.PR_NUMBER); + const e2eRunId = process.env.E2E_RUN_ID; + const publishRunId = process.env.PUBLISH_RUN_ID; + + const e2eUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${e2eRunId}`; + const publishUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${publishRunId}`; + + const body = [ + '## ⏭️ E2E Tests Skipped', + '', + 'The upstream PR Publish workflow failed. E2E tests were **not executed**.', + '', + '### Traceability', + '', + `- **Publish:** [PR Publish #${publishRunId}](${publishUrl}) (failed)`, + `- **E2E:** [E2E Tests #${e2eRunId}](${e2eUrl})`, + ].join('\n'); + + await github.rest.issues.createComment({ + ...context.repo, + issue_number: prNumber, + body: body, + }); From 51188c12446e07c75d663e77cd67449a0f147860 Mon Sep 17 00:00:00 2001 From: Frank Kong Date: Mon, 16 Mar 2026 12:48:35 -0400 Subject: [PATCH 26/35] chore: properly checkout the correct branch for e2e-tests Signed-off-by: Frank Kong --- .github/workflows/e2e-test.yaml | 52 +++++++++++++++++++++++++-------- 1 file changed, 40 insertions(+), 12 deletions(-) diff --git a/.github/workflows/e2e-test.yaml b/.github/workflows/e2e-test.yaml index 1e24d1b..9bc7b36 100644 --- a/.github/workflows/e2e-test.yaml +++ b/.github/workflows/e2e-test.yaml @@ -24,9 +24,13 @@ jobs: if: github.event.workflow_run.conclusion == 'success' permissions: contents: read - pull-requests: write + outputs: + pr_number: ${{ steps.pr-info.outputs.pr_number }} + image: ${{ steps.pr-info.outputs.image }} + short_sha: ${{ steps.pr-info.outputs.short_sha }} + pull_outcome: ${{ steps.pull-image.outcome }} + test_outcome: ${{ steps.e2e-tests.outcome }} steps: - # Download the PR metadata artifact uploaded by PR Publish since we can't get PR info from the github event - name: Download PR metadata uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: @@ -54,6 +58,7 @@ jobs: IMAGE="${REGISTRY}/${REGISTRY_IMAGE}:pr-${PR_NUMBER}" echo "pr_number=${PR_NUMBER}" >> $GITHUB_OUTPUT + echo "commit_sha=${COMMIT_SHA}" >> $GITHUB_OUTPUT echo "image=${IMAGE}" >> $GITHUB_OUTPUT echo "short_sha=${SHORT_SHA}" >> $GITHUB_OUTPUT echo "Resolved PR #${PR_NUMBER}, commit: ${SHORT_SHA}, image: ${IMAGE}" @@ -61,6 +66,7 @@ jobs: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: + ref: ${{ steps.pr-info.outputs.commit_sha }} persist-credentials: false - name: Set up Python @@ -108,16 +114,38 @@ jobs: retention-days: 14 if-no-files-found: ignore + - name: Upload test results + if: always() && steps.e2e-tests.outcome != 'skipped' + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: e2e-test-results + path: e2e-results.xml + retention-days: 1 + if-no-files-found: ignore + + report-results: + runs-on: ubuntu-latest + needs: e2e-test + if: always() && needs.e2e-test.result != 'skipped' + permissions: + pull-requests: write + steps: + - name: Download test results + id: download-results + continue-on-error: true + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + with: + name: e2e-test-results + - name: Generate test summary - if: always() && steps.pr-info.outcome == 'success' env: - IMAGE: ${{ steps.pr-info.outputs.image }} - PR_NUMBER: ${{ steps.pr-info.outputs.pr_number }} + IMAGE: ${{ needs.e2e-test.outputs.image }} + PR_NUMBER: ${{ needs.e2e-test.outputs.pr_number }} PUBLISH_RUN_ID: ${{ github.event.workflow_run.id }} E2E_RUN_ID: ${{ github.run_id }} SERVER_URL: ${{ github.server_url }} REPOSITORY: ${{ github.repository }} - PULL_OUTCOME: ${{ steps.pull-image.outcome }} + PULL_OUTCOME: ${{ needs.e2e-test.outputs.pull_outcome }} run: | python3 << 'PYTHON_SCRIPT' import xml.etree.ElementTree as ET @@ -231,15 +259,15 @@ jobs: PYTHON_SCRIPT - name: Comment on PR - if: always() && steps.pr-info.outcome == 'success' + if: always() uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 env: - PR_NUMBER: ${{ steps.pr-info.outputs.pr_number }} - IMAGE: ${{ steps.pr-info.outputs.image }} + PR_NUMBER: ${{ needs.e2e-test.outputs.pr_number }} + IMAGE: ${{ needs.e2e-test.outputs.image }} E2E_RUN_ID: ${{ github.run_id }} PUBLISH_RUN_ID: ${{ github.event.workflow_run.id }} - TEST_OUTCOME: ${{ steps.e2e-tests.outcome }} - PULL_OUTCOME: ${{ steps.pull-image.outcome }} + TEST_OUTCOME: ${{ needs.e2e-test.outputs.test_outcome }} + PULL_OUTCOME: ${{ needs.e2e-test.outputs.pull_outcome }} with: script: | const prNumber = parseInt(process.env.PR_NUMBER); @@ -325,7 +353,7 @@ jobs: }); - name: Fail workflow if tests failed - if: always() && steps.e2e-tests.outcome == 'failure' + if: always() && needs.e2e-test.outputs.test_outcome == 'failure' run: | echo "E2E tests failed. Failing workflow." exit 1 From e2e8bb21bc15cac9d96632212b6fce6fbfe198c7 Mon Sep 17 00:00:00 2001 From: Frank Kong Date: Mon, 16 Mar 2026 13:30:47 -0400 Subject: [PATCH 27/35] chore: update .gitignore to not ignore fixtures Signed-off-by: Frank Kong --- .gitignore | 6 +++--- tests/e2e/fixtures/todo/config/plugins-list.yaml | 2 ++ .../todo/config/plugins/todo/overlay/scalprum-config.json | 7 +++++++ tests/e2e/fixtures/todo/config/source.json | 1 + 4 files changed, 13 insertions(+), 3 deletions(-) create mode 100644 tests/e2e/fixtures/todo/config/plugins-list.yaml create mode 100644 tests/e2e/fixtures/todo/config/plugins/todo/overlay/scalprum-config.json create mode 100644 tests/e2e/fixtures/todo/config/source.json diff --git a/.gitignore b/.gitignore index e2066f9..b1dc3b7 100644 --- a/.gitignore +++ b/.gitignore @@ -2,9 +2,9 @@ **/__pycache__ # Local Configuration Folders -config/ -source/ -outputs/ +/config/ +/source/ +/outputs/ # VSCode .vscode/ diff --git a/tests/e2e/fixtures/todo/config/plugins-list.yaml b/tests/e2e/fixtures/todo/config/plugins-list.yaml new file mode 100644 index 0000000..373fbe9 --- /dev/null +++ b/tests/e2e/fixtures/todo/config/plugins-list.yaml @@ -0,0 +1,2 @@ +plugins/todo: +plugins/todo-backend: diff --git a/tests/e2e/fixtures/todo/config/plugins/todo/overlay/scalprum-config.json b/tests/e2e/fixtures/todo/config/plugins/todo/overlay/scalprum-config.json new file mode 100644 index 0000000..ec343a0 --- /dev/null +++ b/tests/e2e/fixtures/todo/config/plugins/todo/overlay/scalprum-config.json @@ -0,0 +1,7 @@ +{ + "name": "backstage-community.plugin-todo", + "exposedModules": { + "PluginRoot": "./src/index.ts", + "alpha": "./src/alpha.ts" + } +} diff --git a/tests/e2e/fixtures/todo/config/source.json b/tests/e2e/fixtures/todo/config/source.json new file mode 100644 index 0000000..d1aa07b --- /dev/null +++ b/tests/e2e/fixtures/todo/config/source.json @@ -0,0 +1 @@ +{"repo":"https://github.com/backstage/community-plugins","repo-ref":"3fadf6b0595d1d1804f5c7f2690f4db7b81275f1","workspace-path":"workspaces/todo"} From b7fc64fcaa93f82097b0429d90f3a697df3d772d Mon Sep 17 00:00:00 2001 From: Frank Kong <50030060+Zaperex@users.noreply.github.com> Date: Mon, 16 Mar 2026 15:34:30 -0400 Subject: [PATCH 28/35] chore: fix typo in docs Co-authored-by: Stan Lewis --- examples/example-config-multi-workspace/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/example-config-multi-workspace/README.md b/examples/example-config-multi-workspace/README.md index 4800dc3..dea6f54 100644 --- a/examples/example-config-multi-workspace/README.md +++ b/examples/example-config-multi-workspace/README.md @@ -5,7 +5,7 @@ This directory contains various example plugins to export in multi-workspace mod ## Directory Structure ```bash -example-oconfig-multi-workspace/ +example-config-multi-workspace/ ├── todo/ # Workspace: TODO plugins from community-plugins │ ├── source.json # repo, repo-ref, workspace-path for this workspace │ └── plugins-list.yaml # Plugins to build from the TODO workspace From f9b301e5aa79366fcb8093a870590a5fab6f3141 Mon Sep 17 00:00:00 2001 From: Frank Kong Date: Mon, 16 Mar 2026 17:56:47 -0400 Subject: [PATCH 29/35] chore: update auto-gen logic to create barebones plugins-list.yaml for overlays first Signed-off-by: Frank Kong --- src/rhdh_dynamic_plugin_factory/cli.py | 20 +- src/rhdh_dynamic_plugin_factory/config.py | 109 ++++--- .../plugin_list_config.py | 24 +- tests/test_config_discover_and_build_args.py | 275 ++++++++++++++++++ tests/test_plugin_list_config.py | 53 ++-- 5 files changed, 391 insertions(+), 90 deletions(-) create mode 100644 tests/test_config_discover_and_build_args.py diff --git a/src/rhdh_dynamic_plugin_factory/cli.py b/src/rhdh_dynamic_plugin_factory/cli.py index 2db2084..6f082cf 100644 --- a/src/rhdh_dynamic_plugin_factory/cli.py +++ b/src/rhdh_dynamic_plugin_factory/cli.py @@ -201,8 +201,14 @@ def _process_workspace( repo_path: Path to the repository checkout for this workspace. workspace_path: Relative path from repo_path to the workspace. output_dir: Output directory for build artifacts. - generate_build_args: If True, (re)compute build args for an existing plugins-list.yaml. + generate_build_args: If True, (re)compute build args for a user-provided plugins-list.yaml. """ + was_auto_generated = config.discover_plugins_list( + config_dir=workspace_config_dir, + repo_path=repo_path, + workspace_path=workspace_path, + ) + logger.info("[bold blue]Applying Patches and Overlays[/bold blue]") config.apply_patches_and_overlays( config_dir=workspace_config_dir, @@ -214,12 +220,12 @@ def _process_workspace( full_workspace_path = Path(repo_path).joinpath(workspace_path).absolute() install_dependencies(full_workspace_path) - config.auto_generate_plugins_list( - config_dir=workspace_config_dir, - repo_path=repo_path, - workspace_path=workspace_path, - generate_build_args=generate_build_args, - ) + if was_auto_generated or generate_build_args: + config.populate_plugins_build_args( + config_dir=workspace_config_dir, + repo_path=repo_path, + workspace_path=workspace_path, + ) logger.info("[bold blue]Exporting plugins using RHDH CLI[/bold blue]") config.export_plugins( diff --git a/src/rhdh_dynamic_plugin_factory/config.py b/src/rhdh_dynamic_plugin_factory/config.py index a7c524f..242a17f 100644 --- a/src/rhdh_dynamic_plugin_factory/config.py +++ b/src/rhdh_dynamic_plugin_factory/config.py @@ -249,85 +249,106 @@ def _validate_plugins_list(self) -> None: else: self.logger.debug(f"Using {PLUGIN_LIST_FILE} from: {plugins_file}") - def auto_generate_plugins_list(self, config_dir: Optional[str] = None, - repo_path: Optional[str] = None, - workspace_path: Optional[str] = None, - generate_build_args: bool = False) -> None: - """Auto-generate plugins-list.yaml, or populate build args for an existing one. - - When the file does not exist, a full scan is performed (all plugins - in the workspace are discovered). - - When the file already exists and ``generate_build_args``*`` is ``True``, - build arguments are (re)computed for every plugin listed in the file. + def discover_plugins_list( + self, + config_dir: Optional[str] = None, + repo_path: Optional[str] = None, + workspace_path: Optional[str] = None, + ) -> bool: + """Phase 1: Discover plugins and generate plugins-list.yaml if missing. + + Scans the workspace for ``package.json`` files with a valid + ``backstage.role`` and writes a ``plugins-list.yaml`` containing + only plugin paths (no build arguments). + + This must run **before** overlays are applied, since + ``override-sources.sh`` reads ``plugins-list.yaml`` to decide + which plugin directories receive overlay files. Args: - config_dir: Config directory containing plugins-list.yaml. Defaults to self.config_dir. + config_dir: Config directory for plugins-list.yaml. Defaults to self.config_dir. repo_path: Repository path. Defaults to self.repo_path. workspace_path: Workspace path relative to repo. Defaults to self.workspace_path. - generate_build_args: If True, recompute build args for an existing plugins-list.yaml. + + Returns: + ``True`` if the file was auto-generated, ``False`` if it already existed. Raises: - PluginFactoryError: If auto-generation or build-arg population fails. + PluginFactoryError: If discovery fails. """ config_dir = config_dir or self.config_dir repo_path = repo_path or self.repo_path workspace_path = workspace_path or self.workspace_path - + plugins_file = os.path.join(config_dir, PLUGIN_LIST_FILE) - + if os.path.exists(plugins_file): - if generate_build_args: - self._populate_build_args_for_existing(plugins_file, repo_path, workspace_path) - else: - self.logger.debug(f"[green]{PLUGIN_LIST_FILE} already exists at {plugins_file}. Skipping auto-generation.[/green]") - return - - self.logger.info(f"[bold blue]Auto-generating {PLUGIN_LIST_FILE}[/bold blue]") - + self.logger.debug( + f"[green]{PLUGIN_LIST_FILE} already exists at {plugins_file}. " + f"Skipping discovery.[/green]" + ) + return False + if not os.path.exists(repo_path): raise PluginFactoryError(f"Source code repository does not exist at {repo_path}") workspace_full_path = os.path.abspath(os.path.join(repo_path, workspace_path)) + self.logger.info(f"[bold blue]Discovering plugins in workspace: {workspace_full_path}[/bold blue]") + if not os.path.exists(workspace_full_path): raise PluginFactoryError(f"Plugin workspace does not exist at {workspace_full_path}") - + try: plugin_cfg = PluginListConfig.create_default(workspace_path=Path(workspace_full_path)) plugin_cfg.to_file(Path(plugins_file)) - + plugins: Dict[str, str] = plugin_cfg.get_plugins() if plugins: self.logger.info(f"Generated {PLUGIN_LIST_FILE} with {len(plugins)} plugin(s)") - for plugin_path, build_args in plugins.items(): - if build_args: - self.logger.info(f" - {plugin_path}: {build_args}") - else: - self.logger.info(f" - {plugin_path}") + for plugin_path in plugins: + self.logger.info(f" - {plugin_path}") else: self.logger.warning("No plugins found in workspace") except PluginFactoryError: raise except Exception as e: - raise PluginFactoryError(f"Failed to auto-generate plugins list: {e}") from e + raise PluginFactoryError(f"Failed to discover plugins: {e}") from e + + return True - def _populate_build_args_for_existing( - self, plugins_file: str, repo_path: str, workspace_path: str, + def populate_plugins_build_args( + self, + config_dir: Optional[str] = None, + repo_path: Optional[str] = None, + workspace_path: Optional[str] = None, ) -> None: - """Load an existing plugins-list.yaml, recompute build args, and write it back. + """Compute build arguments for an existing plugins-list.yaml. + + Loads ``plugins-list.yaml``, runs dependency analysis via + :meth:`PluginListConfig.populate_build_args`, and writes the + updated file back. This requires ``node_modules`` to be installed. Args: - plugins_file: Absolute path to the plugins-list.yaml file. - repo_path: Repository root path. - workspace_path: Workspace path relative to repo_path. + config_dir: Config directory containing plugins-list.yaml. Defaults to self.config_dir. + repo_path: Repository path. Defaults to self.repo_path. + workspace_path: Workspace path relative to repo. Defaults to self.workspace_path. Raises: - PluginFactoryError: If the workspace cannot be found or population fails. + PluginFactoryError: If the file is missing, workspace not found, or computation fails. """ - self.logger.warning( - f"[yellow]--generate-build-args: Modifying existing {PLUGIN_LIST_FILE} " - f"to (re)compute build arguments. Your file will be overwritten.[/yellow]" - ) + config_dir = config_dir or self.config_dir + repo_path = repo_path or self.repo_path + workspace_path = workspace_path or self.workspace_path + + plugins_file = os.path.join(config_dir, PLUGIN_LIST_FILE) + + if not os.path.exists(plugins_file): + raise PluginFactoryError( + f"{PLUGIN_LIST_FILE} not found at {plugins_file}. " + f"Cannot compute build arguments without a plugin list." + ) + + self.logger.info(f"[bold blue]Computing build arguments for {plugins_file}[/bold blue]") workspace_full_path = os.path.abspath(os.path.join(repo_path, workspace_path)) if not os.path.exists(workspace_full_path): @@ -341,7 +362,7 @@ def _populate_build_args_for_existing( raise except Exception as e: raise PluginFactoryError( - f"Failed to populate build args for {PLUGIN_LIST_FILE}: {e}" + f"Failed to compute build args for {plugins_file}: {e}" ) from e def discover_source_config(self) -> Optional["SourceConfig"]: diff --git a/src/rhdh_dynamic_plugin_factory/plugin_list_config.py b/src/rhdh_dynamic_plugin_factory/plugin_list_config.py index 8bcb2ff..9e2e95e 100644 --- a/src/rhdh_dynamic_plugin_factory/plugin_list_config.py +++ b/src/rhdh_dynamic_plugin_factory/plugin_list_config.py @@ -74,8 +74,8 @@ def remove_plugin(self, plugin_path: str) -> None: def populate_build_args(self, workspace_path: Path) -> "PluginListConfig": """(Re)compute build arguments for every plugin in the list. - Uses the same dependency analysis as :meth:`create_default` but only - for the plugins already present in ``self.plugins``. All existing + Performs dependency analysis against the bundled RHDH host lockfile + for all plugins already present in ``self.plugins``. All existing build args are overwritten with freshly computed values. A before/after diff is logged for each plugin whose args changed so @@ -180,40 +180,34 @@ def _compute_plugin_build_args( @classmethod def create_default(cls, workspace_path: Path) -> "PluginListConfig": - """Create a default plugin list by scanning workspace for Backstage plugins. + """Discover Backstage plugins in the workspace by scanning for package.json files. Recursively walks *workspace_path* to find ``package.json`` files whose ``backstage.role`` matches one of :pyattr:`VALID_BACKSTAGE_PLUGIN_ROLES`. - For backend plugins, dependency analysis is performed against the - bundled RHDH host lockfile to determine ``--embed-package`` and - ``--shared-package`` arguments. + Only plugin paths are recorded; build arguments are left empty. + Use :meth:`populate_build_args` after dependency installation to + compute build arguments. Args: workspace_path: Absolute path to the workspace root. Returns: - A :class:`PluginListConfig` with discovered plugins and build arg(s) (if any). + A :class:`PluginListConfig` with discovered plugin paths (build args empty). """ plugins: Dict[str, str] = {} - host_packages = cls._get_host_packages() - # Corner case: if the workspace root has a valid backstage role, add it as a plugin root_pkg_json = workspace_path / constants.PKG_JSON if root_pkg_json.is_file(): role = cls._read_backstage_role(root_pkg_json) if role and role in constants.VALID_BACKSTAGE_PLUGIN_ROLES: - plugins["."] = cls._compute_plugin_build_args( - workspace_path, ".", root_pkg_json, host_packages, - ) + plugins["."] = "" for pkg_json_path in cls._find_package_jsons(workspace_path): role = cls._read_backstage_role(pkg_json_path) if role and role in constants.VALID_BACKSTAGE_PLUGIN_ROLES: plugin_dir = pkg_json_path.parent.relative_to(workspace_path).as_posix() - plugins[plugin_dir] = cls._compute_plugin_build_args( - workspace_path, plugin_dir, pkg_json_path, host_packages, - ) + plugins[plugin_dir] = "" sorted_plugins = dict[str, str](sorted(plugins.items())) cls.logger.debug(f"Discovered {len(sorted_plugins)} plugin(s) in {workspace_path}") diff --git a/tests/test_config_discover_and_build_args.py b/tests/test_config_discover_and_build_args.py new file mode 100644 index 0000000..9aed371 --- /dev/null +++ b/tests/test_config_discover_and_build_args.py @@ -0,0 +1,275 @@ +""" +Unit tests for PluginFactoryConfig.discover_plugins_list and +PluginFactoryConfig.populate_plugins_build_args methods. + +Tests the two-phase plugins-list.yaml generation: + Phase 1 (discover_plugins_list): scan workspace, write paths only + Phase 2 (populate_plugins_build_args): compute build args for existing file +""" + +import json +import os +from pathlib import Path + +import pytest +import yaml + +from src.rhdh_dynamic_plugin_factory.config import PluginFactoryConfig +from src.rhdh_dynamic_plugin_factory.constants import PLUGIN_LIST_FILE +from src.rhdh_dynamic_plugin_factory.exceptions import PluginFactoryError +from src.rhdh_dynamic_plugin_factory import constants + + +def _make_plugin_dir(base, rel_path, name, role, dependencies=None): + """Helper to create a plugin directory with a package.json.""" + plugin_dir = base / rel_path + plugin_dir.mkdir(parents=True, exist_ok=True) + pkg = {"name": name, "version": "1.0.0", "backstage": {"role": role}} + if dependencies: + pkg["dependencies"] = dependencies + (plugin_dir / "package.json").write_text(json.dumps(pkg)) + return plugin_dir + + +def _make_node_module(base, dep_name, dependencies=None): + """Helper to create a workspace-root node_modules//package.json entry.""" + nm_dir = base / "node_modules" / dep_name + nm_dir.mkdir(parents=True, exist_ok=True) + pkg = {"name": dep_name, "version": "1.0.0"} + if dependencies: + pkg["dependencies"] = dependencies + (nm_dir / "package.json").write_text(json.dumps(pkg)) + + +class TestDiscoverPluginsList: + """Tests for PluginFactoryConfig.discover_plugins_list (Phase 1).""" + + def test_generates_file_when_missing(self, make_config, setup_test_env): + """When plugins-list.yaml is missing, discover plugins and write the file.""" + config = make_config() + config_dir = setup_test_env["config_dir"] + + plugins_file = Path(config_dir) / PLUGIN_LIST_FILE + plugins_file.unlink() + assert not plugins_file.exists() + + workspace = Path(config.repo_path) + _make_plugin_dir(workspace, "plugins/todo", "@test/plugin-todo", "frontend-plugin") + _make_plugin_dir(workspace, "plugins/todo-backend", "@test/plugin-todo-backend", "backend-plugin") + + result = config.discover_plugins_list() + + assert result is True + assert plugins_file.exists() + + data = yaml.safe_load(plugins_file.read_text()) + assert "plugins/todo" in data + assert "plugins/todo-backend" in data + for val in data.values(): + assert val is None + + def test_skips_when_file_exists(self, make_config, setup_test_env): + """When plugins-list.yaml already exists, do nothing and return False.""" + config = make_config() + config_dir = setup_test_env["config_dir"] + + plugins_file = Path(config_dir) / PLUGIN_LIST_FILE + assert plugins_file.exists() + + original_content = plugins_file.read_text() + result = config.discover_plugins_list() + + assert result is False + assert plugins_file.read_text() == original_content + + def test_raises_when_repo_path_missing(self, make_config, setup_test_env): + """Raises PluginFactoryError when repo_path does not exist.""" + config = make_config(repo_path="/nonexistent/path") + config_dir = setup_test_env["config_dir"] + + plugins_file = Path(config_dir) / PLUGIN_LIST_FILE + plugins_file.unlink() + + with pytest.raises(PluginFactoryError, match="Source code repository does not exist"): + config.discover_plugins_list() + + def test_raises_when_workspace_missing(self, make_config, setup_test_env): + """Raises PluginFactoryError when workspace does not exist under repo_path.""" + config = make_config(workspace_path="nonexistent/workspace") + config_dir = setup_test_env["config_dir"] + + plugins_file = Path(config_dir) / PLUGIN_LIST_FILE + plugins_file.unlink() + + with pytest.raises(PluginFactoryError, match="Plugin workspace does not exist"): + config.discover_plugins_list() + + def test_empty_workspace_still_writes_file(self, make_config, setup_test_env): + """An empty workspace writes a plugins-list.yaml with no entries.""" + config = make_config() + config_dir = setup_test_env["config_dir"] + + plugins_file = Path(config_dir) / PLUGIN_LIST_FILE + plugins_file.unlink() + + result = config.discover_plugins_list() + + assert result is True + assert plugins_file.exists() + assert plugins_file.read_text() == "" + + def test_build_args_are_always_empty(self, make_config, setup_test_env): + """Phase 1 never produces build args, even for backend plugins.""" + config = make_config() + config_dir = setup_test_env["config_dir"] + + plugins_file = Path(config_dir) / PLUGIN_LIST_FILE + plugins_file.unlink() + + workspace = Path(config.repo_path) + _make_plugin_dir( + workspace, "plugins/backend", "@test/my-backend", "backend-plugin", + dependencies={"@backstage/new-experimental": "^0.1.0"}, + ) + + config.discover_plugins_list() + + content = plugins_file.read_text() + assert "--embed-package" not in content + assert "--shared-package" not in content + + def test_uses_explicit_config_dir_param(self, make_config, tmp_path): + """When config_dir is passed explicitly, uses that instead of self.config_dir.""" + config = make_config() + + alt_config_dir = tmp_path / "alt-config" + alt_config_dir.mkdir() + + workspace = Path(config.repo_path) + _make_plugin_dir(workspace, "plugins/todo", "@test/plugin-todo", "frontend-plugin") + + result = config.discover_plugins_list(config_dir=str(alt_config_dir)) + + assert result is True + assert (alt_config_dir / PLUGIN_LIST_FILE).exists() + + +class TestPopulatePluginsBuildArgs: + """Tests for PluginFactoryConfig.populate_plugins_build_args (Phase 2).""" + + def test_computes_build_args_for_existing_file(self, make_config, setup_test_env, monkeypatch): + """Loads plugins-list.yaml, computes build args, writes back.""" + lockfile = Path(setup_test_env["tmp_path"]) / "host-yarn.lock" + lockfile.write_text('"@backstage/core@npm:^1.0.0":\n version: 1.0.0\n') + monkeypatch.setattr(constants, "HOST_LOCKFILE", lockfile) + + config = make_config() + config_dir = setup_test_env["config_dir"] + + workspace = Path(config.repo_path) + _make_plugin_dir( + workspace, "plugins/backend", "@test/my-backend", "backend-plugin", + dependencies={"@backstage/new-experimental": "^0.1.0"}, + ) + _make_plugin_dir( + workspace, "plugins/frontend", "@test/my-frontend", "frontend-plugin", + ) + + plugins_file = Path(config_dir) / PLUGIN_LIST_FILE + plugins_file.write_text("plugins/backend:\nplugins/frontend:\n") + + config.populate_plugins_build_args() + + updated = yaml.safe_load(plugins_file.read_text()) + assert "--embed-package @backstage/new-experimental" in str(updated.get("plugins/backend", "")) + assert updated.get("plugins/frontend") is None + + def test_raises_when_file_missing(self, make_config, setup_test_env): + """Raises PluginFactoryError when plugins-list.yaml does not exist.""" + config = make_config() + config_dir = setup_test_env["config_dir"] + + plugins_file = Path(config_dir) / PLUGIN_LIST_FILE + plugins_file.unlink() + + with pytest.raises(PluginFactoryError, match="not found"): + config.populate_plugins_build_args() + + def test_raises_when_workspace_missing(self, make_config, setup_test_env): + """Raises PluginFactoryError when workspace does not exist.""" + config = make_config(workspace_path="nonexistent/workspace") + + with pytest.raises(PluginFactoryError, match="Plugin workspace does not exist"): + config.populate_plugins_build_args() + + def test_uses_explicit_params(self, make_config, tmp_path, monkeypatch): + """When params are passed explicitly, uses those instead of self defaults.""" + lockfile = tmp_path / "host-yarn.lock" + lockfile.write_text("") + monkeypatch.setattr(constants, "HOST_LOCKFILE", lockfile) + + config = make_config() + + alt_config_dir = tmp_path / "alt-config" + alt_config_dir.mkdir() + + workspace = Path(config.repo_path) + _make_plugin_dir(workspace, "plugins/todo", "@test/plugin-todo", "frontend-plugin") + + plugins_file = alt_config_dir / PLUGIN_LIST_FILE + plugins_file.write_text("plugins/todo:\n") + + config.populate_plugins_build_args(config_dir=str(alt_config_dir)) + + updated = yaml.safe_load(plugins_file.read_text()) + assert "plugins/todo" in updated + + +class TestTwoPhaseFlow: + """Integration tests verifying the Phase 1 -> Phase 2 flow works end-to-end.""" + + def test_discover_then_populate(self, make_config, setup_test_env, monkeypatch): + """Phase 1 discovers plugins, Phase 2 computes build args.""" + lockfile = Path(setup_test_env["tmp_path"]) / "host-yarn.lock" + lockfile.write_text('"@backstage/core@npm:^1.0.0":\n version: 1.0.0\n') + monkeypatch.setattr(constants, "HOST_LOCKFILE", lockfile) + + config = make_config() + config_dir = setup_test_env["config_dir"] + + plugins_file = Path(config_dir) / PLUGIN_LIST_FILE + plugins_file.unlink() + + workspace = Path(config.repo_path) + _make_plugin_dir( + workspace, "plugins/todo", "@test/plugin-todo", "frontend-plugin", + ) + _make_plugin_dir( + workspace, "plugins/todo-backend", "@test/plugin-todo-backend", "backend-plugin", + dependencies={"@backstage/new-experimental": "^0.1.0"}, + ) + + was_generated = config.discover_plugins_list() + assert was_generated is True + + after_phase1 = plugins_file.read_text() + assert "--embed-package" not in after_phase1 + + config.populate_plugins_build_args() + + updated = yaml.safe_load(plugins_file.read_text()) + assert updated.get("plugins/todo") is None + assert "--embed-package @backstage/new-experimental" in str(updated.get("plugins/todo-backend", "")) + + def test_user_provided_file_not_overwritten_by_phase1(self, make_config, setup_test_env): + """Phase 1 does not touch a user-provided plugins-list.yaml.""" + config = make_config() + config_dir = setup_test_env["config_dir"] + + plugins_file = Path(config_dir) / PLUGIN_LIST_FILE + user_content = "plugins/custom-plugin: --user-args\n" + plugins_file.write_text(user_content) + + was_generated = config.discover_plugins_list() + assert was_generated is False + assert plugins_file.read_text() == user_content diff --git a/tests/test_plugin_list_config.py b/tests/test_plugin_list_config.py index fec779c..7af91d5 100644 --- a/tests/test_plugin_list_config.py +++ b/tests/test_plugin_list_config.py @@ -968,8 +968,8 @@ def test_no_deep_backstage_no_embed(self, tmp_path): assert result == "" -class TestCreateDefaultWithEmbedArgs: - """End-to-end tests for create_default with Phase 2 embed/unshare detection.""" +class TestDiscoverThenPopulateBuildArgs: + """End-to-end tests for the two-phase flow: create_default (discovery) + populate_build_args.""" def test_aws_ecs_like_workspace(self, tmp_path, monkeypatch): """Backend plugin with third-party dep that has @backstage/* sub-deps.""" @@ -1001,44 +1001,40 @@ def test_aws_ecs_like_workspace(self, tmp_path, monkeypatch): ) config = PluginListConfig.create_default(workspace) + assert config.get_plugins()["plugins/ecs/frontend"] == "" + assert config.get_plugins()["plugins/ecs/backend"] == "" + + config.populate_build_args(workspace) plugins = config.get_plugins() assert plugins["plugins/ecs/frontend"] == "" assert "--embed-package @aws/aws-core-plugin-for-backstage-common" in plugins["plugins/ecs/backend"] assert "--shared-package" not in plugins["plugins/ecs/backend"] - def test_frontend_plugins_get_no_args(self, tmp_path, monkeypatch): + def test_create_default_never_computes_build_args(self, tmp_path, monkeypatch): + """create_default returns all plugins with empty build args.""" lockfile = tmp_path / "host-yarn.lock" - lockfile.write_text("") + lockfile.write_text('"@backstage/core@npm:^1.0.0":\n version: 1.0.0\n') monkeypatch.setattr(constants, "HOST_LOCKFILE", lockfile) workspace = tmp_path / "workspace" workspace.mkdir() _make_plugin_dir( - workspace, "plugins/todo", - "@backstage-community/plugin-todo", "frontend-plugin", + workspace, "plugins/my-backend", + "@test/my-backend", "backend-plugin", + dependencies={"@backstage/new-experimental": "^0.1.0"}, ) - - config = PluginListConfig.create_default(workspace) - assert config.get_plugins()["plugins/todo"] == "" - - def test_backend_no_deps_no_args(self, tmp_path, monkeypatch): - lockfile = tmp_path / "host-yarn.lock" - lockfile.write_text('"@backstage/core@npm:^1.0.0":\n version: 1.0.0\n') - monkeypatch.setattr(constants, "HOST_LOCKFILE", lockfile) - - workspace = tmp_path / "workspace" - workspace.mkdir() _make_plugin_dir( - workspace, "plugins/todo-backend", - "@backstage-community/plugin-todo-backend", "backend-plugin", + workspace, "plugins/todo", + "@backstage-community/plugin-todo", "frontend-plugin", ) config = PluginListConfig.create_default(workspace) - assert config.get_plugins()["plugins/todo-backend"] == "" + for args in config.get_plugins().values(): + assert args == "" - def test_backstage_dep_not_in_host_unshared_and_embedded(self, tmp_path, monkeypatch): - """@backstage/* direct dep missing from host gets both embed and unshare.""" + def test_populate_adds_args_after_discovery(self, tmp_path, monkeypatch): + """populate_build_args computes args for plugins discovered by create_default.""" lockfile = tmp_path / "host-yarn.lock" lockfile.write_text('"@backstage/core@npm:^1.0.0":\n version: 1.0.0\n') monkeypatch.setattr(constants, "HOST_LOCKFILE", lockfile) @@ -1052,6 +1048,9 @@ def test_backstage_dep_not_in_host_unshared_and_embedded(self, tmp_path, monkeyp ) config = PluginListConfig.create_default(workspace) + assert config.get_plugins()["plugins/my-backend"] == "" + + config.populate_build_args(workspace) args = config.get_plugins()["plugins/my-backend"] assert "--embed-package @backstage/new-experimental" in args assert "--shared-package !@backstage/new-experimental" in args @@ -1071,6 +1070,9 @@ def test_missing_host_lockfile_still_works(self, tmp_path, monkeypatch): ) config = PluginListConfig.create_default(workspace) + assert config.get_plugins()["plugins/backend"] == "" + + config.populate_build_args(workspace) args = config.get_plugins()["plugins/backend"] assert "--embed-package @backstage/catalog-model" in args assert "--shared-package !@backstage/catalog-model" in args @@ -1485,8 +1487,8 @@ def test_sibling_not_in_node_modules_skipped(self, tmp_path): assert result == "" -class TestCreateDefaultWithNativeHandling: - """End-to-end tests for create_default with native module handling.""" +class TestDiscoverThenPopulateWithNativeHandling: + """End-to-end tests for two-phase flow with native module handling.""" def test_native_not_in_host_suppressed(self, tmp_path, monkeypatch): lockfile = tmp_path / "host-yarn.lock" @@ -1510,6 +1512,9 @@ def test_native_not_in_host_suppressed(self, tmp_path, monkeypatch): ) config = PluginListConfig.create_default(workspace) + assert config.get_plugins()["plugins/backend"] == "" + + config.populate_build_args(workspace) args = config.get_plugins()["plugins/backend"] assert "--suppress-native-package cpu-features" in args From 16bbb94afa2e24ac156982e27a452d085abe1269 Mon Sep 17 00:00:00 2001 From: Frank Kong Date: Mon, 16 Mar 2026 18:42:00 -0400 Subject: [PATCH 30/35] chore: add warning when using --env-file podman arg Signed-off-by: Frank Kong --- README.md | 2 ++ default.env | 12 ++++++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 1318554..f3a7d8a 100644 --- a/README.md +++ b/README.md @@ -298,6 +298,8 @@ podman run --rm -it \ This approach keeps your credentials separate from the config directory and can be useful for CI/CD pipelines or when you want to reuse the same environment file across different configurations. +WARNING: `--env-file` will NOT strip quotations from the environmental variables. This means `REGISTRY_URL="quay.io"` will literally resolve to `"quay.io"` instead of `quay.io` which will cause issues with image publishing. + ### Plugin List Auto-Generation If no `plugins-list.yaml` file is provided for a workspace, the factory will scan the **entire** workspace, discover **all** frontend/backend plugins, compute build arguments for backend plugins and generate a `plugins-list.yaml` with the required build arguments. diff --git a/default.env b/default.env index b37e208..8673169 100644 --- a/default.env +++ b/default.env @@ -2,12 +2,12 @@ # These values can be overridden by additional .env files or environment variables # Tooling versions -RHDH_CLI_VERSION="1.10.0" +RHDH_CLI_VERSION=1.10.0 # Registry configuration (required for push operations) # Set these in your .env file or as environment variables -# REGISTRY_URL="quay.io" -# REGISTRY_USERNAME="your-username" -# REGISTRY_PASSWORD="your-password" -# REGISTRY_NAMESPACE="your-namespace" -# REGISTRY_INSECURE="false" \ No newline at end of file +# REGISTRY_URL=quay.io +# REGISTRY_USERNAME=your-username +# REGISTRY_PASSWORD=your-password +# REGISTRY_NAMESPACE=your-namespace +# REGISTRY_INSECURE=false \ No newline at end of file From b43c1e507daef6eb0b9e0bdb223a30b744327590 Mon Sep 17 00:00:00 2001 From: Frank Kong Date: Wed, 18 Mar 2026 12:23:26 -0400 Subject: [PATCH 31/35] chore: update to use latest rhdh-cli by default Signed-off-by: Frank Kong --- README.md | 4 ++-- default.env | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index f3a7d8a..3acf6de 100644 --- a/README.md +++ b/README.md @@ -226,11 +226,11 @@ Note: `source/` in this case refers to the default source code location if not p #### 1. `default.env` (Provided) -This file contains required version settings and defaults for RHDH CLI: +The version of the `rhdh-cli` being used can be set via the `RHDH_CLI_VERSION`, and is set to `latest` by default. Override it in your `.env` file to change the version if you need to use an older cli. ```bash # Tooling versions -RHDH_CLI_VERSION="1.10.0" +RHDH_CLI_VERSION="latest" ``` #### 2. `config/source.json` (Required for remote repositories, unless using `--source-repo`) diff --git a/default.env b/default.env index 8673169..3a5d7ce 100644 --- a/default.env +++ b/default.env @@ -2,7 +2,7 @@ # These values can be overridden by additional .env files or environment variables # Tooling versions -RHDH_CLI_VERSION=1.10.0 +RHDH_CLI_VERSION=latest # Registry configuration (required for push operations) # Set these in your .env file or as environment variables From 15cb1850ff3919f2685db6487c54620a26430a85 Mon Sep 17 00:00:00 2001 From: Frank Kong Date: Mon, 23 Mar 2026 11:17:16 -0400 Subject: [PATCH 32/35] chore: surface native dep yarn build logs Signed-off-by: Frank Kong --- Dockerfile | 2 + src/rhdh_dynamic_plugin_factory/cli.py | 7 +- src/rhdh_dynamic_plugin_factory/utils.py | 44 ++++++++++ tests/test_utils.py | 105 +++++++++++++++++++++++ 4 files changed, 156 insertions(+), 2 deletions(-) create mode 100644 tests/test_utils.py diff --git a/Dockerfile b/Dockerfile index 0a0a9be..123e007 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,6 +22,8 @@ RUN microdnf install -y --nodocs \ brotli-devel openssl-devel buildah bash patch jq fuse-overlayfs \ && microdnf clean all +ENV PYTHON=/usr/bin/python3.12 + COPY requirements.txt . # Install Python Dependencies diff --git a/src/rhdh_dynamic_plugin_factory/cli.py b/src/rhdh_dynamic_plugin_factory/cli.py index 6f082cf..2a8a3b5 100644 --- a/src/rhdh_dynamic_plugin_factory/cli.py +++ b/src/rhdh_dynamic_plugin_factory/cli.py @@ -14,7 +14,7 @@ from .logger import setup_logging, get_logger from .config import PluginFactoryConfig from .source_config import WorkspaceInfo, discover_workspaces, clone_workspaces_with_worktrees - from .utils import run_command_with_streaming, prompt_or_clean_directory + from .utils import run_command_with_streaming, prompt_or_clean_directory, collect_build_logs from .exceptions import PluginFactoryError, ConfigurationError, ExecutionError except ImportError: # For direct script execution, add parent directory to path @@ -23,7 +23,7 @@ from rhdh_dynamic_plugin_factory.logger import setup_logging, get_logger from rhdh_dynamic_plugin_factory.config import PluginFactoryConfig from rhdh_dynamic_plugin_factory.source_config import WorkspaceInfo, discover_workspaces, clone_workspaces_with_worktrees - from rhdh_dynamic_plugin_factory.utils import run_command_with_streaming, prompt_or_clean_directory + from rhdh_dynamic_plugin_factory.utils import run_command_with_streaming, prompt_or_clean_directory, collect_build_logs from rhdh_dynamic_plugin_factory.exceptions import PluginFactoryError, ConfigurationError, ExecutionError logger = get_logger("cli") @@ -169,6 +169,9 @@ def install_dependencies(workspace_path: Path) -> None: env=env ) + if cmd[:2] == ["yarn", "install"]: + collect_build_logs(logger) + if returncode != 0: raise ExecutionError( f"{description} failed with exit code {returncode}", diff --git a/src/rhdh_dynamic_plugin_factory/utils.py b/src/rhdh_dynamic_plugin_factory/utils.py index 8465811..baaf363 100644 --- a/src/rhdh_dynamic_plugin_factory/utils.py +++ b/src/rhdh_dynamic_plugin_factory/utils.py @@ -4,6 +4,7 @@ import shutil import subprocess +import tempfile import threading from pathlib import Path from typing import Optional, Callable @@ -85,6 +86,49 @@ def run_command_with_streaming( return process.returncode +def collect_build_logs(logger, tmp_dir: Optional[Path] = None) -> None: + """Find and display build log files left by failed native package builds. + + Scans a temp directory for build.log files (typically created by yarn when + native dependencies fail to compile) and logs their full contents. + + Args: + logger: Logger instance to use for output. + tmp_dir: Directory to scan for build.log files. Defaults to the + system temp directory (usually /tmp). + """ + search_dir = tmp_dir or Path(tempfile.gettempdir()) + + try: + build_logs = sorted(search_dir.rglob("build.log")) + except OSError as e: + logger.warning(f"[yellow]Could not find build logs in {search_dir}: {e}[/yellow]") + return + + if not build_logs: + logger.warning(f"[yellow]No build logs found in {search_dir}[/yellow]") + return + + logger.warning( + f"[yellow]Found {len(build_logs)} build log(s) that may contain details about the failure:[/yellow]" + ) + + for log_path in build_logs: + try: + contents = log_path.read_text().strip() + except OSError: + logger.warning(f"[yellow]Could not read build log: {log_path}[/yellow]") + continue + + if not contents: + logger.warning(f"[yellow]Empty build log: {log_path}[/yellow]") + continue + + logger.warning(f"[yellow]Build log: {log_path}[/yellow]") + for line in contents.splitlines(): + logger.warning(f" {line}") + + def display_export_results(workspace_path: Path, logger) -> bool: """Display results from export script output files. diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..00f5888 --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,105 @@ +""" +Unit tests for utility functions. +""" + +from src.rhdh_dynamic_plugin_factory.utils import collect_build_logs + + +class TestCollectBuildLogs: + """Tests for collect_build_logs function.""" + + def test_displays_build_log_contents(self, tmp_path, mock_logger): + """Test that build log contents are logged with the full file path.""" + log_dir = tmp_path / "xfs-abc123" + log_dir.mkdir() + log_file = log_dir / "build.log" + log_file.write_text("gyp ERR! build error\ngyp ERR! not ok") + + collect_build_logs(mock_logger, tmp_dir=tmp_path) + + mock_logger.warning.assert_any_call( + f"[yellow]Build log: {log_file}[/yellow]" + ) + mock_logger.warning.assert_any_call(" gyp ERR! build error") + mock_logger.warning.assert_any_call(" gyp ERR! not ok") + + def test_displays_multiple_build_logs(self, tmp_path, mock_logger): + """Test that all build.log files are found and displayed.""" + for name in ("xfs-aaa", "xfs-bbb"): + d = tmp_path / name + d.mkdir() + (d / "build.log").write_text(f"error in {name}") + + collect_build_logs(mock_logger, tmp_dir=tmp_path) + + logged_lines = [call[0][0] for call in mock_logger.warning.call_args_list] + assert any("xfs-aaa" in line and "Build log:" in line for line in logged_lines) + assert any("xfs-bbb" in line and "Build log:" in line for line in logged_lines) + + def test_no_build_logs_found(self, tmp_path, mock_logger): + """Test that a 'no build logs found' message is logged when none exist.""" + collect_build_logs(mock_logger, tmp_dir=tmp_path) + + mock_logger.warning.assert_called_once() + assert "No build logs found" in mock_logger.warning.call_args[0][0] + + def test_warns_on_empty_build_logs(self, tmp_path, mock_logger): + """Test that empty build.log files produce a warning.""" + log_dir = tmp_path / "xfs-empty" + log_dir.mkdir() + log_file = log_dir / "build.log" + log_file.write_text("") + + collect_build_logs(mock_logger, tmp_dir=tmp_path) + + mock_logger.warning.assert_any_call( + f"[yellow]Empty build log: {log_file}[/yellow]" + ) + + def test_handles_unreadable_build_log(self, tmp_path, mock_logger): + """Test that unreadable files are reported but don't cause a crash.""" + log_dir = tmp_path / "xfs-noperm" + log_dir.mkdir() + log_file = log_dir / "build.log" + log_file.write_text("some content") + log_file.chmod(0o000) + + try: + collect_build_logs(mock_logger, tmp_dir=tmp_path) + + logged_lines = [call[0][0] for call in mock_logger.warning.call_args_list] + assert any("Could not read build log" in line for line in logged_lines) + finally: + log_file.chmod(0o644) + + def test_finds_nested_build_logs(self, tmp_path, mock_logger): + """Test that build.log files in deeply nested directories are found.""" + nested = tmp_path / "a" / "b" / "c" + nested.mkdir(parents=True) + log_file = nested / "build.log" + log_file.write_text("nested error") + + collect_build_logs(mock_logger, tmp_dir=tmp_path) + + mock_logger.warning.assert_any_call( + f"[yellow]Build log: {log_file}[/yellow]" + ) + mock_logger.warning.assert_any_call(" nested error") + + def test_defaults_to_system_tmp(self, mock_logger): + """Test that tmp_dir defaults to the system temp directory.""" + collect_build_logs(mock_logger) + # Should not raise; may or may not find logs depending on system state + + def test_summary_message_includes_count(self, tmp_path, mock_logger): + """Test that the summary message shows the correct log count.""" + for i in range(3): + d = tmp_path / f"dir-{i}" + d.mkdir() + (d / "build.log").write_text(f"error {i}") + + collect_build_logs(mock_logger, tmp_dir=tmp_path) + + mock_logger.warning.assert_any_call( + "[yellow]Found 3 build log(s) that may contain details about the failure:[/yellow]" + ) From f99fe07cb135c0ff503523c1c12b126ccfe168ba Mon Sep 17 00:00:00 2001 From: Frank Kong Date: Wed, 1 Apr 2026 14:47:06 -0400 Subject: [PATCH 33/35] chore: fix typos Signed-off-by: Frank Kong --- examples/example-config-multi-workspace/README.md | 4 ++-- src/rhdh_dynamic_plugin_factory/cli.py | 2 +- src/rhdh_dynamic_plugin_factory/source_config.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/example-config-multi-workspace/README.md b/examples/example-config-multi-workspace/README.md index dea6f54..91b300b 100644 --- a/examples/example-config-multi-workspace/README.md +++ b/examples/example-config-multi-workspace/README.md @@ -30,7 +30,7 @@ Each subdirectory containing a `source.json` file is treated as an independent w ```bash podman run --rm -it \ --device /dev/fuse \ - -v ./examples/example-multi-workspace:/config:z \ + -v ./examples/example-config-multi-workspace:/config:z \ -v ./source:/source:z \ -v ./outputs:/outputs:z \ quay.io/rhdh-community/dynamic-plugins-factory:latest @@ -42,7 +42,7 @@ Note: `--workspace-path`, `--source-repo`, and `--source-ref` cannot be used. In ```bash python src/rhdh_dynamic_plugin_factory \ - --config-dir ./examples/example-multi-workspace \ + --config-dir ./examples/example-config-multi-workspace \ --repo-path ./source \ --output-dir ./outputs ``` diff --git a/src/rhdh_dynamic_plugin_factory/cli.py b/src/rhdh_dynamic_plugin_factory/cli.py index 2a8a3b5..3fd35b8 100644 --- a/src/rhdh_dynamic_plugin_factory/cli.py +++ b/src/rhdh_dynamic_plugin_factory/cli.py @@ -126,7 +126,7 @@ def create_parser() -> argparse.ArgumentParser: "--clean", action="store_true", default=False, - help="Clean the source directory before cloning source repository. WARNING: This will all the contents of the source directory." + help="Clean the source directory before cloning source repository. WARNING: This will remove all the contents of the source directory." ) parser.add_argument( "--generate-build-args", diff --git a/src/rhdh_dynamic_plugin_factory/source_config.py b/src/rhdh_dynamic_plugin_factory/source_config.py index f04454a..65387d7 100644 --- a/src/rhdh_dynamic_plugin_factory/source_config.py +++ b/src/rhdh_dynamic_plugin_factory/source_config.py @@ -108,7 +108,7 @@ def resolve_default_ref(repo: str) -> str: ExecutionError: If git ls-remote fails or the default branch cannot be determined. """ logger = get_logger("source_config") - logger.info(f"[cyan]Resolving default branch for {repo}cyan]") + logger.info(f"[cyan]Resolving default branch for {repo}[/cyan]") try: result = subprocess.run( From bf72433c017cd71cef59e0e88f4143b32cdd7c0d Mon Sep 17 00:00:00 2001 From: Frank Kong Date: Tue, 7 Apr 2026 16:23:27 -0400 Subject: [PATCH 34/35] chore: add support for auth.json authentication Signed-off-by: Frank Kong --- README.md | 37 ++- default.env | 13 +- src/rhdh_dynamic_plugin_factory/cli.py | 4 + src/rhdh_dynamic_plugin_factory/config.py | 72 ++++-- tests/conftest.py | 3 +- tests/test_config_load_from_env.py | 47 ++++ tests/test_config_registry.py | 282 +++++++++++++++------- tests/test_multi_workspace.py | 182 +++++++++++++- 8 files changed, 525 insertions(+), 115 deletions(-) diff --git a/README.md b/README.md index 3acf6de..f867751 100644 --- a/README.md +++ b/README.md @@ -272,19 +272,48 @@ If this file is not provided, the factory will auto-generate it by scanning the #### 4. `config/.env` (Optional) -Override default settings to publish to a remote image registry: +Override default settings to publish to a remote image registry. `REGISTRY_URL` and `REGISTRY_NAMESPACE` are always required when `--push-images` is enabled (they are used to construct image tags). + +For authentication, you can either login with username/password or provide a docker/podman `auth.json` file containing your registry credentials. + +**Strategy 1: Username/Password (buildah login)** ```bash -# Registry configuration (required only with --push-images) REGISTRY_URL=quay.io +REGISTRY_NAMESPACE=your_namespace REGISTRY_USERNAME=your_username REGISTRY_PASSWORD=your_password -REGISTRY_NAMESPACE=your_namespace REGISTRY_INSECURE=false +``` + +**Strategy 2: Auth file (container mount)** + +```bash +REGISTRY_URL=quay.io +REGISTRY_NAMESPACE=your_namespace +REGISTRY_AUTH_FILE=/auth.json +``` +Mount your existing auth file into the container and set `REGISTRY_AUTH_FILE`: + +```bash +podman run --rm -it \ + --device /dev/fuse \ + -v ~/.config/containers/auth.json:/auth.json:ro,z \ + -e REGISTRY_AUTH_FILE=/auth.json \ + -v ./config:/config:z \ + quay.io/rhdh-community/dynamic-plugins-factory:latest \ + --workspace-path workspaces/todo \ + --push-images ``` -Alternatively, you can pass the `.env` file directly through podman using the `--env-file` argument instead of placing a `.env` file in the config directory: +The `:ro` (read-only) mount is safe and recommended -- the factory never writes to the auth file. When `REGISTRY_AUTH_FILE` is set, `REGISTRY_USERNAME` and `REGISTRY_PASSWORD` are ignored. + +If you have already authenticated on the host (e.g. via `podman login` or `buildah login`), no additional auth configuration is needed. The factory will log a warning that no explicit auth is configured but will proceed. `buildah push` uses the default credential store automatically. This is only applicable when using the factory outside of a container. + +--- + +Alternatively, you can pass the `.env` file directly through podman using the `--env-file` argument instead of placing a `.env` file in the config directory: ```bash podman run --rm -it \ diff --git a/default.env b/default.env index 3a5d7ce..746982d 100644 --- a/default.env +++ b/default.env @@ -5,9 +5,16 @@ RHDH_CLI_VERSION=latest # Registry configuration (required for push operations) -# Set these in your .env file or as environment variables +# REGISTRY_URL and REGISTRY_NAMESPACE are always required when --push-images is enabled. +# +# Authentication: provide ONE of the following: +# 1. Username/password – factory runs `buildah login` before push +# 2. Auth file – mount a pre-configured auth.json and set REGISTRY_AUTH_FILE +# 3. Pre-authenticated – if already logged in (e.g. `podman login`), no config needed +# # REGISTRY_URL=quay.io # REGISTRY_USERNAME=your-username -# REGISTRY_PASSWORD=your-password +# REGISTRY_PASSWORD=your-password # REGISTRY_NAMESPACE=your-namespace -# REGISTRY_INSECURE=false \ No newline at end of file +# REGISTRY_INSECURE=false +# REGISTRY_AUTH_FILE=/path/to/auth.json \ No newline at end of file diff --git a/src/rhdh_dynamic_plugin_factory/cli.py b/src/rhdh_dynamic_plugin_factory/cli.py index 3fd35b8..456bbbf 100644 --- a/src/rhdh_dynamic_plugin_factory/cli.py +++ b/src/rhdh_dynamic_plugin_factory/cli.py @@ -436,6 +436,10 @@ def _run_single_workspace(args: argparse.Namespace) -> None: ) logger.info(f"Using local repository at: {config.repo_path}") + if config.push_images: + config._validate_registry_fields() + config._buildah_login() + _process_workspace( config=config, workspace_config_dir=str(config.config_dir), diff --git a/src/rhdh_dynamic_plugin_factory/config.py b/src/rhdh_dynamic_plugin_factory/config.py index 242a17f..9c8c442 100644 --- a/src/rhdh_dynamic_plugin_factory/config.py +++ b/src/rhdh_dynamic_plugin_factory/config.py @@ -42,6 +42,7 @@ class PluginFactoryConfig: registry_password: Optional[str] = field(default=None) registry_namespace: Optional[str] = field(default=None) registry_insecure: bool = field(default=False) + registry_auth_file: Optional[str] = field(default=None) use_local: bool = field(default=False) push_images: bool = field(default=False) @@ -54,6 +55,10 @@ def __post_init__(self) -> None: Note: workspace_path is NOT validated here because it may be resolved later from source.json. Validation happens in cli._run() after source configuration discovery. + + Registry fields are NOT validated here. Validation is deferred to + workspace processing level so that multi-workspace mode can load + per-workspace .env files before checking credentials. """ if not self.rhdh_cli_version: raise ConfigurationError("RHDH_CLI_VERSION must be set (usually loaded from default.env)") @@ -61,22 +66,32 @@ def __post_init__(self) -> None: # Validate source arg constraints: --source-ref requires --source-repo if self.source_ref and not self.source_repo: raise ConfigurationError("--source-ref requires --source-repo to be provided") - - if self.push_images: - self._validate_registry_fields() def _validate_registry_fields(self) -> None: - """Validate that all required registry fields are present. + """Validate registry fields required for pushing images. + + REGISTRY_URL and REGISTRY_NAMESPACE are hard requirements (needed to + construct image tags). Authentication is validated as a warning only: + if neither username/password nor REGISTRY_AUTH_FILE is configured, a + warning is logged but execution continues -- the user may be relying + on pre-existing host auth (e.g. a prior ``podman login``). Raises: - ConfigurationError: If any required registry field is missing. + ConfigurationError: If REGISTRY_URL or REGISTRY_NAMESPACE is missing. """ if not self.registry_url: - raise ConfigurationError("REGISTRY_URL environment variable is required when --push-images is enabled") + raise ConfigurationError("REGISTRY_URL is required when --push-images is enabled") if not self.registry_namespace: - raise ConfigurationError("REGISTRY_NAMESPACE environment variable is required when --push-images is enabled") - if not self.registry_username or not self.registry_password: - raise ConfigurationError("REGISTRY_USERNAME and REGISTRY_PASSWORD environment variables are required when --push-images is enabled") + raise ConfigurationError("REGISTRY_NAMESPACE is required when --push-images is enabled") + has_credentials = bool(self.registry_username and self.registry_password) + has_auth_file = bool(self.registry_auth_file) + if not has_credentials and not has_auth_file: + self.logger.warning( + "No explicit registry authentication configured. " + "Push will rely on existing buildah/podman auth " + "(e.g. a prior 'podman login'). If push fails, provide " + "REGISTRY_USERNAME/REGISTRY_PASSWORD or set REGISTRY_AUTH_FILE." + ) def refresh_registry_config(self) -> None: """Re-read registry fields from os.environ and re-login if credentials changed. @@ -93,12 +108,14 @@ def refresh_registry_config(self) -> None: new_password = os.getenv("REGISTRY_PASSWORD") new_namespace = os.getenv("REGISTRY_NAMESPACE") new_insecure = os.getenv("REGISTRY_INSECURE", "false").lower() == "true" + new_auth_file = os.getenv("REGISTRY_AUTH_FILE") - creds_changed = ( + config_changed = ( new_url != self.registry_url or new_username != self.registry_username or new_password != self.registry_password or new_insecure != self.registry_insecure + or new_auth_file != self.registry_auth_file ) self.registry_url = new_url @@ -106,8 +123,9 @@ def refresh_registry_config(self) -> None: self.registry_password = new_password self.registry_namespace = new_namespace self.registry_insecure = new_insecure + self.registry_auth_file = new_auth_file - if self.push_images and creds_changed: + if self.push_images and config_changed: self._validate_registry_fields() self._buildah_login() @@ -119,10 +137,14 @@ def load_from_env(cls, args: argparse.Namespace, env_file: Optional[Path] = None Loads default.env first, then optionally loads additional env file to override defaults or provide additional values. Environment variables take precedence over .env file values. + Registry validation and login are NOT performed here. They are + deferred to workspace processing level so that per-workspace ``.env`` + files are loaded before credentials are checked. + Args: args: Parsed CLI arguments. env_file: Optional additional .env file to merge with defaults. - push_images: Whether to push images to a registry (triggers registry validation and login). + push_images: Whether to push images to a registry. multi_workspace: If True, skip root-level source.json and plugins-list.yaml validation since each workspace manages its own. """ @@ -164,6 +186,7 @@ def load_from_env(cls, args: argparse.Namespace, env_file: Optional[Path] = None registry_password=os.getenv("REGISTRY_PASSWORD"), registry_namespace=os.getenv("REGISTRY_NAMESPACE"), registry_insecure=os.getenv("REGISTRY_INSECURE", "false").lower() == "true", + registry_auth_file=os.getenv("REGISTRY_AUTH_FILE"), use_local=args.use_local, push_images=push_images, ) @@ -172,20 +195,35 @@ def load_from_env(cls, args: argparse.Namespace, env_file: Optional[Path] = None config._validate_source_json() config._validate_plugins_list() - if push_images: - config._buildah_login() - return config def _buildah_login(self) -> None: """Login to the container registry using buildah. - Assumes registry fields have already been validated by __post_init__. + Auth file takes precedence: if ``REGISTRY_AUTH_FILE`` is set, login is + skipped entirely because ``buildah push`` will read credentials from the + file automatically. The file is treated as read-only -- ``buildah login`` + would write to it, so we never call it when an auth file is configured. + + When no credentials and no auth file are present, login is also skipped + under the assumption that the host is already authenticated (e.g. via a + prior ``podman login``). Raises: ExecutionError: If the buildah login command fails. """ - ## TODO: Add support for token logins for ghcr.io registry as well + if self.registry_auth_file: + self.logger.info( + f"Using registry auth file: {self.registry_auth_file} " + f"(skipping buildah login)" + ) + return + if not self.registry_username or not self.registry_password: + self.logger.debug( + "No registry credentials provided, skipping buildah login " + "(relying on existing host auth)" + ) + return try: cmd = [ "buildah", "login", diff --git a/tests/conftest.py b/tests/conftest.py index 09feba4..a6cb473 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -182,7 +182,8 @@ def clean_env(monkeypatch: pytest.MonkeyPatch): "REGISTRY_USERNAME", "REGISTRY_PASSWORD", "REGISTRY_NAMESPACE", - "REGISTRY_INSECURE" + "REGISTRY_INSECURE", + "REGISTRY_AUTH_FILE", ] for var in env_vars: diff --git a/tests/test_config_load_from_env.py b/tests/test_config_load_from_env.py index a11175a..e79ed9b 100644 --- a/tests/test_config_load_from_env.py +++ b/tests/test_config_load_from_env.py @@ -232,3 +232,50 @@ def test_load_from_env_source_json_missing_repo_path_has_content(self, mock_args assert config is not None assert config.config_dir == str(config_dir) assert config.repo_path == str(repo_path) + + def test_load_from_env_push_images_no_validation_no_login(self, mock_args, setup_test_env, monkeypatch): + """push_images=True with no credentials does NOT raise at load time.""" + monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") + + mock_args.config_dir = setup_test_env["config_dir"] + mock_args.repo_path = setup_test_env["source_dir"] + mock_args.workspace_path = "." + + from unittest.mock import patch as mock_patch + with mock_patch.object(PluginFactoryConfig, "_buildah_login") as mock_login: + config = PluginFactoryConfig.load_from_env(mock_args, push_images=True) + mock_login.assert_not_called() + + assert config.push_images is True + + def test_load_from_env_multi_workspace_push_images_no_validation(self, mock_args, tmp_path, monkeypatch): + """multi_workspace=True + push_images=True + no credentials does NOT raise.""" + monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") + + config_dir = tmp_path / "config" + config_dir.mkdir(parents=True, exist_ok=True) + repo_path = tmp_path / "source" + repo_path.mkdir(parents=True, exist_ok=True) + (repo_path / "placeholder").write_text("") + + mock_args.config_dir = str(config_dir) + mock_args.repo_path = str(repo_path) + mock_args.workspace_path = "." + + config = PluginFactoryConfig.load_from_env( + mock_args, push_images=True, multi_workspace=True, + ) + assert config.push_images is True + assert config.registry_url is None + + def test_load_from_env_reads_registry_auth_file(self, mock_args, setup_test_env, monkeypatch): + """REGISTRY_AUTH_FILE env var is read into config.registry_auth_file.""" + monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") + monkeypatch.setenv("REGISTRY_AUTH_FILE", "/auth.json") + + mock_args.config_dir = setup_test_env["config_dir"] + mock_args.repo_path = setup_test_env["source_dir"] + mock_args.workspace_path = "." + + config = PluginFactoryConfig.load_from_env(mock_args) + assert config.registry_auth_file == "/auth.json" diff --git a/tests/test_config_registry.py b/tests/test_config_registry.py index 4436697..95f757e 100644 --- a/tests/test_config_registry.py +++ b/tests/test_config_registry.py @@ -1,8 +1,11 @@ """ Unit tests for PluginFactoryConfig registry validation and buildah login. -Tests the registry configuration validation in __post_init__ and -the _buildah_login functionality. +Tests the registry configuration validation (_validate_registry_fields) +and the _buildah_login functionality. + +Registry validation is no longer called in __post_init__; it is deferred +to workspace processing level. These tests call the methods explicitly. """ import subprocess @@ -14,71 +17,131 @@ class TestRegistryValidation: - """Tests for registry validation in __post_init__ when push_images is True.""" - + """Tests for _validate_registry_fields (called explicitly, not in __post_init__).""" + def test_no_validation_when_push_images_false(self, make_config): - """Test that registry fields are not validated when push_images is False.""" - # Should not raise exceptions even with no registry fields set + """Registry fields are not validated when push_images is False.""" config = make_config() assert config.push_images is False - + + def test_construction_with_push_images_no_registry_fields(self, make_config): + """push_images=True with no registry fields does NOT raise at construction time.""" + config = make_config(push_images=True) + assert config.push_images is True + assert config.registry_url is None + def test_missing_registry_url(self, make_config): - """Test that missing REGISTRY_URL raises ConfigurationError when push_images is True.""" - with pytest.raises(ConfigurationError, match="REGISTRY_URL environment variable is required"): - make_config( - push_images=True, - registry_url=None, - registry_namespace="test-namespace", - registry_username="test-user", - registry_password="test-password", - ) - + """Missing REGISTRY_URL raises ConfigurationError.""" + config = make_config( + push_images=True, + registry_url=None, + registry_namespace="test-namespace", + registry_username="test-user", + registry_password="test-password", + ) + with pytest.raises(ConfigurationError, match="REGISTRY_URL is required"): + config._validate_registry_fields() + def test_missing_registry_namespace(self, make_config): - """Test that missing REGISTRY_NAMESPACE raises ConfigurationError when push_images is True.""" - with pytest.raises(ConfigurationError, match="REGISTRY_NAMESPACE environment variable is required"): - make_config( - push_images=True, - registry_url="quay.io", - registry_namespace=None, - registry_username="test-user", - registry_password="test-password", - ) - - def test_missing_registry_credentials(self, make_config): - """Test that missing credentials raise ConfigurationError when push_images is True.""" - with pytest.raises(ConfigurationError, match="REGISTRY_USERNAME and REGISTRY_PASSWORD environment variables are required"): - make_config( - push_images=True, - registry_url="quay.io", - registry_namespace="test-namespace", - registry_username=None, - registry_password=None, - ) - - def test_missing_registry_username(self, make_config): - """Test that missing username raises ConfigurationError when push_images is True.""" - with pytest.raises(ConfigurationError, match="REGISTRY_USERNAME and REGISTRY_PASSWORD environment variables are required"): - make_config( - push_images=True, - registry_url="quay.io", - registry_namespace="test-namespace", - registry_username=None, - registry_password="test-password", - ) - - def test_missing_registry_password(self, make_config): - """Test that missing password raises ConfigurationError when push_images is True.""" - with pytest.raises(ConfigurationError, match="REGISTRY_USERNAME and REGISTRY_PASSWORD environment variables are required"): - make_config( - push_images=True, - registry_url="quay.io", - registry_namespace="test-namespace", - registry_username="test-user", - registry_password=None, - ) - + """Missing REGISTRY_NAMESPACE raises ConfigurationError.""" + config = make_config( + push_images=True, + registry_url="quay.io", + registry_namespace=None, + registry_username="test-user", + registry_password="test-password", + ) + with pytest.raises(ConfigurationError, match="REGISTRY_NAMESPACE is required"): + config._validate_registry_fields() + + def test_no_auth_warns_but_does_not_raise(self, make_config): + """No credentials and no auth file logs a warning but does NOT raise.""" + config = make_config( + push_images=True, + registry_url="quay.io", + registry_namespace="test-namespace", + registry_username=None, + registry_password=None, + registry_auth_file=None, + ) + with patch.object(config, "logger") as mock_logger: + config._validate_registry_fields() + mock_logger.warning.assert_called_once() + msg = mock_logger.warning.call_args[0][0] + assert "REGISTRY_USERNAME" in msg + assert "REGISTRY_AUTH_FILE" in msg + + def test_valid_with_username_password(self, make_config): + """Username + password passes validation without warning.""" + config = make_config( + push_images=True, + registry_url="quay.io", + registry_namespace="test-namespace", + registry_username="test-user", + registry_password="test-password", + ) + with patch.object(config, "logger") as mock_logger: + config._validate_registry_fields() + mock_logger.warning.assert_not_called() + + def test_valid_with_auth_file_only(self, make_config): + """Auth file set with no username/password passes validation without warning.""" + config = make_config( + push_images=True, + registry_url="quay.io", + registry_namespace="test-namespace", + registry_username=None, + registry_password=None, + registry_auth_file="/auth.json", + ) + with patch.object(config, "logger") as mock_logger: + config._validate_registry_fields() + mock_logger.warning.assert_not_called() + + def test_valid_with_both_auth_methods(self, make_config): + """Both username/password AND auth file set passes validation without warning.""" + config = make_config( + push_images=True, + registry_url="quay.io", + registry_namespace="test-namespace", + registry_username="test-user", + registry_password="test-password", + registry_auth_file="/auth.json", + ) + with patch.object(config, "logger") as mock_logger: + config._validate_registry_fields() + mock_logger.warning.assert_not_called() + + def test_partial_credentials_warns(self, make_config): + """Only username (no password, no auth file) logs warning.""" + config = make_config( + push_images=True, + registry_url="quay.io", + registry_namespace="test-namespace", + registry_username="test-user", + registry_password=None, + registry_auth_file=None, + ) + with patch.object(config, "logger") as mock_logger: + config._validate_registry_fields() + mock_logger.warning.assert_called_once() + + def test_partial_credentials_with_auth_file_no_warning(self, make_config): + """Only username (no password) but auth file set -- no warning.""" + config = make_config( + push_images=True, + registry_url="quay.io", + registry_namespace="test-namespace", + registry_username="test-user", + registry_password=None, + registry_auth_file="/auth.json", + ) + with patch.object(config, "logger") as mock_logger: + config._validate_registry_fields() + mock_logger.warning.assert_not_called() + def test_valid_registry_config(self, make_config): - """Test that valid registry configuration passes validation.""" + """Full registry configuration passes validation.""" config = make_config( push_images=True, registry_url="quay.io", @@ -92,9 +155,9 @@ def test_valid_registry_config(self, make_config): class TestBuildahLogin: """Tests for PluginFactoryConfig._buildah_login method.""" - + def test_successful_buildah_login(self, make_config): - """Test successful buildah login with valid credentials.""" + """Successful buildah login with valid credentials.""" config = make_config( push_images=True, registry_url="quay.io", @@ -103,16 +166,16 @@ def test_successful_buildah_login(self, make_config): registry_password="test-password", registry_insecure=False, ) - + with patch('subprocess.run') as mock_run: mock_run.return_value = MagicMock(returncode=0) - + with patch.object(config, 'logger') as mock_logger: config._buildah_login() - + mock_run.assert_called_once() call_args = mock_run.call_args - + expected_cmd = [ "buildah", "login", "--username", "test-user", @@ -123,13 +186,13 @@ def test_successful_buildah_login(self, make_config): assert call_args[1]['check'] is True assert call_args[1]['stdout'] == subprocess.PIPE assert call_args[1]['stderr'] == subprocess.PIPE - + mock_logger.info.assert_called_with( "Logged in to registry quay.io with buildah." ) - + def test_failed_buildah_login(self, make_config): - """Test that failed buildah login raises ExecutionError.""" + """Failed buildah login raises ExecutionError.""" config = make_config( push_images=True, registry_url="quay.io", @@ -138,7 +201,7 @@ def test_failed_buildah_login(self, make_config): registry_password="wrong-password", registry_insecure=False, ) - + with patch('subprocess.run') as mock_run: mock_error = subprocess.CalledProcessError( returncode=1, @@ -146,15 +209,15 @@ def test_failed_buildah_login(self, make_config): stderr=b"Authentication failed" ) mock_run.side_effect = mock_error - + with pytest.raises(ExecutionError, match="Failed to login to registry quay.io") as exc_info: config._buildah_login() - + assert exc_info.value.step == "buildah login" assert exc_info.value.returncode == 1 - + def test_insecure_registry_flag(self, make_config): - """Test that insecure flag is added to buildah command when registry_insecure is True.""" + """Insecure flag is added to buildah command when registry_insecure is True.""" config = make_config( push_images=True, registry_url="localhost:5000", @@ -163,15 +226,15 @@ def test_insecure_registry_flag(self, make_config): registry_password="test-password", registry_insecure=True, ) - + with patch('subprocess.run') as mock_run: mock_run.return_value = MagicMock(returncode=0) - + config._buildah_login() - + mock_run.assert_called_once() call_args = mock_run.call_args - + expected_cmd = [ "buildah", "login", "--username", "test-user", @@ -180,9 +243,9 @@ def test_insecure_registry_flag(self, make_config): "localhost:5000" ] assert call_args[0][0] == expected_cmd - + def test_secure_registry_default(self, make_config): - """Test that insecure flag is NOT added when registry_insecure is False.""" + """Insecure flag is NOT added when registry_insecure is False.""" config = make_config( push_images=True, registry_url="quay.io", @@ -191,15 +254,15 @@ def test_secure_registry_default(self, make_config): registry_password="test-password", registry_insecure=False, ) - + with patch('subprocess.run') as mock_run: mock_run.return_value = MagicMock(returncode=0) - + config._buildah_login() - + mock_run.assert_called_once() call_args = mock_run.call_args - + expected_cmd = [ "buildah", "login", "--username", "test-user", @@ -208,3 +271,52 @@ def test_secure_registry_default(self, make_config): ] assert call_args[0][0] == expected_cmd assert "--tls-verify=false" not in call_args[0][0] + + def test_skip_login_when_auth_file_set(self, make_config): + """Auth file set -- _buildah_login returns early, no subprocess call.""" + config = make_config( + push_images=True, + registry_url="quay.io", + registry_namespace="test-namespace", + registry_auth_file="/auth.json", + ) + + with patch('subprocess.run') as mock_run: + with patch.object(config, 'logger') as mock_logger: + config._buildah_login() + mock_run.assert_not_called() + mock_logger.info.assert_called_once() + assert "/auth.json" in mock_logger.info.call_args[0][0] + + def test_skip_login_auth_file_with_username_password_present(self, make_config): + """Auth file takes precedence -- login skipped even when credentials are set.""" + config = make_config( + push_images=True, + registry_url="quay.io", + registry_namespace="test-namespace", + registry_username="test-user", + registry_password="test-password", + registry_auth_file="/auth.json", + ) + + with patch('subprocess.run') as mock_run: + config._buildah_login() + mock_run.assert_not_called() + + def test_skip_login_when_no_credentials_and_no_auth_file(self, make_config): + """No credentials and no auth file -- login skipped (pre-authenticated host).""" + config = make_config( + push_images=True, + registry_url="quay.io", + registry_namespace="test-namespace", + registry_username=None, + registry_password=None, + registry_auth_file=None, + ) + + with patch('subprocess.run') as mock_run: + with patch.object(config, 'logger') as mock_logger: + config._buildah_login() + mock_run.assert_not_called() + mock_logger.debug.assert_called_once() + assert "relying on existing host auth" in mock_logger.debug.call_args[0][0] diff --git a/tests/test_multi_workspace.py b/tests/test_multi_workspace.py index f77770d..aca2339 100644 --- a/tests/test_multi_workspace.py +++ b/tests/test_multi_workspace.py @@ -1003,8 +1003,7 @@ def _make_config(self, monkeypatch, tmp_path, push_images=False, **registry_over source_ref=None, ) - with patch("src.rhdh_dynamic_plugin_factory.config.PluginFactoryConfig._buildah_login"): - return PluginFactoryConfig.load_from_env(args, push_images=push_images) + return PluginFactoryConfig.load_from_env(args, push_images=push_images) def test_updates_fields_from_environ(self, tmp_path, monkeypatch): """Test that refresh reads new values from os.environ.""" @@ -1051,8 +1050,8 @@ def test_relogin_skipped_when_push_images_disabled(self, tmp_path, monkeypatch): config.refresh_registry_config() mock_login.assert_not_called() - def test_validation_error_on_missing_fields_after_refresh(self, tmp_path, monkeypatch): - """Test that missing required fields after refresh raise ConfigurationError.""" + def test_validation_error_on_missing_url_after_refresh(self, tmp_path, monkeypatch): + """Missing REGISTRY_URL after refresh raises ConfigurationError.""" config = self._make_config(monkeypatch, tmp_path, push_images=True) monkeypatch.delenv("REGISTRY_URL") @@ -1062,7 +1061,7 @@ def test_validation_error_on_missing_fields_after_refresh(self, tmp_path, monkey config.refresh_registry_config() def test_namespace_change_without_cred_change_no_relogin(self, tmp_path, monkeypatch): - """Test that changing only namespace updates the field but skips re-login.""" + """Changing only namespace updates the field but skips re-login.""" config = self._make_config(monkeypatch, tmp_path, push_images=True) monkeypatch.setenv("REGISTRY_NAMESPACE", "different-ns") @@ -1071,3 +1070,176 @@ def test_namespace_change_without_cred_change_no_relogin(self, tmp_path, monkeyp config.refresh_registry_config() assert config.registry_namespace == "different-ns" mock_login.assert_not_called() + + def test_refresh_reads_auth_file_from_environ(self, tmp_path, monkeypatch): + """REGISTRY_AUTH_FILE in env is picked up by refresh.""" + config = self._make_config(monkeypatch, tmp_path) + assert config.registry_auth_file is None + + monkeypatch.setenv("REGISTRY_AUTH_FILE", "/auth.json") + config.refresh_registry_config() + assert config.registry_auth_file == "/auth.json" + + def test_refresh_auth_file_change_triggers_revalidation(self, tmp_path, monkeypatch): + """Auth file value changing between refreshes triggers re-validation + login.""" + config = self._make_config(monkeypatch, tmp_path, push_images=True) + + monkeypatch.setenv("REGISTRY_AUTH_FILE", "/new-auth.json") + + with patch.object(config, "_buildah_login") as mock_login: + config.refresh_registry_config() + mock_login.assert_called_once() + + def test_refresh_with_auth_file_no_username_password(self, tmp_path, monkeypatch): + """Workspace has auth file + URL + namespace but no credentials -- succeeds.""" + monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") + monkeypatch.setenv("REGISTRY_URL", "quay.io") + monkeypatch.setenv("REGISTRY_NAMESPACE", "ns") + monkeypatch.setenv("REGISTRY_INSECURE", "false") + monkeypatch.delenv("REGISTRY_USERNAME", raising=False) + monkeypatch.delenv("REGISTRY_PASSWORD", raising=False) + + config_dir = tmp_path / "config" + config_dir.mkdir(parents=True, exist_ok=True) + repo_path = tmp_path / "source" + repo_path.mkdir(parents=True, exist_ok=True) + (repo_path / "placeholder").write_text("") + + import argparse + args = argparse.Namespace( + config_dir=config_dir, repo_path=repo_path, workspace_path=".", + use_local=False, push_images=True, log_level="INFO", + source_repo=None, source_ref=None, + ) + config = PluginFactoryConfig.load_from_env(args, push_images=True) + + monkeypatch.setenv("REGISTRY_AUTH_FILE", "/auth.json") + config.refresh_registry_config() + assert config.registry_auth_file == "/auth.json" + + +class TestMultiWorkspaceDeferredCredentials: + """Integration-style tests for multi-workspace deferred credential flow.""" + + def _make_multi_ws_config(self, monkeypatch, tmp_path, env_vars=None): + """Build a config in multi-workspace mode with optional env overrides.""" + monkeypatch.setenv("RHDH_CLI_VERSION", "1.7.2") + if env_vars: + for k, v in env_vars.items(): + monkeypatch.setenv(k, v) + + config_dir = tmp_path / "config" + config_dir.mkdir(parents=True, exist_ok=True) + repo_path = tmp_path / "source" + repo_path.mkdir(parents=True, exist_ok=True) + (repo_path / "placeholder").write_text("") + + import argparse + args = argparse.Namespace( + config_dir=config_dir, repo_path=repo_path, workspace_path=".", + use_local=False, push_images=True, log_level="INFO", + source_repo=None, source_ref=None, + ) + return PluginFactoryConfig.load_from_env( + args, push_images=True, multi_workspace=True, + ) + + def test_root_env_no_credentials_workspace_env_has_credentials(self, tmp_path, monkeypatch): + """Root has URL/NS only; workspace provides username/password.""" + config = self._make_multi_ws_config(monkeypatch, tmp_path, env_vars={ + "REGISTRY_URL": "quay.io", + "REGISTRY_NAMESPACE": "ns", + }) + assert config.registry_username is None + + monkeypatch.setenv("REGISTRY_USERNAME", "ws-user") + monkeypatch.setenv("REGISTRY_PASSWORD", "ws-pass") + + with patch.object(config, "_buildah_login") as mock_login: + config.refresh_registry_config() + mock_login.assert_called_once() + assert config.registry_username == "ws-user" + + def test_root_env_empty_workspaces_provide_all_registry_config(self, tmp_path, monkeypatch): + """Root has NO registry vars; workspace provides everything.""" + config = self._make_multi_ws_config(monkeypatch, tmp_path) + assert config.registry_url is None + + monkeypatch.setenv("REGISTRY_URL", "ghcr.io") + monkeypatch.setenv("REGISTRY_NAMESPACE", "org") + monkeypatch.setenv("REGISTRY_USERNAME", "user") + monkeypatch.setenv("REGISTRY_PASSWORD", "pass") + + with patch.object(config, "_buildah_login") as mock_login: + config.refresh_registry_config() + mock_login.assert_called_once() + assert config.registry_url == "ghcr.io" + + def test_root_env_no_creds_workspace_also_missing_url_fails(self, tmp_path, monkeypatch): + """Root and workspace both missing REGISTRY_URL -- raises ConfigurationError.""" + config = self._make_multi_ws_config(monkeypatch, tmp_path) + + monkeypatch.setenv("REGISTRY_USERNAME", "user") + monkeypatch.setenv("REGISTRY_PASSWORD", "pass") + + with pytest.raises(ConfigurationError, match="REGISTRY_URL"): + config.refresh_registry_config() + + def test_root_env_no_creds_workspace_has_url_but_no_auth_warns(self, tmp_path, monkeypatch): + """Workspace has URL + NS but no credentials/auth file -- warns but succeeds.""" + config = self._make_multi_ws_config(monkeypatch, tmp_path) + + monkeypatch.setenv("REGISTRY_URL", "quay.io") + monkeypatch.setenv("REGISTRY_NAMESPACE", "ns") + + with patch.object(config, "logger") as mock_logger: + with patch.object(config, "_buildah_login"): + config.refresh_registry_config() + warning_calls = [ + c for c in mock_logger.warning.call_args_list + if "No explicit registry authentication" in str(c) + ] + assert len(warning_calls) == 1 + + def test_root_env_no_credentials_workspace_env_has_auth_file(self, tmp_path, monkeypatch): + """Root has nothing; workspace provides auth file + URL + namespace.""" + config = self._make_multi_ws_config(monkeypatch, tmp_path) + + monkeypatch.setenv("REGISTRY_URL", "quay.io") + monkeypatch.setenv("REGISTRY_NAMESPACE", "ns") + monkeypatch.setenv("REGISTRY_AUTH_FILE", "/auth.json") + + config.refresh_registry_config() + assert config.registry_auth_file == "/auth.json" + + def test_mixed_workspaces_different_auth_strategies(self, tmp_path, monkeypatch): + """Three workspaces with different auth: user/pass, auth file, pre-auth.""" + config = self._make_multi_ws_config(monkeypatch, tmp_path) + + # WS-A: username/password + monkeypatch.setenv("REGISTRY_URL", "quay.io") + monkeypatch.setenv("REGISTRY_NAMESPACE", "ns") + monkeypatch.setenv("REGISTRY_USERNAME", "user") + monkeypatch.setenv("REGISTRY_PASSWORD", "pass") + monkeypatch.delenv("REGISTRY_AUTH_FILE", raising=False) + with patch.object(config, "_buildah_login") as mock_login: + config.refresh_registry_config() + mock_login.assert_called_once() + + # WS-B: auth file (clear creds, set auth file) + monkeypatch.delenv("REGISTRY_USERNAME", raising=False) + monkeypatch.delenv("REGISTRY_PASSWORD", raising=False) + monkeypatch.setenv("REGISTRY_AUTH_FILE", "/auth.json") + config.refresh_registry_config() + assert config.registry_auth_file == "/auth.json" + + # WS-C: pre-authenticated (no creds, no auth file) + monkeypatch.delenv("REGISTRY_AUTH_FILE", raising=False) + with patch.object(config, "logger") as mock_logger: + with patch.object(config, "_buildah_login"): + config.refresh_registry_config() + warning_calls = [ + c for c in mock_logger.warning.call_args_list + if "No explicit registry authentication" in str(c) + ] + assert len(warning_calls) == 1 From a95c640e602300bc04af710e51ae8d19393bf1f0 Mon Sep 17 00:00:00 2001 From: Frank Kong Date: Wed, 8 Apr 2026 09:10:34 -0400 Subject: [PATCH 35/35] chore: remove dup logs Signed-off-by: Frank Kong --- src/rhdh_dynamic_plugin_factory/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rhdh_dynamic_plugin_factory/config.py b/src/rhdh_dynamic_plugin_factory/config.py index 9c8c442..789752b 100644 --- a/src/rhdh_dynamic_plugin_factory/config.py +++ b/src/rhdh_dynamic_plugin_factory/config.py @@ -598,7 +598,7 @@ def export_plugins(self, output_dir: str, config_dir: Optional[str] = None, def conditional_stderr_log(line: str) -> None: if "Error" in line: self.logger.error(line) - if "npm warn" in line: + elif "npm warn" in line: self.logger.warning(line) else: self.logger.info(line)