diff --git a/.github/workflows/code-checkers.yml b/.github/workflows/code-checkers.yml index 9412fd2ff..068781984 100644 --- a/.github/workflows/code-checkers.yml +++ b/.github/workflows/code-checkers.yml @@ -15,8 +15,6 @@ jobs: with: persist-credentials: false - uses: ./.github/actions/uv-setup/ - - name: Create a dummy data.py - run: cp data.py-dist data.py - name: Create a dummy vm_data.py run: cp vm_data.py-dist vm_data.py - run: prek -a pyright @@ -30,8 +28,6 @@ jobs: with: persist-credentials: false - uses: ./.github/actions/uv-setup/ - - name: Create a dummy data.py - run: cp data.py-dist data.py - run: prek -a ruff flake8: diff --git a/.github/workflows/jobs-check.yml b/.github/workflows/jobs-check.yml index 58cd835ec..6696c1673 100644 --- a/.github/workflows/jobs-check.yml +++ b/.github/workflows/jobs-check.yml @@ -14,6 +14,4 @@ jobs: - uses: ./.github/actions/uv-setup/ with: dev: false - - name: Create a dummy data.py - run: cp data.py-dist data.py - run: ./jobs.py check diff --git a/.github/workflows/test-sequences.yml b/.github/workflows/test-sequences.yml index 7ff2029f8..25a408652 100644 --- a/.github/workflows/test-sequences.yml +++ b/.github/workflows/test-sequences.yml @@ -14,8 +14,6 @@ jobs: - uses: ./.github/actions/uv-setup/ with: dev: false - - name: Create a dummy data.py - run: cp data.py-dist data.py - name: jobs-check run: | FAILURES="" diff --git a/.github/workflows/unit.yml b/.github/workflows/unit.yml index a37444768..f3f57d482 100644 --- a/.github/workflows/unit.yml +++ b/.github/workflows/unit.yml @@ -14,6 +14,4 @@ jobs: - uses: ./.github/actions/uv-setup/ with: dev: false - - name: Create a dummy data.py - run: cp data.py-dist data.py - run: pytest tests/unit/ diff --git a/.gitignore b/.gitignore index 9d6708b7e..5cee1ac26 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ data.py vm_data.py /scripts/guests/windows/id_rsa.pub .envrc +# Config overrides (user environment-specific) +config.*.toml diff --git a/Makefile b/Makefile index 9029ce3a5..2a9c1a6ce 100644 --- a/Makefile +++ b/Makefile @@ -11,13 +11,10 @@ check: ruff autopep8 flake8 pyright # It runs on all files managed by git (untracked files are not modified) fix: ruff-fix autopep8-fix -data.py: - @test -r data.py || echo "File 'data.py' does not exist. Refer to https://github.com/xcp-ng/xcp-ng-tests#configuration." && exit 1 - vm_data.py: @test -r vm_data.py || echo "File 'vm_data.py' does not exist. Refer to https://github.com/xcp-ng/xcp-ng-tests#configuration." && exit 1 -pyright ruff: data.py vm_data.py +pyright ruff: vm_data.py ruff ruff-fix autopep8 autopep8-fix flake8 pyright: uv run prek -a $@ diff --git a/README.md b/README.md index 82130b26a..7a75216d3 100644 --- a/README.md +++ b/README.md @@ -147,10 +147,108 @@ For Guest UEFI Secure Boot tests, the requirements are: * `efitools` for uefistored (in 8.2) or varstored (in 8.3+) auth var tests * `util-linux` for uefistored (in 8.2) or varstored (in 8.3+) auth var tests in Alpine VMs -Many tests have specific requirements, detailed in a comment at the top of the test file: minimal number of hosts in a pool, number of pools, VMs with specific characteristics (OS, BIOS vs UEFI, additional tools installed in the VM, additional networks in the pool, presence of an unused disk on one host or every host...). Markers, jobs defined in `jobs.py` (`./jobs.py show JOBNAME` will display the requirements and the reference to a VM or VM group), VMs and VM groups defined in `vm-data.py-dist` may all help understanding what tests can run with what VMs. +Many tests have specific requirements, detailed in a comment at the top of the test file: minimal number of hosts in a pool, number of pools, VMs with specific characteristics (OS, BIOS vs UEFI, additional tools installed in the VM, additional networks in the pool, presence of an unused disk on one host or every host...). Markers, jobs defined in `jobs.py` (`./jobs.py show JOBNAME` will display the requirements and the reference to a VM or VM group), VMs and VM groups defined in `vm_data.py` may all help understanding what tests can run with what VMs. ## Configuration -The main configuration file is data.py. Copy data.py-dist to data.py and modify it if needed. + +### Using config.toml + +Test configuration is managed via TOML files. The project includes a `config.toml` file with all default settings. + +#### Default configuration + +The `config.toml` file in the repository contains all default settings for: +- Host SSH credentials and per-host overrides +- Network definitions +- PXE server configuration +- VM images and ISO definitions +- Storage device configurations (NFS, CIFS, CephFS, MooseFS, LVM iSCSI) +- Guest tools (Windows, other) +- Test utilities and SSH keys + +#### Custom configuration + +To customize settings for your environment: + +1. **Create a custom config file** (e.g., `config.local.toml`): + ```toml + [host] + default_user = "root" + default_password = "your-password" + + [host.per_host] + "192.168.1.10" = { user = "custom_user", password = "custom_pass" } + + [pxe] + config_server = "pxe.example.com" + arp_server = "pxe.example.com" + + [storage.nfs] + server = "10.0.0.2" + serverpath = "/mnt/shared" + ``` + +2. **Run tests with custom config**: + ```bash + pytest --config=local --hosts=10.0.0.1 + ``` + + This loads `config.toml` first, then merges `config.local.toml` on top. + +#### Configuration file paths + +- `config.toml` — base config (always loaded, lowest priority) +- `config.local.toml` — local defaults (auto-loaded if exists and no `--config`/`XCPNG_CONFIG` specified) +- `XCPNG_CONFIG=NAME` or `XCPNG_CONFIG=` — env var equivalent of `--config` +- `XCPNG_CONFIG_DIR=DIR` — directory where profile names are looked up + (`config.NAME.toml`); falls back to the repository root +- `--config=NAME` — environment-specific overrides (`config.NAME.toml`) +- `--config=` — a config overlay file from anywhere on disk +- `--config-value KEY=VALUE` — override a single config value (repeatable), + e.g. `--config-value host.default_password=foo`. The key is a dotted path, + with double quotes around segments that contain dots + (`--config-value 'hosts."10.30.0.56".user=root'`). +- `tools.py dump-config` / `migrate-data-py` print only the values that differ + from the base `config.toml`; add `--all` to include the values that are the + same too. +- `tools.py diff-config CONFIG1 CONFIG2` — compare two config files and print + a unified diff (password hashes are ignored); exit code 0 when identical, + 1 when they differ. + +The `--config` flag is optional. When not specified, the `XCPNG_CONFIG` env var +is used if set. When neither is set: +1. `config.toml` is loaded first +2. If `config.local.toml` exists, it is merged on top (auto-detected) + +Overrides stack from lowest to highest priority: +`config.toml` < overlay (`--config`/`XCPNG_CONFIG`) < `XCPNG_TESTS_*` env vars +< `--config-value`. + +When given, the value is resolved to a file in this order: +1. as given (absolute, or relative to the current directory) +2. as a short name (`config.NAME.toml`), in the `XCPNG_CONFIG_DIR` directory + when that env var is set, then in the repository root + +Only values given as paths are resolved relative to the current directory; +short names (`config.NAME.toml`) are looked up in `XCPNG_CONFIG_DIR` or the +repository root, never in the current directory. + +For example, running from another directory: + +```bash +pytest foo -c ../../inventory/cgt1-qcow2.toml +``` + +loads `config.toml` first, then merges `../../inventory/cgt1-qcow2.toml` on top. +The `include` key inside any config file is resolved against the file's own +directory first, then against the main xcp-ng-tests directory, so an overlay +in another directory can reuse files from the tests repository. + +This allows you to: +- Commit `config.toml` with project defaults to version control +- Create `config.local.toml` locally (ignored by git) for your standard environment +- Create `config.prod.toml`, `config.ci.toml`, etc. for other environments and select with `--config=prod` (or `XCPNG_CONFIG=prod`) +- Point `--config` (or `XCPNG_CONFIG`) at an overlay file kept anywhere on disk ## Running tests @@ -165,12 +263,13 @@ pytest tests/misc/test_vm_basic_operations.py --hosts=10.0.0.1 --vm=mini-linux-x Most tests take a `--hosts=yourtesthost` (or `--hosts=host1,host2,...` if they need several pools, e.g. crosspool migration tests). The `--hosts` parameter can be specified several times. Then `pytest` will run the tests on each host or group of hosts, sequentially. +If `--hosts` is not given, the hosts listed in the `[hosts]` section of the config file are used instead. When a test requires a single pool of several hosts, only mention the master host in the `--hosts` option. Some tests accept an optional `--vm=OVA_URL|VM_key|IP_address` parameter. Those are tests that will import a VM before testing stuff on it: -* `OVA_URL` is a URL to download an OVA. It can also be simply a filename, if your `data.py`'s `DEF_VM_URL` is correctly defined. -* `VM_key` refers to a key in `data.py`'s `VM_IMAGES` dict. Example: `mini-linux-x86_64-uefi`. +* `OVA_URL` is a URL to download an OVA. It can also be simply a filename, if your `config.toml`'s `vm.def_url` is correctly defined. +* `VM_key` refers to a key in `config.toml`'s `vm.images` section. Example: `mini-linux-x86_64-uefi`. * `IP_address` allows you to reuse an existing running VM, skipping the whole import, start, wait for VM to be up setup. Can be useful as a development tool. Some tests that accept `--vm` do not support it. If `--vm` is not specified, defaults defined by the tests will be used. The `--vm` parameter can be specified several times. Then pytest will run several instances of the tests sequentially, one for each VM. @@ -237,8 +336,6 @@ We wanted the job definitions to be in this git repository, that's why the job d To use `./jobs.py`, you also need to populate `vm_data.py` to define the VM groups that are necessary to run jobs (unless `--vm` is provided on the command line to override the defaults). -The output of commands below is given as example and may not reflect the current state of the jobs definitions. - #### List jobs ``` $ ./jobs.py list @@ -321,15 +418,18 @@ pytest tests/uefi_sb -m "multi_vms and unix_vm" --hosts=ip_of_poolmaster --vm=ht #### Run a job ``` -usage: jobs.py run [-h] [--print-only] job hosts ... +usage: jobs.py run [-h] [-c PATH] [--config-value KEY=VALUE] [--print-only] job [hosts] ... positional arguments: job name of the job to run. hosts master host(s) of pools to run the tests on, comma-separated. + When omitted, the hosts from the config's [hosts] section are used. pytest_args all additional arguments after the last positional argument will be passed to pytest and replace default job params if needed. optional arguments: -h, --help show this help message and exit + -c PATH, --config PATH config overlay: a .toml file path or profile name + --config-value KEY=VALUE override a config value (repeatable; highest priority) --print-only, -p print the command, but don't run it. Must be specified before positional arguments. ``` @@ -343,6 +443,13 @@ pytest tests/uefi_sb -m "multi_vms and unix_vm" --hosts=ip_of_poolmaster --vm=ht Any parameter added at the end of the command will be passed to `pytest`. Any parameter added that is already defined in the job's "params" (see output of `./jobs.py show`) will replace it, and `--vm` also replaces `--vm[]` in the case of jobs designed to run tests on multiple VMs. +`jobs.py` reads the same configuration as `pytest` and `tools.py`: +`-c`/`--config`, the `XCPNG_CONFIG`/`XCPNG_CONFIG_DIR` env vars and +`--config-value` are supported, and when no hosts are given, `jobs.py run` +falls back to the hosts defined in the config's `[hosts]` section. The +config options must be specified before the positional `job`/`hosts` +arguments (like `--print-only`), otherwise they are forwarded to pytest. + ``` # same, but we override the list of VMs $ ./jobs.py run --print-only sb-unix-multi ip_of_poolmaster --vm=http://path/to/vm4.xva @@ -602,10 +709,10 @@ python scripts/test_install_xcpng.py 10.0.0.2 f0f5f010-80c6-25ae-44a2-1fb154e32d ``` Note: in case of restore, the version must be that of the installer (here 8.2.1), not the version of XCP-ng that will be restored. -The script requires the addressable name or IP of the PXE config server to be defined in `data.py`: -``` -# PXE config server for automated XCP-ng installation -PXE_CONFIG_SERVER = 'pxe' +The script requires the addressable name or IP of the PXE config server to be defined in `config.toml`: +```toml +[pxe] +config_server = "pxe" ``` The `installer` parameter is optional. If you leave it empty it will be automatically defined as `http:///installers/xcp-ng//`. @@ -659,47 +766,45 @@ For each pool target : 2. Get other hosts of the pool * Repeat step `1.` for each host -**Inventory file** +**Inventory** -`update` command can read an inventory file in [TOML v1.0.0](https://toml.io/en/v1.0.0) format: +By default, hosts and repository settings are read from the same config file as +the tests (`config.toml`, auto-merged with `config.local.toml` when present). +A different overlay can be picked with `-c/--config` (or the `XCPNG_CONFIG` +env var), accepting a `.toml` file path (relative to the current directory) or +a profile name (`config.PROFILE.toml`): ```bash -uv run scripts/tools.py update -i my_inventory.toml +uv run scripts/tools.py update +uv run scripts/tools.py update -c ../../inventory/cgt1-qcow2.toml +uv run scripts/tools.py update -c prod ``` -> [!NOTE] -> You can use either `-i/--inventory` or `-H/--hosts`. -> -> **Above flags can't be used together** - -Take a look at an example inventory file: +Hosts are the keys of the `[hosts]` table, and per-host values override the +inventory defaults from the `[tools.update]` table: ```toml -# my_inventory.toml +# config.toml -[default] +[tools.update] repositories = ["xcp-ng-base"] disabled_repositories = ["epel"] -hosting_pool = "A" +hosting_pool = "10.30.0.50" [hosts] - -[hosts."ip_or_hostname-1"] - -[hosts."ip_or_hostname-2"] - -repositories = ["xcp-ng-updates"] -hosting_pool = "B" +"10.30.0.56" = {} +"10.30.0.59" = { repositories = ["xcp-ng-updates"] } # overrides "[tools.update]" ``` -> [!IMPORTANT] -> * `default` is applied to all hosts -> * Config values under `hosts` override values under `default`. For instance, the above inventory would produce -> the following python dict: +> [!NOTE] +> * `tools.update` is applied to all hosts +> * Values set in a `[hosts]` entry override `tools.update`. The example above +> would produce the following python dict: > -> `{'ip_or_hostname-1': {'repositories': ['xcp-ng-base'], 'disabled_repositories': ['epel'], 'hosting_pool': 'A'}, 'ip_or_hostname-2': {'repositories': ['xcp-ng-updates'], 'hosting_pool': 'B'}}` +> `{'10.30.0.56': {'repositories': ['xcp-ng-base'], 'disabled_repositories': ['epel'], 'hosting_pool': '10.30.0.50'}, '10.30.0.59': {'repositories': ['xcp-ng-updates'], 'disabled_repositories': ['epel'], 'hosting_pool': '10.30.0.50'}}` > -> * `disabled_repositories` disables one or more repositories during the update. It can be set under `default` or overridden per host, and can also be passed with the `-x/--disablerepo` flag. +> * `disabled_repositories` disables one or more repositories during the update. It can be set under `tools.update` or overridden per host. > * `*` as a repository value disables **all** repositories (e.g. `disabled_repositories = ["*"]`). > -> * When `--inventory` flag is present, repos passed to `-e` flag won't be considered. +> `-H/--hosts` and the repository flags below can be used to temporarily +> override the config file for one-off runs. diff --git a/config-schema.json b/config-schema.json new file mode 100644 index 000000000..90b9ce6e3 --- /dev/null +++ b/config-schema.json @@ -0,0 +1,1012 @@ +{ + "$defs": { + "AnswerFileDef": { + "additionalProperties": true, + "properties": { + "CONTENTS": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "$ref": "#/$defs/AnswerFileDef" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Contents" + }, + "TAG": { + "title": "Tag", + "type": "string" + } + }, + "title": "AnswerFileDef", + "type": "object" + }, + "CIFSISOConfig": { + "additionalProperties": false, + "properties": { + "cifspassword": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Cifspassword" + }, + "location": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Location" + }, + "type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Type" + }, + "username": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Username" + }, + "vers": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Vers" + } + }, + "title": "CIFSISOConfig", + "type": "object" + }, + "CephFSConfig": { + "additionalProperties": false, + "properties": { + "options": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Options" + }, + "server": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Server" + }, + "serverpath": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Serverpath" + } + }, + "title": "CephFSConfig", + "type": "object" + }, + "GuestToolsConfig": { + "additionalProperties": false, + "properties": { + "download_url": { + "title": "Download Url", + "type": "string" + }, + "installed": { + "additionalProperties": { + "$ref": "#/$defs/InstalledGuestToolDef" + }, + "title": "Installed", + "type": "object" + }, + "other": { + "$ref": "#/$defs/OtherGuestToolDef" + }, + "win": { + "additionalProperties": { + "$ref": "#/$defs/WinGuestToolDef" + }, + "title": "Win", + "type": "object" + } + }, + "title": "GuestToolsConfig", + "type": "object" + }, + "HostConfig": { + "additionalProperties": false, + "properties": { + "default_password": { + "title": "Default Password", + "type": "string" + }, + "default_password_hash": { + "default": "", + "title": "Default Password Hash", + "type": "string" + }, + "default_user": { + "title": "Default User", + "type": "string" + } + }, + "title": "HostConfig", + "type": "object" + }, + "HostOverride": { + "additionalProperties": false, + "properties": { + "disabled_repositories": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Disabled Repositories" + }, + "hosting_pool": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Hosting Pool" + }, + "password": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Password" + }, + "repositories": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Repositories" + }, + "skip_xo_config": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Skip Xo Config" + }, + "user": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "User" + } + }, + "title": "HostOverride", + "type": "object" + }, + "InstallConfig": { + "additionalProperties": false, + "properties": { + "answerfiles": { + "additionalProperties": { + "$ref": "#/$defs/AnswerFileDef" + }, + "title": "Answerfiles", + "type": "object" + }, + "iso_remaster": { + "default": "", + "title": "Iso Remaster", + "type": "string" + }, + "isos": { + "$ref": "#/$defs/InstallIsosConfig" + } + }, + "title": "InstallConfig", + "type": "object" + }, + "InstallIsosConfig": { + "additionalProperties": false, + "properties": { + "base_url": { + "title": "Base Url", + "type": "string" + }, + "cache_dir": { + "title": "Cache Dir", + "type": "string" + }, + "definitions": { + "additionalProperties": { + "$ref": "#/$defs/IsoImageDef" + }, + "title": "Definitions", + "type": "object" + } + }, + "title": "InstallIsosConfig", + "type": "object" + }, + "InstalledGuestToolDef": { + "additionalProperties": false, + "properties": { + "onboarding_phase": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Onboarding Phase" + }, + "package": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Package" + }, + "path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Path" + }, + "testsign_cert": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Testsign Cert" + }, + "type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Type" + }, + "upgradable": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Upgradable" + }, + "vendor_device": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Vendor Device" + } + }, + "title": "InstalledGuestToolDef", + "type": "object" + }, + "IsoImageDef": { + "additionalProperties": false, + "properties": { + "net-only": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Net-Only" + }, + "net-url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Net-Url" + }, + "path": { + "title": "Path", + "type": "string" + }, + "unsigned": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Unsigned" + } + }, + "title": "IsoImageDef", + "type": "object" + }, + "LVMoHBAConfig": { + "additionalProperties": false, + "properties": { + "SCSIid": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Scsiid" + } + }, + "title": "LVMoHBAConfig", + "type": "object" + }, + "LVMoISCSIConfig": { + "additionalProperties": false, + "properties": { + "SCSIid": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Scsiid" + }, + "port": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Port" + }, + "target": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Target" + }, + "targetIQN": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Targetiqn" + } + }, + "title": "LVMoISCSIConfig", + "type": "object" + }, + "LinstorConfig": { + "additionalProperties": false, + "properties": { + "redundancy": { + "title": "Redundancy", + "type": "integer" + } + }, + "title": "LinstorConfig", + "type": "object" + }, + "MooseFSConfig": { + "additionalProperties": false, + "properties": { + "masterhost": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Masterhost" + }, + "masterport": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Masterport" + }, + "rootpath": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Rootpath" + } + }, + "title": "MooseFSConfig", + "type": "object" + }, + "NFS4Config": { + "additionalProperties": false, + "properties": { + "nfsversion": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Nfsversion" + }, + "server": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Server" + }, + "serverpath": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Serverpath" + } + }, + "title": "NFS4Config", + "type": "object" + }, + "NFSConfig": { + "additionalProperties": false, + "properties": { + "server": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Server" + }, + "serverpath": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Serverpath" + } + }, + "title": "NFSConfig", + "type": "object" + }, + "NFSISOConfig": { + "additionalProperties": false, + "properties": { + "location": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Location" + } + }, + "title": "NFSISOConfig", + "type": "object" + }, + "NetworkConfig": { + "additionalProperties": false, + "properties": { + "free_nics": { + "items": { + "type": "string" + }, + "title": "Free Nics", + "type": "array" + }, + "mgmt": { + "title": "Mgmt", + "type": "string" + } + }, + "title": "NetworkConfig", + "type": "object" + }, + "OtherGuestToolDef": { + "additionalProperties": false, + "properties": { + "download": { + "title": "Download", + "type": "boolean" + }, + "name": { + "title": "Name", + "type": "string" + } + }, + "title": "OtherGuestToolDef", + "type": "object" + }, + "PXEConfig": { + "additionalProperties": false, + "properties": { + "arp_server": { + "title": "Arp Server", + "type": "string" + }, + "config_server": { + "title": "Config Server", + "type": "string" + } + }, + "title": "PXEConfig", + "type": "object" + }, + "SSHConfig": { + "additionalProperties": false, + "properties": { + "ignore_banner": { + "title": "Ignore Banner", + "type": "boolean" + }, + "output_max_lines": { + "title": "Output Max Lines", + "type": "integer" + }, + "pubkey": { + "title": "Pubkey", + "type": "string" + } + }, + "title": "SSHConfig", + "type": "object" + }, + "StorageConfig": { + "additionalProperties": false, + "properties": { + "cephfs": { + "$ref": "#/$defs/CephFSConfig" + }, + "cifs_iso": { + "$ref": "#/$defs/CIFSISOConfig" + }, + "linstor": { + "$ref": "#/$defs/LinstorConfig" + }, + "lvmohba": { + "$ref": "#/$defs/LVMoHBAConfig" + }, + "lvmoiscsi": { + "$ref": "#/$defs/LVMoISCSIConfig" + }, + "moosefs": { + "$ref": "#/$defs/MooseFSConfig" + }, + "nfs": { + "$ref": "#/$defs/NFSConfig" + }, + "nfs4": { + "$ref": "#/$defs/NFS4Config" + }, + "nfs_iso": { + "$ref": "#/$defs/NFSISOConfig" + } + }, + "title": "StorageConfig", + "type": "object" + }, + "ToolsConfig": { + "additionalProperties": false, + "properties": { + "update": { + "$ref": "#/$defs/UpdateDefaults" + } + }, + "title": "ToolsConfig", + "type": "object" + }, + "UpdateDefaults": { + "additionalProperties": false, + "properties": { + "disabled_repositories": { + "items": { + "type": "string" + }, + "title": "Disabled Repositories", + "type": "array" + }, + "hosting_pool": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Hosting Pool" + }, + "repositories": { + "items": { + "type": "string" + }, + "title": "Repositories", + "type": "array" + } + }, + "title": "UpdateDefaults", + "type": "object" + }, + "VMConfig": { + "additionalProperties": false, + "properties": { + "cache_imported": { + "title": "Cache Imported", + "type": "boolean" + }, + "def_url": { + "title": "Def Url", + "type": "string" + }, + "default_sr": { + "title": "Default Sr", + "type": "string" + }, + "equivalents": { + "additionalProperties": { + "type": "string" + }, + "title": "Equivalents", + "type": "object" + }, + "images": { + "additionalProperties": { + "type": "string" + }, + "title": "Images", + "type": "object" + } + }, + "title": "VMConfig", + "type": "object" + }, + "WinGuestToolDef": { + "additionalProperties": false, + "properties": { + "download": { + "title": "Download", + "type": "boolean" + }, + "name": { + "title": "Name", + "type": "string" + }, + "onboard_family": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Onboard Family" + }, + "package": { + "title": "Package", + "type": "string" + }, + "testsign_cert": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Testsign Cert" + }, + "xenclean_path": { + "title": "Xenclean Path", + "type": "string" + } + }, + "title": "WinGuestToolDef", + "type": "object" + }, + "XOConfig": { + "additionalProperties": false, + "properties": { + "cli": { + "title": "Cli", + "type": "string" + } + }, + "title": "XOConfig", + "type": "object" + } + }, + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "description": "JSON Schema for validating config.toml, config.local.toml, and config.NAME.toml files. Properties are all optional since overlay files only carry the keys they override; the base config.toml provides the rest.", + "properties": { + "$schema": { + "description": "Path to the JSON schema used by the editor for validation and autocompletion.", + "type": "string" + }, + "dns_server": { + "title": "Dns Server", + "type": "string" + }, + "guest_tools": { + "$ref": "#/$defs/GuestToolsConfig" + }, + "host": { + "$ref": "#/$defs/HostConfig" + }, + "hosts": { + "additionalProperties": { + "$ref": "#/$defs/HostOverride" + }, + "title": "Hosts", + "type": "object" + }, + "include": { + "default": [], + "description": "List of TOML files to load and deep-merge before this file's content. Paths are relative to this file's directory.", + "items": { + "type": "string" + }, + "type": "array" + }, + "install": { + "$ref": "#/$defs/InstallConfig" + }, + "network": { + "$ref": "#/$defs/NetworkConfig" + }, + "objects_name_prefix": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Objects Name Prefix" + }, + "pxe": { + "$ref": "#/$defs/PXEConfig" + }, + "ssh": { + "$ref": "#/$defs/SSHConfig" + }, + "storage": { + "$ref": "#/$defs/StorageConfig" + }, + "tools": { + "$ref": "#/$defs/ToolsConfig" + }, + "vm": { + "$ref": "#/$defs/VMConfig" + }, + "volume_size": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + } + ], + "title": "Volume Size" + }, + "write_volume_align": { + "title": "Write Volume Align", + "type": "integer" + }, + "write_volume_cap": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + } + ], + "title": "Write Volume Cap" + }, + "xo": { + "$ref": "#/$defs/XOConfig" + } + }, + "title": "XCP-ng Tests Configuration Schema", + "type": "object" +} + diff --git a/config.toml b/config.toml new file mode 100644 index 000000000..83871e8f9 --- /dev/null +++ b/config.toml @@ -0,0 +1,277 @@ +"$schema" = "./config-schema.json" +# Configuration file for XCP-ng tests +# To be customized for your environment + +# The following prefix will be added to the `name-label` parameter of XAPI objects +# that the tests will create or import, such as VMs and SRs. +# Default value: [your login/user] +# Prefix for test-created objects in XAPI (empty = use username) +objects_name_prefix = "" + +# This should be a working DNS server that's not used by any VM images. +dns_server = "1.1.1.1" + +# Default size of VDIs created for storage tests. +volume_size = "1 GiB" + +# Maximum amount of data written to a volume during storage tests. +write_volume_cap = "2 GiB" + +# Block size to align span positions to when writing in volumes. +write_volume_align = 1 + +[host] +# Default user and password to connect to a host through XAPI +# Note: this won't be used for SSH. +# You need to have an SSH key into the hosts' /root/.ssh/authorized_keys. +default_user = "root" +default_password = "" + +[hosts] +# Hosts to run tests/tools against. Each key is the address (hostname|ip) of +# the master host of a pool. Settings may also override the global defaults +# below (repositories, disabled_repositories, hosting_pool) per host. +# Connection settings (user/password/skip_xo_config) default to the `host` +# section values. +# "10.0.0.1" = { user = "root", password = "" } +# "testhost1" = { user = "root", password = "", skip_xo_config = true } +# A plain host with default settings: +# "10.3.0.56" = {} +# A host overriding the default repositories: +# "10.3.0.57" = { repositories = ["xcp-ng-updates"], disabled_repositories = ["*"] } + +[tools.update] +# Default settings for the `update` tool, applied to all hosts in the [hosts] +# table. Values set per host in [hosts] override these. +# Repositories to enable when updating. +# repositories = ["xcp-ng-base", "xcp-ng-updates"] +# Repositories to disable when updating. "*" disables all. +# disabled_repositories = ["*"] +# Address (hostname|ip) of hosting pool's master host (nested context). +# hosting_pool = "10.30.0.50" + +[network] +# Network names and descriptions +mgmt = "Pool-wide network associated with eth0" + +# List of NICs available on host for network tests. +# example: free_nics = ["eth1", "eth2"] +free_nics = [] + +[pxe] +# PXE configuration server for automated XCP-ng installation +config_server = "pxe" +# Server on MGMT network, where ARP tables can reveal the MACs +arp_server = "pxe" + +[vm] +# Default VM images location +# Values can be either full URLs or only partial URLs that will be automatically appended to def_url +def_url = "http://pxe/images/" + +# Whether to cache VMs on the test host, that is import them only if not already +# present in the target SR. This also causes the VM to be cloned at the beginning +# of each test module, so that the original VM remains untouched. +# /!\ The VM identifier in cache is simply the URL where it was imported from. +# No checksum or date is checked. +# A cached VM is just a VM which has a special description. +# Example description: "[Cache for http://example.com/images/filename.xva]" +# Delete the VM to remove it from cache. +# This setting affects VMs managed by the `imported_vm` fixture. +cache_imported = false + +# In some cases, we may prefer to favour a local SR to store test VM disks, +# to avoid latency or unstabilities related to network or shared file servers. +# However it's not good practice to make a local SR the default SR for a pool of several hosts. +# Hence this configuration value that you can set to `local` so that our tests use this SR by default. +# This setting affects VMs managed by the `imported_vm` fixture. +# Possible values: +# - 'default': keep using the pool's default SR +# - 'local': use the first local SR found instead +# - A UUID of the SR to be used +default_sr = "default" + +[vm.images] +# VM image definitions: name -> filename or URL +# Values can be either full URLs or only partial URLs appended to vm.def_url +"mini-linux-x86_64-bios" = "alpine-minimal-3.12.0.xva" +"mini-linux-x86_64-uefi" = "alpine-uefi-minimal-3.12.0.xva" + +[vm.equivalents] +# Image equivalences for caching/deduplication +# Maps test image IDs to other IDs they can be substituted with +# "test_id_1" = "test_id_2" + +[xo] +# Path to the xo-cli utility +# Default value: xo-cli found in PATH +cli = "xo-cli" + +[install] +# Installation configuration +# Path to the iso-remaster utility script +# iso_remaster = "/home/user/src/xcpng/xcp/scripts/iso-remaster/iso-remaster.sh" + +[install.answerfiles] +# Base answer files for installation +# Password hash is computed from host.default_password at config load time +# When used, placeholder will be replaced with the actual hash +INSTALL = { TAG = "installation", CONTENTS = [ + { TAG = "root-password", type = "hash", CONTENTS = "" }, + { TAG = "timezone", CONTENTS = "Europe/Paris" }, + { TAG = "keymap", CONTENTS = "us" } +] } +UPGRADE = { TAG = "installation", mode = "upgrade" } +RESTORE = { TAG = "restore" } + +[install.isos] +# Base URL for XCP-ng installer ISOs +base_url = "https://updates.xcp-ng.org/isos/" +# Local cache directory for downloaded ISOs +cache_dir = "/home/user/iso" + +[install.isos.definitions] +# XCP-ng installer ISO definitions +# path can be: +# - absolute filename +# - absolute URL +# - path relative to base_url +# Note the dirname part is ignored when looking in cache_dir, abuse this +# for local-only ISO with things like "locally-built/my.iso" or "xs/8.3.iso". +# If 'net-only' is set to 'True' only source of type URL will be possible. +# By default the parameter is set to False. +"83nightly" = { path = "http://unconfigured.iso", unsigned = true } +"830" = { path = "8.3/xcp-ng-8.3.0.iso" } +"82nightly" = { path = "http://unconfigured.iso", unsigned = true } +"821.1" = { path = "8.2/xcp-ng-8.2.1-20231130.iso" } +"821" = { path = "8.2/xcp-ng-8.2.1.iso" } +"820" = { path = "8.2/xcp-ng-8.2.0.iso" } +"81" = { path = "8.1/xcp-ng-8.1.0-2.iso" } +"80" = { path = "8.0/xcp-ng-8.0.0.iso" } +"76" = { path = "7.6/xcp-ng-7.6.0.iso" } +"75" = { path = "7.5/xcp-ng-7.5.0-2.iso" } +"xs8" = { path = "XenServer8_2024-03-18.iso" } +"ch821.1" = { path = "CitrixHypervisor-8.2.1-2306-install-cd.iso" } +"ch821" = { path = "CitrixHypervisor-8.2.1-install-cd.iso" } + +[guest_tools] +# Guest tools ISO download location +download_url = "http://pxe/isos/" + +[guest_tools.win] +# Definitions of Windows guest tool ISOs to be tested + +[guest_tools.win.stable] +# ISO name on SR or subpath of download_url +name = "guest-tools-win.iso" +# Whether ISO should be downloaded from download_url +download = true +# ISO-relative path of MSI file to be installed +package = "package\\XenDrivers-x64.msi" +# ISO-relative path of XenClean script +xenclean_path = "package\\XenClean\\x64\\Invoke-XenClean.ps1" +# ISO-relative path of root cert file to be installed before guest tools (optional) +testsign_cert = "testsign\\XCP-ng_Test_Signer.crt" +# What's the onboard family of our tools? This is equal to the WinPV VENDOR_NAME value +onboard_family = "XCP-ng" + +[guest_tools.other] +# Definition of ISO containing other guest tools to be tested +# ISO name on SR or subpath of download_url +name = "other-guest-tools-win.iso" +# Whether ISO should be downloaded from download_url +download = false + +[guest_tools.installed] +# Definitions of other guest tools contained in guest_tools.other ISO + +[guest_tools.installed."xcp-ng-9.0.9000"] +# Whether we are installing MSI files ("msi"), bare .inf drivers ("inf") +# or nothing in case of Windows Update (absent or null) +type = "msi" +# ISO-relative path of this guest tool +path = "xcp-ng-9.0.9000" +# "path"-relative path of MSI or driver files to be installed +package = "package\\XenDrivers-x64.msi" +# Relative path of root cert file (optional) +testsign_cert = "testsign\\XCP-ng_Test_Signer.crt" +# Whether this guest tool version wants vendor device to be activated (optional, defaults to False) +# Note: other guest tools may not install correctly with this setting enabled +vendor_device = false +# Can we upgrade automatically from this guest tool to our tools? +upgradable = true +# What is the expected onboarding phase after running XenClean when this tool is installed? (optional) +# See test_xenclean.py ONBOARDING_PHASES for details +onboarding_phase = "see test_xenclean.py ONBOARDING_PHASES" + +[guest_tools.installed.vendor] +# Vendor device-specific guest tools +vendor_device = true +upgradable = false + +[ssh] +# Public keys for a private key available to the test runner +pubkey = """ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMnN/wVdQqHA8KsndfrLS7fktH/IEgxoa533efuXR6rw XCP-ng CI +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDKz9uQOoxq6Q0SQ0XTzQHhDolvuo/7EyrDZsYQbRELhcPJG8MT/o5u3HyJFhIP2+HqBSXXgmqRPJUkwz9wUwb2sUwf44qZm/pyPUWOoxyVtrDXzokU/uiaNKUMhbnfaXMz6Ogovtjua63qld2+ZRXnIgrVtYKtYBeu/qKGVSnf4FTOUKl1w3uKkr59IUwwAO8ay3wVnxXIHI/iJgq6JBgQNHbn3C/SpYU++nqL9G7dMyqGD36QPFuqH/cayL8TjNZ67TgAzsPX8OvmRSqjrv3KFbeSlpS/R4enHkSemhgfc8Z2f49tE7qxWZ6x4Uyp5E6ur37FsRf/tEtKIUJGMRXN XCP-ng CI""" +# Maximum number of SSH output lines to log before truncating +output_max_lines = 20 +# If true, strips SSH banners from output +ignore_banner = false + +[storage] +# Storage device configurations (all optional, leave empty if not used) + +[storage.cifs_iso] +# Default CIFS ISO device config: +# location = r"\\10.0.0.2\" +# username = "" +# cifspassword = "" +# type = "cifs" +# vers = "<1.0> or <3.0>" + +[storage.cephfs] +# CephFS storage configuration +# server = "10.0.0.2" +# serverpath = "/vms" +# options = "name=,secret=" + +[storage.linstor] +# LINSTOR storage configuration +redundancy = 2 + +[storage.lvmohba] +# LVM over HBA storage configuration +# SCSIid = "id" + +[storage.lvmoiscsi] +# LVM over iSCSI storage configuration +# target = "192.168.1.1" +# port = "3260" +# targetIQN = "target.example" +# SCSIid = "id" + +[storage.moosefs] +# MooseFS storage configuration +# masterhost = "mfsmaster" +# masterport = "9421" +# rootpath = "/vms" + +[storage.nfs] +# Default NFS device config: +# URL/Hostname of NFS server +# server = "10.0.0.2" +# Path to shared mountpoint +# serverpath = "/path/to/shared/mount" + +[storage.nfs4] +# Default NFS4+ only device config: +# URL/Hostname of NFS server +# server = "10.0.0.2" +# Path to shared mountpoint +# serverpath = "/path_to_shared_mount" +# nfsversion = "4.1" + +[storage.nfs_iso] +# Default NFS ISO device config: +# URL/Hostname of NFS server and path to shared mountpoint +# location = "10.0.0.2:/path/to/shared/mount" diff --git a/conftest.py b/conftest.py index 9073ac919..256907ff9 100644 --- a/conftest.py +++ b/conftest.py @@ -14,7 +14,6 @@ from cryptography.hazmat.primitives.serialization import SSHCertPrivateKeyTypes from packaging import version -import lib.config as global_config from lib import pxe from lib.common import ( Defer, @@ -30,6 +29,7 @@ vm_image, wait_for, ) +from lib.config_loader import apply_override, config, warn_legacy_data_py from lib.host import Host from lib.netutil import is_ipv6 from lib.pool import Pool @@ -46,13 +46,6 @@ from typing import Any, Dict, Generator, Iterable, Sequence -# Do we cache VMs? -try: - from data import CACHE_IMPORTED_VM -except ImportError: - CACHE_IMPORTED_VM = False -assert CACHE_IMPORTED_VM in [True, False] - class SplitCommaAction(Action): def __call__(self, parser: ArgumentParser, namespace: Namespace, values: str | Sequence[str] | None, option_string: str | None = None) -> None: @@ -66,6 +59,19 @@ def __call__(self, parser: ArgumentParser, namespace: Namespace, values: str | S # pytest hooks def pytest_addoption(parser: pytest.Parser) -> None: + parser.addoption( + "--config", + action="store", + default=None, + help="Config overlay: a .toml file path or profile name (default: config.local.toml or XCPNG_CONFIG)", + ) + parser.addoption( + "--config-value", + action="append", + default=[], + metavar="KEY=VALUE", + help="Override a config value, e.g. host.default_password=foo (repeatable; highest priority)", + ) parser.addoption( "--nest", action="store", @@ -93,13 +99,13 @@ def pytest_addoption(parser: pytest.Parser) -> None: parser.addoption( "--ignore-ssh-banner", action="store_true", - default=False, + default=None, help="Ignore SSH banners when SSH commands are executed" ) parser.addoption( "--ssh-output-max-lines", action="store", - default=20, + default=None, help="Max lines to output in a ssh log (0 if no limit)" ) parser.addoption( @@ -120,39 +126,44 @@ def pytest_addoption(parser: pytest.Parser) -> None: parser.addoption( "--volume-size", action="store", - default="1GiB", + default=None, help="Default volume size for tests." " Accepts sizes like '1GiB', '2.5TiB', or symbolic values 'VHD_MAX', 'QCOW2_MAX'." ) parser.addoption( "--write-volume-cap", action="store", - default="2GiB", + default=None, help="Maximum amount of data written to a volume." " Accepts sizes like '1GiB', '2.5TiB', or symbolic values 'VHD_MAX', 'QCOW2_MAX'." ) parser.addoption( "--write-volume-align", action="store", - default="1", + default=None, help="Block size to align span positions to when writing in volumes." " Accepts sizes like '512', '4KiB', '1MiB'. A value of 1 is equivalent to no alignment." ) def pytest_configure(config: pytest.Config) -> None: - global_config.ignore_ssh_banner = config.getoption('--ignore-ssh-banner') + warn_legacy_data_py() + apply_override(config.getoption("--config"), config.getoption("--config-value")) + from lib.config_loader import config as global_config + ignore_ssh_banner = config.getoption('--ignore-ssh-banner') + if ignore_ssh_banner is not None: + global_config.ssh.ignore_banner = bool(ignore_ssh_banner) ssh_output_max_lines = config.getoption('--ssh-output-max-lines') - assert ssh_output_max_lines is not None - global_config.ssh_output_max_lines = int(ssh_output_max_lines) + if ssh_output_max_lines is not None: + global_config.ssh.output_max_lines = int(ssh_output_max_lines) volume_size = config.getoption('--volume-size') - assert volume_size is not None - global_config.volume_size = parse_size(volume_size) + if volume_size is not None: + global_config.volume_size = parse_size(volume_size) write_volume_cap = config.getoption('--write-volume-cap') - assert write_volume_cap is not None - global_config.write_volume_cap = parse_size(write_volume_cap) + if write_volume_cap is not None: + global_config.write_volume_cap = parse_size(write_volume_cap) write_volume_align = config.getoption('--write-volume-align') - assert write_volume_align is not None - global_config.write_volume_align = parse_size(write_volume_align) + if write_volume_align is not None: + global_config.write_volume_align = parse_size(write_volume_align) def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: if "vm_ref" in metafunc.fixturenames: @@ -338,9 +349,12 @@ def cleanup_hosts() -> None: # a list of master hosts, each from a different pool hosts_args = pytestconfig.getoption("hosts") - assert hosts_args is not None - hosts_split = [hostlist.split(',') for hostlist in hosts_args] - hostname_list = list(itertools.chain(*hosts_split)) + if hosts_args: + hosts_split = [hostlist.split(',') for hostlist in hosts_args] + hostname_list = list(itertools.chain(*hosts_split)) + else: + # no --hosts option: fall back to the hosts defined in the config file + hostname_list = list(config.hosts) try: host_list = [setup_host(hostname_or_ip, config=pytestconfig) @@ -350,7 +364,7 @@ def cleanup_hosts() -> None: raise if not host_list: - pytest.fail("This test requires at least one --hosts parameter") + pytest.fail("This test requires at least one host: pass --hosts or define hosts in the config file") yield host_list cleanup_hosts() @@ -549,16 +563,12 @@ def _host_disks(host: Host, hosts_cli_disks: list[DiskDevName] | None) -> Iterab # LUNs reserved for lvmohba/lvmoiscsi: sort them to the end so they are # only picked if no other disk is available. reserved_wwns: set[str] = set() - try: - import data - for key in ('LVMOHBA_DEVICE_CONFIG', 'LVMOISCSI_DEVICE_CONFIG'): - cfg = getattr(data, key, None) - if isinstance(cfg, dict): - scsiid = cfg.get('SCSIid', '').lower().removeprefix('0x') - if len(scsiid) >= 16: - reserved_wwns.add(scsiid[:16]) - except ImportError: - pass + for cfg_name in ('lvmohba', 'lvmoiscsi'): + cfg = getattr(config.storage, cfg_name, None) + if cfg is not None: + scsiid = (cfg.SCSIid or '').lower().removeprefix('0x') + if len(scsiid) >= 16: + reserved_wwns.add(scsiid[:16]) if reserved_wwns: logging.debug("reserved WWNs (lvmohba/lvmoiscsi): %s", reserved_wwns) ret = { @@ -625,9 +635,9 @@ def imported_vm(host: Host, vm_ref: str) -> Generator[VM, None, None]: name = vm_orig.name() logging.info(">> Reuse VM %s (%s) on host %s" % (vm_ref, name, host)) else: - vm_orig = host.import_vm(vm_ref, host.main_sr_uuid(), use_cache=CACHE_IMPORTED_VM) + vm_orig = host.import_vm(vm_ref, host.main_sr_uuid(), use_cache=config.vm.cache_imported) - if CACHE_IMPORTED_VM: + if config.vm.cache_imported: # Clone the VM before running tests, so that the original VM remains untouched logging.info(">> Clone cached VM before running tests") vm = vm_orig.clone() @@ -638,7 +648,7 @@ def imported_vm(host: Host, vm_ref: str) -> Generator[VM, None, None]: yield vm # teardown - if CACHE_IMPORTED_VM or not is_uuid(vm_ref): + if config.vm.cache_imported or not is_uuid(vm_ref): logging.info("<< Destroy VM") vm.destroy(verify=True) @@ -904,11 +914,11 @@ def second_network(pytestconfig: pytest.Config, host: Host) -> str: @pytest.fixture(scope='module') def nfs_iso_device_config() -> dict[str, Any]: - return global_config.sr_device_config("NFS_ISO_DEVICE_CONFIG", required=['location']) + return config.sr_device_config("NFS_ISO_DEVICE_CONFIG", required=['location']) @pytest.fixture(scope='module') def cifs_iso_device_config() -> dict[str, Any]: - return global_config.sr_device_config("CIFS_ISO_DEVICE_CONFIG") + return config.sr_device_config("CIFS_ISO_DEVICE_CONFIG") @pytest.fixture(scope='module') def nfs_iso_sr(host: Host, nfs_iso_device_config: dict[str, Any]) -> Generator[SR, None, None]: diff --git a/data.py-dist b/data.py-dist deleted file mode 100644 index 183af2b33..000000000 --- a/data.py-dist +++ /dev/null @@ -1,271 +0,0 @@ -# Configuration file, to be adapted to one's needs - -from __future__ import annotations - -import os - -from lib.common import hash_password - -from typing import TYPE_CHECKING, Any - -if TYPE_CHECKING: - from lib.typing import IsoImageDef - -# Default user and password to connect to a host through XAPI -# Note: this won't be used for SSH. -# You need to have an SSH key into the hosts' /root/.ssh/authorized_keys. -HOST_DEFAULT_USER = "root" -HOST_DEFAULT_PASSWORD = "" - -HOST_DEFAULT_PASSWORD_HASH = hash_password(HOST_DEFAULT_PASSWORD) - -# Public keys for a private keys available to the test runner -TEST_SSH_PUBKEY = """ -ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMnN/wVdQqHA8KsndfrLS7fktH/IEgxoa533efuXR6rw XCP-ng CI -ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDKz9uQOoxq6Q0SQ0XTzQHhDolvuo/7EyrDZsYQbRELhcPJG8MT/o5u3HyJFhIP2+HqBSXXgmqRPJUkwz9wUwb2sUwf44qZm/pyPUWOoxyVtrDXzokU/uiaNKUMhbnfaXMz6Ogovtjua63qld2+ZRXnIgrVtYKtYBeu/qKGVSnf4FTOUKl1w3uKkr59IUwwAO8ay3wVnxXIHI/iJgq6JBgQNHbn3C/SpYU++nqL9G7dMyqGD36QPFuqH/cayL8TjNZ67TgAzsPX8OvmRSqjrv3KFbeSlpS/R4enHkSemhgfc8Z2f49tE7qxWZ6x4Uyp5E6ur37FsRf/tEtKIUJGMRXN XCP-ng CI -""" - -# The following prefix will be added to the `name-label` parameter of XAPI objects -# that the tests will create or import, such as VMs and SRs. -# Default value: [your login/user] -# OBJECTS_NAME_PREFIX = "[TEST]" -OBJECTS_NAME_PREFIX = None - -# Override settings for specific hosts -# skip_xo_config allows to not touch XO's configuration regarding the host -# Else the default behaviour is to add the host to XO servers at the beginning -# of the testing session and remove it at the end. -HOSTS: dict[str, dict[str, Any]] = { -# "10.0.0.1": {"user": "root", "password": ""}, -# "testhost1": {"user": "root", "password": "", 'skip_xo_config': True}, -} - -NETWORKS = { - "MGMT": "Pool-wide network associated with eth0", -} - -# PXE config server for automated XCP-ng installation -PXE_CONFIG_SERVER = 'pxe' - -# server on MGMT network, where ARP tables can reveal the MACs -ARP_SERVER = PXE_CONFIG_SERVER - -# Default VM images location -DEF_VM_URL = 'http://pxe/images/' - -# Guest tools ISO download location -ISO_DOWNLOAD_URL = 'http://pxe/isos/' - -# Definitions of Windows guest tool ISOs to be tested -WIN_GUEST_TOOLS_ISOS = { - "stable": { - # ISO name on SR or subpath of ISO_DOWNLOAD_URL - "name": "guest-tools-win.iso", - # Whether ISO should be downloaded from ISO_DOWNLOAD_URL - "download": True, - # ISO-relative path of MSI file to be installed - "package": "package\\XenDrivers-x64.msi", - # ISO-relative path of XenClean script - "xenclean_path": "package\\XenClean\\x64\\Invoke-XenClean.ps1", - # ISO-relative path of root cert file to be installed before guest tools (optional) - "testsign_cert": "testsign\\XCP-ng_Test_Signer.crt", - # What's the onboard family of our tools? This is equal to the WinPV VENDOR_NAME value - "onboard_family": "XCP-ng", - }, - # Add more guest tool ISOs here as needed -} - -# Definition of ISO containing other guest tools to be tested -OTHER_GUEST_TOOLS_ISO = { - "name": "other-guest-tools-win.iso", - "download": False, -} - -# Definitions of other guest tools contained in OTHER_GUEST_TOOLS_ISO -OTHER_GUEST_TOOLS = { - "xcp-ng-9.0.9000": { - # Whether we are installing MSI files ("msi"), bare .inf drivers ("inf") - # or nothing in case of Windows Update (absent or null) - "type": "msi", - # ISO-relative path of this guest tool - "path": "xcp-ng-9.0.9000", - # "path"-relative path of MSI or driver files to be installed - "package": "package\\XenDrivers-x64.msi", - # Relative path of root cert file (optional) - "testsign_cert": "testsign\\XCP-ng_Test_Signer.crt", - # Whether this guest tool version wants vendor device to be activated (optional, defaults to False) - # Note: other guest tools may not install correctly with this setting enabled - "vendor_device": False, - - # Can we upgrade automatically from this guest tool to our tools? - "upgradable": True, - # What is the expected onboarding phase after running XenClean when this tool is installed? (optional) - "onboarding_phase": "see test_xenclean.py ONBOARDING_PHASES", - }, - "vendor": { - "vendor_device": True, - "upgradable": False, - }, -} - -# Tools -TOOLS: dict[str, str] = { -# "iso-remaster": "/home/user/src/xcpng/xcp/scripts/iso-remaster/iso-remaster.sh", -# "xo-cli": "xo-cli", -} - -# Values can be either full URLs or only partial URLs that will be automatically appended to DEF_VM_URL -VM_IMAGES = { - 'mini-linux-x86_64-bios': 'alpine-minimal-3.12.0.xva', - 'mini-linux-x86_64-uefi': 'alpine-uefi-minimal-3.12.0.xva' -} - -ISO_IMAGES_BASE = "https://updates.xcp-ng.org/isos/" -ISO_IMAGES_CACHE = "/home/user/iso" -# ISO_IMAGES path can be: -# - absolute filename -# - absolute URL -# - path relative to ISO_IMAGES_BASE URL -# Note the dirname part is ignored when looking in ISO_IMAGES_CACHE, abuse this -# for local-only ISO with things like "locally-built/my.iso" or "xs/8.3.iso". -# If 'net-only' is set to 'True' only source of type URL will be possible. -# By default the parameter is set to False. -ISO_IMAGES: dict[str, "IsoImageDef"] = { - '83nightly': {'path': os.environ.get("XCPNG83_NIGHTLY", - "http://unconfigured.iso"), - 'unsigned': True}, - # FIXME: no such symlimk + useless without 'net-url' - #'83nightlynet': {'path': "http://pxe/isos/xcp-ng-8.3-ci-netinstall-latest"}, - # 'net-url': 'fake", - # 'net-only': True}, - '830': {'path': "8.3/xcp-ng-8.3.0.iso", - #'net-url': "http://server/installers/xcp-ng/8.3.0", - }, - ## FIXME: only a compensation for the lack of 83nightlynet - #'830net': {'path': "8.3/xcp-ng-8.3.0-netinstall.iso", - # 'net-url': "http://server/installers/xcp-ng/8.3.0", - # 'net-only': True}, - '82nightly': {'path': os.environ.get("XCPNG82_NIGHTLY", - "http://unconfigured.iso"), - 'unsigned': True}, - '821.1': {'path': "8.2/xcp-ng-8.2.1-20231130.iso", - #'net-url': f"http://{PXE_CONFIG_SERVER}/installers/xcp-ng/8.2.1-refreshed/", - }, - '821': {'path': "8.2/xcp-ng-8.2.1.iso"}, - '820': {'path': "8.2/xcp-ng-8.2.0.iso"}, - '81': {'path': "8.1/xcp-ng-8.1.0-2.iso"}, - '80': {'path': "8.0/xcp-ng-8.0.0.iso"}, - '76': {'path': "7.6/xcp-ng-7.6.0.iso"}, - '75': {'path': "7.5/xcp-ng-7.5.0-2.iso"}, - 'xs8': {'path': "XenServer8_2024-03-18.iso"}, - 'ch821.1': {'path': "CitrixHypervisor-8.2.1-2306-install-cd.iso"}, - 'ch821': {'path': "CitrixHypervisor-8.2.1-install-cd.iso"}, -} - -# In some cases, we may prefer to favour a local SR to store test VM disks, -# to avoid latency or unstabilities related to network or shared file servers. -# However it's not good practice to make a local SR the default SR for a pool of several hosts. -# Hence this configuration value that you can set to `local` so that our tests use this SR by default. -# This setting affects VMs managed by the `imported_vm` fixture. -# Possible values: -# - 'default': keep using the pool's default SR -# - 'local': use the first local SR found instead -# - A UUID of the SR to be used -DEFAULT_SR = 'default' - -# Whether to cache VMs on the test host, that is import them only if not already -# present in the target SR. This also causes the VM to be cloned at the beginning -# of each test module, so that the original VM remains untouched. -# /!\ The VM identifier in cache is simply the URL where it was imported from. -# No checksum or date is checked. -# A cached VM is just a VM which has a special description. -# Example description: "[Cache for http://example.com/images/filename.xva]" -# Delete the VM to remove it from cache. -# This setting affects VMs managed by the `imported_vm` fixture. -CACHE_IMPORTED_VM = False - -# Default LINSTOR redundancy configuration for creating SRs. -LINSTOR_REDUNDANCY = 2 - -# Default NFS device config: -NFS_DEVICE_CONFIG: dict[str, str] = { -# 'server': '10.0.0.2', # URL/Hostname of NFS server -# 'serverpath': '/path/to/shared/mount' # Path to shared mountpoint -} - -# Default NFS4+ only device config: -NFS4_DEVICE_CONFIG: dict[str, str] = { -# 'server': '10.0.0.2', # URL/Hostname of NFS server -# 'serverpath': '/path_to_shared_mount' # Path to shared mountpoint -# 'nfsversion': '4.1' -} - -# Default NFS ISO device config: -NFS_ISO_DEVICE_CONFIG: dict[str, str] = { -# 'location': '10.0.0.2:/path/to/shared/mount' # URL/Hostname of NFS server and path to shared mountpoint -} - -# Default CIFS ISO device config: -CIFS_ISO_DEVICE_CONFIG: dict[str, str] = { -# 'location': r'\\10.0.0.2\', -# 'username': '', -# 'cifspassword': '', -# 'type': 'cifs', -# 'vers': '<1.0> or <3.0>' -} - -CEPHFS_DEVICE_CONFIG: dict[str, str] = { -# 'server': '10.0.0.2', -# 'serverpath': '/vms' -} - -MOOSEFS_DEVICE_CONFIG: dict[str, str] = { -# 'masterhost': 'mfsmaster', -# 'masterport': '9421', -# 'rootpath': '/vms' -} - -LVMOISCSI_DEVICE_CONFIG: dict[str, str] = { -# 'target': '192.168.1.1', -# 'port': '3260', -# 'targetIQN': 'target.example', -# 'SCSIid': 'id' -} - -LVMOHBA_DEVICE_CONFIG: dict[str, str] = { -# 'SCSIid': 'wwid' -} - -BASE_ANSWERFILES = dict( - INSTALL={ - "TAG": "installation", - "CONTENTS": ( - {"TAG": "root-password", - "type": "hash", - "CONTENTS": HOST_DEFAULT_PASSWORD_HASH}, - {"TAG": "timezone", - "CONTENTS": "Europe/Paris"}, - {"TAG": "keymap", - "CONTENTS": "us"}, - ), - }, - UPGRADE={ - "TAG": "installation", - "mode": "upgrade", - }, - RESTORE={ - "TAG": "restore", - }, -) - -IMAGE_EQUIVS: dict[str, str] = { -# 'install.test::Nested::install[bios-830-ext]-vm1-607cea0c825a4d578fa5fab56978627d8b2e28bb': -# 'install.test::Nested::install[bios-830-ext]-vm1-addb4ead4da49856e1d2fb3ddf4e31027c6b693b', -} - -# This should be a working DNS server that's not used by any VM images. -TEST_DNS_SERVER = "1.1.1.1" - -# List of NICs available on host for network tests. -# example: HOST_FREE_NICS: list[str] = ['eth1', 'eth2'] -HOST_FREE_NICS: list[str] = [] diff --git a/jobs.py b/jobs.py index 0281c9aa7..f540023b0 100755 --- a/jobs.py +++ b/jobs.py @@ -7,6 +7,7 @@ import sys from lib.commands import ssh +from lib.config_loader import add_config_options, load_config from typing import NotRequired, TypedDict, cast @@ -815,14 +816,30 @@ def extract_tests(cmd: list[str]) -> set[str]: if error: sys.exit(1) +def _config_hosts(args: argparse.Namespace) -> str | None: + """Return the pool masters listed in the config's [hosts], as a comma-separated string, or None.""" + cfg = load_config(override=args.config, config_values=args.config_value) + hosts = list(cfg.hosts.keys()) + return ",".join(hosts) if hosts else None + + def action_run(args: argparse.Namespace) -> None: - cmd = build_pytest_cmd(JOBS[args.job], args.hosts, None, args.pytest_args) + hosts = args.hosts or _config_hosts(args) + if hosts is None: + print("Error: no hosts provided. Pass a comma-separated list of pool masters as the positional " + "hosts argument, or define them in the [hosts] section of the config.", file=sys.stderr) + sys.exit(1) + cmd = build_pytest_cmd(JOBS[args.job], hosts, None, args.pytest_args) + if args.config is not None: + cmd += ["--config", str(args.config)] + for config_value in args.config_value: + cmd += ["--config-value", config_value] print(subprocess.list2cmdline(cmd)) if args.print_only: return # check that enough pool masters have been provided - nb_pools = len(args.hosts.split(",")) + nb_pools = len(hosts.split(",")) job_nb_pools = JOBS[args.job]["nb_pools"] assert isinstance(job_nb_pools, int) if nb_pools < job_nb_pools: @@ -841,35 +858,43 @@ def action_run(args: argparse.Namespace) -> None: def main() -> None: parser = argparse.ArgumentParser(description="Manage test jobs") + common_parser = add_config_options(parser) + subparsers = parser.add_subparsers(dest="action", metavar="action") subparsers.required = True - list_parser = subparsers.add_parser("list", help="list available jobs.") + list_parser = subparsers.add_parser("list", help="list available jobs.", parents=[common_parser]) list_parser.set_defaults(func=action_list) - run_parser = subparsers.add_parser("show", help="show details about a job definition.") - run_parser.add_argument("job", help="name of the job.", choices=JOBS.keys(), metavar="job") - run_parser.set_defaults(func=action_show) + show_parser = subparsers.add_parser("show", help="show details about a job definition.", parents=[common_parser]) + show_parser.add_argument("job", help="name of the job.", choices=JOBS.keys(), metavar="job") + show_parser.set_defaults(func=action_show) - run_parser = subparsers.add_parser("collect", help="show test collection based on the job definition.") - run_parser.add_argument("job", help="name of the job.", choices=JOBS.keys(), metavar="job") - run_parser.add_argument("-v", "--host-version", help="host version to match VM filters.") - run_parser.add_argument("pytest_args", nargs=argparse.REMAINDER, - help="all additional arguments after the last positional argument will " - "be passed to pytest and replace default job params if needed.") - run_parser.set_defaults(func=action_collect) + collect_parser = subparsers.add_parser("collect", help="show test collection based on the job definition.", + parents=[common_parser]) + collect_parser.add_argument("job", help="name of the job.", choices=JOBS.keys(), metavar="job") + collect_parser.add_argument("-v", "--host-version", help="host version to match VM filters.") + collect_parser.add_argument("pytest_args", nargs=argparse.REMAINDER, + help="all additional arguments after the last positional argument will " + "be passed to pytest and replace default job params if needed.") + collect_parser.set_defaults(func=action_collect) - run_parser = subparsers.add_parser("check", help="run sanity checks on the tests and jobs.") - run_parser.set_defaults(func=action_check) + check_parser = subparsers.add_parser( + "check", help="run sanity checks on the tests and jobs.", parents=[common_parser]) + check_parser.set_defaults(func=action_check) - run_parser = subparsers.add_parser("run", help="run a job.") + run_parser = subparsers.add_parser("run", help="run a job.", parents=[common_parser]) run_parser.add_argument("--print-only", "-p", action="store_true", help="print the command, but don't run it. Must be specified before positional arguments.") run_parser.add_argument("job", help="name of the job to run.", choices=JOBS.keys(), metavar="job") - run_parser.add_argument("hosts", help="master host(s) of pools to run the tests on, comma-separated.") + run_parser.add_argument("hosts", nargs="?", default=None, + help="master host(s) of pools to run the tests on, comma-separated. When omitted, the " + "hosts from the config's [hosts] section are used.") run_parser.add_argument("pytest_args", nargs=argparse.REMAINDER, help="all additional arguments after the last positional argument will " - "be passed to pytest and replace default job params if needed.") + "be passed to pytest and replace default job params if needed. " + "Note: -c/--config and --config-value must be specified before the " + "positional arguments.") run_parser.set_defaults(func=action_run) args = parser.parse_args() diff --git a/lib/__init__.py b/lib/__init__.py index e69de29bb..94c9b98fe 100644 --- a/lib/__init__.py +++ b/lib/__init__.py @@ -0,0 +1 @@ +from lib.config_loader import config as config diff --git a/lib/commands.py b/lib/commands.py index 650b72c2a..3af31b136 100644 --- a/lib/commands.py +++ b/lib/commands.py @@ -7,7 +7,7 @@ import subprocess import tempfile -import lib.config as config +from lib import config from lib.netutil import wrap_ip from typing import TYPE_CHECKING, Generic, Literal, TypeVar, overload @@ -67,12 +67,12 @@ def _ellide_log_lines(log: str) -> str: if log == '': return log - if config.ssh_output_max_lines < 1: + if config.ssh.output_max_lines < 1: return "\n{}".format(log) reduced_message = log.split("\n") - if len(reduced_message) > config.ssh_output_max_lines: - reduced_message = reduced_message[:config.ssh_output_max_lines - 1] + if len(reduced_message) > config.ssh.output_max_lines: + reduced_message = reduced_message[:config.ssh.output_max_lines - 1] reduced_message.append("(...)") return "\n{}".format("\n".join(reduced_message)) @@ -111,7 +111,7 @@ def _ssh( # Fetch banner and remove it to avoid stdout/stderr pollution. banner_res = None - if config.ignore_ssh_banner: + if config.ssh.ignore_banner: with tempfile.NamedTemporaryFile(suffix='.log', prefix='ssh_err_banner_', mode='r') as banner_log_file: banner_res = subprocess.run( ['ssh', f'root@{hostname_or_ip}'] + opts + ['-E', banner_log_file.name] + ['\n'], diff --git a/lib/common.py b/lib/common.py index 527b4dd20..c6eadce6b 100644 --- a/lib/common.py +++ b/lib/common.py @@ -20,9 +20,18 @@ from uuid import UUID import requests -from passlib.hash import sha512_crypt from pydantic import TypeAdapter +from lib.config_loader import config +from lib.passwords import hash_password as hash_password +from lib.sizes import QCOW2_MAX as QCOW2_MAX +from lib.sizes import VHD_MAX as VHD_MAX +from lib.sizes import GiB as GiB +from lib.sizes import KiB as KiB +from lib.sizes import MiB as MiB +from lib.sizes import TiB as TiB +from lib.sizes import parse_size as parse_size + from typing import ( TYPE_CHECKING, Any, @@ -37,42 +46,6 @@ if TYPE_CHECKING: from lib.host import Host - -KiB = 2**10 -MiB = KiB**2 -GiB = KiB**3 -TiB = KiB**4 - -VHD_MAX = 2040 * GiB -QCOW2_MAX = 16 * TiB - 2561 * MiB - -_SYMBOLIC_SIZES: dict[str, int] = { - 'VHD_MAX': VHD_MAX, - 'QCOW2_MAX': QCOW2_MAX, -} - -def parse_size(size_str: str) -> int: - """ - Parse a size string like "2.5TiB", "1GiB", "1024", "VHD_MAX", or "QCOW2_MAX". - """ - symbolic = _SYMBOLIC_SIZES.get(size_str.strip().upper()) - if symbolic is not None: - return symbolic - try: - return int(size_str) - except ValueError: - pass - - size_str = size_str.strip() - for unit, multiplier in [('TiB', TiB), ('GiB', GiB), ('MiB', MiB), ('KiB', KiB)]: - if size_str.endswith(unit): - try: - return int(float(size_str[:-len(unit)].strip()) * multiplier) - except ValueError: - pass - - raise ValueError(f"Cannot parse size: {size_str}") - T = TypeVar("T") HostAddress: TypeAlias = str @@ -89,21 +62,15 @@ class PackageManagerEnum(Enum): # Common VM images used in tests def vm_image(vm_key: str) -> str: - from data import DEF_VM_URL, VM_IMAGES - url = VM_IMAGES[vm_key] + url = config.vm.images.get(vm_key) + if url is None: + raise KeyError(f"VM image key {vm_key} not found") if not url.startswith('http'): - url = DEF_VM_URL + url + url = config.vm.def_url + url return url def prefix_object_name(label: str) -> str: - name_prefix = None - try: - from data import OBJECTS_NAME_PREFIX - name_prefix = OBJECTS_NAME_PREFIX - except ImportError: - pass - if name_prefix is None: - name_prefix = f"[{getpass.getuser()}]" + name_prefix = config.objects_name_prefix or f"[{getpass.getuser()}]" return f"{name_prefix} {label}" def shortened_nodeid(nodeid: str) -> str: @@ -420,8 +387,3 @@ def _param_clear(host: Host, xe_prefix: str, uuid: str, param_name: str) -> None """ Common implementation for param_clear. """ args: dict[str, str | bool | dict[str, str]] = {'uuid': uuid, 'param-name': param_name} host.xe(f'{xe_prefix}-param-clear', args) - -def hash_password(password: str) -> str: - """Hash password for /etc/shadow.""" - # XCP-ng uses sha512 with 5000 rounds by default - return sha512_crypt.using(rounds=5000).hash(password) # type: ignore[no-untyped-call] diff --git a/lib/config.py b/lib/config.py deleted file mode 100644 index 3c0752a87..000000000 --- a/lib/config.py +++ /dev/null @@ -1,15 +0,0 @@ -from lib.common import GiB - -ignore_ssh_banner = False -ssh_output_max_lines = 20 -volume_size = 1 * GiB -write_volume_cap = 2 * GiB -write_volume_align = 1 - -def sr_device_config(datakey: str, *, required: list[str] = []) -> dict[str, str]: - import data # import here to avoid depending on this user file for collecting tests - config = getattr(data, datakey) - for required_field in required: - if required_field not in config: - raise Exception(f"{datakey} lacks mandatory {required_field!r}") - return config diff --git a/lib/config_dump.py b/lib/config_dump.py new file mode 100644 index 000000000..af2ff4c9c --- /dev/null +++ b/lib/config_dump.py @@ -0,0 +1,145 @@ +"""Render config dicts as TOML text (used by migrate_data_py and dump-config).""" + +from __future__ import annotations + +import difflib +import json + +import tomli_w +from pygments import highlight +from pygments.formatters import TerminalFormatter +from pygments.lexers import TOMLLexer + +from lib.typing import ConfigDict, JSONType + +from typing import overload + +_TOML_LEXER = TOMLLexer() +_TOML_FORMATTER = TerminalFormatter() + + +def deep_dict_equal(d1: JSONType, d2: JSONType) -> bool: + """Check if two values are deeply equal.""" + if isinstance(d1, dict) and isinstance(d2, dict): + if set(d1.keys()) != set(d2.keys()): + return False + return all(deep_dict_equal(d1[k], d2[k]) for k in d1) + if isinstance(d1, list) and isinstance(d2, list): + return len(d1) == len(d2) and all(deep_dict_equal(a, b) for a, b in zip(d1, d2)) + # For non-container types, check both type and value + return type(d1) is type(d2) and d1 == d2 + + +@overload +def _strip_password_hashes(obj: ConfigDict) -> ConfigDict: + ... + + +@overload +def _strip_password_hashes(obj: JSONType) -> JSONType: + ... + + +def _strip_password_hashes(obj: JSONType) -> JSONType: + """Recursively strip password hashes for comparison purposes. + + - Replaces $6$... hashes with placeholder + - Converts tuples to lists for consistent comparison + """ + if isinstance(obj, str): + if obj.startswith("$6$"): + return "" + return obj + if isinstance(obj, dict): + return {k: _strip_password_hashes(v) for k, v in obj.items()} + if isinstance(obj, (list, tuple)): + return [_strip_password_hashes(item) for item in obj] + return obj + + +def remove_defaults(config: ConfigDict, base: ConfigDict) -> ConfigDict: + """Remove fields from config that have the same value as in base. + + Ignores password hash differences (strips them for comparison). + """ + result: ConfigDict = {} + + for key, value in config.items(): + if key not in base: + # Key not in base, keep it + result[key] = value + elif isinstance(value, dict) and isinstance((base_value := base.get(key)), dict): + # Recursively check nested dicts + nested = remove_defaults(value, base_value) + if nested: # Only add if there's something left + result[key] = nested + else: + # Strip password hashes before comparing + value_stripped = _strip_password_hashes(value) + base_stripped = _strip_password_hashes(base.get(key)) + if not deep_dict_equal(value_stripped, base_stripped): + # Values differ (ignoring password hashes), keep it + result[key] = value + # else: values are the same, skip it + + return result + + +def colorize_toml(text: str) -> str: + """Apply ANSI syntax coloring to TOML text (for TTY display).""" + return highlight(text, _TOML_LEXER, _TOML_FORMATTER) + + +@overload +def _sorted_recursive(value: ConfigDict) -> ConfigDict: + ... + + +@overload +def _sorted_recursive(value: JSONType) -> JSONType: + ... + + +def _sorted_recursive(value: JSONType) -> JSONType: + """Recursively sort dict keys and drop None values for stable output.""" + if isinstance(value, dict): + return { + k: _sorted_recursive(v) for k, v in sorted(value.items()) if v is not None + } + if isinstance(value, list): + return [_sorted_recursive(item) for item in value if item is not None] + return value + + +def render_toml(config: ConfigDict, with_schema: bool = True) -> str: + """Render config dict to TOML text.""" + data = _sorted_recursive(config) + if with_schema: + data = {"$schema": "./config-schema.json", **data} + return tomli_w.dumps(data, multiline_strings=True) + + +def config_diff( + config_a: ConfigDict, + config_b: ConfigDict, + name_a: str = "a", + name_b: str = "b", + as_json: bool = False, +) -> str: + """Return a unified diff between two configs, ignoring password hashes. + + Empty string when the configs are identical (modulo $6$... password hashes). + """ + norm_a = _strip_password_hashes(config_a) + norm_b = _strip_password_hashes(config_b) + if as_json: + text_a = json.dumps(norm_a, indent=2, ensure_ascii=False, sort_keys=True) + text_b = json.dumps(norm_b, indent=2, ensure_ascii=False, sort_keys=True) + else: + text_a = render_toml(norm_a, with_schema=False) + text_b = render_toml(norm_b, with_schema=False) + return "".join(difflib.unified_diff( + text_a.splitlines(keepends=True), + text_b.splitlines(keepends=True), + fromfile=name_a, tofile=name_b, + )) diff --git a/lib/config_loader.py b/lib/config_loader.py new file mode 100644 index 000000000..cf3c21813 --- /dev/null +++ b/lib/config_loader.py @@ -0,0 +1,590 @@ +from __future__ import annotations + +import argparse +import logging +import os +import tomllib +from pathlib import Path + +from pydantic import BaseModel, Field, field_validator, model_validator + +from lib.passwords import hash_password +from lib.sizes import parse_size +from lib.typing import ConfigDict, JSONType + +from typing import cast, overload + +logger = logging.getLogger(__name__) + +class ConfigError(Exception): + """Raised when the TOML configuration cannot be loaded or validated.""" + + +class WarnOnExtraModel(BaseModel): + """Drop unknown config keys, warning about them instead of failing (extra="ignore").""" + model_config = {"extra": "ignore"} + + @model_validator(mode="before") + @classmethod + def _warn_unknown_fields(cls, data: object) -> object: + if not isinstance(data, dict): + return data + if cls.model_config.get("extra") == "allow": + return data + expected = set(cls.model_fields) | { + f.alias for f in cls.model_fields.values() if f.alias + } + unknown = set(data) - expected + if unknown: + logger.warning( + "[%s] Unknown config key(s) ignored (config may be from a newer project version): %s", + cls.__name__, ", ".join(sorted(unknown)), + ) + return data + + +REPO_ROOT = Path(__file__).resolve().parent.parent + + +class HostConfig(WarnOnExtraModel): + default_user: str + default_password: str + default_password_hash: str = "" + + +class HostOverride(WarnOnExtraModel): + user: str | None = None + password: str | None = None + skip_xo_config: bool | None = None + repositories: list[str] | None = None + disabled_repositories: list[str] | None = None + hosting_pool: str | None = None + + +class NetworkConfig(WarnOnExtraModel): + mgmt: str + free_nics: list[str] + + +class PXEConfig(WarnOnExtraModel): + config_server: str + arp_server: str + + +class VMConfig(WarnOnExtraModel): + def_url: str + cache_imported: bool + default_sr: str + images: dict[str, str] + equivalents: dict[str, str] + + +class IsoImageDef(WarnOnExtraModel): + path: str + net_url: str | None = Field(default=None, alias="net-url") + net_only: bool | None = Field(default=None, alias="net-only") + unsigned: bool | None = None + + model_config = {"populate_by_name": True} + + +class InstallIsosConfig(WarnOnExtraModel): + base_url: str + cache_dir: str + definitions: dict[str, IsoImageDef] + + +class AnswerFileDef(WarnOnExtraModel): + model_config = {"extra": "allow"} + + TAG: str + CONTENTS: str | list[AnswerFileDef] | None = None + + +class InstallConfig(WarnOnExtraModel): + answerfiles: dict[str, AnswerFileDef] + isos: InstallIsosConfig + iso_remaster: str = "" + + +class WinGuestToolDef(WarnOnExtraModel): + name: str + download: bool + package: str + xenclean_path: str + testsign_cert: str | None = None + onboard_family: str | None = None + + +class OtherGuestToolDef(WarnOnExtraModel): + name: str + download: bool + + +class InstalledGuestToolDef(WarnOnExtraModel): + type: str | None = None + path: str | None = None + package: str | None = None + testsign_cert: str | None = None + vendor_device: bool | None = None + upgradable: bool | None = None + onboarding_phase: str | None = None + + +class GuestToolsConfig(WarnOnExtraModel): + download_url: str + win: dict[str, WinGuestToolDef] + other: OtherGuestToolDef + installed: dict[str, InstalledGuestToolDef] + + +class XOConfig(WarnOnExtraModel): + cli: str + + +class SSHConfig(WarnOnExtraModel): + pubkey: str + output_max_lines: int + ignore_banner: bool + + +class LinstorConfig(WarnOnExtraModel): + redundancy: int + + +class NFSConfig(WarnOnExtraModel): + server: str | None = None + serverpath: str | None = None + + +class NFS4Config(WarnOnExtraModel): + server: str | None = None + serverpath: str | None = None + nfsversion: str | None = None + + +class NFSISOConfig(WarnOnExtraModel): + location: str | None = None + + +class CIFSISOConfig(WarnOnExtraModel): + location: str | None = None + username: str | None = None + cifspassword: str | None = None + type: str | None = None + vers: str | None = None + + +class CephFSConfig(WarnOnExtraModel): + server: str | None = None + serverpath: str | None = None + options: str | None = None + + +class MooseFSConfig(WarnOnExtraModel): + masterhost: str | None = None + masterport: str | None = None + rootpath: str | None = None + + +class LVMoHBAConfig(WarnOnExtraModel): + SCSIid: str | None = None + + +class LVMoISCSIConfig(WarnOnExtraModel): + target: str | None = None + port: str | None = None + targetIQN: str | None = None + SCSIid: str | None = None + + +class StorageConfig(WarnOnExtraModel): + nfs: NFSConfig + nfs4: NFS4Config + nfs_iso: NFSISOConfig + cifs_iso: CIFSISOConfig + cephfs: CephFSConfig + moosefs: MooseFSConfig + lvmohba: LVMoHBAConfig + lvmoiscsi: LVMoISCSIConfig + linstor: LinstorConfig + + +class UpdateDefaults(WarnOnExtraModel): + repositories: list[str] = Field(default_factory=list) + disabled_repositories: list[str] = Field(default_factory=list) + hosting_pool: str | None = None + + +class ToolsConfig(WarnOnExtraModel): + update: UpdateDefaults = Field(default_factory=UpdateDefaults) + + +class Config(WarnOnExtraModel): + objects_name_prefix: str | None + dns_server: str + host: HostConfig + hosts: dict[str, HostOverride] + tools: ToolsConfig + network: NetworkConfig + pxe: PXEConfig + vm: VMConfig + install: InstallConfig + guest_tools: GuestToolsConfig + xo: XOConfig + ssh: SSHConfig + storage: StorageConfig + volume_size: int + write_volume_cap: int + write_volume_align: int + + @field_validator("volume_size", "write_volume_cap", mode="before") + @classmethod + def parse_size_str(cls, v: int | str) -> int: + if isinstance(v, str): + return parse_size(v) + return v + + @field_validator("objects_name_prefix", mode="before") + @classmethod + def normalize_objects_name_prefix(cls, v: str | None) -> str | None: + """Convert empty string to None.""" + return None if v == "" else v + + def sr_device_config(self, config_key: str, *, required: list[str] | None = None) -> dict[str, str]: + """Get storage config by key name. Validate required fields.""" + if required is None: + required = [] + storage_cfg = getattr(self.storage, config_key.replace("_DEVICE_CONFIG", "").lower(), None) + if storage_cfg is None: + return {} + cfg = storage_cfg.model_dump(exclude_none=True) + for required_field in required: + if required_field not in cfg: + raise ConfigError(f"Storage config '{config_key}' lacks mandatory '{required_field}'") + return cfg + + +def _load_toml_file(path: Path) -> ConfigDict: + """Load TOML file, dropping loader-level pseudo-keys, and return dict.""" + with open(path, "rb") as f: + data = cast(ConfigDict, tomllib.load(f)) + data.pop("$schema", None) + return data + + +def _require_str_list(value: JSONType, what: str) -> list[str]: + """Return ``value`` as a list of strings, raising ConfigError otherwise.""" + if not isinstance(value, list) or not all(isinstance(item, str) for item in value): + raise ConfigError(f"{what} must be a list of file paths") + return [item for item in value if isinstance(item, str)] + + +def _load_toml_with_includes( + path: Path, + _seen: set[Path] | None = None, + fallback_dir: Path = REPO_ROOT, +) -> ConfigDict: + """Load a TOML file and recursively merge its includes. + + Files listed in the root-level ``include`` key (array of strings) + are loaded and deep-merged before the file's own content. + Includes are resolved relative to the including file's directory first, + then relative to the main xcp-ng-tests directory (fallback_dir). + """ + if _seen is None: + _seen = set() + path = path.resolve() + if path in _seen: + raise ConfigError(f"Cyclic include detected: {path}") + _seen.add(path) + + try: + data = _load_toml_file(path) + includes = _require_str_list(data.pop("include", None) or [], f"'include' in {path}") + + result: ConfigDict = {} + for inc in includes: + inc_path = _resolve_include(path.parent, inc, fallback_dir) + included = _load_toml_with_includes(inc_path, _seen, fallback_dir) + result = _merge_dicts(result, included) + + return _merge_dicts(result, data) + finally: + # Track the recursion stack, not all visited files, so diamond + # includes (A -> [B, C], B -> D, C -> D) are allowed while true + # cycles still raise above. + _seen.discard(path) + + +def _resolve_include(base_dir: Path, inc: str, fallback_dir: Path) -> Path: + """Resolve an ``include`` path: relative to base_dir, then fallback_dir.""" + candidate = base_dir / inc + if candidate.is_file(): + return candidate.resolve() + candidate = fallback_dir / inc + if candidate.is_file(): + return candidate.resolve() + raise FileNotFoundError( + f"Included config file not found: {inc} (looked in {base_dir} and {fallback_dir})" + ) + + +def _resolve_config_override(value: str | Path) -> Path: + """Resolve a -c/--config value to a TOML config file path. + + The value is first tried as given (an absolute path, or relative to the + current directory). When it does not match a file, it is treated as a + short name and ``config.NAME.toml`` is looked up in the directory given + by the XCPNG_CONFIG_DIR env var (when set), then in the xcp-ng-tests + repository root. + """ + candidates = [Path(value)] + if "XCPNG_CONFIG_DIR" in os.environ: + candidates.append(Path(os.environ["XCPNG_CONFIG_DIR"]) / f"config.{value}.toml") + candidates.append(REPO_ROOT / f"config.{value}.toml") + for candidate in candidates: + if candidate.is_file(): + return candidate.resolve() + raise FileNotFoundError( + f"Config file not found for {value!r}: " + f"tried {[str(c) for c in candidates]}" + ) + + +def _merge_dicts(base: ConfigDict, override: ConfigDict) -> ConfigDict: + """Deep merge override into base (recursive).""" + for key, value in override.items(): + existing = base.get(key) + if isinstance(existing, dict) and isinstance(value, dict): + base[key] = _merge_dicts(existing, value) + else: + base[key] = value + return base + + +def _parse_env_value(raw: str) -> JSONType: + """Parse env var value as TOML, falling back to plain string.""" + try: + return tomllib.loads(f"x = {raw}")["x"] + except tomllib.TOMLDecodeError: + return raw + + +def _split_key_path(key: str) -> list[str]: + """Split a dotted key path into segments, honoring double-quoted segments. + + e.g. ``hosts."10.30.0.56".user`` -> ['hosts', '10.30.0.56', 'user'] + """ + parts: list[str] = [] + buf: list[str] = [] + in_quotes = False + for ch in key: + if ch == '"': + in_quotes = not in_quotes + elif ch == "." and not in_quotes: + parts.append("".join(buf)) + buf = [] + else: + buf.append(ch) + parts.append("".join(buf)) + return parts + + +def _set_nested_path(branch: ConfigDict, path: list[str], value: JSONType) -> None: + """Set ``value`` at ``path`` inside ``branch`` (creating intermediate dicts).""" + for part in path[:-1]: + node = branch.get(part) + if not isinstance(node, dict): + node = {} + branch[part] = node + branch = node + branch[path[-1]] = value + + +def _apply_env_overrides(data: ConfigDict) -> ConfigDict: + """Override config values from XCPNG_TESTS_* env vars. + + The part after the prefix is split on ``__`` to form the override path. + """ + prefix = "XCPNG_TESTS_" + overrides: ConfigDict = {} + for key, raw in os.environ.items(): + if not key.startswith(prefix): + continue + path = key.removeprefix(prefix).split("__") + value = _parse_env_value(raw) + _set_nested_path(overrides, path, value) + return _merge_dicts(data, overrides) if overrides else data + + +def _apply_config_values(data: ConfigDict, key_values: list[str]) -> ConfigDict: + """Override config values from --config-value KEY=VALUE entries. + + The KEY is a dotted path, with double quotes around segments that contain + dots (e.g. ``hosts."10.30.0.56".user``). Applied after env var overrides, + so it takes precedence over them. + """ + overrides: ConfigDict = {} + for item in key_values: + key, sep, raw_value = item.partition("=") + if not sep: + raise ValueError(f"Invalid --config-value: {item!r} (expected KEY=VALUE)") + path = _split_key_path(key) + if not all(path): + raise ValueError(f"Invalid --config-value key: {key!r}") + value = _parse_env_value(raw_value) + _set_nested_path(overrides, path, value) + return _merge_dicts(data, overrides) if overrides else data + + +@overload +def _replace_password_hash_placeholder(obj: ConfigDict, password_hash: str) -> ConfigDict: + ... + +@overload +def _replace_password_hash_placeholder(obj: JSONType, password_hash: str) -> JSONType: + ... + +def _replace_password_hash_placeholder(obj: JSONType, password_hash: str) -> JSONType: + """Recursively replace placeholders with actual hash.""" + if isinstance(obj, str): + return password_hash if obj == "" else obj + if isinstance(obj, dict): + return {k: _replace_password_hash_placeholder(v, password_hash) for k, v in obj.items()} + if isinstance(obj, list): + return [_replace_password_hash_placeholder(item, password_hash) for item in obj] + return obj + + +def warn_legacy_data_py() -> None: + """Warn if legacy data.py still exists.""" + data_py_path = Path(__file__).parent.parent / "data.py" + if data_py_path.exists(): + logging.warning( + f"Legacy {data_py_path} file found but is NOT used anymore. " + "Configuration is now loaded from TOML files. " + "Please run: uv run scripts/tools.py migrate-data-py\n" + f"And then remove {data_py_path}", + ) + + +def _build_config( + base_data: ConfigDict, + config_values: list[str] | None = None, + apply_value_overrides: bool = True, +) -> Config: + """Apply value/env overrides and password hash replacement, then validate.""" + try: + if apply_value_overrides: + base_data = _apply_env_overrides(base_data) + if config_values: + base_data = _apply_config_values(base_data, config_values) + if "host" in base_data and isinstance(base_data["host"], dict): + host = base_data["host"] + password = host.get("default_password", "") + if not isinstance(password, str): + raise ConfigError("host.default_password must be a string") + password_hash = hash_password(password) + host["default_password_hash"] = password_hash + base_data = _replace_password_hash_placeholder(base_data, password_hash) + return Config.model_validate(base_data) + except Exception as e: + raise ConfigError(f"Config validation failed:\n{e}") from e + + +def load_config( + config_path: Path | None = None, + override: str | Path | None = None, + config_values: list[str] | None = None, + apply_value_overrides: bool = True, +) -> Config: + """Load and validate a merged TOML config with Pydantic, returning a Config. + + The main config.toml at the repo root is always used as the base (lowest + priority). An optional override (a .toml path or a profile name) is merged + on top, then XCPNG_TESTS_* env vars, then --config-value entries (highest + priority). When no override is given, the XCPNG_CONFIG env var is used as + one. Short names are looked up in the XCPNG_CONFIG_DIR directory when it is + set, then in the xcp-ng-tests repository root. + When neither an override nor an explicit base path is given, + config.local.toml at the repo root is auto-merged if it exists. + Short names are looked up in the XCPNG_CONFIG_DIR directory when it is + set, then in the xcp-ng-tests repository root. + apply_value_overrides can be set to False to skip the XCPNG_TESTS_* and + --config-value overrides (used to compute the base config for diffs). + """ + if override is None: + override = os.environ.get("XCPNG_CONFIG") or None + base_path = config_path or REPO_ROOT / "config.toml" + try: + base_data = _load_toml_with_includes(base_path) + except FileNotFoundError as e: + raise ConfigError(f"{e}") from e + if override is not None: + try: + overlay_data = _load_toml_with_includes(_resolve_config_override(override)) + except FileNotFoundError as e: + raise ConfigError(f"{e}") from e + base_data = _merge_dicts(base_data, overlay_data) + elif config_path is None: + default_path = REPO_ROOT / "config.local.toml" + if default_path.exists(): + base_data = _merge_dicts(base_data, _load_toml_with_includes(default_path)) + return _build_config(base_data, config_values, apply_value_overrides) + + +def base_config_dict() -> ConfigDict: + """Return the validated base config.toml, without value/env overrides. + + Used as the reference when computing config deltas (dump-config and + migrate-data-py). + """ + return load_config(config_path=REPO_ROOT / "config.toml", apply_value_overrides=False).model_dump(by_alias=True) + + +def apply_override(config_name: str | None = None, config_values: list[str] | None = None) -> None: + """Load config.toml, merge the overlay (a .toml file path or profile name) on top, update config in place. + + When config_name is None, the XCPNG_CONFIG env var is used if set, else + config.local.toml is auto-merged if it exists. Short names are looked up + in the XCPNG_CONFIG_DIR directory when it is set, then in the xcp-ng-tests + repository root. config_values is passed to load_config (see its docstring). + """ + new = load_config(override=config_name, config_values=config_values) + for field in Config.model_fields: + setattr(config, field, getattr(new, field)) + + +def add_config_options(parser: argparse.ArgumentParser) -> argparse.ArgumentParser: + """Add -c/--config and --config-value to `parser`, and return a parent parser for subcommands. + + Both options are attached to `parser` (so they can be given before the + subcommand) and to a shared ``add_help=False`` parser that subcommands use + as a parent. + """ + common_parser = argparse.ArgumentParser(add_help=False) + for target in (parser, common_parser): + target.add_argument( + "-c", "--config", + type=Path, + default=None, + metavar="PATH", + help="Config overlay: a .toml file path or profile name (default: config.local.toml or XCPNG_CONFIG)", + ) + target.add_argument( + "--config-value", + action="append", + default=[], + metavar="KEY=VALUE", + help="Override a config value, e.g. host.default_password=foo (repeatable; highest priority)", + ) + return common_parser + + +def sr_device_config(config_key: str, *, required: list[str] | None = None) -> dict[str, str]: + """Delegate to config.sr_device_config() for backward compat.""" + return config.sr_device_config(config_key, required=required) + + +config: Config = load_config() diff --git a/lib/config_schema.py b/lib/config_schema.py new file mode 100644 index 000000000..3f2d95dd5 --- /dev/null +++ b/lib/config_schema.py @@ -0,0 +1,71 @@ +"""Build the editor-oriented JSON schema for the config files. + +``config-schema.json`` is derived from the pydantic Config model, but unlike +pydantic's default output it: + +- marks every field optional (no ``required`` arrays): overlay files only carry + the keys they override, so a fully-specified required schema would flag valid + partial configs, +- rejects unknown keys (``additionalProperties: false``), +- documents loader-level pseudo-properties that are not model fields (see + ``_EXTRA_PROPERTIES`` below). +""" +from __future__ import annotations + +from lib.config_loader import Config + +from typing import Any + +# Loader-level pseudo-properties (not pydantic model fields) that config files +# may contain. Each value is merged verbatim into the top-level "properties". +_EXTRA_PROPERTIES: dict[str, Any] = { + "$schema": { + "type": "string", + "description": "Path to the JSON schema used by the editor for validation and autocompletion.", + }, + "include": { + "type": "array", + "items": {"type": "string"}, + "description": "List of TOML files to load and deep-merge before this file's content. " + "Paths are relative to this file's directory.", + "default": [], + }, +} + +# Config fields that accept a human-readable size string ("1 GiB") in addition +# to a plain integer, normalized to an int by the model's parse_size_str +# validator. The pydantic model types them as int, so the schema needs the +# string alternative added by hand. +_SIZE_FIELDS = {"volume_size", "write_volume_cap"} + + +def editor_schema() -> dict[str, Any]: + """Return the editor-oriented schema for the current Config model.""" + schema = Config.model_json_schema() + + def postprocess(obj: object) -> None: + if isinstance(obj, dict): + obj.pop("required", None) + if obj.get("type") == "object": + obj.setdefault("additionalProperties", False) + for value in obj.values(): + postprocess(value) + elif isinstance(obj, list): + for value in obj: + postprocess(value) + + postprocess(schema) + for field in _SIZE_FIELDS & set(schema["properties"]): + prop = schema["properties"][field] + prop["anyOf"] = [{"type": "integer"}, {"type": "string"}] + prop.pop("type", None) + schema["title"] = "XCP-ng Tests Configuration Schema" + schema["description"] = ( + "JSON Schema for validating config.toml, config.local.toml, and config.NAME.toml files. " + "Properties are all optional since overlay files only carry the keys they override; " + "the base config.toml provides the rest." + ) + schema["$schema"] = "http://json-schema.org/draft-07/schema#" + schema.setdefault("additionalProperties", False) + schema["properties"] = {**_EXTRA_PROPERTIES, **schema.get("properties", {})} + return schema diff --git a/lib/host.py b/lib/host.py index 861ede398..80500c976 100644 --- a/lib/host.py +++ b/lib/host.py @@ -27,6 +27,7 @@ wait_for, wait_for_not, ) +from lib.config_loader import config from lib.netutil import wrap_ip from lib.network import Network from lib.pif import PIF @@ -34,7 +35,7 @@ from lib.vm import VM from lib.xo import xo_cli, xo_object_exists -from typing import TYPE_CHECKING, Literal, overload +from typing import TYPE_CHECKING, Literal, TypedDict, overload if TYPE_CHECKING: from lib.pool import Pool @@ -43,15 +44,20 @@ XAPI_CONF_FILE = '/etc/xapi.conf' XAPI_CONF_DIR = '/etc/xapi.conf.d' +class HostData(TypedDict): + user: str + password: str + skip_xo_config: bool -def host_data(hostname_or_ip: str) -> dict[str, str]: - # read from data.py - from data import HOST_DEFAULT_PASSWORD, HOST_DEFAULT_USER, HOSTS - if hostname_or_ip in HOSTS: - h_data = HOSTS[hostname_or_ip] - return h_data - else: - return {'user': HOST_DEFAULT_USER, 'password': HOST_DEFAULT_PASSWORD} + +def host_data(hostname_or_ip: str) -> HostData: + # read from config loader + h = config.hosts.get(hostname_or_ip) + return { + 'user': h.user if h and h.user else config.host.default_user, + 'password': h.password if h and h.password else config.host.default_password, + 'skip_xo_config': h.skip_xo_config if h and h.skip_xo_config is not None else False, + } class Host: xe_prefix = "host" @@ -924,11 +930,8 @@ def local_vm_srs(self) -> list[SR]: return srs def main_sr_uuid(self) -> str: - """ Main SR is the default SR, the first local SR, or a specific SR depending on data.py's DEFAULT_SR. """ - try: - from data import DEFAULT_SR - except ImportError: - DEFAULT_SR = 'default' + """ Main SR is the default SR, the first local SR, or a specific SR depending on config. """ + DEFAULT_SR = config.vm.default_sr sr_uuid = None if DEFAULT_SR == 'local': diff --git a/lib/installer.py b/lib/installer.py index 7307fbad2..0a3aacbcb 100644 --- a/lib/installer.py +++ b/lib/installer.py @@ -4,6 +4,7 @@ import time import xml.etree.ElementTree as ET +from lib import config from lib.commands import ssh from lib.common import wait_for @@ -14,8 +15,7 @@ class InstallationFailed(Exception): class AnswerFile: def __init__(self, kind: str, /): - from data import BASE_ANSWERFILES - defn = BASE_ANSWERFILES[kind] + defn = config.install.answerfiles[kind].model_dump() self.defn = self._normalize_structure(defn) # type: ignore def write_xml(self, filename: str) -> None: diff --git a/lib/passwords.py b/lib/passwords.py new file mode 100644 index 000000000..12a385443 --- /dev/null +++ b/lib/passwords.py @@ -0,0 +1,8 @@ +"""Password hashing helpers.""" + +from passlib.hash import sha512_crypt # type: ignore[import-untyped] + +def hash_password(password: str) -> str: + """Hash password for /etc/shadow.""" + # XCP-ng uses sha512 with 5000 rounds by default + return sha512_crypt.using(rounds=5000).hash(password) diff --git a/lib/pxe.py b/lib/pxe.py index 38b6700d8..adb79566d 100644 --- a/lib/pxe.py +++ b/lib/pxe.py @@ -1,6 +1,6 @@ from __future__ import annotations -from data import ARP_SERVER, PXE_CONFIG_SERVER +from lib import config from lib.commands import scp, ssh PXE_CONFIG_DIR = "/pxe/configs/custom" @@ -21,23 +21,23 @@ def server_push_config(mac_address: str, tmp_local_path: str) -> None: assert mac_address remote_dir = f'{PXE_CONFIG_DIR}/{mac_address}/' server_remove_config(mac_address) - ssh(PXE_CONFIG_SERVER, f'mkdir -p {remote_dir}') - scp(PXE_CONFIG_SERVER, f'{tmp_local_path}/boot.conf', remote_dir) - scp(PXE_CONFIG_SERVER, f'{tmp_local_path}/answerfile.xml', remote_dir) + ssh(config.pxe.config_server, f'mkdir -p {remote_dir}') + scp(config.pxe.config_server, f'{tmp_local_path}/boot.conf', remote_dir) + scp(config.pxe.config_server, f'{tmp_local_path}/answerfile.xml', remote_dir) def server_remove_config(mac_address: str) -> None: assert mac_address # protection against deleting the whole parent dir! remote_dir = f'{PXE_CONFIG_DIR}/{mac_address}/' - ssh(PXE_CONFIG_SERVER, f'rm -rf {remote_dir}') + ssh(config.pxe.config_server, f'rm -rf {remote_dir}') def server_remove_bootconf(mac_address: str) -> None: assert mac_address distant_file = f'{PXE_CONFIG_DIR}/{mac_address}/boot.conf' - ssh(PXE_CONFIG_SERVER, f'rm -rf {distant_file}') + ssh(config.pxe.config_server, f'rm -rf {distant_file}') def arp_addresses_for(mac_address: str) -> list[str]: output = ssh( - ARP_SERVER, + config.pxe.arp_server, f"ip neigh show nud reachable | grep {mac_address} | awk '{{ print $1 }}'" ) candidate_ips = output.splitlines() diff --git a/lib/sizes.py b/lib/sizes.py new file mode 100644 index 000000000..59d8767d9 --- /dev/null +++ b/lib/sizes.py @@ -0,0 +1,37 @@ +"""Size constants and parsing helpers.""" + +KiB = 2**10 +MiB = KiB**2 +GiB = KiB**3 +TiB = KiB**4 + +VHD_MAX = 2040 * GiB +QCOW2_MAX = 16 * TiB - 2561 * MiB + +_SYMBOLIC_SIZES: dict[str, int] = { + 'VHD_MAX': VHD_MAX, + 'QCOW2_MAX': QCOW2_MAX, +} + + +def parse_size(size_str: str) -> int: + """ + Parse a size string like "2.5TiB", "1GiB", "1024", "VHD_MAX", or "QCOW2_MAX". + """ + symbolic = _SYMBOLIC_SIZES.get(size_str.strip().upper()) + if symbolic is not None: + return symbolic + try: + return int(size_str) + except ValueError: + pass + + size_str = size_str.strip() + for unit, multiplier in [('TiB', TiB), ('GiB', GiB), ('MiB', MiB), ('KiB', KiB)]: + if size_str.endswith(unit): + try: + return int(float(size_str[:-len(unit)].strip()) * multiplier) + except ValueError: + pass + + raise ValueError(f"Cannot parse size: {size_str}") diff --git a/lib/tools/cli.py b/lib/tools/cli.py index c8c224d92..d3bf748d3 100644 --- a/lib/tools/cli.py +++ b/lib/tools/cli.py @@ -6,86 +6,138 @@ import argparse import logging -from pathlib import Path +import os +import sys from lib.common import HostAddress +from lib.config_loader import add_config_options, base_config_dict, load_config from lib.tools import logger -from lib.tools.inventory import into_inventory, load_inventory +from lib.tools.inventory import into_inventory, inventory_from_config from lib.tools.tasks.clean import clean_pools from lib.tools.tasks.exec import exec_pools +from lib.tools.tasks.migrate import migrate_data_py from lib.tools.tasks.update import update_pools def _command_update(args: argparse.Namespace) -> None: - if args.inventory: - inventory = load_inventory(args.inventory) - else: + if args.hosts: inventory = into_inventory(args.hosts, args.repos, args.hosting_pool, disabled_repositories=args.disablerepos) + else: + inventory = inventory_from_config(load_config(override=args.config, config_values=args.config_value)) + for host in inventory["hosts"].values(): + if args.repos: + host["repositories"] = args.repos + if args.disablerepos: + host["disabled_repositories"] = args.disablerepos + if args.hosting_pool: + host["hosting_pool"] = args.hosting_pool + if not inventory["hosts"]: + logger.warning("No hosts defined: pass -H/--hosts or define them in the config file") update_pools(inventory, reboot=args.reboot, parallel=args.parallel) def _command_clean(args: argparse.Namespace) -> int: - if args.inventory: - inventory = load_inventory(args.inventory) + if args.hosts: + inventory = into_inventory(args.hosts, [], None) else: - inventory = into_inventory(args.hosts, [], args.hosting_pool) + inventory = inventory_from_config(load_config(override=args.config, config_values=args.config_value)) + if not inventory["hosts"]: + logger.warning("No hosts defined: pass -H/--hosts or define them in the config file") return clean_pools(inventory, dry_run=args.dry_run) def _command_exec(args: argparse.Namespace) -> int: - if args.inventory: - inventory = load_inventory(args.inventory) - else: + if args.hosts: inventory = into_inventory(args.hosts, [], None) + else: + inventory = inventory_from_config(load_config(override=args.config, config_values=args.config_value)) + if not inventory["hosts"]: + logger.warning("No hosts defined: pass -H/--hosts or define them in the config file") command = " ".join(args.command) return exec_pools(inventory, command, parallel=args.parallel, dry_run=args.dry_run, reboot=args.reboot) +def _command_migrate(args: argparse.Namespace) -> int: + return migrate_data_py(data_py=args.data_py, output=args.output, force=args.force, + include_defaults=args.all) + + +def _command_diff_config(args: argparse.Namespace) -> int: + from lib.config_dump import config_diff + + name_a = str(args.config1) + name_b = str(args.config2) + config_a = load_config(override=args.config1, apply_value_overrides=False).model_dump(by_alias=True) + config_b = load_config(override=args.config2, apply_value_overrides=False).model_dump(by_alias=True) + diff = config_diff(config_a, config_b, name_a, name_b, as_json=args.json) + print(diff, end="") + return 1 if diff else 0 + + +def _command_dump_config(args: argparse.Namespace) -> int: + import json + + from lib.config_dump import colorize_toml, remove_defaults, render_toml + + config = load_config(override=args.config, config_values=args.config_value).model_dump(by_alias=True) + if not args.all: + config = remove_defaults(config, base_config_dict()) + if args.json: + print(json.dumps(config, indent=2, ensure_ascii=False)) + else: + out = render_toml(config, with_schema=False) + use_color = args.color if args.color is not None else ( + sys.stdout.isatty() and not os.environ.get("NO_COLOR") + ) + print(colorize_toml(out) if use_color else out) + return 0 + + def cli() -> None: parser = argparse.ArgumentParser( description="Tools that help developers for running recurrent tasks on their XCP-ng sandbox." ) parser.add_argument("-d", "--debug", action="store_true", default=False, help="Enable debug level") + common_parser = add_config_options(parser) subparsers = parser.add_subparsers(required=True, metavar="COMMAND") # subparser - command: update subparser_cmd_update = subparsers.add_parser( name="update", + parents=[common_parser], description="Run update tasks on target pools", help="Run update tasks on target pools", ) - cmd_update_excl_grp = subparser_cmd_update.add_mutually_exclusive_group(required=True) - cmd_update_excl_grp.add_argument( + subparser_cmd_update.add_argument( "-H", "--hosts", type=HostAddress, metavar="HOST", nargs="+", - help="Address (hostname|ip) of the master host in pool", + help="Address (hostname|ip) of the master host in pool (default: hosts defined in the config file)", ) - cmd_update_excl_grp.add_argument("-i", "--inventory", type=Path, help="Use an hosts inventory file") subparser_cmd_update.add_argument( "-e", "--enablerepo", metavar="REPO", action="append", dest="repos", - help="repositories to enable when updating", + help="repositories to enable when updating (overrides the config file)", ) subparser_cmd_update.add_argument( "-x", "--disablerepo", metavar="REPO", action="append", dest="disablerepos", - help="repositories to disable when updating", + help="repositories to disable when updating (overrides the config file)", ) subparser_cmd_update.add_argument( "-P", "--hosting-pool", type=HostAddress, - help="Address (hostname|ip) of hosting pool's master host (nested context)", + help="Address (hostname|ip) of hosting pool's master host (nested context, overrides the config file)", ) subparser_cmd_update.add_argument( "--no-reboot", @@ -105,19 +157,18 @@ def cli() -> None: # subparser - command: clean subparser_cmd_clean = subparsers.add_parser( name="clean", + parents=[common_parser], description="Remove all VMs, snapshorts and VDIs on local storage from target pools", help="Remove all VMs, snapshorts and VDIs on local storage from target pools", ) - cmd_clean_excl_grp = subparser_cmd_clean.add_mutually_exclusive_group(required=True) - cmd_clean_excl_grp.add_argument( + subparser_cmd_clean.add_argument( "-H", "--hosts", type=HostAddress, metavar="HOST", nargs="+", - help="Address (hostname|ip) of the master host in pool", + help="Address (hostname|ip) of the master host in pool (default: hosts defined in the config file)", ) - cmd_clean_excl_grp.add_argument("-i", "--inventory", type=Path, help="Use an hosts inventory file") subparser_cmd_clean.add_argument( "-n", "--dry-run", @@ -130,19 +181,18 @@ def cli() -> None: # subparser - command: exec subparser_cmd_exec = subparsers.add_parser( name="exec", + parents=[common_parser], description="Run the same command on all hosts of target pools", help="Run the same command on all hosts of target pools", ) - cmd_exec_excl_grp = subparser_cmd_exec.add_mutually_exclusive_group(required=True) - cmd_exec_excl_grp.add_argument( + subparser_cmd_exec.add_argument( "-H", "--hosts", type=HostAddress, metavar="HOST", nargs="+", - help="Address (hostname|ip) of the master host in pool", + help="Address (hostname|ip) of the master host in pool (default: hosts defined in the config file)", ) - cmd_exec_excl_grp.add_argument("-i", "--inventory", type=Path, help="Use an hosts inventory file") subparser_cmd_exec.add_argument( "--parallel", action="store_true", @@ -171,6 +221,91 @@ def cli() -> None: ) subparser_cmd_exec.set_defaults(func=_command_exec) + # subparser - command: migrate-data-py + subparser_cmd_migrate = subparsers.add_parser( + name="migrate-data-py", + parents=[common_parser], + description="Convert a legacy data.py file to a TOML config file", + help="Convert a legacy data.py file to a TOML config file", + ) + subparser_cmd_migrate.add_argument( + "data_py", + metavar="DATA_PY", + nargs="?", + default=None, + help="Path to the legacy data.py file (default: data.py in the repo root)", + ) + subparser_cmd_migrate.add_argument( + "--output", + default="config.local.toml", + help="Output file name (default: config.local.toml in the repo root)", + ) + subparser_cmd_migrate.add_argument( + "--force", + action="store_true", + default=False, + help="Overwrite the output file if it already exists", + ) + subparser_cmd_migrate.add_argument( + "--all", + action="store_true", + default=False, + help="Keep the values that are the same as in config.toml instead of writing a delta-only overlay", + ) + subparser_cmd_migrate.set_defaults(func=_command_migrate) + + # subparser - command: dump-config + subparser_cmd_dump = subparsers.add_parser( + name="dump-config", + parents=[common_parser], + description="Print the fully resolved configuration to stdout", + help="Print the fully resolved configuration to stdout", + ) + subparser_cmd_dump.add_argument( + "--json", + action="store_true", + default=False, + help="Dump as JSON instead of TOML", + ) + subparser_cmd_dump.add_argument( + "--color", + action=argparse.BooleanOptionalAction, + default=None, + help="Colorize TOML output (default: auto, only when stdout is a TTY)", + ) + subparser_cmd_dump.add_argument( + "--all", + action="store_true", + default=False, + help="Include the values that are the same as in config.toml", + ) + subparser_cmd_dump.set_defaults(func=_command_dump_config) + + # subparser - command: diff-config + subparser_cmd_diff = subparsers.add_parser( + name="diff-config", + parents=[common_parser], + description="Compare two config files and print their differences", + help="Compare two config files and print their differences", + ) + subparser_cmd_diff.add_argument( + "config1", + metavar="CONFIG1", + help="First config file (.toml path or short name)", + ) + subparser_cmd_diff.add_argument( + "config2", + metavar="CONFIG2", + help="Second config file (.toml path or short name)", + ) + subparser_cmd_diff.add_argument( + "--json", + action="store_true", + default=False, + help="Compare the JSON representations instead of TOML", + ) + subparser_cmd_diff.set_defaults(func=_command_diff_config) + args = parser.parse_args() if args.debug: diff --git a/lib/tools/inventory.py b/lib/tools/inventory.py index 17d116844..8a9623fe2 100644 --- a/lib/tools/inventory.py +++ b/lib/tools/inventory.py @@ -2,10 +2,8 @@ from __future__ import annotations -import tomllib -from pathlib import Path - from lib.common import HostAddress +from lib.config_loader import Config from typing import TypeAlias, TypedDict @@ -20,25 +18,18 @@ class HostConfig(TypedDict): class Inventory(TypedDict): hosts: HostConfigs -def load_inventory(inventory_path: Path) -> Inventory: - """Create an inventory object from loaded inventory file.""" - with open(inventory_path, "rb") as f: - data = tomllib.load(f) - - default = data.get("default", {}) - hosts = data.get("hosts", []) - +def inventory_from_config(config: Config) -> Inventory: + """Create an inventory object from the config's ``[tools.update]`` and ``[hosts]`` tables.""" + default = config.tools.update inventory_hosts: HostConfigs = {} - for h, config in hosts.items(): - repos = config.get("repositories", []) - disabled_repositories = config.get("disabled_repositories", []) - hosting_pool = config.get("hosting_pool", None) - if hosting_pool is None: - hosting_pool = default.get("hosting_pool", None) + for h, config_host in config.hosts.items(): host: HostConfig = { - "repositories": repos or default.get("repositories", []), - "disabled_repositories": disabled_repositories or default.get("disabled_repositories", []), - "hosting_pool": hosting_pool, + "repositories": config_host.repositories if config_host.repositories is not None else default.repositories, + "disabled_repositories": ( + config_host.disabled_repositories if config_host.disabled_repositories is not None + else default.disabled_repositories + ), + "hosting_pool": config_host.hosting_pool if config_host.hosting_pool is not None else default.hosting_pool, } inventory_hosts[h] = host @@ -51,7 +42,7 @@ def into_inventory( hosts: list[HostAddress], repositories: list[str], hosting_pool: HostAddress | None, - disabled_repositories: list[str] = [], + disabled_repositories: list[str] | None = None, ) -> Inventory: """Create an inventory object from arguments. diff --git a/lib/tools/tasks/migrate.py b/lib/tools/tasks/migrate.py new file mode 100644 index 000000000..ab99b840f --- /dev/null +++ b/lib/tools/tasks/migrate.py @@ -0,0 +1,278 @@ +"""Migrate a legacy data.py file to a TOML config file. + +Can be run with: uv run scripts/tools.py migrate-data-py [DATA_PY] +""" +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + +from lib.config_dump import remove_defaults, render_toml +from lib.config_loader import base_config_dict +from lib.typing import ConfigDict + +from typing import Any + +_REPO_ROOT = Path(__file__).resolve().parents[3] + + +def normalize_dict_keys(d: ConfigDict) -> ConfigDict: + """Normalize dict keys by replacing dashes with underscores.""" + return {k.replace("-", "_"): v for k, v in d.items()} + + +def load_data_py(data_py_path: Path, repo_root: Path) -> ConfigDict: + """Load data.py and extract configuration as dict. + + Tries to import as a module first, falls back to exec() for files with custom code. + This allows static type checkers to work even if data.py doesn't exist. + """ + namespace: dict[str, Any] = {} + + # Add repo root to path so imports work + if str(repo_root) not in sys.path: + sys.path.insert(0, str(repo_root)) + + # Try to load as a module using importlib + try: + spec = importlib.util.spec_from_file_location("data", data_py_path) + if spec is None or spec.loader is None: + raise ImportError(f"Could not load spec for {data_py_path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + # Extract all non-private attributes + namespace = { + k: v for k, v in vars(module).items() + if not k.startswith("_") + } + except Exception as e: + # Fallback: execute as raw Python code + print(f"WARNING: importing {data_py_path} failed ({e}); falling back to exec", file=sys.stderr) + with open(data_py_path) as f: + code = f.read() + try: + exec(code, namespace) + except Exception as e: + raise ValueError(f"Failed to execute {data_py_path}: {e}") from e + + # Map data.py variable names to config structure + config: ConfigDict = {} + + # Root-level fields + if "OBJECTS_NAME_PREFIX" in namespace: + config["objects_name_prefix"] = namespace["OBJECTS_NAME_PREFIX"] or "" + if "TEST_DNS_SERVER" in namespace: + config["dns_server"] = namespace["TEST_DNS_SERVER"] + + # Host section + host_config: ConfigDict = {} + # Support both DEFAULT_USER and HOST_DEFAULT_USER naming conventions + if "HOST_DEFAULT_USER" in namespace: + host_config["default_user"] = namespace["HOST_DEFAULT_USER"] + elif "DEFAULT_USER" in namespace: + host_config["default_user"] = namespace["DEFAULT_USER"] + # Support both DEFAULT_PASSWORD and HOST_DEFAULT_PASSWORD naming conventions + if "HOST_DEFAULT_PASSWORD" in namespace: + host_config["default_password"] = namespace["HOST_DEFAULT_PASSWORD"] + elif "DEFAULT_PASSWORD" in namespace: + host_config["default_password"] = namespace["DEFAULT_PASSWORD"] + config["host"] = host_config + + # Hosts section (per-host overrides) + if "HOSTS" in namespace: + config["hosts"] = namespace["HOSTS"] + + # Network section + network_config: ConfigDict = {} + if "NETWORKS" in namespace and "MGMT" in namespace["NETWORKS"]: + network_config["mgmt"] = namespace["NETWORKS"]["MGMT"] + if "HOST_FREE_NICS" in namespace and namespace["HOST_FREE_NICS"]: + network_config["free_nics"] = namespace["HOST_FREE_NICS"] + if network_config: + config["network"] = network_config + + # PXE section + pxe_config: ConfigDict = {} + if "PXE_CONFIG_SERVER" in namespace: + pxe_config["config_server"] = namespace["PXE_CONFIG_SERVER"] + if "ARP_SERVER" in namespace: + pxe_config["arp_server"] = namespace["ARP_SERVER"] + if pxe_config: + config["pxe"] = pxe_config + + # VM section + vm_config: ConfigDict = {} + if "DEF_VM_URL" in namespace: + vm_config["def_url"] = namespace["DEF_VM_URL"] + if "CACHE_IMPORTED_VM" in namespace: + vm_config["cache_imported"] = namespace["CACHE_IMPORTED_VM"] + if "DEFAULT_SR" in namespace: + vm_config["default_sr"] = namespace["DEFAULT_SR"] + + # VM images + if "VM_IMAGES" in namespace: + vm_config["images"] = namespace["VM_IMAGES"] + + # VM equivalences + if "IMAGE_EQUIVS" in namespace: + vm_config["equivalents"] = namespace["IMAGE_EQUIVS"] + if vm_config: + config["vm"] = vm_config + + # Install section + install_config: ConfigDict = {} + if "BASE_ANSWERFILES" in namespace: + install_config["answerfiles"] = namespace["BASE_ANSWERFILES"] + + # Install ISOs + isos_config: ConfigDict = {} + if "ISO_IMAGES_BASE" in namespace: + isos_config["base_url"] = namespace["ISO_IMAGES_BASE"] + if "ISO_IMAGES_CACHE" in namespace: + isos_config["cache_dir"] = namespace["ISO_IMAGES_CACHE"] + if "ISO_IMAGES" in namespace: + isos_config["definitions"] = namespace["ISO_IMAGES"] + if isos_config: + install_config["isos"] = isos_config + if install_config: + config["install"] = install_config + + # Guest tools section + guest_tools_config: ConfigDict = {} + if "ISO_DOWNLOAD_URL" in namespace: + guest_tools_config["download_url"] = namespace["ISO_DOWNLOAD_URL"] + if "WIN_GUEST_TOOLS_ISOS" in namespace: + guest_tools_config["win"] = namespace["WIN_GUEST_TOOLS_ISOS"] + if "OTHER_GUEST_TOOLS_ISO" in namespace: + guest_tools_config["other"] = namespace["OTHER_GUEST_TOOLS_ISO"] + if "OTHER_GUEST_TOOLS" in namespace: + guest_tools_config["installed"] = namespace["OTHER_GUEST_TOOLS"] + if guest_tools_config: + config["guest_tools"] = guest_tools_config + + # SSH section + ssh_config: ConfigDict = {} + if "TEST_SSH_PUBKEY" in namespace: + ssh_config["pubkey"] = namespace["TEST_SSH_PUBKEY"] + if "SSH_OUTPUT_MAX_LINES" in namespace: + ssh_config["output_max_lines"] = namespace["SSH_OUTPUT_MAX_LINES"] + if "IGNORE_SSH_BANNER" in namespace: + ssh_config["ignore_banner"] = namespace["IGNORE_SSH_BANNER"] + if ssh_config: + config["ssh"] = ssh_config + + # iso_remaster tool path goes into install section + if "TOOLS" in namespace: + tools = normalize_dict_keys(namespace["TOOLS"]) + if "iso_remaster" in tools: + install_config["iso_remaster"] = tools["iso_remaster"] + config["install"] = install_config + if "xo_cli" in tools: + config["xo"] = {"cli": tools["xo_cli"]} + + # Storage section + storage: ConfigDict = {} + if "NFS_DEVICE_CONFIG" in namespace: + storage["nfs"] = namespace["NFS_DEVICE_CONFIG"] + if "NFS4_DEVICE_CONFIG" in namespace: + storage["nfs4"] = namespace["NFS4_DEVICE_CONFIG"] + if "NFS_ISO_DEVICE_CONFIG" in namespace: + storage["nfs_iso"] = namespace["NFS_ISO_DEVICE_CONFIG"] + if "CIFS_ISO_DEVICE_CONFIG" in namespace: + storage["cifs_iso"] = namespace["CIFS_ISO_DEVICE_CONFIG"] + if "CEPHFS_DEVICE_CONFIG" in namespace: + storage["cephfs"] = namespace["CEPHFS_DEVICE_CONFIG"] + if "MOOSEFS_DEVICE_CONFIG" in namespace: + storage["moosefs"] = namespace["MOOSEFS_DEVICE_CONFIG"] + if "LVMOHBA_DEVICE_CONFIG" in namespace: + storage["lvmohba"] = namespace["LVMOHBA_DEVICE_CONFIG"] + if "LVMOISCSI_DEVICE_CONFIG" in namespace: + storage["lvmoiscsi"] = namespace["LVMOISCSI_DEVICE_CONFIG"] + if "LINSTOR_REDUNDANCY" in namespace: + storage["linstor"] = {"redundancy": namespace["LINSTOR_REDUNDANCY"]} + if storage: + config["storage"] = storage + + return config + + +def write_toml(config: ConfigDict, output_path: Path) -> None: + """Write config dict to TOML file with $schema attribute.""" + with open(output_path, "w") as f: + f.write(render_toml(config)) + + +def migrate_data_py( + data_py: Path | str | None = None, + output: str = "config.local.toml", + force: bool = False, + include_defaults: bool = False, +) -> int: + """Convert a legacy data.py file to a TOML config file. + + DATA_PY Optional path to the data.py file to migrate. Defaults to + /data.py. May be an absolute or relative path. + output Output file name. If relative, resolved against . + Defaults to config.local.toml. + force Overwrite the output file if it already exists. + include_defaults + Keep the values that are the same as in config.toml instead of + writing a delta-only overlay. + """ + repo_root = _REPO_ROOT + if data_py is not None: + data_py_path = (repo_root / data_py).resolve() + else: + data_py_path = repo_root / "data.py" + + # Validate input file + if not data_py_path.exists(): + print(f"ERROR: {data_py_path} not found", file=sys.stderr) + return 1 + + output_path_raw = Path(output) + output_path = output_path_raw if output_path_raw.is_absolute() else repo_root / output + + # Check if output file exists + if output_path.exists() and not force: + print( + f"ERROR: {output_path} already exists. Use --force to overwrite.", + file=sys.stderr, + ) + return 1 + + # Load base config and data.py + try: + base_config = base_config_dict() + except Exception as e: + print(f"ERROR: Failed to load base config.toml: {e}", file=sys.stderr) + return 1 + + try: + data_config = load_data_py(data_py_path, repo_root) + except Exception as e: + print(f"ERROR: Failed to load {data_py_path}: {e}", file=sys.stderr) + return 1 + + if include_defaults: + out_config = data_config + else: + # Remove defaults + out_config = remove_defaults(data_config, base_config) + if not out_config: + print( + f"INFO: No differences found between {data_py_path} and config.toml", + file=sys.stderr, + ) + print(f" {output_path} would be empty, not creating file", file=sys.stderr) + return 0 + + # Write output + try: + write_toml(out_config, output_path) + print(f"✓ Created {output_path}", file=sys.stdout) + return 0 + except Exception as e: + print(f"ERROR: Failed to write {output_path}: {e}", file=sys.stderr) + return 1 diff --git a/lib/tools/tasks/update.py b/lib/tools/tasks/update.py index 2875c3a03..b2ec3000f 100644 --- a/lib/tools/tasks/update.py +++ b/lib/tools/tasks/update.py @@ -173,6 +173,10 @@ def update_pools(inventory: Inventory, reboot: bool = True, parallel: bool = Fal except NotAMasterHostError: logger.warning(f"[{host}] Skipping: not a master host") + if not pools: + logger.warning("No pool to update") + return + before_packages = _capture_packages(pools) # update master hosts diff --git a/lib/typing.py b/lib/typing.py index 82cd2a6dc..ee43cb15b 100644 --- a/lib/typing.py +++ b/lib/typing.py @@ -1,17 +1,10 @@ import sys -from typing import NotRequired, TypedDict - -if sys.version_info >= (3, 11): - from typing import NotRequired +if sys.version_info >= (3, 12): + from typing import TypeAliasType else: - from typing_extensions import NotRequired + from typing_extensions import TypeAliasType -IsoImageDef = TypedDict('IsoImageDef', - {'path': str, - 'net-url': NotRequired[str], - 'net-only': NotRequired[bool], - 'unsigned': NotRequired[bool], - }) +JSONType = TypeAliasType("JSONType", None | bool | int | float | str | list["JSONType"] | dict[str, "JSONType"]) -JSONType = None | bool | int | float | str | list["JSONType"] | dict[str, "JSONType"] +ConfigDict = dict[str, JSONType] diff --git a/lib/vm.py b/lib/vm.py index b85d01102..4c0f85429 100644 --- a/lib/vm.py +++ b/lib/vm.py @@ -23,6 +23,7 @@ wait_for, wait_for_not, ) +from lib.config_loader import config from lib.snapshot import Snapshot from lib.sr import SR from lib.vbd import VBD @@ -982,5 +983,4 @@ def vm_cache_key_from_def(vm_def: dict[str, str], ref_nodeid: str, test_gitref: nodeid = shortened_nodeid(expand_scope_relative_nodeid(image_test, image_scope, ref_nodeid)) image_key = f"{nodeid}-{image_vm}-{test_gitref}" - from data import IMAGE_EQUIVS - return IMAGE_EQUIVS.get(image_key, image_key) + return config.vm.equivalents.get(image_key, image_key) diff --git a/lib/windows/__init__.py b/lib/windows/__init__.py index 86a3451f0..737200276 100644 --- a/lib/windows/__init__.py +++ b/lib/windows/__init__.py @@ -4,15 +4,16 @@ import time from pathlib import PureWindowsPath -from data import ISO_DOWNLOAD_URL, TEST_DNS_SERVER +from lib import config from lib.commands import SSHCommandFailed from lib.common import strtobool, wait_for +from lib.config_loader import OtherGuestToolDef, WinGuestToolDef from lib.host import Host from lib.sr import SR from lib.vif import VIF from lib.vm import VM -from typing import Any, Generator +from typing import Generator, TypeVar # HACK: I originally thought that using Stop-Computer -Force would cause the SSH session to sometimes fail. # I could never confirm this in the end, but use a slightly delayed shutdown just to be safe anyway. @@ -25,17 +26,17 @@ class PowerAction(enum.Enum): Reboot = "reboot" -def iso_create(host: Host, sr: SR, param: dict[str, Any]) -> Generator[dict[str, Any], None, None]: - if param["download"]: - vdi = host.import_iso(ISO_DOWNLOAD_URL + param["name"], sr) - new_param = param.copy() - new_param["name"] = vdi.name() - yield new_param +T = TypeVar("T", WinGuestToolDef, OtherGuestToolDef) + + +def iso_create(host: Host, sr: SR, param: T) -> Generator[T, None, None]: + if param.download: + vdi = host.import_iso(config.guest_tools.download_url + param.name, sr) + yield param.model_copy(update={"name": vdi.name()}) vdi.destroy() else: yield param - def try_get_and_store_vm_ip_serial(vm: VM, timeout: int) -> bool: domid = vm.param_get("dom-id") logging.debug(f"Domain ID {domid}") @@ -255,18 +256,18 @@ def wait_for_vm_xenvif_offboard(vm: VM) -> None: def set_vm_dns(vm: VM) -> None: - logging.info(f"Set VM DNS to {TEST_DNS_SERVER}") + logging.info(f"Set VM DNS to {config.dns_server}") vif = vm.vifs()[0] - assert TEST_DNS_SERVER not in vif_get_dns(vif) - vif_set_dns(vif, [TEST_DNS_SERVER]) + assert config.dns_server not in vif_get_dns(vif) + vif_set_dns(vif, [config.dns_server]) def check_vm_dns(vm: VM) -> None: # The restore task takes time to fire so wait for it vif = vm.vifs()[0] wait_for( - lambda: TEST_DNS_SERVER in vif_get_dns(vif), - f"Check VM DNS contains {TEST_DNS_SERVER}", + lambda: config.dns_server in vif_get_dns(vif), + f"Check VM DNS contains {config.dns_server}", timeout_secs=300, retry_delay_secs=30, ) diff --git a/lib/windows/guest_tools.py b/lib/windows/guest_tools.py index 376ce847b..29880710e 100644 --- a/lib/windows/guest_tools.py +++ b/lib/windows/guest_tools.py @@ -2,6 +2,7 @@ from pathlib import PureWindowsPath from lib.common import wait_for +from lib.config_loader import WinGuestToolDef from lib.vm import VM from . import ( @@ -13,8 +14,6 @@ wait_for_vm_xenvif_offboard, ) -from typing import Any - ERROR_SUCCESS = 0 ERROR_INSTALL_FAILURE = 1603 ERROR_SUCCESS_REBOOT_INITIATED = 1641 @@ -23,23 +22,23 @@ GUEST_TOOLS_COPY_PATH = "C:\\package.msi" -def install_guest_tools(vm: VM, guest_tools_iso: dict[str, Any], action: PowerAction, +def install_guest_tools(vm: VM, guest_tools_iso: WinGuestToolDef, action: PowerAction, check: bool = True) -> int | None: - insert_cd_safe(vm, guest_tools_iso["name"]) + insert_cd_safe(vm, guest_tools_iso.name) - if guest_tools_iso.get("testsign_cert"): + if guest_tools_iso.testsign_cert: logging.info("Enable testsigning") - rootcert = PureWindowsPath("D:\\") / guest_tools_iso["testsign_cert"] + rootcert = PureWindowsPath("D:\\") / guest_tools_iso.testsign_cert enable_testsign(vm, rootcert) # HACK: Sometimes after rebooting the CD drive just vanishes. Check for it again and # reboot/reinsert CD if needed. if not vm.file_exists("D:/", regular_file=False): logging.warning("CD drive not detected, retrying") - insert_cd_safe(vm, guest_tools_iso["name"]) + insert_cd_safe(vm, guest_tools_iso.name) logging.info("Copy Windows PV drivers to VM") - package_path = PureWindowsPath("D:\\") / guest_tools_iso["package"] + package_path = PureWindowsPath("D:\\") / guest_tools_iso.package vm.execute_powershell_script(f"Copy-Item -Force '{package_path}' '{GUEST_TOOLS_COPY_PATH}'") vm.eject_cd() diff --git a/lib/windows/other_tools.py b/lib/windows/other_tools.py index 96ad628f8..b88b2e44f 100644 --- a/lib/windows/other_tools.py +++ b/lib/windows/other_tools.py @@ -3,27 +3,27 @@ from pathlib import PureWindowsPath from lib.common import strtobool, wait_for +from lib.config_loader import InstalledGuestToolDef from lib.vm import VM from . import WINDOWS_SHUTDOWN_COMMAND, enable_testsign, insert_cd_safe, wait_for_vm_running_and_ssh_up_without_tools -from typing import Any - -def install_other_drivers(vm: VM, other_tools_iso_name: str, param: dict[str, Any]) -> None: - if param.get("vendor_device"): +def install_other_drivers(vm: VM, other_tools_iso_name: str, param: InstalledGuestToolDef) -> None: + if param.vendor_device: assert not strtobool(vm.param_get("has-vendor-device")) vm.param_set("has-vendor-device", True) vm.start() wait_for_vm_running_and_ssh_up_without_tools(vm) - driver_type = param.get("type") + driver_type = param.type if driver_type is not None: + assert param.path is not None and param.package is not None insert_cd_safe(vm, other_tools_iso_name) - if param.get("testsign_cert"): + if param.testsign_cert: logging.info("Enable testsigning") - rootcert = PureWindowsPath("D:\\") / param["path"] / param["testsign_cert"] + rootcert = PureWindowsPath("D:\\") / param.path / param.testsign_cert enable_testsign(vm, rootcert) # HACK: Sometimes after rebooting the CD drive just vanishes. Check for it again and @@ -32,7 +32,7 @@ def install_other_drivers(vm: VM, other_tools_iso_name: str, param: dict[str, An logging.warning("CD drive not detected, retrying") insert_cd_safe(vm, other_tools_iso_name) - package_path = PureWindowsPath("D:\\") / param["path"] / param["package"] + package_path = PureWindowsPath("D:\\") / param.path / param.package install_cmd = "D:\\install-drivers.ps1 " if driver_type == "msi": logging.info(f"Install MSI drivers: {package_path}") diff --git a/lib/xo.py b/lib/xo.py index a09ccdd33..a91de26c3 100644 --- a/lib/xo.py +++ b/lib/xo.py @@ -1,6 +1,6 @@ import json -from data import TOOLS +from lib import config from lib.commands import local_cmd from lib.typing import JSONType @@ -14,7 +14,7 @@ def xo_cli(action: str, args: dict[str, str] = ..., *, check: bool = ..., use_js ... def xo_cli(action: str, args: dict[str, str] = {}, *, check: bool = True, use_json: bool = False) -> JSONType | str: - cmd = [TOOLS.get('xo-cli', 'xo-cli'), action] + cmd = [config.xo.cli, action] if use_json: cmd += ['--json'] cmd += ["%s=%s" % (key, value) for key, value in args.items()] diff --git a/pyproject.toml b/pyproject.toml index 9f8294e3a..bfddac13f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,9 @@ dependencies = [ "pytest-dependency", "requests", "ipdb", + "pygments", + "tomli-w", + "typing-extensions", ] [dependency-groups] @@ -32,7 +35,6 @@ dev = [ "pyright", "ruff", "types-passlib", - "typing-extensions", "zizmor", ] @@ -40,6 +42,7 @@ dev = [ typeCheckingMode = "standard" reportMissingParameterType = "error" reportUnknownParameterType = "error" +exclude = ["data.py", ".venv"] [tool.ty.rules] unused-type-ignore-comment = "ignore" diff --git a/requirements/base.txt b/requirements/base.txt index a59856ebe..fa2840c1b 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -10,3 +10,6 @@ pytest>=9.1.1 pytest-dependency requests ipdb +pygments +tomli-w +typing-extensions diff --git a/requirements/dev.txt b/requirements/dev.txt index 8691ee1ba..a4b3e2a7c 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -10,6 +10,5 @@ pydocstyle pyright ruff types-passlib -typing-extensions zizmor -r base.txt diff --git a/scripts/gen-config-schema.py b/scripts/gen-config-schema.py new file mode 100644 index 000000000..ea3bbbc78 --- /dev/null +++ b/scripts/gen-config-schema.py @@ -0,0 +1,19 @@ +"""Regenerate config-schema.json from the pydantic Config model. + +Run from the repository root: + uv run scripts/gen-config-schema.py +""" +import json +import sys +from pathlib import Path + +# Add root project directory into PYTHONPATH +sys.path.append(str(Path(__file__).absolute().parent.parent)) + +# flake8: noqa: E402 module level import not at top of file +from lib.config_schema import editor_schema + +if __name__ == "__main__": + out = Path(__file__).absolute().parent.parent / "config-schema.json" + out.write_text(json.dumps(editor_schema(), indent=2, sort_keys=True) + "\n") + print(f"Wrote {out}") diff --git a/scripts/install_xcpng.py b/scripts/install_xcpng.py index d2be716a4..03375e6df 100755 --- a/scripts/install_xcpng.py +++ b/scripts/install_xcpng.py @@ -17,7 +17,7 @@ # flake8: noqa: E402 sys.path.append(f"{os.path.abspath(os.path.dirname(__file__))}/..") -from lib import pxe +from lib import config, pxe from lib.commands import SSHCommandFailed, ssh from lib.common import is_uuid, wait_for from lib.host import Host, host_data @@ -49,7 +49,7 @@ def generate_answerfile(directory: str, installer: str, hostname_or_ip: str, tar Europe/Paris {target_hostname} """) @@ -59,7 +59,7 @@ def generate_answerfile(directory: str, installer: str, hostname_or_ip: str, tar {hdd} {installer} """) @@ -168,7 +168,7 @@ def main() -> None: assert host.is_enabled() if not args.installer: - installer = f"http://{pxe.PXE_CONFIG_SERVER}/installers/xcp-ng/{xcp_version}/" + installer = f"http://{config.pxe.config_server}/installers/xcp-ng/{xcp_version}/" else: installer = args.installer diff --git a/tests/guest_tools/win/conftest.py b/tests/guest_tools/win/conftest.py index c5fb63566..200de3d83 100644 --- a/tests/guest_tools/win/conftest.py +++ b/tests/guest_tools/win/conftest.py @@ -4,8 +4,9 @@ import logging -from data import OTHER_GUEST_TOOLS, OTHER_GUEST_TOOLS_ISO, WIN_GUEST_TOOLS_ISOS +from lib import config from lib.common import wait_for +from lib.config_loader import InstalledGuestToolDef, OtherGuestToolDef, WinGuestToolDef from lib.host import Host from lib.snapshot import Snapshot from lib.sr import SR @@ -20,7 +21,16 @@ from lib.windows.guest_tools import install_guest_tools from lib.windows.other_tools import install_other_drivers -from typing import Any, Generator +from typing import Generator + +# Bind config sub-objects at import time. These are used as pytest parameters +# (ids=..., params=...), so they must be fixed when this conftest is imported. +# Sub-conftests are imported during collection, i.e. after pytest_configure has +# applied --config / XCPNG_TESTS_* / --config-value overrides to the global +# `config`, so these reflect the effective configuration. +WIN_GUEST_TOOLS_ISOS = config.guest_tools.win +OTHER_GUEST_TOOLS_ISO = config.guest_tools.other +OTHER_GUEST_TOOLS = config.guest_tools.installed @pytest.fixture(scope="module") def running_windows_vm_without_tools(imported_vm: VM) -> VM: @@ -60,7 +70,7 @@ def running_unsealed_windows_vm(unsealed_windows_vm_and_snapshot: tuple[VM, Snap @pytest.fixture(scope="class") def vm_install_test_tools_per_test_class( - unsealed_windows_vm_and_snapshot: tuple[VM, Snapshot], guest_tools_iso: dict[str, Any] + unsealed_windows_vm_and_snapshot: tuple[VM, Snapshot], guest_tools_iso: WinGuestToolDef ) -> Generator[VM, None, None]: vm, snapshot = unsealed_windows_vm_and_snapshot vm.start() @@ -72,7 +82,7 @@ def vm_install_test_tools_per_test_class( @pytest.fixture -def vm_install_test_tools_no_reboot(running_unsealed_windows_vm: VM, guest_tools_iso: dict[str, Any]) -> VM: +def vm_install_test_tools_no_reboot(running_unsealed_windows_vm: VM, guest_tools_iso: WinGuestToolDef) -> VM: install_guest_tools(running_unsealed_windows_vm, guest_tools_iso, PowerAction.Nothing) return running_unsealed_windows_vm @@ -84,23 +94,23 @@ def vm_install_test_tools_no_reboot(running_unsealed_windows_vm: VM, guest_tools ) def guest_tools_iso( host: Host, request: pytest.FixtureRequest, nfs_iso_sr: SR -) -> Generator[dict[str, Any], None, None]: +) -> Generator[WinGuestToolDef, None, None]: yield from iso_create(host, nfs_iso_sr, request.param) @pytest.fixture(scope="module") -def other_tools_iso(host: Host, nfs_iso_sr: SR) -> Generator[dict[str, Any], None, None]: +def other_tools_iso(host: Host, nfs_iso_sr: SR) -> Generator[OtherGuestToolDef, None, None]: yield from iso_create(host, nfs_iso_sr, OTHER_GUEST_TOOLS_ISO) @pytest.fixture(ids=list(OTHER_GUEST_TOOLS.keys()), params=list(OTHER_GUEST_TOOLS.values())) def vm_install_other_drivers( unsealed_windows_vm_and_snapshot: tuple[VM, Snapshot], - other_tools_iso: dict[str, Any], + other_tools_iso: OtherGuestToolDef, request: pytest.FixtureRequest, -) -> Generator[tuple[VM, dict[str, Any]], None, None]: +) -> Generator[tuple[VM, InstalledGuestToolDef], None, None]: vm, snapshot = unsealed_windows_vm_and_snapshot param = request.param - install_other_drivers(vm, other_tools_iso["name"], param) + install_other_drivers(vm, other_tools_iso.name, param) yield vm, param snapshot.revert() diff --git a/tests/guest_tools/win/test_destructive.py b/tests/guest_tools/win/test_destructive.py index 289c7b3c3..c5d009794 100644 --- a/tests/guest_tools/win/test_destructive.py +++ b/tests/guest_tools/win/test_destructive.py @@ -3,6 +3,7 @@ import logging from lib.commands import SSHCommandFailed +from lib.config_loader import InstalledGuestToolDef, WinGuestToolDef from lib.vm import VM from lib.windows import ( PowerAction, @@ -13,8 +14,6 @@ ) from lib.windows.guest_tools import ERROR_INSTALL_FAILURE, install_guest_tools, uninstall_guest_tools -from typing import Any - # Requirements: # - Same as TestGuestToolsWindowsNondestructive. @@ -42,10 +41,10 @@ def test_uninstall_tools_early(self, vm_install_test_tools_no_reboot: VM) -> Non assert vm.are_windows_tools_uninstalled() def test_install_with_other_tools( - self, vm_install_other_drivers: tuple[VM, dict[str, Any]], guest_tools_iso: dict[str, Any] + self, vm_install_other_drivers: tuple[VM, InstalledGuestToolDef], guest_tools_iso: WinGuestToolDef ) -> None: vm, param = vm_install_other_drivers - if param["upgradable"]: + if param.upgradable: install_guest_tools(vm, guest_tools_iso, PowerAction.Reboot, check=False) assert vm.are_windows_tools_working() else: @@ -61,7 +60,7 @@ def test_uefi_vm_suspend_refused_without_tools(self, running_unsealed_windows_vm # Test of the unplug rework, where the driver must remain activated even if the device ID changes. # Also serves as a "close-enough" test of vendor device toggling. - def test_toggle_device_id(self, running_unsealed_windows_vm: VM, guest_tools_iso: dict[str, Any]) -> None: + def test_toggle_device_id(self, running_unsealed_windows_vm: VM, guest_tools_iso: WinGuestToolDef) -> None: vm = running_unsealed_windows_vm assert vm.param_get("platform", "device_id") == "0002" install_guest_tools(vm, guest_tools_iso, PowerAction.Shutdown, check=False) diff --git a/tests/guest_tools/win/test_xenclean.py b/tests/guest_tools/win/test_xenclean.py index 9853ae4c8..1efe2d3f2 100644 --- a/tests/guest_tools/win/test_xenclean.py +++ b/tests/guest_tools/win/test_xenclean.py @@ -6,6 +6,7 @@ from pathlib import PureWindowsPath from lib.common import wait_for +from lib.config_loader import InstalledGuestToolDef, WinGuestToolDef from lib.vm import VM from lib.windows import ( WINDOWS_SHUTDOWN_COMMAND, @@ -16,7 +17,7 @@ wait_for_vm_xenvif_offboard, ) -from typing import Any, Literal, overload +from typing import Literal, overload # Test uninstallation of other drivers using the XenClean program. @@ -34,16 +35,16 @@ @overload -def run_xenclean(vm: VM, guest_tools_iso: dict[str, Any], onboard: Literal[False] = ...) -> None: # +def run_xenclean(vm: VM, guest_tools_iso: WinGuestToolDef, onboard: Literal[False] = ...) -> None: # ... @overload -def run_xenclean(vm: VM, guest_tools_iso: dict[str, Any], onboard: Literal[True]) -> str: # +def run_xenclean(vm: VM, guest_tools_iso: WinGuestToolDef, onboard: Literal[True]) -> str: # ... -def run_xenclean(vm: VM, guest_tools_iso: dict[str, Any], onboard: bool = False) -> str | None: +def run_xenclean(vm: VM, guest_tools_iso: WinGuestToolDef, onboard: bool = False) -> str | None: """ Run XenClean from the provided guest tools. @@ -51,17 +52,17 @@ def run_xenclean(vm: VM, guest_tools_iso: dict[str, Any], onboard: bool = False) Onboarding is the transition from one guest tool to another, typically driven externally by repeatedly running XenClean. XenClean will exit with one of the exit codes documented in ONBOARDING_PHASES. """ - insert_cd_safe(vm, guest_tools_iso["name"]) + insert_cd_safe(vm, guest_tools_iso.name) logging.info("Run XenClean") - xenclean_path = PureWindowsPath("D:\\") / guest_tools_iso["xenclean_path"] - if guest_tools_iso["xenclean_path"].lower().endswith(".ps1"): + xenclean_path = PureWindowsPath("D:\\") / guest_tools_iso.xenclean_path + if guest_tools_iso.xenclean_path.lower().endswith(".ps1"): assert not onboard, "Onboarding not supported in older versions" xenclean_cmd = f"Set-Location C:\\; {xenclean_path} -NoReboot -Confirm:$false; {WINDOWS_SHUTDOWN_COMMAND}" else: xenclean_cmd = f"Set-Location C:\\; {xenclean_path} -noReboot -noConfirm" if onboard: - onboard_family = guest_tools_iso["onboard_family"] + onboard_family = guest_tools_iso.onboard_family xenclean_cmd += f" -onboard {onboard_family}; Set-Content {ONBOARD_EXIT_CODE_FILE} $LASTEXITCODE -Force" else: xenclean_cmd += "; if ($LASTEXITCODE -ne 0) {{throw}}" @@ -87,9 +88,9 @@ def run_xenclean(vm: VM, guest_tools_iso: dict[str, Any], onboard: bool = False) @pytest.fixture(scope="module") -def onboarding_guest_tools_iso(guest_tools_iso: dict[str, Any]) -> dict[str, Any]: - if not guest_tools_iso.get("onboard_family"): - pytest.skip("Onboarding info not declared in data.py") +def onboarding_guest_tools_iso(guest_tools_iso: WinGuestToolDef) -> WinGuestToolDef: + if not guest_tools_iso.onboard_family: + pytest.skip("Onboarding info not declared in the config") return guest_tools_iso @@ -97,7 +98,7 @@ def onboarding_guest_tools_iso(guest_tools_iso: dict[str, Any]) -> dict[str, Any @pytest.mark.usefixtures("windows_vm") class TestXenClean: def test_xenclean_without_tools( - self, running_unsealed_windows_vm: VM, guest_tools_iso: dict[str, Any] + self, running_unsealed_windows_vm: VM, guest_tools_iso: WinGuestToolDef ) -> None: vm = running_unsealed_windows_vm logging.info("XenClean with empty VM") @@ -105,13 +106,13 @@ def test_xenclean_without_tools( assert vm.are_windows_tools_uninstalled() def test_xenclean_onboard_without_tools(self, running_unsealed_windows_vm: VM, - onboarding_guest_tools_iso: dict[str, Any]) -> None: + onboarding_guest_tools_iso: WinGuestToolDef) -> None: vm = running_unsealed_windows_vm logging.info("XenClean onboard with empty VM") assert run_xenclean(vm, onboarding_guest_tools_iso, onboard=True) == "ReadyForOnboard" def test_xenclean_with_test_tools_early( - self, vm_install_test_tools_no_reboot: VM, guest_tools_iso: dict[str, Any] + self, vm_install_test_tools_no_reboot: VM, guest_tools_iso: WinGuestToolDef ) -> None: vm = vm_install_test_tools_no_reboot logging.info("XenClean with test tools (without reboot)") @@ -119,7 +120,7 @@ def test_xenclean_with_test_tools_early( assert vm.are_windows_tools_uninstalled() def test_xenclean_with_test_tools(self, vm_install_test_tools_no_reboot: VM, - guest_tools_iso: dict[str, Any]) -> None: + guest_tools_iso: WinGuestToolDef) -> None: vm = vm_install_test_tools_no_reboot vm.reboot() # HACK: In some cases, vm.reboot(verify=False) followed by vm.insert_cd() (as called by run_xenclean) @@ -134,7 +135,7 @@ def test_xenclean_with_test_tools(self, vm_install_test_tools_no_reboot: VM, check_vm_dns(vm) def test_xenclean_onboard_with_test_tools(self, vm_install_test_tools_no_reboot: VM, - onboarding_guest_tools_iso: dict[str, Any]) -> None: + onboarding_guest_tools_iso: WinGuestToolDef) -> None: vm = vm_install_test_tools_no_reboot vm.reboot() wait_for_vm_running_and_ssh_up_without_tools(vm) @@ -145,10 +146,10 @@ def test_xenclean_onboard_with_test_tools(self, vm_install_test_tools_no_reboot: assert vm.are_windows_tools_working() def test_xenclean_with_other_tools( - self, vm_install_other_drivers: tuple[VM, dict[str, Any]], guest_tools_iso: dict[str, Any] + self, vm_install_other_drivers: tuple[VM, InstalledGuestToolDef], guest_tools_iso: WinGuestToolDef ) -> None: vm, param = vm_install_other_drivers - if param.get("vendor_device"): + if param.vendor_device: pytest.skip("Skipping XenClean with vendor device present") set_vm_dns(vm) @@ -159,11 +160,11 @@ def test_xenclean_with_other_tools( check_vm_dns(vm) def test_xenclean_onboard_with_other_tools( - self, vm_install_other_drivers: tuple[VM, dict[str, Any]], onboarding_guest_tools_iso: dict[str, Any] + self, vm_install_other_drivers: tuple[VM, InstalledGuestToolDef], onboarding_guest_tools_iso: WinGuestToolDef ) -> None: vm, param = vm_install_other_drivers - onboarding_phase = param.get("onboarding_phase") - if not param.get("onboarding_phase"): + onboarding_phase = param.onboarding_phase + if not param.onboarding_phase: pytest.skip("Skipping XenClean on other tools with no defined onboarding phase") logging.info("XenClean onboard with other tools") diff --git a/tests/install/conftest.py b/tests/install/conftest.py index abf69412c..922da4ee8 100644 --- a/tests/install/conftest.py +++ b/tests/install/conftest.py @@ -7,25 +7,24 @@ import os import tempfile -from data import ARP_SERVER, ISO_IMAGES, ISO_IMAGES_BASE, ISO_IMAGES_CACHE, TEST_SSH_PUBKEY, TOOLS -from lib import installer, pxe +from lib import config, installer, pxe from lib.commands import local_cmd from lib.common import callable_marker, url_download, wait_for from lib.host import Host from lib.installer import AnswerFile from lib.vm import VM -from typing import Generator, Sequence +from typing import Generator, Sequence, cast # Return true if the version of the ISO doesn't support the source type. # Note: this is a quick-win hack, to avoid explicit enumeration of supported # package_source values for each ISO. def skip_package_source(version: str, package_source: str) -> tuple[bool, str]: - if version not in ISO_IMAGES: + if version not in config.install.isos.definitions: return True, "version of ISO {} is unknown".format(version) if package_source == "iso": - if ISO_IMAGES[version].get('net-only', False): + if config.install.isos.definitions[version].net_only or False: return True, "ISO image is net-only while package_source is local" return False, "do not skip" @@ -33,7 +32,7 @@ def skip_package_source(version: str, package_source: str) -> tuple[bool, str]: if package_source == "net": # Net install is not valid if there is no netinstall URL # FIXME: ISO includes a default URL so we should be able to omit net-url - if 'net-url' not in ISO_IMAGES[version]: + if config.install.isos.definitions[version].net_url is None: return True, "net-url required for netinstall was not found for {}".format(version) return False, "do not skip" @@ -82,21 +81,21 @@ def installer_iso(request: pytest.FixtureRequest) -> dict[str, str | bool]: skip, reason = skip_package_source(iso_key, package_source) if skip: pytest.skip(reason) - assert iso_key in ISO_IMAGES, f"ISO_IMAGES does not have a value for {iso_key}" - iso = ISO_IMAGES[iso_key]['path'] + assert iso_key in config.install.isos.definitions, f"install.isos.definitions does not have a value for {iso_key}" + iso = config.install.isos.definitions[iso_key].path if iso.startswith("/"): assert os.path.exists(iso), f"file not found: {iso}" local_iso = iso else: - cached_iso = os.path.join(ISO_IMAGES_CACHE, os.path.basename(iso)) + cached_iso = os.path.join(config.install.isos.cache_dir, os.path.basename(iso)) if not os.path.exists(cached_iso): - url = iso if ":/" in iso else (ISO_IMAGES_BASE + iso) + url = iso if ":/" in iso else (config.install.isos.base_url + iso) logging.info("installer_iso: downloading %r into %r", url, cached_iso) url_download(url, cached_iso) local_iso = cached_iso logging.info("installer_iso: using %r", local_iso) return dict(iso=local_iso, - unsigned=ISO_IMAGES[iso_key].get('unsigned', False), + unsigned=config.install.isos.definitions[iso_key].unsigned or False, ) @pytest.fixture(scope='function') @@ -122,9 +121,8 @@ def remastered_iso(installer_iso: dict[str, str | bool], answerfile: AnswerFile iso_file = str(installer_iso['iso']) unsigned = installer_iso['unsigned'] - assert "iso-remaster" in TOOLS - iso_remaster = TOOLS["iso-remaster"] - assert os.access(iso_remaster, os.X_OK) + assert config.install.iso_remaster, "install.iso_remaster is not configured" + assert os.access(config.install.iso_remaster, os.X_OK) with tempfile.TemporaryDirectory(prefix="remastered-iso-") as isotmp: remastered_iso = os.path.join(isotmp, "image.iso") @@ -153,7 +151,7 @@ def remastered_iso(installer_iso: dict[str, str | bool], answerfile: AnswerFile INSTALLIMG="$1" install -d -m 750 "$INSTALLIMG/root/.ssh" -echo "{TEST_SSH_PUBKEY}" > "$INSTALLIMG/root/.ssh/authorized_keys" +echo "{config.ssh.pubkey}" > "$INSTALLIMG/root/.ssh/authorized_keys" chmod 600 "$INSTALLIMG/root/.ssh/authorized_keys" test ! -e "{answerfile_xml}" || @@ -196,7 +194,7 @@ def remastered_iso(installer_iso: dict[str, str | bool], answerfile: AnswerFile After=network-online.target [Service] Type=oneshot -ExecStart=/bin/sh -c 'while ! /usr/local/sbin/test-pingpxe.sh "{ARP_SERVER}"; do sleep 1 ; done' +ExecStart=/bin/sh -c 'while ! /usr/local/sbin/test-pingpxe.sh "{config.pxe.arp_server}"; do sleep 1 ; done' [Install] WantedBy=default.target EOF @@ -207,7 +205,7 @@ def remastered_iso(installer_iso: dict[str, str | bool], answerfile: AnswerFile #!/bin/sh case "$1" in start) - sh -c 'while ! /usr/local/sbin/test-pingpxe.sh "{ARP_SERVER}"; do sleep 1 ; done' & ;; + sh -c 'while ! /usr/local/sbin/test-pingpxe.sh "{config.pxe.arp_server}"; do sleep 1 ; done' & ;; stop) ;; esac EOF @@ -232,7 +230,7 @@ def remastered_iso(installer_iso: dict[str, str | bool], answerfile: AnswerFile fi mkdir -p "$ROOT/root/.ssh" -echo "{TEST_SSH_PUBKEY}" >> "$ROOT/root/.ssh/authorized_keys" +echo "{config.ssh.pubkey}" >> "$ROOT/root/.ssh/authorized_keys" EOF """ print(script_contents, file=patcher_fd) @@ -261,7 +259,7 @@ def remastered_iso(installer_iso: dict[str, str | bool], answerfile: AnswerFile os.chmod(patcher_fd.fileno(), 0o755) # do remaster - local_cmd([iso_remaster, + local_cmd([config.install.iso_remaster, "--install-patcher", img_patcher_script, "--iso-patcher", iso_patcher_script, iso_file, remastered_iso diff --git a/tests/install/test.py b/tests/install/test.py index d552cc5f7..3376bbaa7 100644 --- a/tests/install/test.py +++ b/tests/install/test.py @@ -3,8 +3,7 @@ import logging from uuid import uuid4 -from data import ISO_IMAGES, NETWORKS -from lib import commands, installer, pxe +from lib import commands, config, installer, pxe from lib.common import safe_split, wait_for from lib.installer import AnswerFile from lib.pif import PIF @@ -14,8 +13,6 @@ from typing import Generator -assert "MGMT" in NETWORKS - # Requirements: # - one XCP-ng host capable of nested virt, with an ISO SR, and a default SR # - the "small_vm" ISO must have in authorized_keys a SSH key accepted by the @@ -83,7 +80,7 @@ class TestNested: dict(name="vm1 extra disk", size="50GiB", device="xvdb", userdevice="1") ], cd_vbd=dict(device="xvdd", userdevice="3"), - vifs=[dict(index=0, network_name=NETWORKS["MGMT"])], + vifs=[dict(index=0, network_name=config.network.mgmt)], )) @pytest.mark.answerfile.with_args( lambda system_disks_names, local_sr, package_source, iso_version: AnswerFile("INSTALL") @@ -91,7 +88,7 @@ class TestNested: .top_append( {"iso": {"TAG": "source", "type": "local"}, "net": {"TAG": "source", "type": "url", - "CONTENTS": ISO_IMAGES[iso_version]['net-url']}, # type: ignore + "CONTENTS": config.install.isos.definitions[iso_version].net_url}, }[package_source], {"TAG": "admin-interface", "name": "eth0", "proto": "dhcp"}, {"TAG": "primary-disk", @@ -340,7 +337,7 @@ def test_boot_inst(self, create_vms: list[VM], lambda system_disks_names, package_source, iso_version: AnswerFile("UPGRADE").top_append( {"iso": {"TAG": "source", "type": "local"}, "net": {"TAG": "source", "type": "url", - "CONTENTS": ISO_IMAGES[iso_version]['net-url']}, # type: ignore + "CONTENTS": config.install.isos.definitions[iso_version].net_url}, }[package_source], {"TAG": "existing-installation", "CONTENTS": system_disks_names[0]}, diff --git a/tests/network/conftest.py b/tests/network/conftest.py index ca9680f24..27bf05870 100644 --- a/tests/network/conftest.py +++ b/tests/network/conftest.py @@ -4,7 +4,7 @@ import logging -from data import HOST_FREE_NICS +from lib import config from lib.bond import Bond from lib.common import PackageManagerEnum from lib.host import Host @@ -66,13 +66,13 @@ def empty_network(host: Host) -> Generator[Network, None, None]: @pytest.fixture(scope='function') def bond_lacp(host: Host, empty_network: Network) -> Generator[Bond, None, None]: - if len(HOST_FREE_NICS) < 2: + if len(config.network.free_nics) < 2: pytest.fail("This fixture needs at least 2 free NICs") pifs = [] logging.info(f"bond: resolve PIFs on {host.hostname_or_ip} using \ {[(pif.network_uuid(), pif.param_get('device')) for pif in host.pifs()]}") - for name in HOST_FREE_NICS[0:2]: + for name in config.network.free_nics[0:2]: [pif] = host.pifs(device=name) pifs.append(pif) @@ -82,13 +82,13 @@ def bond_lacp(host: Host, empty_network: Network) -> Generator[Bond, None, None] @pytest.fixture(scope='function') def bond_activebackup(host: Host, empty_network: Network) -> Generator[Bond, None, None]: - if len(HOST_FREE_NICS) < 2: + if len(config.network.free_nics) < 2: pytest.fail("This fixture needs at least 2 free NICs") pifs = [] logging.info(f"bond: resolve PIFs on {host.hostname_or_ip} using \ {[(pif.network_uuid(), pif.param_get('device')) for pif in host.pifs()]}") - for name in HOST_FREE_NICS[0:2]: + for name in config.network.free_nics[0:2]: [pif] = host.pifs(device=name) pifs.append(pif) @@ -98,13 +98,13 @@ def bond_activebackup(host: Host, empty_network: Network) -> Generator[Bond, Non @pytest.fixture(scope='function') def bond_balanceslb(host: Host, empty_network: Network) -> Generator[Bond, None, None]: - if len(HOST_FREE_NICS) < 2: + if len(config.network.free_nics) < 2: pytest.fail("This fixture needs at least 2 free NICs") pifs = [] logging.info(f"bond: resolve PIFs on {host.hostname_or_ip} using \ {[(pif.network_uuid(), pif.param_get('device')) for pif in host.pifs()]}") - for name in HOST_FREE_NICS[0:2]: + for name in config.network.free_nics[0:2]: [pif] = host.pifs(device=name) pifs.append(pif) diff --git a/tests/storage/linstor/conftest.py b/tests/storage/linstor/conftest.py index 2f3c7800b..fc730c613 100644 --- a/tests/storage/linstor/conftest.py +++ b/tests/storage/linstor/conftest.py @@ -10,18 +10,13 @@ from dataclasses import dataclass import lib.commands as commands -from lib import config +from lib.config_loader import config from lib.host import Host from lib.pool import Pool from lib.sr import SR from lib.vdi import VDI from lib.vm import VM -try: - from data import LINSTOR_REDUNDANCY # type: ignore -except ImportError: - LINSTOR_REDUNDANCY = 2 - # explicit import for package-scope fixtures from pkgfixtures import ( _xfs_config_on_hostA2, @@ -157,7 +152,7 @@ def remove_linstor(host: Host) -> None: @pytest.fixture(scope='package') def linstor_redundancy(pool_with_linstor: Pool) -> int: - return min(len(pool_with_linstor.hosts), LINSTOR_REDUNDANCY) + return min(len(pool_with_linstor.hosts), config.storage.linstor.redundancy) @pytest.fixture(scope='package') def linstor_sr( diff --git a/tests/unit/test_config_loader.py b/tests/unit/test_config_loader.py new file mode 100644 index 000000000..4bdfcfc11 --- /dev/null +++ b/tests/unit/test_config_loader.py @@ -0,0 +1,219 @@ +from __future__ import annotations + +import pytest + +import logging +from pathlib import Path + +from passlib.hash import sha512_crypt + +from lib.config_loader import ( + ConfigError, + _build_config, + _load_toml_file, + _load_toml_with_includes, + _resolve_config_override, + load_config, +) + +from typing import Any + +REPO_ROOT = Path(__file__).parents[2] + + +def _full_config_dict() -> dict[str, Any]: + return load_config().model_dump() + + +def _config_warnings(caplog: pytest.LogCaptureFixture) -> list[str]: + return [ + record.getMessage() + for record in caplog.records + if record.name == "lib.config_loader" and record.levelno >= logging.WARNING + ] + + +def test_base_config_loads() -> None: + cfg = load_config() + assert cfg.host.default_user == "root" + assert cfg.pxe.config_server == "pxe" + assert cfg.volume_size == 2**30 + assert cfg.network.free_nics == [] + + +def test_schema_key_is_ignored() -> None: + assert "$schema" not in _load_toml_file(REPO_ROOT / "config.toml") + assert "$schema" not in _full_config_dict() + + +def test_default_password_hash_matches_password() -> None: + cfg = load_config() + assert sha512_crypt.verify(cfg.host.default_password, cfg.host.default_password_hash) + + +def test_unknown_section_warned(caplog: pytest.LogCaptureFixture) -> None: + data = _full_config_dict() + data["not_a_section"] = 1 + with caplog.at_level(logging.WARNING, logger="lib.config_loader"): + _build_config(data) + assert any("not_a_section" in m for m in _config_warnings(caplog)) + + +def test_unknown_key_warned(caplog: pytest.LogCaptureFixture) -> None: + data = _full_config_dict() + data["host"]["defalt_password"] = "typo" + with caplog.at_level(logging.WARNING, logger="lib.config_loader"): + _build_config(data) + assert any("defalt_password" in m for m in _config_warnings(caplog)) + + +def test_unknown_host_override_key_warned(caplog: pytest.LogCaptureFixture) -> None: + data = _full_config_dict() + data["hosts"]["1.2.3.4"] = {"pasword": "typo"} + with caplog.at_level(logging.WARNING, logger="lib.config_loader"): + _build_config(data) + assert any("pasword" in m for m in _config_warnings(caplog)) + + +def test_unknown_storage_key_warned(caplog: pytest.LogCaptureFixture) -> None: + data = _full_config_dict() + data["storage"]["lvmoiscsi"]["targetIQN"] = "ok" + data["storage"]["lvmoiscsi"]["SCSIid"] = "ok" + data["storage"]["lvmoiscsi"]["targetiqn"] = "typo" + with caplog.at_level(logging.WARNING, logger="lib.config_loader"): + _build_config(data) + assert any("targetiqn" in m for m in _config_warnings(caplog)) + + +def test_iso_alias_keys_not_warned(caplog: pytest.LogCaptureFixture) -> None: + data = _full_config_dict() + data["install"]["isos"]["definitions"]["83net"] = {"path": "x.iso", "net-url": "http://pxe/installers/xcp-ng/8.3"} + with caplog.at_level(logging.WARNING, logger="lib.config_loader"): + _build_config(data) + assert _config_warnings(caplog) == [] + + +def test_answerfiles_extra_keys_not_warned(caplog: pytest.LogCaptureFixture) -> None: + data = _full_config_dict() + data["install"]["answerfiles"]["INSTALL"]["mode"] = "upgrade" + with caplog.at_level(logging.WARNING, logger="lib.config_loader"): + _build_config(data) + assert _config_warnings(caplog) == [] + + +def test_answerfiles_allow_extra_keys() -> None: + data = _full_config_dict() + data["install"]["answerfiles"]["INSTALL"]["mode"] = "upgrade" + assert _build_config(data).install.answerfiles["INSTALL"].model_dump()["mode"] == "upgrade" + + +def test_storage_device_config_delta() -> None: + data = _full_config_dict() + data["storage"]["nfs"] = {"server": "10.0.0.2", "serverpath": "/vms"} + cfg = _build_config(data) + assert cfg.sr_device_config("NFS_DEVICE_CONFIG") == {"server": "10.0.0.2", "serverpath": "/vms"} + assert cfg.sr_device_config("CIFS_ISO_DEVICE_CONFIG") == {} + with pytest.raises(ConfigError): + cfg.sr_device_config("NFS_ISO_DEVICE_CONFIG", required=["location"]) + + +def test_include_merge(tmp_path: Path) -> None: + (tmp_path / "base.toml").write_text("a = 1\n") + (tmp_path / "overlay.toml").write_text('include = ["base.toml"]\nb = 2\n') + assert _load_toml_with_includes(tmp_path / "overlay.toml") == {"a": 1, "b": 2} + + +def test_include_cycle_detected(tmp_path: Path) -> None: + (tmp_path / "a.toml").write_text('include = ["b.toml"]\n') + (tmp_path / "b.toml").write_text('include = ["a.toml"]\n') + with pytest.raises(ConfigError): + _load_toml_with_includes(tmp_path / "a.toml") + + +def test_include_diamond_allowed(tmp_path: Path) -> None: + (tmp_path / "d.toml").write_text("d = 1\n") + (tmp_path / "b.toml").write_text('include = ["d.toml"]\nb = 1\n') + (tmp_path / "c.toml").write_text('include = ["d.toml"]\nc = 1\n') + (tmp_path / "a.toml").write_text('include = ["b.toml", "c.toml"]\n') + assert _load_toml_with_includes(tmp_path / "a.toml") == {"d": 1, "b": 1, "c": 1} + + +def test_env_override_parses_toml(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv('XCPNG_TESTS_network__free_nics', '["eth1"]') + assert load_config().network.free_nics == ["eth1"] + + +def test_env_override_preserves_key_case(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("XCPNG_TESTS_storage__lvmoiscsi__SCSIid", '"wwn-1234567890abcdef"') + assert load_config().storage.lvmoiscsi.SCSIid == "wwn-1234567890abcdef" + + +def test_env_override_preserves_value_case(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("XCPNG_TESTS_network__mgmt", '"MgmtNet"') + assert load_config().network.mgmt == "MgmtNet" + + +def test_inventory_from_config_empty_list_override() -> None: + from lib.tools.inventory import inventory_from_config + data = _full_config_dict() + data["tools"]["update"] = {"repositories": ["xcp-ng-base"], "disabled_repositories": ["epel"]} + data["hosts"] = { + "h1": {"repositories": [], "disabled_repositories": ["*"]}, + "h2": {"repositories": ["xcp-ng-updates"]}, + } + inv = inventory_from_config(_build_config(data)) + assert inv["hosts"]["h1"]["repositories"] == [] + assert inv["hosts"]["h1"]["disabled_repositories"] == ["*"] + assert inv["hosts"]["h2"]["repositories"] == ["xcp-ng-updates"] + assert inv["hosts"]["h2"]["disabled_repositories"] == ["epel"] + + +def test_resolve_config_override_as_path(tmp_path: Path) -> None: + f = tmp_path / "my.toml" + f.write_text("") + assert _resolve_config_override(str(f)) == f.resolve() + + +def test_resolve_config_override_short_name_in_config_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + cfg_dir = tmp_path / "cfg" + cfg_dir.mkdir() + (cfg_dir / "config.prod.toml").write_text("") + monkeypatch.setenv("XCPNG_CONFIG_DIR", str(cfg_dir)) + assert _resolve_config_override("prod") == (cfg_dir / "config.prod.toml").resolve() + + +def test_resolve_config_override_short_name_in_repo_root(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr("lib.config_loader.REPO_ROOT", tmp_path) + monkeypatch.setenv("XCPNG_CONFIG_DIR", str(tmp_path / "empty")) + (tmp_path / "config.local.toml").write_text("") + assert _resolve_config_override("local") == (tmp_path / "config.local.toml").resolve() + + +def test_dump_config_uses_aliases() -> None: + data = _full_config_dict() + data["install"]["isos"]["definitions"]["83net"] = {"path": "x.iso", "net-url": "http://pxe/installers/xcp-ng/8.3"} + defn = _build_config(data).model_dump(by_alias=True)["install"]["isos"]["definitions"]["83net"] + assert "net-url" in defn and defn["net-url"] == "http://pxe/installers/xcp-ng/8.3" + assert "net_url" not in defn + + +def test_config_value_dotted_path(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv('XCPNG_TESTS_host__default_password', '"envpass"') + cfg = load_config(config_values=['host.default_password=clival']) + assert cfg.host.default_password == "clival" + assert sha512_crypt.verify("clival", cfg.host.default_password_hash) + + +def test_config_value_quoted_segment() -> None: + cfg = load_config(config_values=['hosts."10.30.0.56".user=root']) + assert cfg.hosts["10.30.0.56"].user == "root" + + +def test_config_value_parses_toml() -> None: + cfg = load_config(config_values=['network.free_nics=["eth1","eth2"]']) + assert cfg.network.free_nics == ["eth1", "eth2"] + + +def test_config_value_invalid_key() -> None: + with pytest.raises(ConfigError): + load_config(config_values=["no_equals_sign"]) diff --git a/tests/unit/test_config_schema.py b/tests/unit/test_config_schema.py new file mode 100644 index 000000000..40344c8ad --- /dev/null +++ b/tests/unit/test_config_schema.py @@ -0,0 +1,122 @@ +from __future__ import annotations + +import json +import tomllib +from pathlib import Path + +from lib.config_loader import Config +from lib.config_schema import editor_schema +from lib.typing import JSONType + +_SCHEMA_PATH = Path(__file__).parents[2] / "config-schema.json" + + +def _schema() -> dict[str, JSONType]: + return json.loads(_SCHEMA_PATH.read_text()) + + +def _collect_refs(obj: JSONType, refs: list[str]) -> None: + if isinstance(obj, dict): + if "$ref" in obj and isinstance(obj["$ref"], str): + refs.append(obj["$ref"]) + for value in obj.values(): + _collect_refs(value, refs) + elif isinstance(obj, list): + for value in obj: + _collect_refs(value, refs) + + +def _validate_value(value: JSONType, schema: JSONType, defs: dict[str, JSONType], path: str = "$") -> None: + """Assert ``value`` matches ``schema``, for the subset of JSON Schema we generate.""" + if not isinstance(schema, dict): + return + ref = schema.get("$ref") + if isinstance(ref, str): + _validate_value(value, defs[ref[len("#/$defs/"):]], defs, path) + return + any_of = schema.get("anyOf") + if isinstance(any_of, list): + for branch in any_of: + try: + _validate_value(value, branch, defs, path) + return + except AssertionError: + pass + raise AssertionError(f"{path}: {value!r} matches none of {any_of!r}") + expected = schema.get("type") + if expected == "integer": + assert isinstance(value, int) and not isinstance(value, bool), f"{path}: {value!r} is not an integer" + elif expected == "string": + assert isinstance(value, str), f"{path}: {value!r} is not a string" + elif expected == "boolean": + assert isinstance(value, bool), f"{path}: {value!r} is not a boolean" + elif expected == "null": + assert value is None, f"{path}: {value!r} is not null" + elif expected == "array": + assert isinstance(value, list), f"{path}: {value!r} is not an array" + items = schema.get("items", {}) + for i, item in enumerate(value): + _validate_value(item, items, defs, f"{path}[{i}]") + elif expected == "object": + assert isinstance(value, dict), f"{path}: {value!r} is not an object" + properties = schema.get("properties", {}) + assert isinstance(properties, dict) + additional = schema.get("additionalProperties", True) + for key, sub in value.items(): + if key in properties: + _validate_value(sub, properties[key], defs, f"{path}.{key}") + elif additional is False: + raise AssertionError(f"{path}: additional property {key!r} is not allowed") + elif isinstance(additional, dict): + _validate_value(sub, additional, defs, f"{path}.{key}") + + +def test_schema_is_valid_json() -> None: + schema = _schema() + assert "properties" in schema + + +def test_schema_covers_all_model_fields() -> None: + model_props = Config.model_json_schema()["properties"] + schema_props = _schema()["properties"] + assert isinstance(schema_props, dict) + for key in model_props: + assert key in schema_props, f"config-schema.json is missing model field {key!r}" + + +def test_schema_defs_match_model() -> None: + model_defs = Config.model_json_schema()["$defs"] + schema_defs = _schema().get("$defs", {}) + assert isinstance(schema_defs, dict) + assert set(schema_defs) == set(model_defs), ( + "config-schema.json $defs are out of sync with the Config model. " + "Regenerate with: uv run scripts/gen-config-schema.py" + ) + + +def test_schema_refs_resolve() -> None: + schema = _schema() + defs = schema.get("$defs", {}) + assert isinstance(defs, dict) + refs: list[str] = [] + _collect_refs(schema, refs) + for ref in refs: + assert ref.startswith("#/$defs/"), f"unexpected $ref {ref!r}" + assert ref[len("#/$defs/"):] in defs, f"$ref {ref!r} does not resolve to a $def" + + +def test_schema_matches_generator() -> None: + assert _schema() == editor_schema(), ( + "config-schema.json is not what scripts/gen-config-schema.py produces. " + "Regenerate with: uv run scripts/gen-config-schema.py" + ) + + +def test_config_toml_validates_against_schema() -> None: + schema = _schema() + defs = schema.get("$defs", {}) + assert isinstance(defs, dict) + config_toml = Path(__file__).parents[2] / "config.toml" + with open(config_toml, "rb") as f: + data = tomllib.load(f) + _validate_value(data, schema, defs) diff --git a/uv.lock b/uv.lock index 993c15cf7..adaf760aa 100644 --- a/uv.lock +++ b/uv.lock @@ -885,6 +885,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/7b/ce1eafaf1a76852e2ec9b22edecf1daa58175c090266e9f6c64afcd81d91/stack_data-0.6.3-py3-none-any.whl", hash = "sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695", size = 24521, upload-time = "2023-09-30T13:58:03.53Z" }, ] +[[package]] +name = "tomli-w" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/19/75/241269d1da26b624c0d5e110e8149093c759b7a286138f4efd61a60e75fe/tomli_w-1.2.0.tar.gz", hash = "sha256:2dd14fac5a47c27be9cd4c976af5a12d87fb1f0b4512f81d69cce3b35ae25021", size = 7184, upload-time = "2025-01-15T12:07:24.262Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/18/c86eb8e0202e32dd3df50d43d7ff9854f8e0603945ff398974c1d91ac1ef/tomli_w-1.2.0-py3-none-any.whl", hash = "sha256:188306098d013b691fcadc011abd66727d3c414c571bb01b1a174ba8c983cf90", size = 6675, upload-time = "2025-01-15T12:07:22.074Z" }, +] + [[package]] name = "traitlets" version = "5.15.1" @@ -955,9 +964,12 @@ dependencies = [ { name = "passlib" }, { name = "pluggy" }, { name = "pydantic" }, + { name = "pygments" }, { name = "pytest" }, { name = "pytest-dependency" }, { name = "requests" }, + { name = "tomli-w" }, + { name = "typing-extensions" }, ] [package.dev-dependencies] @@ -973,7 +985,6 @@ dev = [ { name = "pyright" }, { name = "ruff" }, { name = "types-passlib" }, - { name = "typing-extensions" }, { name = "zizmor" }, ] @@ -987,9 +998,12 @@ requires-dist = [ { name = "passlib" }, { name = "pluggy", specifier = ">=1.1.0" }, { name = "pydantic" }, + { name = "pygments" }, { name = "pytest", specifier = ">=9.1.1" }, { name = "pytest-dependency" }, { name = "requests" }, + { name = "tomli-w" }, + { name = "typing-extensions" }, ] [package.metadata.requires-dev] @@ -1005,7 +1019,6 @@ dev = [ { name = "pyright" }, { name = "ruff" }, { name = "types-passlib" }, - { name = "typing-extensions" }, { name = "zizmor" }, ]