Skip to content

Commit ad751b7

Browse files
ClausHolbechAristagmuloc
ClausHolbechArista
and
gmuloc
authored
Test(pyavd): Improve testing of schema validation (aristanetworks#5254)
Co-authored-by: gmuloc <[email protected]>
1 parent 6c565d4 commit ad751b7

File tree

11 files changed

+305
-177
lines changed

11 files changed

+305
-177
lines changed

‎.gitignore

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ __pycache__/
1313
/python-avd/vendor/
1414
/python-avd/pyavd.egg-info/
1515
/python-avd/.tox/
16-
/python-avd/.coverage
16+
**/.coverage
1717
/python-avd/.mypy_cache/
1818
# Ignore cloned cloudvision-python repo created during generation of _cv/api
1919
/python-avd/pyavd/_cv/cloudvision-apis/

‎docs/contribution/input-variable-validation.md

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@ The current implementation supports the automatic conversions listed below.
2626
| From (`convert_types`) | To (`type`) |
2727
| ---------------------- | ----------- |
2828
| `bool`, `str` | `int` |
29-
| `int`, `str` | `bool` |
3029
| `bool`, `int` | `str` |
3130

3231
An example of input variable conversion is `bgp_as`. `bgp_as` is expected as a string (`str`) since 32-bit AS numbers can be
@@ -143,12 +142,7 @@ The meta-schema does not allow for other keys to be set in the schema.
143142
| Key | Type | Required | Default | Value Restrictions | Description |
144143
| ----| ---- | -------- | ------- | ------------------ | ----------- |
145144
| <samp>type</samp> | String | True | | Valid Values:<br>- `"bool"` | Type of variable using the Python short names for each type.<br>`bool` for Boolean |
146-
| <samp>convert_types</samp> | List, items: String | | | | List of types to auto-convert from.<br>For type `bool`, auto-conversion is supported from `int` and `str` |
147-
| <samp>&nbsp;&nbsp;- \<str\></samp> | String | | | Valid Values:<br>- `"int"`<br>- `"str"` | |
148145
| <samp>default</samp> | Boolean | | | | Default value |
149-
| <samp>valid_values</samp> | List, items: Boolean | | | | List of valid values |
150-
| <samp>&nbsp;&nbsp;- \<int\></samp> | Boolean | | | | |
151-
| <samp>dynamic_valid_values</samp> | String | | | | Path to variable under the parent dictionary containing valid values.<br>Variable path use dot-notation and variable path must be relative to the parent dictionary.<br>If an element of the variable path is a list, every list item will unpacked.<br>**Note that this is building the schema from values in the *data* being validated!** |
152146
| <samp>display_name</samp> | String | | | Regex Pattern: `"^[^\n]+$"` | Free text display name for forms and documentation (single line) |
153147
| <samp>description</samp> | String | | | Minimum Length: 1 | Free text description for forms and documentation (multi line) |
154148
| <samp>required</samp> | Boolean | | | | Set if variable is required |

‎pyproject.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,5 +128,6 @@ include = [
128128
exclude = [
129129
"python-avd/pyavd/_eos_designs/schema/__init__.py",
130130
"python-avd/pyavd/_eos_designs/eos_designs_facts/schema/protocol.py",
131+
"python-avd/pyavd/_eos_designs/j2templates/compiled_templates",
131132
]
132133
pythonVersion = "3.10"

‎python-avd/pyavd/_schema/avd_meta_schema.json

Lines changed: 0 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -81,28 +81,10 @@
8181
"bool"
8282
]
8383
},
84-
"convert_types": {
85-
"type": "array",
86-
"description": "List of types to auto-convert from.\nFor 'bool' auto-conversion is supported from 'int' and 'str'",
87-
"items": {
88-
"type": "string",
89-
"enum": [
90-
"int",
91-
"str"
92-
]
93-
}
94-
},
9584
"default": {
9685
"description": "Default value",
9786
"type": "boolean"
9887
},
99-
"valid_values": {
100-
"type": "array",
101-
"description": "List of valid values",
102-
"items": {
103-
"type": "boolean"
104-
}
105-
},
10688
"display_name": {
10789
"$ref": "#/$defs/display_name"
10890
},
@@ -112,9 +94,6 @@
11294
"required": {
11395
"$ref": "#/$defs/required"
11496
},
115-
"dynamic_valid_values": {
116-
"$ref": "#/$defs/dynamic_valid_values"
117-
},
11897
"deprecation": {
11998
"$ref": "#/$defs/deprecation"
12099
},

‎python-avd/pyavd/_schema/avddataconverter.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121
SIMPLE_CONVERTERS = {
2222
"str": str,
2323
"int": int,
24-
"bool": bool,
2524
}
2625

2726
if TYPE_CHECKING:

‎python-avd/pyproject.toml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -163,3 +163,7 @@ markers = [
163163
"molecule_scenarios: Create test case fixtures 'molecule_host' or 'molecule_scenario' for the given molecule scenarios.",
164164
]
165165
asyncio_default_fixture_loop_scope = "function"
166+
167+
[tool.pyright]
168+
# Pyright stops at the first parent pyproject.toml it finds so as documented adding base config
169+
extends = "../pyproject.toml"

‎python-avd/tests/pyavd/schema/test_avdschema_bool.py

Lines changed: 17 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -20,41 +20,40 @@
2020
"keys": {
2121
"test_value": {
2222
"type": "bool",
23-
"convert_types": ["int", "str"], # Part of meta schema but not implemented in converter
2423
"default": True,
25-
"valid_values": [True],
26-
"dynamic_valid_values": ["valid_booleans"], # Part of meta schema but not implemented in converter
2724
"required": True,
2825
"description": "Some boolean",
2926
"display_name": "Boolean",
3027
},
3128
},
3229
}
3330

34-
TESTS = [
35-
# (test_value, expected_errors: tuple, expected_error_messages: tuple)
36-
(True, None, None), # Valid value. No errors.
37-
(False, (AvdValidationError,), ("'Validation Error: test_value': 'False' is not one of [True]",)), # Valid value. Not a valid value.
38-
(11.0123, (AvdValidationError,), ("'Validation Error: test_value': Invalid type 'float'. Expected a 'bool'.",)), # Invalid value.
39-
(None, (AvdValidationError,), ("'Validation Error: ': Required key 'test_value' is not set in dict.",)), # Required is set, so None is not ignored.
40-
("11", None, None), # Converted to True. No errors.
41-
("", (AvdValidationError,), ("'Validation Error: test_value': 'False' is not one of [True]",)), # Converted to False. Not a valid value.
42-
(12, None, None), # Converted to True. No errors.
43-
(0, (AvdValidationError,), ("'Validation Error: test_value': 'False' is not one of [True]",)), # Converted to False. Not a valid value.
44-
]
45-
4631

4732
@pytest.fixture(scope="module")
4833
def avd_schema() -> AvdSchema:
4934
return AvdSchema(TEST_SCHEMA)
5035

5136

52-
@pytest.mark.parametrize(("test_value", "expected_errors", "expected_error_messages"), TESTS)
53-
def test_generated_schema(test_value: Any, expected_errors: tuple | None, expected_error_messages: tuple | None, avd_schema: AvdSchema) -> None:
37+
@pytest.mark.parametrize(
38+
("test_value", "expected_errors", "expected_error_messages"),
39+
[
40+
# (test_value, expected_errors: tuple, expected_error_messages: tuple)
41+
pytest.param(True, None, None, id="ok"), # Valid value. No errors.
42+
pytest.param(
43+
11.0123, (AvdValidationError,), ("'Validation Error: test_value': Invalid type 'float'. Expected a 'bool'.",), id="err-invalid-type-float"
44+
),
45+
pytest.param(None, (AvdValidationError,), ("'Validation Error: ': Required key 'test_value' is not set in dict.",), id="err-missing-required-value"),
46+
pytest.param("", (AvdValidationError,), ("'Validation Error: test_value': Invalid type 'str'. Expected a 'bool'.",), id="err-invalid-type-str"),
47+
pytest.param(0, (AvdValidationError,), ("'Validation Error: test_value': Invalid type 'int'. Expected a 'bool'.",), id="err-invalid-type-int"),
48+
],
49+
)
50+
def test_generated_schema(
51+
test_value: Any, expected_errors: tuple[type[AvdValidationError], ...] | None, expected_error_messages: tuple[str, ...] | None, avd_schema: AvdSchema
52+
) -> None:
5453
instance = {"test_value": test_value}
5554
list(avd_schema.convert(instance))
5655
validation_errors = list(avd_schema.validate(instance))
57-
if expected_errors:
56+
if expected_errors and expected_error_messages:
5857
for validation_error in validation_errors:
5958
assert isinstance(validation_error, expected_errors)
6059
assert str(validation_error) in expected_error_messages

‎python-avd/tests/pyavd/schema/test_avdschema_dict.py

Lines changed: 52 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,6 @@
1313
from pyavd._errors import AvdValidationError
1414
from pyavd._schema.avdschema import AvdSchema
1515

16-
# TODO: Test default value with required False.
17-
# Test dynamic keys
18-
1916
TEST_SCHEMA = {
2017
"type": "dict",
2118
"keys": {
@@ -25,39 +22,75 @@
2522
"required": True,
2623
"description": "Some string",
2724
"display_name": "String",
25+
"allow_other_keys": False,
2826
"keys": {
2927
"pri": {"type": "int", "convert_types": ["str"]},
3028
"foo": {"type": "str", "convert_types": ["int"]},
29+
"nested_dict": { # nested_dict is not required, but the key inside is.
30+
"type": "dict",
31+
"keys": {"required_key": {"type": "str", "required": True}},
32+
},
33+
"dynamic": {
34+
"type": "list",
35+
"items": {
36+
"type": "dict",
37+
"keys": {"key": {"type": "str"}},
38+
},
39+
},
3140
},
41+
"dynamic_keys": {"dynamic.key": {"type": "int"}},
3242
},
3343
},
3444
}
3545

36-
TESTS = [
37-
# (test_value, expected_errors: tuple, expected_error_messages: tuple)
38-
({"pri": 1, "foo": "foo1"}, None, None), # Valid value. No errors.
39-
({"pri": "1", "foo": 123}, None, None), # Valid value after conversion. No errors.
40-
({}, None, None), # Valid value. No errors.
41-
(
42-
None,
43-
(AvdValidationError,),
44-
("'Validation Error: ': Required key 'test_value' is not set in dict.",),
45-
), # Required is set, so None is not ignored.
46-
("a", (AvdValidationError,), ("'Validation Error: test_value': Invalid type 'str'. Expected a 'dict'.",)), # Invalid type.
47-
]
48-
4946

5047
@pytest.fixture(scope="module")
5148
def avd_schema() -> AvdSchema:
5249
return AvdSchema(TEST_SCHEMA)
5350

5451

55-
@pytest.mark.parametrize(("test_value", "expected_errors", "expected_error_messages"), TESTS)
56-
def test_generated_schema(test_value: Any, expected_errors: tuple | None, expected_error_messages: tuple | None, avd_schema: AvdSchema) -> None:
52+
@pytest.mark.parametrize(
53+
("test_value", "expected_errors", "expected_error_messages"),
54+
[
55+
pytest.param({"pri": 1, "foo": "foo1"}, None, None, id="ok"),
56+
pytest.param({"pri": "1", "foo": 123}, None, None, id="ok-nested-coercion"),
57+
pytest.param(
58+
{"invalid_key": True},
59+
(AvdValidationError,),
60+
("'Validation Error: test_value': Unexpected key(s) 'invalid_key' found in dict.",),
61+
id="err-invalid-key",
62+
),
63+
pytest.param({}, None, None, id="ok-empty"),
64+
pytest.param(None, (AvdValidationError,), ("'Validation Error: ': Required key 'test_value' is not set in dict.",), id="err-missing-required-value"),
65+
pytest.param(
66+
{"nested_dict": {}},
67+
(AvdValidationError,),
68+
("'Validation Error: test_value.nested_dict': Required key 'required_key' is not set in dict.",),
69+
id="err-missing-nested-required-value",
70+
),
71+
pytest.param("a", (AvdValidationError,), ("'Validation Error: test_value': Invalid type 'str'. Expected a 'dict'.",), id="err-invalid-type-str"),
72+
pytest.param({"mykey": 123, "dynamic": [{"key": "mykey"}]}, None, None, id="ok-dynamic-key-mykey"),
73+
pytest.param(
74+
{"mykey2": 123, "dynamic": [{"key": "mykey"}]},
75+
(AvdValidationError,),
76+
("'Validation Error: test_value': Unexpected key(s) 'mykey2' found in dict.",),
77+
id="err-invalid-dynamic-key-mykey2",
78+
),
79+
pytest.param(
80+
{"mykey": "bar", "dynamic": [{"key": "mykey"}]},
81+
(AvdValidationError,),
82+
("'Validation Error: test_value.mykey': Invalid type 'str'. Expected a 'int'.",),
83+
id="err-invalid-dynamic-key-value-bar",
84+
),
85+
],
86+
)
87+
def test_generated_schema(
88+
test_value: Any, expected_errors: tuple[type[AvdValidationError], ...] | None, expected_error_messages: tuple[str, ...] | None, avd_schema: AvdSchema
89+
) -> None:
5790
instance = {"test_value": test_value}
5891
list(avd_schema.convert(instance))
5992
validation_errors = list(avd_schema.validate(instance))
60-
if expected_errors:
93+
if expected_errors and expected_error_messages:
6194
for validation_error in validation_errors:
6295
assert isinstance(validation_error, expected_errors)
6396
assert str(validation_error) in expected_error_messages

‎python-avd/tests/pyavd/schema/test_avdschema_int.py

Lines changed: 43 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,6 @@
1313
from pyavd._errors import AvdValidationError
1414
from pyavd._schema.avdschema import AvdSchema
1515

16-
# TODO: Test Dynamic valid values.
17-
# Test default value with required False.
18-
1916
TEST_SCHEMA = {
2017
"type": "dict",
2118
"keys": {
@@ -26,48 +23,64 @@
2623
"min": 2,
2724
"max": 20,
2825
"valid_values": [0, 11, 22],
29-
"dynamic_valid_values": ["valid_values"], # Part of meta schema but not implemented in converter
26+
"dynamic_valid_values": ["dynamic.valid_value"], # Part of meta schema but not implemented in converter
3027
"required": True,
3128
"description": "Some integer",
3229
"display_name": "Integer",
3330
},
31+
"dynamic": {"type": "list", "items": {"type": "dict", "keys": {"valid_value": {"type": "int"}}}},
3432
},
3533
}
3634

37-
TESTS = [
38-
# (test_value, expected_errors: tuple, expected_error_messages: tuple)
39-
(11, None, None), # Valid value. No errors.
40-
(
41-
False,
42-
(AvdValidationError,),
43-
("'Validation Error: test_value': '0' is lower than the allowed minimum of 2.",),
44-
), # False is converted to 0 which is valid but below min.
45-
(
46-
True,
47-
(AvdValidationError,),
48-
("'Validation Error: test_value': '1' is lower than the allowed minimum of 2.", "'Validation Error: test_value': '1' is not one of [0, 11, 22]"),
49-
), # True is converted to 1 which is not valid.
50-
("11", None, None), # Converted to 11. No errors.
51-
(11.0123, None, None), # Converted to 11. No errors.
52-
(None, (AvdValidationError,), ("'Validation Error: ': Required key 'test_value' is not set in dict.",)), # Required is set, so None is not ignored.
53-
(12, (AvdValidationError,), ("'Validation Error: test_value': '12' is not one of [0, 11, 22]",)), # Invalid value.
54-
([], (AvdValidationError,), ("'Validation Error: test_value': Invalid type 'list'. Expected a 'int'.",)), # Invalid type.
55-
(0, (AvdValidationError,), ("'Validation Error: test_value': '0' is lower than the allowed minimum of 2.",)), # Valid but below min.
56-
(22, (AvdValidationError,), ("'Validation Error: test_value': '22' is higher than the allowed maximum of 20.",)), # Valid but above max.
57-
]
58-
5935

6036
@pytest.fixture(scope="module")
6137
def avd_schema() -> AvdSchema:
6238
return AvdSchema(TEST_SCHEMA)
6339

6440

65-
@pytest.mark.parametrize(("test_value", "expected_errors", "expected_error_messages"), TESTS)
66-
def test_generated_schema(test_value: Any, expected_errors: tuple | None, expected_error_messages: tuple | None, avd_schema: AvdSchema) -> None:
67-
instance = {"test_value": test_value}
41+
@pytest.mark.parametrize(
42+
("test_value", "dynamic_valid_values", "expected_errors", "expected_error_messages"),
43+
[
44+
pytest.param(11, None, None, None, id="ok-no-coerce-11"), # Valid value. No errors.
45+
pytest.param(
46+
False,
47+
None,
48+
(AvdValidationError,),
49+
("'Validation Error: test_value': '0' is lower than the allowed minimum of 2.",),
50+
id="err-coerce-bool-to-int-below-min-0",
51+
), # False is converted to 0 which is valid but below min.
52+
pytest.param(
53+
True,
54+
None,
55+
(AvdValidationError,),
56+
("'Validation Error: test_value': '1' is lower than the allowed minimum of 2.", "'Validation Error: test_value': '1' is not one of [0, 11, 22]"),
57+
id="err-coerce-bool-to-int-invalid-value-1",
58+
),
59+
pytest.param("11", None, None, None, id="ok-coerce-str-to-int"),
60+
pytest.param(11.0123, None, None, None, id="ok-coerce-float-to-int"),
61+
pytest.param(
62+
None, None, (AvdValidationError,), ("'Validation Error: ': Required key 'test_value' is not set in dict.",), id="err-missing-required-value"
63+
),
64+
pytest.param(12, None, (AvdValidationError,), ("'Validation Error: test_value': '12' is not one of [0, 11, 22]",), id="err-invalid-value-12"),
65+
pytest.param(12, [12], None, None, id="ok-dynamic-valid-value-12"),
66+
pytest.param([], None, (AvdValidationError,), ("'Validation Error: test_value': Invalid type 'list'. Expected a 'int'.",), id="err-invalid-type-list"),
67+
pytest.param(0, None, (AvdValidationError,), ("'Validation Error: test_value': '0' is lower than the allowed minimum of 2.",), id="err-below-min-0"),
68+
pytest.param(
69+
22, None, (AvdValidationError,), ("'Validation Error: test_value': '22' is higher than the allowed maximum of 20.",), id="err-above-max-22"
70+
),
71+
],
72+
)
73+
def test_generated_schema(
74+
test_value: Any,
75+
dynamic_valid_values: list[int] | None,
76+
expected_errors: tuple[type[AvdValidationError], ...] | None,
77+
expected_error_messages: tuple[str, ...] | None,
78+
avd_schema: AvdSchema,
79+
) -> None:
80+
instance = {"test_value": test_value, "dynamic": [{"valid_value": valid_value} for valid_value in dynamic_valid_values or []]}
6881
list(avd_schema.convert(instance))
6982
validation_errors = list(avd_schema.validate(instance))
70-
if expected_errors:
83+
if expected_errors and expected_error_messages:
7184
for validation_error in validation_errors:
7285
assert isinstance(validation_error, expected_errors)
7386
assert str(validation_error) in expected_error_messages

0 commit comments

Comments
 (0)