Merge pull request #2342 from ggbecker/autotailor-validate-datastream-fields

autotailor: Add validation for variable names, rule IDs, and profile IDs

Author: jan-cerny

tests/utils/autotailor_integration_test.sh

@@ -112,6 +112,17 @@ assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www
 assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R3"]/result[text()="notselected"]'
 assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R4"]/result[text()="notselected"]'
 
+# invalid selector for V1 should fail with a descriptive error
+! python3 $autotailor --id-namespace "com.example.www" --var-select V1=invalid $ds $original_profile 2>$stdout
+grep "Selector 'invalid' does not exist" $stdout
+
+# invalid selector for V2 should fail with available selectors listed
+! python3 $autotailor --id-namespace "com.example.www" --var-select V2=invalid $ds $original_profile 2>$stdout
+grep "Available selectors" $stdout
+
+# --no-validate bypasses selector validation
+python3 $autotailor --id-namespace "com.example.www" --no-validate --var-select V1=invalid $ds $original_profile > $tailoring
+
 # use JSON tailoring (P1)
 python3 $autotailor $ds --id-namespace "com.example.www" --json-tailoring $json_tailoring > $tailoring
 $OSCAP xccdf eval --profile JSON_P1 --progress --tailoring-file $tailoring --results $result $ds

tests/utils/test_autotailor.py

@@ -135,3 +135,146 @@ def test_get_datastream_uri():
     uri = t._get_datastream_uri()
     assert uri.startswith("file://")
     assert "relative/path/to/ds.xml" in uri
+
+
+def test_datastream_validator():
+    """Test that DataStreamValidator properly validates IDs."""
+    ds_path = pathlib.Path(__file__).parent.joinpath("data_stream.xml")
+    validator = autotailor.DataStreamValidator(str(ds_path))
+
+    # Test valid profile validation
+    validator.validate_profile("xccdf_com.example.www_profile_P1")
+
+    # Test valid value validation
+    validator.validate_value("xccdf_com.example.www_value_V1")
+    validator.validate_value("xccdf_com.example.www_value_V2")
+
+    # Test valid rule validation
+    validator.validate_rule("xccdf_com.example.www_rule_R1")
+    validator.validate_rule("xccdf_com.example.www_rule_R2")
+    validator.validate_rule("xccdf_com.example.www_rule_R3")
+    validator.validate_rule("xccdf_com.example.www_rule_R4")
+
+    # Test valid group validation
+    validator.validate_group("xccdf_com.example.www_group_G34")
+
+    # Test invalid profile
+    with pytest.raises(ValueError) as e:
+        validator.validate_profile("xccdf_com.example.www_profile_INVALID")
+    assert "Profile ID 'xccdf_com.example.www_profile_INVALID' does not exist" in str(e.value)
+
+    # Test invalid value with suggestion
+    with pytest.raises(ValueError) as e:
+        validator.validate_value("xccdf_com.example.www_value_V3")
+    assert "Value ID 'xccdf_com.example.www_value_V3' does not exist" in str(e.value)
+
+    # Test invalid rule with suggestion
+    with pytest.raises(ValueError) as e:
+        validator.validate_rule("xccdf_com.example.www_rule_R5")
+    assert "Rule ID 'xccdf_com.example.www_rule_R5' does not exist" in str(e.value)
+
+    # Test invalid group
+    with pytest.raises(ValueError) as e:
+        validator.validate_group("xccdf_com.example.www_group_INVALID")
+    assert "Group ID 'xccdf_com.example.www_group_INVALID' does not exist" in str(e.value)
+
+
+def test_profile_with_validator():
+    """Test that Profile uses validator to check IDs."""
+    ds_path = pathlib.Path(__file__).parent.joinpath("data_stream.xml")
+    validator = autotailor.DataStreamValidator(str(ds_path))
+
+    p = autotailor.Profile(validator=validator)
+    p.reverse_dns = "com.example.www"
+
+    # Test valid variable change works
+    p.add_value_change("V1", "30")
+
+    # Test invalid variable name fails
+    with pytest.raises(ValueError) as e:
+        p.add_value_change("INVALID_VAR", "test")
+    assert "Value ID 'xccdf_com.example.www_value_INVALID_VAR' does not exist" in str(e.value)
+
+    # Test valid rule selection works
+    p.select_rule("R1")
+
+    # Test invalid rule selection fails
+    with pytest.raises(ValueError) as e:
+        p.select_rule("INVALID_RULE")
+    assert "Rule ID 'xccdf_com.example.www_rule_INVALID_RULE' does not exist" in str(e.value)
+
+    # Test valid base profile validation
+    p.validate_base_profile("P1")
+
+    # Test invalid base profile fails
+    with pytest.raises(ValueError) as e:
+        p.validate_base_profile("INVALID_PROFILE")
+    assert "Profile ID 'xccdf_com.example.www_profile_INVALID_PROFILE' does not exist" in str(e.value)
+
+
+def test_validator_suggestions():
+    """Test that validator provides helpful suggestions for typos."""
+    ds_path = pathlib.Path(__file__).parent.joinpath("data_stream.xml")
+    validator = autotailor.DataStreamValidator(str(ds_path))
+
+    # Test suggestion for value with typo (V11 instead of V1)
+    with pytest.raises(ValueError) as e:
+        validator.validate_value("xccdf_com.example.www_value_V11")
+    error_msg = str(e.value)
+    assert "Did you mean one of these?" in error_msg
+    assert "xccdf_com.example.www_value_V1" in error_msg
+
+    # Test suggestion for rule with typo (R11 instead of R1)
+    with pytest.raises(ValueError) as e:
+        validator.validate_rule("xccdf_com.example.www_rule_R11")
+    error_msg = str(e.value)
+    assert "Did you mean one of these?" in error_msg
+    assert "xccdf_com.example.www_rule_R1" in error_msg
+
+
+def test_validate_selector():
+    """Test that validate_selector rejects selectors not present in the data stream."""
+    ds_path = pathlib.Path(__file__).parent.joinpath("data_stream.xml")
+    validator = autotailor.DataStreamValidator(str(ds_path))
+
+    # V1 has selector "thirty"; V2 has "some" and "other"
+    validator.validate_selector("xccdf_com.example.www_value_V1", "thirty")
+    validator.validate_selector("xccdf_com.example.www_value_V2", "some")
+    validator.validate_selector("xccdf_com.example.www_value_V2", "other")
+
+    # Invalid selector for V1
+    with pytest.raises(ValueError) as e:
+        validator.validate_selector("xccdf_com.example.www_value_V1", "invalid")
+    error_msg = str(e.value)
+    assert "Selector 'invalid' does not exist for Value 'xccdf_com.example.www_value_V1'" in error_msg
+    assert "thirty" in error_msg
+
+    # Invalid selector for V2 with a close-enough typo triggers a suggestion
+    with pytest.raises(ValueError) as e:
+        validator.validate_selector("xccdf_com.example.www_value_V2", "ther")
+    error_msg = str(e.value)
+    assert "Selector 'ther' does not exist for Value 'xccdf_com.example.www_value_V2'" in error_msg
+    assert "other" in error_msg
+
+
+def test_profile_selector_validation():
+    """Test that Profile validates selectors via -V/--var-select through refine_value."""
+    ds_path = pathlib.Path(__file__).parent.joinpath("data_stream.xml")
+    validator = autotailor.DataStreamValidator(str(ds_path))
+
+    p = autotailor.Profile(validator=validator)
+    p.reverse_dns = "com.example.www"
+
+    # Valid selector passes
+    p.change_selectors(["V1=thirty"])
+    p.change_selectors(["V2=some"])
+
+    # Invalid selector raises
+    with pytest.raises(ValueError) as e:
+        p.change_selectors(["V1=invalid"])
+    assert "Selector 'invalid' does not exist for Value 'xccdf_com.example.www_value_V1'" in str(e.value)
+
+    # Invalid value ID still raises before reaching selector check
+    with pytest.raises(ValueError) as e:
+        p.change_selectors(["NONEXISTENT=thirty"])
+    assert "Value ID 'xccdf_com.example.www_value_NONEXISTENT' does not exist" in str(e.value)