Gracefully handle payload.schema.json

Author: robertwoj-microsoft

.pre-commit-config.yaml

@@ -56,7 +56,7 @@ repos:
         require_serial: true
         files: |
           (?x)(
-          ^src/modules/complianceengine/src/lib/payload.schema.jsonx$|
+          ^src/modules/complianceengine/src/lib/payload.schema.json$|
           ^src/modules/complianceengine/src/lib/procedures/.*\.cpp$|
           ^src/modules/complianceengine/src/lib/procedures/.*\.schema.json$
             )
@@ -70,7 +70,7 @@ repos:
         require_serial: true
         files: |
           (?x)(
-          ^src/modules/complianceengine/src/lib/payload.schema.jsonx$|
+          ^src/modules/complianceengine/src/lib/payload.schema.json$|
           ^src/modules/complianceengine/src/lib/procedures/.*\.h$|
           ^src/modules/complianceengine/src/lib/procedures/.*\.schema.json$
             )

src/modules/complianceengine/src/lib/GenInterface.py

@@ -9,7 +9,7 @@
 
 @dataclass
 class Enum:
-    mapping: dict[str, str] = None
+    mapping: OrderedDict[str, str] = None
     description: str = None
 
 @dataclass
@@ -23,7 +23,7 @@ class Parameter:
 
 @dataclass
 class ProcedureDetails:
-    fields: list[Parameter] = None
+    fields: List[Parameter] = None
     filename: str = None
     line: int = None # Line number of the procedure declaration
 
@@ -96,7 +96,7 @@ def process_params_block(file, lineno: int):
     return fields, lineno
 
 def process_enum_block(file, lineno: int) -> tuple[Enum, int]:
-    result = Enum(mapping=dict[str, str]())
+    result = Enum(mapping=OrderedDict[str, str]())
     inside = False
     label_value = None
     for line in file:
@@ -398,24 +398,8 @@ def generate_procedure_map_impl(model: Model, filename: str):
         f.write("};\n")
         f.write("} // namespace ComplianceEngine\n")
 
-def main():
-    """Main function to process a directory and print results."""
-    basedir = os.path.dirname(os.path.realpath(__file__))
-    print(f"Processing procedures in {basedir}")
-    model = Model(OrderedDict[str, Procedure](), OrderedDict[str, Enum](), set[str]())
-    model.supported_types.add("int")
-    model.supported_types.add("std::string")
-    model.supported_types.add("regex")
-
-    # Iterate over header files and update the model
-    process_directory(model, f"{basedir}/procedures/")
-
-    validate_model(model)
-
-    generate_procedure_map_header(model, f"{basedir}/ProcedureMap2.h")
-    generate_procedure_map_impl(model, f"{basedir}/ProcedureMap2.cpp")
-
-    # Split the model into per-file schemas
+def split_model_by_file(model: Model) -> OrderedDict[str, Model]:
+    """Split the model into per-file schemas."""
     schema_model = OrderedDict[str, Model]()
     for name, details in model.procedures.items():
         if details.audit:
@@ -428,29 +412,105 @@ def main():
             if filename not in schema_model:
                 schema_model[filename] = Model(OrderedDict[str, Procedure](), OrderedDict[str, Enum](), set[str]())
             schema_model[filename].procedures[name] = details
-    for filename, model in schema_model.items():
+    return schema_model
+
+def generate_global_json_schema(split_model: OrderedDict[str, Model], basedir: str):
+    """Generate a global JSON schema file."""
+    print("Reading global payload schema.")
+    with open(f"{basedir}/payload.schema.json", "r") as f:
+        global_schema = json.load(f)
+
+    # Used for model validation
+    definitions = global_schema.get("definitions", {})
+    audits = definitions.get("auditProcedure", {}).get("anyOf", [])
+    audits_dict = dict()
+    for audit in audits:
+        pat = r"procedures/([^/]+)\.schema\.json#/definitions/audit"
+        match = re.match(pat, audit.get("$ref",""))
+        if not match:
+            raise ValueError(f"invalid audit reference {audit}")
+        fname = match.group(1)
+        audits_dict[fname] = audit
+
+    remediations = definitions.get("remediationProcedure", {}).get("anyOf", [])
+    remediations_dict = dict()
+    for remediation in remediations:
+        pat = r"procedures/([^/]+)\.schema\.json#/definitions/remediation"
+        match = re.match(pat, remediation.get("$ref",""))
+        if not match:
+            # This may happend when the missing remediation fallback is present
+            continue
+        fname = match.group(1)
+        remediations_dict[fname] = remediation
+
+    for filename in split_model.keys():
+        if not filename.endswith(".h"):
+            raise ValueError("filename must end with .h")
+        filename = filename[:-2]
+        audits_dict[filename] = {"$ref" : f"procedures/{filename}.schema.json#/definitions/audit" }
+        remediations_dict[filename] = {"$ref" : f"procedures/{filename}.schema.json#/definitions/remediation" }
+
+    audits = []
+    for k, v in sorted(audits_dict.items()):
+        audits.append(v)
+    remediations = []
+    for k, v in sorted(audits_dict.items()):
+        remediations.append(v)
+
+    definitions["auditProcedure"]["anyOf"] = audits
+    definitions["remediationProcedure"]["anyOf"] = remediations
+    # If there's no remediation we fall back to audit
+    fallback = { "$ref": "#definitions/auditProcedure" }
+    if fallback not in definitions["remediationProcedure"]["anyOf"]:
+        definitions["remediationProcedure"]["anyOf"].append( { "$ref": "#definitions/auditProcedure" })
+    print("Dumping global schema.")
+    with open(f"{basedir}/payload.schema.json", "w") as f:
+        json.dump(global_schema, f, indent=2)
+        # Keep EOL check happy
+        f.write("\n")
+
+def generate_json_schema(model: Model, basedir: str):
+    """Generate JSON schema files for each procedure."""
+    split_model = split_model_by_file(model)
+    print("Reading global payload schema.")
+    with open(f"{basedir}/payload.schema.json", "r") as f:
+        global_schema = json.load(f)
+
+    for filename, model in split_model.items():
         print(f"Writing schema for {filename}.")
-        if filename.endswith(".h"):
-            filename = filename[:-2]
+        if not filename.endswith(".h"):
+            raise ValueError("filename must end with .h")
+        filename = filename[:-2]
         json_schema_model = create_file_schema(model)
         with open(f"{basedir}/procedures/{filename}.schema.json","w") as f:
             json.dump(json_schema_model, f, indent=2)
             # Keep EOL check happy
             f.write("\n")
-    # Used for model validation
-    # print("Reading global payload schema.")
-    # schema = None
-    # with open(f"{basedir}/payload.schema.json", "r") as f:
-    #     schema = json.load(f)
-    # schema["definitions"]["auditProcedure"]["anyOf"] = [ {"$ref" : f"procedures/{fname}.schema.json#/definitions/audit" } for fname in result.keys() ]
-    # schema["definitions"]["remediationProcedure"]["anyOf"] = [ {"$ref" : f"procedures/{fname}.schema.json#/definitions/remediation" } for fname in result.keys() ]
-    # # If there's no remediation we fall back to audit
-    # schema["definitions"]["remediationProcedure"]["anyOf"].append( { "$ref": "#definitions/auditProcedure" })
-    # print("Dumping global schema.")
-    # with open(f"{basedir}/payload.schema.json", "w") as f:
-    #     json.dump(schema, f, indent=2)
-    #     # Keep EOL check happy
-    #     f.write("\n")
+
+    generate_global_json_schema(split_model, basedir)
+
+def main():
+    """Main function to process a directory and print results."""
+    basedir = os.path.dirname(os.path.realpath(__file__))
+    model = Model(OrderedDict[str, Procedure](), OrderedDict[str, Enum](), set[str]())
+
+    # Supported types are used in model validation
+    model.supported_types.add("int")
+    model.supported_types.add("std::string")
+    model.supported_types.add("regex")
+
+    # Iterate over header files and update the model
+    process_directory(model, f"{basedir}/procedures/")
+
+    # Test model integrity
+    validate_model(model)
+
+    # Generate C++ procedure map files
+    generate_procedure_map_header(model, f"{basedir}/ProcedureMap2.h")
+    generate_procedure_map_impl(model, f"{basedir}/ProcedureMap2.cpp")
+
+    # Generate JSON schema files
+    generate_json_schema(model, basedir)
 
 if __name__ == "__main__":
     import sys

src/modules/complianceengine/src/lib/GenJSONSchemas.py

@@ -182,7 +182,7 @@ def process_directory(directory_path):
         file_result = process_cpp_file(filepath)
         schema = create_file_schema(file_result)
         definitions = schema.get("definitions")
-        if len(definitions.get("audit").get("anyOf")) > 0 and len(definitions.get("remediation").get("anyOf")) > 0:
+        if len(definitions.get("audit").get("anyOf")) > 0 or len(definitions.get("remediation").get("anyOf")) > 0:
             result[basename] = schema
 
     return result
@@ -203,10 +203,46 @@ def main():
     schema = None
     with open(f"{basedir}/payload.schema.json", "r") as f:
         schema = json.load(f)
-    schema["definitions"]["auditProcedure"]["anyOf"] = [ {"$ref" : f"procedures/{fname}.schema.json#/definitions/audit" } for fname in result.keys() ]
-    schema["definitions"]["remediationProcedure"]["anyOf"] = [ {"$ref" : f"procedures/{fname}.schema.json#/definitions/remediation" } for fname in result.keys() ]
+
+    definitions = schema.get("definitions", {})
+    audits = definitions.get("auditProcedure", {}).get("anyOf", [])
+    audits_dict = dict()
+    for audit in audits:
+        pat = r"procedures/([^/]+)\.schema\.json#/definitions/audit"
+        match = re.match(pat, audit.get("$ref",""))
+        if not match:
+            continue
+        fname = match.group(1)
+        audits_dict[fname] = audit
+
+    remediations = definitions.get("remediationProcedure", {}).get("anyOf", [])
+    remediations_dict = dict()
+    for remediation in remediations:
+        pat = r"procedures/([^/]+)\.schema\.json#/definitions/remediation"
+        match = re.match(pat, remediation.get("$ref",""))
+        if not match:
+            continue
+        fname = match.group(1)
+        remediations_dict[fname] = remediation
+
+    for fname in result.keys():
+        audits_dict[fname] = {"$ref" : f"procedures/{fname}.schema.json#/definitions/audit" }
+        remediations_dict[fname] = {"$ref" : f"procedures/{fname}.schema.json#/definitions/remediation" }
+
+    audits = []
+    for k, v in sorted(audits_dict.items()):
+        audits.append(v)
+    remediations = []
+    for k, v in sorted(audits_dict.items()):
+        remediations.append(v)
+
+    # If there's no remediation we fall back to audit
+    schema["definitions"]["auditProcedure"]["anyOf"] = audits
+    schema["definitions"]["remediationProcedure"]["anyOf"] = remediations
     # If there's no remediation we fall back to audit
-    schema["definitions"]["remediationProcedure"]["anyOf"].append( { "$ref": "#definitions/auditProcedure" })
+    fallback = { "$ref": "#definitions/auditProcedure" }
+    if fallback not in definitions["remediationProcedure"]["anyOf"]:
+        definitions["remediationProcedure"]["anyOf"].append( { "$ref": "#definitions/auditProcedure" })
     print("Dumping global schema.")
     with open(f"{basedir}/payload.schema.json", "w") as f:
         json.dump(schema, f, indent=2)

src/modules/complianceengine/src/lib/payload.schema.json

@@ -205,6 +205,9 @@
         {
           "$ref": "procedures/ExecuteCommandGrep.schema.json#/definitions/audit"
         },
+        {
+          "$ref": "procedures/FileRegexMatch.schema.json#/definitions/audit"
+        },
         {
           "$ref": "procedures/PackageInstalled.schema.json#/definitions/audit"
         },
@@ -219,6 +222,9 @@
         },
         {
           "$ref": "procedures/TestingProcedures.schema.json#/definitions/audit"
+        },
+        {
+          "$ref": "procedures/UfwStatus.schema.json#/definitions/audit"
         }
       ]
     },
@@ -299,6 +305,9 @@
         {
           "$ref": "procedures/ExecuteCommandGrep.schema.json#/definitions/audit"
         },
+        {
+          "$ref": "procedures/FileRegexMatch.schema.json#/definitions/audit"
+        },
         {
           "$ref": "procedures/PackageInstalled.schema.json#/definitions/audit"
         },
@@ -313,6 +322,12 @@
         },
         {
           "$ref": "procedures/TestingProcedures.schema.json#/definitions/audit"
+        },
+        {
+          "$ref": "procedures/UfwStatus.schema.json#/definitions/audit"
+        },
+        {
+          "$ref": "#definitions/auditProcedure"
         }
       ]
     }