Add test_rate_limit to auditd watchdog test cases (#18555)

Add test_rate_limit to auditd watchdod test cases

Why I did it
Auditd watchgod container add ratelimit check in this PR:
sonic-net/sonic-buildimage#22620

Also, json format fix in this PR:
sonic-net/sonic-buildimage#22709

Add new test case to prevent regression

Work item tracking
Microsoft ADO (number only):32313402
How I did it
Add test_rate_limit to auditd watchdod test cases

How to verify it
Pass all test case.

Author: liuh-80

tests/auditd/test_auditd.py

@@ -84,7 +84,8 @@ def test_auditd_watchdog_functionality(duthosts, enum_rand_one_per_hwsku_hostnam
         "auditd_rules",
         "auditd_service",
         "auditd_active",
-        "auditd_reload"
+        "auditd_reload",
+        "rate_limit"
     ]
 
     # Check if all expected keys exist and have the value "OK"
@@ -267,3 +268,45 @@ def test_32bit_failure(duthosts, enum_rand_one_per_hwsku_hostname, check_auditd_
     output = duthost.command(DOCKER_EXEC_CMD.format(container_name) +
                              "'{} {}'".format(NSENTER_CMD, CURL_CMD), module_ignore_errors=True)["stdout"]
     pytest_assert('"auditd_reload":"FAIL ' in output, "Auditd watchdog reports auditd container is healthy")
+
+
+def debug_log(duthost):
+    content = duthost.command(r"sudo cat /etc/audit/rules.d/audit.rules", module_ignore_errors=True)["stdout"]
+    logger.warning("Content of /etc/audit/rules.d/audit.rules: {}".format(content))
+
+    running_config = duthost.command(r"sudo auditctl -s", module_ignore_errors=True)["stdout"]
+    logger.warning("Auditd running config: {}".format(running_config))
+
+
+def read_watchdog(duthost):
+    output = duthost.command(DOCKER_EXEC_CMD.format("auditd_watchdog") +
+                             "'{} {}'".format(NSENTER_CMD, CURL_CMD), module_ignore_errors=True)["stdout"]
+    try:
+        return json.loads(output)
+    except json.JSONDecodeError as e:
+        pytest.fail("Invalid JSON response from auditd watchdog: {} exception: {}".format(output, e))
+
+
+def test_rate_limit(duthosts, enum_rand_one_per_hwsku_hostname):
+    duthost = duthosts[enum_rand_one_per_hwsku_hostname]
+    verify_container_running(duthost, "auditd_watchdog")
+
+    debug_log(duthost)
+    rate_limit_status = read_watchdog(duthost).get("rate_limit")
+    pytest_assert(rate_limit_status == "OK",
+                  "Auditd watchdog check rate limit failed for: {}".format(rate_limit_status))
+
+    # watchdog will report FAIL when auditd running config mismatch with config file
+    duthost.command(r"sudo cp /etc/audit/rules.d/audit.rules /etc/audit.rules_backup")
+    duthost.command(r"sudo sed -i -e '$a\'$'\n''-r 1000' /etc/audit/rules.d/audit.rules")
+    duthost.command(r"sudo auditctl -r 2000")
+
+    debug_log(duthost)
+    rate_limit_status = read_watchdog(duthost).get("rate_limit")
+
+    # revert change before check result, so assert failed will not break next test
+    duthost.command(r"sudo cp /etc/audit.rules_backup /etc/audit/rules.d/audit.rules")
+    duthost.command(r"sudo service auditd restart")
+
+    pytest_assert(rate_limit_status.startswith("FAIL (rate_limit: "),
+                  "Auditd watchdog check rate limit failed for: {}".format(rate_limit_status))