Refactor CI to include ERROR grep

Author: 99Lys

.github/scripts/compare_test_failures.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+set -e
+
+echo "Comparing actual test failures with expected failures..."
+
+# Enable globstar for recursive globbing
+shopt -s globstar
+
+# Extract actual failures from test reports
+actual_failures=$(grep -E "(FAILED tests|ERROR tests)" reports/**/*.txt | awk '{print $2}' | sort)
+
+# Read expected failures
+expected_failures=$(sort .github/expected_failures.txt)
+
+echo "Expected Failures:"
+echo "$expected_failures"
+echo ""
+echo "Actual Failures:"
+echo "$actual_failures"
+echo ""
+
+# Identify unexpected failures (in actual but not in expected)
+unexpected_failures=$(comm -13 <(echo "$expected_failures") <(echo "$actual_failures"))
+
+# Identify missing expected failures (in expected but not in actual)
+missing_failures=$(comm -23 <(echo "$expected_failures") <(echo "$actual_failures"))
+
+# Initialize exit code
+exit_code=0
+
+if [ -n "$unexpected_failures" ]; then
+  echo "Unexpected test failures detected:"
+  echo "$unexpected_failures"
+  exit_code=1
+fi
+
+if [ -n "$missing_failures" ]; then
+  echo "::warning::Expected test failures that did not occur (they passed):"
+  echo "$missing_failures"
+  exit_code=1
+fi
+
+if [ $exit_code -eq 0 ]; then
+  echo "All failed tests are expected, and all expected failures have occurred."
+else
+  echo "Verification failed: There are unexpected or missing test failures."
+fi
+
+exit $exit_code

.github/workflows/ci.yml

@@ -139,43 +139,4 @@ jobs:
           path: reports/
 
       - name: Compare Actual Failures with Expected Failures
-        run: |
-          shopt -s globstar
-          actual_failures=$(grep "FAILED tests" reports/**/*.txt | awk '{print $2}' | sort)
-          expected_failures=$(sort .github/expected_failures.txt)
-
-          echo "Expected Failures:"
-          echo "$expected_failures"
-          echo ""
-          echo "Actual Failures:"
-          echo "$actual_failures"
-          echo ""
-
-          # Identify unexpected failures
-          unexpected_failures=$(comm -13 <(echo "$expected_failures") <(echo "$actual_failures"))
-
-          # Identify missing expected failures
-          missing_failures=$(comm -23 <(echo "$expected_failures") <(echo "$actual_failures"))
-
-          # Initialize exit code
-          exit_code=0
-
-          if [ -n "$unexpected_failures" ]; then
-            echo "Unexpected test failures detected:"
-            echo "$unexpected_failures"
-            exit_code=1
-          fi
-
-          if [ -n "$missing_failures" ]; then
-            echo "Expected test failures that did not occur (they passed):"
-            echo "$missing_failures"
-            exit_code=1
-          fi
-
-          if [ $exit_code -eq 0 ]; then
-            echo "All failed tests are expected, and all expected failures have occurred."
-          else
-            echo "Verification failed: There are unexpected or missing test failures."
-          fi
-
-          exit $exit_code
+        run: bash .github/scripts/compare_test_failures.sh

tests/hooks/test_model_hooks.py

@@ -127,13 +127,13 @@ def check_hooks(self, state, project, host, count=1):
 
 
 class TestPrePostModelHooksDremio(BaseTestPrePost):
-    @pytest.fixture(scope="class")
-    def unique_schema(self, request, prefix) -> str:
-        test_file = request.module.__name__
-        # We only want the last part of the name
-        test_file = test_file.split(".")[-1]
-        unique_schema = f"{BUCKET}.{prefix}_{test_file}"
-        return unique_schema
+    # @pytest.fixture(scope="class")
+    # def unique_schema(self, request, prefix) -> str:
+    #     test_file = request.module.__name__
+    #     # We only want the last part of the name
+    #     test_file = test_file.split(".")[-1]
+    #     unique_schema = f"{BUCKET}.{prefix}_{test_file}"
+    #     return unique_schema
 
     @pytest.fixture(scope="class")
     def project_config_update(self):
@@ -308,13 +308,13 @@ def models(self):
 class TestPrePostModelHooksOnSeedsPlusPrefixedDremio(
     TestPrePostModelHooksOnSeedsDremio
 ):
-    @pytest.fixture(scope="class")
-    def unique_schema(self, request, prefix) -> str:
-        test_file = request.module.__name__
-        # We only want the last part of the name
-        test_file = test_file.split(".")[-1]
-        unique_schema = f"{BUCKET}.{prefix}_{test_file}"
-        return unique_schema
+    # @pytest.fixture(scope="class")
+    # def unique_schema(self, request, prefix) -> str:
+    #     test_file = request.module.__name__
+    #     # We only want the last part of the name
+    #     test_file = test_file.split(".")[-1]
+    #     unique_schema = f"{BUCKET}.{prefix}_{test_file}"
+    #     return unique_schema
 
     @pytest.fixture(scope="class")
     def project_config_update(self):