Fix: Don't show stacktraces when tests fail (#4485)

Author: izeigerman

sqlmesh/core/test/definition.py

@@ -349,18 +349,21 @@ def create_test(
         else:
             _raise_error(f"Model '{name}' is an unsupported model type for testing", path)
 
-        return test_type(
-            body,
-            test_name,
-            t.cast(Model, model),
-            models,
-            engine_adapter,
-            dialect,
-            path,
-            preserve_fixtures,
-            default_catalog,
-            concurrency,
-        )
+        try:
+            return test_type(
+                body,
+                test_name,
+                t.cast(Model, model),
+                models,
+                engine_adapter,
+                dialect,
+                path,
+                preserve_fixtures,
+                default_catalog,
+                concurrency,
+            )
+        except Exception as e:
+            raise TestError(f"Failed to create test {test_name} ({path})\n{str(e)}")
 
     def __str__(self) -> str:
         return f"{self.test_name} ({self.path})"
@@ -905,8 +908,8 @@ def _row_difference(left: pd.DataFrame, right: pd.DataFrame) -> pd.DataFrame:
 
 def _raise_error(msg: str, path: Path | None = None) -> None:
     if path:
-        raise TestError(f"{msg} at {path}")
-    raise TestError(msg)
+        raise TestError(f"Failed to run test at {path}:\n{msg}")
+    raise TestError(f"Failed to run test:\n{msg}")
 
 
 def _normalize_df_value(value: t.Any) -> t.Any:

sqlmesh/core/test/result.py

@@ -17,6 +17,8 @@ class ModelTextTestResult(unittest.TextTestResult):
     def __init__(self, *args: t.Any, **kwargs: t.Any):
         super().__init__(*args, **kwargs)
         self.successes = []
+        self.original_failures: t.List[t.Tuple[unittest.TestCase, ErrorType]] = []
+        self.original_errors: t.List[t.Tuple[unittest.TestCase, ErrorType]] = []
 
     def addSubTest(
         self,
@@ -48,8 +50,9 @@ def addFailure(self, test: unittest.TestCase, err: ErrorType) -> None:
             test: The test case.
             err: A tuple of the form returned by sys.exc_info(), i.e., (type, value, traceback).
         """
-        exctype, value, tb = err
-        self.original_err = (test, err)
+        exctype, value, _ = err
+        self.original_failures.append((test, err))
+        # Intentionally ignore the traceback to hide it from the user
         return super().addFailure(test, (exctype, value, None))  # type: ignore
 
     def addError(self, test: unittest.TestCase, err: ErrorType) -> None:
@@ -59,8 +62,10 @@ def addError(self, test: unittest.TestCase, err: ErrorType) -> None:
             test: The test case.
             err: A tuple of the form returned by sys.exc_info(), i.e., (type, value, traceback).
         """
-        self.original_err = (test, err)
-        return super().addError(test, err)  # type: ignore
+        exctype, value, _ = err
+        self.original_errors.append((test, err))
+        # Intentionally ignore the traceback to hide it from the user
+        return super().addError(test, (exctype, value, None))  # type: ignore
 
     def addSuccess(self, test: unittest.TestCase) -> None:
         """Called when the test case test succeeds.
@@ -105,9 +110,10 @@ def log_test_report(self, test_duration: float) -> None:
             stream.writeln(unittest.TextTestResult.separator2)
             stream.writeln(failure)
 
-        for _, error in errors:
+        for test_case, error in errors:
             stream.writeln(unittest.TextTestResult.separator1)
-            stream.writeln(f"ERROR: {error}")
+            stream.writeln(f"ERROR: {test_case}")
+            stream.writeln(error)
 
         # Output final report
         stream.writeln(unittest.TextTestResult.separator2)

sqlmesh/core/test/runner.py

@@ -150,9 +150,11 @@ def _run_single_test(
             if result.successes:
                 combined_results.addSuccess(result.successes[0])
             elif result.errors:
-                combined_results.addError(result.original_err[0], result.original_err[1])
+                for error_test, error in result.original_errors:
+                    combined_results.addError(error_test, error)
             elif result.failures:
-                combined_results.addFailure(result.original_err[0], result.original_err[1])
+                for failure_test, failure in result.original_failures:
+                    combined_results.addFailure(failure_test, failure)
             elif result.skipped:
                 skipped_args = result.skipped[0]
                 combined_results.addSkip(skipped_args[0], skipped_args[1])

tests/core/test_test.py

@@ -2,7 +2,9 @@
 
 import datetime
 import typing as t
+import io
 from pathlib import Path
+import unittest
 from unittest.mock import call, patch
 from shutil import copyfile
 
@@ -28,6 +30,7 @@
 from sqlmesh.core.macros import MacroEvaluator, macro
 from sqlmesh.core.model import Model, SqlModel, load_sql_based_model, model
 from sqlmesh.core.test.definition import ModelTest, PythonModelTest, SqlModelTest
+from sqlmesh.core.test.result import ModelTextTestResult
 from sqlmesh.utils.errors import ConfigError, SQLMeshError, TestError
 from sqlmesh.utils.yaml import dump as dump_yaml
 from sqlmesh.utils.yaml import load as load_yaml
@@ -1039,7 +1042,8 @@ def test_unknown_column_error() -> None:
             context=Context(config=Config(model_defaults=ModelDefaultsConfig(dialect="duckdb"))),
         ).run(),
         expected_msg=(
-            "sqlmesh.utils.errors.TestError: Detected unknown column(s)\n\n"
+            "sqlmesh.utils.errors.TestError: Failed to run test:\n"
+            "Detected unknown column(s)\n\n"
             "Expected column(s): id, value\n"
             "Unknown column(s): foo\n"
         ),
@@ -2520,3 +2524,61 @@ def upstream_table_python(context, **kwargs):
             context=sushi_context,
         ).run()
     )
+
+
+@pytest.mark.parametrize("is_error", [True, False])
+def test_model_test_text_result_reporting_no_traceback(
+    sushi_context: Context, full_model_with_two_ctes: SqlModel, is_error: bool
+) -> None:
+    test = _create_test(
+        body=load_yaml(
+            """
+test_foo:
+  model: sushi.foo
+  inputs:
+    raw:
+      - id: 1
+  outputs:
+    ctes:
+      source:
+        - id: 1
+      renamed:
+        - fid: 1
+  vars:
+    start: 2022-01-01
+    end: 2022-01-01
+                """
+        ),
+        test_name="test_foo",
+        model=sushi_context.upsert_model(full_model_with_two_ctes),
+        context=sushi_context,
+    )
+    stream = io.StringIO()
+    result = ModelTextTestResult(
+        stream=unittest.runner._WritelnDecorator(stream),  # type: ignore
+        verbosity=1,
+        descriptions=True,
+    )
+
+    try:
+        raise Exception("failure")
+    except Exception as e:
+        assert e.__traceback__ is not None
+        if is_error:
+            result.addError(test, (e.__class__, e, e.__traceback__))
+        else:
+            result.addFailure(test, (e.__class__, e, e.__traceback__))
+
+    result.log_test_report(0)
+
+    stream.seek(0)
+    output = stream.read()
+
+    # Make sure that the traceback is not printed
+    assert "Traceback" not in output
+    assert "File" not in output
+    assert "line" not in output
+
+    prefix = "ERROR" if is_error else "FAIL"
+    assert f"{prefix}: test_foo (None)" in output
+    assert "Exception: failure" in output