feat: fail unused snapshots (#77)

* docs: update contributing

* wip: fail when unused snapshots

* test: unused warn flag

* fix: clarify why unused snapshot was deleted

* cr: fix docs typo

Co-Authored-By: Noah <[email protected]>

* cr: more explicit option

* docs: add options docs

* cr: make instructions clearer

* refactor: option name

* refactor: xor exit status

* refactor: xor in place

* cr: fix typo

* docs: update table formatting

Co-authored-by: Noah <[email protected]>

Author: 2 people

CONTRIBUTING.md

@@ -27,8 +27,6 @@ These are mostly guidelines, not rules. Use your best judgment, and feel free to
 
 - [Commit Messages](#commit-messages)
 - [Code Styleguide](#code-styleguide)
-- [Specs Styleguide](#specs-styleguide)
-- [Docs Styleguide](#docs-styleguide)
 
 [Additional Notes](#additional-notes)
 
@@ -40,14 +38,14 @@ This project and everyone participating in it is governed by our [Code of Conduc
 
 ## What should I know before I get started
 
-### Snapshot Testing
-
-- Javascript snapshot testing [jest](https:/a/jestjs.io/docs/en/snapshot-testing)
-
 ### Python 3
 
 - Python typing and hints: [typing](https://docs.python.org/3/library/typing.html)
 
+### Snapshot Testing
+
+- Javascript snapshot testing [jest](https://jestjs.io/docs/en/snapshot-testing)
+
 ### Releases
 
 - Semantic versioning: [semver](https://semver.org/spec/v2.0.0.html)
@@ -91,12 +89,12 @@ Fill in the relevant sections, clearly linking the issue the change is attemping
 
 ## Styleguides
 
-### Git Commit Messages
+### Commit Messages
 
 Provide semantic commit messages following this [convention](https://www.conventionalcommits.org/en/v1.0.0/#summary).
 This informs the semantic versioning we use to control our [releases](#releases).
 
-### Specs Styleguide
+### Code Styleguide
 
 A linter is available to catch most of our styling concerns.
 This is provided in a pre-commit hook when setting up [local development](#local-development).

README.md

@@ -46,6 +46,15 @@ pytest --snapshot-update
 
 A snapshot file should be generated under a `__snapshots__` directory in the same directory as `test_file.py`. The `__snapshots__` directory and all its children should be committed along with your test code.
 
+### Options
+
+These are the cli options exposed to `pytest` by the plugin.
+
+| Option                   | Description                                                                                                                  |
+| ------------------------ | ---------------------------------------------------------------------------------------------------------------------------- |
+| `--snapshot-update`      | When supplied updates existing snapshots of any run tests, as well as deleting unused and generating new snapshots.          |
+| `--snapshot-warn-unused` | Syrupy default behaviour is to fail the test session when there any unused snapshots. This instructs the plugin not to fail. |
+
 ### Serializers
 
 Syrupy comes with a few built-in serializers for you to choose from. You should also feel free to extend the AbstractSnapshotSerializer if your project has a need not captured by one our built-ins.

src/syrupy/__init__.py

@@ -25,6 +25,13 @@ def pytest_addoption(parser: Any) -> None:
         dest="update_snapshots",
         help="Update snapshots",
     )
+    group.addoption(
+        "--snapshot-warn-unused",
+        action="store_true",
+        default=False,
+        dest="warn_unused_snapshots",
+        help="Do not fail on unused snapshots",
+    )
 
 
 def pytest_assertrepr_compare(op: str, left: Any, right: Any) -> Optional[List[str]]:
@@ -48,7 +55,9 @@ def pytest_sessionstart(session: Any) -> None:
     """
     config = session.config
     session._syrupy = SnapshotSession(
-        update_snapshots=config.option.update_snapshots, base_dir=config.rootdir
+        warn_unused_snapshots=config.option.warn_unused_snapshots,
+        update_snapshots=config.option.update_snapshots,
+        base_dir=config.rootdir,
     )
     session._syrupy.start()
 
@@ -69,15 +78,16 @@ def pytest_collection_finish(session: Any) -> None:
     session._syrupy._ran_items.update(session.items)
 
 
-def pytest_sessionfinish(session: Any) -> None:
+def pytest_sessionfinish(session: Any, exitstatus: int) -> None:
     """
     Add syrupy report to pytest after whole test run finished, before exiting.
     https://docs.pytest.org/en/latest/reference.html#_pytest.hookspec.pytest_sessionfinish
     """
     reporter = session.config.pluginmanager.get_plugin("terminalreporter")
-    session._syrupy.finish()
+    syrupy_exitstatus = session._syrupy.finish()
     for line in session._syrupy.report:
         reporter.write_line(line)
+    session.exitstatus |= syrupy_exitstatus
 
 
 @pytest.fixture

src/syrupy/constants.py

@@ -1,3 +1,5 @@
 SNAPSHOT_DIRNAME = "__snapshots__"
 SNAPSHOT_EMPTY_FILE = {"empty snapshot file"}
 SNAPSHOT_UNKNOWN_FILE = {"unknown snapshot file"}
+
+EXIT_STATUS_FAIL_UNUSED = 1

src/syrupy/session.py

@@ -12,7 +12,10 @@
     Set,
 )
 
-from .constants import SNAPSHOT_UNKNOWN_FILE
+from .constants import (
+    EXIT_STATUS_FAIL_UNUSED,
+    SNAPSHOT_UNKNOWN_FILE,
+)
 from .location import TestLocation
 from .terminal import (
     bold,
@@ -42,7 +45,10 @@ def empty_snapshot_groups() -> "SnapshotGroups":
 
 
 class SnapshotSession:
-    def __init__(self, *, update_snapshots: bool, base_dir: str):
+    def __init__(
+        self, *, warn_unused_snapshots: bool, update_snapshots: bool, base_dir: str
+    ):
+        self.warn_unused_snapshots = warn_unused_snapshots
         self.update_snapshots = update_snapshots
         self.base_dir = base_dir
         self.report: List[str] = []
@@ -60,7 +66,8 @@ def start(self) -> None:
         self._serializers = {}
         self._snapshot_groups = empty_snapshot_groups()
 
-    def finish(self) -> None:
+    def finish(self) -> int:
+        exitstatus = 0
         self._collate_snapshots()
         n_unused = self._count_snapshots(self._snapshot_groups.unused)
         n_written = self._count_snapshots(self._snapshot_groups.created)
@@ -97,12 +104,15 @@ def finish(self) -> None:
             ]
         if n_unused:
             if self.update_snapshots:
-                text_singular = "{} snapshot deleted."
-                text_plural = "{} snapshots deleted."
+                text_singular = "{} unused snapshot deleted."
+                text_plural = "{} unused snapshots deleted."
             else:
                 text_singular = "{} snapshot unused."
                 text_plural = "{} snapshots unused."
-            text_count = warning_style(n_unused)
+            if self.update_snapshots or self.warn_unused_snapshots:
+                text_count = warning_style(n_unused)
+            else:
+                text_count = error_style(n_unused)
             summary_lines += [
                 ngettext(text_singular, text_plural, n_unused).format(text_count)
             ]
@@ -118,15 +128,20 @@ def finish(self) -> None:
                     path_to_file = os.path.relpath(filepath, self.base_dir)
                     deleted_snapshots = ", ".join(map(bold, sorted(snapshots)))
                     self.add_report_line(
-                        f"Deleted {deleted_snapshots} ({path_to_file})"
+                        gettext(f"Deleted {deleted_snapshots} ({path_to_file})")
                     )
             else:
-                self.add_report_line(
-                    gettext(
-                        "Re-run pytest with --snapshot-update"
-                        " to delete the unused snapshots."
-                    )
+                message = gettext(
+                    "Re-run pytest with --snapshot-update"
+                    " to delete the unused snapshots."
                 )
+                if self.warn_unused_snapshots:
+                    message = warning_style(message)
+                else:
+                    message = error_style(message)
+                    exitstatus |= EXIT_STATUS_FAIL_UNUSED
+                self.add_report_line(message)
+        return exitstatus
 
     def add_report_line(self, line: str = "") -> None:
         self.report += [line]

tests/test_integration.py

@@ -41,7 +41,7 @@ def test_collected(snapshot):
     result_stdout = clean_output(result.stdout.str())
     assert "1 snapshot passed" in result_stdout
     assert "1 snapshot updated" in result_stdout
-    assert "1 snapshot deleted" in result_stdout
+    assert "1 unused snapshot deleted" in result_stdout
 
 
 def test_collection_parametrized(testdir):
@@ -83,7 +83,7 @@ def test_collected(snapshot, actual):
     result_stdout = clean_output(result.stdout.str())
     assert "2 snapshots passed" in result_stdout
     assert "snapshot updated" not in result_stdout
-    assert "1 snapshot deleted" in result_stdout
+    assert "1 unused snapshot deleted" in result_stdout
 
 
 @pytest.fixture
@@ -227,6 +227,17 @@ def test_unused_snapshots(stubs):
     assert "snapshots generated" not in result_stdout
     assert "4 snapshots passed" in result_stdout
     assert "1 snapshot unused" in result_stdout
+    assert result.ret == 1
+
+
+def test_unused_snapshots_warning(stubs):
+    _, testdir, tests, _ = stubs
+    testdir.makepyfile(test_file="\n\n".join(tests[k] for k in tests if k != "unused"))
+    result = testdir.runpytest("-v", "--snapshot-warn-unused")
+    result_stdout = clean_output(result.stdout.str())
+    assert "snapshots generated" not in result_stdout
+    assert "4 snapshots passed" in result_stdout
+    assert "1 snapshot unused" in result_stdout
     assert result.ret == 0
 
 
@@ -237,7 +248,7 @@ def test_removed_snapshots(stubs):
     result = testdir.runpytest("-v", "--snapshot-update")
     result_stdout = clean_output(result.stdout.str())
     assert "snapshot unused" not in result_stdout
-    assert "1 snapshot deleted" in result_stdout
+    assert "1 unused snapshot deleted" in result_stdout
     assert result.ret == 0
     assert os.path.isfile(filepath)
 
@@ -249,7 +260,7 @@ def test_removed_snapshot_file(stubs):
     result = testdir.runpytest("-v", "--snapshot-update")
     result_stdout = clean_output(result.stdout.str())
     assert "snapshots unused" not in result_stdout
-    assert "5 snapshots deleted" in result_stdout
+    assert "5 unused snapshots deleted" in result_stdout
     assert result.ret == 0
     assert not os.path.isfile(filepath)
 
@@ -263,7 +274,7 @@ def test_removed_empty_snapshot_file_only(stubs):
     result = testdir.runpytest("-v", "--snapshot-update")
     result_stdout = clean_output(result.stdout.str())
     assert os.path.relpath(filepath) not in result_stdout
-    assert "1 snapshot deleted" in result_stdout
+    assert "1 unused snapshot deleted" in result_stdout
     assert "empty snapshot file" in result_stdout
     assert os.path.relpath(empty_filepath) in result_stdout
     assert result.ret == 0
@@ -280,7 +291,7 @@ def test_removed_hanging_snapshot_file(stubs):
     result = testdir.runpytest("-v", "--snapshot-update")
     result_stdout = clean_output(result.stdout.str())
     assert os.path.relpath(filepath) not in result_stdout
-    assert "1 snapshot deleted" in result_stdout
+    assert "1 unused snapshot deleted" in result_stdout
     assert "unknown snapshot file" in result_stdout
     assert os.path.relpath(hanging_filepath) in result_stdout
     assert result.ret == 0

tests/test_single_serializer.py

@@ -108,5 +108,5 @@ def test_updated_snapshots(stubs, testcases_updated):
     result = testdir.runpytest("-v", "--snapshot-update")
     result_stdout = clean_output(result.stdout.str())
     assert "1 snapshot updated" in result_stdout
-    assert "1 snapshot deleted" in result_stdout
+    assert "1 unused snapshot deleted" in result_stdout
     assert result.ret == 0