Use watchdog timeout to catalog properties (#48)

## Changes
Current implementation does not mark catalogs as being in use for the
watchdog to skip.
This PR fills the gap by adding a RemoveAfter entry to catalog
properties

### Linked issues
PR https://github.com/databrickslabs/watchdog/pull/58

### Tests
Not tested

---------

Co-authored-by: Eric Vergnaud <[email protected]>

Author: ericvergnaud

README.md

@@ -330,7 +330,7 @@ Create and provide a SQL backend for executing statements.
 
 Requires the environment variable `DATABRICKS_WAREHOUSE_ID` to be set.
 
-See also [`make_catalog`](#make_catalog-fixture), [`make_schema`](#make_schema-fixture), [`make_table`](#make_table-fixture), [`make_udf`](#make_udf-fixture), [`sql_exec`](#sql_exec-fixture), [`sql_fetch_all`](#sql_fetch_all-fixture), [`ws`](#ws-fixture), [`env_or_skip`](#env_or_skip-fixture).
+See also [`make_schema`](#make_schema-fixture), [`make_table`](#make_table-fixture), [`make_udf`](#make_udf-fixture), [`sql_exec`](#sql_exec-fixture), [`sql_fetch_all`](#sql_fetch_all-fixture), [`ws`](#ws-fixture), [`env_or_skip`](#env_or_skip-fixture).
 
 
 [[back to top](#python-testing-for-databricks)]
@@ -815,7 +815,7 @@ def test_catalog_fixture(make_catalog, make_schema, make_table):
     logger.info(f"Created new schema: {from_table_1}")
 ```
 
-See also [`ws`](#ws-fixture), [`sql_backend`](#sql_backend-fixture), [`make_random`](#make_random-fixture).
+See also [`ws`](#ws-fixture), [`make_random`](#make_random-fixture), [`watchdog_remove_after`](#watchdog_remove_after-fixture).
 
 
 [[back to top](#python-testing-for-databricks)]
@@ -1104,7 +1104,7 @@ See also [`ws`](#ws-fixture).
 ### `watchdog_remove_after` fixture
 Purge time for test objects, representing the (UTC-based) hour from which objects may be purged.
 
-See also [`make_cluster`](#make_cluster-fixture), [`make_instance_pool`](#make_instance_pool-fixture), [`make_job`](#make_job-fixture), [`make_model`](#make_model-fixture), [`make_pipeline`](#make_pipeline-fixture), [`make_query`](#make_query-fixture), [`make_schema`](#make_schema-fixture), [`make_serving_endpoint`](#make_serving_endpoint-fixture), [`make_table`](#make_table-fixture), [`make_warehouse`](#make_warehouse-fixture), [`watchdog_purge_suffix`](#watchdog_purge_suffix-fixture).
+See also [`make_catalog`](#make_catalog-fixture), [`make_cluster`](#make_cluster-fixture), [`make_instance_pool`](#make_instance_pool-fixture), [`make_job`](#make_job-fixture), [`make_model`](#make_model-fixture), [`make_pipeline`](#make_pipeline-fixture), [`make_query`](#make_query-fixture), [`make_schema`](#make_schema-fixture), [`make_serving_endpoint`](#make_serving_endpoint-fixture), [`make_table`](#make_table-fixture), [`make_warehouse`](#make_warehouse-fixture), [`watchdog_purge_suffix`](#watchdog_purge_suffix-fixture).
 
 
 [[back to top](#python-testing-for-databricks)]

src/databricks/labs/pytester/fixtures/catalog.py

@@ -1,5 +1,7 @@
 import logging
 from collections.abc import Generator, Callable
+from unittest.mock import Mock
+
 from pytest import fixture
 from databricks.labs.blueprint.commands import CommandExecutor
 from databricks.sdk.errors import DatabricksError
@@ -278,7 +280,9 @@ def remove(schema_info: SchemaInfo):
 
 
 @fixture
-def make_catalog(ws, sql_backend, make_random, log_workspace_link) -> Generator[Callable[..., CatalogInfo], None, None]:
+def make_catalog(
+    ws, make_random, watchdog_remove_after, log_workspace_link
+) -> Generator[Callable[..., CatalogInfo], None, None]:
     """
     Create a catalog and return its info. Remove it after the test.
     Returns instance of `databricks.sdk.service.catalog.CatalogInfo`.
@@ -294,11 +298,10 @@ def test_catalog_fixture(make_catalog, make_schema, make_table):
     """
 
     def create() -> CatalogInfo:
-        # Warning: As of 2024-09-04 there is no way to mark this catalog for protection against the watchdog.
-        # Ref: https://github.com/databrickslabs/watchdog/blob/cdc97afdac1567e89d3b39d938f066fd6267b3ba/scan/objects/uc.go#L68
         name = f"dummy_C{make_random(4)}".lower()
-        sql_backend.execute(f"CREATE CATALOG {name}")
-        catalog_info = ws.catalogs.get(name)
+        catalog_info = ws.catalogs.create(name=name, properties={"RemoveAfter": watchdog_remove_after})
+        if isinstance(catalog_info, Mock):
+            catalog_info.name = name
         log_workspace_link(f'{name} catalog', f'explore/data/{name}')
         return catalog_info
 

tests/unit/fixtures/test_catalog.py

@@ -102,11 +102,9 @@ def test_make_table_custom_schema():
 
 
 def test_make_catalog():
-    ctx, _ = call_stateful(make_catalog)
-
-    ctx['ws'].catalogs.get.assert_called_with('dummy_crandom')
-
-    assert ctx['sql_backend'].queries == ['CREATE CATALOG dummy_crandom']
+    ctx, info = call_stateful(make_catalog)
+    ctx['ws'].catalogs.create.assert_called()  # can't specify call params accurately
+    assert info.properties and info.properties.get("RemoveAfter", None)
 
 
 def test_make_udf():