Upgrade databricks sdk requirement to 0.63.0

pyproject.toml

@@ -45,7 +45,7 @@ classifiers = [
 ]
 
 
-dependencies = ["databricks-sdk>=0.58.0,<0.59.0",
+dependencies = ["databricks-sdk>=0.62.0,<0.63.0",
                 "databricks-labs-lsql>=0.16.0,<0.17.0",
                 "databricks-labs-blueprint>=0.11.0,<0.12.0",
                 "PyYAML>=6.0.0,<6.1.0",

tests/integration/assessment/test_dashboards.py

@@ -1,50 +1,11 @@
-import pytest
-from databricks.sdk.service.sql import Dashboard as SdkRedashDashboard
 from databricks.sdk.service.dashboards import Dashboard as SdkLakeviewDashboard
 
 from databricks.labs.ucx.assessment.dashboards import (
     LakeviewDashboardCrawler,
     Dashboard,
-    RedashDashboardCrawler,
 )
 
 
-@pytest.mark.skip(reason="Legacy dashboard creation is no longer supported by Databricks.")
-def test_redash_dashboard_crawler_crawls_dashboards(ws, make_dashboard, inventory_schema, sql_backend) -> None:
-    dashboard: SdkRedashDashboard = make_dashboard()
-    crawler = RedashDashboardCrawler(ws, sql_backend, inventory_schema)
-
-    dashboards = list(crawler.snapshot())
-
-    assert len(dashboards) >= 1
-    assert dashboard.id in {d.id for d in dashboards}, f"Missing dashboard: {dashboard.id}"
-
-
-@pytest.mark.skip(reason="Legacy dashboard creation is no longer supported by Databricks.")
-def test_redash_dashboard_crawler_crawls_dashboard(ws, make_dashboard, inventory_schema, sql_backend) -> None:
-    dashboard: SdkRedashDashboard = make_dashboard()
-    assert dashboard.id
-    make_dashboard()  # Ignore second dashboard
-    crawler = RedashDashboardCrawler(ws, sql_backend, inventory_schema, include_dashboard_ids=[dashboard.id])
-
-    dashboards = list(crawler.snapshot())
-
-    assert dashboards == [Dashboard.from_sdk_redash_dashboard(dashboard)]
-
-
-@pytest.mark.skip(reason="Legacy dashboard creation is no longer supported by Databricks.")
-def test_redash_dashboard_crawler_crawls_dashboards_with_debug_listing_upper_limit(
-    ws, make_dashboard, inventory_schema, sql_backend
-) -> None:
-    for _ in range(2):  # Create two dashboards, expect one to be snapshotted due to upper limit below
-        make_dashboard()
-    crawler = RedashDashboardCrawler(ws, sql_backend, inventory_schema, debug_listing_upper_limit=1)
-
-    dashboards = list(crawler.snapshot())
-
-    assert len(dashboards) == 1
-
-
 def test_lakeview_dashboard_crawler_crawls_dashboards(
     ws, make_lakeview_dashboard, inventory_schema, sql_backend
 ) -> None:
@@ -72,20 +33,6 @@ def test_lakeview_dashboard_crawler_crawls_dashboard(
     assert dashboards == [Dashboard.from_sdk_lakeview_dashboard(dashboard)]
 
 
-@pytest.mark.skip(
-    reason="The user cannot be found using the Dashboard creator user ID when running this test from the CI"
-)
-def test_redash_dashboard_ownership_is_me(runtime_ctx) -> None:
-    """The Redash owner should be the user that creates the dashboard, i.e. who runs this integration test."""
-    sdk_redash_dashboard = runtime_ctx.make_dashboard()
-    dashboard = Dashboard.from_sdk_redash_dashboard(sdk_redash_dashboard)
-
-    owner = runtime_ctx.dashboard_ownership.owner_of(dashboard)
-
-    current_user = runtime_ctx.workspace_client.current_user.me()
-    assert owner == current_user.user_name, f"Invalid owner for dashboard: {dashboard}"
-
-
 def test_lakeview_dashboard_ownership_is_me(runtime_ctx, make_lakeview_dashboard) -> None:
     """Lakeview dashboard do not have a `creator` field, thus we fall back on the parent workspace path owner"""
     sdk_lakeview_dashboard = make_lakeview_dashboard()

tests/integration/assessment/test_workflows.py

@@ -8,7 +8,6 @@
 @retried(on=[NotFound, InvalidParameterValue])
 def test_running_real_assessment_job(
     installation_ctx,
-    make_dashboard,
     sql_backend,
 ) -> None:
     ws_group, _ = installation_ctx.make_ucx_group()

tests/integration/conftest.py

@@ -29,7 +29,7 @@
 from databricks.sdk.service.dashboards import Dashboard as SdkLakeviewDashboard
 from databricks.sdk.service.iam import Group
 from databricks.sdk.service.jobs import Job, SparkPythonTask
-from databricks.sdk.service.sql import Dashboard as SdkRedashDashboard, WidgetPosition, WidgetOptions, LegacyQuery
+from databricks.sdk.service.sql import Dashboard as SdkRedashDashboard, LegacyQuery
 
 from databricks.labs.ucx.__about__ import __version__
 from databricks.labs.ucx.account.workspaces import AccountWorkspaces
@@ -135,66 +135,6 @@ def delete(dashboard: SdkLakeviewDashboard) -> None:
     yield from factory("dashboard", create, delete)
 
 
-@pytest.fixture
-def make_dashboard(
-    ws: WorkspaceClient,
-    make_random: Callable[[int], str],
-    make_query,
-    watchdog_purge_suffix,
-):
-    """Create a legacy dashboard.
-    This fixture is used to test migrating legacy dashboards to Lakeview.
-    """
-
-    def create(query: LegacyQuery | None = None) -> SdkRedashDashboard:
-        if not query:
-            query = make_query()
-        assert query
-        assert query.id
-        viz = ws.query_visualizations_legacy.create(
-            type="table",
-            query_id=query.id,
-            options={
-                "itemsPerPage": 1,
-                "condensed": True,
-                "withRowNumber": False,
-                "version": 2,
-                "columns": [
-                    {"name": "id", "title": "id", "allowSearch": True},
-                ],
-            },
-        )
-
-        dashboard_name = f"ucx_D{make_random(4)}_{watchdog_purge_suffix}"
-        dashboard = ws.dashboards.create(name=dashboard_name, tags=["original_dashboard_tag"])
-        assert dashboard.id is not None
-        ws.dashboard_widgets.create(
-            dashboard_id=dashboard.id,
-            visualization_id=viz.id,
-            width=1,
-            options=WidgetOptions(
-                title="",
-                position=WidgetPosition(
-                    col=0,
-                    row=0,
-                    size_x=3,
-                    size_y=3,
-                ),
-            ),
-        )
-        logger.info(f"Dashboard Created {dashboard_name}: {ws.config.host}/sql/dashboards/{dashboard.id}")
-        return ws.dashboards.get(dashboard.id)  # Dashboard with widget
-
-    def remove(dashboard: SdkRedashDashboard) -> None:
-        try:
-            assert dashboard.id is not None
-            ws.dashboards.delete(dashboard_id=dashboard.id)
-        except RuntimeError as e:
-            logger.info(f"Can't delete dashboard {e}")
-
-    yield from factory("dashboard", create, remove)
-
-
 @pytest.fixture
 def make_dbfs_data_copy(ws, make_cluster, env_or_skip):
     _ = make_cluster  # Need cluster to copy data
@@ -467,7 +407,6 @@ def __init__(  # pylint: disable=too-many-arguments
         make_job_fixture,
         make_notebook_fixture,
         make_query_fixture,
-        make_dashboard_fixture,
         make_lakeview_dashboard_fixture,
         make_cluster_policy_fixture,
         make_cluster_policy_permissions_fixture,
@@ -492,7 +431,6 @@ def __init__(  # pylint: disable=too-many-arguments
         self._make_job = make_job_fixture
         self._make_notebook = make_notebook_fixture
         self._make_query = make_query_fixture
-        self._make_dashboard = make_dashboard_fixture
         self._make_lakeview_dashboard = make_lakeview_dashboard_fixture
         self._make_cluster_policy = make_cluster_policy_fixture
         self._make_cluster_policy_permissions = make_cluster_policy_permissions_fixture
@@ -604,13 +542,6 @@ def make_query(self, **kwargs) -> LegacyQuery:
         self._queries.append(query)
         return query
 
-    def make_dashboard(self, *, query: LegacyQuery | None = None, **kwargs) -> SdkRedashDashboard:
-        dashboard = self._make_dashboard(query=query, **kwargs)
-        if query:
-            self._queries.append(query)
-        self._dashboards.append(dashboard)
-        return dashboard
-
     def make_lakeview_dashboard(self, **kwargs) -> SdkLakeviewDashboard:
         dashboard = self._make_lakeview_dashboard(**kwargs)
         self._lakeview_query_id = "query"  # Hardcoded query name in the `make_lakeview_dashboard` fixture
@@ -856,7 +787,6 @@ def runtime_ctx(  # pylint: disable=too-many-arguments
     make_job,
     make_notebook,
     make_query,
-    make_dashboard,
     make_lakeview_dashboard,
     make_cluster_policy,
     make_cluster_policy_permissions,
@@ -874,7 +804,6 @@ def runtime_ctx(  # pylint: disable=too-many-arguments
         make_job,
         make_notebook,
         make_query,
-        make_dashboard,
         make_lakeview_dashboard,
         make_cluster_policy,
         make_cluster_policy_permissions,
@@ -1018,7 +947,6 @@ def __init__(  # pylint: disable=too-many-arguments, too-many-locals
         make_job_fixture,
         make_notebook_fixture,
         make_query_fixture,
-        make_dashboard_fixture,
         make_lakeview_dashboard_fixture,
         make_cluster_policy,
         make_cluster_policy_permissions,
@@ -1036,7 +964,6 @@ def __init__(  # pylint: disable=too-many-arguments, too-many-locals
             make_job_fixture,
             make_notebook_fixture,
             make_query_fixture,
-            make_dashboard_fixture,
             make_lakeview_dashboard_fixture,
             make_cluster_policy,
             make_cluster_policy_permissions,
@@ -1218,7 +1145,6 @@ def installation_ctx(  # pylint: disable=too-many-arguments,too-many-locals
     make_job,
     make_notebook,
     make_query,
-    make_dashboard,
     make_lakeview_dashboard,
     make_cluster_policy,
     make_cluster_policy_permissions,
@@ -1239,7 +1165,6 @@ def installation_ctx(  # pylint: disable=too-many-arguments,too-many-locals
         make_job,
         make_notebook,
         make_query,
-        make_dashboard,
         make_lakeview_dashboard,
         make_cluster_policy,
         make_cluster_policy_permissions,

tests/integration/source_code/test_queries.py

@@ -1,24 +1,20 @@
-import pytest
 from databricks.labs.lsql.backends import Row
 
-from databricks.labs.ucx.source_code.base import DirectFsAccess, LineageAtom, UsedTable
+from databricks.labs.ucx.source_code.base import DirectFsAccess, LineageAtom
 
 
-@pytest.mark.skip(reason="Legacy dashboard creation is no longer supported by Databricks.")
 def test_query_linter_lints_queries_and_stores_dfsas_and_tables(simple_ctx) -> None:
     query_with_dfsa = simple_ctx.make_query(sql_query="SELECT * from csv.`dbfs://some_folder/some_file.csv`")
-    dashboard_with_dfsa = simple_ctx.make_dashboard(query=query_with_dfsa)
     # Lakeview dashboard expects a string, not a legacy query
-    dashboard_with_used_table = simple_ctx.make_lakeview_dashboard(query="SELECT * FROM some_schema.some_table")
 
     simple_ctx.query_linter.refresh_report()
 
     problems = list(simple_ctx.sql_backend.fetch("SELECT * FROM query_problems", schema=simple_ctx.inventory_database))
     assert problems == [
         Row(
-            dashboard_id=dashboard_with_dfsa.id,
-            dashboard_parent=dashboard_with_dfsa.parent,
-            dashboard_name=dashboard_with_dfsa.name,
+            dashboard_id=None,
+            dashboard_parent=None,
+            dashboard_name=None,
             query_id=query_with_dfsa.id,
             query_parent=query_with_dfsa.parent,
             query_name=query_with_dfsa.name,
@@ -31,18 +27,13 @@ def test_query_linter_lints_queries_and_stores_dfsas_and_tables(simple_ctx) -> N
     # By comparing the element instead of the list the `field(compare=False)` of the dataclass attributes take effect
     assert dfsas == [
         DirectFsAccess(
-            source_id=f"{dashboard_with_dfsa.id}/{query_with_dfsa.id}",
+            source_id=f"no-dashboard-id/{query_with_dfsa.id}",
             source_lineage=[
-                LineageAtom(
-                    object_type="DASHBOARD",
-                    object_id=dashboard_with_dfsa.id,
-                    other={"parent": dashboard_with_dfsa.parent, "name": dashboard_with_dfsa.name},
-                ),
                 LineageAtom(
                     object_type="QUERY",
-                    object_id=f"{dashboard_with_dfsa.id}/{query_with_dfsa.id}",
+                    object_id=f"no-dashboard-id/{query_with_dfsa.id}",
                     other={"name": query_with_dfsa.name},
-                ),
+                )
             ],
             path="dbfs://some_folder/some_file.csv",
             is_read=True,
@@ -54,28 +45,4 @@ def test_query_linter_lints_queries_and_stores_dfsas_and_tables(simple_ctx) -> N
     # By comparing the element instead of the list the `field(compare=False)` of the dataclass attributes take effect
     # The "query" in the source and object id, and "count" in the name are hardcoded in the
     # `make_lakeview_dashboard` fixture
-    assert used_tables == [
-        UsedTable(
-            source_id=f"{dashboard_with_used_table.dashboard_id}/query",
-            source_lineage=[
-                LineageAtom(
-                    object_type="DASHBOARD",
-                    object_id=dashboard_with_used_table.dashboard_id,
-                    other={
-                        "parent": dashboard_with_used_table.parent_path,
-                        "name": dashboard_with_used_table.display_name,
-                    },
-                ),
-                LineageAtom(
-                    object_type="QUERY",
-                    object_id=f"{dashboard_with_used_table.dashboard_id}/query",
-                    other={"name": "count"},
-                ),
-            ],
-            catalog_name="hive_metastore",
-            schema_name="some_schema",
-            table_name="some_table",
-            is_read=True,
-            is_write=False,
-        )
-    ]
+    assert not used_tables