Merge branch 'main' into docs/remove-outdated-teradata-mention

Author: m-abulazm

labs.yml

@@ -2,7 +2,7 @@
 name: lakebridge
 description: Code Transpiler and Data Reconciliation tool for Accelerating Data onboarding to Databricks from EDW, CDW and other ETL sources.
 install:
-  script: src/databricks/labs/lakebridge/base_install.py
+  script: src/databricks/labs/lakebridge/install.py
 uninstall:
   script: src/databricks/labs/lakebridge/uninstall.py
 entrypoint: src/databricks/labs/lakebridge/cli.py

src/databricks/labs/lakebridge/__init__.py

@@ -1,9 +1,26 @@
+import logging
+
 from databricks.sdk.core import with_user_agent_extra, with_product
+from databricks.labs.blueprint.entrypoint import is_in_debug
 from databricks.labs.blueprint.logger import install_logger
 from databricks.labs.lakebridge.__about__ import __version__
 
+# Ensure that anything that imports this (or lower) submodules triggers setup of the blueprint logging.
 install_logger()
 
+
+def initialize_logging() -> None:
+    """Common logging initialisation for non-CLI entry-points."""
+    # This is intended to be used by all the non-CLI entry-points, such as install/uninstall hooks and pipeline tasks.
+    # It emulates the behaviour of the blueprint App() initialisation, except that we don't have handoff from the
+    # Databricks CLI. As such the policy is:
+    #   - The root (and logging system in general) is left alone.
+    #   - If running in the IDE debugger, databricks.* will be set to DEBUG.
+    #   - Otherwise, databricks.* will be set to INFO.
+    databricks_log_level = logging.DEBUG if is_in_debug() else logging.INFO
+    logging.getLogger("databricks").setLevel(databricks_log_level)
+
+
 # Add lakebridge/<version> for projects depending on lakebridge as a library
 with_user_agent_extra("lakebridge", __version__)
 

src/databricks/labs/lakebridge/assessments/configure_assessment.py

@@ -16,7 +16,6 @@
 from databricks.labs.lakebridge.assessments import CONNECTOR_REQUIRED
 
 logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)
 
 
 def _save_to_disk(credential: dict, cred_file: Path) -> None:

src/databricks/labs/lakebridge/assessments/dashboards/execute.py

@@ -19,11 +19,15 @@
     ExtractSchemaValidationCheck,
     build_validation_report_dataframe,
 )
+from databricks.labs.lakebridge import initialize_logging
 
 logger = logging.getLogger(__name__)
 
 
-def main(*argv) -> None:
+def main(*argv: str) -> None:
+    """Lakeview Jobs task entry point: profiler_dashboards"""
+    initialize_logging()
+
     logger.debug(f"Arguments received: {argv}")
     assert len(sys.argv) == 4, f"Invalid number of arguments: {len(sys.argv)}"
     catalog_name = sys.argv[0]

src/databricks/labs/lakebridge/assessments/pipeline.py

@@ -18,7 +18,6 @@
 from databricks.labs.lakebridge.connections.database_manager import DatabaseManager, FetchResult
 
 logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)
 
 DB_NAME = "profiler_extract.db"
 

src/databricks/labs/lakebridge/base_install.py

@@ -16,7 +16,7 @@
 from databricks.sdk import WorkspaceClient
 
 from databricks.labs.blueprint.cli import App
-from databricks.labs.blueprint.entrypoint import is_in_debug
+from databricks.labs.blueprint.entrypoint import get_logger
 from databricks.labs.blueprint.installation import RootJsonValue, JsonObject, JsonValue
 from databricks.labs.blueprint.tui import Prompts
 
@@ -30,7 +30,6 @@
 from databricks.labs.lakebridge.connections.credential_manager import cred_file
 from databricks.labs.lakebridge.helpers.recon_config_utils import ReconConfigPrompts
 from databricks.labs.lakebridge.helpers.telemetry_utils import make_alphanum_or_semver
-from databricks.labs.lakebridge.install import installer
 from databricks.labs.lakebridge.reconcile.runner import ReconcileRunner
 from databricks.labs.lakebridge.lineage import lineage_generator
 from databricks.labs.lakebridge.reconcile.recon_config import RECONCILE_OPERATION_NAME, AGG_RECONCILE_OPERATION_NAME
@@ -48,7 +47,6 @@
 
 # Subclass to allow controlled access to protected methods.
 class Lakebridge(App):
-    _logger_instance: logging.Logger | None = None
 
     def create_workspace_client(self) -> WorkspaceClient:
         """Create a workspace client, with the appropriate product and version information.
@@ -58,15 +56,25 @@ def create_workspace_client(self) -> WorkspaceClient:
         self._patch_databricks_host()
         return self._workspace_client()
 
-    def get_logger(self) -> logging.Logger:
-        if self._logger_instance is None:
-            self._logger_instance = self._logger
-            self._logger_instance.setLevel(logging.INFO)
-        return self._logger_instance
+    def _log_level(self, raw: str) -> int:
+        """Convert the log-level provided by the Databricks CLI into a logging level supported by Python."""
+        log_level = super()._log_level(raw)
+        # Due to an issue in the handoff of the intended logging level from the Databricks CLI to our
+        # application, we can't currently distinguish between --log-level=WARN and nothing at all, where we
+        # prefer (and the application logging expects) INFO.
+        #
+        # Rather than default to only have WARNING logs show, it's preferable to default to INFO and have
+        # --log-level=WARN not work for now.
+        #
+        # See: https://github.com/databrickslabs/lakebridge/issues/2167
+        # TODO: Remove this once #2167 has been resolved.
+        if log_level == logging.WARNING:
+            log_level = logging.INFO
+        return log_level
 
 
 lakebridge = Lakebridge(__file__)
-logger = lakebridge.get_logger()
+logger = get_logger(__file__)
 
 
 def raise_validation_exception(msg: str) -> NoReturn:
@@ -745,6 +753,9 @@ def install_transpile(
     transpiler_repository: TranspilerRepository = TranspilerRepository.user_home(),
 ) -> None:
     """Install or upgrade the Lakebridge transpilers."""
+    # Avoid circular imports.
+    from databricks.labs.lakebridge.install import installer  # pylint: disable=cyclic-import, import-outside-toplevel
+
     is_interactive = interactive_mode(interactive)
     ctx = ApplicationContext(w)
     ctx.add_user_agent_extra("cmd", "install-transpile")
@@ -804,6 +815,9 @@ def configure_reconcile(
     transpiler_repository: TranspilerRepository = TranspilerRepository.user_home(),
 ) -> None:
     """Configure the Lakebridge reconciliation module"""
+    # Avoid circular imports.
+    from databricks.labs.lakebridge.install import installer  # pylint: disable=cyclic-import, import-outside-toplevel
+
     ctx = ApplicationContext(w)
     ctx.add_user_agent_extra("cmd", "configure-reconcile")
     user = w.current_user
@@ -1017,8 +1031,4 @@ def create_profiler_dashboard(
 
 
 if __name__ == "__main__":
-    app = lakebridge
-    logger = app.get_logger()
-    if is_in_debug():
-        logger.setLevel(logging.DEBUG)
-    app()
+    lakebridge()

src/databricks/labs/lakebridge/cli.py

@@ -12,7 +12,6 @@
 from sqlalchemy.orm.session import Session
 
 logger = logging.getLogger(__name__)
-logger.setLevel("INFO")
 
 
 @dataclasses.dataclass

src/databricks/labs/lakebridge/connections/database_manager.py

@@ -1,19 +1,24 @@
 import dataclasses
 import logging
 import os
+import sys
 import webbrowser
 from collections.abc import Callable, Sequence, Set
 from pathlib import Path
 from typing import Any
 
+from databricks.labs.blueprint.entrypoint import get_logger
 from databricks.labs.blueprint.installation import Installation, JsonValue, SerdeError
 from databricks.labs.blueprint.installer import InstallState
 from databricks.labs.blueprint.tui import Prompts
 from databricks.labs.blueprint.wheels import ProductInfo
 from databricks.sdk import WorkspaceClient
+from databricks.sdk.core import with_user_agent_extra
 from databricks.sdk.errors import NotFound, PermissionDenied
 
+from databricks.labs.lakebridge import initialize_logging
 from databricks.labs.lakebridge.__about__ import __version__
+from databricks.labs.lakebridge.cli import lakebridge
 from databricks.labs.lakebridge.config import (
     DatabaseConfig,
     ReconcileConfig,
@@ -430,3 +435,22 @@ def _verify_workspace_client(ws: WorkspaceClient) -> WorkspaceClient:
         setattr(ws.config, '_product_info', ('lakebridge', __version__))
 
     return ws
+
+
+if __name__ == "__main__":
+    with_user_agent_extra("cmd", "install")
+    initialize_logging()
+
+    # Warning: ensures logger for this file is not __main__.
+    logger = get_logger(__file__)
+
+    app_installer = installer(
+        ws=lakebridge.create_workspace_client(),
+        transpiler_repository=TranspilerRepository.user_home(),
+        is_interactive=sys.stdin.isatty(),
+    )
+    if not app_installer.upgrade_installed_transpilers():
+        logger.debug("No existing Lakebridge transpilers detected; assuming fresh installation.")
+
+    logger.info("Successfully Setup Lakebridge Components Locally")
+    logger.info("For more information, please visit https://databrickslabs.github.io/lakebridge/")

src/databricks/labs/lakebridge/install.py

@@ -6,24 +6,20 @@
 from databricks.labs.blueprint.installation import Installation
 from databricks.sdk import WorkspaceClient
 
-from databricks.labs.lakebridge.config import (
-    TableRecon,
-    ReconcileConfig,
-)
-from databricks.labs.lakebridge.reconcile.exception import (
-    ReconciliationException,
-)
+from databricks.labs.lakebridge import initialize_logging
+from databricks.labs.lakebridge.config import ReconcileConfig, TableRecon
+from databricks.labs.lakebridge.reconcile.exception import ReconciliationException
+from databricks.labs.lakebridge.reconcile.recon_config import AGG_RECONCILE_OPERATION_NAME, RECONCILE_OPERATION_NAME
 from databricks.labs.lakebridge.reconcile.trigger_recon_aggregate_service import TriggerReconAggregateService
-from databricks.labs.lakebridge.reconcile.recon_config import (
-    RECONCILE_OPERATION_NAME,
-    AGG_RECONCILE_OPERATION_NAME,
-)
 from databricks.labs.lakebridge.reconcile.trigger_recon_service import TriggerReconService
 
 logger = logging.getLogger(__name__)
 
 
-def main(*argv) -> None:
+def main(*argv: str) -> None:
+    """Lakeview Jobs task entry point: reconcile"""
+    initialize_logging()
+
     logger.debug(f"Arguments received: {argv}")
 
     assert len(sys.argv) == 2, f"Invalid number of arguments: {len(sys.argv)}," f" Operation name must be specified."