Mig assist deps (#247)

it now works when executing 
databricks labs sandbox sql-migration-assistant

can test by running 
```bash
databricks labs install sandbox@mig-assist-deps && databricks labs sandbox sql-migration-assistant
```

Author: robertwhiffin

labs.yml

@@ -4,7 +4,7 @@ install:
   script: install.py
 description: Databricks Labs Sandbox
 entrypoint: cli.py
-min_python: 3.8
+min_python: 3.10
 commands:
   - name: ip-access-list-analyzer
     description: "Analyzer/Fixer for Databricks IP Access Lists"

pyproject.toml

@@ -4,13 +4,20 @@ version = "0.0.1"
 description = "Experimental and low-maturity scripts"
 authors = ["Databricks Labs"]
 packages = [
-    {include = "ip_access_list_analyzer"}
+    {include = "ip_access_list_analyzer"},
+    {include = "sql_migration_assistant"}
 ]
 
 [tool.poetry.dependencies]
-python = ">=3.8,<3.12"
-databricks-sdk = "^0.25.1"
+python = ">=3.10,<3.12"
+databricks-sdk = "0.30.0"
+pyyaml = "^6.0.2"
+mlflow = {extras = ["databricks"], version = "^2.16.0"}
+databricks-labs-blueprint = "0.8.2"
+databricks-labs-lsql = "0.9.0"
 
 [build-system]
-requires = ["poetry-core>=1.0.0"]
+requires = [
+    "poetry-core>=1.0.0"
+]
 build-backend = "poetry.core.masonry.api"

sql_migration_assistant/README.md

@@ -29,7 +29,8 @@ customise the application to their needs and entirely own the IP.
 
 ## Installation Videos
 
-https://github.com/user-attachments/assets/b43372fb-95ea-49cd-9a2c-aec8e0d6700f
+
+https://github.com/user-attachments/assets/e665bcf4-265f-4a47-81eb-60845a72c798
 
 https://github.com/user-attachments/assets/fa622f96-a78c-40b8-9eb9-f6671c4d7b47
 
@@ -38,27 +39,17 @@ https://github.com/user-attachments/assets/1a58a1b5-2dcf-4624-b93f-214735162584
 
 
 Setting Legion up is a simple and automated process. Ensure you have the [Databricks CLI]
-(https://docs.databricks.com/en/dev-tools/cli/index.html) installed and configured with the correct workspace. Install 
-the [Databricks Labs Sandbox](https://github.com/databrickslabs/sandbox). 
+(https://docs.databricks.com/en/dev-tools/cli/index.html) installed and configured with the correct workspace.
 
-First, navigate to where you have installed the Databricks Labs Sandbox. For example
+Once the Databricks CLI has been installed and configured, run the following command to install the Databricks Labs 
+Sandbox and the SQL Migration Assistant.
 ```bash
-cd /Documents/sandbox
-```
-
-You'll need to install the python requirements in the `requirements.txt` file in the root of the project. 
-You may wish to do this in a virtual environment. 
-```bash
-pip install -r sql-migration-assistant/requirements.txt -q
-```
-Run the following command to start the installation process, creating all the necessary resources in your workspace.
-```bash 
-databricks labs sandbox sql-migration-assistant
+databricks labs install sandbox && databricks labs sandbox sql-migration-assistant
 ```
 
 ### What Legion needs - during setup above you will create or choose existing resources for the following:
 
-- A no-isolation shared cluster running the ML runtime (tested on DBR 15.0 ML) to host the front end application.
+- A no-isolation shared cluster to host the front end application.
 - A catalog and schema in Unity Catalog. 
 - A table to store the code intent statements and their embeddings.
 - A vector search endpoint and an embedding model: see docs 

sql_migration_assistant/__init__.py

@@ -2,14 +2,18 @@
 from databricks.sdk import WorkspaceClient
 from databricks.labs.blueprint.tui import Prompts
 import yaml
+from pathlib import Path
 
 
 def hello():
     w = WorkspaceClient(product="sql_migration_assistant", product_version="0.0.1")
     p = Prompts()
     setter_upper = SetUpMigrationAssistant()
     final_config = setter_upper.setup_migration_assistant(w, p)
-    with open("sql_migration_assistant/config.yml", "w") as f:
+    current_path = Path(__file__).parent.resolve()
+
+    local_config = str(current_path) + "/config.yml"
+    with open(local_config, "w") as f:
         yaml.dump(final_config, f)
-    setter_upper.upload_files(w)
+    setter_upper.upload_files(w, current_path)
     setter_upper.launch_review_app(w, final_config)

sql_migration_assistant/docs/Installation Video 1.mov

@@ -10,10 +10,10 @@
 pip install databricks-sdk -U
 
 # COMMAND ----------
-pip install gradio==4.27.0 pyyaml aiohttp databricks-labs-blueprint==0.8.2 databricks-labs-lsql==0.9.0
+pip install gradio==4.27.0 pyyaml aiohttp==3.10.5 databricks-labs-blueprint==0.8.2 databricks-labs-lsql==0.9.0
 
 # COMMAND ----------
-pip install dbtunnel==0.14.6
+pip install fastapi==0.112.2 pydantic==2.8.2 dbtunnel==0.14.6
 
 # COMMAND ----------
 dbutils.library.restartPython()

sql_migration_assistant/docs/Installation Video 2.mov

@@ -14,7 +14,6 @@
 from sql_migration_assistant.utils.upload_files_to_workspace import FileUploader
 from sql_migration_assistant.utils.run_review_app import RunReviewApp
 
-
 class SetUpMigrationAssistant:
 
     # this is a decorator to handle errors and do a retry where user is asked to choose an existing resource
@@ -134,22 +133,26 @@ def setup_migration_assistant(self, w, p):
 
         return config
 
-    def upload_files(self, w):
+    def upload_files(self, w, path):
+        # all this nastiness becomes unnecessary with lakehouse apps, or if we upload a whl it simplifies things.
+        # But for now, this is the way.
+        # TODO - MAKE THIS NICE!!
         logging.info("Uploading files to workspace")
         print("\nUploading files to workspace")
         uploader = FileUploader(w)
         files_to_upload = [
-            "sql_migration_assistant/utils/runindatabricks.py",
-            "sql_migration_assistant/gradio_app.py",
-            "sql_migration_assistant/run_app_from_databricks_notebook.py",
-            "sql_migration_assistant/utils/configloader.py",
-            "sql_migration_assistant/utils/run_review_app.py",
-            "sql_migration_assistant/config.yml",
+            "utils/runindatabricks.py",
+            "gradio_app.py",
+            "run_app_from_databricks_notebook.py",
+            "utils/configloader.py",
+            "utils/run_review_app.py",
+            "config.yml",
         ]
+        files_to_upload = [os.path.join(path, x) for x in files_to_upload]
         files_to_upload.extend(
             [
-                f"sql_migration_assistant/app/{x}"
-                for x in os.listdir("sql_migration_assistant/app")
+                os.path.join(path, "app", x)
+                for x in os.listdir(os.path.join(path, "app"))
                 if x[-3:] == ".py"
             ]
         )

sql_migration_assistant/docs/Installation Video 3.mov

@@ -23,8 +23,10 @@ def __init__(self, w: WorkspaceClient, config: dict):
             "gradio==4.27.0",
             "pyyaml",
             "databricks-sdk==0.30.0",
-            "aiohttp",
+            "aiohttp==3.10.5",
             "databricks-labs-blueprint==0.8.2",
+            "fastapi==0.112.2",
+            "pydantic==2.8.2",
             "dbtunnel==0.14.6",
             "databricks-labs-lsql==0.9.0",
         ]