Merge remote-tracking branch 'origin/main' into jb/aitools-move-only

Author: jamesbroadhead

NEXT_CHANGELOG.md

@@ -11,7 +11,9 @@
 * Marked the default profile in the interactive pickers shown by `databricks auth switch`, `databricks auth logout`, `databricks auth token`, and `databricks auth login`, and moved it to the top of the list. `databricks auth login` and `databricks auth logout` now offer the same selectors as `databricks auth token` and `databricks auth switch` respectively.
 
 ### Bundles
+* Stop applying `presets.name_prefix` (and the dev-mode `[dev <user>]` rename) to `vector_search_endpoints` ([#5209](https://github.com/databricks/cli/pull/5209)).
 
+* Fix `bundle generate` job to preserve nested notebook directory structure ([#4596](https://github.com/databricks/cli/pull/4596))
 * Propagate authentication environment (including `DATABRICKS_CONFIG_PROFILE`) to the `experimental.python` subprocess so bundle validate/deploy no longer fails with a multi-profile host ambiguity error when several profiles in `~/.databrickscfg` share the same host.
 * Fixed `--force-pull` on `bundle summary` and `bundle open` so the flag bypasses the local state cache and reads state from the workspace.
 

acceptance/bundle/debug/list-targets/databricks.yml

@@ -15,7 +15,7 @@ targets:
     mode: development
     workspace:
       host: https://dev.example.com
-  staging: {}
+  staging:
   prod:
     mode: production
     workspace:

acceptance/bundle/generate/job_nested_notebooks/databricks.yml

@@ -0,0 +1,2 @@
+bundle:
+  name: nested_notebooks

acceptance/bundle/generate/job_nested_notebooks/out.job.yml

@@ -0,0 +1,11 @@
+resources:
+  jobs:
+    out:
+      name: dev.my_repo.my_job
+      tasks:
+        - task_key: my_notebook_task
+          notebook_task:
+            notebook_path: src/my_folder/my_notebook.py
+        - task_key: other_notebook_task
+          notebook_task:
+            notebook_path: src/other_folder/other_notebook.py

acceptance/bundle/generate/job_nested_notebooks/out.test.toml

@@ -0,0 +1,9 @@
+File successfully saved to src/my_folder/my_notebook.py
+File successfully saved to src/other_folder/other_notebook.py
+Job configuration successfully saved to out.job.yml
+=== old flattened files should be gone ===
+src/my_notebook.py removed
+src/other_notebook.py removed
+=== new nested files ===
+src/my_folder/my_notebook.py
+src/other_folder/other_notebook.py

acceptance/bundle/generate/job_nested_notebooks/output.txt

@@ -0,0 +1,12 @@
+mkdir -p src
+echo "old" > src/my_notebook.py
+echo "old" > src/other_notebook.py
+
+$CLI bundle generate job --existing-job-id 1234 --config-dir . --key out --force --source-dir src 2>&1 | sort
+
+echo "=== old flattened files should be gone ==="
+test ! -f src/my_notebook.py && echo "src/my_notebook.py removed" || echo "src/my_notebook.py still exists"
+test ! -f src/other_notebook.py && echo "src/other_notebook.py removed" || echo "src/other_notebook.py still exists"
+
+echo "=== new nested files ==="
+find src -type f | sort

acceptance/bundle/generate/job_nested_notebooks/script

@@ -0,0 +1,42 @@
+Ignore = ["src"]
+
+[[Server]]
+Pattern = "GET /api/2.2/jobs/get"
+Response.Body = '''
+{
+    "job_id": 11223344,
+    "settings": {
+        "name": "dev.my_repo.my_job",
+        "tasks": [
+            {
+                "task_key": "my_notebook_task",
+                "notebook_task": {
+                    "notebook_path": "/my_data_product/dev/my_folder/my_notebook"
+                }
+            },
+            {
+                "task_key": "other_notebook_task",
+                "notebook_task": {
+                    "notebook_path": "/my_data_product/dev/other_folder/other_notebook"
+                }
+            }
+        ]
+    }
+}
+'''
+
+[[Server]]
+Pattern = "GET /api/2.0/workspace/get-status"
+Response.Body = '''
+{
+    "object_type": "NOTEBOOK",
+    "language": "PYTHON",
+    "repos_export_format": "SOURCE"
+}
+'''
+
+[[Server]]
+Pattern = "GET /api/2.0/workspace/export"
+Response.Body = '''
+print("Hello, World!")
+'''

acceptance/bundle/generate/job_nested_notebooks/test.toml

@@ -0,0 +1,2 @@
+bundle:
+  name: python_job_and_deploy