From 6f6dbeaf2ab083153ac53079c50d278a90a35f5c Mon Sep 17 00:00:00 2001 From: yi-lyu_data Date: Tue, 6 May 2025 11:58:13 -0700 Subject: [PATCH 1/4] Include dashboard task creation in AI/BI dashboard DABs tutorial --- knowledge_base/dashboard_nyc_taxi/README.md | 57 ++++++++++++++++--- .../resources/nyc_taxi_trip_analysis.job.yml | 19 +++++++ 2 files changed, 69 insertions(+), 7 deletions(-) create mode 100644 knowledge_base/dashboard_nyc_taxi/resources/nyc_taxi_trip_analysis.job.yml diff --git a/knowledge_base/dashboard_nyc_taxi/README.md b/knowledge_base/dashboard_nyc_taxi/README.md index 7c7ff744..d23c00fd 100644 --- a/knowledge_base/dashboard_nyc_taxi/README.md +++ b/knowledge_base/dashboard_nyc_taxi/README.md @@ -1,28 +1,71 @@ # Dashboard for NYC Taxi Trip Analysis -This example demonstrates how to define an AI/BI dashboard in a Databricks Asset Bundle. +This example shows how to define an AI/BI dashboard within a Databricks Asset Bundle, set up a job to capture a snapshot of the dashboard, and email it to a subscriber. -It includes and deploys the sample __NYC Taxi Trip Analysis__ dashboard to a Databricks workspace. +It deploys the sample __NYC Taxi Trip Analysis__ dashboard to a Databricks workspace and configures a daily schedule to run the dashboard and send the snapshot via email. For more information about AI/BI dashboards, please refer to the [documentation](https://docs.databricks.com/dashboards/index.html). ## Prerequisites -* Databricks CLI v0.232.0 or above +- For creating dashboard: Databricks CLI v0.232.0 or above +- For dashboard snapshot task: Databricks CLI v0.250.0 or above ## Usage -Modify `databricks.yml`: -* Update the `host` field under `workspace` to the Databricks workspace to deploy to. -* Update the `warehouse` field under `warehouse_id` to the name of the SQL warehouse to use. +#### Step1: Modify `databricks.yml`: + +- Update the `host` field under `workspace` to the Databricks workspace to deploy to. +- Update the `warehouse` field under `warehouse_id` to the name of the SQL warehouse to use. + +#### Step2: Modify `resources/nyc_taxi_trip_analysis.job.yml`: + +- Update the `user_name` field under `subscribers` to the email of the subscriber of the dashboard + +#### Step3: Deploy the dashboard Run `databricks bundle deploy` to deploy the dashboard. Run `databricks bundle open` to navigate to the deployed dashboard in your browser. Alternatively, run `databricks bundle summary` to display its URL. +Note: +At this moment, we have created the AI/BI dashboard, but the dashboard snapshot job is not fully ready yet since the dashboard id is only generated after the dashboard is created. + +#### Step 4: Update dashboard id in the dashboard snapshot job + +Use the summary command output to locate the generated dashboard ID: + +``` +$userName@GVFX6GR6RN dashboard_nyc_taxi % databricks bundle summary +Name: dashboard_nyc_taxi +Target: dev +Workspace: + Host: $host + User: $user + Path: /Workspace/Users/$user/.bundle/dashboard_nyc_taxi/dev +Resources: + Dashboards: + nyc_taxi_trip_analysis: + Name: [dev $userName] NYC Taxi Trip Analysis + URL: $workspaceUrl/dashboardsv3/$dashboardId/published?o=$workspaceId + Jobs: + my_project_job: + Name: [dev $userName] my_project_job + URL: $workspaceUrl/jobs/$jobId?o=$workspaceId +``` + +Once you've located the dashboard ID, open resources/nyc_taxi_trip_analysis.job.yml and insert the ID accordingly. + +Then, redeploy the bundle: + +``` +databricks bundle deploy +``` + +This sets up the snapshot job to run daily at 8 AM, capture a snapshot of the dashboard, and send it via email to the specified subscriber. ### Visual modification -You can use the Databricks UI to modify the dashboard, but any modifications made through the UI will not be applied to the bundle `.lvdash.json` file unless you explicitly update it. +You can use the Databricks UI to modify the dashboard, but any modifications made through the UI will not be applied to the bundle `.lvdash.json` file unless you explicitly update it. To update the local bundle `.lvdash.json` file, run: diff --git a/knowledge_base/dashboard_nyc_taxi/resources/nyc_taxi_trip_analysis.job.yml b/knowledge_base/dashboard_nyc_taxi/resources/nyc_taxi_trip_analysis.job.yml new file mode 100644 index 00000000..9e511816 --- /dev/null +++ b/knowledge_base/dashboard_nyc_taxi/resources/nyc_taxi_trip_analysis.job.yml @@ -0,0 +1,19 @@ +resources: + jobs: + my_project_job: + name: my_project_job + + schedule: + quartz_cron_expression: '0 0 8 * * ?' # daily at 8am + timezone_id: UTC + pause_status: UNPAUSED + + tasks: + - task_key: dashboard_task + dashboard_task: + subscription: + subscribers: + - user_name: "test@gmail.com" # Replace this with the email of snapshot subscriber email + paused: false + warehouse_id: ${var.warehouse_id} + dashboard_id: "01111111111111111f361f7719111099" # Replace this with the newly generated dashboard id. This will be a UUID string of the dashboard, which could be found from dashboard url. From 852f73bd7becd48aa738777ed6c4ce42e0921db7 Mon Sep 17 00:00:00 2001 From: yi-lyu_data Date: Mon, 19 May 2025 21:10:54 -0700 Subject: [PATCH 2/4] fix comment --- knowledge_base/dashboard_nyc_taxi/README.md | 35 +------------------ .../resources/nyc_taxi_trip_analysis.job.yml | 8 ++--- 2 files changed, 5 insertions(+), 38 deletions(-) diff --git a/knowledge_base/dashboard_nyc_taxi/README.md b/knowledge_base/dashboard_nyc_taxi/README.md index d23c00fd..290d65c0 100644 --- a/knowledge_base/dashboard_nyc_taxi/README.md +++ b/knowledge_base/dashboard_nyc_taxi/README.md @@ -28,40 +28,7 @@ Run `databricks bundle deploy` to deploy the dashboard. Run `databricks bundle open` to navigate to the deployed dashboard in your browser. Alternatively, run `databricks bundle summary` to display its URL. Note: -At this moment, we have created the AI/BI dashboard, but the dashboard snapshot job is not fully ready yet since the dashboard id is only generated after the dashboard is created. - -#### Step 4: Update dashboard id in the dashboard snapshot job - -Use the summary command output to locate the generated dashboard ID: - -``` -$userName@GVFX6GR6RN dashboard_nyc_taxi % databricks bundle summary -Name: dashboard_nyc_taxi -Target: dev -Workspace: - Host: $host - User: $user - Path: /Workspace/Users/$user/.bundle/dashboard_nyc_taxi/dev -Resources: - Dashboards: - nyc_taxi_trip_analysis: - Name: [dev $userName] NYC Taxi Trip Analysis - URL: $workspaceUrl/dashboardsv3/$dashboardId/published?o=$workspaceId - Jobs: - my_project_job: - Name: [dev $userName] my_project_job - URL: $workspaceUrl/jobs/$jobId?o=$workspaceId -``` - -Once you've located the dashboard ID, open resources/nyc_taxi_trip_analysis.job.yml and insert the ID accordingly. - -Then, redeploy the bundle: - -``` -databricks bundle deploy -``` - -This sets up the snapshot job to run daily at 8 AM, capture a snapshot of the dashboard, and send it via email to the specified subscriber. +At this moment, we have created the AI/BI dashboard and sets up the snapshot job to run daily at 8 AM, capture a snapshot of the dashboard, and send it via email to the specified subscriber. ### Visual modification diff --git a/knowledge_base/dashboard_nyc_taxi/resources/nyc_taxi_trip_analysis.job.yml b/knowledge_base/dashboard_nyc_taxi/resources/nyc_taxi_trip_analysis.job.yml index 9e511816..f08232a8 100644 --- a/knowledge_base/dashboard_nyc_taxi/resources/nyc_taxi_trip_analysis.job.yml +++ b/knowledge_base/dashboard_nyc_taxi/resources/nyc_taxi_trip_analysis.job.yml @@ -1,12 +1,11 @@ resources: jobs: - my_project_job: - name: my_project_job + dashboard_snapshot_job: + name: "Daily NYC Taxi Trip Analysis refresh" schedule: quartz_cron_expression: '0 0 8 * * ?' # daily at 8am timezone_id: UTC - pause_status: UNPAUSED tasks: - task_key: dashboard_task @@ -16,4 +15,5 @@ resources: - user_name: "test@gmail.com" # Replace this with the email of snapshot subscriber email paused: false warehouse_id: ${var.warehouse_id} - dashboard_id: "01111111111111111f361f7719111099" # Replace this with the newly generated dashboard id. This will be a UUID string of the dashboard, which could be found from dashboard url. + dashboard_id: ${resources.dashboards.nyc_taxi_trip_analysis.id} + From 279d2a227eed9829e8aa63dab376fd9c3171daee Mon Sep 17 00:00:00 2001 From: yi-lyu_data Date: Mon, 19 May 2025 21:20:46 -0700 Subject: [PATCH 3/4] rephrase readme --- knowledge_base/dashboard_nyc_taxi/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/knowledge_base/dashboard_nyc_taxi/README.md b/knowledge_base/dashboard_nyc_taxi/README.md index 290d65c0..e9338b5e 100644 --- a/knowledge_base/dashboard_nyc_taxi/README.md +++ b/knowledge_base/dashboard_nyc_taxi/README.md @@ -20,7 +20,7 @@ For more information about AI/BI dashboards, please refer to the [documentation] #### Step2: Modify `resources/nyc_taxi_trip_analysis.job.yml`: -- Update the `user_name` field under `subscribers` to the email of the subscriber of the dashboard +- Update the `user_name` field under `subscribers` to the dashboard subscriber's email #### Step3: Deploy the dashboard From 99630d342f4f280a824dc38e7377cb2e426513d8 Mon Sep 17 00:00:00 2001 From: yi-lyu_data Date: Thu, 22 May 2025 14:44:30 -0700 Subject: [PATCH 4/4] Address comments --- knowledge_base/dashboard_nyc_taxi/README.md | 29 ++++++++----------- .../resources/nyc_taxi_trip_analysis.job.yml | 2 +- 2 files changed, 13 insertions(+), 18 deletions(-) diff --git a/knowledge_base/dashboard_nyc_taxi/README.md b/knowledge_base/dashboard_nyc_taxi/README.md index e9338b5e..06d633cd 100644 --- a/knowledge_base/dashboard_nyc_taxi/README.md +++ b/knowledge_base/dashboard_nyc_taxi/README.md @@ -1,34 +1,29 @@ # Dashboard for NYC Taxi Trip Analysis -This example shows how to define an AI/BI dashboard within a Databricks Asset Bundle, set up a job to capture a snapshot of the dashboard, and email it to a subscriber. +This example shows how to define a Databricks Asset Bundle with an AI/BI dashboard and a job that captures a snapshot of the dashboard and emails it to a subscriber. -It deploys the sample __NYC Taxi Trip Analysis__ dashboard to a Databricks workspace and configures a daily schedule to run the dashboard and send the snapshot via email. +It deploys the sample __NYC Taxi Trip Analysis__ dashboard to a Databricks workspace and configures a daily schedule to run the dashboard and send the snapshot in email to a specified email address. For more information about AI/BI dashboards, please refer to the [documentation](https://docs.databricks.com/dashboards/index.html). ## Prerequisites -- For creating dashboard: Databricks CLI v0.232.0 or above -- For dashboard snapshot task: Databricks CLI v0.250.0 or above +This example includes a dashboard snapshot task, which requires Databricks CLI v0.250.0 or above. Creating dashboards in bundles is supported in Databricks CLI v0.232.0 or above. ## Usage -#### Step1: Modify `databricks.yml`: +1. Modify `databricks.yml`: + - Update the `host` field under `workspace` to the Databricks workspace to deploy to. + - Update the `warehouse` field under `warehouse_id` to the name of the SQL warehouse to use. -- Update the `host` field under `workspace` to the Databricks workspace to deploy to. -- Update the `warehouse` field under `warehouse_id` to the name of the SQL warehouse to use. +2. Modify `resources/nyc_taxi_trip_analysis.job.yml`: + - Update the `user_name` field under `subscribers` to the dashboard subscriber's email. -#### Step2: Modify `resources/nyc_taxi_trip_analysis.job.yml`: +3. Deploy the dashboard: + - Run `databricks bundle deploy` to deploy the dashboard. + - Run `databricks bundle open` to navigate to the deployed dashboard in your browser. Alternatively, run `databricks bundle summary` to display its URL. -- Update the `user_name` field under `subscribers` to the dashboard subscriber's email - -#### Step3: Deploy the dashboard - -Run `databricks bundle deploy` to deploy the dashboard. - -Run `databricks bundle open` to navigate to the deployed dashboard in your browser. Alternatively, run `databricks bundle summary` to display its URL. -Note: -At this moment, we have created the AI/BI dashboard and sets up the snapshot job to run daily at 8 AM, capture a snapshot of the dashboard, and send it via email to the specified subscriber. +The AI/BI dashboard is created and the snapshot job is set to run daily at 8 AM, which captures a snapshot of the dashboard, and sends it in email to the specified subscriber. ### Visual modification diff --git a/knowledge_base/dashboard_nyc_taxi/resources/nyc_taxi_trip_analysis.job.yml b/knowledge_base/dashboard_nyc_taxi/resources/nyc_taxi_trip_analysis.job.yml index f08232a8..54d2e0b0 100644 --- a/knowledge_base/dashboard_nyc_taxi/resources/nyc_taxi_trip_analysis.job.yml +++ b/knowledge_base/dashboard_nyc_taxi/resources/nyc_taxi_trip_analysis.job.yml @@ -12,7 +12,7 @@ resources: dashboard_task: subscription: subscribers: - - user_name: "test@gmail.com" # Replace this with the email of snapshot subscriber email + - user_name: ${workspace.current_user.userName} # Replace this with the email of snapshot subscriber email paused: false warehouse_id: ${var.warehouse_id} dashboard_id: ${resources.dashboards.nyc_taxi_trip_analysis.id}