Merge remote-tracking branch 'origin/main'

Author: mwojtyczka

.github/workflows/mlops_demo-bundle-cd-staging.yml

@@ -28,8 +28,8 @@ jobs:
       - name: Validate Bundle For Staging
         id: validate
         run: |
-          databricks bundle validate -t qa
+          databricks bundle validate -t staging
       - name: Deploy Bundle to Staging
         id: deploy
         run: |
-          databricks bundle deploy -t qa
+          databricks bundle deploy -t staging

.github/workflows/mlops_demo-bundle-validate.yml

@@ -31,7 +31,7 @@ jobs:
         env:
           DATABRICKS_TOKEN: ${{ env.STAGING_WORKSPACE_TOKEN }}
         run: |
-          databricks bundle validate -t qa > ../validate_output.txt
+          databricks bundle validate -t staging > ../validate_output.txt
       - name: Create Comment with Bundle Configuration
         uses: actions/github-script@v7
         id: comment

.github/workflows/mlops_demo-run-lint-tests.yml

@@ -53,16 +53,16 @@ jobs:
       - name: Validate Bundle For Test Deployment Target in Staging Workspace
         id: validate
         run: |
-          databricks bundle validate -t qa
+          databricks bundle validate -t ci
       - name: Deploy Bundle to Test Deployment Target in Staging Workspace
         id: deploy
         run: |
-          databricks bundle deploy -t qa
+          databricks bundle deploy -t ci
       - name: Run Feature Engineering Workflow for Test Deployment Target in Staging Workspace
         id: feature_engineering
         run: |
-          databricks bundle run write_feature_table_job -t qa
+          databricks bundle run write_feature_table_job -t ci
       - name: Run Training Workflow for Test Deployment Target in Staging Workspace
         id: training
         run: |
-          databricks bundle run model_training_job -t qa
+          databricks bundle run model_training_job -t ci

mlops_demo/databricks.yml

@@ -18,7 +18,8 @@ include:
   - ./resources/ml-artifacts-resource.yml
   - ./resources/model-workflow-resource.yml
   - ./resources/feature-engineering-workflow-resource.yml
-  - ./resources/monitoring-resource.yml
+  # TODO can only be done once batch inference is run (predictions table exists).
+  #- ./resources/monitoring-resource.yml
 
 # Deployment Target specific values for workspace
 targets:
@@ -28,7 +29,11 @@ targets:
     workspace:
       host: https://e2-demo-field-eng.cloud.databricks.com
 
-  qa:
+  ci:
+    workspace:
+      host: https://e2-demo-field-eng.cloud.databricks.com
+
+  staging:
     workspace:
       host: https://e2-demo-field-eng.cloud.databricks.com
 

mlops_demo/deployment/batch_inference/notebooks/PrepareBatchInferenceInput.py

@@ -17,6 +17,7 @@ def rounded_unix_timestamp(dt, num_minutes=15):
     delta = math.ceil(nsecs / (60 * num_minutes)) * (60 * num_minutes) - nsecs
     return int((dt + timedelta(seconds=delta)).replace(tzinfo=timezone.utc).timestamp())
 
+
 rounded_unix_timestamp_udf = F.udf(rounded_unix_timestamp, IntegerType())
 
 
@@ -38,16 +39,22 @@ def rounded_taxi_data(taxi_data_df):
     taxi_data_df.createOrReplaceTempView("taxi_data")
     return taxi_data_df
 
-
 data = rounded_taxi_data(df)
 
 # COMMAND ----------
 
-data.write.format("delta").mode("overwrite").saveAsTable("qa_mlops_demo.marcin_wojtyczka.feature_store_inference_input")
+# MAGIC %sql
+# MAGIC create schema if not exists staging_mlops_demo.marcin_wojtyczka;
+# MAGIC create schema if not exists prod_mlops_demo.marcin_wojtyczka;
+# MAGIC create schema if not exists ci_mlops_demo.marcin_wojtyczka;
 
 # COMMAND ----------
 
-# MAGIC %sql
-# MAGIC select * from qa_mlops_demo.marcin_wojtyczka.feature_store_inference_input
+data.write.format("delta").mode("overwrite").saveAsTable("staging_mlops_demo.marcin_wojtyczka.feature_store_inference_input")
+data.write.format("delta").mode("overwrite").saveAsTable("prod_mlops_demo.marcin_wojtyczka.feature_store_inference_input")
+data.write.format("delta").mode("overwrite").saveAsTable("ci_mlops_demo.marcin_wojtyczka.feature_store_inference_input")
 
 # COMMAND ----------
+
+# MAGIC %sql
+# MAGIC select * from staging_mlops_demo.marcin_wojtyczka.feature_store_inference_input