Merge pull request #9 from mwojtyczka/update

refactor

Author: mwojtyczka

mlops_demo/deployment/batch_inference/PrepareBatchInferenceInput.py renamed to mlops_demo/deployment/batch_inference/notebooks/PrepareBatchInferenceInput.py

@@ -6,10 +6,9 @@
 import pyspark.sql.functions as F
 from datetime import timedelta, timezone
 import math
-import mlflow.pyfunc
-import pyspark.sql.functions as F
 from pyspark.sql.types import IntegerType
 
+
 def rounded_unix_timestamp(dt, num_minutes=15):
     """
     Ceilings datetime dt to interval num_minutes, then returns the unix timestamp.
@@ -18,27 +17,21 @@ def rounded_unix_timestamp(dt, num_minutes=15):
     delta = math.ceil(nsecs / (60 * num_minutes)) * (60 * num_minutes) - nsecs
     return int((dt + timedelta(seconds=delta)).replace(tzinfo=timezone.utc).timestamp())
 
+
 rounded_unix_timestamp_udf = F.udf(rounded_unix_timestamp, IntegerType())
 
+
 def rounded_taxi_data(taxi_data_df):
     # Round the taxi data timestamp to 15 and 30 minute intervals so we can join with the pickup and dropoff features
     # respectively.
     taxi_data_df = (
         taxi_data_df.withColumn(
             "rounded_pickup_datetime",
-            F.to_timestamp(
-                rounded_unix_timestamp_udf(
-                    taxi_data_df["tpep_pickup_datetime"], F.lit(15)
-                )
-            ),
+            F.to_timestamp(rounded_unix_timestamp_udf(taxi_data_df["tpep_pickup_datetime"], F.lit(15))),
         )
         .withColumn(
             "rounded_dropoff_datetime",
-            F.to_timestamp(
-                rounded_unix_timestamp_udf(
-                    taxi_data_df["tpep_dropoff_datetime"], F.lit(30)
-                )
-            ),
+            F.to_timestamp(rounded_unix_timestamp_udf(taxi_data_df["tpep_dropoff_datetime"], F.lit(30))),
         )
         .drop("tpep_pickup_datetime")
         .drop("tpep_dropoff_datetime")
@@ -59,4 +52,3 @@ def rounded_taxi_data(taxi_data_df):
 # MAGIC select * from qa_mlops_demo.marcin_wojtyczka.feature_store_inference_input
 
 # COMMAND ----------
-

mlops_demo/validation/notebooks/ModelValidation.py

@@ -299,9 +299,7 @@ def get_fs_model(df):
 
     try:
         eval_result = mlflow.evaluate(
-            
             model=get_fs_model,
-            
             data=data,
             targets=targets,
             model_type=model_type,