diff --git a/databricks_notebooks/bulk/Export_All.py b/databricks_notebooks/bulk/Export_All.py index 8970771c..2ef136df 100644 --- a/databricks_notebooks/bulk/Export_All.py +++ b/databricks_notebooks/bulk/Export_All.py @@ -62,15 +62,29 @@ # COMMAND ---------- -from mlflow_export_import.bulk.export_all import export_all - -export_all( - output_dir = output_dir, - stages = stages, - export_latest_versions = export_latest_versions, - run_start_time = run_start_date, - export_permissions = export_permissions, - export_deleted_runs = export_deleted_runs, - notebook_formats = notebook_formats, - use_threads = use_threads -) +# MAGIC %%capture captured +# MAGIC +# MAGIC from mlflow_export_import.bulk.export_all import export_all +# MAGIC +# MAGIC export_all( +# MAGIC output_dir = output_dir, +# MAGIC stages = stages, +# MAGIC export_latest_versions = export_latest_versions, +# MAGIC run_start_time = run_start_date, +# MAGIC export_permissions = export_permissions, +# MAGIC export_deleted_runs = export_deleted_runs, +# MAGIC notebook_formats = notebook_formats, +# MAGIC use_threads = use_threads +# MAGIC ) + +# COMMAND ---------- + +# DBTITLE 1,write log file +filepath = "/mnt/public-blobs/dcoles/mlflow_export_log.txt" + +dbutils.fs.rm(filepath) +dbutils.fs.put(filepath, captured.stdout) + +# COMMAND ---------- + + diff --git a/databricks_notebooks/bulk/Export_Experiments.py b/databricks_notebooks/bulk/Export_Experiments.py index 739d6acc..fc916130 100644 --- a/databricks_notebooks/bulk/Export_Experiments.py +++ b/databricks_notebooks/bulk/Export_Experiments.py @@ -1,8 +1,8 @@ # Databricks notebook source # MAGIC %md ## Export Experiments -# MAGIC +# MAGIC # MAGIC Export multiple experiments and all their runs. -# MAGIC +# MAGIC # MAGIC Widgets # MAGIC * `1. Experiments` - comma delimited list of either experiment IDs or experiment names. `all` will export all experiments. # MAGIC * `2. Output directory` - shared directory between source and destination workspaces. @@ -53,6 +53,22 @@ # COMMAND ---------- +# DBTITLE 1,set up log file +import os +from datetime import datetime +import pytz + +cst = pytz.timezone('US/Central') +now = datetime.now(tz=cst) +date = now.strftime("%Y-%m-%d-%H:%M:%S") + +logfile = f"export_experiments.{date}.log" +os.environ["MLFLOW_EXPORT_IMPORT_LOG_OUTPUT_FILE"] = logfile + +print("Logging to", logfile) + +# COMMAND ---------- + assert_widget(experiments, "1. Experiments") assert_widget(output_dir, "2. Output directory") diff --git a/databricks_notebooks/bulk/Export_Models.py b/databricks_notebooks/bulk/Export_Models.py index 7e7c3a9e..106d166a 100644 --- a/databricks_notebooks/bulk/Export_Models.py +++ b/databricks_notebooks/bulk/Export_Models.py @@ -1,8 +1,8 @@ # Databricks notebook source # MAGIC %md ## Export Models -# MAGIC +# MAGIC # MAGIC Export specified models, their version runs and the experiments that the runs belong to. -# MAGIC +# MAGIC # MAGIC Widgets # MAGIC * `1. Models` - comma seperated registered model names to be exported. `all` will export all models. # MAGIC * `2. Output directory` - shared directory between source and destination workspaces. @@ -13,7 +13,7 @@ # MAGIC * `7. Export deleted runs` # MAGIC * `8. Notebook formats` # MAGIC * `9. Use threads` -# MAGIC +# MAGIC # MAGIC See: https://github.com/mlflow/mlflow-export-import/blob/master/README_bulk.md#registered-models. # COMMAND ---------- @@ -69,6 +69,22 @@ # COMMAND ---------- +# DBTITLE 1,set up log file +import os +from datetime import datetime +import pytz + +cst = pytz.timezone('US/Central') +now = datetime.now(tz=cst) +date = now.strftime("%Y-%m-%d-%H:%M:%S") + +logfile = f"export_models.{date}.log" +os.environ["MLFLOW_EXPORT_IMPORT_LOG_OUTPUT_FILE"] = logfile + +print("Logging to", logfile) + +# COMMAND ---------- + assert_widget(models, "1. Models") assert_widget(output_dir, "2. Output directory") @@ -91,33 +107,3 @@ notebook_formats = notebook_formats, use_threads = use_threads ) - -# COMMAND ---------- - -# MAGIC %md ### Display exported files - -# COMMAND ---------- - -# MAGIC %sh -# MAGIC echo $OUTPUT_DIR -# MAGIC ls -l $OUTPUT_DIR - -# COMMAND ---------- - -# MAGIC %sh cat $OUTPUT_DIR/manifest.json - -# COMMAND ---------- - -# MAGIC %sh ls -l $OUTPUT_DIR/models - -# COMMAND ---------- - -# MAGIC %sh cat $OUTPUT_DIR/models/models.json - -# COMMAND ---------- - -# MAGIC %sh ls -l $OUTPUT_DIR/experiments - -# COMMAND ---------- - -# MAGIC %sh cat $OUTPUT_DIR/experiments/experiments.json diff --git a/databricks_notebooks/bulk/Import_Experiments.py b/databricks_notebooks/bulk/Import_Experiments.py index 035d5858..70fd2b19 100644 --- a/databricks_notebooks/bulk/Import_Experiments.py +++ b/databricks_notebooks/bulk/Import_Experiments.py @@ -1,12 +1,12 @@ # Databricks notebook source # MAGIC %md ## Import Experiments -# MAGIC +# MAGIC # MAGIC Widgets # MAGIC * `1. Input directory` - directory of exported experiments. # MAGIC * `2. Import source tags` # MAGIC * `3. Experiment rename file` - Experiment rename file. # MAGIC * `4. Use threads` - use multi-threaded import. -# MAGIC +# MAGIC # MAGIC See https://github.com/mlflow/mlflow-export-import/blob/master/README_bulk.md#Import-experiments. # COMMAND ---------- @@ -15,6 +15,22 @@ # COMMAND ---------- +# DBTITLE 1,set up log file +import os +from datetime import datetime +import pytz + +cst = pytz.timezone('US/Central') +now = datetime.now(tz=cst) +date = now.strftime("%Y-%m-%d-%H:%M:%S") + +logfile = f"import_experiments.{date}.log" +os.environ["MLFLOW_EXPORT_IMPORT_LOG_OUTPUT_FILE"] = logfile + +print("Logging to", logfile) + +# COMMAND ---------- + dbutils.widgets.text("1. Input directory", "") input_dir = dbutils.widgets.get("1. Input directory") input_dir = input_dir.replace("dbfs:","/dbfs") @@ -40,6 +56,8 @@ # COMMAND ---------- +#%%capture captured + from mlflow_export_import.bulk.import_experiments import import_experiments import_experiments( @@ -48,3 +66,7 @@ experiment_renames = experiment_rename_file, use_threads = use_threads ) + +# COMMAND ---------- + + diff --git a/databricks_notebooks/bulk/Import_Models.py b/databricks_notebooks/bulk/Import_Models.py index 4f475944..024a4a4e 100644 --- a/databricks_notebooks/bulk/Import_Models.py +++ b/databricks_notebooks/bulk/Import_Models.py @@ -1,6 +1,6 @@ # Databricks notebook source # MAGIC %md ## Import Models -# MAGIC +# MAGIC # MAGIC Widgets # MAGIC * `1. Input directory` - directory of exported models. # MAGIC * `2. Delete model` - delete the current contents of model @@ -8,7 +8,7 @@ # MAGIC * `4. Experiment rename file` - Experiment rename file. # MAGIC * `5. Import source tags` # MAGIC * `6. Use threads` - use multi-threaded import -# MAGIC +# MAGIC # MAGIC See https://github.com/mlflow/mlflow-export-import/blob/master/README_bulk.md#Import-registered-models # COMMAND ---------- @@ -47,17 +47,35 @@ # COMMAND ---------- +# DBTITLE 1,set up log file +import os +from datetime import datetime +import pytz + +cst = pytz.timezone('US/Central') +now = datetime.now(tz=cst) +date = now.strftime("%Y-%m-%d-%H:%M:%S") + +logfile = f"import_models.{date}.log" +os.environ["MLFLOW_EXPORT_IMPORT_LOG_OUTPUT_FILE"] = logfile + +print("Logging to", logfile) + +# COMMAND ---------- + assert_widget(input_dir, "1. Input directory") # COMMAND ---------- -from mlflow_export_import.bulk.import_models import import_all +from mlflow_export_import.bulk.import_models import import_models -import_all( - input_dir = input_dir, - delete_model = delete_model, - import_source_tags = import_source_tags, - experiment_renames = experiment_rename_file, - model_renames = model_rename_file, - use_threads = use_threads +import_models( + input_dir = input_dir, + delete_model = delete_model, + use_src_user_id = True, + verbose=True, + import_source_tags = import_source_tags, + experiment_renames = experiment_rename_file, + model_renames = model_rename_file, + use_threads = use_threads ) diff --git a/databricks_notebooks/bulk/export_common.py b/databricks_notebooks/bulk/export_common.py new file mode 100644 index 00000000..a0a004bb --- /dev/null +++ b/databricks_notebooks/bulk/export_common.py @@ -0,0 +1,2 @@ +# Databricks notebook source +# MAGIC %run ./Common diff --git a/databricks_notebooks/bulk/import_common.py b/databricks_notebooks/bulk/import_common.py new file mode 100644 index 00000000..dfb4fc59 --- /dev/null +++ b/databricks_notebooks/bulk/import_common.py @@ -0,0 +1,14 @@ +# Databricks notebook source +# DBTITLE 1,install mlflow-export-import from local +# MAGIC %pip install ../../../mlflow-export-import --use-feature=in-tree-build + +# COMMAND ---------- + +def assert_widget(value, name): + if len(value.rstrip())==0: + raise Exception(f"ERROR: '{name}' widget is required") + +# COMMAND ---------- + +import mlflow +mlflow_client = mlflow.client.MlflowClient() diff --git a/mlflow_export_import/bulk/export_experiments.py b/mlflow_export_import/bulk/export_experiments.py index 671d0c38..e53567ed 100644 --- a/mlflow_export_import/bulk/export_experiments.py +++ b/mlflow_export_import/bulk/export_experiments.py @@ -99,7 +99,7 @@ def export_experiments( export_results = [] futures = [] notebook_formats = utils.string_to_list(notebook_formats) - with ThreadPoolExecutor(max_workers=max_workers) as executor: + with ThreadPoolExecutor() as executor: for exp_id_or_name in experiments: run_ids = experiments_dct.get(exp_id_or_name, None) future = executor.submit(_export_experiment, diff --git a/mlflow_export_import/bulk/export_models.py b/mlflow_export_import/bulk/export_models.py index b74dc1da..2aa123ab 100644 --- a/mlflow_export_import/bulk/export_models.py +++ b/mlflow_export_import/bulk/export_models.py @@ -104,7 +104,7 @@ def _export_models( notebook_formats = utils.string_to_list(notebook_formats), futures = [] - with ThreadPoolExecutor(max_workers=max_workers) as executor: + with ThreadPoolExecutor() as executor: for model_name in model_names: dir = os.path.join(output_dir, model_name) future = executor.submit(export_model, diff --git a/mycode/.gitignore b/mycode/.gitignore new file mode 100644 index 00000000..4ead4ed0 --- /dev/null +++ b/mycode/.gitignore @@ -0,0 +1,5 @@ +export_* +exported_* +import_* +imported_* +scratch* \ No newline at end of file diff --git a/mycode/archive/cli-import-models-v2.py b/mycode/archive/cli-import-models-v2.py new file mode 100644 index 00000000..8f904941 --- /dev/null +++ b/mycode/archive/cli-import-models-v2.py @@ -0,0 +1,102 @@ +# Databricks notebook source +# MAGIC %md ### Manually loop over models instead of using `import-models`, which is unreliable +# MAGIC +# MAGIC TODO: needs work with model and experiment names + +# COMMAND ---------- + +# DBTITLE 1,install latest pkg from github +# MAGIC %sh +# MAGIC #pip install mlflow-export-import +# MAGIC pip install git+https:///github.com/mlflow/mlflow-export-import/#egg=mlflow-export-import + +# COMMAND ---------- + +# DBTITLE 1,we can see the s3 mount from %sh :) +# MAGIC %sh ls /dbfs/mnt/ccidsdatascidatalake/ + +# COMMAND ---------- + +# DBTITLE 1,set env vars +import os +from datetime import datetime +import pytz + +cst = pytz.timezone('US/Central') +now = datetime.now(tz=cst) +date = now.strftime("%Y%m%d_%H%M") + +logfile = f"import_models_{date}.log" +os.environ["MLFLOW_EXPORT_IMPORT_LOG_OUTPUT_FILE"] = logfile + +os.environ["MLFLOW_EXPORT_IMPORT_LOG_FORMAT"]="%(threadName)s-%(levelname)s-%(message)s" + +os.environ["MLFLOW_TRACKING_URI"]="databricks" + +with open("/dbfs/FileStore/shared_uploads/darrell.coles@crowncastle.com/aws_databricks_credentials") as f: + os.environ["DATABRICKS_HOST"] = f.readline().strip("\n") + os.environ["DATABRICKS_TOKEN"] = f.readline().strip("\n") + +# COMMAND ---------- + +# DBTITLE 1,verify env vars +# MAGIC %sh +# MAGIC echo $MLFLOW_EXPORT_IMPORT_LOG_OUTPUT_FILE +# MAGIC echo $MLFLOW_EXPORT_IMPORT_LOG_FORMAT +# MAGIC echo $MLFLOW_TRACKING_URI +# MAGIC #echo $DATABRICKS_HOST +# MAGIC #echo $DATABRICKS_TOKEN + +# COMMAND ---------- + +# DBTITLE 0,`export-models` options +# MAGIC %sh +# MAGIC import-model --help + +# COMMAND ---------- + +# DBTITLE 1,make model-experiment file +import json + +input_dir = "/dbfs/mnt/ccidsdatascidatalake/mlflow-migration-models/models" + +def get_most_recent_experiment_name(model): + try: + newest = max(model["mlflow"]["registered_model"]["versions"], key=lambda v:v["last_updated_timestamp"]) + return newest["_experiment_name"] + except: + return None + +with open(f"{input_dir}/models.json") as f: + models = json.load(f) + model_names = models["info"]["model_names"] + +with open("model_experiment","w") as model_experiment: + for m in model_names: + with open(f"{input_dir}/{m}/model.json") as f: + model = json.load(f) + experiment_name = get_most_recent_experiment_name(model) + if experiment_name: + model_experiment.write(f"{m},{experiment_name}\n") + +# COMMAND ---------- + +# DBTITLE 1,cli execution +# MAGIC %sh +# MAGIC +# MAGIC input_dir=/dbfs/mnt/ccidsdatascidatalake/mlflow-migration-models/models +# MAGIC +# MAGIC while IFS=, read model experiment; +# MAGIC do +# MAGIC import-model \ +# MAGIC --input-dir '$input_dir/$model' \ +# MAGIC --model '$model' \ +# MAGIC --experiment-name '$experiment' \ +# MAGIC --delete-model True \ +# MAGIC --import-source-tags True \ +# MAGIC --verbose True; +# MAGIC done < model_experiment + +# COMMAND ---------- + + diff --git a/mycode/aws-model-hashes b/mycode/aws-model-hashes new file mode 100644 index 00000000..7630ae31 --- /dev/null +++ b/mycode/aws-model-hashes @@ -0,0 +1,69 @@ +alv_regressor: bc08c046ef468213921d1ceaea551d35 +churn-arima: Model file not found +churn-model: cfae6c5c021a01238fab224158046823 +FDM: cba24da13caf82c694dcc443d2abb6dc +five-year-alv: 3d1e4337f348a53d54701fff0e1f8b9f +nearby_location_classifier: Model file not found +nearby_transmitter_classificiation_1km: cd36bab1df90665e6957ec5dfa6d973e +pycaret_small_cell_clf: ce16bf9bbcfe0226860c1d77c03bf737 +random-forest-small-cell-classifier: 02c70a5af3406809b34f93a72b2c14f2 +rm_sos_model: b79915de67585a407af1bc188cf59350 +SC_TAM_xgb_ATT_14: Model file not found +SC_TAM_xgb_CC_ATT_13: c9b1bff151699b358199e8c3f29dd0fd +SC_TAM_xgb_CC_ATT_14: 2fdfd9d04f790e59e39568ee4bb720b1 +SC_TAM_xgb_CC_ATT_15: 9fb0a58f88c76f442a2ae95d1c5f5572 +SC_TAM_xgb_CC_ATT_16: b126e73954f0147d5db0f2f9f4e5d41e +SC_TAM_xgb_CC_TMO_13: 86867c9f74a829b49e2615fcb025c4f2 +SC_TAM_xgb_CC_TMO_14: fa57802a4515434b3f477446341476ef +SC_TAM_xgb_CC_TMO_15: 455b2ff6a2937013bd4b3f7ea57866f9 +SC_TAM_xgb_CC_TMO_16: 898a81cff0b8faa63d7cde8ed306f977 +SC_TAM_xgb_CC_total_13: 0568cd270cb84ac7a5e3ba192b430bfb +SC_TAM_xgb_CC_total_14: 46be3c00931fcd55320befb4a8a7f69b +SC_TAM_xgb_CC_total_15: 3073035766c0fb6db6bbd0cfc3928c9d +SC_TAM_xgb_CC_total_16: 33c815846d6c4d77ee3095b2348707be +SC_TAM_xgb_CC_VZW_13: 10099377eb81084545b7035707bc09c7 +SC_TAM_xgb_CC_VZW_14: 09dd59a9721327c9e8d7a8c1b2929b9d +SC_TAM_xgb_CC_VZW_15: e38cfe16baeef765596bb0130d3d6061 +SC_TAM_xgb_CC_VZW_16: 1cb7e9c8ae4535979b2d47a47999cad8 +SC_TAM_xgb_TMO_14: Model file not found +SC_TAM_xgb_total_14: Model file not found +SC_TAM_xgb_VZW_14: Model file not found +small_cell_classifier: e88cbd08d01e0bbe8e5ff6608541455a +small_cell_classifier_fs: Model file not found +ta_step_size_classifier: afaf962b88719e786d59597e0553921c +TAM v1 MVP: Model file not found +tam_RF_CC_12: 4eaa75365903f7a30946194a9b3cb94c +tam_RF_CC_13: c8d6a83d36cb4d28eb9f83dd829c99b3 +tam_RF_CC_14: 67508cac03d341de200abaa958b77a31 +tam_RF_CC_15: a0ac24c68a95df7687ee3d28eeabea36 +tam_RF_CC_16: a842437542bb380e02f6b9fa1922e2f4 +tam_RF_CC_17: 524709a93e2063b113343085f05e1091 +tamByMNO_RF_CC_ATT_12: 4f8bc9bd6f0dbda8bbf0d6838d8ac5ad +tamByMNO_RF_CC_ATT_13: e11fe2bbd705870bca8b8d793c83327f +tamByMNO_RF_CC_ATT_14: 25fbcb74d2d3bfd55339874713dd7541 +tamByMNO_RF_CC_ATT_15: d9ead39efbea06ac569c8477b6840fdf +tamByMNO_RF_CC_ATT_16: 408133539c2cbc02bbdfce3eae38923c +tamByMNO_RF_CC_ATT_17: c99b6dfa4e66de57faa9a32d128ee6ee +tamByMNO_RF_CC_TMO_13: b5c04f4f050f1a3640ca9963cb469db6 +tamByMNO_RF_CC_TMO_14: cb5fe7361fc17881ea60bfa0fd23060b +tamByMNO_RF_CC_TMO_15: 8bed2612506e7be6c7da54fde3dd5017 +tamByMNO_RF_CC_TMO_16: 18e3849545c621bb83203cfe00f6cf92 +tamByMNO_RF_CC_TMO_17: e27df7ef79937a2b779e194adbd1413e +tamByMNO_RF_CC_VZW_13: 6c913d63f1420c10ebd15662a7f68b1d +tamByMNO_RF_CC_VZW_14: da88b8c761d0c5b33d20b535c3d3ae0b +tamByMNO_RF_CC_VZW_15: 30ffdb42cb9f0e50f9a176fa56649a65 +tamByMNO_RF_CC_VZW_16: 508aeed31329c6bea3f8e6c76334987c +tamByMNO_RF_CC_VZW_17: 433e554237d605ae97f39bfce6e66f66 +timing-advance-step-size-ternary-classifier: b1b0b19abd834a86b582253577acd646 +tsa-gt-reg: aeb0e6623cadfd936b781b9bb4da39e3 +tsa-mono-reg: 1503821b4c9689fde4726471b77989aa +tsa-sst-reg: b8aecdddfd906087b5335b5b9f96a9a3 +wireless_throughput_model_AT&T: b443b41808ce9eaa6a4eb566f0093302 +wireless_throughput_model_AT&T_log: 6db791d5c8b1b5b7b8e8ee396d37cd4e +wireless_throughput_model_T-Mobile: 8650f744e0440ae9474978948505ac50 +wireless_throughput_model_T-Mobile_log: 09f16d64726d3d721240eda91271ef07 +wireless_throughput_model_Verizon: e6311060a594a4250e9bfaba97688695 +wireless_throughput_model_Verizon_log: 3023cc6114e2f16cdf89a2c1b22d99d1 +wireless_throughput_Verizon: Model file not found +ws_activity_classifier: c159a33ed4788b4e5925befe8da66efe +xgboost_dev: Model file not found \ No newline at end of file diff --git a/mycode/aws-model-hashes-uc b/mycode/aws-model-hashes-uc new file mode 100644 index 00000000..b7257b6c --- /dev/null +++ b/mycode/aws-model-hashes-uc @@ -0,0 +1,68 @@ +ds_nonprod.migrated_models.wireless_throughput_model_verizon_log: 3023cc6114e2f16cdf89a2c1b22d99d1 +ds_nonprod.migrated_models.tam_rf_cc_15: a0ac24c68a95df7687ee3d28eeabea36 +ds_nonprod.migrated_models.wireless_throughput_verizon: Model not found +ds_nonprod.migrated_models.churn-arima: Model not found +ds_nonprod.migrated_models.tam_rf_cc_12: 4eaa75365903f7a30946194a9b3cb94c +ds_nonprod.migrated_models.tambymno_rf_cc_vzw_16: 508aeed31329c6bea3f8e6c76334987c +ds_nonprod.migrated_models.sc_tam_xgb_cc_tmo_16: 898a81cff0b8faa63d7cde8ed306f977 +ds_nonprod.migrated_models.ta_step_size_classifier: afaf962b88719e786d59597e0553921c +ds_nonprod.migrated_models.ws_activity_classifier: c159a33ed4788b4e5925befe8da66efe +ds_nonprod.migrated_models.random-forest-small-cell-classifier: 02c70a5af3406809b34f93a72b2c14f2 +ds_nonprod.migrated_models.tambymno_rf_cc_vzw_15: 30ffdb42cb9f0e50f9a176fa56649a65 +ds_nonprod.migrated_models.tam_rf_cc_17: 524709a93e2063b113343085f05e1091 +ds_nonprod.migrated_models.tambymno_rf_cc_tmo_15: 8bed2612506e7be6c7da54fde3dd5017 +ds_nonprod.migrated_models.sc_tam_xgb_total_14: Model not found +ds_nonprod.migrated_models.tambymno_rf_cc_att_15: d9ead39efbea06ac569c8477b6840fdf +ds_nonprod.migrated_models.tam_rf_cc_13: c8d6a83d36cb4d28eb9f83dd829c99b3 +ds_nonprod.migrated_models.wireless_throughput_model_at&t: b443b41808ce9eaa6a4eb566f0093302 +ds_nonprod.migrated_models.five-year-alv: 3d1e4337f348a53d54701fff0e1f8b9f +ds_nonprod.migrated_models.sc_tam_xgb_cc_vzw_14: 09dd59a9721327c9e8d7a8c1b2929b9d +ds_nonprod.migrated_models.sc_tam_xgb_cc_tmo_15: 455b2ff6a2937013bd4b3f7ea57866f9 +ds_nonprod.migrated_models.tam_rf_cc_14: 67508cac03d341de200abaa958b77a31 +ds_nonprod.migrated_models.tambymno_rf_cc_tmo_16: 18e3849545c621bb83203cfe00f6cf92 +ds_nonprod.migrated_models.sc_tam_xgb_cc_tmo_14: fa57802a4515434b3f477446341476ef +ds_nonprod.migrated_models.sc_tam_xgb_cc_vzw_15: e38cfe16baeef765596bb0130d3d6061 +ds_nonprod.migrated_models.sc_tam_xgb_cc_total_14: 46be3c00931fcd55320befb4a8a7f69b +ds_nonprod.migrated_models.sc_tam_xgb_vzw_14: Model not found +ds_nonprod.migrated_models.tambymno_rf_cc_att_12: 4f8bc9bd6f0dbda8bbf0d6838d8ac5ad +ds_nonprod.migrated_models.sc_tam_xgb_cc_att_15: 9fb0a58f88c76f442a2ae95d1c5f5572 +ds_nonprod.migrated_models.tambymno_rf_cc_tmo_17: e27df7ef79937a2b779e194adbd1413e +ds_nonprod.migrated_models.tambymno_rf_cc_att_14: 25fbcb74d2d3bfd55339874713dd7541 +ds_nonprod.migrated_models.tsa-mono-reg: Model not found +ds_nonprod.migrated_models.wireless_throughput_model_at&t_log: 6db791d5c8b1b5b7b8e8ee396d37cd4e +ds_nonprod.migrated_models.tambymno_rf_cc_att_17: c99b6dfa4e66de57faa9a32d128ee6ee +ds_nonprod.migrated_models.xgboost_dev: Model not found +ds_nonprod.migrated_models.sc_tam_xgb_cc_total_16: 33c815846d6c4d77ee3095b2348707be +ds_nonprod.migrated_models.rm_sos_model: b79915de67585a407af1bc188cf59350 +ds_nonprod.migrated_models.tambymno_rf_cc_vzw_14: da88b8c761d0c5b33d20b535c3d3ae0b +ds_nonprod.migrated_models.sc_tam_xgb_cc_total_15: 3073035766c0fb6db6bbd0cfc3928c9d +ds_nonprod.migrated_models.nearby_location_classifier: Model not found +ds_nonprod.migrated_models.small_cell_classifier_fs: Model not found +ds_nonprod.migrated_models.sc_tam_xgb_cc_att_16: b126e73954f0147d5db0f2f9f4e5d41e +ds_nonprod.migrated_models.sc_tam_xgb_cc_total_13: 0568cd270cb84ac7a5e3ba192b430bfb +ds_nonprod.migrated_models.sc_tam_xgb_cc_att_14: 2fdfd9d04f790e59e39568ee4bb720b1 +ds_nonprod.migrated_models.tambymno_rf_cc_tmo_14: cb5fe7361fc17881ea60bfa0fd23060b +ds_nonprod.migrated_models.timing-advance-step-size-ternary-classifier: b1b0b19abd834a86b582253577acd646 +ds_nonprod.migrated_models.tsa-sst-reg: Model not found +ds_nonprod.migrated_models.tambymno_rf_cc_att_16: 408133539c2cbc02bbdfce3eae38923c +ds_nonprod.migrated_models.sc_tam_xgb_cc_vzw_16: 1cb7e9c8ae4535979b2d47a47999cad8 +ds_nonprod.migrated_models.tambymno_rf_cc_att_13: e11fe2bbd705870bca8b8d793c83327f +ds_nonprod.migrated_models.sc_tam_xgb_tmo_14: Model not found +ds_nonprod.migrated_models.tambymno_rf_cc_vzw_13: 6c913d63f1420c10ebd15662a7f68b1d +ds_nonprod.migrated_models.tsa-gt-reg: Model not found +ds_nonprod.migrated_models.fdm: cba24da13caf82c694dcc443d2abb6dc +ds_nonprod.migrated_models.nearby_transmitter_classificiation_1km: cd36bab1df90665e6957ec5dfa6d973e +ds_nonprod.migrated_models.tambymno_rf_cc_tmo_13: b5c04f4f050f1a3640ca9963cb469db6 +ds_nonprod.migrated_models.wireless_throughput_model_t-mobile_log: 09f16d64726d3d721240eda91271ef07 +ds_nonprod.migrated_models.sc_tam_xgb_cc_tmo_13: 86867c9f74a829b49e2615fcb025c4f2 +ds_nonprod.migrated_models.wireless_throughput_model_verizon: Model not found +ds_nonprod.migrated_models.tambymno_rf_cc_vzw_17: 433e554237d605ae97f39bfce6e66f66 +ds_nonprod.migrated_models.alv_regressor: bc08c046ef468213921d1ceaea551d35 +ds_nonprod.migrated_models.sc_tam_xgb_cc_att_13: c9b1bff151699b358199e8c3f29dd0fd +ds_nonprod.migrated_models.churn-model: Model not found +ds_nonprod.migrated_models.sc_tam_xgb_att_14: Model not found +ds_nonprod.migrated_models.wireless_throughput_model_t-mobile: 8650f744e0440ae9474978948505ac50 +ds_nonprod.migrated_models.sc_tam_xgb_cc_vzw_13: 10099377eb81084545b7035707bc09c7 +ds_nonprod.migrated_models.tam_rf_cc_16: a842437542bb380e02f6b9fa1922e2f4 +ds_nonprod.migrated_models.pycaret_small_cell_clf: Model not found +ds_nonprod.migrated_models.small_cell_classifier: 375a1ac8c163d8e70980abc6ab01292a \ No newline at end of file diff --git a/mycode/azure-model-hashes b/mycode/azure-model-hashes new file mode 100644 index 00000000..4408ba4d --- /dev/null +++ b/mycode/azure-model-hashes @@ -0,0 +1,70 @@ +alv_regressor: bc08c046ef468213921d1ceaea551d35 +churn-arima: Model file not found +churn-model: cfae6c5c021a01238fab224158046823 +FDM: cba24da13caf82c694dcc443d2abb6dc +five-year-alv: 3d1e4337f348a53d54701fff0e1f8b9f +nearby_location_classifier: Model file not found +nearby_transmitter_classificiation_1km: 4f7014a660f5dd7b0bdf519a1cc8733e +pycaret_small_cell_clf: ce16bf9bbcfe0226860c1d77c03bf737 +random-forest-small-cell-classifier: 02c70a5af3406809b34f93a72b2c14f2 +rfds_vgg: Model file not found +rm_sos_model: b79915de67585a407af1bc188cf59350 +SC_TAM_xgb_ATT_14: Model file not found +SC_TAM_xgb_CC_ATT_13: c9b1bff151699b358199e8c3f29dd0fd +SC_TAM_xgb_CC_ATT_14: e780e65b67f3d5f8ee10d33c26bca286 +SC_TAM_xgb_CC_ATT_15: 9fb0a58f88c76f442a2ae95d1c5f5572 +SC_TAM_xgb_CC_ATT_16: 582919c7b5a1ec7006ca740e41eb1c8a +SC_TAM_xgb_CC_TMO_13: 86867c9f74a829b49e2615fcb025c4f2 +SC_TAM_xgb_CC_TMO_14: a91791267a14d95e94572c37ba6c4925 +SC_TAM_xgb_CC_TMO_15: 455b2ff6a2937013bd4b3f7ea57866f9 +SC_TAM_xgb_CC_TMO_16: d2337a992ab0bf48f38f5d6b608c0a49 +SC_TAM_xgb_CC_total_13: 0568cd270cb84ac7a5e3ba192b430bfb +SC_TAM_xgb_CC_total_14: 46be3c00931fcd55320befb4a8a7f69b +SC_TAM_xgb_CC_total_15: 3073035766c0fb6db6bbd0cfc3928c9d +SC_TAM_xgb_CC_total_16: 33c815846d6c4d77ee3095b2348707be +SC_TAM_xgb_CC_VZW_13: 5bcd17f601967212a59132710d116b72 +SC_TAM_xgb_CC_VZW_14: 3bc0d66861a84ab7a66b20588cacfcf4 +SC_TAM_xgb_CC_VZW_15: e38cfe16baeef765596bb0130d3d6061 +SC_TAM_xgb_CC_VZW_16: a4bdc13587d437ed12dd3021e506ad16 +SC_TAM_xgb_TMO_14: Model file not found +SC_TAM_xgb_total_14: Model file not found +SC_TAM_xgb_VZW_14: Model file not found +small_cell_classifier: e88cbd08d01e0bbe8e5ff6608541455a +small_cell_classifier_fs: Model file not found +ta_step_size_classifier: afaf962b88719e786d59597e0553921c +TAM v1 MVP: Model file not found +tam_RF_CC_12: 4eaa75365903f7a30946194a9b3cb94c +tam_RF_CC_13: c8d6a83d36cb4d28eb9f83dd829c99b3 +tam_RF_CC_14: 67508cac03d341de200abaa958b77a31 +tam_RF_CC_15: a0ac24c68a95df7687ee3d28eeabea36 +tam_RF_CC_16: a842437542bb380e02f6b9fa1922e2f4 +tam_RF_CC_17: 524709a93e2063b113343085f05e1091 +tamByMNO_RF_CC_ATT_12: 4f8bc9bd6f0dbda8bbf0d6838d8ac5ad +tamByMNO_RF_CC_ATT_13: d90498883b237ef6b71932ebf3dfee75 +tamByMNO_RF_CC_ATT_14: 25fbcb74d2d3bfd55339874713dd7541 +tamByMNO_RF_CC_ATT_15: d9ead39efbea06ac569c8477b6840fdf +tamByMNO_RF_CC_ATT_16: 38d12e47d3a9774b73749c3f31460521 +tamByMNO_RF_CC_ATT_17: c99b6dfa4e66de57faa9a32d128ee6ee +tamByMNO_RF_CC_TMO_13: c7c0fa3532104107db03b4cc46da53a3 +tamByMNO_RF_CC_TMO_14: cb5fe7361fc17881ea60bfa0fd23060b +tamByMNO_RF_CC_TMO_15: 8bed2612506e7be6c7da54fde3dd5017 +tamByMNO_RF_CC_TMO_16: 18e3849545c621bb83203cfe00f6cf92 +tamByMNO_RF_CC_TMO_17: e27df7ef79937a2b779e194adbd1413e +tamByMNO_RF_CC_VZW_13: 33f8b87ffcfc246912cff23b9c46929e +tamByMNO_RF_CC_VZW_14: da88b8c761d0c5b33d20b535c3d3ae0b +tamByMNO_RF_CC_VZW_15: 30ffdb42cb9f0e50f9a176fa56649a65 +tamByMNO_RF_CC_VZW_16: 508aeed31329c6bea3f8e6c76334987c +tamByMNO_RF_CC_VZW_17: 433e554237d605ae97f39bfce6e66f66 +timing-advance-step-size-ternary-classifier: b1b0b19abd834a86b582253577acd646 +tsa-gt-reg: aeb0e6623cadfd936b781b9bb4da39e3 +tsa-mono-reg: 1503821b4c9689fde4726471b77989aa +tsa-sst-reg: b8aecdddfd906087b5335b5b9f96a9a3 +wireless_throughput_model_AT&T: 99c32a9286879b6057b75e60384c30ca +wireless_throughput_model_AT&T_log: 6db791d5c8b1b5b7b8e8ee396d37cd4e +wireless_throughput_model_T-Mobile: e6e83b20667bd5726ba39c183ece3527 +wireless_throughput_model_T-Mobile_log: 09f16d64726d3d721240eda91271ef07 +wireless_throughput_model_Verizon: e6311060a594a4250e9bfaba97688695 +wireless_throughput_model_Verizon_log: 3023cc6114e2f16cdf89a2c1b22d99d1 +wireless_throughput_Verizon: Model file not found +ws_activity_classifier: c159a33ed4788b4e5925befe8da66efe +xgboost_dev: Model file not found \ No newline at end of file diff --git a/mycode/cli-export-experiments.py b/mycode/cli-export-experiments.py new file mode 100644 index 00000000..cd03bb7f --- /dev/null +++ b/mycode/cli-export-experiments.py @@ -0,0 +1,78 @@ +# Databricks notebook source +# MAGIC %md ## Libs + +# COMMAND ---------- + +# DBTITLE 1,install latest pkg from github +# MAGIC %sh +# MAGIC #pip install mlflow-export-import +# MAGIC pip install git+https:///github.com/mlflow/mlflow-export-import/#egg=mlflow-export-import + +# COMMAND ---------- + +# DBTITLE 1,we can see the s3 mount from %sh :) +# MAGIC %sh ls /dbfs/mnt/aws-ds-non-prod + +# COMMAND ---------- + +# MAGIC %run ./credentials + +# COMMAND ---------- + +# MAGIC %md ## Setup + +# COMMAND ---------- + +# DBTITLE 1,variables +dbutils.widgets.dropdown("platform","",["", "azure", "aws"]) +platform = dbutils.widgets.get("platform") + +credentials_path = get_credentials_path(platform) + +# COMMAND ---------- + +# DBTITLE 1,set env vars +import os +from datetime import datetime +import pytz + +cst = pytz.timezone('US/Central') +now = datetime.now(tz=cst) +date = now.strftime("%Y%m%d_%H%M") + +logfile = f"export_experiments_{date}.log" +os.environ["MLFLOW_EXPORT_IMPORT_LOG_OUTPUT_FILE"] = logfile + +os.environ["MLFLOW_EXPORT_IMPORT_LOG_FORMAT"]="%(threadName)s-%(levelname)s-%(message)s" + +os.environ["MLFLOW_TRACKING_URI"]="databricks" + +with open(credentials_path) as f: + os.environ["DATABRICKS_HOST"] = f.readline().strip("\n") + os.environ["DATABRICKS_TOKEN"] = f.readline().strip("\n") + +# COMMAND ---------- + +# DBTITLE 1,verify env vars +# MAGIC %sh +# MAGIC echo $MLFLOW_EXPORT_IMPORT_LOG_OUTPUT_FILE +# MAGIC echo $MLFLOW_EXPORT_IMPORT_LOG_FORMAT +# MAGIC echo $MLFLOW_TRACKING_URI +# MAGIC # echo $DATABRICKS_HOST +# MAGIC # echo $DATABRICKS_TOKEN + +# COMMAND ---------- + +# MAGIC %md ## Execute + +# COMMAND ---------- + +# DBTITLE 1,cli execution | all +# MAGIC %sh +# MAGIC export-experiments \ +# MAGIC --output-dir /dbfs/mnt/aws-ds-non-prod/mlflow-migration-experiments/experiments \ +# MAGIC --experiments all \ +# MAGIC --run-start-time 2024-11-09 \ +# MAGIC --export-permissions True \ +# MAGIC --notebook-formats SOURCE \ +# MAGIC --use-threads True diff --git a/mycode/cli-export-models.py b/mycode/cli-export-models.py new file mode 100644 index 00000000..a863d799 --- /dev/null +++ b/mycode/cli-export-models.py @@ -0,0 +1,90 @@ +# Databricks notebook source +# MAGIC %md ## Libs + +# COMMAND ---------- + +# DBTITLE 1,install latest pkg from github +# MAGIC %sh +# MAGIC #pip install mlflow-export-import +# MAGIC pip install git+https:///github.com/mlflow/mlflow-export-import/#egg=mlflow-export-import + +# COMMAND ---------- + +# MAGIC %run ./credentials + +# COMMAND ---------- + +# DBTITLE 1,we can see the s3 mount from %sh :) +# MAGIC %sh ls /dbfs/mnt/aws-ds-non-prod + +# COMMAND ---------- + +# MAGIC %md ## Setup +# MAGIC Make sure the mount to the target s3 bucket is set up. See the **mount** notebook in this directory + +# COMMAND ---------- + +# DBTITLE 1,variables +dbutils.widgets.dropdown("platform","",["", "azure", "aws"]) +platform = dbutils.widgets.get("platform") + +credentials_path = get_credentials_path(platform) + +# COMMAND ---------- + +# DBTITLE 1,set env vars +import os +from datetime import datetime +import pytz + +cst = pytz.timezone('US/Central') +now = datetime.now(tz=cst) +date = now.strftime("%Y%m%d_%H%M") + +logfile = f"export_models_{date}.log" +os.environ["MLFLOW_EXPORT_IMPORT_LOG_OUTPUT_FILE"] = logfile + +os.environ["MLFLOW_EXPORT_IMPORT_LOG_FORMAT"]="%(threadName)s-%(levelname)s-%(message)s" + +os.environ["MLFLOW_TRACKING_URI"]="databricks" + +with open(credentials_path) as f: + os.environ["DATABRICKS_HOST"] = f.readline().strip("\n") + os.environ["DATABRICKS_TOKEN"] = f.readline().strip("\n") + +# COMMAND ---------- + +# DBTITLE 1,verify env vars +# MAGIC %sh +# MAGIC echo $MLFLOW_EXPORT_IMPORT_LOG_OUTPUT_FILE +# MAGIC echo $MLFLOW_EXPORT_IMPORT_LOG_FORMAT +# MAGIC echo $MLFLOW_TRACKING_URI +# MAGIC # echo $DATABRICKS_HOST +# MAGIC # echo $DATABRICKS_TOKEN + +# COMMAND ---------- + +# DBTITLE 0,`export-models` options +# MAGIC %sh +# MAGIC export-models --help + +# COMMAND ---------- + +# MAGIC %md ## Execute + +# COMMAND ---------- + +# DBTITLE 1,cli execution +# MAGIC %sh +# MAGIC export-models \ +# MAGIC --output-dir /dbfs/mnt/aws-ds-non-prod/mlflow-migration-models-01 \ +# MAGIC --models all \ +# MAGIC --stages 'Production' \ +# MAGIC --export-permissions True \ +# MAGIC --notebook-formats SOURCE \ +# MAGIC --export-version-model True \ +# MAGIC --use-threads True + +# COMMAND ---------- + + diff --git a/mycode/cli-import-experiments.py b/mycode/cli-import-experiments.py new file mode 100644 index 00000000..b549676f --- /dev/null +++ b/mycode/cli-import-experiments.py @@ -0,0 +1,90 @@ +# Databricks notebook source +# MAGIC %md ## Libs + +# COMMAND ---------- + +# DBTITLE 1,install latest pkg from github +# MAGIC %sh +# MAGIC #pip install mlflow-export-import +# MAGIC pip install git+https:///github.com/mlflow/mlflow-export-import/#egg=mlflow-export-import + +# COMMAND ---------- + +# MAGIC %run ./credentials + +# COMMAND ---------- + +# MAGIC %md ## Setup + +# COMMAND ---------- + +# DBTITLE 1,we can see the s3 mount from %sh :) +# MAGIC %fs ls /mnt/datalake + +# COMMAND ---------- + +# DBTITLE 1,variables +dbutils.widgets.dropdown("platform","",["", "azure", "aws"]) +platform = dbutils.widgets.get("platform") + +credentials_path = get_credentials_path(platform) + +# COMMAND ---------- + +# DBTITLE 1,set env vars +import os +from datetime import datetime +import pytz + +cst = pytz.timezone('US/Central') +now = datetime.now(tz=cst) +date = now.strftime("%Y%m%d_%H%M") + +logfile = f"import_experiments_{date}.log" +os.environ["MLFLOW_EXPORT_IMPORT_LOG_OUTPUT_FILE"] = logfile + +os.environ["MLFLOW_EXPORT_IMPORT_LOG_FORMAT"]="%(threadName)s-%(levelname)s-%(message)s" + +os.environ["MLFLOW_TRACKING_URI"]="databricks" + +with open(credentials_path) as f: + os.environ["DATABRICKS_HOST"] = f.readline().strip("\n") + os.environ["DATABRICKS_TOKEN"] = f.readline().strip("\n") + +# COMMAND ---------- + +# DBTITLE 1,verify env vars +# MAGIC %sh +# MAGIC echo $MLFLOW_EXPORT_IMPORT_LOG_OUTPUT_FILE +# MAGIC echo $MLFLOW_EXPORT_IMPORT_LOG_FORMAT +# MAGIC echo $MLFLOW_TRACKING_URI +# MAGIC # echo $DATABRICKS_HOST +# MAGIC # echo $DATABRICKS_TOKEN + +# COMMAND ---------- + +# MAGIC %md ## Execution + +# COMMAND ---------- + +# DBTITLE 0,`export-models` options +# MAGIC %sh +# MAGIC import-experiments --help + +# COMMAND ---------- + +# DBTITLE 1,cli execution +# MAGIC %sh +# MAGIC import-experiments \ +# MAGIC --input-dir /dbfs/mnt/datalake/mlflow-migration-02/experiments \ +# MAGIC --import-source-tags True \ +# MAGIC --import-permissions True \ +# MAGIC --use-threads True + +# COMMAND ---------- + +# MAGIC %sh ls /dbfs/mnt/datalake + +# COMMAND ---------- + + diff --git a/mycode/cli-import-models.py b/mycode/cli-import-models.py new file mode 100644 index 00000000..69bc8450 --- /dev/null +++ b/mycode/cli-import-models.py @@ -0,0 +1,184 @@ +# Databricks notebook source +# MAGIC %md ## Libs + +# COMMAND ---------- + +# DBTITLE 1,install latest pkg from github +# MAGIC %sh +# MAGIC #pip install mlflow-export-import +# MAGIC pip install git+https:///github.com/mlflow/mlflow-export-import/#egg=mlflow-export-import + +# COMMAND ---------- + +# DBTITLE 1,we can see the s3 mount from %sh :) +# MAGIC %sh ls /dbfs/mnt/aws-ds-non-prod + +# COMMAND ---------- + +# MAGIC %run ./credentials + +# COMMAND ---------- + +# MAGIC %md ## Setup + +# COMMAND ---------- + +dbutils.widgets.dropdown("platform","",["", "azure", "aws"]) +platform = dbutils.widgets.get("platform") + +credentials_path = get_credentials_path(platform) + +# COMMAND ---------- + +# DBTITLE 1,set env vars +import os +from datetime import datetime +import pytz + +cst = pytz.timezone('US/Central') +now = datetime.now(tz=cst) +date = now.strftime("%Y%m%d_%H%M") + +logfile = f"import_models_{date}.log" +os.environ["MLFLOW_EXPORT_IMPORT_LOG_OUTPUT_FILE"] = logfile + +os.environ["MLFLOW_EXPORT_IMPORT_LOG_FORMAT"]="%(threadName)s-%(levelname)s-%(message)s" + +os.environ["MLFLOW_TRACKING_URI"]="databricks" + +with open(credentials_path) as f: + os.environ["DATABRICKS_HOST"] = f.readline().strip("\n") + os.environ["DATABRICKS_TOKEN"] = f.readline().strip("\n") + +# COMMAND ---------- + +# DBTITLE 1,verify env vars +# MAGIC %sh +# MAGIC echo $MLFLOW_EXPORT_IMPORT_LOG_OUTPUT_FILE +# MAGIC echo $MLFLOW_EXPORT_IMPORT_LOG_FORMAT +# MAGIC echo $MLFLOW_TRACKING_URI +# MAGIC echo $DATABRICKS_HOST +# MAGIC echo $DATABRICKS_TOKEN + +# COMMAND ---------- + +# DBTITLE 0,`export-models` options +# MAGIC %sh +# MAGIC import-models --help + +# COMMAND ---------- + +# MAGIC %md ## Execution + +# COMMAND ---------- + +# DBTITLE 1,cli execution +# MAGIC %sh +# MAGIC import-models \ +# MAGIC --input-dir /dbfs/mnt/aws-ds-non-prod/mlflow-migration-models-01 \ +# MAGIC --delete-model True \ +# MAGIC --import-permissions True \ +# MAGIC --import-source-tags True \ +# MAGIC --verbose True +# MAGIC --use-threads False + +# COMMAND ---------- + +# MAGIC %md ## Appendices + +# COMMAND ---------- + +# DBTITLE 1,count models in workspace model registry +import mlflow +len(mlflow.search_registered_models()) + +# COMMAND ---------- + +# MAGIC %sh +# MAGIC import-model --help + +# COMMAND ---------- + +# MAGIC %sh +# MAGIC import-model \ +# MAGIC --input-dir /dbfs/mnt/ccidsdatascidatalake/mlflow-migration-models/models/pycaret_small_cell_clf \ +# MAGIC --model pycaret_small_cell_clf \ +# MAGIC --experiment-name /Users/darrell.coles@crowncastle.com/databricks_automl/pycaret-sc-clf-test-1 \ +# MAGIC --delete-model True \ +# MAGIC # --import-permissions True \ +# MAGIC --import-source-tags True \ +# MAGIC --verbose True + +# COMMAND ---------- + +# MAGIC %sh cat /dbfs/mnt/ccidsdatascidatalake/mlflow-migration-models/models/models.json + +# COMMAND ---------- + +def get_most_recent_experiment_name(model): + try: + newest = max(model["mlflow"]["registered_model"]["versions"], key=lambda v:v["last_updated_timestamp"]) + return newest["_experiment_name"] + except: + return None + +# COMMAND ---------- + +import json + +input_dir = "/dbfs/mnt/ccidsdatascidatalake/mlflow-migration-models/models" + +with open(f"{input_dir}/models.json") as f: + models = json.load(f) + model_names = models["info"]["model_names"] + +with open("model_experiment","w") as model_experiment: + for m in model_names: + with open(f"{input_dir}/{m}/model.json") as f: + model = json.load(f) + experiment_name = get_most_recent_experiment_name(model) + if experiment_name: + model_experiment.write(f"{m},{experiment_name}\n") + +# COMMAND ---------- + +# MAGIC %sh cat model_experiment + +# COMMAND ---------- + +# MAGIC %sh +# MAGIC +# MAGIC input_dir=/dbfs/mnt/ccidsdatascidatalake/mlflow-migration-models/models +# MAGIC +# MAGIC while IFS=, read model experiment; +# MAGIC do +# MAGIC import-model \ +# MAGIC --input-dir $input_dir/$model \ +# MAGIC --model $model \ +# MAGIC --experiment-name $experiment_name \ +# MAGIC --delete-model True \ +# MAGIC --import-source-tags True \ +# MAGIC --verbose True; +# MAGIC done < model_experiment + +# COMMAND ---------- + +# MAGIC %sh +# MAGIC while IFS=, read field1 field2; +# MAGIC do +# MAGIC echo model: $field1 +# MAGIC echo experiment: $field2 +# MAGIC done < model_experiment + +# COMMAND ---------- + +import os +os.environ + +# COMMAND ---------- + +# MAGIC %sh $elas + +# COMMAND ---------- + + diff --git a/mycode/compare-model-hashes-uc.py b/mycode/compare-model-hashes-uc.py new file mode 100644 index 00000000..a6e2107b --- /dev/null +++ b/mycode/compare-model-hashes-uc.py @@ -0,0 +1,60 @@ +# Databricks notebook source +# DBTITLE 1,azure model hashes +with open("azure-model-hashes", "r") as f: + lines = f.readlines() + +az_model_hashes = dict() +for line in lines: + k, v = line.lower().strip("\n").split(": ") + if v=="model file not found": + v=None + az_model_hashes[k] = v + +# COMMAND ---------- + +# DBTITLE 1,aws model hashes +with open("aws-model-hashes-uc", "r") as f: + lines = f.readlines() + +aws_model_hashes = dict() +for line in lines: + k, v = line.lower().replace("ds_nonprod.migrated_models.","").strip("\n").split(": ") + if v=='model not found': + v=None + aws_model_hashes[k] = v + +# COMMAND ---------- + +aws_model_hashes + +# COMMAND ---------- + +# DBTITLE 1,num registered models on each platform +len(az_model_hashes), len(aws_model_hashes) + +# COMMAND ---------- + +# DBTITLE 1,missing models +set(az_model_hashes.keys()) - set(aws_model_hashes.keys()), set(aws_model_hashes.keys()) - set(az_model_hashes.keys()) + +# COMMAND ---------- + +import pandas as pd + +azure_df = pd.DataFrame(az_model_hashes, index=["azure"]) +aws_df = pd.DataFrame(aws_model_hashes, index=["aws"]) +model_hashes_pdf = pd.concat((azure_df, aws_df)).T + +model_hashes_pdf["match"] = model_hashes_pdf.azure == model_hashes_pdf.aws + +model_hashes = spark.createDataFrame(model_hashes_pdf.reset_index()) + +display(model_hashes) + +# COMMAND ---------- + +display(model_hashes.where("not match and azure is not null and aws is not null")) + +# COMMAND ---------- + + diff --git a/mycode/compare-model-hashes.py b/mycode/compare-model-hashes.py new file mode 100644 index 00000000..bb4c21cc --- /dev/null +++ b/mycode/compare-model-hashes.py @@ -0,0 +1,52 @@ +# Databricks notebook source +# DBTITLE 1,azure model hashes +with open("azure-model-hashes", "r") as f: + lines = f.readlines() + +az_model_hashes = dict() +for line in lines: + k, v = line.strip("\n").split(": ") + az_model_hashes[k] = v + +# COMMAND ---------- + +# DBTITLE 1,aws model hashes +with open("aws-model-hashes", "r") as f: + lines = f.readlines() + +aws_model_hashes = dict() +for line in lines: + k, v = line.strip("\n").split(": ") + aws_model_hashes[k] = v + +# COMMAND ---------- + +# DBTITLE 1,num registered models on each platform +len(az_model_hashes), len(aws_model_hashes) + +# COMMAND ---------- + +# DBTITLE 1,missing models +set(az_model_hashes.keys()) - set(aws_model_hashes.keys()), set(aws_model_hashes.keys()) - set(az_model_hashes.keys()) + +# COMMAND ---------- + +import pandas as pd + +azure_df = pd.DataFrame(az_model_hashes, index=["azure"]) +aws_df = pd.DataFrame(aws_model_hashes, index=["aws"]) +model_hashes_pdf = pd.concat((azure_df, aws_df)).T + +model_hashes_pdf["match"] = model_hashes_pdf.azure == model_hashes_pdf.aws + +model_hashes = spark.createDataFrame(model_hashes_pdf.reset_index()) + +display(model_hashes) + +# COMMAND ---------- + +display(model_hashes.where("not match")) + +# COMMAND ---------- + + diff --git a/mycode/count-exported-mlflow-objects.py b/mycode/count-exported-mlflow-objects.py new file mode 100644 index 00000000..a12a6ac3 --- /dev/null +++ b/mycode/count-exported-mlflow-objects.py @@ -0,0 +1,98 @@ +# Databricks notebook source +import re +import pandas as pd +import numpy as np + +# COMMAND ---------- + +# MAGIC %md ## Experiments + +# COMMAND ---------- + +# MAGIC %sh +# MAGIC find /dbfs/mnt/ccidsdatascidatalake/mlflow-migration-models/experiments -type d -maxdepth 2 > exported_experiments + +# COMMAND ---------- + +exp_pat = "^.*\/dbfs\/mnt\/datalake\/mlflow-migration-models\/experiments\/(.*)$" + +filepath = "exported_experiments" +capture = [re.findall(exp_pat, line) for line in open(filepath)] + +# COMMAND ---------- + +def parse_exp_run(s): + res = s.split("/") + return res if len(res)==2 else [*res,None] + +parsed = np.array([parse_exp_run(c[0]) for c in capture if c]) + +df_parsed = pd.DataFrame(dict(experiment_id=parsed[:,0], run_id=parsed[:,1])) + +exp_df = df_parsed.groupby("experiment_id").run_id.count().to_frame() +exp_df.columns = ["n_runs"] + +# COMMAND ---------- + +# MAGIC %md ## Models + +# COMMAND ---------- + +# MAGIC %sh +# MAGIC ls /dbfs/mnt/datalake/mlflow-migration-models/models/ > exported_models + +# COMMAND ---------- + +with open("exported_models") as f: + n_models = len(f.readlines()) + +# COMMAND ---------- + +# MAGIC %md ## Report + +# COMMAND ---------- + +print("There are", exp_df.shape[0], "total experiments") +print("There are", exp_df.query("n_runs > 0").shape[0], "experiments with at least one run") +print("There are", exp_df.query("n_runs == 0").shape[0], "experiments with no runs") +print("There are", exp_df.n_runs.sum(), "runs") +print("There are", n_models, "registered models") + +# COMMAND ---------- + +# import os + +# def exp_id(exp): +# try: +# return exp.experiment_id +# except: +# return "none" + +# def exp_id_after_import(exp): +# try: +# return exp.experiment_id +# except: +# return "none" + +# def email(exp): +# try: +# return exp.tags['mlflow.ownerEmail'] +# except: +# return "none" + +# def n_runs(exp): +# try: +# return len(mlflow.search_runs(exp_id(exp))) +# except: +# 0 + +# result = [] +# for exp in mlflow.search_experiments(): +# result.append([os.path.basename(exp.name), email(exp), n_runs(exp)]) + +# df = spark.createDataFrame(result,["experiment_name","owner","n_runs"]) +# display(df) + +# COMMAND ---------- + + diff --git a/mycode/count-legacy-production-models.py b/mycode/count-legacy-production-models.py new file mode 100644 index 00000000..72d82334 --- /dev/null +++ b/mycode/count-legacy-production-models.py @@ -0,0 +1,28 @@ +# Databricks notebook source +import mlflow +from mlflow import MlflowClient +client = MlflowClient() + +# COMMAND ---------- + +# DBTITLE 1,count all models +len(client.search_registered_models()) + +# COMMAND ---------- + +# DBTITLE 1,count models with prod stage +count = 0 + +for registered_model in mlflow.search_registered_models(): + has_prod_stg = False + if "Production" in [version.current_stage for version in registered_model.latest_versions]: + has_prod_stg = True + count +=1 + if not has_prod_stg: + print(registered_model.name, "DOES NOT HAVE a Production stage") + +print("\n", count, "models have Production stages all together") + +# COMMAND ---------- + + diff --git a/mycode/count-objects-in-mlflow.py b/mycode/count-objects-in-mlflow.py new file mode 100644 index 00000000..356bd7d2 --- /dev/null +++ b/mycode/count-objects-in-mlflow.py @@ -0,0 +1,90 @@ +# Databricks notebook source +import mlflow +import pandas as pd +import numpy as np +from datetime import datetime + +# COMMAND ---------- + +wij = dbutils.widgets + +wij.text("start_time","") +start_time = wij.get("start_time") +start_time = start_time if start_time else None + +print("start_time:", start_time) + +# COMMAND ---------- + +# DBTITLE 1,inspection functions +def exp_id(exp): + try: + return exp.experiment_id + except: + return "none" + +def exp_id_after_import(exp): + try: + return exp.experiment_id + except: + return "none" + +def email(exp): + try: + return exp.tags['mlflow.ownerEmail'] + except: + return "none" + +def run_start_time(run): + """ + Parameters + ========== + run (pandas.Series): run parameters + + Returns + ======= + datetime + """ + return datetime( + year=run.start_time.year, + month=run.start_time.month, + day=run.start_time.day) + +def n_runs(exp, start_time=None): + """ + Parameters + ========== + start_time (str): date; format 'YYYY-MM-DD' + """ + runs = mlflow.search_runs(exp.experiment_id) + try: + if start_time: + year, month, day = start_time.split("-") + dt_thresh = datetime(year=int(year), month=int(month), day=int(day)) + return len([1 for _, run in runs.iterrows() if run_start_time(run) >= dt_thresh]) + else: + return len(runs) + except: + 0 + +# COMMAND ---------- + +# DBTITLE 1,experiment stats +data = np.array([(exp_id(exp), n_runs(exp, start_time), email(exp)) for exp in mlflow.search_experiments()]) + +df = pd.DataFrame(dict(experiment_id=data[:,0], n_runs=data[:,1], owner_email=data[:,2])) +df["n_runs"] = df.n_runs.astype(int) + +print("There are", df.shape[0], "total experiments") +print("There are", df.query("n_runs > 0").shape[0], "experiments with at least one run") +print("There are", df.query("n_runs == 0").shape[0], "experiments with no runs") +print("There are", df.n_runs.sum(), "runs") + +# COMMAND ---------- + +# DBTITLE 1,model stats +print("There are", len(mlflow.search_registered_models()), "registered models") + +# COMMAND ---------- + + diff --git a/mycode/credentials.py b/mycode/credentials.py new file mode 100644 index 00000000..35cb7836 --- /dev/null +++ b/mycode/credentials.py @@ -0,0 +1,10 @@ +# Databricks notebook source +def get_credentials_path(platform): + + az_creds_path = "/dbfs/FileStore/shared_uploads/darrell.coles@crowncastle.com/azure_databricks_credentials" + aws_creds_path = "/Workspace/Users/darrell.coles@crowncastle.com/aws_databricks_credentials" + + match platform: + case "": raise ValueError("platform must be specified") + case "azure": return az_creds_path + case "aws": return aws_creds_path diff --git a/mycode/delete-all-mlflow-experiments.py b/mycode/delete-all-mlflow-experiments.py new file mode 100644 index 00000000..3bc4717c --- /dev/null +++ b/mycode/delete-all-mlflow-experiments.py @@ -0,0 +1,20 @@ +# Databricks notebook source +import mlflow +mlflow.__version__ + +# COMMAND ---------- + +for exp in mlflow.search_experiments(): + eid = exp.experiment_id + try: + mlflow.delete_experiment(experiment_id=eid) + except: + print(f"Experiment {eid} not deleted") + +# COMMAND ---------- + +mlflow.search_experiments() + +# COMMAND ---------- + + diff --git a/mycode/delete-all-mlflow-registered-models.py b/mycode/delete-all-mlflow-registered-models.py new file mode 100644 index 00000000..0cac72e2 --- /dev/null +++ b/mycode/delete-all-mlflow-registered-models.py @@ -0,0 +1,27 @@ +# Databricks notebook source +from mlflow import MlflowClient + +# COMMAND ---------- + +dbutils.widgets.dropdown("mlflow_model_registry","",["", "unity_catalog","workspace"]) +model_registry = dbutils.widgets.get("mlflow_model_registry") + +client = MlflowClient(registry_uri="databricks-uc") if model_registry == "unity_catalog" else MlflowClient() + +# COMMAND ---------- + +for model in client.search_registered_models(): + model_name = model.name + if model_registry == "unity_catalog": + if not model_name.startswith("ds_nonprod.migrated_models"): + continue + else: + # transition all stages to 'Archived' in workspace registry + dbutils.notebook.run("./set-all-registered-model-stages-to-archived", -1, {"model_name": model_name}) + + print(f"Deleting model: {model_name}") + client.delete_registered_model(model_name) + +# COMMAND ---------- + + diff --git a/mycode/get-model-hash.py b/mycode/get-model-hash.py new file mode 100644 index 00000000..0aaabaf7 --- /dev/null +++ b/mycode/get-model-hash.py @@ -0,0 +1,86 @@ +# Databricks notebook source +# MAGIC %md +# MAGIC This notebook takes the name of a registered mlflow model and hashes its `champion` or `Production` version, depending on whether it's in Azure (Databricks Workspace) or AWS (Unity Catalog). +# MAGIC +# MAGIC 1. The directory of the `champion`/`Production` model is exported to the local machine with MLflow artifact utilities +# MAGIC 1. From this directory, hash the first file that ends with `pkl` + +# COMMAND ---------- + +# MAGIC %md ## Libs + +# COMMAND ---------- + +import mlflow +from mlflow import MlflowClient +from hashlib import md5 +import os + +# COMMAND ---------- + +# MAGIC %md ## Setup + +# COMMAND ---------- + +# DBTITLE 1,utils +def hash(path: str): + with open(path,"rb") as f: + return md5(f.read()).hexdigest() + +def hash_model_directory(model_dir): + try: + fname = [file for file in os.listdir(model_dir) if file.endswith("pkl")][0] + model_hash = hash(model_dir+fname) + result = f"{model_name}: {model_hash}" + except: + result = f"{model_name}: Model file not found" + return result + +# COMMAND ---------- + +# DBTITLE 1,variables +dbutils.widgets.text("registered-model-name","") +model_name = dbutils.widgets.get("registered-model-name") + +dbutils.widgets.dropdown("mlflow_model_registry", "workspace",["workspace", "unity_catalog"]) +model_registry = dbutils.widgets.get("mlflow_model_registry") + +# COMMAND ---------- + +# DBTITLE 1,dependent variables +if model_registry == "workspace": + mlflow.set_registry_uri("databricks") +elif model_registry == "unity_catalog": + mlflow.set_registry_uri("databricks-uc") +else: + raise Exception("Invalid model registry") + +if model_registry == "workspace": + client=mlflow.tracking.MlflowClient() + try: + version = client.get_latest_versions(model_name, stages=["Production"])[0].version + model_uri = f"models:/{model_name}/{version}" + except: + model_uri = "" +else: + model_uri = f"models:/{model_name}@champion" + +# COMMAND ---------- + +# MAGIC %md ## Execute + +# COMMAND ---------- + +try: + model_dir = mlflow.artifacts.download_artifacts(model_uri) + result = hash_model_directory(model_dir) +except: + result = f"{model_name}: Model not found" + +# COMMAND ---------- + +dbutils.notebook.exit(result) + +# COMMAND ---------- + + diff --git a/mycode/get-model-hashes.py b/mycode/get-model-hashes.py new file mode 100644 index 00000000..39fa7463 --- /dev/null +++ b/mycode/get-model-hashes.py @@ -0,0 +1,29 @@ +# Databricks notebook source +# DBTITLE 1,Libs +import mlflow + +# COMMAND ---------- + +# DBTITLE 1,variables +dbutils.widgets.dropdown("mlflow_model_registry", "workspace",["workspace", "unity_catalog"]) +model_registry = dbutils.widgets.get("mlflow_model_registry") + +# set up mlflow model registry uri +if model_registry == "workspace": + mlflow.set_registry_uri("databricks") +elif model_registry == "unity_catalog": + mlflow.set_registry_uri("databricks-uc") +else: + raise Exception("Invalid model registry") + +# COMMAND ---------- + +# DBTITLE 1,execute +for model in mlflow.search_registered_models(): + if (model_registry == "unity_catalog" and model.name.startswith("ds_nonprod.migrated_models")) or model_registry == "workspace": + result = dbutils.notebook.run("get-model-hash", -1, {"registered-model-name": model.name, "mlflow_model_registry": model_registry}) + print(result) + +# COMMAND ---------- + + diff --git a/mycode/manual-import-model.py b/mycode/manual-import-model.py new file mode 100644 index 00000000..ad5a5c13 --- /dev/null +++ b/mycode/manual-import-model.py @@ -0,0 +1,150 @@ +# Databricks notebook source +# MAGIC %md ### Manually import models that failed with the `import-models` utility +# MAGIC +# MAGIC You can find which models failed by searching the import_models log for "the source run_id was probably deleted" + +# COMMAND ---------- + +# MAGIC %md ## Libs + +# COMMAND ---------- + +# DBTITLE 1,install latest pkg from github +# MAGIC %sh +# MAGIC #pip install mlflow-export-import +# MAGIC pip install git+https:///github.com/mlflow/mlflow-export-import/#egg=mlflow-export-import + +# COMMAND ---------- + +# MAGIC %run ./credentials + +# COMMAND ---------- + +import os + +# COMMAND ---------- + +# MAGIC %md ## Setup + +# COMMAND ---------- + +# DBTITLE 1,variables +dbutils.widgets.dropdown("platform","",["", "azure", "aws"]) +platform = dbutils.widgets.get("platform") + +credentials_path = get_credentials_path(platform) + +# COMMAND ---------- + +# DBTITLE 1,set model env vars +wij = dbutils.widgets + +wij.text("input_dir","/dbfs/mnt/datalake/mlflow-migration-models/models") +input_dir = wij.get("input_dir") + +wij.text("model_name", "churn-model") +model_name = wij.get("model_name") + +# set env variables for shell command +os.environ["INPUT_DIR"]=input_dir +os.environ["MODEL_NAME"]=model_name + +# COMMAND ---------- + +# DBTITLE 1,set MLflow env vars +from datetime import datetime +import pytz + +cst = pytz.timezone('US/Central') +now = datetime.now(tz=cst) +date = now.strftime("%Y%m%d_%H%M") + +logfile = f"import_model_{model_name}_{date}.log" +os.environ["MLFLOW_EXPORT_IMPORT_LOG_OUTPUT_FILE"] = logfile + +os.environ["MLFLOW_EXPORT_IMPORT_LOG_FORMAT"]="%(threadName)s-%(levelname)s-%(message)s" + +os.environ["MLFLOW_TRACKING_URI"]="databricks" + +with open(credentials_path) as f: + os.environ["DATABRICKS_HOST"] = f.readline().strip("\n") + os.environ["DATABRICKS_TOKEN"] = f.readline().strip("\n") + +# COMMAND ---------- + +# DBTITLE 1,set experiment env vars +import json + +def get_most_recent_experiment_name(model: dict): + try: + newest = max(model["mlflow"]["registered_model"]["versions"], key=lambda v:v["last_updated_timestamp"]) + return newest["_experiment_name"] + except: + return None + +with open(f"{input_dir}/{model_name}/model.json") as f: + model = json.load(f) + os.environ["EXPERIMENT_NAME"] = get_most_recent_experiment_name(model).replace(" ","_") + +# COMMAND ---------- + +# DBTITLE 1,remove failed model registrations before import +import json +import os + +fname = f"{input_dir}/{model_name}/model.json" + +# read model.json +with open(fname, "r") as f: + d = json.load(f) + +# remove failed versions +for version in d["mlflow"]["registered_model"]["versions"]: + if version["status"] == 'FAILED_REGISTRATION': + d["mlflow"]["registered_model"]["versions"].remove(version) + +# rename old json +os.rename(fname, f"{fname}_old") + +# write clean json +with open(fname, "w") as f: + d = json.dump(d, f) + +# COMMAND ---------- + +# DBTITLE 1,verify env vars +# MAGIC %sh +# MAGIC echo $MLFLOW_EXPORT_IMPORT_LOG_OUTPUT_FILE +# MAGIC echo $MLFLOW_EXPORT_IMPORT_LOG_FORMAT +# MAGIC echo $MLFLOW_TRACKING_URI +# MAGIC echo $INPUT_DIR +# MAGIC echo $MODEL_NAME +# MAGIC echo $EXPERIMENT_NAME +# MAGIC #echo $DATABRICKS_HOST +# MAGIC #echo $DATABRICKS_TOKEN + +# COMMAND ---------- + +# MAGIC %md ## Execute + +# COMMAND ---------- + +# DBTITLE 0,`export-models` options +# MAGIC %sh +# MAGIC import-model --help + +# COMMAND ---------- + +# DBTITLE 1,cli execution +# MAGIC %sh +# MAGIC import-model \ +# MAGIC --input-dir $INPUT_DIR/$MODEL_NAME \ +# MAGIC --model $MODEL_NAME \ +# MAGIC --experiment-name $EXPERIMENT_NAME \ +# MAGIC --delete-model True \ +# MAGIC # --import-source-tags True \ +# MAGIC --verbose True + +# COMMAND ---------- + + diff --git a/mycode/models-by-owner.py b/mycode/models-by-owner.py new file mode 100644 index 00000000..5c69bb5f --- /dev/null +++ b/mycode/models-by-owner.py @@ -0,0 +1,27 @@ +# Databricks notebook source +import mlflow +import pyspark.sql.functions as F +import pandas as pd + +# COMMAND ---------- + +registered_models = mlflow.search_registered_models() + +model_name, user_id = [], [] +for model in registered_models: + model_name.append(model.name) + user_id.append(model.latest_versions[0].user_id) + +user_models = pd.DataFrame({'model': model_name, 'user': user_id}) + +# COMMAND ---------- + +display( + spark.createDataFrame(user_models) + .groupBy("user") + .agg(F.collect_set("model").alias("models")) + ) + +# COMMAND ---------- + + diff --git a/mycode/mount.py b/mycode/mount.py new file mode 100644 index 00000000..b55005a7 --- /dev/null +++ b/mycode/mount.py @@ -0,0 +1,25 @@ +# Databricks notebook source +# MAGIC %md +# MAGIC I added the AWS keys to Databricks secrets using the databricks cli. For Azure DEV, the profile name is **azure-dev**. +# MAGIC +# MAGIC Raj had to give me the AWS keys because I don't have permission to create AWS IAM users with S3 access. +# MAGIC +# MAGIC ### Reference +# MAGIC - https://docs.databricks.com/aws/en/dbfs/mounts#mount-a-bucket-using-aws-keys +# MAGIC - https://docs.databricks.com/aws/en/security/secrets#create-a-secret-scope +# MAGIC - https://docs.databricks.com/aws/en/security/secrets#create-a-secret + +# COMMAND ---------- + +# DBTITLE 1,s3 bucket in ds-non-prod +access_key = dbutils.secrets.get(scope = "aws", key = "aws-access-key") +secret_key = dbutils.secrets.get(scope = "aws", key = "aws-secret-key") +encoded_secret_key = secret_key.replace("/", "%2F") +aws_bucket_name = "s3-us-east-2-dev-ds-non-prod-s3-app-ovisg" +mount_name = "aws-ds-non-prod" + +dbutils.fs.mount(f"s3a://{access_key}:{encoded_secret_key}@{aws_bucket_name}", f"/mnt/{mount_name}") +display(dbutils.fs.ls(f"/mnt/{mount_name}")) + + + diff --git a/mycode/set-all-registered-model-stages-to-archived.py b/mycode/set-all-registered-model-stages-to-archived.py new file mode 100644 index 00000000..160a6780 --- /dev/null +++ b/mycode/set-all-registered-model-stages-to-archived.py @@ -0,0 +1,23 @@ +# Databricks notebook source +from mlflow import MlflowClient + +# COMMAND ---------- + +dbutils.widgets.text("model_name","") +model_name = dbutils.widgets.get("model_name") + +# COMMAND ---------- + +client = MlflowClient() + +for mv in client.search_model_versions(f"name='{model_name}'"): + version = int(mv.version) + if mv.current_stage != "Archived": + try: + client.transition_model_version_stage(name=model_name, version=version, stage="Archived") + except: + print(f"'{model_name}' version {version} state transition failed") + +# COMMAND ---------- + +