Skip to content

Enhance model saving with joblib and update dependencies #3789

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Gnanasundaramsampath-sys wants to merge 11 commits into
base: main
Choose a base branch
from
Open

Enhance model saving with joblib and update dependencies #3789

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
4c4cc4a
Enhance model saving with joblib and update dependencies
Gnanasundaramsampath-sys Mar 9, 2026
4117e9b
Update dependencies and environment configuration
Gnanasundaramsampath-sys Mar 9, 2026
983549d
Update model argument and environment in notebook
Gnanasundaramsampath-sys Mar 9, 2026
1e24636
Remove model output and command from quickstart notebook
Gnanasundaramsampath-sys Mar 9, 2026
6ce1549
Fix escape characters in quickstart notebook
Gnanasundaramsampath-sys Mar 9, 2026
3b6947f
Add model argument to parser in quickstart notebook
Gnanasundaramsampath-sys Mar 9, 2026
df12610
Fix formatting in quickstart.ipynb
Gnanasundaramsampath-sys Mar 9, 2026
90940d7
Fix output import in quickstart notebook
Gnanasundaramsampath-sys Mar 9, 2026
1a823fc
Fix import statement for Output in quickstart notebook
Gnanasundaramsampath-sys Mar 9, 2026
3843415
Fix output model type in quickstart notebook
Gnanasundaramsampath-sys Mar 9, 2026
26168a7
Fix formatting in quickstart notebook
Gnanasundaramsampath-sys Mar 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 38 additions & 33 deletions tutorials/get-started-notebooks/pipeline.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -234,10 +234,12 @@
" - scipy=1.7.1\n",
" - pandas>=1.1,<1.2\n",
" - pip:\n",
" - inference-schema[numpy-support]==1.3.0\n",
" - inference-schema[numpy-support]==1.8.0\n",
" - xlrd==2.0.1\n",
" - mlflow== 2.4.1\n",
" - azureml-mlflow==1.51.0"
" - mlflow==2.16.2\n",
" - azureml-mlflow==1.51.0\n",
" - azureml-ai-monitoring\n",
" - azureml-inference-server-http==1.3.4"
]
},
{
Expand Down Expand Up @@ -269,20 +271,26 @@
"from azure.ai.ml.entities import Environment\n",
"\n",
"custom_env_name = \"aml-scikit-learn\"\n",
"\n",
"pipeline_job_env = Environment(\n",
" name=custom_env_name,\n",
" description=\"Custom environment for Credit Card Defaults pipeline\",\n",
" tags={\"scikit-learn\": \"0.24.2\"},\n",
" conda_file=os.path.join(dependencies_dir, \"conda.yaml\"),\n",
" image=\"mcr.microsoft.com/azureml/openmpi4.1.0-ubuntu20.04:latest\",\n",
" version=\"0.2.0\",\n",
")\n",
"pipeline_job_env = ml_client.environments.create_or_update(pipeline_job_env)\n",
"\n",
"print(\n",
" f\"Environment with name {pipeline_job_env.name} is registered to workspace, the environment version is {pipeline_job_env.version}\"\n",
")"
"env_version = \"0.2.0\"\n",
"\n",
"# Check if the environment already exists\n",
"try:\n",
" pipeline_job_env = ml_client.environments.get(name=custom_env_name, version=env_version)\n",
" print(f\"Environment with name {pipeline_job_env.name} already exists, using existing version {pipeline_job_env.version}\")\n",
"except Exception:\n",
" # If it doesn't exist, create it\n",
" pipeline_job_env = Environment(\n",
" name=custom_env_name,\n",
" description=\"Custom environment for Credit Card Defaults pipeline\",\n",
" tags={\"scikit-learn\": \"0.24.2\"},\n",
" conda_file=os.path.join(dependencies_dir, \"conda.yaml\"),\n",
" image=\"mcr.microsoft.com/azureml/openmpi4.1.0-ubuntu20.04:latest\",\n",
" version=env_version,\n",
" )\n",
" pipeline_job_env = ml_client.environments.create_or_update(pipeline_job_env)\n",
" print(\n",
" f\"Environment with name {pipeline_job_env.name} is registered to workspace, the environment version is {pipeline_job_env.version}\"\n",
" )"
]
},
{
Expand Down Expand Up @@ -543,6 +551,7 @@
"import os\n",
"import pandas as pd\n",
"import mlflow\n",
"import joblib\n",
"\n",
"\n",
"def select_first_file(path):\n",
Expand All @@ -560,7 +569,7 @@
"mlflow.start_run()\n",
"\n",
"# enable autologging\n",
"mlflow.sklearn.autolog()\n",
"mlflow.sklearn.autolog(log_models=False)\n",
"\n",
"os.makedirs(\"./outputs\", exist_ok=True)\n",
"\n",
Expand Down Expand Up @@ -607,19 +616,15 @@
"\n",
" print(classification_report(y_test, y_pred))\n",
"\n",
" # Registering the model to the workspace\n",
" print(\"Registering the model via MLFlow\")\n",
" mlflow.sklearn.log_model(\n",
" sk_model=clf,\n",
" registered_model_name=args.registered_model_name,\n",
" artifact_path=args.registered_model_name,\n",
" )\n",
" print(\"Saving the model to the component output folder\")\n",
" os.makedirs(args.model, exist_ok=True)\n",
"\n",
" # Saving the model to a file\n",
" mlflow.sklearn.save_model(\n",
" sk_model=clf,\n",
" path=os.path.join(args.model, \"trained_model\"),\n",
" )\n",
" model_path = os.path.join(args.model, \"model.pkl\")\n",
" joblib.dump(clf, model_path)\n",
"\n",
" mlflow.log_artifact(model_path, artifact_path=\"model_output\")\n",
"\n",
" print(f\"Model saved to: {model_path}\")\n",
"\n",
" # Stop Logging\n",
" mlflow.end_run()\n",
Expand Down Expand Up @@ -924,9 +929,9 @@
"name": "python310-sdkv2"
},
"kernelspec": {
"display_name": "Python 3.10 - SDK v2",
"display_name": ".venv",
"language": "python",
"name": "python310-sdkv2"
"name": "python3"
},
"language_info": {
"codemirror_mode": {
Expand All @@ -938,7 +943,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.6"
"version": "3.9.5"
},
"nteract": {
"version": "nteract-front-end@1.0.0"
Expand Down
27 changes: 24 additions & 3 deletions tutorials/get-started-notebooks/quickstart.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -178,9 +178,11 @@
"import pandas as pd\n",
"import mlflow\n",
"import mlflow.sklearn\n",
"import joblib\n",
"from sklearn.ensemble import GradientBoostingClassifier\n",
"from sklearn.metrics import classification_report\n",
"from sklearn.model_selection import train_test_split\n",
"from azure.ai.ml.entities import Environment, ManagedOnlineDeployment\n",
"\n",
"def main():\n",
" \"\"\"Main function of the script.\"\"\"\n",
Expand All @@ -192,13 +194,14 @@
" parser.add_argument(\"--n_estimators\", required=False, default=100, type=int)\n",
" parser.add_argument(\"--learning_rate\", required=False, default=0.1, type=float)\n",
" parser.add_argument(\"--registered_model_name\", type=str, help=\"model name\")\n",
" parser.add_argument(\"--model\", type=str, default=\"./outputs\", help=\"output model directory\")\n",
" args = parser.parse_args()\n",
" \n",
" # Start Logging\n",
" mlflow.start_run()\n",
"\n",
" # enable autologging\n",
" mlflow.sklearn.autolog()\n",
" #mlflow.sklearn.autolog()\n",
"\n",
" ###################\n",
" #<prepare the data>\n",
Expand Down Expand Up @@ -264,12 +267,14 @@
" 'pip<=21.3.1',\n",
" {\n",
" 'pip': [\n",
" 'mlflow==2.17.0',\n",
" 'mlflow==2.16.2',\n",
" 'cloudpickle==2.2.1',\n",
" 'pandas==1.5.3',\n",
" 'psutil==5.8.0',\n",
" 'scikit-learn==1.5.2',\n",
" 'numpy==1.26.4',\n",
" 'azureml-ai-monitoring',\n",
" 'azureml-inference-server-http==1.3.4',\n",
" ]\n",
" }\n",
" ],\n",
Expand All @@ -281,12 +286,23 @@
" artifact_path=args.registered_model_name,\n",
" conda_env=conda_env,\n",
" )\n",
" # Save the model to the component output folder\n",
" print(\"Saving the model to the component output folder\")\n",
"\n",
" # Saving the model to a file\n",
" mlflow.sklearn.save_model(\n",
" sk_model=clf,\n",
" path=os.path.join(args.registered_model_name, \"trained_model\"),\n",
" )\n",
" os.makedirs(args.model, exist_ok=True)\n",
"\n",
" model_path = os.path.join(args.model, \"model.pkl\")\n",
" joblib.dump(clf, model_path)\n",
"\n",
" # Log the saved model as an artifact in MLflow\n",
" mlflow.log_artifact(model_path, artifact_path=\"model_output\")\n",
"\n",
" print(f\"Model saved to: {model_path}\")\n",
" ###########################\n",
" #</save and register model>\n",
" ###########################\n",
Expand Down Expand Up @@ -339,6 +355,7 @@
"source": [
"from azure.ai.ml import command\n",
"from azure.ai.ml import Input\n",
"from azure.ai.ml import Output\n",
"\n",
"registered_model_name = \"credit_defaults_model\"\n",
"\n",
Expand All @@ -352,6 +369,9 @@
" learning_rate=0.25,\n",
" registered_model_name=registered_model_name,\n",
" ),\n",
" outputs=dict(\n",
" model=Output(type=\"uri_folder\")\n",
" ),\n",
" code=\"./src/\", # location of source code\n",
" command=\"python main.py --data ${{inputs.data}} --test_train_ratio ${{inputs.test_train_ratio}} --learning_rate ${{inputs.learning_rate}} --registered_model_name ${{inputs.registered_model_name}}\",\n",
" environment=\"azureml://registries/azureml/environments/sklearn-1.5/labels/latest\",\n",
Expand Down Expand Up @@ -564,7 +584,8 @@
" name=\"blue\",\n",
" endpoint_name=online_endpoint_name,\n",
" model=model,\n",
" instance_type=\"Standard_DS3_v2\",\n",
" environment=\"azureml://registries/azureml/environments/sklearn-1.5/labels/latest\",\n",
" instance_type=\"Standard_DS2_v2\",\n",
" instance_count=1,\n",
")\n",
"\n",
Expand Down
Loading