diff --git a/kubeflow/trainer/backends/container/utils.py b/kubeflow/trainer/backends/container/utils.py
index 097762cbe..f7a5a468a 100644
--- a/kubeflow/trainer/backends/container/utils.py
+++ b/kubeflow/trainer/backends/container/utils.py
@@ -104,7 +104,7 @@ def build_pip_install_cmd(trainer: types.CustomTrainer) -> str:
     extras = " ".join(f"--extra-index-url {u}" for u in index_urls[1:])
     quoted = " ".join(f'"{p}"' for p in pkgs)
     return (
-        "PIP_DISABLE_PIP_VERSION_CHECK=1 pip install --no-warn-script-location "
+        "PIP_DISABLE_PIP_VERSION_CHECK=1 PIP_BREAK_SYSTEM_PACKAGES=1 pip install --no-warn-script-location "
         f"--index-url {main_idx} {extras} {quoted} && "
     )
 
diff --git a/kubeflow/trainer/backends/kubernetes/utils.py b/kubeflow/trainer/backends/kubernetes/utils.py
index c35674a77..a3dad97c7 100644
--- a/kubeflow/trainer/backends/kubernetes/utils.py
+++ b/kubeflow/trainer/backends/kubernetes/utils.py
@@ -294,10 +294,10 @@ def get_script_for_python_packages(
         LOG_FILE="{install_log_file}"
         rm -f "$LOG_FILE"
 
-        if PIP_DISABLE_PIP_VERSION_CHECK=1 python -m pip install --quiet \\
+        if PIP_DISABLE_PIP_VERSION_CHECK=1 PIP_BREAK_SYSTEM_PACKAGES=1 python -m pip install --quiet \\
             --no-warn-script-location $PIP_OPTS --user $PACKAGES >"$LOG_FILE" 2>&1; then
             echo "Successfully installed Python packages: $PACKAGES"
-        elif PIP_DISABLE_PIP_VERSION_CHECK=1 python -m pip install --quiet \\
+        elif PIP_DISABLE_PIP_VERSION_CHECK=1 PIP_BREAK_SYSTEM_PACKAGES=1 python -m pip install --quiet \\
             --no-warn-script-location $PIP_OPTS $PACKAGES >>"$LOG_FILE" 2>&1; then
             echo "Successfully installed Python packages: $PACKAGES"
         else
diff --git a/kubeflow/trainer/backends/kubernetes/utils_test.py b/kubeflow/trainer/backends/kubernetes/utils_test.py
index a4c483d1c..6166e520a 100644
--- a/kubeflow/trainer/backends/kubernetes/utils_test.py
+++ b/kubeflow/trainer/backends/kubernetes/utils_test.py
@@ -209,10 +209,10 @@ def test_get_resources_per_node(test_case: TestCase):
                 'LOG_FILE="pip_install.log"\n'
                 'rm -f "$LOG_FILE"\n'
                 "\n"
-                "if PIP_DISABLE_PIP_VERSION_CHECK=1 python -m pip install --quiet \\\n"
+                "if PIP_DISABLE_PIP_VERSION_CHECK=1 PIP_BREAK_SYSTEM_PACKAGES=1 python -m pip install --quiet \\\n"
                 '    --no-warn-script-location $PIP_OPTS --user $PACKAGES >"$LOG_FILE" 2>&1; then\n'
                 '    echo "Successfully installed Python packages: $PACKAGES"\n'
-                "elif PIP_DISABLE_PIP_VERSION_CHECK=1 python -m pip install --quiet \\\n"
+                "elif PIP_DISABLE_PIP_VERSION_CHECK=1 PIP_BREAK_SYSTEM_PACKAGES=1 python -m pip install --quiet \\\n"
                 '    --no-warn-script-location $PIP_OPTS $PACKAGES >>"$LOG_FILE" 2>&1; then\n'
                 '    echo "Successfully installed Python packages: $PACKAGES"\n'
                 "else\n"
@@ -239,10 +239,10 @@ def test_get_resources_per_node(test_case: TestCase):
                 'LOG_FILE="pip_install.log"\n'
                 'rm -f "$LOG_FILE"\n'
                 "\n"
-                "if PIP_DISABLE_PIP_VERSION_CHECK=1 python -m pip install --quiet \\\n"
+                "if PIP_DISABLE_PIP_VERSION_CHECK=1 PIP_BREAK_SYSTEM_PACKAGES=1 python -m pip install --quiet \\\n"
                 '    --no-warn-script-location $PIP_OPTS --user $PACKAGES >"$LOG_FILE" 2>&1; then\n'
                 '    echo "Successfully installed Python packages: $PACKAGES"\n'
-                "elif PIP_DISABLE_PIP_VERSION_CHECK=1 python -m pip install --quiet \\\n"
+                "elif PIP_DISABLE_PIP_VERSION_CHECK=1 PIP_BREAK_SYSTEM_PACKAGES=1 python -m pip install --quiet \\\n"
                 '    --no-warn-script-location $PIP_OPTS $PACKAGES >>"$LOG_FILE" 2>&1; then\n'
                 '    echo "Successfully installed Python packages: $PACKAGES"\n'
                 "else\n"
@@ -273,10 +273,10 @@ def test_get_resources_per_node(test_case: TestCase):
                 'LOG_FILE="pip_install.log"\n'
                 'rm -f "$LOG_FILE"\n'
                 "\n"
-                "if PIP_DISABLE_PIP_VERSION_CHECK=1 python -m pip install --quiet \\\n"
+                "if PIP_DISABLE_PIP_VERSION_CHECK=1 PIP_BREAK_SYSTEM_PACKAGES=1 python -m pip install --quiet \\\n"
                 '    --no-warn-script-location $PIP_OPTS --user $PACKAGES >"$LOG_FILE" 2>&1; then\n'
                 '    echo "Successfully installed Python packages: $PACKAGES"\n'
-                "elif PIP_DISABLE_PIP_VERSION_CHECK=1 python -m pip install --quiet \\\n"
+                "elif PIP_DISABLE_PIP_VERSION_CHECK=1 PIP_BREAK_SYSTEM_PACKAGES=1 python -m pip install --quiet \\\n"
                 '    --no-warn-script-location $PIP_OPTS $PACKAGES >>"$LOG_FILE" 2>&1; then\n'
                 '    echo "Successfully installed Python packages: $PACKAGES"\n'
                 "else\n"
@@ -303,10 +303,10 @@ def test_get_resources_per_node(test_case: TestCase):
                 'LOG_FILE="pip_install.log"\n'
                 'rm -f "$LOG_FILE"\n'
                 "\n"
-                "if PIP_DISABLE_PIP_VERSION_CHECK=1 python -m pip install --quiet \\\n"
+                "if PIP_DISABLE_PIP_VERSION_CHECK=1 PIP_BREAK_SYSTEM_PACKAGES=1 python -m pip install --quiet \\\n"
                 '    --no-warn-script-location $PIP_OPTS --user $PACKAGES >"$LOG_FILE" 2>&1; then\n'
                 '    echo "Successfully installed Python packages: $PACKAGES"\n'
-                "elif PIP_DISABLE_PIP_VERSION_CHECK=1 python -m pip install --quiet \\\n"
+                "elif PIP_DISABLE_PIP_VERSION_CHECK=1 PIP_BREAK_SYSTEM_PACKAGES=1 python -m pip install --quiet \\\n"
                 '    --no-warn-script-location $PIP_OPTS $PACKAGES >>"$LOG_FILE" 2>&1; then\n'
                 '    echo "Successfully installed Python packages: $PACKAGES"\n'
                 "else\n"
@@ -457,10 +457,10 @@ def test_get_script_for_python_packages(test_case):
                     'LOG_FILE="pip_install.log"\n'
                     'rm -f "$LOG_FILE"\n'
                     "\n"
-                    "if PIP_DISABLE_PIP_VERSION_CHECK=1 python -m pip install --quiet \\\n"
+                    "if PIP_DISABLE_PIP_VERSION_CHECK=1 PIP_BREAK_SYSTEM_PACKAGES=1 python -m pip install --quiet \\\n"
                     '    --no-warn-script-location $PIP_OPTS --user $PACKAGES >"$LOG_FILE" 2>&1; then\n'
                     '    echo "Successfully installed Python packages: $PACKAGES"\n'
-                    "elif PIP_DISABLE_PIP_VERSION_CHECK=1 python -m pip install --quiet \\\n"
+                    "elif PIP_DISABLE_PIP_VERSION_CHECK=1 PIP_BREAK_SYSTEM_PACKAGES=1 python -m pip install --quiet \\\n"
                     '    --no-warn-script-location $PIP_OPTS $PACKAGES >>"$LOG_FILE" 2>&1; then\n'
                     '    echo "Successfully installed Python packages: $PACKAGES"\n'
                     "else\n"