Update dependencies to PyTorch 2.9.0 and Lightning 2.6, adjust related configurations and documentation

speediedan · speediedan · commit 35dc7c4e15d8 · 2025-08-15T12:12:07.000-07:00
diff --git a/.azure-pipelines/gpu-tests.yml b/.azure-pipelines/gpu-tests.yml
@@ -46,7 +46,7 @@ jobs:
     strategy:
       matrix:
         'PyTorch | latest':
-          image: "speediedan/finetuning-scheduler:py3.12-pt2.8.0-pl2.8-azpl-init"
+          image: "speediedan/finetuning-scheduler:py3.12-pt2.9.0-pl2.6-azpl-init"
           scope: ""
     # how long to run the job before automatically cancelling
     timeoutInMinutes: "100"
diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -39,9 +39,9 @@ python collect_env_details.py
 You can also fill out the list below manually.
 -->
 
-- Fine-Tuning Scheduler Version (e.g., 2.8.0):
-- Lightning Version (e.g., 2.8.0):
-- PyTorch Version (e.g., 2.8.0):
+- Fine-Tuning Scheduler Version (e.g., 2.9.0):
+- Lightning Version (e.g., 2.6.0):
+- PyTorch Version (e.g., 2.9.0):
 - Python version (e.g., 3.12):
 - OS (e.g., Linux):
 - CUDA/cuDNN version:
diff --git a/.github/workflows/release-docker.yml b/.github/workflows/release-docker.yml
@@ -31,9 +31,9 @@ jobs:
       matrix:
         # initially building only the latest supported configuration
         python_version: ["3.12"]
-        pytorch_version: ["2.8.0"]
+        pytorch_version: ["2.9.0"]
         cust_base: ["cu12.8.1-"]
-        pl_version: ["2.8"]
+        pl_version: ["2.6"]
     steps:
       - name: Checkout
         uses: actions/checkout@v4
diff --git a/CITATION.cff b/CITATION.cff
@@ -6,7 +6,7 @@ date-released: 2022-02-04
 authors:
   - family-names: "Dale"
     given-names: "Dan"
-version: 2.8.0
+version: 2.9.0
 identifiers:
   - description: "Fine-Tuning Scheduler (all versions)"
     type: doi
diff --git a/README.md b/README.md
@@ -173,7 +173,7 @@ To ensure maximum stability, the latest Lightning patch release fully tested wit
 <details>
   <summary>Current build statuses for Fine-Tuning Scheduler </summary>
 
-| System / (PyTorch/Python ver) |                                                                                                        2.3.1/3.9                                                                                                         |                                                                                                              2.8.0/3.9, 2.8.0/3.12                                                                                                               |
+| System / (PyTorch/Python ver) |                                                                                                        2.3.1/3.9                                                                                                         |                                                                                                              2.9.0/3.9, 2.9.0/3.12                                                                                                               |
 | :---------------------------: | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: |
 |      Linux \[GPUs\*\*\]       |                                                                                                            -                                                                                                             | [![Build Status](https://dev.azure.com//speediedan/finetuning-scheduler/_apis/build/status/Multi-GPU%20&%20Example%20Tests?branchName=main)](https://dev.azure.com/speediedan/finetuning-scheduler/_build/latest?definitionId=1&branchName=main) |
 |     Linux (Ubuntu 22.04)      | [![Test](https://github.com/speediedan/finetuning-scheduler/actions/workflows/ci_test-full.yml/badge.svg?branch=main&event=push)](https://github.com/speediedan/finetuning-scheduler/actions/workflows/ci_test-full.yml) |             [![Test](https://github.com/speediedan/finetuning-scheduler/actions/workflows/ci_test-full.yml/badge.svg?branch=main&event=push)](https://github.com/speediedan/finetuning-scheduler/actions/workflows/ci_test-full.yml)             |
diff --git a/dockers/base-cuda/Dockerfile b/dockers/base-cuda/Dockerfile
@@ -17,7 +17,7 @@ ARG OS_VER=ubuntu22.04
 FROM nvidia/cuda:${CUDA_VERSION}-devel-${OS_VER}
 
 ARG PYTHON_VERSION=3.12
-ARG PYTORCH_VERSION=2.8.0
+ARG PYTORCH_VERSION=2.9.0
 ARG CUST_BUILD=0
 ARG MKL_THREADING_LAYER=GNU
 
@@ -91,9 +91,9 @@ RUN \
         # ... pytorch patch version
         # pip install torch==1.11.1+cu113 torchvision==0.11.3+cu113 -f https://download.pytorch.org/whl/cu113/torch_stable.html; \
         # ... pytorch nightly dev version
-        pip install --pre torch==2.9.0.dev20250815 --index-url https://download.pytorch.org/whl/nightly/cu128; \
+        pip install --pre torch==2.9.0.dev20250811 --index-url https://download.pytorch.org/whl/nightly/cu128; \
         # ... test channel
-        # pip install --pre torch==2.8.0 --index-url https://download.pytorch.org/whl/test/cu128; \
+        # pip install --pre torch==2.9.0 --index-url https://download.pytorch.org/whl/test/cu128; \
     fi && \
     # We avoid installing Lightning and other dependencies here as they are usually upgraded anyway later in
     # CI but we may re-enable in the future.
diff --git a/dockers/docker_images_main.sh b/dockers/docker_images_main.sh
@@ -43,7 +43,7 @@ maybe_build(){
 build_eval(){
 	# latest PyTorch image supported by release
 	# see CUDA_ARCHES_FULL_VERSION for the full version of the pytorch-provided toolkit
-	declare -A iv=(["cuda"]="12.8.1" ["python"]="3.12" ["pytorch"]="2.8.0" ["lightning"]="2.8" ["cust_build"]="1")
+	declare -A iv=(["cuda"]="12.8.1" ["python"]="3.12" ["pytorch"]="2.9.0" ["lightning"]="2.6" ["cust_build"]="1")
 	export latest_pt="base-cu${iv["cuda"]}-py${iv["python"]}-pt${iv["pytorch"]}-pl${iv["lightning"]}"
 	export latest_azpl="py${iv["python"]}-pt${iv["pytorch"]}-pl${iv["lightning"]}-azpl-init"
 	maybe_build iv "${latest_pt}" "${latest_azpl}"
diff --git a/dockers/docker_images_release.sh b/dockers/docker_images_release.sh
@@ -41,7 +41,7 @@ maybe_build(){
 
 build_eval(){
 	# latest PyTorch image supported by release
-	declare -A iv=(["cuda"]="12.8.1" ["python"]="3.12" ["pytorch"]="2.8.0" ["lightning"]="2.8" ["cust_build"]="0")
+	declare -A iv=(["cuda"]="12.8.1" ["python"]="3.12" ["pytorch"]="2.8.0" ["lightning"]="2.5" ["cust_build"]="0")
 	export latest_pt="base-cu${iv["cuda"]}-py${iv["python"]}-pt${iv["pytorch"]}-pl${iv["lightning"]}"
 	export latest_azpl="py${iv["python"]}-pt${iv["pytorch"]}-pl${iv["lightning"]}-azpl-init"
 	maybe_build iv "${latest_pt}" "${latest_azpl}"
diff --git a/dockers/fts-az-base/Dockerfile b/dockers/fts-az-base/Dockerfile
@@ -11,8 +11,8 @@
 # limitations under the License.
 
 ARG PYTHON_VERSION=3.12
-ARG PYTORCH_VERSION=2.8.0
-ARG LIGHTNING_VERSION=2.8
+ARG PYTORCH_VERSION=2.9.0
+ARG LIGHTNING_VERSION=2.6
 ARG CUST_BASE
 
 FROM speediedan/finetuning-scheduler:base-${CUST_BASE}py${PYTHON_VERSION}-pt${PYTORCH_VERSION}-pl${LIGHTNING_VERSION}
diff --git a/dockers/release/Dockerfile b/dockers/release/Dockerfile
@@ -11,8 +11,8 @@
 # limitations under the License.
 
 ARG PYTHON_VERSION=3.12
-ARG PYTORCH_VERSION=2.8.0
-ARG LIGHTNING_VERSION=2.8
+ARG PYTORCH_VERSION=2.9.0
+ARG LIGHTNING_VERSION=2.6
 ARG CUST_BASE
 
 FROM speediedan/finetuning-scheduler:base-${CUST_BASE}py${PYTHON_VERSION}-pt${PYTORCH_VERSION}-pl${LIGHTNING_VERSION}
diff --git a/requirements/lightning_pin.txt b/requirements/lightning_pin.txt
@@ -1 +1 @@
-947f9391656198b7bd0808e59a5e1756d36cc386
+8d1a73475e0f11ed3b3c5907da54fc993717d492
diff --git a/scripts/build_fts_env.sh b/scripts/build_fts_env.sh
@@ -98,19 +98,19 @@ base_env_build(){
                 if [[ -n ${torchvision_dev_ver} ]]; then
                     torchvision_dev_ver=${torch_dev_ver}
                 fi
-                pip install ${pip_install_flags} --pre torch==2.9.0.${torch_dev_ver} --index-url https://download.pytorch.org/whl/nightly/cu129
+                pip install ${pip_install_flags} --pre torch==2.9.0.${torch_dev_ver} --index-url https://download.pytorch.org/whl/nightly/cu128
             elif [[ $torch_test_channel -eq 1 ]]; then
-                pip install ${pip_install_flags} --pre torch==2.8.0 --index-url https://download.pytorch.org/whl/test/cu128
+                pip install ${pip_install_flags} --pre torch==2.9.0 --index-url https://download.pytorch.org/whl/test/cu128
             else
                 pip install ${pip_install_flags} torch torchvision --index-url https://download.pytorch.org/whl/cu128
             fi
             ;;
         fts_release)
             clear_activate_env python3.12
             if [[ $torch_test_channel -eq 1 ]]; then
-                pip install ${pip_install_flags} --pre torch torchvision --index-url https://download.pytorch.org/whl/test/cu126
+                pip install ${pip_install_flags} --pre torch torchvision --index-url https://download.pytorch.org/whl/test/cu128
             else
-                pip install ${pip_install_flags} torch torchvision --index-url https://download.pytorch.org/whl/cu126
+                pip install ${pip_install_flags} torch torchvision --index-url https://download.pytorch.org/whl/cu128
             fi
             ;;
         fts_latest_pt_oldest | fts_latest_pt2_5_x | fts_release_pt2_5_x)
diff --git a/src/finetuning_scheduler/__about__.py b/src/finetuning_scheduler/__about__.py
@@ -1,7 +1,7 @@
 import time
 
 _this_year = time.strftime("%Y")
-__version__ = "2.8.0.dev0"
+__version__ = "2.9.0.dev0"
 __author__ = "Dan Dale"
 __author_email__ = "danny.dale@gmail.com"
 __license__ = "Apache-2.0"
diff --git a/src/fts_examples/cli_experiment_utils.py b/src/fts_examples/cli_experiment_utils.py
@@ -9,7 +9,6 @@
 from torch.optim import Optimizer
 from torch.utils.data import Dataset, DataLoader
 import lightning as L
-from lightning.fabric.accelerators.cuda import is_cuda_available
 from lightning.fabric.utilities import rank_zero_warn
 from lightning.pytorch.cli import LightningCLI
 from lightning.pytorch.utilities.exceptions import MisconfigurationException
@@ -87,22 +86,34 @@ def get_env_info():
     run_lambda = collect_env.run
     pip_version, pip_list_output = get_pip_packages(run_lambda)
 
-    if collect_env.TORCH_AVAILABLE:
-        version_str = torch.__version__
-        debug_mode_str = str(torch.version.debug)
-        cuda_available_str = str(is_cuda_available())
-        cuda_version_str = torch.version.cuda
-        if not hasattr(torch.version, "hip") or torch.version.hip is None:  # cuda version
-            hip_compiled_version = hip_runtime_version = miopen_runtime_version = "N/A"
-        else:  # HIP version
-            cfg = torch._C._show_config().split("\n")
-            hip_runtime_version = [s.rsplit(None, 1)[-1] for s in cfg if "HIP Runtime" in s][0]
-            miopen_runtime_version = [s.rsplit(None, 1)[-1] for s in cfg if "MIOpen" in s][0]
-            cuda_version_str = "N/A"
-            hip_compiled_version = torch.version.hip
-    else:
-        version_str = debug_mode_str = cuda_available_str = cuda_version_str = "N/A"
+    version_str = torch.__version__
+    debug_mode_str = str(torch.version.debug)
+    cuda_available_str = str(torch.cuda.is_available())
+    cuda_version_str = torch.version.cuda
+    xpu_available_str = str(torch.xpu.is_available())
+    if torch.xpu.is_available():
+        xpu_available_str = (
+            f"{xpu_available_str}\n"
+            + f"XPU used to build PyTorch: {torch.version.xpu}\n"
+            + f"Intel GPU driver version:\n{collect_env.get_intel_gpu_driver_version(run_lambda)}\n"
+            + f"Intel GPU models onboard:\n{collect_env.get_intel_gpu_onboard(run_lambda)}\n"
+            + f"Intel GPU models detected:\n{collect_env.get_intel_gpu_detected(run_lambda)}"
+        )
+    if (
+        not hasattr(torch.version, "hip") or torch.version.hip is None
+    ):  # cuda version
         hip_compiled_version = hip_runtime_version = miopen_runtime_version = "N/A"
+    else:  # HIP version
+
+        def get_version_or_na(cfg, prefix):
+            _lst = [s.rsplit(None, 1)[-1] for s in cfg if prefix in s]
+            return _lst[0] if _lst else "N/A"
+
+        cfg = torch._C._show_config().split("\n")
+        hip_runtime_version = get_version_or_na(cfg, "HIP Runtime")
+        miopen_runtime_version = get_version_or_na(cfg, "MIOpen")
+        cuda_version_str = "N/A"
+        hip_compiled_version = torch.version.hip
 
     sys_version = sys.version.replace("\n", " ")
 
@@ -130,6 +141,7 @@ def get_env_info():
         "cmake_version": collect_env.get_cmake_version(run_lambda),
         "caching_allocator_config": collect_env.get_cachingallocator_config(),
         "is_xnnpack_available": collect_env.is_xnnpack_available(),
+        "is_xpu_available": xpu_available_str,
         "cpu_info": collect_env.get_cpu_info(run_lambda),
     }
     # get_cuda_module_loading_config() initializes CUDA which we want to avoid so we bypass this inspection
diff --git a/tests/helpers/expected_warns.py b/tests/helpers/expected_warns.py
@@ -22,6 +22,7 @@
     "`max_epochs` was not",
     "The dirpath has changed from",
     "Conversion of an array with ndim > 0",  # still needed with python 3.9 and torch 2.4.0
+    "Please use the new API settings to control TF32 behavior",  # TODO: temporarily required with 20250811 nightly
     WORKER_WARNING,
 ]
 
