|
| 1 | +# Copyright 2025 Google LLC |
| 2 | +# |
| 3 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | +# you may not use this file except in compliance with the License. |
| 5 | +# You may obtain a copy of the License at |
| 6 | +# |
| 7 | +# https://www.apache.org/licenses/LICENSE-2.0 |
| 8 | +# |
| 9 | +# Unless required by applicable law or agreed to in writing, software |
| 10 | +# distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | +# See the License for the specific language governing permissions and |
| 13 | +# limitations under the License. |
| 14 | +"""Initialization of Pathways-on-Cloud utilities.""" |
| 15 | + |
| 16 | +import datetime |
| 17 | +import logging |
| 18 | +import os |
| 19 | + |
| 20 | +import jax |
| 21 | +from pathwaysutils import profiling |
| 22 | +from pathwaysutils import proxy_backend |
| 23 | +from pathwaysutils.persistence import orbax_handler |
| 24 | + |
| 25 | + |
| 26 | +_logger = logging.getLogger(__name__) |
| 27 | +_initialization_count = 0 |
| 28 | + |
| 29 | + |
| 30 | +# This is a brittle implementation since the platforms value is not necessarily |
| 31 | +# which backend is ultimately selected |
| 32 | +def is_pathways_backend_used() -> bool: |
| 33 | + """Returns whether Pathways backend is used. |
| 34 | +
|
| 35 | + This function checks the JAX platforms configuration to determine whether |
| 36 | + Pathways is used. If the platforms configuration contains the string "proxy", |
| 37 | + Pathways is used. This is a brittle implementation since the platforms value |
| 38 | + is not necessarily which backend is ultimately selected or there may be more |
| 39 | + than one platform specified and another may have higher priority. |
| 40 | + """ |
| 41 | + return jax.config.jax_platforms and "proxy" in jax.config.jax_platforms |
| 42 | + |
| 43 | + |
| 44 | +def _is_persistence_enabled() -> bool: |
| 45 | + """Returns whether persistence is enabled. |
| 46 | +
|
| 47 | + This function checks the environment variable ENABLE_PATHWAYS_PERSISTENCE to |
| 48 | + determine whether persistence is enabled. If the variable is set to "1", |
| 49 | + persistence is enabled. If the variable is set to "0" or unset, persistence is |
| 50 | + disabled. |
| 51 | +
|
| 52 | + Returns: |
| 53 | + True if persistence is enabled, False otherwise. |
| 54 | + """ |
| 55 | + if "ENABLE_PATHWAYS_PERSISTENCE" in os.environ: |
| 56 | + if os.environ["ENABLE_PATHWAYS_PERSISTENCE"] == "1": |
| 57 | + return True |
| 58 | + if os.environ["ENABLE_PATHWAYS_PERSISTENCE"] == "0": |
| 59 | + return False |
| 60 | + else: |
| 61 | + raise ValueError( |
| 62 | + "ENABLE_PATHWAYS_PERSISTENCE must be set to 1/0 or unset, got: " |
| 63 | + + os.environ["ENABLE_PATHWAYS_PERSISTENCE"] |
| 64 | + ) |
| 65 | + return False |
| 66 | + |
| 67 | + |
| 68 | +def initialize() -> None: |
| 69 | + """Initializes pathwaysutils. |
| 70 | +
|
| 71 | + This function is called by the user to initialize pathwaysutils. It is |
| 72 | + responsible for setting up the logging, profiling, and persistence handlers |
| 73 | + through various monkey patching functions. It is also responsible for |
| 74 | + registering the proxy backend factory. |
| 75 | + """ |
| 76 | + global _initialization_count |
| 77 | + _initialization_count += 1 |
| 78 | + |
| 79 | + # Ignoring the second call to initialize() is a temporary measure so that this |
| 80 | + # debug log is not triggered for customers who are following our instructions |
| 81 | + # and using the new initialize() function only once but have already had the |
| 82 | + # legacy initialization triggered. |
| 83 | + if _initialization_count > 1: |
| 84 | + _logger.debug("Already initialized. Ignoring duplicate call.") |
| 85 | + return |
| 86 | + |
| 87 | + _logger.debug("Starting initialize.") |
| 88 | + |
| 89 | + if is_pathways_backend_used(): |
| 90 | + _logger.debug("Detected Pathways-on-Cloud backend. Applying changes.") |
| 91 | + proxy_backend.register_backend_factory() |
| 92 | + profiling.monkey_patch_jax() |
| 93 | + # TODO: b/365549911 - Remove when OCDBT-compatible |
| 94 | + if _is_persistence_enabled(): |
| 95 | + orbax_handler.register_pathways_handlers(datetime.timedelta(hours=1)) |
| 96 | + |
| 97 | + # Turn off JAX compilation cache because Pathways handles its own |
| 98 | + # compilation cache. |
| 99 | + jax.config.update("jax_enable_compilation_cache", False) |
| 100 | + |
| 101 | + else: |
| 102 | + _logger.debug( |
| 103 | + "Did not detect Pathways-on-Cloud backend. No changes applied." |
| 104 | + ) |
0 commit comments