|
| 1 | +# SPDX-License-Identifier: LGPL-3.0-or-later |
| 2 | +"""Backend-independent resolution of the training-step schedule. |
| 3 | +
|
| 4 | +A run length is expressed either directly in optimizer steps |
| 5 | +(``training.numb_steps``) or as a number of passes over the training data |
| 6 | +(``training.numb_epoch`` for single-task runs, ``training.num_epoch_dict`` |
| 7 | +for multi-task runs). Converting epochs into steps requires exactly one |
| 8 | +backend-specific quantity, the epoch length of a task, which callers supply |
| 9 | +through the ``epoch_length`` callback. Validation of the mutually exclusive |
| 10 | +options, the multi-task step split and the resulting task sampling weights |
| 11 | +are shared by every backend. |
| 12 | +""" |
| 13 | + |
| 14 | +from __future__ import ( |
| 15 | + annotations, |
| 16 | +) |
| 17 | + |
| 18 | +import logging |
| 19 | +from dataclasses import ( |
| 20 | + dataclass, |
| 21 | +) |
| 22 | +from typing import ( |
| 23 | + TYPE_CHECKING, |
| 24 | + Any, |
| 25 | +) |
| 26 | + |
| 27 | +import numpy as np |
| 28 | + |
| 29 | +from deepmd.dpmodel.utils.training_utils import ( |
| 30 | + resolve_model_prob, |
| 31 | + resolve_model_prob_from_epochs, |
| 32 | +) |
| 33 | + |
| 34 | +if TYPE_CHECKING: |
| 35 | + from collections.abc import ( |
| 36 | + Callable, |
| 37 | + Mapping, |
| 38 | + Sequence, |
| 39 | + ) |
| 40 | + |
| 41 | +log = logging.getLogger(__name__) |
| 42 | + |
| 43 | +__all__ = ["StepSchedule", "resolve_step_schedule"] |
| 44 | + |
| 45 | + |
| 46 | +@dataclass(frozen=True) |
| 47 | +class StepSchedule: |
| 48 | + """Resolved run length and task sampling weights. |
| 49 | +
|
| 50 | + Attributes |
| 51 | + ---------- |
| 52 | + num_steps : int |
| 53 | + Total number of optimizer steps of the run. |
| 54 | + model_prob : np.ndarray | None |
| 55 | + Probability of selecting each task at a training step, with shape |
| 56 | + ``(ntasks,)`` and ordered like the ``model_keys`` passed to |
| 57 | + :func:`resolve_step_schedule`. ``None`` for single-task runs. |
| 58 | + """ |
| 59 | + |
| 60 | + num_steps: int |
| 61 | + model_prob: np.ndarray | None = None |
| 62 | + |
| 63 | + |
| 64 | +def resolve_step_schedule( |
| 65 | + training_params: Mapping[str, Any], |
| 66 | + *, |
| 67 | + multi_task: bool, |
| 68 | + model_keys: Sequence[str], |
| 69 | + training_data: Mapping[str, Any], |
| 70 | + epoch_length: Callable[[str], int], |
| 71 | + broadcast: Callable[[list[int]], Sequence[int]] | None = None, |
| 72 | + rank: int = 0, |
| 73 | +) -> StepSchedule: |
| 74 | + """Resolve the run length and the task sampling weights. |
| 75 | +
|
| 76 | + Parameters |
| 77 | + ---------- |
| 78 | + training_params : Mapping[str, Any] |
| 79 | + The normalized ``training`` section. ``numb_steps``, ``numb_epoch``, |
| 80 | + ``num_epoch_dict`` and ``model_prob`` are read from it. |
| 81 | + multi_task : bool |
| 82 | + Whether the run trains several task branches. Single-task runs accept |
| 83 | + ``numb_epoch``, multi-task runs accept ``num_epoch_dict``. |
| 84 | + model_keys : Sequence[str] |
| 85 | + Task keys in the order used by the trainer. Single-task runs pass the |
| 86 | + one key under which their data is registered. |
| 87 | + training_data : Mapping[str, Any] |
| 88 | + Training data of every task, keyed like ``model_keys``. Only consulted |
| 89 | + when multi-task sampling weights fall back to the per-task data size. |
| 90 | + epoch_length : Callable[[str], int] |
| 91 | + Maps a task key to the number of steps *this rank* performs during one |
| 92 | + epoch of that task. Backends whose data pipeline is sharded across |
| 93 | + ranks report their per-rank batch count directly; backends that |
| 94 | + replicate the dataset on every rank divide the dataset-wide batch count |
| 95 | + by the world size, so that an epoch always denotes one pass over the |
| 96 | + whole dataset. |
| 97 | + broadcast : Callable[[list[int]], Sequence[int]], optional |
| 98 | + Replaces the epoch lengths by rank 0's values. Epoch lengths follow |
| 99 | + from floating-point sampling weights and may otherwise differ by one |
| 100 | + unit across ranks, which would desynchronize the run length and |
| 101 | + deadlock later collective calls. |
| 102 | + rank : int, optional |
| 103 | + Process rank, used to restrict informational logging to the chief. |
| 104 | +
|
| 105 | + Returns |
| 106 | + ------- |
| 107 | + StepSchedule |
| 108 | + The resolved run length and, for multi-task runs, the task sampling |
| 109 | + weights. |
| 110 | +
|
| 111 | + Raises |
| 112 | + ------ |
| 113 | + ValueError |
| 114 | + If the run length is unspecified, over-specified, or non-positive. |
| 115 | + """ |
| 116 | + keys = list(model_keys) |
| 117 | + num_steps = training_params.get("numb_steps") |
| 118 | + num_epoch = training_params.get("numb_epoch") |
| 119 | + num_epoch_dict = training_params.get("num_epoch_dict") or {} |
| 120 | + |
| 121 | + if not multi_task: |
| 122 | + if num_steps is not None and num_epoch is not None: |
| 123 | + raise ValueError( |
| 124 | + "training.numb_steps and training.num_epoch are mutually exclusive." |
| 125 | + ) |
| 126 | + if num_steps is not None: |
| 127 | + return StepSchedule(num_steps=int(num_steps)) |
| 128 | + if num_epoch is None: |
| 129 | + raise ValueError( |
| 130 | + "Either training.numb_steps or training.num_epoch must be set." |
| 131 | + ) |
| 132 | + num_epoch = float(num_epoch) |
| 133 | + if num_epoch <= 0.0: |
| 134 | + raise ValueError("training.num_epoch must be positive.") |
| 135 | + (total_numb_batch,) = _epoch_lengths(keys, epoch_length, broadcast) |
| 136 | + steps = int(np.ceil(num_epoch * total_numb_batch)) |
| 137 | + if rank == 0: |
| 138 | + log.info( |
| 139 | + "Computed num_steps=%d from num_epoch=%s and total_numb_batch=%d.", |
| 140 | + steps, |
| 141 | + num_epoch, |
| 142 | + total_numb_batch, |
| 143 | + ) |
| 144 | + return StepSchedule(num_steps=steps) |
| 145 | + |
| 146 | + if num_epoch_dict: |
| 147 | + if num_steps is not None: |
| 148 | + raise ValueError( |
| 149 | + "training.numb_steps and training.num_epoch_dict " |
| 150 | + "are mutually exclusive." |
| 151 | + ) |
| 152 | + per_task_total = _epoch_lengths(keys, epoch_length, broadcast) |
| 153 | + model_prob, steps, per_task_steps = resolve_model_prob_from_epochs( |
| 154 | + keys, |
| 155 | + num_epoch_dict, |
| 156 | + np.asarray(per_task_total, dtype=np.float64), |
| 157 | + ) |
| 158 | + if rank == 0: |
| 159 | + log.info( |
| 160 | + "Computed model_prob=%s and num_steps=%d from num_epoch_dict=%s " |
| 161 | + "with per-task target steps: %s.", |
| 162 | + model_prob, |
| 163 | + steps, |
| 164 | + dict(num_epoch_dict), |
| 165 | + {k: int(np.ceil(v)) for k, v in per_task_steps.items()}, |
| 166 | + ) |
| 167 | + return StepSchedule(num_steps=steps, model_prob=model_prob) |
| 168 | + |
| 169 | + if num_steps is None: |
| 170 | + raise ValueError( |
| 171 | + "Either training.numb_steps (multi-task only) or " |
| 172 | + "training.num_epoch_dict must be set." |
| 173 | + ) |
| 174 | + model_prob = resolve_model_prob( |
| 175 | + keys, |
| 176 | + training_params.get("model_prob"), |
| 177 | + training_data, |
| 178 | + rank=rank, |
| 179 | + ) |
| 180 | + return StepSchedule(num_steps=int(num_steps), model_prob=model_prob) |
| 181 | + |
| 182 | + |
| 183 | +def _epoch_lengths( |
| 184 | + model_keys: Sequence[str], |
| 185 | + epoch_length: Callable[[str], int], |
| 186 | + broadcast: Callable[[list[int]], Sequence[int]] | None, |
| 187 | +) -> list[int]: |
| 188 | + """Collect the per-task epoch lengths agreed upon by every rank.""" |
| 189 | + lengths = [int(epoch_length(model_key)) for model_key in model_keys] |
| 190 | + if broadcast is not None: |
| 191 | + lengths = [int(value) for value in broadcast(lengths)] |
| 192 | + for model_key, length in zip(model_keys, lengths, strict=True): |
| 193 | + if length <= 0: |
| 194 | + raise ValueError( |
| 195 | + f"Number of training batches per epoch must be positive for " |
| 196 | + f"task '{model_key}', got {length}." |
| 197 | + ) |
| 198 | + return lengths |
0 commit comments