diff --git a/runtime/onert/api/python/package/infer/session.py b/runtime/onert/api/python/package/infer/session.py index 0b81c025b41..ec8f5f26962 100644 --- a/runtime/onert/api/python/package/infer/session.py +++ b/runtime/onert/api/python/package/infer/session.py @@ -1,6 +1,7 @@ from typing import List, Union, Tuple, Dict import numpy as np import time +import warnings from contextlib import contextmanager from ..native.libnnfw_api_pybind import infer, tensorinfo @@ -22,19 +23,6 @@ def __init__(self, path: str, backends: str = "cpu") -> None: super().__init__(infer.nnfw_session(path, backends)) self._prepared: bool = False - # TODO: Revise this after discussion to properly support dynamic shapes - # This is a temporary workaround to prevent prepare() errors when tensorinfo dims include -1 - original_infos: List[tensorinfo] = self.get_inputs_tensorinfo() - fixed_infos: List[tensorinfo] = [] - for info in original_infos: - dims = list(info.dims) - # replace -1 with 1 - dims = [1 if d == -1 else d for d in dims] - info.dims = dims # assume setter accepts a list - fixed_infos.append(info) - # update tensorinfo in session - self.update_inputs_tensorinfo(fixed_infos) - def update_inputs_tensorinfo(self, new_infos: List[tensorinfo]) -> None: """ Update all input tensors' tensorinfo at once. @@ -89,13 +77,6 @@ def infer( """ metrics: Dict[str, float] = {} - # Check if the session is prepared. If not, call prepare() and set_outputs() once. - if not self._prepared: - with self._time_block(metrics, 'prepare_time_ms', measure): - self.session.prepare() - self.set_outputs(self.session.output_size()) - self._prepared = True - # Verify that the number of provided inputs matches the session's expected input count. expected_input_size: int = self.session.input_size() if len(inputs_array) != expected_input_size: @@ -103,6 +84,43 @@ def infer( f"Expected {expected_input_size} input(s), but received {len(inputs_array)}." ) + # Check if the session is prepared. If not, call prepare() and set_outputs() once. + if not self._prepared: + with self._time_block(metrics, 'prepare_time_ms', measure): + # On first call, fix any -1 dims to real input shapes and validate + original_infos = self.get_inputs_tensorinfo() + fixed_infos = [] + for idx, info in enumerate(original_infos): + input_shape = inputs_array[idx].shape + new_dims = [] + static_dim_changed = False + # only the first `info.rank` entries matter + for j, d in enumerate(info.dims[:info.rank]): + if d == -1: + # replace dynamic dim with actual incoming shape + new_dims.append(input_shape[j]) + elif d == input_shape[j]: + # static dim must match the provided array + new_dims.append(d) + else: + static_dim_changed = True + + if static_dim_changed: + warnings.warn( + f"infer() called with input {idx}'s shape={input_shape}, " + f"which differs from model’s expected shape={tuple(info.dims)}. " + "Ensure this is intended.", UserWarning) + + info.dims = new_dims + fixed_infos.append(info) + + # Update tensorinfo to optimize using it + self.update_inputs_tensorinfo(fixed_infos) + + self.session.prepare() + self.set_outputs(self.session.output_size()) + self._prepared = True + # Configure input buffers using the current session's input size and provided data. with self._time_block(metrics, 'io_time_ms', measure): self.set_inputs(expected_input_size, inputs_array) diff --git a/runtime/onert/sample/minimal-python/src/dynamic_shape_inference.py b/runtime/onert/sample/minimal-python/src/dynamic_shape_inference.py new file mode 100644 index 00000000000..350ff99b337 --- /dev/null +++ b/runtime/onert/sample/minimal-python/src/dynamic_shape_inference.py @@ -0,0 +1,37 @@ +import numpy as np +import random +import sys +from onert import infer + + +def main(nnpackage_path, backends="cpu"): + # Create session and load nnpackage + session = infer.session(nnpackage_path, backends) + + # Prepare input. Here we just allocate dummy input arrays. + input_infos = session.get_inputs_tensorinfo() + + # Call infer() 10 times + for i in range(10): + dummy_inputs = [] + for info in input_infos: + # Retrieve the dimensions list from tensorinfo property. + dims = list(info.dims) + # Replace -1 with a random value between 1 and 10 + dims = [random.randint(1, 10) if d == -1 else d for d in dims] + # Build the shape tuple from tensorinfo dimensions. + shape = tuple(dims[:info.rank]) + # Create a dummy numpy array filled with uniform random values in [0,1). + dummy_inputs.append( + np.random.uniform(low=0.0, high=1.0, size=shape).astype(info.dtype)) + + outputs = session.infer(dummy_inputs) + print(f"Inference run {i+1}/10 completed.") + + print(f"nnpackage {nnpackage_path.split('/')[-1]} runs successfully.") + return + + +if __name__ == "__main__": + argv = sys.argv[1:] + main(*argv) diff --git a/runtime/onert/sample/minimal-python/src/static_shape_inference.py b/runtime/onert/sample/minimal-python/src/static_shape_inference.py index d08094f5550..4901d7e6327 100644 --- a/runtime/onert/sample/minimal-python/src/static_shape_inference.py +++ b/runtime/onert/sample/minimal-python/src/static_shape_inference.py @@ -18,12 +18,7 @@ def main(nnpackage_path, backends="cpu"): # We copy the current info and modify the rank and dims. # (Note: Depending on your model, you may want to modify additional dimensions.) new_shape = [10] + list(info.dims[1:info.rank]) - info.rank = len(new_shape) - for i, dim in enumerate(new_shape): - info.dims[i] = dim - # For any remaining dimensions up to NNFW_MAX_RANK, set them to a default (1). - for i in range(len(new_shape), len(info.dims)): - info.dims[i] = 1 + info.dims = new_shape new_input_infos.append(info) # Update all input tensorinfos in the session at once.