From b539db695faaa0621d21832b2412f6f227f86947 Mon Sep 17 00:00:00 2001 From: ragmani Date: Tue, 22 Apr 2025 10:58:16 +0000 Subject: [PATCH 1/3] [onert/python] Support dynamic shapes(on-the-fly) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit support dynamic shapes and add dynamic_shape_inference sample. - Remove the old “replace -1 with 1” placeholder in the constructor - In `infer()`, on first call: - Inspect each `tensorinfo.dims` and for any `-1` replace it with the matching `inputs_array[i].shape[j]` - Validate that any non-`-1` dims match the actual input shape, raising on mismatch - Call `update_inputs_tensorinfo()`, then `prepare()` and `set_outputs()` once - Bundle the above into the “auto‑dynamic” path instead of hard‑coding 1’s - Add `dynamic_shape_inference.py` sample to show 10 runs with random shapes ONE-DCO-1.0-Signed-off-by: ragmani --- .../onert/api/python/package/infer/session.py | 52 ++++++++++++------- .../src/dynamic_shape_inference.py | 40 ++++++++++++++ 2 files changed, 72 insertions(+), 20 deletions(-) create mode 100644 runtime/onert/sample/minimal-python/src/dynamic_shape_inference.py diff --git a/runtime/onert/api/python/package/infer/session.py b/runtime/onert/api/python/package/infer/session.py index 0b81c025b41..a46db2f175c 100644 --- a/runtime/onert/api/python/package/infer/session.py +++ b/runtime/onert/api/python/package/infer/session.py @@ -22,19 +22,6 @@ def __init__(self, path: str, backends: str = "cpu") -> None: super().__init__(infer.nnfw_session(path, backends)) self._prepared: bool = False - # TODO: Revise this after discussion to properly support dynamic shapes - # This is a temporary workaround to prevent prepare() errors when tensorinfo dims include -1 - original_infos: List[tensorinfo] = self.get_inputs_tensorinfo() - fixed_infos: List[tensorinfo] = [] - for info in original_infos: - dims = list(info.dims) - # replace -1 with 1 - dims = [1 if d == -1 else d for d in dims] - info.dims = dims # assume setter accepts a list - fixed_infos.append(info) - # update tensorinfo in session - self.update_inputs_tensorinfo(fixed_infos) - def update_inputs_tensorinfo(self, new_infos: List[tensorinfo]) -> None: """ Update all input tensors' tensorinfo at once. @@ -89,13 +76,6 @@ def infer( """ metrics: Dict[str, float] = {} - # Check if the session is prepared. If not, call prepare() and set_outputs() once. - if not self._prepared: - with self._time_block(metrics, 'prepare_time_ms', measure): - self.session.prepare() - self.set_outputs(self.session.output_size()) - self._prepared = True - # Verify that the number of provided inputs matches the session's expected input count. expected_input_size: int = self.session.input_size() if len(inputs_array) != expected_input_size: @@ -103,6 +83,38 @@ def infer( f"Expected {expected_input_size} input(s), but received {len(inputs_array)}." ) + # Check if the session is prepared. If not, call prepare() and set_outputs() once. + if not self._prepared: + with self._time_block(metrics, 'prepare_time_ms', measure): + # On first call, fix any -1 dims to real input shapes and validate + original_infos = self.get_inputs_tensorinfo() + fixed_infos = [] + for idx, info in enumerate(original_infos): + input_shape = inputs_array[idx].shape + new_dims = [] + # only the first `info.rank` entries matter + for j, d in enumerate(info.dims[:info.rank]): + if d == -1: + # replace dynamic dim with actual incoming shape + new_dims.append(input_shape[j]) + elif d == input_shape[j]: + # static dim must match the provided array + new_dims.append(d) + else: + raise ValueError( + f"Input #{idx} dim {j} mismatch: " + f"tensorinfo={d}, actual input shape={input_shape[j]}") + # Preserve any trailing dims beyond rank + # new_dims += list(info.dims[info.rank:]) + info.dims = new_dims + fixed_infos.append(info) + + # Update tensorinfo to optimize using it + self.update_inputs_tensorinfo(fixed_infos) + self.session.prepare() + self.set_outputs(self.session.output_size()) + self._prepared = True + # Configure input buffers using the current session's input size and provided data. with self._time_block(metrics, 'io_time_ms', measure): self.set_inputs(expected_input_size, inputs_array) diff --git a/runtime/onert/sample/minimal-python/src/dynamic_shape_inference.py b/runtime/onert/sample/minimal-python/src/dynamic_shape_inference.py new file mode 100644 index 00000000000..d6b8537a860 --- /dev/null +++ b/runtime/onert/sample/minimal-python/src/dynamic_shape_inference.py @@ -0,0 +1,40 @@ +import numpy as np +import random +import sys +from onert import infer + + +def main(nnpackage_path, backends="cpu"): + # Create session and load nnpackage + session = infer.session(nnpackage_path, backends) + + # Prepare input. Here we just allocate dummy input arrays. + input_infos = session.get_inputs_tensorinfo() + + # Call infer() 10 times + for i in range(10): + dummy_inputs = [] + for info in input_infos: + # Retrieve the dimensions list from tensorinfo property. + dims = list(info.dims) + # Replace -1 with a random value between 1 and 10 + dims = [random.randint(1, 10) if d == -1 else d for d in dims] + # Build the shape tuple from tensorinfo dimensions. + print(dims) + shape = tuple(dims[:info.rank]) + # Create a dummy numpy array filled with uniform random values in [0,1). + dummy_inputs.append( + np.random.uniform(low=0.0, high=1.0, size=shape).astype(info.dtype)) + + print(dummy_inputs) + outputs = session.infer(dummy_inputs) + print(outputs) + print(f"Inference run {i+1}/10 completed.") + + print(f"nnpackage {nnpackage_path.split('/')[-1]} runs successfully.") + return + + +if __name__ == "__main__": + argv = sys.argv[1:] + main(*argv) From a696f38d1758a52ae23df22ab1f1dbabb91de309 Mon Sep 17 00:00:00 2001 From: ragmani Date: Thu, 24 Apr 2025 05:49:44 +0000 Subject: [PATCH 2/3] Remove unnecessary code ONE-DCO-1.0-Signed-off-by: ragmani --- runtime/onert/api/python/package/infer/session.py | 1 + .../sample/minimal-python/src/dynamic_shape_inference.py | 3 --- .../sample/minimal-python/src/static_shape_inference.py | 7 +------ 3 files changed, 2 insertions(+), 9 deletions(-) diff --git a/runtime/onert/api/python/package/infer/session.py b/runtime/onert/api/python/package/infer/session.py index a46db2f175c..8b3848132d5 100644 --- a/runtime/onert/api/python/package/infer/session.py +++ b/runtime/onert/api/python/package/infer/session.py @@ -111,6 +111,7 @@ def infer( # Update tensorinfo to optimize using it self.update_inputs_tensorinfo(fixed_infos) + self.session.prepare() self.set_outputs(self.session.output_size()) self._prepared = True diff --git a/runtime/onert/sample/minimal-python/src/dynamic_shape_inference.py b/runtime/onert/sample/minimal-python/src/dynamic_shape_inference.py index d6b8537a860..350ff99b337 100644 --- a/runtime/onert/sample/minimal-python/src/dynamic_shape_inference.py +++ b/runtime/onert/sample/minimal-python/src/dynamic_shape_inference.py @@ -20,15 +20,12 @@ def main(nnpackage_path, backends="cpu"): # Replace -1 with a random value between 1 and 10 dims = [random.randint(1, 10) if d == -1 else d for d in dims] # Build the shape tuple from tensorinfo dimensions. - print(dims) shape = tuple(dims[:info.rank]) # Create a dummy numpy array filled with uniform random values in [0,1). dummy_inputs.append( np.random.uniform(low=0.0, high=1.0, size=shape).astype(info.dtype)) - print(dummy_inputs) outputs = session.infer(dummy_inputs) - print(outputs) print(f"Inference run {i+1}/10 completed.") print(f"nnpackage {nnpackage_path.split('/')[-1]} runs successfully.") diff --git a/runtime/onert/sample/minimal-python/src/static_shape_inference.py b/runtime/onert/sample/minimal-python/src/static_shape_inference.py index d08094f5550..4901d7e6327 100644 --- a/runtime/onert/sample/minimal-python/src/static_shape_inference.py +++ b/runtime/onert/sample/minimal-python/src/static_shape_inference.py @@ -18,12 +18,7 @@ def main(nnpackage_path, backends="cpu"): # We copy the current info and modify the rank and dims. # (Note: Depending on your model, you may want to modify additional dimensions.) new_shape = [10] + list(info.dims[1:info.rank]) - info.rank = len(new_shape) - for i, dim in enumerate(new_shape): - info.dims[i] = dim - # For any remaining dimensions up to NNFW_MAX_RANK, set them to a default (1). - for i in range(len(new_shape), len(info.dims)): - info.dims[i] = 1 + info.dims = new_shape new_input_infos.append(info) # Update all input tensorinfos in the session at once. From 307b680b6f0bd6e0fdc925339ddd5ecba43af3f4 Mon Sep 17 00:00:00 2001 From: ragmani Date: Tue, 29 Apr 2025 08:39:58 +0000 Subject: [PATCH 3/3] Change shape-mismatch error to warning in infer() --- runtime/onert/api/python/package/infer/session.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/runtime/onert/api/python/package/infer/session.py b/runtime/onert/api/python/package/infer/session.py index 8b3848132d5..ec8f5f26962 100644 --- a/runtime/onert/api/python/package/infer/session.py +++ b/runtime/onert/api/python/package/infer/session.py @@ -1,6 +1,7 @@ from typing import List, Union, Tuple, Dict import numpy as np import time +import warnings from contextlib import contextmanager from ..native.libnnfw_api_pybind import infer, tensorinfo @@ -92,6 +93,7 @@ def infer( for idx, info in enumerate(original_infos): input_shape = inputs_array[idx].shape new_dims = [] + static_dim_changed = False # only the first `info.rank` entries matter for j, d in enumerate(info.dims[:info.rank]): if d == -1: @@ -101,11 +103,14 @@ def infer( # static dim must match the provided array new_dims.append(d) else: - raise ValueError( - f"Input #{idx} dim {j} mismatch: " - f"tensorinfo={d}, actual input shape={input_shape[j]}") - # Preserve any trailing dims beyond rank - # new_dims += list(info.dims[info.rank:]) + static_dim_changed = True + + if static_dim_changed: + warnings.warn( + f"infer() called with input {idx}'s shape={input_shape}, " + f"which differs from model’s expected shape={tuple(info.dims)}. " + "Ensure this is intended.", UserWarning) + info.dims = new_dims fixed_infos.append(info)