From 2e8fce344f418655691302af3f8395b4892ac4a5 Mon Sep 17 00:00:00 2001 From: ragmani Date: Thu, 17 Apr 2025 10:24:08 +0000 Subject: [PATCH 1/4] [onert/python] Support static shape modification across inference API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit supports static shape modification across inference API and samples. - common/basesession.py: add typed get_inputs_tensorinfo/get_outputs_tensorinfo helpers - infer/session.py: - normalize any `-1` dims to `1` then call `update_inputs_tensorinfo` in `__init__` - remove obsolete `compile()` method - provide `update_inputs_tensorinfo` and `run_inference` with full type hints - samples: - minimal sample: build dummy inputs from tensorinfo and use `run_inference` - static_shape_inference example: demonstrate modifying input tensorinfo (e.g. batch size → 10) and running inference with static shapes ONE-DCO-1.0-Signed-off-by: ragmani --- .../api/python/package/common/basesession.py | 31 ++++++- .../onert/api/python/package/infer/session.py | 92 ++++++++++++++----- .../sample/minimal-python/src/minimal.py | 14 ++- .../src/static_shape_inference.py | 52 +++++++++++ 4 files changed, 160 insertions(+), 29 deletions(-) create mode 100644 runtime/onert/sample/minimal-python/src/static_shape_inference.py diff --git a/runtime/onert/api/python/package/common/basesession.py b/runtime/onert/api/python/package/common/basesession.py index 1521efb452b..7d55a884f0b 100644 --- a/runtime/onert/api/python/package/common/basesession.py +++ b/runtime/onert/api/python/package/common/basesession.py @@ -1,6 +1,7 @@ +from typing import List import numpy as np -from ..native import libnnfw_api_pybind +from ..native.libnnfw_api_pybind import infer, tensorinfo def num_elems(tensor_info): @@ -52,6 +53,32 @@ def _recreate_session(self, backend_session): del self.session # Clean up the existing session self.session = backend_session + def get_inputs_tensorinfo(self) -> List[tensorinfo]: + """ + Retrieve tensorinfo for all input tensors. + + Returns: + list[tensorinfo]: A list of tensorinfo objects for each input. + """ + num_inputs: int = self.session.input_size() + infos: List[tensorinfo] = [] + for i in range(num_inputs): + infos.append(self.session.input_tensorinfo(i)) + return infos + + def get_outputs_tensorinfo(self) -> List[tensorinfo]: + """ + Retrieve tensorinfo for all output tensors. + + Returns: + list[tensorinfo]: A list of tensorinfo objects for each output. + """ + num_outputs: int = self.session.output_size() + infos: List[tensorinfo] = [] + for i in range(num_outputs): + infos.append(self.session.output_tensorinfo(i)) + return infos + def set_inputs(self, size, inputs_array=[]): """ Set the input tensors for the session. @@ -97,4 +124,4 @@ def set_outputs(self, size): def tensorinfo(): - return libnnfw_api_pybind.infer.nnfw_tensorinfo() + return infer.nnfw_tensorinfo() diff --git a/runtime/onert/api/python/package/infer/session.py b/runtime/onert/api/python/package/infer/session.py index e0ef4f7f8bb..5eeb32ea40c 100644 --- a/runtime/onert/api/python/package/infer/session.py +++ b/runtime/onert/api/python/package/infer/session.py @@ -1,4 +1,7 @@ -from ..native import libnnfw_api_pybind +from typing import List, Any +import numpy as np + +from ..native.libnnfw_api_pybind import infer, tensorinfo from ..common.basesession import BaseSession @@ -6,41 +9,82 @@ class session(BaseSession): """ Class for inference using nnfw_session. """ - def __init__(self, path: str = None, backends: str = "cpu"): + def __init__(self, path: str, backends: str = "cpu") -> None: """ Initialize the inference session. + Args: path (str): Path to the model file or nnpackage directory. backends (str): Backends to use, default is "cpu". """ - if path is not None: - super().__init__(libnnfw_api_pybind.infer.nnfw_session(path, backends)) - self.session.prepare() - self.set_outputs(self.session.output_size()) - else: - super().__init__() + super().__init__(infer.nnfw_session(path, backends)) + self._prepared: bool = False - def compile(self, path: str, backends: str = "cpu"): + # Replace any dynamic dimension (-1) with 1 before static shape inference, + # because nnfw_session cannot perform static shape inference on input dimensions set to -1. + original_infos: List[tensorinfo] = self.get_inputs_tensorinfo() + fixed_infos: List[tensorinfo] = [] + for info in original_infos: + dims = list(info.dims) + # replace -1 with 1 + dims = [1 if d == -1 else d for d in dims] + info.dims = dims # assume setter accepts a list + fixed_infos.append(info) + # update tensorinfo in session + self.update_inputs_tensorinfo(fixed_infos) + + def update_inputs_tensorinfo(self, new_infos: List[tensorinfo]) -> None: """ - Prepare the session by recreating it with new parameters. + Update all input tensors' tensorinfo at once. + Args: - path (str): Path to the model file or nnpackage directory. Defaults to the existing path. - backends (str): Backends to use. Defaults to the existing backends. + new_infos (list[tensorinfo]): A list of updated tensorinfo objects for the inputs. + + Raises: + ValueError: If the number of new_infos does not match the session's input size. """ - # Update parameters if provided - if path is None: - raise ValueError("path must not be None.") - # Recreate the session with updated parameters - self._recreate_session(libnnfw_api_pybind.infer.nnfw_session(path, backends)) - # Prepare the new session - self.session.prepare() - self.set_outputs(self.session.output_size()) - - def inference(self): + num_inputs: int = self.session.input_size() + if len(new_infos) != num_inputs: + raise ValueError( + f"Expected {num_inputs} input tensorinfo(s), but got {len(new_infos)}.") + for i, info in enumerate(new_infos): + self.session.set_input_tensorinfo(i, info) + + def run_inference(self, inputs_array: List[np.ndarray]) -> List[np.ndarray]: """ - Perform model and get outputs + Run a complete inference cycle: + - If the session has not been prepared or outputs have not been set, call prepare() and set_outputs(). + - Automatically configure input buffers based on the provided numpy arrays. + - Execute the inference session. + - Return the output tensors with proper multi-dimensional shapes. + + This method supports both static and dynamic shape modification: + - If update_inputs_tensorinfo() has been called before running inference, the model is compiled + with the fixed static input shape. + - Otherwise, the input shapes can be adjusted dynamically. + + Args: + inputs_array (list[np.ndarray]): List of numpy arrays representing the input data. + Returns: - list: Outputs from the model. + list[np.ndarray]: A list containing the output numpy arrays. """ + # Check if the session is prepared. If not, call prepare() and set_outputs() once. + if not self._prepared: + self.session.prepare() + self.set_outputs(self.session.output_size()) + self._prepared = True + + # Verify that the number of provided inputs matches the session's expected input count. + expected_input_size: int = self.session.input_size() + if len(inputs_array) != expected_input_size: + raise ValueError( + f"Expected {expected_input_size} input(s), but received {len(inputs_array)}." + ) + + # Configure input buffers using the current session's input size and provided data. + self.set_inputs(expected_input_size, inputs_array) + # Execute the inference. self.session.run() + # Return the output buffers. return self.outputs diff --git a/runtime/onert/sample/minimal-python/src/minimal.py b/runtime/onert/sample/minimal-python/src/minimal.py index 2ae3f249fcd..6f5032187a9 100644 --- a/runtime/onert/sample/minimal-python/src/minimal.py +++ b/runtime/onert/sample/minimal-python/src/minimal.py @@ -1,4 +1,5 @@ from onert import infer +import numpy as np import sys @@ -8,10 +9,17 @@ def main(nnpackage_path, backends="cpu"): session = infer.session(nnpackage_path, backends) # Prepare input. Here we just allocate dummy input arrays. - input_size = session.input_size() - session.set_inputs(input_size) + input_infos = session.get_inputs_tensorinfo() + dummy_inputs = [] + for info in input_infos: + # Retrieve the dimensions list from tensorinfo property. + dims = list(info.dims) + # Build the shape tuple from tensorinfo dimensions. + shape = tuple(dims[:info.rank]) + # Create a dummy numpy array filled with zeros. + dummy_inputs.append(np.zeros(shape, dtype=info.dtype)) - outputs = session.inference() + outputs = session.run_inference(dummy_inputs) print(f"nnpackage {nnpackage_path.split('/')[-1]} runs successfully.") return diff --git a/runtime/onert/sample/minimal-python/src/static_shape_inference.py b/runtime/onert/sample/minimal-python/src/static_shape_inference.py new file mode 100644 index 00000000000..530f82bb2d9 --- /dev/null +++ b/runtime/onert/sample/minimal-python/src/static_shape_inference.py @@ -0,0 +1,52 @@ +from onert import infer +import numpy as np +import sys + + +def main(nnpackage_path, backends="cpu"): + # Create session and load the nnpackage + sess = infer.session(nnpackage_path, backends) + + # Retrieve the current tensorinfo for all inputs. + current_input_infos = sess.get_inputs_tensorinfo() + + # Create new tensorinfo objects with a static shape modification. + # For this example, assume we change the first dimension (e.g., batch size) to 10. + new_input_infos = [] + for info in current_input_infos: + # For example, if the current shape is (?, 4), update it to (10, 4). + # We copy the current info and modify the rank and dims. + # (Note: Depending on your model, you may want to modify additional dimensions.) + new_shape = [10] + list(info.dims[1:info.rank]) + info.rank = len(new_shape) + for i, dim in enumerate(new_shape): + info.dims[i] = dim + # For any remaining dimensions up to NNFW_MAX_RANK, set them to a default (1). + for i in range(len(new_shape), len(info.dims)): + info.dims[i] = 1 + new_input_infos.append(info) + + # Update all input tensorinfos in the session at once. + # This will call prepare() and set_outputs() internally. + sess.update_inputs_tensorinfo(new_input_infos) + + # Create dummy input arrays based on the new static shapes. + dummy_inputs = [] + for info in new_input_infos: + # Build the shape tuple from tensorinfo dimensions. + shape = tuple(info.dims[:info.rank]) + # Create a dummy numpy array filled with zeros. + dummy_inputs.append(np.zeros(shape, dtype=info.dtype)) + + # Run inference with the new static input shapes. + outputs = sess.run_inference(dummy_inputs) + + print( + f"Static shape modification sample: nnpackage {nnpackage_path.split('/')[-1]} runs successfully." + ) + return + + +if __name__ == "__main__": + argv = sys.argv[1:] + main(*argv) From 03a35ccff70741fed82356c66de83ee05eaff843 Mon Sep 17 00:00:00 2001 From: ragmani Date: Fri, 18 Apr 2025 01:16:53 +0000 Subject: [PATCH 2/4] Rename run_inference to infer --- runtime/onert/api/python/package/infer/session.py | 2 +- runtime/onert/sample/minimal-python/src/minimal.py | 2 +- .../onert/sample/minimal-python/src/static_shape_inference.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/runtime/onert/api/python/package/infer/session.py b/runtime/onert/api/python/package/infer/session.py index 5eeb32ea40c..916d7f2d4fc 100644 --- a/runtime/onert/api/python/package/infer/session.py +++ b/runtime/onert/api/python/package/infer/session.py @@ -50,7 +50,7 @@ def update_inputs_tensorinfo(self, new_infos: List[tensorinfo]) -> None: for i, info in enumerate(new_infos): self.session.set_input_tensorinfo(i, info) - def run_inference(self, inputs_array: List[np.ndarray]) -> List[np.ndarray]: + def infer(self, inputs_array: List[np.ndarray]) -> List[np.ndarray]: """ Run a complete inference cycle: - If the session has not been prepared or outputs have not been set, call prepare() and set_outputs(). diff --git a/runtime/onert/sample/minimal-python/src/minimal.py b/runtime/onert/sample/minimal-python/src/minimal.py index 6f5032187a9..aafeb5de52f 100644 --- a/runtime/onert/sample/minimal-python/src/minimal.py +++ b/runtime/onert/sample/minimal-python/src/minimal.py @@ -19,7 +19,7 @@ def main(nnpackage_path, backends="cpu"): # Create a dummy numpy array filled with zeros. dummy_inputs.append(np.zeros(shape, dtype=info.dtype)) - outputs = session.run_inference(dummy_inputs) + outputs = session.infer(dummy_inputs) print(f"nnpackage {nnpackage_path.split('/')[-1]} runs successfully.") return diff --git a/runtime/onert/sample/minimal-python/src/static_shape_inference.py b/runtime/onert/sample/minimal-python/src/static_shape_inference.py index 530f82bb2d9..d08094f5550 100644 --- a/runtime/onert/sample/minimal-python/src/static_shape_inference.py +++ b/runtime/onert/sample/minimal-python/src/static_shape_inference.py @@ -39,7 +39,7 @@ def main(nnpackage_path, backends="cpu"): dummy_inputs.append(np.zeros(shape, dtype=info.dtype)) # Run inference with the new static input shapes. - outputs = sess.run_inference(dummy_inputs) + outputs = sess.infer(dummy_inputs) print( f"Static shape modification sample: nnpackage {nnpackage_path.split('/')[-1]} runs successfully." From 35207393ddb103c7f35c6d4ab58367133127caa8 Mon Sep 17 00:00:00 2001 From: ragmani Date: Fri, 18 Apr 2025 05:09:43 +0000 Subject: [PATCH 3/4] Add checking negative dim in update_inputs_tensorinfo --- runtime/onert/api/python/package/infer/session.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/runtime/onert/api/python/package/infer/session.py b/runtime/onert/api/python/package/infer/session.py index 916d7f2d4fc..1b2ca5e773b 100644 --- a/runtime/onert/api/python/package/infer/session.py +++ b/runtime/onert/api/python/package/infer/session.py @@ -41,13 +41,20 @@ def update_inputs_tensorinfo(self, new_infos: List[tensorinfo]) -> None: new_infos (list[tensorinfo]): A list of updated tensorinfo objects for the inputs. Raises: - ValueError: If the number of new_infos does not match the session's input size. + ValueError: If the number of new_infos does not match the session's input size, + or if any tensorinfo contains a negative dimension. """ num_inputs: int = self.session.input_size() if len(new_infos) != num_inputs: raise ValueError( f"Expected {num_inputs} input tensorinfo(s), but got {len(new_infos)}.") + for i, info in enumerate(new_infos): + # Check for any negative dimension in the specified rank + if any(d < 0 for d in info.dims[:info.rank]): + raise ValueError( + f"Input tensorinfo at index {i} contains negative dimension(s): " + f"{info.dims[:info.rank]}") self.session.set_input_tensorinfo(i, info) def infer(self, inputs_array: List[np.ndarray]) -> List[np.ndarray]: From 5f4a3f5c5cea47489f48417a721612ad6ff95226 Mon Sep 17 00:00:00 2001 From: ragmani Date: Fri, 18 Apr 2025 05:17:56 +0000 Subject: [PATCH 4/4] Add a TODO comment --- runtime/onert/api/python/package/infer/session.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runtime/onert/api/python/package/infer/session.py b/runtime/onert/api/python/package/infer/session.py index 1b2ca5e773b..0f9daa12e7e 100644 --- a/runtime/onert/api/python/package/infer/session.py +++ b/runtime/onert/api/python/package/infer/session.py @@ -20,8 +20,8 @@ def __init__(self, path: str, backends: str = "cpu") -> None: super().__init__(infer.nnfw_session(path, backends)) self._prepared: bool = False - # Replace any dynamic dimension (-1) with 1 before static shape inference, - # because nnfw_session cannot perform static shape inference on input dimensions set to -1. + # TODO: Revise this after discussion to properly support dynamic shapes + # This is a temporary workaround to prevent prepare() errors when tensorinfo dims include -1 original_infos: List[tensorinfo] = self.get_inputs_tensorinfo() fixed_infos: List[tensorinfo] = [] for info in original_infos: