Skip to content

Commit d31ce6b

Browse files
ismukhinismukhin
committed
add python api benchmarks
1 parent 0d2ca8c commit d31ce6b

12 files changed

+453
-0
lines changed

‎src/benchmark/config_parser_factory.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
from frameworks.tvm.tvm_parameters_parser import TVMParametersParser
1414
from frameworks.ncnn.ncnn_parameters_parser import NcnnParametersParser
1515
from frameworks.spektral.spektral_parameters_parser import SpektralParametersParser
16+
from frameworks.executorch.executorch_parameters_parser import ExecuTorchParametersParser
1617

1718

1819
def get_parameters_parser(framework):
@@ -54,4 +55,6 @@ def get_parameters_parser(framework):
5455
return CppParametersParser()
5556
if framework == KnownFrameworks.executorch_cpp:
5657
return CppParametersParser()
58+
if framework == KnownFrameworks.executorch:
59+
return ExecuTorchParametersParser()
5760
raise NotImplementedError(f'Unknown framework {framework}')

‎src/benchmark/frameworks/executorch/__init__.py‎

Whitespace-only changes.

‎src/benchmark/frameworks/executorch/executorch_parameters_parser.py‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
from ..config_parser.dependent_parameters_parser import DependentParametersParser
2+
from ..config_parser.framework_parameters_parser import FrameworkParameters
3+
4+
5+
class ExecuTorchParametersParser(DependentParametersParser):
6+
def parse_parameters(self, curr_test):
7+
CONFIG_FRAMEWORK_DEPENDENT_TAG = 'FrameworkDependent'
8+
CONFIG_FRAMEWORK_DEPENDENT_INPUT_NAME_TAG = 'InputName'
9+
CONFIG_FRAMEWORK_DEPENDENT_INPUT_SHAPE_TAG = 'InputShape'
10+
CONFIG_FRAMEWORK_DEPENDENT_NORMALIZE_TAG = 'Normalize'
11+
CONFIG_FRAMEWORK_DEPENDENT_MEAN_TAG = 'Mean'
12+
CONFIG_FRAMEWORK_DEPENDENT_STD_TAG = 'Std'
13+
CONFIG_FRAMEWORK_DEPENDENT_CHANNEL_SWAP_TAG = 'ChannelSwap'
14+
CONFIG_FRAMEWORK_DEPENDENT_LAYOUT_TAG = 'Layout'
15+
16+
dep_parameters_tag = curr_test.getElementsByTagName(CONFIG_FRAMEWORK_DEPENDENT_TAG)[0]
17+
18+
_input_name = dep_parameters_tag.getElementsByTagName(
19+
CONFIG_FRAMEWORK_DEPENDENT_INPUT_NAME_TAG)[0].firstChild
20+
_input_shape = dep_parameters_tag.getElementsByTagName(
21+
CONFIG_FRAMEWORK_DEPENDENT_INPUT_SHAPE_TAG)[0].firstChild
22+
_normalize = dep_parameters_tag.getElementsByTagName(
23+
CONFIG_FRAMEWORK_DEPENDENT_NORMALIZE_TAG)[0].firstChild
24+
_mean = dep_parameters_tag.getElementsByTagName(
25+
CONFIG_FRAMEWORK_DEPENDENT_MEAN_TAG)[0].firstChild
26+
_std = dep_parameters_tag.getElementsByTagName(
27+
CONFIG_FRAMEWORK_DEPENDENT_STD_TAG)[0].firstChild
28+
_channel_swap = dep_parameters_tag.getElementsByTagName(
29+
CONFIG_FRAMEWORK_DEPENDENT_CHANNEL_SWAP_TAG)[0].firstChild
30+
_layout = dep_parameters_tag.getElementsByTagName(
31+
CONFIG_FRAMEWORK_DEPENDENT_LAYOUT_TAG)[0].firstChild
32+
33+
return ExecuTorchParameters(
34+
input_name=_input_name.data if _input_name else None,
35+
input_shape=_input_shape.data if _input_shape else None,
36+
normalize=_normalize.data if _normalize else None,
37+
mean=_mean.data if _mean else None,
38+
std=_std.data if _std else None,
39+
channel_swap=_channel_swap.data if _channel_swap else None,
40+
layout=_layout.data if _layout else None,
41+
)
42+
43+
44+
class ExecuTorchParameters(FrameworkParameters):
45+
def __init__(self, input_name, input_shape,
46+
normalize, mean, std, channel_swap,
47+
layout):
48+
self.input_name = None
49+
self.input_shape = None
50+
self.normalize = None
51+
self.mean = None
52+
self.std = None
53+
self.channel_swap = None
54+
self.layout = None
55+
56+
if self._parameter_is_not_none(input_name):
57+
self.input_name = input_name
58+
if self._parameter_is_not_none(input_shape):
59+
self.input_shape = input_shape
60+
if self._parameter_is_not_none(normalize):
61+
self.normalize = normalize
62+
if self._parameter_is_not_none(mean):
63+
self.mean = mean
64+
if self._parameter_is_not_none(std):
65+
self.std = std
66+
if self._parameter_is_not_none(channel_swap):
67+
self.channel_swap = channel_swap
68+
if self._parameter_is_not_none(layout):
69+
self.layout = layout

‎src/benchmark/frameworks/executorch/executorch_process.py‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
from pathlib import Path
2+
3+
from ..processes import ProcessHandler
4+
5+
6+
class ExecuTorchProcess(ProcessHandler):
7+
benchmark_app_name = 'executorch_python_benchmark'
8+
launcher_latency_units = 'seconds'
9+
10+
def __init__(self, test, executor, log):
11+
super().__init__(test, executor, log)
12+
self.path_to_script = Path.joinpath(self.inference_script_root, 'inference_executorch.py')
13+
14+
@staticmethod
15+
def create_process(test, executor, log):
16+
return ExecuTorchProcess(test, executor, log)
17+
18+
def get_performance_metrics(self):
19+
return self.get_performance_metrics_from_json_report()
20+
21+
def _fill_command_line(self):
22+
python = ProcessHandler.get_cmd_python_version()
23+
dataset = self._test.dataset.path
24+
input_shape = self._test.dep_parameters.input_shape
25+
layout = self._test.dep_parameters.layout
26+
batch_size = self._test.indep_parameters.batch_size
27+
iteration = self._test.indep_parameters.iteration
28+
name = self._test.model.name
29+
model = self._test.model.model
30+
common_params = (f'-i {dataset} -is {input_shape} -b {batch_size} '
31+
f'-ni {iteration} --report_path {self.report_path} '
32+
f'--layout {layout} ')
33+
34+
input_name = self._test.dep_parameters.input_name
35+
common_params = ExecuTorchProcess._add_optional_argument_to_cmd_line(
36+
common_params, '--input_name', input_name)
37+
38+
normalize = self._test.dep_parameters.normalize
39+
if normalize == 'True':
40+
common_params = ExecuTorchProcess._add_flag_to_cmd_line(
41+
common_params, '--norm')
42+
43+
mean = self._test.dep_parameters.mean
44+
common_params = ExecuTorchProcess._add_optional_argument_to_cmd_line(
45+
common_params, '--mean', mean)
46+
47+
std = self._test.dep_parameters.std
48+
common_params = ExecuTorchProcess._add_optional_argument_to_cmd_line(
49+
common_params, '--std', std)
50+
51+
channel_swap = self._test.dep_parameters.channel_swap
52+
common_params = ExecuTorchProcess._add_optional_argument_to_cmd_line(
53+
common_params, '--channel_swap', channel_swap)
54+
55+
device = self._test.indep_parameters.device
56+
common_params = ExecuTorchProcess._add_optional_argument_to_cmd_line(
57+
common_params, '--device', device)
58+
59+
common_params = ExecuTorchProcess._add_optional_argument_to_cmd_line(
60+
common_params, '-m', model)
61+
62+
common_params = ExecuTorchProcess._add_optional_argument_to_cmd_line(
63+
common_params, '-mn', name)
64+
65+
command_line = f'{python} {self.path_to_script} {common_params}'
66+
67+
return command_line

‎src/benchmark/frameworks/executorch/executorch_test.py‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
from collections import OrderedDict
2+
3+
from ..config_parser.test_reporter import Test
4+
5+
6+
class ExecuTorchTest(Test):
7+
def __init__(self, model, dataset, indep_parameters, dep_parameters):
8+
super().__init__(model, dataset, indep_parameters, dep_parameters)
9+
10+
def get_report(self, process):
11+
parameters = OrderedDict()
12+
parameters.update({'Device': self.indep_parameters.device})
13+
parameters.update({'Iteration count': self.indep_parameters.iteration})
14+
other_param = self._get_optional_parameters_string(parameters)
15+
16+
report_res = {
17+
'task': self.model.task,
18+
'model': self.model.name,
19+
'dataset': self.dataset.name,
20+
'source_framework': self.model.source_framework,
21+
'inference_framework': self.indep_parameters.inference_framework,
22+
'precision': self.model.precision,
23+
'batch_size': self.indep_parameters.batch_size,
24+
'mode': 'Sync',
25+
'framework_params': other_param,
26+
}
27+
28+
return report_res

‎src/benchmark/frameworks/executorch/executorch_wrapper.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
from .executorch_process import ExecuTorchProcess
2+
from .executorch_test import ExecuTorchTest
3+
from ..framework_wrapper import FrameworkWrapper
4+
from ..known_frameworks import KnownFrameworks
5+
6+
7+
class ExecuTorchWrapper(FrameworkWrapper):
8+
framework_name = KnownFrameworks.executorch
9+
10+
@staticmethod
11+
def create_process(test, executor, log, **kwargs):
12+
return ExecuTorchProcess.create_process(test, executor, log)
13+
14+
@staticmethod
15+
def create_test(model, dataset, indep_parameters, dep_parameters):
16+
return ExecuTorchTest(model, dataset, indep_parameters, dep_parameters)

‎src/benchmark/frameworks/framework_wrapper_registry.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
from .spektral.spektral_wrapper import SpektralWrapper
2121
from .rknn.rknn_wrapper import RknnWrapper
2222
from .executorch_cpp.executorch_cpp_wrapper import ExecuTorchCppWrapper
23+
from .executorch.executorch_wrapper import ExecuTorchWrapper
2324

2425

2526
class FrameworkWrapperRegistry(metaclass=Singleton):
@@ -60,3 +61,4 @@ def _get_wrappers(self):
6061
self._framework_wrappers[SpektralWrapper.framework_name] = SpektralWrapper()
6162
self._framework_wrappers[RknnWrapper.framework_name] = RknnWrapper()
6263
self._framework_wrappers[ExecuTorchCppWrapper.framework_name] = ExecuTorchCppWrapper()
64+
self._framework_wrappers[ExecuTorchWrapper.framework_name] = ExecuTorchWrapper()

‎src/benchmark/frameworks/known_frameworks.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,3 +18,4 @@ class KnownFrameworks:
1818
rknn = 'RKNN'
1919
ncnn = 'ncnn'
2020
executorch_cpp = 'ExecuTorch Cpp'
21+
executorch = 'ExecuTorch'

0 commit comments

Comments
 (0)