Skip to content

Commit 4a22c0c

Browse files
rpanchamvaibhavjainwizdbasunag
authored and
fege
committed
Add Testsuite for Triton KServe Models (Rest and gRPC) (opendatahub-io#414)
* Add ONNX test case with config files (gRPC/REST templates, conftest, constants) and supporting utils * Incorporate review comments Signed-off-by: Vaibhav Jain <vajain@redhat.com> * Fix pre-commit errors Signed-off-by: Vaibhav Jain <vajain@redhat.com> * Incorporate review comments Signed-off-by: Vaibhav Jain <vajain@redhat.com> --------- Signed-off-by: Vaibhav Jain <vajain@redhat.com> Co-authored-by: Vaibhav Jain <vajain@redhat.com> Co-authored-by: Debarati Basu-Nag <dbasunag@redhat.com>
1 parent e189794 commit 4a22c0c

19 files changed

+2828
-123
lines changed

‎conftest.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,11 @@ def pytest_addoption(parser: Parser) -> None:
112112
default=os.environ.get("MLSERVER_RUNTIME_IMAGE"),
113113
help="Specify the runtime image to use for the tests",
114114
)
115+
runtime_group.addoption(
116+
"--triton-runtime-image",
117+
default=os.environ.get("TRITON_RUNTIME_IMAGE"),
118+
help="Specify the runtime image to use for the tests",
119+
)
115120

116121
# Upgrade options
117122
upgrade_group.addoption(

‎tests/conftest.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -239,6 +239,14 @@ def mlserver_runtime_image(pytestconfig: pytest.Config) -> str | None:
239239
return runtime_image
240240

241241

242+
@pytest.fixture(scope="session")
243+
def triton_runtime_image(pytestconfig: pytest.Config) -> str | None:
244+
runtime_image = pytestconfig.option.triton_runtime_image
245+
if not runtime_image:
246+
return None
247+
return runtime_image
248+
249+
242250
@pytest.fixture(scope="session")
243251
def use_unprivileged_client(pytestconfig: pytest.Config) -> bool:
244252
_use_unprivileged_client = py_config.get("use_unprivileged_client")

‎tests/model_serving/conftest.py‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,122 @@
1+
from typing import Generator, Any
2+
13
import pytest
24
from _pytest.fixtures import FixtureRequest
5+
from kubernetes.dynamic import DynamicClient
6+
from ocp_resources.namespace import Namespace
7+
from ocp_resources.secret import Secret
8+
from contextlib import contextmanager
9+
10+
11+
@pytest.fixture(scope="session")
12+
def root_dir(pytestconfig: pytest.Config) -> Any:
13+
"""
14+
Provides the root directory path of the pytest project for the entire test session.
15+
16+
Args:
17+
pytestconfig (pytest.Config): The pytest configuration object.
18+
19+
Returns:
20+
Any: The root path of the pytest project.
21+
"""
22+
return pytestconfig.rootpath
23+
24+
25+
@pytest.fixture(scope="class")
26+
def protocol(request: FixtureRequest) -> str:
27+
"""
28+
Provides the protocol type parameter for the test class.
29+
30+
Args:
31+
request (pytest.FixtureRequest): The pytest fixture request object.
32+
33+
Returns:
34+
str: The protocol type specified in the test parameter.
35+
"""
36+
return request.param["protocol_type"]
337

438

539
@pytest.fixture(scope="session")
640
def s3_models_storage_uri(request: FixtureRequest, models_s3_bucket_name: str) -> str:
741
return f"s3://{models_s3_bucket_name}/{request.param['model-dir']}/"
42+
43+
44+
@pytest.fixture(scope="class")
45+
def kserve_s3_secret(
46+
admin_client: DynamicClient,
47+
model_namespace: Namespace,
48+
aws_access_key_id: str,
49+
aws_secret_access_key: str,
50+
models_s3_bucket_region: str,
51+
models_s3_bucket_endpoint: str,
52+
) -> Secret:
53+
"""
54+
Creates and yields a Kubernetes Secret configured for S3 access in KServe.
55+
56+
Args:
57+
admin_client (DynamicClient): Kubernetes dynamic client.
58+
model_namespace (Namespace): Namespace where the secret will be created.
59+
aws_access_key_id (str): AWS access key ID.
60+
aws_secret_access_key (str): AWS secret access key.
61+
models_s3_bucket_region (str): AWS S3 bucket region.
62+
models_s3_bucket_endpoint (str): AWS S3 bucket endpoint URL.
63+
64+
Yields:
65+
Secret: A Kubernetes Secret configured with the provided AWS credentials and S3 endpoint.
66+
"""
67+
with kserve_s3_endpoint_secret(
68+
admin_client=admin_client,
69+
name="mlserver-models-bucket-secret",
70+
namespace=model_namespace.name,
71+
aws_access_key=aws_access_key_id,
72+
aws_secret_access_key=aws_secret_access_key,
73+
aws_s3_region=models_s3_bucket_region,
74+
aws_s3_endpoint=models_s3_bucket_endpoint,
75+
) as secret:
76+
yield secret
77+
78+
79+
@contextmanager
80+
def kserve_s3_endpoint_secret(
81+
admin_client: DynamicClient,
82+
name: str,
83+
namespace: str,
84+
aws_access_key: str,
85+
aws_secret_access_key: str,
86+
aws_s3_endpoint: str,
87+
aws_s3_region: str,
88+
) -> Generator[Secret, Any, Any]:
89+
"""
90+
Context manager that creates a temporary Kubernetes Secret for KServe
91+
to access an S3-compatible storage endpoint.
92+
93+
Args:
94+
admin_client (DynamicClient): Kubernetes dynamic client for resource operations.
95+
name (str): Name of the Secret resource.
96+
namespace (str): Kubernetes namespace in which to create the Secret.
97+
aws_access_key (str): AWS access key ID for authentication.
98+
aws_secret_access_key (str): AWS secret access key for authentication.
99+
aws_s3_endpoint (str): S3 endpoint URL (e.g., https://s3.example.com).
100+
aws_s3_region (str): AWS region for the S3 service.
101+
102+
Yields:
103+
Secret: The created Kubernetes Secret object within the context.
104+
"""
105+
with Secret(
106+
client=admin_client,
107+
name=name,
108+
namespace=namespace,
109+
annotations={
110+
"serving.kserve.io/s3-endpoint": (aws_s3_endpoint.replace("https://", "").replace("http://", "")),
111+
"serving.kserve.io/s3-region": aws_s3_region,
112+
"serving.kserve.io/s3-useanoncredential": "false",
113+
"serving.kserve.io/s3-verifyssl": "0",
114+
"serving.kserve.io/s3-usehttps": "1",
115+
},
116+
string_data={
117+
"AWS_ACCESS_KEY_ID": aws_access_key,
118+
"AWS_SECRET_ACCESS_KEY": aws_secret_access_key,
119+
},
120+
wait_for_resource=True,
121+
) as secret:
122+
yield secret

‎tests/model_serving/model_runtime/__init__.py‎

Whitespace-only changes.

‎tests/model_serving/model_runtime/mlserver/conftest.py‎

Lines changed: 0 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,6 @@
3030
TEMPLATE_MAP,
3131
TEMPLATE_FILE_PATH,
3232
)
33-
from tests.model_serving.model_runtime.mlserver.utils import kserve_s3_endpoint_secret
3433

3534
from utilities.constants import (
3635
KServeDeploymentType,
@@ -48,20 +47,6 @@
4847
LOGGER = get_logger(name=__name__)
4948

5049

51-
@pytest.fixture(scope="session")
52-
def root_dir(pytestconfig: pytest.Config) -> Any:
53-
"""
54-
Provides the root directory path of the pytest project for the entire test session.
55-
56-
Args:
57-
pytestconfig (pytest.Config): The pytest configuration object.
58-
59-
Returns:
60-
Any: The root path of the pytest project.
61-
"""
62-
return pytestconfig.rootpath
63-
64-
6550
@pytest.fixture(scope="class")
6651
def mlserver_grpc_serving_runtime_template(admin_client: DynamicClient) -> Generator[Template, None, None]:
6752
"""
@@ -102,20 +87,6 @@ def mlserver_rest_serving_runtime_template(admin_client: DynamicClient) -> Gener
10287
yield tp
10388

10489

105-
@pytest.fixture(scope="class")
106-
def protocol(request: pytest.FixtureRequest) -> str:
107-
"""
108-
Provides the protocol type parameter for the test class.
109-
110-
Args:
111-
request (pytest.FixtureRequest): The pytest fixture request object.
112-
113-
Returns:
114-
str: The protocol type specified in the test parameter.
115-
"""
116-
return request.param["protocol_type"]
117-
118-
11990
@pytest.fixture(scope="class")
12091
def mlserver_serving_runtime(
12192
request: pytest.FixtureRequest,
@@ -229,41 +200,6 @@ def mlserver_model_service_account(admin_client: DynamicClient, kserve_s3_secret
229200
yield sa
230201

231202

232-
@pytest.fixture(scope="class")
233-
def kserve_s3_secret(
234-
admin_client: DynamicClient,
235-
model_namespace: Namespace,
236-
aws_access_key_id: str,
237-
aws_secret_access_key: str,
238-
models_s3_bucket_region: str,
239-
models_s3_bucket_endpoint: str,
240-
) -> Secret:
241-
"""
242-
Creates and yields a Kubernetes Secret configured for S3 access in KServe.
243-
244-
Args:
245-
admin_client (DynamicClient): Kubernetes dynamic client.
246-
model_namespace (Namespace): Namespace where the secret will be created.
247-
aws_access_key_id (str): AWS access key ID.
248-
aws_secret_access_key (str): AWS secret access key.
249-
models_s3_bucket_region (str): AWS S3 bucket region.
250-
models_s3_bucket_endpoint (str): AWS S3 bucket endpoint URL.
251-
252-
Yields:
253-
Secret: A Kubernetes Secret configured with the provided AWS credentials and S3 endpoint.
254-
"""
255-
with kserve_s3_endpoint_secret(
256-
admin_client=admin_client,
257-
name="mlserver-models-bucket-secret",
258-
namespace=model_namespace.name,
259-
aws_access_key=aws_access_key_id,
260-
aws_secret_access_key=aws_secret_access_key,
261-
aws_s3_region=models_s3_bucket_region,
262-
aws_s3_endpoint=models_s3_bucket_endpoint,
263-
) as secret:
264-
yield secret
265-
266-
267203
@pytest.fixture
268204
def mlserver_response_snapshot(snapshot: Any) -> Any:
269205
"""

‎tests/model_serving/model_runtime/mlserver/utils.py‎

Lines changed: 5 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -8,21 +8,16 @@
88
- Validating responses against snapshots
99
"""
1010

11-
import os
12-
import json
1311
import base64
12+
import json
13+
import os
1414
import subprocess
15-
from contextlib import contextmanager
16-
from typing import Generator, Any
15+
from typing import Any
1716

18-
import requests
1917
import portforward
20-
21-
from kubernetes.dynamic import DynamicClient
22-
from ocp_resources.secret import Secret
18+
import requests
2319
from ocp_resources.inference_service import InferenceService
2420

25-
from utilities.constants import KServeDeploymentType, Protocols
2621
from tests.model_serving.model_runtime.mlserver.constant import (
2722
MLSERVER_GRPC_REMOTE_PORT,
2823
LOCAL_HOST_URL,
@@ -33,52 +28,7 @@
3328
NON_DETERMINISTIC_OUTPUT,
3429
HUGGING_FACE_FRAMEWORK,
3530
)
36-
37-
38-
@contextmanager
39-
def kserve_s3_endpoint_secret(
40-
admin_client: DynamicClient,
41-
name: str,
42-
namespace: str,
43-
aws_access_key: str,
44-
aws_secret_access_key: str,
45-
aws_s3_endpoint: str,
46-
aws_s3_region: str,
47-
) -> Generator[Secret, Any, Any]:
48-
"""
49-
Context manager that creates a temporary Kubernetes Secret for KServe
50-
to access an S3-compatible storage endpoint.
51-
52-
Args:
53-
admin_client (DynamicClient): Kubernetes dynamic client for resource operations.
54-
name (str): Name of the Secret resource.
55-
namespace (str): Kubernetes namespace in which to create the Secret.
56-
aws_access_key (str): AWS access key ID for authentication.
57-
aws_secret_access_key (str): AWS secret access key for authentication.
58-
aws_s3_endpoint (str): S3 endpoint URL (e.g., https://s3.example.com).
59-
aws_s3_region (str): AWS region for the S3 service.
60-
61-
Yields:
62-
Secret: The created Kubernetes Secret object within the context.
63-
"""
64-
with Secret(
65-
client=admin_client,
66-
name=name,
67-
namespace=namespace,
68-
annotations={
69-
"serving.kserve.io/s3-endpoint": (aws_s3_endpoint.replace("https://", "").replace("http://", "")),
70-
"serving.kserve.io/s3-region": aws_s3_region,
71-
"serving.kserve.io/s3-useanoncredential": "false",
72-
"serving.kserve.io/s3-verifyssl": "0",
73-
"serving.kserve.io/s3-usehttps": "1",
74-
},
75-
string_data={
76-
"AWS_ACCESS_KEY_ID": aws_access_key,
77-
"AWS_SECRET_ACCESS_KEY": aws_secret_access_key,
78-
},
79-
wait_for_resource=True,
80-
) as secret:
81-
yield secret
31+
from utilities.constants import KServeDeploymentType, Protocols
8232

8333

8434
def send_rest_request(url: str, input_data: dict[str, Any], verify: bool = False) -> Any:

‎tests/model_serving/model_runtime/triton/__init__.py‎

Whitespace-only changes.

‎tests/model_serving/model_runtime/triton/basic_model_deployment/__init__.py‎

Whitespace-only changes.

‎tests/model_serving/model_runtime/triton/basic_model_deployment/__snapshots__/test_onnx_model/TestONNXModel.test_onnx_inference[densenetonnx-raw-grpc-deployment].json‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
{
2+
"id": "test1",
3+
"modelName": "densenetonnx",
4+
"modelVersion": "1",
5+
"outputs": [
6+
{
7+
"datatype": "FP32",
8+
"name": "fc6_1",
9+
"shape": [
10+
"1000"
11+
]
12+
}
13+
],
14+
"rawOutputContents": [
15+
"j7HswQiaFMH951jB5HoZwuU0nsGTYEfAvhMFwiJTScGe8h7B0nWHwXFKF8Feet7AVUkkwR3FrcFv5k/BFYgIwVCV88DQwj7BONwfwT3M7sEpmG7BcdxvwfaEP8GSPKTBsBtUwRqGucFSEgTCG6r7wc/gpcH1uczBHLTcwfSwvsFBJ6bBYjBewforrkDCkozBcp/fvx5Kzz/ghA9BjYVkwfHtXMH0UhXBDyaSwG+RfcEhMofBfjxKweHBKEBsMUPBDe3pwVAzwMEeniLBG+SNQOHTUEH5Xu5AhHxPQEyTDUCMWvlBuSjlwAyNasFPOgU9oPLYQQwBPsFdenTBDO2pQDTrI8Ex1ofBdHUfQZEIAEDXwco+Gq7uwc7TScF6AMW/3lcEwWMkUEA4sa3AUAgVQUzG6b89EA9AIG/TwCD6C0HA3wvCWqkJwi4QdMDnaljBKU4RwqsdA8IUDobBWhNfwWu2h8GbstvBBqB/wUp/C8IUIojAqMVDwSwT6D8CebrB2quNwCfPz8DGwGHBvuumwAbL3sAZnQnCm0/QwdfLAcIO6ZPBS9G0wXPHCsIrYr/Ber1Wwfv0CsBygCTCAvVRwYOJ60C24uPAteurwQwQvME+AQbCSVmRvod3LcBwDonAy6WNwY0sFj+lN41A6OlxwQ3EAsHB9PrBekWFwfVmz8EBjaPBUXaXwRuDTcHKCQbCbwK8wclGZsCPvwTC5mdCwaBDQcHaQszAYKUCwtiLpUDXW5jB1vzcPzJBQcH2DmvAzprYwVIIsMF+a5nB34G6wW5h8cGSyePBe9C6wbxc2zymyelAUhJXQXxlTcCYCCvAfcapwAL+KkLyJrG+CerdwMvCMkDjgojA1EquwEpx2cD5SqbBEXR2wP/U1sEfQZrBC1VhwXkFGsF4TMnB/G5+wUKBp8E13jbBmVu/vzJWJkHkNrW+GK9bwZTbDcEOkZPBPNgLwoP2776f44HBcqoNQWUno7/LdxrAKc4NwNoUCkEFUEBAitUSwUqGTsA5gTlAgL5SwCYelkDtRJy8wEE1wb72Uj7nXvxABunhQDhm8D8hAolBnaaEwIDCV0EyMq+9zwWuQEXiO8HO+6TBfGk0wUIHk8EzDp3BpxykwfcO2r/MtsrBBj4mwc/CLcG4e3HBFbcJQdYhwMDVbR3Bz1VmQP/3S0GJiylB662nwFysW0BdTB3BGPTxwIDTrUFQTyi/eVFiQShpF0H9o5NBPP0CQY07p8CBNI9AvhQiwQgcmEAWwNnAL9fyP2o9TsFK9mJBzqOTvyZ4jb5kFs7AJp4VwcIrukA+es5ApY2BwSSw/cCbl5xAFv+HP/+iNUAcXDrBFG2swOZX1kFmpZnAwlN/wdPbm0Az8CzBaJD8QAtgzUAGaPs/4+CaP+fC4b5meIZBCKq0QQjKp0EqwWZBsfelQcyvrcH9hp7BV87awQozYMEbijnBBUWGwe5278DplybBVPHJwTmOPsB0Y8DBCQEzwUSSnMG0xI9B32YVQVCVmz+qO7/BtbR/P2cp5sEwi/7AfQMswXLxhcEOpJrBjEKTwa8vicFyUmVAhfe2wcaF88FTKb/BisjBwUg2m8E/ktLBJdADwcFVqsBcjWFAm7VNQQ+MkMGJ5qLBjzfOwHhhq0CeNvbADDGDwZu7uEHdT0VBL8agQatztsAiUrTA4YetwL/tEMKUzJrBZn12wdv7WUBzlOrAzHcbQfBpkMGgz5RBiO2ywT8YE8Ef8r+/N+uKQcNIq7/q+ubBAztBwFmGsMAUqNhA8VrHv3tL1MH3bjrBfc0Two36BsJS25TBTo/IwZpUZ8GCqN/B1W+0wbA+HMId+hPCCukCwt2ELMIgx/XBS9nOwabfEMKecP7BubcFwkF9fMEbvjPBpyK0wfFnoMEd46vBcESWwfNGU8EoUgTBWGUOwuV+n8HA4AvC4YMzwWtMLMG22zjBQmzNwfmDCcFXQR/BIZpBwSLh+8FiWOzBmjvgwK15GMGx2OzBVjkdwlFrBsLOFoXBeUbgwAn128G0qzXBVeXVwZqnMsCTXYXB8Ne7wYJNy8Fct5fBQrFWwbdCB8IA3ZjBdh1TwVsdFMLGl0TBzAOKwQv4gMENhsLBOu+ZwfLBvMEEW/FB6cBXwfe8rECl+YlAa4tDQfUL2MFu7RxBpBJGQYhUV8GDWZlBrBoewbDhg0KMwKXBamduQeDGiEEdyO4+Mlz9QGRUQEBYVtvAjuQdQgwSt0F1hGU/+EEKwYKIhjwLWq9BdpDjQeiwpL9hs6rBW+FXQajYfkH6/ilAfo7xQHSeM8Fe2wVCnT4FQVSc7sEX2P/BagK2v16wrUFHsZ9AaB+FQZaXj0Cf93BBVs6xQZQ3fUHDa5nAHAbtQGAbUz8jyH9B6KUOQWwbIcGqEMnBPMEMwEGmEMFbPTXBo+6qQG0nVkHf+Za/0myoQZPdMkLaXazBCm5PQVeYTsG9NttA96qjQNTxqEEspttBbXwTwfMUyr+TH1jB522FQb3CjEHE9blBiBwPwbmXMsEpV51ASCiEwS4KQkE4xADBJoAvwd/gG0DkIThCWpo7vpMEw0B7QDRBhA3pwER+h0CP08tB0+gKQZKZykAgfJtBgp07QeyfxkAgC2HBadQawXvJB8HjsYJBkkKPQBwAtcHpIw3B+fu3QVCmtUC7gojBcCuFwb9Eqr/lKQBCyoZxQipOVkKyvBxCyUH0QGTuVUGMSEJAFwtvwcWdaMA3XHNBMO/ZQCU+PcGooae/xrYJQrFBnsDSvd1BuHBkwcdTX0Fg+dVB6d3LQaeSAUGaoD6/uhfDwHh5BELAARlCt6bYPzrTnMHHu2VCOZSXQeFpHkJoSBw/ZubDQYrP6UGB6JjBA3WRwcoOw0FCFXO+2QLYwY7/+UG64MhB78EqQlo0wkBmPfJBR+GBQSSSq8F7hitBeXyYQWoVTUIG5sxBIv9/QQcEl0Hh7TDBMh1eQUmPGkFfpZ1AJJLxvnV7L0ItDLRBrXMrwK9A1cCWI9NAZcMtwWCjbkF04tVABze8QXgWkMBVhJDBgJxTQQ9pK0GLj/JBlzsXQCSF7kEt0xnALs9MwcM0vkF9SWFBlm5JQONpQMHfU6ZBOwKAQdldUMACuOtBgd9UQaUGAUEYRW9AlyuSQbam1b5uapZATkgNQYoq7UDvdL7AXCSPP3YZScFCXLdAnNqbwaTLwUHvcslBv7HRQer9WEHCzcJAJh/iwY4Px0HvUxtCQzOLvw6DIL/fbrZAGFHGQUhnE0EM4co//V7WQPaCEkGaeLe/htV4QYG6AkJQIh9B1HL6QKKQcEIC0XVB81ZMwdWbl0AM28zAhRWjQYFShcHI0DBBJxWcQQnZ28EHaTZB8I4BwaSYlEHZU0BCaq1WQaHgEsIaAcBBF7geQUMBGsBCZOhBfk/2wCTKpMHtnRhC1cixQZHeAkFTNdNBYDItvtMy1UEtz8ZB7OrswCPxC8G8lQFCgsjHQaO4xsB7STrBkkqBQZQmIkHmtiBAAUUZQWQUpUGuTqxBfGC2wdzNGEFilmrB2/7nwPBu/UHEvuTARyZOQaPiREEC3qnAyetSwf8LJUHJZQW/+ZShQM56vEFEufhAmWcGwgRzskFu4IdAg35LPlyxikHm26hB2TsGQkkIU8F494xBixZtQEuchEF7ex0/+m1AwT+OocDdzpLAmgD7waPq3L7jJIBBFO4WP2Tut7/e3YQ+vXdJwI1l3cCH1f/AgoOmQIrgaMBY3alBif6wwLBwa7xDSsU/X8ucPz6ZI0EiPL7AEWXSQBbhtcA+HetBKKWHQSqJ6sCxoLFBBjGkQGFAAsJddTrBbAC/QbJweUGe8AU/90MkQXBthEHZYVNCnVU3QpPgZMFDbARCzcoewQwO20EkbBzB5CzcQY45vsBd/BVBQba1PmSu3EFgpmNBx17XwPSO8UEGqUBBRhYgwAWWukDrjyPBdSWmwYja2EFsPVfBmqUwQd0BmkGQppZBTIq6QXxqtcGjoLM/a7nUPkLXc0H1495A5ZL6wM6MT0IRhOS/7PJfwV8WJ8DoLJpBDcwoQUqe08Gk09tBg3sdQSaUAMLa9MvAUhsIQoI5A0Ik0grAdlmnQeFIm0EtNzLBQkKkv8Dsej8dsfVBjosPQXB5XsGdRB5AfcrBwJJht0HaXINAvjonwUjtBkJwxxNCEV99QeqeWsFq94NBS60GQUpZuUEahzHAng5fQWlPyED7KGBBjAGvQfSrnMGWk2rABBqPQVAe1MDwPifAwhADweiiP0GusaRB7T4VQBRFPsHOIkPAl/WHQWkKdEGyBJK/w0GeQdRmG79K31xBRC7+P3m3hkE+JRLBpLioQbPjJcFDZgRBbQiiwKqElUBXlDtCshkEQZn+XsF/7oFAaAHSQZeZokFaF19BlD/ywO7RyUHuWPhBqt3IQc65+MBO5SdBwD0tQTlfRMF/7pbBflZ9wZ+G28H/U/1BFOl0QRFvfkEx5qBAm4W8QX5Gg0AqZS9BlWL0QE7zQ0Ho+z5BVfA4QretvMFo2M9AKJ5qQuwqOEHbDxlC9Tj8QevsQkBuA9JANUyrQHjpAcGxhVPA6TSkwRnwK0JkieBAmsIcv3IrDkG8DYnAgLAbQXb1zz8XsBfBKqZnwfVPOUJdVJTA+3ekQfzFFcGBBrVBo28fQF+sqsEJNzBB0KRxwc+f0EG8G4/Bh6ylQWqSgsBo86LAN1O4Qc6glUHyWKk/z1ehwVowW0FpEXTB8I4tQccZzUB4bW/ACyo9QeFc8kHPgAzBufISQTr2k0EnqYBBrNTLQafAFEKAxSZChk/RQGO7AcH5HyNCiYC7Qcl4Q0Ez2FFBiUciwXfkjEEH7bVBdqTWQfqEzT+tLu5Am44kwaE6CMLyB/3BCNUGQPXLQUKINg9BvJfcQfTl40EwEtpBFRolwVwyUkFOuDRB4qulQMRm/ECQZfXAizALwZSIVUEK04BAXDzWQK8+kEHIXahBIcivwP+cCEETpBZATg/5vgO1wEG/ehZBS9tCvzQ6OMFGYavBiOItQFbhlD84x1rAXtGWvu4DG8FKBXBBc7CmQe8flUGj4rVBNZiKQTCbV0GafxHBqSkEQjsY6kCxcb1AJfQGQdV2b8A92AlBi650QR3MzcBBeghB/qsGwQDxRkBaIhvBz08IQpVeLELtzn9C+RQAQkDFxsCsf+NAOh3jP12qDMFmb4DBcTwJwZ6XQcH1HyhBpIAdvzIwJcHiFTHBg2aAwEQXl0Fb+1rAIB51wGl/oEH4CArCz2CiQb/9ib+zk4jBK4Fhv2gTu8EdAIHBVEv5wSF4BcIWeF/BYBGxQOPlvcEElpJA52tTwQ=="
16+
]
17+
}

0 commit comments

Comments
 (0)