Skip to content

Commit 6965b58

Browse files
alvarolopezalvarolopez
committed
feat: add utility to check V2 and V3 compatibility
1 parent eeda50e commit 6965b58

File tree

4 files changed

+298
-0
lines changed

4 files changed

+298
-0
lines changed

‎deepaas/api/__init__.py

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,15 @@ def get_fastapi_app(
9999
response_model=responses.VersionsAndLinks,
100100
)
101101

102+
# Add a redirect from the old swagger.json to the new openapi.json
103+
APP.add_api_route(
104+
f"{base_path}/swagger.json",
105+
APP.openapi,
106+
methods=["GET"],
107+
summary="Get OpenAPI schema",
108+
tags=["version"],
109+
)
110+
102111
return APP
103112

104113

‎deepaas/cmd/deepaas_test_v2_v3.py

Lines changed: 287 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,287 @@
1+
#!/usr/bin/env python
2+
# -*- coding: utf-8 -*-
3+
4+
# Copyright 2018 Spanish National Research Council (CSIC)
5+
#
6+
# Licensed under the Apache License, Version 2.0 (the "License"); you may
7+
# not use this file except in compliance with the License. You may obtain
8+
# a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
14+
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
15+
# License for the specific language governing permissions and limitations
16+
# under the License.
17+
18+
import argparse
19+
import json
20+
import sys
21+
import requests
22+
23+
24+
def get_openapi_spec(base_url):
25+
"""Retrieve the OpenAPI spec from the given base URL."""
26+
response = requests.get(f"{base_url}/swagger.json", timeout=5)
27+
response.raise_for_status() # Raise an error for bad responses
28+
return response.json()
29+
30+
31+
def compare_endpoints(old_spec, new_spec, model_name):
32+
"""Compare endpoints in both OpenAPI specs.
33+
34+
This method ignoring methods that have been removed.
35+
"""
36+
old_paths = set(old_spec['paths'].keys())
37+
new_paths = set(new_spec['paths'].keys())
38+
39+
removed_paths = [f'/v2/models/{model_name}/train']
40+
for path in removed_paths:
41+
old_paths.discard(path)
42+
43+
# Find paths that are in new but not in old
44+
missing_in_old = new_paths - old_paths
45+
46+
if missing_in_old:
47+
print("The following endpoints are missing in the old API:")
48+
for path in missing_in_old:
49+
print(f" - {path}")
50+
51+
# Check if all old paths (except removed) are in the new API
52+
missing_in_new = old_paths - new_paths
53+
54+
if missing_in_new:
55+
print("The following endpoints are missing in the new API:")
56+
for path in missing_in_new:
57+
print(f" - {path}")
58+
59+
return old_paths.intersection(new_paths)
60+
61+
62+
def test_endpoint(old_url, new_url, method, endpoint, debug=False):
63+
"""Test both endpoints and check if responses are equivalent."""
64+
old_response = requests.request(method, old_url + endpoint)
65+
new_response = requests.request(method, new_url + endpoint)
66+
67+
# If both return 405 Method Not Allowed, that's expected for some method/endpoint combinations
68+
if old_response.status_code == 405 and new_response.status_code == 405:
69+
return True
70+
71+
if old_response.status_code != new_response.status_code:
72+
error_msg = (
73+
f"Status code mismatch for {method} {endpoint}: "
74+
f"Old: {old_response.status_code}, New: {new_response.status_code}"
75+
)
76+
if debug:
77+
try:
78+
old_body = old_response.json()
79+
new_body = new_response.json()
80+
error_msg += f"\nOld response: {json.dumps(old_body, indent=2)}"
81+
error_msg += f"\nNew response: {json.dumps(new_body, indent=2)}"
82+
except ValueError:
83+
old_text = old_response.text[:500] + "..." if len(old_response.text) > 500 else old_response.text
84+
new_text = new_response.text[:500] + "..." if len(new_response.text) > 500 else new_response.text
85+
error_msg += f"\nOld response: {old_text}"
86+
error_msg += f"\nNew response: {new_text}"
87+
88+
raise AssertionError(error_msg)
89+
90+
# Only compare JSON responses if status code indicates success
91+
if 200 <= old_response.status_code < 300:
92+
try:
93+
old_data = old_response.json()
94+
new_data = new_response.json()
95+
96+
if old_data != new_data:
97+
error_msg = (
98+
f"Response mismatch for {method} {endpoint}"
99+
)
100+
if debug:
101+
error_msg += f"\nOld response: {json.dumps(old_data, indent=2)}"
102+
error_msg += f"\nNew response: {json.dumps(new_data, indent=2)}"
103+
104+
# Find differences in keys
105+
old_keys = set(old_data.keys() if isinstance(old_data, dict) else range(len(old_data)))
106+
new_keys = set(new_data.keys() if isinstance(new_data, dict) else range(len(new_data)))
107+
108+
if old_keys != new_keys:
109+
only_in_old = old_keys - new_keys
110+
only_in_new = new_keys - old_keys
111+
112+
if only_in_old:
113+
error_msg += f"\nKeys only in old response: {sorted(only_in_old)}"
114+
if only_in_new:
115+
error_msg += f"\nKeys only in new response: {sorted(only_in_new)}"
116+
117+
# Show differing values for common keys
118+
if isinstance(old_data, dict) and isinstance(new_data, dict):
119+
common_keys = old_keys.intersection(new_keys)
120+
different_values = {k: (old_data[k], new_data[k]) for k in common_keys if old_data[k] != new_data[k]}
121+
122+
if different_values:
123+
error_msg += "\nDiffering values for common keys:"
124+
for k, (old_val, new_val) in different_values.items():
125+
error_msg += f"\n {k}:"
126+
error_msg += f"\n Old: {old_val}"
127+
error_msg += f"\n New: {new_val}"
128+
129+
raise AssertionError(error_msg)
130+
except ValueError as e:
131+
if debug:
132+
raise AssertionError(f"JSON parsing failed: {e}\nOld: {old_response.text}\nNew: {new_response.text}")
133+
else:
134+
raise AssertionError(f"JSON parsing failed: {e}")
135+
136+
return True
137+
138+
139+
def run_tests(old_api_url, new_api_url, model_name, debug=False):
140+
"""Run all tests and return results."""
141+
test_results = {
142+
"passed": 0,
143+
"failed": 0,
144+
"errors": []
145+
}
146+
147+
# Retrieve OpenAPI specifications
148+
try:
149+
old_openapi_spec = get_openapi_spec(old_api_url)
150+
new_openapi_spec = get_openapi_spec(new_api_url)
151+
except requests.exceptions.RequestException as e:
152+
test_results["errors"].append(f"Failed to retrieve OpenAPI specs: {e}")
153+
return test_results
154+
155+
# Compare endpoints and get common paths
156+
common_paths = compare_endpoints(old_openapi_spec, new_openapi_spec, model_name)
157+
158+
# Testing the endpoints
159+
for endpoint in common_paths:
160+
# We will loop through all HTTP methods available in the old API
161+
for method in ['GET', 'POST', 'PUT', 'DELETE']:
162+
try:
163+
test_endpoint(old_api_url, new_api_url, method, endpoint, debug)
164+
print(f"\033[92mTest passed for {method}\033[0m {endpoint}")
165+
test_results["passed"] += 1
166+
except Exception as e:
167+
print(f"\033[91mTest failed for {method}\033[0m {endpoint}")
168+
test_results["failed"] += 1
169+
test_results["errors"].append(f"{method} {endpoint}: {str(e)}")
170+
if debug:
171+
print(f" Error details: {str(e)}")
172+
173+
return test_results
174+
175+
176+
# For pytest integration
177+
def test_api_endpoints(old_api_url, new_api_url, model_name, debug=False):
178+
"""Pytest-compatible test function."""
179+
import pytest
180+
181+
# Retrieve OpenAPI specifications
182+
old_openapi_spec = get_openapi_spec(old_api_url)
183+
new_openapi_spec = get_openapi_spec(new_api_url)
184+
185+
# Compare endpoints and get common paths
186+
common_paths = compare_endpoints(old_openapi_spec, new_openapi_spec, model_name)
187+
188+
# Testing the endpoints
189+
for endpoint in common_paths:
190+
# We will loop through all HTTP methods available in the old API
191+
for method in ['GET', 'POST', 'PUT', 'DELETE']:
192+
try:
193+
assert test_endpoint(old_api_url, new_api_url, method, endpoint, debug)
194+
except AssertionError as e:
195+
pytest.fail(str(e))
196+
197+
198+
def main():
199+
"""Main function to test the API endpoints."""
200+
parser = argparse.ArgumentParser(
201+
description="DEEPaaS API v2 to v3 migration test script"
202+
)
203+
parser.add_argument(
204+
"--old-api-base-url",
205+
default="http://127.0.0.1:5000",
206+
help="Base URL of the old API",
207+
)
208+
parser.add_argument(
209+
"--new-api-base-url",
210+
default="http://127.0.0.1:5001",
211+
help="Base URL of the new API",
212+
)
213+
parser.add_argument(
214+
"--model-name",
215+
default="demo_app",
216+
help="Name of the model to test. To make testing easier, you should provide "
217+
"the name of the model you want to test.",
218+
)
219+
parser.add_argument(
220+
"--debug",
221+
action="store_true",
222+
help="Show detailed debug information for failed tests"
223+
)
224+
225+
args = parser.parse_args()
226+
227+
results = run_tests(
228+
args.old_api_base_url,
229+
args.new_api_base_url,
230+
args.model_name,
231+
args.debug
232+
)
233+
234+
# Print summary
235+
print("\nTest Summary:")
236+
print(f"\tPassed: \033[92m{results['passed']}\033[0m")
237+
print(f"\tFailed: \033[91m{results['failed']}\033[0m")
238+
239+
if results["failed"] > 0 and args.debug:
240+
print("\nDetailed Errors:")
241+
for i, error in enumerate(results["errors"], 1):
242+
print(f"\n{i}. {error}")
243+
244+
# Return non-zero exit code if any tests failed
245+
if results["failed"] > 0:
246+
sys.exit(1)
247+
248+
249+
# Pytest integration functions
250+
def pytest_addoption(parser):
251+
"""Add command line options to pytest."""
252+
parser.addoption("--old-api-base-url", default="http://127.0.0.1:5000",
253+
help="Base URL of the old API")
254+
parser.addoption("--new-api-base-url", default="http://127.0.0.1:5001",
255+
help="Base URL of the new API")
256+
parser.addoption("--model-name", default="demo_app",
257+
help="Name of the model to test")
258+
parser.addoption("--debug", action="store_true",
259+
help="Show detailed debug information for failed tests")
260+
261+
262+
def pytest_generate_tests(metafunc):
263+
"""Generate test configurations."""
264+
if set(["old_api_url", "new_api_url", "model_name", "debug"]).intersection(metafunc.fixturenames):
265+
old_api_url = metafunc.config.getoption("old_api_base_url")
266+
new_api_url = metafunc.config.getoption("new_api_base_url")
267+
model_name = metafunc.config.getoption("model_name")
268+
debug = metafunc.config.getoption("debug")
269+
270+
params = {}
271+
if "old_api_url" in metafunc.fixturenames:
272+
params["old_api_url"] = [old_api_url]
273+
if "new_api_url" in metafunc.fixturenames:
274+
params["new_api_url"] = [new_api_url]
275+
if "model_name" in metafunc.fixturenames:
276+
params["model_name"] = [model_name]
277+
if "debug" in metafunc.fixturenames:
278+
params["debug"] = [debug]
279+
280+
metafunc.parametrize(",".join(params.keys()), list(zip(*params.values())))
281+
282+
283+
if __name__ == "__main__":
284+
main()
285+
else:
286+
# When imported as a module, provide the necessary pytest hooks and test functions
287+
__all__ = ["test_api_endpoints", "pytest_addoption", "pytest_generate_tests"]

‎deepaas/model/v2/wrapper.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import contextlib
2020
import functools
2121
import io
22+
2223
# import os
2324
# import tempfile
2425

‎pyproject.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ classifiers = [
4141
[tool.poetry.scripts]
4242
deepaas-run = "deepaas.cmd.run:main"
4343
deepaas-cli = "deepaas.cmd.cli:main"
44+
deepaas-test-v2-v3 = "deepaas.cmd.deepaas_test_v2_v3:main"
4445

4546
[tool.poetry.plugins] # Optional super table
4647

0 commit comments

Comments
 (0)