Move 36 check test suites onto the agnostic HTTP doubles

amazon_msk/tests/conftest.py

@@ -8,7 +8,7 @@
 import pytest
 
 from datadog_checks.dev import docker_run
-from datadog_checks.dev.http import MockResponse
+from datadog_checks.dev.http import MockHTTPResponse
 
 from . import common
 
@@ -25,19 +25,15 @@ def dd_environment():
         yield common.INSTANCE, common.E2E_METADATA
 
 
-def mock_requests_get(url, *args, **kwargs):
+def route_metrics_fixture(url, *args, **kwargs):
     exporter_type = 'jmx' if urlparse(url).port == common.JMX_PORT else 'node'
-    return MockResponse(file_path=common.get_metrics_fixture_path(exporter_type))
+    return MockHTTPResponse(file_path=common.get_metrics_fixture_path(exporter_type))
 
 
 @pytest.fixture
-def mock_data():
-    # Mock requests.get because it is used internally within boto3
-    with (
-        mock.patch('requests.get', side_effect=mock_requests_get, autospec=True),
-        mock.patch('requests.Session.get', side_effect=mock_requests_get),
-    ):
-        yield
+def mock_data(mock_openmetrics_http):
+    mock_openmetrics_http.get.side_effect = route_metrics_fixture
+    yield
 
 
 @pytest.fixture

apache/tests/conftest.py

@@ -4,16 +4,19 @@
 
 import os
 
-import mock
 import pytest
-import requests
 
 from datadog_checks.apache import Apache
+from datadog_checks.base.utils.http import create_http_client
 from datadog_checks.dev import docker_run
 from datadog_checks.dev.conditions import CheckEndpoints, WaitFor
+from datadog_checks.dev.http import MockHTTPResponse
 
 from .common import AUTO_STATUS_URL, BASE_URL, CHECK_NAME, HERE, STATUS_CONFIG, STATUS_URL
 
+# Library-agnostic HTTP client shared by the e2e helpers and fixtures below.
+http_client = create_http_client({}, {})
+
 
 @pytest.fixture(scope="session")
 def dd_environment():
@@ -33,34 +36,34 @@ def dd_environment():
 
 def generate_metrics():
     for _ in range(0, 100):
-        requests.get(BASE_URL)
+        http_client.get(BASE_URL)
 
 
 def check_status_page_ready():
     """
     Some status info we need for metrics do not appear immediately.
     This check help waiting for the full status page.
     """
-    resp = requests.get(AUTO_STATUS_URL)
+    resp = http_client.get(AUTO_STATUS_URL)
     data = resp.content.decode('utf-8')
     assert 'ReqPerSec: ' in data
     assert 'CPULoad: ' in data
 
 
 @pytest.fixture
-def mock_hide_server_version():
-    req = mock.MagicMock()
-    with mock.patch('datadog_checks.base.utils.http.requests.Session', return_value=req):
-
-        def mock_requests_get_headers(*args, **kwargs):
-            r = requests.get(*args, **kwargs)
-            old_iter = r.iter_lines
-            r.iter_lines = mock.MagicMock()
-            r.iter_lines.return_value = (l for l in old_iter(decode_unicode=True) if 'ServerVersion' not in l)
-            return r
-
-        req.get = mock_requests_get_headers
-        yield
+def mock_hide_server_version(mock_http):
+    def filter_server_version(url, *args, **kwargs):
+        r = http_client.get(url, **kwargs)
+        content = '\n'.join(line for line in r.text.splitlines() if 'ServerVersion' not in line)
+        return MockHTTPResponse(
+            content=content,
+            status_code=r.status_code,
+            headers=dict(r.headers),
+            url=r.url,
+        )
+
+    mock_http.get.side_effect = filter_server_version
+    yield
 
 
 @pytest.fixture

azure_iot_edge/tests/test_check.py

@@ -5,12 +5,12 @@
 from typing import Callable  # noqa: F401
 
 import pytest
-import requests
 
 from datadog_checks.azure_iot_edge import AzureIoTEdgeCheck
 from datadog_checks.azure_iot_edge.types import Instance  # noqa: F401
 from datadog_checks.base.stubs.aggregator import AggregatorStub  # noqa: F401
 from datadog_checks.base.stubs.datadog_agent import DatadogAgentStub  # noqa: F401
+from datadog_checks.base.utils.http_exceptions import HTTPConnectionError
 from datadog_checks.dev.utils import get_metadata_metrics
 
 from . import common
@@ -104,7 +104,7 @@ def test_prometheus_endpoint_down(aggregator, mock_instance, option, url, servic
 
     check = AzureIoTEdgeCheck('azure_iot_edge', {}, [instance])
 
-    with pytest.raises(requests.ConnectionError):
+    with pytest.raises(HTTPConnectionError):
         check.check(instance)
 
     aggregator.assert_service_check(service_check, AzureIoTEdgeCheck.CRITICAL)

azure_iot_edge/tests/test_e2e.py

@@ -5,10 +5,10 @@
 from typing import Callable  # noqa: F401
 
 import pytest
-import requests
 
 from datadog_checks.azure_iot_edge import AzureIoTEdgeCheck
 from datadog_checks.base.stubs.aggregator import AggregatorStub  # noqa: F401
+from datadog_checks.base.utils.http_exceptions import HTTPSSLError
 
 from . import common
 
@@ -40,7 +40,7 @@ def test_bad_url_e2e(e2e_instance, dd_agent_check):
     bad_url_agent_instance = copy.deepcopy(e2e_instance)
     bad_url_agent_instance['edge_agent_prometheus_url'] = bad_url_hub_instance['edge_agent_prometheus_url'][:-2]
 
-    with pytest.raises(requests.exceptions.SSLError):
+    with pytest.raises(HTTPSSLError):
         dd_agent_check(bad_url_hub_instance, rate=True)
 
     aggregator = dd_agent_check(bad_url_agent_instance, rate=True)

bentoml/tests/test_unit.py

@@ -5,7 +5,7 @@
 
 from datadog_checks.base.constants import ServiceCheck
 from datadog_checks.bentoml import BentomlCheck
-from datadog_checks.dev.http import MockResponse
+from datadog_checks.dev.http import MockHTTPResponse
 from datadog_checks.dev.utils import get_metadata_metrics
 
 from .common import (
@@ -16,8 +16,8 @@
 )
 
 
-def test_bentoml_mock_metrics(dd_run_check, aggregator, mock_http_response):
-    mock_http_response(file_path=get_fixture_path('metrics.txt'))
+def test_bentoml_mock_metrics(dd_run_check, aggregator, mock_http):
+    mock_http.get.return_value = MockHTTPResponse(file_path=get_fixture_path('metrics.txt'))
 
     check = BentomlCheck('bentoml', {}, [OM_MOCKED_INSTANCE])
     dd_run_check(check)
@@ -34,23 +34,29 @@ def test_bentoml_mock_metrics(dd_run_check, aggregator, mock_http_response):
     aggregator.assert_service_check('bentoml.openmetrics.health', ServiceCheck.OK)
 
 
-def test_bentoml_mock_invalid_endpoint(dd_run_check, aggregator, mock_http_response):
-    mock_http_response(status_code=503)
+def test_bentoml_mock_invalid_endpoint(dd_run_check, aggregator, mock_http):
+    mock_http.get.return_value = MockHTTPResponse(status_code=503)
     check = BentomlCheck('bentoml', {}, [OM_MOCKED_INSTANCE])
     with pytest.raises(Exception):
         dd_run_check(check)
 
     aggregator.assert_service_check('bentoml.openmetrics.health', ServiceCheck.CRITICAL)
 
 
-def test_bentoml_mock_valid_endpoint_invalid_health(dd_run_check, aggregator, mock_http_response_per_endpoint):
-    mock_http_response_per_endpoint(
-        {
-            'http://bentoml:3000/metrics': [MockResponse(file_path=get_fixture_path('metrics.txt'))],
-            'http://bentoml:3000//livez': [MockResponse(status_code=500)],
-            'http://bentoml:3000//readyz': [MockResponse(status_code=500)],
-        }
-    )
+def test_bentoml_mock_valid_endpoint_invalid_health(dd_run_check, aggregator, mock_http):
+    responses = {
+        'http://bentoml:3000/metrics': MockHTTPResponse(file_path=get_fixture_path('metrics.txt')),
+        'http://bentoml:3000//livez': MockHTTPResponse(status_code=500),
+        'http://bentoml:3000//readyz': MockHTTPResponse(status_code=500),
+    }
+
+    def get_response(url: str, **_kwargs: object) -> MockHTTPResponse:
+        try:
+            return responses[url]
+        except KeyError:
+            raise ValueError(f'Endpoint {url} not found in mocked responses') from None
+
+    mock_http.get.side_effect = get_response
 
     check = BentomlCheck('bentoml', {}, [OM_MOCKED_INSTANCE])
     dd_run_check(check)

cert_manager/tests/test_cert_manager.py

@@ -3,22 +3,18 @@
 # Licensed under a 3-clause BSD style license (see LICENSE)
 import os
 
-import mock
 import pytest
 
 from datadog_checks.cert_manager import CertManagerCheck
-from datadog_checks.dev.http import MockResponse
+from datadog_checks.dev.http import MockHTTPResponse
 
 from .common import ACME_METRICS, CERT_METRICS, CONTROLLER_METRICS, MOCK_INSTANCE
 
 
 @pytest.fixture()
-def error_metrics():
-    with mock.patch(
-        'requests.Session.get',
-        return_value=mock.MagicMock(status_code=502, headers={'Content-Type': "text/plain"}),
-    ):
-        yield
+def mock_http_error_response(mock_http):
+    mock_http.get.return_value = MockHTTPResponse(status_code=502, headers={'Content-Type': "text/plain"})
+    yield
 
 
 @pytest.mark.unit
@@ -28,14 +24,14 @@ def test_config():
 
 
 @pytest.mark.unit
-def test_check(aggregator, dd_run_check):
+def test_check(aggregator, dd_run_check, mock_http):
     check = CertManagerCheck('cert_manager', {}, [MOCK_INSTANCE])
 
     def mock_requests_get(url, *args, **kwargs):
-        return MockResponse(file_path=os.path.join(os.path.dirname(__file__), 'fixtures', 'cert_manager.txt'))
+        return MockHTTPResponse(file_path=os.path.join(os.path.dirname(__file__), 'fixtures', 'cert_manager.txt'))
 
-    with mock.patch('requests.Session.get', side_effect=mock_requests_get, autospec=True):
-        dd_run_check(check)
+    mock_http.get.side_effect = mock_requests_get
+    dd_run_check(check)
 
     expected_metrics = dict(CERT_METRICS)
     expected_metrics.update(CONTROLLER_METRICS)
@@ -55,7 +51,7 @@ def mock_requests_get(url, *args, **kwargs):
 
 
 @pytest.mark.unit
-def test_openmetrics_error(aggregator, instance, error_metrics):
+def test_openmetrics_error(aggregator, instance, mock_http_error_response):
     check = CertManagerCheck('cert_manager', {}, [MOCK_INSTANCE])
     with pytest.raises(Exception):
         check.check(MOCK_INSTANCE)

cilium/tests/conftest.py

@@ -3,13 +3,13 @@
 # Licensed under a 3-clause BSD style license (see LICENSE)
 import os
 
-import mock
 import pytest
 
 from datadog_checks.base.utils.common import get_docker_hostname
 from datadog_checks.cilium import CiliumCheck
 from datadog_checks.dev import run_command
 from datadog_checks.dev.conditions import WaitFor
+from datadog_checks.dev.http import MockHTTPResponse
 from datadog_checks.dev.kind import kind_run
 from datadog_checks.dev.kube_port_forward import port_forward
 from datadog_checks.dev.utils import get_active_env
@@ -247,28 +247,18 @@ def operator_instance_use_openmetrics():
 
 
 @pytest.fixture()
-def mock_agent_data():
+def mock_agent_data(mock_openmetrics_http):
     f_name = os.path.join(os.path.dirname(__file__), "fixtures", "agent_metrics.txt")
     with open(f_name, "r") as f:
         text_data = f.read()
-    with mock.patch(
-        'requests.Session.get',
-        return_value=mock.MagicMock(
-            status_code=200, iter_lines=lambda **kwargs: text_data.split("\n"), headers={"Content-Type": "text/plain"}
-        ),
-    ):
-        yield
+    mock_openmetrics_http.get.return_value = MockHTTPResponse(content=text_data, headers={"Content-Type": "text/plain"})
+    yield
 
 
 @pytest.fixture()
-def mock_operator_data():
+def mock_operator_data(mock_openmetrics_http):
     f_name = os.path.join(os.path.dirname(__file__), "fixtures", "operator_metrics.txt")
     with open(f_name, "r") as f:
         text_data = f.read()
-    with mock.patch(
-        'requests.Session.get',
-        return_value=mock.MagicMock(
-            status_code=200, iter_lines=lambda **kwargs: text_data.split("\n"), headers={"Content-Type": "text/plain"}
-        ),
-    ):
-        yield
+    mock_openmetrics_http.get.return_value = MockHTTPResponse(content=text_data, headers={"Content-Type": "text/plain"})
+    yield

crio/tests/test_crio.py

@@ -3,7 +3,6 @@
 # Licensed under a 3-clause BSD style license (see LICENSE)
 import os
 
-import mock
 import pytest
 
 from datadog_checks.base import AgentCheck
@@ -14,17 +13,11 @@
 
 
 @pytest.fixture()
-def mock_data():
-    f_name = os.path.join(os.path.dirname(__file__), 'fixtures', 'metrics.txt')
-    with open(f_name, 'r') as f:
-        text_data = f.read()
-    with mock.patch(
-        'requests.Session.get',
-        return_value=mock.MagicMock(
-            status_code=200, iter_lines=lambda **kwargs: text_data.split("\n"), headers={'Content-Type': "text/plain"}
-        ),
-    ):
-        yield
+def mock_data(mock_openmetrics_http, mock_response):
+    mock_openmetrics_http.get.return_value = mock_response(
+        file_path=os.path.join(os.path.dirname(__file__), 'fixtures', 'metrics.txt'),
+        headers={'Content-Type': 'text/plain'},
+    )
 
 
 def test_crio(aggregator, mock_data, instance):

datadog_cluster_agent/tests/conftest.py

@@ -4,9 +4,10 @@
 import os
 from copy import deepcopy
 
-import mock
 import pytest
 
+from datadog_checks.dev.http import MockHTTPResponse
+
 INSTANCE = {'prometheus_url': 'http://localhost:5000/metrics'}
 
 
@@ -21,14 +22,9 @@ def instance():
 
 
 @pytest.fixture()
-def mock_metrics_endpoint():
+def mock_metrics_endpoint(mock_openmetrics_http):
     f_name = os.path.join(os.path.dirname(__file__), 'fixtures', 'metrics.txt')
     with open(f_name, 'r') as f:
         text_data = f.read()
-    with mock.patch(
-        'requests.Session.get',
-        return_value=mock.MagicMock(
-            status_code=200, iter_lines=lambda **kwargs: text_data.split("\n"), headers={'Content-Type': "text/plain"}
-        ),
-    ):
-        yield
+    mock_openmetrics_http.get.return_value = MockHTTPResponse(content=text_data, headers={'Content-Type': 'text/plain'})
+    yield