Allow HTTPS requests to use tls_ciphers (DataDog#20179)

* Add core TLS context changes

* Fix base package tests

* Run formatter

* Turn requests.X mocks into requests.Session.X

* Fix airflow unit tests

* Run formatter

* Add tls_ciphers to TLS Remote fetch_intermediate_certs

* Rework Adapter class

* Revert TLS protocol change

* Fix error in TlsContextAdapter

* Stop using TlsContextWrapper in RequestsWrapper

* Run formatter

* Add test for ciphers, switch naming to SSL and fix hole in persistent session logic

* Run formatter

* Naming

* Naming and test fixes

* Add unit test for debugging

* Fix consul test

* Fix consul config assertion

* Fail SSLContext certificate loading with WARNING instead of ERROR

* Lint

* Change mocks from requests.method to requests.Session.method

* Lint

* Fix couchbase tests

* Fix amazon_msk tests - mocking requests.get because the boto3 lib uses it

* Fix apache tests by reworking mock with requests.Session

* Fix arangodb tests by updating mock side effect

* Fix couch tests by updating mock with requests.Session

* Fix remaining occurrences of mock.patch("datadog_checks.base.utils.http.requests")

* Fix gitlab tests

* Fix harbor tests

* Fix hdfs_namenode tests

* Fix http_check test

* Fix mapreduce tests

* Fix proxysql test

* Fix spark tests

* Fix spark ssl test by fixing REMAPPER

* Fix twistlock tests

* Fix vault tests

* Fix vertica tests

* Fix yarn tests

* Fix avi_vantage test

* Fix http warning test

* Extract options to tls_config logic in separate function

* Add test for ssl_context initialization

* Run linter

* Fix falco test

* Fix `tls_verify` parsing in SSL context creation logic

* Add tests with custom server

* Fix request method use

* Fixing vsphere tests WIP

* Fix vsphere tests

* Run formatter

* Fix vault tests again

* Try fixing amazon_msk tests as they do not fail locally

* Try fixing amazon_msk tests as they do not fail locally 2

* Run formatter

* Try fixing amazon_msk tests as they do not fail locally 3

* Try fixing amazon_msk tests as they do not fail locally 4

* Try fixing amazon_msk tests as they do not fail locally 5

* Try fixing amazon_msk tests as they do not fail locally 6

* Try fixing http_check test

* Fix datadog_checks_dev test

* Fix datadog_checks_base test

* Try fixing tls_cipher test

* Run formatter

* Fix datadog_checks_base test 2

* Fix datadog_checks_base test 3

* Run formatter

* Fix datadog_checks_base test final

* Add changelog

* Update base.py

* Use create_ssl_context in tls_remote

* Apply suggestions from code review

Co-authored-by: Ilia Kurenkov <ilia.kurenkov@datadoghq.com>

* Add _mount_new_ssl_adapter method

* Implement suggestions from code review

* Use ChainMaps

* Implement caching for HTTPS adapters

* Use pydantic for TlsConfig class

* Minor improvements and fixes

* Add tests

* Run formatter

* Allow tls_ca_cert to be bool for legacy compatibility

* Fix datadog_checks_base tests

* Fix default tls_verify value in go_expvar

* Fix proxysql and vertica tests

* Run formatter and fix linting

* Add go_expvar changelog

* Fix base tests

* Fix intermediate_certs_tests for windows

* Fix intermediate_certs_tests for windows in a better way

* tweak how we create session and mount adapter

* Remove unused func and mounting https adapters when creating session

* Mount default https adapter in property

* Apply suggestions from code review

Co-authored-by: Ilia Kurenkov <ilia.kurenkov@datadoghq.com>

* Update datadog_checks_base/datadog_checks/base/utils/http.py

Co-authored-by: Ilia Kurenkov <ilia.kurenkov@datadoghq.com>

* Add TypeError for wrong TLS option types

* Run formatter

* Fix kubelet typo in test

* Run the ACTUAL formatter

* remove shallow network module

* Update datadog_checks_base/datadog_checks/base/utils/http.py

Co-authored-by: Ilia Kurenkov <ilia.kurenkov@datadoghq.com>

* Make SSLContextAdapter private and simplify super() calls

---------

Co-authored-by: Ilia Kurenkov <ilia.kurenkov@datadoghq.com>

Author: dkirov-dd

airflow/tests/test_unit.py

@@ -31,10 +31,11 @@ def test_service_checks_healthy_exp(aggregator, json_resp, expected_healthy_stat
     check = AirflowCheck('airflow', common.FULL_CONFIG, [instance])
 
     with mock.patch('datadog_checks.airflow.airflow.AirflowCheck._get_version', return_value=None):
-        with mock.patch('datadog_checks.base.utils.http.requests') as req:
+        mock_session = mock.MagicMock()
+        with mock.patch('datadog_checks.base.utils.http.requests.Session', return_value=mock_session):
             mock_resp = mock.MagicMock(status_code=200)
             mock_resp.json.side_effect = [json_resp]
-            req.get.return_value = mock_resp
+            mock_session.get.return_value = mock_resp
 
             check.check(None)
 
@@ -59,13 +60,14 @@ def test_service_checks_healthy_stable(
     check = AirflowCheck('airflow', common.FULL_CONFIG, [instance])
 
     with mock.patch('datadog_checks.airflow.airflow.AirflowCheck._get_version', return_value='2.6.2'):
-        with mock.patch('datadog_checks.base.utils.http.requests') as req:
+        mock_session = mock.MagicMock()
+        with mock.patch('datadog_checks.base.utils.http.requests.Session', return_value=mock_session):
             mock_resp = mock.MagicMock(status_code=200)
             mock_resp.json.side_effect = [
                 {'metadatabase': {'status': metadb_status}, 'scheduler': {'status': scheduler_status}},
                 {'status': 'OK'},
             ]
-            req.get.return_value = mock_resp
+            mock_session.get.return_value = mock_resp
 
             check.check(None)
 
@@ -80,7 +82,8 @@ def test_dag_total_tasks(aggregator, task_instance):
     check = AirflowCheck('airflow', common.FULL_CONFIG, [instance])
 
     with mock.patch('datadog_checks.airflow.airflow.AirflowCheck._get_version', return_value='2.6.2'):
-        with mock.patch('datadog_checks.base.utils.http.requests') as req:
+        req = mock.MagicMock()
+        with mock.patch('datadog_checks.base.utils.http.requests.Session', return_value=req):
             mock_resp = mock.MagicMock(status_code=200)
             mock_resp.json.side_effect = [
                 {'metadatabase': {'status': 'healthy'}, 'scheduler': {'status': 'healthy'}},
@@ -98,7 +101,8 @@ def test_dag_task_ongoing_duration(aggregator, task_instance):
     check = AirflowCheck('airflow', common.FULL_CONFIG, [instance])
 
     with mock.patch('datadog_checks.airflow.airflow.AirflowCheck._get_version', return_value='2.6.2'):
-        with mock.patch('datadog_checks.base.utils.http.requests') as req:
+        req = mock.MagicMock()
+        with mock.patch('datadog_checks.base.utils.http.requests.Session', return_value=req):
             mock_resp = mock.MagicMock(status_code=200)
             mock_resp.json.side_effect = [
                 {'metadatabase': {'status': 'healthy'}, 'scheduler': {'status': 'healthy'}},
@@ -142,7 +146,8 @@ def test_config_collect_ongoing_duration(collect_ongoing_duration, should_call_m
     check = AirflowCheck('airflow', common.FULL_CONFIG, [instance])
 
     with mock.patch('datadog_checks.airflow.airflow.AirflowCheck._get_version', return_value='2.6.2'):
-        with mock.patch('datadog_checks.base.utils.http.requests') as req:
+        req = mock.MagicMock()
+        with mock.patch('datadog_checks.base.utils.http.requests.Session', return_value=req):
             mock_resp = mock.MagicMock(status_code=200)
             mock_resp.json.side_effect = [
                 {'metadatabase': {'status': 'healthy'}, 'scheduler': {'status': 'healthy'}},

amazon_msk/tests/conftest.py

@@ -32,7 +32,11 @@ def mock_requests_get(url, *args, **kwargs):
 
 @pytest.fixture
 def mock_data():
-    with mock.patch('requests.get', side_effect=mock_requests_get, autospec=True):
+    # Mock requests.get because it is used internally within boto3
+    with (
+        mock.patch('requests.get', side_effect=mock_requests_get, autospec=True),
+        mock.patch('requests.Session.get', side_effect=mock_requests_get),
+    ):
         yield
 
 

apache/tests/conftest.py

@@ -49,7 +49,8 @@ def check_status_page_ready():
 
 @pytest.fixture
 def mock_hide_server_version():
-    with mock.patch('datadog_checks.base.utils.http.requests') as req:
+    req = mock.MagicMock()
+    with mock.patch('datadog_checks.base.utils.http.requests.Session', return_value=req):
 
         def mock_requests_get_headers(*args, **kwargs):
             r = requests.get(*args, **kwargs)

arangodb/tests/test_arangodb.py

@@ -52,11 +52,11 @@ def test_invalid_endpoint(aggregator, instance_invalid_endpoint, dd_run_check):
 def test_check(instance, dd_run_check, aggregator, tag_condition, base_tags):
     check = ArangodbCheck('arangodb', {}, [instance])
 
-    def mock_requests_get(url, *args, **kwargs):
+    def mock_requests_get(session, url, *args, **kwargs):
         fixture = url.rsplit('/', 1)[-1]
         return MockResponse(file_path=os.path.join(os.path.dirname(__file__), 'fixtures', tag_condition, fixture))
 
-    with mock.patch('requests.get', side_effect=mock_requests_get, autospec=True):
+    with mock.patch('requests.Session.get', side_effect=mock_requests_get, autospec=True):
         dd_run_check(check)
 
     aggregator.assert_service_check(

avi_vantage/tests/conftest.py

@@ -58,29 +58,31 @@ def _get_metrics(metrics_folder=NO_TENANT_METRICS_FOLDER, endpoint=None):
 
 @pytest.fixture
 def mock_client():
-    with mock.patch('datadog_checks.base.utils.http.requests') as req:
+    def mock_get(url: AnyStr, *__: Any, **___: Any):
+        parsed = urlparse(url)
+        resource = [part for part in parsed.path.split('/') if len(part) > 0][-1]
+        query_params = parsed.query
 
-        def get(url: AnyStr, *_: Any, **__: Any):
-            parsed = urlparse(url)
-            resource = [part for part in parsed.path.split('/') if len(part) > 0][-1]
-            query_params = parsed.query
+        path = {}
 
-            path = {}
-
-            path['tenant=admin'] = ADMIN_TENANT_METRICS_FOLDER
-            path['tenant=admin%2Ctenant_a%2Ctenant_b'] = MULTIPLE_TENANTS_METRICS_FOLDER
-
-            if query_params:
-                return MockResponse(
-                    file_path=os.path.join(HERE, 'compose', 'fixtures', path[query_params], f'{resource}_metrics')
-                )
+        path['tenant=admin'] = ADMIN_TENANT_METRICS_FOLDER
+        path['tenant=admin%2Ctenant_a%2Ctenant_b'] = MULTIPLE_TENANTS_METRICS_FOLDER
 
+        if query_params:
             return MockResponse(
-                file_path=os.path.join(HERE, 'compose', 'fixtures', NO_TENANT_METRICS_FOLDER, f'{resource}_metrics')
+                file_path=os.path.join(HERE, 'compose', 'fixtures', path[query_params], f'{resource}_metrics')
             )
 
-        req.Session = mock.MagicMock(return_value=mock.MagicMock(get=get))
-        yield
+        return MockResponse(
+            file_path=os.path.join(HERE, 'compose', 'fixtures', NO_TENANT_METRICS_FOLDER, f'{resource}_metrics')
+        )
+
+    def mock_post(url: AnyStr, *__: Any, **___: Any):
+        return mock.MagicMock(status_code=200, content=b'{"results": []}')
+
+    with mock.patch('datadog_checks.base.utils.http.RequestsWrapper.get', side_effect=mock_get):
+        with mock.patch('datadog_checks.base.utils.http.RequestsWrapper.post', new=mock_post):
+            yield
 
 
 @pytest.fixture(scope='session')

cert_manager/tests/test_cert_manager.py

@@ -15,7 +15,7 @@
 @pytest.fixture()
 def error_metrics():
     with mock.patch(
-        'requests.get',
+        'requests.Session.get',
         return_value=mock.MagicMock(status_code=502, headers={'Content-Type': "text/plain"}),
     ):
         yield
@@ -34,7 +34,7 @@ def test_check(aggregator, dd_run_check):
     def mock_requests_get(url, *args, **kwargs):
         return MockResponse(file_path=os.path.join(os.path.dirname(__file__), 'fixtures', 'cert_manager.txt'))
 
-    with mock.patch('requests.get', side_effect=mock_requests_get, autospec=True):
+    with mock.patch('requests.Session.get', side_effect=mock_requests_get, autospec=True):
         dd_run_check(check)
 
     expected_metrics = dict(CERT_METRICS)

cilium/tests/conftest.py

@@ -203,7 +203,7 @@ def mock_agent_data():
     with open(f_name, "r") as f:
         text_data = f.read()
     with mock.patch(
-        "requests.get",
+        'requests.Session.get',
         return_value=mock.MagicMock(
             status_code=200, iter_lines=lambda **kwargs: text_data.split("\n"), headers={"Content-Type": "text/plain"}
         ),
@@ -217,7 +217,7 @@ def mock_operator_data():
     with open(f_name, "r") as f:
         text_data = f.read()
     with mock.patch(
-        "requests.get",
+        'requests.Session.get',
         return_value=mock.MagicMock(
             status_code=200, iter_lines=lambda **kwargs: text_data.split("\n"), headers={"Content-Type": "text/plain"}
         ),

citrix_hypervisor/tests/conftest.py

@@ -50,5 +50,5 @@ def mock_requests_get(url, *args, **kwargs):
 
 @pytest.fixture
 def mock_responses():
-    with mock.patch('requests.get', side_effect=mock_requests_get):
+    with mock.patch('requests.Session.get', side_effect=mock_requests_get):
         yield

consul/tests/test_unit.py

@@ -158,7 +158,7 @@ def test_get_nodes_with_service_critical(aggregator):
 def test_consul_request(aggregator, instance, mocker):
     consul_check = ConsulCheck(common.CHECK_NAME, {}, [consul_mocks.MOCK_CONFIG])
     mocker.patch("datadog_checks.base.utils.serialization.json.loads")
-    with mock.patch("datadog_checks.consul.consul.requests.get") as mock_requests_get:
+    with mock.patch("datadog_checks.consul.consul.requests.Session.get") as mock_requests_get:
         consul_check.consul_request("foo")
         url = "{}/{}".format(instance["url"], "foo")
         aggregator.assert_service_check("consul.can_connect", ConsulCheck.OK, tags=["url:{}".format(url)], count=1)
@@ -549,8 +549,10 @@ def test_config(test_case, extra_config, expected_http_kwargs, mocker):
     check = ConsulCheck(common.CHECK_NAME, {}, instances=[instance])
     mocker.patch("datadog_checks.base.utils.serialization.json.loads")
 
-    with mock.patch('datadog_checks.base.utils.http.requests') as r:
-        r.get.return_value = mock.MagicMock(status_code=200)
+    with mock.patch('datadog_checks.base.utils.http.requests.Session') as session:
+        mock_session = mock.MagicMock()
+        session.return_value = mock_session
+        mock_session.get.return_value = mock.MagicMock(status_code=200)
 
         check.check(None)
 
@@ -564,4 +566,4 @@ def test_config(test_case, extra_config, expected_http_kwargs, mocker):
             'allow_redirects': mock.ANY,
         }
         http_wargs.update(expected_http_kwargs)
-        r.get.assert_called_with('/v1/status/leader', **http_wargs)
+        mock_session.get.assert_called_with('/v1/status/leader', **http_wargs)

couch/tests/test_unit.py

@@ -30,7 +30,8 @@ def test_config(test_case, extra_config, expected_http_kwargs):
     instance.update(extra_config)
     check = CouchDb(common.CHECK_NAME, {}, instances=[instance])
 
-    with mock.patch('datadog_checks.base.utils.http.requests') as r:
+    r = mock.MagicMock()
+    with mock.patch('datadog_checks.base.utils.http.requests.Session', return_value=r):
         r.get.return_value = mock.MagicMock(status_code=200, content='{}')
 
         check.check(instance)