Fix lint errors and dependency compatibility issues in test workflow

.github/workflows/test.yml

@@ -68,5 +68,6 @@ jobs:
     - name: Setup extension
       run: |
         ckan -c test.ini db init
+        ckan -c test.ini harvester initdb
     - name: Run tests
       run: pytest --ckan-ini=test.ini --cov=ckanext.harvest --disable-warnings ckanext/harvest/tests

ckanext/harvest/harvesters/ckanharvester.py

@@ -4,7 +4,6 @@
 from requests.exceptions import HTTPError, RequestException
 
 import datetime
-from urllib3.contrib import pyopenssl
 
 from six.moves.urllib.parse import urlencode
 from ckan import model
@@ -41,8 +40,6 @@ def _get_content(self, url):
         if api_key:
             headers['Authorization'] = api_key
 
-        pyopenssl.inject_into_urllib3()
-
         try:
             http_request = requests.get(url, headers=headers)
         except HTTPError as e:

ckanext/harvest/tests/harvesters/test_ckanharvester.py

@@ -46,7 +46,7 @@ def test_gather_normal(self):
         obj_ids = harvester.gather_stage(job)
 
         assert job.gather_errors == []
-        assert type(obj_ids) == list
+        assert isinstance(obj_ids, list)
         assert len(obj_ids) == len(mock_ckan.DATASETS)
         harvest_object = harvest_model.HarvestObject.get(obj_ids[0])
         assert harvest_object.guid == mock_ckan.DATASETS[0]['id']
@@ -320,11 +320,10 @@ def test_default_extras_invalid(self):
                 config=json.dumps(config))
         assert 'default_extras must be a dictionary' in str(harvest_context.value)
 
-    @patch('ckanext.harvest.harvesters.ckanharvester.pyopenssl.inject_into_urllib3')
     @patch('ckanext.harvest.harvesters.ckanharvester.CKANHarvester.config')
     @patch('ckanext.harvest.harvesters.ckanharvester.requests.get', side_effect=RequestException('Test.value'))
     def test_get_content_handles_request_exception(
-        self, mock_requests_get, mock_config, mock_pyopenssl_inject
+        self, mock_requests_get, mock_config
     ):
         mock_config.return_value = {}
 
@@ -342,11 +341,10 @@ def __init__(self):
             self.request = Mock()
             self.request.url = "http://test.example.gov.uk"
 
-    @patch('ckanext.harvest.harvesters.ckanharvester.pyopenssl.inject_into_urllib3')
     @patch('ckanext.harvest.harvesters.ckanharvester.CKANHarvester.config')
     @patch('ckanext.harvest.harvesters.ckanharvester.requests.get', side_effect=MockHTTPError())
     def test_get_content_handles_http_error(
-        self, mock_requests_get, mock_config, mock_pyopenssl_inject
+        self, mock_requests_get, mock_config
     ):
         mock_config.return_value = {}
 

requirements.txt

@@ -1,6 +1,7 @@
 ckantoolkit>=0.0.7
 pika>=1.1.0
-pyOpenSSL==18.0.0
+pyOpenSSL>=23.2.0
+cryptography>=41.0.0
 redis
 requests>=2.11.1
 six>=1.12.0