diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c06f1cd2b..2a78eac97 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,15 +2,11 @@ on: [pull_request] jobs: lint: runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: [ 2.7, 3.9 ] steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 with: - python-version: ${{ matrix.python-version }} + python-version: "3.10" - name: Install requirements run: pip install flake8 pycodestyle - name: Check syntax @@ -18,29 +14,16 @@ jobs: flake8 . --count --max-line-length=127 --show-source --statistics test: - name: CKAN + name: CKAN 2.11 runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - include: - - ckan-container-version: "2.9-py2" - ckan-postgres-version: "2.9" - ckan-solr-version: "2.9-solr8" - - ckan-container-version: "2.9" - ckan-postgres-version: "2.9" - ckan-solr-version: "2.9-solr8" - - ckan-container-version: "2.10" - ckan-postgres-version: "2.10" - ckan-solr-version: "2.10" - container: - image: openknowledge/ckan-dev:${{ matrix.ckan-container-version }} + image: ckan/ckan-dev:2.11-py3.10 + options: --user root services: solr: - image: ckan/ckan-solr:${{ matrix.ckan-solr-version }} + image: ckan/ckan-solr:2.11-solr9 postgres: - image: ckan/ckan-postgres-dev:${{ matrix.ckan-postgres-version }} + image: ckan/ckan-postgres-dev:2.11 env: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres @@ -56,10 +39,10 @@ jobs: CKAN_REDIS_URL: redis://redis:6379/1 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v6 - name: Install requirements run: | - pip install -r pip-requirements.txt + pip install -r requirements.txt pip install -r dev-requirements.txt pip install -e . # Replace default path to CKAN core config file with the one on the container @@ -67,5 +50,7 @@ jobs: - name: Setup extension run: | ckan -c test.ini db init + ckan -c test.ini harvester initdb - name: Run tests - run: pytest --ckan-ini=test.ini --cov=ckanext.harvest --disable-warnings ckanext/harvest/tests + run: | + pytest --ckan-ini=test.ini --cov=ckanext.harvest --disable-warnings ckanext/harvest/tests/ diff --git a/ckanext/harvest/harvesters/ckanharvester.py b/ckanext/harvest/harvesters/ckanharvester.py index 9a3298972..cfca9ed13 100644 --- a/ckanext/harvest/harvesters/ckanharvester.py +++ b/ckanext/harvest/harvesters/ckanharvester.py @@ -4,7 +4,6 @@ from requests.exceptions import HTTPError, RequestException import datetime -from urllib3.contrib import pyopenssl from six.moves.urllib.parse import urlencode from ckan import model @@ -41,8 +40,6 @@ def _get_content(self, url): if api_key: headers['Authorization'] = api_key - pyopenssl.inject_into_urllib3() - try: http_request = requests.get(url, headers=headers) except HTTPError as e: @@ -442,12 +439,12 @@ def import_stage(self, harvest_object): log.error('Could not get remote group %s', group_) continue - for key in ['packages', 'created', 'users', 'groups', 'tags', 'extras', 'display_name']: + for key in ['packages', 'created', 'users', 'groups', 'tags', 'extras', 'display_name', 'id']: group.pop(key, None) - get_action('group_create')(base_context.copy(), group) - log.info('Group %s has been newly created', group_) - validated_groups.append({'id': group['id'], 'name': group['name']}) + created_group = get_action('group_create')(base_context.copy(), group) + log.info('Group %s has been newly created', created_group) + validated_groups.append({'id': created_group['id'], 'name': created_group['name']}) package_dict['groups'] = validated_groups diff --git a/ckanext/harvest/logic/auth/__init__.py b/ckanext/harvest/logic/auth/__init__.py index 1c4a16072..52a49d05c 100644 --- a/ckanext/harvest/logic/auth/__init__.py +++ b/ckanext/harvest/logic/auth/__init__.py @@ -1,6 +1,15 @@ from ckan.plugins import toolkit as pt from ckanext.harvest import model as harvest_model +try: + from flask import request, has_request_context +except ImportError: + # Flask not available (shouldn't happen in CKAN 2.11+) + request = None + + def has_request_context(): + return False + def user_is_sysadmin(context): ''' @@ -17,6 +26,72 @@ def user_is_sysadmin(context): return user_obj.sysadmin +def load_user_from_flask_request(context): + """ + Load user from Flask request environ into context if not already present. + + In CKAN 2.11, when accessing harvest forms, authorization checks happen + before the user is loaded into context, even though REMOTE_USER is set + in the Flask request environ. This helper ensures the user is loaded + for proper authorization checks. + + Args: + context: CKAN context dict + + Returns: + None (modifies context in-place) + """ + # Use has_request_context() instead of `if not request`: the Flask + # `request` LocalProxy raises RuntimeError when evaluated outside an + # active request context (e.g. background queue, CLI). + if not has_request_context(): + return + + user = context.get('user', '') + if not user: + try: + user = request.environ.get('REMOTE_USER', '') + if user: + context['user'] = user + # Load user object into context for sysadmin checks + model = context.get('model') + if model: + user_obj = model.User.get(user) + # Mirror CKAN's own _get_user contract: only inject active + # users so a deleted/blocked account whose name lingers in + # REMOTE_USER (e.g. from a still-valid session cookie) does + # not retain auth privileges. + if user_obj and getattr(user_obj, 'state', None) == 'active': + context['auth_user_obj'] = user_obj + except Exception: + # Intentionally ignore failures when loading user from request + # so that authorization can safely fall back to CKAN defaults + pass + + +def is_harvest_form_view(): + """ + True only for GET requests serving the harvest creation/edit forms. + + Used to safely identify the "form view" case in CKAN 2.11 where + package_create/package_update auth fires before the form renders with + a sparse data_dict. Anchored on the request path so unrelated callers + that happen to pass an empty data_dict do not bypass auth. + """ + if not has_request_context(): + return False + try: + if request.method != 'GET': + return False + from ckanext.harvest import utils + path = (request.path or '').rstrip('/') + prefix = '/{0}/'.format(utils.DATASET_TYPE_NAME) + return path.endswith('/{0}/new'.format(utils.DATASET_TYPE_NAME)) \ + or (prefix + 'edit/') in (path + '/') + except Exception: + return False + + def _get_object(context, data_dict, name, class_name): ''' return the named item if in the data_dict, or get it from diff --git a/ckanext/harvest/logic/auth/create.py b/ckanext/harvest/logic/auth/create.py index 2ee2ae0b3..efb63f241 100644 --- a/ckanext/harvest/logic/auth/create.py +++ b/ckanext/harvest/logic/auth/create.py @@ -1,5 +1,42 @@ from ckan.plugins import toolkit as pt -from ckanext.harvest.logic.auth import user_is_sysadmin +from ckanext.harvest.logic.auth import ( + user_is_sysadmin, + load_user_from_flask_request, + is_harvest_form_view, +) +import ckan.logic.auth.create as create_auth + + +def package_create(context, data_dict): + """ + Authorization for creating packages (harvest and regular types). + + In CKAN 2.11, this auth function is called before showing the creation form. + When viewing forms (empty data_dict), allow sysadmins to proceed. + """ + # Load user from Flask request if not already in context (CKAN 2.11 Flask issue) + load_user_from_flask_request(context) + + package_type = data_dict.get('type', '') + + # Check if user is sysadmin + try: + is_sysadmin = user_is_sysadmin(context) + + # Allow sysadmins for harvest packages, or for the GET render of the + # harvest creation form (CKAN 2.11 calls this auth before the view + # renders with a sparse data_dict). The form-view case is gated on + # the request path so an empty data_dict from any other caller does + # not silently skip CKAN's normal package_create auth chain. + if is_sysadmin and (package_type == 'harvest' or is_harvest_form_view()): + return {'success': True} + except Exception: + # Intentionally ignore failures in sysadmin check so that authorization + # can fall back to CKAN's default package_create logic below + pass + + # For non-sysadmins or non-harvest packages, use CKAN's default auth + return create_auth.package_create(context, data_dict) def harvest_source_create(context, data_dict): @@ -35,7 +72,8 @@ def harvest_job_create(context, data_dict): context['package'] = pkg try: - pt.check_access('package_update', context, data_dict) + # Pass the package id to package_update for authorization check + pt.check_access('package_update', context, {'id': pkg.id}) return {'success': True} except pt.NotAuthorized: return {'success': False, diff --git a/ckanext/harvest/logic/auth/update.py b/ckanext/harvest/logic/auth/update.py index 419a1620a..d27608a61 100644 --- a/ckanext/harvest/logic/auth/update.py +++ b/ckanext/harvest/logic/auth/update.py @@ -1,5 +1,42 @@ from ckan.plugins import toolkit as pt -from ckanext.harvest.logic.auth import user_is_sysadmin +from ckanext.harvest.logic.auth import ( + user_is_sysadmin, + load_user_from_flask_request, + is_harvest_form_view, +) +import ckan.logic.auth.update as update_auth + + +def package_update(context, data_dict): + """ + Custom package_update authorization for CKAN 2.11 compatibility. + + This function handles the same issue as package_create: in CKAN 2.11, + when viewing the edit form (/harvest/edit/), Flask's authorization + check happens before the user is loaded into context, causing 403 errors. + + See package_create in logic/auth/create.py for detailed explanation. + """ + # Load user from Flask request if not already in context (CKAN 2.11 Flask issue) + load_user_from_flask_request(context) + + # For sysadmins viewing harvest edit forms, allow access + user_obj = context.get('auth_user_obj') + # Use getattr to safely check sysadmin attribute (auth_user_obj might be AnonymousUser) + if user_obj and getattr(user_obj, 'sysadmin', False): + package = context.get('package') + package_type = package.type if package else data_dict.get('type') + + # Allow sysadmins for harvest packages, or for the GET render of a + # harvest edit form (CKAN 2.11 calls this auth before the view renders + # with a sparse data_dict). The form-view case is gated on the request + # path so a missing 'name' from any other caller does not silently + # skip CKAN's normal package_update auth chain. + if package_type == 'harvest' or is_harvest_form_view(): + return {'success': True} + + # Delegate to CKAN's default package_update auth for all other cases + return update_auth.package_update(context, data_dict) def harvest_source_update(context, data_dict): diff --git a/ckanext/harvest/model/__init__.py b/ckanext/harvest/model/__init__.py index 69ecafb5d..a2a944251 100644 --- a/ckanext/harvest/model/__init__.py +++ b/ckanext/harvest/model/__init__.py @@ -11,7 +11,6 @@ from sqlalchemy.orm import backref, relation from sqlalchemy.exc import InvalidRequestError -from ckan import model from ckan.model.meta import metadata, mapper, Session from ckan.model.types import make_uuid from ckan.model.domain_object import DomainObject @@ -47,55 +46,62 @@ def setup(): define_harvester_tables() log.debug('Harvest tables defined in memory') - if not model.package_table.exists(): + # Check if database is initialized by using Inspector + from ckan.model.meta import engine + try: + inspector = Inspector.from_engine(engine) + existing_tables = inspector.get_table_names() + except Exception as e: + log.debug('Harvest table creation deferred: %s', e) + return + + if 'package' not in existing_tables: log.debug('Harvest table creation deferred') return - if not harvest_source_table.exists(): + if 'harvest_source' not in existing_tables: # Create each table individually rather than # using metadata.create_all() - harvest_source_table.create() - harvest_job_table.create() - harvest_object_table.create() - harvest_gather_error_table.create() - harvest_object_error_table.create() - harvest_object_extra_table.create() - harvest_log_table.create() + harvest_source_table.create(bind=engine) + harvest_job_table.create(bind=engine) + harvest_object_table.create(bind=engine) + harvest_gather_error_table.create(bind=engine) + harvest_object_error_table.create(bind=engine) + harvest_object_extra_table.create(bind=engine) + harvest_log_table.create(bind=engine) log.debug('Harvest tables created') else: - from ckan.model.meta import engine log.debug('Harvest tables already exist') # Check if existing tables need to be updated - inspector = Inspector.from_engine(engine) # Check if harvest_log table exist - needed for existing users - if 'harvest_log' not in inspector.get_table_names(): - harvest_log_table.create() + if 'harvest_log' not in existing_tables: + harvest_log_table.create(bind=engine) # Check if harvest_object has a index index_names = [index['name'] for index in inspector.get_indexes("harvest_object")] if "harvest_job_id_idx" not in index_names: log.debug('Creating index for harvest_object') - Index("harvest_job_id_idx", harvest_object_table.c.harvest_job_id).create() + Index("harvest_job_id_idx", harvest_object_table.c.harvest_job_id).create(bind=engine) if "harvest_source_id_idx" not in index_names: log.debug('Creating index for harvest source') - Index("harvest_source_id_idx", harvest_object_table.c.harvest_source_id).create() + Index("harvest_source_id_idx", harvest_object_table.c.harvest_source_id).create(bind=engine) if "package_id_idx" not in index_names: log.debug('Creating index for package') - Index("package_id_idx", harvest_object_table.c.package_id).create() + Index("package_id_idx", harvest_object_table.c.package_id).create(bind=engine) if "guid_idx" not in index_names: log.debug('Creating index for guid') - Index("guid_idx", harvest_object_table.c.guid).create() + Index("guid_idx", harvest_object_table.c.guid).create(bind=engine) index_names = [index['name'] for index in inspector.get_indexes("harvest_object_extra")] if "harvest_object_id_idx" not in index_names: log.debug('Creating index for harvest_object_extra') - Index("harvest_object_id_idx", harvest_object_extra_table.c.harvest_object_id).create() + Index("harvest_object_id_idx", harvest_object_extra_table.c.harvest_object_id).create(bind=engine) class HarvestError(Exception): diff --git a/ckanext/harvest/plugin/__init__.py b/ckanext/harvest/plugin/__init__.py index c2d0df63a..a99b95133 100644 --- a/ckanext/harvest/plugin/__init__.py +++ b/ckanext/harvest/plugin/__init__.py @@ -358,7 +358,9 @@ def organization_facets(self, facets_dict, organization_type, package_type): ]) -def _get_logic_functions(module_root, logic_functions={}): +def _get_logic_functions(module_root, logic_functions=None): + if logic_functions is None: + logic_functions = {} for module_name in ['get', 'create', 'update', 'patch', 'delete']: module_path = '%s.%s' % (module_root, module_name,) @@ -369,9 +371,26 @@ def _get_logic_functions(module_root, logic_functions={}): module = getattr(module, part) for key, value in module.__dict__.items(): - if not key.startswith('_') and (hasattr(value, '__call__') - and (value.__module__ == module_path)): - logic_functions[key] = value + # Skip items that start with underscore + if key.startswith('_'): + continue + + # CKAN 2.11 Flask compatibility: + # Flask's LocalProxy objects (like 'request') raise RuntimeError + # when accessed outside of request context. Also, some imported + # modules don't have __module__ attribute. We need to handle both. + try: + is_callable = hasattr(value, '__call__') + # Check if value has __module__ attribute before accessing it + has_module_attr = hasattr(value, '__module__') + + if is_callable and has_module_attr: + has_correct_module = (value.__module__ == module_path) + if has_correct_module: + logic_functions[key] = value + except (RuntimeError, AttributeError): + # Skip Flask LocalProxy objects and other problematic imports + continue return logic_functions diff --git a/ckanext/harvest/templates/source/admin_base.html b/ckanext/harvest/templates/source/admin_base.html index ce65c99f0..40ffb6297 100644 --- a/ckanext/harvest/templates/source/admin_base.html +++ b/ckanext/harvest/templates/source/admin_base.html @@ -13,7 +13,7 @@ {% else %} {% set locale = h.dump_json({'content': _('This will re-run the harvesting for this source. Any updates at the source will overwrite the local datasets. Sources with a large number of datasets may take a significant amount of time to finish harvesting. Please confirm you would like us to start reharvesting.')}) %} - {{ _('Reharvest') }} @@ -22,7 +22,7 @@ {% endif %} {% if harvest_source.status and harvest_source.status.last_job and (harvest_source.status.last_job.status == 'Running') %} - + {{ _('Stop') }} @@ -30,7 +30,7 @@ {% endif %} {% set locale = h.dump_json({'content': _('Warning: This will remove all datasets for this source, as well as all previous job reports. Are you sure you want to continue?')}) %} - {{ _('Clear') }} diff --git a/ckanext/harvest/tests/harvesters/test_ckanharvester.py b/ckanext/harvest/tests/harvesters/test_ckanharvester.py index cfda4b615..3db3145c0 100644 --- a/ckanext/harvest/tests/harvesters/test_ckanharvester.py +++ b/ckanext/harvest/tests/harvesters/test_ckanharvester.py @@ -46,7 +46,7 @@ def test_gather_normal(self): obj_ids = harvester.gather_stage(job) assert job.gather_errors == [] - assert type(obj_ids) == list + assert isinstance(obj_ids, list) assert len(obj_ids) == len(mock_ckan.DATASETS) harvest_object = harvest_model.HarvestObject.get(obj_ids[0]) assert harvest_object.guid == mock_ckan.DATASETS[0]['id'] @@ -168,8 +168,8 @@ def test_remote_groups_create(self): harvester=CKANHarvester(), config=json.dumps(config)) assert 'dataset1-id' in results_by_guid - # Check that the remote group was created locally - call_action('group_show', {}, id=mock_ckan.GROUPS[0]['id']) + # Check that the remote group was created locally (by name, not id, since id is auto-generated in CKAN 2.11) + call_action('group_show', {}, id=mock_ckan.GROUPS[0]['name']) def test_harvest_info_in_package_show(self): results_by_guid = run_harvest( @@ -185,8 +185,8 @@ def test_harvest_info_in_package_show(self): assert 'harvest_source_title' in extras_dict def test_remote_groups_only_local(self): - # Create an existing group - Group(id='group1-id', name='group1') + # Create an existing group (without specifying ID - CKAN 2.11 auto-generates IDs) + local_group = Group(name='group1') config = {'remote_groups': 'only_local'} results_by_guid = run_harvest( @@ -197,7 +197,12 @@ def test_remote_groups_only_local(self): # Check that the dataset was added to the existing local group dataset = call_action('package_show', {}, id=mock_ckan.DATASETS[0]['id']) - assert dataset['groups'][0]['id'] == mock_ckan.DATASETS[0]['groups'][0]['id'] + + # In CKAN 2.11, group IDs are auto-generated and won't match the remote ID + # The harvester should match by name and use the local group's ID + assert len(dataset['groups']) == 1 + assert dataset['groups'][0]['name'] == 'group1' + assert dataset['groups'][0]['id'] == local_group['id'] # Check that the other remote group was not created locally with pytest.raises(toolkit.ObjectNotFound): @@ -255,11 +260,13 @@ def test_default_tags_invalid(self): assert 'default_tags must be a list of dictionaries' in str(harvest_context.value) def test_default_groups(self): - Group(id='group1-id', name='group1') - Group(id='group2-id', name='group2') - Group(id='group3-id', name='group3') + # Create groups without custom IDs (CKAN 2.11 auto-generates IDs) + Group(name='group1') + group2 = Group(name='group2') + Group(name='group3') - config = {'default_groups': ['group2-id', 'group3'], + # Use group names (or the auto-generated IDs) for default_groups config + config = {'default_groups': [group2['id'], 'group3'], 'remote_groups': 'only_local'} tmp_c = toolkit.c try: @@ -278,10 +285,11 @@ def test_default_groups(self): group_names = set(group['name'] for group in groups) # group1 comes from the harvested dataset # group2 & 3 come from the default_groups - assert group_names, set(('group1', 'group2' == 'group3')) + assert group_names == set(('group1', 'group2', 'group3')) def test_default_groups_invalid(self): - Group(id='group2-id', name='group2') + # Create group without custom ID (CKAN 2.11 auto-generates IDs) + Group(name='group2') # should be list of strings config = {'default_groups': [{'name': 'group2'}]} @@ -320,11 +328,10 @@ def test_default_extras_invalid(self): config=json.dumps(config)) assert 'default_extras must be a dictionary' in str(harvest_context.value) - @patch('ckanext.harvest.harvesters.ckanharvester.pyopenssl.inject_into_urllib3') @patch('ckanext.harvest.harvesters.ckanharvester.CKANHarvester.config') @patch('ckanext.harvest.harvesters.ckanharvester.requests.get', side_effect=RequestException('Test.value')) def test_get_content_handles_request_exception( - self, mock_requests_get, mock_config, mock_pyopenssl_inject + self, mock_requests_get, mock_config ): mock_config.return_value = {} @@ -342,11 +349,10 @@ def __init__(self): self.request = Mock() self.request.url = "http://test.example.gov.uk" - @patch('ckanext.harvest.harvesters.ckanharvester.pyopenssl.inject_into_urllib3') @patch('ckanext.harvest.harvesters.ckanharvester.CKANHarvester.config') @patch('ckanext.harvest.harvesters.ckanharvester.requests.get', side_effect=MockHTTPError()) def test_get_content_handles_http_error( - self, mock_requests_get, mock_config, mock_pyopenssl_inject + self, mock_requests_get, mock_config ): mock_config.return_value = {} diff --git a/ckanext/harvest/tests/test_action.py b/ckanext/harvest/tests/test_action.py index d2927d599..2f10d5951 100644 --- a/ckanext/harvest/tests/test_action.py +++ b/ckanext/harvest/tests/test_action.py @@ -176,7 +176,7 @@ def test_update(self): "name": "test-source-action-updated", "title": "Test source action updated", "notes": "Test source action desc updated", - "source_type": "test", + "source_type": "test-for-action", "frequency": "MONTHLY", "config": json.dumps({"custom_option": ["c", "d"]}) }) @@ -185,7 +185,7 @@ def test_update(self): for key in set(('url', 'name', 'title', 'notes', 'source_type', 'frequency', 'config')): - assert source_dict[key], result[key] == "Key: %s" % key + assert source_dict[key] == result[key], "Key: %s" % key # Check that source was actually updated source = harvest_model.HarvestSource.get(result['id']) diff --git a/ckanext/harvest/tests/test_blueprint.py b/ckanext/harvest/tests/test_blueprint.py index 2ec13a720..7fa2e57d3 100644 --- a/ckanext/harvest/tests/test_blueprint.py +++ b/ckanext/harvest/tests/test_blueprint.py @@ -16,9 +16,11 @@ def _assert_in_body(string, response): @pytest.mark.usefixtures('clean_db', 'clean_index', 'harvest_setup') class TestBlueprint(): - def setup(self): + def setup_method(self): sysadmin = factories.Sysadmin() - self.extra_environ = {'REMOTE_USER': sysadmin['name'].encode('ascii')} + # CKAN 2.11 uses Flask - REMOTE_USER should be string, not bytes + self.extra_environ = {'REMOTE_USER': sysadmin['name']} + self.sysadmin = sysadmin def test_index_page_is_rendered(self, app): @@ -64,7 +66,7 @@ def test_admin_page_rendered(self, app): job = harvest_factories.HarvestJob(source=source_obj) sysadmin = factories.Sysadmin() - env = {"REMOTE_USER": sysadmin['name'].encode('ascii')} + env = {"REMOTE_USER": sysadmin['name']} url = url_for('harvest_admin', id=source_obj.id) @@ -89,7 +91,7 @@ def test_job_page_rendered(self, app): job = harvest_factories.HarvestJob() sysadmin = factories.Sysadmin() - env = {"REMOTE_USER": sysadmin['name'].encode('ascii')} + env = {"REMOTE_USER": sysadmin['name']} url = url_for('harvest_job_list', source=job['source_id']) @@ -102,7 +104,7 @@ def test_job_show_last_page_rendered(self, app): job = harvest_factories.HarvestJob() sysadmin = factories.Sysadmin() - env = {"REMOTE_USER": sysadmin['name'].encode('ascii')} + env = {"REMOTE_USER": sysadmin['name']} url = url_for('harvest_job_show_last', source=job['source_id']) diff --git a/ckanext/harvest/tests/test_timeouts.py b/ckanext/harvest/tests/test_timeouts.py index b39537494..460ea9176 100644 --- a/ckanext/harvest/tests/test_timeouts.py +++ b/ckanext/harvest/tests/test_timeouts.py @@ -1,5 +1,4 @@ from datetime import datetime, timedelta -from nose.tools import assert_equal, assert_in import pytest from ckan.tests import factories as ckan_factories from ckan import model @@ -25,14 +24,14 @@ def test_timeout_jobs(self): ob2 = self.add_object(job=job, source=source, state='COMPLETE', minutes_ago=5) self.add_object(job=job, source=source, state='COMPLETE', minutes_ago=15) - assert_equal(job.get_last_finished_object(), ob2) - assert_equal(job.get_last_action_time(), ob2.import_finished) + assert job.get_last_finished_object() == ob2 + assert job.get_last_action_time() == ob2.import_finished gather_errors = self.run(timeout=3, source=source, job=job) - assert_equal(len(gather_errors), 1) - assert_equal(job.status, 'Finished') + assert len(gather_errors) == 1 + assert job.status == 'Finished' gather_error = gather_errors[0] - assert_in('timeout', gather_error.message) + assert 'timeout' in gather_error.message def test_no_timeout_jobs(self): """ Test a job that don't raise timeout """ @@ -42,12 +41,12 @@ def test_no_timeout_jobs(self): ob2 = self.add_object(job=job, source=source, state='COMPLETE', minutes_ago=5) self.add_object(job=job, source=source, state='COMPLETE', minutes_ago=15) - assert_equal(job.get_last_finished_object(), ob2) - assert_equal(job.get_last_action_time(), ob2.import_finished) + assert job.get_last_finished_object() == ob2 + assert job.get_last_action_time() == ob2.import_finished gather_errors = self.run(timeout=7, source=source, job=job) - assert_equal(len(gather_errors), 0) - assert_equal(job.status, 'Finished') + assert len(gather_errors) == 0 + assert job.status == 'Finished' def test_no_objects_job(self): """ Test a job that don't raise timeout """ @@ -56,8 +55,8 @@ def test_no_objects_job(self): job.gather_finished = datetime.utcnow() job.save() - assert_equal(job.get_last_finished_object(), None) - assert_equal(job.get_last_action_time(), job.gather_finished) + assert job.get_last_finished_object() is None + assert job.get_last_action_time() == job.gather_finished def test_no_gathered_job(self): """ Test a job that don't raise timeout """ @@ -66,8 +65,8 @@ def test_no_gathered_job(self): job.gather_finished = None job.save() - assert_equal(job.get_last_finished_object(), None) - assert_equal(job.get_last_action_time(), job.created) + assert job.get_last_finished_object() is None + assert job.get_last_action_time() == job.created def test_gather_get_last_action_time(self): """ Test get_last_action_time at gather stage """ @@ -77,8 +76,8 @@ def test_gather_get_last_action_time(self): self.add_object(job=job, source=source, state='WAITING') ob3 = self.add_object(job=job, source=source, state='WAITING') - assert_equal(job.get_last_gathered_object(), ob3) - assert_equal(job.get_last_action_time(), ob3.gathered) + assert job.get_last_gathered_object() == ob3 + assert job.get_last_action_time() == ob3.gathered def run(self, timeout, source, job): """ Run the havester_job_run and return the errors """ @@ -125,7 +124,7 @@ def get_source(self): job.save() jobs = source.get_jobs(status='Running') - assert_in(job, jobs) + assert job in jobs return source, job diff --git a/pip-requirements.txt b/requirements.txt similarity index 62% rename from pip-requirements.txt rename to requirements.txt index 54ef8b935..f97a4921a 100644 --- a/pip-requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ ckantoolkit>=0.0.7 pika>=1.1.0 -pyOpenSSL==18.0.0 +pyOpenSSL>=23.2.0 +cryptography>=41.0.0 redis requests>=2.11.1 six>=1.12.0