Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions rero_mef/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -840,6 +840,10 @@ def harvestname(name, from_date, until_date, arguments, quiet, enqueue, test_md5
actions = ", ".join([f"{action.value}={count}" for action, count in action_count.items()])
mef_actions = ", ".join([f"{action.value}={count}" for action, count in mef_action_count.items()])
click.echo(f"Count: {count} agent: {actions} mef: {mef_actions}")
if count == -1:
# A failed harvest must fail its job, or the scheduler reports it as a success.
click.secho(f"Harvest {name} failed", fg="red", err=True)
sys.exit(1)


@oaiharvester.command()
Expand Down
6 changes: 6 additions & 0 deletions rero_mef/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@

from celery.schedules import crontab
from invenio_records_rest.facets import terms_filter
from sentry_sdk.scrubber import DEFAULT_DENYLIST, EventScrubber

from .agents import AgentViafRecord
from .agents.gnd.models import AgentGndIdentifier
Expand Down Expand Up @@ -265,6 +266,11 @@
RERO_OAI_RETRY_STATUS_CODES = [423, 429, 503]
#: Seconds to wait when the response states no `Retry-After`.
RERO_OAI_RETRY_AFTER = 60
#: Sentry sends the local variables of a failure, the OAI parameters with the GND access token among them. Its
#: default scrubber neither knows that key nor looks into nested dictionaries.
LOGGING_SENTRY_INIT_KWARGS = {
"event_scrubber": EventScrubber(denylist=[*DEFAULT_DENYLIST, "accesstoken", "access_token"], recursive=True)
}

MAX_RESULT_WINDOW = 100000000
#: Largest page of the REST search endpoints. Whole exports go through `/updated` or deep paging.
Expand Down
17 changes: 14 additions & 3 deletions rero_mef/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import gc
import json
import os
import re
import time
from copy import deepcopy
from datetime import UTC, datetime, timedelta
Expand Down Expand Up @@ -84,6 +85,16 @@ def _log_oai_failure(name, spec, dates, err):
)


ACCESS_TOKEN_PARAM = re.compile(r"(accessToken=)[^&\s]+")


def _without_access_token(err):
# A request error states the URL it failed on, the access token of the source included.
if isinstance(err, requests.RequestException):
return type(err)(ACCESS_TOKEN_PARAM.sub(r"\1***", str(err)), response=err.response)
return err


# Hours can not be retrieved by get_info_by_oai_name
# TIME_FORMAT = '%Y-%m-%dT%H:%M:%SZ'
TIME_FORMAT = "%Y-%m-%d"
Expand Down Expand Up @@ -214,9 +225,9 @@ def _next_response(self):
oaiexceptions.OAIError,
XMLSyntaxError,
) as err:
last_error = err
last_error = _without_access_token(err)
# Transient failures are breadcrumbs only; the caller reports the final one.
current_app.logger.warning(f"Sickle harvest attempt {count + 1}/5 failed: {err}")
current_app.logger.warning(f"Sickle harvest attempt {count + 1}/5 failed: {last_error}")
# Wait 60 seconds before retrying to avoid overwhelming the server
if count < 4:
sleep(60)
Expand Down Expand Up @@ -546,7 +557,7 @@ def oai_get_record(id_, name, transformation, access_token=None, identifier=None

if access_token:
params["accessToken"] = access_token
full_url = f"{full_url}&accessToken={access_token}"
full_url = f"{full_url}&accessToken=***"

try:
record = request.GetRecord(**params)
Expand Down
23 changes: 23 additions & 0 deletions tests/unit/test_harvest_cli.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
# SPDX-FileCopyrightText: Fondation RERO+
# SPDX-License-Identifier: AGPL-3.0-or-later

"""Test the exit status of the harvest command, which a scheduled job reports."""

from unittest import mock

import pytest
from click.testing import CliRunner
from flask import Flask
from flask.cli import ScriptInfo

from rero_mef.cli import harvestname


@pytest.mark.parametrize(("count", "exit_code"), [(3, 0), (0, 0), (-1, 1)])
def test_harvest_exit_code(count, exit_code):
harvest_task = mock.Mock(return_value=(count, {}, {}))
with mock.patch("rero_mef.cli.obj_or_import_string", return_value=harvest_task):
result = CliRunner().invoke(
harvestname, ["-n", "concepts.gnd"], obj=ScriptInfo(create_app=lambda: Flask("test"))
)
assert result.exit_code == exit_code
40 changes: 39 additions & 1 deletion tests/unit/test_oai_failure_logging.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
alerted on in Sentry.
"""

import json
import logging
from unittest import mock

Expand All @@ -15,6 +16,7 @@
import sentry_sdk
from flask import Flask

from rero_mef.config import LOGGING_SENTRY_INIT_KWARGS
from rero_mef.utils import MyOAIItemIterator, _log_oai_failure

DATES = {"from": "2026-10-09", "until": "2026-10-11"}
Expand All @@ -33,7 +35,7 @@ def sentry_events():
def capture(event, hint):
events.append(event)

sentry_sdk.init(dsn="https://key@sentry.invalid/1", before_send=capture)
sentry_sdk.init(dsn="https://key@sentry.invalid/1", before_send=capture, **LOGGING_SENTRY_INIT_KWARGS)
yield events
sentry_sdk.init()

Expand Down Expand Up @@ -67,6 +69,21 @@ def test_failure_without_http_response_has_no_status_tag(app_context, sentry_eve
assert event["tags"]["oai.set"] == ""


def test_sentry_event_has_no_access_token(app_context, sentry_events):
def harvest(params, access_token):
raise http_error(502)

try:
harvest({"verb": "ListRecords", "accessToken": "secret"}, access_token="secret")
except requests.HTTPError as err:
_log_oai_failure("concepts.gnd", None, DATES, err)

[event] = sentry_events
frame_vars = [frame["vars"] for frame in event["exception"]["values"][0]["stacktrace"]["frames"]]
assert frame_vars[-1]
assert "secret" not in json.dumps(frame_vars, default=str)


def test_retry_attempts_are_warnings(app_context, caplog):
iterator = MyOAIItemIterator.__new__(MyOAIItemIterator)
iterator.params = {"verb": "ListRecords"}
Expand All @@ -78,3 +95,24 @@ def test_retry_attempts_are_warnings(app_context, caplog):

assert [record.levelno for record in caplog.records] == [logging.WARNING] * 5
assert all(record.exc_info is None for record in caplog.records)


def test_access_token_stays_out_of_the_logs(app_context, caplog):
url = "https://services.dnb.de/oai/repository?verb=ListRecords&accessToken=secret&set=authorities"
response = requests.Response()
response.status_code = 502
iterator = MyOAIItemIterator.__new__(MyOAIItemIterator)
iterator.params = {"verb": "ListRecords", "accessToken": "secret"}
iterator.resumption_token = None
iterator.sickle = mock.Mock(
harvest=mock.Mock(side_effect=requests.HTTPError(f"502 Server Error for url: {url}", response=response))
)

with mock.patch("rero_mef.utils.sleep"), pytest.raises(requests.HTTPError) as error:
iterator._next_response()

assert "secret" not in str(error.value)
assert "accessToken=***&set=authorities" in str(error.value)
assert error.value.response is response
assert error.value.__context__ is None
assert "secret" not in caplog.text
Loading