stats: add feature flag for indexing stats data onto records

invenio_app_ils/circulation/errors.py

@@ -0,0 +1,12 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2025 CERN.
+#
+# Invenio-App-Ils is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+
+"""Circulation exceptions."""
+
+
+class LoanTransitionEventsIndexMissingError(Exception):
+    """Error raised when the loan transition events index is missing."""

invenio_app_ils/circulation/indexer.py

@@ -17,6 +17,7 @@
 from invenio_pidstore.errors import PIDDeletedError
 from invenio_search import current_search_client
 
+from invenio_app_ils.circulation.errors import LoanTransitionEventsIndexMissingError
 from invenio_app_ils.circulation.utils import resolve_item_from_loan
 from invenio_app_ils.documents.api import DOCUMENT_PID_TYPE
 from invenio_app_ils.indexer import ReferencedRecordsIndexer
@@ -129,44 +130,40 @@ def index_stats_fields_for_loan(loan_dict):
 
     # Document availability during loan request
     stat_events_index_name = "events-stats-loan-transitions"
-    if current_search_client.indices.exists(index=stat_events_index_name):
-        loan_pid = loan_dict["pid"]
-        search_body = {
-            "query": {
-                "bool": {
-                    "must": [
-                        {"term": {"trigger": "request"}},
-                        {"term": {"pid_value": loan_pid}},
-                    ],
-                }
-            },
-        }
-
-        search_result = current_search_client.search(
-            index=stat_events_index_name, body=search_body
+    if not current_search_client.indices.exists(index=stat_events_index_name):
+        raise LoanTransitionEventsIndexMissingError()
+
+    loan_pid = loan_dict["pid"]
+    search_body = {
+        "query": {
+            "bool": {
+                "must": [
+                    {"term": {"trigger": "request"}},
+                    {"term": {"pid_value": loan_pid}},
+                ],
+            }
+        },
+    }
+
+    search_result = current_search_client.search(
+        index=stat_events_index_name, body=search_body
+    )
+    hits = search_result["hits"]["hits"]
+    if len(hits) == 1:
+        request_transition_event = hits[0]["_source"]
+        available_items_during_request_count = request_transition_event[
+            "extra_data"
+        ]["available_items_during_request_count"]
+        stats["available_items_during_request"] = (
+            available_items_during_request_count > 0
         )
-        hits = search_result["hits"]["hits"]
-        if len(hits) == 1:
-            request_transition_event = hits[0]["_source"]
-            available_items_during_request_count = request_transition_event[
-                "extra_data"
-            ]["available_items_during_request_count"]
-            stats["available_items_during_request"] = (
-                available_items_during_request_count > 0
-            )
-        elif len(hits) > 1:
-            raise ValueError(
-                f"Multiple request transition events for loan {loan_pid}."
-                "Expected zero or one."
-            )
-    else:
-        current_app.logger.error(
-            "Stats events index '{stat_events_index_name}' does not exist. "
-            "This is normal during initial setup or if no events have been processed yet. "
-            "No data is lost, as soon as the events are processed, " \
-            "the loan wil lbe reindex and the the stat will be available."
+    elif len(hits) > 1:
+        raise ValueError(
+            f"Multiple request transition events for loan {loan_pid}."
+            "Expected zero or one."
         )
 
+
     if not "extra_data" in loan_dict:
         loan_dict["extra_data"] = {}
     loan_dict["extra_data"]["stats"] = stats

invenio_app_ils/config.py

@@ -1247,6 +1247,7 @@ def _(x):
 
 # Feature Toggles
 ILS_SELF_CHECKOUT_ENABLED = False
+ILS_EXTEND_INDICES_WITH_STATS_ENABLED = False
 
 # Use default frontpage
 THEME_FRONTPAGE = False

invenio_app_ils/ext.py

@@ -330,4 +330,5 @@ def before_loan_index_hook(sender, json=None, record=None, index=None, **kwargs)
     :param kwargs: Any other parameters.
     """
     index_extra_fields_for_loan(json)
-    index_stats_fields_for_loan(json)
+    if current_app.config["ILS_EXTEND_INDICES_WITH_STATS_ENABLED"]:
+        index_stats_fields_for_loan(json)

tests/api/circulation/test_loan_bulk_extend.py

@@ -19,7 +19,7 @@
 from invenio_app_ils.circulation.api import bulk_extend_loans
 from invenio_app_ils.items.api import ITEM_PID_TYPE, Item
 from invenio_app_ils.proxies import current_app_ils
-from tests.api.conftest import _create_records
+from tests.api.conftest import create_records
 from tests.helpers import user_login
 
 pid_start_value = 200
@@ -108,8 +108,8 @@ def _prepare_data(db, repeated_run_number=0):
     loans = create_loans(new_pid_start_value, new_pid_end_value)
     items = create_items(new_pid_start_value, new_pid_end_value)
 
-    bulk_index_items = _create_records(db, items, Item, ITEM_PID_TYPE)
-    bulk_index_loans = _create_records(db, loans, Loan, CIRCULATION_LOAN_PID_TYPE)
+    bulk_index_items = create_records(db, items, Item, ITEM_PID_TYPE)
+    bulk_index_loans = create_records(db, loans, Loan, CIRCULATION_LOAN_PID_TYPE)
 
     for item in bulk_index_items:
         current_app_ils.item_indexer.index(item)

tests/api/conftest.py

@@ -74,7 +74,7 @@ def json_headers():
     ]
 
 
-def _create_records(db, objs, cls, pid_type):
+def create_records(db, objs, cls, pid_type):
     """Create records and index."""
     recs = []
     for obj in objs:
@@ -89,42 +89,42 @@ def _create_records(db, objs, cls, pid_type):
 def testdata(app, db, search_clear, users):
     """Create, index and return test data."""
     data = load_json_from_datadir("locations.json")
-    locations = _create_records(db, data, Location, LOCATION_PID_TYPE)
+    locations = create_records(db, data, Location, LOCATION_PID_TYPE)
 
     data = load_json_from_datadir("internal_locations.json")
-    int_locs = _create_records(db, data, InternalLocation, INTERNAL_LOCATION_PID_TYPE)
+    int_locs = create_records(db, data, InternalLocation, INTERNAL_LOCATION_PID_TYPE)
 
     data = load_json_from_datadir("series.json")
-    series = _create_records(db, data, Series, SERIES_PID_TYPE)
+    series = create_records(db, data, Series, SERIES_PID_TYPE)
 
     data = load_json_from_datadir("documents.json")
-    documents = _create_records(db, data, Document, DOCUMENT_PID_TYPE)
+    documents = create_records(db, data, Document, DOCUMENT_PID_TYPE)
 
     data = load_json_from_datadir("items.json")
-    items = _create_records(db, data, Item, ITEM_PID_TYPE)
+    items = create_records(db, data, Item, ITEM_PID_TYPE)
 
     data = load_json_from_datadir("eitems.json")
-    eitems = _create_records(db, data, EItem, EITEM_PID_TYPE)
+    eitems = create_records(db, data, EItem, EITEM_PID_TYPE)
 
     data = load_json_from_datadir("loans.json")
-    loans = _create_records(db, data, Loan, CIRCULATION_LOAN_PID_TYPE)
+    loans = create_records(db, data, Loan, CIRCULATION_LOAN_PID_TYPE)
 
     data = load_json_from_datadir("acq_providers.json")
-    acq_providers = _create_records(db, data, Provider, PROVIDER_PID_TYPE)
+    acq_providers = create_records(db, data, Provider, PROVIDER_PID_TYPE)
 
     data = load_json_from_datadir("acq_orders.json")
-    acq_orders = _create_records(db, data, Order, ORDER_PID_TYPE)
+    acq_orders = create_records(db, data, Order, ORDER_PID_TYPE)
 
     data = load_json_from_datadir("ill_providers.json")
-    ill_providers = _create_records(db, data, Provider, PROVIDER_PID_TYPE)
+    ill_providers = create_records(db, data, Provider, PROVIDER_PID_TYPE)
 
     data = load_json_from_datadir("ill_borrowing_requests.json")
-    ill_brw_reqs = _create_records(
+    ill_brw_reqs = create_records(
         db, data, BorrowingRequest, BORROWING_REQUEST_PID_TYPE
     )
 
     data = load_json_from_datadir("document_requests.json")
-    doc_reqs = _create_records(db, data, DocumentRequest, DOCUMENT_REQUEST_PID_TYPE)
+    doc_reqs = create_records(db, data, DocumentRequest, DOCUMENT_REQUEST_PID_TYPE)
 
     # index
     ri = RecordIndexer()
@@ -166,7 +166,7 @@ def testdata(app, db, search_clear, users):
 def testdata_most_loaned(db, testdata):
     """Create, index and return test data for most loans tests."""
     most_loaned = load_json_from_datadir("loans_most_loaned.json")
-    recs = _create_records(db, most_loaned, Loan, CIRCULATION_LOAN_PID_TYPE)
+    recs = create_records(db, most_loaned, Loan, CIRCULATION_LOAN_PID_TYPE)
 
     ri = RecordIndexer()
     for rec in recs:
@@ -184,23 +184,6 @@ def testdata_most_loaned(db, testdata):
     }
 
 
-@pytest.fixture()
-def testdata_loan_histogram(db, testdata):
-    """Create, index and return test data for loans histogram."""
-    loans_histogram = load_json_from_datadir("loans_histogram.json")
-    recs = _create_records(db, loans_histogram, Loan, CIRCULATION_LOAN_PID_TYPE)
-
-    ri = RecordIndexer()
-    for rec in recs:
-        ri.index(rec)
-
-    current_search.flush_and_refresh(index="loans")
-
-    testdata["loans_histogram"] = loans_histogram
-
-    return testdata
-
-
 @pytest.fixture()
 def item_record(app):
     """Fixture to return an Item payload."""

tests/api/ils/stats/conftest.py

@@ -6,8 +6,21 @@
 
 """Pytest fixtures and plugins for ILS stats."""
 
+
+import datetime
+
 import pytest
+from invenio_circulation.api import Loan
+from invenio_circulation.pidstore.pids import CIRCULATION_LOAN_PID_TYPE
+from invenio_indexer.api import RecordIndexer
+from invenio_search import current_search
 from invenio_stats import current_stats
+from invenio_stats.tasks import process_events
+
+from tests.api.conftest import create_records
+from tests.helpers import (
+    load_json_from_datadir,
+)
 
 
 @pytest.fixture()
@@ -17,3 +30,54 @@ def empty_event_queues():
         queue = current_stats.events[event].queue
         queue.queue.declare()
         queue.consume()
+
+
+@pytest.fixture()
+def testdata_loan_histogram(db, testdata):
+    """Create, index and return test data for loans histogram."""
+    loans_histogram = load_json_from_datadir("loans_histogram.json")
+    recs = create_records(db, loans_histogram, Loan, CIRCULATION_LOAN_PID_TYPE)
+
+    ri = RecordIndexer()
+    for rec in recs:
+        ri.index(rec)
+
+    current_search.flush_and_refresh(index="loans")
+
+    testdata["loans_histogram"] = loans_histogram
+
+    return testdata
+
+
+@pytest.fixture()
+def with_stats_index_extensions(app, ensure_loan_transitions_index):
+    """Enable indices to be extended with stats data."""
+    app.config["ILS_EXTEND_INDICES_WITH_STATS_ENABLED"] = True
+    yield
+    app.config["ILS_EXTEND_INDICES_WITH_STATS_ENABLED"] = False
+
+
+@pytest.fixture()
+def ensure_loan_transitions_index(app, empty_event_queues):
+    """Ensure the loan-transitions events index exists.
+
+    The loan indexer requires this index to be present when indexing loans
+    with stats extensions enabled. This fixture publishes a dummy event
+    and processes it to trigger index creation.
+    """
+    index_name = "events-stats-loan-transitions"
+
+    # Publish a dummy loan transition event to trigger index creation
+    dummy_event = {
+        "timestamp": datetime.datetime.now(datetime.timezone.utc)
+        .replace(tzinfo=None)
+        .isoformat(),
+        "trigger": "extend",
+        "pid_value": "loanid-1",
+        "unique_id": "loanid-1__extend",
+    }
+    current_stats.publish("loan-transitions", [dummy_event])
+
+    # Process events to create the index
+    process_events(["loan-transitions"])
+    current_search.flush_and_refresh(index=index_name)

tests/api/ils/stats/test_loan_stats.py

@@ -198,6 +198,7 @@ def test_loan_stats_histogram_group_by_document_availability(
     json_headers,
     testdata_loan_histogram,
     loan_params,
+    with_stats_index_extensions,
 ):
     """Test that the availability of an item during loan request can be used for grouping loans in the histogram."""
 
@@ -266,6 +267,7 @@ def test_loan_stats_indexed_fields(
     empty_event_queues,
     empty_search,
     testdata_loan_histogram,
+    with_stats_index_extensions,
 ):
     """Test loan time ranges being indexed onto loans