Merge branch 'main' into OPS-5571/history-ordering-ui-fixes

Author: johndeange

backend/data_tools/data/agreements_and_blin_data.json5

@@ -161,9 +161,12 @@ def get(self) -> Response:
             request_schema = AgreementRequestSchema()
             data = request_schema.load(request.args.to_dict(flat=False))
 
+            include_procurement_list = data.pop("include_procurement", [False])
+            include_procurement = include_procurement_list[0] if include_procurement_list else False
+
             logger.debug("Beginning agreement queries")
             service = AgreementsService(current_app.db_session)
-            agreements, metadata = service.get_list(agreement_classes, data)
+            agreements, metadata = service.get_list(agreement_classes, data, include_procurement)
             logger.debug("Agreement queries complete")
 
             logger.debug("Serializing results")
@@ -202,6 +205,8 @@ def get(self) -> Response:
                 "limit": metadata["limit"],
                 "offset": metadata["offset"],
                 "totals": metadata["totals"],
+                "procurement_overview": metadata["procurement_overview"],
+                "procurement_step_summary": metadata["procurement_step_summary"],
             }
 
             return make_response_with_headers(response_data)

backend/ops_api/ops/resources/agreements.py

@@ -153,6 +153,7 @@ class Meta:
     only_my = fields.List(fields.Boolean(), required=False)
     award_type = fields.List(fields.Enum(AgreementClassification), required=False)
     exact_match = fields.List(fields.Boolean(), required=False, load_default=[True])
+    include_procurement = fields.List(fields.Boolean(), required=False, load_default=[False])
 
 
 class AgreementFiltersQueryParametersSchema(Schema):
@@ -224,7 +225,7 @@ class AgreementListResponse(FyObligatedMixin, AgreementData):
     project = fields.Nested(ProjectSchema())
     product_service_code = fields.Nested(ProductServiceCodeSchema)
     budget_line_items = fields.List(
-        fields.Nested(BudgetLineItemResponseSchema, only=["id", "amount", "fees", "status", "is_obe"]),
+        fields.Nested(BudgetLineItemResponseSchema, only=["id", "amount", "fees", "status", "is_obe", "fiscal_year"]),
         allow_none=True,
     )
     procurement_shop = fields.Nested(ProcurementShopSchema)

backend/ops_api/ops/schemas/agreements.py

@@ -8,7 +8,7 @@
 from loguru import logger
 from sqlalchemy import Select, distinct, func, or_, select, union
 from sqlalchemy.exc import IntegrityError
-from sqlalchemy.orm import Session
+from sqlalchemy.orm import Session, selectinload
 
 from models import (
     CAN,
@@ -25,6 +25,7 @@
     OpsEventType,
     Portfolio,
     PortfolioTeamLeaders,
+    ProcurementShop,
     Project,
     ResearchMethodology,
     ServicesComponent,
@@ -33,6 +34,7 @@
     Vendor,
 )
 from models.agreements import AgreementType
+from models.procurement_tracker import ProcurementTrackerStatus
 from models.utils.fiscal_year import get_current_fiscal_year
 from ops_api.ops.schemas.agreements import AgreementListFilterOptionResponseSchema
 from ops_api.ops.services.change_requests import ChangeRequestService
@@ -414,14 +416,18 @@ def get(self, id: int) -> Agreement:
         return agreement
 
     def get_list(
-        self, agreement_classes: list[Type[Agreement]], data: dict[str, Any]
+        self,
+        agreement_classes: list[Type[Agreement]],
+        data: dict[str, Any],
+        include_procurement: bool = False,
     ) -> tuple[list[Agreement], dict[str, Any]]:
         """
         Get list of agreements with optional filtering and pagination.
 
         Args:
             agreement_classes: List of Agreement subclasses to query (e.g., ContractAgreement, GrantAgreement)
             data: Dictionary containing filter parameters including limit and offset
+            include_procurement: When True, eager-load BLIs/trackers and compute procurement metrics
 
         Returns:
             Tuple of (paginated agreements list, metadata dict with count/limit/offset)
@@ -432,7 +438,7 @@ def get_list(
         # Collect all agreements across types using existing resource helpers
         all_results = []
         for agreement_cls in agreement_classes:
-            agreements = _get_agreements(self.db_session, agreement_cls, data)
+            agreements = _get_agreements(self.db_session, agreement_cls, data, include_procurement)
             all_results.extend(agreements)
 
         # Filter by award_type (computed property, must be done post-query)
@@ -453,6 +459,16 @@ def get_list(
         # Calculate aggregate totals before pagination (for summary cards)
         totals = _compute_agreement_totals(all_results)
 
+        # Calculate procurement overview and step summary before pagination (only when requested)
+        procurement_overview = None
+        procurement_step_summary = None
+        if include_procurement:
+            overview_fiscal_year = (
+                filters.fiscal_year[0] if filters.fiscal_year and len(filters.fiscal_year) == 1 else None
+            )
+            procurement_overview = _compute_procurement_overview(all_results, overview_fiscal_year)
+            procurement_step_summary = _compute_procurement_step_summary(all_results, overview_fiscal_year)
+
         # Apply pagination slicing
         if filters.limit is not None and filters.offset is not None:
             limit_value = filters.limit[0]
@@ -468,6 +484,8 @@ def get_list(
             "limit": limit_value,
             "offset": offset_value,
             "totals": totals,
+            "procurement_overview": procurement_overview,
+            "procurement_step_summary": procurement_step_summary,
         }
 
         return paginated_results, metadata
@@ -828,6 +846,13 @@ def _validate_special_topics(db_session: Session, special_topics: List[SpecialTo
         raise ValidationError({"special_topics": [f"Special Topic IDs do not exist: {invalid_special_topic_ids}"]})
 
 
+def _percent(value, total):
+    """Return ``value / total`` as a rounded whole-number percentage, or 0.0 when *total* is zero."""
+    if total == 0:
+        return 0.0
+    return float(round((float(value) / float(total)) * 100))
+
+
 def _compute_agreement_totals(all_results: list[Agreement]) -> dict[str, Any]:
     """Compute aggregate totals across all filtered agreements for summary cards."""
     totals = {
@@ -879,8 +904,113 @@ def _compute_agreement_totals(all_results: list[Agreement]) -> dict[str, Any]:
     return totals
 
 
-def _get_agreements(session: Session, agreement_cls: Type[Agreement], data: dict[str, Any]) -> Sequence[Agreement]:
-    query = _build_base_query(agreement_cls)
+def _compute_procurement_overview(all_results: list[Agreement], fiscal_year: int | None) -> dict[str, Any]:
+    """Compute procurement overview data grouped by BLI status for a given fiscal year.
+
+    Returns amount and agreement count breakdowns for PLANNED, IN_EXECUTION, and OBLIGATED statuses.
+    """
+    tracked_statuses = [
+        BudgetLineItemStatus.PLANNED,
+        BudgetLineItemStatus.IN_EXECUTION,
+        BudgetLineItemStatus.OBLIGATED,
+    ]
+
+    amount_by_status: dict[BudgetLineItemStatus, Decimal] = {s: Decimal("0") for s in tracked_statuses}
+    agreements_by_status: dict[BudgetLineItemStatus, set[int]] = {s: set() for s in tracked_statuses}
+
+    for agreement in all_results:
+        for bli in agreement.budget_line_items:
+            if fiscal_year is not None and bli.fiscal_year != fiscal_year:
+                continue
+            if bli.status in amount_by_status:
+                amount_by_status[bli.status] += (bli.amount or Decimal("0")) + bli.fees
+                agreements_by_status[bli.status].add(agreement.id)
+
+    total_amount = sum(amount_by_status.values(), Decimal("0"))
+    tracked_agreement_ids = set().union(*agreements_by_status.values())
+    total_agreements = len(tracked_agreement_ids)
+
+    status_data = []
+    for status in tracked_statuses:
+        amount = amount_by_status[status]
+        agreement_count = len(agreements_by_status[status])
+        status_data.append(
+            {
+                "status": status.name,
+                "label": status.value,
+                "amount": float(amount),
+                "amount_percent": _percent(amount, total_amount),
+                "agreements": agreement_count,
+                "agreements_percent": _percent(agreement_count, total_agreements),
+            }
+        )
+
+    return {
+        "status_data": status_data,
+        "total_amount": float(total_amount),
+        "total_agreements": total_agreements,
+    }
+
+
+def _get_active_step(agreement: Agreement) -> int | None:
+    """Return the active procurement tracker step number for an agreement, or None."""
+    for tracker in agreement.procurement_trackers:
+        if tracker.status == ProcurementTrackerStatus.ACTIVE and tracker.active_step_number is not None:
+            return tracker.active_step_number
+    return None
+
+
+def _compute_procurement_step_summary(all_results: list[Agreement], fiscal_year: int | None) -> dict[str, Any]:
+    """Compute procurement step summary: agreement counts and dollar amounts per step (1-6)."""
+    amount_by_step: dict[int, Decimal] = {step: Decimal("0") for step in range(1, 7)}
+    agreements_by_step: dict[int, int] = {step: 0 for step in range(1, 7)}
+
+    for agreement in all_results:
+        active_step = _get_active_step(agreement)
+        if active_step is None or active_step < 1 or active_step > 6:
+            continue
+
+        executing_total = Decimal("0")
+        has_executing_blis = False
+        for bli in agreement.budget_line_items:
+            if bli.status != BudgetLineItemStatus.IN_EXECUTION:
+                continue
+            if fiscal_year is not None and bli.fiscal_year != fiscal_year:
+                continue
+            has_executing_blis = True
+            executing_total += (bli.amount or Decimal("0")) + bli.fees
+
+        if not has_executing_blis:
+            continue
+
+        agreements_by_step[active_step] += 1
+        amount_by_step[active_step] += executing_total
+
+    total_agreement_count = sum(agreements_by_step.values())
+
+    step_data = [
+        {
+            "step": step,
+            "agreements": agreements_by_step[step],
+            "agreements_percent": _percent(agreements_by_step[step], total_agreement_count),
+            "amount": float(amount_by_step[step]),
+        }
+        for step in range(1, 7)
+    ]
+
+    return {
+        "step_data": step_data,
+        "total_agreement_count": total_agreement_count,
+    }
+
+
+def _get_agreements(
+    session: Session,
+    agreement_cls: Type[Agreement],
+    data: dict[str, Any],
+    include_procurement: bool = False,
+) -> Sequence[Agreement]:
+    query = _build_base_query(agreement_cls, include_procurement)
     query = _apply_filters(query, agreement_cls, data)
 
     logger.debug(f"query: {query}")
@@ -889,14 +1019,17 @@ def _get_agreements(session: Session, agreement_cls: Type[Agreement], data: dict
     return _filter_by_ownership(all_results, data.get("only_my", []))
 
 
-def _build_base_query(agreement_cls: Type[Agreement]) -> Select[tuple[Agreement]]:
-    return (
-        select(agreement_cls)
-        .distinct()
-        .join(BudgetLineItem, isouter=True)
-        .join(CAN, isouter=True)
-        .order_by(agreement_cls.id)
-    )
+def _build_base_query(agreement_cls: Type[Agreement], include_procurement: bool = False) -> Select[tuple[Agreement]]:
+    query = select(agreement_cls).distinct().join(BudgetLineItem, isouter=True).join(CAN, isouter=True)
+
+    if include_procurement:
+        query = query.options(
+            selectinload(agreement_cls.budget_line_items).selectinload(BudgetLineItem.procurement_shop_fee),
+            selectinload(agreement_cls.procurement_trackers),
+            selectinload(agreement_cls.procurement_shop).selectinload(ProcurementShop.procurement_shop_fees),
+        )
+
+    return query.order_by(agreement_cls.id)
 
 
 def _apply_filters(query: Select[Agreement], agreement_cls: Type[Agreement], data: dict[str, Any]) -> Select[Agreement]:

backend/ops_api/ops/services/agreements.py

@@ -118,7 +118,7 @@ def test_agreements_get_all_by_budget_line_status(auth_client, loaded_db, app_ct
         query_string={"budget_line_status": BudgetLineItemStatus.DRAFT.name},
     )
     assert response.status_code == 200
-    assert len(response.json["data"]) == len(agreements)
+    assert response.json["count"] == len(agreements)
 
     # determine how many agreements in the DB are in budget line status "OBLIGATED"
     stmt = (
@@ -134,7 +134,7 @@ def test_agreements_get_all_by_budget_line_status(auth_client, loaded_db, app_ct
         query_string={"budget_line_status": BudgetLineItemStatus.OBLIGATED.name},
     )
     assert response.status_code == 200
-    assert len(response.json["data"]) == len(agreements)
+    assert response.json["count"] == len(agreements)
 
 
 def test_agreements_get_all_by_portfolio(auth_client, loaded_db, app_ctx):
@@ -594,20 +594,30 @@ def test_agreement_search(auth_client, loaded_db):
     assert response.status_code == 200
     assert len(response.json["data"]) == 0
 
+    # determine how many agreements match "contract" search
+    stmt = select(Agreement).distinct().where(Agreement.name.ilike("%contract%"))
+    expected_count = len(loaded_db.scalars(stmt).all())
+    assert expected_count > 0
+
     response = auth_client.get(
         url_for("api.agreements-group"),
         query_string={"search": "contract"},
     )
     assert response.status_code == 200
-    assert len(response.json["data"]) == 5
+    assert response.json["count"] == expected_count
+
+    # determine how many agreements match "Contract #" search
+    stmt = select(Agreement).distinct().where(Agreement.name.ilike("%Contract #%"))
+    expected_count = len(loaded_db.scalars(stmt).all())
+    assert expected_count > 0
 
     response = auth_client.get(
         url_for("api.agreements-group"),
         query_string={"search": "Contract #"},
     )
 
     assert response.status_code == 200
-    assert len(response.json["data"]) == 4
+    assert response.json["count"] == expected_count
 
 
 def test_agreement_name_filter_partial_match(auth_client, loaded_db):
@@ -621,23 +631,31 @@ def test_agreement_name_filter_partial_match(auth_client, loaded_db):
     assert len(response.json["data"]) == 0
 
     # Test partial match with lowercase "contract" should match agreements with "Contract" in name
+    stmt = select(Agreement).distinct().where(Agreement.name.ilike("%contract%"))
+    expected_count = len(loaded_db.scalars(stmt).all())
+    assert expected_count > 0
+
     response = auth_client.get(
         url_for("api.agreements-group"),
         query_string={"name": "contract", "exact_match": "false"},
     )
     assert response.status_code == 200
-    assert len(response.json["data"]) == 5
+    assert response.json["count"] == expected_count
     # Verify all results contain "contract" in their name (case-insensitive)
     for agreement in response.json["data"]:
         assert "contract" in agreement["name"].lower()
 
     # Test partial match with "Contract #" should match agreements starting with "Contract #"
+    stmt = select(Agreement).distinct().where(Agreement.name.ilike("%Contract #%"))
+    expected_count = len(loaded_db.scalars(stmt).all())
+    assert expected_count > 0
+
     response = auth_client.get(
         url_for("api.agreements-group"),
         query_string={"name": "Contract #", "exact_match": "false"},
     )
     assert response.status_code == 200
-    assert len(response.json["data"]) == 4
+    assert response.json["count"] == expected_count
     # Verify all results contain "Contract #" in their name (case-insensitive)
     for agreement in response.json["data"]:
         assert "contract #" in agreement["name"].lower()