[DEV-14462] - Update gtas loader

usaspending_api/references/management/commands/load_gtas.py

@@ -47,13 +47,13 @@
 class Command(mixins.ETLMixin, BaseCommand):
     help = "Drop and recreate all GTAS reference data"
 
-    def handle(self, *args, **options):
+    def handle(self, *args, **options) -> None:
         logger.info("Starting ETL script")
         self.process_data()
         logger.info("GTAS ETL finished successfully!")
 
     @transaction.atomic()
-    def process_data(self):
+    def process_data(self) -> None:
         broker_cursor = connections[settings.BROKER_DB_ALIAS].cursor()
 
         logger.info("Extracting data from Broker")
@@ -78,53 +78,77 @@ def process_data(self):
         logger.info("Committing transaction to database")
 
     @property
-    def broker_fetch_sql(self):
+    def broker_fetch_sql(self) -> str:
         return f"""
             SELECT
                 fiscal_year,
                 period AS fiscal_period,
                 {self.column_statements}
                 disaster_emergency_fund_code AS disaster_emergency_fund_id,
                 CONCAT(
-                    CASE WHEN sf.allocation_transfer_agency is not null THEN CONCAT(sf.allocation_transfer_agency, '-') ELSE null END,
+                    CASE WHEN sf.allocation_transfer_agency is not null
+                    THEN CONCAT(sf.allocation_transfer_agency, '-') ELSE null END,
                     sf.agency_identifier, '-',
-                    CASE WHEN sf.beginning_period_of_availa is not null THEN CONCAT(sf.beginning_period_of_availa, '/', sf.ending_period_of_availabil) ELSE sf.availability_type_code END,
+                    CASE WHEN sf.beginning_period_of_availa is not null
+                    THEN CONCAT(sf.beginning_period_of_availa, '/', sf.ending_period_of_availabil)
+                    ELSE sf.availability_type_code END,
                     '-', sf.main_account_code, '-', sf.sub_account_code)
-                AS tas_rendering_label
+                AS tas_rendering_label,
+                budget_object_class,
+                program_activity_reporting_key AS program_activity_reporting_key_id,
+                prior_year_adjustment,
+                by_direct_reimbursable_fun,
+                bea_category
             FROM
                 sf_133 sf
             GROUP BY
                 fiscal_year,
                 fiscal_period,
                 disaster_emergency_fund_code,
-                tas_rendering_label
+                tas_rendering_label,
+                budget_object_class,
+                program_activity_reporting_key_id,
+                prior_year_adjustment,
+                by_direct_reimbursable_fun,
+                bea_category
             ORDER BY
                 fiscal_year,
                 fiscal_period;
         """
 
     @property
-    def column_statements(self):
+    def column_statements(self) -> str:
         simple_fields = [
-            f"COALESCE(SUM(CASE WHEN line IN ({','.join([str(elem) for elem in val])}) THEN sf.amount ELSE 0 END), 0.0) AS {key},"
+            f"""COALESCE(SUM(
+                CASE WHEN line IN ({','.join([str(elem) for elem in val])})
+                THEN sf.amount ELSE 0 END
+            ), 0.0) AS {key},"""
             for key, val in DERIVED_COLUMNS.items()
         ]
         inverted_fields = [
-            f"COALESCE(SUM(CASE WHEN line IN ({','.join([str(elem) for elem in val])}) THEN sf.amount * -1 ELSE 0 END), 0.0) AS {key},"
+            f"""COALESCE(SUM(
+                CASE WHEN line IN ({','.join([str(elem) for elem in val])})
+                THEN sf.amount * -1 ELSE 0 END
+            ), 0.0) AS {key},"""
             for key, val in INVERTED_DERIVED_COLUMNS.items()
         ]
         year_specific_fields = [
             f"""COALESCE(SUM(CASE
-                        WHEN line IN ({','.join([str(elem) for elem in val["before_year"]])}) AND fiscal_year < {val["change_year"]} THEN sf.amount * -1
-                        WHEN line IN ({','.join([str(elem) for elem in val["year_and_after"]])}) AND fiscal_year >= {val["change_year"]} THEN sf.amount * -1
-                        ELSE 0
-                    END), 0.0) AS {key},"""
+                    WHEN line IN ({','.join([str(elem) for elem in val["before_year"]])})
+                        AND fiscal_year < {val["change_year"]}
+                    THEN sf.amount * -1
+                    WHEN line IN ({','.join([str(elem) for elem in val["year_and_after"]])})
+                        AND fiscal_year >= {val["change_year"]}
+                    THEN sf.amount * -1
+                    ELSE 0
+                    END
+                ), 0.0) AS {key},"""
             for key, val in DERIVED_COLUMNS_DYNAMIC.items()
         ]
         return "\n".join(simple_fields + inverted_fields + year_specific_fields)
 
     @property
-    def tas_fk_sql(self):
+    def tas_fk_sql(self) -> str:
         return f"""
             UPDATE {GTAS_TABLE}
             SET treasury_account_identifier = tas.treasury_account_identifier
@@ -134,5 +158,5 @@ def tas_fk_sql(self):
                 AND {GTAS_TABLE}.treasury_account_identifier IS DISTINCT FROM tas.treasury_account_identifier"""
 
     @property
-    def financing_account_sql(self):
+    def financing_account_sql(self) -> str:
         return f"""DELETE FROM {GTAS_TABLE} WHERE treasury_account_identifier IS NULL"""

usaspending_api/references/migrations/0073_gtassf133balances_bea_category_and_more.py

@@ -0,0 +1,46 @@
+# Generated by Django 4.2.23 on 2026-03-10 17:49
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("references", "0072_naics_year_retired"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="gtassf133balances",
+            name="bea_category",
+            field=models.TextField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name="gtassf133balances",
+            name="budget_object_class",
+            field=models.TextField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name="gtassf133balances",
+            name="by_direct_reimbursable_fun",
+            field=models.TextField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name="gtassf133balances",
+            name="prior_year_adjustment",
+            field=models.TextField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name="gtassf133balances",
+            name="program_activity_reporting_key",
+            field=models.ForeignKey(
+                blank=True,
+                db_column="program_activity_reporting_key_code",
+                null=True,
+                on_delete=django.db.models.deletion.DO_NOTHING,
+                related_name="gtas",
+                to="references.programactivitypark",
+            ),
+        ),
+    ]

usaspending_api/references/models/gtas_sf133_balances.py

@@ -34,6 +34,18 @@ class GTASSF133Balances(models.Model):
         related_name="gtas",
     )
     tas_rendering_label = models.TextField(null=True, db_index=True)
+    budget_object_class = models.TextField(blank=True, null=True)
+    program_activity_reporting_key = models.ForeignKey(
+        "references.ProgramActivityPark",
+        on_delete=models.DO_NOTHING,
+        blank=True,
+        null=True,
+        db_column="program_activity_reporting_key_code",
+        related_name="gtas",
+    )
+    prior_year_adjustment = models.TextField(blank=True, null=True)
+    by_direct_reimbursable_fun = models.TextField(blank=True, null=True)
+    bea_category = models.TextField(blank=True, null=True)
     create_date = models.DateTimeField(auto_now_add=True)
     update_date = models.DateTimeField(auto_now=True)
     anticipated_prior_year_obligation_recoveries = models.DecimalField(max_digits=23, decimal_places=2)

usaspending_api/references/tests/data/broker_gtas.json

@@ -1,10 +1,11 @@
 {
-    "SELECT fiscal_year, period AS fiscal_period, COALESCE(SUM(CASE WHEN line IN (1033) THEN sf.amount ELSE 0 END), 0.0) AS anticipated_prior_year_obligation_recoveries, COALESCE(SUM(CASE WHEN line IN (1020) THEN sf.amount ELSE 0 END), 0.0) AS adjustments_to_unobligated_balance_brought_forward_fyb, COALESCE(SUM(CASE WHEN line IN (1340,1440) THEN sf.amount ELSE 0 END), 0.0) AS borrowing_authority_amount, COALESCE(SUM(CASE WHEN line IN (1160,1180,1260,1280) THEN sf.amount ELSE 0 END), 0.0) AS budget_authority_appropriation_amount_cpe, COALESCE(SUM(CASE WHEN line IN (1000) THEN sf.amount ELSE 0 END), 0.0) AS budget_authority_unobligated_balance_brought_forward_cpe, COALESCE(SUM(CASE WHEN line IN (1540,1640) THEN sf.amount ELSE 0 END), 0.0) AS contract_authority_amount, COALESCE(SUM(CASE WHEN line IN (1021,1033) THEN sf.amount ELSE 0 END), 0.0) AS deobligations_or_recoveries_or_refunds_from_prior_year_cpe, COALESCE(SUM(CASE WHEN line IN (2190) THEN sf.amount ELSE 0 END), 0.0) AS obligations_incurred, COALESCE(SUM(CASE WHEN line IN (2190) THEN sf.amount ELSE 0 END), 0.0) AS obligations_incurred_total_cpe, COALESCE(SUM(CASE WHEN line IN (1340,1440,1540,1640,1750,1850) THEN sf.amount ELSE 0 END), 0.0) AS other_budgetary_resources_amount_cpe, COALESCE(SUM(CASE WHEN line IN (1061) THEN sf.amount ELSE 0 END), 0.0) AS prior_year_paid_obligation_recoveries, COALESCE(SUM(CASE WHEN line IN (1750,1850) THEN sf.amount ELSE 0 END), 0.0) AS spending_authority_from_offsetting_collections_amount, COALESCE(SUM(CASE WHEN line IN (1910) THEN sf.amount ELSE 0 END), 0.0) AS total_budgetary_resources_cpe, COALESCE(SUM(CASE WHEN line IN (2490) THEN sf.amount ELSE 0 END), 0.0) AS unobligated_balance_cpe, COALESCE(SUM(CASE WHEN line IN (2500) THEN sf.amount ELSE 0 END), 0.0) AS status_of_budgetary_resources_total_cpe, COALESCE(SUM(CASE WHEN line IN (3020) THEN sf.amount * -1 ELSE 0 END), 0.0) AS gross_outlay_amount_by_tas_cpe, COALESCE(SUM(CASE WHEN line IN (1010,1011,1012,1013,1014,1015,1016,1017,1018,1019,1020,1021,1022,1023,1024,1025,1026,1027,1028,1029,1030,1031,1032,1033,1034,1035,1036,1037,1038,1039,1040,1041,1042) AND fiscal_year < 2021 THEN sf.amount * -1 WHEN line IN (1010,1011,1012,1013,1014,1015,1016,1017,1018,1019,1020,1021,1022,1023,1024,1025,1026,1027,1028,1029,1030,1031,1032,1033,1034,1035,1036,1037,1038,1039,1040,1041,1042,1043,1044,1045,1046,1047,1048,1049,1050,1051,1052,1053,1054,1055,1056,1057,1058,1059,1060,1061,1062,1063,1064,1065) AND fiscal_year >= 2021 THEN sf.amount * -1 ELSE 0 END), 0.0) AS adjustments_to_unobligated_balance_brought_forward_cpe, disaster_emergency_fund_code AS disaster_emergency_fund_id, CONCAT( CASE WHEN sf.allocation_transfer_agency is not null THEN CONCAT(sf.allocation_transfer_agency, '-') ELSE null END, sf.agency_identifier, '-', CASE WHEN sf.beginning_period_of_availa is not null THEN CONCAT(sf.beginning_period_of_availa, '/', sf.ending_period_of_availabil) ELSE sf.availability_type_code END, '-', sf.main_account_code, '-', sf.sub_account_code) AS tas_rendering_label FROM sf_133 sf GROUP BY fiscal_year, fiscal_period, disaster_emergency_fund_code, tas_rendering_label ORDER BY fiscal_year, fiscal_period;": [
+    "SELECT fiscal_year, period AS fiscal_period, COALESCE(SUM(CASE WHEN line IN (1033) THEN sf.amount ELSE 0 END), 0.0) AS anticipated_prior_year_obligation_recoveries, COALESCE(SUM(CASE WHEN line IN (1020) THEN sf.amount ELSE 0 END), 0.0) AS adjustments_to_unobligated_balance_brought_forward_fyb, COALESCE(SUM(CASE WHEN line IN (1340,1440) THEN sf.amount ELSE 0 END), 0.0) AS borrowing_authority_amount, COALESCE(SUM(CASE WHEN line IN (1160,1180,1260,1280) THEN sf.amount ELSE 0 END), 0.0) AS budget_authority_appropriation_amount_cpe, COALESCE(SUM(CASE WHEN line IN (1000) THEN sf.amount ELSE 0 END), 0.0) AS budget_authority_unobligated_balance_brought_forward_cpe, COALESCE(SUM(CASE WHEN line IN (1540,1640) THEN sf.amount ELSE 0 END), 0.0) AS contract_authority_amount, COALESCE(SUM(CASE WHEN line IN (1021,1033) THEN sf.amount ELSE 0 END), 0.0) AS deobligations_or_recoveries_or_refunds_from_prior_year_cpe, COALESCE(SUM(CASE WHEN line IN (2190) THEN sf.amount ELSE 0 END), 0.0) AS obligations_incurred, COALESCE(SUM(CASE WHEN line IN (2190) THEN sf.amount ELSE 0 END), 0.0) AS obligations_incurred_total_cpe, COALESCE(SUM(CASE WHEN line IN (1340,1440,1540,1640,1750,1850) THEN sf.amount ELSE 0 END), 0.0) AS other_budgetary_resources_amount_cpe, COALESCE(SUM(CASE WHEN line IN (1061) THEN sf.amount ELSE 0 END), 0.0) AS prior_year_paid_obligation_recoveries, COALESCE(SUM(CASE WHEN line IN (1750,1850) THEN sf.amount ELSE 0 END), 0.0) AS spending_authority_from_offsetting_collections_amount, COALESCE(SUM(CASE WHEN line IN (1910) THEN sf.amount ELSE 0 END), 0.0) AS total_budgetary_resources_cpe, COALESCE(SUM(CASE WHEN line IN (2490) THEN sf.amount ELSE 0 END), 0.0) AS unobligated_balance_cpe, COALESCE(SUM(CASE WHEN line IN (2500) THEN sf.amount ELSE 0 END), 0.0) AS status_of_budgetary_resources_total_cpe, COALESCE(SUM(CASE WHEN line IN (3020) THEN sf.amount * -1 ELSE 0 END), 0.0) AS gross_outlay_amount_by_tas_cpe, COALESCE(SUM(CASE WHEN line IN (1010,1011,1012,1013,1014,1015,1016,1017,1018,1019,1020,1021,1022,1023,1024,1025,1026,1027,1028,1029,1030,1031,1032,1033,1034,1035,1036,1037,1038,1039,1040,1041,1042) AND fiscal_year < 2021 THEN sf.amount * -1 WHEN line IN (1010,1011,1012,1013,1014,1015,1016,1017,1018,1019,1020,1021,1022,1023,1024,1025,1026,1027,1028,1029,1030,1031,1032,1033,1034,1035,1036,1037,1038,1039,1040,1041,1042,1043,1044,1045,1046,1047,1048,1049,1050,1051,1052,1053,1054,1055,1056,1057,1058,1059,1060,1061,1062,1063,1064,1065) AND fiscal_year >= 2021 THEN sf.amount * -1 ELSE 0 END), 0.0) AS adjustments_to_unobligated_balance_brought_forward_cpe, disaster_emergency_fund_code AS disaster_emergency_fund_id, CONCAT( CASE WHEN sf.allocation_transfer_agency is not null THEN CONCAT(sf.allocation_transfer_agency, '-') ELSE null END, sf.agency_identifier, '-', CASE WHEN sf.beginning_period_of_availa is not null THEN CONCAT(sf.beginning_period_of_availa, '/', sf.ending_period_of_availabil) ELSE sf.availability_type_code END, '-', sf.main_account_code, '-', sf.sub_account_code) AS tas_rendering_label, budget_object_class, program_activity_reporting_key AS program_activity_reporting_key_id, prior_year_adjustment, by_direct_reimbursable_fun, bea_category FROM sf_133 sf GROUP BY fiscal_year, fiscal_period, disaster_emergency_fund_code, tas_rendering_label, budget_object_class, program_activity_reporting_key_id, prior_year_adjustment, by_direct_reimbursable_fun, bea_category ORDER BY fiscal_year, fiscal_period;": [
         {
             "fiscal_year": 1600,
             "fiscal_period": "-1",
             "budget_authority_unobligated_balance_brought_forward_cpe": "-11",
             "adjustments_to_unobligated_balance_brought_forward_cpe": "-11",
+            "adjustments_to_unobligated_balance_brought_forward_cpe": "-11",
             "obligations_incurred_total_cpe": "-10",
             "budget_authority_appropriation_amount_cpe": "-11",
             "borrowing_authority_amount": "-11",
@@ -21,7 +22,12 @@
             "tas_rendering_label": "999-X-111",
             "anticipated_prior_year_obligation_recoveries": "-111",
             "prior_year_paid_obligation_recoveries": "-110",
-            "adjustments_to_unobligated_balance_brought_forward_fyb": "-11"
+            "adjustments_to_unobligated_balance_brought_forward_fyb": "-11",
+            "budget_object_class": "test",
+            "program_activity_reporting_key_id":"test",
+            "prior_year_adjustment": "test",
+            "by_direct_reimbursable_fun":"test",
+            "bea_category": "test"
         },
         {
             "fiscal_year": 1600,
@@ -44,7 +50,12 @@
             "tas_rendering_label": "999-X-111",
             "anticipated_prior_year_obligation_recoveries": "-121",
             "prior_year_paid_obligation_recoveries": "-120",
-            "adjustments_to_unobligated_balance_brought_forward_fyb": "-12"
+            "adjustments_to_unobligated_balance_brought_forward_fyb": "-12",
+            "budget_object_class": "test",
+            "program_activity_reporting_key_id":"test",
+            "prior_year_adjustment": "test",
+            "by_direct_reimbursable_fun":"test",
+            "bea_category": "test"
         },
         {
             "fiscal_year": 1601,
@@ -67,7 +78,12 @@
             "tas_rendering_label": "999-X-111",
             "anticipated_prior_year_obligation_recoveries": "-131",
             "prior_year_paid_obligation_recoveries": "-130",
-            "adjustments_to_unobligated_balance_brought_forward_fyb": "-13"
+            "adjustments_to_unobligated_balance_brought_forward_fyb": "-13",
+            "budget_object_class": "test",
+            "program_activity_reporting_key_id":"test",
+            "prior_year_adjustment": "test",
+            "by_direct_reimbursable_fun":"test",
+            "bea_category": "test"
         }
     ]
 }

usaspending_api/references/tests/integration/test_load_gtas_mgmt_cmd.py

@@ -1,7 +1,6 @@
-import pytest
-
 from unittest.mock import MagicMock
 
+import pytest
 from django.conf import settings
 from django.core.management import call_command
 from model_bakery import baker
@@ -18,6 +17,7 @@ def test_program_activity_fresh_load(monkeypatch):
 
     baker.make("references.DisasterEmergencyFundCode", code="A")
     baker.make("accounts.TreasuryAppropriationAccount", tas_rendering_label="999-X-111")
+    baker.make("references.ProgramActivityPark", code="test")
 
     data_broker_mock = MagicMock()
     data_broker_mock.cursor.return_value = PhonyCursor("usaspending_api/references/tests/data/broker_gtas.json")
@@ -52,6 +52,11 @@ def test_program_activity_fresh_load(monkeypatch):
                 -111,
                 -110,
                 -11.00,
+                "test",
+                "test",
+                "test",
+                "test",
+                "test",
             ),
             (
                 1600,
@@ -72,6 +77,11 @@ def test_program_activity_fresh_load(monkeypatch):
                 -121,
                 -120,
                 -12.00,
+                "test",
+                "test",
+                "test",
+                "test",
+                "test",
             ),
             (
                 1601,
@@ -92,6 +102,11 @@ def test_program_activity_fresh_load(monkeypatch):
                 -131,
                 -130,
                 -13.00,
+                "test",
+                "test",
+                "test",
+                "test",
+                "test",
             ),
         ],
     }
@@ -118,6 +133,11 @@ def test_program_activity_fresh_load(monkeypatch):
                 "anticipated_prior_year_obligation_recoveries",
                 "prior_year_paid_obligation_recoveries",
                 "adjustments_to_unobligated_balance_brought_forward_cpe",
+                "budget_object_class",
+                "program_activity_reporting_key_id",
+                "prior_year_adjustment",
+                "by_direct_reimbursable_fun",
+                "bea_category",
             ).order_by("-budget_authority_unobligated_balance_brought_forward_cpe")
         ),
     }