diff --git a/usaspending_api/references/management/commands/load_gtas.py b/usaspending_api/references/management/commands/load_gtas.py index 4875324cc0..401d8694c4 100644 --- a/usaspending_api/references/management/commands/load_gtas.py +++ b/usaspending_api/references/management/commands/load_gtas.py @@ -47,13 +47,13 @@ class Command(mixins.ETLMixin, BaseCommand): help = "Drop and recreate all GTAS reference data" - def handle(self, *args, **options): + def handle(self, *args, **options) -> None: logger.info("Starting ETL script") self.process_data() logger.info("GTAS ETL finished successfully!") @transaction.atomic() - def process_data(self): + def process_data(self) -> None: broker_cursor = connections[settings.BROKER_DB_ALIAS].cursor() logger.info("Extracting data from Broker") @@ -78,7 +78,7 @@ def process_data(self): logger.info("Committing transaction to database") @property - def broker_fetch_sql(self): + def broker_fetch_sql(self) -> str: return f""" SELECT fiscal_year, @@ -86,45 +86,69 @@ def broker_fetch_sql(self): {self.column_statements} disaster_emergency_fund_code AS disaster_emergency_fund_id, CONCAT( - CASE WHEN sf.allocation_transfer_agency is not null THEN CONCAT(sf.allocation_transfer_agency, '-') ELSE null END, + CASE WHEN sf.allocation_transfer_agency is not null + THEN CONCAT(sf.allocation_transfer_agency, '-') ELSE null END, sf.agency_identifier, '-', - CASE WHEN sf.beginning_period_of_availa is not null THEN CONCAT(sf.beginning_period_of_availa, '/', sf.ending_period_of_availabil) ELSE sf.availability_type_code END, + CASE WHEN sf.beginning_period_of_availa is not null + THEN CONCAT(sf.beginning_period_of_availa, '/', sf.ending_period_of_availabil) + ELSE sf.availability_type_code END, '-', sf.main_account_code, '-', sf.sub_account_code) - AS tas_rendering_label + AS tas_rendering_label, + budget_object_class, + program_activity_reporting_key AS program_activity_reporting_key_id, + prior_year_adjustment, + by_direct_reimbursable_fun, + bea_category FROM sf_133 sf GROUP BY fiscal_year, fiscal_period, disaster_emergency_fund_code, - tas_rendering_label + tas_rendering_label, + budget_object_class, + program_activity_reporting_key_id, + prior_year_adjustment, + by_direct_reimbursable_fun, + bea_category ORDER BY fiscal_year, fiscal_period; """ @property - def column_statements(self): + def column_statements(self) -> str: simple_fields = [ - f"COALESCE(SUM(CASE WHEN line IN ({','.join([str(elem) for elem in val])}) THEN sf.amount ELSE 0 END), 0.0) AS {key}," + f"""COALESCE(SUM( + CASE WHEN line IN ({','.join([str(elem) for elem in val])}) + THEN sf.amount ELSE 0 END + ), 0.0) AS {key},""" for key, val in DERIVED_COLUMNS.items() ] inverted_fields = [ - f"COALESCE(SUM(CASE WHEN line IN ({','.join([str(elem) for elem in val])}) THEN sf.amount * -1 ELSE 0 END), 0.0) AS {key}," + f"""COALESCE(SUM( + CASE WHEN line IN ({','.join([str(elem) for elem in val])}) + THEN sf.amount * -1 ELSE 0 END + ), 0.0) AS {key},""" for key, val in INVERTED_DERIVED_COLUMNS.items() ] year_specific_fields = [ f"""COALESCE(SUM(CASE - WHEN line IN ({','.join([str(elem) for elem in val["before_year"]])}) AND fiscal_year < {val["change_year"]} THEN sf.amount * -1 - WHEN line IN ({','.join([str(elem) for elem in val["year_and_after"]])}) AND fiscal_year >= {val["change_year"]} THEN sf.amount * -1 - ELSE 0 - END), 0.0) AS {key},""" + WHEN line IN ({','.join([str(elem) for elem in val["before_year"]])}) + AND fiscal_year < {val["change_year"]} + THEN sf.amount * -1 + WHEN line IN ({','.join([str(elem) for elem in val["year_and_after"]])}) + AND fiscal_year >= {val["change_year"]} + THEN sf.amount * -1 + ELSE 0 + END + ), 0.0) AS {key},""" for key, val in DERIVED_COLUMNS_DYNAMIC.items() ] return "\n".join(simple_fields + inverted_fields + year_specific_fields) @property - def tas_fk_sql(self): + def tas_fk_sql(self) -> str: return f""" UPDATE {GTAS_TABLE} SET treasury_account_identifier = tas.treasury_account_identifier @@ -134,5 +158,5 @@ def tas_fk_sql(self): AND {GTAS_TABLE}.treasury_account_identifier IS DISTINCT FROM tas.treasury_account_identifier""" @property - def financing_account_sql(self): + def financing_account_sql(self) -> str: return f"""DELETE FROM {GTAS_TABLE} WHERE treasury_account_identifier IS NULL""" diff --git a/usaspending_api/references/migrations/0073_gtassf133balances_bea_category_and_more.py b/usaspending_api/references/migrations/0073_gtassf133balances_bea_category_and_more.py new file mode 100644 index 0000000000..3d7f269c97 --- /dev/null +++ b/usaspending_api/references/migrations/0073_gtassf133balances_bea_category_and_more.py @@ -0,0 +1,46 @@ +# Generated by Django 4.2.23 on 2026-03-10 17:49 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("references", "0072_naics_year_retired"), + ] + + operations = [ + migrations.AddField( + model_name="gtassf133balances", + name="bea_category", + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name="gtassf133balances", + name="budget_object_class", + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name="gtassf133balances", + name="by_direct_reimbursable_fun", + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name="gtassf133balances", + name="prior_year_adjustment", + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name="gtassf133balances", + name="program_activity_reporting_key", + field=models.ForeignKey( + blank=True, + db_column="program_activity_reporting_key_code", + null=True, + on_delete=django.db.models.deletion.DO_NOTHING, + related_name="gtas", + to="references.programactivitypark", + ), + ), + ] diff --git a/usaspending_api/references/models/gtas_sf133_balances.py b/usaspending_api/references/models/gtas_sf133_balances.py index 02023b1197..4afac4d298 100644 --- a/usaspending_api/references/models/gtas_sf133_balances.py +++ b/usaspending_api/references/models/gtas_sf133_balances.py @@ -34,6 +34,18 @@ class GTASSF133Balances(models.Model): related_name="gtas", ) tas_rendering_label = models.TextField(null=True, db_index=True) + budget_object_class = models.TextField(blank=True, null=True) + program_activity_reporting_key = models.ForeignKey( + "references.ProgramActivityPark", + on_delete=models.DO_NOTHING, + blank=True, + null=True, + db_column="program_activity_reporting_key_code", + related_name="gtas", + ) + prior_year_adjustment = models.TextField(blank=True, null=True) + by_direct_reimbursable_fun = models.TextField(blank=True, null=True) + bea_category = models.TextField(blank=True, null=True) create_date = models.DateTimeField(auto_now_add=True) update_date = models.DateTimeField(auto_now=True) anticipated_prior_year_obligation_recoveries = models.DecimalField(max_digits=23, decimal_places=2) diff --git a/usaspending_api/references/tests/data/broker_gtas.json b/usaspending_api/references/tests/data/broker_gtas.json index 134f28e5a5..4fa4c96b8f 100644 --- a/usaspending_api/references/tests/data/broker_gtas.json +++ b/usaspending_api/references/tests/data/broker_gtas.json @@ -1,10 +1,11 @@ { - "SELECT fiscal_year, period AS fiscal_period, COALESCE(SUM(CASE WHEN line IN (1033) THEN sf.amount ELSE 0 END), 0.0) AS anticipated_prior_year_obligation_recoveries, COALESCE(SUM(CASE WHEN line IN (1020) THEN sf.amount ELSE 0 END), 0.0) AS adjustments_to_unobligated_balance_brought_forward_fyb, COALESCE(SUM(CASE WHEN line IN (1340,1440) THEN sf.amount ELSE 0 END), 0.0) AS borrowing_authority_amount, COALESCE(SUM(CASE WHEN line IN (1160,1180,1260,1280) THEN sf.amount ELSE 0 END), 0.0) AS budget_authority_appropriation_amount_cpe, COALESCE(SUM(CASE WHEN line IN (1000) THEN sf.amount ELSE 0 END), 0.0) AS budget_authority_unobligated_balance_brought_forward_cpe, COALESCE(SUM(CASE WHEN line IN (1540,1640) THEN sf.amount ELSE 0 END), 0.0) AS contract_authority_amount, COALESCE(SUM(CASE WHEN line IN (1021,1033) THEN sf.amount ELSE 0 END), 0.0) AS deobligations_or_recoveries_or_refunds_from_prior_year_cpe, COALESCE(SUM(CASE WHEN line IN (2190) THEN sf.amount ELSE 0 END), 0.0) AS obligations_incurred, COALESCE(SUM(CASE WHEN line IN (2190) THEN sf.amount ELSE 0 END), 0.0) AS obligations_incurred_total_cpe, COALESCE(SUM(CASE WHEN line IN (1340,1440,1540,1640,1750,1850) THEN sf.amount ELSE 0 END), 0.0) AS other_budgetary_resources_amount_cpe, COALESCE(SUM(CASE WHEN line IN (1061) THEN sf.amount ELSE 0 END), 0.0) AS prior_year_paid_obligation_recoveries, COALESCE(SUM(CASE WHEN line IN (1750,1850) THEN sf.amount ELSE 0 END), 0.0) AS spending_authority_from_offsetting_collections_amount, COALESCE(SUM(CASE WHEN line IN (1910) THEN sf.amount ELSE 0 END), 0.0) AS total_budgetary_resources_cpe, COALESCE(SUM(CASE WHEN line IN (2490) THEN sf.amount ELSE 0 END), 0.0) AS unobligated_balance_cpe, COALESCE(SUM(CASE WHEN line IN (2500) THEN sf.amount ELSE 0 END), 0.0) AS status_of_budgetary_resources_total_cpe, COALESCE(SUM(CASE WHEN line IN (3020) THEN sf.amount * -1 ELSE 0 END), 0.0) AS gross_outlay_amount_by_tas_cpe, COALESCE(SUM(CASE WHEN line IN (1010,1011,1012,1013,1014,1015,1016,1017,1018,1019,1020,1021,1022,1023,1024,1025,1026,1027,1028,1029,1030,1031,1032,1033,1034,1035,1036,1037,1038,1039,1040,1041,1042) AND fiscal_year < 2021 THEN sf.amount * -1 WHEN line IN (1010,1011,1012,1013,1014,1015,1016,1017,1018,1019,1020,1021,1022,1023,1024,1025,1026,1027,1028,1029,1030,1031,1032,1033,1034,1035,1036,1037,1038,1039,1040,1041,1042,1043,1044,1045,1046,1047,1048,1049,1050,1051,1052,1053,1054,1055,1056,1057,1058,1059,1060,1061,1062,1063,1064,1065) AND fiscal_year >= 2021 THEN sf.amount * -1 ELSE 0 END), 0.0) AS adjustments_to_unobligated_balance_brought_forward_cpe, disaster_emergency_fund_code AS disaster_emergency_fund_id, CONCAT( CASE WHEN sf.allocation_transfer_agency is not null THEN CONCAT(sf.allocation_transfer_agency, '-') ELSE null END, sf.agency_identifier, '-', CASE WHEN sf.beginning_period_of_availa is not null THEN CONCAT(sf.beginning_period_of_availa, '/', sf.ending_period_of_availabil) ELSE sf.availability_type_code END, '-', sf.main_account_code, '-', sf.sub_account_code) AS tas_rendering_label FROM sf_133 sf GROUP BY fiscal_year, fiscal_period, disaster_emergency_fund_code, tas_rendering_label ORDER BY fiscal_year, fiscal_period;": [ + "SELECT fiscal_year, period AS fiscal_period, COALESCE(SUM(CASE WHEN line IN (1033) THEN sf.amount ELSE 0 END), 0.0) AS anticipated_prior_year_obligation_recoveries, COALESCE(SUM(CASE WHEN line IN (1020) THEN sf.amount ELSE 0 END), 0.0) AS adjustments_to_unobligated_balance_brought_forward_fyb, COALESCE(SUM(CASE WHEN line IN (1340,1440) THEN sf.amount ELSE 0 END), 0.0) AS borrowing_authority_amount, COALESCE(SUM(CASE WHEN line IN (1160,1180,1260,1280) THEN sf.amount ELSE 0 END), 0.0) AS budget_authority_appropriation_amount_cpe, COALESCE(SUM(CASE WHEN line IN (1000) THEN sf.amount ELSE 0 END), 0.0) AS budget_authority_unobligated_balance_brought_forward_cpe, COALESCE(SUM(CASE WHEN line IN (1540,1640) THEN sf.amount ELSE 0 END), 0.0) AS contract_authority_amount, COALESCE(SUM(CASE WHEN line IN (1021,1033) THEN sf.amount ELSE 0 END), 0.0) AS deobligations_or_recoveries_or_refunds_from_prior_year_cpe, COALESCE(SUM(CASE WHEN line IN (2190) THEN sf.amount ELSE 0 END), 0.0) AS obligations_incurred, COALESCE(SUM(CASE WHEN line IN (2190) THEN sf.amount ELSE 0 END), 0.0) AS obligations_incurred_total_cpe, COALESCE(SUM(CASE WHEN line IN (1340,1440,1540,1640,1750,1850) THEN sf.amount ELSE 0 END), 0.0) AS other_budgetary_resources_amount_cpe, COALESCE(SUM(CASE WHEN line IN (1061) THEN sf.amount ELSE 0 END), 0.0) AS prior_year_paid_obligation_recoveries, COALESCE(SUM(CASE WHEN line IN (1750,1850) THEN sf.amount ELSE 0 END), 0.0) AS spending_authority_from_offsetting_collections_amount, COALESCE(SUM(CASE WHEN line IN (1910) THEN sf.amount ELSE 0 END), 0.0) AS total_budgetary_resources_cpe, COALESCE(SUM(CASE WHEN line IN (2490) THEN sf.amount ELSE 0 END), 0.0) AS unobligated_balance_cpe, COALESCE(SUM(CASE WHEN line IN (2500) THEN sf.amount ELSE 0 END), 0.0) AS status_of_budgetary_resources_total_cpe, COALESCE(SUM(CASE WHEN line IN (3020) THEN sf.amount * -1 ELSE 0 END), 0.0) AS gross_outlay_amount_by_tas_cpe, COALESCE(SUM(CASE WHEN line IN (1010,1011,1012,1013,1014,1015,1016,1017,1018,1019,1020,1021,1022,1023,1024,1025,1026,1027,1028,1029,1030,1031,1032,1033,1034,1035,1036,1037,1038,1039,1040,1041,1042) AND fiscal_year < 2021 THEN sf.amount * -1 WHEN line IN (1010,1011,1012,1013,1014,1015,1016,1017,1018,1019,1020,1021,1022,1023,1024,1025,1026,1027,1028,1029,1030,1031,1032,1033,1034,1035,1036,1037,1038,1039,1040,1041,1042,1043,1044,1045,1046,1047,1048,1049,1050,1051,1052,1053,1054,1055,1056,1057,1058,1059,1060,1061,1062,1063,1064,1065) AND fiscal_year >= 2021 THEN sf.amount * -1 ELSE 0 END), 0.0) AS adjustments_to_unobligated_balance_brought_forward_cpe, disaster_emergency_fund_code AS disaster_emergency_fund_id, CONCAT( CASE WHEN sf.allocation_transfer_agency is not null THEN CONCAT(sf.allocation_transfer_agency, '-') ELSE null END, sf.agency_identifier, '-', CASE WHEN sf.beginning_period_of_availa is not null THEN CONCAT(sf.beginning_period_of_availa, '/', sf.ending_period_of_availabil) ELSE sf.availability_type_code END, '-', sf.main_account_code, '-', sf.sub_account_code) AS tas_rendering_label, budget_object_class, program_activity_reporting_key AS program_activity_reporting_key_id, prior_year_adjustment, by_direct_reimbursable_fun, bea_category FROM sf_133 sf GROUP BY fiscal_year, fiscal_period, disaster_emergency_fund_code, tas_rendering_label, budget_object_class, program_activity_reporting_key_id, prior_year_adjustment, by_direct_reimbursable_fun, bea_category ORDER BY fiscal_year, fiscal_period;": [ { "fiscal_year": 1600, "fiscal_period": "-1", "budget_authority_unobligated_balance_brought_forward_cpe": "-11", "adjustments_to_unobligated_balance_brought_forward_cpe": "-11", + "adjustments_to_unobligated_balance_brought_forward_cpe": "-11", "obligations_incurred_total_cpe": "-10", "budget_authority_appropriation_amount_cpe": "-11", "borrowing_authority_amount": "-11", @@ -21,7 +22,12 @@ "tas_rendering_label": "999-X-111", "anticipated_prior_year_obligation_recoveries": "-111", "prior_year_paid_obligation_recoveries": "-110", - "adjustments_to_unobligated_balance_brought_forward_fyb": "-11" + "adjustments_to_unobligated_balance_brought_forward_fyb": "-11", + "budget_object_class": "test", + "program_activity_reporting_key_id":"test", + "prior_year_adjustment": "test", + "by_direct_reimbursable_fun":"test", + "bea_category": "test" }, { "fiscal_year": 1600, @@ -44,7 +50,12 @@ "tas_rendering_label": "999-X-111", "anticipated_prior_year_obligation_recoveries": "-121", "prior_year_paid_obligation_recoveries": "-120", - "adjustments_to_unobligated_balance_brought_forward_fyb": "-12" + "adjustments_to_unobligated_balance_brought_forward_fyb": "-12", + "budget_object_class": "test", + "program_activity_reporting_key_id":"test", + "prior_year_adjustment": "test", + "by_direct_reimbursable_fun":"test", + "bea_category": "test" }, { "fiscal_year": 1601, @@ -67,7 +78,12 @@ "tas_rendering_label": "999-X-111", "anticipated_prior_year_obligation_recoveries": "-131", "prior_year_paid_obligation_recoveries": "-130", - "adjustments_to_unobligated_balance_brought_forward_fyb": "-13" + "adjustments_to_unobligated_balance_brought_forward_fyb": "-13", + "budget_object_class": "test", + "program_activity_reporting_key_id":"test", + "prior_year_adjustment": "test", + "by_direct_reimbursable_fun":"test", + "bea_category": "test" } ] } \ No newline at end of file diff --git a/usaspending_api/references/tests/integration/test_load_gtas_mgmt_cmd.py b/usaspending_api/references/tests/integration/test_load_gtas_mgmt_cmd.py index de70c3df2a..34a37fee4c 100644 --- a/usaspending_api/references/tests/integration/test_load_gtas_mgmt_cmd.py +++ b/usaspending_api/references/tests/integration/test_load_gtas_mgmt_cmd.py @@ -1,7 +1,6 @@ -import pytest - from unittest.mock import MagicMock +import pytest from django.conf import settings from django.core.management import call_command from model_bakery import baker @@ -18,6 +17,7 @@ def test_program_activity_fresh_load(monkeypatch): baker.make("references.DisasterEmergencyFundCode", code="A") baker.make("accounts.TreasuryAppropriationAccount", tas_rendering_label="999-X-111") + baker.make("references.ProgramActivityPark", code="test") data_broker_mock = MagicMock() data_broker_mock.cursor.return_value = PhonyCursor("usaspending_api/references/tests/data/broker_gtas.json") @@ -52,6 +52,11 @@ def test_program_activity_fresh_load(monkeypatch): -111, -110, -11.00, + "test", + "test", + "test", + "test", + "test", ), ( 1600, @@ -72,6 +77,11 @@ def test_program_activity_fresh_load(monkeypatch): -121, -120, -12.00, + "test", + "test", + "test", + "test", + "test", ), ( 1601, @@ -92,6 +102,11 @@ def test_program_activity_fresh_load(monkeypatch): -131, -130, -13.00, + "test", + "test", + "test", + "test", + "test", ), ], } @@ -118,6 +133,11 @@ def test_program_activity_fresh_load(monkeypatch): "anticipated_prior_year_obligation_recoveries", "prior_year_paid_obligation_recoveries", "adjustments_to_unobligated_balance_brought_forward_cpe", + "budget_object_class", + "program_activity_reporting_key_id", + "prior_year_adjustment", + "by_direct_reimbursable_fun", + "bea_category", ).order_by("-budget_authority_unobligated_balance_brought_forward_cpe") ), }