diff --git a/usaspending_api/references/management/commands/load_gtas.py b/usaspending_api/references/management/commands/load_gtas.py index 401d8694c4..6d467bcae9 100644 --- a/usaspending_api/references/management/commands/load_gtas.py +++ b/usaspending_api/references/management/commands/load_gtas.py @@ -94,23 +94,18 @@ def broker_fetch_sql(self) -> str: ELSE sf.availability_type_code END, '-', sf.main_account_code, '-', sf.sub_account_code) AS tas_rendering_label, - budget_object_class, - program_activity_reporting_key AS program_activity_reporting_key_id, - prior_year_adjustment, - by_direct_reimbursable_fun, - bea_category + STRING_AGG(budget_object_class, ',') AS budget_object_class, + STRING_AGG(program_activity_reporting_key, ',') AS program_activity_reporting_key_id, + STRING_AGG(prior_year_adjustment, ',') AS prior_year_adjustment, + STRING_AGG(by_direct_reimbursable_fun, ',') AS by_direct_reimbursable_fun, + STRING_AGG(bea_category, ',') AS bea_category FROM sf_133 sf GROUP BY fiscal_year, fiscal_period, disaster_emergency_fund_code, - tas_rendering_label, - budget_object_class, - program_activity_reporting_key_id, - prior_year_adjustment, - by_direct_reimbursable_fun, - bea_category + tas_rendering_label ORDER BY fiscal_year, fiscal_period; diff --git a/usaspending_api/references/tests/data/broker_gtas.json b/usaspending_api/references/tests/data/broker_gtas.json index 4fa4c96b8f..55f8d368a2 100644 --- a/usaspending_api/references/tests/data/broker_gtas.json +++ b/usaspending_api/references/tests/data/broker_gtas.json @@ -1,5 +1,5 @@ { - "SELECT fiscal_year, period AS fiscal_period, COALESCE(SUM(CASE WHEN line IN (1033) THEN sf.amount ELSE 0 END), 0.0) AS anticipated_prior_year_obligation_recoveries, COALESCE(SUM(CASE WHEN line IN (1020) THEN sf.amount ELSE 0 END), 0.0) AS adjustments_to_unobligated_balance_brought_forward_fyb, COALESCE(SUM(CASE WHEN line IN (1340,1440) THEN sf.amount ELSE 0 END), 0.0) AS borrowing_authority_amount, COALESCE(SUM(CASE WHEN line IN (1160,1180,1260,1280) THEN sf.amount ELSE 0 END), 0.0) AS budget_authority_appropriation_amount_cpe, COALESCE(SUM(CASE WHEN line IN (1000) THEN sf.amount ELSE 0 END), 0.0) AS budget_authority_unobligated_balance_brought_forward_cpe, COALESCE(SUM(CASE WHEN line IN (1540,1640) THEN sf.amount ELSE 0 END), 0.0) AS contract_authority_amount, COALESCE(SUM(CASE WHEN line IN (1021,1033) THEN sf.amount ELSE 0 END), 0.0) AS deobligations_or_recoveries_or_refunds_from_prior_year_cpe, COALESCE(SUM(CASE WHEN line IN (2190) THEN sf.amount ELSE 0 END), 0.0) AS obligations_incurred, COALESCE(SUM(CASE WHEN line IN (2190) THEN sf.amount ELSE 0 END), 0.0) AS obligations_incurred_total_cpe, COALESCE(SUM(CASE WHEN line IN (1340,1440,1540,1640,1750,1850) THEN sf.amount ELSE 0 END), 0.0) AS other_budgetary_resources_amount_cpe, COALESCE(SUM(CASE WHEN line IN (1061) THEN sf.amount ELSE 0 END), 0.0) AS prior_year_paid_obligation_recoveries, COALESCE(SUM(CASE WHEN line IN (1750,1850) THEN sf.amount ELSE 0 END), 0.0) AS spending_authority_from_offsetting_collections_amount, COALESCE(SUM(CASE WHEN line IN (1910) THEN sf.amount ELSE 0 END), 0.0) AS total_budgetary_resources_cpe, COALESCE(SUM(CASE WHEN line IN (2490) THEN sf.amount ELSE 0 END), 0.0) AS unobligated_balance_cpe, COALESCE(SUM(CASE WHEN line IN (2500) THEN sf.amount ELSE 0 END), 0.0) AS status_of_budgetary_resources_total_cpe, COALESCE(SUM(CASE WHEN line IN (3020) THEN sf.amount * -1 ELSE 0 END), 0.0) AS gross_outlay_amount_by_tas_cpe, COALESCE(SUM(CASE WHEN line IN (1010,1011,1012,1013,1014,1015,1016,1017,1018,1019,1020,1021,1022,1023,1024,1025,1026,1027,1028,1029,1030,1031,1032,1033,1034,1035,1036,1037,1038,1039,1040,1041,1042) AND fiscal_year < 2021 THEN sf.amount * -1 WHEN line IN (1010,1011,1012,1013,1014,1015,1016,1017,1018,1019,1020,1021,1022,1023,1024,1025,1026,1027,1028,1029,1030,1031,1032,1033,1034,1035,1036,1037,1038,1039,1040,1041,1042,1043,1044,1045,1046,1047,1048,1049,1050,1051,1052,1053,1054,1055,1056,1057,1058,1059,1060,1061,1062,1063,1064,1065) AND fiscal_year >= 2021 THEN sf.amount * -1 ELSE 0 END), 0.0) AS adjustments_to_unobligated_balance_brought_forward_cpe, disaster_emergency_fund_code AS disaster_emergency_fund_id, CONCAT( CASE WHEN sf.allocation_transfer_agency is not null THEN CONCAT(sf.allocation_transfer_agency, '-') ELSE null END, sf.agency_identifier, '-', CASE WHEN sf.beginning_period_of_availa is not null THEN CONCAT(sf.beginning_period_of_availa, '/', sf.ending_period_of_availabil) ELSE sf.availability_type_code END, '-', sf.main_account_code, '-', sf.sub_account_code) AS tas_rendering_label, budget_object_class, program_activity_reporting_key AS program_activity_reporting_key_id, prior_year_adjustment, by_direct_reimbursable_fun, bea_category FROM sf_133 sf GROUP BY fiscal_year, fiscal_period, disaster_emergency_fund_code, tas_rendering_label, budget_object_class, program_activity_reporting_key_id, prior_year_adjustment, by_direct_reimbursable_fun, bea_category ORDER BY fiscal_year, fiscal_period;": [ + "SELECT fiscal_year, period AS fiscal_period, COALESCE(SUM( CASE WHEN line IN (1033) THEN sf.amount ELSE 0 END ), 0.0) AS anticipated_prior_year_obligation_recoveries,COALESCE(SUM( CASE WHEN line IN (1020) THEN sf.amount ELSE 0 END ), 0.0) AS adjustments_to_unobligated_balance_brought_forward_fyb,COALESCE(SUM( CASE WHEN line IN (1340,1440) THEN sf.amount ELSE 0 END ), 0.0) AS borrowing_authority_amount,COALESCE(SUM( CASE WHEN line IN (1160,1180,1260,1280) THEN sf.amount ELSE 0 END ), 0.0) AS budget_authority_appropriation_amount_cpe, COALESCE(SUM( CASE WHEN line IN (1000) THEN sf.amount ELSE 0 END ), 0.0) AS budget_authority_unobligated_balance_brought_forward_cpe,COALESCE(SUM( CASE WHEN line IN (1540,1640) THEN sf.amount ELSE 0 END ), 0.0) AS contract_authority_amount,COALESCE(SUM( CASE WHEN line IN (1021,1033) THEN sf.amount ELSE 0 END ), 0.0) AS deobligations_or_recoveries_or_refunds_from_prior_year_cpe,COALESCE(SUM( CASE WHEN line IN (2190) THEN sf.amount ELSE 0 END ), 0.0) AS obligations_incurred,COALESCE(SUM( CASE WHEN line IN (2190) THEN sf.amount ELSE 0 END ), 0.0) AS obligations_incurred_total_cpe,COALESCE(SUM( CASE WHEN line IN (1340,1440,1540,1640,1750,1850) THEN sf.amount ELSE 0 END ), 0.0) AS other_budgetary_resources_amount_cpe, COALESCE(SUM( CASE WHEN line IN (1061) THEN sf.amount ELSE 0 END ), 0.0) AS prior_year_paid_obligation_recoveries,COALESCE(SUM( CASE WHEN line IN (1750,1850) THEN sf.amount ELSE 0 END ), 0.0) AS spending_authority_from_offsetting_collections_amount,COALESCE(SUM( CASE WHEN line IN (1910) THEN sf.amount ELSE 0 END ), 0.0) AS total_budgetary_resources_cpe,COALESCE(SUM( CASE WHEN line IN (2490) THEN sf.amount ELSE 0 END ), 0.0) AS unobligated_balance_cpe,COALESCE(SUM( CASE WHEN line IN (2500) THEN sf.amount ELSE 0 END ), 0.0) AS status_of_budgetary_resources_total_cpe,COALESCE(SUM( CASE WHEN line IN (3020) THEN sf.amount * -1 ELSE 0 END ), 0.0) AS gross_outlay_amount_by_tas_cpe,COALESCE(SUM(CASE WHEN line IN (1010,1011,1012,1013,1014,1015,1016,1017,1018,1019,1020,1021,1022,1023,1024,1025,1026,1027,1028,1029,1030,1031,1032,1033,1034,1035,1036,1037,1038,1039,1040,1041,1042) AND fiscal_year < 2021 THEN sf.amount * -1 WHEN line IN (1010,1011,1012,1013,1014,1015,1016,1017,1018,1019,1020,1021,1022,1023,1024,1025,1026,1027,1028,1029,1030,1031,1032,1033,1034,1035,1036,1037,1038,1039,1040,1041,1042,1043,1044,1045,1046,1047,1048,1049,1050,1051,1052,1053,1054,1055,1056,1057,1058,1059,1060,1061,1062,1063,1064,1065) AND fiscal_year >= 2021 THEN sf.amount * -1 ELSE 0 END ), 0.0) AS adjustments_to_unobligated_balance_brought_forward_cpe, disaster_emergency_fund_code AS disaster_emergency_fund_id, CONCAT( CASE WHEN sf.allocation_transfer_agency is not null THEN CONCAT(sf.allocation_transfer_agency, '-') ELSE null END, sf.agency_identifier, '-', CASE WHEN sf.beginning_period_of_availa is not null THEN CONCAT(sf.beginning_period_of_availa, '/', sf.ending_period_of_availabil) ELSE sf.availability_type_code END, '-', sf.main_account_code, '-', sf.sub_account_code) AS tas_rendering_label, STRING_AGG(budget_object_class, ',') AS budget_object_class, STRING_AGG(program_activity_reporting_key, ',') AS program_activity_reporting_key_id, STRING_AGG(prior_year_adjustment, ',') AS prior_year_adjustment, STRING_AGG(by_direct_reimbursable_fun, ',') AS by_direct_reimbursable_fun, STRING_AGG(bea_category, ',') AS bea_category FROM sf_133 sf GROUP BY fiscal_year, fiscal_period, disaster_emergency_fund_code, tas_rendering_label ORDER BY fiscal_year, fiscal_period;": [ { "fiscal_year": 1600, "fiscal_period": "-1", diff --git a/usaspending_api/references/tests/integration/test_load_gtas_mgmt_cmd.py b/usaspending_api/references/tests/integration/test_load_gtas_mgmt_cmd.py index 34a37fee4c..b0fb06e56a 100644 --- a/usaspending_api/references/tests/integration/test_load_gtas_mgmt_cmd.py +++ b/usaspending_api/references/tests/integration/test_load_gtas_mgmt_cmd.py @@ -20,6 +20,8 @@ def test_program_activity_fresh_load(monkeypatch): baker.make("references.ProgramActivityPark", code="test") data_broker_mock = MagicMock() + # TODO this mock results in the broker query essentially going untested. We should instead use fixtures + # for the broker data and test that the query works as expected. data_broker_mock.cursor.return_value = PhonyCursor("usaspending_api/references/tests/data/broker_gtas.json") mock_connections = { settings.DEFAULT_DB_ALIAS: MagicMock(),