BFD-4238: Re-map beneficiary history (#2772)

Co-authored-by: Brick Green <86254221+brick-green@users.noreply.github.com>

Author: aschey-forpeople

apps/bfd-pipeline/bfd-pipeline-idr/bfd.sql

@@ -2,7 +2,7 @@ DROP SCHEMA IF EXISTS idr CASCADE;
 
 CREATE SCHEMA idr;
 CREATE TABLE idr.beneficiary(
-    bene_sk BIGINT NOT NULL PRIMARY KEY, 
+    bene_sk BIGINT NOT NULL, 
     bene_xref_efctv_sk BIGINT NOT NULL, 
     bene_xref_efctv_sk_computed BIGINT NOT NULL GENERATED ALWAYS 
         AS (CASE WHEN bene_xref_efctv_sk = 0 THEN bene_sk ELSE bene_xref_efctv_sk END) STORED,
@@ -30,26 +30,12 @@ CREATE TABLE idr.beneficiary(
     idr_insrt_ts TIMESTAMPTZ NOT NULL,
     idr_updt_ts TIMESTAMPTZ NOT NULL,
     bfd_created_ts TIMESTAMPTZ NOT NULL,
-    bfd_updated_ts TIMESTAMPTZ NOT NULL
-);
-
-CREATE INDEX ON idr.beneficiary(bene_mbi_id);
-
-CREATE TABLE idr.beneficiary_history(
-    bene_sk BIGINT NOT NULL,
-    bene_xref_efctv_sk BIGINT NOT NULL,
-    bene_xref_efctv_sk_computed BIGINT NOT NULL GENERATED ALWAYS
-        AS (CASE WHEN bene_xref_efctv_sk = 0 THEN bene_sk ELSE bene_xref_efctv_sk END) STORED,
-    bene_mbi_id VARCHAR(11) NOT NULL,
-    idr_trans_efctv_ts TIMESTAMPTZ NOT NULL,
-    idr_trans_obslt_ts TIMESTAMPTZ NOT NULL,
-    idr_insrt_ts TIMESTAMPTZ NOT NULL,
-    idr_updt_ts TIMESTAMPTZ NOT NULL,
-    bfd_created_ts TIMESTAMPTZ NOT NULL,
     bfd_updated_ts TIMESTAMPTZ NOT NULL,
     PRIMARY KEY(bene_sk, idr_trans_efctv_ts)
 );
 
+CREATE INDEX ON idr.beneficiary(bene_mbi_id);
+
 CREATE TABLE idr.beneficiary_mbi_id (
     bene_mbi_id VARCHAR(11) NOT NULL,
     bene_mbi_efctv_dt DATE NOT NULL,

apps/bfd-pipeline/bfd-pipeline-idr/load_synthetic.py

@@ -8,7 +8,6 @@
 from loader import get_connection_string
 
 tables = [
-    {"csv_name": "SYNTHETIC_BENE.csv", "table": "v2_mdcr_bene"},
     {"csv_name": "SYNTHETIC_BENE_HSTRY.csv", "table": "v2_mdcr_bene_hstry"},
     {"csv_name": "SYNTHETIC_BENE_MBI_ID.csv", "table": "v2_mdcr_bene_mbi_id"},
     {"csv_name": "SYNTHETIC_BENE_XREF.csv", "table": "v2_mdcr_bene_xref"},

apps/bfd-pipeline/bfd-pipeline-idr/mock-idr.sql

@@ -1,8 +1,9 @@
 DROP SCHEMA IF EXISTS cms_vdm_view_mdcr_prd CASCADE;
 
 CREATE SCHEMA cms_vdm_view_mdcr_prd;
-CREATE TABLE cms_vdm_view_mdcr_prd.v2_mdcr_bene (
-    bene_sk BIGINT NOT NULL PRIMARY KEY, 
+
+CREATE TABLE cms_vdm_view_mdcr_prd.v2_mdcr_bene_hstry (
+     bene_sk BIGINT NOT NULL, 
     bene_xref_efctv_sk BIGINT NOT NULL, 
     bene_mbi_id VARCHAR(11),
     bene_1st_name VARCHAR(30),
@@ -26,16 +27,6 @@ CREATE TABLE cms_vdm_view_mdcr_prd.v2_mdcr_bene (
     idr_trans_efctv_ts TIMESTAMPTZ NOT NULL,
     idr_trans_obslt_ts TIMESTAMPTZ NOT NULL,
     idr_insrt_ts TIMESTAMPTZ NOT NULL,
-    idr_updt_ts TIMESTAMPTZ
-);
-
-CREATE TABLE cms_vdm_view_mdcr_prd.v2_mdcr_bene_hstry (
-    bene_sk BIGINT NOT NULL,
-    bene_xref_efctv_sk BIGINT NOT NULL,
-    bene_mbi_id VARCHAR(11),
-    idr_trans_efctv_ts TIMESTAMPTZ NOT NULL,
-    idr_trans_obslt_ts TIMESTAMPTZ NOT NULL,
-    idr_insrt_ts TIMESTAMPTZ NOT NULL,
     idr_updt_ts TIMESTAMPTZ,
     PRIMARY KEY(bene_sk, idr_trans_efctv_ts)
 );

apps/bfd-pipeline/bfd-pipeline-idr/model.py

@@ -174,35 +174,6 @@ class IdrBeneficiary(IdrBaseModel):
     bene_line_5_adr: Annotated[str, BeforeValidator(transform_null_string)]
     bene_line_6_adr: Annotated[str, BeforeValidator(transform_null_string)]
     cntct_lang_cd: Annotated[str, BeforeValidator(transform_default_string)]
-    idr_trans_efctv_ts: datetime
-    idr_trans_obslt_ts: datetime
-    idr_insrt_ts: Annotated[datetime, {BATCH_TIMESTAMP: True}]
-    idr_updt_ts: Annotated[
-        datetime, {UPDATE_TIMESTAMP: True}, BeforeValidator(transform_null_date_to_min)
-    ]
-
-    @staticmethod
-    def table() -> str:
-        return "idr.beneficiary"
-
-    @staticmethod
-    def computed_keys() -> list[str]:
-        return ["bene_xref_efctv_sk_computed"]
-
-    @staticmethod
-    def _current_fetch_query() -> str:
-        return """
-            SELECT {COLUMNS}
-            FROM cms_vdm_view_mdcr_prd.v2_mdcr_bene
-            {WHERE_CLAUSE}
-            {ORDER_BY}
-        """
-
-
-class IdrBeneficiaryHistory(IdrBaseModel):
-    bene_sk: Annotated[int, {PRIMARY_KEY: True}]
-    bene_xref_efctv_sk: int
-    bene_mbi_id: Annotated[str, BeforeValidator(transform_null_string)]
     idr_trans_efctv_ts: Annotated[datetime, {PRIMARY_KEY: True}]
     idr_trans_obslt_ts: datetime
     idr_insrt_ts: Annotated[datetime, {BATCH_TIMESTAMP: True}]
@@ -212,7 +183,7 @@ class IdrBeneficiaryHistory(IdrBaseModel):
 
     @staticmethod
     def table() -> str:
-        return "idr.beneficiary_history"
+        return "idr.beneficiary"
 
     @staticmethod
     def computed_keys() -> list[str]:

apps/bfd-pipeline/bfd-pipeline-idr/pipeline.py

@@ -9,7 +9,6 @@
     IdrBeneficiary,
     IdrBeneficiaryEntitlement,
     IdrBeneficiaryEntitlementReason,
-    IdrBeneficiaryHistory,
     IdrBeneficiaryMbiId,
     IdrBeneficiaryStatus,
     IdrBeneficiaryThirdParty,
@@ -102,7 +101,6 @@ def run_pipeline(data_extractor: Extractor, connection_string: str) -> None:
     load_all(
         data_extractor,
         connection_string,
-        IdrBeneficiaryHistory,
         IdrBeneficiaryMbiId,
     )
 

apps/bfd-pipeline/bfd-pipeline-idr/test_pipeline.py

@@ -52,12 +52,6 @@ def test_pipeline(self, psql_url: str) -> None:
         assert rows[1]["bene_sk"] == 70288544
         assert rows[1]["bene_mbi_id"] == "3BR5F18GJ10"
 
-        cur = conn.execute("select * from idr.beneficiary_history order by bene_sk")
-        assert cur.rowcount == 4
-        rows = cur.fetchmany(1)
-        assert rows[0]["bene_sk"] == 70288544
-        assert rows[0]["bene_mbi_id"] == "3BR5F18GJ10"
-
         cur = conn.execute("select * from idr.beneficiary_mbi_id order by bene_mbi_id")
         assert cur.rowcount == 19
         rows = cur.fetchmany(1)
@@ -67,7 +61,7 @@ def test_pipeline(self, psql_url: str) -> None:
         time.sleep(0.05)
         conn.execute(
             """
-            UPDATE cms_vdm_view_mdcr_prd.v2_mdcr_bene
+            UPDATE cms_vdm_view_mdcr_prd.v2_mdcr_bene_hstry
             SET bene_mbi_id = '1S000000000', idr_insrt_ts=%(timestamp)s
             WHERE bene_sk = 53965935
             """,

apps/bfd-pipeline/bfd-pipeline-idr/test_samples1/SYNTHETIC_BENE.csv

@@ -1,5 +1,21 @@
-BENE_SK,BENE_XREF_EFCTV_SK,BENE_MBI_ID,IDR_TRANS_EFCTV_TS,IDR_INSRT_TS,IDR_UPDT_TS,IDR_TRANS_OBSLT_TS
-117817790,117817790,6B93F00KU13,2022-10-15 12:13:56.334055,2023-03-26 20:43:51.604489,2024-07-01 12:56:24.602335,9999-12-31T00:00:00.000000+0000
-272113945,272113945,4IH5AA7WA20,2021-06-20 12:27:27.556274,2022-10-02 20:58:38.274968,2023-07-19 12:55:55.935648,9999-12-31T00:00:00.000000+0000
-272113945,272113945,2IN9P05GF18,2024-11-02 20:24:56.155747,2025-05-22 11:46:35.388957,2025-05-31 14:16:26.009645,9999-12-31T00:00:00.000000+0000
-70288544,70288544,3BR5F18GJ10,2019-02-14 22:41:10.493733,2022-09-14 00:16:10.866654,2024-02-02 03:34:57.891775,9999-12-31T00:00:00.000000+0000
+BENE_SK,BENE_XREF_EFCTV_SK,BENE_MBI_ID,BENE_LAST_NAME,BENE_1ST_NAME,BENE_MIDL_NAME,BENE_BRTH_DT,BENE_DEATH_DT,BENE_VRFY_DEATH_DAY_SW,BENE_SEX_CD,BENE_RACE_CD,BENE_LINE_1_ADR,BENE_LINE_2_ADR,BENE_LINE_3_ADR,BENE_LINE_4_ADR,BENE_LINE_5_ADR,BENE_LINE_6_ADR,GEO_ZIP_PLC_NAME,GEO_ZIP5_CD,GEO_USPS_STATE_CD,CNTCT_LANG_CD,IDR_TRANS_EFCTV_TS,IDR_INSRT_TS,IDR_UPDT_TS,IDR_TRANS_OBSLT_TS
+311552434,425540338,8ON8E48PV29,Sheep,Caroline,Snuffleupagus,1972-08-19,,~,2,~,1970 S University Blvd,,,,,,Mobile,36609,AL,~,2019-07-08 12:46:02.436650,2025-07-10 09:22:45.906696,2025-07-10 11:08:43.021284,9999-12-31T00:00:00.000000+0000
+425540338,425540338,8ON8E48PV29,Sheep,Caroline,Snuffleupagus,1972-08-19,,~,2,~,1970 S University Blvd,,,,,,Mobile,36609,AL,~,2022-01-16 12:18:27.595331,2025-01-11 17:40:01.531385,2025-01-28 01:15:20.158628,9999-12-31T00:00:00.000000+0000
+857996786,857996786,7L45AM2GV52,Sheep,Bluey,,1949-06-05,,~,2,8,2904 61st,,,,,,Galveston,77551,TX,ENG,2025-03-05 09:37:08.630838,2025-05-03 04:30:52.905225,2025-06-12 08:02:27.598221,9999-12-31T00:00:00.000000+0000
+53965935,53965935,3LQ6D75DA70,Erdapfel,Kartoffel,Le Petit Prince,1935-07-10,2024-10-27,N,2,2,1970 S University Blvd,,,,,,Mobile,36609,AL,SPA,2021-02-01 22:01:05.240222,2023-02-21 21:48:40.555322,2023-08-30 02:17:51.244994,9999-12-31T00:00:00.000000+0000
+517422110,102369345,5O05DK1XP31,Jones,Bluey,,1932-12-20,2021-05-10,N,2,8,1970 S University Blvd,,,,,,Mobile,36609,AL,~,2023-11-12 20:37:44.020260,2025-04-08 14:00:22.157491,2025-05-01 03:09:29.056006,9999-12-31T00:00:00.000000+0000
+102369345,102369345,5O05DK1XP31,Jones,Bluey,,1932-12-20,2021-05-10,N,2,8,1970 S University Blvd,,,,,,Mobile,36609,AL,~,2019-04-27 23:45:43.283025,2020-08-12 11:41:17.232129,2021-05-10 08:30:17.154918,9999-12-31T00:00:00.000000+0000
+147278642,147278642,4LU7H76KC23,Jones,Le Petit Prince,Chilli,1972-11-25,,~,1,4,2904 61st,,,,,,Galveston,77551,TX,~,2024-10-31 14:38:12.519112,2024-11-18 21:00:13.893840,2025-05-05 03:54:05.413744,9999-12-31T00:00:00.000000+0000
+505371082,834197189,8O24T49MX97,Erdapfel,Alex,,1941-07-14,,~,1,2,3728 Broadway,Avenue J,,,,,Galveston,77550,TX,SPA,2017-09-27 22:05:38.715156,2022-10-17 21:23:19.976331,2023-12-01 04:19:04.565871,9999-12-31T00:00:00.000000+0000
+834197189,834197189,8O24T49MX97,Erdapfel,Alex,,1941-07-14,,~,1,2,3728 Broadway,Avenue J,,,,,Galveston,77550,TX,SPA,2025-05-15 13:32:00.210842,2025-05-22 15:29:54.199964,2025-07-16 04:03:23.713558,9999-12-31T00:00:00.000000+0000
+354920192,354920192,2O42YN4KP33,Sheep,Bandit,,1955-03-25,,~,2,4,1970 S University Blvd,,,,,,Mobile,36609,AL,~,2023-11-05 16:30:38.002895,2024-10-08 01:07:49.674837,2025-06-30 03:44:29.747099,9999-12-31T00:00:00.000000+0000
+433775608,433775608,2OA5NR1UH88,Erdapfel,Chilli,,1950-03-24,,~,1,8,1970 S University Blvd,,,,,,Mobile,36609,AL,SPA,2024-04-16 18:25:58.096717,2024-06-07 01:17:56.443537,2024-12-27 14:01:40.318225,9999-12-31T00:00:00.000000+0000
+117817790,117817790,6B93F00KU13,Heeler,Snuffleupagus,Kartoffel,1929-02-12,,~,2,2,3728 Broadway,Avenue J,,,,,Galveston,77550,TX,ENG,2021-07-25 08:48:38.041088,2024-03-24 20:29:42.974335,2024-12-02 04:46:58.471123,9999-12-31T00:00:00.000000+0000
+445272343,445272343,8BV8QX8PV95,Smith,Kartoffel,,1957-03-10,,~,1,8,2904 61st,,,,,,Galveston,77551,TX,SPA,2023-07-13 23:49:00.738083,2024-02-08 15:31:08.963883,2024-12-04 20:16:27.097487,9999-12-31T00:00:00.000000+0000
+963814581,963814581,1IR9UF0WG76,Heeler,Frankie,,1912-04-20,,~,1,4,777 Brockton Avenue,,,,,,Abington,2351,MA,ENG,2024-09-09 06:34:45.974120,2025-04-13 17:34:32.319522,2025-06-18 09:52:54.505704,9999-12-31T00:00:00.000000+0000
+156171824,272113945,4IH5AA7WA20,Erdapfel,Bingo,,1912-05-27,,~,1,6,1970 S University Blvd,,,,,,Mobile,36609,AL,~,2017-10-26 19:10:23.805462,2019-10-06 12:12:47.941338,2024-08-01 21:27:49.203404,9999-12-31T00:00:00.000000+0000
+272113945,272113945,4IH5AA7WA20,Erdapfel,Bingo,,1912-05-27,,~,1,6,1970 S University Blvd,,,,,,Mobile,36609,AL,~,2023-07-18 05:57:34.749530,2023-11-13 19:44:50.609209,2025-04-28 21:16:26.520700,9999-12-31T00:00:00.000000+0000
+70288544,70288544,3BR5F18GJ10,Jones,Bingo,,1945-10-26,,~,1,~,777 Brockton Avenue,,,,,,Abington,2351,MA,ENG,2018-03-05 19:26:30.486252,2020-05-16 23:57:54.643451,2021-02-09 12:19:47.586092,9999-12-31T00:00:00.000000+0000
+827500309,124574998,5BP8KV9HX07,Heeler,Bandit,Elmo,1929-06-01,,~,1,1,2904 61st,,,,,,Galveston,77551,TX,SPA,2018-01-26 17:11:18.745006,2021-02-10 10:55:52.239463,2021-11-23 09:04:15.409995,9999-12-31T00:00:00.000000+0000
+124574998,124574998,5BP8KV9HX07,Heeler,Bandit,Elmo,1929-06-01,,~,1,1,2904 61st,,,,,,Galveston,77551,TX,SPA,2024-05-06 22:49:20.488602,2024-10-14 07:13:22.721314,2025-05-02 08:34:56.188964,9999-12-31T00:00:00.000000+0000
+411208587,411208587,3IG3QK5FU73,Smith,Kartoffel,,1940-11-30,2022-09-18,Y,1,1,14152 ND-27,,,,,,Lisbon,58054,ND,~,2022-11-07 05:07:35.245732,2023-10-16 18:35:13.615041,2024-10-23 12:20:35.219603,9999-12-31T00:00:00.000000+0000