update generators

Author: alex-dzeda

apps/bfd-model/bfd-model-idr/sample-data/generator/claims_generator.py

@@ -743,7 +743,7 @@ def add_meta_timestamps(obj, clm, max_date):
 def main():
     parser = argparse.ArgumentParser(description='Generate Synthetic Data for Ingestion by the BFD v3 pipeline.')
     parser.add_argument('--sushi', '-s', action='store_true', help='Generate new StructureDefinitions. Use when testing locally if new .fsh files have been added.')
-    parser.add_argument('--benes', '-b', type=str, help='Pull BENE_SKs from the input file. Expected format is that of SYNTHETIC_BENE.csv')
+    parser.add_argument('--benes', '-b', type=str, help='Pull BENE_SKs from the input file. Expected format is that of SYNTHETIC_BENE_HSTRY.csv')
 
     args = parser.parse_args()
     if(args.sushi):

apps/bfd-model/bfd-model-idr/sample-data/generator/generator_util.py

@@ -18,8 +18,8 @@ def __init__(self):
         self.fake = Faker()
         self.used_bene_sk = []
         self.used_mbi = []
-        self.bene_table = []
         self.bene_hstry_table = []
+        self.bene_xref_table = []
         self.mbi_table = {}
         self.address_options = []
         self.mdcr_stus = []
@@ -108,6 +108,29 @@ def gen_bene_sk(self):
         if bene_sk in self.used_bene_sk:
             return self.gen_bene_sk()
         return bene_sk
+    
+    def generate_bene_xref(self, new_bene_sk, old_bene_sk):
+        #10% chance for invalid xref.
+        kill_cred_cd = 1 if random.randint(1, 10) == 1 else 2
+        
+        efctv_ts = self.fake.date_time_between_dates(
+            datetime.date(year=2017, month=5, day=20),
+            datetime.datetime.now() - datetime.timedelta(days=1)
+        )
+        insrt_ts = self.fake.date_time_between_dates(efctv_ts, datetime.datetime.now() - datetime.timedelta(days=1))
+        updt_ts = self.fake.date_time_between_dates(insrt_ts, datetime.datetime.now() - datetime.timedelta(days=1))
+        
+        xref_row = {
+            "BENE_SK": str(new_bene_sk),
+            "BENE_XREF": str(old_bene_sk),
+            "BENE_KILL_CRED_CD": str(kill_cred_cd),
+            "IDR_TRANS_EFCTV_TS": str(efctv_ts),
+            "IDR_INSRT_TS": str(insrt_ts),
+            "IDR_UPDT_TS": str(updt_ts),
+            "IDR_TRANS_OBSLT_TS": "9999-12-31T00:00:00.000000+0000"
+        }
+        
+        self.bene_xref_table.append(xref_row)
 
     def gen_address(self):
         return self.address_options[random.randint(0, len(self.address_options) - 1)]
@@ -260,45 +283,31 @@ def generate_coverages(self, patient):
     def save_output_files(self):
         Path("out").mkdir(exist_ok=True)
 
-        df = pd.json_normalize(self.bene_table)
-        df = df[
-            [
-                "BENE_SK",
-                "BENE_XREF_EFCTV_SK",
-                "BENE_MBI_ID",
-                "BENE_LAST_NAME",
-                "BENE_1ST_NAME",
-                "BENE_MIDL_NAME",
-                "BENE_BRTH_DT",
-                "BENE_DEATH_DT",
-                "BENE_VRFY_DEATH_DAY_SW",
-                "BENE_SEX_CD",
-                "BENE_RACE_CD",
-                "BENE_LINE_1_ADR",
-                "BENE_LINE_2_ADR",
-                "BENE_LINE_3_ADR",
-                "BENE_LINE_4_ADR",
-                "BENE_LINE_5_ADR",
-                "BENE_LINE_6_ADR",
-                "GEO_ZIP_PLC_NAME",
-                "GEO_ZIP5_CD",
-                "GEO_USPS_STATE_CD",
-                "CNTCT_LANG_CD",
-                "IDR_TRANS_EFCTV_TS",
-                "IDR_INSRT_TS",
-                "IDR_UPDT_TS",
-                "IDR_TRANS_OBSLT_TS",
-            ]
-        ]
-        df.to_csv("out/SYNTHETIC_BENE.csv", index=False)
-
         df = pd.json_normalize(self.bene_hstry_table)
         if(df.size>0):
             df = df[
                 [
                     "BENE_SK",
                     "BENE_XREF_EFCTV_SK",
                     "BENE_MBI_ID",
+                    "BENE_LAST_NAME",
+                    "BENE_1ST_NAME",
+                    "BENE_MIDL_NAME",
+                    "BENE_BRTH_DT",
+                    "BENE_DEATH_DT",
+                    "BENE_VRFY_DEATH_DAY_SW",
+                    "BENE_SEX_CD",
+                    "BENE_RACE_CD",
+                    "BENE_LINE_1_ADR",
+                    "BENE_LINE_2_ADR",
+                    "BENE_LINE_3_ADR",
+                    "BENE_LINE_4_ADR",
+                    "BENE_LINE_5_ADR",
+                    "BENE_LINE_6_ADR",
+                    "GEO_ZIP_PLC_NAME",
+                    "GEO_ZIP5_CD",
+                    "GEO_USPS_STATE_CD",
+                    "CNTCT_LANG_CD",
                     "IDR_TRANS_EFCTV_TS",
                     "IDR_INSRT_TS",
                     "IDR_UPDT_TS",
@@ -332,3 +341,7 @@ def save_output_files(self):
         df = pd.json_normalize(self.mdcr_rsn)
         df.to_csv("out/SYNTHETIC_BENE_MDCR_ENTLMT_RSN.csv", index=False)
 
+        df = pd.json_normalize(self.bene_xref_table)
+        if(df.size>0):
+            df.to_csv("out/SYNTHETIC_BENE_XREF.csv", index=False)
+        

apps/bfd-model/bfd-model-idr/sample-data/generator/patient_generator.py

@@ -15,7 +15,7 @@
 # Command line argument parsing
 parser = argparse.ArgumentParser(description='Generate synthetic patient data')
 parser.add_argument('--benes', type=str, help='Path to CSV file containing beneficiary data')
-parser.add_argument('--claims', action='store_true', help='Automatically generate claims after patient generation using the generated SYNTHETIC_BENE.csv file')
+parser.add_argument('--claims', action='store_true', help='Automatically generate claims after patient generation using the generated SYNTHETIC_BENE_HSTRY.csv file')
 args = parser.parse_args()
 
 patients_to_generate = 15
@@ -177,9 +177,13 @@
         if idx == 0:
             continue
         prior_patient = copy.deepcopy(patient)
+        old_bene_sk = prior_patient["BENE_SK"]
         pt_bene_sk = generator.gen_bene_sk()
         prior_patient["BENE_SK"] = str(pt_bene_sk)
         generator.used_bene_sk.append(pt_bene_sk)
+        
+        generator.generate_bene_xref(pt_bene_sk, old_bene_sk)
+        
         generator.set_timestamps(
             prior_patient, datetime.date(year=2017, month=5, day=20)
         )
@@ -188,20 +192,20 @@
         past_year_date = datetime.date.today() - datetime.timedelta(days=random.randint(30, 365))
         prior_patient["IDR_TRANS_OBSLT_TS"] = f"{past_year_date}T00:00:00.000000+0000"
 
-        generator.bene_table.append(prior_patient)
+        generator.bene_hstry_table.append(prior_patient)
 
-    generator.bene_table.append(patient)
+    generator.bene_hstry_table.append(patient)
 
 generator.save_output_files()
 
 # If --claims flag is provided, automatically call claims_generator.py
 if args.claims:
     print("Generating claims for generated benes")
     try:
-        # Call claims_generator.py with the generated SYNTHETIC_BENE.csv file
+        # Call claims_generator.py with the generated SYNTHETIC_BENE_HSTRY.csv file
         result = subprocess.run([
             sys.executable, 'claims_generator.py', 
-            '--benes', 'out/SYNTHETIC_BENE.csv'
+            '--benes', 'out/SYNTHETIC_BENE_HSTRY.csv'
         ], check=True, capture_output=True, text=True)
 
         print("Claims generation completed successfully!")