Skip to content

Commit 526e1b2

Browse files
Merge remote-tracking branch 'origin/master' into BFD-4167__idr-pipeline-terraservice
2 parents ecdd96a + cf4d941 commit 526e1b2

59 files changed

Lines changed: 926 additions & 1089 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

apps/bfd-db-migrator/pom.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
<parent>
66
<groupId>gov.cms.bfd</groupId>
77
<artifactId>bfd-parent</artifactId>
8-
<version>2.213.0-SNAPSHOT</version>
8+
<version>2.216.0-SNAPSHOT</version>
99
</parent>
1010

1111
<artifactId>bfd-db-migrator</artifactId>

apps/bfd-model/bfd-model-codebook-data/pom.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
<parent>
55
<groupId>gov.cms.bfd</groupId>
66
<artifactId>bfd-model-parent</artifactId>
7-
<version>2.213.0-SNAPSHOT</version>
7+
<version>2.216.0-SNAPSHOT</version>
88
</parent>
99

1010
<artifactId>bfd-model-codebook-data</artifactId>

apps/bfd-model/bfd-model-codebook-library/pom.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
<parent>
55
<groupId>gov.cms.bfd</groupId>
66
<artifactId>bfd-model-parent</artifactId>
7-
<version>2.213.0-SNAPSHOT</version>
7+
<version>2.216.0-SNAPSHOT</version>
88
</parent>
99

1010
<artifactId>bfd-model-codebook-library</artifactId>

apps/bfd-model/bfd-model-codebook-plugin/pom.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
<parent>
55
<groupId>gov.cms.bfd</groupId>
66
<artifactId>bfd-model-parent</artifactId>
7-
<version>2.213.0-SNAPSHOT</version>
7+
<version>2.216.0-SNAPSHOT</version>
88
</parent>
99

1010
<artifactId>bfd-model-codebook-plugin</artifactId>

apps/bfd-model/bfd-model-dsl-codegen/bfd-model-dsl-codegen-library/pom.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
<parent>
55
<groupId>gov.cms.bfd</groupId>
66
<artifactId>bfd-model-dsl-codegen-parent</artifactId>
7-
<version>2.213.0-SNAPSHOT</version>
7+
<version>2.216.0-SNAPSHOT</version>
88
</parent>
99

1010
<artifactId>bfd-model-dsl-codegen-library</artifactId>

apps/bfd-model/bfd-model-dsl-codegen/bfd-model-dsl-codegen-plugin/pom.xml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,11 @@
55
<parent>
66
<groupId>gov.cms.bfd</groupId>
77
<artifactId>bfd-model-dsl-codegen-parent</artifactId>
8-
<version>2.213.0-SNAPSHOT</version>
8+
<version>2.216.0-SNAPSHOT</version>
99
</parent>
1010

1111
<artifactId>bfd-model-dsl-codegen-plugin</artifactId>
12-
<version>2.213.0-SNAPSHOT</version>
12+
<version>2.216.0-SNAPSHOT</version>
1313
<packaging>maven-plugin</packaging>
1414

1515
<description>

apps/bfd-model/bfd-model-dsl-codegen/pom.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
<parent>
55
<groupId>gov.cms.bfd</groupId>
66
<artifactId>bfd-model-parent</artifactId>
7-
<version>2.213.0-SNAPSHOT</version>
7+
<version>2.216.0-SNAPSHOT</version>
88
</parent>
99

1010
<artifactId>bfd-model-dsl-codegen-parent</artifactId>

apps/bfd-model/bfd-model-idr/README.md

Lines changed: 33 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,28 @@ pass along the resource url with -r
7474
pass along --test to run conformance tests
7575

7676

77-
To generate synthetic claims data, the claims_generator.py script is used. As of 5/22/25, it will only generate inpatient institutional claims and their PAC equivalent claim type codes.
77+
To generate synthetic patient data, the patient_generator.py script is used.
78+
To utilize it:
79+
```sh
80+
uv run patient_generator.py
81+
```
82+
83+
The script supports several options:
84+
- `--benes <csv_file>`: Use a CSV file containing beneficiary data to populate fields. Empty fields will be filled with random data.
85+
- `--claims`: Automatically generate claims after patient generation using the generated SYNTHETIC_BENE_HSTRY.csv file.
86+
87+
The files output will be in the out folder:
88+
SYNTHETIC_BENE_HSTRY.csv
89+
SYNTHETIC_BENE_MBI_ID.csv
90+
SYNTHETIC_BENE_MDCR_ENTLMT_RSN.csv
91+
SYNTHETIC_BENE_MDCR_ENTLMT.csv
92+
SYNTHETIC_BENE_MDCR_STUS.csv
93+
SYNTHETIC_BENE_TP.csv
94+
SYNTHETIC_BENE_XREF.csv
95+
96+
The patient generator creates synthetic beneficiary data with realistic but SYNTHETIC MBIs, coverage information, and historical records. It can generate multiple MBI versions per beneficiary and handles beneficiary cross-references with kill credit switches.
97+
98+
To generate synthetic claims data, the claims_generator.py script is used.
7899
To utilize it:
79100
```sh
80101
uv run claims_generator.py \
@@ -83,31 +104,22 @@ uv run claims_generator.py \
83104
```
84105

85106
--sushi is not strictly needed, if you have a local copy of the compiled shorthand files, but recommended to reduce drift. To specify a list of benes, pass in a .csv file containing a column named BENE_SK.
86-
The files output will be in the outputs folder, there are several files:
87-
SYNTHETIC_CLM_DCMTN.csv
88-
SYNTHETIC_CLM_LINE_INSTNL.csv
89-
SYNTHETIC_CLM_INSTNL.csv
107+
The files output will be in the out folder, there are several files:
108+
SYNTHETIC_CLM.csv
109+
SYNTHETIC_CLM_LINE.csv
110+
SYNTHETIC_CLM_VAL.csv
90111
SYNTHETIC_CLM_DT_SGNTR.csv
91112
SYNTHETIC_CLM_PROD.csv
92-
SYNTHETIC_CLM_VAL.csv
93-
SYNTHETIC_CLM_LINE.csv
94-
SYNTHETIC_CLM.csv
113+
SYNTHETIC_CLM_INSTNL.csv
114+
SYNTHETIC_CLM_LINE_INSTNL.csv
115+
SYNTHETIC_CLM_DCMTN.csv
116+
SYNTHETIC_CLM_FISS.csv
117+
SYNTHETIC_CLM_PRFNL.csv
118+
SYNTHETIC_CLM_LINE_PRFNL.csv
119+
SYNTHETIC_CLM_ANSI_SGNTR.csv
95120

96121
These files represent the schema of the tables the information is sourced from, although for tables other than CLM_DT_SGNTR, the CLM_UNIQ_ID is propagated instead of the 5 part unique key from the IDR.
97122

98-
To generate synthetic patient data, the patient_generator.py script is used.
99-
To utilize it:
100-
```sh
101-
uv run patient_generator.py
102-
```
103-
The files output will be in the outputs folder:
104-
SYNTHETIC_BENE_HSTRY.csv
105-
SYNTHETIC_BENE_MBI_ID.csv
106-
SYNTHETIC_BENE_MDCR_ENTLMT_RSN.csv
107-
SYNTHETIC_BENE_MDCR_ENTLMT.csv
108-
SYNTHETIC_BENE_MDCR_STUS.csv
109-
SYNTHETIC_BENE.csv
110-
111123
Data Dictionary Notes:
112124
Generally, the data dictionary will source definitions from the IDR's table definitions. There are instances where this may not be the definition we wish to publish. To overwrite the definition from the IDR, or populate a definition not available from the IDR, populate the "definition" key for the relevant concept in the relevant StructureDefinition.
113125

apps/bfd-model/bfd-model-idr/sample-data/generator/claims_generator.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -743,7 +743,7 @@ def add_meta_timestamps(obj, clm, max_date):
743743
def main():
744744
parser = argparse.ArgumentParser(description='Generate Synthetic Data for Ingestion by the BFD v3 pipeline.')
745745
parser.add_argument('--sushi', '-s', action='store_true', help='Generate new StructureDefinitions. Use when testing locally if new .fsh files have been added.')
746-
parser.add_argument('--benes', '-b', type=str, help='Pull BENE_SKs from the input file. Expected format is that of SYNTHETIC_BENE.csv')
746+
parser.add_argument('--benes', '-b', type=str, help='Pull BENE_SKs from the input file. Expected format is that of SYNTHETIC_BENE_HSTRY.csv')
747747

748748
args = parser.parse_args()
749749
if(args.sushi):

apps/bfd-model/bfd-model-idr/sample-data/generator/generator_util.py

Lines changed: 57 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,8 @@ def __init__(self):
1818
self.fake = Faker()
1919
self.used_bene_sk = []
2020
self.used_mbi = []
21-
self.bene_table = []
2221
self.bene_hstry_table = []
22+
self.bene_xref_table = []
2323
self.mbi_table = {}
2424
self.address_options = []
2525
self.mdcr_stus = []
@@ -108,6 +108,29 @@ def gen_bene_sk(self):
108108
if bene_sk in self.used_bene_sk:
109109
return self.gen_bene_sk()
110110
return bene_sk
111+
112+
def generate_bene_xref(self, new_bene_sk, old_bene_sk):
113+
#10% chance for invalid xref.
114+
kill_cred_cd = 1 if random.randint(1, 10) == 1 else 2
115+
116+
efctv_ts = self.fake.date_time_between_dates(
117+
datetime.date(year=2017, month=5, day=20),
118+
datetime.datetime.now() - datetime.timedelta(days=1)
119+
)
120+
insrt_ts = self.fake.date_time_between_dates(efctv_ts, datetime.datetime.now() - datetime.timedelta(days=1))
121+
updt_ts = self.fake.date_time_between_dates(insrt_ts, datetime.datetime.now() - datetime.timedelta(days=1))
122+
123+
xref_row = {
124+
"BENE_SK": str(new_bene_sk),
125+
"BENE_XREF": str(old_bene_sk),
126+
"BENE_KILL_CRED_CD": str(kill_cred_cd),
127+
"IDR_TRANS_EFCTV_TS": str(efctv_ts),
128+
"IDR_INSRT_TS": str(insrt_ts),
129+
"IDR_UPDT_TS": str(updt_ts),
130+
"IDR_TRANS_OBSLT_TS": "9999-12-31T00:00:00.000000+0000"
131+
}
132+
133+
self.bene_xref_table.append(xref_row)
111134

112135
def gen_address(self):
113136
return self.address_options[random.randint(0, len(self.address_options) - 1)]
@@ -126,6 +149,7 @@ def create_base_patient(self):
126149
patient = {}
127150
self.set_timestamps(patient, datetime.date(year=2017, month=5, day=20))
128151
patient["CNTCT_LANG_CD"] = random.choice(["~", "ENG", "SPA"])
152+
patient["IDR_LTST_TRANS_FLG"] = "Y"
129153
address = self.gen_address()
130154
for component in address:
131155
patient[component] = address[component]
@@ -173,12 +197,15 @@ def handle_mbis(self, patient, num_mbis, custom_first_mbi=None):
173197
)
174198
mbi_obj["BENE_MBI_OBSLT_DT"] = obslt_dt.strftime("%Y-%m-%d")
175199

176-
# Create historical entry for the OLD MBI (not the new one)
177-
historical_patient = copy.deepcopy(patient)
178-
historical_patient["BENE_MBI_ID"] = previous_mbi if previous_mbi else patient["BENE_MBI_ID"]
179-
# Set the obsolescence timestamp for the historical entry
180-
self.set_timestamps(historical_patient, obslt_dt)
181-
self.bene_hstry_table.append(historical_patient)
200+
if previous_mbi and previous_mbi != current_mbi:
201+
historical_patient = copy.deepcopy(patient)
202+
historical_patient["BENE_MBI_ID"] = previous_mbi
203+
historical_patient["IDR_LTST_TRANS_FLG"] = "N"
204+
205+
self.set_timestamps(historical_patient, obslt_dt)
206+
historical_patient["IDR_TRANS_OBSLT_TS"] = str(obslt_dt) + "T00:00:00.000000+0000"
207+
self.bene_hstry_table.append(historical_patient)
208+
182209

183210
previous_obslt_dt = obslt_dt # Store for next iteration
184211
else:
@@ -260,45 +287,32 @@ def generate_coverages(self, patient):
260287
def save_output_files(self):
261288
Path("out").mkdir(exist_ok=True)
262289

263-
df = pd.json_normalize(self.bene_table)
264-
df = df[
265-
[
266-
"BENE_SK",
267-
"BENE_XREF_EFCTV_SK",
268-
"BENE_MBI_ID",
269-
"BENE_LAST_NAME",
270-
"BENE_1ST_NAME",
271-
"BENE_MIDL_NAME",
272-
"BENE_BRTH_DT",
273-
"BENE_DEATH_DT",
274-
"BENE_VRFY_DEATH_DAY_SW",
275-
"BENE_SEX_CD",
276-
"BENE_RACE_CD",
277-
"BENE_LINE_1_ADR",
278-
"BENE_LINE_2_ADR",
279-
"BENE_LINE_3_ADR",
280-
"BENE_LINE_4_ADR",
281-
"BENE_LINE_5_ADR",
282-
"BENE_LINE_6_ADR",
283-
"GEO_ZIP_PLC_NAME",
284-
"GEO_ZIP5_CD",
285-
"GEO_USPS_STATE_CD",
286-
"CNTCT_LANG_CD",
287-
"IDR_TRANS_EFCTV_TS",
288-
"IDR_INSRT_TS",
289-
"IDR_UPDT_TS",
290-
"IDR_TRANS_OBSLT_TS",
291-
]
292-
]
293-
df.to_csv("out/SYNTHETIC_BENE.csv", index=False)
294-
295290
df = pd.json_normalize(self.bene_hstry_table)
296291
if(df.size>0):
297292
df = df[
298293
[
299294
"BENE_SK",
300295
"BENE_XREF_EFCTV_SK",
301296
"BENE_MBI_ID",
297+
"BENE_LAST_NAME",
298+
"BENE_1ST_NAME",
299+
"BENE_MIDL_NAME",
300+
"BENE_BRTH_DT",
301+
"BENE_DEATH_DT",
302+
"BENE_VRFY_DEATH_DAY_SW",
303+
"BENE_SEX_CD",
304+
"BENE_RACE_CD",
305+
"BENE_LINE_1_ADR",
306+
"BENE_LINE_2_ADR",
307+
"BENE_LINE_3_ADR",
308+
"BENE_LINE_4_ADR",
309+
"BENE_LINE_5_ADR",
310+
"BENE_LINE_6_ADR",
311+
"GEO_ZIP_PLC_NAME",
312+
"GEO_ZIP5_CD",
313+
"GEO_USPS_STATE_CD",
314+
"CNTCT_LANG_CD",
315+
"IDR_LTST_TRANS_FLG",
302316
"IDR_TRANS_EFCTV_TS",
303317
"IDR_INSRT_TS",
304318
"IDR_UPDT_TS",
@@ -332,3 +346,7 @@ def save_output_files(self):
332346
df = pd.json_normalize(self.mdcr_rsn)
333347
df.to_csv("out/SYNTHETIC_BENE_MDCR_ENTLMT_RSN.csv", index=False)
334348

349+
df = pd.json_normalize(self.bene_xref_table)
350+
if(df.size>0):
351+
df.to_csv("out/SYNTHETIC_BENE_XREF.csv", index=False)
352+

0 commit comments

Comments
 (0)