BFD-4654: Python Code Cleanup and Upgrade Python to 3.14

apps/bfd-model-idr/augment_sample_resources.py

@@ -1,10 +1,9 @@
 import json
 import sys
 from dataclasses import asdict, dataclass, field
+from datetime import UTC, datetime
 from decimal import Decimal
 from pathlib import Path
-from typing import Optional
-from datetime import datetime, timezone
 
 import pandas as pd
 
@@ -92,19 +91,19 @@ def convert_to_decimal(val: str | None) -> Decimal:
 
 @dataclass
 class Provider:
-    PRVDR_SK: Optional[str] = None
-    PRVDR_ID_QLFYR_CD: Optional[str] = None
-    NPI_TYPE: Optional[str] = None
-    careTeamType: Optional[str] = None
-    careTeamSequenceNumber: Optional[str] = None
-    PRVDR_LAST_OR_LGL_NAME: Optional[str] = None
-    PRVDR_1ST_NAME: Optional[str] = None
-    PRVDR_CARETEAM_NAME: Optional[str] = None
-    specialtyCode: Optional[str] = None
-    PRVDR_OSCAR_NUM: Optional[str] = None
-    CLM_BLG_PRVDR_ZIP5_CD: Optional[str] = None
-    CLM_PRVDR_GNRC_ID_NUM: Optional[str] = None
-    CLM_BLG_PRVDR_TAX_NUM: Optional[str] = None
+    PRVDR_SK: str | None = None
+    PRVDR_ID_QLFYR_CD: str | None = None
+    NPI_TYPE: str | None = None
+    careTeamType: str | None = None
+    careTeamSequenceNumber: str | None = None
+    PRVDR_LAST_OR_LGL_NAME: str | None = None
+    PRVDR_1ST_NAME: str | None = None
+    PRVDR_CARETEAM_NAME: str | None = None
+    specialtyCode: str | None = None
+    PRVDR_OSCAR_NUM: str | None = None
+    CLM_BLG_PRVDR_ZIP5_CD: str | None = None
+    CLM_PRVDR_GNRC_ID_NUM: str | None = None
+    CLM_BLG_PRVDR_TAX_NUM: str | None = None
 
 populate_fields_except_na = [
     "PRVDR_LGL_NAME",
@@ -409,7 +408,7 @@ def next_row_num(supporting_info):
 
 filename = "out/temporary-sample.json"
 
-cur_sample_data["lastUpdated"] = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+cur_sample_data["lastUpdated"] = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
 
 with Path(filename).open("w") as f:
     json.dump(cur_sample_data, f, indent=4)

apps/bfd-model-idr/claims_generator.py

@@ -7,9 +7,8 @@
 from typing import Any
 
 import click
-import tqdm
-
 import field_constants as f
+import tqdm
 from claims_adj import AdjudicatedGeneratorUtil
 from claims_other import OtherGeneratorUtil
 from claims_pac import PacGeneratorUtil
@@ -787,7 +786,7 @@ def generate(
     clm_line_rx_per_clm_uniq_id = {str(row[f.CLM_UNIQ_ID]): row for row in files[CLM_LINE_RX]}
     clm_dcmtns_per_fpk = partition_rows(
         llist=files[CLM_DCMTN],
-        part_by=lambda x: four_part_key(x),
+        part_by=four_part_key,
     )
     dsprtnt_clm_val_per_fpk = {
         four_part_key(x): x for x in files[CLM_VAL] if int(x[f.CLM_VAL_CD]) == 18
@@ -797,30 +796,30 @@ def generate(
     }
     proc_clm_prod_per_fpk = partition_rows(
         llist=files[CLM_PROD],
-        part_by=lambda x: four_part_key(x),
+        part_by=four_part_key,
         filter_by=lambda x: x[f.CLM_PROD_TYPE_CD] == "S",
     )
     diag_clm_prod_per_fpk = partition_rows(
         llist=files[CLM_PROD],
-        part_by=lambda x: four_part_key(x),
+        part_by=four_part_key,
         filter_by=lambda x: x[f.CLM_PROD_TYPE_CD] != "S",
     )
     clm_dt_sgntr_per_sk = {str(row[f.CLM_DT_SGNTR_SK]): row for row in files[CLM_DT_SGNTR]}
     clm_instnl_per_fpk = {four_part_key(row): row for row in files[CLM_INSTNL]}
     clm_prfnls_per_fpk = partition_rows(
         llist=files[CLM_PRFNL],
-        part_by=lambda x: four_part_key(x),
+        part_by=four_part_key,
     )
     clm_line_instnls_per_fpk = partition_rows(
-        llist=files[CLM_LINE_INSTNL], part_by=lambda x: four_part_key(x)
+        llist=files[CLM_LINE_INSTNL], part_by=four_part_key
     )
     clm_line_prfnls_per_fpk = partition_rows(
-        llist=files[CLM_LINE_PRFNL], part_by=lambda x: four_part_key(x)
+        llist=files[CLM_LINE_PRFNL], part_by=four_part_key
     )
     clm_fiss_per_fpk = {four_part_key(row): row for row in files[CLM_FISS]}
     clm_lctn_hstry_per_fpk = {four_part_key(row): row for row in files[CLM_LCTN_HSTRY]}
     clm_line_dcmtns_per_clk = partition_rows(
-        llist=files[CLM_LINE_DCMTN], part_by=lambda x: four_part_key(x)
+        llist=files[CLM_LINE_DCMTN], part_by=four_part_key
     )
 
     # pac CLM generation is random per-generation iteration, so the number of pac CLMs rows is
@@ -1026,17 +1025,9 @@ def generate(
             for file_pac_clm in existing_pac_clms
         ]
         init_pac_clms_tbls = (
-            pac_clms_tbls_from_file
-            if pac_clms_tbls_from_file
-            else [
-                x
-                for x in all_adj_clms_tbls
-                if probability(0.5)
-                and int(x[CLM][0][f.CLM_TYPE_CD])
-                not in (1, 2, 3, 4)  # obviously we don't have pac claims for PD claims
-            ]
-            if gen_new_pac_clms
-            else []
+            pac_clms_tbls_from_file or 
+            ([x for x in all_adj_clms_tbls if probability(0.5) and int(x[CLM][0][f.CLM_TYPE_CD]) not in (1, 2, 3, 4)]
+             if gen_new_pac_clms else [])
         )
         for claims_tbls in init_pac_clms_tbls:
             pac_clm = pac_util.gen_pac_clm(init_clm=claims_tbls[CLM][0])

apps/bfd-model-idr/claims_other.py

@@ -2,10 +2,8 @@
 from datetime import date, datetime
 from typing import Any
 
-import pandas as pd
-from faker import Faker
-
 import field_constants as f
+import pandas as pd
 from claims_static import (
     AVAILABLE_FAMILY_NAMES,
     AVAILABLE_GIVEN_NAMES,
@@ -15,6 +13,7 @@
     AVAILABLE_PROVIDER_TYPE_CODES,
     NOW,
 )
+from faker import Faker
 from generator_util import (
     CLM_ANSI_SGNTR,
     RowAdapter,

apps/bfd-model-idr/claims_util.py

@@ -1,10 +1,9 @@
 import random
 from datetime import date, datetime
 
-from faker import Faker
-
 import field_constants as f
 from claims_static import NOW
+from faker import Faker
 from generator_util import RowAdapter
 
 _faker = Faker()

apps/bfd-model-idr/compile_resources.py

@@ -2,9 +2,10 @@
 import json
 import subprocess
 import sys
-import requests
 from pathlib import Path
 
+import requests
+
 MATCHBOX_SERVER = "http://localhost:8080/matchboxv3"
 
 
@@ -126,9 +127,10 @@ def run_conformance_test(input_file, output_file):
 
         errors = []
         if "issue" in outcome:
-            for issue in outcome["issue"]:
-                if issue.get("severity") in ["error", "fatal"]:
-                    errors.append(issue)
+            errors.extend(
+                issue for issue in outcome["issue"]
+                if issue.get("severity") in ["error", "fatal"]
+            )
 
         if errors:
             print(f"Validation failed with {len(errors)} errors. Note, not all errors are bad:")

apps/bfd-model-idr/dd_helper_script.py

@@ -1,7 +1,9 @@
-'''
+"""
 This script can be used, with the requisite copybooks from the IDR, to do some comparisons for accuracy on given profiles/applicability in BFD v3.
 This will be used as part of future work to enhance the IDR copybooks and improve the quality of our data dictionary.
-'''
+"""
+from collections import Counter  # ✅ Solution 3: Add this import
+
 import pandas as pd
 import yaml
 
@@ -39,27 +41,25 @@
 
 #If we pull more than the current CLM_FISS / CLM_MCS fields (1 and 0, respectively) then we'll need to profile 
 #the individual claim types within PAC data. Otherwise, this just updates the source
-for cur_source in otherSources:
+for cur_source in other_sources:  # ✅ Solution 1: Change otherSources to other_sources
     df = pd.read_excel(other_sources[cur_source], sheet_name='Claim Header',header=3, usecols=['Target Table','Target Column'])
     for _, row in df.iterrows():
         element_concatenated = f"{str(row['Target Table']).strip()}.{str(row['Target Column']).strip()}"
         #there are newlines in some, we should ask IDR to consider changing the DD structure?
         if row['Target Table'] != '-' and row['Target Column'] != '-' and "\n" not in element_concatenated:
             if element_concatenated in applies_to and cur_source not in applies_to[element_concatenated]['sources']:
                 applies_to[element_concatenated]['sources'].append(cur_source)
-            pass
     df = pd.read_excel(other_sources[cur_source], sheet_name='Claim Line',header=3, usecols=['Target Table','Target Column'])
     for _, row in df.iterrows():
         element_concatenated = f"{str(row['Target Table']).strip()}.{str(row['Target Column']).strip()}"
         #there are newlines in some, we should ask IDR to consider changing the DD structure?
         if row['Target Table'] != '-' and row['Target Column'] != '-' and "\n" not in element_concatenated:
-            if element_concatenated in appliesTo and cur_source not in applies_to[element_concatenated]['sources']:
+            if element_concatenated in applies_to and cur_source not in applies_to[element_concatenated]['sources']:  # ✅ Solution 2: Change appliesTo to applies_to
                 applies_to[element_concatenated]['sources'].append(cur_source)
-            pass
 
 #this is more of a heuristic. for example, HCPCS_5_MDFR_CD appears to be missing from the copybook but it's there in reality. 
 profile_divergence_counter = 0
-with open(eob_dict_yaml, "r") as f:
+with open(eob_dict_yaml) as f:
     data = yaml.safe_load(f)
 for i in data:
     if 'sourceView' in i:
@@ -72,7 +72,7 @@
 print("remaining diverging for profiles:",profile_divergence_counter)
 
 source_divergence_counter = 0
-with open(eob_dict_yaml, "r") as f:
+with open(eob_dict_yaml) as f:
     data = yaml.safe_load(f)
 for i in data:
     if 'sourceView' in i:

apps/bfd-model-idr/gen_dd.py

@@ -77,7 +77,7 @@
             data = yaml.safe_load(file)
             current_resource_type = file_name[0 : len(file_name) - 5]
             for entry in data:
-                if "suppressInDD" in entry and entry["suppressInDD"]:
+                if entry.get("suppressInDD"):
                     continue
                 if "fhirPath" in entry:
                     entry["appliesTo"].sort()

apps/bfd-model-idr/load_splitter.py

@@ -5,9 +5,8 @@
 from pathlib import Path
 
 import click
-import tqdm
-
 import field_constants as f
+import tqdm
 from claims_util import four_part_key
 from generator_util import (
     BENE_DUAL,
@@ -113,42 +112,42 @@ def main(clm_batches_size: int, out: Path):
     )
     clm_dcmtns_per_fpk = partition_rows(
         llist=files_to_split[CLM_DCMTN],
-        part_by=lambda x: four_part_key(x),
+        part_by=four_part_key,
     )
     clm_vals_per_fpk = partition_rows(
         llist=files_to_split[CLM_VAL],
-        part_by=lambda x: four_part_key(x),
+        part_by=four_part_key,
     )
     clm_prods_per_fpk = partition_rows(
         llist=files_to_split[CLM_PROD],
-        part_by=lambda x: four_part_key(x),
+        part_by=four_part_key,
     )
     clm_dt_sgntrs_per_sk = partition_rows(
         llist=files_to_split[CLM_DT_SGNTR],
         part_by=lambda x: str(x[f.CLM_DT_SGNTR_SK]),
     )
     clm_instnls_per_fpk = partition_rows(
         llist=files_to_split[CLM_INSTNL],
-        part_by=lambda x: four_part_key(x),
+        part_by=four_part_key,
     )
     clm_prfnls_per_fpk = partition_rows(
         llist=files_to_split[CLM_PRFNL],
-        part_by=lambda x: four_part_key(x),
+        part_by=four_part_key,
     )
     clm_line_instnls_per_fpk = partition_rows(
-        llist=files_to_split[CLM_LINE_INSTNL], part_by=lambda x: four_part_key(x)
+        llist=files_to_split[CLM_LINE_INSTNL], part_by=four_part_key
     )
     clm_line_prfnls_per_fpk = partition_rows(
-        llist=files_to_split[CLM_LINE_PRFNL], part_by=lambda x: four_part_key(x)
+        llist=files_to_split[CLM_LINE_PRFNL], part_by=four_part_key
     )
     clm_fiss_per_fpk = partition_rows(
-        llist=files_to_split[CLM_FISS], part_by=lambda x: four_part_key(x)
+        llist=files_to_split[CLM_FISS], part_by=four_part_key
     )
     clm_lctn_hstrys_per_fpk = partition_rows(
-        llist=files_to_split[CLM_LCTN_HSTRY], part_by=lambda x: four_part_key(x)
+        llist=files_to_split[CLM_LCTN_HSTRY], part_by=four_part_key
     )
     clm_line_dcmtns_per_fpk = partition_rows(
-        llist=files_to_split[CLM_LINE_DCMTN], part_by=lambda x: four_part_key(x)
+        llist=files_to_split[CLM_LINE_DCMTN], part_by=four_part_key
     )
 
     batched_tables: dict[int, dict[str, list[RowAdapter]]] = defaultdict(lambda: defaultdict(list))

apps/bfd-model-idr/patient_generator.py

@@ -6,7 +6,6 @@
 
 import tqdm
 from faker import Faker
-
 from generator_util import (
     BENE_DUAL,
     BENE_ENTLMT,

pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "bfd-workspace"
 version = "0.1.0"
-requires-python="==3.13.*"
+requires-python="==3.14.*"
 description = "BFD Python workspace for unified dependency management"
 
 dependencies = [
@@ -22,10 +22,10 @@ members = [
     # NOTE: apps/** and ops/** will include the every subdirectory and fail on those without pyproject.toml
     # Apps directory
     "apps/bfd-model-idr", 
-#    "apps/bfd-pipeline-idr", # 3.14 isolated uv workspace for now
+    "apps/bfd-pipeline-idr", # 3.14 isolated uv workspace for now
     "apps/bfd-server-ng/codegen",
     "apps/utils/locust_tests",
-#    "apps/utils/samhsa-regression-tests", # 3.14 isolated uv workspace for now
+    "apps/utils/samhsa-regression-tests", # 3.14 isolated uv workspace for now
     "apps/utils/synthetic-load-converter",
     # Ops directory add new members here as needed
     "ops/images/bfd-platform-mount-certstores",
@@ -34,7 +34,7 @@ members = [
     "ops/services/02-eft/lambda_src/sftp_outbound_transfer",
     "ops/services/03-eft-o11y/lambda_src/outbound_slack_notifier",
     "ops/services/04-ccw-pipeline/lambda-src/ccw-runner",
-#    "ops/services/04-idr-pipeline/lambda_src/consume-idr-events", # 3.14 isolated uv workspace for now
+    "ops/services/04-idr-pipeline/lambda_src/consume-idr-events", # 3.14 isolated uv workspace for now
     "ops/services/04-server/lambda_src/regression-wrapper",
     "ops/services/05-server-insights/lambda_src/cw-to-flattened-json",
     "ops/services/06-ccw-pipeline-alarms/lambda_src/manifests-verifier",