BFD-4655: Update BFD synthetic data with Clear test users (#3083)

Co-authored-by: aschey-forpeople <162160982+aschey-forpeople@users.noreply.github.com>
Co-authored-by: Austin Schey <austin.schey@forpeople.us>
Co-authored-by: Alex Dzeda <alex@e14s.com>

Author: 3 people

apps/bfd-model-idr/synthetic-data/SYNTHETIC_BENE_HSTRY.csv

@@ -95,7 +95,7 @@ def __init__(
         self.insert_timer = Timer("insert", model, partition)
         self.commit_timer = Timer("commit", model, partition)
         self.load_type = load_type
-        self.enable_load_progress = load_mode == LoadMode.IDR or force_load_progress()
+        self.enable_load_progress = should_track_load_progress(load_mode)
 
     def load(
         self,
@@ -352,3 +352,8 @@ def _convert_date(date_field: date | datetime) -> datetime:
     if type(date_field) is datetime:
         return date_field.replace(tzinfo=UTC)
     return datetime.combine(date_field, datetime.min.time()).replace(tzinfo=UTC)
+
+
+def should_track_load_progress(load_mode: LoadMode) -> bool:
+    # Whether to read/write load progress, which is diabled for synthetic and testing loads.
+    return load_mode == LoadMode.IDR or force_load_progress()

apps/bfd-pipeline-idr/loader.py

@@ -139,19 +139,19 @@ def repl_interstate(match: re.Match[str]) -> str:
         (r"\bBZN\b", "BOX"),
         (r"\bB0X\b", "BOX"),
         (r"\bRUTA RURAL\b", "RR"),
-        (r"\bIH(\d+[A-Z]?)\b", r"INTERSTATE \1"),
-        (r"\bI\s?(\d+[A-Z]?)(?:\s+(?:HIGHWAY|HWY))?\b", repl_interstate),
-        (r"\bUS(?:\s+HIGHWAY)?\s+(\d+[A-Z]?)(?:\s+(?:HIGHWAY|HWY))?\b", r"US HIGHWAY \1"),
+        (r"\bIH(\d{1,4}[A-Z]?)\b", r"INTERSTATE \1"),
+        (r"\bI\s?(\d{1,4}[A-Z]?)(?:\s+(?:HIGHWAY|HWY))?\b", repl_interstate),
+        (r"\bUS(?:\s+HIGHWAY)?\s+(\d{1,4}[A-Z]?)(?:\s+(?:HIGHWAY|HWY))?\b", r"US HIGHWAY \1"),
         (r"\bBYP ROAD\b", "BYPASS ROAD"),
         (r"\bKY\s+(\d{1,4})(?:\s+(?:HIGHWAY|HWY))?\b", r"KY HIGHWAY \1"),
         (r"\bCNTY\b", "COUNTY"),
-        (r"\bCR\s+(\d+[A-Z]?)(?:\s+(?:ROAD|RD))?\b", r"COUNTY ROAD \1"),
+        (r"\bCR\s+(\d{1,4}[A-Z]?)(?:\s+(?:ROAD|RD))?\b", r"COUNTY ROAD \1"),
         (r"\bHWY\b", "HIGHWAY"),
         (r"\bRD\b", "ROAD"),
         (r"\bRT\b", "ROUTE"),
         (r"\bRTE\b", "ROUTE"),
         (r"\bSR\s+([A-Z]+)(?:\s+(?:ROUTE|RT|RTE|ROAD|RD))?\b", r"STATE ROUTE \1"),
-        (r"\bSR\s+(\d+[A-Z]?)(?:\s+(?:ROAD|RD|ROUTE|RT|RTE))?\b", r"STATE ROAD \1"),
+        (r"\bSR\s+(\d{1,4}[A-Z]?)(?:\s+(?:ROAD|RD|ROUTE|RT|RTE))?\b", r"STATE ROAD \1"),
         (r"\bSR\b", "STATE ROAD"),
         (r"\bTSR\b", "TOWNSHIP ROAD"),
     ]
@@ -169,8 +169,9 @@ def normalize_text(text: str) -> str:
     # Mask floating point periods to preserve them (e.g. 39.2)
     text = re.sub(r"(\d)\.(\d)", r"\1_DOT_\2", text)
 
-    # Remove specific punctuation: *, ., (, ), ", :, ;, ', &, @
-    text = re.sub(r'[*.,()":;\'&@]', "", text)
+    # Remove unnecessary punctuation
+    # Preserve commas since they reduce ambiguity in address components
+    text = re.sub(r'[*.()":;\'&@]', "", text)
 
     # Restore floating point periods
     text = text.replace("_DOT_", ".")
@@ -306,16 +307,9 @@ def normalize_address(address_str: str) -> str:
             else:
                 formatted_lines.append(line)
 
-    normalized_lines: list[str] = []
-    country_line = ""
-    for line in formatted_lines:
-        if line.strip().upper() in COUNTRIES:
-            country_line = line
-        else:
-            normalized_lines.append(line)
-
-    if country_line:
-        normalized_lines.append(country_line)
+    normalized_lines = [
+        line for line in formatted_lines if line.strip() and line.strip().upper() not in COUNTRIES
+    ]
 
     res = "\n".join(normalized_lines)
 
@@ -565,7 +559,7 @@ def get_val_simple(obj: object) -> str:
         if "SubaddressIdentifier" in tokens:
             street_parts.append(tokens["SubaddressIdentifier"])
 
-    # City, State, ZIP — Moved up for boundary standalone passes
+    # City, State, ZIP — Only move to last_line_parts if they appear together
     if "StateName" in tokens:
         state_val_dict = _format_state(tokens["StateName"])
         if "PlaceName" in tokens:
@@ -595,50 +589,6 @@ def get_val_simple(obj: object) -> str:
         final_lines.append(_apply_pr_exceptions(" ".join(line1_parts)))
     if line2_parts:
         final_lines.append(_apply_pr_exceptions(" ".join(line2_parts)))
-    if last_line_parts:
-        # Construct City State Zip carefully
-        city = last_line_parts[0] if len(last_line_parts) > 0 else ""
-        state = last_line_parts[1] if len(last_line_parts) > 1 else ""
-        zip_code_val = last_line_parts[2] if len(last_line_parts) > 2 else ""
-
-        last_line = ""
-        if city:
-            last_line += city
-
-        # Handle Canada spacing
-        if state:
-            state_parts = state.split()
-            if (
-                state_parts[0]
-                in (
-                    "AB",
-                    "BC",
-                    "MB",
-                    "NB",
-                    "NL",
-                    "NT",
-                    "NS",
-                    "NU",
-                    "ON",
-                    "PE",
-                    "QC",
-                    "SK",
-                    "YT",
-                )
-                and len(state_parts) > 1
-            ):
-                # usaddress merged part of Zip into StateName
-                # e.g., 'ON K1A'
-                state = state_parts[0]
-                zip_code_val = " ".join(state_parts[1:]) + (
-                    " " + zip_code_val if zip_code_val else ""
-                )
-
-            last_line += (" " if last_line else "") + state
-
-        if zip_code_val:
-            last_line += (" " if last_line else "") + zip_code_val
-        final_lines.append(last_line)
 
     return "\n".join(final_lines)
 
@@ -684,9 +634,26 @@ def _format_from_raw(raw_parsed: list[tuple[str, str]]) -> str:
             if can_swap and not any(v.isdigit() for v, _label in raw_parsed[:-1]):
                 raw_parsed = [raw_parsed[-1], *raw_parsed[:-1]]
 
+    # Identify and fix lines that are just City State Zip incorrectly tagged
+    if len(raw_parsed) >= 2:
+        v_state, l_state = raw_parsed[-2]
+        _, l_zip = raw_parsed[-1]
+        is_state_abbr = v_state.upper() in STATES.values()
+        is_state_name = v_state.upper() in STATES
+        if l_zip == "ZipCode" and (is_state_abbr or is_state_name) and l_state != "StateName":
+            raw_parsed[-2] = (v_state, "StateName")
+            for i in range(len(raw_parsed) - 2):
+                if raw_parsed[i][1] in ("StreetName", "PlaceName", "StreetNamePostType"):
+                    raw_parsed[i] = (raw_parsed[i][0], "PlaceName")
+
     has_street_name = any(label == "StreetName" for _val, label in raw_parsed)
     reconstructed: list[str] = []
+    geo_labels = ("PlaceName", "StateName", "ZipCode", "CountryName")
+    has_other = any(p_label not in geo_labels for _, p_label in raw_parsed)
+
     for p_val, p_label in raw_parsed:
+        if p_label in geo_labels and not has_other:
+            continue
         # apply directional checks
         res_val = p_val
         if "Directional" in p_label:
@@ -698,6 +665,7 @@ def _format_from_raw(raw_parsed: list[tuple[str, str]]) -> str:
         # apply state checks
         elif p_label == "StateName":
             res_val = _format_state(p_val)
+
         # apply secondary unit
         elif p_label == "OccupancyType":
             res_val = _format_secondary_unit(p_val)
@@ -722,11 +690,6 @@ def _apply_pr_exceptions(text: str) -> str:
     if not lines:
         return text
 
-    last_line_val = ""
-    # If the last line contains a State and Zip code, isolate it from reordering
-    if re.search(r"\b[A-Z]{2}\b\s+\d{5}", lines[-1].upper()):
-        last_line_val = lines.pop(-1)
-
     urb_lines: list[str] = []
     condo_lines: list[str] = []
     street_lines_list: list[str] = []
@@ -767,8 +730,7 @@ def _apply_pr_exceptions(text: str) -> str:
                 break
 
     lines = urb_lines + condo_lines + street_lines_list + other_lines + postal_lines
-    if last_line_val:
-        lines.append(last_line_val)
+
     text = "\n".join(lines)
 
     # Standardize Spanish boxes on the combined text (e.g., APARTADO -> PO BOX)