1+ #!/usr/bin/env python3
2+
3+ import re
4+ import os
5+ from pathlib import Path
6+ from ulid import ULID
7+ import yaml
8+ import toml
9+ from collections import defaultdict
10+ from typing import Dict , List , Set
11+
12+ def parse_member_name (full_name : str ) -> Dict [str , any ]:
13+ """Parse House member full name into components."""
14+ name_parts = {}
15+
16+ # Clean up the full name - remove leading/trailing whitespace and commas
17+ clean_name = full_name .strip ().strip (',' )
18+
19+ # Split by comma to get last name and rest
20+ if ',' in clean_name :
21+ parts = [p .strip () for p in clean_name .split (',' , 1 )]
22+ last_name = parts [0 ].strip ()
23+
24+ # Store last name (convert to title case for consistency)
25+ name_parts ['last_name' ] = last_name .title ()
26+
27+ # Parse first and middle name from the rest
28+ if len (parts ) > 1 :
29+ rest = parts [1 ].strip ()
30+
31+ # Split the rest into words
32+ words = rest .split ()
33+ if words :
34+ # First word is the first name
35+ name_parts ['first_name' ] = words [0 ].title ()
36+
37+ # If there are more words, they're middle names/initials
38+ if len (words ) > 1 :
39+ middle_parts = words [1 :]
40+ # Join all middle parts
41+ middle = ' ' .join (middle_parts )
42+ # Remove periods from initials
43+ middle = middle .replace ('.' , '' )
44+ name_parts ['middle_name' ] = middle .title ()
45+
46+ return name_parts
47+
48+ def extract_congresses_from_file (file_path : Path ) -> Set [int ]:
49+ """Extract congress numbers from a House member file."""
50+ congresses = set ()
51+
52+ with open (file_path , 'r' ) as f :
53+ data = toml .load (f )
54+
55+ # Check for memberships array (newer format)
56+ if 'memberships' in data :
57+ for membership in data ['memberships' ]:
58+ if 'congress' in membership :
59+ congresses .add (membership ['congress' ])
60+
61+ # Check for principalAuthoredBills array (older format)
62+ if 'principalAuthoredBills' in data :
63+ for bill_group in data ['principalAuthoredBills' ]:
64+ if 'congress' in bill_group :
65+ congresses .add (bill_group ['congress' ])
66+
67+ return congresses
68+
69+ def load_existing_mappings (mapping_file : Path ) -> Dict [str , str ]:
70+ """Load existing author ID to ULID mappings from YAML file."""
71+ if mapping_file .exists ():
72+ with open (mapping_file , 'r' ) as f :
73+ return yaml .safe_load (f ) or {}
74+ return {}
75+
76+ def save_mappings (mapping_file : Path , mappings : Dict [str , str ]):
77+ """Save author ID to ULID mappings to YAML file."""
78+ # Sort mappings by key for consistent output
79+ sorted_mappings = dict (sorted (mappings .items ()))
80+ with open (mapping_file , 'w' ) as f :
81+ yaml .dump (sorted_mappings , f , default_flow_style = False , sort_keys = False )
82+
83+ def normalize_name_for_comparison (full_name : str ) -> str :
84+ """Normalize a name for comparison purposes."""
85+ # Remove all non-alphanumeric characters and convert to lowercase
86+ normalized = re .sub (r'[^a-zA-Z0-9]' , '' , full_name ).lower ()
87+ return normalized
88+
89+ def main ():
90+ # Setup paths
91+ base_dir = Path (__file__ ).parent
92+ input_dir = base_dir / 'house' / 'members' / 'all'
93+ output_dir = base_dir / 'house' / 'members' / 'person'
94+
95+ # Create output directory if it doesn't exist
96+ output_dir .mkdir (parents = True , exist_ok = True )
97+
98+ # Load existing mappings
99+ mapping_file = output_dir / 'house-website-key-mapping.yml'
100+ author_mappings = load_existing_mappings (mapping_file )
101+
102+ # Dictionary to group members by normalized name
103+ member_groups = defaultdict (list ) # normalized_name -> [(author_id, id, file_path)]
104+
105+ # First pass: read all files and group by normalized name
106+ print ("Reading House member files..." )
107+ for file_path in sorted (input_dir .glob ('*.toml' )):
108+ with open (file_path , 'r' ) as f :
109+ data = toml .load (f )
110+
111+ author_id = data .get ('authorId' )
112+ member_id = data .get ('id' )
113+ full_name = data .get ('fullName' , '' )
114+
115+ if author_id and full_name :
116+ normalized = normalize_name_for_comparison (full_name )
117+ member_groups [normalized ].append ((author_id , member_id , file_path ))
118+
119+ print (f"Found { len (member_groups )} )
120+
121+ # Process each unique member
122+ processed_ulids = set ()
123+
124+ for normalized_name , member_list in sorted (member_groups .items ()):
125+ # Get or create ULID for this person
126+ primary_author_id = sorted ([author_id for author_id , _ , _ in member_list ])[0 ]
127+
128+ if primary_author_id not in author_mappings :
129+ ulid = str (ULID ())
130+ # Map all author IDs for this person to the same ULID
131+ for author_id , _ , _ in member_list :
132+ author_mappings [author_id ] = ulid
133+ else :
134+ ulid = author_mappings [primary_author_id ]
135+ # Ensure all author IDs map to the same ULID
136+ for author_id , _ , _ in member_list :
137+ author_mappings [author_id ] = ulid
138+
139+ # Skip if we've already processed this ULID
140+ if ulid in processed_ulids :
141+ continue
142+ processed_ulids .add (ulid )
143+
144+ # Collect all congress IDs and author IDs for this person
145+ all_congress_ids = []
146+ all_author_ids = []
147+ all_congresses = set ()
148+
149+ # Use the first file's data as the primary source
150+ primary_file = member_list [0 ][2 ]
151+ with open (primary_file , 'r' ) as f :
152+ primary_data = toml .load (f )
153+
154+ for author_id , member_id , file_path in member_list :
155+ all_author_ids .append (author_id )
156+ if member_id :
157+ all_congress_ids .append (member_id )
158+
159+ # Extract congresses from this file
160+ congresses = extract_congresses_from_file (file_path )
161+ all_congresses .update (congresses )
162+
163+ # Parse name components
164+ full_name = primary_data .get ('fullName' , '' )
165+ name_parts = parse_member_name (full_name )
166+
167+ # Get nickname/aliases
168+ aliases = []
169+ nick_name = primary_data .get ('nickName' , '' ).strip ()
170+ if nick_name and nick_name != full_name :
171+ # Clean up the nickname - remove "HON." prefix if present
172+ nick_name = re .sub (r'^HON\.\s*' , '' , nick_name , flags = re .IGNORECASE ).strip ()
173+ # Only add if it's different from the parsed names
174+ if nick_name and nick_name not in name_parts .get ('first_name' , '' ), full_name ]:
175+ # Extract just the nickname part if it contains the full name
176+ # For example: "JERNIE JETT V. NISAY" -> we want "JERNIE JETT"
177+ if name_parts .get ('first_name' ):
178+ # Check if nickname contains first name
179+ first_name = name_parts ['first_name' ]
180+ if first_name in nick_name :
181+ # Extract the actual nickname part
182+ words = nick_name .split ()
183+ # Keep words until we hit the last name
184+ nickname_parts = []
185+ for word in words :
186+ if word .upper () == name_parts .get ('last_name' , '' ).upper ():
187+ break
188+ nickname_parts .append (word )
189+ if nickname_parts :
190+ actual_nickname = ' ' .join (nickname_parts ).title ()
191+ if actual_nickname != first_name :
192+ aliases .append (actual_nickname )
193+ else :
194+ aliases .append (nick_name .title ())
195+
196+ # Create the person TOML data
197+ person_data = {
198+ 'id' : ulid ,
199+ 'congress_website_primary_keys' : sorted (all_congress_ids ),
200+ 'congress_website_author_keys' : sorted (all_author_ids ),
201+ 'full_name' : full_name
202+ }
203+
204+ # Add name parts
205+ if 'last_name' in name_parts :
206+ person_data ['last_name' ] = name_parts ['last_name' ]
207+ if 'first_name' in name_parts :
208+ person_data ['first_name' ] = name_parts ['first_name' ]
209+ if 'middle_name' in name_parts :
210+ person_data ['middle_name' ] = name_parts ['middle_name' ]
211+
212+ # Add aliases if present
213+ if aliases :
214+ person_data ['aliases' ] = aliases
215+
216+ # Add congresses
217+ if all_congresses :
218+ person_data ['congresses' ] = sorted (list (all_congresses ))
219+
220+ # Write the person TOML file
221+ output_file = output_dir / f"{ ulid }
222+ with open (output_file , 'w' ) as f :
223+ toml .dump (person_data , f )
224+
225+ # Save mappings
226+ print ("Saving mappings..." )
227+ save_mappings (mapping_file , author_mappings )
228+
229+ # Count unique files
230+ unique_member_ulids = len (processed_ulids )
231+
232+ print (f"\n Done!" )
233+ print (f"- Created { unique_member_ulids } { output_dir } )
234+ print (f"- Mapping saved to { mapping_file } { len (author_mappings )} )
235+
236+ if __name__ == "__main__" :
237+ main ()
0 commit comments