fix Philter regex

.gitignore

@@ -1 +1,5 @@
+data/phi/
+data/i2b2_anno/
+data/i2b2_notes/
 .DS_Store
+__pycache__

build/lib/philter_ucsf/coordinate_map.py

@@ -230,7 +230,7 @@ def get_complement(self, filename, text):
 		complement_coordinates = list(set(text_coordinates) - set(current_map_coordinates))
 
 		# Remove punctuation from complement coordinates
-		punctuation_matcher = re.compile(r"[^a-zA-Z0-9*]")
+		punctuation_matcher = re.compile(r"[^À-ÿa-zA-Z0-9*]")
 		for i in range(0, len(text)):
 			if punctuation_matcher.match(text[i]):
 				if i in complement_coordinates:

build/lib/philter_ucsf/eval_scrubber_i2b2.py

@@ -69,7 +69,7 @@ def comparison(filename, file1path, file2path, allpositive_dict):
     #annotation_note = re.sub(r'[\/\-\:\~\_]', ' ', annotation_note)
 
     # get a list of sentences within the note , returns a list of lists  [[sent1],[sent2]] 
-    phi_reduced_note = re.sub(r'\[[A-Z]+\]','', phi_reduced_note)
+    phi_reduced_note = re.sub(r'\[[À-ÞA-Z]+\]','', phi_reduced_note)
     phi_reduced_note = re.sub(r'\n',' ', phi_reduced_note)
     phi_reduced_note = re.sub(r'\#{5}\sDOCUMENT.*','',phi_reduced_note)
     phi_reduced_note = re.sub(r'[\/\-\:\~\_]', ' ', phi_reduced_note)

build/lib/philter_ucsf/eval_scrubber_ucsf.py

@@ -70,7 +70,7 @@ def comparison(filename, file1path, file2path, allpositive_dict):
     #annotation_note = re.sub(r'[\/\-\:\~\_]', ' ', annotation_note)
 
     # get a list of sentences within the note , returns a list of lists  [[sent1],[sent2]] 
-    phi_reduced_note = re.sub(r'\[[A-Z]+\]','', phi_reduced_note)
+    phi_reduced_note = re.sub(r'\[[À-ÞA-Z]+\]','', phi_reduced_note)
     phi_reduced_note = re.sub(r'\n',' ', phi_reduced_note)
     phi_reduced_note = re.sub(r'\#{5}\sDOCUMENT.*','',phi_reduced_note)
     phi_reduced_note = re.sub(r'[\/\-\:\~\_]', ' ', phi_reduced_note)

build/lib/philter_ucsf/filters/regex/addresses/at_address_noindicator.txt

@@ -1 +1 @@
-\b(?<=at\s)(\d+)\s{1,4}(?!bpm\b)(?!ccs\b)(?!cc\b)(?!pm\b)(?!am\b)(?!noon\b)([A-Z][a-zA-Z]+\s{1,4}[A-Z][a-zA-Z]+|[A-Z][a-zA-Z]+|[A-Z])\b(?!\.m\.)
+\b(?<=at\s)(\d+)\s{1,4}(?!bpm\b)(?!ccs\b)(?!cc\b)(?!pm\b)(?!am\b)(?!noon\b)([À-ÞA-Z][À-ÿa-zA-Z]+\s{1,4}[À-ÞA-Z][À-ÿa-zA-Z]+|[À-ÞA-Z][À-ÿa-zA-Z]+|[À-ÞA-Z])\b(?!\.m\.)

build/lib/philter_ucsf/filters/regex/addresses/at_address_noindicator_transformed.txt

@@ -1 +1 @@
-\b(?<=at\s)(\d+)\s{1,4}(?!bpm\b)(?!ccs\b)(?!cc\b)(?!pm\b)(?!am\b)(?!noon\b)([A-Z][a-zA-Z]+\s{1,4}[A-Z][a-zA-Z]+|[A-Z][a-zA-Z]+|[A-Z])\b(?!\.m\.)
+\b(?<=at\s)(\d+)\s{1,4}(?!bpm\b)(?!ccs\b)(?!cc\b)(?!pm\b)(?!am\b)(?!noon\b)([À-ÞA-Z][À-ÿa-zA-Z]+\s{1,4}[À-ÞA-Z][À-ÿa-zA-Z]+|[À-ÞA-Z][À-ÿa-zA-Z]+|[À-ÞA-Z])\b(?!\.m\.)

build/lib/philter_ucsf/filters/regex/addresses/at_street_dash_street.txt

@@ -1 +1 @@
-\b(?<=(?i)at\s)([A-Z][a-zA-Z]+|((?i)("""+day_numbering+r""")))((?i)\s("""+address_indicator+r"""))\/([A-Z][a-zA-Z]+|((?i)("""+day_numbering+r""")))((?i)\s("""+address_indicator+r"""))\b
+\b(?<=(?i)at\s)([À-ÞA-Z][À-ÿa-zA-Z]+|((?i)("""+day_numbering+r""")))((?i)\s("""+address_indicator+r"""))\/([À-ÞA-Z][À-ÿa-zA-Z]+|((?i)("""+day_numbering+r""")))((?i)\s("""+address_indicator+r"""))\b

build/lib/philter_ucsf/filters/regex/addresses/at_street_dash_street_transformed.txt

@@ -1 +1 @@
-\b(?<=(?i)at\s)([A-Z][a-zA-Z]+|((?i)(1st|2nd|3rd|4th|5th|6th|7th|8th|9th|10th|11th|12th|13th|14th|15th|16th|17th|18th|19th|20th|21st|22nd|23rd|24th|25th|26th|27th|28th|29th|30th|31st|32nd|33rd|34th|35th|36th|37th|38th|39th|40th|41st|42nd|43rd|44th|45th|46th|47th|48th|49th|50th)))((?i)\s(alley|alley|ally|aly|anex|annex|annx|anx|apartment|apt|avenue|ave|aven|avenu|avenue|avn|avnue|bayou|bayou|beach|beach|bnd|bluff|bluf|bluff|bluffs|boulevard|boul|boulevard|blvd|boulv|box|branch|brnch|branch|bridge|brg|bridge|brook|brook|brooks|burg|burgs|byps|canyon|canyon|cnyn|cape|cpe|causeway|causwa|cswy|cent|centr|centre|cnter|cntr|ctr|circle|circ|circl|circle|crcl|crcle|circles|cliff|cliff|cliffs|cliffs|club|club|commons|corner|corner|corners|cors|crse|court|ct|courts|cts|cove|cv|coves|creek|crk|crescent|cres|crsent|crsnt|crest|crossing|crssng|xing|crossroad|crossroads|dale|dl|dam|divide|divide|dv|dvd|drive|driv|dr|drv|drives|east|estate|estate|estates|ests|expressway|expr|express|expressway|expw|expy|extn|extnsn|fls|ferry|frry|fry|flat|flt|flats|flts|ford|frd|fords|forest|forests|frst|forge|forge|frg|forges|fork|frk|forks|frks|fort|frt|ft|freeway|freewy|frway|frwy|fwy|garden|gardn|grden|grdn|gardens|gdns|grdns|gateway|gatewy|gatway|gtway|gtwy|glen|gln|glens|green|grn|greens|grove|grove|grv|groves|harbor|harbor|harbr|hbr|hrbor|harbors|haven|hvn|heights|hts|highway|highwy|hiway|hiwy|hway|hwy|hill|hl|hills|hls|hollow|hollow|hollows|holw|holws|iss|isle|isles|knoll|knol|knoll|knolls|knolls|lake|lake|lakes|lakes|land|landing|lndg|lndng|lane|ln|loaf|loaf|lock|lock|locks|locks|ldge|lodg|lodge|loop|loops|mall|manor|manor|manors|mnrs|meadow|meadows|mdws|meadows|medows|mews|mill|mills|mission|mssn|motorway|north|orchard|orchard|orchrd|oval|ovl|overpass|park|prk|parks|parkway|parkwy|pkway|pkwy|pky|parkways|pkwys|pass|passage|pike|pikes|pine|pines|pnes|place|plain|pln|plains|plns|plaza|plz|plza|port|prt|ports|prts|prairie|prairie|prr|ramp|ranch|ranches|rnch|rnchs|rapids|rpds|rst|ridge|rdge|ridge|ridges|ridges|river|river|rvr|rivr|road|road|rd|roads|rds|route|row|run|shoal|shoal|shoals|shoals|shore|shore|shr|shores|shores|shrs|skyway|south|spring|spng|spring|sprng|springs|spngs|springs|sprngs|spur|spurs|square|sqr|sqre|squ|square|squares|squares|station|station|statn|stn|stravenue|strav|straven|stravenue|stravn|strvn|strvnue|stream|streme|strm|street|strt|st|str|streets|suite|summit|sumit|sumitt|summit|terrace|terr|terrace|throughway|trce|track|tracks|trak|trk|trks|trafficway|trail|trails|trl|trls|trailer|trlr|trlrs|tunnel|tunl|tunls|tunnel|tunnels|tunnl|turnpike|turnpike|turnpk|underpass|union|union|unions|valley|vally|vlly|vly|valleys|vlys|viaduct|viadct|viaduct|village|villag|village|villg|villiage|vlg|villages|vlgs|ville|vl|vista|vist|vista|vst|vsta|way|way|ways|west|wls))\/([A-Z][a-zA-Z]+|((?i)(1st|2nd|3rd|4th|5th|6th|7th|8th|9th|10th|11th|12th|13th|14th|15th|16th|17th|18th|19th|20th|21st|22nd|23rd|24th|25th|26th|27th|28th|29th|30th|31st|32nd|33rd|34th|35th|36th|37th|38th|39th|40th|41st|42nd|43rd|44th|45th|46th|47th|48th|49th|50th)))((?i)\s(alley|alley|ally|aly|anex|annex|annx|anx|apartment|apt|avenue|ave|aven|avenu|avenue|avn|avnue|bayou|bayou|beach|beach|bnd|bluff|bluf|bluff|bluffs|boulevard|boul|boulevard|blvd|boulv|box|branch|brnch|branch|bridge|brg|bridge|brook|brook|brooks|burg|burgs|byps|canyon|canyon|cnyn|cape|cpe|causeway|causwa|cswy|cent|centr|centre|cnter|cntr|ctr|circle|circ|circl|circle|crcl|crcle|circles|cliff|cliff|cliffs|cliffs|club|club|commons|corner|corner|corners|cors|crse|court|ct|courts|cts|cove|cv|coves|creek|crk|crescent|cres|crsent|crsnt|crest|crossing|crssng|xing|crossroad|crossroads|dale|dl|dam|divide|divide|dv|dvd|drive|driv|dr|drv|drives|east|estate|estate|estates|ests|expressway|expr|express|expressway|expw|expy|extn|extnsn|fls|ferry|frry|fry|flat|flt|flats|flts|ford|frd|fords|forest|forests|frst|forge|forge|frg|forges|fork|frk|forks|frks|fort|frt|ft|freeway|freewy|frway|frwy|fwy|garden|gardn|grden|grdn|gardens|gdns|grdns|gateway|gatewy|gatway|gtway|gtwy|glen|gln|glens|green|grn|greens|grove|grove|grv|groves|harbor|harbor|harbr|hbr|hrbor|harbors|haven|hvn|heights|hts|highway|highwy|hiway|hiwy|hway|hwy|hill|hl|hills|hls|hollow|hollow|hollows|holw|holws|iss|isle|isles|knoll|knol|knoll|knolls|knolls|lake|lake|lakes|lakes|land|landing|lndg|lndng|lane|ln|loaf|loaf|lock|lock|locks|locks|ldge|lodg|lodge|loop|loops|mall|manor|manor|manors|mnrs|meadow|meadows|mdws|meadows|medows|mews|mill|mills|mission|mssn|motorway|north|orchard|orchard|orchrd|oval|ovl|overpass|park|prk|parks|parkway|parkwy|pkway|pkwy|pky|parkways|pkwys|pass|passage|pike|pikes|pine|pines|pnes|place|plain|pln|plains|plns|plaza|plz|plza|port|prt|ports|prts|prairie|prairie|prr|ramp|ranch|ranches|rnch|rnchs|rapids|rpds|rst|ridge|rdge|ridge|ridges|ridges|river|river|rvr|rivr|road|road|rd|roads|rds|route|row|run|shoal|shoal|shoals|shoals|shore|shore|shr|shores|shores|shrs|skyway|south|spring|spng|spring|sprng|springs|spngs|springs|sprngs|spur|spurs|square|sqr|sqre|squ|square|squares|squares|station|station|statn|stn|stravenue|strav|straven|stravenue|stravn|strvn|strvnue|stream|streme|strm|street|strt|st|str|streets|suite|summit|sumit|sumitt|summit|terrace|terr|terrace|throughway|trce|track|tracks|trak|trk|trks|trafficway|trail|trails|trl|trls|trailer|trlr|trlrs|tunnel|tunl|tunls|tunnel|tunnels|tunnl|turnpike|turnpike|turnpk|underpass|union|union|unions|valley|vally|vlly|vly|valleys|vlys|viaduct|viadct|viaduct|village|villag|village|villg|villiage|vlg|villages|vlgs|ville|vl|vista|vist|vista|vst|vsta|way|way|ways|west|wls))\b
+\b(?<=(?i)at\s)([À-ÞA-Z][À-ÿa-zA-Z]+|((?i)(1st|2nd|3rd|4th|5th|6th|7th|8th|9th|10th|11th|12th|13th|14th|15th|16th|17th|18th|19th|20th|21st|22nd|23rd|24th|25th|26th|27th|28th|29th|30th|31st|32nd|33rd|34th|35th|36th|37th|38th|39th|40th|41st|42nd|43rd|44th|45th|46th|47th|48th|49th|50th)))((?i)\s(alley|alley|ally|aly|anex|annex|annx|anx|apartment|apt|avenue|ave|aven|avenu|avenue|avn|avnue|bayou|bayou|beach|beach|bnd|bluff|bluf|bluff|bluffs|boulevard|boul|boulevard|blvd|boulv|box|branch|brnch|branch|bridge|brg|bridge|brook|brook|brooks|burg|burgs|byps|canyon|canyon|cnyn|cape|cpe|causeway|causwa|cswy|cent|centr|centre|cnter|cntr|ctr|circle|circ|circl|circle|crcl|crcle|circles|cliff|cliff|cliffs|cliffs|club|club|commons|corner|corner|corners|cors|crse|court|ct|courts|cts|cove|cv|coves|creek|crk|crescent|cres|crsent|crsnt|crest|crossing|crssng|xing|crossroad|crossroads|dale|dl|dam|divide|divide|dv|dvd|drive|driv|dr|drv|drives|east|estate|estate|estates|ests|expressway|expr|express|expressway|expw|expy|extn|extnsn|fls|ferry|frry|fry|flat|flt|flats|flts|ford|frd|fords|forest|forests|frst|forge|forge|frg|forges|fork|frk|forks|frks|fort|frt|ft|freeway|freewy|frway|frwy|fwy|garden|gardn|grden|grdn|gardens|gdns|grdns|gateway|gatewy|gatway|gtway|gtwy|glen|gln|glens|green|grn|greens|grove|grove|grv|groves|harbor|harbor|harbr|hbr|hrbor|harbors|haven|hvn|heights|hts|highway|highwy|hiway|hiwy|hway|hwy|hill|hl|hills|hls|hollow|hollow|hollows|holw|holws|iss|isle|isles|knoll|knol|knoll|knolls|knolls|lake|lake|lakes|lakes|land|landing|lndg|lndng|lane|ln|loaf|loaf|lock|lock|locks|locks|ldge|lodg|lodge|loop|loops|mall|manor|manor|manors|mnrs|meadow|meadows|mdws|meadows|medows|mews|mill|mills|mission|mssn|motorway|north|orchard|orchard|orchrd|oval|ovl|overpass|park|prk|parks|parkway|parkwy|pkway|pkwy|pky|parkways|pkwys|pass|passage|pike|pikes|pine|pines|pnes|place|plain|pln|plains|plns|plaza|plz|plza|port|prt|ports|prts|prairie|prairie|prr|ramp|ranch|ranches|rnch|rnchs|rapids|rpds|rst|ridge|rdge|ridge|ridges|ridges|river|river|rvr|rivr|road|road|rd|roads|rds|route|row|run|shoal|shoal|shoals|shoals|shore|shore|shr|shores|shores|shrs|skyway|south|spring|spng|spring|sprng|springs|spngs|springs|sprngs|spur|spurs|square|sqr|sqre|squ|square|squares|squares|station|station|statn|stn|stravenue|strav|straven|stravenue|stravn|strvn|strvnue|stream|streme|strm|street|strt|st|str|streets|suite|summit|sumit|sumitt|summit|terrace|terr|terrace|throughway|trce|track|tracks|trak|trk|trks|trafficway|trail|trails|trl|trls|trailer|trlr|trlrs|tunnel|tunl|tunls|tunnel|tunnels|tunnl|turnpike|turnpike|turnpk|underpass|union|union|unions|valley|vally|vlly|vly|valleys|vlys|viaduct|viadct|viaduct|village|villag|village|villg|villiage|vlg|villages|vlgs|ville|vl|vista|vist|vista|vst|vsta|way|way|ways|west|wls))\/([À-ÞA-Z][À-ÿa-zA-Z]+|((?i)(1st|2nd|3rd|4th|5th|6th|7th|8th|9th|10th|11th|12th|13th|14th|15th|16th|17th|18th|19th|20th|21st|22nd|23rd|24th|25th|26th|27th|28th|29th|30th|31st|32nd|33rd|34th|35th|36th|37th|38th|39th|40th|41st|42nd|43rd|44th|45th|46th|47th|48th|49th|50th)))((?i)\s(alley|alley|ally|aly|anex|annex|annx|anx|apartment|apt|avenue|ave|aven|avenu|avenue|avn|avnue|bayou|bayou|beach|beach|bnd|bluff|bluf|bluff|bluffs|boulevard|boul|boulevard|blvd|boulv|box|branch|brnch|branch|bridge|brg|bridge|brook|brook|brooks|burg|burgs|byps|canyon|canyon|cnyn|cape|cpe|causeway|causwa|cswy|cent|centr|centre|cnter|cntr|ctr|circle|circ|circl|circle|crcl|crcle|circles|cliff|cliff|cliffs|cliffs|club|club|commons|corner|corner|corners|cors|crse|court|ct|courts|cts|cove|cv|coves|creek|crk|crescent|cres|crsent|crsnt|crest|crossing|crssng|xing|crossroad|crossroads|dale|dl|dam|divide|divide|dv|dvd|drive|driv|dr|drv|drives|east|estate|estate|estates|ests|expressway|expr|express|expressway|expw|expy|extn|extnsn|fls|ferry|frry|fry|flat|flt|flats|flts|ford|frd|fords|forest|forests|frst|forge|forge|frg|forges|fork|frk|forks|frks|fort|frt|ft|freeway|freewy|frway|frwy|fwy|garden|gardn|grden|grdn|gardens|gdns|grdns|gateway|gatewy|gatway|gtway|gtwy|glen|gln|glens|green|grn|greens|grove|grove|grv|groves|harbor|harbor|harbr|hbr|hrbor|harbors|haven|hvn|heights|hts|highway|highwy|hiway|hiwy|hway|hwy|hill|hl|hills|hls|hollow|hollow|hollows|holw|holws|iss|isle|isles|knoll|knol|knoll|knolls|knolls|lake|lake|lakes|lakes|land|landing|lndg|lndng|lane|ln|loaf|loaf|lock|lock|locks|locks|ldge|lodg|lodge|loop|loops|mall|manor|manor|manors|mnrs|meadow|meadows|mdws|meadows|medows|mews|mill|mills|mission|mssn|motorway|north|orchard|orchard|orchrd|oval|ovl|overpass|park|prk|parks|parkway|parkwy|pkway|pkwy|pky|parkways|pkwys|pass|passage|pike|pikes|pine|pines|pnes|place|plain|pln|plains|plns|plaza|plz|plza|port|prt|ports|prts|prairie|prairie|prr|ramp|ranch|ranches|rnch|rnchs|rapids|rpds|rst|ridge|rdge|ridge|ridges|ridges|river|river|rvr|rivr|road|road|rd|roads|rds|route|row|run|shoal|shoal|shoals|shoals|shore|shore|shr|shores|shores|shrs|skyway|south|spring|spng|spring|sprng|springs|spngs|springs|sprngs|spur|spurs|square|sqr|sqre|squ|square|squares|squares|station|station|statn|stn|stravenue|strav|straven|stravenue|stravn|strvn|strvnue|stream|streme|strm|street|strt|st|str|streets|suite|summit|sumit|sumitt|summit|terrace|terr|terrace|throughway|trce|track|tracks|trak|trk|trks|trafficway|trail|trails|trl|trls|trailer|trlr|trlrs|tunnel|tunl|tunls|tunnel|tunnels|tunnl|turnpike|turnpike|turnpk|underpass|union|union|unions|valley|vally|vlly|vly|valleys|vlys|viaduct|viadct|viaduct|village|villag|village|villg|villiage|vlg|villages|vlgs|ville|vl|vista|vist|vista|vst|vsta|way|way|ways|west|wls))\b

build/lib/philter_ucsf/filters/regex/addresses/at_street_number_dash_street.txt

@@ -1 +1 @@
-\b(?<=(?i)at\s)\d+\s([A-Z][a-zA-Z]+(\s)?){1,3}\/("""+day_numbering+r""")\b
+\b(?<=(?i)at\s)\d+\s([À-ÞA-Z][À-ÿa-zA-Z]+(\s)?){1,3}\/("""+day_numbering+r""")\b

build/lib/philter_ucsf/filters/regex/addresses/at_street_number_dash_street_transformed.txt

@@ -1 +1 @@
-\b(?<=(?i)at\s)\d+\s([A-Z][a-zA-Z]+(\s)?){1,3}\/(1st|2nd|3rd|4th|5th|6th|7th|8th|9th|10th|11th|12th|13th|14th|15th|16th|17th|18th|19th|20th|21st|22nd|23rd|24th|25th|26th|27th|28th|29th|30th|31st|32nd|33rd|34th|35th|36th|37th|38th|39th|40th|41st|42nd|43rd|44th|45th|46th|47th|48th|49th|50th)\b
+\b(?<=(?i)at\s)\d+\s([À-ÞA-Z][À-ÿa-zA-Z]+(\s)?){1,3}\/(1st|2nd|3rd|4th|5th|6th|7th|8th|9th|10th|11th|12th|13th|14th|15th|16th|17th|18th|19th|20th|21st|22nd|23rd|24th|25th|26th|27th|28th|29th|30th|31st|32nd|33rd|34th|35th|36th|37th|38th|39th|40th|41st|42nd|43rd|44th|45th|46th|47th|48th|49th|50th)\b

build/lib/philter_ucsf/filters/regex/addresses/box_room.txt

@@ -1 +1 @@
-\b((?i)(suite|ste)?(\.)?([ ]{1,2})?\d+([ ]{1,2})?box(:)?[ ]{1,2}\d+(,)?[ ]{1,2}[A-Z](\-)?\d+)\b
+\b((?i)(suite|ste)?(\.)?([ ]{1,2})?\d+([ ]{1,2})?box(:)?[ ]{1,2}\d+(,)?[ ]{1,2}[À-ÞA-Z](\-)?\d+)\b

build/lib/philter_ucsf/filters/regex/addresses/box_room_transformed.txt

@@ -1 +1 @@
-\b((?i)(suite|ste)?(\.)?([ ]{1,2})?\d+([ ]{1,2})?box(:)?[ ]{1,2}\d+(,)?[ ]{1,2}[A-Z](\-)?\d+)\b
+\b((?i)(suite|ste)?(\.)?([ ]{1,2})?\d+([ ]{1,2})?box(:)?[ ]{1,2}\d+(,)?[ ]{1,2}[À-ÞA-Z](\-)?\d+)\b

build/lib/philter_ucsf/filters/regex/addresses/city.txt

@@ -1 +1 @@
-\b([A-Z][a-zA-Z]+\s{1,4}[A-Z][a-zA-Z]+|[A-Z][a-zA-Z]+)\s{1,4}(City|CITY|Street|STREET)\b
+\b([À-ÞA-Z][À-ÿa-zA-Z]+\s{1,4}[À-ÞA-Z][À-ÿa-zA-Z]+|[À-ÞA-Z][À-ÿa-zA-Z]+)\s{1,4}(City|CITY|Street|STREET)\b

build/lib/philter_ucsf/filters/regex/addresses/city_state.txt

@@ -1 +1 @@
-\b(?!(NT\,\sND|MR\,\sLA)\b)(([A-Z][a-zA-Z]+\s{1,4}[A-Z][a-zA-Z]+|[A-Z][a-zA-Z]+)(\,\s{1,4})("""+state_name+r"""))\b
+\b(?!(NT\,\sND|MR\,\sLA)\b)(([À-ÞA-Z][À-ÿa-zA-Z]+\s{1,4}[À-ÞA-Z][À-ÿa-zA-Z]+|[À-ÞA-Z][À-ÿa-zA-Z]+)(\,\s{1,4})("""+state_name+r"""))\b

build/lib/philter_ucsf/filters/regex/addresses/city_state_transformed.txt

@@ -1 +1 @@
-\b(?!(NT\,\sND|MR\,\sLA)\b)(([A-Z][a-zA-Z]+\s{1,4}[A-Z][a-zA-Z]+|[A-Z][a-zA-Z]+)(\,\s{1,4})((A|a)rizona|AZ|(V|v)irginia|VA|(M|m)innesota|MN|(A|a)laska|AK|(N|n)ew (Y|y)ork|NY|(T|t)exas|TX|(V|v)ermont|VT|(U|u)tah|UT|(N|n)ew (J|j)ersey|NJ|(N|n)orth (D|d)akota|ND|(S|s)outh (D|d)akota|SD|(M|m)issouri|MO|(W|w)ashington (D|d).(C|c).|(G|g)eorgia|GA|(M|m)assachusetts|MA|(P|p)uerto (R|r)ico|(M|m)ichigan|MI|(I|i)owa|IA|(N|n)orth (C|c)arolina|NC|(S|s)outh (C|c)arolina|SC|(N|n)evada|NV|(C|c)olorado|CO|(O|o)hio|OH|(H|h)awaii|HI|(N|n)ebraska|NE|(N|n)ew (H|h)ampshire|NH|(W|w)ashington|WA|(T|t)ennessee|TN|(A|a)rkansas|AR|(L|l)ouisiana|LA|(M|m)ississippi|MS|(O|o)regon|OR|(A|a)labama|AL|(W|w)yoming|WY|(W|w)isconsin|WI|(O|o)klahoma|OK|(F|f)lorida|FL|(R|r)hode (I|i)sland|RI|(I|i)ndiana|IN|(C|c)alifornia|CA|(K|k)ansas|KS|(D|d)elaware|DE|(M|m)aryland|(I|i)daho|ID|(P|p)ennsylvania|PA|(K|k)entucky|KY|(C|c)onnecticut|CT|(M|m)ontana|MT|(I|i)llinois|IL|(M|m)aine|ME))\b
+\b(?!(NT\,\sND|MR\,\sLA)\b)(([À-ÞA-Z][À-ÿa-zA-Z]+\s{1,4}[À-ÞA-Z][À-ÿa-zA-Z]+|[À-ÞA-Z][À-ÿa-zA-Z]+)(\,\s{1,4})((A|a)rizona|AZ|(V|v)irginia|VA|(M|m)innesota|MN|(A|a)laska|AK|(N|n)ew (Y|y)ork|NY|(T|t)exas|TX|(V|v)ermont|VT|(U|u)tah|UT|(N|n)ew (J|j)ersey|NJ|(N|n)orth (D|d)akota|ND|(S|s)outh (D|d)akota|SD|(M|m)issouri|MO|(W|w)ashington (D|d).(C|c).|(G|g)eorgia|GA|(M|m)assachusetts|MA|(P|p)uerto (R|r)ico|(M|m)ichigan|MI|(I|i)owa|IA|(N|n)orth (C|c)arolina|NC|(S|s)outh (C|c)arolina|SC|(N|n)evada|NV|(C|c)olorado|CO|(O|o)hio|OH|(H|h)awaii|HI|(N|n)ebraska|NE|(N|n)ew (H|h)ampshire|NH|(W|w)ashington|WA|(T|t)ennessee|TN|(A|a)rkansas|AR|(L|l)ouisiana|LA|(M|m)ississippi|MS|(O|o)regon|OR|(A|a)labama|AL|(W|w)yoming|WY|(W|w)isconsin|WI|(O|o)klahoma|OK|(F|f)lorida|FL|(R|r)hode (I|i)sland|RI|(I|i)ndiana|IN|(C|c)alifornia|CA|(K|k)ansas|KS|(D|d)elaware|DE|(M|m)aryland|(I|i)daho|ID|(P|p)ennsylvania|PA|(K|k)entucky|KY|(C|c)onnecticut|CT|(M|m)ontana|MT|(I|i)llinois|IL|(M|m)aine|ME))\b

build/lib/philter_ucsf/filters/regex/addresses/city_state_zip.txt

@@ -1 +1 @@
-\b(\d+\s[A-Z]\s)?([a-zA-Z]+\s{1,4}[a-zA-Z]+|[a-zA-Z]+)(\:\s{1,4}|\,\s{1,4}|\.\s{1,4}|\s{1,4})("""+state_name+r""")(\:\s{1,4}|\,\s{1,4}|\.\s{1,4}|\s{1,4})[0-9]{5}(\-[0-9]{4})?\b
+\b(\d+\s[À-ÞA-Z]\s)?([À-ÿa-zA-Z]+\s{1,4}[À-ÿa-zA-Z]+|[À-ÿa-zA-Z]+)(\:\s{1,4}|\,\s{1,4}|\.\s{1,4}|\s{1,4})("""+state_name+r""")(\:\s{1,4}|\,\s{1,4}|\.\s{1,4}|\s{1,4})[0-9]{5}(\-[0-9]{4})?\b