Skip to content

2020 update #33

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
paulsalama wants to merge 4 commits into
base: master
Choose a base branch
from
Open

2020 update #33

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
fd89100
add 2019 and fix url
toolness Sep 1, 2019
2105b7b
2020 data and new tax bill
paulsalama Jun 13, 2020
c76c7b7
Update regex parsing for new tax bill format
paulsalama Jun 13, 2020
5cf16d9
Fix download_direct bug & Clarify
paulsalama Jun 13, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 11 additions & 3 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ Under the following terms:

You'll need the following:

- Linux, BSD, or MacOSX machine
- Linux, BSD, or MacOSX machine (or [Windows Subsystem for Linux](https://docs.microsoft.com/en-us/windows/wsl/install-win10))
- Python 2.6 or greater (not tested on 3)
- `virtualenv` or `virtualenvwrapper` (in order to install requirements without
using `sudo`
Expand Down Expand Up @@ -79,14 +79,22 @@ Make sure to put the street name in single quotes.

### To download a single tax bill for many BBLS:

python download_direct.py YYYYMMDD /path/to/bbls.csv > path/to/log.txt 2>&1 &
1. Create a `csv` of BBLs to download, with each separated by a new line. (E.g. all BBLs with 3/6+ units from [PLUTO](https://www1.nyc.gov/site/planning/data-maps/open-data/dwn-pluto-mappluto.page))

2. Then run:

```python download_direct.py YYYYMMDD [SOA/NPV] /path/to/input/bbls.csv > path/to/log.log 2>&1 &```

_more specific example_

python download_direct.py 20200606 SOA ./input/MN_bbls.csv > ./output/log$(date +"%Y%m%d_%H.%M.%S").log 2>&1 | tee -a ./output/wget-log

### To parse the raw data into a CSV

You'll probably want to background this too, as it takes a while. The text PDF
bills are turned into txt files using `pdftotext`.

python parse.py /path/to/input >/path/to/output.csv 2>/path/to/log.txt &
python parse.py ./data/ >/path/to/output.csv 2>/path/to/log.log &

The structure of the CSV is as follows, including types:

Expand Down
25 changes: 15 additions & 10 deletions download_direct.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,17 @@
LOGGER.addHandler(HANDLER)

PERIODS = {
#'20120817 - Quarterly Property Tax Bill.pdf',
('20200606', 'SOA'): 'June 6, 2020 - Quarterly Property Tax Bill.pdf',
('20200222', 'SOA'): 'February 22, 2020 - Quarterly Property Tax Bill.pdf',
('20200115', 'NOPV'): 'January 15, 2020 - Notice of Property Value.pdf',
('20191205', 'SOA'): 'December 5, 2019 - Quarterly Property Tax Bill.pdf',
('20190829', 'SOA'): 'August 29, 2019 - Quarterly Property Tax Bill.pdf',
('20190605', 'SOA'): 'June 5, 2019 - Quarterly Property Tax Bill.pdf',
#Note - new url will only work for Tax Bills above. Use previous url for below.
('20190201', 'SOA'): 'February 1, 2019 - Quarterly Property Tax Bill.pdf',
('20190115', 'NOPV'): 'January 15, 2019 - Notice of Property Value.pdf',
('20181116', 'SOA'): 'November 16, 2018 - Quarterly Property Tax Bill.pdf',
('20180824', 'SOA'): 'August 24, 2018 - Quarterly Property Tax Bill.pdf',
('20180601', 'SOA'): 'June 1, 2018 - Quarterly Property Tax Bill.pdf',
('20180223', 'SOA'): 'February 23, 2018 - Quarterly Property Tax Bill.pdf',
('20180115', 'NOPV'): 'January 15, 2018 - Notice of Property Value.pdf',
Expand Down Expand Up @@ -112,22 +122,17 @@ def main(period, doc_type, borough, block, lot, *_):
filenames = os.listdir(bbldir)
nostatement_fname = 'nostatement.' + period + '.txt'
if (docname in filenames) or (docname.replace('.pdf', '.txt') in filenames):
LOGGER.info(u'Already downloaded "%s" for BBL %s, skipping',
docname, bbl)
LOGGER.info(u'Already downloaded "%s" for BBL %s, skipping', docname, bbl)
return
elif docname + '.pdf' in filenames:
subprocess.check_call('mv "{bbldir}/{docname}.pdf" "{bbldir}/{docname}"'.format(
bbldir=bbldir, docname=docname), shell=True)
LOGGER.info(u'Already downloaded "%s" for BBL %s, skipping (fixed path)',
docname, bbl)
subprocess.check_call('mv "{bbldir}/{docname}.pdf" "{bbldir}/{docname}"'.format(bbldir=bbldir, docname=docname), shell=True)
LOGGER.info(u'Already downloaded "%s" for BBL %s, skipping (fixed path)', docname, bbl)
return
elif nostatement_fname in filenames:
LOGGER.info(u'There is no "%s" for BBL %s, skipping', docname, bbl)
return

url = 'https://nycprop.nyc.gov/nycproperty/StatementSearch?' + \
'bbl={bbl}&stmtDate={period}&stmtType={doc_type}'.format(
period=period, bbl=bbl, doc_type=doc_type)
url = 'https://a836-edms.nyc.gov/dctm-rest/repositories/dofedmspts/' + 'StatementSearch?bbl={bbl}&stmtDate={period}&stmtType={doc_type}'.format(period=period, bbl=bbl, doc_type=doc_type)

filename = os.path.join(bbldir, docname)
LOGGER.info('Saving %s for %s', filename, bbl)
Expand Down
61 changes: 51 additions & 10 deletions parse.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,7 @@
}
OWNER_ADDRESS_AREA = re.compile(
r'Owner name:(.*)Property address:(.*)Borough, block & lot:(.*)'
r'(Outstanding\s+Charges|Statement\s+Billing\s+Summary)', re.DOTALL + re.IGNORECASE
)
r'(Statement\s+Billing\s+Summary|Statement\s+Details)', re.DOTALL + re.IGNORECASE) #r'(Outstanding\s+Charges|Statement\s+Billing\s+Summary)'
PROPERTY_TAX_DETAIL_AREA = re.compile(
'(Annual [Pp]roperty [Tt]ax [Dd]etail|How We Calculated Your Property Tax).*'
'[Aa]nnual [Pp]roperty [Tt]ax.*?\n', re.DOTALL + re.IGNORECASE)
Expand All @@ -60,9 +59,10 @@
SPLIT_X_RE = re.compile(r'[\sX$]{2,}')
UNITS_RE = re.compile(r'(\d+ [Uu]nits)')
SECTIONS_RE = re.compile(r'(Charges You Can Pre-pay|' # prepayment
r'Amount Not Due [bB]ut That Can [bB]e Paid Early|' #prepayment
r'If you want to pay everything .* please pay|' #prepayment r'Amount Not Due [bB]ut That Can [bB]e Paid Early|'
r'If you pay everything .* you would save|' #prepayment
r'Tax Year Charges Remaining|' # prepayment
r'Current Amount Due|' # due
r'Total Amount Due|' # due r'Current Amount Due|'
r'Current Charges|' # due
r'Overpayments/[Cc]redits|' # ?
r'Payment Agreement|' # ?
Expand All @@ -83,7 +83,6 @@ def parseamount(string):
"""
return float(string.replace(',', '').replace('$', '').replace('*', '').replace('X', ''))


def split(string, with_x=False):
"""
Split a string by any time there are multiple spaces
Expand Down Expand Up @@ -165,6 +164,16 @@ def extract_statement_pdf(text): #pylint: disable=too-many-locals,too-many-branc
elif cell0.startswith('estimated market value'):
key = cell0
value = parseamount(cells[1])
elif cell0.startswith('billable assessed value'):
key = cell0
value = parseamount(cells[1])
elif cell0.startswith('taxable value'):
section = 'details'
key = cell0
splitcells = cells[1].split(' ',3)
meta = parseamount(splitcells[0]) #parseamount(cells[1])
value = splitcells[2]
exemptions_flag = True
elif cell0.startswith('tax before exemptions and abatements'):
section = 'details'
key = cell0
Expand All @@ -176,6 +185,8 @@ def extract_statement_pdf(text): #pylint: disable=too-many-locals,too-many-branc
key = cell0
meta = parseamount(cells[1])
value = parseamount(cells[-1])
if value == meta:
meta = None
abatements_flag = True
elif cell0 == 'annual property tax':
section = 'details'
Expand All @@ -193,6 +204,17 @@ def extract_statement_pdf(text): #pylint: disable=too-many-locals,too-many-branc
elif cell0 == 'revocation':
revocation_flag = True
continue
elif cell0.startswith('damp'):
section = 'details'
key = "Damp / Article XI"
value = parseamount(cells[-1])
elif cell0.startswith('article'):
if cells[1]== 'i':
section = 'details'
key = "Article XI"
value = parseamount(cells[-1])
else:
continue
elif revocation_flag:
section = 'details-revocation'
key = cell0
Expand Down Expand Up @@ -267,6 +289,7 @@ def extract_statement_pdf(text): #pylint: disable=too-many-locals,too-many-branc
meta = None
stabilization_due_date = None
activity_date = None
due_date = None
value = None
apts = None

Expand Down Expand Up @@ -306,8 +329,25 @@ def extract_statement_pdf(text): #pylint: disable=too-many-locals,too-many-branc

if line == '':
continue
elif 'rent stabilization fee' in cells[0].lower():
continue
elif cells[0].lower().startswith('rent stabilization'):
#print(cells[0].lower())
if 'rent stabilization fee- chg' in cells[0].lower():
rent_line = RENT_LINE_RE.split(line) #line
#print(rent_line, len(rent_line))
key = ' '.join(rent_line[0:3])[:-1]
if len(rent_line) < 7:
raise Exception('Unable to parse rent stabilization line')
elif len(rent_line) == 7:
apts = int(rent_line[4]) #int(rent_line[2])
due_date = parsedate(rent_line[5]) #parsedate(rent_line[3])
value = parseamount(rent_line[-1]) #parseamount(rent_line[len(rent_line)-1])
else:
apts = int(rent_line[4]) #int(rent_line[2])
due_date = parsedate(rent_line[5]) #parsedate(rent_line[3])
meta = rent_line[6] #meta = ' '.join(rent_line[4:len(rent_line)-1])
value = parseamount(rent_line[7]) #parseamount(rent_line[len(rent_line)-1])
else:
continue
elif cells[0].startswith('Activity Date'):
continue
elif cells[0] == 'Housing-Rent Stabilization':
Expand All @@ -334,6 +374,9 @@ def extract_statement_pdf(text): #pylint: disable=too-many-locals,too-many-branc
activity_date = parsedate(cells[1])
except: #pylint: disable=bare-except
meta = cells[1]
if key.find('Tax')>0 or key.find('Fee')>0 or key.find('Chg')>0:
due_date = activity_date
activity_date=None
value = parseamount(cells[2])
elif len(cells) == 4:
key = cells[0]
Expand All @@ -349,8 +392,6 @@ def extract_statement_pdf(text): #pylint: disable=too-many-locals,too-many-branc
continue
elif 'Due to this change,' in line:
continue
#import pdb
#pdb.set_trace()

yield {
'key': key,
Expand Down Expand Up @@ -380,7 +421,7 @@ def _html_rent_stabilized(html):
for section in STABILIZED_RE.finditer(html, re.DOTALL):
section_type = section.group(1)
section_text = section.group()
for match in re.finditer(r'(Housing-Rent Stabilization.*?)<', section_text, re.IGNORECASE):
for match in re.finditer(r'((Housing-Rent Stabilization)|(Rent Stabilization Fee- Chg).*?)<', section_text, re.IGNORECASE):
housing_rent, stabilization, num_apts, date, id1, id2, payment = \
re.split(r'\s+', match.group(1).strip())
yield {
Expand Down