Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions downloadMetadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
2. Write rows to sqlite db, ON DUPLICATE KEY UPDATE VALUES

"""

from __future__ import print_function, division
from datetime import datetime
import requests
import requests_cache
Expand Down Expand Up @@ -56,14 +56,14 @@ def compileResultsList(results_list=[],start=0, limit=1000):

def formatTimestamp(timestamp):
try:
return datetime.fromtimestamp(timestamp/1000).strftime("%Y-%m-%d")
return datetime.fromtimestamp(timestamp//1000).strftime("%Y-%m-%d")
except TypeError:
return None

results_list = compileResultsList()

print "got",len(results_list),"total document rows"
print "writing rows to sqlite database ..."
print("got",len(results_list),"total document rows")
print("writing rows to sqlite database ...")

with db.transaction():
for result in results_list:
Expand Down
1 change: 1 addition & 0 deletions generatePDFList.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
2. Write out file for wget to use
"""

from __future__ import print_function, division
from hrcemail_common import *

pdf_base = "https://foia.state.gov/searchapp/"
Expand Down
1 change: 1 addition & 0 deletions hrcemail_common.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
#!/usr/bin/env python
# encoding: utf-8
from __future__ import print_function, division

from peewee import *
db = SqliteDatabase("hrcemail.sqlite")
Expand Down
3 changes: 2 additions & 1 deletion pdfTextToDatabase.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
4. Write contents back to the database

"""
from __future__ import print_function, division

from hrcemail_common import *
import string
Expand All @@ -31,7 +32,7 @@ def extract(filename):

for docID in docIDs:
filename = docID.docID
print "Working on",filename
print("Working on",filename)
output_string = extract(filename)
insert_query = Document.update(docText = output_string).where(Document.docID == filename)
insert_query.execute()
3 changes: 2 additions & 1 deletion zipPDFs.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
3. Write the PDFs into each zip file from pdfs/

"""
from __future__ import print_function, division

from hrcemail_common import *
import zipfile
Expand All @@ -17,7 +18,7 @@
docClasses = (Document.select(fn.Distinct(Document.documentClass).alias("docClass")))

for docClass in docClasses:
print "working on ",docClass.docClass
print("working on ",docClass.docClass)
if not os.path.isfile("zips/"+docClass.docClass+".zip"):
with zipfile.ZipFile("zips/"+docClass.docClass+".zip", "w") as zf:
docIDs = (Document.select(Document.docID).where(Document.documentClass == docClass.docClass))
Expand Down