protocol parsing fixes

Author: OriHoch

airflow/knesset_data_pipelines/committees/parsed_document_committee_sessions.py

@@ -9,10 +9,10 @@
 
 
 def parse_retry_error(type_, error, document_session_id, group_type_id, application_desc, stats):
-    document_session_id = str(document_session_id)
-    group_type_id = str(group_type_id)
-    application_desc = str(application_desc)
     if error:
+        document_session_id = str(document_session_id)
+        group_type_id = str(group_type_id)
+        application_desc = str(application_desc)
         ext = 'txt' if type_ == 'text' else 'csv'
         download_filepath = os.path.join(
             config.KNESSET_PIPELINES_DATA_PATH,

committees/download_document_committee_session.py

@@ -2,6 +2,7 @@
 import logging, os, time
 from datapackage_pipelines_knesset.retry_get_response_content import get_retry_response_content
 import crcmod, base64
+import sys
 
 
 def get_crc32c(filename):
@@ -30,6 +31,7 @@ def process_row(row, row_index, resource_descriptor, resource_index, parameters,
                     row.update(download_error='reached limit, skipping download')
                     stats["download: skipped files"] += 1
                 else:
+                    print(f'download_document_committee_session: {row_index} {row}', file=sys.stderr)
                     error_string, content = None, ''
                     try:
                         content = get_retry_response_content(row["FilePath"], None, None, None, retry_num=1, num_retries=10,
@@ -52,6 +54,7 @@ def process_row(row, row_index, resource_descriptor, resource_index, parameters,
                         stats["download: downloaded files"] += 1
                         row.update(download_filename=rel_filename, download_filesize=os.path.getsize(filename),
                                    download_crc32c=get_crc32c(filename))
+                    print(f'download_document_committee_session: done {row_index}', file=sys.stderr)
     return row
 
 

committees/parse_meeting_protocols.py

@@ -6,6 +6,7 @@
 import crcmod, base64
 import hashlib
 import knesset_data
+import sys
 
 
 BASE_HASH_OBJ = hashlib.md5()
@@ -55,7 +56,7 @@ def process_row(row, row_index, resource_descriptor, resource_index, parameters,
             and (row["FilePath"].lower().endswith('.doc') or row["FilePath"].lower().endswith('.docx'))):
                 document_id = "{}-{}-{}".format(row["GroupTypeID"], row["DocumentCommitteeSessionID"], row["ApplicationDesc"])
                 original_filename, ext, output_filename, full_output_filename, download_filename, full_output_hash_filename = get_filenames(row, parameters)
-                if os.path.exists(download_filename) and row.get('download_crc32c'):
+                if os.path.exists(download_filename) and row.get('download_crc32c') and os.path.getsize(download_filename) > 0:
                     m = BASE_HASH_OBJ.copy()
                     m.update(row['download_crc32c'].encode())
                     new_cache_hash = m.hexdigest()
@@ -74,6 +75,7 @@ def process_row(row, row_index, resource_descriptor, resource_index, parameters,
                         row[t + "_error"] = 'reached files-limit, skipping'
                         stats[t + ": skipped files"] += 1
                     else:
+                        print(f'parse_meeting_protocols({t}): {row_index} old_cache_hash={old_cache_hash} new_cache_hash={new_cache_hash} {row}', file=sys.stderr)
                         error_string = None
                         try:
                             with open(download_filename, "rb") as f:
@@ -106,6 +108,7 @@ def process_row(row, row_index, resource_descriptor, resource_index, parameters,
                             stats[t + ": parsed files"] += 1
                             with open(full_output_hash_filename, 'w') as f:
                                 f.write(new_cache_hash)
+                        print(f'parse_meeting_protocols({t}): done {row_index} {row}', file=sys.stderr)
                 else:
                     row[t + "_error"] = 'missing download file'
                     stats[t + ': missing download files'] += 1