Adds functionality to add markdown_used

Author: robvandijk

ocd_backend/enrichers/text_enricher/__init__.py

@@ -11,7 +11,7 @@
 from ocd_backend.enrichers import BaseEnricher
 from ocd_backend.exceptions import SkipEnrichment
 from ocd_backend.log import get_source_logger
-from ocd_backend.settings import RESOLVER_BASE_URL, RETRY_MAX_RETRIES, OCR_VERSION
+from ocd_backend.settings import RESOLVER_BASE_URL, RETRY_MAX_RETRIES, OCR_VERSION, MARKDOWN_VERSION
 from ocd_backend.models.postgres_database import PostgresDatabase
 from ocd_backend.models.serializers import PostgresSerializer
 from ocd_backend.utils.file_parsing import file_parser, make_temp_pdf_fname, md_file_parser, md_file_parser_using_ocr, parse_result_is_empty, rewrite_problematic_pdfs, force_ocr
@@ -146,7 +146,7 @@ def enrich_item(self, item, metadata):
                             item.md_text = md_file_parser_using_ocr(path, item.original_url)
                             ocr_used = OCR_VERSION
 
-                    ori_document = OriDocument(path, item, ocr_used=ocr_used, metadata=metadata)
+                    ori_document = OriDocument(path, item, ocr_used=ocr_used, markdown_used=MARKDOWN_VERSION, metadata=metadata)
                     try:
                         ori_document.store()
                     except sa.exc.IntegrityError as e:

ocd_backend/models/postgres_models.py

@@ -20,6 +20,7 @@ class StoredDocument(Base):
     content_type = Column(String, nullable=False)
     file_size = Column(BigInteger, nullable=False)
     ocr_used = Column(String, nullable=True)
+    markdown_used = Column(String, nullable=True)
     created_at = Column(DateTime, nullable=False)
     updated_at = Column(DateTime, nullable=False)
 

ocd_backend/requirements.txt

@@ -9,9 +9,9 @@ lxml==5.3.0
 kombu==5.4.2
 # msgpack-python==0.4.2
 nose2==0.15.1
-pymupdf @ git+https://github.com/openstate/PyMuPDF.git@366458a
+pymupdf @ git+https://github.com/openstate/PyMuPDF.git@366458a # When changing this, also change settings.MARKDOWN_VERSION
 pdftotext==3.0.0
-pymupdf4llm==0.0.24
+pymupdf4llm==0.0.24 # When changing this, also change settings.MARKDOWN_VERSION
 Pillow==9.3.0
 psycopg2==2.9.10
 PyLD==2.0.4

ocd_backend/settings.py

@@ -286,6 +286,7 @@ def get_ocr_version():
     # See also requirements.txt
     return f"tesserocr==2.7.1,{tesseract_version}"
 OCR_VERSION = get_ocr_version()
+MARKDOWN_VERSION = f"openstate/PyMuPDF@366458a-pymupdf4llm==0.0.24"
 
 # Allow any settings to be defined in local_settings.py which should be
 # ignored in your version control system allowing for settings to be

ocd_backend/utils/ori_document.py

@@ -16,7 +16,7 @@
 log = get_source_logger('document_storage')
 
 class OriDocument():
-    def __init__(self, temp_path, item, metadata, ocr_used = None):
+    def __init__(self, temp_path, item, metadata, ocr_used = None, markdown_used = None):
         self.temp_path = temp_path
         self.file_name = item.file_name if hasattr(item, 'file_name') else None
         self.md_text = item.md_text
@@ -25,6 +25,7 @@ def __init__(self, temp_path, item, metadata, ocr_used = None):
         self.last_changed_at = self.get_last_changed_at(item)
         self.source, self.supplier = self.get_source_and_supplier(item)
         self.ocr_used = ocr_used
+        self.markdown_used = markdown_used
         self.metadata = metadata
 
         self.metadata['content_type'] = item.content_type
@@ -33,6 +34,7 @@ def __init__(self, temp_path, item, metadata, ocr_used = None):
         self.metadata['last_changed_at'] = self.last_changed_at.isoformat() if self.last_changed_at else ''
         self.metadata['original_url'] = item.original_url if hasattr(item, 'original_url') else ''
         self.metadata['ocr_used'] = ocr_used if ocr_used else ''
+        self.metadata['markdown_used'] = markdown_used if markdown_used else ''
 
         database = PostgresDatabase(serializer=PostgresSerializer)
         self.session = database.Session()
@@ -61,6 +63,7 @@ def exists_and_not_changed(self):
         if self.stored_document.last_changed_at != self.last_changed_at or \
             self.stored_document.file_size != self.file_size or \
             self.stored_document.ocr_used != self.ocr_used or \
+            (self.stored_document.markdown_used and self.stored_document.markdown_used != self.markdown_used) or \
             self.stored_document.file_size != self.file_size:
             return False
 
@@ -95,6 +98,7 @@ def store_in_db(self):
             self.stored_document.last_changed_at = self.last_changed_at
             self.stored_document.file_size = self.file_size
             self.stored_document.ocr_used = self.ocr_used
+            self.stored_document.markdown_used = self.markdown_used
             self.stored_document.updated_at = time_now
         else:
             content_type = magic.from_file(self.temp_path, mime=True)
@@ -107,6 +111,7 @@ def store_in_db(self):
                 content_type=content_type,
                 file_size=self.file_size,
                 ocr_used=self.ocr_used,
+                markdown_used=self.markdown_used,
                 created_at=time_now,
                 updated_at=time_now
             )

ocd_frontend/models/postgres_models.py

@@ -20,6 +20,7 @@ class StoredDocument(Base):
     content_type = Column(String, nullable=False)
     file_size = Column(BigInteger, nullable=False)
     ocr_used = Column(String, nullable=True)
+    markdown_used = Column(String, nullable=True)
     created_at = Column(DateTime, nullable=False)
     updated_at = Column(DateTime, nullable=False)