feat(tutor): use tika to extract from pdf (#80)

Author: sandragjacinto

src/app/api/api_v1/endpoints/tutor.py

@@ -12,7 +12,6 @@
 from src.app.services.tutor.agents import TEMPLATES
 from src.app.services.tutor.models import (
     ExtractorOutputList,
-    SummariesOutputModel,
     SyllabusFeedback,
     SyllabusResponse,
     SyllabusResponseAgent,
@@ -48,7 +47,7 @@
 @router.post("/files/content")
 async def extract_files_content(
     files: Annotated[list[UploadFile], File()],
-) -> SummariesOutputModel | None:
+) -> ExtractorOutputList | None:
     files_content = await get_files_content(files)
     files_content_str = ("__DOCUMENT_SEPARATOR__").join(files_content)
 
@@ -67,7 +66,7 @@ async def extract_files_content(
         summaries = await chatfactory.chat_client.completion(messages=messages)
         assert isinstance(summaries, str)
         json_summaries = extract_json_from_response(summaries)
-        summaries_output = SummariesOutputModel(**json_summaries)
+        summaries_output = ExtractorOutputList(**json_summaries)
 
         return summaries_output
 

src/app/services/llm_proxy.py

@@ -57,6 +57,8 @@ async def completion(
         response_format: Optional[Union[dict, Type[BaseModel]]] = None,
     ) -> dict | str:
 
+        logger.info("starting completion with model_name=%s", self.model)
+
         if self.is_azure_model:
             return await self.az_completion(messages)
 

src/app/services/tutor/models.py

@@ -14,10 +14,6 @@ class ExtractorOutputList(BaseModel):
     extracts: list[ExtractorOutput]
 
 
-class SummariesOutputModel(BaseModel):
-    summaries: list[str]
-
-
 class TutorSearchResponse(BaseModel):
     extracts: list[ExtractorOutput]
     nb_results: int

src/app/services/tutor/utils.py

@@ -2,12 +2,11 @@
 
 from docx import Document as DocxReader
 from fastapi import HTTPException, UploadFile
-
-# from src.app.services.pdf_extractor import extract_txt_from_pdf_with_tika
 from pypdf import PdfReader
 from qdrant_client.models import ScoredPoint
 
 from src.app.api.dependencies import get_settings
+from src.app.services.pdf_extractor import extract_txt_from_pdf_with_tika
 from src.app.utils.decorators import log_time_and_error_sync
 
 settings = get_settings()
@@ -97,11 +96,14 @@ async def get_file_content(file: UploadFile) -> str:
 
 
 async def _extract_pdf_content(file) -> str:
-    reader = PdfReader(file.file)
-    return "\n".join(page.extract_text() or "" for page in reader.pages)
-    # content = extract_txt_from_pdf_with_tika(file.file, settings.TIKA_URL_BASE)
+    content = ""
+    try:
+        content = extract_txt_from_pdf_with_tika(file.file, settings.TIKA_URL_BASE)
+    except Exception:
+        reader = PdfReader(file.file)
+        content = "\n".join(page.extract_text() or "" for page in reader.pages)
 
-    # return content
+    return content
 
 
 async def _extract_text_content(file) -> str: