initial commit

Author: potter-potter

pyproject.toml

@@ -23,10 +23,10 @@ dependencies = [
     "uvicorn",
     "fastapi",
     "click",
-    "unstructured-ingest",
+    "utic-public-types",
     "opentelemetry-instrumentation-fastapi",
     "opentelemetry-exporter-otlp-proto-grpc",
-    "dataclasses-json"
+    "dataclasses-json",
 ]
 dynamic = [
  "version",

test/api/test_api.py

@@ -4,7 +4,7 @@
 import pytest
 from fastapi.testclient import TestClient
 from pydantic import BaseModel
-from unstructured_ingest.data_types.file_data import (
+from utic_public_types.plugins.file_data import (
     BatchFileData,
     BatchItem,
     FileData,

test/assets/dataclass_response.py

@@ -2,7 +2,7 @@
 from pathlib import Path
 from typing import Any, Optional, TypedDict, Union
 
-from unstructured_ingest.data_types.file_data import BatchFileData, FileData
+from utic_public_types.plugins.file_data import BatchFileData, FileData
 
 
 class SampleFunctionResponse(TypedDict):

test/assets/exception_status_code.py

@@ -1,7 +1,7 @@
 """Test assets for testing exception handling with various status_code scenarios."""
 
 from fastapi import HTTPException
-from unstructured_ingest.error import UnstructuredIngestError
+from utic_public_types.plugins.errors import UnstructuredIngestError
 
 
 class ExceptionWithNoneStatusCode(Exception):

test/assets/filedata_meta.py

@@ -2,7 +2,7 @@
 from typing import Optional, Union
 
 from pydantic import BaseModel
-from unstructured_ingest.data_types.file_data import BatchFileData, FileData, SourceIdentifiers
+from utic_public_types.plugins.file_data import BatchFileData, FileData, SourceIdentifiers
 
 from unstructured_platform_plugins.schema import FileDataMeta, NewRecord
 

test/test_schema.py

@@ -6,7 +6,7 @@
 import pytest
 from pydantic import BaseModel
 from typing_extensions import TypedDict
-from unstructured_ingest.data_types.file_data import FileData
+from utic_public_types.plugins.file_data import FileData
 
 import unstructured_platform_plugins.schema.json_schema as js
 from unstructured_platform_plugins.schema.model import is_valid_input_dict, is_valid_response_dict

test/test_utils.py

@@ -4,7 +4,7 @@
 
 import pytest
 from pydantic import BaseModel
-from unstructured_ingest.data_types.file_data import FileData, SourceIdentifiers
+from utic_public_types.plugins.file_data import FileData, SourceIdentifiers
 from uvicorn.importer import import_from_string
 
 from unstructured_platform_plugins.etl_uvicorn import utils

unstructured_platform_plugins/etl_uvicorn/api_generator.py

@@ -9,10 +9,10 @@
 from fastapi import FastAPI, HTTPException, status
 from fastapi.responses import StreamingResponse
 from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
-from pydantic import BaseModel, Field, create_model
+from pydantic import BaseModel, Field, ValidationError, create_model
 from starlette.responses import RedirectResponse
-from unstructured_ingest.data_types.file_data import BatchFileData, FileData, file_data_from_dict
-from unstructured_ingest.error import UnstructuredIngestError
+from utic_public_types.plugins.file_data import BatchFileData, FileData
+from utic_public_types.plugins.errors import UnstructuredIngestError
 from uvicorn.config import LOG_LEVELS
 from uvicorn.importer import import_from_string
 
@@ -32,12 +32,19 @@
 
 FileDataType = Union[FileData, BatchFileData]
 
+logger = logging.getLogger("uvicorn.error")
 
-class EtlApiException(Exception):
-    pass
+
+def file_data_from_dict(data: dict) -> FileData:
+    try:
+        return BatchFileData.model_validate(data)
+    except ValidationError:
+        logger.debug(f"{data} not valid for batch file data")
+    return FileData.model_validate(data)
 
 
-logger = logging.getLogger("uvicorn.error")
+class EtlApiException(Exception):
+    pass
 
 
 class MessageChannels(BaseModel):

unstructured_platform_plugins/schema/filedata_meta.py

@@ -1,7 +1,7 @@
 from typing import Any
 
 from pydantic import BaseModel, Field
-from unstructured_ingest.data_types.file_data import FileData
+from utic_public_types.plugins.file_data import FileData
 
 
 class NewRecord(BaseModel):

unstructured_platform_plugins/schema/json_schema.py

@@ -8,7 +8,7 @@
 
 from pydantic import BaseModel, create_model
 from pydantic.fields import FieldInfo, PydanticUndefined
-from unstructured_ingest.data_types.file_data import BatchFileData, FileData
+from utic_public_types.plugins.file_data import BatchFileData, FileData
 
 from unstructured_platform_plugins.schema.utils import TypedParameter
 from unstructured_platform_plugins.type_hints import get_type_hints