bugfix/FileData Literal edge case (#18)

Author: rbiseck3

CHANGELOG.md

@@ -1,3 +1,13 @@
+## 0.0.2
+
+### Enhancements
+
+### Features
+
+### Fixes
+
+* **FileData Literal not handled** FileData content was updated to use Literal rather than Enum. This case needed to be added. 
+
 ## 0.0.1
 
 ### Enhancements

requirements/cli.in

@@ -1,4 +1,4 @@
 uvicorn
 fastapi
 click
-unstructured
+unstructured-ingest

requirements/cli.txt

@@ -1,8 +1,8 @@
 #
-# This file is autogenerated by pip-compile with Python 3.10
+# This file is autogenerated by pip-compile with Python 3.11
 # by the following command:
 #
-#    pip-compile requirements//cli.in
+#    pip-compile cli.in
 #
 annotated-types==0.7.0
     # via pydantic
@@ -15,7 +15,7 @@ backoff==2.2.1
     # via unstructured
 beautifulsoup4==4.12.3
     # via unstructured
-certifi==2024.6.2
+certifi==2024.7.4
     # via
     #   httpcore
     #   httpx
@@ -29,7 +29,7 @@ charset-normalizer==3.3.2
     #   unstructured-client
 click==8.1.7
     # via
-    #   -r requirements//cli.in
+    #   -r cli.in
     #   nltk
     #   typer
     #   uvicorn
@@ -45,10 +45,8 @@ email-validator==2.2.0
     # via fastapi
 emoji==2.12.1
     # via unstructured
-exceptiongroup==1.2.1
-    # via anyio
-fastapi==0.111.0
-    # via -r requirements//cli.in
+fastapi==0.111.1
+    # via -r cli.in
 fastapi-cli==0.0.4
     # via fastapi
 filetype==1.2.0
@@ -101,25 +99,32 @@ nest-asyncio==1.6.0
 nltk==3.8.1
     # via unstructured
 numpy==1.26.4
-    # via unstructured
+    # via
+    #   pandas
+    #   unstructured
 ordered-set==4.1.0
     # via deepdiff
-orjson==3.10.6
-    # via fastapi
 packaging==24.1
     # via
     #   marshmallow
     #   unstructured-client
-pydantic==2.8.0
+pandas==2.2.2
+    # via unstructured-ingest
+psutil==6.0.0
+    # via unstructured
+pydantic==2.8.2
     # via fastapi
-pydantic-core==2.20.0
+pydantic-core==2.20.1
     # via pydantic
 pygments==2.18.0
     # via rich
-pypdf==4.2.0
+pypdf==4.3.1
     # via unstructured-client
 python-dateutil==2.9.0.post0
-    # via unstructured-client
+    # via
+    #   pandas
+    #   unstructured-client
+    #   unstructured-ingest
 python-dotenv==1.0.1
     # via uvicorn
 python-iso639==2024.4.27
@@ -128,11 +133,13 @@ python-magic==0.4.27
     # via unstructured
 python-multipart==0.0.9
     # via fastapi
+pytz==2024.1
+    # via pandas
 pyyaml==6.0.1
     # via uvicorn
-rapidfuzz==3.9.4
+rapidfuzz==3.9.5
     # via unstructured
-regex==2024.5.15
+regex==2024.7.24
     # via nltk
 requests==2.32.3
     # via
@@ -168,34 +175,33 @@ typer==0.12.3
     # via fastapi-cli
 typing-extensions==4.12.2
     # via
-    #   anyio
     #   emoji
     #   fastapi
     #   pydantic
     #   pydantic-core
-    #   pypdf
     #   typer
     #   typing-inspect
     #   unstructured
     #   unstructured-client
-    #   uvicorn
 typing-inspect==0.9.0
     # via
     #   dataclasses-json
     #   unstructured-client
-ujson==5.10.0
-    # via fastapi
-unstructured==0.14.9
-    # via -r requirements//cli.in
-unstructured-client==0.23.8
+tzdata==2024.1
+    # via pandas
+unstructured==0.15.0
+    # via unstructured-ingest
+unstructured-client==0.25.0
     # via unstructured
+unstructured-ingest==0.0.0
+    # via -r cli.in
 urllib3==2.2.2
     # via
     #   requests
     #   unstructured-client
-uvicorn[standard]==0.30.1
+uvicorn[standard]==0.30.3
     # via
-    #   -r requirements//cli.in
+    #   -r cli.in
     #   fastapi
 uvloop==0.19.0
     # via uvicorn

requirements/constraints.txt

@@ -0,0 +1 @@
+unstructured-ingest==0.0.0

test/test_schema.py

@@ -7,7 +7,7 @@
 import pytest
 from pydantic import BaseModel
 from typing_extensions import TypedDict
-from unstructured.ingest.v2.interfaces import FileData
+from unstructured_ingest.v2.interfaces import FileData
 
 import unstructured_platform_plugins.schema.json_schema as js
 from unstructured_platform_plugins.etl_uvicorn.utils import get_input_schema
@@ -507,7 +507,7 @@ def fn(a: FileData) -> list[FileData]:
                         ],
                         "default": None,
                     },
-                    "doc_type": {"type": "string", "enum": ["batch", "file"], "default": "file"},
+                    "doc_type": {"type": "string", "default": "file"},
                     "metadata": {
                         "type": "object",
                         "properties": {
@@ -599,7 +599,7 @@ def fn(a: FileData) -> list[FileData]:
                     ],
                     "default": None,
                 },
-                "doc_type": {"type": "string", "enum": ["batch", "file"], "default": "file"},
+                "doc_type": {"type": "string", "default": "file"},
                 "metadata": {
                     "type": "object",
                     "properties": {

test/test_utils.py

@@ -4,7 +4,7 @@
 
 import pytest
 from pydantic import BaseModel
-from unstructured.ingest.v2.interfaces import FileData
+from unstructured_ingest.v2.interfaces import FileData
 from uvicorn.importer import import_from_string
 
 from unstructured_platform_plugins.etl_uvicorn import utils

unstructured_platform_plugins/__version__.py

@@ -1 +1 @@
-__version__ = "0.0.1"  # pragma: no cover
+__version__ = "0.0.2"  # pragma: no cover

unstructured_platform_plugins/etl_uvicorn/api_generator.py

@@ -122,8 +122,8 @@ async def run_job(request: input_schema_model) -> InvokeResponse:
 
     else:
 
-        @fastapi_app.post("/invoke", response_model=response_type)
-        async def run_job() -> response_type:
+        @fastapi_app.post("/invoke", response_model=InvokeResponse)
+        async def run_job() -> InvokeResponse:
             logger.debug(f"invoking function without inputs: {func}")
             return await wrap_fn(
                 func=func,

unstructured_platform_plugins/etl_uvicorn/utils.py

@@ -1,6 +1,7 @@
 import inspect
 from dataclasses import is_dataclass
 from enum import EnumMeta
+from inspect import Parameter
 from types import GenericAlias, NoneType
 from typing import Any, Callable, Optional
 
@@ -58,6 +59,10 @@ def get_input_schema(func: Callable, omit: Optional[list[str]] = None) -> dict:
     parameters = get_typed_parameters(func)
     if omit:
         parameters = [p for p in parameters if p.name not in omit]
+    # Omit self if wrapping method
+    parameters = [
+        p for p in parameters if not (p.param_type is Parameter.empty and p.name == "self")
+    ]
     return parameters_to_json_schema(parameters)
 
 

unstructured_platform_plugins/schema/json_schema.py

@@ -4,11 +4,11 @@
 from inspect import Parameter
 from pathlib import Path
 from types import GenericAlias, NoneType, UnionType
-from typing import Any, Optional, Type, Union, _UnionGenericAlias
+from typing import Any, Literal, Optional, Type, Union, _UnionGenericAlias
 
 from pydantic import BaseModel, create_model
 from pydantic.fields import FieldInfo, PydanticUndefined
-from unstructured.ingest.v2.interfaces import FileData
+from unstructured_ingest.v2.interfaces import FileData
 
 from unstructured_platform_plugins.schema.utils import TypedParameter
 from unstructured_platform_plugins.type_hints import get_type_hints
@@ -37,6 +37,13 @@ def is_typed_dict(val: Any) -> bool:
 
 
 def type_to_json_schema(t: Type, args: Optional[tuple[Any, ...]] = None) -> dict:
+    if t is Literal:
+        arg_types = [type(a) for a in args]
+        if len(set(arg_types)) != 1:
+            raise TypeError(
+                "Literal must be of a single type. Defined with: {}".format(", ".join(arg_types))
+            )
+        t = arg_types[0]
     resp = {"type": types_map[t]}
     if t is list and args:
         list_type = args[0]