Merge pull request #392 from maxfordham/pydantic-model-creation-update-from-edittsv

Updated pydantic model generation in edittsv lifecycle

Author: Arshadwaqas115

docs/TestArrayTransposed.xlsx

@@ -1,5 +1,5 @@
-from ipyautoui.automapschema import pydantic_model_from_json_schema
-from ipyautoui.automapschema import pydantic_model_file_from_json_schema
+from ipyautoui._utils import pydantic_model_from_json_schema
+from ipyautoui._utils import pydantic_model_file_from_json_schema
 from ipyautoui.custom.edittsv import EditTsvWithDiff
 from ipyautoui.custom.edittsv_with_diff_and_key_mapping import EditTsvWithDiffAndKeyMapping
 import pathlib

docs/digital-schedules-issue.py

@@ -1,5 +1,5 @@
-from ipyautoui.automapschema import pydantic_model_from_json_schema
-from ipyautoui.automapschema import pydantic_model_file_from_json_schema
+from ipyautoui._utils import pydantic_model_from_json_schema
+from ipyautoui._utils import pydantic_model_file_from_json_schema
 from ipyautoui.custom.edittsv import EditTsvWithDiff
 import pathlib
 import numpy as np

docs/digital-schedules-phantom-changes-issue.py

@@ -21,24 +21,24 @@ These docs requires a python kernel to run. Try on Binder [![Binder](https://myb
 
 ```{python}
 from ipyautoui.custom.editgrid import EditGrid
+from ipyautoui.custom.edittsv import EditTsvFileUpload
 import typing as ty
 from pydantic import BaseModel, Field, RootModel
 from ipyautoui import AutoUi
 
 
 class DataFrameCols(BaseModel):
-    string: str = Field("string", json_schema_extra=dict(column_width=100))
-    integer: int = Field(1, json_schema_extra=dict(column_width=80))
-    floater: float = Field(3.1415, json_schema_extra=dict(column_width=70, global_decimal_places=3))
-    something_else: float = Field(324, json_schema_extra=dict(column_width=100))
-    nullable_string: ty.Optional[str] = None
+    string: str = Field("string", json_schema_extra=dict(column_width=100, name="string"))
+    integer: int = Field(1, json_schema_extra=dict(column_width=80, name="integer"))
+    floater: float = Field(3.1415, json_schema_extra=dict(column_width=70, global_decimal_places=3, name="floater"))
+    something_else: float = Field(324, json_schema_extra=dict(column_width=100, name="something_else"))
 
 class TestDataFrame(RootModel):
     """a description of TestDataFrame"""
 
-    root: ty.List[DataFrameCols] = Field(json_schema_extra=dict(format="dataframe"))
+    root: ty.List[DataFrameCols] = Field(json_schema_extra=dict(format="dataframe", datagrid_index_name=("name", )))
     
-egrid = EditGrid(schema=TestDataFrame, show_ui_io=True)
+egrid = EditGrid(schema=TestDataFrame, show_ui_io=True, ui_io=EditTsvFileUpload)
 egrid
 ```
 

docs/editgrid.qmd

@@ -20,7 +20,10 @@
 from markdown import markdown
 from math import log10, floor
 from pydantic import BaseModel, field_validator, Field, ValidationInfo
+from datamodel_code_generator import DataModelType, InputFileType, generate
+from tempfile import TemporaryDirectory
 from IPython.display import display, Markdown
+import sys
 
 logger = logging.getLogger(__name__)
 frozenmap = immutables.Map
@@ -527,3 +530,46 @@ def is_null(v) -> bool:
         return False
     else:
         return pd.isnull(v)
+
+def pydantic_model_file_from_json_schema(json_schema, fpth):
+    return generate(
+            json.dumps(json_schema, ensure_ascii=False),
+            input_file_type=InputFileType.JsonSchema,
+            input_filename="example.json",
+            output=fpth,
+            output_model_type=DataModelType.PydanticV2BaseModel,
+            capitalise_enum_members=True,
+            field_include_all_keys=True
+        )
+
+def pydantic_model_from_json_schema(json_schema: dict) -> ty.Type[BaseModel]:
+    load = json_schema["title"].replace(" ", "") if "title" in json_schema else "Model"
+
+    with TemporaryDirectory() as temporary_directory_name:
+        temporary_directory = pathlib.Path(temporary_directory_name)
+        file_path = "model.py"
+        module_name = file_path.split(".")[0]
+        output = pathlib.Path(temporary_directory / file_path)
+        
+        pydantic_model_file_from_json_schema(json_schema, output)
+
+        #HACK refer to https://github.com/koxudaxi/datamodel-code-generator/issues/2534 for official fix, then remove the PATCH LOGIC once that is resolved
+        # --- NEW PATCH LOGIC ---
+        if json_schema.get("title") == "Project Building Area":
+            text = output.read_text()
+
+            # Replace Enum → IntEnum in TargetYear only
+            text = text.replace("class TargetYear(Enum):", "class TargetYear(IntEnum):")
+
+            # Ensure IntEnum is imported
+            if "from enum import IntEnum" not in text:
+                text = text.replace("from enum import Enum", "from enum import Enum, IntEnum")
+
+            output.write_text(text)
+        # --- END PATCH LOGIC ---
+
+        spec = importlib.util.spec_from_file_location(module_name, output)
+        module = importlib.util.module_from_spec(spec)
+        sys.modules[module_name] = module
+        spec.loader.exec_module(module)
+    return getattr(module, load)

src/ipyautoui/_utils.py

@@ -12,62 +12,13 @@
 from ipyautoui.custom.filechooser import FileChooser
 from ipyautoui.custom.date_string import DatePickerString, NaiveDatetimePickerString
 from ipyautoui.autobox import AutoBox
-from tempfile import TemporaryDirectory
-import pathlib
-from datamodel_code_generator import DataModelType, InputFileType, generate
-import json
-import importlib.util
-import sys
 
 import logging
 
 logger = logging.getLogger(__name__)
 
-def pydantic_model_file_from_json_schema(json_schema, fpth):
-    return generate(
-            json.dumps(json_schema, ensure_ascii=False),
-            input_file_type=InputFileType.JsonSchema,
-            input_filename="example.json",
-            output=fpth,
-            output_model_type=DataModelType.PydanticV2BaseModel,
-            capitalise_enum_members=True,
-            field_include_all_keys=True
-        )
-
-def pydantic_model_from_json_schema(json_schema: dict) -> ty.Type[BaseModel]:
-    load = json_schema["title"].replace(" ", "") if "title" in json_schema else "Model"
-
-    with TemporaryDirectory() as temporary_directory_name:
-        temporary_directory = pathlib.Path(temporary_directory_name)
-        file_path = "model.py"
-        module_name = file_path.split(".")[0]
-        output = pathlib.Path(temporary_directory / file_path)
-        
-        pydantic_model_file_from_json_schema(json_schema, output)
-
-        #HACK refer to https://github.com/koxudaxi/datamodel-code-generator/issues/2534 for official fix, then remove the PATCH LOGIC once that is resolved
-        # --- NEW PATCH LOGIC ---
-        if json_schema.get("title") == "Project Building Area":
-            text = output.read_text()
-
-            # Replace Enum → IntEnum in TargetYear only
-            text = text.replace("class TargetYear(Enum):", "class TargetYear(IntEnum):")
-
-            # Ensure IntEnum is imported
-            if "from enum import IntEnum" not in text:
-                text = text.replace("from enum import Enum", "from enum import Enum, IntEnum")
-
-            output.write_text(text)
-        # --- END PATCH LOGIC ---
-
-        spec = importlib.util.spec_from_file_location(module_name, output)
-        module = importlib.util.module_from_spec(spec)
-        sys.modules[module_name] = module
-        spec.loader.exec_module(module)
-    return getattr(module, load)
-
 def _init_model_schema(
-    schema=None, by_alias=False, generate_pydantic_model_from_json_schema = True
+    schema=None, by_alias=False
 ) -> tuple[ty.Optional[ty.Type[BaseModel]], dict]:
     if schema is None:
         return None, {
@@ -76,17 +27,9 @@ def _init_model_schema(
             "items": {"properties": {}},
         }
     if isinstance(schema, dict):
-        if generate_pydantic_model_from_json_schema:
-            schema = replace_refs(schema, merge_props=True)
-            schema = {k: v for k, v in schema.items() if k != "$defs"}
-            model = pydantic_model_from_json_schema(schema)
-        else:
-            model = None
-            schema = replace_refs(schema, merge_props=True)
-            schema = {k: v for k, v in schema.items() if k != "$defs"}
-        # IDEA: Possible implementations -@jovyan at 8/24/2022, 12:05:02 PM
-        # jsonschema_to_pydantic
-        # https://koxudaxi.github.io/datamodel-code-generator/using_as_module/
+        model = None
+        schema = replace_refs(schema, merge_props=True)
+        schema = {k: v for k, v in schema.items() if k != "$defs"}
     else:
         model = schema  # the "model" passed is a pydantic model
         schema = model.model_json_schema(by_alias=by_alias).copy()

src/ipyautoui/automapschema.py

@@ -514,7 +514,6 @@ def update_from_schema(
         by_alias: bool = False,
         by_title: bool = True,
         order: ty.Optional[tuple] = None,
-        generate_pydantic_model_from_json_schema: bool = False,
         **kwargs,
     ):
         self.__init__(
@@ -523,7 +522,6 @@ def update_from_schema(
             by_alias=by_alias,
             by_title=by_title,
             order=order,
-            generate_pydantic_model_from_json_schema=generate_pydantic_model_from_json_schema,
             **kwargs,
         )
 
@@ -606,7 +604,6 @@ def __init__(
         by_alias: bool = False,
         by_title: bool = True,
         order: ty.Optional[tuple] = None,
-        generate_pydantic_model_from_json_schema: bool = False,
         **kwargs,
     ):
         # accept schema or pydantic schema
@@ -615,7 +612,7 @@ def __init__(
         )
         self.by_title = by_title
         self.selection_mode = MAP_TRANSPOSED_SELECTION_MODE[self.transposed]
-        self.model, self.schema = asch._init_model_schema(schema, by_alias=by_alias, generate_pydantic_model_from_json_schema=generate_pydantic_model_from_json_schema)
+        self.model, self.schema = asch._init_model_schema(schema, by_alias=by_alias)
         # ^ generates gridschema
         self.gridschema.get_traits = self.datagrid_trait_names
         _data = self._init_data(data)

src/ipyautoui/custom/autogrid.py

@@ -289,7 +289,6 @@ def __init__(
         title: str = None,
         description: str = None,
         show_title: bool = True,
-        generate_pydantic_model_from_json_schema: bool = False,
         show_ui_io: bool = False,
         **kwargs,
     ):  # TODO: use **kwargs to pass attributes to EditGrid as in AutoObject and AutoArray
@@ -302,7 +301,6 @@ def __init__(
         self.by_title = by_title
         self.by_alias = by_alias
         self.datahandler = datahandler
-        self.generate_pydantic_model_from_json_schema = generate_pydantic_model_from_json_schema
 
         self.ui_io = None
         self._ui_io_factory = None
@@ -346,7 +344,7 @@ def update_from_schema(
     ):
         value = None if value is None or value == [{}] else pd.DataFrame(value)
         self.grid.update_from_schema(
-            schema, data=value, by_alias=self.by_alias, generate_pydantic_model_from_json_schema=self.generate_pydantic_model_from_json_schema, **kwargs
+            schema, data=value, by_alias=self.by_alias, **kwargs
         )
         self._init_ui_callables(
             ui_add=ui_add, ui_edit=ui_edit, ui_delete=ui_delete, ui_copy=ui_copy, ui_io=ui_io
@@ -367,7 +365,7 @@ def _init_autogrid(
             None if value is None or value == [{}] else pd.DataFrame(value)
         )
         self.grid = AutoGrid(
-            schema, data=getvalue(value), generate_pydantic_model_from_json_schema=self.generate_pydantic_model_from_json_schema, by_alias=self.by_alias, **kwargs
+            schema, data=getvalue(value), by_alias=self.by_alias, **kwargs
         )
 
     def _init_ui_callables(
@@ -403,24 +401,24 @@ def _missing_model_ui():
 
             if ui_io is None:
                 def _factory():
-                    if self.model is None:
+                    if self.schema is None:
                         return _missing_model_ui()
                     return EditTsvWithDiff(
-                        model=self.model, fn_upload=self.fn_upload, transposed=self.transposed, by_alias = self.by_alias
+                        schema=self.schema, fn_upload=self.fn_upload, transposed=self.transposed, by_alias = self.by_alias
                     )
                 self._ui_io_factory = _factory
             else:
                 def _factory_custom():
-                    if self.model is None:
+                    if self.schema is None:
                         return _missing_model_ui()
                     try:
                         return ui_io(
-                            model=self.model, fn_upload=self.fn_upload, transposed=self.transposed, by_alias = self.by_alias
+                            schema=self.schema, fn_upload=self.fn_upload, transposed=self.transposed, by_alias = self.by_alias
                         )
                     except Exception as e:
                         raise RuntimeError(
                             f"Failed to initialize ui_io '{ui_io.__name__}'."
-                            " Required traits are: `model`, `fn_upload`, `transposed`."
+                            " Required traits are: `schema`, `fn_upload`, `transposed`."
                             f" Original error: {e}"
                         ) from e
                 self._ui_io_factory = _factory_custom
@@ -430,7 +428,7 @@ def _factory_custom():
     def _init_ui_io(self, ui_io):
         if ui_io is not None and self.ui_io_initialised:
             self.ui_io = ui_io(
-                model=self.model, fn_upload=self.fn_upload, transposed=self.transposed, by_alias = self.by_alias
+                schema=self.schema, fn_upload=self.fn_upload, transposed=self.transposed, by_alias = self.by_alias
             )
 
     def _ensure_ui_io_initialised(self):

src/ipyautoui/custom/editgrid.py

@@ -2,7 +2,7 @@
     EditGrid,
     DataHandler,
 )
-from ipyautoui.custom.edittsv import Changes
+from ipyautoui.custom.edittsv import Changes, EditTsvFileUpload
 import pathlib
 import traitlets as tr
 import json
@@ -183,21 +183,21 @@ def handle_crud(self, changes: Changes):
     AUTO_GRID_DEFAULT_VALUE = data
 
     class DataFrameCols(BaseModel):
-        id: int = Field(1, json_schema_extra=dict(column_width=80, section="a"))
+        id: int = Field(1, json_schema_extra=dict(column_width=80, section="a", name= "id"))
         string: str = Field(
-            "string", json_schema_extra=dict(column_width=400, section="a")
+            "string", json_schema_extra=dict(column_width=400, section="a", name= "string")
         )
-        integer: int = Field(1, json_schema_extra=dict(column_width=80, section="a"))
+        integer: int = Field(1, json_schema_extra=dict(column_width=80, section="a", name= "integer"))
         floater: float = Field(
-            None, json_schema_extra=dict(column_width=70, section="b")
+            None, json_schema_extra=dict(column_width=70, section="b", name= "floater")
         )
 
     class TestDataFrame(RootModel):
         """a description of TestDataFrame"""
         root: ty.List[DataFrameCols] = Field(
             default=AUTO_GRID_DEFAULT_VALUE,
             json_schema_extra=dict(
-                format="dataframe", datagrid_index_name=("section", "title")
+                format="dataframe", datagrid_index_name=("section", "title", "name")
             ),
         )
 
@@ -214,7 +214,8 @@ class TestDataFrame(RootModel):
         global_decimal_places=1,
         column_width={"String": 400},
         fpth=json_path,
-        show_ui_io=True
+        show_ui_io=True,
+        ui_io=EditTsvFileUpload
     )
     display(edit_grid_file)
 
@@ -223,4 +224,3 @@ class TestDataFrame(RootModel):
 
 # -
 
-

src/ipyautoui/custom/editgridfile.py

@@ -11,13 +11,15 @@
 from IPython.display import clear_output
 from deepdiff import DeepDiff
 from deepdiff.helper import COLORED_COMPACT_VIEW  # COLORED_VIEW,
+from ipyautoui.automapschema import _init_model_schema
 from ipyautoui.custom.filedownload import MakeFileAndDownload
 from ipyautoui.custom.fileupload import TempFileUploadProcessor
 from copy import deepcopy
 import typing as ty
 import xlsxdatagrid as xdg
 from pathlib import Path
 from ipyautoui.watch_validate import pydantic_validate
+from ipyautoui._utils import pydantic_model_from_json_schema
 
 from ipyautoui.constants import BUTTON_WIDTH_MIN
 
@@ -132,7 +134,7 @@ def default_fn_upload(value):
 
 class EditTsv(CopyToClipboard):
     _value = tr.List(value=None, trait=tr.Dict, allow_none=True)
-    model = tr.Type(klass=BaseModel)
+    model = tr.Type(klass=BaseModel, default_value=None, allow_none=True)
     by_alias = tr.Bool(default_value=False)
     errors = tr.List(value=[], trait=tr.Dict)
     fn_upload = tr.Callable(default_value=default_fn_upload)
@@ -145,6 +147,7 @@ class EditTsv(CopyToClipboard):
     header_depth = tr.Int(default_value=1)
     disable_text_editing = tr.Bool(default_value=True)
     filename_suffix = tr.Unicode(default_value="", allow_none=True)
+    schema = tr.Dict()
 
     @tr.observe("upload_status")
     def upload_status_onchange(self, on_change):
@@ -195,6 +198,10 @@ def disable_text_editing_onchange(self, on_change):
             self.text.disabled = False
 
     def __init__(self, **kwargs):
+        #If model json schema is given, generate model from it, otherwise use given model
+        self.model, self.schema = _init_model_schema(kwargs.get("schema"))
+        if self.model is None:
+            self.model = pydantic_model_from_json_schema(self.schema)
         self.vbx_errors = w.VBox()
         self.bn_upload_text = w.Button(
             icon="save", disabled=True, layout={"width": BUTTON_WIDTH_MIN}