Transition to new flat JSON format

execution_tests/loop_condition_with_cmd_line_args/execution_test.py

@@ -40,8 +40,8 @@ def test_execution(self):
             output_value = from_database(value_rows[0].value, value_rows[0].type)
         expected_x = [f"T{i:03}" for i in range(31)]
         expected_y = [float(i) for i in range(31)]
-        self.assertEqual(output_value, Map(expected_x, expected_y))
+        self.assertEqual(output_value, Map(expected_x, expected_y, index_name="col_1"))
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     unittest.main()

requirements.txt

@@ -1,4 +1,4 @@
--e git+https://github.com/spine-tools/Spine-Database-API.git#egg=spinedb_api
+-e git+https://github.com/spine-tools/Spine-Database-API.git@WIP-data-transition#egg=spinedb_api
 -e git+https://github.com/spine-tools/spine-engine.git#egg=spine_engine
--e git+https://github.com/spine-tools/spine-items.git#egg=spine_items
+-e git+https://github.com/spine-tools/spine-items.git@WIP-data-transition#egg=spine_items
 -e .

spinetoolbox/spine_db_editor/mvcmodels/compound_models.py

@@ -20,7 +20,6 @@
 from PySide6.QtGui import QFont
 from spinedb_api import DatabaseMapping
 from spinedb_api.db_mapping_base import PublicItem
-from spinedb_api.parameter_value import join_value_and_type
 from spinedb_api.temp_id import TempId
 from ...fetch_parent import FlexibleFetchParent
 from ...helpers import DBMapPublicItems, parameter_identifier, rows_to_row_count_tuples
@@ -686,7 +685,7 @@ def get_set_data_delayed(self, index: QModelIndex) -> Callable[tuple[bytes, str]
         sub_model = self.sub_model_at_row(index.row())
         id_ = self.item_at_row(index.row())
         return lambda value_and_type, sub_model=sub_model, id_=id_: sub_model.update_items_in_db(
-            [{"id": id_, sub_model.value_field: join_value_and_type(*value_and_type)}]
+            [{"id": id_, sub_model.value_field: value_and_type[0], sub_model.type_field: value_and_type[1]}]
         )
 
 

spinetoolbox/spine_db_editor/mvcmodels/empty_models.py

@@ -229,7 +229,7 @@ def _entity_class_name_candidates(self, db_map: DatabaseMapping, item: list) ->
         raise NotImplementedError()
 
     def _make_item(self, row: int) -> dict:
-        return dict(zip(self.field_map.values(), self._main_data[row]))
+        return {key: value for key, value in zip(self.field_map.values(), self._main_data[row]) if value is not None}
 
     def _make_db_map_data(self, rows: Iterable[int]) -> DBMapDictItems:
         """
@@ -245,11 +245,10 @@ def _make_db_map_data(self, rows: Iterable[int]) -> DBMapDictItems:
         db_map_cache = _TempDBMapCache(self.db_mngr)
         for row in rows:
             item = self._make_item(row)
-            database = item.pop("database")
+            database = item.pop("database", "")
             db_map = db_map_cache.get(database)
             if db_map is None:
                 continue
-            item = {k: v for k, v in item.items() if v is not None}
             db_map_data.setdefault(db_map, []).append(item)
         return db_map_data
 
@@ -342,7 +341,7 @@ def _entity_class_name_candidates_by_parameter(cls, db_map: DatabaseMapping, row
             return []
         return [x["entity_class_name"] for x in db_map.find_parameter_definitions(name=name)]
 
-    def get_set_data_delayed(self, index: QModelIndex) -> Callable[[tuple[bytes, Optional[str]]], None]:
+    def get_set_data_delayed(self, index: QModelIndex) -> Callable[[Optional[bytes | str]], None]:
         """Returns a function that ParameterValueEditor can call to set data for the given index at any later time,
         even if the model changes.
         """
@@ -359,14 +358,14 @@ def __init__(self, model: MinimalTableModel, index: QModelIndex):
         self._model.modelReset.connect(self._invalidate)
         self._valid = True
 
-    def __call__(self, value_and_type: tuple[bytes, Optional[str]]) -> None:
+    def __call__(self, blob: Optional[bytes | str]) -> None:
         self._model.rowsInserted.disconnect(self._rows_inserted)
         self._model.rowsRemoved.disconnect(self._rows_removed)
         self._model.modelReset.disconnect(self._invalidate)
         if not self._valid:
             return
         index = self._model.index(self._row, self._column)
-        self._model.batch_set_data([index], [load_db_value(*value_and_type)])
+        self._model.batch_set_data([index], [load_db_value(blob)])
 
     def _rows_inserted(self, _: QModelIndex, first: int, last: int) -> None:
         if first > self._row:

spinetoolbox/spine_db_editor/mvcmodels/pivot_table_models.py

@@ -22,7 +22,8 @@
 from PySide6.QtGui import QFont
 from spinedb_api import DatabaseMapping
 from spinedb_api.helpers import name_from_elements
-from spinedb_api.parameter_value import IndexedValue, join_value_and_type, split_value_and_type
+from spinedb_api.incomplete_values import join_value_and_type, split_value_and_type
+from spinedb_api.parameter_value import IndexedValue
 from spinedb_api.temp_id import TempId
 from spinetoolbox.fetch_parent import FlexibleFetchParent
 from spinetoolbox.helpers import DB_ITEM_SEPARATOR, parameter_identifier, plain_to_tool_tip

spinetoolbox/spine_db_editor/mvcmodels/single_and_empty_model_mixins.py

@@ -11,16 +11,20 @@
 ######################################################################################################################
 
 """Miscellaneous mixins for parameter models."""
-from typing import Optional
-from spinedb_api import DatabaseMapping
-from spinedb_api.parameter_value import split_value_and_type
+from json import JSONDecodeError
+from spinedb_api.incomplete_values import split_value_and_type
 
 
 class SplitValueAndTypeMixin:
     def _convert_to_db(self, item: dict) -> dict:
         item = super()._convert_to_db(item)
-        if self.value_field in item:
-            value, value_type = split_value_and_type(item[self.value_field])
-            item[self.value_field] = value
-            item[self.type_field] = value_type
+        if self.value_field in item and not self.type_field in item:
+            value = item.pop(self.value_field)
+            try:
+                value_blob, value_type = split_value_and_type(value)
+            except JSONDecodeError:
+                item["parsed_value"] = value
+            else:
+                item[self.value_field] = value_blob
+                item[self.type_field] = value_type
         return item

spinetoolbox/spine_db_editor/mvcmodels/single_models.py

@@ -111,7 +111,7 @@ def update_items_in_db(self, items: list[dict]) -> None:
         items_to_upd = []
         for item in items:
             item_to_upd = self._convert_to_db(item)
-            if tuple(item_to_upd.keys()) != ("id",):
+            if len(item_to_upd) > 1:
                 items_to_upd.append(item_to_upd)
         if items_to_upd:
             self.db_mngr.update_items(self._parent.item_type, {self.db_map: items_to_upd})

spinetoolbox/spine_db_editor/widgets/custom_delegates.py

@@ -17,7 +17,7 @@
 from PySide6.QtGui import QColor, QFont, QFontMetrics, QIcon, QRegularExpressionValidator
 from PySide6.QtWidgets import QStyledItemDelegate
 from spinedb_api import to_database
-from spinedb_api.parameter_value import join_value_and_type
+from spinedb_api.incomplete_values import join_value_and_type
 from spinetoolbox.spine_db_editor.widgets.custom_editors import (
     BooleanSearchBarEditor,
     CheckListEditor,
@@ -31,7 +31,7 @@
     SearchBarEditorWithCreation,
 )
 from ...font import TOOLBOX_FONT
-from ...helpers import DB_ITEM_SEPARATOR, object_icon
+from ...helpers import object_icon
 from ...mvcmodels.shared import DB_MAP_ROLE, INVALID_TYPE, PARAMETER_TYPE_VALIDATION_ROLE, PARSED_ROLE
 from ...spine_db_manager import SpineDBManager
 from ...widgets.custom_delegates import CheckBoxDelegate, RankDelegate

spinetoolbox/spine_db_manager.py

@@ -49,15 +49,8 @@
 from spinedb_api.db_mapping_base import PublicItem
 from spinedb_api.exception import NothingToCommit
 from spinedb_api.helpers import remove_credentials_from_url
-from spinedb_api.parameter_value import (
-    MapIndex,
-    Value,
-    deep_copy_value,
-    dump_db_value,
-    join_value_and_type,
-    load_db_value,
-    split_value_and_type,
-)
+from spinedb_api.incomplete_values import dump_db_value, join_value_and_type, split_value_and_type
+from spinedb_api.parameter_value import MapIndex, Value, deep_copy_value, load_db_value
 from spinedb_api.spine_io.exporters.excel import export_spine_database_to_xlsx
 from spinedb_api.temp_id import TempId
 from .database_display_names import NameRegistry
@@ -1407,7 +1400,9 @@ def _get_data_for_export(
         data = {}
         for db_map, item_ids in db_map_item_ids.items():
             with db_map:
-                for key, items in export_data(db_map, parse_value=load_db_value, **item_ids).items():
+                for key, items in export_data(
+                    db_map, parse_value=lambda value, _: load_db_value(value), **item_ids
+                ).items():
                     data.setdefault(key, []).extend(items)
         return data
 

spinetoolbox/widgets/custom_qtableview.py

@@ -16,6 +16,7 @@
 import ctypes
 import io
 from itertools import cycle
+import json
 import locale
 from numbers import Number
 from operator import methodcaller
@@ -33,7 +34,8 @@
     from_database,
     to_database,
 )
-from spinedb_api.parameter_value import FLOAT_VALUE_TYPE, join_value_and_type, split_value_and_type
+from spinedb_api.incomplete_values import join_value_and_type, split_value_and_type
+from spinedb_api.parameter_value import FLOAT_VALUE_TYPE
 from ..mvcmodels.empty_row_model import EmptyRowModel
 from ..mvcmodels.minimal_table_model import MinimalTableModel
 from .paste_excel import EXCEL_CLIPBOARD_MIME_TYPE, clipboard_excel_as_table
@@ -862,12 +864,16 @@ def _convert_pasted_data(data, length):
                         except SpineDBAPIError:
                             pass
                     try:
-                        value = from_database(*split_value_and_type(cell))
-                        table_row.append(value)
-                        continue
-                    except ParameterValueFormatError:
-                        pass
-                    table_row.append(cell)
+                        value_and_type = split_value_and_type(cell)
+                    except json.JSONDecodeError:
+                        table_row.append(cell)
+                    else:
+                        try:
+                            value = from_database(*value_and_type)
+                        except ParameterValueFormatError:
+                            table_row.append(cell)
+                        else:
+                            table_row.append(value)
                 pasted_table.append(table_row)
         return pasted_table
 

spinetoolbox/widgets/parameter_value_editor.py

@@ -60,10 +60,10 @@ def __init__(self, index, parent=None, plain=False):
     def _set_data(self, value):
         """See base class."""
         try:
-            value_type_tup = to_database(value)
+            value_and_type = to_database(value)
         except ParameterValueFormatError as error:
             message = f"Cannot set value: {error}"
             QMessageBox.warning(self, "Parameter Value error", message)
             return False
-        self.set_data_delayed(value_type_tup)
+        self.set_data_delayed(value_and_type)
         return True

spinetoolbox/widgets/paste_excel.py

@@ -12,7 +12,7 @@
 """This module contains utilities for pasting data from Excel."""
 from xml.etree import ElementTree
 from spinedb_api import DateTime, to_database
-from spinedb_api.parameter_value import join_value_and_type
+from spinedb_api.incomplete_values import join_value_and_type
 
 # Ideally, we would use openpyxl to deal with Excel data.
 # However, openpyxl is geared towards handling data from file,
@@ -34,10 +34,10 @@ def _convert_from_excel(data):
     """
 
     def date_time_to_database(x):
-        return bytes(join_value_and_type(*to_database(DateTime(x))), encoding="utf-8")
+        return join_value_and_type(*to_database(DateTime(x)))
 
     def bool_to_database(x):
-        return bytes(join_value_and_type(*to_database(bool(x))), encoding="utf-8")
+        return join_value_and_type(*to_database(bool(x)))
 
     convert_function = {
         "Boolean": bool_to_database,

tests/spine_db_editor/mvcmodels/test_emptyParameterModels.py

@@ -22,8 +22,9 @@
     import_relationship_parameters,
     import_relationships,
 )
-from spinedb_api.parameter_value import join_value_and_type, to_database
-from spinetoolbox.helpers import DB_ITEM_SEPARATOR, signal_waiter
+from spinedb_api.incomplete_values import join_value_and_type
+from spinedb_api.parameter_value import to_database
+from spinetoolbox.helpers import signal_waiter
 from spinetoolbox.spine_db_editor.mvcmodels.empty_models import EmptyParameterDefinitionModel, EmptyParameterValueModel
 from tests.mock_helpers import MockSpineDBManager, TestCaseWithQApplication, fetch_model, q_object
 
@@ -82,7 +83,10 @@ def test_do_not_add_invalid_object_parameter_values(self):
             model.set_undo_stack(self._undo_stack)
             fetch_model(model)
             self.assertTrue(
-                model.batch_set_data(_empty_indexes(model), ["fish", ("nemo",), "water", "Base", "salty", "mock_db"])
+                model.batch_set_data(
+                    _empty_indexes(model),
+                    ["fish", ("nemo",), "water", "Base", join_value_and_type(*to_database("salty")), "mock_db"],
+                )
             )
             values = [x for x in self._db_map.get_items("parameter_value") if not x["dimension_id_list"]]
             self.assertEqual(values, [])