Change kd.slice([]).get_schema() == kd.NONE

felbro · copybara-github · commit 951a792a9064 · 2025-01-21T09:44:51.000-08:00
Additionally:
* kd.list() -&gt; LIST[OBJECT]
* kd.list([]) -&gt; LIST[NONE]
* kd.slice([]) -&gt; NONE
* kd.dict() -&gt; DICT[OBJECT, OBJECT]
* kd.dict({}) -&gt; DICT[NONE, NONE]

I have not addressed the intended `from_py([]) -&gt; LIST[NONE]` as `from_py` requires more generic changes (e.g. `from_py([1]).get_schema() == OBJECT` right now).

PiperOrigin-RevId: 717956799
Change-Id: I10f6c2912e6762f6c2b92c41c49a9efcca046892
diff --git a/koladata/internal/schema_utils.cc b/koladata/internal/schema_utils.cc
@@ -338,8 +338,8 @@ bool VerifySchemaForItemIds(const internal::DataItem& schema_item) {
 }
 
 absl::Status VerifyDictKeySchema(const internal::DataItem& schema_item) {
-  if (schema_item == kNone || schema_item == kFloat32 ||
-      schema_item == kFloat64 || schema_item == kExpr) {
+  if (schema_item == kFloat32 || schema_item == kFloat64 ||
+      schema_item == kExpr) {
     return absl::InvalidArgumentError(
         absl::StrFormat("dict keys cannot be %v", schema_item));
   }
diff --git a/koladata/internal/schema_utils_test.cc b/koladata/internal/schema_utils_test.cc
@@ -516,13 +516,12 @@ TEST(SchemaUtilsTest, VerifySchemaForItemIds) {
 
 TEST(SchemaUtilsTest, VerifyDictKeySchema) {
   EXPECT_THAT(VerifyDictKeySchema(DataItem(schema::kInt32)), IsOk());
+  EXPECT_THAT(VerifyDictKeySchema(DataItem(schema::kNone)), IsOk());
   EXPECT_THAT(VerifyDictKeySchema(DataItem(schema::kFloat32)),
               StatusIs(absl::StatusCode::kInvalidArgument,
                        HasSubstr("dict keys cannot be FLOAT32")));
   EXPECT_THAT(VerifyDictKeySchema(DataItem(schema::kFloat64)),
               StatusIs(absl::StatusCode::kInvalidArgument));
-  EXPECT_THAT(VerifyDictKeySchema(DataItem(schema::kNone)),
-              StatusIs(absl::StatusCode::kInvalidArgument));
   EXPECT_THAT(VerifyDictKeySchema(DataItem(schema::kExpr)),
               StatusIs(absl::StatusCode::kInvalidArgument));
 }
diff --git a/koladata/operators/slices.cc b/koladata/operators/slices.cc
@@ -140,7 +140,7 @@ absl::StatusOr<DataSlice> ConcatOrStackImpl(bool stack, int64_t ndim,
     return DataSlice::Create(
         internal::DataSliceImpl::CreateEmptyAndUnknownType(0),
         DataSlice::JaggedShape::FlatFromSize(0),
-        internal::DataItem(schema::kObject), nullptr);
+        internal::DataItem(schema::kNone), nullptr);
   }
 
   const int64_t rank = args[0].GetShape().rank();
diff --git a/py/koladata/ext/experimental/parallel_call/call.py b/py/koladata/ext/experimental/parallel_call/call.py
@@ -391,7 +391,9 @@ def _schedule_call(
   """
   inputs = bind_args(fn, args, kwargs)
   debug = parent_debug.get_bag().obj(
-      name=debug_name, start_time=kd.item(time.time(), kd.FLOAT64), children=[]
+      name=debug_name,
+      start_time=kd.item(time.time(), kd.FLOAT64),
+      children=kd.list(),
   )
   parent_debug.children.append(debug)
   fn_call = _FunctorCall(fn, inputs, debug)
@@ -435,7 +437,7 @@ def call_multithreaded_with_debug(
   kwargs = {k: kd.optools.as_qvalue(v) for k, v in kwargs.items()}
   repo = task_repository.TaskRepository()
   debug_db = kd.bag()
-  root_debug = debug_db.obj(children=[])
+  root_debug = debug_db.obj(children=kd.list())
   final_result_task = _schedule_call(
       repo, fn, args, kwargs, root_debug, '<root>'
   )
diff --git a/py/koladata/functions/tests/BUILD b/py/koladata/functions/tests/BUILD
@@ -605,6 +605,7 @@ py_test(
     name = "to_py_test",
     srcs = ["to_py_test.py"],
     deps = [
+        "//py/koladata/exceptions",
         "//py/koladata/functions",
         "//py/koladata/operators:kde_operators",
         "//py/koladata/types:data_slice",
diff --git a/py/koladata/functions/tests/concat_lists_test.py b/py/koladata/functions/tests/concat_lists_test.py
@@ -12,8 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""Tests for kd.concat_lists."""
-
 from absl.testing import absltest
 from absl.testing import parameterized
 from koladata.functions import functions as fns
@@ -45,7 +43,7 @@ def test_mutability(self):
   @parameterized.parameters(
       (
           (),
-          fns.list([]),
+          fns.list(),
       ),
       (
           (fns.list([1, 2, 3]),),
diff --git a/py/koladata/functions/tests/dict_test.py b/py/koladata/functions/tests/dict_test.py
@@ -53,6 +53,12 @@ def test_empty(self):
         d[keys], ds([1, None, 1], schema_constants.OBJECT).with_bag(d.get_bag())
     )
 
+  def test_no_input_values(self):
+    d = fns.dict({})
+    self.assertIsInstance(d, dict_item.DictItem)
+    testing.assert_dicts_keys_equal(d, ds([]))
+    testing.assert_dicts_values_equal(d, ds([]))
+
   def test_dict_with_uuid(self):
     db = fns.bag()
     testing.assert_equal(
diff --git a/py/koladata/functions/tests/from_proto_test.py b/py/koladata/functions/tests/from_proto_test.py
@@ -29,7 +29,7 @@ class FromProtoTest(absltest.TestCase):
 
   def test_zero_messages(self):
     x = fns.from_proto([])
-    testing.assert_equal(x.no_bag(), ds([]))
+    testing.assert_equal(x.no_bag(), ds([], schema_constants.OBJECT))
 
   def test_single_none(self):
     x = fns.from_proto(None)
diff --git a/py/koladata/functions/tests/from_py_test.py b/py/koladata/functions/tests/from_py_test.py
@@ -272,6 +272,10 @@ def test_list_from_dim(self):
     l3 = fns.from_py([1, 2], from_dim=1)
     testing.assert_equal(l3, ds([1, 2]))
 
+  def test_empty_from_dim(self):
+    l0 = fns.from_py([], from_dim=1)
+    testing.assert_equal(l0, ds([]).with_bag(l0.get_bag()))
+
   def test_list_from_dim_with_schema(self):
     input_list = [[1, 2.0], [3, 4]]
 
diff --git a/py/koladata/functions/tests/list_like_test.py b/py/koladata/functions/tests/list_like_test.py
@@ -37,7 +37,9 @@ def test_mutability(self):
   def test_item(self):
     l = fns.list_like(ds(1)).fork_db()
     self.assertIsInstance(l, list_item.ListItem)
-    testing.assert_equal(l[:], ds([]).with_bag(l.get_bag()))
+    testing.assert_equal(
+        l[:], ds([], schema_constants.OBJECT).with_bag(l.get_bag())
+    )
     l.append(1)
     testing.assert_equal(
         l[:], ds([1], schema_constants.OBJECT).with_bag(l.get_bag())
@@ -54,12 +56,15 @@ def test_empty_item(self):
     l = fns.list_like(ds(None)).fork_db()
     testing.assert_equal(l.no_bag(), ds(None, l.get_schema().no_bag()))
     l.append(1)
-    testing.assert_equal(l[:].no_bag(), ds([]))
+    testing.assert_equal(l[:].no_bag(), ds([], schema_constants.OBJECT))
 
   def test_slice(self):
     l = fns.list_like(ds([[1, None], [3]])).fork_db()
     self.assertIsInstance(l, data_slice.DataSlice)
-    testing.assert_equal(l[:], ds([[[], []], [[]]]).with_bag(l.get_bag()))
+    testing.assert_equal(
+        l[:],
+        ds([[[], []], [[]]], schema_constants.OBJECT).with_bag(l.get_bag()),
+    )
     l.append(1)
     testing.assert_equal(
         l[:],
@@ -69,9 +74,15 @@ def test_slice(self):
   def test_all_missing_slice(self):
     l = fns.list_like(ds([[None, None], [None]])).fork_db()
     self.assertIsInstance(l, data_slice.DataSlice)
-    testing.assert_equal(l[:], ds([[[], []], [[]]]).with_bag(l.get_bag()))
+    testing.assert_equal(
+        l[:],
+        ds([[[], []], [[]]], schema_constants.OBJECT).with_bag(l.get_bag()),
+    )
     l.append(1)
-    testing.assert_equal(l[:], ds([[[], []], [[]]]).with_bag(l.get_bag()))
+    testing.assert_equal(
+        l[:],
+        ds([[[], []], [[]]], schema_constants.OBJECT).with_bag(l.get_bag()),
+    )
 
   def test_slice_with_scalar_items(self):
     l = fns.list_like(ds([[1, None], [3]]), 1).fork_db()
diff --git a/py/koladata/functions/tests/list_shaped_as_test.py b/py/koladata/functions/tests/list_shaped_as_test.py
@@ -39,7 +39,9 @@ def test_mutability(self):
   def test_item(self):
     l = fns.list_shaped_as(ds(1))
     self.assertIsInstance(l, list_item.ListItem)
-    testing.assert_equal(l[:], ds([]).with_bag(l.get_bag()))
+    testing.assert_equal(
+        l[:], ds([], schema_constants.OBJECT).with_bag(l.get_bag())
+    )
 
   def test_item_with_items(self):
     l = fns.list_shaped_as(ds(1), [1, 2])
@@ -49,7 +51,10 @@ def test_item_with_items(self):
   def test_slice(self):
     l = fns.list_shaped_as(ds([['a', 'b'], ['c']])).fork_db()
     self.assertIsInstance(l, data_slice.DataSlice)
-    testing.assert_equal(l[:], ds([[[], []], [[]]]).with_bag(l.get_bag()))
+    testing.assert_equal(
+        l[:],
+        ds([[[], []], [[]]], schema_constants.OBJECT).with_bag(l.get_bag()),
+    )
     l.append(1)
     testing.assert_equal(
         l[:],
diff --git a/py/koladata/functions/tests/list_shaped_test.py b/py/koladata/functions/tests/list_shaped_test.py
@@ -41,7 +41,9 @@ def test_item(self):
     shape = jagged_shape.create_shape()
     l = fns.list_shaped(shape).fork_db()
     self.assertIsInstance(l, list_item.ListItem)
-    testing.assert_equal(l[:], ds([]).with_bag(l.get_bag()))
+    testing.assert_equal(
+        l[:], ds([], schema_constants.OBJECT).with_bag(l.get_bag())
+    )
     l.append(1)
     testing.assert_equal(
         l[:], ds([1], schema_constants.OBJECT).with_bag(l.get_bag())
@@ -59,7 +61,10 @@ def test_slice(self):
     shape = jagged_shape.create_shape([2], [2, 1])
     l = fns.list_shaped(shape).fork_db()
     self.assertIsInstance(l, data_slice.DataSlice)
-    testing.assert_equal(l[:], ds([[[], []], [[]]]).with_bag(l.get_bag()))
+    testing.assert_equal(
+        l[:],
+        ds([[[], []], [[]]], schema_constants.OBJECT).with_bag(l.get_bag()),
+    )
     l.append(1)
     testing.assert_equal(
         l[:],
diff --git a/py/koladata/functions/tests/list_test.py b/py/koladata/functions/tests/list_test.py
@@ -43,12 +43,19 @@ def test_mutability(self):
   def test_empty(self):
     l = fns.list().fork_db()
     self.assertIsInstance(l, list_item.ListItem)
-    testing.assert_equal(l[:], ds([]).with_bag(l.get_bag()))
+    testing.assert_equal(
+        l[:], ds([], schema_constants.OBJECT).with_bag(l.get_bag())
+    )
     l.append(1)
     testing.assert_equal(
         l[:], ds([1], schema_constants.OBJECT).with_bag(l.get_bag())
     )
 
+  def test_no_values(self):
+    l = fns.list([]).fork_db()
+    self.assertIsInstance(l, list_item.ListItem)
+    testing.assert_equal(l[:], ds([]).with_bag(l.get_bag()))
+
   def test_list_with_uuid(self):
     db = fns.bag()
     testing.assert_equal(
diff --git a/py/koladata/functions/tests/new_test.py b/py/koladata/functions/tests/new_test.py
@@ -415,7 +415,7 @@ def test_universal_converter_list(self):
   def test_universal_converter_empty_list(self):
     l = fns.new([])
     testing.assert_equal(
-        l.get_schema().get_attr('__items__').no_bag(), schema_constants.OBJECT
+        l.get_schema().get_attr('__items__').no_bag(), schema_constants.NONE
     )
     testing.assert_equal(l[:].no_bag(), ds([]))
 
@@ -440,10 +440,10 @@ def test_universal_converter_dict(self):
   def test_universal_converter_empty_dict(self):
     d = fns.new({})
     testing.assert_equal(
-        d.get_schema().get_attr('__keys__').no_bag(), schema_constants.OBJECT
+        d.get_schema().get_attr('__keys__').no_bag(), schema_constants.NONE
     )
     testing.assert_equal(
-        d.get_schema().get_attr('__values__').no_bag(), schema_constants.OBJECT
+        d.get_schema().get_attr('__values__').no_bag(), schema_constants.NONE
     )
     testing.assert_dicts_keys_equal(d, ds([]))
     testing.assert_equal(d[ds([])].no_bag(), ds([]))
diff --git a/py/koladata/functions/tests/obj_test.py b/py/koladata/functions/tests/obj_test.py
@@ -190,7 +190,7 @@ def test_universal_converter_empty_list(self):
         l.get_obj_schema().get_attr('__items__').no_bag(),
         schema_constants.OBJECT,
     )
-    testing.assert_equal(l[:].no_bag(), ds([]))
+    testing.assert_equal(l[:].no_bag(), ds([], schema_constants.OBJECT))
 
   def test_universal_converter_dict(self):
     d = fns.obj({'a': 42, 'b': ds(37, schema_constants.INT64)})
@@ -223,8 +223,8 @@ def test_universal_converter_empty_dict(self):
         d.get_obj_schema().get_attr('__values__').no_bag(),
         schema_constants.OBJECT,
     )
-    testing.assert_dicts_keys_equal(d, ds([]))
-    testing.assert_equal(d[ds([])].no_bag(), ds([]))
+    testing.assert_dicts_keys_equal(d, ds([], schema_constants.OBJECT))
+    testing.assert_equal(d[ds([])].no_bag(), ds([], schema_constants.OBJECT))
 
   def test_universal_converter_entity(self):
     with self.subTest('item'):
diff --git a/py/koladata/functions/tests/to_py_test.py b/py/koladata/functions/tests/to_py_test.py
@@ -15,6 +15,7 @@
 import dataclasses
 
 from absl.testing import absltest
+from koladata.exceptions import exceptions
 from koladata.functions import functions as fns
 from koladata.operators import kde_operators
 from koladata.types import data_slice
@@ -29,7 +30,14 @@ class ToPyTest(absltest.TestCase):
 
   def test_list(self):
     root = fns.container()
-    root.list_value = []
+    root.none_list_value = []
+    root.none_list_value.append(None)
+    with self.assertRaisesRegex(
+        exceptions.KodaError, 'the schema for List item is incompatible'
+    ):
+      root.none_list_value.append(1)
+
+    root.list_value = fns.list()
     root.list_value.append(1)
     root.list_value.append(2)
     nested_values = ds([[1, 2], [3, 4, 5]])
@@ -41,12 +49,13 @@ def test_list(self):
         'list_value': [1, 2],
         'nested_list': [[1, 2], [3, 4, 5]],
         'empty_list': [],
+        'none_list_value': [None],
     }
     self.assertEqual(expected, py_obj)
 
   def test_dict(self):
     root = fns.container()
-    root.dict_value = {}
+    root.dict_value = fns.dict()
     root.dict_value['key_1'] = 'value_1'
     root.dict_value['key_2'] = 'value_2'
     root.empty_dict = {}
@@ -60,7 +69,7 @@ def test_dict(self):
 
   def test_dict_with_obj_keys(self):
     root = fns.container()
-    root.dict_value = {}
+    root.dict_value = fns.dict()
     k = fns.obj(a=1)
     root.dict_value[k] = 1
     py_obj = fns.to_py(root)
diff --git a/py/koladata/operators/tests/BUILD b/py/koladata/operators/tests/BUILD
@@ -3062,6 +3062,7 @@ py_test(
         "//py/koladata/expr:view",
         "//py/koladata/operators:kde_operators",
         "//py/koladata/operators:optools",
+        "//py/koladata/operators:slices",
         "//py/koladata/operators/tests/util:qtypes",
         "//py/koladata/testing",
         "//py/koladata/types:data_bag",
@@ -3083,6 +3084,7 @@ py_test(
         "//py/koladata/expr:view",
         "//py/koladata/operators:kde_operators",
         "//py/koladata/operators:optools",
+        "//py/koladata/operators:slices",
         "//py/koladata/operators/tests/util:qtypes",
         "//py/koladata/testing",
         "//py/koladata/types:data_bag",
diff --git a/py/koladata/operators/tests/core_attrs_test.py b/py/koladata/operators/tests/core_attrs_test.py
@@ -119,7 +119,7 @@ def test_entity_as_obj_conflict(self):
 
   def test_empty_slice(self):
     entity = kde.new_like(ds([])).eval()
-    db = kde.core.attrs(entity, x=ds([])).eval()
+    db = kde.core.attrs(entity, x=ds([], schema_constants.OBJECT)).eval()
     expected_db = bag()
     expected_db[entity.get_schema()].x = schema_constants.OBJECT
     testing.assert_equivalent(db, expected_db)
diff --git a/py/koladata/operators/tests/json_to_json_test.py b/py/koladata/operators/tests/json_to_json_test.py
@@ -75,7 +75,7 @@ class JsonToJsonTest(parameterized.TestCase):
           ds('[[1], [2, 3]]'),
       ),
       (
-          fns.dict({}),
+          fns.dict(),
           {},
           ds('{}'),
       ),
diff --git a/py/koladata/operators/tests/lists_explode_test.py b/py/koladata/operators/tests/lists_explode_test.py
@@ -38,7 +38,7 @@
 
 
 db = data_bag.DataBag.empty()
-LIST0 = db.list([])
+LIST0 = db.list()
 LIST1 = db.list([1, 2, 3])
 LIST2 = db.list([[1], [2, 3]])
 LIST3 = db.list([[None], [None, None]])
diff --git a/py/koladata/operators/tests/objs_new_test.py b/py/koladata/operators/tests/objs_new_test.py
@@ -137,7 +137,7 @@ def test_converter_empty_list(self):
         l.get_obj_schema().get_attr('__items__').no_bag(),
         schema_constants.OBJECT,
     )
-    testing.assert_equal(l[:].no_bag(), ds([]))
+    testing.assert_equal(l[:].no_bag(), ds([], schema_constants.OBJECT))
 
   def test_converter_dict(self):
     d = kde.objs.new(
@@ -160,8 +160,8 @@ def test_converter_empty_dict(self):
         d.get_obj_schema().get_attr('__values__').no_bag(),
         schema_constants.OBJECT,
     )
-    testing.assert_dicts_keys_equal(d, ds([]))
-    testing.assert_equal(d[ds([])].no_bag(), ds([]))
+    testing.assert_dicts_keys_equal(d, ds([], schema_constants.OBJECT))
+    testing.assert_equal(d[ds([])].no_bag(), ds([], schema_constants.OBJECT))
 
   def test_converter_entity(self):
     with self.subTest('item'):
diff --git a/py/koladata/operators/tests/py_map_py_on_cond_test.py b/py/koladata/operators/tests/py_map_py_on_cond_test.py
@@ -89,7 +89,7 @@ def test_empty_input(self):
     self.assertIsNone(res.get_bag())
 
     res = expr_eval.eval(kde.py.map_py_on_cond(yes_fn, None, cond, x=val))
-    testing.assert_equal(res.no_bag(), ds([], schema_constants.OBJECT))
+    testing.assert_equal(res.no_bag(), ds([]))
     self.assertIsNone(res.get_bag())
 
     res = expr_eval.eval(
diff --git a/py/koladata/operators/tests/py_map_py_on_present_test.py b/py/koladata/operators/tests/py_map_py_on_present_test.py
@@ -67,7 +67,7 @@ def test_empty_input(self):
     self.assertIsNone(res.get_bag())
 
     res = expr_eval.eval(kde.py.map_py_on_present(yes_fn, x=val))
-    testing.assert_equal(res.no_bag(), ds([], schema_constants.OBJECT))
+    testing.assert_equal(res.no_bag(), ds([]))
     self.assertIsNone(res.get_bag())
 
     res = expr_eval.eval(
diff --git a/py/koladata/operators/tests/py_map_py_on_selected_test.py b/py/koladata/operators/tests/py_map_py_on_selected_test.py
diff --git a/py/koladata/operators/tests/py_map_py_test.py b/py/koladata/operators/tests/py_map_py_test.py
diff --git a/py/koladata/operators/tests/slices_concat_test.py b/py/koladata/operators/tests/slices_concat_test.py
diff --git a/py/koladata/operators/tests/slices_inverse_select_test.py b/py/koladata/operators/tests/slices_inverse_select_test.py
diff --git a/py/koladata/operators/tests/slices_stack_test.py b/py/koladata/operators/tests/slices_stack_test.py
diff --git a/py/koladata/types/boxing.cc b/py/koladata/types/boxing.cc
diff --git a/py/koladata/types/boxing_test.py b/py/koladata/types/boxing_test.py
diff --git a/py/koladata/types/data_bag_test.py b/py/koladata/types/data_bag_test.py
diff --git a/py/koladata/types/data_slice_test.py b/py/koladata/types/data_slice_test.py
diff --git a/py/koladata/types/list_item_test.py b/py/koladata/types/list_item_test.py

Original file line number	Diff line number	Diff line change
`@@ -338,8 +338,8 @@ bool VerifySchemaForItemIds(const internal::DataItem& schema_item) {`
`338`	`338`	`}`
`339`	`339`
`340`	`340`	`absl::Status VerifyDictKeySchema(const internal::DataItem& schema_item) {`
`341`		`- if (schema_item == kNone \|\| schema_item == kFloat32 \|\|`
`342`		`- schema_item == kFloat64 \|\| schema_item == kExpr) {`
	`341`	`+ if (schema_item == kFloat32 \|\| schema_item == kFloat64 \|\|`
	`342`	`+ schema_item == kExpr) {`
`343`	`343`	`return absl::InvalidArgumentError(`
`344`	`344`	`absl::StrFormat("dict keys cannot be %v", schema_item));`
`345`	`345`	`}`
Original file line number	Diff line number	Diff line change
`@@ -140,7 +140,7 @@ absl::StatusOr<DataSlice> ConcatOrStackImpl(bool stack, int64_t ndim,`
`140`	`140`	`return DataSlice::Create(`
`141`	`141`	`internal::DataSliceImpl::CreateEmptyAndUnknownType(0),`
`142`	`142`	`DataSlice::JaggedShape::FlatFromSize(0),`
`143`		`- internal::DataItem(schema::kObject), nullptr);`
	`143`	`+ internal::DataItem(schema::kNone), nullptr);`
`144`	`144`	`}`
`145`	`145`
`146`	`146`	`const int64_t rank = args[0].GetShape().rank();`