From d90d216d911d9acc3fc8b565b64d6963d119144c Mon Sep 17 00:00:00 2001 From: "Carlson, Eric Thomas" Date: Wed, 2 Feb 2022 23:16:52 -0500 Subject: [PATCH 1/8] Add failing test with a dataframe that has datetimes in it --- tests/API1/testjsonserialization.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/API1/testjsonserialization.py b/tests/API1/testjsonserialization.py index f4b6d070f..8e04f7510 100644 --- a/tests/API1/testjsonserialization.py +++ b/tests/API1/testjsonserialization.py @@ -42,7 +42,7 @@ class TestSet(param.Parameterized): __test__ = False numpy_params = ['r','y'] - pandas_params = ['s','t','u','z'] + pandas_params = ['s','t','u','z','ab'] conditionally_unsafe = ['f', 'o'] a = param.Integer(default=5, doc='Example doc', bounds=(2,30), inclusive_bounds=(True, False)) @@ -73,6 +73,9 @@ class TestSet(param.Parameterized): y = None if np is None else param.Array(default=None) z = None if pd is None else param.DataFrame(default=None, allow_None=True) aa = param.Tuple(default=None, allow_None=True, length=1) + ab = None if pd is None else param.DataFrame(default=pd.DataFrame( + {'A':[datetime.datetime(year,1,1) for year in range(2020,2023)], 'B':[1.1,2.2,3.3]}), + columns=(1,4), rows=(2,5)) test = TestSet(a=29) From d8a4c8c9c0c5e3e2f12f748d38e0811cc24e741f Mon Sep 17 00:00:00 2001 From: "Carlson, Eric Thomas" Date: Wed, 2 Feb 2022 23:20:03 -0500 Subject: [PATCH 2/8] Add new dataframe ser/de implementations, tests pass --- param/__init__.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/param/__init__.py b/param/__init__.py index b0f509321..5b569d8d9 100644 --- a/param/__init__.py +++ b/param/__init__.py @@ -1567,14 +1567,16 @@ def _validate(self, val): def serialize(cls, value): if value is None: return 'null' - return value.to_dict('records') + import json + return json.loads(value.to_json(orient='table')) @classmethod def deserialize(cls, value): if value == 'null': return None - from pandas import DataFrame as pdDFrame - return pdDFrame(value) + import json + import pandas as pd + return pd.read_json(json.dumps(value), orient='table') class Series(ClassSelector): From 8a00687d76bef626bae3c8adfcd0c5886c332a86 Mon Sep 17 00:00:00 2001 From: "Carlson, Eric Thomas" Date: Thu, 3 Feb 2022 12:49:11 -0500 Subject: [PATCH 3/8] Add load from array test that breaks pandas --- tests/API1/testjsonserialization.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/API1/testjsonserialization.py b/tests/API1/testjsonserialization.py index 8e04f7510..e9d714864 100644 --- a/tests/API1/testjsonserialization.py +++ b/tests/API1/testjsonserialization.py @@ -42,7 +42,7 @@ class TestSet(param.Parameterized): __test__ = False numpy_params = ['r','y'] - pandas_params = ['s','t','u','z','ab'] + pandas_params = ['s','t','u','z','ab','ac'] conditionally_unsafe = ['f', 'o'] a = param.Integer(default=5, doc='Example doc', bounds=(2,30), inclusive_bounds=(True, False)) @@ -76,6 +76,9 @@ class TestSet(param.Parameterized): ab = None if pd is None else param.DataFrame(default=pd.DataFrame( {'A':[datetime.datetime(year,1,1) for year in range(2020,2023)], 'B':[1.1,2.2,3.3]}), columns=(1,4), rows=(2,5)) + ac = None if pd is None else param.DataFrame(default=pd.DataFrame( + [[1,2,3],[4,5,6],[7,8,9]]), + columns=(1,4), rows=(2,5)) test = TestSet(a=29) From a2661067620fa4bd10af319f5bb7a0e22ed048fd Mon Sep 17 00:00:00 2001 From: "Carlson, Eric Thomas" Date: Thu, 3 Feb 2022 14:37:34 -0500 Subject: [PATCH 4/8] Fix operation for matrix-defined dataframes, allow dates to serialize to strings --- param/__init__.py | 4 ++-- tests/API1/testjsonserialization.py | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/param/__init__.py b/param/__init__.py index 5b569d8d9..1e66cb34b 100644 --- a/param/__init__.py +++ b/param/__init__.py @@ -1568,7 +1568,7 @@ def serialize(cls, value): if value is None: return 'null' import json - return json.loads(value.to_json(orient='table')) + return json.loads(value.to_json(date_format='iso')) @classmethod def deserialize(cls, value): @@ -1576,7 +1576,7 @@ def deserialize(cls, value): return None import json import pandas as pd - return pd.read_json(json.dumps(value), orient='table') + return pd.read_json(json.dumps(value)) class Series(ClassSelector): diff --git a/tests/API1/testjsonserialization.py b/tests/API1/testjsonserialization.py index e9d714864..b0c3098f4 100644 --- a/tests/API1/testjsonserialization.py +++ b/tests/API1/testjsonserialization.py @@ -192,13 +192,16 @@ def test_numpy_instance_serialization(self): @pd_skip def test_pandas_instance_serialization(self): + from pandas.api.types import is_datetime64_any_dtype as is_datetime serialized = test.param.serialize_parameters(subset=test.pandas_params, mode=self.mode) deserialized = TestSet.param.deserialize_parameters(serialized, mode=self.mode) for pname in test.pandas_params: if getattr(test, pname) is None: self.assertTrue(deserialized[pname] is None) else: - self.assertTrue(getattr(test, pname).equals(deserialized[pname])) + test_df = getattr(test, pname) + non_date_cols = [c for c in test_df.columns if not is_datetime(test_df[c])] + self.assertTrue(test_df[non_date_cols].equals(deserialized[pname][non_date_cols])) From 18882d4035a33ac5d7a85373476d200df56071c2 Mon Sep 17 00:00:00 2001 From: "Carlson, Eric Thomas" Date: Thu, 3 Feb 2022 16:11:30 -0500 Subject: [PATCH 5/8] Add verification of datetime serialization roundtrips to pandas test --- tests/API1/testjsonserialization.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/API1/testjsonserialization.py b/tests/API1/testjsonserialization.py index b0c3098f4..fb8a8200b 100644 --- a/tests/API1/testjsonserialization.py +++ b/tests/API1/testjsonserialization.py @@ -200,8 +200,15 @@ def test_pandas_instance_serialization(self): self.assertTrue(deserialized[pname] is None) else: test_df = getattr(test, pname) - non_date_cols = [c for c in test_df.columns if not is_datetime(test_df[c])] - self.assertTrue(test_df[non_date_cols].equals(deserialized[pname][non_date_cols])) + deser_df = deserialized[pname].copy() + + date_cols = [c for c in test_df.columns if is_datetime(test_df[c])] + if date_cols: + for c in date_cols: + src_tz = test_df.loc[0, c].tz + deser_df[c] = pd.to_datetime(deser_df[c]).dt.tz_convert(src_tz) + + self.assertTrue(test_df.equals(deser_df)) From f59a0228d9c199e35aef5fecb3548983f94507b4 Mon Sep 17 00:00:00 2001 From: "Carlson, Eric Thomas" Date: Thu, 3 Feb 2022 20:38:33 -0500 Subject: [PATCH 6/8] Remove redundant "if" statement --- tests/API1/testjsonserialization.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/API1/testjsonserialization.py b/tests/API1/testjsonserialization.py index fb8a8200b..bf8d569c8 100644 --- a/tests/API1/testjsonserialization.py +++ b/tests/API1/testjsonserialization.py @@ -203,10 +203,9 @@ def test_pandas_instance_serialization(self): deser_df = deserialized[pname].copy() date_cols = [c for c in test_df.columns if is_datetime(test_df[c])] - if date_cols: - for c in date_cols: - src_tz = test_df.loc[0, c].tz - deser_df[c] = pd.to_datetime(deser_df[c]).dt.tz_convert(src_tz) + for c in date_cols: + src_tz = test_df.loc[0, c].tz + deser_df[c] = pd.to_datetime(deser_df[c]).dt.tz_convert(src_tz) self.assertTrue(test_df.equals(deser_df)) From 5501d0e54e614f045e6fb91efa4d9c8e87fffc78 Mon Sep 17 00:00:00 2001 From: "Carlson, Eric Thomas" Date: Fri, 4 Feb 2022 08:21:11 -0500 Subject: [PATCH 7/8] Globally import json instead of inline --- param/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/param/__init__.py b/param/__init__.py index 1e66cb34b..8415f2ea3 100644 --- a/param/__init__.py +++ b/param/__init__.py @@ -24,6 +24,7 @@ import re import datetime as dt import collections +import json from .parameterized import ( Parameterized, Parameter, String, ParameterizedFunction, ParamOverrides, @@ -1567,14 +1568,12 @@ def _validate(self, val): def serialize(cls, value): if value is None: return 'null' - import json return json.loads(value.to_json(date_format='iso')) @classmethod def deserialize(cls, value): if value == 'null': return None - import json import pandas as pd return pd.read_json(json.dumps(value)) From 7e5098c1f1dec7c8f79d27b41cbb1f5f72dd916c Mon Sep 17 00:00:00 2001 From: maximlt Date: Wed, 12 Apr 2023 11:04:27 +0200 Subject: [PATCH 8/8] update tests --- tests/API1/testjsonserialization.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/API1/testjsonserialization.py b/tests/API1/testjsonserialization.py index b509de0df..39f22f71d 100644 --- a/tests/API1/testjsonserialization.py +++ b/tests/API1/testjsonserialization.py @@ -249,10 +249,10 @@ def test_pandas_instance_serialization(self): date_cols = [c for c in test_df.columns if is_datetime(test_df[c])] for c in date_cols: - src_tz = test_df.loc[0, c].tz - deser_df[c] = pd.to_datetime(deser_df[c]).dt.tz_convert(src_tz) + deser_df[c] = pd.to_datetime(deser_df[c]) + + pd.testing.assert_frame_equal(test_df, deser_df) - self.assertTrue(test_df.equals(deser_df)) def test_serialize_calendar_date_range_class(self): self._test_serialize(TestSet, 'ab')