Merge pull request #3197 from perspective-dev/date-timezone

Fix `date` type timezone error

Author: texodus

docs/md/how_to/python/table.md

@@ -79,3 +79,12 @@ Python `datetime` objects are serialized to strings before parsing. Naive
 `datetime` objects (without `tzinfo`) produce strings without timezone
 information and are therefore treated as UTC. Timezone-aware `datetime` objects
 include their offset in the serialized string, which is used to convert to UTC.
+
+`"date"` values are timezone-agnostic calendar days with no time component.
+They are _output_ as integer timestamps at _UTC midnight_ of the calendar day
+(equivalent to Arrow `date32` day arithmetic), and integer timestamp _input_ to
+a `"date"` column is likewise interpreted as UTC. The host process timezone
+never affects `"date"` values — a `Viewer` renders them in UTC, recovering the
+stored calendar day exactly. Datetime expression functions such as
+`bucket("x", 'D')`, `day_of_week("x")` and `hour_of_day("x")` also compute in
+UTC.

rust/perspective-js/test/js/constructors/date_timezone.spec.ts

@@ -0,0 +1,111 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+import { test, expect } from "@perspective-dev/test";
+import perspective from "../perspective_client";
+
+const JAN_1_2024_UTC = 1704067200000;
+
+test.describe("Date timezone invariance", function () {
+    let SAVED_TZ: string | undefined;
+
+    test.beforeEach(() => {
+        SAVED_TZ = process.env.TZ;
+        process.env.TZ = "Europe/Amsterdam";
+    });
+
+    test.afterEach(() => {
+        process.env.TZ = SAVED_TZ;
+    });
+
+    test("date strings serialize to UTC midnight epoch ms", async function () {
+        const table = await perspective.table({ d: "date" });
+        await table.update({ d: ["2024-01-01"] });
+        const view = await table.view();
+        const cols = (await view.to_columns()) as { d: number[] };
+        expect(cols.d).toEqual([JAN_1_2024_UTC]);
+        await view.delete();
+        await table.delete();
+    });
+
+    test("date epoch ms input round-trips exactly", async function () {
+        const table = await perspective.table({ d: "date" });
+        await table.update({ d: [JAN_1_2024_UTC] });
+        const view = await table.view();
+        const cols = (await view.to_columns()) as { d: number[] };
+        expect(cols.d).toEqual([JAN_1_2024_UTC]);
+        await view.delete();
+        await table.delete();
+    });
+
+    test("formatted output prints the stored calendar day", async function () {
+        const table = await perspective.table({ d: "date" });
+        await table.update({ d: ["2024-01-01"] });
+        const view = await table.view();
+        expect(await view.to_columns_string({ formatted: true })).toEqual(
+            '{"d":["2024-01-01"]}',
+        );
+        expect(await view.to_csv()).toEqual('"d"\n2024-01-01\n');
+        await view.delete();
+        await table.delete();
+    });
+
+    test("day_bucket stays on the UTC calendar day at the day boundary", async function () {
+        const table = await perspective.table({ t: "datetime" });
+
+        // 23:59 UTC is already the next day in Europe/Amsterdam
+        await table.update({ t: ["2020-01-31 23:59:00"] });
+        const view = await table.view({
+            expressions: { bucket: `bucket("t", 'D')` },
+        });
+
+        const cols = (await view.to_columns()) as { bucket: number[] };
+
+        // 2020-01-31T00:00:00Z
+        expect(cols.bucket).toEqual([1580428800000]);
+        await view.delete();
+        await table.delete();
+    });
+
+    test("day_of_week and hour_of_day compute in UTC", async function () {
+        const table = await perspective.table({ t: "datetime" });
+        await table.update({ t: ["2020-01-31 23:59:00"] });
+        const view = await table.view({
+            expressions: {
+                dow: `day_of_week("t")`,
+                hod: `hour_of_day("t")`,
+            },
+        });
+
+        const cols = (await view.to_columns()) as {
+            dow: string[];
+            hod: number[];
+        };
+
+        // Friday 23:00 UTC, not Saturday 00:59 Amsterdam
+        expect(cols.dow).toEqual(["6 Friday"]);
+        expect(cols.hod).toEqual([23]);
+        await view.delete();
+        await table.delete();
+    });
+
+    test("JSON date output agrees with Arrow date32 day arithmetic", async function () {
+        const table = await perspective.table({ d: "date" });
+        await table.update({ d: ["2024-01-01"] });
+        const view = await table.view();
+        const cols = (await view.to_columns()) as { d: number[] };
+        expect(cols.d[0] % 86400000).toEqual(0);
+        expect(cols.d[0] / 86400000).toEqual(19723); // days since epoch
+        await view.delete();
+        await table.delete();
+    });
+});

rust/perspective-python/docs/lib.md

@@ -153,15 +153,14 @@ table = perspective.table(data, index="index")
 
 #### Time Zone Handling
 
-When parsing `"datetime"` strings, times are assumed _local time_ unless an
-explicit timezone offset is parsed. All `"datetime"` columns (regardless of
-input time zone) are _output_ to the user as `datetime.datetime` objects in
-_local time_ according to the Python runtime.
-
-This behavior is consistent with Perspective's behavior in JavaScript. For more
-details, see this in-depth
-[explanation](https://github.com/perspective-dev/perspective/pull/867) of
-`perspective-python` semantics around time zone handling.
+When parsing `"datetime"` strings, times without an explicit timezone offset are
+interpreted as _UTC_; strings with an offset are converted to UTC. All
+`"datetime"` values are stored internally as milliseconds since the Unix epoch
+and are _output_ as integer timestamps from methods like `to_columns()` and
+`to_json()`. `"date"` values are timezone-agnostic calendar days, output as
+integer timestamps at _UTC midnight_ of the calendar day. The host process
+timezone never affects engine results; localization is the display layer's
+responsibility.
 
 ### Callbacks and Events
 

rust/perspective-python/perspective/tests/conftest.py

@@ -10,7 +10,7 @@
 #  ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
 #  ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
 
-from datetime import datetime, date
+from datetime import datetime, date, timezone
 
 import numpy as np
 import pandas as pd
@@ -165,10 +165,23 @@ def make_dictionary_arrow(names, data, types=None, legacy=False):
 
     @staticmethod
     def to_timestamp(obj):
-        """Return an integer timestamp based on a date/datetime object."""
+        """Return an integer timestamp based on a date/datetime object.
+
+        A `date` is a timezone-agnostic calendar day and serializes to epoch
+        ms at *UTC* midnight regardless of the host process timezone, so it
+        is converted here in UTC. A naive `datetime` is a wall-clock reading
+        in the process-local timezone (callers express expected instants
+        this way, e.g. `aware.astimezone(TZ).replace(tzinfo=None)`), so it is
+        converted via local `timestamp()`.
+        """
         classname = obj.__class__.__name__
         if classname == "date":
-            return int(datetime(obj.year, obj.month, obj.day).timestamp() * 1000)
+            return int(
+                datetime(
+                    obj.year, obj.month, obj.day, tzinfo=timezone.utc
+                ).timestamp()
+                * 1000
+            )
         elif classname == "datetime":
             return int(obj.timestamp() * 1000)
         else: