Fix dates issue on windows

Author: Vasil Pashov

cpp/arcticdb/CMakeLists.txt

@@ -523,9 +523,9 @@ set(arcticdb_srcs
         util/offset_string.cpp
         util/sparse_utils.cpp
         util/string_utils.cpp
-        util/timer.cpp
         util/trace.cpp
         util/type_handler.cpp
+        util/format_date.cpp
         version/key_block.hpp
         version/key_block.cpp
         version/local_versioned_engine.cpp

cpp/arcticdb/pipeline/write_frame.cpp

@@ -20,6 +20,7 @@
 #include <arcticdb/stream/incompletes.hpp>
 #include <arcticdb/async/task_scheduler.hpp>
 #include <arcticdb/util/format_date.hpp>
+
 #include <vector>
 #include <array>
 #include <ranges>

cpp/arcticdb/util/format_date.cpp

@@ -0,0 +1,30 @@
+#define BOOST_DATE_TIME_POSIX_TIME_STD_CONFIG // Allows using nanoseconds in boost.
+#include <arcticdb/util/format_date.hpp>
+#include <boost/date_time/posix_time/time_formatters.hpp>
+#include <boost/date_time/posix_time/posix_time_duration.hpp>
+#include <boost/date_time/posix_time/conversion.hpp>
+
+namespace arcticdb::util {
+
+    std::string format_timestamp(const entity::timestamp ts) {
+        // Use boost as it can handle nanoseconds both on all OS's.
+        // std::std::chrono::time_point<std::chrono::system_clock> does not handle nanoseconds on Windows and Mac
+        const boost::posix_time::ptime epoch = boost::posix_time::from_time_t(0);
+        const boost::posix_time::ptime time = epoch + boost::posix_time::nanoseconds{ts};
+        // Custom formatting seems to work best compared to other options.
+        // * using std::put_time(std::gmtime(...)) throws on Windows when pre-epoch dates are used
+        // * using boosts time_facet requires the facet used for formatting to be allocated on the heap for each
+        //   formatting call (because it requires calling std::stringstream::imbue which takes onwership of the passed
+        //   pointer
+        return fmt::format(
+            "{}-{:02}-{:02} {:02}:{:02}:{:02}.{:09}",
+            int{time.date().year()},
+            int{time.date().month()},
+            int{time.date().day()},
+            time.time_of_day().hours(),
+            time.time_of_day().minutes(),
+            time.time_of_day().seconds(),
+            time.time_of_day().fractional_seconds()
+        );
+    }
+}

cpp/arcticdb/util/format_date.hpp

@@ -7,28 +7,11 @@
 
 #pragma once
 
-#include <fmt/format.h>
-#include <chrono>
-#include <ctime>
-#include <iomanip>
+#include <string>
+#include <arcticdb/entity/types.hpp>
 
 namespace arcticdb::util {
 
-using Clock = std::chrono::system_clock;
-using TimePoint = std::chrono::time_point<Clock>;
-
-inline std::string format_timestamp(entity::timestamp ts) {
-    std::stringstream ss;
-#if defined(_WIN32) || defined(__APPLE__)
-    auto time_point = Clock::time_point(std::chrono::duration_cast<std::chrono::seconds>(std::chrono::nanoseconds(ts)));
-#else
-    auto time_point = Clock::time_point(std::chrono::nanoseconds(ts));
-#endif
-    std::time_t t = Clock::to_time_t(time_point);
-    auto fraction = time_point - std::chrono::time_point_cast<std::chrono::seconds>(time_point);
-    ss << std::put_time(std::gmtime(&t), "%F %T.")
-       << std::chrono::duration_cast<std::chrono::milliseconds>(fraction).count();
-    return ss.str();
-}
+std::string format_timestamp(entity::timestamp ts);
 
 }

cpp/arcticdb/util/storage_lock.hpp

@@ -13,6 +13,7 @@
 #include <arcticdb/stream/index.hpp>
 #include <arcticdb/util/exponential_backoff.hpp>
 #include <arcticdb/util/configs_map.hpp>
+#include <arcticdb/util/format_date.hpp>
 
 #include <fmt/std.h>
 #include <mutex>
@@ -163,7 +164,12 @@ class StorageLock {
                 ts_ = 0;
                 log::lock().info("Lock timed out, giving up after {}", wait_ms);
                 mutex_.unlock();
-                throw StorageLockTimeout{fmt::format("Storage lock {} timeout out after {} ms. Lock held since {} (UTC)", name_, total_wait, date_and_time(*read_ts))};
+                throw StorageLockTimeout{fmt::format(
+                        "Storage lock {} timeout out after {} ms. Lock held since {} (UTC)",
+                        name_,
+                        total_wait,
+                        util::format_timestamp(*read_ts)
+                )};
             }
         }
         ts_ = create_ref_key(store);

cpp/arcticdb/util/test/test_format_date.cpp

@@ -10,15 +10,20 @@
 #include <arcticdb/util/format_date.hpp>
 
 TEST(FormatDate, ZeroTs) {
-    using namespace arcticdb;
-    ASSERT_EQ("1970-01-01 00:00:00.0", util::format_timestamp(0));
+    ASSERT_EQ("1970-01-01 00:00:00.000000000", arcticdb::util::format_timestamp(0));
+}
+
+TEST(FormatDate, PrependZero) {
+    ASSERT_EQ("2025-06-09 08:06:09.000000000", arcticdb::util::format_timestamp(1749456369000000000));
+    ASSERT_EQ("2025-06-09 08:06:09.000000001", arcticdb::util::format_timestamp(1749456369000000000 + 1));
+    ASSERT_EQ("2025-06-09 00:00:00.000000000", arcticdb::util::format_timestamp(1749427200000000000));
+}
+
+TEST(FormatDate, PreEpoch) {
+    ASSERT_EQ("1969-12-31 23:59:59.999999999", arcticdb::util::format_timestamp(-1));
+    ASSERT_EQ("1969-12-31 23:59:59.000000000", arcticdb::util::format_timestamp(-1'000'000'000));
 }
 
 TEST(FormatDate, April2821) {
-    using namespace arcticdb;
-#ifdef __linux__
-    ASSERT_EQ("2021-04-28 16:11:35.213", util::format_timestamp(1619626295213000000));
-#else
-    ASSERT_EQ("2021-04-28 16:11:35.0", util::format_timestamp(1619626295213000000));
-#endif
+    ASSERT_EQ("2021-04-28 16:11:35.213000000", arcticdb::util::format_timestamp(1619626295213000000));
 }

cpp/arcticdb/util/timer.cpp

@@ -292,7 +292,4 @@ arcticdb::ScopedTimerTotal timer2{#name, [&data](auto totals) { \
     std::copy(std::begin(totals), std::end(totals), std::back_inserter(data)); \
 }};
 
-[[nodiscard]] std::string date_and_time(int64_t ts);
-
-
 } //namespace arcticdb

cpp/arcticdb/util/timer.hpp

@@ -28,6 +28,8 @@
 #include <arcticdb/version/version_utils.hpp>
 #include <arcticdb/entity/merge_descriptors.hpp>
 #include <arcticdb/processing/component_manager.hpp>
+#include <arcticdb/util/format_date.hpp>
+
 #include <ranges>
 
 namespace arcticdb::version_store {
@@ -1262,9 +1264,9 @@ static void check_incompletes_index_ranges_dont_overlap(const std::shared_ptr<Pi
             sorting::check<ErrorCode::E_UNSORTED_DATA>(
                 !last_existing_index_value.has_value() || key.start_time() >= *last_existing_index_value,
                 "Cannot append staged segments to existing data as incomplete segment contains index value < existing data (in UTC): {} <= {}",
-                date_and_time(key.start_time()),
+                util::format_timestamp(key.start_time()),
                 // Should never reach "" but the standard mandates that all function arguments are evaluated
-                last_existing_index_value ? date_and_time(*last_existing_index_value) : ""
+                last_existing_index_value ? util::format_timestamp(*last_existing_index_value) : ""
             );
             auto [_, inserted] = unique_timestamp_ranges.emplace(key.start_time(), key.end_time());
             // This is correct because incomplete segments aren't column sliced
@@ -1273,7 +1275,7 @@ static void check_incompletes_index_ranges_dont_overlap(const std::shared_ptr<Pi
                     // -1 as end_time is stored as 1 greater than the last index value in the segment
                     inserted || key.end_time() -1 == key.start_time(),
                     "Cannot finalize staged data as 2 or more incomplete segments cover identical index values (in UTC): ({}, {})",
-                    date_and_time(key.start_time()), date_and_time(key.end_time()));
+                    util::format_timestamp(key.start_time()), util::format_timestamp(key.end_time()));
         }
 
         for (auto it = unique_timestamp_ranges.begin(); it != unique_timestamp_ranges.end(); it++) {
@@ -1283,10 +1285,11 @@ static void check_incompletes_index_ranges_dont_overlap(const std::shared_ptr<Pi
                         // -1 as end_time is stored as 1 greater than the last index value in the segment
                         next_it->first >= it->second - 1,
                         "Cannot finalize staged data as incomplete segment index values overlap one another (in UTC): ({}, {}) intersects ({}, {})",
-                        date_and_time(it->first),
-                        date_and_time(it->second - 1),
-                        date_and_time(next_it->first),
-                        date_and_time(next_it->second - 1));
+                        util::format_timestamp(it->first),
+                        util::format_timestamp(it->second - 1),
+                        util::format_timestamp(next_it->first),
+                        util::format_timestamp(next_it->second - 1)
+                );
             }
         }
     }
@@ -1903,8 +1906,8 @@ VersionedItem sort_merge_impl(
                 sorting::check<ErrorCode::E_UNSORTED_DATA>(
                     last_index_on_disc <= incomplete_start,
                     "Cannot append staged segments to existing data as incomplete segment contains index value {} < existing data {}",
-                    date_and_time(incomplete_start),
-                    date_and_time(last_index_on_disc)
+                    util::format_timestamp(incomplete_start),
+                    util::format_timestamp(last_index_on_disc)
                 );
             }
             pipeline_context->total_rows_ = num_versioned_rows + get_slice_rowcounts(segments);

cpp/arcticdb/version/version_core.cpp

@@ -15,8 +15,8 @@
 
 COLUMN_DTYPE = ["float", "int", "uint"]
 ALL_AGGREGATIONS = ["sum", "mean", "min", "max", "first", "last", "count"]
-MIN_DATE = np.datetime64('1969-06-01')
-MAX_DATE = np.datetime64('1970-06-01')
+MIN_DATE = np.datetime64('1969-01-01')
+MAX_DATE = np.datetime64('1971-01-01')
 
 pytestmark = pytest.mark.pipeline