diff --git a/fixtures/server_byte_exact/06_number_format_regimes.json b/fixtures/server_byte_exact/06_number_format_regimes.json new file mode 100644 index 00000000..f8dd9500 --- /dev/null +++ b/fixtures/server_byte_exact/06_number_format_regimes.json @@ -0,0 +1,71 @@ +{ + "config": { + "pipeline_config": { + "operators": { + "compute": { + "type_name": "transform_by_lua", + "lua_script": "function f()\n return item_score * 2\nend", + "function_for_item": "f", + "function_for_common": "", + "$metadata": { + "common_input": [], + "common_output": [], + "item_input": [ + "item_score" + ], + "item_output": [ + "doubled" + ] + } + } + } + }, + "pipeline_group": { + "main": { + "pipeline": [ + "compute" + ] + } + }, + "flow_contract": { + "common_input": [], + "item_input": [ + "item_score" + ], + "common_output": [], + "item_output": [ + "item_score", + "doubled" + ] + } + }, + "request": { + "common": {}, + "items": [ + { + "item_score": 5e+19 + }, + { + "item_score": 5e+20 + }, + { + "item_score": 7.5e+20 + }, + { + "item_score": 5e-08 + }, + { + "item_score": 5e-07 + }, + { + "item_score": 5.000000000000001e+19 + }, + { + "item_score": 5000000000000000.0 + }, + { + "item_score": -5e+19 + } + ] + } +} diff --git a/llmdoc/architecture/dag-engine.md b/llmdoc/architecture/dag-engine.md index cfe43f4d..d879d3db 100644 --- a/llmdoc/architecture/dag-engine.md +++ b/llmdoc/architecture/dag-engine.md @@ -812,18 +812,27 @@ Pine-Java 注册全部内置算子（`AllOperators.java`），与 Pine-Go `pine- ### 跨运行时格式兼容（GoFormat） -`GoFormat.java` 提供静态方法复制 Go 标准库数值格式化行为： +`GoFormat.java` 提供静态方法复制 Go 标准库数值格式化行为。它有**四个**格式化入口，各自对应 Go 侧不同的函数、阈值不同、**不可互换**： - `sprint(Object)` — 等效 Go `fmt.Sprint`；nil → `""`，magnitude < 1e6 的整数值 float → 无小数点（阈值 1e6 匹配 Go 切换科学计数法的边界） - `formatFloatF(double)` — 等效 Go `strconv.FormatFloat(d, 'f', -1, 64)` - `formatG(double)` — 等效 Go `fmt.Sprintf("%g", d)`；保留完整精度 +- `formatJsonNumber(double)` — 等效 Go `encoding/json` 对 float64 的输出，即 `strconv.FormatFloat(d, 'e'|'f', -1, 64)`：`|x| < 1e-6` 或 `|x| >= 1e21` 走 `'e'`（科学计数），否则走 `'f'`（平铺小数）；precision `-1` 表示最短往返表示。这条路径**只服务 JSON 序列化**，与上面三个入口没有调用关系 - magnitude ∈ [1e6, 1e7) 时 `formatG` 将科学计数法表示转换为定点表示，匹配 Go `%g` 在该区间的输出 - `sprint` 支持 `List` 和数组类型，输出 `"[a b c]"` 空格分隔格式（匹配 Go `fmt.Sprint` 对 slice 的行为） - `formatG` 将 `Infinity` / `-Infinity` 输出为 `"+Inf"` / `"-Inf"`（Go 惯例） - `formatG` 对 magnitude ∈ [1e-4, 1e-3) 的小数通过 `BigDecimal.toPlainString()` 转换为定点表示 - `sprint`、`formatFloatF`、`formatG` 均保留 `-0.0` 的符号位（输出 `"-0"` 而非 `"0"`），通过 `Double.doubleToRawLongBits` 在各自的整数快捷路径前检测 -消费者：`TransformResourceLookup`（key coerce）、`TransformRedisGet`（key 拼接）、`FilterCondition`（条件比较值格式化，替代旧的 `formatValue` 方法）、`ReorderShuffle`（salt 格式化，替代旧的 `formatFloatG` 方法）。第六轮 parity 审计中移除了 `FilterCondition.formatValue` 和 `ReorderShuffle.formatFloatG`，统一使用 `GoFormat` 作为跨算子格式化单一事实源。 +前三个入口的消费者：`TransformResourceLookup`（key coerce）、`TransformRedisGet`（key 拼接）、`FilterCondition`（条件比较值格式化，替代旧的 `formatValue` 方法）、`ReorderShuffle`（salt 格式化，替代旧的 `formatFloatG` 方法）。第六轮 parity 审计中移除了 `FilterCondition.formatValue` 和 `ReorderShuffle.formatFloatG`，统一使用 `GoFormat` 作为跨算子格式化单一事实源。 + +`formatJsonNumber` 的消费链是**独立的一条**：`GoFormat.createGoCompatMapper()` 里注册的 Jackson `Double` 序列化器调用它，该 mapper 的消费者是 `RunCli`（CLI 输出）、`PineServer`（HTTP `/execute` 等响应体）与 `MetricsCollectorTest`（测试侧）。也就是说 `/execute` 响应里的数字字面量**一条都不经过 `sprint` / `formatFloatF` / `formatG`**。 + +这个分裂正是 issue #180 能长期存在的条件：读文档的人会以为 GoFormat 是格式化的单一事实源，而 JSON 数字实际走的是序列化器自带的一套 double 逻辑；讽刺之处在于 `formatFloatF` 对 1e20 本来就给出正确答案（内部有 `BigDecimal.toPlainString` 平铺），只是 JSON 路径从来没调它。修复后的契约： + +- 序列化器对**所有** `Double` 值（含 `0` 与 `-0.0`）都走 `writeRawValue(formatJsonNumber(...))`，不再有任何一条分支落回 Jackson `writeNumber` +- 测试 `GoJsonNumberParityTest.formatJsonNumberMatchesTheSerializer` 钉住「序列化器不得持有格式化规则的第二份拷贝」 +- 跨语言等价实现见 `pine-cpp/src/config/json.cpp`（`go_format_json_number`）；两处实现共同依赖的实测事实见 `llmdoc/reference/number-formatting-parity.md` ### 资源管理 diff --git a/llmdoc/guides/ci-quality-baseline.md b/llmdoc/guides/ci-quality-baseline.md index d3adb960..59a32267 100644 --- a/llmdoc/guides/ci-quality-baseline.md +++ b/llmdoc/guides/ci-quality-baseline.md @@ -189,6 +189,34 @@ Nightly diff-fuzz artifact 分歧定位顺序：(a) 下载 artifact，解压 `di 判据：本地复现走弯路超过 30 分钟时，条件反射式切"从末端逐算子截断"策略，不要继续深挖末端错误路径。 +### 校验通道能钉住的属性（归一化 vs 字节级） + +差分 fuzz 与 cross-validate 大部分通道在比对前做**归一化**，因此有整类属性对它们结构上不可见。新增契约时必须先问「哪条通道会红」，而不是「测试是否全绿」。issue #180（JSON 数字格式跨运行时分歧）暴露的通道能力如下。 + +**differential-fuzz：归一化抹掉 key 顺序与绝大多数数字字面量差异。** `scripts/differential-fuzz.py` 的 `normalize_json` 做 `json.loads` → `_normalize_value` → `json.dumps(sort_keys=True)`。`sort_keys=True` 使 key 顺序整个维度不可见（issue #183 因此从未被抓到）；`_normalize_value` 只对 `float` 分支做 `round(v, 10)` 与小量级归零，`int` 分支原样穿过。 + +**#180 能被 fuzz 报出来靠的是 Python 的 int/float 类型分裂，不是设计出来的检出能力**：Go 输出 `100000000000000000000` 被 `json.loads` 解析成 `int`（原样穿过），Java 输出 `1.0E20` 解析成 `float` 再 re-dump 成 `1e+20`，两串才不相等。推论：**只有至少一侧输出整数形状字面量（无小数点无指数）时，数字格式分歧才可见**。实测的可见性分档： + +| 分歧 | 归一化后可见 | +|------|------| +| Go `100000000000000000000` vs Java `1.0E20` | 可见 | +| Go `100000000000000020000` vs C++ `100000000000000016384` | 可见 | +| Go `9007199254740992` vs Java `9.007199254740992E15` | 可见 | +| Go `0.0000001` vs Java `1.0E-7` | 不可见 | +| Go `1e+21` vs Java `1.0E21` | 不可见 | +| C++ `1e-07` vs Go `1e-7` | 不可见 | +| 第 11 位起的精度差（`1.2345678901234567` vs `...68`） | 不可见（`round(v,10)` 抹掉） | + +#180 实际有 15 个分歧，fuzz 结构上只能看见其中一部分。 + +**`scripts/cross-validate/09-raw-byte.sh` 标题写 "no normalization"，实际有回落。** 字节比较失败后会用 `normalize_json` 再比一次，相等就打 `[W]` 警告并**计为 pass**（`09-raw-byte.sh:115-126`）。这是 key 顺序差异被有意容忍的地方，同时也意味着它不能钉住字节级数字格式。 + +**`scripts/cross-validate/14-byte-exact-execute.sh` 是唯一真字节通道**：curl 响应体直接 `==`，无任何回落。#180 给它补了 `fixtures/server_byte_exact/06_number_format_regimes.json`，覆盖 Go 各个格式化区间（输入 doubled 后分别落在 1e20 / 1e21 / 1.5e21 / 1e-7 / 1e-6 / 最短往返差异 / 1e16 / -1e20）。双向 mutation 验证过有牙：Java 序列化器改回 `writeNumber` → Go-vs-Java 变红；C++ 改回 `chars_format::fixed` → Go-vs-C++ 变红。 + +**原有的 `04_number_precision.json` 名字看起来正好覆盖数字精度，实际不可能抓到 #180**：输入 `100000 / 1000001 / 0.5`，×2 后全部落在 ±2^53 内的整数值区间——恰好是 Java 旧代码唯一处理对的区间。一个名叫 `number_precision` 却漏掉所有真正分歧量级的 gate，比没有 gate 更糟：它读起来像已覆盖。 + +**纪律：声称「字节级对等」的属性，必须有一条不做任何归一化的通道覆盖。** 归一化通道只能证明「语义等价」，不能证明字节等价。这与 `guides/cross-layer-validation.md` 的「fixture 比对器语义决定该层能钉住的属性」是同一条原则。当前字节通道覆盖面窄的待决策项记在 `llmdoc/memory/doc-gaps.md`。 + ### Daily sanitized-fuzz（ASan/TSan 深度诊断） `.github/workflows/daily-sanitized-fuzz.yml` 每日 schedule 运行 pine-cpp 的 ASan+UBSan 与 TSan 两个 sanitizer-instrumented differential-fuzz pass，复用同一份 `scripts/differential-fuzz.py`： @@ -341,6 +369,8 @@ Pine-Java 通过 Sonatype Central Portal 发布到 Maven Central（release profi - Differential-fuzz 脚本：`scripts/differential-fuzz.py`、`scripts/differential-fuzz.sh` - DAG differential-fuzz 脚本：`scripts/dag-differential-fuzz.py` - Cross-validate section 列表：`scripts/cross-validate/` +- Cross-validate raw-byte（带归一化回落）：`scripts/cross-validate/09-raw-byte.sh` +- Cross-validate 唯一真字节通道：`scripts/cross-validate/14-byte-exact-execute.sh`、`fixtures/server_byte_exact/` - Cross-validate metrics-parity section：`scripts/cross-validate/13-metrics-parity.sh` - Cross-validate pine-cpp 预构建：`scripts/cross-validate/_prebuild.sh` - 跨引擎 benchmark：`scripts/cross-engine-bench.py`、`scripts/cross-engine-bench-cli.sh`、`scripts/bench-generate-fixtures.py` diff --git a/llmdoc/index.md b/llmdoc/index.md index e92906fa..401106d7 100644 --- a/llmdoc/index.md +++ b/llmdoc/index.md @@ -12,14 +12,14 @@ ## architecture/ -- `llmdoc/architecture/dag-engine.md` — 核心引擎架构：配置编译流水线、DAG 推导规则（三标记 + auto-inject 模型：ConsumesRowSet/MutatesRowSet/AdditiveWritesRowSet 标记与 item 字段自动注入）、调度模型、DataFrame 语义（含 InputFieldSpec 三态模型：Nullable/Strict/Defaulted）、算子类型约束、行集依赖行为，以及引擎级 option / 根级配置注入（含 debug nullable 三态继承）、Server struct 生命周期与 context 传播、服务端 reload 集成与 HTTP middleware 包装边界、双通道运行时观测、ExecutionError/PanicError 因果链（三运行时 cause chain parity）、资源数据型（snapshot 导出）/句柄型（borrow 借用，如 redis_connection）区分、Pine-Java 完整功能对等描述、接受的跨引擎设计差异归档（如 issue #91 Lua VM pool 上限/GC 回收语义不对等：指标层 5 元组对等 + 端到端 calibrated 持平 + 生产无 OOM 痛点 + 跨语言机制无可移植近似 → 接受差异，重启触发条件为生产 RSS 单调爬升数据；issue #169 Java routeHandler 抛/返回二分 vs Go 统一 err 传 Egress）、跨运行时 operator-visible input 排除集合契约（值访问、init 字段名元数据、debug/trace 快照三个消费面，issue #174）。 +- `llmdoc/architecture/dag-engine.md` — 核心引擎架构：配置编译流水线、DAG 推导规则（三标记 + auto-inject 模型：ConsumesRowSet/MutatesRowSet/AdditiveWritesRowSet 标记与 item 字段自动注入）、调度模型、DataFrame 语义（含 InputFieldSpec 三态模型：Nullable/Strict/Defaulted）、算子类型约束、行集依赖行为，以及引擎级 option / 根级配置注入（含 debug nullable 三态继承）、Server struct 生命周期与 context 传播、服务端 reload 集成与 HTTP middleware 包装边界、双通道运行时观测、ExecutionError/PanicError 因果链（三运行时 cause chain parity）、资源数据型（snapshot 导出）/句柄型（borrow 借用，如 redis_connection）区分、Pine-Java 完整功能对等描述、接受的跨引擎设计差异归档（如 issue #91 Lua VM pool 上限/GC 回收语义不对等：指标层 5 元组对等 + 端到端 calibrated 持平 + 生产无 OOM 痛点 + 跨语言机制无可移植近似 → 接受差异，重启触发条件为生产 RSS 单调爬升数据；issue #169 Java routeHandler 抛/返回二分 vs Go 统一 err 传 Egress）、跨运行时 operator-visible input 排除集合契约（值访问、init 字段名元数据、debug/trace 快照三个消费面，issue #174）、GoFormat 四个格式化入口与各自独立的消费链（`sprint`/`formatFloatF`/`formatG` 供算子层，`formatJsonNumber` 单独服务 JSON 序列化并经 `createGoCompatMapper` → `RunCli`/`PineServer`；四者阈值不同不可互换，`/execute` 数字字面量一条都不走前三个，issue #180）。 - `llmdoc/architecture/apple-compiler.md` — Python DSL 架构：Flow 声明 API、SubFlow 契约声明与编译期强制（`common_input`/`common_output`/`item_input`/`item_output` 在 issue #78 落地为 subtree-scoped 字段覆盖 + 死代码校验，未声明契约的 SubFlow 自动继承外层；`required_resources` 沿用 issue #37 校验）、编译流水线（含 step 8b `_validate_subflow_contracts`）、校验规则（含 `validate_write_without_read` 对 `AdditiveWritesRowSet` 算子的同字段豁免，issue #72）、控制流降级（含 `_rename_field` Lua `_G[]` 语法处理）、资源声明处理、根级配置字段扩展路径（如 `storage_mode`、`log_prefix`、`debug`），以及 row-set 标记三元组（`consumes_row_set` / `mutates_row_set` / `additive_writes_row_set`）通过 `apple_generated/markers.py` 表填充 `OpCall`、True-OR widen 合并语义、`_apply` 与 `_add_op` 的刻意非对称（仅 `consumes_row_set` 暴露给 DSL 调用点）。 - `llmdoc/architecture/pine-cpp-runtime.md` — Pine-C++ 运行时架构：作为标杆运行时的定位、错误/fixture parity 契约、CLI 与 HTTP 入口（含 HTTP/1.1 keep-alive / read-header-timeout / idle-timeout / max-body-size / middleware / graceful shutdown / 客户端断连取消 eventfd 零延迟唤醒 / custom Route 与 Watch 的"黑盒行为对等、实现结构自由"决策——不做 Go/Java 嵌入 API、可测校验逻辑抽 socket-free routes.cpp）、codegen 入口（`-schema-json` schema 导出 + `-output` 发射完整 Apple DSL 产物集与 Go/Java 字节级一致 + `-doc-dir` 发射算子文档 markdown 与 pine-go byte-equal、`OperatorSchema.metadata` 字段显式声明、`format_g` 对 |d| > LLONG_MAX 的 UB 守卫与 Ryu/Grisu 路由点、ResourceSchema 全局注册表与 `reset_resource_schema_registry`/`reset_all_resource_registries` 拆分语义）、`metrics::Provider` 与 `resource::Manager` 对等（`ResourceValue` 数据 `Variant` XOR 句柄 `shared_ptr` 双通道，数据型走 `snapshot()`、句柄型走 `borrow()`，RAII 拆除）、Frame 多态基类 + ColumnFrame/RowFrame 双物理实现（C++23，per-call 锁形态与 Go/Java 对齐、`pine::SharedMutex` 备件）、Column 类型层级、`PINE_REGISTER_OPERATOR_T` 注册模型、ValidateOutput 类型约束、NaN/Inf 校验、PanicError stacktrace、外部 stop_token 取消、ready-queue DAG 调度器（双隔离线程池 + in-degree 原子追踪）、observe_log/pine-debug 日志、Redis client 失败收敛与 SIGPIPE 守卫（`MSG_NOSIGNAL` + AUTH/SELECT close fd）+ per-command 指标 `run_command` 模板（错误类型分层 known follow-up）、`OperatorOutput` 缓冲区复用（`node_body` 用 `thread_local` 而非对象池——ready-queue 不迁移半完成节点故省掉 Get/Put 记账；acquire 与 node body 尾部**两侧都 reset**，承重的是尾部那次（在 `try/catch` 之外、成功与抛异常都覆盖），acquire 侧现为 defence in depth；`reset()` 用 `clear()` 保住 heap capacity 并对超 `kRetainLimit` 的容器释放；spine 上限不含元素 payload，issue #122）。 ## guides/ - `llmdoc/guides/standard-workflow.md` — 标准工作流程：llmdoc 加载、plan mode 对齐、任务跟踪、逐步验证、文档同步、review-driven scope expansion 接受、并行 worker 改动收集（worktree diff→apply 收集模式 + 新公开入口/状态机修复需重扫整个状态机：可达性、并发交错、发布顺序 + 新公开入口上线后审计既有全局状态的归属：sync.Once/静态 CAS/System property/全局 setter 在多实例下该属于谁） + 示例代码的契约要求：examples/ 受全部生产契约约束、文档命令须真实执行过、冒烟覆盖负空间（含正面案例：`pine-go/benchmarks/` 是独立 module，实测才拦住 `cd pine-go && go test ./benchmarks/` 这条错误复现命令）、示例纳入默认构建防 rot；另含三条硬规则：临时改动/mutation 期间禁用一切 git 恢复只用文件级备份 `cp`（取代并加强旧建议 `git checkout -- `，同型事故第二次）、`git add ` 是单域隔离纪律的粒度漏洞（同一文件承载多域改动时先做完一件再提交，混了用 `reset --soft` + 备份逐次施加补救）、动长期无人维护的文件要预期承担整文件既存 lint 债（pre-commit 是 staged-file 粒度非 diff 粒度，清理单独 commit）。 -- `llmdoc/guides/ci-quality-baseline.md` — CI 工程质量基线：lint（含 Java checkstyle `failOnViolation=true` + `OneStatementPerLine`；C++ `cpp-lint` job 实际只做 `-Werror` 严格构建 + whitespace/tab/结尾换行卫生 + 相邻字面量拼接排查，clang-format 无 CI job、纯本地约定）/ test / coverage / fuzz / differential-fuzz（含新增 fuzz 维度必须验证信号到达比对面：有效可见率 vs 形状出现率、flow_contract 投影盲区教训、red-before/green-after 验证配方、mutation 验证两步判据：先证明 mutation 真的改变了被测语义再判断测试有没有牙，「测试没红」的第一解释是 mutation 无效——`vector = {}` 走 `operator=(initializer_list)` → `assign()` 从不缩容、与 `clear()` 容量语义等价，须用真移动赋值） / daily sanitized-fuzz（ASan/TSan 深度诊断，与 nightly Release 10k 轮吞吐互补，`--time-budget-seconds` 内层 pacing + 外层纯 hang 保护两层 timeout 设计） / cross-validate / nightly cross-runtime benchmark / release-gate 架构与接入约定（含 pine-cpp 的 4 个 CI job 与 cross-validate cpp 二进制注入路径），统一任务入口 Makefile 体系（顶层 + `pine-go/` Makefile 封装跨四语言 fmt/lint/test/bench/codegen/版本管理，CI 与本地共用同一命令序列、`make bench` 默认 `pine_bench` tag），CI apt 依赖安装约定（`scripts/ci-apt-install.sh` 重试 wrapper + 包清单瘦身，防慢镜像单发击穿，历史 #125/#164），以及本地 `.githooks/` 体系（`pre-commit` staged-only 格式 gate + `pre-push` 工程级 lint + 自包装 CI watch）、differential-fuzz artifact triage playbook：末端错误常是下游放大点、从 pipeline 尾部向前截断定位真实 frame 分歧 op。 +- `llmdoc/guides/ci-quality-baseline.md` — CI 工程质量基线：lint（含 Java checkstyle `failOnViolation=true` + `OneStatementPerLine`；C++ `cpp-lint` job 实际只做 `-Werror` 严格构建 + whitespace/tab/结尾换行卫生 + 相邻字面量拼接排查，clang-format 无 CI job、纯本地约定）/ test / coverage / fuzz / differential-fuzz（含新增 fuzz 维度必须验证信号到达比对面：有效可见率 vs 形状出现率、flow_contract 投影盲区教训、red-before/green-after 验证配方、mutation 验证两步判据：先证明 mutation 真的改变了被测语义再判断测试有没有牙，「测试没红」的第一解释是 mutation 无效——`vector = {}` 走 `operator=(initializer_list)` → `assign()` 从不缩容、与 `clear()` 容量语义等价，须用真移动赋值） / daily sanitized-fuzz（ASan/TSan 深度诊断，与 nightly Release 10k 轮吞吐互补，`--time-budget-seconds` 内层 pacing + 外层纯 hang 保护两层 timeout 设计） / cross-validate / nightly cross-runtime benchmark / release-gate 架构与接入约定（含 pine-cpp 的 4 个 CI job 与 cross-validate cpp 二进制注入路径），统一任务入口 Makefile 体系（顶层 + `pine-go/` Makefile 封装跨四语言 fmt/lint/test/bench/codegen/版本管理，CI 与本地共用同一命令序列、`make bench` 默认 `pine_bench` tag），CI apt 依赖安装约定（`scripts/ci-apt-install.sh` 重试 wrapper + 包清单瘦身，防慢镜像单发击穿，历史 #125/#164），以及本地 `.githooks/` 体系（`pre-commit` staged-only 格式 gate + `pre-push` 工程级 lint + 自包装 CI watch）、differential-fuzz artifact triage playbook：末端错误常是下游放大点、从 pipeline 尾部向前截断定位真实 frame 分歧 op，以及「校验通道能钉住的属性（归一化 vs 字节级）」：fuzz `normalize_json` 的 `sort_keys=True` 抹掉 key 顺序整维度、`round(v,10)` + int/float 类型分裂使数字格式分歧只在至少一侧输出整数形状字面量时可见（附 8 类分歧可见性实测表）、`09-raw-byte.sh` 标题写 "no normalization" 实际有归一化回落并把差异降级为 `[W]` 计入 pass、`14-byte-exact-execute.sh` 是唯一真字节通道但 fixture 少（`04_number_precision.json` 名字像覆盖精度实则只覆盖已对的 ±2^53 整数区间），配纪律「声称字节级对等的属性必须有一条不做任何归一化的通道覆盖」。 - `llmdoc/guides/investigation-to-fix-testing.md` — 从调查到修复的测试策略：按缺陷类型选择测试层、最小修复面原则、跟进上游 issue 与临时止血方法论（跨 issue 根因归属不顺 follow-up 措辞、临时止血阈值用 probe 实测标定）。 - `llmdoc/guides/cross-layer-validation.md` — 跨层语义校验：JSON 边界类型枚举、codegen 语义验证（含跨引擎 markdown / Python 产物 byte-equal gate）、边界值 E2E、隐含 metadata 契约检测、扩展点对等验证（能力等价 + 编程扩展点的 demo-routes 黑盒验证模式 + 执行路径可观测副作用增量比对 + 可选测试臂 fail-closed）、fixture 比对器语义决定该层能钉住的属性（值级对等 vs 类型身份的分层边界，层边界应显式写下）。 - `llmdoc/guides/benchmark-hygiene.md` — Benchmark 噪声卫生：跑前/跑后 load 与残留进程检查、同日同机对照纪律、±5-7% 二进制布局噪声与 perf stat 交叉验证、calibrated stddev 33-36ms 来源校准（DAG 调度抖动 + LuaJ JIT warmup + 网络抖动主导，GC pause 非主要源）、fixture 代表性（calibrated 为性能决策唯一裁判）、合成 guardrail fixture 与 calibrated 的分工（`transform_heavy_1000` 钉死 `storage_mode: column` 补列存批量列访问路径无 nightly 守护者的缺口，只守「整体崩了」量级、细粒度回归归 `BenchmarkStorageAB_TransformHeavy_*`，其 delta 不得作为收益声明；issue 标题措辞不是规格——#160 标题误写 calibrated）、把 in-process microbench 形状搬成 e2e fixture 要重查投影/序列化段（空 `item_output` 让 `projectMap` 投影成 `{}`、抹掉序列化成本、transform 链变死写入）、microbench 访问模式戒律、逐 op 删除归因法、测量路径对称性（PureVM vs CallOnly vs Boundary 不可互推）、用户可见文档不写性能倍数只写定性判据 + 可复现入口。 @@ -27,6 +27,7 @@ ## reference/ - `llmdoc/reference/operator-contract.md` — 算子开发参考：接口、Schema 注册契约、批量列访问 API 契约（`ItemColumn`/`itemColumn`/`item_column` 三引擎对照、与逐元素 `Item()` 含 defaults 语义一致、只读/仅当次 Execute 有效、扫描型热循环优先批量）、批量列写 API 契约（`SetItemColumnFloat64`/`setItemColumnDouble`/`set_item_column_double`、stage 2b 顺序语义列写覆盖逐元素写、整列或全无、NaN 批量校验消息对等、列存 adopt 零拷贝/行存 scatter、所有权转移）、可选的 metadata/debug/logger/metrics/stats 钩子、算子日志规范（`LoggerAware` 三引擎入口 + 引擎实例级 log_prefix + 用户可控字符串绝不拼进 printf 格式串）、类型/输出限制、保留 JSON 键、命名规范、网络调用安全约束（SSRF 防护、LimitReader、fail_on_error 模式）、Redis 算子句柄型资源借用契约（`transform_redis_get`/`transform_redis_set` 按 `resource_name` 借用 `redis_connection`、借用失败静默降级；`redis_connection` cascade-safety 五参数：`{dial,read,write,pool}_timeout_ms` + `pool_size`、Jedis socketTimeout 折叠 max(read,write)、cpp pool_timeout_ms no-op；`metrics_name` 资源级指标开关；failed-path 静默降级审计契约——新增 client/资源失败路径必须走完 `fail_on_error=false → connected()==false` 链路）、Lua bridge 标量分派必须用真实类型标签而非 coercion 谓词（luaj `is*` = coercion vs gopher-lua/wangshu = 标签，issue #175）、Lua pool baseline reset 仅覆盖字符串键 globals 契约（wangshu godoc 权威表述、三运行时一致、issue #177 修 Java `snapshotKeys` coercion 谓词残留）。 +- `llmdoc/reference/number-formatting-parity.md` — 跨语言数值格式化的两条实测事实（复刻 Go double 输出时必读）：C++ `std::to_chars` 只有 `chars_format::scientific` 保证最短往返（`chars_format::fixed` 与**不带 format 参数的默认 overload** 在量级大到不需要指数时都退化成精确打印，`1.0000000000000002e20` → `100000000000000016384`），故 pine-cpp `go_format_json_number` 统一从 scientific 取数字再自己摆小数点、两种输出形式共用同一数字来源；Go `encoding/json` 相对 `strconv.FormatFloat(d,'e',-1,64)` **只对负指数去掉一个前导零**（`1e-07`→`1e-7`、`1e-09`→`1e-9`，而 `1e+21` / `1e+100` / `1e-100` 都不动），靠推理必写错。含「跨语言等价函数隐含语义不等价、只能实测」的纪律与 "verified against X rather than inferred" 注释要求（issue #180）。 - `llmdoc/reference/apple-control-template-syntax.md` — Apple DSL 控制流条件参考：`if_` / `elseif_` 需要使用 `{{field_name}}` 模板语法显式标记字段引用，编译器据此提取依赖并在发射 Lua 前去掉模板标记。 - `llmdoc/reference/metrics-observability.md` — 可插拔观测参考：跨运行时 `Provider` 契约（pine-go 规范 + pine-cpp/pine-java 对等）、引擎/调度器/Lua pool 指标注入、`/stats` 组合响应（含 `/stats.http` 与 `/stats.resources` 子树 schema）、内置 HTTP metrics middleware（各运行时 default-on）、资源级指标 fan-out（Tee）路由与 Collector 契约（4 个连接池/探针指标 + 2 个 per-command 指标 `pine_redis_command_*`、status taxonomy ok/timeout/pool_timeout/error、lifecycle 命令过滤、cpp 错误类型分层 known follow-up）、Prometheus 适配边界。 - `llmdoc/reference/dag-visualization.md` — DAG 可视化参考：`RenderDAG` / `WithCollapse` API、SubFlow 折叠规则、`GET /dag` 参数与 DOT/Mermaid 输出约定。 @@ -37,7 +38,7 @@ `memory/` 下有 `reflections/` 与 `decisions/` 两个子目录（各自分节列在下方），以及下列直接位于顶层的文件： -- `llmdoc/memory/doc-gaps.md` — 跨任务累积的文档与质量检查缺口跟踪（已确认存在、不属任何单次任务、需单独排期决策的条目；与 reflections「单次任务教训」和 decisions「已定下的取舍」分工）：clang-format 无 CI job（`cpp-lint` 只做 `-Werror` + 卫生检查 + 字面量拼接排查，格式仅由可绕过的本地 pre-commit hook 守，待决策是否加 fmt-check job 并锁版本）、`projectMap` 空列表投影语义在 fixture 编写视角无落点（同一陷阱已三次现身，待决策是否给 `reference/` 加独立契约条目）、issue #179 `storage_mode` 非法值兜底跨运行时分歧（已知分歧，未修，待决策修还是归档为 accepted difference）。 +- `llmdoc/memory/doc-gaps.md` — 跨任务累积的文档与质量检查缺口跟踪（已确认存在、不属任何单次任务、需单独排期决策的条目；与 reflections「单次任务教训」和 decisions「已定下的取舍」分工）：clang-format 无 CI job（`cpp-lint` 只做 `-Werror` + 卫生检查 + 字面量拼接排查，格式仅由可绕过的本地 pre-commit hook 守，待决策是否加 fmt-check job 并锁版本）、`projectMap` 空列表投影语义在 fixture 编写视角无落点（同一陷阱已三次现身，待决策是否给 `reference/` 加独立契约条目）、issue #179 `storage_mode` 非法值兜底跨运行时分歧（已知分歧，未修，待决策修还是归档为 accepted difference）、字节级对等校验通道覆盖面太窄（14 号通道仅 5 个 fixture 而字节对等是全局契约，待决策是扩 fixture 还是取消 09 号通道的归一化回落——后者是 issue #183 的前置条件）、issue #183 Java object key 插入顺序 vs Go 排序（已开 issue、**未修**，含 Go UTF-8 字节序 vs Java `String.compareTo` UTF-16 code unit 序的实现陷阱与 pine-cpp 侧未比对）。 ## memory/reflections/ @@ -123,6 +124,7 @@ - `llmdoc/memory/reflections/lua-type-tag-dispatch-and-fuzz-blindspot.md` — issue #175 复盘：luaj coercion 谓词 vs 真实类型标签分派、fuzzer flow_contract 投影盲区（值级 bug 对差分比较不可见）、有效可见率 vs 形状出现率、fixture 比对器分层边界、snapshotKeys 预存问题待开 issue。 - `llmdoc/memory/reflections/cpp-output-pool-and-storage-mode-guardrail.md` — issue #122（pine-cpp `OperatorOutput` thread_local 复用 + 两侧 reset 与容量上限的多轮收敛过程）+ issue #160（合成列存护栏 fixture `transform_heavy_1000` 与 `storage_mode` 用户文档）联合复盘，另修三处 nightly benchmark 接线缺陷（报表路径不匹配致 artifact 长期为空但 job 全绿 / `--modes` 空转 / `bench-compare` 列数过期）。核心教训：mutation 没红的第一解释应是「mutation 无效」而非「测试没牙」（`vector = {}` 走 `assign()` 不缩容，与 `clear()` 容量语义等价）、mutation 期间禁用 git 恢复只能用文件备份 `cp`（同型事故第二次）、同一文件多域改动时 `git add ` 是单域隔离漏洞、clang-format 在 CI 里没有对应 job（真实覆盖缺口）、用户文档不写性能倍数只写定性判据 + 可复现入口、microbench 形状搬 e2e fixture 须重查投影/序列化段。issue #179（`storage_mode` 非法值兜底跨运行时分歧）仅记录未修。 - `llmdoc/memory/reflections/skip-field-lazy-input-and-pool-baseline-keys.md` — issue #174 + #177 联合复盘：lazy proxy vs materialized input 在跨运行时 skip 契约上的实现分裂（pine-go/pine-java `BuildInput` materialize-time 剔除 vs pine-cpp `OperatorInput` proxy 读路径 gate `excluded_common`）、Lua pool baseline 仅覆盖字符串键的四运行时一致契约（wangshu godoc 权威表述、非 bug 而是文档化负空间）、`TransformByLua.java` 最后一处 coercion 谓词清理（`snapshotKeys` 从 `isstring()` 迁移到 `type() == TSTRING`）、fuzz artifact triage playbook（末端错误误导、需从 pipeline 末端向前截断定位真实 frame 分歧 op）。 +- `llmdoc/memory/reflections/json-number-format-parity-180.md` — issue #180（JSON 数字格式跨运行时对等）复盘：issue 标题「1e20 分歧」把范围说小一个量级（20 值探针实测 15/20 分歧，缺陷是三个互相独立的、分布在 pine-java 与 pine-cpp 两侧）、`to_chars` 默认 overload 非最短往返致试错两轮、Go `encoding/json` 指数负号不对称必须实测、同一 double 在仓库里有多条格式化路径（JSON 序列化器自带规则第二份拷贝、文档把 GoFormat 写成单一事实源却漏了 JSON 路径正是缺陷长期存在的条件）、校验层核实结论（#180 被抓到靠 Python int/float 类型分裂而非设计出的检出能力、09 号通道 `[W]` 降级、`04_number_precision.json` 恰好只覆盖已对区间）、顺带发现 issue #183 判为不同缺陷单独开（**未修**）、三个环境坑（JaCoCo `-Dtest` major version 70、`/tmp` 短名 permission、stderr `[pine:debug]` 污染输出）。 ## memory/decisions/ diff --git a/llmdoc/memory/doc-gaps.md b/llmdoc/memory/doc-gaps.md index a0c363f1..322dcf90 100644 --- a/llmdoc/memory/doc-gaps.md +++ b/llmdoc/memory/doc-gaps.md @@ -23,3 +23,15 @@ - **现状**：已知分歧，见 issue #179，**未修**。仅在 issue 中记录，无稳定文档条目 - **待决策**：修（三方对齐兜底行为 + error fixture）还是归档为 accepted design difference（则需进 `architecture/dag-engine.md` 的接受差异段并给出理由）。在决定之前，稳定文档不得表述为已解决 + +### 字节级对等的校验通道覆盖面太窄 + +- **现状**：`scripts/cross-validate/14-byte-exact-execute.sh` 是唯一不做任何归一化的通道，但只有 5 个 fixture（`fixtures/server_byte_exact/`）；而「字节级对等」是全局契约，覆盖面与声明严重不匹配。`09-raw-byte.sh` 标题写 "no normalization"，实际在字节比较失败后回落到 `normalize_json` 再比一次，相等就打 `[W]` 并计为 pass（`09-raw-byte.sh:115-126`）；`scripts/differential-fuzz.py` 的 `normalize_json` 用 `sort_keys=True` + `round(v,10)`，key 顺序整维度与大部分数字拼写差异都不可见 +- **已做**：`guides/ci-quality-baseline.md` 新增「校验通道能钉住的属性（归一化 vs 字节级）」节，写清各通道的可见性边界与那条纪律；issue #180 给 14 号通道补了 `06_number_format_regimes.json` +- **待决策**：两条路径任选或并行——(a) 继续扩 `fixtures/server_byte_exact/`，把「字节级」声明真正覆盖到主要响应形状；(b) 把 `09-raw-byte.sh` 的归一化回落改成硬失败。(b) 会立刻暴露 issue #183 的 key 顺序分歧，因此**它是 #183 的前置条件**：先决定通道方案，再修 #183，否则修完没有回归门 + +### issue #183：Java object key 插入顺序 vs Go 排序（已记录、已开 issue、未修） + +- **现状**：Go `encoding/json` 对 map key 排序输出，pine-java Jackson 序列化 `LinkedHashMap` 保留插入顺序，两者 JSON key 顺序不一致。已在干净 master 的 worktree 上复现，确认既存且与数字格式无关，**未修** +- **为何长期不可见**：见上一条——fuzz 的 `sort_keys=True` 与 09 号通道的 `[W]` 降级都把这个维度抹掉了 +- **待决策 / 前置**：修它之前必须先解决校验通道问题（上一条）。实现层已知陷阱：Go 的 sort 是 **UTF-8 字节序**，Java `String.compareTo` 是 **UTF-16 code unit 序**，对 BMP 外字符（surrogate pair）会分歧，必须显式给字节序 comparator，否则只是把分歧点从 ASCII 挪到 emoji。另外 pine-cpp 侧的 key 顺序尚未与另两方比对过（#180 期间只对了 go/java 这一对），需补三方比对 diff --git a/llmdoc/memory/reflections/json-number-format-parity-180.md b/llmdoc/memory/reflections/json-number-format-parity-180.md new file mode 100644 index 00000000..e0dda218 --- /dev/null +++ b/llmdoc/memory/reflections/json-number-format-parity-180.md @@ -0,0 +1,227 @@ +# [JSON 数字格式跨运行时对等修复（issue #180）] + +分支 `fix/180-go-java-json-number-parity`（基于 `origin/master` = `ab2dfd5f`），两个 commit： +`70762f19 fix(cpp)` + `95fa8f6f fix(java)`。顺带开了 issue #183（未修）。 + +## Task + +修 issue #180：差分 fuzz 报出 Go 与 Java 的 JSON 数字格式分歧（`1e20` → Go 输出 +`100000000000000000000`、Java 输出 `1.0E20`）。目标是让三运行时的 JSON 数字字面量 +与 Go `encoding/json` 字节一致。 + +## Expected vs Actual + +- Expected：一个边界值的特例。issue 标题写的是 "divergence at 1e20"，读起来像 + Java 少处理了一个阈值，改一处即可。 +- Actual：**三个互相独立的缺陷，分布在两个运行时**。用 20 个精选 double 做三运行时 + 比对，20 个里 **15 个分歧**。1e20 只是最大那个缺口里的一个点。 + +## What Went Wrong + +### 1. issue 标题把缺陷范围说小了一个量级 + +三类缺陷各自的真实范围： + +- **Java**（`GoFormat.createGoCompatMapper` 里的 Double 序列化器）：只特殊处理 + `-0.0` 和「整数值且在 ±2^53 内」两种情况，其余全部落到 Jackson `writeNumber`， + 按 `Double.toString` 格式化。Go 的 plain-decimal 区间一直延伸到 1e21，所以 + **2^53 到 1e21 整个区间**都错，`< 1e-6` 的小数也全错拼法。 +- **C++ 缺陷一**（`pine-cpp/src/config/json.cpp` `go_format_json_number`）： + `std::to_chars` 配 `chars_format::fixed` 打印的是**精确值**，而 Go + `strconv.FormatFloat(d, 'f', -1, 64)` 的 precision=-1 意为「最短往返」。 + `1.0000000000000002e20` → C++ `100000000000000016384`、Go + `100000000000000020000`。 +- **C++ 缺陷二**：`chars_format::scientific` 把指数补到两位（`1e-07`），Go 的 + `encoding/json` 会去掉一个前导零。 + +如果照标题只改 1e20 一处，会留下 14 个分歧。 + +### 2. 「哪个 `to_chars` 模式是最短往返」反直觉，试错两轮 + +第一轮假设「不带 format 参数的默认 `to_chars` overload 就是最短往返」，据此改完 +实测仍有 1 个分歧。探针结果： + +``` +d = 1.0000000000000002e20 +to_chars 默认 -> 100000000000000016384 (精确值，不是最短往返) +chars_format::fixed -> 100000000000000016384 +chars_format::scientific -> 1.0000000000000002e+20 (最短往返) +``` + +**默认 overload 在量级大到不需要指数时会退化成精确打印**，只有 `scientific` 保证 +最短往返。最终方案：统一从 `scientific` 取数字，再自己摆小数点 +（`go_json_to_fixed` / `go_json_to_scientific` 共用同一个数字来源）。 + +### 3. Go 指数格式有不对称，必须实测 + +`strconv.FormatFloat(d, 'e', -1, 64)` 输出 `1e-07`，而 `encoding/json` 输出 +`1e-7`。实测确认规则是**只对负指数去掉一个前导零**： + +``` +strconv 'e': 1e-07 -> json: 1e-7 +strconv 'e': 1e-09 -> json: 1e-9 +strconv 'e': 1e+21 -> json: 1e+21 (不动) +strconv 'e': 1e+100 -> json: 1e+100 (不动) +strconv 'e': 1e-100 -> json: 1e-100 (三位，不动) +``` + +靠推理一定写错（容易写成两边都 trim，或干脆不 trim）。两处实现的注释都写明 +"verified against encoding/json rather than inferred"。 + +### 4. 同一个 double 在仓库里有多条格式化路径 + +pine-java 有 `GoFormat.sprint` / `formatFloatF` / `formatG` 三个格式化器， +**JSON 输出路径一条都没用**——它走 `createGoCompatMapper` 里装的 Jackson Double +序列化器。而 `formatFloatF` 对 1e20 本来就会给出正确答案（内部就有 +`BigDecimal.toPlainString` 平铺逻辑），只是 JSON 路径从来没调它。 + +即：同一个 double，仓库里有两套独立实现给出不同字符串。本次新增的 +`formatJsonNumber` 是第四条，所以测试里加了 `formatJsonNumberMatchesTheSerializer` +钉住「序列化器不得持有规则的第二份拷贝」。 + +`llmdoc/architecture/dag-engine.md`「跨运行时格式兼容（GoFormat）」节列了三个 +格式化器和消费者清单，**完全没提 JSON 输出路径**。读文档的人会以为 GoFormat 是 +格式化的单一事实源——这个文档缺口正是缺陷能长期存在的条件。 + +### 5. 校验层：#180 被抓到是侥幸，机制比预想的窄（已核实，纠正原推测） + +原先的推测是「fuzz 有另一条不归一化的字节通道」。核实后不是。 + +`scripts/differential-fuzz.py:1257` 的 `normalize_json` 做 +`json.loads` → `_normalize_value` → `json.dumps(sort_keys=True)`。#180 能被报出来 +的真实原因是 **Python 的 int/float 类型分裂**： + +``` +Go 输出 100000000000000000000 -> json.loads 得到 int -> _normalize_value 不动 +Java 输出 1.0E20 -> json.loads 得到 float -> re-dump 成 1e+20 +``` + +`_normalize_value` 只对 `float` 分支做 `round(v, 10)` 和小量级归零，`int` 原样 +穿过。所以只有**至少一侧输出整数形状字面量（无小数点无指数）**时分歧才可见。 +实测各类分歧在归一化后的可见性： + +``` +Go 100000000000000000000 vs Java 1.0E20 -> 可见 +Go 100000000000000020000 vs C++ 100000000000000016384 -> 可见 +Go 9007199254740992 vs Java 9.007199254740992E15 -> 可见 +Go 0.0000001 vs Java 1.0E-7 -> 不可见 +Go 0.000001234 vs Java 1.234E-6 -> 不可见 +Go 1e+21 vs Java 1.0E21 -> 不可见 +C++ 1e-07 vs Go 1e-7 -> 不可见 +1.2345678901234567 vs 1.2345678901234568 -> 不可见（round 10 抹掉） +``` + +也就是说 15 个分歧里 fuzz 结构上只能看见其中一部分，且看见的那部分靠的是 +Python 类型系统的副作用，不是设计出来的检出能力。 + +cross-validate 侧的两条通道也都没拦住： + +- `scripts/cross-validate/09-raw-byte.sh` 标题写 "no normalization"，但字节比较 + 失败后会回落到 `normalize_json` 再比一次，相等就**记 `[W]` 警告并计为 pass**。 + key 顺序差异因此被有意容忍——这就是 #183 长期不可见的直接原因。 +- `scripts/cross-validate/14-byte-exact-execute.sh` 是真正的字节通道（`curl` 响应 + 体直接 `==`），本任务前只有 4 个 fixture（本任务新增第 5 个）。其中 `04_number_precision.json` 名字看起来 + 正好覆盖本缺陷，实际输入是 `100000 / 1000001 / 0.5`，×2 后全部落在 + ±2^53 内的整数值区间——**恰好是 Java 旧代码唯一处理对的那个区间**。 + +结论：**声称「字节级对等」的校验，实际上在归一化之后比较**，校验强度与声明不符。 +修 #183 之前必须先决定字节通道怎么补，否则修完没有回归门。 + +### 6. 顺带发现 #183，判为不同缺陷、单独开 issue + +用 600 个 double（边界值 + 随机 bit pattern）做三运行时字节比对：go 与 cpp 字节 +完全相同（md5 `bf229ccbf23c82e75ac1beaa20991bc7`），java 字节数相同（18669）但 +md5 不同。逐字段查完发现 **600 个数字字面量全部一致**，差异纯在 object key 顺序 +（Go `encoding/json` 排序 vs pine-java Jackson 序列化 `LinkedHashMap` 保留插入 +顺序）。在干净 master 的 worktree 上重建 pine-java 复现，确认既存且与数字格式无关， +故开 #183，**本次未修**。 + +（正面教训：字节不同 ≠ 正在修的东西还没修好。先定位差异落在哪个维度，再决定归属。） + +### 7. 三个环境坑 + +- `mvn -o test -Dtest=GoJsonNumberParityTest` 挂在 JaCoCo：`Unsupported class file + major version 70`（JaCoCo 0.8.13 不认当前 JDK 的 class 文件）。改走 + `make java-test` 全量跑正常。**单测过滤路径与 make target 走的不是同一套 profile， + 前者失败不代表测试有问题。** +- 临时文件用 `/tmp/g.json` 这种极短名踩到 `Permission denied`（`/tmp` 下已有其他 + 用户的同名文件）。应用 `mktemp -d` 或带任务前缀。 +- 有一次把 java stderr 里的 `[pine:debug]` 行一起重定向进输出文件，JSON 解析失败， + 误以为 java 还在输出 `1.0E20`。**比对输出前先确认捕获的是纯净 stdout。** + +## Root Cause + +1. **范围来自实测，不来自 issue 标题。** 标题给的是「症状的一个实例」，不是「缺陷 + 的范围」。本次先写 20 值探针才发现 15/20 分歧；先扫边界空间再动手是必需步骤， + 不是可选的谨慎。 +2. **格式化规则的第二份拷贝。** JSON 序列化器自带一套 double 处理逻辑，而不是调用 + 已有的格式化模块，两份实现独立漂移。文档把 GoFormat 描述成单一事实源、却没列 + JSON 路径，让这个分裂在读文档时不可见。 +3. **跨语言标准库「等价函数」的隐含语义不等价。** `to_chars` 默认 overload 与 + `FormatFloat(-1)` 名义上都是「合理的默认」，实际一个是精确值一个是最短往返； + `strconv 'e'` 与 `encoding/json` 的指数补零规则也不同。这类差异只能实测。 +4. **校验是按「解析后的对象」比的，声明是「字节级」。** 归一化（`sort_keys` + + `round(v,10)` + int/float 分裂）把 key 顺序整维度、以及一部分数字拼写差异 + 直接抹掉。真字节通道存在但 fixture 覆盖太窄，且那个名叫 `number_precision` 的 + fixture 恰好只覆盖已经对的区间。 + +## Missing Docs or Signals + +- `architecture/dag-engine.md` 的 GoFormat 节没有 JSON 输出路径。缺的信号正是 + 「格式化不止这三个入口」。 +- 没有任何稳定文档记录「C++ 只有 `chars_format::scientific` 保证最短往返」和 + 「Go json 指数负号不对称」。下一个碰数字格式化的人会重新试错两轮。 +- `guides/ci-quality-baseline.md` 的 differential-fuzz 节描述了归一化机制，但没有 + 写清**归一化抹掉了哪些维度**（key 顺序、数字拼写、float 第 11 位起的差异）， + 也没有把 09-raw-byte 的 `[W]` 降级和 14-byte-exact 的 fixture 覆盖面写成一张 + 「哪条通道能钉住哪个属性」的表。这与 `guides/cross-layer-validation.md` 已有的 + 「fixture 比对器语义决定该层能钉住的属性」是同一条原则，只是没落到 fuzz 上。 + +## Promotion Candidates + +- **进 `guides/`（或 `reference/`）：跨语言数值格式化的两条实测事实**——C++ + `std::to_chars` 只有 `chars_format::scientific` 保证 shortest round-trip + （默认 overload 与 `fixed` 在大量级下退化为精确打印）；Go `encoding/json` 相对 + `strconv.FormatFloat('e', -1)` 只 trim 负指数的一个前导零。两条都必须实测， + 注释里要留「verified against X rather than inferred」。 +- **必须修稳定文档：`architecture/dag-engine.md` 的 GoFormat 节**——补第四个入口 + `formatJsonNumber` 及其消费者（`createGoCompatMapper` → `RunCli` / `PineServer`）， + 并写明四者阈值不同、不可互换；同时写明 JSON 路径不走 `formatFloatF`。 +- **进 `guides/ci-quality-baseline.md`：校验通道能钉住的属性表**——differential-fuzz + 归一化抹掉 key 顺序与部分数字拼写（含 int/float 分裂导致的检出偏斜）； + 09-raw-byte 把 key-order-only 差异降级为警告；14-byte-exact 是唯一真字节通道但 + 本任务前只有 4 个 fixture，新增后 5 个。配一条纪律：**声称字节级对等的属性，必须有一条不归一化的 + 通道覆盖**。 +- **进 `memory/doc-gaps.md`：字节级对等校验缺口**——待决策项，是给 fuzz 加不归一化 + 字节通道，还是扩 `fixtures/server_byte_exact/`，还是取消 09 的 `[W]` 降级。 + 这个决策是 #183 的前置条件。 +- 不提升：JaCoCo `-Dtest` 坑、`/tmp` 短名、stderr 污染三条留在本篇即可；若再现 + 第二次再考虑进 `guides/standard-workflow.md`。 + +## Follow-up + +1. 修 #183 之前先决定字节通道方案（见上述 doc-gap 条目）。#183 的实现层还有一个 + 已记录的陷阱：Go 的 sort 按 UTF-8 字节序、Java `String.compareTo` 按 UTF-16 + code unit，BMP 外字符（surrogate pair）会分歧，必须显式给字节序 comparator， + 否则只是把分歧点从 ASCII 挪到 emoji。 +2. 给 `fixtures/server_byte_exact/` 补一个真正跨区间的数字 fixture（覆盖 + 2^53~1e21 plain-decimal 段、`< 1e-6` 科学计数段、`>= 1e21` 段、1e-7/1e-9 指数 + trim、三位指数），当前 `04_number_precision.json` 只覆盖了原本就对的区间。 +3. 调用 `recorder` 落地上面三处稳定文档修改（dag-engine GoFormat 节、 + ci-quality-baseline 通道表、guides 数值格式化事实），并在 doc-gaps 开条目。 +4. #183 的 pine-cpp 侧未测（只对了 go/java 这一对），需补测三方 key 顺序。 + +## 验证情况（本次已完成） + +- 20 值定向探针：三运行时 0/20 分歧（修复前 15/20） +- 600 值广谱探针（边界 + 随机 bit pattern）：go 与 cpp 字节全同 + （md5 `bf229ccbf23c82e75ac1beaa20991bc7`）；三方数字字面量 0 分歧 +- #180 原始 artifact（`.code-review/artifacts/issue180-divergence-000664/`）重跑： + 三运行时归一化文档 md5 全同（`3bd709debbee27ab8e384a7397d3a6eb`） +- `make cpp-test` 246 用例；`make java-test` 322 用例（新增 7）；`make lint`、 + `make codegen-check`、`make test` 全过 +- `make cross-validate` 55/55；`make differential-fuzz` 1000/1000 + （row=604/0 column=396/0） +- mutation 双向验证：C++ 两个机制各自独立变红（改回 `chars_format::fixed` → + 最短往返断言红；去掉指数 trim → `1e-7`/`1e-9` 断言红）；Java 序列化器改回 + `writeNumber` → 7 个测试全红 diff --git a/llmdoc/reference/number-formatting-parity.md b/llmdoc/reference/number-formatting-parity.md new file mode 100644 index 00000000..1cd84be4 --- /dev/null +++ b/llmdoc/reference/number-formatting-parity.md @@ -0,0 +1,177 @@ +# 跨语言数值格式化实测事实 + +本文件记录复刻 Go 数值格式化时**必须实测、不能靠推理**的两条事实。两条都是 issue #180 期间用探针实测得到的，各自让一轮实现失败过。 + +适用场景：在 pine-cpp / pine-java（或任何第四运行时）里复刻 Go `strconv` / `encoding/json` 的 double 输出，或修改已有的格式化路径。 + +参照的实现落点： + +- pine-java：`pine-java/src/main/java/page/liam/pine/GoFormat.java`（`formatJsonNumber`） +- pine-cpp：`pine-cpp/src/config/json.cpp`（`go_format_json_number` / `go_json_to_fixed` / `go_json_to_scientific`） +- Go 侧规则来源：`encoding/json` 的 `floatEncoder` + +## 事实一：C++ `std::to_chars` 只有 `chars_format::scientific` 保证最短往返 + +Go `strconv.FormatFloat(d, 'f'|'e', -1, 64)` 的 `precision = -1` 意为「能往返的最少位数」。C++ 侧对应关系不是逐个 format 平移： + +- `chars_format::fixed` 打印的是**精确值**，不是最短往返 +- **不带 format 参数的默认 overload，在量级大到不需要指数时也退化成精确打印**——这条最反直觉，issue #180 第一轮就是按「默认 overload = 最短往返」写的，改完仍剩 1 个分歧 +- 只有 `chars_format::scientific` 保证最短往返 + +实测（`d = 1.0000000000000002e20`）： + +``` +to_chars 默认 overload -> 100000000000000016384 精确值 +chars_format::fixed -> 100000000000000016384 精确值 +chars_format::scientific -> 1.0000000000000002e+20 最短往返 +Go FormatFloat(d,'f',-1,64)-> 100000000000000020000 +``` + +因此 pine-cpp 的 `go_format_json_number` 统一从 `chars_format::scientific` 取数字串，再由 `go_json_to_fixed` / `go_json_to_scientific` 自己摆小数点：两种输出形式（平铺小数 / 科学计数）**共用同一个数字来源**，避免两条形式各自取数字导致的精度分叉。 + +## 事实二：Go `encoding/json` 的指数格式相对 `strconv` 有一处不对称 + +`strconv.FormatFloat(d, 'e', -1, 64)` 把指数补到至少两位（`1e-07`），而 `encoding/json` **只对负指数去掉一个前导零**，正指数与三位数指数一概不动。 + +实测： + +``` +strconv 'e': 1e-07 -> json: 1e-7 去掉一个前导零 +strconv 'e': 1e-09 -> json: 1e-9 去掉一个前导零 +strconv 'e': 1e+21 -> json: 1e+21 不动 +strconv 'e': 1e+100 -> json: 1e+100 不动 +strconv 'e': 1e-100 -> json: 1e-100 三位数，不动 +``` + +靠推理很容易写成两边都 trim 或都不 trim，两种都错。这条规则不对称到没法从文档反推，必须跑一次 Go 程序对照。 + +## 纪律 + +两处实现的注释都显式标注了 "verified against encoding/json rather than inferred"。给这类跨语言等价函数写实现时，注释要说明**依据是实测还是推理**——下一个改这段代码的人需要知道哪些常量不能靠「看起来更合理」去调整。 + +更一般的判据：跨语言标准库里名字/角色对应的「等价函数」，其隐含语义常常不等价（`to_chars` 默认 overload 与 `FormatFloat(-1)` 名义上都是「合理默认」，实际一个精确值一个最短往返）。这类差异只能实测。 + +## 相关 + +- 格式化入口划分与消费链：`llmdoc/architecture/dag-engine.md`「跨运行时格式兼容（GoFormat）」节 +- 哪条校验通道能钉住字节级数字格式：`llmdoc/guides/ci-quality-baseline.md`「校验通道能钉住的属性」节 +- 完整过程记录：`llmdoc/memory/reflections/json-number-format-parity-180.md` + +## 非有限值（NaN / ±Inf）：刻意不对等，记为 accepted difference + +Go 的 `encoding/json` 对非有限 float64 直接报 `UnsupportedValueError`，**根本不产出字节**。 +所以这里没有"与 Go 一致"这个选项，三运行时各自的取舍如下： + +| 运行时 | 输出 | 是否合法 JSON | +|---|---|---| +| pine-go | 拒绝序列化（error） | — | +| pine-cpp | `inf` / `-inf` / `nan`（裸 token） | 否 | +| pine-java | `"Infinity"` / `"-Infinity"` / `"NaN"`（带引号字符串） | 是 | + +**为什么不统一**：正常路径上三者都不会走到这里——写入侧有 NaN/Inf 校验 +（pine-cpp 在 `engine.cpp` 的 `validate_output`、pine-go 在 `row_frame.go`）。 +唯一能绕过校验的入口是**请求里直接带非有限数值**，而这条路上 pine-go 与 +pine-cpp 都在解析阶段就拒绝整个请求（C++ 的 `from_chars` 返回 +`result_out_of_range`），只有 Jackson 会把 `1e400` 静默 coerce 成 `Infinity`。 +也就是说分歧的成因在**请求解析层**，不在数字格式化层，修格式化不能消除它。 + +**pine-java 选带引号字符串**的理由是：那条路上已经不可能与 Go 字节对等 +（Go 会拒绝请求），剩下唯一可争取的属性是"响应仍是可解析的 JSON"。曾经有一版让 +序列化器无条件走 `formatJsonNumber` 的输出，结果写出裸 `+Inf`，把整份响应变成 +不可解析——比不对等更糟。`GoFormat.formatJsonNumber` 现在对非有限输入直接抛 +`IllegalArgumentException`，逼调用方自己决定，而不是编造一个"看起来像 Go"的表示。 + +**pine-cpp 保留 `inf` / `nan`** 是为了不在修 #180 时顺带改动既有行为——它与 +`ab2dfd5f` 之前逐字节一致。修复只解决了一个真实缺陷：`to_chars` 对非有限输入 +**返回成功**并写出 `inf`，导致这些字母流进 decompose 逻辑被当成尾数/指数数字， +输出 `i.nfe+2`（既不合法也不是原来的 `inf`）。 + +**若要真正统一**，得先解决请求解析层的分歧（让 pine-java 也拒绝非有限请求）， +那是独立议题，不在 #180 范围内。`metrics_collector.cpp` 的 `/stats` 路径曾被标为「理论上可序列化非有限指标值、未实测」——现已查清：那两处调用点（`metrics_collector.cpp:195,208`）走的就是 `go_format_json_number`，而该函数现在对非有限输入前置返回 `nan` / `inf`，与上表 pine-cpp 一行一致。也就是说 `/stats` 不构成额外暴露面，行为与 `/execute` 相同。 + +## Java 侧最短往返：为什么最终选了「不优化」的实现 + +`GoFormat.shortestRoundTrip` 从 precision 1 逐位向上试，返回第一个能往返的候选。 +**没有快速路径，刻意如此**，这是三轮审查换来的结论： + +前后加过两版快速路径（「若少一位就无法往返，则 `Double.toString` 已最短，直接返回」）， +逻辑本身正确、输出也始终与 Go 一致，但每一版都有一整类输入被守卫无声排除： + +| 版本 | 声称 | 实测 | +|---|---|---| +| v1 | 「normal double 首次尝试即命中」 | 恰好相反，normal 是最慢的一类，约 90× | +| v2 | 「快速路径覆盖 normal double」 | 整数值 double **0% 命中**（`Double.toString(1.0)` = `"1.0"`，尾随零被算作有效位） | +| v3 | 「前导零计数只是放宽搜索范围」 | `[0.001, 1)` **0% 命中**，8065 ns/value vs `[1,1000)` 的 1104 | + +三次的共同机制是**基准样本恰好避开了会反驳注释的那个区间**： +`nextDouble()*1000` 有 99.9% 落在 `|d| ≥ 1`，也就是快速路径唯一真正生效的地方。 +所以「测出来是快的」和「注释说的那类输入是快的」是两件事。 + +**结论与实测代价**：这条路径挂在 `/execute` 响应序列化上，代价是真实的——实测 30000 个 double 的响应，**约 18–25×**（三次独立测量：103 ms / 5.5 ms = 18.6×、71 ms / 3.1 ms = 22.7×、54.7 ms / 2.2 ms = 25.2×，同样是 30000 个 `nextDouble()*1000`、openjdk 26，差异来自 warmup 轮数与取中位数的方式）。**这是一个区间，不是阈值**——绝对毫秒数和倍数都会随预热策略移动，复测得到 25× 不代表回归。要当门用就得先固定 warmup / 取样 / 统计口径（不是「慢几微秒」，早先注释里那句「不是热点瓶颈」未经测量，已删）。选择接受这个代价的理由不是它便宜，而是三轮审查证明**在这段代码上加快速路径的失败率是 3/3**，每次都以「注释说错自己覆盖哪些输入」的形式出现，而正确性是硬契约、吞吐不是。若这个 18.6× 在实际负载里成为问题，那是一个独立的、有明确验收标准的优化任务（附下面的分区间基准要求），而不是顺手加个守卫。现在的实现无法说错「它覆盖哪些输入」，因为它对所有输入 +一视同仁。若将来真要优化，**必须分别基准 `[0.001,1)`、`[1,1000)`、整数值三类**—— +只从其中任一类取样都会印证你已有的判断。 + +## 一个易踩的坑：缩短必须作用在 `Double.toString` 选出的数字上 + +不能用 `new BigDecimal(double)` + `MathContext` 去舍入**精确二进制展开**： +`MathContext` 在真值上做 HALF_UP，而 Go 报告的是**离该 double 最近的**那位数字。 +实测分歧（数量随取样分布变化：按量级定向取样时 200k 中 50+ 例，纯随机 bit pattern 则 8–13 例；机制与下面四个值本身不依赖分布）： + +``` +bits=431f64571af9dce5 go 2209012388886329.2 舍入精确值 → ...329.3 +bits=42e3866babcd3a24 go 171744423733713.12 舍入精确值 → ...713.13 +``` + +`Double.toString` 选的数字本来就是对的，唯一的问题是**可能太多**——所以正确做法是 +`new BigDecimal(Double.toString(d))` 再逐位缩短。由 +`digitsComeFromDoubleToStringNotFromRoundingTheExactValue` 钉住。 + +## 另一条既存不对等：resource lookup key 上三个运行时互不相同 + +`formatFloatF`（Java）/ `go_format_lookup_key`（C++）/ `strconv.FormatFloat(d,'f',-1,64)`（Go） +是 `transform_resource_lookup` 的 key 派生函数，与 JSON 输出**是不同的路径**。对次正规值 +三者输出互不相同（实测 `Double.MIN_VALUE`）： + +| 运行时 | 输出 | 长度 | +|---|---|---| +| pine-go | `0.000...005`（完整平铺） | 326 | +| pine-java | `0.000...049`（`Double.toString` 非最短，多一位） | 327 | +| pine-cpp | `5e-324` | 6 | + +C++ 那支的成因单独说一下，因为不看代码想不到：`go_format_lookup_key` 用 `char buf[64]` +配 `to_chars(chars_format::fixed)`，326 字符**装不下**，`to_chars` 返回 +`value_too_large`，于是走了 `go_format_g` 兜底——而那个兜底会输出科学计数法。也就是说 +它不是「少了几位」，是整个格式换了。 + +**可达性**：请求里带 `5e-324` 能活着到这里（Jackson 解析出 bits=1），所以不是理论问题。 + +**为什么不在 #180 里修**：#180 的范围是 JSON 输出字节；key 派生函数是另一条路径， +且改它对任何已经按当前形式建过索引的数据是行为变更。现状由 +`formatFloatFSubnormalDivergenceIsPinnedNotFixed` 钉住 Java 侧的 327， +使后续修它必须显式改掉一个失败断言。 + +**后续要收这条时的注意点**：不要只改 Java 的位数——C++ 的 `buf[64]` 必须一起扩， +否则「统一」之后 C++ 仍在输出 `5e-324`。这一点是第九轮审查提出的，它在自己的 +finding 之外额外查了 C++ 分支，而我之前的文档只写了 Java vs Go。 + +## 如果将来给 pine-cpp 加 float32 路径 + +现在不存在这条路径——`Variant::value_t` 只有 `double`，所以 C++ 侧没有 float32 分歧， +这一节是**给未来的警告**，不是待修项。 + +真要加的时候，**不要把 float32 加宽成 double 再调 `go_format_json_number`**。Go 用 +`strconv.AppendFloat(..., 32)`，数字是「对 float32 最短往返」，加宽会把窄类型原本藏住的 +二进制噪声抖出来： + +``` +float32 0.1 Go: 0.1 加宽后: 0.10000000149011612 +float32 1e20 Go: 100000000000000000000 加宽后: 100000002004087730000 +``` + +pine-java 侧的 `formatJsonNumber(float)` 就是踩过这个坑之后的写法：单独一条 +`shortestRoundTrip(float)` 按 float 精度缩短，**且阈值要拿缩短后的十进制去比、不是拿加宽的 +double 去比**（`bits=897988541` 加宽是 `9.999999974752427e-07` 低于 1e-6，缩短后是 `1e-06` +不低于，Go 输出 `0.000001`）。C++ 侧要加的话，同样两点都得照做。 + +另外注意 `Float.toString` / `Double.toString` 都**不是**次正规的最短表示（float32 有 9 个 +bit pattern、double 有 8 个），这是两条 `shortestRoundTrip` 存在的唯一理由。 diff --git a/pine-cpp/src/config/json.cpp b/pine-cpp/src/config/json.cpp index b8ab9c32..dab94e61 100644 --- a/pine-cpp/src/config/json.cpp +++ b/pine-cpp/src/config/json.cpp @@ -385,26 +385,204 @@ class Parser { } // namespace +namespace { + +// Splits a shortest-round-trip rendering from std::to_chars into a sign, a +// digit string with no decimal point, and a decimal exponent such that the +// value is sign * 0. * 10^exp10. +// +// Accepts both fixed and scientific input shapes. Only scientific arrives today +// — the single call site hardcodes chars_format::scientific — but the parsing is +// kept general so the function is about decimal renderings rather than about one +// caller's current choice. An earlier version of this comment said to_chars +// "picks fixed or scientific on its own", which stopped being true when the call +// site was pinned to scientific. +struct DecimalParts { + bool negative = false; + std::string digits; // significant digits, no '.', no leading zeros + int exp10 = 0; // number of digits before the decimal point +}; + +DecimalParts go_json_decompose(const std::string& s) { + DecimalParts p; + std::size_t i = 0; + if (i < s.size() && (s[i] == '-' || s[i] == '+')) { + p.negative = (s[i] == '-'); + ++i; + } + std::string mantissa; + int point_pos = -1; + for (; i < s.size(); ++i) { + if (s[i] == '.') { + point_pos = static_cast(mantissa.size()); + continue; + } + if (s[i] == 'e' || s[i] == 'E') { + ++i; + int sign = 1; + if (i < s.size() && (s[i] == '-' || s[i] == '+')) { + sign = (s[i] == '-') ? -1 : 1; + ++i; + } + int e = 0; + for (; i < s.size(); ++i) { + e = e * 10 + (s[i] - '0'); + } + p.exp10 = sign * e; + break; + } + mantissa.push_back(s[i]); + } + if (point_pos < 0) { + point_pos = static_cast(mantissa.size()); + } + // exp10 so far holds only the explicit exponent; add the point position. + p.exp10 += point_pos; + // Both normalizations below are UNREACHABLE from the current single call + // site, which always passes chars_format::scientific output: that form emits + // neither leading nor trailing zeros in the mantissa (checked over 3.3M + // samples), and deleting either loop leaves 810k renderings byte-identical. + // They are kept because this function's contract is "decompose a decimal + // rendering", not "decompose what to_chars(scientific) happens to produce" — + // a second caller passing fixed-format input would need them. If that never + // arrives, they are safe to delete. + // + // Strip leading zeros, adjusting the exponent as we go (0.001 -> digits "1"). + std::size_t lead = 0; + while (lead < mantissa.size() && mantissa[lead] == '0') { + ++lead; + --p.exp10; + } + mantissa.erase(0, lead); + // Strip trailing zeros: they carry no information in this representation. + while (!mantissa.empty() && mantissa.back() == '0') { + mantissa.pop_back(); + } + p.digits = mantissa; + return p; +} + +// Renders as Go's strconv.FormatFloat(d, 'f', -1, 64) does: plain decimal, no +// exponent, shortest digits zero-filled out to the decimal point. +std::string go_json_to_fixed(const std::string& shortest) { + DecimalParts p = go_json_decompose(shortest); + if (p.digits.empty()) { + return p.negative ? "-0" : "0"; + } + std::string out; + if (p.negative) { + out.push_back('-'); + } + int nd = static_cast(p.digits.size()); + if (p.exp10 <= 0) { + out += "0."; + out.append(static_cast(-p.exp10), '0'); + out += p.digits; + } else if (p.exp10 >= nd) { + out += p.digits; + out.append(static_cast(p.exp10 - nd), '0'); + } else { + out.append(p.digits, 0, static_cast(p.exp10)); + out.push_back('.'); + out.append(p.digits, static_cast(p.exp10), std::string::npos); + } + return out; +} + +// Renders as Go's encoding/json does for the scientific branch: strconv's +// 'e' format, then ONE leading zero trimmed from a two-digit negative +// exponent. Go's json does exactly that and nothing more, so 1e-7 prints as +// "1e-7" while 1e+21 keeps its "+21" and 1e-100 keeps all three digits +// (verified against encoding/json, not inferred). +std::string go_json_to_scientific(const std::string& shortest) { + DecimalParts p = go_json_decompose(shortest); + if (p.digits.empty()) { + return p.negative ? "-0" : "0"; + } + std::string out; + if (p.negative) { + out.push_back('-'); + } + out.push_back(p.digits[0]); + if (p.digits.size() > 1) { + out.push_back('.'); + out.append(p.digits, 1, std::string::npos); + } + // 0. * 10^exp10 == . * 10^(exp10-1) + const int e = p.exp10 - 1; + const bool negative_exponent = e < 0; + out.push_back('e'); + out.push_back(negative_exponent ? '-' : '+'); + + // strconv pads the exponent to at least two digits ("1e-07"), and + // encoding/json then strips one leading zero back off — but only for negative + // exponents. So "1e-7" is trimmed while "1e+21" keeps "+21" and three-digit + // exponents are untouched either way. Verified against encoding/json rather + // than inferred; the asymmetry is easy to get wrong in both directions. + // + // No padding branch is needed for the positive side: this function is only + // reached when |d| >= 1e21 or |d| < 1e-6, so a positive exponent is never + // below 21 and is already two digits. A pad guarded on `!negative_exponent` + // was unreachable — confirmed by deleting it and diffing 688k renderings + // against Go byte for byte with no change. + out += std::to_string(negative_exponent ? -e : e); + return out; +} + +} // namespace + // go_format_json_number formats a double matching Go's encoding/json byte-for-byte. // Go rule (encoding/json/encode.go floatEncoder): for float64, use 'f' format // (fixed-point, shortest digits) when 1e-6 <= |x| < 1e21, else 'e' (scientific, // shortest). Both with precision=-1 (shortest representation). Diverges from // Go's fmt.Sprintf("%g") which uses different thresholds — keep them separate. std::string go_format_json_number(double d) { + // Non-finite first, before to_chars. There is no Go byte sequence to match: + // encoding/json errors out on NaN/Inf, so no choice here is "correct" and the + // aim is only to avoid making things worse. These are the exact strings the + // pre-#180 implementation produced (raw to_chars output), kept verbatim so + // this change does not alter behaviour beyond fixing the corruption below. + // Callers are expected to reject non-finite before serializing — engine.cpp + // validates on the write path — so reaching here means that guard was bypassed. + // + // The check must be here rather than relying on the `ec` fallback further + // down: to_chars SUCCEEDS on these inputs and writes "inf" / "-inf" / "nan", + // so the error path never fires. Those letters then reached + // go_json_decompose, which read 'i' as a mantissa digit and 'f' as an + // exponent digit and emitted "i.nfe+2" — still invalid JSON, but now + // corrupted rather than merely non-standard. + if (std::isnan(d)) { + return "nan"; + } + if (std::isinf(d)) { + return d < 0 ? "-inf" : "inf"; + } if (d == 0.0) { return std::signbit(d) ? "-0" : "0"; } double abs_d = std::abs(d); bool use_scientific = (abs_d < 1e-6) || (abs_d >= 1e21); char buf[64]; - auto fmt = use_scientific ? std::chars_format::scientific : std::chars_format::fixed; - auto [ptr, ec] = std::to_chars(buf, buf + sizeof(buf), d, fmt); + + // Always source the digits from chars_format::scientific, then reposition the + // decimal point ourselves. Go uses strconv.FormatFloat(d, 'f'|'e', -1, 64), + // where precision -1 means "fewest digits that round-trip". + // + // Neither of the other to_chars modes gives that. chars_format::fixed prints + // the value exactly, and so does the default overload once the magnitude is + // large enough to render without an exponent — both turn + // 1.0000000000000002e20 into 100000000000000016384 where Go emits + // 100000000000000020000. Same double, different bytes: that is issue #180. + // Only the scientific form is guaranteed shortest-round-trip, so it is the + // single source of digits for both output shapes. + auto [ptr, ec] = std::to_chars(buf, buf + sizeof(buf), d, std::chars_format::scientific); if (ec != std::errc()) { std::ostringstream oss; oss << std::setprecision(17) << d; return oss.str(); } - return std::string(buf, ptr); + std::string shortest(buf, ptr); + return use_scientific ? go_json_to_scientific(shortest) : go_json_to_fixed(shortest); } namespace { diff --git a/pine-cpp/tests/test_format_g.cpp b/pine-cpp/tests/test_format_g.cpp index ef2431b0..02b44355 100644 --- a/pine-cpp/tests/test_format_g.cpp +++ b/pine-cpp/tests/test_format_g.cpp @@ -1,8 +1,11 @@ #include +#include + #include "operators/_helpers.hpp" using pine::operators::go_format_g; +using pine::operators::go_format_lookup_key; TEST_CASE("go_format_g matches Go strconv.FormatFloat('g', -1, 64) at long-integer boundaries") { // Each line below was captured from `strconv.FormatFloat(v, 'g', -1, 64)` @@ -50,3 +53,17 @@ TEST_CASE("go_format_lookup_key matches Go FormatInt / FormatFloat('f', -1)") { CHECK(go_format_lookup_key(1e-5).find('E') == std::string::npos); CHECK(go_format_lookup_key(1e-5) == "0.00001"); } + +TEST_CASE("go_format_lookup_key: subnormal falls back to scientific (known divergence)") { + // Pins the C++ half of the three-way resource-key divergence documented in + // llmdoc/reference/number-formatting-parity.md. Go emits 326 characters and + // pine-java 327; this returns "5e-324" because go_format_lookup_key's + // char buf[64] cannot hold a 326-character expansion, so to_chars reports + // value_too_large and the go_format_g fallback emits scientific notation. + // + // Asserted so that widening the buffer is a deliberate act with a failing + // test to update. Without this, someone unifying the key derivation could + // change the Java side, watch its pinned test go red, fix it, and never learn + // that C++ still disagrees. + CHECK(go_format_lookup_key(std::numeric_limits::denorm_min()) == "5e-324"); +} diff --git a/pine-cpp/tests/test_json.cpp b/pine-cpp/tests/test_json.cpp index 2dbcb293..fa1aae56 100644 --- a/pine-cpp/tests/test_json.cpp +++ b/pine-cpp/tests/test_json.cpp @@ -2,6 +2,8 @@ #include +#include + using namespace pine; TEST_CASE("parse_json: scalar values") { @@ -148,3 +150,85 @@ TEST_CASE("FlatMap::reserve after partial insertion preserves entries (L6)") { Variant v(std::move(obj)); CHECK(dump_json(v, 0) == R"({"alpha":1,"bravo":2,"charlie":3,"delta":4})"); } + +TEST_CASE("dump_json: numbers match Go encoding/json byte for byte (#180)") { + // Every expected string below was produced by running json.Marshal on the + // same float64 in Go, not derived from the spec by hand. Go's rule is + // strconv.FormatFloat(d, 'f'|'e', -1, 64) with 'e' chosen when |x| < 1e-6 or + // |x| >= 1e21, and precision -1 meaning shortest round-trip. + auto emit = [](double d) { return dump_json(Variant(d), 0); }; + + SUBCASE("plain decimal range keeps every digit, no exponent") { + CHECK(emit(0.0) == "0"); + CHECK(emit(1.0) == "1"); + CHECK(emit(1.5) == "1.5"); + CHECK(emit(1e6) == "1000000"); + CHECK(emit(1e15) == "1000000000000000"); + // Beyond 2^53 but still below 1e21: Go stays in plain decimal. An earlier + // Java guard stopped at 2^53 and fell back to "1.0E16" here, which is the + // divergence #180 reported. + CHECK(emit(1e16) == "10000000000000000"); + CHECK(emit(1e18) == "1000000000000000000"); + CHECK(emit(1e20) == "100000000000000000000"); + CHECK(emit(-1e20) == "-100000000000000000000"); + } + + SUBCASE("shortest round-trip, not the exact binary expansion") { + // 1.0000000000000002e20 is exactly 100000000000000016384. Go prints the + // shortest digits that round-trip and zero-fills, so the trailing digits + // are 20000, not 16384. std::to_chars with chars_format::fixed prints the + // exact value and got this wrong before the fix. + CHECK(emit(1.0000000000000002e20) == "100000000000000020000"); + } + + SUBCASE("scientific above 1e21") { + CHECK(emit(1e21) == "1e+21"); + CHECK(emit(1e22) == "1e+22"); + CHECK(emit(1.5e21) == "1.5e+21"); + CHECK(emit(-1e21) == "-1e+21"); + } + + SUBCASE("small magnitudes: 1e-6 is the boundary, and it is inclusive") { + CHECK(emit(1e-5) == "0.00001"); + CHECK(emit(1e-6) == "0.000001"); + // Below 1e-6 switches to scientific. Note the exponent: strconv pads to two + // digits ("1e-07") and encoding/json then strips one leading zero from + // NEGATIVE exponents only, so this is "1e-7" while 1e+21 above keeps "+21". + CHECK(emit(1e-7) == "1e-7"); + CHECK(emit(1e-9) == "1e-9"); + CHECK(emit(1e-10) == "1e-10"); + } + + SUBCASE("three-digit exponents keep all digits, no trimming") { + CHECK(emit(1e100) == "1e+100"); + CHECK(emit(1e-100) == "1e-100"); + } + + SUBCASE("negative zero keeps its sign bit") { + CHECK(emit(-0.0) == "-0"); + } + + SUBCASE("non-finite is not mangled into a corrupt token") { + // Go's encoding/json refuses NaN/Inf, so there is no reference byte + // sequence; these are the strings the pre-#180 implementation produced and + // callers are expected to reject non-finite before serializing. + // + // What matters is that they are not silently corrupted. to_chars SUCCEEDS + // on non-finite input and writes "inf"/"nan", so the errc fallback never + // fires; those letters used to reach the decompose helper, which read 'i' + // as a mantissa digit and 'f' as an exponent digit and emitted "i.nfe+2". + const double inf = std::numeric_limits::infinity(); + CHECK(emit(inf) == "inf"); + CHECK(emit(-inf) == "-inf"); + CHECK(emit(std::numeric_limits::quiet_NaN()) == "nan"); + // The isnan guard's only observable effect is normalizing the sign: without + // it, -NaN reaches to_chars and comes back as "-nan". + CHECK(emit(-std::numeric_limits::quiet_NaN()) == "nan"); + } + + SUBCASE("subnormals render shortest, matching Go") { + // Verified against json.Marshal over the first 1e6 bit patterns. + CHECK(emit(std::numeric_limits::denorm_min()) == "5e-324"); + CHECK(emit(-std::numeric_limits::denorm_min()) == "-5e-324"); + } +} diff --git a/pine-java/src/main/java/page/liam/pine/GoFormat.java b/pine-java/src/main/java/page/liam/pine/GoFormat.java index 2c93bb3c..1eaf222a 100644 --- a/pine-java/src/main/java/page/liam/pine/GoFormat.java +++ b/pine-java/src/main/java/page/liam/pine/GoFormat.java @@ -68,10 +68,236 @@ public static String sprint(Object v) { return v.toString(); } + + + /** + * Replicates Go's encoding/json number output for a float64, byte for byte. + * + *

Go's rule (encoding/json/encode.go floatEncoder): render with + * strconv.FormatFloat(d, fmt, -1, 64), choosing 'e' when the magnitude is + * below 1e-6 or at/above 1e21 and 'f' otherwise. Precision -1 means the + * fewest digits that round-trip. Double.toString is that for normal + * doubles but NOT for subnormals, so the digits come from + * {@link #shortestRoundTrip} and only their placement differs here. + * + *

This is deliberately separate from {@link #formatFloatF} (always + * decimal, used for Lua/field formatting) and from the %g emulation. The + * three have different thresholds and are not interchangeable — conflating + * the JSON path with the others is how issue #180 arose. + * + *

One quirk is load-bearing and was verified against encoding/json + * rather than inferred: strconv pads exponents to two digits ("1e-07"), + * and json then strips a single leading zero from NEGATIVE exponents only. + * So 1e-7 prints as "1e-7", while 1e+21 keeps "+21" and 1e-100 keeps all + * three digits. + */ + + + public static String formatJsonNumber(double d) { + if (Double.isNaN(d) || Double.isInfinite(d)) { + // Go's encoding/json refuses these outright (UnsupportedValueError), + // so there is no Go byte sequence to match and this method has no + // correct answer to give. It throws rather than inventing one: the + // caller decides, and the serializer keeps Jackson's quoted-string + // form because that is the only shape that stays parseable JSON. + // + // An earlier version returned formatFloatF(d) here, i.e. "+Inf", + // which the serializer then wrote as a bare token — invalid JSON. + // Reachable in practice: the write path validates NaN/Inf, but a + // request carrying 1e400 does not go through it, and Jackson + // silently coerces that to Infinity on parse where Go and C++ both + // reject it. See the isNonFinite tests. + throw new IllegalArgumentException( + "NaN/Infinity has no Go encoding/json representation: " + d); + } + if (d == 0.0) { + return (Double.doubleToRawLongBits(d) == Double.doubleToRawLongBits(-0.0)) ? "-0" : "0"; + } + // new BigDecimal(String) is exact; new BigDecimal(double) would + // reintroduce the full binary expansion we are trying to avoid. + return formatDecimal(new java.math.BigDecimal(shortestRoundTrip(d)).stripTrailingZeros(), + Math.abs(d), d < 0); + } + /** + * Go's encoding/json output for a float32, byte for byte. + * + *

Go calls strconv.AppendFloat with bitSize=32, so the digits are the + * shortest that round-trip through a float32 — NOT through a double. That + * distinction is the whole reason this method exists: widening first and + * formatting as a double surfaces the binary noise the narrower type was + * hiding. float32 0.1 must print "0.1", but (double) 0.1f is + * 0.10000000149011612, and 1e20f widens to 100000002004087730000 where Go + * emits 100000000000000000000. + * + *

Float.toString supplies the digits, but is not shortest for subnormals + * — the same defect the double path has with Double.toString. It renders + * Float.MIN_VALUE as "1.4E-45" when "1E-45" round-trips, and Go emits the + * latter. Exhaustively over the float32 subnormals (bits 1..0x7FFFFF), nine + * bit patterns render with more digits than needed — 1, 2, 3, 4, 6, 7, 21, + * 29 and 71 — counted by output bytes changing, the same convention the + * double figure above uses. So the digits go through + * shortestRoundTrip(float) first, which shortens against float precision. + * Placement and thresholds are then identical to the double case, which is + * why this delegates rather than duplicating them. + */ + public static String formatJsonNumber(float f) { + if (Float.isNaN(f) || Float.isInfinite(f)) { + throw new IllegalArgumentException( + "NaN/Infinity has no Go encoding/json representation: " + f); + } + if (f == 0.0f) { + return (Float.floatToRawIntBits(f) == Float.floatToRawIntBits(-0.0f)) ? "-0" : "0"; + } + // Re-parse the shortest float digits as a decimal, then run the same + // placement rules as the double path over exactly those digits. + // + // The threshold is compared against the SHORTENED decimal, not against + // the widened double. Go's floatEncoder tests abs(float64(f)) — but it + // does so after strconv has already produced the 32-bit shortest form, + // and for one float32 near the boundary the two disagree: bits + // 897988541 widens to 9.999999974752427e-07, which is below 1e-6, while + // its shortest float rendering is 1e-06, which is not. Go prints + // 0.000001; comparing the widened double gives 1e-6. + java.math.BigDecimal shortened = + new java.math.BigDecimal(shortestRoundTrip(f)).stripTrailingZeros(); + return formatDecimal(shortened, shortened.abs().doubleValue(), f < 0); + } + /** + * Shortest decimal string that round-trips to {@code d}, which is what Go's + * precision -1 means. + * + *

{@code Double.toString} is NOT that string in general: it renders + * {@code Double.MIN_VALUE} as "4.9E-324" when the single digit "5E-324" + * already round-trips to the same bits, and Go emits the latter. + * + *

Eight bit patterns in bits 1..200000 render with more digits than + * needed, producing eight distinct Double.toString strings (4.9E-324, + * 9.9E-324, 4.9E-323, 5.9E-323, 6.9E-323, 7.9E-323, 8.9E-323, 9.9E-323). + * The count is stated per bit pattern over that scan range, since counting + * by decimal target or over a wider enumeration gives a different number. + * + *

So search: try one significant digit, then two, and return the first + * rendering that parses back to the identical double. The first hit is by + * construction the shortest, since the candidates are generated in + * increasing length. + * + *

Deliberately unoptimized. Earlier versions added a fast path that + * skipped the search when Double.toString was already minimal, guarded by a + * digit count. Three review rounds each found a different input class that + * the guard silently excluded — integer-valued doubles, then everything + * below 1.0 — because the digit count and the benchmark sample disagreed + * about which values mattered. Each iteration was correct on output and + * wrong on the claim in its own comment. The loop below cannot be wrong + * about which inputs it covers, because it covers all of them the same way. + * If this ever needs to be faster, benchmark [0.001,1), [1,1000) and + * integer-valued separately: a sample drawn from any one of them will + * confirm whatever you already believe. + */ + private static String shortestRoundTrip(double d) { + String repr = Double.toString(d); + // Shorten only the digits Double.toString chose. Rounding the exact + // binary value instead (BigDecimal(double) with a MathContext) picks a + // different last digit for some values, because MathContext rounds + // HALF_UP on the true expansion while Go's shortest algorithm reports + // the digit nearest the double: 2209012388886329.2 in Go against + // ...329.3 that way. The count depends entirely on how you sample: + // 11-13 over 200k uniform random bit patterns, and far more when drawing + // by magnitude. The mechanism does not depend on the draw; see + // llmdoc/reference/number-formatting-parity.md. + // Double.toString's digits are already the correct ones; the only thing + // wrong with them is that there can be too many. + java.math.BigDecimal exact = new java.math.BigDecimal(repr); + for (int precision = 1; precision < 17; precision++) { + String candidate = exact.round(new java.math.MathContext(precision)).toString(); + if (Double.parseDouble(candidate) == d) { + return candidate; + } + } + return repr; + } + /** + * Shortest decimal string that round-trips to {@code f} through FLOAT + * precision. Mirrors the double overload, including the reason it operates + * on Float.toString's digits rather than on the exact binary expansion. + */ + private static String shortestRoundTrip(float f) { + String repr = Float.toString(f); + java.math.BigDecimal exact = new java.math.BigDecimal(repr); + for (int precision = 1; precision < 9; precision++) { + String candidate = exact.round(new java.math.MathContext(precision)).toString(); + if (Float.parseFloat(candidate) == f) { + return candidate; + } + } + return repr; + } + /** + * Places the decimal point for an already-shortest set of digits, applying + * Go's encoding/json thresholds. Shared by the double and float paths: the + * bit width only affects which digits are shortest, never where the point + * goes or how the exponent is spelled. + * + * @param bd shortest round-tripping digits for the value + * @param abs magnitude, deciding fixed versus scientific + * @param negative whether to emit a leading '-' (passed separately so -0.0 + * and negative zero-scale values are handled by the caller) + */ + private static String formatDecimal(java.math.BigDecimal bd, double abs, boolean negative) { + boolean scientific = abs < 1e-6 || abs >= 1e21; + if (!scientific) { + return bd.toPlainString(); + } + String digits = bd.unscaledValue().abs().toString(); + int exp10 = digits.length() - bd.scale() - 1; + StringBuilder sb = new StringBuilder(); + if (negative) { + sb.append('-'); + } + sb.append(digits.charAt(0)); + if (digits.length() > 1) { + sb.append('.').append(digits, 1, digits.length()); + } + sb.append('e'); + if (exp10 < 0) { + sb.append('-'); + } else { + sb.append('+'); + } + // No zero-padding branch for positive exponents, deliberately. strconv + // pads exponents to two digits and encoding/json un-pads negatives back + // to one — but the scientific branch is only entered when |d| >= 1e21 or + // |d| < 1e-6, so a positive exponent is never below 21 and is already + // two digits. Verified: across 688k sampled doubles Go never emits a + // single-digit positive exponent, and adding the pad changes no output. + sb.append(Math.abs(exp10)); + return sb.toString(); + } + /** * Replicates Go's strconv.FormatFloat(d, 'f', -1, 64). * Always uses decimal notation (no scientific notation). - * Uses Double.toString for shortest round-trip representation. + * Uses Double.toString, which is shortest-round-trip for normal doubles but + * NOT for subnormals: MIN_VALUE renders "4.9E-324" where "5E-324" + * round-trips, so the plain-decimal expansion here comes out one character + * longer than Go's (327 vs 326). + * + *

KNOWN DIVERGENCE, deliberately not fixed here. The sole caller is + * TransformResourceLookup's key coercion, and a request-supplied 5e-324 does + * survive Jackson parsing and reach it, so this is reachable rather than + * theoretical. All THREE runtimes disagree on that key, not just Java: Go + * emits 326 characters, Java 327, and pine-cpp emits "5e-324" because + * go_format_lookup_key's 64-byte to_chars buffer cannot hold a 326-character + * expansion, so it returns value_too_large and falls back to scientific + * notation. Whoever unifies this must fix the C++ buffer too, not only the + * Java digit count. It is pre-existing and outside issue #180 (which is + * about JSON output bytes), and changing a key-derivation function is a + * behaviour change for anything already keyed on the current form. + * + *

An earlier version of this comment claimed the callers "never see + * subnormals" and listed salt and condition formatting among them. Both were + * wrong: salt uses formatG, conditions use sprint, and the one real caller is + * reachable from a request. formatJsonNumber needs exact Go parity and uses + * shortestRoundTrip instead. */ public static String formatFloatF(double d) { if (Double.doubleToRawLongBits(d) == Double.doubleToRawLongBits(-0.0)) { @@ -243,26 +469,98 @@ private int[] initEsc() { } }); SimpleModule module = new SimpleModule(); - module.addSerializer(Double.class, new StdSerializer(Double.class) { + // Registered for the boxed, primitive and array forms of both widths. + // Jackson dispatches on the declared type, so a Double.class-only + // registration left primitive `double`, double[], and every float form + // on Jackson's default path emitting "1.0E20" — the shape of issue #180. + // + // Scope of this claim, stated precisely because earlier versions of this + // comment overreached: these six registrations cover every carrier that + // a frame value can take on a response path. Frame values are Double or + // Float (pine-go row_frame.go, pine-java DataFrame/ColumnFrame), and + // arrays and primitives are covered so the mapper does not depend on + // which of those forms a caller happens to declare. + // + // NOT covered, deliberately: JsonNode carriers (DoubleNode, FloatNode, + // DecimalNode) and BigDecimal, which still emit Jackson's default form. + // Checked rather than assumed — readTree appears only in Config and + // ResourceManager, both parsing configuration on the way IN, and no + // response is assembled from a JsonNode. If a future change serializes a + // JsonNode outward, these need registering too. + StdSerializer goDoubleSerializer = new StdSerializer(Double.class) { @Override public void serialize(Double value, JsonGenerator gen, SerializerProvider provider) throws IOException { - if (Double.doubleToRawLongBits(value) == Double.doubleToRawLongBits(-0.0)) { - // Go encoding/json preserves the sign bit on negative zero: - // json.Marshal(math.Copysign(0, -1)) emits "-0". - // Jackson's writeNumber(-0.0) emits "-0.0", so we have to - // write the raw literal to match Go byte-for-byte. - gen.writeRawValue("-0"); - } else if (!Double.isNaN(value) && !Double.isInfinite(value) - && value == Math.floor(value) - && value >= -9.007199254740992e15 - && value <= 9.007199254740992e15) { - // Go json.Encoder omits the trailing ".0" for integer-valued - // doubles (e.g. 1.0 → "1") since it serializes via %g/strconv. - // Match that exactly. - gen.writeNumber((long) value.doubleValue()); - } else { - gen.writeNumber(value.doubleValue()); + // Every FINITE value goes through formatJsonNumber. Delegating + // any of those to Jackson's writeNumber is what caused issue + // #180: it formats via Double.toString, so everything the old + // guard did not catch fell out as "1.0E20" where Go emits + // "100000000000000000000". The guard only covered + // integer-valued doubles within +-2^53, i.e. a small slice of + // the range Go renders in plain decimal (up to 1e21). + double d = value.doubleValue(); + if (Double.isNaN(d) || Double.isInfinite(d)) { + // No Go equivalent exists — encoding/json errors out on + // these. Keep Jackson's quoted-string form ("Infinity", + // "-Infinity", "NaN"): it is the only rendering that leaves + // the response parseable, which matters because a request + // carrying 1e400 reaches here without passing the write + // path's NaN/Inf validation. Parity is already broken + // upstream in that case (Go and C++ reject the request + // outright), so the goal here is valid JSON, not byte + // equality with a Go output that does not exist. + gen.writeNumber(d); + return; + } + gen.writeRawValue(formatJsonNumber(d)); + } + }; + module.addSerializer(Double.class, goDoubleSerializer); + module.addSerializer(Double.TYPE, goDoubleSerializer); + // double[] needs its own registration: Jackson serializes primitive + // arrays with a dedicated ArraySerializer that writes elements directly + // rather than delegating to a per-element serializer, so neither of the + // registrations above reaches them. + // Float gets the same three registrations. It is an accepted frame value + // type in all three runtimes (pine-go row_frame.go's `case float32`, + // pine-java DataFrame/ColumnFrame's `instanceof Float`), so a custom + // operator writing one reaches the serializer even though no built-in + // operator does today — the same "closing a hole" reasoning as + // Double.TYPE above, and the reason the type list has to be complete + // rather than just covering the paths that exist. + StdSerializer goFloatSerializer = new StdSerializer(Float.class) { + @Override + public void serialize(Float value, JsonGenerator gen, SerializerProvider provider) + throws IOException { + float f = value.floatValue(); + if (Float.isNaN(f) || Float.isInfinite(f)) { + gen.writeNumber(f); + return; + } + gen.writeRawValue(formatJsonNumber(f)); + } + }; + module.addSerializer(Float.class, goFloatSerializer); + module.addSerializer(Float.TYPE, goFloatSerializer); + module.addSerializer(float[].class, new StdSerializer(float[].class) { + @Override + public void serialize(float[] values, JsonGenerator gen, SerializerProvider provider) + throws IOException { + gen.writeStartArray(); + for (float v : values) { + goFloatSerializer.serialize(v, gen, provider); + } + gen.writeEndArray(); + } + }); + module.addSerializer(double[].class, new StdSerializer(double[].class) { + @Override + public void serialize(double[] values, JsonGenerator gen, SerializerProvider provider) + throws IOException { + gen.writeStartArray(); + for (double v : values) { + goDoubleSerializer.serialize(v, gen, provider); } + gen.writeEndArray(); } }); m.registerModule(module); diff --git a/pine-java/src/test/java/page/liam/pine/GoJsonNumberParityTest.java b/pine-java/src/test/java/page/liam/pine/GoJsonNumberParityTest.java new file mode 100644 index 00000000..ab80b758 --- /dev/null +++ b/pine-java/src/test/java/page/liam/pine/GoJsonNumberParityTest.java @@ -0,0 +1,329 @@ +package page.liam.pine; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.LinkedHashMap; +import java.util.Map; +import org.junit.jupiter.api.Test; + +/** + * Pins pine-java's JSON number output to Go's encoding/json, byte for byte + * (issue #180). + * + *

Every expected string here was produced by running json.Marshal on the + * same float64 in Go, not derived from the spec by hand. Go's rule is + * strconv.FormatFloat(d, 'f'|'e', -1, 64), choosing 'e' when |x| < 1e-6 or + * |x| >= 1e21, with precision -1 meaning shortest round-trip. + * + *

The divergence #180 reported was Java emitting "1.0E20" where Go emits + * "100000000000000000000". The serializer only special-cased integer-valued + * doubles within +-2^53 and let Jackson's writeNumber handle everything else, + * and writeNumber formats via Double.toString. + */ +class GoJsonNumberParityTest { + + private static final ObjectMapper MAPPER = GoFormat.createGoCompatMapper(); + + private static String emit(double d) throws Exception { + Map m = new LinkedHashMap<>(); + m.put("v", d); + String json = MAPPER.writeValueAsString(m); + // {"v":} + return json.substring(json.indexOf(':') + 1, json.length() - 1); + } + + @Test + void plainDecimalRangeKeepsEveryDigit() throws Exception { + assertEquals("0", emit(0.0)); + assertEquals("1", emit(1.0)); + assertEquals("1.5", emit(1.5)); + assertEquals("1000000", emit(1e6)); + assertEquals("1000000000000000", emit(1e15)); + // Past 2^53 but below 1e21: Go stays in plain decimal. This is exactly + // the band the old +-2^53 guard failed to cover. + assertEquals("10000000000000000", emit(1e16)); + assertEquals("1000000000000000000", emit(1e18)); + assertEquals("100000000000000000000", emit(1e20)); + assertEquals("-100000000000000000000", emit(-1e20)); + } + + @Test + void shortestRoundTripNotExactBinaryExpansion() throws Exception { + // 1.0000000000000002e20 is exactly 100000000000000016384, but Go prints + // the shortest round-tripping digits and zero-fills: ...20000. + assertEquals("100000000000000020000", emit(1.0000000000000002e20)); + } + + @Test + void scientificAtOrAbove1e21() throws Exception { + assertEquals("1e+21", emit(1e21)); + assertEquals("1e+22", emit(1e22)); + assertEquals("1.5e+21", emit(1.5e21)); + assertEquals("-1e+21", emit(-1e21)); + } + + @Test + void smallMagnitudeBoundaryAt1eMinus6IsInclusive() throws Exception { + assertEquals("0.00001", emit(1e-5)); + assertEquals("0.000001", emit(1e-6)); + // Below 1e-6 goes scientific. strconv pads the exponent to two digits + // ("1e-07"); encoding/json then strips one leading zero from NEGATIVE + // exponents only, so this is "1e-7" while 1e+21 above keeps "+21". + assertEquals("1e-7", emit(1e-7)); + assertEquals("1e-9", emit(1e-9)); + assertEquals("1e-10", emit(1e-10)); + } + + @Test + void threeDigitExponentsAreNotTrimmed() throws Exception { + assertEquals("1e+100", emit(1e100)); + assertEquals("1e-100", emit(1e-100)); + } + + @Test + void negativeZeroKeepsItsSignBit() throws Exception { + assertEquals("-0", emit(-0.0)); + } + + @Test + void subnormalsUseShortestRoundTripNotDoubleToString() throws Exception { + // Double.toString is documented as emitting enough digits to uniquely + // identify the value, and for normal doubles it is also the shortest + // such rendering. For subnormals it is not: MIN_VALUE comes out as + // "4.9E-324" when "5E-324" already round-trips, and Go emits 5e-324. + // Building the BigDecimal straight from Double.toString inherited that. + assertEquals("5e-324", emit(Double.MIN_VALUE)); + assertEquals("-5e-324", emit(-Double.MIN_VALUE)); + assertEquals("1e-323", emit(Double.longBitsToDouble(2L))); + assertEquals("5e-323", emit(Double.longBitsToDouble(10L))); + // Values Double.toString already renders shortest must not change. + assertEquals("1.5e-323", emit(Double.longBitsToDouble(3L))); + assertEquals("4.4e-323", emit(Double.longBitsToDouble(9L))); + } + + @Test + void shortestRoundTripHoldsAcrossTheSubnormalRange() throws Exception { + // Every candidate must parse back to the identical double, and must be + // no longer than what Double.toString would have produced. + for (long bits = 1; bits <= 20000; bits++) { + double d = Double.longBitsToDouble(bits); + String s = GoFormat.formatJsonNumber(d); + assertEquals(d, Double.parseDouble(s), "round-trip failed for bits " + bits); + } + } + + @Test + void nonFiniteStaysQuotedSoTheResponseRemainsParseable() throws Exception { + // Go's encoding/json refuses NaN/Infinity, so there is no byte sequence + // to match here and byte parity is not the goal — valid JSON is. This + // path is reachable: the write path validates NaN/Inf, but a request + // carrying 1e400 does not go through it, and Jackson coerces that to + // Infinity on parse (Go and C++ reject the request outright instead). + // + // An earlier version of the serializer wrote formatJsonNumber's output + // unconditionally, which emitted a bare +Inf token and made the whole + // response unparseable. + assertEquals("\"Infinity\"", emit(Double.POSITIVE_INFINITY)); + assertEquals("\"-Infinity\"", emit(Double.NEGATIVE_INFINITY)); + assertEquals("\"NaN\"", emit(Double.NaN)); + } + + @Test + void wholeDocumentStaysParseableWithNonFiniteValues() throws Exception { + Map m = new LinkedHashMap<>(); + m.put("inf", Double.POSITIVE_INFINITY); + m.put("nan", Double.NaN); + m.put("ok", 1e20); + String json = MAPPER.writeValueAsString(m); + // Must round-trip through a strict parser. + new ObjectMapper().readTree(json); + org.junit.jupiter.api.Assertions.assertTrue(json.contains("\"inf\":\"Infinity\""), json); + org.junit.jupiter.api.Assertions.assertTrue(json.contains("\"ok\":100000000000000000000"), json); + } + + @Test + void formatJsonNumberRefusesNonFiniteRatherThanInventingBytes() { + for (double d : new double[] {Double.NaN, Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY}) { + org.junit.jupiter.api.Assertions.assertThrows(IllegalArgumentException.class, + () -> GoFormat.formatJsonNumber(d), + "formatJsonNumber must not fabricate a representation for " + d); + } + } + + @Test + void integerValuedDoublesDropTheFractionalPart() throws Exception { + // Double.toString writes "1.0"; Go writes "1". + assertEquals("1", emit(1.0)); + assertEquals("42", emit(42.0)); + assertEquals("100", emit(100.0)); + assertEquals("100000000000000000000", emit(1e20)); + assertEquals("-42", emit(-42.0)); + } + + @Test + void negativeNaNNormalizesLikePositiveNaN() throws Exception { + // The isNaN guard's only observable effect: without it -NaN renders as + // "-nan" in C++ and would diverge here too. + assertEquals("\"NaN\"", emit(Double.longBitsToDouble(0xFFF8000000000000L))); + } + + @Test + void digitsComeFromDoubleToStringNotFromRoundingTheExactValue() throws Exception { + // Shortening must operate on the digits Double.toString chose. Rounding + // the exact binary expansion instead (BigDecimal(double) plus a + // MathContext) selects a different final digit for some values, because + // MathContext rounds HALF_UP on the true value while Go reports the + // digit nearest the double. These four all came out one ulp-of-the-last + // -digit high that way. The count depends on how you sample: 13 over + // 200k uniform random bit patterns, 50+ when sampling by magnitude. + // The mechanism and these four values do not depend on the draw. + assertEquals("2209012388886329.2", emit(Double.longBitsToDouble(0x431f64571af9dce5L))); + assertEquals("-1300666636127457.2", emit(Double.longBitsToDouble(0xc3127bcc33453385L))); + assertEquals("897344844809170.2", emit(Double.longBitsToDouble(0x4309810b05ba1e92L))); + assertEquals("171744423733713.12", emit(Double.longBitsToDouble(0x42e3866babcd3a24L))); + } + + @Test + void formatFloatFSubnormalDivergenceIsPinnedNotFixed() throws Exception { + // formatFloatF still uses Double.toString, so a subnormal expands one + // character longer than Go's (327 vs 326). Documented as a known + // divergence rather than fixed: its only caller is resource-lookup key + // coercion, and changing a key-derivation function is a behaviour change + // for anything already keyed on the current form. Pinned here so the + // number is a recorded fact rather than a surprise, and so that fixing + // it later is a deliberate act with a failing test to update. + assertEquals(327, GoFormat.formatFloatF(Double.MIN_VALUE).length()); + // formatJsonNumber, which does need Go parity, is unaffected. + assertEquals("5e-324", emit(Double.MIN_VALUE)); + } + + @Test + void smallPlainDecimalsRenderExactly() throws Exception { + // Values just above the 1e-6 threshold, where Double.toString writes + // placeholder zeros. This asserts the rendered bytes, not the internal + // digit count: over-counting significant digits only widens + // shortestRoundTrip's search and yields the same result, so there is no + // observable property to assert about the count itself. + assertEquals("0.001234", emit(0.001234)); + assertEquals("0.0001", emit(0.0001)); + assertEquals("0.001", emit(0.001)); + assertEquals("0.000001", emit(1e-6)); + } + + @Test + void boxedPrimitiveAndArrayDoublesAllUseTheGoFormatter() throws Exception { + // Jackson dispatches on the declared type. A Double.class-only + // registration left primitive double fields and double[] on Jackson's + // default path, emitting "1.0E20" — the exact shape of issue #180. + // Nothing on /execute reaches those today (Variant boxes everything), + // but the mapper should be right regardless of which paths exist. + // Asserted per field rather than as a whole document: Jackson's key + // order is not Go's (issue #183), which is a separate matter from the + // number bytes under test here. + String json = MAPPER.writeValueAsString(new PrimitiveHolder()); + org.junit.jupiter.api.Assertions.assertTrue( + json.contains("\"boxed\":100000000000000000000"), json); + org.junit.jupiter.api.Assertions.assertTrue( + json.contains("\"primitive\":100000000000000000000"), json); + org.junit.jupiter.api.Assertions.assertTrue( + json.contains("\"array\":[100000000000000000000,1e+21]"), json); + } + + /** Exercises all three declared shapes Jackson dispatches on separately. */ + public static final class PrimitiveHolder { + public Double getBoxed() { + return 1e20; + } + + public double getPrimitive() { + return 1e20; + } + + public double[] getArray() { + return new double[] {1e20, 1e21}; + } + } + + @Test + void float32UsesThirtyTwoBitShortestRoundTrip() throws Exception { + // Go formats float32 with bitSize=32, so the digits are shortest for + // FLOAT, not for double. Widening first surfaces the binary noise the + // narrower type was hiding: (double) 0.1f is 0.10000000149011612 and + // 1e20f widens to 100000002004087730000, where Go emits 0.1 and + // 100000000000000000000. + assertEquals("0.1", GoFormat.formatJsonNumber(0.1f)); + assertEquals("100000000000000000000", GoFormat.formatJsonNumber(1e20f)); + assertEquals("1e-7", GoFormat.formatJsonNumber(1e-7f)); + assertEquals("3.4e+38", GoFormat.formatJsonNumber(3.4e38f)); + assertEquals("-0", GoFormat.formatJsonNumber(-0.0f)); + // Float.toString is not shortest for subnormals either: it renders + // MIN_VALUE as "1.4E-45" where "1E-45" round-trips through float. + assertEquals("1e-45", GoFormat.formatJsonNumber(Float.MIN_VALUE)); + assertEquals("3e-45", GoFormat.formatJsonNumber(Float.intBitsToFloat(2))); + // The 1e-6 threshold is applied to the SHORTENED decimal, not the + // widened double: this value widens to 9.999999974752427e-07 (below the + // threshold) but shortens to 1e-06 (not below), and Go prints plain. + assertEquals("0.000001", GoFormat.formatJsonNumber(Float.intBitsToFloat(897988541))); + } + + @Test + void floatShapesAllUseTheGoFormatter() throws Exception { + String json = MAPPER.writeValueAsString(new FloatHolder()); + org.junit.jupiter.api.Assertions.assertTrue( + json.contains("\"boxed\":100000000000000000000"), json); + // 1e-7f, not 0.1f: Jackson's default writeNumber(float) also emits "0.1", + // so that value cannot tell the registered path from the default one and + // the assertion had no teeth. Go renders float32 1e-7 as "1e-7" while + // Jackson gives "1.0E-7". + org.junit.jupiter.api.Assertions.assertTrue(json.contains("\"primitive\":1e-7"), json); + org.junit.jupiter.api.Assertions.assertTrue( + json.contains("\"array\":[100000000000000000000,1e-7]"), json); + } + + /** float counterpart of PrimitiveHolder. */ + public static final class FloatHolder { + public Float getBoxed() { + return 1e20f; + } + + public float getPrimitive() { + return 1e-7f; + } + + public float[] getArray() { + return new float[] {1e20f, 1e-7f}; + } + } + + @Test + void jsonNodeCarriersAreDocumentedAsUncovered() throws Exception { + // Pins the boundary of the type coverage rather than the coverage + // itself. DoubleNode and BigDecimal bypass the registered serializers, + // which is acceptable only because no response is assembled from a + // JsonNode (readTree appears only in Config and ResourceManager, both + // parsing input). This asserts the current uncovered behaviour so that + // if someone later serializes a JsonNode outward, they meet a failing + // test that points at the comment explaining what to register. + com.fasterxml.jackson.databind.node.DoubleNode node = + com.fasterxml.jackson.databind.node.DoubleNode.valueOf(1e20); + assertEquals("1.0E20", MAPPER.writeValueAsString(node)); + assertEquals("1E+20", MAPPER.writeValueAsString(new java.math.BigDecimal("1e20"))); + // The covered carriers, for contrast. + assertEquals("100000000000000000000", emit(1e20)); + } + + @Test + void formatJsonNumberMatchesTheSerializer() throws Exception { + // The serializer must not carry its own second copy of the rule. + double[] vals = { + 0.0, -0.0, 1.0, 1.5, 1e6, 1e15, 1e16, 1e18, 1e20, 1e21, 1e22, + 1.5e21, -1e20, -1e21, 1e-5, 1e-6, 1e-7, 1e-9, 1e-10, 1e100, 1e-100, + 1.0000000000000002e20, 3.141592653589793, -2.718281828459045, + }; + for (double d : vals) { + assertEquals(GoFormat.formatJsonNumber(d), emit(d), + "serializer diverged from formatJsonNumber for " + d); + } + } +}