Skip to content

Commit c59f86b

Browse files
authored
fix: avoid linear objectHas scans for large inline imports (#1070)
Motivation: PR #1061 was closed against the obsolete `fixjson` base, but the issue still needs a clean PR on the current upstream `master`. PR #1060 added the `InlineScanMax` fallback for `valueRaw`, yet repeated `std.objectHas` / `std.objectHasAll` checks over large imported strict-JSON objects still reached `containsVisibleKey` / `containsKey` and scanned uncapped inline field arrays. That kept membership-heavy workloads at O(N^2). Modification: - Rebase the PR #1061 follow-up onto current `upstream/master` (`1db39dde`). - Apply `Obj.InlineScanMax` to `Val.Obj.containsKey` and `Val.Obj.containsVisibleKey`, so large inline objects fall through to the existing lazy `getValue0` `LinkedHashMap` path. - Reuse `Obj.InlineScanMax` in inline-object producers, keeping construction and lookup thresholds on one source of truth. - Add a benchmark regression covering repeated `std.objectHas` / `std.objectHasAll` calls over imported JSON object fields. Result: | Benchmark | Current `master` | This branch | Change | | --- | ---: | ---: | ---: | | repeated `std.objectHas` / `std.objectHasAll` over large imported JSON object fields | `97.942 ms/op` | `21.117 ms/op` | `4.64x` faster (`-78%`) | Large inline imported objects now pay one lazy map build above the scan threshold, then reuse O(1) map probes for repeated membership checks. Small inline objects keep the cheap linear scan path. References: - Replaces #1061. - Follow-up to #1060.
1 parent dda53fa commit c59f86b

5 files changed

Lines changed: 25 additions & 10 deletions

File tree

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
// Regression guard for large imported JSON objects queried repeatedly via objectHas/objectHasAll.
2+
// Large imported objects may use uncapped inline arrays, so repeated membership checks must use
3+
// the cached lookup map above the inline scan threshold instead of doing O(N) scans per key.
4+
local data = import "json_import_merge_data.json";
5+
std.assertEqual(
6+
[
7+
std.length([k for k in std.objectFields(d) if std.objectHas(d, k)]) +
8+
std.length([k for k in std.objectFieldsAll(d) if std.objectHasAll(d, k)]) +
9+
(if std.objectHas(d, "missing") || std.objectHasAll(d, "missing") then 1 else 0)
10+
for d in data
11+
],
12+
[2000 for d in data]
13+
)
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
true

sjsonnet/src/sjsonnet/Evaluator.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1783,7 +1783,7 @@ class Evaluator(
17831783
var inlineKeys: Array[String] = null
17841784
var inlineMembers: Array[Val.Obj.Member] = null
17851785
var fieldCount = 0
1786-
val maxInlineFields = 8
1786+
val maxInlineFields = Val.Obj.InlineScanMax
17871787

17881788
// Shared field-tracking logic: manages singleKey → inlineKeys → builder transitions.
17891789
// Handles duplicate key detection at each tier.

sjsonnet/src/sjsonnet/Val.scala

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1816,12 +1816,11 @@ object Val {
18161816
private[sjsonnet] var currentDebugStats: DebugStats = _
18171817

18181818
/**
1819-
* Field count at/below which `valueRaw` resolves an inline-array object by linear scan. Above
1820-
* it (e.g. large imported strict-JSON objects, whose inline arrays are uncapped — see
1821-
* Importer), `valueRaw` instead lazily builds the `value0` LinkedHashMap via `getValue0` for
1822-
* O(1) lookups. This keeps the scan for small objects (and never builds a map for the
1823-
* import-and-emit path, which materializes via direct inline iteration) while avoiding O(N^2)
1824-
* when a large imported object gains a `super` and must be resolved per-key.
1819+
* Field count at/below which inline-array object key lookups use a linear scan. Above it (e.g.
1820+
* large imported strict-JSON objects, whose inline arrays are uncapped — see Importer), lookups
1821+
* lazily build the `value0` LinkedHashMap via `getValue0` for O(1) probes. This keeps the scan
1822+
* for small objects (and never builds a map for the import-and-emit path, which materializes
1823+
* via direct inline iteration) while avoiding O(N^2) repeated per-key lookups.
18251824
*/
18261825
private[sjsonnet] final val InlineScanMax = 8
18271826

@@ -2301,7 +2300,9 @@ object Val {
23012300

23022301
@inline def containsKey(k: String): Boolean = {
23032302
if (singleFieldKey != null && `super` == null) singleFieldKey.equals(k)
2304-
else if (inlineFieldKeys != null && `super` == null) {
2303+
else if (
2304+
inlineFieldKeys != null && `super` == null && inlineFieldKeys.length <= Obj.InlineScanMax
2305+
) {
23052306
val keys = inlineFieldKeys
23062307
val n = keys.length
23072308
var i = 0
@@ -2319,7 +2320,7 @@ object Val {
23192320
@inline def containsVisibleKey(k: String): Boolean = {
23202321
if (static || `super` != null) {
23212322
getAllKeys.get(k) == java.lang.Boolean.FALSE
2322-
} else if (inlineFieldKeys != null) {
2323+
} else if (inlineFieldKeys != null && inlineFieldKeys.length <= Obj.InlineScanMax) {
23232324
val keys = inlineFieldKeys
23242325
val members = inlineFieldMembers
23252326
val n = keys.length

sjsonnet/src/sjsonnet/stdlib/ObjectModule.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -209,7 +209,7 @@ object ObjectModule extends AbstractFunctionModule {
209209
singleFieldKey = keys(0),
210210
singleFieldMember = new MapWithKeyMember(func, obj, keys(0), pos)
211211
)
212-
} else if (n <= 8) {
212+
} else if (n <= Val.Obj.InlineScanMax) {
213213
val members = new Array[Val.Obj.Member](n)
214214
var i = 0
215215
while (i < n) {

0 commit comments

Comments
 (0)