fix(plugins): correct plugin-storage query/count/index/order on Postgres

.changeset/plugin-storage-postgres-jsonb.md

@@ -0,0 +1,7 @@
+---
+"emdash": patch
+---
+
+Fixes plugin storage `query()`, `count()`, ordering, and index creation on Postgres. Queries and unique/expression indexes on stored JSON fields previously failed with an "operator does not exist" error; numeric range/equality/`in` filters compared values as text (so `stock >= 10` also matched `9`), causing over-counting; and `orderBy` on a numeric field sorted lexically (`10, 100, 9`) instead of numerically. Numeric guards now compare and order numerically, and counts are returned as numbers.
+
+Numeric filters are now type-guarded on both dialects: a stored value that isn't a JSON number is excluded from a numeric comparison (evaluates to no-match) instead of being compared as text — previously such a value could match, and on Postgres an untyped cast would have thrown. Numeric predicates on Postgres are served by a sequential scan (the per-field expression index is text-typed, since field types aren't known when indexes are created). SQLite behavior is otherwise unchanged.

packages/core/src/database/dialect-helpers.ts

@@ -229,3 +229,74 @@ export function jsonExtractExpr(db: Kysely<any>, column: string, path: string):
 	}
 	return `json_extract(${column}, '$.${path}')`;
 }
+
+/**
+ * SQL expression for extracting a field from the plugin-storage `data` column.
+ *
+ * Unlike `jsonExtractExpr` (for real `json`/`jsonb` content columns),
+ * `_plugin_storage.data` is a plain `text` column. On Postgres the JSON
+ * operator `->>` has no overload for `text` — `text ->> 'x'` raises
+ * `operator does not exist: text ->> unknown` — so the column must be cast to
+ * `jsonb` first. The extracted value is still `text`, so a numeric comparison
+ * (`stock >= 10`) would compare lexically (`'9' >= '10'` is TRUE) and silently
+ * over-count / oversell; pass `{ numeric: true }` for a numeric comparison.
+ *
+ * The numeric form is a **type-guarded** cast, not a bare `::numeric`. A bare
+ * cast throws `invalid input syntax for type numeric` on Postgres the moment a
+ * single scanned row stores a non-number in that field (documents are
+ * schemaless), aborting the whole query — and it would diverge from SQLite,
+ * which silently coerces. Guarding with `jsonb_typeof`/`json_type` makes the
+ * comparison **total and parity-correct on both dialects**: a non-number
+ * stored value yields `NULL` (no match) instead of an error.
+ *
+ * The field name is validated (`/^[a-zA-Z][a-zA-Z0-9_]*$/`) before
+ * interpolation, so the casts wrap only a safe identifier and add no injection
+ * surface.
+ *
+ * sqlite text:      json_extract(data, '$.field')
+ * sqlite numeric:   CASE WHEN json_type(data, '$.field') IN ('integer', 'real')
+ *                     THEN json_extract(data, '$.field') END
+ * postgres text:    (data::jsonb)->>'field'
+ * postgres numeric: CASE WHEN jsonb_typeof((data::jsonb)->'field') = 'number'
+ *                     THEN ((data::jsonb)->>'field')::numeric END
+ */
+// eslint-disable-next-line @typescript-eslint/no-explicit-any -- accepts any Kysely instance
+export function pluginDataExtractExpr(
+	db: Kysely<any>,
+	field: string,
+	options?: { numeric?: boolean },
+): string {
+	validateJsonFieldName(field, "plugin storage field name");
+	if (isPostgres(db)) {
+		const text = `(data::jsonb)->>'${field}'`;
+		if (!options?.numeric) return text;
+		return `CASE WHEN jsonb_typeof((data::jsonb)->'${field}') = 'number' THEN (${text})::numeric END`;
+	}
+	const extract = `json_extract(data, '$.${field}')`;
+	if (!options?.numeric) return extract;
+	return `CASE WHEN json_type(data, '$.${field}') IN ('integer', 'real') THEN ${extract} END`;
+}
+
+/**
+ * SQL expression for ordering plugin-storage rows by a `data` field.
+ *
+ * `ORDER BY` has no bound operand to infer numeric-vs-text from, so extracting
+ * as text (`->>'field'`) would sort a numeric field lexically on Postgres
+ * (`[10, 100, 9]`) while SQLite's `json_extract` sorts it numerically —
+ * a cross-dialect divergence. Ordering over the **jsonb-native value**
+ * (`->'field'`, note the single arrow) fixes this: the jsonb btree ordering is
+ * numeric among numbers, lexical among strings, and total across heterogeneous
+ * values (it never throws). SQLite's `json_extract` already orders numerically,
+ * so it is unchanged.
+ *
+ * sqlite:   json_extract(data, '$.field')
+ * postgres: (data::jsonb)->'field'
+ */
+// eslint-disable-next-line @typescript-eslint/no-explicit-any -- accepts any Kysely instance
+export function pluginDataOrderExpr(db: Kysely<any>, field: string): string {
+	validateJsonFieldName(field, "plugin storage order field name");
+	if (isPostgres(db)) {
+		return `(data::jsonb)->'${field}'`;
+	}
+	return `json_extract(data, '$.${field}')`;
+}

packages/core/src/database/repositories/plugin-storage.ts

@@ -7,15 +7,15 @@
  * @see PLUGIN-SYSTEM.md § Plugin Storage > Full API Reference
  */
 
-import type { Kysely } from "kysely";
+import type { Kysely, RawBuilder, SqlBool } from "kysely";
 import { sql } from "kysely";
 
 import {
 	buildWhereClause,
 	validateWhereClause,
 	validateOrderByClause,
 	getIndexedFields,
-	jsonExtract,
+	jsonOrderExtract,
 } from "../../plugins/storage-query.js";
 import type {
 	StorageCollection,
@@ -27,6 +27,37 @@ import { withTransaction } from "../transaction.js";
 import type { Database } from "../types.js";
 import { encodeCursor, decodeCursor } from "./types.js";
 
+/**
+ * Turn a `buildWhereClause` result (`?`-placeholder SQL + ordered params) into a
+ * single boolean expression suitable for Kysely's `.where()`.
+ *
+ * The `?` placeholders are spliced back into value fragments (`sql`${param}``,
+ * which parameterizes safely) interleaved with the raw SQL between them. The
+ * whole condition is returned as a boolean expression and passed directly to
+ * `.where()` — it must NOT be wrapped in an `= 1` comparison. Postgres parses
+ * `<cond> = 1` as a chained comparison (`a >= $1 = 1`), a syntax error, and even
+ * parenthesized `(a >= $1) = 1` is `boolean = integer`, which Postgres rejects.
+ * A bare boolean expression is valid on both SQLite and Postgres.
+ */
+function buildRawWhereExpression(whereResult: {
+	sql: string;
+	params: unknown[];
+}): RawBuilder<SqlBool> {
+	const parts: RawBuilder<unknown>[] = [];
+	let paramIndex = 0;
+	const sqlParts = whereResult.sql.split("?");
+	for (let i = 0; i < sqlParts.length; i++) {
+		if (i > 0) {
+			parts.push(sql`${whereResult.params[paramIndex++]}`);
+		}
+		const chunk = sqlParts[i];
+		if (chunk) {
+			parts.push(sql.raw(chunk));
+		}
+	}
+	return sql<SqlBool>`${sql.join(parts, sql.raw(""))}`;
+}
+
 /**
  * Plugin Storage Repository
  *
@@ -211,19 +242,7 @@ export class PluginStorageRepository<T = unknown> implements StorageCollection<T
 		// Add JSON extraction WHERE conditions
 		const whereResult = buildWhereClause(this.db, where);
 		if (whereResult.sql) {
-			// Use sql template to add the raw WHERE conditions with params
-			const whereSqlParts: ReturnType<typeof sql>[] = [];
-			let paramIndex = 0;
-			const sqlParts = whereResult.sql.split("?");
-			for (let i = 0; i < sqlParts.length; i++) {
-				if (i > 0) {
-					whereSqlParts.push(sql`${whereResult.params[paramIndex++]}`);
-				}
-				if (sqlParts[i]) {
-					whereSqlParts.push(sql.raw(sqlParts[i]));
-				}
-			}
-			query = query.where(({ eb }) => eb(sql.join(whereSqlParts, sql.raw("")), "=", sql.raw("1")));
+			query = query.where(buildRawWhereExpression(whereResult));
 		}
 
 		// Handle cursor-based pagination — throws on invalid cursor.
@@ -237,7 +256,9 @@ export class PluginStorageRepository<T = unknown> implements StorageCollection<T
 		// Build ORDER BY using sql template
 		if (Object.keys(orderBy).length > 0) {
 			for (const [field, direction] of Object.entries(orderBy)) {
-				const extract = jsonExtract(this.db, field);
+				// Order over the jsonb-native value on Postgres so numeric fields sort
+				// numerically, not lexically. See pluginDataOrderExpr.
+				const extract = jsonOrderExtract(this.db, field);
 				const orderExpr =
 					direction === "desc" ? sql`${sql.raw(extract)} desc` : sql`${sql.raw(extract)} asc`;
 				query = query.orderBy(orderExpr);
@@ -289,26 +310,14 @@ export class PluginStorageRepository<T = unknown> implements StorageCollection<T
 		if (where && Object.keys(where).length > 0) {
 			const whereResult = buildWhereClause(this.db, where);
 			if (whereResult.sql) {
-				// Use sql template to add the raw WHERE conditions with params
-				const whereSqlParts: ReturnType<typeof sql>[] = [];
-				let paramIndex = 0;
-				const sqlParts = whereResult.sql.split("?");
-				for (let i = 0; i < sqlParts.length; i++) {
-					if (i > 0) {
-						whereSqlParts.push(sql`${whereResult.params[paramIndex++]}`);
-					}
-					if (sqlParts[i]) {
-						whereSqlParts.push(sql.raw(sqlParts[i]));
-					}
-				}
-				query = query.where(({ eb }) =>
-					eb(sql.join(whereSqlParts, sql.raw("")), "=", sql.raw("1")),
-				);
+				query = query.where(buildRawWhereExpression(whereResult));
 			}
 		}
 
 		const result = await query.executeTakeFirst();
-		return result?.count ?? 0;
+		// Postgres returns COUNT(*) (bigint) as a string via node-postgres; coerce
+		// so this always satisfies its Promise<number> contract on both dialects.
+		return Number(result?.count ?? 0);
 	}
 }
 

packages/core/src/plugins/storage-indexes.ts

@@ -9,7 +9,7 @@
 import type { Kysely, RawBuilder } from "kysely";
 import { sql } from "kysely";
 
-import { jsonExtractExpr, isPostgres } from "../database/dialect-helpers.js";
+import { pluginDataExtractExpr, isPostgres } from "../database/dialect-helpers.js";
 import type { Database } from "../database/types.js";
 import {
 	validateIdentifier,
@@ -58,13 +58,25 @@ export function generateCreateIndexSql(
 
 	// Build the indexed expressions
 	// Fields are validated above, safe to interpolate into json path
+	//
+	// The index expression is TEXT (`->>`), matching equality/startsWith/text
+	// predicates. Field TYPES are not known at index-creation time — the plugin
+	// manifest declares field names, not types — so we cannot build a numeric
+	// index speculatively. Consequence on Postgres: a numeric predicate compiles
+	// to a type-guarded `::numeric` expression (see pluginDataExtractExpr) that a
+	// text btree can't satisfy, so numeric range/equality guards fall back to a
+	// sequential scan. Acceptable for the small per-plugin/collection document
+	// sets this store targets; revisit with a typed-field manifest if needed.
 	const expressions = fields
 		.map((field) => {
+			// Index the extracted value as text (no numeric cast): uniqueness is
+			// defined on the textual JSON value, and on Postgres the `data` column
+			// is `text`, so pluginDataExtractExpr adds the required `::jsonb` cast.
 			if (isPostgres(db)) {
 				// Postgres expression indexes need parens around the expression
-				return `(${jsonExtractExpr(db, "data", field)})`;
+				return `(${pluginDataExtractExpr(db, field)})`;
 			}
-			return jsonExtractExpr(db, "data", field);
+			return pluginDataExtractExpr(db, field);
 		})
 		.join(", ");
 

packages/core/src/plugins/storage-query.ts

@@ -8,8 +8,7 @@
 
 import type { Kysely } from "kysely";
 
-import { jsonExtractExpr } from "../database/dialect-helpers.js";
-import { validateJsonFieldName } from "../database/validate.js";
+import { pluginDataExtractExpr, pluginDataOrderExpr } from "../database/dialect-helpers.js";
 import type { WhereClause, WhereValue, RangeFilter, InFilter, StartsWithFilter } from "./types.js";
 
 /**
@@ -117,15 +116,33 @@ export function validateOrderByClause(
 }
 
 /**
- * SQL expression for extracting JSON field.
+ * SQL expression for extracting a queryable field from the `_plugin_storage.data`
+ * column.
  *
- * Validates the field name before interpolation to prevent SQL injection
- * via crafted JSON path expressions.
+ * Delegates to `pluginDataExtractExpr`, which validates the field name before
+ * interpolation and applies the dialect-correct extraction: a `::jsonb` cast on
+ * Postgres (the `data` column is `text`) plus an optional `::numeric` cast so
+ * numeric comparisons don't fall back to lexical text ordering.
  */
 // eslint-disable-next-line @typescript-eslint/no-explicit-any -- accepts any Kysely instance
-export function jsonExtract(db: Kysely<any>, field: string): string {
-	validateJsonFieldName(field, "query field name");
-	return jsonExtractExpr(db, "data", field);
+export function jsonExtract(
+	db: Kysely<any>,
+	field: string,
+	options?: { numeric?: boolean },
+): string {
+	return pluginDataExtractExpr(db, field, options);
+}
+
+/**
+ * SQL expression for ordering by a `_plugin_storage.data` field.
+ *
+ * Delegates to `pluginDataOrderExpr`, which orders over the jsonb-native value
+ * on Postgres so numeric fields sort numerically (not lexically) while staying
+ * total across heterogeneous data. SQLite keeps `json_extract` (already numeric).
+ */
+// eslint-disable-next-line @typescript-eslint/no-explicit-any -- accepts any Kysely instance
+export function jsonOrderExtract(db: Kysely<any>, field: string): string {
+	return pluginDataOrderExpr(db, field);
 }
 
 /**
@@ -137,33 +154,44 @@ export function buildCondition(
 	field: string,
 	value: WhereValue,
 ): { sql: string; params: unknown[] } {
-	const extract = jsonExtract(db, field);
+	// Numeric-vs-text is decided per condition from the JS type of the bound
+	// value. On Postgres a text extract compared to a bound number would sort
+	// lexically (`'9' >= '10'` is TRUE); a `::numeric` cast on the extract fixes
+	// it. String/boolean operands keep text comparison. SQLite is unaffected —
+	// `json_extract` already returns a typed value.
+	const extractFor = (numeric: boolean): string => jsonExtract(db, field, { numeric });
 
 	if (value === null) {
-		return { sql: `${extract} IS NULL`, params: [] };
+		return { sql: `${extractFor(false)} IS NULL`, params: [] };
 	}
 
-	if (typeof value === "string" || typeof value === "number") {
-		return { sql: `${extract} = ?`, params: [value] };
+	if (typeof value === "number") {
+		return { sql: `${extractFor(true)} = ?`, params: [value] };
+	}
+
+	if (typeof value === "string") {
+		return { sql: `${extractFor(false)} = ?`, params: [value] };
 	}
 
 	if (typeof value === "boolean") {
 		// JSON booleans are stored as true/false strings
-		return { sql: `${extract} = ?`, params: [value] };
+		return { sql: `${extractFor(false)} = ?`, params: [value] };
 	}
 
 	if (isInFilter(value)) {
+		const numeric = value.in.length > 0 && value.in.every((v) => typeof v === "number");
 		const placeholders = value.in.map(() => "?").join(", ");
 		return {
-			sql: `${extract} IN (${placeholders})`,
+			sql: `${extractFor(numeric)} IN (${placeholders})`,
 			params: value.in,
 		};
 	}
 
 	if (isStartsWithFilter(value)) {
-		// ESCAPE '\' works on both SQLite and PostgreSQL.
+		// ESCAPE '\' works on both SQLite and PostgreSQL. startsWith is a string
+		// operation, so always compare as text.
 		return {
-			sql: `${extract} LIKE ? ESCAPE '\\'`,
+			sql: `${extractFor(false)} LIKE ? ESCAPE '\\'`,
 			params: [`${escapeLikePattern(value.startsWith)}%`],
 		};
 	}
@@ -172,22 +200,17 @@ export function buildCondition(
 		const conditions: string[] = [];
 		const params: unknown[] = [];
 
-		if (value.gt !== undefined) {
-			conditions.push(`${extract} > ?`);
-			params.push(value.gt);
-		}
-		if (value.gte !== undefined) {
-			conditions.push(`${extract} >= ?`);
-			params.push(value.gte);
-		}
-		if (value.lt !== undefined) {
-			conditions.push(`${extract} < ?`);
-			params.push(value.lt);
-		}
-		if (value.lte !== undefined) {
-			conditions.push(`${extract} <= ?`);
-			params.push(value.lte);
-		}
+		// Each bound is cast to numeric only when its own operand is a number, so
+		// a mixed range (e.g. a string lower bound) stays correct per side.
+		const pushBound = (op: string, bound: string | number): void => {
+			conditions.push(`${extractFor(typeof bound === "number")} ${op} ?`);
+			params.push(bound);
+		};
+
+		if (value.gt !== undefined) pushBound(">", value.gt);
+		if (value.gte !== undefined) pushBound(">=", value.gte);
+		if (value.lt !== undefined) pushBound("<", value.lt);
+		if (value.lte !== undefined) pushBound("<=", value.lte);
 
 		return {
 			sql: conditions.join(" AND "),
@@ -239,7 +262,7 @@ export function buildOrderByClause(
 	const clauses: string[] = [];
 
 	for (const [field, direction] of Object.entries(orderBy)) {
-		clauses.push(`${jsonExtract(db, field)} ${direction.toUpperCase()}`);
+		clauses.push(`${jsonOrderExtract(db, field)} ${direction.toUpperCase()}`);
 	}
 
 	if (clauses.length === 0) {