From 0c3782ea3f3ce0ae9b212bea7053aa5309a0eaee Mon Sep 17 00:00:00 2001 From: Isaac Van Doren Date: Thu, 16 Apr 2026 15:00:52 -0500 Subject: [PATCH 01/12] Add issues: write permission to notify job --- .github/workflows/schema-compat.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/schema-compat.yml b/.github/workflows/schema-compat.yml index 0eb71b4..cc0bf75 100644 --- a/.github/workflows/schema-compat.yml +++ b/.github/workflows/schema-compat.yml @@ -91,6 +91,8 @@ jobs: name: Notify on Failure runs-on: ubuntu-latest needs: [schema-compat, integration-tests] + permissions: + issues: write if: failure() steps: - name: Create GitHub Issue From b25d8978ee28f5d6c6771348916d9a2b7c27591a Mon Sep 17 00:00:00 2001 From: Isaac Van Doren Date: Thu, 16 Apr 2026 15:02:04 -0500 Subject: [PATCH 02/12] test workflow on PR --- .github/workflows/schema-compat.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/schema-compat.yml b/.github/workflows/schema-compat.yml index cc0bf75..f5bfbaa 100644 --- a/.github/workflows/schema-compat.yml +++ b/.github/workflows/schema-compat.yml @@ -17,6 +17,9 @@ on: version: description: "ParadeDB version to check against" required: true + pull_request: + branches: [main] + concurrency: group: schema-compat-${{ github.head_ref || github.ref }} From fcd77404317c46fe23975a43f6f1622c5faf4500 Mon Sep 17 00:00:00 2001 From: Isaac Van Doren Date: Thu, 16 Apr 2026 15:04:11 -0500 Subject: [PATCH 03/12] Revert "test workflow on PR" This reverts commit 2d42fbf4b6d6472e65fc7553b3f117cd35fc9996. --- .github/workflows/schema-compat.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/schema-compat.yml b/.github/workflows/schema-compat.yml index f5bfbaa..cc0bf75 100644 --- a/.github/workflows/schema-compat.yml +++ b/.github/workflows/schema-compat.yml @@ -17,9 +17,6 @@ on: version: description: "ParadeDB version to check against" required: true - pull_request: - branches: [main] - concurrency: group: schema-compat-${{ github.head_ref || github.ref }} From bbd126f537990b8d0b7ca5ba63996dc8777ce08a Mon Sep 17 00:00:00 2001 From: Isaac Van Doren Date: Thu, 16 Apr 2026 15:12:56 -0500 Subject: [PATCH 04/12] Update apiignore.json --- apiignore.json | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/apiignore.json b/apiignore.json index 6e049eb..1dbd20f 100644 --- a/apiignore.json +++ b/apiignore.json @@ -73,6 +73,16 @@ "pdb.whitespace_out", "pdb.whitespace_recv", "pdb.whitespace_send", + "pdb.edge_ngram_in", + "pdb.edge_ngram_out", + "pdb.edge_ngram_recv", + "pdb.edge_ngram_send", + "pdb.json_to_edge_ngram", + "pdb.jsonb_to_edge_ngram", + "pdb.text_array_to_edge_ngram", + "pdb.tokenize_edge_ngram", + "pdb.uuid_to_edge_ngram", + "pdb.varchar_array_to_edge_ngram", "pdb.bigint_array_to_alias", "pdb.bigint_to_alias", @@ -211,6 +221,7 @@ "types": [ "pdb.icu", - "pdb.proximityclause" + "pdb.proximityclause", + "pdb.edge_ngram" ] } From 63292202fa8f5f1272fe76d0d84c298e344a2720 Mon Sep 17 00:00:00 2001 From: Isaac Van Doren Date: Mon, 20 Apr 2026 13:21:20 -0500 Subject: [PATCH 05/12] Use standard python scripts for schema/api checks --- .github/workflows/ci.yml | 12 ++- .github/workflows/schema-compat.yml | 7 +- api.json => api.json5 | 0 apiignore.json => apiignore.json5 | 0 scripts/check_api_coverage.py | 136 ++++++++++++++++++++++++++++ scripts/check_api_coverage.rb | 127 -------------------------- scripts/check_schema_compat.py | 45 +++++---- 7 files changed, 175 insertions(+), 152 deletions(-) rename api.json => api.json5 (100%) rename apiignore.json => apiignore.json5 (100%) create mode 100644 scripts/check_api_coverage.py delete mode 100755 scripts/check_api_coverage.rb mode change 100755 => 100644 scripts/check_schema_compat.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6b0116c..31c0858 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,6 +38,16 @@ jobs: - name: Install RuboCop run: gem install rubocop --no-document + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.13" + + - name: Set up uv + uses: astral-sh/setup-uv@v7 + with: + enable-cache: true + - name: Run RuboCop lint cops run: | rubocop --version @@ -48,7 +58,7 @@ jobs: Rakefile - name: Check API coverage - run: ruby scripts/check_api_coverage.rb + run: uv run --with json5 scripts/check_api_coverage.py - name: Gem install smoke test run: bash scripts/smoke_gem_install.sh diff --git a/.github/workflows/schema-compat.yml b/.github/workflows/schema-compat.yml index cc0bf75..79255fd 100644 --- a/.github/workflows/schema-compat.yml +++ b/.github/workflows/schema-compat.yml @@ -38,10 +38,15 @@ jobs: with: python-version: "3.13" + - name: Set up uv + uses: astral-sh/setup-uv@v7 + with: + enable-cache: true + - name: Run Schema Compatibility Check env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: python scripts/check_schema_compat.py ${{ env.PARADEDB_VERSION }} + run: uv run --with json5 scripts/check_schema_compat.py ${{ env.PARADEDB_VERSION }} integration-tests: name: Integration Tests diff --git a/api.json b/api.json5 similarity index 100% rename from api.json rename to api.json5 diff --git a/apiignore.json b/apiignore.json5 similarity index 100% rename from apiignore.json rename to apiignore.json5 diff --git a/scripts/check_api_coverage.py b/scripts/check_api_coverage.py new file mode 100644 index 0000000..7f83f65 --- /dev/null +++ b/scripts/check_api_coverage.py @@ -0,0 +1,136 @@ +"""Validate that api.json5 matches the ORM wrapper surface.""" + +from __future__ import annotations + +import re +import sys +from pathlib import Path + +import json5 + +ROOT = Path(__file__).resolve().parents[1] +API_JSON = ROOT / "api.json5" +PDB_SYMBOL_RE = re.compile(r"\bpdb\.[A-Za-z_][A-Za-z0-9_]*\b") + + +def load_json(path: Path) -> object: + try: + return json5.loads(path.read_text(encoding="utf-8")) + except FileNotFoundError as exc: + raise FileNotFoundError(f"{path} not found") from exc + except ValueError as exc: + raise ValueError(f"invalid JSON5 in {path}: {exc}") from exc + + +def source_paths() -> list[Path]: + paths: list[Path] = [] + for directory in ("lib", "spec"): + paths.extend(ROOT.glob(f"{directory}/**/*.rb")) + return sorted(paths) + + +def read_source(paths: list[Path]) -> str: + try: + return "\n".join(path.read_text(encoding="utf-8") for path in paths) + except OSError as exc: + raise OSError(f"failed to read {exc.filename}: {exc.strerror}") from exc + + +def _quoted(name: str) -> str: + return rf"""['"]{re.escape(name)}['"]""" + + +def main() -> int: + try: + api = load_json(API_JSON) + except (FileNotFoundError, ValueError) as exc: + print(f"❌ {exc}", file=sys.stderr) + return 1 + + if not isinstance(api, dict): + print("❌ api.json5 must contain a JSON object.", file=sys.stderr) + return 1 + + try: + operators = api["operators"] + functions = api["functions"] + types = api["types"] + except KeyError as exc: + print(f"❌ api.json5 missing required section: {exc}", file=sys.stderr) + return 1 + + if not all(isinstance(section, dict) for section in (operators, functions, types)): + print( + "❌ api.json5 sections operators/functions/types must all be objects.", + file=sys.stderr, + ) + return 1 + + expected_operator_symbols = {str(value) for value in operators.values()} + expected_pdb_symbols = { + *(str(value) for value in functions.values()), + *(str(value) for value in types.values()), + } + + referenced_pdb_symbols: set[str] = set() + referenced_operator_symbols: set[str] = set() + + try: + paths = source_paths() + source = read_source(paths) + except OSError as exc: + print(f"❌ {exc}", file=sys.stderr) + return 1 + + observed_pdb_symbols = set(PDB_SYMBOL_RE.findall(source)) + referenced_pdb_symbols = { + symbol for symbol in expected_pdb_symbols if symbol in observed_pdb_symbols + } + referenced_operator_symbols = { + symbol + for symbol in expected_operator_symbols + if re.search(_quoted(symbol), source) + } + + missing_pdb_symbols = sorted(expected_pdb_symbols - referenced_pdb_symbols) + missing_operator_symbols = sorted( + expected_operator_symbols - referenced_operator_symbols + ) + + issues: list[str] = [] + if missing_pdb_symbols: + issues.append( + "api.json5 pdb.* symbols not referenced by the codebase: " + + ", ".join(missing_pdb_symbols) + ) + if missing_operator_symbols: + issues.append( + "api.json5 operators not referenced by ORM wrappers: " + + ", ".join(missing_operator_symbols) + ) + + if issues: + print("❌ API coverage check failed:", file=sys.stderr) + for issue in issues: + print(f" - {issue}", file=sys.stderr) + print( + "\nUpdate api.json5 or the codebase so they stay in sync.", + file=sys.stderr, + ) + return 1 + + print("✅ API coverage check passed.") + covered_total = len(expected_pdb_symbols & referenced_pdb_symbols) + len( + expected_operator_symbols & referenced_operator_symbols + ) + expected_total = len(expected_pdb_symbols) + len(expected_operator_symbols) + print(f" api symbols referenced: {covered_total}/{expected_total}") + print( + f" source files: {len(paths)}, concrete API references checked: " + f"{len(referenced_pdb_symbols) + len(referenced_operator_symbols)}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check_api_coverage.rb b/scripts/check_api_coverage.rb deleted file mode 100755 index 7e7897d..0000000 --- a/scripts/check_api_coverage.rb +++ /dev/null @@ -1,127 +0,0 @@ -#!/usr/bin/env ruby -# frozen_string_literal: true - -require "json" -require "pathname" -require "set" - -ROOT = Pathname(__dir__).join("..").expand_path -API_JSON = ROOT.join("api.json") -APIIGNORE_JSON = ROOT.join("apiignore.json") -TOKENIZER_TYPE_KEY_PREFIX = "PDB_TYPE_TOKENIZER_" - -def load_json(path) - JSON.parse(path.read) -rescue Errno::ENOENT - raise "#{path} not found" -rescue JSON::ParserError => e - raise "invalid JSON in #{path}: #{e.message}" -end - -def flatten_ignore(section, kind:) - case section - when nil - Set.new - when Array - Set.new(section.map(&:to_s)) - when Hash - Set.new(section.values.flatten.map(&:to_s)) - else - raise "apiignore #{kind} section must be an Array or object of Arrays" - end -end - -def source_paths - files = Dir.glob(ROOT.join("lib/**/*.rb").to_s).sort - rakefile = ROOT.join("Rakefile") - files << rakefile.to_s if rakefile.file? - files -end - -def read_sources - source_paths.to_h { |path| [path, File.read(path)] } -end - -def missing_quoted_tokens(expected_tokens, source_text) - expected_tokens.select do |token| - !source_text.match?(/["']#{Regexp.escape(token)}["']/) - end.sort -end - -def missing_symbols(expected_symbols, source_text) - expected_symbols.select { |symbol| !source_text.include?(symbol) }.sort -end - -def main - api = load_json(API_JSON) - apiignore = APIIGNORE_JSON.file? ? load_json(APIIGNORE_JSON) : {} - - operators = Set.new(api.fetch("operators").values.map(&:to_s)) - functions = Set.new(api.fetch("functions").values.map(&:to_s)) - all_types_by_key = api.fetch("types").transform_keys(&:to_s).transform_values(&:to_s) - - tokenizer_types = Set.new( - all_types_by_key - .select { |name, _| name.start_with?(TOKENIZER_TYPE_KEY_PREFIX) } - .values - ) - static_types = Set.new( - all_types_by_key - .reject { |name, _| name.start_with?(TOKENIZER_TYPE_KEY_PREFIX) } - .values - ) - - ignored_operators = flatten_ignore(apiignore["operators"], kind: "operators") - ignored_functions = flatten_ignore(apiignore["functions"], kind: "functions") - ignored_types = flatten_ignore(apiignore["types"], kind: "types") - - sources = read_sources - source_text = sources.values.join("\n") - - missing_ops = missing_quoted_tokens(operators, source_text) - missing_functions = missing_symbols(functions, source_text) - missing_static_types = missing_symbols(static_types, source_text) - - tokenizer_sql = sources.find { |path, _| path.end_with?("lib/parade_db/tokenizer_sql.rb") }&.last.to_s - dynamic_tokenizer_supported = tokenizer_sql.include?('"pdb.#{function_name}"') - missing_tokenizer_types = dynamic_tokenizer_supported ? [] : tokenizer_types.to_a.sort - - referenced_symbols = Set.new(source_text.scan(/\bpdb\.[a-zA-Z_][a-zA-Z0-9_]*\b/)) - allowed_symbols = functions | static_types | tokenizer_types | ignored_functions | ignored_types - untracked_symbols = (referenced_symbols - allowed_symbols).to_a.sort - - issues = [] - - unless missing_ops.empty? - issues << "operators declared in api.json but not found in Ruby wrappers: #{missing_ops.join(', ')}" - end - - unless missing_functions.empty? - issues << "functions declared in api.json but not found in Ruby wrappers: #{missing_functions.join(', ')}" - end - - unless missing_static_types.empty? - issues << "types declared in api.json but not found in Ruby wrappers: #{missing_static_types.join(', ')}" - end - - unless missing_tokenizer_types.empty? - issues << "tokenizer types require dynamic tokenizer qualification, but support was not detected: #{missing_tokenizer_types.join(', ')}" - end - - unless untracked_symbols.empty? - issues << "pdb.* symbols used in code but missing from api.json/apiignore.json: #{untracked_symbols.join(', ')}" - end - - if issues.empty? - puts "✅ API coverage check passed." - puts " operators: #{operators.size}, functions: #{functions.size}, types: #{all_types_by_key.size}, referenced symbols: #{referenced_symbols.size}" - return 0 - end - - warn "❌ API coverage check failed:" - issues.each { |issue| warn " - #{issue}" } - warn "\nUpdate api.json, apiignore.json, or wrapper usage so they stay in sync." - 1 -end - -exit(main) diff --git a/scripts/check_schema_compat.py b/scripts/check_schema_compat.py old mode 100755 new mode 100644 index 0d8c215..f5d6004 --- a/scripts/check_schema_compat.py +++ b/scripts/check_schema_compat.py @@ -1,21 +1,20 @@ -#!/usr/bin/env python3 """ -Check compatibility between rails-paradedb's api.json and a released pg_search schema. +Check compatibility between the ORM's api.json5 and a released pg_search schema. The schema is downloaded from the corresponding ParadeDB GitHub release and checked in both directions: - Forward: every symbol in api.json is present in the schema (detects removals/renames). - Reverse: every pdb.* symbol in the schema is either in api.json or in apiignore.json + Forward: every symbol in api.json5 is present in the schema (detects removals/renames). + Reverse: every pdb.* symbol in the schema is either in api.json5 or in apiignore.json5 (surfaces new paradedb APIs that haven't been wrapped yet). Example: python scripts/check_schema_compat.py 0.22.0 -The ignore list is read automatically from apiignore.json (repo root) if it exists. +The ignore list is read automatically from apiignore.json5 (repo root) if it exists. """ + import argparse -import json import re import shutil import subprocess @@ -23,9 +22,11 @@ import tempfile from pathlib import Path +import json5 + _ROOT_DIR = Path(__file__).resolve().parent.parent -_API_FILE = _ROOT_DIR / "api.json" -_IGNORE_FILE = _ROOT_DIR / "apiignore.json" +_API_FILE = _ROOT_DIR / "api.json5" +_IGNORE_FILE = _ROOT_DIR / "apiignore.json5" _SCHEMA_FILE_NAME = "pg_search.schema.sql" @@ -35,8 +36,8 @@ def normalize(sql: str) -> str: def extract_from_api(path: Path) -> dict: - """Read api.json and return {functions: [...], operators: [...], types: [...]}.""" - data = json.loads(path.read_text()) + """Read api.json5 and return {functions: [...], operators: [...], types: [...]}.""" + data = json5.loads(path.read_text()) return { "functions": sorted(set(data["functions"].values())), "operators": sorted(set(data["operators"].values())), @@ -115,8 +116,7 @@ def normalize_version(version: str) -> str: def parse_args(argv: list[str]) -> argparse.Namespace: parser = argparse.ArgumentParser( description=( - "Download pg_search.schema.sql for a ParadeDB release and check it " - "against this repo's api.json." + "Download pg_search.schema.sql for a ParadeDB release and check it against this repo's api.json5." ) ) parser.add_argument("version", help="ParadeDB version to check, for example 0.22.0") @@ -177,17 +177,17 @@ def run_checks(schema_path: Path, api_path: Path) -> int: print(f"❌ Schema file not found: {schema_path}", file=sys.stderr) return 1 if not api_path.exists(): - print(f"❌ api.json not found: {api_path}", file=sys.stderr) + print(f"❌ api.json5 not found: {api_path}", file=sys.stderr) return 1 schema = normalize(schema_path.read_text()) deps = extract_from_api(api_path) - ignored = json.loads(_IGNORE_FILE.read_text()) if _IGNORE_FILE.exists() else {} + ignored = json5.loads(_IGNORE_FILE.read_text()) if _IGNORE_FILE.exists() else {} rc = 0 # ------------------------------------------------------------------ - # Forward check: every symbol in api.json must exist in the schema. + # Forward check: every symbol in api.json5 must exist in the schema. # ------------------------------------------------------------------ missing: list[tuple[str, str]] = [] for fn in deps.get("functions", []): @@ -203,21 +203,21 @@ def run_checks(schema_path: Path, api_path: Path) -> int: total_api = sum(len(v) for v in deps.values() if isinstance(v, list)) if missing: print( - f"❌ Forward check: {len(missing)}/{total_api} api.json symbols missing from schema:" + f"❌ Forward check: {len(missing)}/{total_api} api.json5 symbols missing from schema:" ) for kind, name in missing: print(f" {kind}: {name}") print( "\nThese symbols were removed or renamed in this version of pg_search.\n" - "Update rails-paradedb to handle the API change, then update api.json." + "Update the ORM to handle the API change, then update api.json5." ) rc = 1 else: - print(f"✅ Forward check: all {total_api} api.json symbols present in schema.") + print(f"✅ Forward check: all {total_api} api.json5 symbols present in schema.") # ------------------------------------------------------------------ - # Reverse check: every pdb.* symbol in the schema must be in api.json - # or explicitly ignored in apiignore.json. + # Reverse check: every pdb.* symbol in the schema must be in api.json5 + # or explicitly ignored in apiignore.json5. # ------------------------------------------------------------------ schema_symbols = scan_schema_symbols(schema) uncovered: list[tuple[str, str]] = [] @@ -231,13 +231,12 @@ def run_checks(schema_path: Path, api_path: Path) -> int: total_schema = sum(len(v) for v in schema_symbols.values()) if uncovered: print( - f"\n⚠️ Reverse check: {len(uncovered)} schema symbols not covered by api.json:" + f"\n⚠️ Reverse check: {len(uncovered)} schema symbols not covered by api.json5:" ) for kind, name in uncovered: print(f" {kind}: {name}") print( - "\nThese are paradedb APIs not yet wrapped by rails-paradedb.\n" - "Either add them to api.json or add them to apiignore.json." + "\nThese are paradedb APIs not yet wrapped.\nEither add them to api.json5 or add them to apiignore.json5." ) rc = 1 else: From 5913fc7cc682f1f9aacb50136c13086e790d0fab Mon Sep 17 00:00:00 2001 From: Isaac Van Doren Date: Mon, 20 Apr 2026 14:02:52 -0500 Subject: [PATCH 06/12] Add test covering all tokenizers --- api.json5 | 4 ++- apiignore.json5 | 2 -- spec/user_api_behavior_integration_spec.rb | 31 ++++++++++++++++++++++ 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/api.json5 b/api.json5 index c65c010..478b248 100644 --- a/api.json5 +++ b/api.json5 @@ -48,6 +48,8 @@ "PDB_TYPE_TOKENIZER_SIMPLE": "pdb.simple", "PDB_TYPE_TOKENIZER_SOURCE_CODE": "pdb.source_code", "PDB_TYPE_TOKENIZER_UNICODE_WORDS": "pdb.unicode_words", - "PDB_TYPE_TOKENIZER_WHITESPACE": "pdb.whitespace" + "PDB_TYPE_TOKENIZER_WHITESPACE": "pdb.whitespace", + "PDB_TYPE_TOKENIZER_ICU": "pdb.icu", + "PDB_TYPE_TOKENIZER_EDGE_NGRAM": "pdb.edge_ngram", } } diff --git a/apiignore.json5 b/apiignore.json5 index 1dbd20f..1b54912 100644 --- a/apiignore.json5 +++ b/apiignore.json5 @@ -220,8 +220,6 @@ ], "types": [ - "pdb.icu", "pdb.proximityclause", - "pdb.edge_ngram" ] } diff --git a/spec/user_api_behavior_integration_spec.rb b/spec/user_api_behavior_integration_spec.rb index 256d649..032fb93 100644 --- a/spec/user_api_behavior_integration_spec.rb +++ b/spec/user_api_behavior_integration_spec.rb @@ -54,6 +54,37 @@ class BehaviorRangeItem < ActiveRecord::Base assert_equal [1, 2, 6], ids end + it "matching all with supported tokenizers matches raw SQL" do + [ + ["pdb.whitespace", "whitespace"], + ["pdb.whitespace('alias=my_column')", "whitespace('alias=my_column')"], + ["pdb.unicode_words", "unicode_words"], + ["pdb.literal", "literal"], + ["pdb.literal_normalized", "literal_normalized"], + ["pdb.ngram(3,3)", "ngram(3,3)"], + ["pdb.ngram(3,3,'positions=true')", "ngram(3,3,'positions=true')"], + ["pdb.edge_ngram(2,5)", "edge_ngram(2,5)"], + ["pdb.simple", "simple"], + # ["pdb.regex_pattern('.*')", "regex_pattern('.*')"], # rejected by the current tokenizer validator + ["pdb.chinese_compatible", "chinese_compatible"], + ["pdb.lindera('chinese')", "lindera('chinese')"], + ["pdb.icu", "icu"], + ["pdb.jieba", "jieba"], + ["pdb.source_code", "source_code"] + ].each do |expected, tokenizer| + relation = BehaviorProduct.search(:description) + .matching_all("running shoes", tokenizer: tokenizer) + .order(:id) + + relation.load + + assert_sql_equal <<~SQL, relation.to_sql + SELECT products.* FROM products + WHERE ("products"."description" &&& 'running shoes'::#{expected}) + ORDER BY "products"."id" ASC + SQL + end + end it "matching with tokenizer + fuzzy distance raises argument error" do error = assert_raises(ArgumentError) do BehaviorProduct.search(:description) From f9503d134f12da64a3724b4464c86d7b61730a95 Mon Sep 17 00:00:00 2001 From: Isaac Van Doren Date: Mon, 20 Apr 2026 15:37:23 -0500 Subject: [PATCH 07/12] Implement function based tokenizer api and use it for queries --- lib/parade_db.rb | 1 + lib/parade_db/arel/builder.rb | 7 +- lib/parade_db/tokenizer.rb | 95 ++++++++++++++++++++++ spec/arel_builder_unit_spec.rb | 6 +- spec/arel_predications_unit_spec.rb | 2 +- spec/arel_visitor_spec.rb | 4 +- spec/guards_integration_spec.rb | 24 +----- spec/user_api_behavior_integration_spec.rb | 38 ++++----- spec/user_api_unit_spec.rb | 6 +- 9 files changed, 131 insertions(+), 52 deletions(-) create mode 100644 lib/parade_db/tokenizer.rb diff --git a/lib/parade_db.rb b/lib/parade_db.rb index a1184f8..35f9f43 100644 --- a/lib/parade_db.rb +++ b/lib/parade_db.rb @@ -12,6 +12,7 @@ require_relative "parade_db/model" require_relative "parade_db/search_methods" require_relative "parade_db/railtie" +require_relative "parade_db/tokenizer" module ParadeDB FacetQueryError = Errors::FacetQueryError diff --git a/lib/parade_db/arel/builder.rb b/lib/parade_db/arel/builder.rb index db81f01..3249f93 100644 --- a/lib/parade_db/arel/builder.rb +++ b/lib/parade_db/arel/builder.rb @@ -302,12 +302,11 @@ def apply_fuzzy(node, distance:, prefix:, transposition_cost_one:, bridge_to_que def apply_tokenizer(node, tokenizer) return node if tokenizer.nil? - unless tokenizer.is_a?(String) - raise ArgumentError, "tokenizer must be a string" + unless tokenizer.is_a?(Tokenizer) + raise ArgumentError, "tokenizer must be a Tokenizer" end - normalized = normalize_tokenizer(tokenizer) - Nodes::TokenizerCast.new(node, normalized) + return Nodes::TokenizerCast.new(node, tokenizer.render()) end def apply_slop(node, slop) diff --git a/lib/parade_db/tokenizer.rb b/lib/parade_db/tokenizer.rb new file mode 100644 index 0000000..8bad541 --- /dev/null +++ b/lib/parade_db/tokenizer.rb @@ -0,0 +1,95 @@ +class Tokenizer + attr_reader :name, :positional_args, :options + + def initialize(name, positional_args, options) + @name = name + @positional_args = positional_args + @options = options + end + + def render() + if options.nil? && positional_args.nil? + return "pdb.#{name}" + end + + args = [] + if !positional_args.nil? + args.concat(positional_args.map { |x| render_positional_arg(x) }) + end + if !options.nil? + args.concat(options.map {|k, v| quote_term("#{k}=#{v}")}) + end + + return "pdb.#{name}(#{args.join(",")})" + end + + def self.whitespace(options: nil) + new("whitespace", nil, options) + end + + def self.unicode_words(options: nil) + new("unicode_words", nil, options) + end + + def self.ngram(min_gram, max_gram, options: nil) + new("ngram", [min_gram, max_gram], options) + end + + def self.simple(options: nil) + new("simple", nil, options) + end + + def self.literal(options: nil) + new("literal", nil, options) + end + + def self.literal_normalized(options: nil) + new("literal_normalized", nil, options) + end + + def self.edge_ngram(min_gram, max_gram, options: nil) + new("edge_ngram", [min_gram, max_gram], options) + end + + def self.regex_pattern(pattern, options: nil) + new("regex_pattern", [pattern], options) + end + + def self.chinese_compatible(options: nil) + new("chinese_compatible", nil, options) + end + + def self.lindera(dictionary, options: nil) + new("lindera", [dictionary], options) + end + + def self.icu(options: nil) + new("icu", nil, options) + end + + def self.jieba(options: nil) + new("jieba", nil, options) + end + + def self.source_code(options: nil) + new("source_code", nil, options) + end + + private + + def quote_term(value) + escaped = value.gsub("'", "''") + "'#{escaped}'" + end + + def render_positional_arg(value) + case value + when true, false, Numeric + value.to_s + when String + quote_term(value) + else + raise InvalidArgumentError, "Unsupported tokenizer arg type: #{value.class}" + end + end +end diff --git a/spec/arel_builder_unit_spec.rb b/spec/arel_builder_unit_spec.rb index 830afe0..19267de 100644 --- a/spec/arel_builder_unit_spec.rb +++ b/spec/arel_builder_unit_spec.rb @@ -59,11 +59,11 @@ def sql(node) assert_equal %("products"."description" &&& 'shoes'::pdb.boost(2.5)), sql(node) end it "match with tokenizer override" do - node = @builder.match(:description, "running shoes", tokenizer: "whitespace") + node = @builder.match(:description, "running shoes", tokenizer: Tokenizer.whitespace()) assert_equal %("products"."description" &&& 'running shoes'::pdb.whitespace), sql(node) end it "match with tokenizer override and args" do - node = @builder.match(:description, "running shoes", tokenizer: "whitespace('lowercase=false')") + node = @builder.match(:description, "running shoes", tokenizer: Tokenizer.whitespace(options: {lowercase: false})) assert_equal %("products"."description" &&& 'running shoes'::pdb.whitespace('lowercase=false')), sql(node) end it "match without boost" do @@ -95,7 +95,7 @@ def sql(node) assert_equal %("products"."description" ### 'running shoes'::pdb.slop(10)), sql(node) end it "phrase with tokenizer override" do - node = @builder.phrase(:description, "running shoes", tokenizer: "whitespace") + node = @builder.phrase(:description, "running shoes", tokenizer: Tokenizer.whitespace()) assert_equal %("products"."description" ### 'running shoes'::pdb.whitespace), sql(node) end it "phrase with slop and constant score bridges through query" do diff --git a/spec/arel_predications_unit_spec.rb b/spec/arel_predications_unit_spec.rb index eb8f883..93dd525 100644 --- a/spec/arel_predications_unit_spec.rb +++ b/spec/arel_predications_unit_spec.rb @@ -64,7 +64,7 @@ def sql(node) assert_equal %("products"."description" ### 'running shoes'::pdb.slop(10)), sql(node) end it "pdb_phrase with tokenizer" do - node = @t[:description].pdb_phrase("running shoes", tokenizer: "whitespace") + node = @t[:description].pdb_phrase("running shoes", tokenizer: Tokenizer.whitespace()) assert_equal %("products"."description" ### 'running shoes'::pdb.whitespace), sql(node) end it "pdb_phrase with pretokenized array" do diff --git a/spec/arel_visitor_spec.rb b/spec/arel_visitor_spec.rb index 5f0a995..e244bee 100644 --- a/spec/arel_visitor_spec.rb +++ b/spec/arel_visitor_spec.rb @@ -19,7 +19,7 @@ def sql(node) assert_equal %("products"."description" ||| 'wireless bluetooth'), sql(node) end it "match any with tokenizer override" do - node = @builder.match_any(:description, "running shoes", tokenizer: "whitespace") + node = @builder.match_any(:description, "running shoes", tokenizer: Tokenizer.whitespace()) assert_equal %("products"."description" ||| 'running shoes'::pdb.whitespace), sql(node) end it "phrase with slop" do @@ -27,7 +27,7 @@ def sql(node) assert_equal %("products"."description" ### 'running shoes'::pdb.slop(2)), sql(node) end it "phrase with tokenizer" do - node = @builder.phrase(:description, "running shoes", tokenizer: "whitespace") + node = @builder.phrase(:description, "running shoes", tokenizer: Tokenizer.whitespace()) assert_equal %("products"."description" ### 'running shoes'::pdb.whitespace), sql(node) end it "phrase with array" do diff --git a/spec/guards_integration_spec.rb b/spec/guards_integration_spec.rb index d5d30d7..08e051a 100644 --- a/spec/guards_integration_spec.rb +++ b/spec/guards_integration_spec.rb @@ -101,30 +101,14 @@ def builder node = builder.match(:description, "shoes", boost: nil) refute_nil node end - it "match tokenizer rejects non-string" do - error = assert_raises(ArgumentError) { builder.match(:description, "shoes", tokenizer: 123) } - assert_includes error.message, "tokenizer must be a string" - end - it "match tokenizer rejects invalid expression" do + it "match tokenizer rejects strings" do error = assert_raises(ArgumentError) { builder.match(:description, "shoes", tokenizer: "whitespace;drop") } - assert_includes error.message, "invalid tokenizer expression" + assert_includes error.message, "tokenizer must be a Tokenizer" end it "phrase slop rejects non numeric" do error = assert_raises(ArgumentError) { builder.phrase(:description, "running shoes", slop: "lots") } assert_includes error.message, "slop must be numeric" end - it "phrase tokenizer rejects non-string" do - error = assert_raises(ArgumentError) { builder.phrase(:description, "running shoes", tokenizer: 123) } - assert_includes error.message, "tokenizer must be a string" - end - it "phrase tokenizer rejects invalid expression" do - error = assert_raises(ArgumentError) { builder.phrase(:description, "running shoes", tokenizer: "whitespace;drop") } - assert_includes error.message, "invalid tokenizer expression" - end - it "phrase array rejects tokenizer" do - error = assert_raises(ArgumentError) { builder.phrase(:description, %w[running shoes], tokenizer: "whitespace") } - assert_includes error.message, "tokenizer is not supported for pretokenized phrase arrays" - end it "phrase slop accepts integer" do node = builder.phrase(:description, "running shoes", slop: 2) refute_nil node @@ -143,13 +127,13 @@ def builder end it "matching_any rejects tokenizer combined with fuzzy options" do error = assert_raises(ArgumentError) do - builder.match_any(:description, "shoes", tokenizer: "whitespace", distance: 1) + builder.match_any(:description, "shoes", tokenizer: Tokenizer.whitespace(), distance: 1) end assert_includes error.message, "tokenizer cannot be combined with fuzzy options" end it "matching_all rejects tokenizer combined with fuzzy options" do error = assert_raises(ArgumentError) do - builder.match(:description, "shoes", tokenizer: "whitespace", prefix: true) + builder.match(:description, "shoes", tokenizer: Tokenizer.whitespace(), prefix: true) end assert_includes error.message, "tokenizer cannot be combined with fuzzy options" end diff --git a/spec/user_api_behavior_integration_spec.rb b/spec/user_api_behavior_integration_spec.rb index 032fb93..3794104 100644 --- a/spec/user_api_behavior_integration_spec.rb +++ b/spec/user_api_behavior_integration_spec.rb @@ -48,7 +48,7 @@ class BehaviorRangeItem < ActiveRecord::Base end it "matching with tokenizer override executes" do ids = BehaviorProduct.search(:description) - .matching_any("running shoes", tokenizer: "whitespace") + .matching_any("running shoes", tokenizer: Tokenizer.whitespace()) .order(:id) .pluck(:id) @@ -56,21 +56,21 @@ class BehaviorRangeItem < ActiveRecord::Base end it "matching all with supported tokenizers matches raw SQL" do [ - ["pdb.whitespace", "whitespace"], - ["pdb.whitespace('alias=my_column')", "whitespace('alias=my_column')"], - ["pdb.unicode_words", "unicode_words"], - ["pdb.literal", "literal"], - ["pdb.literal_normalized", "literal_normalized"], - ["pdb.ngram(3,3)", "ngram(3,3)"], - ["pdb.ngram(3,3,'positions=true')", "ngram(3,3,'positions=true')"], - ["pdb.edge_ngram(2,5)", "edge_ngram(2,5)"], - ["pdb.simple", "simple"], - # ["pdb.regex_pattern('.*')", "regex_pattern('.*')"], # rejected by the current tokenizer validator - ["pdb.chinese_compatible", "chinese_compatible"], - ["pdb.lindera('chinese')", "lindera('chinese')"], - ["pdb.icu", "icu"], - ["pdb.jieba", "jieba"], - ["pdb.source_code", "source_code"] + ["pdb.whitespace", Tokenizer.whitespace()], + ["pdb.whitespace('alias=my_column')", Tokenizer.whitespace(options: {alias: "my_column"})], + ["pdb.unicode_words", Tokenizer.unicode_words()], + ["pdb.literal", Tokenizer.literal()], + ["pdb.literal_normalized", Tokenizer.literal_normalized()], + ["pdb.ngram(3,3)", Tokenizer.ngram(3, 3)], + ["pdb.ngram(3,3,'positions=true')", Tokenizer.ngram(3, 3, options: {positions: true})], + ["pdb.edge_ngram(2,5)", Tokenizer.edge_ngram(2, 5)], + ["pdb.simple", Tokenizer.simple()], + ["pdb.regex_pattern('.*')", Tokenizer.regex_pattern('.*')], + ["pdb.chinese_compatible", Tokenizer.chinese_compatible()], + ["pdb.lindera('chinese')", Tokenizer.lindera('chinese')], + ["pdb.icu", Tokenizer.icu()], + ["pdb.jieba", Tokenizer.jieba()], + ["pdb.source_code", Tokenizer.source_code()] ].each do |expected, tokenizer| relation = BehaviorProduct.search(:description) .matching_all("running shoes", tokenizer: tokenizer) @@ -88,7 +88,7 @@ class BehaviorRangeItem < ActiveRecord::Base it "matching with tokenizer + fuzzy distance raises argument error" do error = assert_raises(ArgumentError) do BehaviorProduct.search(:description) - .matching_any("runing shose", tokenizer: "whitespace", distance: 1) + .matching_any("runing shose", tokenizer: Tokenizer.whitespace(), distance: 1) .order(:id) .pluck(:id) end @@ -97,7 +97,7 @@ class BehaviorRangeItem < ActiveRecord::Base it "matching with tokenizer + fuzzy constant score raises argument error" do error = assert_raises(ArgumentError) do BehaviorProduct.search(:description) - .matching_any("runing shose", tokenizer: "whitespace", distance: 1, constant_score: 1.0) + .matching_any("runing shose", tokenizer: Tokenizer.whitespace(), distance: 1, constant_score: 1.0) .order(:id) .pluck(:id) end @@ -475,7 +475,7 @@ class BehaviorRangeItem < ActiveRecord::Base SQL relation = BehaviorProduct.search(:description) - .phrase("running shoes", tokenizer: "whitespace") + .phrase("running shoes", tokenizer: Tokenizer.whitespace()) .order(:id) expected_ids = [1, 5] diff --git a/spec/user_api_unit_spec.rb b/spec/user_api_unit_spec.rb index efa5f75..30d4af9 100644 --- a/spec/user_api_unit_spec.rb +++ b/spec/user_api_unit_spec.rb @@ -76,11 +76,11 @@ class Category < ActiveRecord::Base assert_sql_equal %(SELECT products.* FROM products WHERE ("products"."description" ||| 'wireless bluetooth')), sql end it "matching all with tokenizer override" do - sql = Product.search(:description).matching_all("running shoes", tokenizer: "whitespace").to_sql + sql = Product.search(:description).matching_all("running shoes", tokenizer: Tokenizer.whitespace()).to_sql assert_sql_equal %(SELECT products.* FROM products WHERE ("products"."description" &&& 'running shoes'::pdb.whitespace)), sql end it "matching all with tokenizer args" do - sql = Product.search(:description).matching_all("running shoes", tokenizer: "whitespace('lowercase=false')").to_sql + sql = Product.search(:description).matching_all("running shoes", tokenizer: Tokenizer.whitespace(options: {lowercase: false})).to_sql assert_sql_equal %(SELECT products.* FROM products WHERE ("products"."description" &&& 'running shoes'::pdb.whitespace('lowercase=false'))), sql end it "excluding terms" do @@ -112,7 +112,7 @@ class Category < ActiveRecord::Base assert_sql_equal %(SELECT products.* FROM products WHERE ("products"."description" ### 'running shoes'::pdb.slop(2))), sql end it "phrase with tokenizer" do - sql = Product.search(:description).phrase("running shoes", tokenizer: "whitespace").to_sql + sql = Product.search(:description).phrase("running shoes", tokenizer: Tokenizer.whitespace()).to_sql assert_sql_equal %(SELECT products.* FROM products WHERE ("products"."description" ### 'running shoes'::pdb.whitespace)), sql end it "phrase with pretokenized array" do From bef918f707e3f34b69f976bf82ae21725c69349f Mon Sep 17 00:00:00 2001 From: Isaac Van Doren Date: Mon, 20 Apr 2026 16:28:22 -0500 Subject: [PATCH 08/12] Use function based approach in index creation --- CHANGELOG.md | 5 + examples/autocomplete/model.rb | 12 +- examples/faceted_search/model.rb | 6 +- examples/hybrid_rrf/model.rb | 6 +- examples/more_like_this/model.rb | 6 +- examples/rag/model.rb | 6 +- lib/parade_db/index.rb | 131 +++------------------ lib/parade_db/migration_helpers.rb | 23 ++-- spec/index_dsl_unit_spec.rb | 91 ++++---------- spec/index_migration_integration_spec.rb | 53 ++++----- spec/index_runtime_features_unit_spec.rb | 12 +- spec/key_field_runtime_integration_spec.rb | 2 +- 12 files changed, 113 insertions(+), 240 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cc3e212..2b62a61 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. The format ## [Unreleased] +### Changed + +- BM25 index definitions and schema dumps now specify field tokenizers with + `Tokenizer.*(...)`, matching query APIs like `matching_all` + ## [0.6.0] - 2026-04-14 ### Added diff --git a/examples/autocomplete/model.rb b/examples/autocomplete/model.rb index 9a1d053..87c57ff 100644 --- a/examples/autocomplete/model.rb +++ b/examples/autocomplete/model.rb @@ -18,9 +18,9 @@ class MockItemIndex < ParadeDB::Index id: nil, description: nil, rating: nil, - category: { tokenizer: :literal }, - "metadata->>'color'" => { tokenizer: :literal, alias: "metadata_color" }, - "metadata->>'location'" => { tokenizer: :literal, alias: "metadata_location" } + category: { tokenizer: Tokenizer.literal() }, + "metadata->>'color'" => { tokenizer: Tokenizer.literal(options: {alias: "metadata_color"}) }, + "metadata->>'location'" => { tokenizer: Tokenizer.literal(options: {alias: "metadata_location"}) } } end @@ -39,10 +39,10 @@ class AutocompleteItemIndex < ParadeDB::Index id: nil, description: { tokenizers: [ - { tokenizer: :unicode_words }, - { tokenizer: :ngram, named_args: { min: 3, max: 8 }, alias: "description_ngram" } + Tokenizer.unicode_words(), + Tokenizer.ngram(3, 8, options: {alias: "description_ngram"}) ] }, - category: { tokenizer: :literal } + category: { tokenizer: Tokenizer.literal() } } end diff --git a/examples/faceted_search/model.rb b/examples/faceted_search/model.rb index e9738a9..08afce2 100644 --- a/examples/faceted_search/model.rb +++ b/examples/faceted_search/model.rb @@ -18,8 +18,8 @@ class MockItemIndex < ParadeDB::Index id: nil, description: nil, rating: nil, - category: { tokenizer: :literal }, - "metadata->>'color'" => { tokenizer: :literal, alias: "metadata_color" }, - "metadata->>'location'" => { tokenizer: :literal, alias: "metadata_location" } + category: { tokenizer: Tokenizer.literal() }, + "metadata->>'color'" => { tokenizer: Tokenizer.literal(options: {alias: "metadata_color"}) }, + "metadata->>'location'" => { tokenizer: Tokenizer.literal(options: {alias: "metadata_location"}) } } end diff --git a/examples/hybrid_rrf/model.rb b/examples/hybrid_rrf/model.rb index eb659fe..9f617f3 100644 --- a/examples/hybrid_rrf/model.rb +++ b/examples/hybrid_rrf/model.rb @@ -26,8 +26,8 @@ class MockItemIndex < ParadeDB::Index id: nil, description: nil, rating: nil, - category: { tokenizer: :literal }, - "metadata->>'color'" => { tokenizer: :literal, alias: "metadata_color" }, - "metadata->>'location'" => { tokenizer: :literal, alias: "metadata_location" } + category: { tokenizer: Tokenizer.literal() }, + "metadata->>'color'" => { tokenizer: Tokenizer.literal(options: {alias: "metadata_color"}) }, + "metadata->>'location'" => { tokenizer: Tokenizer.literal(options: {alias: "metadata_location"}) } } end diff --git a/examples/more_like_this/model.rb b/examples/more_like_this/model.rb index 47b5e47..57184c4 100644 --- a/examples/more_like_this/model.rb +++ b/examples/more_like_this/model.rb @@ -18,8 +18,8 @@ class MockItemIndex < ParadeDB::Index id: nil, description: nil, rating: nil, - category: { tokenizer: :literal }, - "metadata->>'color'" => { tokenizer: :literal, alias: "metadata_color" }, - "metadata->>'location'" => { tokenizer: :literal, alias: "metadata_location" } + category: { tokenizer: Tokenizer.literal() }, + "metadata->>'color'" => { tokenizer: Tokenizer.literal(options: {alias: "metadata_color"}) }, + "metadata->>'location'" => { tokenizer: Tokenizer.literal(options: {alias: "metadata_location"}) } } end diff --git a/examples/rag/model.rb b/examples/rag/model.rb index 47b5e47..57184c4 100644 --- a/examples/rag/model.rb +++ b/examples/rag/model.rb @@ -18,8 +18,8 @@ class MockItemIndex < ParadeDB::Index id: nil, description: nil, rating: nil, - category: { tokenizer: :literal }, - "metadata->>'color'" => { tokenizer: :literal, alias: "metadata_color" }, - "metadata->>'location'" => { tokenizer: :literal, alias: "metadata_location" } + category: { tokenizer: Tokenizer.literal() }, + "metadata->>'color'" => { tokenizer: Tokenizer.literal(options: {alias: "metadata_color"}) }, + "metadata->>'location'" => { tokenizer: Tokenizer.literal(options: {alias: "metadata_location"}) } } end diff --git a/lib/parade_db/index.rb b/lib/parade_db/index.rb index 9778876..4041c03 100644 --- a/lib/parade_db/index.rb +++ b/lib/parade_db/index.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require_relative "tokenizer" + module ParadeDB class Index class << self @@ -47,117 +49,29 @@ def validate! class TokenizerParser TOKENIZER_EXPRESSION = /\A[a-zA-Z_][a-zA-Z0-9_]*(?:(?:::|\.)[a-zA-Z_][a-zA-Z0-9_]*)*(?:\(\s*[a-zA-Z0-9_'".,=\s:-]*\s*\))?\z/.freeze - TOKENIZER_SINGLE_KEYS = %i[tokenizer args named_args filters stemmer alias].freeze + TOKENIZER_SINGLE_KEYS = %i[tokenizer alias].freeze class << self - def parse(source_name, tokenizer_spec) - case tokenizer_spec - when Symbol, String - [build_tokenized_entry(source_name, tokenizer_spec.to_s, {})] - when Hash - tokenizer_spec.map do |tokenizer, opts| - case opts - when Hash - build_tokenized_entry(source_name, tokenizer.to_s, normalize_options(opts)) - when Symbol, String - build_tokenized_entry(source_name, tokenizer.to_s, normalize_positional_option(opts)) - else - raise InvalidIndexDefinition, - "tokenizer options for #{source_name}.#{tokenizer} must be a Hash, Symbol, or String" - end - end - else - raise InvalidIndexDefinition, - "invalid tokenizer definition for #{source_name}: #{tokenizer_spec.inspect}" - end - end - - private - - def parse_structured_tokenizer_config(source_name, config, context:) - unless config.is_a?(Hash) - raise InvalidIndexDefinition, "#{context} for #{source_name.inspect} must be a Hash" + def parse(source_name, tokenizer, context:) + unless tokenizer.is_a?(Tokenizer) + raise InvalidIndexDefinition, "#{context} for #{source_name.inspect} must be a Tokenizer" end - tokenizer = config[:tokenizer] || config["tokenizer"] - if tokenizer.nil? - raise InvalidIndexDefinition, "#{context} for #{source_name.inspect} requires :tokenizer" - end - - tokenizer_name = tokenizer.to_s - validate_tokenizer_name!(source_name, tokenizer_name) - - args = config[:args] || config["args"] - named_args = config[:named_args] || config["named_args"] - filters = config[:filters] || config["filters"] - stemmer = config[:stemmer] || config["stemmer"] - alias_name = config[:alias] || config["alias"] - options = {} - if args - unless args.respond_to?(:to_ary) - raise InvalidIndexDefinition, "args for #{source_name.inspect} must be an Array" - end - options[:__positional] = args.to_ary - end - - if named_args - unless named_args.is_a?(Hash) - raise InvalidIndexDefinition, "named_args for #{source_name.inspect} must be a Hash" - end - named_args.each { |key, value| options[key.to_sym] = value } - end - - if filters - unless filters.respond_to?(:to_ary) - raise InvalidIndexDefinition, "filters for #{source_name.inspect} must be an Array" - end - filters.to_ary.each do |name| - filter_key = name.to_s - if filter_key == "stemmer" && stemmer - options[:stemmer] = stemmer - else - key = filter_key.to_sym - options[key] = true unless options.key?(key) - end - end - end - - options[:stemmer] = stemmer if stemmer && !options.key?(:stemmer) - options[:alias] = alias_name.to_s if alias_name - - build_tokenized_entry(source_name, tokenizer_name, options) - end - - def normalize_options(opts) - opts.each_with_object({}) do |(key, value), memo| - memo[key.to_sym] = value - end - end - - def normalize_positional_option(option) - { __positional: [option.to_s] } - end + options[:__positional] = tokenizer.positional_args.dup unless tokenizer.positional_args.nil? + tokenizer.options&.each { |key, value| options[key.to_sym] = value } - def build_tokenized_entry(source_name, tokenizer, options) - validate_tokenizer_name!(source_name, tokenizer) unless tokenizer.nil? key = options[:alias]&.to_s || source_name DefinitionCompiler::Entry.new( source: source_name, expression: expression?(source_name), - tokenizer: tokenizer, + tokenizer: tokenizer.name, options: options, query_key: key ) end - def validate_tokenizer_name!(source_name, tokenizer) - return if TOKENIZER_EXPRESSION.match?(tokenizer) - - raise InvalidIndexDefinition, - "invalid tokenizer name #{tokenizer.inspect} for #{source_name}. " \ - "Expected identifier form like simple, pdb::simple, or pdb::ngram(2, 5, alias=field_alias)." - end + private def expression?(value) value.match?(/[^a-zA-Z0-9_]/) @@ -260,33 +174,22 @@ def build_entries(raw_fields) elsif tokenizers if single_tokenizer_keys_present raise InvalidIndexDefinition, - "field #{source_name.inspect} cannot mix :tokenizers with :tokenizer/:args/:named_args/:filters/:stemmer/:alias" + "field #{source_name.inspect} cannot mix :tokenizers with :tokenizer/:alias" end unless tokenizers.respond_to?(:to_ary) && !tokenizers.to_ary.empty? raise InvalidIndexDefinition, "field #{source_name.inspect} :tokenizers must be a non-empty Array" end tokenizers.to_ary.each_with_index do |tokenizer_config, idx| - entry = TokenizerParser.send( - :parse_structured_tokenizer_config, - source_name, - tokenizer_config, - context: "tokenizers[#{idx}]" - ) + entry = TokenizerParser.parse(source_name, tokenizer_config, context: "tokenizers[#{idx}]") entries << entry end - elsif single_tokenizer_keys_present - unless normalized[:tokenizer] - raise InvalidIndexDefinition, - "field #{source_name.inspect} specifies tokenizer configuration but no :tokenizer" - end - entry = TokenizerParser.send( - :parse_structured_tokenizer_config, - source_name, - select_keys(normalized, TokenizerParser::TOKENIZER_SINGLE_KEYS), - context: "tokenizer config" - ) + elsif normalized.key?(:tokenizer) + entry = TokenizerParser.parse(source_name, normalized[:tokenizer], context: "tokenizer") entries << entry + elsif single_tokenizer_keys_present + raise InvalidIndexDefinition, + "field #{source_name.inspect} specifies tokenizer configuration but no :tokenizer" else entries << Entry.new( source: source_name, diff --git a/lib/parade_db/migration_helpers.rb b/lib/parade_db/migration_helpers.rb index f587e06..18cb91f 100644 --- a/lib/parade_db/migration_helpers.rb +++ b/lib/parade_db/migration_helpers.rb @@ -336,7 +336,7 @@ def bm25_index_to_ruby(row) elsif entries.length == 1 "#{source_ruby} #{bm25_tokenizer_config_ruby(entries.first)}" else - configs = entries.map { |e| bm25_tokenizer_config_ruby(e) } + configs = entries.map { |e| bm25_tokenizer_ruby_from_entry(e) } "#{source_ruby} { tokenizers: [#{configs.join(', ')}] }" end end @@ -678,6 +678,10 @@ def bm25_parse_tokenizer(tokenizer_sql_str) end def bm25_tokenizer_config_ruby(entry) + "{ tokenizer: #{bm25_tokenizer_ruby_from_entry(entry)} }" + end + + def bm25_tokenizer_ruby_from_entry(entry) opts = entry[:options].dup positional_args = Array(opts.delete(:__positional)) alias_val = opts.delete(:alias) @@ -685,16 +689,19 @@ def bm25_tokenizer_config_ruby(entry) max_val = opts.delete(:max) positional_args = [min_val, max_val] + positional_args if min_val && max_val - parts = ["tokenizer: #{entry[:tokenizer].to_sym.inspect}"] - parts << "args: #{positional_args.inspect}" unless positional_args.empty? - parts << "alias: #{alias_val.inspect}" if alias_val + opts[:alias] = alias_val if alias_val + + bm25_tokenizer_ruby(entry[:tokenizer], positional_args, opts) + end - unless opts.empty? - named_pairs = opts.map { |k, v| "#{k.inspect} => #{v.inspect}" }.join(", ") - parts << "named_args: { #{named_pairs} }" + def bm25_tokenizer_ruby(name, positional_args, options) + if name.match?(/\A[a-z_][a-z0-9_]*\z/) && Tokenizer.respond_to?(name) + args = positional_args.map { |arg| ruby_literal(arg) } + args << "options: #{ruby_hash_literal(options)}" unless options.empty? + return "Tokenizer.#{name}(#{args.join(', ')})" end - "{ #{parts.join(', ')} }" + "Tokenizer.new(#{name.inspect}, #{ruby_literal(positional_args.empty? ? nil : positional_args)}, #{ruby_literal(options.empty? ? nil : options)})" end def split_sql_arguments(args_sql) diff --git a/spec/index_dsl_unit_spec.rb b/spec/index_dsl_unit_spec.rb index f0d76fa..ee9f98d 100644 --- a/spec/index_dsl_unit_spec.rb +++ b/spec/index_dsl_unit_spec.rb @@ -12,8 +12,8 @@ id: {}, description: { tokenizers: [ - { tokenizer: :literal }, - { tokenizer: :simple, alias: "description_simple", filters: [:lowercase] } + Tokenizer.literal(), + Tokenizer.simple(options: {alias: "description_simple", lowercase: true}) ] } } @@ -32,7 +32,7 @@ self.where = "archived_at IS NULL" self.fields = { id: {}, - description: { tokenizer: :simple } + description: { tokenizer: Tokenizer.simple() } } end @@ -54,7 +54,7 @@ self.key_field = :id self.fields = { id: {}, - description: { tokenizer: :simple } + description: { tokenizer: Tokenizer.simple() } } end @@ -73,8 +73,8 @@ self.fields = { id: {}, description: { - tokenizers: [{ tokenizer: :literal }], - tokenizer: :simple + tokenizers: [Tokenizer.literal()], + tokenizer: Tokenizer.simple() } } end @@ -83,18 +83,18 @@ assert_includes error.message, "cannot mix" end - it "rejects tokenizer config without tokenizer key" do + it "rejects non-Tokenizer tokenizer config" do klass = Class.new(ParadeDB::Index) do self.table_name = :products self.key_field = :id self.fields = { id: {}, - description: { filters: [:lowercase] } + description: { tokenizer: :simple } } end error = assert_raises(ParadeDB::InvalidIndexDefinition) { klass.compiled_definition } - assert_includes error.message, "no :tokenizer" + assert_includes error.message, "must be a Tokenizer" end it "compiles a valid index definition" do @@ -103,9 +103,9 @@ self.key_field = :id self.fields = { id: {}, - description: { tokenizer: :simple }, - category: { tokenizer: :literal }, - "metadata->>'title'" => { tokenizer: :simple, alias: "metadata_title" } + description: { tokenizer: Tokenizer.simple() }, + category: { tokenizer: Tokenizer.literal() }, + "metadata->>'title'" => { tokenizer: Tokenizer.simple(options: {alias: "metadata_title"}) } } end @@ -134,8 +134,8 @@ id: {}, description: { tokenizers: [ - { tokenizer: :simple }, - { tokenizer: :literal } + Tokenizer.simple(), + Tokenizer.literal() ] } } @@ -154,8 +154,8 @@ id: {}, description: { tokenizers: [ - { tokenizer: :simple, alias: "description_simple" }, - { tokenizer: :literal, alias: "description_exact" } + Tokenizer.simple(options: {alias: "description_simple"}), + Tokenizer.literal(options: {alias: "description_exact"}) ] } } @@ -174,7 +174,7 @@ self.key_field = :id self.fields = { id: {}, - description: { tokenizer: :ngram, named_args: { min: 2, max: 5 } } + description: { tokenizer: Tokenizer.ngram(2, 5) } } end @@ -186,71 +186,35 @@ SQL end - it "rejects partial ngram bounds in named_args" do - klass = Class.new(ParadeDB::Index) do - self.table_name = :products - self.key_field = :id - self.fields = { - id: {}, - description: { tokenizer: :ngram, named_args: { min: 2 } } - } - end - - error = assert_raises(ArgumentError) do - ActiveRecord::Base.connection.send(:build_create_sql, klass.compiled_definition, if_not_exists: false) - end - assert_includes error.message, ":min and :max must be provided together" - end - - it "rejects max-only ngram bounds in named_args" do - klass = Class.new(ParadeDB::Index) do - self.table_name = :products - self.key_field = :id - self.fields = { - id: {}, - description: { tokenizer: :ngram, named_args: { max: 5 } } - } - end - - error = assert_raises(ArgumentError) do - ActiveRecord::Base.connection.send(:build_create_sql, klass.compiled_definition, if_not_exists: false) - end - assert_includes error.message, ":min and :max must be provided together" - end - - it "renders qualified and inline tokenizer forms" do + it "rejects non-Tokenizer values in tokenizers arrays" do klass = Class.new(ParadeDB::Index) do self.table_name = :products self.key_field = :id self.fields = { id: {}, - description: { tokenizer: "pdb::xyz" }, - "metadata->>'title'" => { tokenizer: "pdb::abc(12, \"fafda\")" } + description: { tokenizers: [Tokenizer.literal(), :simple] } } end - sql = ActiveRecord::Base.connection.send(:build_create_sql, klass.compiled_definition, if_not_exists: false) - assert_sql_equal <<~SQL, sql - CREATE INDEX products_bm25_idx ON products - USING bm25 (id, (description::pdb::xyz), ((metadata->>'title')::pdb::abc(12, "fafda"))) - WITH (key_field='id') - SQL + error = assert_raises(ParadeDB::InvalidIndexDefinition) { klass.compiled_definition } + assert_includes error.message, "must be a Tokenizer" end - it "prefixes unqualified inline tokenizers with pdb namespace" do + it "renders custom tokenizer objects" do klass = Class.new(ParadeDB::Index) do self.table_name = :products self.key_field = :id self.fields = { id: {}, - description: { tokenizer: "ngram(2, 5)" } + description: { tokenizer: Tokenizer.new("pdb::xyz", nil, nil) }, + "metadata->>'title'" => { tokenizer: Tokenizer.new("pdb::abc", [12, "fafda"], nil) } } end sql = ActiveRecord::Base.connection.send(:build_create_sql, klass.compiled_definition, if_not_exists: false) assert_sql_equal <<~SQL, sql CREATE INDEX products_bm25_idx ON products - USING bm25 (id, (description::pdb.ngram(2, 5))) + USING bm25 (id, (description::pdb::xyz), ((metadata->>'title')::pdb::abc(12, 'fafda'))) WITH (key_field='id') SQL end @@ -262,10 +226,7 @@ self.fields = { id: {}, description: { - tokenizer: :ngram, - args: [2, 5], - named_args: { prefix_only: true }, - alias: "description_ngram" + tokenizer: Tokenizer.ngram(2, 5, options: {prefix_only: true, alias: "description_ngram"}) } } end diff --git a/spec/index_migration_integration_spec.rb b/spec/index_migration_integration_spec.rb index 4b4376f..f104c0e 100644 --- a/spec/index_migration_integration_spec.rb +++ b/spec/index_migration_integration_spec.rb @@ -16,11 +16,11 @@ class IndexMigrationBookIndex < ParadeDB::Index id: {}, title: { tokenizers: [ - { tokenizer: :literal }, - { tokenizer: :simple, alias: "title_simple", filters: [:lowercase] } + Tokenizer.literal(), + Tokenizer.simple(options: {alias: "title_simple", lowercase: true}) ] }, - author: { tokenizer: :literal }, + author: { tokenizer: Tokenizer.literal() }, metadata: { fast: true, expand_dots: false } } end @@ -31,7 +31,7 @@ class IndexMigrationBookByNameIndex < ParadeDB::Index self.index_name = :books_by_name_bm25_idx self.fields = { id: {}, - title: { tokenizer: :simple } + title: { tokenizer: Tokenizer.simple() } } end @@ -95,8 +95,8 @@ class IndexMigrationBookByNameIndex < ParadeDB::Index id: {}, title: { tokenizers: [ - { tokenizer: :literal }, - { tokenizer: :simple } + Tokenizer.literal(), + Tokenizer.simple() ] } } @@ -144,7 +144,7 @@ class IndexMigrationBookByNameIndex < ParadeDB::Index self.where = "author IS NOT NULL" self.fields = { id: {}, - title: { tokenizer: :simple }, + title: { tokenizer: Tokenizer.simple() }, author: {} } end @@ -167,7 +167,7 @@ class IndexMigrationBookByNameIndex < ParadeDB::Index :books, fields: { id: {}, - title: { tokenizer: :simple } + title: { tokenizer: Tokenizer.simple() } }, key_field: :id, name: :books_custom_bm25_idx, @@ -187,7 +187,7 @@ class IndexMigrationBookByNameIndex < ParadeDB::Index :books, fields: { id: {}, - title: { tokenizer: :simple } + title: { tokenizer: Tokenizer.simple() } }, key_field: :id, name: :books_partial_bm25_idx, @@ -214,7 +214,7 @@ class IndexMigrationBookByNameIndex < ParadeDB::Index :books, fields: { id: {}, - title: { tokenizer: :simple } + title: { tokenizer: Tokenizer.simple() } }, key_field: :id, name: :books_concurrent_bm25_idx, @@ -252,7 +252,7 @@ class IndexMigrationBookByNameIndex < ParadeDB::Index :books, fields: { id: {}, - title: { tokenizer: :simple } + title: { tokenizer: Tokenizer.simple() } }, key_field: :id, name: :books_custom_bm25_idx, @@ -292,7 +292,7 @@ class IndexMigrationBookByNameIndex < ParadeDB::Index self.key_field = :id self.fields = { id: {}, - title: { tokenizer: :simple } + title: { tokenizer: Tokenizer.simple() } } end v2 = Class.new(ParadeDB::Index) do @@ -300,8 +300,8 @@ class IndexMigrationBookByNameIndex < ParadeDB::Index self.key_field = :id self.fields = { id: {}, - title: { tokenizer: :simple }, - author: { tokenizer: :literal } + title: { tokenizer: Tokenizer.simple() }, + author: { tokenizer: Tokenizer.literal() } } end @@ -348,7 +348,7 @@ class IndexMigrationBookByNameIndex < ParadeDB::Index :books, fields: { id: {}, - title: { tokenizer: :simple } + title: { tokenizer: Tokenizer.simple() } }, key_field: :id, name: :books_concurrent_bm25_idx, @@ -376,7 +376,7 @@ class IndexMigrationBookByNameIndex < ParadeDB::Index :books, fields: { id: {}, - title: { tokenizer: :simple, alias: "title_simple" } + title: { tokenizer: Tokenizer.simple(options: {alias: "title_simple"}) } }, key_field: :id, index_options: { target_segment_count: 17 }, @@ -395,7 +395,7 @@ class IndexMigrationBookByNameIndex < ParadeDB::Index end assert_equal <<~RUBY.strip, add_stmt.to_s.strip - add_bm25_index :books, fields: { id: {}, title: { tokenizer: :simple, alias: "title_simple" } }, key_field: :id, name: "books_bm25_idx", index_options: { :target_segment_count => 17 } + add_bm25_index :books, fields: { id: {}, title: { tokenizer: Tokenizer.simple(options: { :alias => "title_simple" }) } }, key_field: :id, name: "books_bm25_idx", index_options: { :target_segment_count => 17 } RUBY expect(schema).not_to match(/add_index.*books_bm25_idx/) expect(schema).not_to match(/t\.index.*books_bm25_idx/) @@ -411,8 +411,7 @@ class IndexMigrationBookByNameIndex < ParadeDB::Index self.fields = { id: {}, "(metadata->>'title')::text": { - tokenizer: :simple, - alias: "metadata_title" + tokenizer: Tokenizer.simple(options: {alias: "metadata_title"}) } } end @@ -471,8 +470,8 @@ class IndexMigrationBookByNameIndex < ParadeDB::Index id: {}, title: { tokenizers: [ - { tokenizer: :literal }, - { tokenizer: :simple, alias: "title_simple" } + Tokenizer.literal(), + Tokenizer.simple(options: {alias: "title_simple"}) ] } }, @@ -491,7 +490,7 @@ class IndexMigrationBookByNameIndex < ParadeDB::Index end assert_equal <<~RUBY.strip, add_stmt.to_s.strip - add_bm25_index :books, fields: { id: {}, title: { tokenizers: [{ tokenizer: :literal }, { tokenizer: :simple, alias: "title_simple" }] } }, key_field: :id, name: "books_bm25_idx", index_options: { :target_segment_count => 17 } + add_bm25_index :books, fields: { id: {}, title: { tokenizers: [Tokenizer.literal(), Tokenizer.simple(options: { :alias => "title_simple" })] } }, key_field: :id, name: "books_bm25_idx", index_options: { :target_segment_count => 17 } RUBY conn.remove_bm25_index(:books, if_exists: true) @@ -508,9 +507,7 @@ class IndexMigrationBookByNameIndex < ParadeDB::Index fields: { id: {}, "(metadata->>'title')::text": { - tokenizer: :simple, - alias: "metadata_title_text", - filters: [:lowercase] + tokenizer: Tokenizer.simple(options: {alias: "metadata_title_text", lowercase: true}) } }, key_field: :id, @@ -525,7 +522,7 @@ class IndexMigrationBookByNameIndex < ParadeDB::Index end assert_equal <<~RUBY.strip, add_stmt.to_s.strip - add_bm25_index :books, fields: { id: {}, "metadata ->> 'title'::text" => { tokenizer: :simple, alias: "metadata_title_text", named_args: { :lowercase => true } } }, key_field: :id, name: "books_bm25_idx" + add_bm25_index :books, fields: { id: {}, "metadata ->> 'title'::text" => { tokenizer: Tokenizer.simple(options: { :lowercase => true, :alias => "metadata_title_text" }) } }, key_field: :id, name: "books_bm25_idx" RUBY conn.remove_bm25_index(:books, if_exists: true) @@ -541,7 +538,7 @@ class IndexMigrationBookByNameIndex < ParadeDB::Index :books, fields: { id: {}, - title: { tokenizer: :simple, alias: "title_simple" } + title: { tokenizer: Tokenizer.simple(options: {alias: "title_simple"}) } }, key_field: :id, where: "author IS NOT NULL", @@ -558,7 +555,7 @@ class IndexMigrationBookByNameIndex < ParadeDB::Index end assert_equal <<~RUBY.strip, add_stmt.to_s.strip - add_bm25_index :books, fields: { id: {}, title: { tokenizer: :simple, alias: "title_simple" } }, key_field: :id, name: "books_bm25_idx", where: "author IS NOT NULL" + add_bm25_index :books, fields: { id: {}, title: { tokenizer: Tokenizer.simple(options: { :alias => "title_simple" }) } }, key_field: :id, name: "books_bm25_idx", where: "author IS NOT NULL" RUBY conn.remove_bm25_index(:books, if_exists: true) diff --git a/spec/index_runtime_features_unit_spec.rb b/spec/index_runtime_features_unit_spec.rb index c3757ce..8cef369 100644 --- a/spec/index_runtime_features_unit_spec.rb +++ b/spec/index_runtime_features_unit_spec.rb @@ -99,7 +99,7 @@ self.fields = { id: {}, category: {}, - description: { tokenizer: :simple, alias: "description_simple" } + description: { tokenizer: Tokenizer.simple(options: {alias: "description_simple"}) } } end) @@ -121,7 +121,7 @@ self.fields = { id: {}, category: {}, - description: { tokenizer: :simple, alias: "description_simple" } + description: { tokenizer: Tokenizer.simple(options: {alias: "description_simple"}) } } end) @@ -167,7 +167,7 @@ self.key_field = :id self.fields = { id: {}, - description: { tokenizer: :simple, alias: "description_simple" } + description: { tokenizer: Tokenizer.simple(options: {alias: "description_simple"}) } } end) @@ -192,7 +192,7 @@ self.key_field = :id self.fields = { id: {}, - description: { tokenizer: :literal, alias: "description_exact" } + description: { tokenizer: Tokenizer.literal(options: {alias: "description_exact"}) } } end) @@ -214,7 +214,7 @@ self.key_field = :id self.fields = { id: {}, - description: { tokenizer: :simple, alias: "description_simple" } + description: { tokenizer: Tokenizer.simple(options: {alias: "description_simple"}) } } end) @@ -243,7 +243,7 @@ self.key_field = :id self.fields = { id: {}, - description: { tokenizer: :simple, alias: "description_simple" } + description: { tokenizer: Tokenizer.simple(options: {alias: "description_simple"}) } } end) diff --git a/spec/key_field_runtime_integration_spec.rb b/spec/key_field_runtime_integration_spec.rb index 8c3e4fa..c86ccbb 100644 --- a/spec/key_field_runtime_integration_spec.rb +++ b/spec/key_field_runtime_integration_spec.rb @@ -12,7 +12,7 @@ class RuntimeKeyDocIndex < ParadeDB::Index self.key_field = :external_id self.fields = { external_id: {}, - body: { tokenizer: :simple }, + body: { tokenizer: Tokenizer.simple() }, tag: {} } end From b0f065934ffdbc8981bf6ea45a9830f8353ba6b3 Mon Sep 17 00:00:00 2001 From: Isaac Van Doren Date: Mon, 20 Apr 2026 16:30:05 -0500 Subject: [PATCH 09/12] Update changelog --- CHANGELOG.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b62a61..441a3ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,8 +6,7 @@ All notable changes to this project will be documented in this file. The format ### Changed -- BM25 index definitions and schema dumps now specify field tokenizers with - `Tokenizer.*(...)`, matching query APIs like `matching_all` +- **BREAKING**: Use function based approach for specifying tokenizers: `Tokenizer.simple(options: {alias: "description_simple"})` ## [0.6.0] - 2026-04-14 From caaeb40a8dd7630960d0ca0cefa7f6bdbc3f3778 Mon Sep 17 00:00:00 2001 From: Isaac Van Doren Date: Mon, 20 Apr 2026 16:35:13 -0500 Subject: [PATCH 10/12] update to 0.23 in CI --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 31c0858..664e6cc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -96,7 +96,7 @@ jobs: services: paradedb: - image: paradedb/paradedb:0.22.0-pg18 + image: paradedb/paradedb:0.23.0-pg18 env: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres From cbaa01ce89b91d069da26e59e7bd899f64488108 Mon Sep 17 00:00:00 2001 From: Isaac Van Doren Date: Tue, 21 Apr 2026 08:41:38 -0500 Subject: [PATCH 11/12] Revert "Add issues: write permission to notify job" This reverts commit 0c3782ea3f3ce0ae9b212bea7053aa5309a0eaee. --- .github/workflows/schema-compat.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/schema-compat.yml b/.github/workflows/schema-compat.yml index 79255fd..4df51dd 100644 --- a/.github/workflows/schema-compat.yml +++ b/.github/workflows/schema-compat.yml @@ -96,8 +96,6 @@ jobs: name: Notify on Failure runs-on: ubuntu-latest needs: [schema-compat, integration-tests] - permissions: - issues: write if: failure() steps: - name: Create GitHub Issue From c0cb1a25245c774ca0014029bc7612806807c80b Mon Sep 17 00:00:00 2001 From: Isaac Van Doren Date: Tue, 21 Apr 2026 08:42:26 -0500 Subject: [PATCH 12/12] Add comment about tokenizer functions to apiignore.json5 --- apiignore.json5 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apiignore.json5 b/apiignore.json5 index 1b54912..386dabd 100644 --- a/apiignore.json5 +++ b/apiignore.json5 @@ -21,6 +21,9 @@ "pdb.alias_out_safe", "pdb.alias_recv", "pdb.alias_send", + + // These functions are implementation details of tokenizers that the ORM + // doesn't need to interact with "pdb.chinese_compatible_in", "pdb.chinese_compatible_out", "pdb.chinese_compatible_recv",