feat: Implement function-based tokenizer API

.github/workflows/ci.yml

@@ -38,6 +38,16 @@ jobs:
       - name: Install RuboCop
         run: gem install rubocop --no-document
 
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.13"
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+
       - name: Run RuboCop lint cops
         run: |
           rubocop --version
@@ -48,7 +58,7 @@ jobs:
             Rakefile
 
       - name: Check API coverage
-        run: ruby scripts/check_api_coverage.rb
+        run: uv run --with json5 scripts/check_api_coverage.py
 
       - name: Gem install smoke test
         run: bash scripts/smoke_gem_install.sh
@@ -86,7 +96,7 @@ jobs:
 
     services:
       paradedb:
-        image: paradedb/paradedb:0.22.0-pg18
+        image: paradedb/paradedb:0.23.0-pg18
         env:
           POSTGRES_USER: postgres
           POSTGRES_PASSWORD: postgres

.github/workflows/schema-compat.yml

@@ -38,10 +38,15 @@ jobs:
         with:
           python-version: "3.13"
 
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+
       - name: Run Schema Compatibility Check
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: python scripts/check_schema_compat.py ${{ env.PARADEDB_VERSION }}
+        run: uv run --with json5  scripts/check_schema_compat.py ${{ env.PARADEDB_VERSION }}
 
   integration-tests:
     name: Integration Tests
@@ -91,6 +96,8 @@ jobs:
     name: Notify on Failure
     runs-on: ubuntu-latest
     needs: [schema-compat, integration-tests]
+    permissions:
+      issues: write
     if: failure()
     steps:
       - name: Create GitHub Issue

CHANGELOG.md

@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file. The format
 
 ## [Unreleased]
 
+### Changed
+
+- **BREAKING**: Use function based approach for specifying tokenizers: `Tokenizer.simple(options: {alias: "description_simple"})`
+
 ## [0.6.0] - 2026-04-14
 
 ### Added

api.json → api.json5

@@ -48,6 +48,8 @@
     "PDB_TYPE_TOKENIZER_SIMPLE":             "pdb.simple",
     "PDB_TYPE_TOKENIZER_SOURCE_CODE":        "pdb.source_code",
     "PDB_TYPE_TOKENIZER_UNICODE_WORDS":      "pdb.unicode_words",
-    "PDB_TYPE_TOKENIZER_WHITESPACE":         "pdb.whitespace"
+    "PDB_TYPE_TOKENIZER_WHITESPACE":         "pdb.whitespace",
+    "PDB_TYPE_TOKENIZER_ICU":                "pdb.icu",
+    "PDB_TYPE_TOKENIZER_EDGE_NGRAM":         "pdb.edge_ngram",
   }
 }

apiignore.json → apiignore.json5

@@ -73,6 +73,16 @@
     "pdb.whitespace_out",
     "pdb.whitespace_recv",
     "pdb.whitespace_send",
+    "pdb.edge_ngram_in",
+    "pdb.edge_ngram_out",
+    "pdb.edge_ngram_recv",
+    "pdb.edge_ngram_send",
+    "pdb.json_to_edge_ngram",
+    "pdb.jsonb_to_edge_ngram",
+    "pdb.text_array_to_edge_ngram",
+    "pdb.tokenize_edge_ngram",
+    "pdb.uuid_to_edge_ngram",
+    "pdb.varchar_array_to_edge_ngram",
 
     "pdb.bigint_array_to_alias",
     "pdb.bigint_to_alias",
@@ -210,7 +220,6 @@
   ],
 
   "types": [
-    "pdb.icu",
-    "pdb.proximityclause"
+    "pdb.proximityclause",
   ]
 }

examples/autocomplete/model.rb

@@ -18,9 +18,9 @@ class MockItemIndex < ParadeDB::Index
     id: nil,
     description: nil,
     rating: nil,
-    category: { tokenizer: :literal },
-    "metadata->>'color'" => { tokenizer: :literal, alias: "metadata_color" },
-    "metadata->>'location'" => { tokenizer: :literal, alias: "metadata_location" }
+    category: { tokenizer: Tokenizer.literal() },
+    "metadata->>'color'" => { tokenizer: Tokenizer.literal(options: {alias: "metadata_color"}) },
+    "metadata->>'location'" => { tokenizer: Tokenizer.literal(options: {alias: "metadata_location"}) }
   }
 end
 
@@ -39,10 +39,10 @@ class AutocompleteItemIndex < ParadeDB::Index
     id: nil,
     description: {
       tokenizers: [
-        { tokenizer: :unicode_words },
-        { tokenizer: :ngram, named_args: { min: 3, max: 8 }, alias: "description_ngram" }
+        Tokenizer.unicode_words(),
+        Tokenizer.ngram(3, 8, options: {alias: "description_ngram"})
       ]
     },
-    category: { tokenizer: :literal }
+    category: { tokenizer: Tokenizer.literal() }
   }
 end

examples/faceted_search/model.rb

@@ -18,8 +18,8 @@ class MockItemIndex < ParadeDB::Index
     id: nil,
     description: nil,
     rating: nil,
-    category: { tokenizer: :literal },
-    "metadata->>'color'" => { tokenizer: :literal, alias: "metadata_color" },
-    "metadata->>'location'" => { tokenizer: :literal, alias: "metadata_location" }
+    category: { tokenizer: Tokenizer.literal() },
+    "metadata->>'color'" => { tokenizer: Tokenizer.literal(options: {alias: "metadata_color"}) },
+    "metadata->>'location'" => { tokenizer: Tokenizer.literal(options: {alias: "metadata_location"}) }
   }
 end

examples/hybrid_rrf/model.rb

@@ -26,8 +26,8 @@ class MockItemIndex < ParadeDB::Index
     id: nil,
     description: nil,
     rating: nil,
-    category: { tokenizer: :literal },
-    "metadata->>'color'" => { tokenizer: :literal, alias: "metadata_color" },
-    "metadata->>'location'" => { tokenizer: :literal, alias: "metadata_location" }
+    category: { tokenizer: Tokenizer.literal() },
+    "metadata->>'color'" => { tokenizer: Tokenizer.literal(options: {alias: "metadata_color"}) },
+    "metadata->>'location'" => { tokenizer: Tokenizer.literal(options: {alias: "metadata_location"}) }
   }
 end

examples/more_like_this/model.rb

@@ -18,8 +18,8 @@ class MockItemIndex < ParadeDB::Index
     id: nil,
     description: nil,
     rating: nil,
-    category: { tokenizer: :literal },
-    "metadata->>'color'" => { tokenizer: :literal, alias: "metadata_color" },
-    "metadata->>'location'" => { tokenizer: :literal, alias: "metadata_location" }
+    category: { tokenizer: Tokenizer.literal() },
+    "metadata->>'color'" => { tokenizer: Tokenizer.literal(options: {alias: "metadata_color"}) },
+    "metadata->>'location'" => { tokenizer: Tokenizer.literal(options: {alias: "metadata_location"}) }
   }
 end

examples/rag/model.rb

@@ -18,8 +18,8 @@ class MockItemIndex < ParadeDB::Index
     id: nil,
     description: nil,
     rating: nil,
-    category: { tokenizer: :literal },
-    "metadata->>'color'" => { tokenizer: :literal, alias: "metadata_color" },
-    "metadata->>'location'" => { tokenizer: :literal, alias: "metadata_location" }
+    category: { tokenizer: Tokenizer.literal() },
+    "metadata->>'color'" => { tokenizer: Tokenizer.literal(options: {alias: "metadata_color"}) },
+    "metadata->>'location'" => { tokenizer: Tokenizer.literal(options: {alias: "metadata_location"}) }
   }
 end

lib/parade_db.rb

@@ -12,6 +12,7 @@
 require_relative "parade_db/model"
 require_relative "parade_db/search_methods"
 require_relative "parade_db/railtie"
+require_relative "parade_db/tokenizer"
 
 module ParadeDB
   FacetQueryError = Errors::FacetQueryError

lib/parade_db/arel/builder.rb

@@ -302,12 +302,11 @@ def apply_fuzzy(node, distance:, prefix:, transposition_cost_one:, bridge_to_que
       def apply_tokenizer(node, tokenizer)
         return node if tokenizer.nil?
 
-        unless tokenizer.is_a?(String)
-          raise ArgumentError, "tokenizer must be a string"
+        unless tokenizer.is_a?(Tokenizer)
+          raise ArgumentError, "tokenizer must be a Tokenizer"
         end
 
-        normalized = normalize_tokenizer(tokenizer)
-        Nodes::TokenizerCast.new(node, normalized)
+        return Nodes::TokenizerCast.new(node, tokenizer.render())
       end
 
       def apply_slop(node, slop)

lib/parade_db/index.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative "tokenizer"
+
 module ParadeDB
   class Index
     class << self
@@ -47,117 +49,29 @@ def validate!
 
     class TokenizerParser
       TOKENIZER_EXPRESSION = /\A[a-zA-Z_][a-zA-Z0-9_]*(?:(?:::|\.)[a-zA-Z_][a-zA-Z0-9_]*)*(?:\(\s*[a-zA-Z0-9_'".,=\s:-]*\s*\))?\z/.freeze
-      TOKENIZER_SINGLE_KEYS = %i[tokenizer args named_args filters stemmer alias].freeze
+      TOKENIZER_SINGLE_KEYS = %i[tokenizer alias].freeze
 
       class << self
-        def parse(source_name, tokenizer_spec)
-          case tokenizer_spec
-          when Symbol, String
-            [build_tokenized_entry(source_name, tokenizer_spec.to_s, {})]
-          when Hash
-            tokenizer_spec.map do |tokenizer, opts|
-              case opts
-              when Hash
-                build_tokenized_entry(source_name, tokenizer.to_s, normalize_options(opts))
-              when Symbol, String
-                build_tokenized_entry(source_name, tokenizer.to_s, normalize_positional_option(opts))
-              else
-                raise InvalidIndexDefinition,
-                      "tokenizer options for #{source_name}.#{tokenizer} must be a Hash, Symbol, or String"
-              end
-            end
-          else
-            raise InvalidIndexDefinition,
-                  "invalid tokenizer definition for #{source_name}: #{tokenizer_spec.inspect}"
-          end
-        end
-
-        private
-
-        def parse_structured_tokenizer_config(source_name, config, context:)
-          unless config.is_a?(Hash)
-            raise InvalidIndexDefinition, "#{context} for #{source_name.inspect} must be a Hash"
+        def parse(source_name, tokenizer, context:)
+          unless tokenizer.is_a?(Tokenizer)
+            raise InvalidIndexDefinition, "#{context} for #{source_name.inspect} must be a Tokenizer"
           end
 
-          tokenizer = config[:tokenizer] || config["tokenizer"]
-          if tokenizer.nil?
-            raise InvalidIndexDefinition, "#{context} for #{source_name.inspect} requires :tokenizer"
-          end
-
-          tokenizer_name = tokenizer.to_s
-          validate_tokenizer_name!(source_name, tokenizer_name)
-
-          args = config[:args] || config["args"]
-          named_args = config[:named_args] || config["named_args"]
-          filters = config[:filters] || config["filters"]
-          stemmer = config[:stemmer] || config["stemmer"]
-          alias_name = config[:alias] || config["alias"]
-
           options = {}
-          if args
-            unless args.respond_to?(:to_ary)
-              raise InvalidIndexDefinition, "args for #{source_name.inspect} must be an Array"
-            end
-            options[:__positional] = args.to_ary
-          end
-
-          if named_args
-            unless named_args.is_a?(Hash)
-              raise InvalidIndexDefinition, "named_args for #{source_name.inspect} must be a Hash"
-            end
-            named_args.each { |key, value| options[key.to_sym] = value }
-          end
-
-          if filters
-            unless filters.respond_to?(:to_ary)
-              raise InvalidIndexDefinition, "filters for #{source_name.inspect} must be an Array"
-            end
-            filters.to_ary.each do |name|
-              filter_key = name.to_s
-              if filter_key == "stemmer" && stemmer
-                options[:stemmer] = stemmer
-              else
-                key = filter_key.to_sym
-                options[key] = true unless options.key?(key)
-              end
-            end
-          end
-
-          options[:stemmer] = stemmer if stemmer && !options.key?(:stemmer)
-          options[:alias] = alias_name.to_s if alias_name
-
-          build_tokenized_entry(source_name, tokenizer_name, options)
-        end
-
-        def normalize_options(opts)
-          opts.each_with_object({}) do |(key, value), memo|
-            memo[key.to_sym] = value
-          end
-        end
-
-        def normalize_positional_option(option)
-          { __positional: [option.to_s] }
-        end
+          options[:__positional] = tokenizer.positional_args.dup unless tokenizer.positional_args.nil?
+          tokenizer.options&.each { |key, value| options[key.to_sym] = value }
 
-        def build_tokenized_entry(source_name, tokenizer, options)
-          validate_tokenizer_name!(source_name, tokenizer) unless tokenizer.nil?
           key = options[:alias]&.to_s || source_name
           DefinitionCompiler::Entry.new(
             source: source_name,
             expression: expression?(source_name),
-            tokenizer: tokenizer,
+            tokenizer: tokenizer.name,
             options: options,
             query_key: key
           )
         end
 
-        def validate_tokenizer_name!(source_name, tokenizer)
-          return if TOKENIZER_EXPRESSION.match?(tokenizer)
-
-          raise InvalidIndexDefinition,
-                "invalid tokenizer name #{tokenizer.inspect} for #{source_name}. " \
-                "Expected identifier form like simple, pdb::simple, or pdb::ngram(2, 5, alias=field_alias)."
-        end
+        private
 
         def expression?(value)
           value.match?(/[^a-zA-Z0-9_]/)
@@ -260,33 +174,22 @@ def build_entries(raw_fields)
             elsif tokenizers
               if single_tokenizer_keys_present
                 raise InvalidIndexDefinition,
-                      "field #{source_name.inspect} cannot mix :tokenizers with :tokenizer/:args/:named_args/:filters/:stemmer/:alias"
+                      "field #{source_name.inspect} cannot mix :tokenizers with :tokenizer/:alias"
               end
               unless tokenizers.respond_to?(:to_ary) && !tokenizers.to_ary.empty?
                 raise InvalidIndexDefinition, "field #{source_name.inspect} :tokenizers must be a non-empty Array"
               end
 
               tokenizers.to_ary.each_with_index do |tokenizer_config, idx|
-                entry = TokenizerParser.send(
-                  :parse_structured_tokenizer_config,
-                  source_name,
-                  tokenizer_config,
-                  context: "tokenizers[#{idx}]"
-                )
+                entry = TokenizerParser.parse(source_name, tokenizer_config, context: "tokenizers[#{idx}]")
                 entries << entry
               end
-            elsif single_tokenizer_keys_present
-              unless normalized[:tokenizer]
-                raise InvalidIndexDefinition,
-                      "field #{source_name.inspect} specifies tokenizer configuration but no :tokenizer"
-              end
-              entry = TokenizerParser.send(
-                :parse_structured_tokenizer_config,
-                source_name,
-                select_keys(normalized, TokenizerParser::TOKENIZER_SINGLE_KEYS),
-                context: "tokenizer config"
-              )
+            elsif normalized.key?(:tokenizer)
+              entry = TokenizerParser.parse(source_name, normalized[:tokenizer], context: "tokenizer")
               entries << entry
+            elsif single_tokenizer_keys_present
+              raise InvalidIndexDefinition,
+                    "field #{source_name.inspect} specifies tokenizer configuration but no :tokenizer"
             else
               entries << Entry.new(
                 source: source_name,