Skip to content

Commit d14ecc4

Browse files
philippemnoelisaacvando
philippemnoel
and
isaacvando
authored
chore: namespace Tokenizer as ParadeDB::Tokenizer (#108)
Co-authored-by: Isaac Van Doren <isaacjvandoren@gmail.com>
1 parent 2ef3723 commit d14ecc4

20 files changed

Lines changed: 181 additions & 175 deletions

‎CHANGELOG.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file. The format
44

55
## [Unreleased]
66

7+
### Changed
8+
9+
- **BREAKING**: The `Tokenizer` class is now namespaced as `ParadeDB::Tokenizer`. Update references from `Tokenizer.simple(...)` to `ParadeDB::Tokenizer.simple(...)`. Schema dumps (`schema.rb`) now emit the fully-qualified constant.
10+
711
## [0.7.0] - 2026-04-21
812

913
### Changed

‎examples/autocomplete/model.rb‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,9 @@ class MockItemIndex < ParadeDB::Index
1818
id: nil,
1919
description: nil,
2020
rating: nil,
21-
category: { tokenizer: Tokenizer.literal() },
22-
"metadata->>'color'" => { tokenizer: Tokenizer.literal(options: {alias: "metadata_color"}) },
23-
"metadata->>'location'" => { tokenizer: Tokenizer.literal(options: {alias: "metadata_location"}) }
21+
category: { tokenizer: ParadeDB::Tokenizer.literal() },
22+
"metadata->>'color'" => { tokenizer: ParadeDB::Tokenizer.literal(options: {alias: "metadata_color"}) },
23+
"metadata->>'location'" => { tokenizer: ParadeDB::Tokenizer.literal(options: {alias: "metadata_location"}) }
2424
}
2525
end
2626

@@ -39,10 +39,10 @@ class AutocompleteItemIndex < ParadeDB::Index
3939
id: nil,
4040
description: {
4141
tokenizers: [
42-
Tokenizer.unicode_words(),
43-
Tokenizer.ngram(3, 8, options: {alias: "description_ngram"})
42+
ParadeDB::Tokenizer.unicode_words(),
43+
ParadeDB::Tokenizer.ngram(3, 8, options: {alias: "description_ngram"})
4444
]
4545
},
46-
category: { tokenizer: Tokenizer.literal() }
46+
category: { tokenizer: ParadeDB::Tokenizer.literal() }
4747
}
4848
end

‎examples/faceted_search/model.rb‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,8 @@ class MockItemIndex < ParadeDB::Index
1818
id: nil,
1919
description: nil,
2020
rating: nil,
21-
category: { tokenizer: Tokenizer.literal() },
22-
"metadata->>'color'" => { tokenizer: Tokenizer.literal(options: {alias: "metadata_color"}) },
23-
"metadata->>'location'" => { tokenizer: Tokenizer.literal(options: {alias: "metadata_location"}) }
21+
category: { tokenizer: ParadeDB::Tokenizer.literal() },
22+
"metadata->>'color'" => { tokenizer: ParadeDB::Tokenizer.literal(options: {alias: "metadata_color"}) },
23+
"metadata->>'location'" => { tokenizer: ParadeDB::Tokenizer.literal(options: {alias: "metadata_location"}) }
2424
}
2525
end

‎examples/hybrid_rrf/model.rb‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,8 @@ class MockItemIndex < ParadeDB::Index
2626
id: nil,
2727
description: nil,
2828
rating: nil,
29-
category: { tokenizer: Tokenizer.literal() },
30-
"metadata->>'color'" => { tokenizer: Tokenizer.literal(options: {alias: "metadata_color"}) },
31-
"metadata->>'location'" => { tokenizer: Tokenizer.literal(options: {alias: "metadata_location"}) }
29+
category: { tokenizer: ParadeDB::Tokenizer.literal() },
30+
"metadata->>'color'" => { tokenizer: ParadeDB::Tokenizer.literal(options: {alias: "metadata_color"}) },
31+
"metadata->>'location'" => { tokenizer: ParadeDB::Tokenizer.literal(options: {alias: "metadata_location"}) }
3232
}
3333
end

‎examples/more_like_this/model.rb‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,8 @@ class MockItemIndex < ParadeDB::Index
1818
id: nil,
1919
description: nil,
2020
rating: nil,
21-
category: { tokenizer: Tokenizer.literal() },
22-
"metadata->>'color'" => { tokenizer: Tokenizer.literal(options: {alias: "metadata_color"}) },
23-
"metadata->>'location'" => { tokenizer: Tokenizer.literal(options: {alias: "metadata_location"}) }
21+
category: { tokenizer: ParadeDB::Tokenizer.literal() },
22+
"metadata->>'color'" => { tokenizer: ParadeDB::Tokenizer.literal(options: {alias: "metadata_color"}) },
23+
"metadata->>'location'" => { tokenizer: ParadeDB::Tokenizer.literal(options: {alias: "metadata_location"}) }
2424
}
2525
end

‎examples/rag/model.rb‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,8 @@ class MockItemIndex < ParadeDB::Index
1818
id: nil,
1919
description: nil,
2020
rating: nil,
21-
category: { tokenizer: Tokenizer.literal() },
22-
"metadata->>'color'" => { tokenizer: Tokenizer.literal(options: {alias: "metadata_color"}) },
23-
"metadata->>'location'" => { tokenizer: Tokenizer.literal(options: {alias: "metadata_location"}) }
21+
category: { tokenizer: ParadeDB::Tokenizer.literal() },
22+
"metadata->>'color'" => { tokenizer: ParadeDB::Tokenizer.literal(options: {alias: "metadata_color"}) },
23+
"metadata->>'location'" => { tokenizer: ParadeDB::Tokenizer.literal(options: {alias: "metadata_location"}) }
2424
}
2525
end

‎lib/parade_db/arel/builder.rb‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -302,7 +302,7 @@ def apply_fuzzy(node, distance:, prefix:, transposition_cost_one:, bridge_to_que
302302
def apply_tokenizer(node, tokenizer)
303303
return node if tokenizer.nil?
304304

305-
unless tokenizer.is_a?(Tokenizer)
305+
unless tokenizer.is_a?(ParadeDB::Tokenizer)
306306
raise ArgumentError, "tokenizer must be a Tokenizer"
307307
end
308308

‎lib/parade_db/index.rb‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ class TokenizerParser
5353

5454
class << self
5555
def parse(source_name, tokenizer, context:)
56-
unless tokenizer.is_a?(Tokenizer)
56+
unless tokenizer.is_a?(ParadeDB::Tokenizer)
5757
raise InvalidIndexDefinition, "#{context} for #{source_name.inspect} must be a Tokenizer"
5858
end
5959

‎lib/parade_db/migration_helpers.rb‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -695,13 +695,13 @@ def bm25_tokenizer_ruby_from_entry(entry)
695695
end
696696

697697
def bm25_tokenizer_ruby(name, positional_args, options)
698-
if name.match?(/\A[a-z_][a-z0-9_]*\z/) && Tokenizer.respond_to?(name)
698+
if name.match?(/\A[a-z_][a-z0-9_]*\z/) && ParadeDB::Tokenizer.respond_to?(name)
699699
args = positional_args.map { |arg| ruby_literal(arg) }
700700
args << "options: #{ruby_hash_literal(options)}" unless options.empty?
701-
return "Tokenizer.#{name}(#{args.join(', ')})"
701+
return "ParadeDB::Tokenizer.#{name}(#{args.join(', ')})"
702702
end
703703

704-
"Tokenizer.new(#{name.inspect}, #{ruby_literal(positional_args.empty? ? nil : positional_args)}, #{ruby_literal(options.empty? ? nil : options)})"
704+
"ParadeDB::Tokenizer.new(#{name.inspect}, #{ruby_literal(positional_args.empty? ? nil : positional_args)}, #{ruby_literal(options.empty? ? nil : options)})"
705705
end
706706

707707
def split_sql_arguments(args_sql)

‎lib/parade_db/tokenizer.rb‎

Lines changed: 74 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -1,95 +1,97 @@
1-
class Tokenizer
2-
attr_reader :name, :positional_args, :options
1+
module ParadeDB
2+
class Tokenizer
3+
attr_reader :name, :positional_args, :options
4+
5+
def initialize(name, positional_args, options)
6+
@name = name
7+
@positional_args = positional_args
8+
@options = options
9+
end
310

4-
def initialize(name, positional_args, options)
5-
@name = name
6-
@positional_args = positional_args
7-
@options = options
8-
end
11+
def render()
12+
if options.nil? && positional_args.nil?
13+
return "pdb.#{name}"
14+
end
915

10-
def render()
11-
if options.nil? && positional_args.nil?
12-
return "pdb.#{name}"
13-
end
16+
args = []
17+
if !positional_args.nil?
18+
args.concat(positional_args.map { |x| render_positional_arg(x) })
19+
end
20+
if !options.nil?
21+
args.concat(options.map {|k, v| quote_term("#{k}=#{v}")})
22+
end
1423

15-
args = []
16-
if !positional_args.nil?
17-
args.concat(positional_args.map { |x| render_positional_arg(x) })
18-
end
19-
if !options.nil?
20-
args.concat(options.map {|k, v| quote_term("#{k}=#{v}")})
24+
return "pdb.#{name}(#{args.join(",")})"
2125
end
2226

23-
return "pdb.#{name}(#{args.join(",")})"
24-
end
25-
26-
def self.whitespace(options: nil)
27-
new("whitespace", nil, options)
28-
end
27+
def self.whitespace(options: nil)
28+
new("whitespace", nil, options)
29+
end
2930

30-
def self.unicode_words(options: nil)
31-
new("unicode_words", nil, options)
32-
end
31+
def self.unicode_words(options: nil)
32+
new("unicode_words", nil, options)
33+
end
3334

34-
def self.ngram(min_gram, max_gram, options: nil)
35-
new("ngram", [min_gram, max_gram], options)
36-
end
35+
def self.ngram(min_gram, max_gram, options: nil)
36+
new("ngram", [min_gram, max_gram], options)
37+
end
3738

38-
def self.simple(options: nil)
39-
new("simple", nil, options)
40-
end
39+
def self.simple(options: nil)
40+
new("simple", nil, options)
41+
end
4142

42-
def self.literal(options: nil)
43-
new("literal", nil, options)
44-
end
43+
def self.literal(options: nil)
44+
new("literal", nil, options)
45+
end
4546

46-
def self.literal_normalized(options: nil)
47-
new("literal_normalized", nil, options)
48-
end
47+
def self.literal_normalized(options: nil)
48+
new("literal_normalized", nil, options)
49+
end
4950

50-
def self.edge_ngram(min_gram, max_gram, options: nil)
51-
new("edge_ngram", [min_gram, max_gram], options)
52-
end
51+
def self.edge_ngram(min_gram, max_gram, options: nil)
52+
new("edge_ngram", [min_gram, max_gram], options)
53+
end
5354

54-
def self.regex_pattern(pattern, options: nil)
55-
new("regex_pattern", [pattern], options)
56-
end
55+
def self.regex_pattern(pattern, options: nil)
56+
new("regex_pattern", [pattern], options)
57+
end
5758

58-
def self.chinese_compatible(options: nil)
59-
new("chinese_compatible", nil, options)
60-
end
59+
def self.chinese_compatible(options: nil)
60+
new("chinese_compatible", nil, options)
61+
end
6162

62-
def self.lindera(dictionary, options: nil)
63-
new("lindera", [dictionary], options)
64-
end
63+
def self.lindera(dictionary, options: nil)
64+
new("lindera", [dictionary], options)
65+
end
6566

66-
def self.icu(options: nil)
67-
new("icu", nil, options)
68-
end
67+
def self.icu(options: nil)
68+
new("icu", nil, options)
69+
end
6970

70-
def self.jieba(options: nil)
71-
new("jieba", nil, options)
72-
end
71+
def self.jieba(options: nil)
72+
new("jieba", nil, options)
73+
end
7374

74-
def self.source_code(options: nil)
75-
new("source_code", nil, options)
76-
end
75+
def self.source_code(options: nil)
76+
new("source_code", nil, options)
77+
end
7778

78-
private
79+
private
7980

80-
def quote_term(value)
81-
escaped = value.gsub("'", "''")
82-
"'#{escaped}'"
83-
end
81+
def quote_term(value)
82+
escaped = value.gsub("'", "''")
83+
"'#{escaped}'"
84+
end
8485

85-
def render_positional_arg(value)
86-
case value
87-
when true, false, Numeric
88-
value.to_s
89-
when String
90-
quote_term(value)
91-
else
92-
raise InvalidArgumentError, "Unsupported tokenizer arg type: #{value.class}"
86+
def render_positional_arg(value)
87+
case value
88+
when true, false, Numeric
89+
value.to_s
90+
when String
91+
quote_term(value)
92+
else
93+
raise InvalidArgumentError, "Unsupported tokenizer arg type: #{value.class}"
94+
end
9395
end
9496
end
9597
end

0 commit comments

Comments
 (0)