|
18 | 18 | [java.util Locale] |
19 | 19 | [ai.djl.util PairList])) |
20 | 20 |
|
21 | | -(defn- ^Path as-path [x] |
| 21 | +(defn- as-path ^Path [x] |
22 | 22 | (cond |
23 | 23 | (instance? Path x) x |
24 | 24 | (instance? File x) (.toPath ^File x) |
|
105 | 105 | [(if (keyword? key) (name key) (str key)) (str value)])) |
106 | 106 | (merge (:options opts) (:raw-options opts)))) |
107 | 107 |
|
108 | | -(defn- ^java.util.Map constructor-options [opts] |
| 108 | +(defn- constructor-options ^java.util.Map [opts] |
109 | 109 | (merge (djl-options opts) (raw-options opts))) |
110 | 110 |
|
111 | | -(defn ^HuggingFaceTokenizer$Builder builder |
| 111 | +(defn builder |
112 | 112 | "Create a DJL tokenizer builder configured from wrapper opts. |
113 | 113 | Entries in `:options` or `:raw-options` pass through verbatim by DJL option |
114 | 114 | name, such as `modelMaxLength`, `stripAccents`, and `addPrefixSpace`; keyword |
115 | 115 | keys are converted with `name`. Raw entries override translated wrapper opts. |
116 | 116 | `:manager` attaches the built tokenizer to a caller-supplied `NDManager`." |
117 | | - ([] |
| 117 | + (^HuggingFaceTokenizer$Builder [] |
118 | 118 | (HuggingFaceTokenizer/builder)) |
119 | | - ([opts] |
| 119 | + (^HuggingFaceTokenizer$Builder [opts] |
120 | 120 | (let [builder (HuggingFaceTokenizer/builder)] |
121 | 121 | (.configure builder (constructor-options opts)) |
122 | 122 | (when-let [manager (:manager opts)] |
123 | 123 | (.optManager builder manager)) |
124 | 124 | builder))) |
125 | 125 |
|
126 | | -(defn- ^TokenizerConfig tokenizer-config [opts] |
| 126 | +(defn- tokenizer-config ^TokenizerConfig [opts] |
127 | 127 | (some-> (:tokenizer-config opts) as-path TokenizerConfig/load)) |
128 | 128 |
|
129 | 129 | (defn from-file |
|
296 | 296 | :overflow (mapv enc->map (.getOverflowing e)) |
297 | 297 | :exceed-max-length? (.exceedMaxLength e)}) |
298 | 298 |
|
299 | | -(defn- ^Encoding raw-encode |
300 | | - ([^HuggingFaceTokenizer t ^String text] |
| 299 | +(defn- raw-encode |
| 300 | + (^Encoding [^HuggingFaceTokenizer t ^String text] |
301 | 301 | (.encode t text)) |
302 | | - ([^HuggingFaceTokenizer t ^String text pair-or-opts] |
| 302 | + (^Encoding [^HuggingFaceTokenizer t ^String text pair-or-opts] |
303 | 303 | (if (map? pair-or-opts) |
304 | 304 | (let [{:keys [add-special-tokens? with-overflowing-tokens?] |
305 | 305 | :or {add-special-tokens? true with-overflowing-tokens? false}} pair-or-opts] |
306 | 306 | (.encode t text (boolean add-special-tokens?) |
307 | 307 | (boolean with-overflowing-tokens?))) |
308 | 308 | (.encode t text ^String pair-or-opts))) |
309 | | - ([^HuggingFaceTokenizer t ^String text ^String text-pair |
| 309 | + (^Encoding [^HuggingFaceTokenizer t ^String text ^String text-pair |
310 | 310 | {:keys [add-special-tokens? with-overflowing-tokens?] |
311 | 311 | :or {add-special-tokens? true with-overflowing-tokens? false}}] |
312 | 312 | (.encode t text text-pair (boolean add-special-tokens?) |
|
0 commit comments