filipino-tokenizer/pyproject.toml at main · JpCurada/filipino-tokenizer · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39

[build-system]
requires = ["setuptools>=68", "wheel", "setuptools-rust>=1.5.2"]
build-backend = "setuptools.build_meta"

[project]
name = "filipino-tokenizer"
version = "0.4.1"
description = "Morphology-aware BPE tokenizer for Philippine languages (Tagalog)"
readme = "README.md"
license = { file = "LICENSE" }
requires-python = ">=3.10"
keywords = ["nlp", "tokenizer", "tagalog", "filipino", "bpe", "morphology"]
classifiers = [
    "Development Status :: 3 - Alpha",
    "Intended Audience :: Science/Research",
    "License :: OSI Approved :: MIT License",
    "Programming Language :: Python :: 3",
    "Programming Language :: Python :: 3.10",
    "Programming Language :: Python :: 3.11",
    "Programming Language :: Python :: 3.12",
    "Programming Language :: Python :: 3.13",
    "Topic :: Scientific/Engineering :: Artificial Intelligence",
    "Topic :: Text Processing :: Linguistic",
]

[project.optional-dependencies]
hf = ["transformers>=4.30"]

[project.urls]
Homepage = "https://github.com/JpCurada/filipino-tokenizer"
Repository = "https://github.com/JpCurada/filipino-tokenizer"
Issues = "https://github.com/JpCurada/filipino-tokenizer/issues"

[tool.setuptools.packages.find]
include = ["filipino_tokenizer*"]

[tool.setuptools.package-data]
"filipino_tokenizer" = ["data/*.json", "data/pretrained/*.json", "data/pretrained/*.txt"]