init

Author: jaco-bro

README.md

@@ -1,2 +1,23 @@
-# tokenizer
+# Tokenizer
 A Zig library for tokenizing text using PCRE2 regular expressions.
+
+## Requirement
+zig v0.13.0
+
+## Install
+```bash
+git clone https://github.com/jaco-bro/tokenizer
+cd tokenizer
+zig build exe --release=fast
+```
+
+## Usage
+- `zig-out/bin/tokenizer_exe [--model MODEL_NAME] COMMAND INPUT` 
+- `zig build run -- [--model MODEL_NAME] COMMAND INPUT` 
+
+```bash
+zig build run -- --encode "hello world"
+zig build run -- --decode "{14990, 1879}"
+zig build run -- --model "phi-4-4bit" --encode "hello world"
+zig build run -- --model "phi-4-4bit" --decode "15339 1917"
+```

build.zig

@@ -0,0 +1,49 @@
+const std = @import("std");
+
+pub fn build(b: *std.Build) !void {
+    const target = b.standardTargetOptions(.{});
+    const optimize = b.standardOptimizeOption(.{});
+    const pcre2_dep = b.dependency("pcre2", .{
+        .target = target,
+        .optimize = optimize,
+    });
+    // Exe
+    const exe = b.addExecutable(.{
+        .name = "tokenizer_exe",
+        .root_source_file = b.path("src/tokenizer.zig"),
+        .target = target,
+        .optimize = .ReleaseFast,
+    });
+    exe.linkLibrary(pcre2_dep.artifact("pcre2-8"));
+    const exe_install = b.addInstallArtifact(exe, .{});
+    const exe_step = b.step("exe", "Install executable");
+    exe_step.dependOn(&exe_install.step);
+    // Run exe
+    const exe_run = b.addRunArtifact(exe);
+    if (b.args) |args| exe_run.addArgs(args);
+    const run_step = b.step("run", "Run tokenizer app");
+    run_step.dependOn(&exe_run.step);
+    // Tests
+    const tst = b.addTest(.{
+        .root_source_file = b.path("src/tokenizer.zig"),
+        .target = target,
+        .optimize = optimize,
+    });
+    tst.linkLibrary(pcre2_dep.artifact("pcre2-8"));
+    // Run test
+    const run_test = b.addRunArtifact(tst);
+    const test_step = b.step("test", "Run tests");
+    test_step.dependOn(&run_test.step);
+    // Lib
+    const lib = b.addSharedLibrary(.{
+        .name = "tokenizer",
+        .root_source_file = b.path("src/tokenizer.zig"),
+        .target = target,
+        .optimize = optimize,
+    });
+    lib.linkLibrary(pcre2_dep.artifact("pcre2-8"));
+    const lib_install = b.addInstallArtifact(lib, .{});
+    const lib_step = b.step("lib", "Install library only");
+    lib_step.dependOn(&lib_install.step);
+    b.getInstallStep().dependOn(&lib_install.step);
+}

build.zig.zon

@@ -0,0 +1,17 @@
+.{
+    .name = "tokenizer",
+    .version = "0.0.1",
+    .dependencies = .{
+        .pcre2 = .{
+            .url = "https://github.com/PCRE2Project/pcre2/archive/refs/tags/pcre2-10.45.tar.gz",
+            .hash = "1220c927731a4b31f5ebf8b4f33f0efe6fa1561cdcf026f39cbee512e08ad24fb785",
+        },
+    },
+    .paths = .{
+        "build.zig",
+        "build.zig.zon",
+        "src",
+        "LICENSE",
+        "README.md",
+    },
+}