fix: revert tokenizer version

Author: riccardopinosio

go.mod

@@ -5,7 +5,7 @@ go 1.22
 require (
 	github.com/bodaay/HuggingFaceModelDownloader v0.0.0-20240307153905-2f38356a6d6c
 	github.com/json-iterator/go v1.1.12
-	github.com/knights-analytics/tokenizers v0.11.0
+	github.com/knights-analytics/tokenizers v0.10.0
 	github.com/mattn/go-isatty v0.0.20
 	github.com/stretchr/testify v1.9.0
 	github.com/urfave/cli/v2 v2.27.1

go.sum

@@ -47,8 +47,8 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
-github.com/knights-analytics/tokenizers v0.11.0 h1:AVdRxXL7p2UdrgfWOd1ZF3D1DzRVLqVs3tm8W0iGpTk=
-github.com/knights-analytics/tokenizers v0.11.0/go.mod h1:2ngD6H76uCx8TzGUnsdkji1fI6ehQDzU5RgdO6KQenc=
+github.com/knights-analytics/tokenizers v0.10.0 h1:3IBHb2PQh8qGrmtqbtkkO8XYiL+vnaF1ns19lXtSy6Q=
+github.com/knights-analytics/tokenizers v0.10.0/go.mod h1:2ngD6H76uCx8TzGUnsdkji1fI6ehQDzU5RgdO6KQenc=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=