Update tokenizers requirement (#10077)

Author: n1t0

setup.py

@@ -132,7 +132,7 @@
     "tensorflow-cpu>=2.3",
     "tensorflow>=2.3",
     "timeout-decorator",
-    "tokenizers==0.10.1rc1",
+    "tokenizers>=0.10.1,<0.11",
     "torch>=1.0",
     "tqdm>=4.27",
     "unidic>=1.0.2",

src/transformers/dependency_versions_table.py

@@ -45,7 +45,7 @@
     "tensorflow-cpu": "tensorflow-cpu>=2.3",
     "tensorflow": "tensorflow>=2.3",
     "timeout-decorator": "timeout-decorator",
-    "tokenizers": "tokenizers==0.10.1rc1",
+    "tokenizers": "tokenizers>=0.10.1,<0.11",
     "torch": "torch>=1.0",
     "tqdm": "tqdm>=4.27",
     "unidic": "unidic>=1.0.2",