Skip to content

fix Tokenizer class LLaMATokenizer does not exist or is not currently imported. #8

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: openthaigpt-master
Choose a base branch
from
Open

fix Tokenizer class LLaMATokenizer does not exist or is not currently imported. #8

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 28 additions & 4 deletions lm_eval/models/huggingface.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import torch
import torch.nn.functional as F
import transformers
from transformers import LlamaForCausalLM, LlamaTokenizer
import peft
from peft import __version__ as PEFT_VERSION
from pathlib import Path
Expand Down Expand Up @@ -323,6 +324,22 @@ def _create_auto_model_peft(
)
return model

# def _create_auto_tokenizer(
# self,
# *,
# pretrained: str,
# revision: str,
# subfolder: str,
# tokenizer: Optional[str] = None,
# ) -> transformers.PreTrainedTokenizer:
# """Returns a pre-trained tokenizer from a pre-trained tokenizer configuration."""
# tokenizer = self.AUTO_TOKENIZER_CLASS.from_pretrained(
# pretrained if tokenizer is None else tokenizer,
# revision=revision + ("/" + subfolder if subfolder is not None else ""),
# )
# tokenizer.pad_token = tokenizer.eos_token
# return tokenizer

def _create_auto_tokenizer(
self,
*,
Expand All @@ -332,13 +349,20 @@ def _create_auto_tokenizer(
tokenizer: Optional[str] = None,
) -> transformers.PreTrainedTokenizer:
"""Returns a pre-trained tokenizer from a pre-trained tokenizer configuration."""
tokenizer = self.AUTO_TOKENIZER_CLASS.from_pretrained(
pretrained if tokenizer is None else tokenizer,
revision=revision + ("/" + subfolder if subfolder is not None else ""),
)
try:
tokenizer = self.AUTO_TOKENIZER_CLASS.from_pretrained(
pretrained if tokenizer is None else tokenizer,
revision=revision + ("/" + subfolder if subfolder is not None else ""),
)
except:
# LLaMATokenizer not found, using default tokenizer
tokenizer = LlamaTokenizer.from_pretrained(pretrained,
revision=revision + ("/" + subfolder if subfolder is not None else ""),)

tokenizer.pad_token = tokenizer.eos_token
return tokenizer


@property
def add_special_tokens(self) -> bool:
"""Whether to include special tokens in encoded text. This should be
Expand Down