Skip to content

Commit 3917907

Browse files
committed
fix isort ci
1 parent b1b42f9 commit 3917907

File tree

1 file changed

+4
-0
lines changed

1 file changed

+4
-0
lines changed

examples/scripts/preprocess_data.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,12 @@
2727
from primus.backends.megatron.training.tokenizer.tokenizer import (
2828
_add_extra_tokenizer_args as _add_tokenizer_args,
2929
)
30+
31+
# isort: off
3032
from primus.backends.megatron.training.tokenizer.tokenizer import build_tokenizer
3133

34+
# isort: on
35+
3236

3337
# https://stackoverflow.com/questions/33139531/preserve-empty-lines-with-nltks-punkt-tokenizer
3438
class CustomLanguageVars(PunktLanguageVars):

0 commit comments

Comments
 (0)