SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

fix: switch from slow to generic tokenizer class (#15122)

Browse Source
This commit is contained in:
Leandro von Werra
2022-01-12 15:12:43 +01:00
committed by GitHub
parent 27b819b0e3
commit aa0135f2e0
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
examples/research_projects/codeparrot/scripts/bpe_training.py
View File

@@ -2,7 +2,7 @@ from datasets import load_dataset
from tqdm import tqdm from tqdm import tqdm
from arguments import TokenizerTrainingArguments from arguments import TokenizerTrainingArguments
from transformers import GPT2Tokenizer, HfArgumentParser from transformers import AutoTokenizer, HfArgumentParser
from transformers.models.gpt2.tokenization_gpt2 import bytes_to_unicode from transformers.models.gpt2.tokenization_gpt2 import bytes_to_unicode
@@ -17,7 +17,7 @@ parser = HfArgumentParser(TokenizerTrainingArguments)
args = parser.parse_args() args = parser.parse_args()
# Base tokenizer # Base tokenizer
tokenizer = GPT2Tokenizer.from_pretrained(args.base_tokenizer) tokenizer = AutoTokenizer.from_pretrained(args.base_tokenizer)
base_vocab = list(bytes_to_unicode().values()) base_vocab = list(bytes_to_unicode().values())
# Load dataset # Load dataset