SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

LongformerTokenizerFast (#4547)

Browse Source
This commit is contained in:
Suraj Patil
2020-05-26 01:33:55 +05:30
committed by GitHub
parent c9c385c522
commit 5139733623
2 changed files with 11 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
src/transformers/tokenization_longformer.py
View File

@@ -15,7 +15,7 @@
import logging
from .tokenization_roberta import RobertaTokenizer
from .tokenization_roberta import RobertaTokenizer, RobertaTokenizerFast
logger = logging.getLogger(__name__)
@@ -40,3 +40,12 @@ class LongformerTokenizer(RobertaTokenizer):
"vocab_file": {m: vocab_url for m in _all_longformer_models},
"merges_file": {m: merges_url for m in _all_longformer_models},
}
class LongformerTokenizerFast(RobertaTokenizerFast):
# merges and vocab same as Roberta
max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
pretrained_vocab_files_map = {
"vocab_file": {m: vocab_url for m in _all_longformer_models},
"merges_file": {m: merges_url for m in _all_longformer_models},
}