SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

remove token_type_ids from TokenizerBertGeneration output (#10070)

Browse Source
This commit is contained in:
demSd
2021-02-08 19:05:32 +01:00
committed by GitHub
parent e4bf9910dc
commit 84acf0c7bb
1 changed files with 1 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/transformers/models/bert_generation/tokenization_bert_generation.py
View File

@@ -58,6 +58,7 @@ class BertGenerationTokenizer(PreTrainedTokenizer):
pretrained_vocab_files_map = {"vocab_file": {"bert_for_seq_generation": tokenizer_url}} pretrained_vocab_files_map = {"vocab_file": {"bert_for_seq_generation": tokenizer_url}}
max_model_input_sizes = {"bert_for_seq_generation": 512} max_model_input_sizes = {"bert_for_seq_generation": 512}
prefix_tokens: List[int] = [] prefix_tokens: List[int] = []
model_input_names = ["input_ids", "attention_mask"]
def __init__( def __init__(
self, self,