SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Add missing comment quotes (#17379)

Browse Source
This commit is contained in:
Leon Derczynski
2022-06-29 12:16:36 +02:00
committed by GitHub
parent e113c5cb64
commit b8142753f9
1 changed files with 3 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
templates/adding_a_new_model/cookiecutter-template-{{cookiecutter.modelname}}/tokenization_{{cookiecutter.lowercase_modelname}}.py
View File

@@ -144,14 +144,14 @@ class {{cookiecutter.camelcase_modelname}}Tokenizer(PreTrainedTokenizer):
unk_token = AddedToken(unk_token, lstrip=False, rstrip=False) if isinstance(unk_token, str) else unk_token unk_token = AddedToken(unk_token, lstrip=False, rstrip=False) if isinstance(unk_token, str) else unk_token
super().__init__(bos_token=bos_token, eos_token=eos_token, unk_token=unk_token, **kwargs) super().__init__(bos_token=bos_token, eos_token=eos_token, unk_token=unk_token, **kwargs)
"Initialisation" """ Initialisation """
@property @property
def vocab_size(self): def vocab_size(self):
"Returns vocab size" """ Returns vocab size """
def get_vocab(self): def get_vocab(self):
"Returns vocab as a dict" """ Returns vocab as a dict """
def _tokenize(self, text): def _tokenize(self, text):
""" Returns a tokenized string. """ """ Returns a tokenized string. """