SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Remove six.

Browse Source
This commit is contained in:
Aymeric Augustin
2019-12-22 17:56:09 +01:00
parent 6b2200fc88
commit 8af25b1664
19 changed files with 61 additions and 167 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/transformers/tokenization_ctrl.py
View File

@@ -204,11 +204,11 @@ class CTRLTokenizer(PreTrainedTokenizer):
return split_tokens
def _convert_token_to_id(self, token):
""" Converts a token (str/unicode) in an id using the vocab. """
""" Converts a token (str) in an id using the vocab. """
return self.encoder.get(token, self.encoder.get(self.unk_token))
def _convert_id_to_token(self, index):
"""Converts an index (integer) in a token (string/unicode) using the vocab."""
"""Converts an index (integer) in a token (str) using the vocab."""
return self.decoder.get(index, self.unk_token)
def convert_tokens_to_string(self, tokens):