Decode documentaton
This commit is contained in:
@@ -912,6 +912,11 @@ class PreTrainedTokenizer(object):
|
||||
Converts a sequence of ids (integer) in a string, using the tokenizer and vocabulary
|
||||
with options to remove special tokens and clean up tokenization spaces.
|
||||
Similar to doing ``self.convert_tokens_to_string(self.convert_ids_to_tokens(token_ids))``.
|
||||
|
||||
Args:
|
||||
token_ids: list of tokenized input ids. Can be obtained using the `encode` or `encode_plus` methods.
|
||||
skip_special_tokens: if set to True, will replace special tokens.
|
||||
clean_up_tokenization_spaces: if set to True, will clean up the tokenization spaces.
|
||||
"""
|
||||
filtered_tokens = self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user