diff --git a/transformers/tokenization_utils.py b/transformers/tokenization_utils.py index de3f48f4c3..a712703190 100644 --- a/transformers/tokenization_utils.py +++ b/transformers/tokenization_utils.py @@ -912,6 +912,11 @@ class PreTrainedTokenizer(object): Converts a sequence of ids (integer) in a string, using the tokenizer and vocabulary with options to remove special tokens and clean up tokenization spaces. Similar to doing ``self.convert_tokens_to_string(self.convert_ids_to_tokens(token_ids))``. + + Args: + token_ids: list of tokenized input ids. Can be obtained using the `encode` or `encode_plus` methods. + skip_special_tokens: if set to True, will replace special tokens. + clean_up_tokenization_spaces: if set to True, will clean up the tokenization spaces. """ filtered_tokens = self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens)