diff --git a/src/transformers/tokenization_utils_base.py b/src/transformers/tokenization_utils_base.py index 1836a2dd3d..b6e3498674 100644 --- a/src/transformers/tokenization_utils_base.py +++ b/src/transformers/tokenization_utils_base.py @@ -3054,12 +3054,13 @@ class PreTrainedTokenizerBase(SpecialTokensMixin): def convert_tokens_to_string(self, tokens: List[str]) -> str: """ - Converts a sequence of token ids in a single string. The most simple way to do it is ``" ".join(tokens)`` but - we often want to remove sub-word tokenization artifacts at the same time + Converts a sequence of tokens in a single string. The most simple way to do it is ``" ".join(tokens)`` but + we often want to remove sub-word tokenization artifacts at the same time. Args: tokens (:obj:`List[str]`): The token to join in a string. - Return: The joined tokens. + Returns: + :obj:`str`: The joined tokens. """ raise NotImplementedError