diff --git a/src/transformers/tokenization_utils_base.py b/src/transformers/tokenization_utils_base.py index b6e3498674..a895df01c0 100644 --- a/src/transformers/tokenization_utils_base.py +++ b/src/transformers/tokenization_utils_base.py @@ -3054,11 +3054,12 @@ class PreTrainedTokenizerBase(SpecialTokensMixin): def convert_tokens_to_string(self, tokens: List[str]) -> str: """ - Converts a sequence of tokens in a single string. The most simple way to do it is ``" ".join(tokens)`` but - we often want to remove sub-word tokenization artifacts at the same time. + Converts a sequence of tokens in a single string. The most simple way to do it is ``" ".join(tokens)`` but we + often want to remove sub-word tokenization artifacts at the same time. Args: tokens (:obj:`List[str]`): The token to join in a string. + Returns: :obj:`str`: The joined tokens. """