From a0dfc2d30fa08aa605b37c712bd8c8137d7f145f Mon Sep 17 00:00:00 2001 From: cronoik Date: Sat, 20 Feb 2021 21:21:33 +0100 Subject: [PATCH] fixes #10303 (#10304) --- src/transformers/tokenization_utils_base.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/transformers/tokenization_utils_base.py b/src/transformers/tokenization_utils_base.py index 1836a2dd3d..b6e3498674 100644 --- a/src/transformers/tokenization_utils_base.py +++ b/src/transformers/tokenization_utils_base.py @@ -3054,12 +3054,13 @@ class PreTrainedTokenizerBase(SpecialTokensMixin): def convert_tokens_to_string(self, tokens: List[str]) -> str: """ - Converts a sequence of token ids in a single string. The most simple way to do it is ``" ".join(tokens)`` but - we often want to remove sub-word tokenization artifacts at the same time + Converts a sequence of tokens in a single string. The most simple way to do it is ``" ".join(tokens)`` but + we often want to remove sub-word tokenization artifacts at the same time. Args: tokens (:obj:`List[str]`): The token to join in a string. - Return: The joined tokens. + Returns: + :obj:`str`: The joined tokens. """ raise NotImplementedError