This commit is contained in:
cronoik
2021-02-20 21:21:33 +01:00
committed by GitHub
parent 9a7e63729f
commit a0dfc2d30f

View File

@@ -3054,12 +3054,13 @@ class PreTrainedTokenizerBase(SpecialTokensMixin):
def convert_tokens_to_string(self, tokens: List[str]) -> str:
"""
Converts a sequence of token ids in a single string. The most simple way to do it is ``" ".join(tokens)`` but
we often want to remove sub-word tokenization artifacts at the same time
Converts a sequence of tokens in a single string. The most simple way to do it is ``" ".join(tokens)`` but
we often want to remove sub-word tokenization artifacts at the same time.
Args:
tokens (:obj:`List[str]`): The token to join in a string.
Return: The joined tokens.
Returns:
:obj:`str`: The joined tokens.
"""
raise NotImplementedError