@@ -3054,12 +3054,13 @@ class PreTrainedTokenizerBase(SpecialTokensMixin):
|
|||||||
|
|
||||||
def convert_tokens_to_string(self, tokens: List[str]) -> str:
|
def convert_tokens_to_string(self, tokens: List[str]) -> str:
|
||||||
"""
|
"""
|
||||||
Converts a sequence of token ids in a single string. The most simple way to do it is ``" ".join(tokens)`` but
|
Converts a sequence of tokens in a single string. The most simple way to do it is ``" ".join(tokens)`` but
|
||||||
we often want to remove sub-word tokenization artifacts at the same time
|
we often want to remove sub-word tokenization artifacts at the same time.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
tokens (:obj:`List[str]`): The token to join in a string.
|
tokens (:obj:`List[str]`): The token to join in a string.
|
||||||
Return: The joined tokens.
|
Returns:
|
||||||
|
:obj:`str`: The joined tokens.
|
||||||
"""
|
"""
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user