A few fixes in the documentation (#10033)
This commit is contained in:
@@ -101,7 +101,7 @@ def _is_start_of_word(text):
|
||||
return bool(_is_control(first_char) | _is_punctuation(first_char) | _is_whitespace(first_char))
|
||||
|
||||
|
||||
@add_end_docstrings(INIT_TOKENIZER_DOCSTRING, """ .. automethod:: __call__""")
|
||||
@add_end_docstrings(INIT_TOKENIZER_DOCSTRING)
|
||||
class PreTrainedTokenizer(PreTrainedTokenizerBase):
|
||||
"""
|
||||
Base class for all slow tokenizers.
|
||||
|
||||
@@ -2056,7 +2056,7 @@ class PreTrainedTokenizerBase(SpecialTokensMixin):
|
||||
Whether or not to add the special tokens associated with the corresponding model.
|
||||
kwargs (additional keyword arguments, `optional`):
|
||||
Will be passed to the underlying model specific encode method. See details in
|
||||
:meth:`~transformers.PreTrainedTokenizer.__call__`
|
||||
:meth:`~transformers.PreTrainedTokenizerBase.__call__`
|
||||
|
||||
Returns:
|
||||
:obj:`List[str]`: The list of tokens.
|
||||
|
||||
@@ -56,12 +56,7 @@ TOKENIZER_CONFIG_FILE = "tokenizer_config.json"
|
||||
ADDED_TOKENS_FILE = "added_tokens.json"
|
||||
|
||||
|
||||
@add_end_docstrings(
|
||||
INIT_TOKENIZER_DOCSTRING,
|
||||
"""
|
||||
.. automethod:: __call__
|
||||
""",
|
||||
)
|
||||
@add_end_docstrings(INIT_TOKENIZER_DOCSTRING)
|
||||
class PreTrainedTokenizerFast(PreTrainedTokenizerBase):
|
||||
"""
|
||||
Base class for all fast tokenizers (wrapping HuggingFace tokenizers library).
|
||||
|
||||
Reference in New Issue
Block a user