A few fixes in the documentation (#10033)

This commit is contained in:
Sylvain Gugger
2021-02-08 05:02:01 -05:00
committed by GitHub
parent 04fd783cc5
commit 45aaf5f7ab
6 changed files with 12 additions and 17 deletions

View File

@@ -101,7 +101,7 @@ def _is_start_of_word(text):
return bool(_is_control(first_char) | _is_punctuation(first_char) | _is_whitespace(first_char))
@add_end_docstrings(INIT_TOKENIZER_DOCSTRING, """ .. automethod:: __call__""")
@add_end_docstrings(INIT_TOKENIZER_DOCSTRING)
class PreTrainedTokenizer(PreTrainedTokenizerBase):
"""
Base class for all slow tokenizers.

View File

@@ -2056,7 +2056,7 @@ class PreTrainedTokenizerBase(SpecialTokensMixin):
Whether or not to add the special tokens associated with the corresponding model.
kwargs (additional keyword arguments, `optional`):
Will be passed to the underlying model specific encode method. See details in
:meth:`~transformers.PreTrainedTokenizer.__call__`
:meth:`~transformers.PreTrainedTokenizerBase.__call__`
Returns:
:obj:`List[str]`: The list of tokens.

View File

@@ -56,12 +56,7 @@ TOKENIZER_CONFIG_FILE = "tokenizer_config.json"
ADDED_TOKENS_FILE = "added_tokens.json"
@add_end_docstrings(
INIT_TOKENIZER_DOCSTRING,
"""
.. automethod:: __call__
""",
)
@add_end_docstrings(INIT_TOKENIZER_DOCSTRING)
class PreTrainedTokenizerFast(PreTrainedTokenizerBase):
"""
Base class for all fast tokenizers (wrapping HuggingFace tokenizers library).