SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

A few fixes in the documentation (#10033)

Browse Source
This commit is contained in:
Sylvain Gugger
2021-02-08 05:02:01 -05:00
committed by GitHub
parent 04fd783cc5
commit 45aaf5f7ab
6 changed files with 12 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.circleci/deploy.sh
View File

@@ -56,5 +56,5 @@ deploy_doc "eb0e0ce" v3.4.0
deploy_doc "818878d" v3.5.1 deploy_doc "818878d" v3.5.1
deploy_doc "c781171" v4.0.0 deploy_doc "c781171" v4.0.0
deploy_doc "bfa4ccf" v4.1.1 deploy_doc "bfa4ccf" v4.1.1
deploy_doc "7d9a9d0" # v4.2.1 Latest stable release deploy_doc "7d9a9d0" # v4.2.2 Latest stable release
deploy_doc "4cd22r1" v4.3.0 # Pre-release deploy_doc "4cd22r1" v4.3.0 # Pre-release

4
docs/source/_static/js/custom.js
View File

@@ -1,11 +1,11 @@
// These two things need to be updated at each release for the version selector. // These two things need to be updated at each release for the version selector.
// Last stable version // Last stable version
const stableVersion = "v4.2.1" const stableVersion = "v4.2.2"
// Dictionary doc folder to label. The last stable version should have an empty key. // Dictionary doc folder to label. The last stable version should have an empty key.
const versionMapping = { const versionMapping = {
"master": "master", "master": "master",
"v4.3.0": "v4.3.0 (pre)", "v4.3.0": "v4.3.0 (pre)",
"": "v4.2.0/v4.2.1 (stable)", "": "v4.2.0/v4.2.1/v4.2.2 (stable)",
"v4.1.1": "v4.1.0/v4.1.1", "v4.1.1": "v4.1.0/v4.1.1",
"v4.0.1": "v4.0.0/v4.0.1", "v4.0.1": "v4.0.0/v4.0.1",
"v3.5.1": "v3.5.0/v3.5.1", "v3.5.1": "v3.5.0/v3.5.1",

12
docs/source/main_classes/tokenizer.rst
View File

@@ -54,9 +54,9 @@ PreTrainedTokenizer
.. autoclass:: transformers.PreTrainedTokenizer .. autoclass:: transformers.PreTrainedTokenizer
:special-members: __call__ :special-members: __call__
:members: :members: batch_decode, convert_ids_to_tokens, convert_tokens_to_ids, convert_tokens_to_string, decode, encode,
get_added_vocab, get_special_tokens_mask, num_special_tokens_to_add, prepare_for_tokenization, tokenize,
.. automethod:: encode vocab_size
PreTrainedTokenizerFast PreTrainedTokenizerFast
@@ -64,9 +64,9 @@ PreTrainedTokenizerFast
.. autoclass:: transformers.PreTrainedTokenizerFast .. autoclass:: transformers.PreTrainedTokenizerFast
:special-members: __call__ :special-members: __call__
:members: :members: batch_decode, convert_ids_to_tokens, convert_tokens_to_ids, convert_tokens_to_string, decode, encode,
get_added_vocab, get_special_tokens_mask, num_special_tokens_to_add,
.. automethod:: encode set_truncation_and_padding,tokenize, vocab_size
BatchEncoding BatchEncoding

2
src/transformers/tokenization_utils.py
View File

@@ -101,7 +101,7 @@ def _is_start_of_word(text):
return bool(_is_control(first_char) | _is_punctuation(first_char) | _is_whitespace(first_char)) return bool(_is_control(first_char) | _is_punctuation(first_char) | _is_whitespace(first_char))
@add_end_docstrings(INIT_TOKENIZER_DOCSTRING, """ .. automethod:: __call__""") @add_end_docstrings(INIT_TOKENIZER_DOCSTRING)
class PreTrainedTokenizer(PreTrainedTokenizerBase): class PreTrainedTokenizer(PreTrainedTokenizerBase):
""" """
Base class for all slow tokenizers. Base class for all slow tokenizers.

2
src/transformers/tokenization_utils_base.py
View File

@@ -2056,7 +2056,7 @@ class PreTrainedTokenizerBase(SpecialTokensMixin):
Whether or not to add the special tokens associated with the corresponding model. Whether or not to add the special tokens associated with the corresponding model.
kwargs (additional keyword arguments, `optional`): kwargs (additional keyword arguments, `optional`):
Will be passed to the underlying model specific encode method. See details in Will be passed to the underlying model specific encode method. See details in
:meth:`~transformers.PreTrainedTokenizer.__call__` :meth:`~transformers.PreTrainedTokenizerBase.__call__`
Returns: Returns:
:obj:`List[str]`: The list of tokens. :obj:`List[str]`: The list of tokens.

7
src/transformers/tokenization_utils_fast.py
View File

@@ -56,12 +56,7 @@ TOKENIZER_CONFIG_FILE = "tokenizer_config.json"
ADDED_TOKENS_FILE = "added_tokens.json" ADDED_TOKENS_FILE = "added_tokens.json"
@add_end_docstrings( @add_end_docstrings(INIT_TOKENIZER_DOCSTRING)
INIT_TOKENIZER_DOCSTRING,
"""
.. automethod:: __call__
""",
)
class PreTrainedTokenizerFast(PreTrainedTokenizerBase): class PreTrainedTokenizerFast(PreTrainedTokenizerBase):
""" """
Base class for all fast tokenizers (wrapping HuggingFace tokenizers library). Base class for all fast tokenizers (wrapping HuggingFace tokenizers library).