From d8b641c8393f1f7ce0ec0d0a1f7c1cefcd966971 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Fri, 27 Sep 2019 17:22:01 -0400 Subject: [PATCH] 6 -> 8 models --- docs/source/quickstart.md | 2 +- transformers/tokenization_utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/quickstart.md b/docs/source/quickstart.md index 4eb04c17d2..ccba75e7c0 100644 --- a/docs/source/quickstart.md +++ b/docs/source/quickstart.md @@ -33,7 +33,7 @@ A few other goals: The library is build around three type of classes for each models: -- **model classes** which are PyTorch models (`torch.nn.Modules`) of the 6 models architectures currently provided in the library, e.g. `BertModel` +- **model classes** which are PyTorch models (`torch.nn.Modules`) of the 8 models architectures currently provided in the library, e.g. `BertModel` - **configuration classes** which store all the parameters required to build a model, e.g. `BertConfig`. You don't always need to instantiate these your-self, in particular if you are using a pretrained model without any modification, creating the model will automatically take care of instantiating the configuration (which is part of the model) - **tokenizer classes** which store the vocabulary for each model and provide methods for encoding/decoding strings in list of token embeddings indices to be fed to a model, e.g. `BertTokenizer` diff --git a/transformers/tokenization_utils.py b/transformers/tokenization_utils.py index e8ffff3cb9..1e20588f83 100644 --- a/transformers/tokenization_utils.py +++ b/transformers/tokenization_utils.py @@ -430,7 +430,7 @@ class PreTrainedTokenizer(object): - tokenizer instantiation positional and keywords inputs (e.g. do_lower_case for Bert). This won't save modifications other than (added tokens and special token mapping) you may have - applied to the tokenizer after the instantion (e.g. modifying tokenizer.do_lower_case after creation). + applied to the tokenizer after the instantiation (e.g. modifying tokenizer.do_lower_case after creation). This method make sure the full tokenizer can then be re-loaded using the :func:`~transformers.PreTrainedTokenizer.from_pretrained` class method. """