SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Change to use relative imports in some files & Add python prompt symbols to example codes (#7202)

Browse Source 
* Move 'from transformers' statements to relative imports in some files

* Add python prompt symbols in front of the example codes

* Reformat the code

* Add one missing space

Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com>

Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com>
This commit is contained in:
Sohee Yang
2020-09-18 01:30:45 +09:00
committed by GitHub
parent 0fe6e435b6
commit e643a29722
20 changed files with 267 additions and 270 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/transformers/integrations.py
View File

@@ -3,8 +3,8 @@ import os
import numpy as np import numpy as np
from transformers.trainer_utils import PREFIX_CHECKPOINT_DIR, BestRun from .trainer_utils import PREFIX_CHECKPOINT_DIR, BestRun
from transformers.utils import logging from .utils import logging
logger = logging.get_logger(__name__) logger = logging.get_logger(__name__)

80
src/transformers/modeling_auto.py
View File

@@ -524,10 +524,10 @@ class AutoModel:
Examples:: Examples::
from transformers import AutoConfig, AutoModel >>> from transformers import AutoConfig, AutoModel
# Download configuration from S3 and cache. >>> # Download configuration from S3 and cache.
config = AutoConfig.from_pretrained('bert-base-uncased') >>> config = AutoConfig.from_pretrained('bert-base-uncased')
model = AutoModel.from_config(config) >>> model = AutoModel.from_config(config)
""" """
for config_class, model_class in MODEL_MAPPING.items(): for config_class, model_class in MODEL_MAPPING.items():
if isinstance(config, config_class): if isinstance(config, config_class):
@@ -618,10 +618,10 @@ class AutoModelForPreTraining:
Examples:: Examples::
from transformers import AutoConfig, AutoModelForPreTraining >>> from transformers import AutoConfig, AutoModelForPreTraining
# Download configuration from S3 and cache. >>> # Download configuration from S3 and cache.
config = AutoConfig.from_pretrained('bert-base-uncased') >>> config = AutoConfig.from_pretrained('bert-base-uncased')
model = AutoModelForPreTraining.from_config(config) >>> model = AutoModelForPreTraining.from_config(config)
""" """
for config_class, model_class in MODEL_FOR_PRETRAINING_MAPPING.items(): for config_class, model_class in MODEL_FOR_PRETRAINING_MAPPING.items():
if isinstance(config, config_class): if isinstance(config, config_class):
@@ -718,10 +718,10 @@ class AutoModelWithLMHead:
Examples:: Examples::
from transformers import AutoConfig, AutoModelWithLMHead >>> from transformers import AutoConfig, AutoModelWithLMHead
# Download configuration from S3 and cache. >>> # Download configuration from S3 and cache.
config = AutoConfig.from_pretrained('bert-base-uncased') >>> config = AutoConfig.from_pretrained('bert-base-uncased')
model = AutoModelWithLMHead.from_config(config) >>> model = AutoModelWithLMHead.from_config(config)
""" """
warnings.warn( warnings.warn(
"The class `AutoModelWithLMHead` is deprecated and will be removed in a future version. Please use " "The class `AutoModelWithLMHead` is deprecated and will be removed in a future version. Please use "
@@ -824,10 +824,10 @@ class AutoModelForCausalLM:
Examples:: Examples::
from transformers import AutoConfig, AutoModelForCausalLM >>> from transformers import AutoConfig, AutoModelForCausalLM
# Download configuration from S3 and cache. >>> # Download configuration from S3 and cache.
config = AutoConfig.from_pretrained('gpt2') >>> config = AutoConfig.from_pretrained('gpt2')
model = AutoModelForCausalLM.from_config(config) >>> model = AutoModelForCausalLM.from_config(config)
""" """
for config_class, model_class in MODEL_FOR_CAUSAL_LM_MAPPING.items(): for config_class, model_class in MODEL_FOR_CAUSAL_LM_MAPPING.items():
if isinstance(config, config_class): if isinstance(config, config_class):
@@ -918,10 +918,10 @@ class AutoModelForMaskedLM:
Examples:: Examples::
from transformers import AutoConfig, AutoModelForMaskedLM >>> from transformers import AutoConfig, AutoModelForMaskedLM
# Download configuration from S3 and cache. >>> # Download configuration from S3 and cache.
config = AutoConfig.from_pretrained('bert-base-uncased') >>> config = AutoConfig.from_pretrained('bert-base-uncased')
model = AutoModelForMaskedLM.from_config(config) >>> model = AutoModelForMaskedLM.from_config(config)
""" """
for config_class, model_class in MODEL_FOR_MASKED_LM_MAPPING.items(): for config_class, model_class in MODEL_FOR_MASKED_LM_MAPPING.items():
if isinstance(config, config_class): if isinstance(config, config_class):
@@ -1012,10 +1012,10 @@ class AutoModelForSeq2SeqLM:
Examples:: Examples::
from transformers import AutoConfig, AutoModelForSeq2SeqLM >>> from transformers import AutoConfig, AutoModelForSeq2SeqLM
# Download configuration from S3 and cache. >>> # Download configuration from S3 and cache.
config = AutoConfig.from_pretrained('t5') >>> config = AutoConfig.from_pretrained('t5')
model = AutoModelForSeq2SeqLM.from_config(config) >>> model = AutoModelForSeq2SeqLM.from_config(config)
""" """
for config_class, model_class in MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING.items(): for config_class, model_class in MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING.items():
if isinstance(config, config_class): if isinstance(config, config_class):
@@ -1110,10 +1110,10 @@ class AutoModelForSequenceClassification:
Examples:: Examples::
from transformers import AutoConfig, AutoModelForSequenceClassification >>> from transformers import AutoConfig, AutoModelForSequenceClassification
# Download configuration from S3 and cache. >>> # Download configuration from S3 and cache.
config = AutoConfig.from_pretrained('bert-base-uncased') >>> config = AutoConfig.from_pretrained('bert-base-uncased')
model = AutoModelForSequenceClassification.from_config(config) >>> model = AutoModelForSequenceClassification.from_config(config)
""" """
for config_class, model_class in MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING.items(): for config_class, model_class in MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING.items():
if isinstance(config, config_class): if isinstance(config, config_class):
@@ -1207,10 +1207,10 @@ class AutoModelForQuestionAnswering:
Examples:: Examples::
from transformers import AutoConfig, AutoModelForQuestionAnswering >>> from transformers import AutoConfig, AutoModelForQuestionAnswering
# Download configuration from S3 and cache. >>> # Download configuration from S3 and cache.
config = AutoConfig.from_pretrained('bert-base-uncased') >>> config = AutoConfig.from_pretrained('bert-base-uncased')
model = AutoModelForQuestionAnswering.from_config(config) >>> model = AutoModelForQuestionAnswering.from_config(config)
""" """
for config_class, model_class in MODEL_FOR_QUESTION_ANSWERING_MAPPING.items(): for config_class, model_class in MODEL_FOR_QUESTION_ANSWERING_MAPPING.items():
if isinstance(config, config_class): if isinstance(config, config_class):
@@ -1306,10 +1306,10 @@ class AutoModelForTokenClassification:
Examples:: Examples::
from transformers import AutoConfig, AutoModelForTokenClassification >>> from transformers import AutoConfig, AutoModelForTokenClassification
# Download configuration from S3 and cache. >>> # Download configuration from S3 and cache.
config = AutoConfig.from_pretrained('bert-base-uncased') >>> config = AutoConfig.from_pretrained('bert-base-uncased')
model = AutoModelForTokenClassification.from_config(config) >>> model = AutoModelForTokenClassification.from_config(config)
""" """
for config_class, model_class in MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING.items(): for config_class, model_class in MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING.items():
if isinstance(config, config_class): if isinstance(config, config_class):
@@ -1406,10 +1406,10 @@ class AutoModelForMultipleChoice:
Examples:: Examples::
from transformers import AutoConfig, AutoModelForMultipleChoice >>> from transformers import AutoConfig, AutoModelForMultipleChoice
# Download configuration from S3 and cache. >>> # Download configuration from S3 and cache.
config = AutoConfig.from_pretrained('bert-base-uncased') >>> config = AutoConfig.from_pretrained('bert-base-uncased')
model = AutoModelForMultipleChoice.from_config(config) >>> model = AutoModelForMultipleChoice.from_config(config)
""" """
for config_class, model_class in MODEL_FOR_MULTIPLE_CHOICE_MAPPING.items(): for config_class, model_class in MODEL_FOR_MULTIPLE_CHOICE_MAPPING.items():
if isinstance(config, config_class): if isinstance(config, config_class):

42
src/transformers/modeling_bart.py
View File

@@ -76,18 +76,18 @@ BART_START_DOCSTRING = r"""
BART_GENERATION_EXAMPLE = r""" BART_GENERATION_EXAMPLE = r"""
Summarization example:: Summarization example::
from transformers import BartTokenizer, BartForConditionalGeneration, BartConfig >>> from transformers import BartTokenizer, BartForConditionalGeneration, BartConfig
# see ``examples/summarization/bart/run_eval.py`` for a longer example >>> # see ``examples/summarization/bart/run_eval.py`` for a longer example
model = BartForConditionalGeneration.from_pretrained('facebook/bart-large-cnn') >>> model = BartForConditionalGeneration.from_pretrained('facebook/bart-large-cnn')
tokenizer = BartTokenizer.from_pretrained('facebook/bart-large-cnn') >>> tokenizer = BartTokenizer.from_pretrained('facebook/bart-large-cnn')
ARTICLE_TO_SUMMARIZE = "My friends are cool but they eat too many carbs." >>> ARTICLE_TO_SUMMARIZE = "My friends are cool but they eat too many carbs."
inputs = tokenizer([ARTICLE_TO_SUMMARIZE], max_length=1024, return_tensors='pt') >>> inputs = tokenizer([ARTICLE_TO_SUMMARIZE], max_length=1024, return_tensors='pt')
# Generate Summary >>> # Generate Summary
summary_ids = model.generate(inputs['input_ids'], num_beams=4, max_length=5, early_stopping=True) >>> summary_ids = model.generate(inputs['input_ids'], num_beams=4, max_length=5, early_stopping=True)
print([tokenizer.decode(g, skip_special_tokens=True, clean_up_tokenization_spaces=False) for g in summary_ids]) >>> print([tokenizer.decode(g, skip_special_tokens=True, clean_up_tokenization_spaces=False) for g in summary_ids])
""" """
@@ -1023,21 +1023,21 @@ class BartForConditionalGeneration(PretrainedBartModel):
Conditional generation example:: Conditional generation example::
# Mask filling only works for bart-large >>> # Mask filling only works for bart-large
from transformers import BartTokenizer, BartForConditionalGeneration >>> from transformers import BartTokenizer, BartForConditionalGeneration
tokenizer = BartTokenizer.from_pretrained('facebook/bart-large') >>> tokenizer = BartTokenizer.from_pretrained('facebook/bart-large')
TXT = "My friends are <mask> but they eat too many carbs." >>> TXT = "My friends are <mask> but they eat too many carbs."
model = BartForConditionalGeneration.from_pretrained('facebook/bart-large') >>> model = BartForConditionalGeneration.from_pretrained('facebook/bart-large')
input_ids = tokenizer([TXT], return_tensors='pt')['input_ids'] >>> input_ids = tokenizer([TXT], return_tensors='pt')['input_ids']
logits = model(input_ids).logits >>> logits = model(input_ids).logits
masked_index = (input_ids[0] == tokenizer.mask_token_id).nonzero().item() >>> masked_index = (input_ids[0] == tokenizer.mask_token_id).nonzero().item()
probs = logits[0, masked_index].softmax(dim=0) >>> probs = logits[0, masked_index].softmax(dim=0)
values, predictions = probs.topk(5) >>> values, predictions = probs.topk(5)
tokenizer.decode(predictions).split() >>> tokenizer.decode(predictions).split()
# ['good', 'great', 'all', 'really', 'very'] >>> # ['good', 'great', 'all', 'really', 'very']
""" """
if "lm_labels" in unused: if "lm_labels" in unused:
warnings.warn( warnings.warn(

46
src/transformers/modeling_dpr.py
View File

@@ -425,11 +425,11 @@ class DPRContextEncoder(DPRPretrainedContextEncoder):
Examples:: Examples::
from transformers import DPRContextEncoder, DPRContextEncoderTokenizer >>> from transformers import DPRContextEncoder, DPRContextEncoderTokenizer
tokenizer = DPRContextEncoderTokenizer.from_pretrained('facebook/dpr-ctx_encoder-single-nq-base') >>> tokenizer = DPRContextEncoderTokenizer.from_pretrained('facebook/dpr-ctx_encoder-single-nq-base')
model = DPRContextEncoder.from_pretrained('facebook/dpr-ctx_encoder-single-nq-base', return_dict=True) >>> model = DPRContextEncoder.from_pretrained('facebook/dpr-ctx_encoder-single-nq-base', return_dict=True)
input_ids = tokenizer("Hello, is my dog cute ?", return_tensors='pt')["input_ids"] >>> input_ids = tokenizer("Hello, is my dog cute ?", return_tensors='pt')["input_ids"]
embeddings = model(input_ids).pooler_output >>> embeddings = model(input_ids).pooler_output
""" """
output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
@@ -503,11 +503,11 @@ class DPRQuestionEncoder(DPRPretrainedQuestionEncoder):
Examples:: Examples::
from transformers import DPRQuestionEncoder, DPRQuestionEncoderTokenizer >>> from transformers import DPRQuestionEncoder, DPRQuestionEncoderTokenizer
tokenizer = DPRQuestionEncoderTokenizer.from_pretrained('facebook/dpr-question_encoder-single-nq-base') >>> tokenizer = DPRQuestionEncoderTokenizer.from_pretrained('facebook/dpr-question_encoder-single-nq-base')
model = DPRQuestionEncoder.from_pretrained('facebook/dpr-question_encoder-single-nq-base', return_dict=True) >>> model = DPRQuestionEncoder.from_pretrained('facebook/dpr-question_encoder-single-nq-base', return_dict=True)
input_ids = tokenizer("Hello, is my dog cute ?", return_tensors='pt')["input_ids"] >>> input_ids = tokenizer("Hello, is my dog cute ?", return_tensors='pt')["input_ids"]
embeddings = model(input_ids).pooler_output >>> embeddings = model(input_ids).pooler_output
""" """
output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
output_hidden_states = ( output_hidden_states = (
@@ -579,19 +579,19 @@ class DPRReader(DPRPretrainedReader):
Examples:: Examples::
from transformers import DPRReader, DPRReaderTokenizer >>> from transformers import DPRReader, DPRReaderTokenizer
tokenizer = DPRReaderTokenizer.from_pretrained('facebook/dpr-reader-single-nq-base') >>> tokenizer = DPRReaderTokenizer.from_pretrained('facebook/dpr-reader-single-nq-base')
model = DPRReader.from_pretrained('facebook/dpr-reader-single-nq-base', return_dict=True) >>> model = DPRReader.from_pretrained('facebook/dpr-reader-single-nq-base', return_dict=True)
encoded_inputs = tokenizer( >>> encoded_inputs = tokenizer(
questions=["What is love ?"], ... questions=["What is love ?"],
titles=["Haddaway"], ... titles=["Haddaway"],
texts=["'What Is Love' is a song recorded by the artist Haddaway"], ... texts=["'What Is Love' is a song recorded by the artist Haddaway"],
return_tensors='pt' ... return_tensors='pt'
) ... )
outputs = model(**encoded_inputs) >>> outputs = model(**encoded_inputs)
start_logits = outputs.stat_logits >>> start_logits = outputs.stat_logits
end_logits = outputs.end_logits >>> end_logits = outputs.end_logits
relevance_logits = outputs.relevance_logits >>> relevance_logits = outputs.relevance_logits
""" """
output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions

24
src/transformers/modeling_openai.py
View File

@@ -655,21 +655,21 @@ class OpenAIGPTDoubleHeadsModel(OpenAIGPTPreTrainedModel):
Examples:: Examples::
from transformers import OpenAIGPTTokenizer, OpenAIGPTDoubleHeadsModel >>> from transformers import OpenAIGPTTokenizer, OpenAIGPTDoubleHeadsModel
import torch >>> import torch
tokenizer = OpenAIGPTTokenizer.from_pretrained('openai-gpt') >>> tokenizer = OpenAIGPTTokenizer.from_pretrained('openai-gpt')
model = OpenAIGPTDoubleHeadsModel.from_pretrained('openai-gpt', return_dict=True) >>> model = OpenAIGPTDoubleHeadsModel.from_pretrained('openai-gpt', return_dict=True)
tokenizer.add_special_tokens({'cls_token': '[CLS]'}) # Add a [CLS] to the vocabulary (we should train it also!) >>> tokenizer.add_special_tokens({'cls_token': '[CLS]'}) # Add a [CLS] to the vocabulary (we should train it also!)
model.resize_token_embeddings(len(tokenizer)) >>> model.resize_token_embeddings(len(tokenizer))
choices = ["Hello, my dog is cute [CLS]", "Hello, my cat is cute [CLS]"] >>> choices = ["Hello, my dog is cute [CLS]", "Hello, my cat is cute [CLS]"]
input_ids = torch.tensor([tokenizer.encode(s) for s in choices]).unsqueeze(0) # Batch size 1, 2 choices >>> input_ids = torch.tensor([tokenizer.encode(s) for s in choices]).unsqueeze(0) # Batch size 1, 2 choices
mc_token_ids = torch.tensor([input_ids.size(-1)-1, input_ids.size(-1)-1]).unsqueeze(0) # Batch size 1 >>> mc_token_ids = torch.tensor([input_ids.size(-1)-1, input_ids.size(-1)-1]).unsqueeze(0) # Batch size 1
outputs = model(input_ids, mc_token_ids=mc_token_ids) >>> outputs = model(input_ids, mc_token_ids=mc_token_ids)
lm_logits = outputs.lm_logits >>> lm_logits = outputs.lm_logits
mc_logits = outputs.mc_logits >>> mc_logits = outputs.mc_logits
""" """
return_dict = return_dict if return_dict is not None else self.config.use_return_dict return_dict = return_dict if return_dict is not None else self.config.use_return_dict
if "lm_labels" in kwargs: if "lm_labels" in kwargs:

14
src/transformers/modeling_tf_albert.py
View File

@@ -827,13 +827,13 @@ class TFAlbertForPreTraining(TFAlbertPreTrainedModel):
Return: Return:
Examples:: Examples::
import tensorflow as tf >>> import tensorflow as tf
from transformers import AlbertTokenizer, TFAlbertForPreTraining >>> from transformers import AlbertTokenizer, TFAlbertForPreTraining
tokenizer = AlbertTokenizer.from_pretrained('albert-base-v2') >>> tokenizer = AlbertTokenizer.from_pretrained('albert-base-v2')
model = TFAlbertForPreTraining.from_pretrained('albert-base-v2') >>> model = TFAlbertForPreTraining.from_pretrained('albert-base-v2')
input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 >>> input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1
outputs = model(input_ids) >>> outputs = model(input_ids)
prediction_scores, sop_scores = outputs[:2] >>> prediction_scores, sop_scores = outputs[:2]
""" """
return_dict = kwargs.get("return_dict") return_dict = kwargs.get("return_dict")
return_dict = return_dict if return_dict is not None else self.albert.return_dict return_dict = return_dict if return_dict is not None else self.albert.return_dict

80
src/transformers/modeling_tf_auto.py
View File

@@ -448,10 +448,10 @@ class TFAutoModel(object):
Examples:: Examples::
from transformers import AutoConfig, TFAutoModel >>> from transformers import AutoConfig, TFAutoModel
# Download configuration from S3 and cache. >>> # Download configuration from S3 and cache.
config = TFAutoConfig.from_pretrained('bert-base-uncased') >>> config = TFAutoConfig.from_pretrained('bert-base-uncased')
model = TFAutoModel.from_config(config) >>> model = TFAutoModel.from_config(config)
""" """
for config_class, model_class in TF_MODEL_MAPPING.items(): for config_class, model_class in TF_MODEL_MAPPING.items():
if isinstance(config, config_class): if isinstance(config, config_class):
@@ -542,10 +542,10 @@ class TFAutoModelForPreTraining(object):
Examples:: Examples::
from transformers import AutoConfig, TFAutoModelForPreTraining >>> from transformers import AutoConfig, TFAutoModelForPreTraining
# Download configuration from S3 and cache. >>> # Download configuration from S3 and cache.
config = AutoConfig.from_pretrained('bert-base-uncased') >>> config = AutoConfig.from_pretrained('bert-base-uncased')
model = TFAutoModelForPreTraining.from_config(config) >>> model = TFAutoModelForPreTraining.from_config(config)
""" """
for config_class, model_class in TF_MODEL_FOR_PRETRAINING_MAPPING.items(): for config_class, model_class in TF_MODEL_FOR_PRETRAINING_MAPPING.items():
if isinstance(config, config_class): if isinstance(config, config_class):
@@ -642,10 +642,10 @@ class TFAutoModelWithLMHead(object):
Examples:: Examples::
from transformers import AutoConfig, TFAutoModelWithLMHead >>> from transformers import AutoConfig, TFAutoModelWithLMHead
# Download configuration from S3 and cache. >>> # Download configuration from S3 and cache.
config = AutoConfig.from_pretrained('bert-base-uncased') >>> config = AutoConfig.from_pretrained('bert-base-uncased')
model = TFAutoModelWithLMHead.from_config(config) >>> model = TFAutoModelWithLMHead.from_config(config)
""" """
warnings.warn( warnings.warn(
"The class `TFAutoModelWithLMHead` is deprecated and will be removed in a future version. Please use " "The class `TFAutoModelWithLMHead` is deprecated and will be removed in a future version. Please use "
@@ -750,10 +750,10 @@ class TFAutoModelForCausalLM:
Examples:: Examples::
from transformers import AutoConfig, TFAutoModelForCausalLM >>> from transformers import AutoConfig, TFAutoModelForCausalLM
# Download configuration from S3 and cache. >>> # Download configuration from S3 and cache.
config = AutoConfig.from_pretrained('gpt2') >>> config = AutoConfig.from_pretrained('gpt2')
model = TFAutoModelForCausalLM.from_config(config) >>> model = TFAutoModelForCausalLM.from_config(config)
""" """
for config_class, model_class in TF_MODEL_FOR_CAUSAL_LM_MAPPING.items(): for config_class, model_class in TF_MODEL_FOR_CAUSAL_LM_MAPPING.items():
if isinstance(config, config_class): if isinstance(config, config_class):
@@ -844,10 +844,10 @@ class TFAutoModelForMaskedLM:
Examples:: Examples::
from transformers import AutoConfig, TFAutoModelForMaskedLM >>> from transformers import AutoConfig, TFAutoModelForMaskedLM
# Download configuration from S3 and cache. >>> # Download configuration from S3 and cache.
config = AutoConfig.from_pretrained('bert-base-uncased') >>> config = AutoConfig.from_pretrained('bert-base-uncased')
model = TFAutoModelForMaskedLM.from_config(config) >>> model = TFAutoModelForMaskedLM.from_config(config)
""" """
for config_class, model_class in TF_MODEL_FOR_MASKED_LM_MAPPING.items(): for config_class, model_class in TF_MODEL_FOR_MASKED_LM_MAPPING.items():
if isinstance(config, config_class): if isinstance(config, config_class):
@@ -938,10 +938,10 @@ class TFAutoModelForSeq2SeqLM:
Examples:: Examples::
from transformers import AutoConfig, TFAutoModelForSeq2SeqLM >>> from transformers import AutoConfig, TFAutoModelForSeq2SeqLM
# Download configuration from S3 and cache. >>> # Download configuration from S3 and cache.
config = AutoConfig.from_pretrained('t5') >>> config = AutoConfig.from_pretrained('t5')
model = TFAutoModelForSeq2SeqLM.from_config(config) >>> model = TFAutoModelForSeq2SeqLM.from_config(config)
""" """
for config_class, model_class in TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING.items(): for config_class, model_class in TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING.items():
if isinstance(config, config_class): if isinstance(config, config_class):
@@ -1036,10 +1036,10 @@ class TFAutoModelForSequenceClassification(object):
Examples:: Examples::
from transformers import AutoConfig, TFAutoModelForSequenceClassification >>> from transformers import AutoConfig, TFAutoModelForSequenceClassification
# Download configuration from S3 and cache. >>> # Download configuration from S3 and cache.
config = AutoConfig.from_pretrained('bert-base-uncased') >>> config = AutoConfig.from_pretrained('bert-base-uncased')
model = TFAutoModelForSequenceClassification.from_config(config) >>> model = TFAutoModelForSequenceClassification.from_config(config)
""" """
for config_class, model_class in TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING.items(): for config_class, model_class in TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING.items():
if isinstance(config, config_class): if isinstance(config, config_class):
@@ -1133,10 +1133,10 @@ class TFAutoModelForQuestionAnswering(object):
Examples:: Examples::
from transformers import AutoConfig, TFAutoModelForQuestionAnswering >>> from transformers import AutoConfig, TFAutoModelForQuestionAnswering
# Download configuration from S3 and cache. >>> # Download configuration from S3 and cache.
config = AutoConfig.from_pretrained('bert-base-uncased') >>> config = AutoConfig.from_pretrained('bert-base-uncased')
model = TFAutoModelForQuestionAnswering.from_config(config) >>> model = TFAutoModelForQuestionAnswering.from_config(config)
""" """
for config_class, model_class in TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING.items(): for config_class, model_class in TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING.items():
if isinstance(config, config_class): if isinstance(config, config_class):
@@ -1230,10 +1230,10 @@ class TFAutoModelForTokenClassification:
Examples:: Examples::
from transformers import AutoConfig, TFAutoModelForTokenClassification >>> from transformers import AutoConfig, TFAutoModelForTokenClassification
# Download configuration from S3 and cache. >>> # Download configuration from S3 and cache.
config = AutoConfig.from_pretrained('bert-base-uncased') >>> config = AutoConfig.from_pretrained('bert-base-uncased')
model = TFAutoModelForTokenClassification.from_config(config) >>> model = TFAutoModelForTokenClassification.from_config(config)
""" """
for config_class, model_class in TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING.items(): for config_class, model_class in TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING.items():
if isinstance(config, config_class): if isinstance(config, config_class):
@@ -1328,10 +1328,10 @@ class TFAutoModelForMultipleChoice:
Examples:: Examples::
from transformers import AutoConfig, TFAutoModelForMultipleChoice >>> from transformers import AutoConfig, TFAutoModelForMultipleChoice
# Download configuration from S3 and cache. >>> # Download configuration from S3 and cache.
config = AutoConfig.from_pretrained('bert-base-uncased') >>> config = AutoConfig.from_pretrained('bert-base-uncased')
model = TFAutoModelForMultipleChoice.from_config(config) >>> model = TFAutoModelForMultipleChoice.from_config(config)
""" """
for config_class, model_class in TF_MODEL_FOR_MULTIPLE_CHOICE_MAPPING.items(): for config_class, model_class in TF_MODEL_FOR_MULTIPLE_CHOICE_MAPPING.items():
if isinstance(config, config_class): if isinstance(config, config_class):

32
src/transformers/modeling_tf_bert.py
View File

@@ -792,14 +792,14 @@ class TFBertForPreTraining(TFBertPreTrainedModel):
Examples:: Examples::
import tensorflow as tf >>> import tensorflow as tf
from transformers import BertTokenizer, TFBertForPreTraining >>> from transformers import BertTokenizer, TFBertForPreTraining
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') >>> tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
model = TFBertForPreTraining.from_pretrained('bert-base-uncased') >>> model = TFBertForPreTraining.from_pretrained('bert-base-uncased')
input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 >>> input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1
outputs = model(input_ids) >>> outputs = model(input_ids)
prediction_scores, seq_relationship_scores = outputs[:2] >>> prediction_scores, seq_relationship_scores = outputs[:2]
""" """
return_dict = kwargs.get("return_dict") return_dict = kwargs.get("return_dict")
@@ -1004,18 +1004,18 @@ class TFBertForNextSentencePrediction(TFBertPreTrainedModel):
Examples:: Examples::
import tensorflow as tf >>> import tensorflow as tf
from transformers import BertTokenizer, TFBertForNextSentencePrediction >>> from transformers import BertTokenizer, TFBertForNextSentencePrediction
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') >>> tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
model = TFBertForNextSentencePrediction.from_pretrained('bert-base-uncased') >>> model = TFBertForNextSentencePrediction.from_pretrained('bert-base-uncased')
prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced." >>> prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced."
next_sentence = "The sky is blue due to the shorter wavelength of blue light." >>> next_sentence = "The sky is blue due to the shorter wavelength of blue light."
encoding = tokenizer(prompt, next_sentence, return_tensors='tf') >>> encoding = tokenizer(prompt, next_sentence, return_tensors='tf')
logits = model(encoding['input_ids'], token_type_ids=encoding['token_type_ids'])[0] >>> logits = model(encoding['input_ids'], token_type_ids=encoding['token_type_ids'])[0]
assert logits[0][0] < logits[0][1] # the next sentence was random >>> assert logits[0][0] < logits[0][1] # the next sentence was random
""" """
return_dict = kwargs.get("return_dict") return_dict = kwargs.get("return_dict")
return_dict = return_dict if return_dict is not None else self.bert.return_dict return_dict = return_dict if return_dict is not None else self.bert.return_dict

17
src/transformers/modeling_tf_electra.py
View File

@@ -3,9 +3,8 @@ from typing import Optional, Tuple
import tensorflow as tf import tensorflow as tf
from transformers import ElectraConfig
from .activations_tf import get_tf_activation from .activations_tf import get_tf_activation
from .configuration_electra import ElectraConfig
from .file_utils import ( from .file_utils import (
MULTIPLE_CHOICE_DUMMY_INPUTS, MULTIPLE_CHOICE_DUMMY_INPUTS,
ModelOutput, ModelOutput,
@@ -501,14 +500,14 @@ class TFElectraForPreTraining(TFElectraPreTrainedModel):
Examples:: Examples::
import tensorflow as tf >>> import tensorflow as tf
from transformers import ElectraTokenizer, TFElectraForPreTraining >>> from transformers import ElectraTokenizer, TFElectraForPreTraining
tokenizer = ElectraTokenizer.from_pretrained('google/electra-small-discriminator') >>> tokenizer = ElectraTokenizer.from_pretrained('google/electra-small-discriminator')
model = TFElectraForPreTraining.from_pretrained('google/electra-small-discriminator') >>> model = TFElectraForPreTraining.from_pretrained('google/electra-small-discriminator')
input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 >>> input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1
outputs = model(input_ids) >>> outputs = model(input_ids)
scores = outputs[0] >>> scores = outputs[0]
""" """
return_dict = return_dict if return_dict is not None else self.electra.config.return_dict return_dict = return_dict if return_dict is not None else self.electra.config.return_dict

3
src/transformers/modeling_tf_lxmert.py
View File

@@ -23,8 +23,6 @@ from typing import Dict, Optional, Tuple
import tensorflow as tf import tensorflow as tf
from transformers import BatchEncoding
from .activations_tf import get_tf_activation from .activations_tf import get_tf_activation
from .configuration_lxmert import LxmertConfig from .configuration_lxmert import LxmertConfig
from .file_utils import ( from .file_utils import (
@@ -35,6 +33,7 @@ from .file_utils import (
replace_return_docstrings, replace_return_docstrings,
) )
from .modeling_tf_utils import TFPreTrainedModel, get_initializer, keras_serializable, shape_list from .modeling_tf_utils import TFPreTrainedModel, get_initializer, keras_serializable, shape_list
from .tokenization_utils_base import BatchEncoding
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)

22
src/transformers/modeling_tf_utils.py
View File

@@ -504,17 +504,17 @@ class TFPreTrainedModel(tf.keras.Model, TFModelUtilsMixin, TFGenerationMixin):
Examples:: Examples::
from transformers import BertConfig, TFBertModel >>> from transformers import BertConfig, TFBertModel
# Download model and configuration from S3 and cache. >>> # Download model and configuration from S3 and cache.
model = TFBertModel.from_pretrained('bert-base-uncased') >>> model = TFBertModel.from_pretrained('bert-base-uncased')
# Model was saved using `save_pretrained('./test/saved_model/')` (for example purposes, not runnable). >>> # Model was saved using `save_pretrained('./test/saved_model/')` (for example purposes, not runnable).
model = TFBertModel.from_pretrained('./test/saved_model/') >>> model = TFBertModel.from_pretrained('./test/saved_model/')
# Update configuration during loading. >>> # Update configuration during loading.
model = TFBertModel.from_pretrained('bert-base-uncased', output_attentions=True) >>> model = TFBertModel.from_pretrained('bert-base-uncased', output_attentions=True)
assert model.config.output_attentions == True >>> assert model.config.output_attentions == True
# Loading from a Pytorch model file instead of a TensorFlow checkpoint (slower, for example purposes, not runnable). >>> # Loading from a Pytorch model file instead of a TensorFlow checkpoint (slower, for example purposes, not runnable).
config = BertConfig.from_json_file('./pt_model/my_pt_model_config.json') >>> config = BertConfig.from_json_file('./pt_model/my_pt_model_config.json')
model = TFBertModel.from_pretrained('./pt_model/my_pytorch_model.bin', from_pt=True, config=config) >>> model = TFBertModel.from_pretrained('./pt_model/my_pytorch_model.bin', from_pt=True, config=config)
""" """
config = kwargs.pop("config", None) config = kwargs.pop("config", None)

26
src/transformers/modeling_tf_xlnet.py
View File

@@ -1201,25 +1201,25 @@ class TFXLNetLMHeadModel(TFXLNetPreTrainedModel, TFCausalLanguageModelingLoss):
Examples:: Examples::
import tensorflow as tf >>> import tensorflow as tf
import numpy as np >>> import numpy as np
from transformers import XLNetTokenizer, TFXLNetLMHeadModel >>> from transformers import XLNetTokenizer, TFXLNetLMHeadModel
tokenizer = XLNetTokenizer.from_pretrained('xlnet-large-cased') >>> tokenizer = XLNetTokenizer.from_pretrained('xlnet-large-cased')
model = TFXLNetLMHeadModel.from_pretrained('xlnet-large-cased') >>> model = TFXLNetLMHeadModel.from_pretrained('xlnet-large-cased')
# We show how to setup inputs to predict a next token using a bi-directional context. >>> # We show how to setup inputs to predict a next token using a bi-directional context.
input_ids = tf.constant(tokenizer.encode("Hello, my dog is very <mask>", add_special_tokens=True))[None, :] # We will predict the masked token >>> input_ids = tf.constant(tokenizer.encode("Hello, my dog is very <mask>", add_special_tokens=True))[None, :] # We will predict the masked token
perm_mask = np.zeros((1, input_ids.shape[1], input_ids.shape[1])) >>> perm_mask = np.zeros((1, input_ids.shape[1], input_ids.shape[1]))
perm_mask[:, :, -1] = 1.0 # Previous tokens don't see last token >>> perm_mask[:, :, -1] = 1.0 # Previous tokens don't see last token
target_mapping = np.zeros((1, 1, input_ids.shape[1])) # Shape [1, 1, seq_length] => let's predict one token >>> target_mapping = np.zeros((1, 1, input_ids.shape[1])) # Shape [1, 1, seq_length] => let's predict one token
target_mapping[0, 0, -1] = 1.0 # Our first (and only) prediction will be the last token of the sequence (the masked token) >>> target_mapping[0, 0, -1] = 1.0 # Our first (and only) prediction will be the last token of the sequence (the masked token)
outputs = model(input_ids, perm_mask=tf.constant(perm_mask, dtype=tf.float32), target_mapping=tf.constant(target_mapping, dtype=tf.float32)) >>> outputs = model(input_ids, perm_mask=tf.constant(perm_mask, dtype=tf.float32), target_mapping=tf.constant(target_mapping, dtype=tf.float32))
next_token_logits = outputs[0] # Output has shape [target_mapping.size(0), target_mapping.size(1), config.vocab_size] >>> next_token_logits = outputs[0] # Output has shape [target_mapping.size(0), target_mapping.size(1), config.vocab_size]
""" """
return_dict = return_dict if return_dict is not None else self.transformer.return_dict return_dict = return_dict if return_dict is not None else self.transformer.return_dict

24
src/transformers/modeling_utils.py
View File

@@ -804,17 +804,17 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin, GenerationMixin):
Examples:: Examples::
from transformers import BertConfig, BertModel >>> from transformers import BertConfig, BertModel
# Download model and configuration from S3 and cache. >>> # Download model and configuration from S3 and cache.
model = BertModel.from_pretrained('bert-base-uncased') >>> model = BertModel.from_pretrained('bert-base-uncased')
# Model was saved using `save_pretrained('./test/saved_model/')` (for example purposes, not runnable). >>> # Model was saved using `save_pretrained('./test/saved_model/')` (for example purposes, not runnable).
model = BertModel.from_pretrained('./test/saved_model/') >>> model = BertModel.from_pretrained('./test/saved_model/')
# Update configuration during loading. >>> # Update configuration during loading.
model = BertModel.from_pretrained('bert-base-uncased', output_attentions=True) >>> model = BertModel.from_pretrained('bert-base-uncased', output_attentions=True)
assert model.config.output_attentions == True >>> assert model.config.output_attentions == True
# Loading from a TF checkpoint file instead of a PyTorch model (slower, for example purposes, not runnable). >>> # Loading from a TF checkpoint file instead of a PyTorch model (slower, for example purposes, not runnable).
config = BertConfig.from_json_file('./tf_model/my_tf_model_config.json') >>> config = BertConfig.from_json_file('./tf_model/my_tf_model_config.json')
model = BertModel.from_pretrained('./tf_model/my_tf_checkpoint.ckpt.index', from_tf=True, config=config) >>> model = BertModel.from_pretrained('./tf_model/my_tf_checkpoint.ckpt.index', from_tf=True, config=config)
""" """
config = kwargs.pop("config", None) config = kwargs.pop("config", None)
state_dict = kwargs.pop("state_dict", None) state_dict = kwargs.pop("state_dict", None)
@@ -931,7 +931,7 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin, GenerationMixin):
else: else:
# Load from our TensorFlow 2.0 checkpoints # Load from our TensorFlow 2.0 checkpoints
try: try:
from transformers import load_tf2_checkpoint_in_pytorch_model from .modeling_tf_pytorch_utils import load_tf2_checkpoint_in_pytorch_model
model = load_tf2_checkpoint_in_pytorch_model(model, resolved_archive_file, allow_missing_keys=True) model = load_tf2_checkpoint_in_pytorch_model(model, resolved_archive_file, allow_missing_keys=True)
except ImportError: except ImportError:

46
src/transformers/modeling_xlnet.py
View File

@@ -1360,34 +1360,34 @@ class XLNetLMHeadModel(XLNetPreTrainedModel):
Examples:: Examples::
from transformers import XLNetTokenizer, XLNetLMHeadModel >>> from transformers import XLNetTokenizer, XLNetLMHeadModel
import torch >>> import torch
tokenizer = XLNetTokenizer.from_pretrained('xlnet-large-cased') >>> tokenizer = XLNetTokenizer.from_pretrained('xlnet-large-cased')
model = XLNetLMHeadModel.from_pretrained('xlnet-large-cased', return_dict=True) >>> model = XLNetLMHeadModel.from_pretrained('xlnet-large-cased', return_dict=True)
# We show how to setup inputs to predict a next token using a bi-directional context. >>> # We show how to setup inputs to predict a next token using a bi-directional context.
input_ids = torch.tensor(tokenizer.encode("Hello, my dog is very <mask>", add_special_tokens=False)).unsqueeze(0) # We will predict the masked token >>> input_ids = torch.tensor(tokenizer.encode("Hello, my dog is very <mask>", add_special_tokens=False)).unsqueeze(0) # We will predict the masked token
perm_mask = torch.zeros((1, input_ids.shape[1], input_ids.shape[1]), dtype=torch.float) >>> perm_mask = torch.zeros((1, input_ids.shape[1], input_ids.shape[1]), dtype=torch.float)
perm_mask[:, :, -1] = 1.0 # Previous tokens don't see last token >>> perm_mask[:, :, -1] = 1.0 # Previous tokens don't see last token
target_mapping = torch.zeros((1, 1, input_ids.shape[1]), dtype=torch.float) # Shape [1, 1, seq_length] => let's predict one token >>> target_mapping = torch.zeros((1, 1, input_ids.shape[1]), dtype=torch.float) # Shape [1, 1, seq_length] => let's predict one token
target_mapping[0, 0, -1] = 1.0 # Our first (and only) prediction will be the last token of the sequence (the masked token) >>> target_mapping[0, 0, -1] = 1.0 # Our first (and only) prediction will be the last token of the sequence (the masked token)
outputs = model(input_ids, perm_mask=perm_mask, target_mapping=target_mapping) >>> outputs = model(input_ids, perm_mask=perm_mask, target_mapping=target_mapping)
next_token_logits = outputs[0] # Output has shape [target_mapping.size(0), target_mapping.size(1), config.vocab_size] >>> next_token_logits = outputs[0] # Output has shape [target_mapping.size(0), target_mapping.size(1), config.vocab_size]
# The same way can the XLNetLMHeadModel be used to be trained by standard auto-regressive language modeling. >>> # The same way can the XLNetLMHeadModel be used to be trained by standard auto-regressive language modeling.
input_ids = torch.tensor(tokenizer.encode("Hello, my dog is very <mask>", add_special_tokens=False)).unsqueeze(0) # We will predict the masked token >>> input_ids = torch.tensor(tokenizer.encode("Hello, my dog is very <mask>", add_special_tokens=False)).unsqueeze(0) # We will predict the masked token
labels = torch.tensor(tokenizer.encode("cute", add_special_tokens=False)).unsqueeze(0) >>> labels = torch.tensor(tokenizer.encode("cute", add_special_tokens=False)).unsqueeze(0)
assert labels.shape[0] == 1, 'only one word will be predicted' >>> assert labels.shape[0] == 1, 'only one word will be predicted'
perm_mask = torch.zeros((1, input_ids.shape[1], input_ids.shape[1]), dtype=torch.float) >>> perm_mask = torch.zeros((1, input_ids.shape[1], input_ids.shape[1]), dtype=torch.float)
perm_mask[:, :, -1] = 1.0 # Previous tokens don't see last token as is done in standard auto-regressive lm training >>> perm_mask[:, :, -1] = 1.0 # Previous tokens don't see last token as is done in standard auto-regressive lm training
target_mapping = torch.zeros((1, 1, input_ids.shape[1]), dtype=torch.float) # Shape [1, 1, seq_length] => let's predict one token >>> target_mapping = torch.zeros((1, 1, input_ids.shape[1]), dtype=torch.float) # Shape [1, 1, seq_length] => let's predict one token
target_mapping[0, 0, -1] = 1.0 # Our first (and only) prediction will be the last token of the sequence (the masked token) >>> target_mapping[0, 0, -1] = 1.0 # Our first (and only) prediction will be the last token of the sequence (the masked token)
outputs = model(input_ids, perm_mask=perm_mask, target_mapping=target_mapping, labels=labels) >>> outputs = model(input_ids, perm_mask=perm_mask, target_mapping=target_mapping, labels=labels)
loss = outputs.loss >>> loss = outputs.loss
next_token_logits = outputs.logits # Logits have shape [target_mapping.size(0), target_mapping.size(1), config.vocab_size] >>> next_token_logits = outputs.logits # Logits have shape [target_mapping.size(0), target_mapping.size(1), config.vocab_size]
""" """
return_dict = return_dict if return_dict is not None else self.config.use_return_dict return_dict = return_dict if return_dict is not None else self.config.use_return_dict
use_cache = self.training or (use_cache if use_cache is not None else self.config.use_cache) use_cache = self.training or (use_cache if use_cache is not None else self.config.use_cache)

18
src/transformers/pipelines.py
View File

@@ -2661,18 +2661,18 @@ def pipeline(
Examples:: Examples::
from transformers import pipeline, AutoModelForTokenClassification, AutoTokenizer >>> from transformers import pipeline, AutoModelForTokenClassification, AutoTokenizer
# Sentiment analysis pipeline >>> # Sentiment analysis pipeline
pipeline('sentiment-analysis') >>> pipeline('sentiment-analysis')
# Question answering pipeline, specifying the checkpoint identifier >>> # Question answering pipeline, specifying the checkpoint identifier
pipeline('question-answering', model='distilbert-base-cased-distilled-squad', tokenizer='bert-base-cased') >>> pipeline('question-answering', model='distilbert-base-cased-distilled-squad', tokenizer='bert-base-cased')
# Named entity recognition pipeline, passing in a specific model and tokenizer >>> # Named entity recognition pipeline, passing in a specific model and tokenizer
model = AutoModelForTokenClassification.from_pretrained("dbmdz/bert-large-cased-finetuned-conll03-english") >>> model = AutoModelForTokenClassification.from_pretrained("dbmdz/bert-large-cased-finetuned-conll03-english")
tokenizer = AutoTokenizer.from_pretrained("bert-base-cased") >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
pipeline('ner', model=model, tokenizer=tokenizer) >>> pipeline('ner', model=model, tokenizer=tokenizer)
""" """
# Retrieve the task # Retrieve the task
if task not in SUPPORTED_TASKS: if task not in SUPPORTED_TASKS:

16
src/transformers/testing_utils.py
View File

@@ -297,15 +297,15 @@ class CaptureLogger:
Example: Example:
from transformers import logging >>> from transformers import logging
from transformers.testing_utils import CaptureLogger >>> from transformers.testing_utils import CaptureLogger
msg = "Testing 1, 2, 3" >>> msg = "Testing 1, 2, 3"
logging.set_verbosity_info() >>> logging.set_verbosity_info()
logger = logging.get_logger("transformers.tokenization_bart") >>> logger = logging.get_logger("transformers.tokenization_bart")
with CaptureLogger(logger) as cl: >>> with CaptureLogger(logger) as cl:
logger.info(msg) ... logger.info(msg)
assert cl.out, msg+"\n" >>> assert cl.out, msg+"\n"
""" """
def __init__(self, logger): def __init__(self, logger):

14
src/transformers/tokenization_auto.py
View File

@@ -185,16 +185,16 @@ class AutoTokenizer:
Examples:: Examples::
from transformers import AutoTokenizer >>> from transformers import AutoTokenizer
# Download vocabulary from S3 and cache. >>> # Download vocabulary from S3 and cache.
tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased') >>> tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased')
# Download vocabulary from S3 (user-uploaded) and cache. >>> # Download vocabulary from S3 (user-uploaded) and cache.
tokenizer = AutoTokenizer.from_pretrained('dbmdz/bert-base-german-cased') >>> tokenizer = AutoTokenizer.from_pretrained('dbmdz/bert-base-german-cased')
# If vocabulary files are in a directory (e.g. tokenizer was saved using `save_pretrained('./test/saved_model/')`) >>> # If vocabulary files are in a directory (e.g. tokenizer was saved using `save_pretrained('./test/saved_model/')`)
tokenizer = AutoTokenizer.from_pretrained('./test/bert_saved_model/') >>> tokenizer = AutoTokenizer.from_pretrained('./test/bert_saved_model/')
""" """
config = kwargs.pop("config", None) config = kwargs.pop("config", None)

24
src/transformers/tokenization_dpr.py
View File

@@ -278,18 +278,18 @@ class CustomDPRReaderTokenizerMixin:
Examples:: Examples::
from transformers import DPRReader, DPRReaderTokenizer >>> from transformers import DPRReader, DPRReaderTokenizer
tokenizer = DPRReaderTokenizer.from_pretrained('facebook/dpr-reader-single-nq-base') >>> tokenizer = DPRReaderTokenizer.from_pretrained('facebook/dpr-reader-single-nq-base')
model = DPRReader.from_pretrained('facebook/dpr-reader-single-nq-base') >>> model = DPRReader.from_pretrained('facebook/dpr-reader-single-nq-base')
encoded_inputs = tokenizer( >>> encoded_inputs = tokenizer(
questions=["What is love ?"], ... questions=["What is love ?"],
titles=["Haddaway"], ... titles=["Haddaway"],
texts=["'What Is Love' is a song recorded by the artist Haddaway"], ... texts=["'What Is Love' is a song recorded by the artist Haddaway"],
return_tensors='pt' ... return_tensors='pt'
) ... )
outputs = model(**encoded_inputs) >>> outputs = model(**encoded_inputs)
predicted_spans = tokenizer.decode_best_spans(encoded_inputs, outputs) >>> predicted_spans = tokenizer.decode_best_spans(encoded_inputs, outputs)
print(predicted_spans[0].text) # best span >>> print(predicted_spans[0].text) # best span
""" """
input_ids = reader_input["input_ids"] input_ids = reader_input["input_ids"]

3
src/transformers/tokenization_pegasus.py
View File

@@ -14,9 +14,8 @@
# limitations under the License. # limitations under the License.
from typing import Dict, List, Optional from typing import Dict, List, Optional
from transformers.tokenization_reformer import ReformerTokenizer
from .file_utils import add_start_docstrings from .file_utils import add_start_docstrings
from .tokenization_reformer import ReformerTokenizer
from .tokenization_utils_base import PREPARE_SEQ2SEQ_BATCH_DOCSTRING, BatchEncoding from .tokenization_utils_base import PREPARE_SEQ2SEQ_BATCH_DOCSTRING, BatchEncoding

2
src/transformers/trainer.py
View File

@@ -60,7 +60,7 @@ _use_apex = False
# Check if Pytorch version >= 1.6 to switch between Native AMP and Apex # Check if Pytorch version >= 1.6 to switch between Native AMP and Apex
if version.parse(torch.__version__) < version.parse("1.6"): if version.parse(torch.__version__) < version.parse("1.6"):
from transformers.file_utils import is_apex_available from .file_utils import is_apex_available
if is_apex_available(): if is_apex_available():
from apex import amp from apex import amp