From e643a297228c8cb2c189fe4c93e11125f938d20b Mon Sep 17 00:00:00 2001 From: Sohee Yang Date: Fri, 18 Sep 2020 01:30:45 +0900 Subject: [PATCH] Change to use relative imports in some files & Add python prompt symbols to example codes (#7202) * Move 'from transformers' statements to relative imports in some files * Add python prompt symbols in front of the example codes * Reformat the code * Add one missing space Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> --- src/transformers/integrations.py | 4 +- src/transformers/modeling_auto.py | 80 ++++++++++++------------ src/transformers/modeling_bart.py | 42 ++++++------- src/transformers/modeling_dpr.py | 46 +++++++------- src/transformers/modeling_openai.py | 24 +++---- src/transformers/modeling_tf_albert.py | 14 ++--- src/transformers/modeling_tf_auto.py | 80 ++++++++++++------------ src/transformers/modeling_tf_bert.py | 32 +++++----- src/transformers/modeling_tf_electra.py | 17 +++-- src/transformers/modeling_tf_lxmert.py | 3 +- src/transformers/modeling_tf_utils.py | 22 +++---- src/transformers/modeling_tf_xlnet.py | 26 ++++---- src/transformers/modeling_utils.py | 24 +++---- src/transformers/modeling_xlnet.py | 46 +++++++------- src/transformers/pipelines.py | 18 +++--- src/transformers/testing_utils.py | 16 ++--- src/transformers/tokenization_auto.py | 14 ++--- src/transformers/tokenization_dpr.py | 24 +++---- src/transformers/tokenization_pegasus.py | 3 +- src/transformers/trainer.py | 2 +- 20 files changed, 267 insertions(+), 270 deletions(-) diff --git a/src/transformers/integrations.py b/src/transformers/integrations.py index d1299282a0..8db09950d2 100644 --- a/src/transformers/integrations.py +++ b/src/transformers/integrations.py @@ -3,8 +3,8 @@ import os import numpy as np -from transformers.trainer_utils import PREFIX_CHECKPOINT_DIR, BestRun -from transformers.utils import logging +from .trainer_utils import PREFIX_CHECKPOINT_DIR, BestRun +from .utils import logging logger = logging.get_logger(__name__) diff --git a/src/transformers/modeling_auto.py b/src/transformers/modeling_auto.py index 0ac47a4e9e..e482b170f6 100644 --- a/src/transformers/modeling_auto.py +++ b/src/transformers/modeling_auto.py @@ -524,10 +524,10 @@ class AutoModel: Examples:: - from transformers import AutoConfig, AutoModel - # Download configuration from S3 and cache. - config = AutoConfig.from_pretrained('bert-base-uncased') - model = AutoModel.from_config(config) + >>> from transformers import AutoConfig, AutoModel + >>> # Download configuration from S3 and cache. + >>> config = AutoConfig.from_pretrained('bert-base-uncased') + >>> model = AutoModel.from_config(config) """ for config_class, model_class in MODEL_MAPPING.items(): if isinstance(config, config_class): @@ -618,10 +618,10 @@ class AutoModelForPreTraining: Examples:: - from transformers import AutoConfig, AutoModelForPreTraining - # Download configuration from S3 and cache. - config = AutoConfig.from_pretrained('bert-base-uncased') - model = AutoModelForPreTraining.from_config(config) + >>> from transformers import AutoConfig, AutoModelForPreTraining + >>> # Download configuration from S3 and cache. + >>> config = AutoConfig.from_pretrained('bert-base-uncased') + >>> model = AutoModelForPreTraining.from_config(config) """ for config_class, model_class in MODEL_FOR_PRETRAINING_MAPPING.items(): if isinstance(config, config_class): @@ -718,10 +718,10 @@ class AutoModelWithLMHead: Examples:: - from transformers import AutoConfig, AutoModelWithLMHead - # Download configuration from S3 and cache. - config = AutoConfig.from_pretrained('bert-base-uncased') - model = AutoModelWithLMHead.from_config(config) + >>> from transformers import AutoConfig, AutoModelWithLMHead + >>> # Download configuration from S3 and cache. + >>> config = AutoConfig.from_pretrained('bert-base-uncased') + >>> model = AutoModelWithLMHead.from_config(config) """ warnings.warn( "The class `AutoModelWithLMHead` is deprecated and will be removed in a future version. Please use " @@ -824,10 +824,10 @@ class AutoModelForCausalLM: Examples:: - from transformers import AutoConfig, AutoModelForCausalLM - # Download configuration from S3 and cache. - config = AutoConfig.from_pretrained('gpt2') - model = AutoModelForCausalLM.from_config(config) + >>> from transformers import AutoConfig, AutoModelForCausalLM + >>> # Download configuration from S3 and cache. + >>> config = AutoConfig.from_pretrained('gpt2') + >>> model = AutoModelForCausalLM.from_config(config) """ for config_class, model_class in MODEL_FOR_CAUSAL_LM_MAPPING.items(): if isinstance(config, config_class): @@ -918,10 +918,10 @@ class AutoModelForMaskedLM: Examples:: - from transformers import AutoConfig, AutoModelForMaskedLM - # Download configuration from S3 and cache. - config = AutoConfig.from_pretrained('bert-base-uncased') - model = AutoModelForMaskedLM.from_config(config) + >>> from transformers import AutoConfig, AutoModelForMaskedLM + >>> # Download configuration from S3 and cache. + >>> config = AutoConfig.from_pretrained('bert-base-uncased') + >>> model = AutoModelForMaskedLM.from_config(config) """ for config_class, model_class in MODEL_FOR_MASKED_LM_MAPPING.items(): if isinstance(config, config_class): @@ -1012,10 +1012,10 @@ class AutoModelForSeq2SeqLM: Examples:: - from transformers import AutoConfig, AutoModelForSeq2SeqLM - # Download configuration from S3 and cache. - config = AutoConfig.from_pretrained('t5') - model = AutoModelForSeq2SeqLM.from_config(config) + >>> from transformers import AutoConfig, AutoModelForSeq2SeqLM + >>> # Download configuration from S3 and cache. + >>> config = AutoConfig.from_pretrained('t5') + >>> model = AutoModelForSeq2SeqLM.from_config(config) """ for config_class, model_class in MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING.items(): if isinstance(config, config_class): @@ -1110,10 +1110,10 @@ class AutoModelForSequenceClassification: Examples:: - from transformers import AutoConfig, AutoModelForSequenceClassification - # Download configuration from S3 and cache. - config = AutoConfig.from_pretrained('bert-base-uncased') - model = AutoModelForSequenceClassification.from_config(config) + >>> from transformers import AutoConfig, AutoModelForSequenceClassification + >>> # Download configuration from S3 and cache. + >>> config = AutoConfig.from_pretrained('bert-base-uncased') + >>> model = AutoModelForSequenceClassification.from_config(config) """ for config_class, model_class in MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING.items(): if isinstance(config, config_class): @@ -1207,10 +1207,10 @@ class AutoModelForQuestionAnswering: Examples:: - from transformers import AutoConfig, AutoModelForQuestionAnswering - # Download configuration from S3 and cache. - config = AutoConfig.from_pretrained('bert-base-uncased') - model = AutoModelForQuestionAnswering.from_config(config) + >>> from transformers import AutoConfig, AutoModelForQuestionAnswering + >>> # Download configuration from S3 and cache. + >>> config = AutoConfig.from_pretrained('bert-base-uncased') + >>> model = AutoModelForQuestionAnswering.from_config(config) """ for config_class, model_class in MODEL_FOR_QUESTION_ANSWERING_MAPPING.items(): if isinstance(config, config_class): @@ -1306,10 +1306,10 @@ class AutoModelForTokenClassification: Examples:: - from transformers import AutoConfig, AutoModelForTokenClassification - # Download configuration from S3 and cache. - config = AutoConfig.from_pretrained('bert-base-uncased') - model = AutoModelForTokenClassification.from_config(config) + >>> from transformers import AutoConfig, AutoModelForTokenClassification + >>> # Download configuration from S3 and cache. + >>> config = AutoConfig.from_pretrained('bert-base-uncased') + >>> model = AutoModelForTokenClassification.from_config(config) """ for config_class, model_class in MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING.items(): if isinstance(config, config_class): @@ -1406,10 +1406,10 @@ class AutoModelForMultipleChoice: Examples:: - from transformers import AutoConfig, AutoModelForMultipleChoice - # Download configuration from S3 and cache. - config = AutoConfig.from_pretrained('bert-base-uncased') - model = AutoModelForMultipleChoice.from_config(config) + >>> from transformers import AutoConfig, AutoModelForMultipleChoice + >>> # Download configuration from S3 and cache. + >>> config = AutoConfig.from_pretrained('bert-base-uncased') + >>> model = AutoModelForMultipleChoice.from_config(config) """ for config_class, model_class in MODEL_FOR_MULTIPLE_CHOICE_MAPPING.items(): if isinstance(config, config_class): diff --git a/src/transformers/modeling_bart.py b/src/transformers/modeling_bart.py index 63dfeb361b..e48e430d34 100644 --- a/src/transformers/modeling_bart.py +++ b/src/transformers/modeling_bart.py @@ -76,18 +76,18 @@ BART_START_DOCSTRING = r""" BART_GENERATION_EXAMPLE = r""" Summarization example:: - from transformers import BartTokenizer, BartForConditionalGeneration, BartConfig + >>> from transformers import BartTokenizer, BartForConditionalGeneration, BartConfig - # see ``examples/summarization/bart/run_eval.py`` for a longer example - model = BartForConditionalGeneration.from_pretrained('facebook/bart-large-cnn') - tokenizer = BartTokenizer.from_pretrained('facebook/bart-large-cnn') + >>> # see ``examples/summarization/bart/run_eval.py`` for a longer example + >>> model = BartForConditionalGeneration.from_pretrained('facebook/bart-large-cnn') + >>> tokenizer = BartTokenizer.from_pretrained('facebook/bart-large-cnn') - ARTICLE_TO_SUMMARIZE = "My friends are cool but they eat too many carbs." - inputs = tokenizer([ARTICLE_TO_SUMMARIZE], max_length=1024, return_tensors='pt') + >>> ARTICLE_TO_SUMMARIZE = "My friends are cool but they eat too many carbs." + >>> inputs = tokenizer([ARTICLE_TO_SUMMARIZE], max_length=1024, return_tensors='pt') - # Generate Summary - summary_ids = model.generate(inputs['input_ids'], num_beams=4, max_length=5, early_stopping=True) - print([tokenizer.decode(g, skip_special_tokens=True, clean_up_tokenization_spaces=False) for g in summary_ids]) + >>> # Generate Summary + >>> summary_ids = model.generate(inputs['input_ids'], num_beams=4, max_length=5, early_stopping=True) + >>> print([tokenizer.decode(g, skip_special_tokens=True, clean_up_tokenization_spaces=False) for g in summary_ids]) """ @@ -1023,21 +1023,21 @@ class BartForConditionalGeneration(PretrainedBartModel): Conditional generation example:: - # Mask filling only works for bart-large - from transformers import BartTokenizer, BartForConditionalGeneration - tokenizer = BartTokenizer.from_pretrained('facebook/bart-large') - TXT = "My friends are but they eat too many carbs." + >>> # Mask filling only works for bart-large + >>> from transformers import BartTokenizer, BartForConditionalGeneration + >>> tokenizer = BartTokenizer.from_pretrained('facebook/bart-large') + >>> TXT = "My friends are but they eat too many carbs." - model = BartForConditionalGeneration.from_pretrained('facebook/bart-large') - input_ids = tokenizer([TXT], return_tensors='pt')['input_ids'] - logits = model(input_ids).logits + >>> model = BartForConditionalGeneration.from_pretrained('facebook/bart-large') + >>> input_ids = tokenizer([TXT], return_tensors='pt')['input_ids'] + >>> logits = model(input_ids).logits - masked_index = (input_ids[0] == tokenizer.mask_token_id).nonzero().item() - probs = logits[0, masked_index].softmax(dim=0) - values, predictions = probs.topk(5) + >>> masked_index = (input_ids[0] == tokenizer.mask_token_id).nonzero().item() + >>> probs = logits[0, masked_index].softmax(dim=0) + >>> values, predictions = probs.topk(5) - tokenizer.decode(predictions).split() - # ['good', 'great', 'all', 'really', 'very'] + >>> tokenizer.decode(predictions).split() + >>> # ['good', 'great', 'all', 'really', 'very'] """ if "lm_labels" in unused: warnings.warn( diff --git a/src/transformers/modeling_dpr.py b/src/transformers/modeling_dpr.py index 0aaba24dcf..1ba940bd54 100644 --- a/src/transformers/modeling_dpr.py +++ b/src/transformers/modeling_dpr.py @@ -425,11 +425,11 @@ class DPRContextEncoder(DPRPretrainedContextEncoder): Examples:: - from transformers import DPRContextEncoder, DPRContextEncoderTokenizer - tokenizer = DPRContextEncoderTokenizer.from_pretrained('facebook/dpr-ctx_encoder-single-nq-base') - model = DPRContextEncoder.from_pretrained('facebook/dpr-ctx_encoder-single-nq-base', return_dict=True) - input_ids = tokenizer("Hello, is my dog cute ?", return_tensors='pt')["input_ids"] - embeddings = model(input_ids).pooler_output + >>> from transformers import DPRContextEncoder, DPRContextEncoderTokenizer + >>> tokenizer = DPRContextEncoderTokenizer.from_pretrained('facebook/dpr-ctx_encoder-single-nq-base') + >>> model = DPRContextEncoder.from_pretrained('facebook/dpr-ctx_encoder-single-nq-base', return_dict=True) + >>> input_ids = tokenizer("Hello, is my dog cute ?", return_tensors='pt')["input_ids"] + >>> embeddings = model(input_ids).pooler_output """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions @@ -503,11 +503,11 @@ class DPRQuestionEncoder(DPRPretrainedQuestionEncoder): Examples:: - from transformers import DPRQuestionEncoder, DPRQuestionEncoderTokenizer - tokenizer = DPRQuestionEncoderTokenizer.from_pretrained('facebook/dpr-question_encoder-single-nq-base') - model = DPRQuestionEncoder.from_pretrained('facebook/dpr-question_encoder-single-nq-base', return_dict=True) - input_ids = tokenizer("Hello, is my dog cute ?", return_tensors='pt')["input_ids"] - embeddings = model(input_ids).pooler_output + >>> from transformers import DPRQuestionEncoder, DPRQuestionEncoderTokenizer + >>> tokenizer = DPRQuestionEncoderTokenizer.from_pretrained('facebook/dpr-question_encoder-single-nq-base') + >>> model = DPRQuestionEncoder.from_pretrained('facebook/dpr-question_encoder-single-nq-base', return_dict=True) + >>> input_ids = tokenizer("Hello, is my dog cute ?", return_tensors='pt')["input_ids"] + >>> embeddings = model(input_ids).pooler_output """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( @@ -579,19 +579,19 @@ class DPRReader(DPRPretrainedReader): Examples:: - from transformers import DPRReader, DPRReaderTokenizer - tokenizer = DPRReaderTokenizer.from_pretrained('facebook/dpr-reader-single-nq-base') - model = DPRReader.from_pretrained('facebook/dpr-reader-single-nq-base', return_dict=True) - encoded_inputs = tokenizer( - questions=["What is love ?"], - titles=["Haddaway"], - texts=["'What Is Love' is a song recorded by the artist Haddaway"], - return_tensors='pt' - ) - outputs = model(**encoded_inputs) - start_logits = outputs.stat_logits - end_logits = outputs.end_logits - relevance_logits = outputs.relevance_logits + >>> from transformers import DPRReader, DPRReaderTokenizer + >>> tokenizer = DPRReaderTokenizer.from_pretrained('facebook/dpr-reader-single-nq-base') + >>> model = DPRReader.from_pretrained('facebook/dpr-reader-single-nq-base', return_dict=True) + >>> encoded_inputs = tokenizer( + ... questions=["What is love ?"], + ... titles=["Haddaway"], + ... texts=["'What Is Love' is a song recorded by the artist Haddaway"], + ... return_tensors='pt' + ... ) + >>> outputs = model(**encoded_inputs) + >>> start_logits = outputs.stat_logits + >>> end_logits = outputs.end_logits + >>> relevance_logits = outputs.relevance_logits """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions diff --git a/src/transformers/modeling_openai.py b/src/transformers/modeling_openai.py index 4b0001a1de..f6128f34c0 100644 --- a/src/transformers/modeling_openai.py +++ b/src/transformers/modeling_openai.py @@ -655,21 +655,21 @@ class OpenAIGPTDoubleHeadsModel(OpenAIGPTPreTrainedModel): Examples:: - from transformers import OpenAIGPTTokenizer, OpenAIGPTDoubleHeadsModel - import torch + >>> from transformers import OpenAIGPTTokenizer, OpenAIGPTDoubleHeadsModel + >>> import torch - tokenizer = OpenAIGPTTokenizer.from_pretrained('openai-gpt') - model = OpenAIGPTDoubleHeadsModel.from_pretrained('openai-gpt', return_dict=True) - tokenizer.add_special_tokens({'cls_token': '[CLS]'}) # Add a [CLS] to the vocabulary (we should train it also!) - model.resize_token_embeddings(len(tokenizer)) + >>> tokenizer = OpenAIGPTTokenizer.from_pretrained('openai-gpt') + >>> model = OpenAIGPTDoubleHeadsModel.from_pretrained('openai-gpt', return_dict=True) + >>> tokenizer.add_special_tokens({'cls_token': '[CLS]'}) # Add a [CLS] to the vocabulary (we should train it also!) + >>> model.resize_token_embeddings(len(tokenizer)) - choices = ["Hello, my dog is cute [CLS]", "Hello, my cat is cute [CLS]"] - input_ids = torch.tensor([tokenizer.encode(s) for s in choices]).unsqueeze(0) # Batch size 1, 2 choices - mc_token_ids = torch.tensor([input_ids.size(-1)-1, input_ids.size(-1)-1]).unsqueeze(0) # Batch size 1 + >>> choices = ["Hello, my dog is cute [CLS]", "Hello, my cat is cute [CLS]"] + >>> input_ids = torch.tensor([tokenizer.encode(s) for s in choices]).unsqueeze(0) # Batch size 1, 2 choices + >>> mc_token_ids = torch.tensor([input_ids.size(-1)-1, input_ids.size(-1)-1]).unsqueeze(0) # Batch size 1 - outputs = model(input_ids, mc_token_ids=mc_token_ids) - lm_logits = outputs.lm_logits - mc_logits = outputs.mc_logits + >>> outputs = model(input_ids, mc_token_ids=mc_token_ids) + >>> lm_logits = outputs.lm_logits + >>> mc_logits = outputs.mc_logits """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict if "lm_labels" in kwargs: diff --git a/src/transformers/modeling_tf_albert.py b/src/transformers/modeling_tf_albert.py index e6b9a9c92f..b2d13956b6 100644 --- a/src/transformers/modeling_tf_albert.py +++ b/src/transformers/modeling_tf_albert.py @@ -827,13 +827,13 @@ class TFAlbertForPreTraining(TFAlbertPreTrainedModel): Return: Examples:: - import tensorflow as tf - from transformers import AlbertTokenizer, TFAlbertForPreTraining - tokenizer = AlbertTokenizer.from_pretrained('albert-base-v2') - model = TFAlbertForPreTraining.from_pretrained('albert-base-v2') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 - outputs = model(input_ids) - prediction_scores, sop_scores = outputs[:2] + >>> import tensorflow as tf + >>> from transformers import AlbertTokenizer, TFAlbertForPreTraining + >>> tokenizer = AlbertTokenizer.from_pretrained('albert-base-v2') + >>> model = TFAlbertForPreTraining.from_pretrained('albert-base-v2') + >>> input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 + >>> outputs = model(input_ids) + >>> prediction_scores, sop_scores = outputs[:2] """ return_dict = kwargs.get("return_dict") return_dict = return_dict if return_dict is not None else self.albert.return_dict diff --git a/src/transformers/modeling_tf_auto.py b/src/transformers/modeling_tf_auto.py index e18ed2712c..ba389bafac 100644 --- a/src/transformers/modeling_tf_auto.py +++ b/src/transformers/modeling_tf_auto.py @@ -448,10 +448,10 @@ class TFAutoModel(object): Examples:: - from transformers import AutoConfig, TFAutoModel - # Download configuration from S3 and cache. - config = TFAutoConfig.from_pretrained('bert-base-uncased') - model = TFAutoModel.from_config(config) + >>> from transformers import AutoConfig, TFAutoModel + >>> # Download configuration from S3 and cache. + >>> config = TFAutoConfig.from_pretrained('bert-base-uncased') + >>> model = TFAutoModel.from_config(config) """ for config_class, model_class in TF_MODEL_MAPPING.items(): if isinstance(config, config_class): @@ -542,10 +542,10 @@ class TFAutoModelForPreTraining(object): Examples:: - from transformers import AutoConfig, TFAutoModelForPreTraining - # Download configuration from S3 and cache. - config = AutoConfig.from_pretrained('bert-base-uncased') - model = TFAutoModelForPreTraining.from_config(config) + >>> from transformers import AutoConfig, TFAutoModelForPreTraining + >>> # Download configuration from S3 and cache. + >>> config = AutoConfig.from_pretrained('bert-base-uncased') + >>> model = TFAutoModelForPreTraining.from_config(config) """ for config_class, model_class in TF_MODEL_FOR_PRETRAINING_MAPPING.items(): if isinstance(config, config_class): @@ -642,10 +642,10 @@ class TFAutoModelWithLMHead(object): Examples:: - from transformers import AutoConfig, TFAutoModelWithLMHead - # Download configuration from S3 and cache. - config = AutoConfig.from_pretrained('bert-base-uncased') - model = TFAutoModelWithLMHead.from_config(config) + >>> from transformers import AutoConfig, TFAutoModelWithLMHead + >>> # Download configuration from S3 and cache. + >>> config = AutoConfig.from_pretrained('bert-base-uncased') + >>> model = TFAutoModelWithLMHead.from_config(config) """ warnings.warn( "The class `TFAutoModelWithLMHead` is deprecated and will be removed in a future version. Please use " @@ -750,10 +750,10 @@ class TFAutoModelForCausalLM: Examples:: - from transformers import AutoConfig, TFAutoModelForCausalLM - # Download configuration from S3 and cache. - config = AutoConfig.from_pretrained('gpt2') - model = TFAutoModelForCausalLM.from_config(config) + >>> from transformers import AutoConfig, TFAutoModelForCausalLM + >>> # Download configuration from S3 and cache. + >>> config = AutoConfig.from_pretrained('gpt2') + >>> model = TFAutoModelForCausalLM.from_config(config) """ for config_class, model_class in TF_MODEL_FOR_CAUSAL_LM_MAPPING.items(): if isinstance(config, config_class): @@ -844,10 +844,10 @@ class TFAutoModelForMaskedLM: Examples:: - from transformers import AutoConfig, TFAutoModelForMaskedLM - # Download configuration from S3 and cache. - config = AutoConfig.from_pretrained('bert-base-uncased') - model = TFAutoModelForMaskedLM.from_config(config) + >>> from transformers import AutoConfig, TFAutoModelForMaskedLM + >>> # Download configuration from S3 and cache. + >>> config = AutoConfig.from_pretrained('bert-base-uncased') + >>> model = TFAutoModelForMaskedLM.from_config(config) """ for config_class, model_class in TF_MODEL_FOR_MASKED_LM_MAPPING.items(): if isinstance(config, config_class): @@ -938,10 +938,10 @@ class TFAutoModelForSeq2SeqLM: Examples:: - from transformers import AutoConfig, TFAutoModelForSeq2SeqLM - # Download configuration from S3 and cache. - config = AutoConfig.from_pretrained('t5') - model = TFAutoModelForSeq2SeqLM.from_config(config) + >>> from transformers import AutoConfig, TFAutoModelForSeq2SeqLM + >>> # Download configuration from S3 and cache. + >>> config = AutoConfig.from_pretrained('t5') + >>> model = TFAutoModelForSeq2SeqLM.from_config(config) """ for config_class, model_class in TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING.items(): if isinstance(config, config_class): @@ -1036,10 +1036,10 @@ class TFAutoModelForSequenceClassification(object): Examples:: - from transformers import AutoConfig, TFAutoModelForSequenceClassification - # Download configuration from S3 and cache. - config = AutoConfig.from_pretrained('bert-base-uncased') - model = TFAutoModelForSequenceClassification.from_config(config) + >>> from transformers import AutoConfig, TFAutoModelForSequenceClassification + >>> # Download configuration from S3 and cache. + >>> config = AutoConfig.from_pretrained('bert-base-uncased') + >>> model = TFAutoModelForSequenceClassification.from_config(config) """ for config_class, model_class in TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING.items(): if isinstance(config, config_class): @@ -1133,10 +1133,10 @@ class TFAutoModelForQuestionAnswering(object): Examples:: - from transformers import AutoConfig, TFAutoModelForQuestionAnswering - # Download configuration from S3 and cache. - config = AutoConfig.from_pretrained('bert-base-uncased') - model = TFAutoModelForQuestionAnswering.from_config(config) + >>> from transformers import AutoConfig, TFAutoModelForQuestionAnswering + >>> # Download configuration from S3 and cache. + >>> config = AutoConfig.from_pretrained('bert-base-uncased') + >>> model = TFAutoModelForQuestionAnswering.from_config(config) """ for config_class, model_class in TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING.items(): if isinstance(config, config_class): @@ -1230,10 +1230,10 @@ class TFAutoModelForTokenClassification: Examples:: - from transformers import AutoConfig, TFAutoModelForTokenClassification - # Download configuration from S3 and cache. - config = AutoConfig.from_pretrained('bert-base-uncased') - model = TFAutoModelForTokenClassification.from_config(config) + >>> from transformers import AutoConfig, TFAutoModelForTokenClassification + >>> # Download configuration from S3 and cache. + >>> config = AutoConfig.from_pretrained('bert-base-uncased') + >>> model = TFAutoModelForTokenClassification.from_config(config) """ for config_class, model_class in TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING.items(): if isinstance(config, config_class): @@ -1328,10 +1328,10 @@ class TFAutoModelForMultipleChoice: Examples:: - from transformers import AutoConfig, TFAutoModelForMultipleChoice - # Download configuration from S3 and cache. - config = AutoConfig.from_pretrained('bert-base-uncased') - model = TFAutoModelForMultipleChoice.from_config(config) + >>> from transformers import AutoConfig, TFAutoModelForMultipleChoice + >>> # Download configuration from S3 and cache. + >>> config = AutoConfig.from_pretrained('bert-base-uncased') + >>> model = TFAutoModelForMultipleChoice.from_config(config) """ for config_class, model_class in TF_MODEL_FOR_MULTIPLE_CHOICE_MAPPING.items(): if isinstance(config, config_class): diff --git a/src/transformers/modeling_tf_bert.py b/src/transformers/modeling_tf_bert.py index 9cf6fbb7c4..be1e25f546 100644 --- a/src/transformers/modeling_tf_bert.py +++ b/src/transformers/modeling_tf_bert.py @@ -792,14 +792,14 @@ class TFBertForPreTraining(TFBertPreTrainedModel): Examples:: - import tensorflow as tf - from transformers import BertTokenizer, TFBertForPreTraining + >>> import tensorflow as tf + >>> from transformers import BertTokenizer, TFBertForPreTraining - tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') - model = TFBertForPreTraining.from_pretrained('bert-base-uncased') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 - outputs = model(input_ids) - prediction_scores, seq_relationship_scores = outputs[:2] + >>> tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + >>> model = TFBertForPreTraining.from_pretrained('bert-base-uncased') + >>> input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 + >>> outputs = model(input_ids) + >>> prediction_scores, seq_relationship_scores = outputs[:2] """ return_dict = kwargs.get("return_dict") @@ -1004,18 +1004,18 @@ class TFBertForNextSentencePrediction(TFBertPreTrainedModel): Examples:: - import tensorflow as tf - from transformers import BertTokenizer, TFBertForNextSentencePrediction + >>> import tensorflow as tf + >>> from transformers import BertTokenizer, TFBertForNextSentencePrediction - tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') - model = TFBertForNextSentencePrediction.from_pretrained('bert-base-uncased') + >>> tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + >>> model = TFBertForNextSentencePrediction.from_pretrained('bert-base-uncased') - prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced." - next_sentence = "The sky is blue due to the shorter wavelength of blue light." - encoding = tokenizer(prompt, next_sentence, return_tensors='tf') + >>> prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced." + >>> next_sentence = "The sky is blue due to the shorter wavelength of blue light." + >>> encoding = tokenizer(prompt, next_sentence, return_tensors='tf') - logits = model(encoding['input_ids'], token_type_ids=encoding['token_type_ids'])[0] - assert logits[0][0] < logits[0][1] # the next sentence was random + >>> logits = model(encoding['input_ids'], token_type_ids=encoding['token_type_ids'])[0] + >>> assert logits[0][0] < logits[0][1] # the next sentence was random """ return_dict = kwargs.get("return_dict") return_dict = return_dict if return_dict is not None else self.bert.return_dict diff --git a/src/transformers/modeling_tf_electra.py b/src/transformers/modeling_tf_electra.py index 9128ac13a4..0c4ed4115a 100644 --- a/src/transformers/modeling_tf_electra.py +++ b/src/transformers/modeling_tf_electra.py @@ -3,9 +3,8 @@ from typing import Optional, Tuple import tensorflow as tf -from transformers import ElectraConfig - from .activations_tf import get_tf_activation +from .configuration_electra import ElectraConfig from .file_utils import ( MULTIPLE_CHOICE_DUMMY_INPUTS, ModelOutput, @@ -501,14 +500,14 @@ class TFElectraForPreTraining(TFElectraPreTrainedModel): Examples:: - import tensorflow as tf - from transformers import ElectraTokenizer, TFElectraForPreTraining + >>> import tensorflow as tf + >>> from transformers import ElectraTokenizer, TFElectraForPreTraining - tokenizer = ElectraTokenizer.from_pretrained('google/electra-small-discriminator') - model = TFElectraForPreTraining.from_pretrained('google/electra-small-discriminator') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 - outputs = model(input_ids) - scores = outputs[0] + >>> tokenizer = ElectraTokenizer.from_pretrained('google/electra-small-discriminator') + >>> model = TFElectraForPreTraining.from_pretrained('google/electra-small-discriminator') + >>> input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + >>> outputs = model(input_ids) + >>> scores = outputs[0] """ return_dict = return_dict if return_dict is not None else self.electra.config.return_dict diff --git a/src/transformers/modeling_tf_lxmert.py b/src/transformers/modeling_tf_lxmert.py index ce96da0e08..92d1578567 100644 --- a/src/transformers/modeling_tf_lxmert.py +++ b/src/transformers/modeling_tf_lxmert.py @@ -23,8 +23,6 @@ from typing import Dict, Optional, Tuple import tensorflow as tf -from transformers import BatchEncoding - from .activations_tf import get_tf_activation from .configuration_lxmert import LxmertConfig from .file_utils import ( @@ -35,6 +33,7 @@ from .file_utils import ( replace_return_docstrings, ) from .modeling_tf_utils import TFPreTrainedModel, get_initializer, keras_serializable, shape_list +from .tokenization_utils_base import BatchEncoding logger = logging.getLogger(__name__) diff --git a/src/transformers/modeling_tf_utils.py b/src/transformers/modeling_tf_utils.py index 0ee2875744..afccec7bc5 100644 --- a/src/transformers/modeling_tf_utils.py +++ b/src/transformers/modeling_tf_utils.py @@ -504,17 +504,17 @@ class TFPreTrainedModel(tf.keras.Model, TFModelUtilsMixin, TFGenerationMixin): Examples:: - from transformers import BertConfig, TFBertModel - # Download model and configuration from S3 and cache. - model = TFBertModel.from_pretrained('bert-base-uncased') - # Model was saved using `save_pretrained('./test/saved_model/')` (for example purposes, not runnable). - model = TFBertModel.from_pretrained('./test/saved_model/') - # Update configuration during loading. - model = TFBertModel.from_pretrained('bert-base-uncased', output_attentions=True) - assert model.config.output_attentions == True - # Loading from a Pytorch model file instead of a TensorFlow checkpoint (slower, for example purposes, not runnable). - config = BertConfig.from_json_file('./pt_model/my_pt_model_config.json') - model = TFBertModel.from_pretrained('./pt_model/my_pytorch_model.bin', from_pt=True, config=config) + >>> from transformers import BertConfig, TFBertModel + >>> # Download model and configuration from S3 and cache. + >>> model = TFBertModel.from_pretrained('bert-base-uncased') + >>> # Model was saved using `save_pretrained('./test/saved_model/')` (for example purposes, not runnable). + >>> model = TFBertModel.from_pretrained('./test/saved_model/') + >>> # Update configuration during loading. + >>> model = TFBertModel.from_pretrained('bert-base-uncased', output_attentions=True) + >>> assert model.config.output_attentions == True + >>> # Loading from a Pytorch model file instead of a TensorFlow checkpoint (slower, for example purposes, not runnable). + >>> config = BertConfig.from_json_file('./pt_model/my_pt_model_config.json') + >>> model = TFBertModel.from_pretrained('./pt_model/my_pytorch_model.bin', from_pt=True, config=config) """ config = kwargs.pop("config", None) diff --git a/src/transformers/modeling_tf_xlnet.py b/src/transformers/modeling_tf_xlnet.py index 1bdbee8705..00ece9aaaa 100644 --- a/src/transformers/modeling_tf_xlnet.py +++ b/src/transformers/modeling_tf_xlnet.py @@ -1201,25 +1201,25 @@ class TFXLNetLMHeadModel(TFXLNetPreTrainedModel, TFCausalLanguageModelingLoss): Examples:: - import tensorflow as tf - import numpy as np - from transformers import XLNetTokenizer, TFXLNetLMHeadModel + >>> import tensorflow as tf + >>> import numpy as np + >>> from transformers import XLNetTokenizer, TFXLNetLMHeadModel - tokenizer = XLNetTokenizer.from_pretrained('xlnet-large-cased') - model = TFXLNetLMHeadModel.from_pretrained('xlnet-large-cased') + >>> tokenizer = XLNetTokenizer.from_pretrained('xlnet-large-cased') + >>> model = TFXLNetLMHeadModel.from_pretrained('xlnet-large-cased') - # We show how to setup inputs to predict a next token using a bi-directional context. - input_ids = tf.constant(tokenizer.encode("Hello, my dog is very ", add_special_tokens=True))[None, :] # We will predict the masked token + >>> # We show how to setup inputs to predict a next token using a bi-directional context. + >>> input_ids = tf.constant(tokenizer.encode("Hello, my dog is very ", add_special_tokens=True))[None, :] # We will predict the masked token - perm_mask = np.zeros((1, input_ids.shape[1], input_ids.shape[1])) - perm_mask[:, :, -1] = 1.0 # Previous tokens don't see last token + >>> perm_mask = np.zeros((1, input_ids.shape[1], input_ids.shape[1])) + >>> perm_mask[:, :, -1] = 1.0 # Previous tokens don't see last token - target_mapping = np.zeros((1, 1, input_ids.shape[1])) # Shape [1, 1, seq_length] => let's predict one token - target_mapping[0, 0, -1] = 1.0 # Our first (and only) prediction will be the last token of the sequence (the masked token) + >>> target_mapping = np.zeros((1, 1, input_ids.shape[1])) # Shape [1, 1, seq_length] => let's predict one token + >>> target_mapping[0, 0, -1] = 1.0 # Our first (and only) prediction will be the last token of the sequence (the masked token) - outputs = model(input_ids, perm_mask=tf.constant(perm_mask, dtype=tf.float32), target_mapping=tf.constant(target_mapping, dtype=tf.float32)) + >>> outputs = model(input_ids, perm_mask=tf.constant(perm_mask, dtype=tf.float32), target_mapping=tf.constant(target_mapping, dtype=tf.float32)) - next_token_logits = outputs[0] # Output has shape [target_mapping.size(0), target_mapping.size(1), config.vocab_size] + >>> next_token_logits = outputs[0] # Output has shape [target_mapping.size(0), target_mapping.size(1), config.vocab_size] """ return_dict = return_dict if return_dict is not None else self.transformer.return_dict diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index e67c2f7bd1..899884875b 100755 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -804,17 +804,17 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin, GenerationMixin): Examples:: - from transformers import BertConfig, BertModel - # Download model and configuration from S3 and cache. - model = BertModel.from_pretrained('bert-base-uncased') - # Model was saved using `save_pretrained('./test/saved_model/')` (for example purposes, not runnable). - model = BertModel.from_pretrained('./test/saved_model/') - # Update configuration during loading. - model = BertModel.from_pretrained('bert-base-uncased', output_attentions=True) - assert model.config.output_attentions == True - # Loading from a TF checkpoint file instead of a PyTorch model (slower, for example purposes, not runnable). - config = BertConfig.from_json_file('./tf_model/my_tf_model_config.json') - model = BertModel.from_pretrained('./tf_model/my_tf_checkpoint.ckpt.index', from_tf=True, config=config) + >>> from transformers import BertConfig, BertModel + >>> # Download model and configuration from S3 and cache. + >>> model = BertModel.from_pretrained('bert-base-uncased') + >>> # Model was saved using `save_pretrained('./test/saved_model/')` (for example purposes, not runnable). + >>> model = BertModel.from_pretrained('./test/saved_model/') + >>> # Update configuration during loading. + >>> model = BertModel.from_pretrained('bert-base-uncased', output_attentions=True) + >>> assert model.config.output_attentions == True + >>> # Loading from a TF checkpoint file instead of a PyTorch model (slower, for example purposes, not runnable). + >>> config = BertConfig.from_json_file('./tf_model/my_tf_model_config.json') + >>> model = BertModel.from_pretrained('./tf_model/my_tf_checkpoint.ckpt.index', from_tf=True, config=config) """ config = kwargs.pop("config", None) state_dict = kwargs.pop("state_dict", None) @@ -931,7 +931,7 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin, GenerationMixin): else: # Load from our TensorFlow 2.0 checkpoints try: - from transformers import load_tf2_checkpoint_in_pytorch_model + from .modeling_tf_pytorch_utils import load_tf2_checkpoint_in_pytorch_model model = load_tf2_checkpoint_in_pytorch_model(model, resolved_archive_file, allow_missing_keys=True) except ImportError: diff --git a/src/transformers/modeling_xlnet.py b/src/transformers/modeling_xlnet.py index a0ed8b0c55..903f67923a 100755 --- a/src/transformers/modeling_xlnet.py +++ b/src/transformers/modeling_xlnet.py @@ -1360,34 +1360,34 @@ class XLNetLMHeadModel(XLNetPreTrainedModel): Examples:: - from transformers import XLNetTokenizer, XLNetLMHeadModel - import torch + >>> from transformers import XLNetTokenizer, XLNetLMHeadModel + >>> import torch - tokenizer = XLNetTokenizer.from_pretrained('xlnet-large-cased') - model = XLNetLMHeadModel.from_pretrained('xlnet-large-cased', return_dict=True) + >>> tokenizer = XLNetTokenizer.from_pretrained('xlnet-large-cased') + >>> model = XLNetLMHeadModel.from_pretrained('xlnet-large-cased', return_dict=True) - # We show how to setup inputs to predict a next token using a bi-directional context. - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is very ", add_special_tokens=False)).unsqueeze(0) # We will predict the masked token - perm_mask = torch.zeros((1, input_ids.shape[1], input_ids.shape[1]), dtype=torch.float) - perm_mask[:, :, -1] = 1.0 # Previous tokens don't see last token - target_mapping = torch.zeros((1, 1, input_ids.shape[1]), dtype=torch.float) # Shape [1, 1, seq_length] => let's predict one token - target_mapping[0, 0, -1] = 1.0 # Our first (and only) prediction will be the last token of the sequence (the masked token) + >>> # We show how to setup inputs to predict a next token using a bi-directional context. + >>> input_ids = torch.tensor(tokenizer.encode("Hello, my dog is very ", add_special_tokens=False)).unsqueeze(0) # We will predict the masked token + >>> perm_mask = torch.zeros((1, input_ids.shape[1], input_ids.shape[1]), dtype=torch.float) + >>> perm_mask[:, :, -1] = 1.0 # Previous tokens don't see last token + >>> target_mapping = torch.zeros((1, 1, input_ids.shape[1]), dtype=torch.float) # Shape [1, 1, seq_length] => let's predict one token + >>> target_mapping[0, 0, -1] = 1.0 # Our first (and only) prediction will be the last token of the sequence (the masked token) - outputs = model(input_ids, perm_mask=perm_mask, target_mapping=target_mapping) - next_token_logits = outputs[0] # Output has shape [target_mapping.size(0), target_mapping.size(1), config.vocab_size] + >>> outputs = model(input_ids, perm_mask=perm_mask, target_mapping=target_mapping) + >>> next_token_logits = outputs[0] # Output has shape [target_mapping.size(0), target_mapping.size(1), config.vocab_size] - # The same way can the XLNetLMHeadModel be used to be trained by standard auto-regressive language modeling. - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is very ", add_special_tokens=False)).unsqueeze(0) # We will predict the masked token - labels = torch.tensor(tokenizer.encode("cute", add_special_tokens=False)).unsqueeze(0) - assert labels.shape[0] == 1, 'only one word will be predicted' - perm_mask = torch.zeros((1, input_ids.shape[1], input_ids.shape[1]), dtype=torch.float) - perm_mask[:, :, -1] = 1.0 # Previous tokens don't see last token as is done in standard auto-regressive lm training - target_mapping = torch.zeros((1, 1, input_ids.shape[1]), dtype=torch.float) # Shape [1, 1, seq_length] => let's predict one token - target_mapping[0, 0, -1] = 1.0 # Our first (and only) prediction will be the last token of the sequence (the masked token) + >>> # The same way can the XLNetLMHeadModel be used to be trained by standard auto-regressive language modeling. + >>> input_ids = torch.tensor(tokenizer.encode("Hello, my dog is very ", add_special_tokens=False)).unsqueeze(0) # We will predict the masked token + >>> labels = torch.tensor(tokenizer.encode("cute", add_special_tokens=False)).unsqueeze(0) + >>> assert labels.shape[0] == 1, 'only one word will be predicted' + >>> perm_mask = torch.zeros((1, input_ids.shape[1], input_ids.shape[1]), dtype=torch.float) + >>> perm_mask[:, :, -1] = 1.0 # Previous tokens don't see last token as is done in standard auto-regressive lm training + >>> target_mapping = torch.zeros((1, 1, input_ids.shape[1]), dtype=torch.float) # Shape [1, 1, seq_length] => let's predict one token + >>> target_mapping[0, 0, -1] = 1.0 # Our first (and only) prediction will be the last token of the sequence (the masked token) - outputs = model(input_ids, perm_mask=perm_mask, target_mapping=target_mapping, labels=labels) - loss = outputs.loss - next_token_logits = outputs.logits # Logits have shape [target_mapping.size(0), target_mapping.size(1), config.vocab_size] + >>> outputs = model(input_ids, perm_mask=perm_mask, target_mapping=target_mapping, labels=labels) + >>> loss = outputs.loss + >>> next_token_logits = outputs.logits # Logits have shape [target_mapping.size(0), target_mapping.size(1), config.vocab_size] """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict use_cache = self.training or (use_cache if use_cache is not None else self.config.use_cache) diff --git a/src/transformers/pipelines.py b/src/transformers/pipelines.py index c280d1c026..0fc221d61e 100755 --- a/src/transformers/pipelines.py +++ b/src/transformers/pipelines.py @@ -2661,18 +2661,18 @@ def pipeline( Examples:: - from transformers import pipeline, AutoModelForTokenClassification, AutoTokenizer + >>> from transformers import pipeline, AutoModelForTokenClassification, AutoTokenizer - # Sentiment analysis pipeline - pipeline('sentiment-analysis') + >>> # Sentiment analysis pipeline + >>> pipeline('sentiment-analysis') - # Question answering pipeline, specifying the checkpoint identifier - pipeline('question-answering', model='distilbert-base-cased-distilled-squad', tokenizer='bert-base-cased') + >>> # Question answering pipeline, specifying the checkpoint identifier + >>> pipeline('question-answering', model='distilbert-base-cased-distilled-squad', tokenizer='bert-base-cased') - # Named entity recognition pipeline, passing in a specific model and tokenizer - model = AutoModelForTokenClassification.from_pretrained("dbmdz/bert-large-cased-finetuned-conll03-english") - tokenizer = AutoTokenizer.from_pretrained("bert-base-cased") - pipeline('ner', model=model, tokenizer=tokenizer) + >>> # Named entity recognition pipeline, passing in a specific model and tokenizer + >>> model = AutoModelForTokenClassification.from_pretrained("dbmdz/bert-large-cased-finetuned-conll03-english") + >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-cased") + >>> pipeline('ner', model=model, tokenizer=tokenizer) """ # Retrieve the task if task not in SUPPORTED_TASKS: diff --git a/src/transformers/testing_utils.py b/src/transformers/testing_utils.py index cda9e0f889..49e1dff952 100644 --- a/src/transformers/testing_utils.py +++ b/src/transformers/testing_utils.py @@ -297,15 +297,15 @@ class CaptureLogger: Example: - from transformers import logging - from transformers.testing_utils import CaptureLogger + >>> from transformers import logging + >>> from transformers.testing_utils import CaptureLogger - msg = "Testing 1, 2, 3" - logging.set_verbosity_info() - logger = logging.get_logger("transformers.tokenization_bart") - with CaptureLogger(logger) as cl: - logger.info(msg) - assert cl.out, msg+"\n" + >>> msg = "Testing 1, 2, 3" + >>> logging.set_verbosity_info() + >>> logger = logging.get_logger("transformers.tokenization_bart") + >>> with CaptureLogger(logger) as cl: + ... logger.info(msg) + >>> assert cl.out, msg+"\n" """ def __init__(self, logger): diff --git a/src/transformers/tokenization_auto.py b/src/transformers/tokenization_auto.py index 2e7f38193f..2b25ad68a7 100644 --- a/src/transformers/tokenization_auto.py +++ b/src/transformers/tokenization_auto.py @@ -185,16 +185,16 @@ class AutoTokenizer: Examples:: - from transformers import AutoTokenizer + >>> from transformers import AutoTokenizer - # Download vocabulary from S3 and cache. - tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased') + >>> # Download vocabulary from S3 and cache. + >>> tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased') - # Download vocabulary from S3 (user-uploaded) and cache. - tokenizer = AutoTokenizer.from_pretrained('dbmdz/bert-base-german-cased') + >>> # Download vocabulary from S3 (user-uploaded) and cache. + >>> tokenizer = AutoTokenizer.from_pretrained('dbmdz/bert-base-german-cased') - # If vocabulary files are in a directory (e.g. tokenizer was saved using `save_pretrained('./test/saved_model/')`) - tokenizer = AutoTokenizer.from_pretrained('./test/bert_saved_model/') + >>> # If vocabulary files are in a directory (e.g. tokenizer was saved using `save_pretrained('./test/saved_model/')`) + >>> tokenizer = AutoTokenizer.from_pretrained('./test/bert_saved_model/') """ config = kwargs.pop("config", None) diff --git a/src/transformers/tokenization_dpr.py b/src/transformers/tokenization_dpr.py index 3d75028e6c..41728d455d 100644 --- a/src/transformers/tokenization_dpr.py +++ b/src/transformers/tokenization_dpr.py @@ -278,18 +278,18 @@ class CustomDPRReaderTokenizerMixin: Examples:: - from transformers import DPRReader, DPRReaderTokenizer - tokenizer = DPRReaderTokenizer.from_pretrained('facebook/dpr-reader-single-nq-base') - model = DPRReader.from_pretrained('facebook/dpr-reader-single-nq-base') - encoded_inputs = tokenizer( - questions=["What is love ?"], - titles=["Haddaway"], - texts=["'What Is Love' is a song recorded by the artist Haddaway"], - return_tensors='pt' - ) - outputs = model(**encoded_inputs) - predicted_spans = tokenizer.decode_best_spans(encoded_inputs, outputs) - print(predicted_spans[0].text) # best span + >>> from transformers import DPRReader, DPRReaderTokenizer + >>> tokenizer = DPRReaderTokenizer.from_pretrained('facebook/dpr-reader-single-nq-base') + >>> model = DPRReader.from_pretrained('facebook/dpr-reader-single-nq-base') + >>> encoded_inputs = tokenizer( + ... questions=["What is love ?"], + ... titles=["Haddaway"], + ... texts=["'What Is Love' is a song recorded by the artist Haddaway"], + ... return_tensors='pt' + ... ) + >>> outputs = model(**encoded_inputs) + >>> predicted_spans = tokenizer.decode_best_spans(encoded_inputs, outputs) + >>> print(predicted_spans[0].text) # best span """ input_ids = reader_input["input_ids"] diff --git a/src/transformers/tokenization_pegasus.py b/src/transformers/tokenization_pegasus.py index 67ae882d8c..441f23d3ca 100644 --- a/src/transformers/tokenization_pegasus.py +++ b/src/transformers/tokenization_pegasus.py @@ -14,9 +14,8 @@ # limitations under the License. from typing import Dict, List, Optional -from transformers.tokenization_reformer import ReformerTokenizer - from .file_utils import add_start_docstrings +from .tokenization_reformer import ReformerTokenizer from .tokenization_utils_base import PREPARE_SEQ2SEQ_BATCH_DOCSTRING, BatchEncoding diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py index 7cb7f77920..bb17635171 100755 --- a/src/transformers/trainer.py +++ b/src/transformers/trainer.py @@ -60,7 +60,7 @@ _use_apex = False # Check if Pytorch version >= 1.6 to switch between Native AMP and Apex if version.parse(torch.__version__) < version.parse("1.6"): - from transformers.file_utils import is_apex_available + from .file_utils import is_apex_available if is_apex_available(): from apex import amp