SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Update repo to isort v5 (#6686)

Browse Source 
* Run new isort

* More changes

* Update CI, CONTRIBUTING and benchmarks
This commit is contained in:
Sylvain Gugger
2020-08-24 11:03:01 -04:00
committed by GitHub
parent d329c9b05d
commit a573777901
89 changed files with 615 additions and 632 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.circleci/config.yml
View File

@@ -235,8 +235,7 @@ jobs:
- v0.3-code_quality-{{ checksum "setup.py" }}
- v0.3-{{ checksum "setup.py" }}
- run: pip install --upgrade pip
# we need a version of isort with https://github.com/timothycrosley/isort/pull/1000
- run: pip install git+git://github.com/timothycrosley/isort.git@e63ae06ec7d70b06df9e528357650281a3d3ec22#egg=isort
- run: pip install isort
- run: pip install .[tf,torch,quality]
- save_cache:
key: v0.3-code_quality-{{ checksum "setup.py" }}

6
CONTRIBUTING.md
View File

@@ -134,12 +134,6 @@ Follow these steps to start contributing:
it with `pip uninstall transformers` before reinstalling it in editable
mode with the `-e` flag.)
Right now, we need an unreleased version of `isort` to avoid a
[bug](https://github.com/timothycrosley/isort/pull/1000):
```bash
$ pip install -U git+git://github.com/timothycrosley/isort.git@e63ae06ec7d70b06df9e528357650281a3d3ec22#egg=isort
```
5. Develop the features on your branch.
As you work on the features, you should make sure that the test suite

4
Makefile
View File

@@ -4,7 +4,7 @@
quality:
black --check --line-length 119 --target-version py35 examples templates tests src utils
isort --check-only --recursive examples templates tests src utils
isort --check-only examples templates tests src utils
flake8 examples templates tests src utils
python utils/check_repo.py
@@ -12,7 +12,7 @@ quality:
style:
black --line-length 119 --target-version py35 examples templates tests src utils
isort --recursive examples templates tests src utils
isort examples templates tests src utils
# Run tests for the library

2
examples/adversarial/utils_hans.py
View File

@@ -20,8 +20,8 @@ from dataclasses import dataclass
from typing import List, Optional, Union
import tqdm
from filelock import FileLock
from filelock import FileLock
from transformers import (
BartTokenizer,
BartTokenizerFast,

2
examples/multiple-choice/utils_multiple_choice.py
View File

@@ -26,8 +26,8 @@ from enum import Enum
from typing import List, Optional
import tqdm
from filelock import FileLock
from filelock import FileLock
from transformers import PreTrainedTokenizer, is_tf_available, is_torch_available

3
examples/seq2seq/bertabs/run_summarization.py
View File

@@ -44,9 +44,10 @@ def evaluate(args):
reference_summaries = []
generated_summaries = []
import rouge
import nltk
import rouge
nltk.download("punkt")
rouge_evaluator = rouge.Rouge(
metrics=["rouge-n", "rouge-l"],

28
examples/seq2seq/distillation.py
View File

@@ -15,27 +15,27 @@ from transformers import BartConfig, BartForConditionalGeneration, MBartTokenize
try:
from .finetune import SummarizationModule, TranslationModule
from .initialization_utils import init_student, copy_layers
from .utils import (
use_task_specific_params,
pickle_load,
freeze_params,
assert_all_frozen,
any_requires_grad,
calculate_bleu_score,
)
from .finetune import main as ft_main
from .initialization_utils import copy_layers, init_student
from .utils import (
any_requires_grad,
assert_all_frozen,
calculate_bleu_score,
freeze_params,
pickle_load,
use_task_specific_params,
)
except ImportError:
from finetune import SummarizationModule, TranslationModule
from finetune import main as ft_main
from initialization_utils import init_student, copy_layers
from initialization_utils import copy_layers, init_student
from utils import (
use_task_specific_params,
pickle_load,
freeze_params,
assert_all_frozen,
any_requires_grad,
assert_all_frozen,
calculate_bleu_score,
freeze_params,
pickle_load,
use_task_specific_params,
)

55
examples/seq2seq/finetune.py
View File

@@ -17,44 +17,43 @@ from transformers import MarianTokenizer, MBartTokenizer, T5ForConditionalGenera
try:
from .utils import (
assert_all_frozen,
use_task_specific_params,
lmap,
flatten_list,
pickle_save,
save_git_info,
save_json,
freeze_params,
calculate_rouge,
get_git_info,
ROUGE_KEYS,
calculate_bleu_score,
Seq2SeqDataset,
TranslationDataset,
label_smoothed_nll_loss,
)
from .callbacks import Seq2SeqLoggingCallback, get_checkpoint_callback, get_early_stopping_callback
except ImportError:
from utils import (
from .utils import (
ROUGE_KEYS,
Seq2SeqDataset,
TranslationDataset,
assert_all_frozen,
use_task_specific_params,
lmap,
calculate_bleu_score,
calculate_rouge,
flatten_list,
freeze_params,
get_git_info,
label_smoothed_nll_loss,
lmap,
pickle_save,
save_git_info,
save_json,
freeze_params,
calculate_rouge,
get_git_info,
ROUGE_KEYS,
calculate_bleu_score,
label_smoothed_nll_loss,
use_task_specific_params,
)
except ImportError:
from callbacks import Seq2SeqLoggingCallback, get_checkpoint_callback, get_early_stopping_callback
from utils import (
ROUGE_KEYS,
Seq2SeqDataset,
TranslationDataset,
assert_all_frozen,
calculate_bleu_score,
calculate_rouge,
flatten_list,
freeze_params,
get_git_info,
label_smoothed_nll_loss,
lmap,
pickle_save,
save_git_info,
save_json,
use_task_specific_params,
)
logger = logging.getLogger(__name__)

4
examples/seq2seq/run_eval.py
View File

@@ -9,9 +9,9 @@ from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
try:
from .utils import calculate_rouge, use_task_specific_params, calculate_bleu_score, trim_batch
from .utils import calculate_bleu_score, calculate_rouge, trim_batch, use_task_specific_params
except ImportError:
from utils import calculate_rouge, use_task_specific_params, calculate_bleu_score, trim_batch
from utils import calculate_bleu_score, calculate_rouge, trim_batch, use_task_specific_params
DEFAULT_DEVICE = "cuda" if torch.cuda.is_available() else "cpu"

2
examples/test_examples.py
View File

@@ -35,8 +35,8 @@ sys.path.extend(SRC_DIRS)
if SRC_DIRS is not None:
import run_generation
import run_glue
import run_pl_glue
import run_language_modeling
import run_pl_glue
import run_squad

1
examples/token-classification/utils_ner.py
View File

@@ -23,7 +23,6 @@ from enum import Enum
from typing import List, Optional, Union
from filelock import FileLock
from transformers import PreTrainedTokenizer, is_tf_available, is_torch_available

1
setup.cfg
View File

@@ -1,4 +1,5 @@
[isort]
default_section = FIRSTPARTY
ensure_newline_before_comments = True
force_grid_wrap = 0
include_trailing_comma = True

7
setup.py
View File

@@ -91,12 +91,7 @@ extras["all"] = extras["serving"] + ["tensorflow", "torch"]
extras["testing"] = ["pytest", "pytest-xdist", "timeout-decorator", "psutil"]
# sphinx-rtd-theme==0.5.0 introduced big changes in the style.
extras["docs"] = ["recommonmark", "sphinx", "sphinx-markdown-tables", "sphinx-rtd-theme==0.4.3", "sphinx-copybutton"]
extras["quality"] = [
"black",
# "isort",
"isort @ git+git://github.com/timothycrosley/isort.git@e63ae06ec7d70b06df9e528357650281a3d3ec22#egg=isort",
"flake8",
]
extras["quality"] = ["black", "isort >= 5", "flake8"]
extras["dev"] = extras["testing"] + extras["quality"] + extras["ja"] + ["scikit-learn", "tensorflow", "torch"]
setup(

562
src/transformers/__init__.py
View File

@@ -189,241 +189,246 @@ if is_sklearn_available():
# Modeling
if is_torch_available():
# Benchmarks
from .benchmark.benchmark import PyTorchBenchmark
from .benchmark.benchmark_args import PyTorchBenchmarkArguments
from .data.data_collator import (
DataCollator,
DataCollatorForLanguageModeling,
DataCollatorForPermutationLanguageModeling,
DataCollatorWithPadding,
default_data_collator,
)
from .data.datasets import (
GlueDataset,
GlueDataTrainingArguments,
LineByLineTextDataset,
SquadDataset,
SquadDataTrainingArguments,
TextDataset,
)
from .generation_utils import top_k_top_p_filtering
from .modeling_utils import PreTrainedModel, prune_layer, Conv1D, apply_chunking_to_forward
from .modeling_auto import (
AutoModel,
AutoModelForPreTraining,
AutoModelForSequenceClassification,
AutoModelForQuestionAnswering,
AutoModelWithLMHead,
AutoModelForCausalLM,
AutoModelForMaskedLM,
AutoModelForSeq2SeqLM,
AutoModelForTokenClassification,
AutoModelForMultipleChoice,
MODEL_MAPPING,
MODEL_FOR_PRETRAINING_MAPPING,
MODEL_WITH_LM_HEAD_MAPPING,
MODEL_FOR_CAUSAL_LM_MAPPING,
MODEL_FOR_MASKED_LM_MAPPING,
MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
MODEL_FOR_QUESTION_ANSWERING_MAPPING,
MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
)
from .modeling_mobilebert import (
MobileBertPreTrainedModel,
MobileBertModel,
MobileBertForPreTraining,
MobileBertForSequenceClassification,
MobileBertForQuestionAnswering,
MobileBertForMaskedLM,
MobileBertForNextSentencePrediction,
MobileBertForMultipleChoice,
MobileBertForTokenClassification,
load_tf_weights_in_mobilebert,
MOBILEBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
MobileBertLayer,
)
from .modeling_bert import (
BertPreTrainedModel,
BertModel,
BertForPreTraining,
BertForMaskedLM,
BertLMHeadModel,
BertForNextSentencePrediction,
BertForSequenceClassification,
BertForMultipleChoice,
BertForTokenClassification,
BertForQuestionAnswering,
load_tf_weights_in_bert,
BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
BertLayer,
)
from .modeling_openai import (
OpenAIGPTPreTrainedModel,
OpenAIGPTModel,
OpenAIGPTLMHeadModel,
OpenAIGPTDoubleHeadsModel,
load_tf_weights_in_openai_gpt,
OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
)
from .modeling_transfo_xl import (
TransfoXLPreTrainedModel,
TransfoXLModel,
TransfoXLLMHeadModel,
AdaptiveEmbedding,
load_tf_weights_in_transfo_xl,
TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_LIST,
)
from .modeling_gpt2 import (
GPT2PreTrainedModel,
GPT2Model,
GPT2LMHeadModel,
GPT2DoubleHeadsModel,
load_tf_weights_in_gpt2,
GPT2_PRETRAINED_MODEL_ARCHIVE_LIST,
)
from .modeling_ctrl import CTRLPreTrainedModel, CTRLModel, CTRLLMHeadModel, CTRL_PRETRAINED_MODEL_ARCHIVE_LIST
from .modeling_xlnet import (
XLNetPreTrainedModel,
XLNetModel,
XLNetLMHeadModel,
XLNetForSequenceClassification,
XLNetForTokenClassification,
XLNetForMultipleChoice,
XLNetForQuestionAnsweringSimple,
XLNetForQuestionAnswering,
load_tf_weights_in_xlnet,
XLNET_PRETRAINED_MODEL_ARCHIVE_LIST,
)
from .modeling_xlm import (
XLMPreTrainedModel,
XLMModel,
XLMWithLMHeadModel,
XLMForSequenceClassification,
XLMForTokenClassification,
XLMForQuestionAnswering,
XLMForQuestionAnsweringSimple,
XLMForMultipleChoice,
XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
)
from .modeling_pegasus import PegasusForConditionalGeneration
from .modeling_bart import (
PretrainedBartModel,
BartForSequenceClassification,
BartModel,
BartForConditionalGeneration,
BartForQuestionAnswering,
BART_PRETRAINED_MODEL_ARCHIVE_LIST,
)
from .modeling_mbart import MBartForConditionalGeneration
from .modeling_marian import MarianMTModel
from .tokenization_marian import MarianTokenizer
from .modeling_roberta import (
RobertaForMaskedLM,
RobertaForCausalLM,
RobertaModel,
RobertaForSequenceClassification,
RobertaForMultipleChoice,
RobertaForTokenClassification,
RobertaForQuestionAnswering,
ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST,
)
from .modeling_distilbert import (
DistilBertPreTrainedModel,
DistilBertForMaskedLM,
DistilBertModel,
DistilBertForMultipleChoice,
DistilBertForSequenceClassification,
DistilBertForQuestionAnswering,
DistilBertForTokenClassification,
DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
)
from .modeling_camembert import (
CamembertForMaskedLM,
CamembertModel,
CamembertForSequenceClassification,
CamembertForMultipleChoice,
CamembertForTokenClassification,
CamembertForQuestionAnswering,
CamembertForCausalLM,
CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
)
from .modeling_encoder_decoder import EncoderDecoderModel
from .modeling_t5 import (
T5PreTrainedModel,
T5Model,
T5ForConditionalGeneration,
load_tf_weights_in_t5,
T5_PRETRAINED_MODEL_ARCHIVE_LIST,
)
from .modeling_albert import (
AlbertPreTrainedModel,
AlbertModel,
AlbertForPreTraining,
ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
AlbertForMaskedLM,
AlbertForMultipleChoice,
AlbertForSequenceClassification,
AlbertForPreTraining,
AlbertForQuestionAnswering,
AlbertForSequenceClassification,
AlbertForTokenClassification,
AlbertModel,
AlbertPreTrainedModel,
load_tf_weights_in_albert,
ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
)
from .modeling_xlm_roberta import (
XLMRobertaForMaskedLM,
XLMRobertaModel,
XLMRobertaForMultipleChoice,
XLMRobertaForSequenceClassification,
XLMRobertaForTokenClassification,
XLMRobertaForQuestionAnswering,
XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST,
from .modeling_auto import (
MODEL_FOR_CAUSAL_LM_MAPPING,
MODEL_FOR_MASKED_LM_MAPPING,
MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
MODEL_FOR_PRETRAINING_MAPPING,
MODEL_FOR_QUESTION_ANSWERING_MAPPING,
MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
MODEL_MAPPING,
MODEL_WITH_LM_HEAD_MAPPING,
AutoModel,
AutoModelForCausalLM,
AutoModelForMaskedLM,
AutoModelForMultipleChoice,
AutoModelForPreTraining,
AutoModelForQuestionAnswering,
AutoModelForSeq2SeqLM,
AutoModelForSequenceClassification,
AutoModelForTokenClassification,
AutoModelWithLMHead,
)
from .modeling_mmbt import ModalEmbeddings, MMBTModel, MMBTForClassification
from .modeling_flaubert import (
FlaubertModel,
FlaubertWithLMHeadModel,
FlaubertForSequenceClassification,
FlaubertForTokenClassification,
FlaubertForQuestionAnswering,
FlaubertForQuestionAnsweringSimple,
FlaubertForTokenClassification,
FlaubertForMultipleChoice,
FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
from .modeling_bart import (
BART_PRETRAINED_MODEL_ARCHIVE_LIST,
BartForConditionalGeneration,
BartForQuestionAnswering,
BartForSequenceClassification,
BartModel,
PretrainedBartModel,
)
from .modeling_electra import (
ElectraForPreTraining,
ElectraForMaskedLM,
ElectraForTokenClassification,
ElectraPreTrainedModel,
ElectraForMultipleChoice,
ElectraForSequenceClassification,
ElectraForQuestionAnswering,
ElectraModel,
load_tf_weights_in_electra,
ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST,
from .modeling_bert import (
BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
BertForMaskedLM,
BertForMultipleChoice,
BertForNextSentencePrediction,
BertForPreTraining,
BertForQuestionAnswering,
BertForSequenceClassification,
BertForTokenClassification,
BertLayer,
BertLMHeadModel,
BertModel,
BertPreTrainedModel,
load_tf_weights_in_bert,
)
from .modeling_reformer import (
ReformerAttention,
ReformerLayer,
ReformerModel,
ReformerForMaskedLM,
ReformerModelWithLMHead,
ReformerForSequenceClassification,
ReformerForQuestionAnswering,
REFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
from .modeling_camembert import (
CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
CamembertForCausalLM,
CamembertForMaskedLM,
CamembertForMultipleChoice,
CamembertForQuestionAnswering,
CamembertForSequenceClassification,
CamembertForTokenClassification,
CamembertModel,
)
from .modeling_longformer import (
LongformerModel,
LongformerForMaskedLM,
LongformerForSequenceClassification,
LongformerForMultipleChoice,
LongformerForTokenClassification,
LongformerForQuestionAnswering,
LongformerSelfAttention,
LONGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
from .modeling_ctrl import CTRL_PRETRAINED_MODEL_ARCHIVE_LIST, CTRLLMHeadModel, CTRLModel, CTRLPreTrainedModel
from .modeling_distilbert import (
DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
DistilBertForMaskedLM,
DistilBertForMultipleChoice,
DistilBertForQuestionAnswering,
DistilBertForSequenceClassification,
DistilBertForTokenClassification,
DistilBertModel,
DistilBertPreTrainedModel,
)
from .modeling_dpr import (
DPRContextEncoder,
DPRPretrainedContextEncoder,
DPRPretrainedQuestionEncoder,
DPRPretrainedReader,
DPRContextEncoder,
DPRQuestionEncoder,
DPRReader,
)
from .modeling_retribert import (
RetriBertPreTrainedModel,
RetriBertModel,
RETRIBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
from .modeling_electra import (
ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST,
ElectraForMaskedLM,
ElectraForMultipleChoice,
ElectraForPreTraining,
ElectraForQuestionAnswering,
ElectraForSequenceClassification,
ElectraForTokenClassification,
ElectraModel,
ElectraPreTrainedModel,
load_tf_weights_in_electra,
)
from .modeling_encoder_decoder import EncoderDecoderModel
from .modeling_flaubert import (
FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
FlaubertForMultipleChoice,
FlaubertForQuestionAnswering,
FlaubertForQuestionAnsweringSimple,
FlaubertForSequenceClassification,
FlaubertForTokenClassification,
FlaubertModel,
FlaubertWithLMHeadModel,
)
from .modeling_gpt2 import (
GPT2_PRETRAINED_MODEL_ARCHIVE_LIST,
GPT2DoubleHeadsModel,
GPT2LMHeadModel,
GPT2Model,
GPT2PreTrainedModel,
load_tf_weights_in_gpt2,
)
from .modeling_longformer import (
LONGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
LongformerForMaskedLM,
LongformerForMultipleChoice,
LongformerForQuestionAnswering,
LongformerForSequenceClassification,
LongformerForTokenClassification,
LongformerModel,
LongformerSelfAttention,
)
from .modeling_marian import MarianMTModel
from .modeling_mbart import MBartForConditionalGeneration
from .modeling_mmbt import MMBTForClassification, MMBTModel, ModalEmbeddings
from .modeling_mobilebert import (
MOBILEBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
MobileBertForMaskedLM,
MobileBertForMultipleChoice,
MobileBertForNextSentencePrediction,
MobileBertForPreTraining,
MobileBertForQuestionAnswering,
MobileBertForSequenceClassification,
MobileBertForTokenClassification,
MobileBertLayer,
MobileBertModel,
MobileBertPreTrainedModel,
load_tf_weights_in_mobilebert,
)
from .modeling_openai import (
OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
OpenAIGPTDoubleHeadsModel,
OpenAIGPTLMHeadModel,
OpenAIGPTModel,
OpenAIGPTPreTrainedModel,
load_tf_weights_in_openai_gpt,
)
from .modeling_pegasus import PegasusForConditionalGeneration
from .modeling_reformer import (
REFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
ReformerAttention,
ReformerForMaskedLM,
ReformerForQuestionAnswering,
ReformerForSequenceClassification,
ReformerLayer,
ReformerModel,
ReformerModelWithLMHead,
)
from .modeling_retribert import RETRIBERT_PRETRAINED_MODEL_ARCHIVE_LIST, RetriBertModel, RetriBertPreTrainedModel
from .modeling_roberta import (
ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST,
RobertaForCausalLM,
RobertaForMaskedLM,
RobertaForMultipleChoice,
RobertaForQuestionAnswering,
RobertaForSequenceClassification,
RobertaForTokenClassification,
RobertaModel,
)
from .modeling_t5 import (
T5_PRETRAINED_MODEL_ARCHIVE_LIST,
T5ForConditionalGeneration,
T5Model,
T5PreTrainedModel,
load_tf_weights_in_t5,
)
from .modeling_transfo_xl import (
TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_LIST,
AdaptiveEmbedding,
TransfoXLLMHeadModel,
TransfoXLModel,
TransfoXLPreTrainedModel,
load_tf_weights_in_transfo_xl,
)
from .modeling_utils import Conv1D, PreTrainedModel, apply_chunking_to_forward, prune_layer
from .modeling_xlm import (
XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
XLMForMultipleChoice,
XLMForQuestionAnswering,
XLMForQuestionAnsweringSimple,
XLMForSequenceClassification,
XLMForTokenClassification,
XLMModel,
XLMPreTrainedModel,
XLMWithLMHeadModel,
)
from .modeling_xlm_roberta import (
XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST,
XLMRobertaForMaskedLM,
XLMRobertaForMultipleChoice,
XLMRobertaForQuestionAnswering,
XLMRobertaForSequenceClassification,
XLMRobertaForTokenClassification,
XLMRobertaModel,
)
from .modeling_xlnet import (
XLNET_PRETRAINED_MODEL_ARCHIVE_LIST,
XLNetForMultipleChoice,
XLNetForQuestionAnswering,
XLNetForQuestionAnsweringSimple,
XLNetForSequenceClassification,
XLNetForTokenClassification,
XLNetLMHeadModel,
XLNetModel,
XLNetPreTrainedModel,
load_tf_weights_in_xlnet,
)
# Optimization
@@ -436,61 +441,18 @@ if is_torch_available():
get_linear_schedule_with_warmup,
get_polynomial_decay_schedule_with_warmup,
)
from .tokenization_marian import MarianTokenizer
# Trainer
from .trainer import Trainer, set_seed, torch_distributed_zero_first, EvalPrediction
from .data.data_collator import (
default_data_collator,
DataCollator,
DataCollatorForLanguageModeling,
DataCollatorForPermutationLanguageModeling,
DataCollatorWithPadding,
)
from .data.datasets import (
GlueDataset,
TextDataset,
LineByLineTextDataset,
GlueDataTrainingArguments,
SquadDataset,
SquadDataTrainingArguments,
)
# Benchmarks
from .benchmark.benchmark import PyTorchBenchmark
from .benchmark.benchmark_args import PyTorchBenchmarkArguments
from .trainer import EvalPrediction, Trainer, set_seed, torch_distributed_zero_first
# TensorFlow
if is_tf_available():
from .generation_tf_utils import tf_top_k_top_p_filtering
from .modeling_tf_utils import (
shape_list,
TFPreTrainedModel,
TFSequenceSummary,
TFSharedEmbeddings,
)
from .modeling_tf_auto import (
TF_MODEL_MAPPING,
TF_MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
TF_MODEL_FOR_PRETRAINING_MAPPING,
TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
TF_MODEL_WITH_LM_HEAD_MAPPING,
TF_MODEL_FOR_CAUSAL_LM_MAPPING,
TF_MODEL_FOR_MASKED_LM_MAPPING,
TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
TFAutoModel,
TFAutoModelForMultipleChoice,
TFAutoModelForPreTraining,
TFAutoModelForQuestionAnswering,
TFAutoModelForSequenceClassification,
TFAutoModelForTokenClassification,
TFAutoModelWithLMHead,
TFAutoModelForCausalLM,
TFAutoModelForMaskedLM,
TFAutoModelForSeq2SeqLM,
)
from .benchmark.benchmark_args_tf import TensorFlowBenchmarkArguments
# Benchmarks
from .benchmark.benchmark_tf import TensorFlowBenchmark
from .generation_tf_utils import tf_top_k_top_p_filtering
from .modeling_tf_albert import (
TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
TFAlbertForMaskedLM,
@@ -503,11 +465,31 @@ if is_tf_available():
TFAlbertModel,
TFAlbertPreTrainedModel,
)
from .modeling_tf_auto import (
TF_MODEL_FOR_CAUSAL_LM_MAPPING,
TF_MODEL_FOR_MASKED_LM_MAPPING,
TF_MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
TF_MODEL_FOR_PRETRAINING_MAPPING,
TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
TF_MODEL_MAPPING,
TF_MODEL_WITH_LM_HEAD_MAPPING,
TFAutoModel,
TFAutoModelForCausalLM,
TFAutoModelForMaskedLM,
TFAutoModelForMultipleChoice,
TFAutoModelForPreTraining,
TFAutoModelForQuestionAnswering,
TFAutoModelForSeq2SeqLM,
TFAutoModelForSequenceClassification,
TFAutoModelForTokenClassification,
TFAutoModelWithLMHead,
)
from .modeling_tf_bert import (
TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
TFBertEmbeddings,
TFBertLMHeadModel,
TFBertForMaskedLM,
TFBertForMultipleChoice,
TFBertForNextSentencePrediction,
@@ -515,28 +497,26 @@ if is_tf_available():
TFBertForQuestionAnswering,
TFBertForSequenceClassification,
TFBertForTokenClassification,
TFBertLMHeadModel,
TFBertMainLayer,
TFBertModel,
TFBertPreTrainedModel,
)
from .modeling_tf_camembert import (
TF_CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
TFCamembertForMaskedLM,
TFCamembertModel,
TFCamembertForMultipleChoice,
TFCamembertForQuestionAnswering,
TFCamembertForSequenceClassification,
TFCamembertForTokenClassification,
TFCamembertModel,
)
from .modeling_tf_ctrl import (
TF_CTRL_PRETRAINED_MODEL_ARCHIVE_LIST,
TFCTRLLMHeadModel,
TFCTRLModel,
TFCTRLPreTrainedModel,
)
from .modeling_tf_distilbert import (
TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
TFDistilBertForMaskedLM,
@@ -548,7 +528,6 @@ if is_tf_available():
TFDistilBertModel,
TFDistilBertPreTrainedModel,
)
from .modeling_tf_electra import (
TF_ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST,
TFElectraForMaskedLM,
@@ -560,17 +539,15 @@ if is_tf_available():
TFElectraModel,
TFElectraPreTrainedModel,
)
from .modeling_tf_flaubert import (
TF_FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
TFFlaubertForMultipleChoice,
TFFlaubertForQuestionAnsweringSimple,
TFFlaubertForSequenceClassification,
TFFlaubertForTokenClassification,
TFFlaubertWithLMHeadModel,
TFFlaubertModel,
TFFlaubertWithLMHeadModel,
)
from .modeling_tf_gpt2 import (
TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST,
TFGPT2DoubleHeadsModel,
@@ -579,29 +556,26 @@ if is_tf_available():
TFGPT2Model,
TFGPT2PreTrainedModel,
)
from .modeling_tf_longformer import (
TF_LONGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
TFLongformerModel,
TFLongformerForMaskedLM,
TFLongformerForQuestionAnswering,
TFLongformerModel,
TFLongformerSelfAttention,
)
from .modeling_tf_mobilebert import (
TF_MOBILEBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
TFMobileBertModel,
TFMobileBertPreTrainedModel,
TFMobileBertForPreTraining,
TFMobileBertForSequenceClassification,
TFMobileBertForQuestionAnswering,
TFMobileBertForMaskedLM,
TFMobileBertForNextSentencePrediction,
TFMobileBertForMultipleChoice,
TFMobileBertForNextSentencePrediction,
TFMobileBertForPreTraining,
TFMobileBertForQuestionAnswering,
TFMobileBertForSequenceClassification,
TFMobileBertForTokenClassification,
TFMobileBertMainLayer,
TFMobileBertModel,
TFMobileBertPreTrainedModel,
)
from .modeling_tf_openai import (
TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
TFOpenAIGPTDoubleHeadsModel,
@@ -610,7 +584,6 @@ if is_tf_available():
TFOpenAIGPTModel,
TFOpenAIGPTPreTrainedModel,
)
from .modeling_tf_roberta import (
TF_ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST,
TFRobertaForMaskedLM,
@@ -622,14 +595,12 @@ if is_tf_available():
TFRobertaModel,
TFRobertaPreTrainedModel,
)
from .modeling_tf_t5 import (
TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST,
TFT5ForConditionalGeneration,
TFT5Model,
TFT5PreTrainedModel,
)
from .modeling_tf_transfo_xl import (
TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_LIST,
TFAdaptiveEmbedding,
@@ -638,19 +609,18 @@ if is_tf_available():
TFTransfoXLModel,
TFTransfoXLPreTrainedModel,
)
from .modeling_tf_utils import TFPreTrainedModel, TFSequenceSummary, TFSharedEmbeddings, shape_list
from .modeling_tf_xlm import (
TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
TFXLMForMultipleChoice,
TFXLMForQuestionAnsweringSimple,
TFXLMForSequenceClassification,
TFXLMForTokenClassification,
TFXLMWithLMHeadModel,
TFXLMMainLayer,
TFXLMModel,
TFXLMPreTrainedModel,
TFXLMWithLMHeadModel,
)
from .modeling_tf_xlm_roberta import (
TF_XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST,
TFXLMRobertaForMaskedLM,
@@ -660,7 +630,6 @@ if is_tf_available():
TFXLMRobertaForTokenClassification,
TFXLMRobertaModel,
)
from .modeling_tf_xlnet import (
TF_XLNET_PRETRAINED_MODEL_ARCHIVE_LIST,
TFXLNetForMultipleChoice,
@@ -674,20 +643,11 @@ if is_tf_available():
)
# Optimization
from .optimization_tf import (
AdamWeightDecay,
create_optimizer,
GradientAccumulator,
WarmUp,
)
from .optimization_tf import AdamWeightDecay, GradientAccumulator, WarmUp, create_optimizer
# Trainer
from .trainer_tf import TFTrainer
# Benchmarks
from .benchmark.benchmark_tf import TensorFlowBenchmark
from .benchmark.benchmark_args_tf import TensorFlowBenchmarkArguments
if not is_tf_available() and not is_torch_available():
logger.warning(

12
src/transformers/benchmark/benchmark.py
View File

@@ -22,14 +22,9 @@ import logging
import timeit
from typing import Callable, Optional
from transformers import (
MODEL_MAPPING,
MODEL_WITH_LM_HEAD_MAPPING,
PretrainedConfig,
is_py3nvml_available,
is_torch_available,
)
from ..configuration_utils import PretrainedConfig
from ..file_utils import is_py3nvml_available, is_torch_available
from ..modeling_auto import MODEL_MAPPING, MODEL_WITH_LM_HEAD_MAPPING
from .benchmark_utils import (
Benchmark,
Memory,
@@ -42,6 +37,7 @@ from .benchmark_utils import (
if is_torch_available():
import torch
from .benchmark_args import PyTorchBenchmarkArguments

14
src/transformers/benchmark/benchmark_tf.py
View File

@@ -24,14 +24,9 @@ import timeit
from functools import wraps
from typing import Callable, Optional
from transformers import (
TF_MODEL_MAPPING,
TF_MODEL_WITH_LM_HEAD_MAPPING,
PretrainedConfig,
is_py3nvml_available,
is_tf_available,
)
from ..configuration_utils import PretrainedConfig
from ..file_utils import is_py3nvml_available, is_tf_available
from ..modeling_tf_auto import TF_MODEL_MAPPING, TF_MODEL_WITH_LM_HEAD_MAPPING
from .benchmark_utils import (
Benchmark,
Memory,
@@ -44,9 +39,10 @@ from .benchmark_utils import (
if is_tf_available():
import tensorflow as tf
from .benchmark_args_tf import TensorFlowBenchmarkArguments
from tensorflow.python.framework.errors_impl import ResourceExhaustedError
from .benchmark_args_tf import TensorFlowBenchmarkArguments
if is_py3nvml_available():
import py3nvml.py3nvml as nvml

4
src/transformers/commands/serving.py
View File

@@ -8,11 +8,11 @@ from transformers.pipelines import SUPPORTED_TASKS, pipeline
try:
from uvicorn import run
from fastapi import FastAPI, HTTPException, Body
from fastapi import Body, FastAPI, HTTPException
from fastapi.routing import APIRoute
from pydantic import BaseModel
from starlette.responses import JSONResponse
from uvicorn import run
_serve_dependencies_installed = True
except (ImportError, AttributeError):

1
src/transformers/commands/user.py
View File

@@ -5,7 +5,6 @@ from getpass import getpass
from typing import List, Union
from requests.exceptions import HTTPError
from transformers.commands import BaseTransformersCLICommand
from transformers.hf_api import HfApi, HfFolder

8
src/transformers/convert_graph_to_onnx.py
View File

@@ -273,7 +273,9 @@ def convert_tensorflow(nlp: Pipeline, opset: int, output: Path):
try:
import tensorflow as tf
from keras2onnx import convert_keras, save_model, __version__ as k2ov
from keras2onnx import __version__ as k2ov
from keras2onnx import convert_keras, save_model
print(f"Using framework TensorFlow: {tf.version.VERSION}, keras2onnx: {k2ov}")
@@ -340,7 +342,7 @@ def optimize(onnx_model_path: Path) -> Path:
Returns: Path where the optimized model binary description has been saved
"""
from onnxruntime import SessionOptions, InferenceSession
from onnxruntime import InferenceSession, SessionOptions
# Generate model name with suffix "optimized"
opt_model_path = generate_identified_filename(onnx_model_path, "-optimized")
@@ -364,7 +366,7 @@ def quantize(onnx_model_path: Path) -> Path:
"""
try:
import onnx
from onnxruntime.quantization import quantize, QuantizationMode
from onnxruntime.quantization import QuantizationMode, quantize
onnx_model = onnx.load(onnx_model_path.as_posix())

25
src/transformers/convert_pytorch_checkpoint_to_tf2.py
View File

@@ -78,28 +78,29 @@ from transformers.file_utils import hf_bucket_url
if is_torch_available():
import torch
import numpy as np
import torch
from transformers import (
AlbertForPreTraining,
BertForPreTraining,
BertForQuestionAnswering,
BertForSequenceClassification,
CamembertForMaskedLM,
CTRLLMHeadModel,
DistilBertForMaskedLM,
DistilBertForQuestionAnswering,
ElectraForPreTraining,
FlaubertWithLMHeadModel,
GPT2LMHeadModel,
XLNetLMHeadModel,
XLMWithLMHeadModel,
XLMRobertaForMaskedLM,
TransfoXLLMHeadModel,
OpenAIGPTLMHeadModel,
RobertaForMaskedLM,
RobertaForSequenceClassification,
CamembertForMaskedLM,
FlaubertWithLMHeadModel,
DistilBertForMaskedLM,
DistilBertForQuestionAnswering,
CTRLLMHeadModel,
AlbertForPreTraining,
T5ForConditionalGeneration,
ElectraForPreTraining,
TransfoXLLMHeadModel,
XLMRobertaForMaskedLM,
XLMWithLMHeadModel,
XLNetLMHeadModel,
)

3
src/transformers/data/datasets/glue.py
View File

@@ -6,9 +6,10 @@ from enum import Enum
from typing import List, Optional, Union
import torch
from filelock import FileLock
from torch.utils.data.dataset import Dataset
from filelock import FileLock
from ...tokenization_bart import BartTokenizer, BartTokenizerFast
from ...tokenization_roberta import RobertaTokenizer, RobertaTokenizerFast
from ...tokenization_utils import PreTrainedTokenizer

3
src/transformers/data/datasets/language_modeling.py
View File

@@ -4,9 +4,10 @@ import pickle
import time
import torch
from filelock import FileLock
from torch.utils.data.dataset import Dataset
from filelock import FileLock
from ...tokenization_utils import PreTrainedTokenizer

3
src/transformers/data/datasets/squad.py
View File

@@ -6,9 +6,10 @@ from enum import Enum
from typing import Dict, List, Optional, Union
import torch
from filelock import FileLock
from torch.utils.data.dataset import Dataset
from filelock import FileLock
from ...modeling_auto import MODEL_FOR_QUESTION_ANSWERING_MAPPING
from ...tokenization_utils import PreTrainedTokenizer
from ..processors.squad import SquadFeatures, SquadV1Processor, SquadV2Processor, squad_convert_examples_to_features

3
src/transformers/data/metrics/__init__.py
View File

@@ -15,8 +15,9 @@
# limitations under the License.
try:
from sklearn.metrics import f1_score, matthews_corrcoef
from scipy.stats import pearsonr, spearmanr
from sklearn.metrics import matthews_corrcoef, f1_score
_has_sklearn = True
except (AttributeError, ImportError):

5
src/transformers/data/test_generation_utils.py
View File

@@ -11,10 +11,7 @@ from transformers.testing_utils import require_torch
if is_torch_available():
import torch
from transformers import (
MarianConfig,
MarianMTModel,
)
from transformers import MarianConfig, MarianMTModel
@require_torch

3
src/transformers/file_utils.py
View File

@@ -24,9 +24,10 @@ from urllib.parse import urlparse
from zipfile import ZipFile, is_zipfile
import numpy as np
from tqdm.auto import tqdm
import requests
from filelock import FileLock
from tqdm.auto import tqdm
from . import __version__

3
src/transformers/hf_api.py
View File

@@ -19,9 +19,10 @@ import os
from os.path import expanduser
from typing import Dict, List, Optional, Tuple
import requests
from tqdm import tqdm
import requests
ENDPOINT = "https://huggingface.co"

1
src/transformers/modeling_electra.py
View File

@@ -50,6 +50,7 @@ def load_tf_weights_in_electra(model, config, tf_checkpoint_path, discriminator_
"""
try:
import re
import numpy as np
import tensorflow as tf
except ImportError:

1
src/transformers/modeling_gpt2.py
View File

@@ -65,6 +65,7 @@ def load_tf_weights_in_gpt2(model, config, gpt2_checkpoint_path):
"""
try:
import re
import tensorflow as tf
except ImportError:
logger.error(

1
src/transformers/modeling_mobilebert.py
View File

@@ -68,6 +68,7 @@ def load_tf_weights_in_mobilebert(model, config, tf_checkpoint_path):
"""
try:
import re
import numpy as np
import tensorflow as tf
except ImportError:

1
src/transformers/modeling_openai.py
View File

@@ -62,6 +62,7 @@ def load_tf_weights_in_openai_gpt(model, config, openai_checkpoint_folder_path):
""" Load tf pre-trained weights in a pytorch model (from NumPy arrays here)
"""
import re
import numpy as np
if ".ckpt" in openai_checkpoint_folder_path:

1
src/transformers/modeling_t5.py
View File

@@ -66,6 +66,7 @@ def load_tf_weights_in_t5(model, config, tf_checkpoint_path):
"""
try:
import re
import numpy as np
import tensorflow as tf
except ImportError:

2
src/transformers/modeling_tf_pytorch_utils.py
View File

@@ -108,8 +108,8 @@ def load_pytorch_weights_in_tf2_model(tf_model, pt_state_dict, tf_inputs=None, a
""" Load pytorch state_dict in a TF 2.0 model.
"""
try:
import torch # noqa: F401
import tensorflow as tf # noqa: F401
import torch # noqa: F401
from tensorflow.python.keras import backend as K
except ImportError:
logger.error(

38
src/transformers/pipelines.py
View File

@@ -43,39 +43,41 @@ from .tokenization_utils_base import BatchEncoding, PaddingStrategy
if is_tf_available():
import tensorflow as tf
from .modeling_tf_auto import (
TFAutoModel,
TFAutoModelForSequenceClassification,
TFAutoModelForQuestionAnswering,
TFAutoModelForTokenClassification,
TFAutoModelWithLMHead,
TF_MODEL_WITH_LM_HEAD_MAPPING,
TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
TF_MODEL_WITH_LM_HEAD_MAPPING,
TFAutoModel,
TFAutoModelForCausalLM,
TFAutoModelForQuestionAnswering,
TFAutoModelForSequenceClassification,
TFAutoModelForTokenClassification,
TFAutoModelWithLMHead,
)
if is_torch_available():
import torch
from .modeling_auto import (
AutoModel,
AutoModelForSequenceClassification,
AutoModelForQuestionAnswering,
AutoModelForTokenClassification,
AutoModelForSeq2SeqLM,
AutoModelForCausalLM,
AutoModelForMaskedLM,
MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
MODEL_FOR_MASKED_LM_MAPPING,
MODEL_FOR_QUESTION_ANSWERING_MAPPING,
MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
MODEL_FOR_MASKED_LM_MAPPING,
MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
AutoModel,
AutoModelForCausalLM,
AutoModelForMaskedLM,
AutoModelForQuestionAnswering,
AutoModelForSeq2SeqLM,
AutoModelForSequenceClassification,
AutoModelForTokenClassification,
)
if TYPE_CHECKING:
from .modeling_utils import PreTrainedModel
from .modeling_tf_utils import TFPreTrainedModel
from .modeling_utils import PreTrainedModel
logger = logging.getLogger(__name__)

1
src/transformers/tokenization_transfo_xl.py
View File

@@ -27,6 +27,7 @@ from collections import Counter, OrderedDict
from typing import Optional
import numpy as np
from tokenizers import Tokenizer
from tokenizers.implementations import BaseTokenizer
from tokenizers.models import WordLevel

1
src/transformers/tokenization_utils_base.py
View File

@@ -28,6 +28,7 @@ from enum import Enum
from typing import Any, Dict, List, NamedTuple, Optional, Sequence, Tuple, Union
import numpy as np
from tokenizers import AddedToken
from tokenizers import Encoding as EncodingFast

1
templates/adding_a_new_model/modeling_xxx.py
View File

@@ -63,6 +63,7 @@ def load_tf_weights_in_xxx(model, config, tf_checkpoint_path):
"""
try:
import re
import numpy as np
import tensorflow as tf
except ImportError:

5
templates/adding_a_new_model/tests/test_modeling_tf_xxx.py
View File

@@ -25,13 +25,14 @@ from .utils import CACHE_DIR, require_tf, slow
if is_tf_available():
import tensorflow as tf
from transformers.modeling_tf_xxx import (
TFXxxModel,
TFXxxForMaskedLM,
TFXxxForMultipleChoice,
TFXxxForQuestionAnswering,
TFXxxForSequenceClassification,
TFXxxForTokenClassification,
TFXxxForQuestionAnswering,
TFXxxModel,
)

6
templates/adding_a_new_model/tests/test_modeling_xxx.py
View File

@@ -25,14 +25,14 @@ from .utils import require_torch, require_torch_and_cuda, slow, torch_device
if is_torch_available():
from transformers import (
AutoModelForMaskedLM,
AutoTokenizer,
XxxConfig,
XxxModel,
XxxForMaskedLM,
XxxForQuestionAnswering,
XxxForSequenceClassification,
XxxForTokenClassification,
AutoModelForMaskedLM,
AutoTokenizer,
XxxModel,
)
from transformers.file_utils import cached_property

3
tests/test_activations.py
View File

@@ -5,9 +5,10 @@ from transformers.testing_utils import require_torch
if is_torch_available():
from transformers.activations import _gelu_python, get_activation, gelu_new
import torch
from transformers.activations import _gelu_python, gelu_new, get_activation
@require_torch
class TestActivations(unittest.TestCase):

5
tests/test_benchmark.py
View File

@@ -8,10 +8,7 @@ from transformers.testing_utils import require_torch, torch_device
if is_torch_available():
from transformers import (
PyTorchBenchmarkArguments,
PyTorchBenchmark,
)
from transformers import PyTorchBenchmark, PyTorchBenchmarkArguments
@require_torch

1
tests/test_benchmark_tf.py
View File

@@ -9,6 +9,7 @@ from transformers.testing_utils import require_tf
if is_tf_available():
import tensorflow as tf
from transformers import TensorFlowBenchmark, TensorFlowBenchmarkArguments

1
tests/test_hf_api.py
View File

@@ -20,7 +20,6 @@ import unittest
import requests
from requests.exceptions import HTTPError
from transformers.hf_api import HfApi, HfFolder, ModelInfo, PresignedUrl, S3Obj

6
tests/test_modeling_albert.py
View File

@@ -26,13 +26,13 @@ from .test_modeling_common import ModelTesterMixin, ids_tensor, random_attention
if is_torch_available():
from transformers import (
AlbertConfig,
AlbertModel,
AlbertForPreTraining,
AlbertForMaskedLM,
AlbertForMultipleChoice,
AlbertForPreTraining,
AlbertForQuestionAnswering,
AlbertForSequenceClassification,
AlbertForTokenClassification,
AlbertForQuestionAnswering,
AlbertModel,
)
from transformers.modeling_albert import ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST

52
tests/test_modeling_auto.py
View File

@@ -23,42 +23,42 @@ from transformers.testing_utils import DUMMY_UNKWOWN_IDENTIFIER, SMALL_MODEL_IDE
if is_torch_available():
from transformers import (
AutoConfig,
BertConfig,
GPT2Config,
T5Config,
AutoModel,
BertModel,
AutoModelForPreTraining,
BertForPreTraining,
AutoModelForCausalLM,
GPT2LMHeadModel,
AutoModelWithLMHead,
AutoModelForMaskedLM,
BertForMaskedLM,
RobertaForMaskedLM,
AutoModelForSeq2SeqLM,
T5ForConditionalGeneration,
AutoModelForSequenceClassification,
BertForSequenceClassification,
AutoModelForPreTraining,
AutoModelForQuestionAnswering,
BertForQuestionAnswering,
AutoModelForSeq2SeqLM,
AutoModelForSequenceClassification,
AutoModelForTokenClassification,
AutoModelWithLMHead,
BertConfig,
BertForMaskedLM,
BertForPreTraining,
BertForQuestionAnswering,
BertForSequenceClassification,
BertForTokenClassification,
BertModel,
GPT2Config,
GPT2LMHeadModel,
RobertaForMaskedLM,
T5Config,
T5ForConditionalGeneration,
)
from transformers.modeling_auto import (
MODEL_FOR_CAUSAL_LM_MAPPING,
MODEL_FOR_MASKED_LM_MAPPING,
MODEL_FOR_PRETRAINING_MAPPING,
MODEL_FOR_QUESTION_ANSWERING_MAPPING,
MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
MODEL_MAPPING,
MODEL_WITH_LM_HEAD_MAPPING,
)
from transformers.modeling_bert import BERT_PRETRAINED_MODEL_ARCHIVE_LIST
from transformers.modeling_gpt2 import GPT2_PRETRAINED_MODEL_ARCHIVE_LIST
from transformers.modeling_t5 import T5_PRETRAINED_MODEL_ARCHIVE_LIST
from transformers.modeling_auto import (
MODEL_MAPPING,
MODEL_FOR_PRETRAINING_MAPPING,
MODEL_FOR_QUESTION_ANSWERING_MAPPING,
MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
MODEL_WITH_LM_HEAD_MAPPING,
MODEL_FOR_CAUSAL_LM_MAPPING,
MODEL_FOR_MASKED_LM_MAPPING,
MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
)
@require_torch

15
tests/test_modeling_bart.py
View File

@@ -28,24 +28,25 @@ from .test_modeling_common import ModelTesterMixin, ids_tensor
if is_torch_available():
import torch
from transformers import (
AutoModel,
AutoModelForSequenceClassification,
AutoTokenizer,
BartModel,
BartForConditionalGeneration,
BartForSequenceClassification,
BartForQuestionAnswering,
BartConfig,
BartForConditionalGeneration,
BartForQuestionAnswering,
BartForSequenceClassification,
BartModel,
BartTokenizer,
BartTokenizerFast,
pipeline,
)
from transformers.modeling_bart import (
shift_tokens_right,
invert_mask,
_prepare_bart_decoder_inputs,
SinusoidalPositionalEmbedding,
_prepare_bart_decoder_inputs,
invert_mask,
shift_tokens_right,
)
PGE_ARTICLE = """ PG&E stated it scheduled the blackouts in response to forecasts for high winds amid dry conditions. The aim is to reduce the risk of wildfires. Nearly 800 thousand customers were scheduled to be affected by the shutoffs which were expected to last through at least midday tomorrow."""

1
tests/test_modeling_camembert.py
View File

@@ -21,6 +21,7 @@ from transformers.testing_utils import require_torch, slow, torch_device
if is_torch_available():
import torch
from transformers import CamembertModel

14
tests/test_modeling_common.py
View File

@@ -29,19 +29,19 @@ if is_torch_available():
import torch
from transformers import (
AdaptiveEmbedding,
PretrainedConfig,
PreTrainedModel,
BertConfig,
BertModel,
BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
MODEL_FOR_QUESTION_ANSWERING_MAPPING,
MODEL_FOR_CAUSAL_LM_MAPPING,
MODEL_FOR_MASKED_LM_MAPPING,
MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
MODEL_FOR_QUESTION_ANSWERING_MAPPING,
MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
AdaptiveEmbedding,
BertConfig,
BertModel,
PretrainedConfig,
PreTrainedModel,
top_k_top_p_filtering,
)

3
tests/test_modeling_ctrl.py
View File

@@ -24,7 +24,8 @@ from .test_modeling_common import ModelTesterMixin, ids_tensor, random_attention
if is_torch_available():
import torch
from transformers import CTRLConfig, CTRLModel, CTRL_PRETRAINED_MODEL_ARCHIVE_LIST, CTRLLMHeadModel
from transformers import CTRL_PRETRAINED_MODEL_ARCHIVE_LIST, CTRLConfig, CTRLLMHeadModel, CTRLModel
class CTRLModelTester:

6
tests/test_modeling_distilbert.py
View File

@@ -25,14 +25,14 @@ from .test_modeling_common import ModelTesterMixin, ids_tensor, random_attention
if is_torch_available():
from transformers import (
DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
DistilBertConfig,
DistilBertModel,
DistilBertForMaskedLM,
DistilBertForMultipleChoice,
DistilBertForTokenClassification,
DistilBertForQuestionAnswering,
DistilBertForSequenceClassification,
DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
DistilBertForTokenClassification,
DistilBertModel,
)
class DistilBertModelTester(object):

8
tests/test_modeling_electra.py
View File

@@ -26,13 +26,13 @@ from .test_modeling_common import ModelTesterMixin, ids_tensor, random_attention
if is_torch_available():
from transformers import (
ElectraConfig,
ElectraModel,
ElectraForMaskedLM,
ElectraForTokenClassification,
ElectraForPreTraining,
ElectraForMultipleChoice,
ElectraForSequenceClassification,
ElectraForPreTraining,
ElectraForQuestionAnswering,
ElectraForSequenceClassification,
ElectraForTokenClassification,
ElectraModel,
)
from transformers.modeling_electra import ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST

19
tests/test_modeling_encoder_decoder.py
View File

@@ -27,18 +27,19 @@ from .test_modeling_roberta import RobertaModelTester
if is_torch_available():
from transformers import (
BertModel,
BertLMHeadModel,
GPT2LMHeadModel,
RobertaModel,
RobertaForCausalLM,
EncoderDecoderModel,
EncoderDecoderConfig,
)
import numpy as np
import torch
from transformers import (
BertLMHeadModel,
BertModel,
EncoderDecoderConfig,
EncoderDecoderModel,
GPT2LMHeadModel,
RobertaForCausalLM,
RobertaModel,
)
@require_torch
class EncoderDecoderMixin:

6
tests/test_modeling_flaubert.py
View File

@@ -26,13 +26,13 @@ from .test_modeling_common import ModelTesterMixin, ids_tensor, random_attention
if is_torch_available():
from transformers import (
FlaubertConfig,
FlaubertModel,
FlaubertWithLMHeadModel,
FlaubertForMultipleChoice,
FlaubertForQuestionAnswering,
FlaubertForQuestionAnsweringSimple,
FlaubertForSequenceClassification,
FlaubertForTokenClassification,
FlaubertForMultipleChoice,
FlaubertModel,
FlaubertWithLMHeadModel,
)
from transformers.modeling_flaubert import FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST

7
tests/test_modeling_gpt2.py
View File

@@ -25,12 +25,13 @@ from .test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, r
if is_torch_available():
import torch
from transformers import (
GPT2Config,
GPT2Model,
GPT2_PRETRAINED_MODEL_ARCHIVE_LIST,
GPT2LMHeadModel,
GPT2Config,
GPT2DoubleHeadsModel,
GPT2LMHeadModel,
GPT2Model,
)

7
tests/test_modeling_longformer.py
View File

@@ -25,14 +25,15 @@ from .test_modeling_common import ModelTesterMixin, ids_tensor, random_attention
if is_torch_available():
import torch
from transformers import (
LongformerConfig,
LongformerModel,
LongformerForMaskedLM,
LongformerForMultipleChoice,
LongformerForQuestionAnswering,
LongformerForSequenceClassification,
LongformerForTokenClassification,
LongformerForQuestionAnswering,
LongformerForMultipleChoice,
LongformerModel,
LongformerSelfAttention,
)

9
tests/test_modeling_marian.py
View File

@@ -24,18 +24,19 @@ from transformers.testing_utils import require_torch, slow, torch_device
if is_torch_available():
import torch
from transformers import (
AutoTokenizer,
MarianConfig,
AutoConfig,
AutoModelWithLMHead,
MarianTokenizer,
AutoTokenizer,
MarianConfig,
MarianMTModel,
MarianTokenizer,
)
from transformers.convert_marian_to_pytorch import (
ORG_NAME,
convert_hf_name_to_opus_name,
convert_opus_name_to_hf_name,
ORG_NAME,
)
from transformers.pipelines import TranslationPipeline

5
tests/test_modeling_mbart.py
View File

@@ -9,12 +9,13 @@ from .test_modeling_bart import TOLERANCE, _assert_tensors_equal, _long_tensor
if is_torch_available():
import torch
from transformers import (
AutoModelForSeq2SeqLM,
AutoTokenizer,
BatchEncoding,
MBartConfig,
MBartForConditionalGeneration,
BatchEncoding,
AutoTokenizer,
)

5
tests/test_modeling_mobilebert.py
View File

@@ -25,16 +25,17 @@ from .test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, r
if is_torch_available():
import torch
from transformers import (
MobileBertConfig,
MobileBertModel,
MobileBertForMaskedLM,
MobileBertForMultipleChoice,
MobileBertForNextSentencePrediction,
MobileBertForPreTraining,
MobileBertForQuestionAnswering,
MobileBertForSequenceClassification,
MobileBertForTokenClassification,
MobileBertForMultipleChoice,
MobileBertModel,
)

7
tests/test_modeling_openai.py
View File

@@ -25,12 +25,13 @@ from .test_modeling_common import ModelTesterMixin, ids_tensor
if is_torch_available():
import torch
from transformers import (
OpenAIGPTConfig,
OpenAIGPTModel,
OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
OpenAIGPTLMHeadModel,
OpenAIGPTConfig,
OpenAIGPTDoubleHeadsModel,
OpenAIGPTLMHeadModel,
OpenAIGPTModel,
)

11
tests/test_modeling_reformer.py
View File

@@ -23,18 +23,19 @@ from .test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, r
if is_torch_available():
import torch
from transformers import (
REFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
ReformerConfig,
ReformerForMaskedLM,
ReformerForQuestionAnswering,
ReformerForSequenceClassification,
ReformerLayer,
ReformerModel,
ReformerModelWithLMHead,
ReformerForSequenceClassification,
ReformerTokenizer,
ReformerLayer,
ReformerForQuestionAnswering,
REFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
)
import torch
class ReformerModelTester:

10
tests/test_modeling_roberta.py
View File

@@ -25,18 +25,22 @@ from .test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, r
if is_torch_available():
import torch
from transformers import (
RobertaConfig,
RobertaModel,
RobertaForCausalLM,
RobertaForMaskedLM,
RobertaForMultipleChoice,
RobertaForQuestionAnswering,
RobertaForSequenceClassification,
RobertaForTokenClassification,
RobertaModel,
)
from transformers.modeling_roberta import (
ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST,
RobertaEmbeddings,
create_position_ids_from_input_ids,
)
from transformers.modeling_roberta import RobertaEmbeddings, create_position_ids_from_input_ids
from transformers.modeling_roberta import ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST
class RobertaModelTester:

3
tests/test_modeling_t5.py
View File

@@ -28,7 +28,8 @@ from .test_modeling_common import ModelTesterMixin, ids_tensor
if is_torch_available():
import torch
from transformers import T5Config, T5Model, T5ForConditionalGeneration
from transformers import T5Config, T5ForConditionalGeneration, T5Model
from transformers.modeling_t5 import T5_PRETRAINED_MODEL_ARCHIVE_LIST
from transformers.tokenization_t5 import T5Tokenizer

9
tests/test_modeling_tf_albert.py
View File

@@ -25,15 +25,16 @@ from .test_modeling_tf_common import TFModelTesterMixin, ids_tensor
if is_tf_available():
import tensorflow as tf
from transformers.modeling_tf_albert import (
TFAlbertModel,
TFAlbertForPreTraining,
TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
TFAlbertForMaskedLM,
TFAlbertForMultipleChoice,
TFAlbertForSequenceClassification,
TFAlbertForPreTraining,
TFAlbertForQuestionAnswering,
TFAlbertForSequenceClassification,
TFAlbertForTokenClassification,
TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
TFAlbertModel,
)

40
tests/test_modeling_tf_auto.py
View File

@@ -27,36 +27,36 @@ if is_tf_available():
GPT2Config,
T5Config,
TFAutoModel,
TFBertModel,
TFAutoModelForCausalLM,
TFAutoModelForMaskedLM,
TFAutoModelForPreTraining,
TFBertForPreTraining,
TFAutoModelForQuestionAnswering,
TFAutoModelForSeq2SeqLM,
TFAutoModelForSequenceClassification,
TFAutoModelWithLMHead,
TFBertForMaskedLM,
TFRobertaForMaskedLM,
TFAutoModelForSequenceClassification,
TFBertForSequenceClassification,
TFAutoModelForQuestionAnswering,
TFBertForPreTraining,
TFBertForQuestionAnswering,
TFAutoModelForCausalLM,
TFBertForSequenceClassification,
TFBertModel,
TFGPT2LMHeadModel,
TFAutoModelForMaskedLM,
TFAutoModelForSeq2SeqLM,
TFRobertaForMaskedLM,
TFT5ForConditionalGeneration,
)
from transformers.modeling_tf_auto import (
TF_MODEL_FOR_CAUSAL_LM_MAPPING,
TF_MODEL_FOR_MASKED_LM_MAPPING,
TF_MODEL_FOR_PRETRAINING_MAPPING,
TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
TF_MODEL_MAPPING,
TF_MODEL_WITH_LM_HEAD_MAPPING,
)
from transformers.modeling_tf_bert import TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST
from transformers.modeling_tf_gpt2 import TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST
from transformers.modeling_tf_t5 import TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST
from transformers.modeling_tf_auto import (
TF_MODEL_MAPPING,
TF_MODEL_FOR_PRETRAINING_MAPPING,
TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
TF_MODEL_WITH_LM_HEAD_MAPPING,
TF_MODEL_FOR_CAUSAL_LM_MAPPING,
TF_MODEL_FOR_MASKED_LM_MAPPING,
TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
)
@require_tf

11
tests/test_modeling_tf_bert.py
View File

@@ -25,16 +25,17 @@ from .test_modeling_tf_common import TFModelTesterMixin, ids_tensor
if is_tf_available():
import tensorflow as tf
from transformers.modeling_tf_bert import (
TFBertModel,
TFBertLMHeadModel,
TFBertForMaskedLM,
TFBertForMultipleChoice,
TFBertForNextSentencePrediction,
TFBertForPreTraining,
TFBertForSequenceClassification,
TFBertForMultipleChoice,
TFBertForTokenClassification,
TFBertForQuestionAnswering,
TFBertForSequenceClassification,
TFBertForTokenClassification,
TFBertLMHeadModel,
TFBertModel,
)

3
tests/test_modeling_tf_camembert.py
View File

@@ -20,8 +20,9 @@ from transformers.testing_utils import require_tf, slow
if is_tf_available():
import tensorflow as tf
import numpy as np
import tensorflow as tf
from transformers import TFCamembertModel

17
tests/test_modeling_tf_common.py
View File

@@ -28,20 +28,20 @@ from transformers.testing_utils import _tf_gpu_memory_limit, require_tf, slow
if is_tf_available():
import tensorflow as tf
import numpy as np
import tensorflow as tf
from transformers import (
tf_top_k_top_p_filtering,
TFAdaptiveEmbedding,
TFSharedEmbeddings,
TF_MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
TF_MODEL_FOR_CAUSAL_LM_MAPPING,
TF_MODEL_FOR_MASKED_LM_MAPPING,
TF_MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
TFAdaptiveEmbedding,
TFSharedEmbeddings,
tf_top_k_top_p_filtering,
)
if _tf_gpu_memory_limit is not None:
@@ -260,6 +260,7 @@ class TFModelTesterMixin:
return
import torch
import transformers
config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()

3
tests/test_modeling_tf_ctrl.py
View File

@@ -25,7 +25,8 @@ from .test_modeling_tf_common import TFModelTesterMixin, ids_tensor
if is_tf_available():
import tensorflow as tf
from transformers.modeling_tf_ctrl import TFCTRLModel, TFCTRLLMHeadModel, TF_CTRL_PRETRAINED_MODEL_ARCHIVE_LIST
from transformers.modeling_tf_ctrl import TF_CTRL_PRETRAINED_MODEL_ARCHIVE_LIST, TFCTRLLMHeadModel, TFCTRLModel
class TFCTRLModelTester(object):

7
tests/test_modeling_tf_distilbert.py
View File

@@ -25,14 +25,15 @@ from .test_modeling_tf_common import TFModelTesterMixin, ids_tensor
if is_tf_available():
import tensorflow as tf
from transformers.modeling_tf_distilbert import (
TFDistilBertModel,
TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
TFDistilBertForMaskedLM,
TFDistilBertForMultipleChoice,
TFDistilBertForQuestionAnswering,
TFDistilBertForSequenceClassification,
TFDistilBertForTokenClassification,
TFDistilBertForMultipleChoice,
TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
TFDistilBertModel,
)

4
tests/test_modeling_tf_electra.py
View File

@@ -27,13 +27,13 @@ if is_tf_available():
import tensorflow as tf
from transformers.modeling_tf_electra import (
TFElectraModel,
TFElectraForMaskedLM,
TFElectraForMultipleChoice,
TFElectraForPreTraining,
TFElectraForQuestionAnswering,
TFElectraForSequenceClassification,
TFElectraForTokenClassification,
TFElectraForQuestionAnswering,
TFElectraModel,
)

12
tests/test_modeling_tf_flaubert.py
View File

@@ -23,18 +23,18 @@ from .test_modeling_tf_common import TFModelTesterMixin, ids_tensor
if is_tf_available():
import tensorflow as tf
import numpy as np
import tensorflow as tf
from transformers import (
TF_FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
FlaubertConfig,
TFFlaubertForMultipleChoice,
TFFlaubertForQuestionAnsweringSimple,
TFFlaubertForSequenceClassification,
TFFlaubertForTokenClassification,
TFFlaubertModel,
TFFlaubertWithLMHeadModel,
TFFlaubertForSequenceClassification,
TFFlaubertForQuestionAnsweringSimple,
TFFlaubertForTokenClassification,
TFFlaubertForMultipleChoice,
TF_FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
)

7
tests/test_modeling_tf_gpt2.py
View File

@@ -25,11 +25,12 @@ from .test_modeling_tf_common import TFModelTesterMixin, ids_tensor
if is_tf_available():
import tensorflow as tf
from transformers.modeling_tf_gpt2 import (
TFGPT2Model,
TFGPT2LMHeadModel,
TFGPT2DoubleHeadsModel,
TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST,
TFGPT2DoubleHeadsModel,
TFGPT2LMHeadModel,
TFGPT2Model,
shape_list,
)

3
tests/test_modeling_tf_longformer.py
View File

@@ -25,11 +25,12 @@ from .test_modeling_tf_common import TFModelTesterMixin, ids_tensor
if is_tf_available():
import tensorflow as tf
from transformers import (
LongformerConfig,
TFLongformerModel,
TFLongformerForMaskedLM,
TFLongformerForQuestionAnswering,
TFLongformerModel,
TFLongformerSelfAttention,
)

9
tests/test_modeling_tf_mobilebert.py
View File

@@ -25,15 +25,16 @@ from .test_modeling_tf_common import TFModelTesterMixin, ids_tensor
if is_tf_available():
import tensorflow as tf
from transformers.modeling_tf_mobilebert import (
TFMobileBertModel,
TFMobileBertForMaskedLM,
TFMobileBertForMultipleChoice,
TFMobileBertForNextSentencePrediction,
TFMobileBertForPreTraining,
TFMobileBertForSequenceClassification,
TFMobileBertForMultipleChoice,
TFMobileBertForTokenClassification,
TFMobileBertForQuestionAnswering,
TFMobileBertForSequenceClassification,
TFMobileBertForTokenClassification,
TFMobileBertModel,
)

7
tests/test_modeling_tf_openai.py
View File

@@ -25,11 +25,12 @@ from .test_modeling_tf_common import TFModelTesterMixin, ids_tensor
if is_tf_available():
import tensorflow as tf
from transformers.modeling_tf_openai import (
TFOpenAIGPTModel,
TFOpenAIGPTLMHeadModel,
TFOpenAIGPTDoubleHeadsModel,
TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
TFOpenAIGPTDoubleHeadsModel,
TFOpenAIGPTLMHeadModel,
TFOpenAIGPTModel,
)

11
tests/test_modeling_tf_roberta.py
View File

@@ -24,16 +24,17 @@ from .test_modeling_tf_common import TFModelTesterMixin, ids_tensor
if is_tf_available():
import tensorflow as tf
import numpy
import tensorflow as tf
from transformers.modeling_tf_roberta import (
TFRobertaModel,
TF_ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST,
TFRobertaForMaskedLM,
TFRobertaForMultipleChoice,
TFRobertaForQuestionAnswering,
TFRobertaForSequenceClassification,
TFRobertaForTokenClassification,
TFRobertaForQuestionAnswering,
TFRobertaForMultipleChoice,
TF_ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST,
TFRobertaModel,
)

3
tests/test_modeling_tf_t5.py
View File

@@ -26,7 +26,8 @@ from .test_modeling_tf_common import TFModelTesterMixin, ids_tensor
if is_tf_available():
import tensorflow as tf
from transformers import TFT5Model, TFT5ForConditionalGeneration, T5Tokenizer
from transformers import T5Tokenizer, TFT5ForConditionalGeneration, TFT5Model
class TFT5ModelTester:

7
tests/test_modeling_tf_transfo_xl.py
View File

@@ -26,11 +26,8 @@ from .test_modeling_tf_common import TFModelTesterMixin, ids_tensor
if is_tf_available():
import tensorflow as tf
from transformers import (
TFTransfoXLModel,
TFTransfoXLLMHeadModel,
TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_LIST,
)
from transformers import TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_LIST, TFTransfoXLLMHeadModel, TFTransfoXLModel
class TFTransfoXLModelTester:

13
tests/test_modeling_tf_xlm.py
View File

@@ -25,15 +25,16 @@ from .test_modeling_tf_common import TFModelTesterMixin, ids_tensor
if is_tf_available():
import tensorflow as tf
from transformers import (
XLMConfig,
TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
TFXLMForMultipleChoice,
TFXLMForQuestionAnsweringSimple,
TFXLMForSequenceClassification,
TFXLMForTokenClassification,
TFXLMModel,
TFXLMWithLMHeadModel,
TFXLMForSequenceClassification,
TFXLMForQuestionAnsweringSimple,
TFXLMForTokenClassification,
TFXLMForMultipleChoice,
TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
XLMConfig,
)

3
tests/test_modeling_tf_xlm_roberta.py
View File

@@ -20,8 +20,9 @@ from transformers.testing_utils import require_tf, slow
if is_tf_available():
import tensorflow as tf
import numpy as np
import tensorflow as tf
from transformers import TFXLMRobertaModel

10
tests/test_modeling_tf_xlnet.py
View File

@@ -28,13 +28,13 @@ if is_tf_available():
import tensorflow as tf
from transformers.modeling_tf_xlnet import (
TFXLNetModel,
TFXLNetLMHeadModel,
TF_XLNET_PRETRAINED_MODEL_ARCHIVE_LIST,
TFXLNetForMultipleChoice,
TFXLNetForQuestionAnsweringSimple,
TFXLNetForSequenceClassification,
TFXLNetForTokenClassification,
TFXLNetForQuestionAnsweringSimple,
TFXLNetForMultipleChoice,
TF_XLNET_PRETRAINED_MODEL_ARCHIVE_LIST,
TFXLNetLMHeadModel,
TFXLNetModel,
)

3
tests/test_modeling_transfo_xl.py
View File

@@ -25,7 +25,8 @@ from .test_modeling_common import ModelTesterMixin, ids_tensor
if is_torch_available():
import torch
from transformers import TransfoXLConfig, TransfoXLModel, TransfoXLLMHeadModel
from transformers import TransfoXLConfig, TransfoXLLMHeadModel, TransfoXLModel
from transformers.modeling_transfo_xl import TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_LIST

11
tests/test_modeling_xlm.py
View File

@@ -25,15 +25,16 @@ from .test_modeling_common import ModelTesterMixin, ids_tensor, random_attention
if is_torch_available():
import torch
from transformers import (
XLMConfig,
XLMForMultipleChoice,
XLMForQuestionAnswering,
XLMForQuestionAnsweringSimple,
XLMForSequenceClassification,
XLMForTokenClassification,
XLMModel,
XLMWithLMHeadModel,
XLMForTokenClassification,
XLMForQuestionAnswering,
XLMForSequenceClassification,
XLMForQuestionAnsweringSimple,
XLMForMultipleChoice,
)
from transformers.modeling_xlm import XLM_PRETRAINED_MODEL_ARCHIVE_LIST

1
tests/test_modeling_xlm_roberta.py
View File

@@ -22,6 +22,7 @@ from transformers.testing_utils import slow
if is_torch_available():
import torch
from transformers import XLMRobertaModel

8
tests/test_modeling_xlnet.py
View File

@@ -29,13 +29,13 @@ if is_torch_available():
from transformers import (
XLNetConfig,
XLNetModel,
XLNetLMHeadModel,
XLNetForMultipleChoice,
XLNetForSequenceClassification,
XLNetForTokenClassification,
XLNetForQuestionAnswering,
XLNetForQuestionAnsweringSimple,
XLNetForSequenceClassification,
XLNetForTokenClassification,
XLNetLMHeadModel,
XLNetModel,
)
from transformers.modeling_xlnet import XLNET_PRETRAINED_MODEL_ARCHIVE_LIST

3
tests/test_optimization_tf.py
View File

@@ -8,7 +8,8 @@ if is_tf_available():
import tensorflow as tf
from tensorflow.python.eager import context
from tensorflow.python.framework import ops
from transformers import create_optimizer, GradientAccumulator
from transformers import GradientAccumulator, create_optimizer
@require_tf

7
tests/test_tokenization_common.py
View File

@@ -28,11 +28,7 @@ from transformers.tokenization_utils import AddedToken
if TYPE_CHECKING:
from transformers import (
PretrainedConfig,
PreTrainedModel,
TFPreTrainedModel,
)
from transformers import PretrainedConfig, PreTrainedModel, TFPreTrainedModel
def merge_model_tokenizer_mappings(
@@ -1398,6 +1394,7 @@ class TokenizerTesterMixin:
@require_torch
def test_torch_encode_plus_sent_to_model(self):
import torch
from transformers import MODEL_MAPPING, TOKENIZER_MAPPING
MODEL_TOKENIZER_MAPPING = merge_model_tokenizer_mappings(MODEL_MAPPING, TOKENIZER_MAPPING)

3
tests/test_tokenization_reformer.py
View File

@@ -232,7 +232,8 @@ class ReformerTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
@require_torch
def test_torch_encode_plus_sent_to_model(self):
import torch
from transformers import ReformerModel, ReformerConfig
from transformers import ReformerConfig, ReformerModel
# Build sequence
first_ten_tokens = list(self.big_tokenizer.get_vocab().keys())[:10]

2
tests/test_tokenization_transfo_xl.py
View File

@@ -24,7 +24,7 @@ from .test_tokenization_common import TokenizerTesterMixin
if is_torch_available():
from transformers.tokenization_transfo_xl import TransfoXLTokenizer, VOCAB_FILES_NAMES
from transformers.tokenization_transfo_xl import VOCAB_FILES_NAMES, TransfoXLTokenizer
@require_torch