SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Fast transformers import part 1 (#9441)

Browse Source 
* Don't import libs to check they are available

* Don't import integrations at init

* Add importlib_metdata to deps

* Remove old vars references

* Avoid syntax error

* Adapt testing utils

* Try to appease torchhub

* Add dependency

* Remove more private variables

* Fix typo

* Another typo

* Refine the tf availability test
This commit is contained in:
Sylvain Gugger
2021-01-06 12:17:24 -05:00
committed by GitHub
parent c89f1bc92e
commit 0c96262f7d
13 changed files with 280 additions and 360 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
tests/test_tokenization_camembert.py
View File

@@ -18,14 +18,15 @@ import os
import unittest
from transformers import CamembertTokenizer, CamembertTokenizerFast
from transformers.testing_utils import _torch_available, require_sentencepiece, require_tokenizers
from transformers.file_utils import is_torch_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers
from .test_tokenization_common import TokenizerTesterMixin
SAMPLE_VOCAB = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures/test_sentencepiece.model")
FRAMEWORK = "pt" if _torch_available else "tf"
FRAMEWORK = "pt" if is_torch_available() else "tf"
@require_sentencepiece

7
tests/test_tokenization_marian.py
View File

@@ -21,10 +21,11 @@ from pathlib import Path
from shutil import copyfile
from transformers import BatchEncoding, MarianTokenizer
from transformers.testing_utils import _sentencepiece_available, _torch_available, require_sentencepiece
from transformers.file_utils import is_sentencepiece_available, is_torch_available
from transformers.testing_utils import require_sentencepiece
if _sentencepiece_available:
if is_sentencepiece_available():
from transformers.models.marian.tokenization_marian import save_json, vocab_files_names
from .test_tokenization_common import TokenizerTesterMixin
@@ -35,7 +36,7 @@ SAMPLE_SP = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures/t
mock_tokenizer_config = {"target_lang": "fi", "source_lang": "en"}
zh_code = ">>zh<<"
ORG_NAME = "Helsinki-NLP/"
FRAMEWORK = "pt" if _torch_available else "tf"
FRAMEWORK = "pt" if is_torch_available() else "tf"
@require_sentencepiece

10
tests/test_tokenization_mbart.py
View File

@@ -16,17 +16,13 @@ import tempfile
import unittest
from transformers import SPIECE_UNDERLINE, BatchEncoding, MBartTokenizer, MBartTokenizerFast, is_torch_available
from transformers.testing_utils import (
_sentencepiece_available,
require_sentencepiece,
require_tokenizers,
require_torch,
)
from transformers.file_utils import is_sentencepiece_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch
from .test_tokenization_common import TokenizerTesterMixin
if _sentencepiece_available:
if is_sentencepiece_available():
from .test_tokenization_xlm_roberta import SAMPLE_VOCAB

6
tests/test_tokenization_t5.py
View File

@@ -17,15 +17,15 @@
import unittest
from transformers import SPIECE_UNDERLINE, BatchEncoding, T5Tokenizer, T5TokenizerFast
from transformers.file_utils import cached_property
from transformers.testing_utils import _torch_available, get_tests_dir, require_sentencepiece, require_tokenizers
from transformers.file_utils import cached_property, is_torch_available
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers
from .test_tokenization_common import TokenizerTesterMixin
SAMPLE_VOCAB = get_tests_dir("fixtures/test_sentencepiece.model")
FRAMEWORK = "pt" if _torch_available else "tf"
FRAMEWORK = "pt" if is_torch_available() else "tf"
@require_sentencepiece