SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

[pegasus] Faster tokenizer tests (#7672)

Browse Source
This commit is contained in:
Stas Bekman
2020-10-09 08:10:32 -07:00
committed by GitHub
parent bc00b37a0d
commit b0f05e0c4c
8 changed files with 51 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
tests/test_tokenization_t5.py
View File

@@ -14,19 +14,18 @@
# limitations under the License.
import os
import unittest
from transformers import BatchEncoding
from transformers.file_utils import cached_property
from transformers.testing_utils import _torch_available
from transformers.testing_utils import _torch_available, get_tests_dir
from transformers.tokenization_t5 import T5Tokenizer, T5TokenizerFast
from transformers.tokenization_xlnet import SPIECE_UNDERLINE
from .test_tokenization_common import TokenizerTesterMixin
SAMPLE_VOCAB = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures/test_sentencepiece.model")
SAMPLE_VOCAB = get_tests_dir("fixtures/test_sentencepiece.model")
FRAMEWORK = "pt" if _torch_available else "tf"