SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Make Barthez tokenizer tests a bit faster (#10399)

Browse Source 
* Make Barthez tokenizer tests a bit faster

* Quality
This commit is contained in:
Sylvain Gugger
2021-02-25 11:42:25 -05:00
committed by GitHub
parent b040e6efc1
commit 26f8b2cb10
4 changed files with 6 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
tests/test_tokenization_mbart50.py
View File

@@ -17,14 +17,12 @@ import tempfile
import unittest
from transformers import SPIECE_UNDERLINE, BatchEncoding, MBart50Tokenizer, MBart50TokenizerFast, is_torch_available
from transformers.file_utils import is_sentencepiece_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch
from .test_tokenization_common import TokenizerTesterMixin
if is_sentencepiece_available():
SAMPLE_VOCAB = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures/test_sentencepiece.model")
SAMPLE_VOCAB = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures/test_sentencepiece.model")
if is_torch_available():