SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Make Barthez tokenizer tests a bit faster (#10399)

Browse Source 
* Make Barthez tokenizer tests a bit faster

* Quality
This commit is contained in:
Sylvain Gugger
2021-02-25 11:42:25 -05:00
committed by GitHub
parent b040e6efc1
commit 26f8b2cb10
4 changed files with 6 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
tests/test_tokenization_mbart.py
View File

@@ -12,18 +12,17 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import tempfile
import unittest
from transformers import SPIECE_UNDERLINE, BatchEncoding, MBartTokenizer, MBartTokenizerFast, is_torch_available
from transformers.file_utils import is_sentencepiece_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch
from .test_tokenization_common import TokenizerTesterMixin
if is_sentencepiece_available():
from .test_tokenization_xlm_roberta import SAMPLE_VOCAB
SAMPLE_VOCAB = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures/test_sentencepiece.model")
if is_torch_available():