Replace CommonTestCases for tokenizers with a mixin.

This is the same change as for (TF)CommonTestCases for modeling.
This commit is contained in:
Aymeric Augustin
2019-12-22 15:34:15 +01:00
parent a3c5883f2c
commit 00204f2b4c
16 changed files with 412 additions and 411 deletions

View File

@@ -15,11 +15,12 @@
from __future__ import absolute_import, division, print_function, unicode_literals
import os
import unittest
from io import open
from transformers import is_torch_available
from .test_tokenization_commo import CommonTestCases
from .test_tokenization_common import TokenizerTesterMixin
from .utils import require_torch
@@ -28,7 +29,7 @@ if is_torch_available():
@require_torch
class TransfoXLTokenizationTest(CommonTestCases.CommonTokenizerTester):
class TransfoXLTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
tokenizer_class = TransfoXLTokenizer if is_torch_available() else None