SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Replace CommonTestCases for tokenizers with a mixin.

Browse Source 
This is the same change as for (TF)CommonTestCases for modeling.
This commit is contained in:
Aymeric Augustin
2019-12-22 15:34:15 +01:00
parent a3c5883f2c
commit 00204f2b4c
16 changed files with 412 additions and 411 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
tests/test_tokenization_bert_japanese.py
View File

@@ -15,6 +15,7 @@
from __future__ import absolute_import, division, print_function, unicode_literals
import os
import unittest
from io import open
from transformers.tokenization_bert import WordpieceTokenizer
@@ -25,12 +26,12 @@ from transformers.tokenization_bert_japanese import (
MecabTokenizer,
)
from .test_tokenization_commo import CommonTestCases
from .test_tokenization_common import TokenizerTesterMixin
from .utils import custom_tokenizers, slow
@custom_tokenizers
class BertJapaneseTokenizationTest(CommonTestCases.CommonTokenizerTester):
class BertJapaneseTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
tokenizer_class = BertJapaneseTokenizer
@@ -130,7 +131,7 @@ class BertJapaneseTokenizationTest(CommonTestCases.CommonTokenizerTester):
assert encoded_pair == [2] + text + [3] + text_2 + [3]
class BertJapaneseCharacterTokenizationTest(CommonTestCases.CommonTokenizerTester):
class BertJapaneseCharacterTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
tokenizer_class = BertJapaneseTokenizer