Fix past CI (#20967)
* Fix for Past CI * make style * clean up * unindent 2 blocks Co-authored-by: ydshieh <ydshieh@users.noreply.github.com>
This commit is contained in:
@@ -4,15 +4,15 @@ from tempfile import TemporaryDirectory
|
||||
|
||||
from transformers import AutoConfig, TFAutoModel, is_tensorflow_text_available, is_tf_available
|
||||
from transformers.models.bert.tokenization_bert import BertTokenizer
|
||||
from transformers.testing_utils import require_tensorflow_text, slow
|
||||
from transformers.testing_utils import require_tensorflow_text, require_tf, slow
|
||||
|
||||
|
||||
if is_tensorflow_text_available():
|
||||
from transformers.models.bert import TFBertTokenizer
|
||||
|
||||
if is_tf_available():
|
||||
import tensorflow as tf
|
||||
|
||||
if is_tensorflow_text_available():
|
||||
from transformers.models.bert import TFBertTokenizer
|
||||
|
||||
|
||||
TOKENIZER_CHECKPOINTS = ["bert-base-uncased", "bert-base-cased"]
|
||||
TINY_MODEL_CHECKPOINT = "hf-internal-testing/tiny-bert-tf-only"
|
||||
@@ -32,6 +32,7 @@ if is_tf_available():
|
||||
return out["pooler_output"]
|
||||
|
||||
|
||||
@require_tf
|
||||
@require_tensorflow_text
|
||||
class BertTokenizationTest(unittest.TestCase):
|
||||
# The TF tokenizers are usually going to be used as pretrained tokenizers from existing model checkpoints,
|
||||
|
||||
@@ -4,15 +4,15 @@ from tempfile import TemporaryDirectory
|
||||
|
||||
from transformers import AutoConfig, TFGPT2LMHeadModel, is_keras_nlp_available, is_tf_available
|
||||
from transformers.models.gpt2.tokenization_gpt2 import GPT2Tokenizer
|
||||
from transformers.testing_utils import require_keras_nlp, slow
|
||||
from transformers.testing_utils import require_keras_nlp, require_tf, slow
|
||||
|
||||
|
||||
if is_keras_nlp_available():
|
||||
from transformers.models.gpt2 import TFGPT2Tokenizer
|
||||
|
||||
if is_tf_available():
|
||||
import tensorflow as tf
|
||||
|
||||
if is_keras_nlp_available():
|
||||
from transformers.models.gpt2 import TFGPT2Tokenizer
|
||||
|
||||
|
||||
TOKENIZER_CHECKPOINTS = ["gpt2"]
|
||||
TINY_MODEL_CHECKPOINT = "gpt2"
|
||||
@@ -40,6 +40,7 @@ if is_tf_available():
|
||||
return outputs
|
||||
|
||||
|
||||
@require_tf
|
||||
@require_keras_nlp
|
||||
class GPTTokenizationTest(unittest.TestCase):
|
||||
# The TF tokenizers are usually going to be used as pretrained tokenizers from existing model checkpoints,
|
||||
|
||||
@@ -38,7 +38,14 @@ from transformers.models.layoutlmv2.tokenization_layoutlmv2 import (
|
||||
_is_punctuation,
|
||||
_is_whitespace,
|
||||
)
|
||||
from transformers.testing_utils import is_pt_tf_cross_test, require_pandas, require_tokenizers, require_torch, slow
|
||||
from transformers.testing_utils import (
|
||||
is_pt_tf_cross_test,
|
||||
require_detectron2,
|
||||
require_pandas,
|
||||
require_tokenizers,
|
||||
require_torch,
|
||||
slow,
|
||||
)
|
||||
|
||||
from ...test_tokenization_common import (
|
||||
SMALL_TRAINING_CORPUS,
|
||||
@@ -1264,6 +1271,7 @@ class LayoutLMv2TokenizationTest(TokenizerTesterMixin, unittest.TestCase):
|
||||
self.assertEqual(sum(tokens_with_offsets["special_tokens_mask"]), added_tokens)
|
||||
|
||||
@require_torch
|
||||
@require_detectron2
|
||||
@slow
|
||||
def test_torch_encode_plus_sent_to_model(self):
|
||||
import torch
|
||||
|
||||
Reference in New Issue
Block a user