Make OpenAIGPTTokenizer work with SpaCy 2.x and 3.x (#15019)
* Make OpenAIGPTTokenizer work with SpaCy 3.x SpaCy 3.x introduced an API change to creating the tokenizer that breaks OpenAIGPTTokenizer. The old API for creating the tokenizer in SpaCy 2.x no longer works under SpaCy 3.x, but the new API for creating the tokenizer in SpaCy 3.x DOES work under SpaCy 2.x. Switching to the new API should allow OpenAIGPTTokenizer to work under both SpaCy 2.x and SpaCy 3.x versions. * Add is_spacy_available and is_ftfy_available methods to file utils * Add spacy and ftfy unittest decorator to testing utils * Add tests for OpenAIGPTTokenizer that require spacy and ftfy * Modify CircleCI config to run tests that require spacy and ftfy * Remove unneeded unittest decorators are reuse test code * Run make fixup
This commit is contained in:
@@ -34,6 +34,7 @@ from .file_utils import (
|
||||
is_detectron2_available,
|
||||
is_faiss_available,
|
||||
is_flax_available,
|
||||
is_ftfy_available,
|
||||
is_keras2onnx_available,
|
||||
is_librosa_available,
|
||||
is_onnx_available,
|
||||
@@ -46,6 +47,7 @@ from .file_utils import (
|
||||
is_scatter_available,
|
||||
is_sentencepiece_available,
|
||||
is_soundfile_availble,
|
||||
is_spacy_available,
|
||||
is_tensorflow_probability_available,
|
||||
is_tf_available,
|
||||
is_timm_available,
|
||||
@@ -412,6 +414,26 @@ def require_vision(test_case):
|
||||
return test_case
|
||||
|
||||
|
||||
def require_ftfy(test_case):
|
||||
"""
|
||||
Decorator marking a test that requires ftfy. These tests are skipped when ftfy isn't installed.
|
||||
"""
|
||||
if not is_ftfy_available():
|
||||
return unittest.skip("test requires ftfy")(test_case)
|
||||
else:
|
||||
return test_case
|
||||
|
||||
|
||||
def require_spacy(test_case):
|
||||
"""
|
||||
Decorator marking a test that requires SpaCy. These tests are skipped when SpaCy isn't installed.
|
||||
"""
|
||||
if not is_spacy_available():
|
||||
return unittest.skip("test requires spacy")(test_case)
|
||||
else:
|
||||
return test_case
|
||||
|
||||
|
||||
def require_torch_multi_gpu(test_case):
|
||||
"""
|
||||
Decorator marking a test that requires a multi-GPU setup (in PyTorch). These tests are skipped on a machine without
|
||||
|
||||
Reference in New Issue
Block a user