[RoFormer] Fix some issues (#12397)

* add RoFormerTokenizerFast into AutoTokenizer

* fix typo in roformer docs

* make onnx export happy

* update RoFormerConfig embedding_size

* use jieba not rjieba

* fix 12244 and make test_alignement passed

* update ARCHIVE_MAP

* make style & quality & fixup

* update

* make style & quality & fixup

* make style quality fixup

* update

* suggestion from LysandreJik

Co-authored-by: Lysandre Debut <lysandre@huggingface.co>

* make style

* use rjieba

Co-authored-by: Lysandre Debut <lysandre@huggingface.co>
This commit is contained in:
yujun
2021-07-06 15:31:57 +08:00
committed by GitHub
parent f5b0c1ecf0
commit 626a0a0147
11 changed files with 97 additions and 53 deletions

View File

@@ -35,6 +35,7 @@ from .file_utils import (
is_flax_available,
is_onnx_available,
is_pandas_available,
is_rjieba_available,
is_scatter_available,
is_sentencepiece_available,
is_soundfile_availble,
@@ -223,6 +224,16 @@ def require_git_lfs(test_case):
return test_case
def require_rjieba(test_case):
"""
Decorator marking a test that requires rjieba. These tests are skipped when rjieba isn't installed.
"""
if not is_rjieba_available():
return unittest.skip("test requires rjieba")(test_case)
else:
return test_case
def require_onnx(test_case):
if not is_onnx_available():
return unittest.skip("test requires ONNX")(test_case)