Support additional dictionaries for BERT Japanese tokenizers (#6515)
* Update BERT Japanese tokenizers * Update CircleCI config to download unidic * Specify to use the latest dictionary packages
This commit is contained in:
2
setup.py
2
setup.py
@@ -65,7 +65,7 @@ if stale_egg_info.exists():
|
||||
|
||||
extras = {}
|
||||
|
||||
extras["ja"] = ["fugashi>=1.0", "ipadic>=1.0,<2.0"]
|
||||
extras["ja"] = ["fugashi>=1.0", "ipadic>=1.0.0,<2.0", "unidic_lite>=1.0.7", "unidic>=1.0.2"]
|
||||
extras["sklearn"] = ["scikit-learn"]
|
||||
|
||||
# keras2onnx and onnxconverter-common version is specific through a commit until 1.7.0 lands on pypi
|
||||
|
||||
Reference in New Issue
Block a user