Improve documentation coverage for Herbert (#9428)

* first commit * changed XLMTokenizer to HerbertTokenizer in code example
2021-01-06 22:13:43 +08:00
parent b972c1bfb0
commit be898998bb
3 changed files with 72 additions and 3 deletions
--- a/utils/check_repo.py
+++ b/utils/check_repo.py
@@ -402,9 +402,6 @@ SHOULD_HAVE_THEIR_OWN_PAGE = [
    "BertJapaneseTokenizer",
    "CharacterTokenizer",
    "MecabTokenizer",
-    # Herbert
-    "HerbertTokenizer",
-    "HerbertTokenizerFast",
    # Phoebus
    "PhobertTokenizer",
    # Benchmarks