SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Fix: Fixed directory path for utils folder in test_tokenization_utils.py (#32601)

Browse Source 
* Removed un-necessary expressions.

* Fixed directory path for utils folder in test_tokenization_utils.py
This commit is contained in:
Sai-Suraj-27
2024-08-13 21:18:15 +05:30
committed by GitHub
parent cc25757a44
commit c3cd9d807e
2 changed files with 1 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
tests/utils/test_tokenization_utils.py
View File

@@ -36,7 +36,7 @@ from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tok
from transformers.tokenization_utils import ExtensionsTrie, Trie
sys.path.append(str(Path(__file__).parent.parent / "utils"))
sys.path.append(str(Path(__file__).parent.parent.parent / "utils"))
from test_module.custom_tokenization import CustomTokenizer # noqa E402
@@ -253,7 +253,6 @@ class TrieTest(unittest.TestCase):
trie.add("Hello 友達")
self.assertEqual(trie.data, {"H": {"e": {"l": {"l": {"o": {" ": {"友": {"達": {"": 1}}}}}}}}})
trie.add("Hello")
trie.data
self.assertEqual(trie.data, {"H": {"e": {"l": {"l": {"o": {"": 1, " ": {"友": {"達": {"": 1}}}}}}}}})
def test_trie_split(self):