SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

fix: use correct var names for check_tokenizers script (#33702)

Browse Source
This commit is contained in:
Nicola De Angeli
2024-09-26 17:24:46 +02:00
committed by GitHub
parent 46841d3eb2
commit c716fc0e48
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
scripts/check_tokenizers.py
View File

@@ -88,7 +88,7 @@ def check_details(line, spm_ids, tok_ids, slow, fast):
if tok_ids[first + k : first + k + min_width] == spm_ids[first + i : first + i + min_width]
]
for j in possible_matches:
if check_diff(spm_ids[first : first + i], tok_ids[first : first + j], sp, tok) and check_details(
if check_diff(spm_ids[first : first + i], tok_ids[first : first + j], slow, fast) and check_details(
line,
spm_ids[first + i : last],
tok_ids[first + j : last],