SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

tiny tweak to allow BatchEncoding.token_to_char when token doesn't correspond to chars (#15901)

Browse Source 
* tweak to allow BatchEncoding.char_to_token(0)

* update docstring

* remote trailing whitespace

* make fixup

* make value checking for span_indices explicit

Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com>

Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com>
This commit is contained in:
ghlai9665
2022-04-21 07:07:54 -05:00
committed by GitHub
parent cb7e166428
commit daf520b033
1 changed files with 5 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
src/transformers/tokenization_utils_base.py
View File

@@ -503,7 +503,8 @@ class BatchEncoding(UserDict):
the sequence.
Returns:
[`~tokenization_utils_base.CharSpan`]: Span of characters in the original string.
[`~tokenization_utils_base.CharSpan`]: Span of characters in the original string, or None, if the token
(e.g. <s>, </s>) doesn't correspond to any chars in the origin string.
"""
if not self._encodings:
@@ -513,7 +514,9 @@ class BatchEncoding(UserDict):
else:
batch_index = 0
token_index = batch_or_token_index
return CharSpan(*(self._encodings[batch_index].token_to_chars(token_index)))
span_indices = self._encodings[batch_index].token_to_chars(token_index)
return CharSpan(*span_indices) if span_indices is not None else None
def char_to_token(
self, batch_or_char_index: int, char_index: Optional[int] = None, sequence_index: int = 0