From 3ec8171bedff6139a23dff192b9e8af33c1fca9a Mon Sep 17 00:00:00 2001 From: Ceyda Cinarel <15624271+cceyda@users.noreply.github.com> Date: Thu, 9 Mar 2023 06:21:46 +0900 Subject: [PATCH] Bug fix: token classification pipeline while passing offset_mapping (#22034) fix slow tokenizers with passing offset_mapping --- src/transformers/pipelines/token_classification.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/transformers/pipelines/token_classification.py b/src/transformers/pipelines/token_classification.py index 5dc243d1ac..f3c78d3498 100644 --- a/src/transformers/pipelines/token_classification.py +++ b/src/transformers/pipelines/token_classification.py @@ -304,7 +304,9 @@ class TokenClassificationPipeline(Pipeline): start_ind = start_ind.item() end_ind = end_ind.item() word_ref = sentence[start_ind:end_ind] - if getattr(self.tokenizer._tokenizer.model, "continuing_subword_prefix", None): + if getattr(self.tokenizer, "_tokenizer", None) and getattr( + self.tokenizer._tokenizer.model, "continuing_subword_prefix", None + ): # This is a BPE, word aware tokenizer, there is a correct way # to fuse tokens is_subword = len(word) != len(word_ref)