From 9535bf1977b33ebaf34363cf4d0b305157194015 Mon Sep 17 00:00:00 2001
From: Sam Shleifer <sshleifer@gmail.com>
Date: Thu, 14 May 2020 13:50:47 -0400
Subject: [PATCH] Tokenizer.batch_decode convenience method (#4159)

---
 src/transformers/tokenization_marian.py | 3 ---
 src/transformers/tokenization_utils.py  | 3 +++
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/transformers/tokenization_marian.py b/src/transformers/tokenization_marian.py
index bd56382cb7..cb2dab5248 100644
--- a/src/transformers/tokenization_marian.py
+++ b/src/transformers/tokenization_marian.py
@@ -124,9 +124,6 @@ class MarianTokenizer(PreTrainedTokenizer):
         # We don't expect to process pairs, but leave the pair logic for API consistency
         return token_ids_0 + token_ids_1 + [self.eos_token_id]
 
-    def batch_decode(self, token_ids, **kwargs) -> List[str]:
-        return [self.decode(ids, **kwargs) for ids in token_ids]
-
     def prepare_translation_batch(
         self,
         src_texts: List[str],
diff --git a/src/transformers/tokenization_utils.py b/src/transformers/tokenization_utils.py
index b7f8e755e8..5eba847bd0 100644
--- a/src/transformers/tokenization_utils.py
+++ b/src/transformers/tokenization_utils.py
@@ -2183,6 +2183,9 @@ class PreTrainedTokenizer(SpecialTokensMixin):
         else:
             return text
 
+    def batch_decode(self, sequences: List[List[int]], **kwargs) -> List[str]:
+        return [self.decode(seq, **kwargs) for seq in sequences]
+
     @staticmethod
     def clean_up_tokenization(out_string: str) -> str:
         """ Clean up a list of simple English tokenization artifacts like spaces before punctuations and abreviated forms.