Convert tokenizer outputs for Keras in doc example (#20732)

* Convert tokenizer outputs for Keras in doc example * Das deutsche Beispiel auch korrigieren
2022-12-12 16:14:04 +00:00
parent 0ba94aceb6
commit c1b9a11dd4
2 changed files with 4 additions and 0 deletions
--- a/docs/source/en/training.mdx
+++ b/docs/source/en/training.mdx
@@ -185,6 +185,8 @@ from transformers import AutoTokenizer

 tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
 tokenized_data = tokenizer(dataset["text"], return_tensors="np", padding=True)
+# Tokenizer returns a BatchEncoding, but we convert that to a dict for Keras
+tokenized_data = dict(tokenized_data)

 labels = np.array(dataset["label"])  # Label is already an array of 0 and 1
 ```