From 306af132d7059a8528345a2aab4f64215c87723b Mon Sep 17 00:00:00 2001 From: thomwolf Date: Fri, 30 Aug 2019 11:30:51 +0200 Subject: [PATCH] update readme to mention add_special_tokens more clearly in example --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4e57de5842..dc51629b4a 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ for model_class, tokenizer_class, pretrained_weights in MODELS: model = model_class.from_pretrained(pretrained_weights) # Encode text - input_ids = torch.tensor([tokenizer.encode("Here is some text to encode")]) + input_ids = torch.tensor([tokenizer.encode("Here is some text to encode", add_special_tokens=True)]) # Add special tokens takes care of adding [CLS], [SEP], ... tokens in the right way for each model. with torch.no_grad(): last_hidden_states = model(input_ids)[0] # Models outputs are now tuples