SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

update readme to mention add_special_tokens more clearly in example

Browse Source
This commit is contained in:
thomwolf
2019-08-30 11:30:51 +02:00
parent 50e6daf83a
commit 306af132d7
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@@ -93,7 +93,7 @@ for model_class, tokenizer_class, pretrained_weights in MODELS:
model = model_class.from_pretrained(pretrained_weights) model = model_class.from_pretrained(pretrained_weights)
# Encode text # Encode text
input_ids = torch.tensor([tokenizer.encode("Here is some text to encode")]) input_ids = torch.tensor([tokenizer.encode("Here is some text to encode", add_special_tokens=True)]) # Add special tokens takes care of adding [CLS], [SEP], <s>... tokens in the right way for each model.
with torch.no_grad(): with torch.no_grad():
last_hidden_states = model(input_ids)[0] # Models outputs are now tuples last_hidden_states = model(input_ids)[0] # Models outputs are now tuples