cleaning up example docstrings
This commit is contained in:
thomwolf
@@ -84,12 +84,12 @@ def bertTokenizer(*args, **kwargs):
|
||||
Default: ["[UNK]", "[SEP]", "[PAD]", "[CLS]", "[MASK]"]
|
||||
|
||||
Example:
|
||||
>>> import torch
|
||||
>>> sentence = 'Hello, World!'
|
||||
>>> tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'bertTokenizer', 'bert-base-cased', do_basic_tokenize=False)
|
||||
>>> toks = tokenizer.tokenize(sentence)
|
||||
import torch
|
||||
sentence = 'Hello, World!'
|
||||
tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'bertTokenizer', 'bert-base-cased', do_basic_tokenize=False)
|
||||
toks = tokenizer.tokenize(sentence)
|
||||
['Hello', '##,', 'World', '##!']
|
||||
>>> ids = tokenizer.convert_tokens_to_ids(toks)
|
||||
ids = tokenizer.convert_tokens_to_ids(toks)
|
||||
[8667, 28136, 1291, 28125]
|
||||
"""
|
||||
tokenizer = BertTokenizer.from_pretrained(*args, **kwargs)
|
||||
@@ -105,20 +105,20 @@ def bertModel(*args, **kwargs):
|
||||
|
||||
Example:
|
||||
# Load the tokenizer
|
||||
>>> import torch
|
||||
>>> tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'bertTokenizer', 'bert-base-cased', do_basic_tokenize=False)
|
||||
import torch
|
||||
tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'bertTokenizer', 'bert-base-cased', do_basic_tokenize=False)
|
||||
# Prepare tokenized input
|
||||
>>> text = "[CLS] Who was Jim Henson ? [SEP] Jim Henson was a puppeteer [SEP]"
|
||||
>>> tokenized_text = tokenizer.tokenize(text)
|
||||
>>> indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text)
|
||||
>>> segments_ids = [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1]
|
||||
>>> tokens_tensor = torch.tensor([indexed_tokens])
|
||||
>>> segments_tensors = torch.tensor([segments_ids])
|
||||
text = "[CLS] Who was Jim Henson ? [SEP] Jim Henson was a puppeteer [SEP]"
|
||||
tokenized_text = tokenizer.tokenize(text)
|
||||
indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text)
|
||||
segments_ids = [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1]
|
||||
tokens_tensor = torch.tensor([indexed_tokens])
|
||||
segments_tensors = torch.tensor([segments_ids])
|
||||
# Load bertModel
|
||||
>>> model = torch.hub.load('huggingface/pytorch-transformers', 'bertModel', 'bert-base-cased')
|
||||
>>> model.eval()
|
||||
model = torch.hub.load('huggingface/pytorch-transformers', 'bertModel', 'bert-base-cased')
|
||||
model.eval()
|
||||
# Predict hidden states features for each layer
|
||||
>>> with torch.no_grad():
|
||||
with torch.no_grad():
|
||||
encoded_layers, _ = model(tokens_tensor, segments_tensors)
|
||||
"""
|
||||
model = BertModel.from_pretrained(*args, **kwargs)
|
||||
@@ -134,20 +134,20 @@ def bertForNextSentencePrediction(*args, **kwargs):
|
||||
|
||||
Example:
|
||||
# Load the tokenizer
|
||||
>>> import torch
|
||||
>>> tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'bertTokenizer', 'bert-base-cased', do_basic_tokenize=False)
|
||||
import torch
|
||||
tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'bertTokenizer', 'bert-base-cased', do_basic_tokenize=False)
|
||||
# Prepare tokenized input
|
||||
>>> text = "[CLS] Who was Jim Henson ? [SEP] Jim Henson was a puppeteer [SEP]"
|
||||
>>> tokenized_text = tokenizer.tokenize(text)
|
||||
>>> indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text)
|
||||
>>> segments_ids = [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1]
|
||||
>>> tokens_tensor = torch.tensor([indexed_tokens])
|
||||
>>> segments_tensors = torch.tensor([segments_ids])
|
||||
text = "[CLS] Who was Jim Henson ? [SEP] Jim Henson was a puppeteer [SEP]"
|
||||
tokenized_text = tokenizer.tokenize(text)
|
||||
indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text)
|
||||
segments_ids = [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1]
|
||||
tokens_tensor = torch.tensor([indexed_tokens])
|
||||
segments_tensors = torch.tensor([segments_ids])
|
||||
# Load bertForNextSentencePrediction
|
||||
>>> model = torch.hub.load('huggingface/pytorch-transformers', 'bertForNextSentencePrediction', 'bert-base-cased')
|
||||
>>> model.eval()
|
||||
model = torch.hub.load('huggingface/pytorch-transformers', 'bertForNextSentencePrediction', 'bert-base-cased')
|
||||
model.eval()
|
||||
# Predict the next sentence classification logits
|
||||
>>> with torch.no_grad():
|
||||
with torch.no_grad():
|
||||
next_sent_classif_logits = model(tokens_tensor, segments_tensors)
|
||||
"""
|
||||
model = BertForNextSentencePrediction.from_pretrained(*args, **kwargs)
|
||||
@@ -164,17 +164,17 @@ def bertForPreTraining(*args, **kwargs):
|
||||
|
||||
Example:
|
||||
# Load the tokenizer
|
||||
>>> import torch
|
||||
>>> tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'bertTokenizer', 'bert-base-cased', do_basic_tokenize=False)
|
||||
import torch
|
||||
tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'bertTokenizer', 'bert-base-cased', do_basic_tokenize=False)
|
||||
# Prepare tokenized input
|
||||
>>> text = "[CLS] Who was Jim Henson ? [SEP] Jim Henson was a puppeteer [SEP]"
|
||||
>>> tokenized_text = tokenizer.tokenize(text)
|
||||
>>> segments_ids = [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1]
|
||||
>>> tokens_tensor = torch.tensor([indexed_tokens])
|
||||
>>> segments_tensors = torch.tensor([segments_ids])
|
||||
text = "[CLS] Who was Jim Henson ? [SEP] Jim Henson was a puppeteer [SEP]"
|
||||
tokenized_text = tokenizer.tokenize(text)
|
||||
segments_ids = [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1]
|
||||
tokens_tensor = torch.tensor([indexed_tokens])
|
||||
segments_tensors = torch.tensor([segments_ids])
|
||||
# Load bertForPreTraining
|
||||
>>> model = torch.hub.load('huggingface/pytorch-transformers', 'bertForPreTraining', 'bert-base-cased')
|
||||
>>> masked_lm_logits_scores, seq_relationship_logits = model(tokens_tensor, segments_tensors)
|
||||
model = torch.hub.load('huggingface/pytorch-transformers', 'bertForPreTraining', 'bert-base-cased')
|
||||
masked_lm_logits_scores, seq_relationship_logits = model(tokens_tensor, segments_tensors)
|
||||
"""
|
||||
model = BertForPreTraining.from_pretrained(*args, **kwargs)
|
||||
return model
|
||||
@@ -188,25 +188,25 @@ def bertForMaskedLM(*args, **kwargs):
|
||||
|
||||
Example:
|
||||
# Load the tokenizer
|
||||
>>> import torch
|
||||
>>> tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'bertTokenizer', 'bert-base-cased', do_basic_tokenize=False)
|
||||
import torch
|
||||
tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'bertTokenizer', 'bert-base-cased', do_basic_tokenize=False)
|
||||
# Prepare tokenized input
|
||||
>>> text = "[CLS] Who was Jim Henson ? [SEP] Jim Henson was a puppeteer [SEP]"
|
||||
>>> tokenized_text = tokenizer.tokenize(text)
|
||||
>>> masked_index = 8
|
||||
>>> tokenized_text[masked_index] = '[MASK]'
|
||||
>>> indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text)
|
||||
>>> segments_ids = [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1]
|
||||
>>> tokens_tensor = torch.tensor([indexed_tokens])
|
||||
>>> segments_tensors = torch.tensor([segments_ids])
|
||||
text = "[CLS] Who was Jim Henson ? [SEP] Jim Henson was a puppeteer [SEP]"
|
||||
tokenized_text = tokenizer.tokenize(text)
|
||||
masked_index = 8
|
||||
tokenized_text[masked_index] = '[MASK]'
|
||||
indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text)
|
||||
segments_ids = [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1]
|
||||
tokens_tensor = torch.tensor([indexed_tokens])
|
||||
segments_tensors = torch.tensor([segments_ids])
|
||||
# Load bertForMaskedLM
|
||||
>>> model = torch.hub.load('huggingface/pytorch-transformers', 'bertForMaskedLM', 'bert-base-cased')
|
||||
>>> model.eval()
|
||||
model = torch.hub.load('huggingface/pytorch-transformers', 'bertForMaskedLM', 'bert-base-cased')
|
||||
model.eval()
|
||||
# Predict all tokens
|
||||
>>> with torch.no_grad():
|
||||
with torch.no_grad():
|
||||
predictions = model(tokens_tensor, segments_tensors)
|
||||
>>> predicted_index = torch.argmax(predictions[0, masked_index]).item()
|
||||
>>> predicted_token = tokenizer.convert_ids_to_tokens([predicted_index])[0]
|
||||
predicted_index = torch.argmax(predictions[0, masked_index]).item()
|
||||
predicted_token = tokenizer.convert_ids_to_tokens([predicted_index])[0]
|
||||
'henson'
|
||||
"""
|
||||
model = BertForMaskedLM.from_pretrained(*args, **kwargs)
|
||||
@@ -230,24 +230,24 @@ def bertForSequenceClassification(*args, **kwargs):
|
||||
|
||||
Example:
|
||||
# Load the tokenizer
|
||||
>>> import torch
|
||||
>>> tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'bertTokenizer', 'bert-base-cased', do_basic_tokenize=False)
|
||||
import torch
|
||||
tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'bertTokenizer', 'bert-base-cased', do_basic_tokenize=False)
|
||||
# Prepare tokenized input
|
||||
>>> text = "[CLS] Who was Jim Henson ? [SEP] Jim Henson was a puppeteer [SEP]"
|
||||
>>> tokenized_text = tokenizer.tokenize(text)
|
||||
>>> indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text)
|
||||
>>> segments_ids = [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1]
|
||||
>>> tokens_tensor = torch.tensor([indexed_tokens])
|
||||
>>> segments_tensors = torch.tensor([segments_ids])
|
||||
text = "[CLS] Who was Jim Henson ? [SEP] Jim Henson was a puppeteer [SEP]"
|
||||
tokenized_text = tokenizer.tokenize(text)
|
||||
indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text)
|
||||
segments_ids = [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1]
|
||||
tokens_tensor = torch.tensor([indexed_tokens])
|
||||
segments_tensors = torch.tensor([segments_ids])
|
||||
# Load bertForSequenceClassification
|
||||
>>> model = torch.hub.load('huggingface/pytorch-transformers', 'bertForSequenceClassification', 'bert-base-cased', num_labels=2)
|
||||
>>> model.eval()
|
||||
model = torch.hub.load('huggingface/pytorch-transformers', 'bertForSequenceClassification', 'bert-base-cased', num_labels=2)
|
||||
model.eval()
|
||||
# Predict the sequence classification logits
|
||||
>>> with torch.no_grad():
|
||||
with torch.no_grad():
|
||||
seq_classif_logits = model(tokens_tensor, segments_tensors)
|
||||
# Or get the sequence classification loss
|
||||
>>> labels = torch.tensor([1])
|
||||
>>> seq_classif_loss = model(tokens_tensor, segments_tensors, labels=labels) # set model.train() before if training this loss
|
||||
labels = torch.tensor([1])
|
||||
seq_classif_loss = model(tokens_tensor, segments_tensors, labels=labels) # set model.train() before if training this loss
|
||||
"""
|
||||
model = BertForSequenceClassification.from_pretrained(*args, **kwargs)
|
||||
return model
|
||||
@@ -265,24 +265,24 @@ def bertForMultipleChoice(*args, **kwargs):
|
||||
|
||||
Example:
|
||||
# Load the tokenizer
|
||||
>>> import torch
|
||||
>>> tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'bertTokenizer', 'bert-base-cased', do_basic_tokenize=False)
|
||||
import torch
|
||||
tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'bertTokenizer', 'bert-base-cased', do_basic_tokenize=False)
|
||||
# Prepare tokenized input
|
||||
>>> text = "[CLS] Who was Jim Henson ? [SEP] Jim Henson was a puppeteer [SEP]"
|
||||
>>> tokenized_text = tokenizer.tokenize(text)
|
||||
>>> indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text)
|
||||
>>> segments_ids = [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1]
|
||||
>>> tokens_tensor = torch.tensor([indexed_tokens, indexed_tokens]).unsqueeze(0)
|
||||
>>> segments_tensors = torch.tensor([segments_ids, segments_ids]).unsqueeze(0)
|
||||
text = "[CLS] Who was Jim Henson ? [SEP] Jim Henson was a puppeteer [SEP]"
|
||||
tokenized_text = tokenizer.tokenize(text)
|
||||
indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text)
|
||||
segments_ids = [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1]
|
||||
tokens_tensor = torch.tensor([indexed_tokens, indexed_tokens]).unsqueeze(0)
|
||||
segments_tensors = torch.tensor([segments_ids, segments_ids]).unsqueeze(0)
|
||||
# Load bertForMultipleChoice
|
||||
>>> model = torch.hub.load('huggingface/pytorch-transformers', 'bertForMultipleChoice', 'bert-base-cased', num_choices=2)
|
||||
>>> model.eval()
|
||||
model = torch.hub.load('huggingface/pytorch-transformers', 'bertForMultipleChoice', 'bert-base-cased', num_choices=2)
|
||||
model.eval()
|
||||
# Predict the multiple choice logits
|
||||
>>> with torch.no_grad():
|
||||
with torch.no_grad():
|
||||
multiple_choice_logits = model(tokens_tensor, segments_tensors)
|
||||
# Or get the multiple choice loss
|
||||
>>> labels = torch.tensor([1])
|
||||
>>> multiple_choice_loss = model(tokens_tensor, segments_tensors, labels=labels) # set model.train() before if training this loss
|
||||
labels = torch.tensor([1])
|
||||
multiple_choice_loss = model(tokens_tensor, segments_tensors, labels=labels) # set model.train() before if training this loss
|
||||
"""
|
||||
model = BertForMultipleChoice.from_pretrained(*args, **kwargs)
|
||||
return model
|
||||
@@ -298,25 +298,25 @@ def bertForQuestionAnswering(*args, **kwargs):
|
||||
|
||||
Example:
|
||||
# Load the tokenizer
|
||||
>>> import torch
|
||||
>>> tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'bertTokenizer', 'bert-base-cased', do_basic_tokenize=False)
|
||||
import torch
|
||||
tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'bertTokenizer', 'bert-base-cased', do_basic_tokenize=False)
|
||||
# Prepare tokenized input
|
||||
>>> text = "[CLS] Who was Jim Henson ? [SEP] Jim Henson was a puppeteer [SEP]"
|
||||
>>> tokenized_text = tokenizer.tokenize(text)
|
||||
>>> indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text)
|
||||
>>> segments_ids = [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1]
|
||||
>>> tokens_tensor = torch.tensor([indexed_tokens])
|
||||
>>> segments_tensors = torch.tensor([segments_ids])
|
||||
text = "[CLS] Who was Jim Henson ? [SEP] Jim Henson was a puppeteer [SEP]"
|
||||
tokenized_text = tokenizer.tokenize(text)
|
||||
indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text)
|
||||
segments_ids = [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1]
|
||||
tokens_tensor = torch.tensor([indexed_tokens])
|
||||
segments_tensors = torch.tensor([segments_ids])
|
||||
# Load bertForQuestionAnswering
|
||||
>>> model = torch.hub.load('huggingface/pytorch-transformers', 'bertForQuestionAnswering', 'bert-base-cased')
|
||||
>>> model.eval()
|
||||
model = torch.hub.load('huggingface/pytorch-transformers', 'bertForQuestionAnswering', 'bert-base-cased')
|
||||
model.eval()
|
||||
# Predict the start and end positions logits
|
||||
>>> with torch.no_grad():
|
||||
with torch.no_grad():
|
||||
start_logits, end_logits = model(tokens_tensor, segments_tensors)
|
||||
# Or get the total loss which is the sum of the CrossEntropy loss for the start and end token positions
|
||||
>>> start_positions, end_positions = torch.tensor([12]), torch.tensor([14])
|
||||
start_positions, end_positions = torch.tensor([12]), torch.tensor([14])
|
||||
# set model.train() before if training this loss
|
||||
>>> multiple_choice_loss = model(tokens_tensor, segments_tensors, start_positions=start_positions, end_positions=end_positions)
|
||||
multiple_choice_loss = model(tokens_tensor, segments_tensors, start_positions=start_positions, end_positions=end_positions)
|
||||
"""
|
||||
model = BertForQuestionAnswering.from_pretrained(*args, **kwargs)
|
||||
return model
|
||||
@@ -337,24 +337,24 @@ def bertForTokenClassification(*args, **kwargs):
|
||||
|
||||
Example:
|
||||
# Load the tokenizer
|
||||
>>> import torch
|
||||
>>> tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'bertTokenizer', 'bert-base-cased', do_basic_tokenize=False)
|
||||
import torch
|
||||
tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'bertTokenizer', 'bert-base-cased', do_basic_tokenize=False)
|
||||
# Prepare tokenized input
|
||||
>>> text = "[CLS] Who was Jim Henson ? [SEP] Jim Henson was a puppeteer [SEP]"
|
||||
>>> tokenized_text = tokenizer.tokenize(text)
|
||||
>>> indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text)
|
||||
>>> segments_ids = [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1]
|
||||
>>> tokens_tensor = torch.tensor([indexed_tokens])
|
||||
>>> segments_tensors = torch.tensor([segments_ids])
|
||||
text = "[CLS] Who was Jim Henson ? [SEP] Jim Henson was a puppeteer [SEP]"
|
||||
tokenized_text = tokenizer.tokenize(text)
|
||||
indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text)
|
||||
segments_ids = [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1]
|
||||
tokens_tensor = torch.tensor([indexed_tokens])
|
||||
segments_tensors = torch.tensor([segments_ids])
|
||||
# Load bertForTokenClassification
|
||||
>>> model = torch.hub.load('huggingface/pytorch-transformers', 'bertForTokenClassification', 'bert-base-cased', num_labels=2)
|
||||
>>> model.eval()
|
||||
model = torch.hub.load('huggingface/pytorch-transformers', 'bertForTokenClassification', 'bert-base-cased', num_labels=2)
|
||||
model.eval()
|
||||
# Predict the token classification logits
|
||||
>>> with torch.no_grad():
|
||||
with torch.no_grad():
|
||||
classif_logits = model(tokens_tensor, segments_tensors)
|
||||
# Or get the token classification loss
|
||||
>>> labels = torch.tensor([[0, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 1, 0, 0, 1, 0]])
|
||||
>>> classif_loss = model(tokens_tensor, segments_tensors, labels=labels) # set model.train() before if training this loss
|
||||
labels = torch.tensor([[0, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 1, 0, 0, 1, 0]])
|
||||
classif_loss = model(tokens_tensor, segments_tensors, labels=labels) # set model.train() before if training this loss
|
||||
"""
|
||||
model = BertForTokenClassification.from_pretrained(*args, **kwargs)
|
||||
return model
|
||||
|
||||
@@ -52,11 +52,11 @@ def gpt2Tokenizer(*args, **kwargs):
|
||||
Default: None
|
||||
|
||||
Example:
|
||||
>>> import torch
|
||||
>>> tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'gpt2Tokenizer', 'gpt2')
|
||||
import torch
|
||||
tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'gpt2Tokenizer', 'gpt2')
|
||||
|
||||
>>> text = "Who was Jim Henson ?"
|
||||
>>> indexed_tokens = tokenizer.encode(tokenized_text)
|
||||
text = "Who was Jim Henson ?"
|
||||
indexed_tokens = tokenizer.encode(tokenized_text)
|
||||
"""
|
||||
tokenizer = GPT2Tokenizer.from_pretrained(*args, **kwargs)
|
||||
return tokenizer
|
||||
@@ -71,24 +71,24 @@ def gpt2Model(*args, **kwargs):
|
||||
|
||||
Example:
|
||||
# Load the tokenizer
|
||||
>>> import torch
|
||||
>>> tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'gpt2Tokenizer', 'gpt2')
|
||||
import torch
|
||||
tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'gpt2Tokenizer', 'gpt2')
|
||||
|
||||
# Prepare tokenized input
|
||||
>>> text_1 = "Who was Jim Henson ?"
|
||||
>>> text_2 = "Jim Henson was a puppeteer"
|
||||
>>> indexed_tokens_1 = tokenizer.encode(text_1)
|
||||
>>> indexed_tokens_2 = tokenizer.encode(text_2)
|
||||
>>> tokens_tensor_1 = torch.tensor([indexed_tokens_1])
|
||||
>>> tokens_tensor_2 = torch.tensor([indexed_tokens_2])
|
||||
text_1 = "Who was Jim Henson ?"
|
||||
text_2 = "Jim Henson was a puppeteer"
|
||||
indexed_tokens_1 = tokenizer.encode(text_1)
|
||||
indexed_tokens_2 = tokenizer.encode(text_2)
|
||||
tokens_tensor_1 = torch.tensor([indexed_tokens_1])
|
||||
tokens_tensor_2 = torch.tensor([indexed_tokens_2])
|
||||
|
||||
# Load gpt2Model
|
||||
>>> model = torch.hub.load('huggingface/pytorch-transformers', 'gpt2Model', 'gpt2')
|
||||
>>> model.eval()
|
||||
model = torch.hub.load('huggingface/pytorch-transformers', 'gpt2Model', 'gpt2')
|
||||
model.eval()
|
||||
|
||||
# Predict hidden states features for each layer
|
||||
# past can be used to reuse precomputed hidden state in a subsequent predictions
|
||||
>>> with torch.no_grad():
|
||||
with torch.no_grad():
|
||||
hidden_states_1, past = model(tokens_tensor_1)
|
||||
hidden_states_2, past = model(tokens_tensor_2, past=past)
|
||||
"""
|
||||
@@ -104,31 +104,31 @@ def gpt2LMHeadModel(*args, **kwargs):
|
||||
|
||||
Example:
|
||||
# Load the tokenizer
|
||||
>>> import torch
|
||||
>>> tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'gpt2Tokenizer', 'gpt2')
|
||||
import torch
|
||||
tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'gpt2Tokenizer', 'gpt2')
|
||||
|
||||
# Prepare tokenized input
|
||||
>>> text_1 = "Who was Jim Henson ?"
|
||||
>>> text_2 = "Jim Henson was a puppeteer"
|
||||
>>> indexed_tokens_1 = tokenizer.encode(text_1)
|
||||
>>> indexed_tokens_2 = tokenizer.encode(text_2)
|
||||
>>> tokens_tensor_1 = torch.tensor([indexed_tokens_1])
|
||||
>>> tokens_tensor_2 = torch.tensor([indexed_tokens_2])
|
||||
text_1 = "Who was Jim Henson ?"
|
||||
text_2 = "Jim Henson was a puppeteer"
|
||||
indexed_tokens_1 = tokenizer.encode(text_1)
|
||||
indexed_tokens_2 = tokenizer.encode(text_2)
|
||||
tokens_tensor_1 = torch.tensor([indexed_tokens_1])
|
||||
tokens_tensor_2 = torch.tensor([indexed_tokens_2])
|
||||
|
||||
# Load gpt2LMHeadModel
|
||||
>>> model = torch.hub.load('huggingface/pytorch-transformers', 'gpt2LMHeadModel', 'gpt2')
|
||||
>>> model.eval()
|
||||
model = torch.hub.load('huggingface/pytorch-transformers', 'gpt2LMHeadModel', 'gpt2')
|
||||
model.eval()
|
||||
|
||||
# Predict hidden states features for each layer
|
||||
# past can be used to reuse precomputed hidden state in a subsequent predictions
|
||||
>>> with torch.no_grad():
|
||||
with torch.no_grad():
|
||||
predictions_1, past = model(tokens_tensor_1)
|
||||
predictions_2, past = model(tokens_tensor_2, past=past)
|
||||
|
||||
# Get the predicted last token
|
||||
>>> predicted_index = torch.argmax(predictions_2[0, -1, :]).item()
|
||||
>>> predicted_token = tokenizer.decode([predicted_index])
|
||||
>>> assert predicted_token == ' who'
|
||||
predicted_index = torch.argmax(predictions_2[0, -1, :]).item()
|
||||
predicted_token = tokenizer.decode([predicted_index])
|
||||
assert predicted_token == ' who'
|
||||
"""
|
||||
model = GPT2LMHeadModel.from_pretrained(*args, **kwargs)
|
||||
return model
|
||||
@@ -143,25 +143,25 @@ def gpt2DoubleHeadsModel(*args, **kwargs):
|
||||
|
||||
Example:
|
||||
# Load the tokenizer
|
||||
>>> import torch
|
||||
>>> tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'gpt2Tokenizer', 'gpt2')
|
||||
import torch
|
||||
tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'gpt2Tokenizer', 'gpt2')
|
||||
|
||||
# Prepare tokenized input
|
||||
>>> text1 = "Who was Jim Henson ? Jim Henson was a puppeteer"
|
||||
>>> text2 = "Who was Jim Henson ? Jim Henson was a mysterious young man"
|
||||
>>> tokenized_text1 = tokenizer.tokenize(text1)
|
||||
>>> tokenized_text2 = tokenizer.tokenize(text2)
|
||||
>>> indexed_tokens1 = tokenizer.convert_tokens_to_ids(tokenized_text1)
|
||||
>>> indexed_tokens2 = tokenizer.convert_tokens_to_ids(tokenized_text2)
|
||||
>>> tokens_tensor = torch.tensor([[indexed_tokens1, indexed_tokens2]])
|
||||
>>> mc_token_ids = torch.LongTensor([[len(tokenized_text1)-1, len(tokenized_text2)-1]])
|
||||
text1 = "Who was Jim Henson ? Jim Henson was a puppeteer"
|
||||
text2 = "Who was Jim Henson ? Jim Henson was a mysterious young man"
|
||||
tokenized_text1 = tokenizer.tokenize(text1)
|
||||
tokenized_text2 = tokenizer.tokenize(text2)
|
||||
indexed_tokens1 = tokenizer.convert_tokens_to_ids(tokenized_text1)
|
||||
indexed_tokens2 = tokenizer.convert_tokens_to_ids(tokenized_text2)
|
||||
tokens_tensor = torch.tensor([[indexed_tokens1, indexed_tokens2]])
|
||||
mc_token_ids = torch.LongTensor([[len(tokenized_text1)-1, len(tokenized_text2)-1]])
|
||||
|
||||
# Load gpt2DoubleHeadsModel
|
||||
>>> model = torch.hub.load('huggingface/pytorch-transformers', 'gpt2DoubleHeadsModel', 'gpt2')
|
||||
>>> model.eval()
|
||||
model = torch.hub.load('huggingface/pytorch-transformers', 'gpt2DoubleHeadsModel', 'gpt2')
|
||||
model.eval()
|
||||
|
||||
# Predict hidden states features for each layer
|
||||
>>> with torch.no_grad():
|
||||
with torch.no_grad():
|
||||
lm_logits, multiple_choice_logits, presents = model(tokens_tensor, mc_token_ids)
|
||||
"""
|
||||
model = GPT2DoubleHeadsModel.from_pretrained(*args, **kwargs)
|
||||
|
||||
@@ -76,12 +76,12 @@ def openAIGPTTokenizer(*args, **kwargs):
|
||||
Default: None
|
||||
|
||||
Example:
|
||||
>>> import torch
|
||||
>>> tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'openAIGPTTokenizer', 'openai-gpt')
|
||||
import torch
|
||||
tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'openAIGPTTokenizer', 'openai-gpt')
|
||||
|
||||
>>> text = "Who was Jim Henson ? Jim Henson was a puppeteer"
|
||||
>>> tokenized_text = tokenizer.tokenize(text)
|
||||
>>> indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text)
|
||||
text = "Who was Jim Henson ? Jim Henson was a puppeteer"
|
||||
tokenized_text = tokenizer.tokenize(text)
|
||||
indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text)
|
||||
[763, 509, 4265, 2298, 945, 257, 4265, 2298, 945, 509, 246, 10148, 39041, 483]
|
||||
"""
|
||||
tokenizer = OpenAIGPTTokenizer.from_pretrained(*args, **kwargs)
|
||||
@@ -97,21 +97,21 @@ def openAIGPTModel(*args, **kwargs):
|
||||
|
||||
Example:
|
||||
# Load the tokenizer
|
||||
>>> import torch
|
||||
>>> tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'openAIGPTTokenizer', 'openai-gpt')
|
||||
import torch
|
||||
tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'openAIGPTTokenizer', 'openai-gpt')
|
||||
|
||||
# Prepare tokenized input
|
||||
>>> text = "Who was Jim Henson ? Jim Henson was a puppeteer"
|
||||
>>> tokenized_text = tokenizer.tokenize(text)
|
||||
>>> indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text)
|
||||
>>> tokens_tensor = torch.tensor([indexed_tokens])
|
||||
text = "Who was Jim Henson ? Jim Henson was a puppeteer"
|
||||
tokenized_text = tokenizer.tokenize(text)
|
||||
indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text)
|
||||
tokens_tensor = torch.tensor([indexed_tokens])
|
||||
|
||||
# Load openAIGPTModel
|
||||
>>> model = torch.hub.load('huggingface/pytorch-transformers', 'openAIGPTModel', 'openai-gpt')
|
||||
>>> model.eval()
|
||||
model = torch.hub.load('huggingface/pytorch-transformers', 'openAIGPTModel', 'openai-gpt')
|
||||
model.eval()
|
||||
|
||||
# Predict hidden states features for each layer
|
||||
>>> with torch.no_grad():
|
||||
with torch.no_grad():
|
||||
hidden_states = model(tokens_tensor)
|
||||
"""
|
||||
model = OpenAIGPTModel.from_pretrained(*args, **kwargs)
|
||||
@@ -126,26 +126,26 @@ def openAIGPTLMHeadModel(*args, **kwargs):
|
||||
|
||||
Example:
|
||||
# Load the tokenizer
|
||||
>>> import torch
|
||||
>>> tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'openAIGPTTokenizer', 'openai-gpt')
|
||||
import torch
|
||||
tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'openAIGPTTokenizer', 'openai-gpt')
|
||||
|
||||
# Prepare tokenized input
|
||||
>>> text = "Who was Jim Henson ? Jim Henson was a puppeteer"
|
||||
>>> tokenized_text = tokenizer.tokenize(text)
|
||||
>>> indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text)
|
||||
>>> tokens_tensor = torch.tensor([indexed_tokens])
|
||||
text = "Who was Jim Henson ? Jim Henson was a puppeteer"
|
||||
tokenized_text = tokenizer.tokenize(text)
|
||||
indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text)
|
||||
tokens_tensor = torch.tensor([indexed_tokens])
|
||||
|
||||
# Load openAIGPTLMHeadModel
|
||||
>>> model = torch.hub.load('huggingface/pytorch-transformers', 'openAIGPTLMHeadModel', 'openai-gpt')
|
||||
>>> model.eval()
|
||||
model = torch.hub.load('huggingface/pytorch-transformers', 'openAIGPTLMHeadModel', 'openai-gpt')
|
||||
model.eval()
|
||||
|
||||
# Predict hidden states features for each layer
|
||||
>>> with torch.no_grad():
|
||||
with torch.no_grad():
|
||||
predictions = model(tokens_tensor)
|
||||
|
||||
# Get the predicted last token
|
||||
>>> predicted_index = torch.argmax(predictions[0, -1, :]).item()
|
||||
>>> predicted_token = tokenizer.convert_ids_to_tokens([predicted_index])[0]
|
||||
predicted_index = torch.argmax(predictions[0, -1, :]).item()
|
||||
predicted_token = tokenizer.convert_ids_to_tokens([predicted_index])[0]
|
||||
'.</w>'
|
||||
"""
|
||||
model = OpenAIGPTLMHeadModel.from_pretrained(*args, **kwargs)
|
||||
@@ -161,25 +161,25 @@ def openAIGPTDoubleHeadsModel(*args, **kwargs):
|
||||
|
||||
Example:
|
||||
# Load the tokenizer
|
||||
>>> import torch
|
||||
>>> tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'openAIGPTTokenizer', 'openai-gpt')
|
||||
import torch
|
||||
tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'openAIGPTTokenizer', 'openai-gpt')
|
||||
|
||||
# Prepare tokenized input
|
||||
>>> text1 = "Who was Jim Henson ? Jim Henson was a puppeteer"
|
||||
>>> text2 = "Who was Jim Henson ? Jim Henson was a mysterious young man"
|
||||
>>> tokenized_text1 = tokenizer.tokenize(text1)
|
||||
>>> tokenized_text2 = tokenizer.tokenize(text2)
|
||||
>>> indexed_tokens1 = tokenizer.convert_tokens_to_ids(tokenized_text1)
|
||||
>>> indexed_tokens2 = tokenizer.convert_tokens_to_ids(tokenized_text2)
|
||||
>>> tokens_tensor = torch.tensor([[indexed_tokens1, indexed_tokens2]])
|
||||
>>> mc_token_ids = torch.LongTensor([[len(tokenized_text1)-1, len(tokenized_text2)-1]])
|
||||
text1 = "Who was Jim Henson ? Jim Henson was a puppeteer"
|
||||
text2 = "Who was Jim Henson ? Jim Henson was a mysterious young man"
|
||||
tokenized_text1 = tokenizer.tokenize(text1)
|
||||
tokenized_text2 = tokenizer.tokenize(text2)
|
||||
indexed_tokens1 = tokenizer.convert_tokens_to_ids(tokenized_text1)
|
||||
indexed_tokens2 = tokenizer.convert_tokens_to_ids(tokenized_text2)
|
||||
tokens_tensor = torch.tensor([[indexed_tokens1, indexed_tokens2]])
|
||||
mc_token_ids = torch.LongTensor([[len(tokenized_text1)-1, len(tokenized_text2)-1]])
|
||||
|
||||
# Load openAIGPTDoubleHeadsModel
|
||||
>>> model = torch.hub.load('huggingface/pytorch-transformers', 'openAIGPTDoubleHeadsModel', 'openai-gpt')
|
||||
>>> model.eval()
|
||||
model = torch.hub.load('huggingface/pytorch-transformers', 'openAIGPTDoubleHeadsModel', 'openai-gpt')
|
||||
model.eval()
|
||||
|
||||
# Predict hidden states features for each layer
|
||||
>>> with torch.no_grad():
|
||||
with torch.no_grad():
|
||||
lm_logits, multiple_choice_logits = model(tokens_tensor, mc_token_ids)
|
||||
"""
|
||||
model = OpenAIGPTDoubleHeadsModel.from_pretrained(*args, **kwargs)
|
||||
|
||||
@@ -45,12 +45,12 @@ def transformerXLTokenizer(*args, **kwargs):
|
||||
* transfo-xl-wt103
|
||||
|
||||
Example:
|
||||
>>> import torch
|
||||
>>> tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'transformerXLTokenizer', 'transfo-xl-wt103')
|
||||
import torch
|
||||
tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'transformerXLTokenizer', 'transfo-xl-wt103')
|
||||
|
||||
>>> text = "Who was Jim Henson ?"
|
||||
>>> tokenized_text = tokenizer.tokenize(tokenized_text)
|
||||
>>> indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text)
|
||||
text = "Who was Jim Henson ?"
|
||||
tokenized_text = tokenizer.tokenize(tokenized_text)
|
||||
indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text)
|
||||
"""
|
||||
tokenizer = TransfoXLTokenizer.from_pretrained(*args, **kwargs)
|
||||
return tokenizer
|
||||
@@ -63,26 +63,26 @@ def transformerXLModel(*args, **kwargs):
|
||||
|
||||
Example:
|
||||
# Load the tokenizer
|
||||
>>> import torch
|
||||
>>> tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'transformerXLTokenizer', 'transfo-xl-wt103')
|
||||
import torch
|
||||
tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'transformerXLTokenizer', 'transfo-xl-wt103')
|
||||
|
||||
# Prepare tokenized input
|
||||
>>> text_1 = "Who was Jim Henson ?"
|
||||
>>> text_2 = "Jim Henson was a puppeteer"
|
||||
>>> tokenized_text_1 = tokenizer.tokenize(text_1)
|
||||
>>> tokenized_text_2 = tokenizer.tokenize(text_2)
|
||||
>>> indexed_tokens_1 = tokenizer.convert_tokens_to_ids(tokenized_text_1)
|
||||
>>> indexed_tokens_2 = tokenizer.convert_tokens_to_ids(tokenized_text_2)
|
||||
>>> tokens_tensor_1 = torch.tensor([indexed_tokens_1])
|
||||
>>> tokens_tensor_2 = torch.tensor([indexed_tokens_2])
|
||||
text_1 = "Who was Jim Henson ?"
|
||||
text_2 = "Jim Henson was a puppeteer"
|
||||
tokenized_text_1 = tokenizer.tokenize(text_1)
|
||||
tokenized_text_2 = tokenizer.tokenize(text_2)
|
||||
indexed_tokens_1 = tokenizer.convert_tokens_to_ids(tokenized_text_1)
|
||||
indexed_tokens_2 = tokenizer.convert_tokens_to_ids(tokenized_text_2)
|
||||
tokens_tensor_1 = torch.tensor([indexed_tokens_1])
|
||||
tokens_tensor_2 = torch.tensor([indexed_tokens_2])
|
||||
|
||||
# Load transformerXLModel
|
||||
>>> model = torch.hub.load('huggingface/pytorch-transformers', 'transformerXLModel', 'transfo-xl-wt103')
|
||||
>>> model.eval()
|
||||
model = torch.hub.load('huggingface/pytorch-transformers', 'transformerXLModel', 'transfo-xl-wt103')
|
||||
model.eval()
|
||||
|
||||
# Predict hidden states features for each layer
|
||||
# We can re-use the memory cells in a subsequent call to attend a longer context
|
||||
>>> with torch.no_grad():
|
||||
with torch.no_grad():
|
||||
hidden_states_1, mems_1 = model(tokens_tensor_1)
|
||||
hidden_states_2, mems_2 = model(tokens_tensor_2, mems=mems_1)
|
||||
"""
|
||||
@@ -98,33 +98,33 @@ def transformerXLLMHeadModel(*args, **kwargs):
|
||||
|
||||
Example:
|
||||
# Load the tokenizer
|
||||
>>> import torch
|
||||
>>> tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'transformerXLTokenizer', 'transfo-xl-wt103')
|
||||
import torch
|
||||
tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'transformerXLTokenizer', 'transfo-xl-wt103')
|
||||
|
||||
# Prepare tokenized input
|
||||
>>> text_1 = "Who was Jim Henson ?"
|
||||
>>> text_2 = "Jim Henson was a puppeteer"
|
||||
>>> tokenized_text_1 = tokenizer.tokenize(text_1)
|
||||
>>> tokenized_text_2 = tokenizer.tokenize(text_2)
|
||||
>>> indexed_tokens_1 = tokenizer.convert_tokens_to_ids(tokenized_text_1)
|
||||
>>> indexed_tokens_2 = tokenizer.convert_tokens_to_ids(tokenized_text_2)
|
||||
>>> tokens_tensor_1 = torch.tensor([indexed_tokens_1])
|
||||
>>> tokens_tensor_2 = torch.tensor([indexed_tokens_2])
|
||||
text_1 = "Who was Jim Henson ?"
|
||||
text_2 = "Jim Henson was a puppeteer"
|
||||
tokenized_text_1 = tokenizer.tokenize(text_1)
|
||||
tokenized_text_2 = tokenizer.tokenize(text_2)
|
||||
indexed_tokens_1 = tokenizer.convert_tokens_to_ids(tokenized_text_1)
|
||||
indexed_tokens_2 = tokenizer.convert_tokens_to_ids(tokenized_text_2)
|
||||
tokens_tensor_1 = torch.tensor([indexed_tokens_1])
|
||||
tokens_tensor_2 = torch.tensor([indexed_tokens_2])
|
||||
|
||||
# Load transformerXLLMHeadModel
|
||||
>>> model = torch.hub.load('huggingface/pytorch-transformers', 'transformerXLLMHeadModel', 'transfo-xl-wt103')
|
||||
>>> model.eval()
|
||||
model = torch.hub.load('huggingface/pytorch-transformers', 'transformerXLLMHeadModel', 'transfo-xl-wt103')
|
||||
model.eval()
|
||||
|
||||
# Predict hidden states features for each layer
|
||||
# We can re-use the memory cells in a subsequent call to attend a longer context
|
||||
>>> with torch.no_grad():
|
||||
with torch.no_grad():
|
||||
predictions_1, mems_1 = model(tokens_tensor_1)
|
||||
predictions_2, mems_2 = model(tokens_tensor_2, mems=mems_1)
|
||||
|
||||
# Get the predicted last token
|
||||
>>> predicted_index = torch.argmax(predictions_2[0, -1, :]).item()
|
||||
>>> predicted_token = tokenizer.convert_ids_to_tokens([predicted_index])[0]
|
||||
>>> assert predicted_token == 'who'
|
||||
predicted_index = torch.argmax(predictions_2[0, -1, :]).item()
|
||||
predicted_token = tokenizer.convert_ids_to_tokens([predicted_index])[0]
|
||||
assert predicted_token == 'who'
|
||||
"""
|
||||
model = TransfoXLLMHeadModel.from_pretrained(*args, **kwargs)
|
||||
return model
|
||||
|
||||
@@ -17,16 +17,16 @@ xlm_start_docstring = """
|
||||
|
||||
Example:
|
||||
# Load the tokenizer
|
||||
>>> import torch
|
||||
>>> tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'xlmTokenizer', 'xlm-mlm-en-2048')
|
||||
import torch
|
||||
tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'xlmTokenizer', 'xlm-mlm-en-2048')
|
||||
|
||||
# Prepare tokenized input
|
||||
>>> text_1 = "Who was Jim Henson ?"
|
||||
>>> text_2 = "Jim Henson was a puppeteer"
|
||||
>>> indexed_tokens_1 = tokenizer.encode(text_1)
|
||||
>>> indexed_tokens_2 = tokenizer.encode(text_2)
|
||||
>>> tokens_tensor_1 = torch.tensor([indexed_tokens_1])
|
||||
>>> tokens_tensor_2 = torch.tensor([indexed_tokens_2])
|
||||
text_1 = "Who was Jim Henson ?"
|
||||
text_2 = "Jim Henson was a puppeteer"
|
||||
indexed_tokens_1 = tokenizer.encode(text_1)
|
||||
indexed_tokens_2 = tokenizer.encode(text_2)
|
||||
tokens_tensor_1 = torch.tensor([indexed_tokens_1])
|
||||
tokens_tensor_2 = torch.tensor([indexed_tokens_2])
|
||||
"""
|
||||
|
||||
# A lot of models share the same param doc. Use a decorator
|
||||
@@ -76,11 +76,11 @@ def xlmTokenizer(*args, **kwargs):
|
||||
Default: None
|
||||
|
||||
Example:
|
||||
>>> import torch
|
||||
>>> tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'xlmTokenizer', 'xlm-mlm-en-2048')
|
||||
import torch
|
||||
tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'xlmTokenizer', 'xlm-mlm-en-2048')
|
||||
|
||||
>>> text = "Who was Jim Henson ?"
|
||||
>>> indexed_tokens = tokenizer.encode(tokenized_text)
|
||||
text = "Who was Jim Henson ?"
|
||||
indexed_tokens = tokenizer.encode(tokenized_text)
|
||||
"""
|
||||
tokenizer = XLMTokenizer.from_pretrained(*args, **kwargs)
|
||||
return tokenizer
|
||||
@@ -91,11 +91,11 @@ def xlmTokenizer(*args, **kwargs):
|
||||
def xlmModel(*args, **kwargs):
|
||||
"""
|
||||
# Load xlmModel
|
||||
>>> model = torch.hub.load('huggingface/pytorch-transformers', 'xlmModel', 'xlm-mlm-en-2048')
|
||||
>>> model.eval()
|
||||
model = torch.hub.load('huggingface/pytorch-transformers', 'xlmModel', 'xlm-mlm-en-2048')
|
||||
model.eval()
|
||||
|
||||
# Predict hidden states features for each layer
|
||||
>>> with torch.no_grad():
|
||||
with torch.no_grad():
|
||||
hidden_states_1, mems = model(tokens_tensor_1)
|
||||
hidden_states_2, mems = model(tokens_tensor_2, past=mems)
|
||||
"""
|
||||
@@ -108,26 +108,26 @@ def xlmModel(*args, **kwargs):
|
||||
def xlmLMHeadModel(*args, **kwargs):
|
||||
"""
|
||||
# Prepare tokenized input
|
||||
>>> text_1 = "Who was Jim Henson ?"
|
||||
>>> text_2 = "Jim Henson was a puppeteer"
|
||||
>>> indexed_tokens_1 = tokenizer.encode(text_1)
|
||||
>>> indexed_tokens_2 = tokenizer.encode(text_2)
|
||||
>>> tokens_tensor_1 = torch.tensor([indexed_tokens_1])
|
||||
>>> tokens_tensor_2 = torch.tensor([indexed_tokens_2])
|
||||
text_1 = "Who was Jim Henson ?"
|
||||
text_2 = "Jim Henson was a puppeteer"
|
||||
indexed_tokens_1 = tokenizer.encode(text_1)
|
||||
indexed_tokens_2 = tokenizer.encode(text_2)
|
||||
tokens_tensor_1 = torch.tensor([indexed_tokens_1])
|
||||
tokens_tensor_2 = torch.tensor([indexed_tokens_2])
|
||||
|
||||
# Load xlnetLMHeadModel
|
||||
>>> model = torch.hub.load('huggingface/pytorch-transformers', 'xlnetLMHeadModel', 'xlm-mlm-en-2048')
|
||||
>>> model.eval()
|
||||
model = torch.hub.load('huggingface/pytorch-transformers', 'xlnetLMHeadModel', 'xlm-mlm-en-2048')
|
||||
model.eval()
|
||||
|
||||
# Predict hidden states features for each layer
|
||||
>>> with torch.no_grad():
|
||||
with torch.no_grad():
|
||||
predictions_1, mems = model(tokens_tensor_1)
|
||||
predictions_2, mems = model(tokens_tensor_2, mems=mems)
|
||||
|
||||
# Get the predicted last token
|
||||
>>> predicted_index = torch.argmax(predictions_2[0, -1, :]).item()
|
||||
>>> predicted_token = tokenizer.decode([predicted_index])
|
||||
>>> assert predicted_token == ' who'
|
||||
predicted_index = torch.argmax(predictions_2[0, -1, :]).item()
|
||||
predicted_token = tokenizer.decode([predicted_index])
|
||||
assert predicted_token == ' who'
|
||||
"""
|
||||
model = XLMWithLMHeadModel.from_pretrained(*args, **kwargs)
|
||||
return model
|
||||
@@ -142,25 +142,25 @@ def xlmLMHeadModel(*args, **kwargs):
|
||||
|
||||
# Example:
|
||||
# # Load the tokenizer
|
||||
# >>> import torch
|
||||
# >>> tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'xlnetTokenizer', 'xlm-mlm-en-2048')
|
||||
# import torch
|
||||
# tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'xlnetTokenizer', 'xlm-mlm-en-2048')
|
||||
|
||||
# # Prepare tokenized input
|
||||
# >>> text1 = "Who was Jim Henson ? Jim Henson was a puppeteer"
|
||||
# >>> text2 = "Who was Jim Henson ? Jim Henson was a mysterious young man"
|
||||
# >>> tokenized_text1 = tokenizer.tokenize(text1)
|
||||
# >>> tokenized_text2 = tokenizer.tokenize(text2)
|
||||
# >>> indexed_tokens1 = tokenizer.convert_tokens_to_ids(tokenized_text1)
|
||||
# >>> indexed_tokens2 = tokenizer.convert_tokens_to_ids(tokenized_text2)
|
||||
# >>> tokens_tensor = torch.tensor([[indexed_tokens1, indexed_tokens2]])
|
||||
# >>> mc_token_ids = torch.LongTensor([[len(tokenized_text1)-1, len(tokenized_text2)-1]])
|
||||
# text1 = "Who was Jim Henson ? Jim Henson was a puppeteer"
|
||||
# text2 = "Who was Jim Henson ? Jim Henson was a mysterious young man"
|
||||
# tokenized_text1 = tokenizer.tokenize(text1)
|
||||
# tokenized_text2 = tokenizer.tokenize(text2)
|
||||
# indexed_tokens1 = tokenizer.convert_tokens_to_ids(tokenized_text1)
|
||||
# indexed_tokens2 = tokenizer.convert_tokens_to_ids(tokenized_text2)
|
||||
# tokens_tensor = torch.tensor([[indexed_tokens1, indexed_tokens2]])
|
||||
# mc_token_ids = torch.LongTensor([[len(tokenized_text1)-1, len(tokenized_text2)-1]])
|
||||
|
||||
# # Load xlnetForSequenceClassification
|
||||
# >>> model = torch.hub.load('huggingface/pytorch-transformers', 'xlnetForSequenceClassification', 'xlm-mlm-en-2048')
|
||||
# >>> model.eval()
|
||||
# model = torch.hub.load('huggingface/pytorch-transformers', 'xlnetForSequenceClassification', 'xlm-mlm-en-2048')
|
||||
# model.eval()
|
||||
|
||||
# # Predict sequence classes logits
|
||||
# >>> with torch.no_grad():
|
||||
# with torch.no_grad():
|
||||
# lm_logits, mems = model(tokens_tensor)
|
||||
# """
|
||||
# model = XLNetForSequenceClassification.from_pretrained(*args, **kwargs)
|
||||
|
||||
@@ -53,11 +53,11 @@ def xlnetTokenizer(*args, **kwargs):
|
||||
Default: None
|
||||
|
||||
Example:
|
||||
>>> import torch
|
||||
>>> tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'xlnetTokenizer', 'xlnet-large-cased')
|
||||
import torch
|
||||
tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'xlnetTokenizer', 'xlnet-large-cased')
|
||||
|
||||
>>> text = "Who was Jim Henson ?"
|
||||
>>> indexed_tokens = tokenizer.encode(tokenized_text)
|
||||
text = "Who was Jim Henson ?"
|
||||
indexed_tokens = tokenizer.encode(tokenized_text)
|
||||
"""
|
||||
tokenizer = XLNetTokenizer.from_pretrained(*args, **kwargs)
|
||||
return tokenizer
|
||||
@@ -72,23 +72,23 @@ def xlnetModel(*args, **kwargs):
|
||||
|
||||
Example:
|
||||
# Load the tokenizer
|
||||
>>> import torch
|
||||
>>> tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'xlnetTokenizer', 'xlnet-large-cased')
|
||||
import torch
|
||||
tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'xlnetTokenizer', 'xlnet-large-cased')
|
||||
|
||||
# Prepare tokenized input
|
||||
>>> text_1 = "Who was Jim Henson ?"
|
||||
>>> text_2 = "Jim Henson was a puppeteer"
|
||||
>>> indexed_tokens_1 = tokenizer.encode(text_1)
|
||||
>>> indexed_tokens_2 = tokenizer.encode(text_2)
|
||||
>>> tokens_tensor_1 = torch.tensor([indexed_tokens_1])
|
||||
>>> tokens_tensor_2 = torch.tensor([indexed_tokens_2])
|
||||
text_1 = "Who was Jim Henson ?"
|
||||
text_2 = "Jim Henson was a puppeteer"
|
||||
indexed_tokens_1 = tokenizer.encode(text_1)
|
||||
indexed_tokens_2 = tokenizer.encode(text_2)
|
||||
tokens_tensor_1 = torch.tensor([indexed_tokens_1])
|
||||
tokens_tensor_2 = torch.tensor([indexed_tokens_2])
|
||||
|
||||
# Load xlnetModel
|
||||
>>> model = torch.hub.load('huggingface/pytorch-transformers', 'xlnetModel', 'xlnet-large-cased')
|
||||
>>> model.eval()
|
||||
model = torch.hub.load('huggingface/pytorch-transformers', 'xlnetModel', 'xlnet-large-cased')
|
||||
model.eval()
|
||||
|
||||
# Predict hidden states features for each layer
|
||||
>>> with torch.no_grad():
|
||||
with torch.no_grad():
|
||||
hidden_states_1, mems = model(tokens_tensor_1)
|
||||
hidden_states_2, mems = model(tokens_tensor_2, past=mems)
|
||||
"""
|
||||
@@ -106,30 +106,30 @@ def xlnetLMHeadModel(*args, **kwargs):
|
||||
|
||||
Example:
|
||||
# Load the tokenizer
|
||||
>>> import torch
|
||||
>>> tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'xlnetTokenizer', 'xlnet-large-cased')
|
||||
import torch
|
||||
tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'xlnetTokenizer', 'xlnet-large-cased')
|
||||
|
||||
# Prepare tokenized input
|
||||
>>> text_1 = "Who was Jim Henson ?"
|
||||
>>> text_2 = "Jim Henson was a puppeteer"
|
||||
>>> indexed_tokens_1 = tokenizer.encode(text_1)
|
||||
>>> indexed_tokens_2 = tokenizer.encode(text_2)
|
||||
>>> tokens_tensor_1 = torch.tensor([indexed_tokens_1])
|
||||
>>> tokens_tensor_2 = torch.tensor([indexed_tokens_2])
|
||||
text_1 = "Who was Jim Henson ?"
|
||||
text_2 = "Jim Henson was a puppeteer"
|
||||
indexed_tokens_1 = tokenizer.encode(text_1)
|
||||
indexed_tokens_2 = tokenizer.encode(text_2)
|
||||
tokens_tensor_1 = torch.tensor([indexed_tokens_1])
|
||||
tokens_tensor_2 = torch.tensor([indexed_tokens_2])
|
||||
|
||||
# Load xlnetLMHeadModel
|
||||
>>> model = torch.hub.load('huggingface/pytorch-transformers', 'xlnetLMHeadModel', 'xlnet-large-cased')
|
||||
>>> model.eval()
|
||||
model = torch.hub.load('huggingface/pytorch-transformers', 'xlnetLMHeadModel', 'xlnet-large-cased')
|
||||
model.eval()
|
||||
|
||||
# Predict hidden states features for each layer
|
||||
>>> with torch.no_grad():
|
||||
with torch.no_grad():
|
||||
predictions_1, mems = model(tokens_tensor_1)
|
||||
predictions_2, mems = model(tokens_tensor_2, mems=mems)
|
||||
|
||||
# Get the predicted last token
|
||||
>>> predicted_index = torch.argmax(predictions_2[0, -1, :]).item()
|
||||
>>> predicted_token = tokenizer.decode([predicted_index])
|
||||
>>> assert predicted_token == ' who'
|
||||
predicted_index = torch.argmax(predictions_2[0, -1, :]).item()
|
||||
predicted_token = tokenizer.decode([predicted_index])
|
||||
assert predicted_token == ' who'
|
||||
"""
|
||||
model = XLNetLMHeadModel.from_pretrained(*args, **kwargs)
|
||||
return model
|
||||
@@ -144,25 +144,25 @@ def xlnetLMHeadModel(*args, **kwargs):
|
||||
|
||||
# Example:
|
||||
# # Load the tokenizer
|
||||
# >>> import torch
|
||||
# >>> tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'xlnetTokenizer', 'xlnet-large-cased')
|
||||
# import torch
|
||||
# tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'xlnetTokenizer', 'xlnet-large-cased')
|
||||
|
||||
# # Prepare tokenized input
|
||||
# >>> text1 = "Who was Jim Henson ? Jim Henson was a puppeteer"
|
||||
# >>> text2 = "Who was Jim Henson ? Jim Henson was a mysterious young man"
|
||||
# >>> tokenized_text1 = tokenizer.tokenize(text1)
|
||||
# >>> tokenized_text2 = tokenizer.tokenize(text2)
|
||||
# >>> indexed_tokens1 = tokenizer.convert_tokens_to_ids(tokenized_text1)
|
||||
# >>> indexed_tokens2 = tokenizer.convert_tokens_to_ids(tokenized_text2)
|
||||
# >>> tokens_tensor = torch.tensor([[indexed_tokens1, indexed_tokens2]])
|
||||
# >>> mc_token_ids = torch.LongTensor([[len(tokenized_text1)-1, len(tokenized_text2)-1]])
|
||||
# text1 = "Who was Jim Henson ? Jim Henson was a puppeteer"
|
||||
# text2 = "Who was Jim Henson ? Jim Henson was a mysterious young man"
|
||||
# tokenized_text1 = tokenizer.tokenize(text1)
|
||||
# tokenized_text2 = tokenizer.tokenize(text2)
|
||||
# indexed_tokens1 = tokenizer.convert_tokens_to_ids(tokenized_text1)
|
||||
# indexed_tokens2 = tokenizer.convert_tokens_to_ids(tokenized_text2)
|
||||
# tokens_tensor = torch.tensor([[indexed_tokens1, indexed_tokens2]])
|
||||
# mc_token_ids = torch.LongTensor([[len(tokenized_text1)-1, len(tokenized_text2)-1]])
|
||||
|
||||
# # Load xlnetForSequenceClassification
|
||||
# >>> model = torch.hub.load('huggingface/pytorch-transformers', 'xlnetForSequenceClassification', 'xlnet-large-cased')
|
||||
# >>> model.eval()
|
||||
# model = torch.hub.load('huggingface/pytorch-transformers', 'xlnetForSequenceClassification', 'xlnet-large-cased')
|
||||
# model.eval()
|
||||
|
||||
# # Predict sequence classes logits
|
||||
# >>> with torch.no_grad():
|
||||
# with torch.no_grad():
|
||||
# lm_logits, mems = model(tokens_tensor)
|
||||
# """
|
||||
# model = XLNetForSequenceClassification.from_pretrained(*args, **kwargs)
|
||||
|
||||
Reference in New Issue
Block a user