SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Fixed horizon_length for PPLM (#13886)

Browse Source 
* fixed horizon_length

* fixed horizon_length

* fix style
This commit is contained in:
jacksukk
2021-10-15 09:46:09 +08:00
committed by GitHub
parent 5b317f7ea4
commit d5b82bb70c
1 changed files with 8 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
examples/research_projects/pplm/run_pplm.py
View File

@@ -181,7 +181,14 @@ def perturb_past(
for _ in range(horizon_length): for _ in range(horizon_length):
inputs_embeds = torch.matmul(curr_probs, wte.weight.data) inputs_embeds = torch.matmul(curr_probs, wte.weight.data)
lm_output = model(past_key_values=curr_unpert_past, inputs_embeds=inputs_embeds) lm_output = model(past_key_values=curr_unpert_past, inputs_embeds=inputs_embeds)
curr_unpert_past, curr_all_hidden = lm_output["past_key_values"], lm_output["hidden_states"] curr_all_logits, curr_unpert_past, curr_all_hidden = (
lm_output["logits"],
lm_output["past_key_values"],
lm_output["hidden_states"],
)
curr_logits = curr_all_logits[:, -1, :]
curr_probs = nn.functional.softmax(curr_logits, dim=-1)
curr_probs = torch.unsqueeze(curr_probs, dim=1)
curr_hidden = curr_all_hidden[-1] curr_hidden = curr_all_hidden[-1]
new_accumulated_hidden = new_accumulated_hidden + torch.sum(curr_hidden, dim=1) new_accumulated_hidden = new_accumulated_hidden + torch.sum(curr_hidden, dim=1)