SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Fix docs and bad word tokens generation_utils.py (#6387)

Browse Source 
* fix

* fix2

* fix3
This commit is contained in:
Zhu Baohe
2020-08-13 19:12:16 +08:00
committed by GitHub
parent 0ed7c00ba6
commit 9d94aecd51
2 changed files with 6 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/transformers/generation_tf_utils.py
View File

@@ -163,7 +163,7 @@ class TFGenerationMixin:
model = TFAutoModelWithLMHead.from_pretrained('distilgpt2') # Download model and configuration from S3 and cache. model = TFAutoModelWithLMHead.from_pretrained('distilgpt2') # Download model and configuration from S3 and cache.
input_context = 'The dog' input_context = 'The dog'
input_ids = tokenizer.encode(input_context, return_tensors='tf') # encode input context input_ids = tokenizer.encode(input_context, return_tensors='tf') # encode input context
outputs = model.generate(input_ids=input_ids, max_length=40, temperature=0.7, num_return_sequences=3) # 3 generate sequences using by sampling outputs = model.generate(input_ids=input_ids, max_length=40, temperature=0.7, num_return_sequences=3, do_sample=True) # generate 3 candidates using sampling
for i in range(3): # 3 output sequences were generated for i in range(3): # 3 output sequences were generated
print('Generated {}: {}'.format(i, tokenizer.decode(outputs[i], skip_special_tokens=True))) print('Generated {}: {}'.format(i, tokenizer.decode(outputs[i], skip_special_tokens=True)))
@@ -936,8 +936,8 @@ def calc_banned_bad_words_ids(prev_input_ids, bad_words_ids):
if len(tokens) == 0: if len(tokens) == 0:
# if bad word tokens is just one token always ban it # if bad word tokens is just one token always ban it
return True return True
if len(tokens) > len(prev_input_ids): if len(tokens) > len(prev_tokens):
# if bad word tokens are longer then prev input_ids they can't be equal # if bad word tokens are longer than prev tokens they can't be equal
return False return False
if prev_tokens[-len(tokens) :] == tokens: if prev_tokens[-len(tokens) :] == tokens:

6
src/transformers/generation_utils.py
View File

@@ -226,7 +226,7 @@ class GenerationMixin:
model = AutoModelWithLMHead.from_pretrained('distilgpt2') # Download model and configuration from S3 and cache. model = AutoModelWithLMHead.from_pretrained('distilgpt2') # Download model and configuration from S3 and cache.
input_context = 'The dog' input_context = 'The dog'
input_ids = tokenizer.encode(input_context, return_tensors='pt') # encode input context input_ids = tokenizer.encode(input_context, return_tensors='pt') # encode input context
outputs = model.generate(input_ids=input_ids, max_length=40, temperature=0.7, num_return_sequences=3) # 3 generate sequences using by sampling outputs = model.generate(input_ids=input_ids, max_length=40, temperature=0.7, num_return_sequences=3, do_sample=True) # generate 3 candidates using sampling
for i in range(3): # 3 output sequences were generated for i in range(3): # 3 output sequences were generated
print('Generated {}: {}'.format(i, tokenizer.decode(outputs[i], skip_special_tokens=True))) print('Generated {}: {}'.format(i, tokenizer.decode(outputs[i], skip_special_tokens=True)))
@@ -876,8 +876,8 @@ def calc_banned_bad_words_ids(prev_input_ids: Iterable[int], bad_words_ids: Iter
if len(tokens) == 0: if len(tokens) == 0:
# if bad word tokens is just one token always ban it # if bad word tokens is just one token always ban it
return True return True
if len(tokens) > len(prev_input_ids): if len(tokens) > len(prev_tokens):
# if bad word tokens are longer then prev input_ids they can't be equal # if bad word tokens are longer than prev tokens they can't be equal
return False return False
if prev_tokens[-len(tokens) :] == tokens: if prev_tokens[-len(tokens) :] == tokens: