SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

add special tokens to pretrain configs of respective lm head models

Browse Source
This commit is contained in:
Patrick von Platen
2020-02-25 16:37:56 +01:00
parent e693cd1e87
commit e645dcbb70
4 changed files with 5 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
tests/test_modeling_transfo_xl.py
View File

@@ -214,14 +214,8 @@ class TransfoXLModelTest(ModelTesterMixin, unittest.TestCase):
self.assertIsNotNone(model)
def prepare_generation_special_tokens():
return {"eos_token_id": 0}
class TransfoXLModelLanguageGenerationTest(unittest.TestCase):
special_tokens = prepare_generation_special_tokens()
@slow
def test_lm_generate_transfo_xl_wt103(self):
model = TransfoXLLMHeadModel.from_pretrained("transfo-xl-wt103")
@@ -578,6 +572,5 @@ class TransfoXLModelLanguageGenerationTest(unittest.TestCase):
torch.manual_seed(0)
output_ids = model.generate(input_ids, eos_token_ids=self.special_tokens["eos_token_id"], max_length=200)
output_ids = model.generate(input_ids, max_length=200)
self.assertListEqual(output_ids[0].tolist(), expected_output_ids)