Upgrade examples to pl=0.8.1(#5146)

2020-06-22 20:40:10 -04:00
parent 06b60c8b05
commit f5c2a122e3
11 changed files with 53 additions and 150 deletions
--- a/examples/summarization/utils.py
+++ b/examples/summarization/utils.py
@@ -45,8 +45,10 @@ def encode_file(
            max_length=max_length,
            pad_to_max_length=pad_to_max_length,
            add_prefix_space=True,
+            truncation=True,
            return_tensors=return_tensors,
        )
+        assert tokenized.input_ids.shape[1] == max_length
        examples.append(tokenized)
    torch.save(lmap(dict, examples), cache_path.open("wb"))
    return examples