SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Remove unused variables in examples.

Browse Source
This commit is contained in:
Aymeric Augustin
2019-12-23 22:23:44 +01:00
parent 072750f4dc
commit 81422c4e6d
4 changed files with 2 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
examples/contrib/run_transfo_xl.py
View File

@@ -28,7 +28,7 @@ import time
import torch
from transformers import TransfoXLCorpus, TransfoXLLMHeadModel, TransfoXLTokenizer
from transformers import TransfoXLCorpus, TransfoXLLMHeadModel
logging.basicConfig(
@@ -73,9 +73,7 @@ def main():
# The pre-processing involve computing word frequencies to prepare the Adaptive input and SoftMax
# and tokenizing the dataset
# The pre-processed corpus is a convertion (using the conversion script )
tokenizer = TransfoXLTokenizer.from_pretrained(args.model_name)
corpus = TransfoXLCorpus.from_pretrained(args.model_name)
ntokens = len(corpus.vocab)
va_iter = corpus.get_iterator("valid", args.batch_size, args.tgt_len, device=device, ext_len=args.ext_len)
te_iter = corpus.get_iterator("test", args.batch_size, args.tgt_len, device=device, ext_len=args.ext_len)