SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

[seq2seq] make it easier to run the scripts (#7274)

Browse Source
This commit is contained in:
Stas Bekman
2020-09-24 12:23:48 -07:00
committed by GitHub
parent 8d3bb781ee
commit eadd870b2f
18 changed files with 50 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
examples/seq2seq/test_datasets.py
View File

@@ -6,14 +6,13 @@ import numpy as np
import pytest
from torch.utils.data import DataLoader
from pack_dataset import pack_data_dir
from save_len_file import save_len_file
from test_seq2seq_examples import ARTICLES, BART_TINY, MARIAN_TINY, MBART_TINY, SUMMARIES, T5_TINY, make_test_data_dir
from transformers import AutoTokenizer
from transformers.modeling_bart import shift_tokens_right
from transformers.testing_utils import slow
from .pack_dataset import pack_data_dir
from .save_len_file import save_len_file
from .test_seq2seq_examples import ARTICLES, BART_TINY, MARIAN_TINY, MBART_TINY, SUMMARIES, T5_TINY, make_test_data_dir
from .utils import FAIRSEQ_AVAILABLE, DistributedSortishSampler, LegacySeq2SeqDataset, Seq2SeqDataset
from utils import FAIRSEQ_AVAILABLE, DistributedSortishSampler, LegacySeq2SeqDataset, Seq2SeqDataset
BERT_BASE_CASED = "bert-base-cased"