SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

add progress bar for convert_examples_to_features

Browse Source 
It takes considerate amount of time (~10 min) to parse the examples to features, it is good to have a progress-bar to track this
This commit is contained in:
Oren Amsalem
2019-11-05 10:34:27 +02:00
committed by GitHub
parent d2e2577dd3
commit d7906165a3
1 changed files with 2 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
examples/utils_squad.py
View File

@@ -23,6 +23,7 @@ import logging
import math import math
import collections import collections
from io import open from io import open
from tqdm import tqdm
from transformers.tokenization_bert import BasicTokenizer, whitespace_tokenize from transformers.tokenization_bert import BasicTokenizer, whitespace_tokenize
@@ -202,7 +203,7 @@ def convert_examples_to_features(examples, tokenizer, max_seq_length,
# f = np.zeros((max_N, max_M), dtype=np.float32) # f = np.zeros((max_N, max_M), dtype=np.float32)
features = [] features = []
for (example_index, example) in enumerate(examples): for (example_index, example) in enumerate(tqdm(examples)):
# if example_index % 100 == 0: # if example_index % 100 == 0:
# logger.info('Converting %s/%s pos %s neg %s', example_index, len(examples), cnt_pos, cnt_neg) # logger.info('Converting %s/%s pos %s neg %s', example_index, len(examples), cnt_pos, cnt_neg)