From d7906165a329c17e6e49d5069e9b21fa37d50773 Mon Sep 17 00:00:00 2001 From: Oren Amsalem Date: Tue, 5 Nov 2019 10:34:27 +0200 Subject: [PATCH] add progress bar for convert_examples_to_features It takes considerate amount of time (~10 min) to parse the examples to features, it is good to have a progress-bar to track this --- examples/utils_squad.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/utils_squad.py b/examples/utils_squad.py index 6d1c86493d..c04dacf6d3 100644 --- a/examples/utils_squad.py +++ b/examples/utils_squad.py @@ -23,6 +23,7 @@ import logging import math import collections from io import open +from tqdm import tqdm from transformers.tokenization_bert import BasicTokenizer, whitespace_tokenize @@ -202,7 +203,7 @@ def convert_examples_to_features(examples, tokenizer, max_seq_length, # f = np.zeros((max_N, max_M), dtype=np.float32) features = [] - for (example_index, example) in enumerate(examples): + for (example_index, example) in enumerate(tqdm(examples)): # if example_index % 100 == 0: # logger.info('Converting %s/%s pos %s neg %s', example_index, len(examples), cnt_pos, cnt_neg)