Remove tqdm logging when using pipelines. (#3833)
Introduce tqdm_enabled parameter on squad_convert_examples_to_features() default to True and set to False in QA pipelines.
This commit is contained in:
@@ -263,7 +263,15 @@ def squad_convert_example_to_features_init(tokenizer_for_convert):
|
||||
|
||||
|
||||
def squad_convert_examples_to_features(
|
||||
examples, tokenizer, max_seq_length, doc_stride, max_query_length, is_training, return_dataset=False, threads=1
|
||||
examples,
|
||||
tokenizer,
|
||||
max_seq_length,
|
||||
doc_stride,
|
||||
max_query_length,
|
||||
is_training,
|
||||
return_dataset=False,
|
||||
threads=1,
|
||||
tqdm_enabled=True,
|
||||
):
|
||||
"""
|
||||
Converts a list of examples into a list of features that can be directly given as input to a model.
|
||||
@@ -316,12 +324,15 @@ def squad_convert_examples_to_features(
|
||||
p.imap(annotate_, examples, chunksize=32),
|
||||
total=len(examples),
|
||||
desc="convert squad examples to features",
|
||||
disable=not tqdm_enabled,
|
||||
)
|
||||
)
|
||||
new_features = []
|
||||
unique_id = 1000000000
|
||||
example_index = 0
|
||||
for example_features in tqdm(features, total=len(features), desc="add example index and unique id"):
|
||||
for example_features in tqdm(
|
||||
features, total=len(features), desc="add example index and unique id", disable=not tqdm_enabled
|
||||
):
|
||||
if not example_features:
|
||||
continue
|
||||
for example_feature in example_features:
|
||||
|
||||
Reference in New Issue
Block a user