Remove tqdm logging when using pipelines. (#3833)
Introduce tqdm_enabled parameter on squad_convert_examples_to_features() default to True and set to False in QA pipelines.
This commit is contained in:
@@ -263,7 +263,15 @@ def squad_convert_example_to_features_init(tokenizer_for_convert):
|
|||||||
|
|
||||||
|
|
||||||
def squad_convert_examples_to_features(
|
def squad_convert_examples_to_features(
|
||||||
examples, tokenizer, max_seq_length, doc_stride, max_query_length, is_training, return_dataset=False, threads=1
|
examples,
|
||||||
|
tokenizer,
|
||||||
|
max_seq_length,
|
||||||
|
doc_stride,
|
||||||
|
max_query_length,
|
||||||
|
is_training,
|
||||||
|
return_dataset=False,
|
||||||
|
threads=1,
|
||||||
|
tqdm_enabled=True,
|
||||||
):
|
):
|
||||||
"""
|
"""
|
||||||
Converts a list of examples into a list of features that can be directly given as input to a model.
|
Converts a list of examples into a list of features that can be directly given as input to a model.
|
||||||
@@ -316,12 +324,15 @@ def squad_convert_examples_to_features(
|
|||||||
p.imap(annotate_, examples, chunksize=32),
|
p.imap(annotate_, examples, chunksize=32),
|
||||||
total=len(examples),
|
total=len(examples),
|
||||||
desc="convert squad examples to features",
|
desc="convert squad examples to features",
|
||||||
|
disable=not tqdm_enabled,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
new_features = []
|
new_features = []
|
||||||
unique_id = 1000000000
|
unique_id = 1000000000
|
||||||
example_index = 0
|
example_index = 0
|
||||||
for example_features in tqdm(features, total=len(features), desc="add example index and unique id"):
|
for example_features in tqdm(
|
||||||
|
features, total=len(features), desc="add example index and unique id", disable=not tqdm_enabled
|
||||||
|
):
|
||||||
if not example_features:
|
if not example_features:
|
||||||
continue
|
continue
|
||||||
for example_feature in example_features:
|
for example_feature in example_features:
|
||||||
|
|||||||
@@ -958,6 +958,7 @@ class QuestionAnsweringPipeline(Pipeline):
|
|||||||
kwargs["doc_stride"],
|
kwargs["doc_stride"],
|
||||||
kwargs["max_question_len"],
|
kwargs["max_question_len"],
|
||||||
False,
|
False,
|
||||||
|
tqdm_enabled=False,
|
||||||
)
|
)
|
||||||
for example in examples
|
for example in examples
|
||||||
]
|
]
|
||||||
|
|||||||
Reference in New Issue
Block a user