From 2c05b8a56c595b094b34f32f30bb477725f8a8fd Mon Sep 17 00:00:00 2001 From: Funtowicz Morgan Date: Mon, 20 Apr 2020 20:58:52 +0000 Subject: [PATCH] Remove tqdm logging when using pipelines. (#3833) Introduce tqdm_enabled parameter on squad_convert_examples_to_features() default to True and set to False in QA pipelines. --- src/transformers/data/processors/squad.py | 15 +++++++++++++-- src/transformers/pipelines.py | 1 + 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/transformers/data/processors/squad.py b/src/transformers/data/processors/squad.py index 782af453e3..2ed78ccb12 100644 --- a/src/transformers/data/processors/squad.py +++ b/src/transformers/data/processors/squad.py @@ -263,7 +263,15 @@ def squad_convert_example_to_features_init(tokenizer_for_convert): def squad_convert_examples_to_features( - examples, tokenizer, max_seq_length, doc_stride, max_query_length, is_training, return_dataset=False, threads=1 + examples, + tokenizer, + max_seq_length, + doc_stride, + max_query_length, + is_training, + return_dataset=False, + threads=1, + tqdm_enabled=True, ): """ Converts a list of examples into a list of features that can be directly given as input to a model. @@ -316,12 +324,15 @@ def squad_convert_examples_to_features( p.imap(annotate_, examples, chunksize=32), total=len(examples), desc="convert squad examples to features", + disable=not tqdm_enabled, ) ) new_features = [] unique_id = 1000000000 example_index = 0 - for example_features in tqdm(features, total=len(features), desc="add example index and unique id"): + for example_features in tqdm( + features, total=len(features), desc="add example index and unique id", disable=not tqdm_enabled + ): if not example_features: continue for example_feature in example_features: diff --git a/src/transformers/pipelines.py b/src/transformers/pipelines.py index 3f8450ac14..761930c367 100755 --- a/src/transformers/pipelines.py +++ b/src/transformers/pipelines.py @@ -958,6 +958,7 @@ class QuestionAnsweringPipeline(Pipeline): kwargs["doc_stride"], kwargs["max_question_len"], False, + tqdm_enabled=False, ) for example in examples ]