From 87e124c245b0206a695b48c13e5262557f81fb5e Mon Sep 17 00:00:00 2001 From: David LaPalomento Date: Tue, 11 Aug 2020 05:31:26 -0400 Subject: [PATCH] Warn if debug requested without TPU fixes (#6308) (#6390) * Warn if debug requested without TPU fixes (#6308) Check whether a PyTorch compatible TPU is available before attempting to print TPU metrics after training has completed. This way, users who apply `--debug` without reading the documentation aren't suprised by a stacktrace. * Style Co-authored-by: Lysandre --- src/transformers/trainer.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py index d8aeddb853..cf4a392817 100755 --- a/src/transformers/trainer.py +++ b/src/transformers/trainer.py @@ -624,8 +624,14 @@ class Trainer: train_iterator.close() break if self.args.tpu_metrics_debug or self.args.debug: - # tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.) - xm.master_print(met.metrics_report()) + if is_torch_tpu_available(): + # tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.) + xm.master_print(met.metrics_report()) + else: + logger.warning( + "You enabled PyTorch/XLA debug metrics but you don't have a TPU " + "configured. Check your training configuration if this is unexpected." + ) if self.tb_writer: self.tb_writer.close()