DataParallel for SQuAD + fix XLM

2019-12-10 19:21:20 +00:00
parent e6cff60b4c
commit dc4e9e5cb3
3 changed files with 15 additions and 2 deletions
--- a/transformers/data/metrics/squad_metrics.py
+++ b/transformers/data/metrics/squad_metrics.py
@@ -695,7 +695,12 @@ def compute_predictions_log_probs(
            tok_text = " ".join(tok_text.split())
            orig_text = " ".join(orig_tokens)

-            final_text = get_final_text(tok_text, orig_text, tokenizer.do_lower_case,
+            if hasattr(tokenizer, "do_lower_case"):
+                do_lower_case = tokenizer.do_lower_case
+            else:
+                do_lower_case = tokenizer.do_lowercase_and_remove_accent
+
+            final_text = get_final_text(tok_text, orig_text, do_lower_case,
                                        verbose_logging)

            if final_text in seen_predictions: