From 1761d2091a898103a07fc2cd4c02e28bc5b72be0 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Tue, 24 Sep 2019 14:59:10 -0400 Subject: [PATCH] Check to see if the models have the same results when in eval mode (pt) or when training=False (tf) --- pytorch_transformers/tests/modeling_common_test.py | 10 ++++++++++ pytorch_transformers/tests/modeling_tf_common_test.py | 8 ++++++++ 2 files changed, 18 insertions(+) diff --git a/pytorch_transformers/tests/modeling_common_test.py b/pytorch_transformers/tests/modeling_common_test.py index ff321193a5..5e30cd1e32 100644 --- a/pytorch_transformers/tests/modeling_common_test.py +++ b/pytorch_transformers/tests/modeling_common_test.py @@ -68,6 +68,16 @@ class CommonTestCases: self.assertIn(param.data.mean().item(), [0.0, 1.0], msg="Parameter {} of model {} seems not properly initialized".format(name, model_class)) + def test_determinism(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + model = model_class(config) + model.eval() + first, second = model(inputs_dict["input_ids"])[0], model(inputs_dict["input_ids"])[0] + self.assertEqual(first.ne(second).sum().item(), 0) + + def test_attention_outputs(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() diff --git a/pytorch_transformers/tests/modeling_tf_common_test.py b/pytorch_transformers/tests/modeling_tf_common_test.py index ecd1e387f9..5e7d29cb7f 100644 --- a/pytorch_transformers/tests/modeling_tf_common_test.py +++ b/pytorch_transformers/tests/modeling_tf_common_test.py @@ -298,6 +298,14 @@ class TFCommonTestCases: # self.assertGreater(len(params_not_tied), len(params_tied)) # self.assertEqual(len(params_tied_2), len(params_tied)) + def test_determinism(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + model = model_class(config) + first, second = model(inputs_dict, training=False)[0], model(inputs_dict, training=False)[0] + self.assertTrue(tf.math.equal(first, second).numpy().all()) + def ids_tensor(shape, vocab_size, rng=None, name=None, dtype=None): """Creates a random int32 tensor of the shape within the vocab size."""