From 4a3f1a686fcda27efc19b8b3a87b62a338c2ad86 Mon Sep 17 00:00:00 2001 From: Mohamed Abu El-Nasr <64566340+abuelnasr0@users.noreply.github.com> Date: Thu, 10 Oct 2024 15:44:05 +0300 Subject: [PATCH] check if eigenvalues of covariance matrix are complex. (#34037) check if eigenvalues of covariance complex for psd checking --- src/transformers/modeling_utils.py | 3 ++- tests/models/reformer/test_modeling_reformer.py | 4 ---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 32561dad41..cb0d743b0a 100755 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -2441,8 +2441,9 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin, GenerationMixin, PushToHubMix covariance = old_centered_embeddings.T @ old_centered_embeddings / old_num_tokens # Check if the covariance is positive definite. + eigenvalues = torch.linalg.eigvals(covariance) is_covariance_psd = bool( - (covariance == covariance.T).all() and (torch.linalg.eigvals(covariance).real >= 0).all() + (covariance == covariance.T).all() and not torch.is_complex(eigenvalues) and (eigenvalues > 0).all() ) if is_covariance_psd: # If covariances is positive definite, a distribution can be created. and we can sample new weights from it. diff --git a/tests/models/reformer/test_modeling_reformer.py b/tests/models/reformer/test_modeling_reformer.py index cc36b01287..774831791f 100644 --- a/tests/models/reformer/test_modeling_reformer.py +++ b/tests/models/reformer/test_modeling_reformer.py @@ -694,10 +694,6 @@ class ReformerLocalAttnModelTest(ReformerTesterMixin, GenerationTesterMixin, Mod self.model_tester.seq_length = original_sequence_length return test_inputs - @unittest.skip(reason="Resizing sometimes goes bad") # not worth investigating for now (it's not a popular model) - def test_resize_tokens_embeddings(self): - pass - @require_torch class ReformerLSHAttnModelTest(