SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Fix TypeError: Object of type int64 is not JSON serializable (#24340)

Browse Source 
* Fix TypeError: Object of type int64 is not JSON serializable

* Convert numpy.float64 and numpy.int64 to float and int for json serialization

* Black reformatted examples/pytorch/token-classification/run_ner_no_trainer.py

* * make style
This commit is contained in:
Xiaoli Wang
2023-06-27 19:15:49 +08:00
committed by GitHub
parent ac19871ce2
commit 239ace152b
44 changed files with 74 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
tests/generation/test_framework_agnostic.py
View File

@@ -647,7 +647,7 @@ class GenerationIntegrationTestsMixin:
generated_tokens = model.generate(**tokens, eos_token_id=eos_token_id, **generation_kwargs)
unpadded_correct_condition = expectation == len(generated_tokens[0])
padded_correct_condition = expectation < len(generated_tokens[0]) and all(
[token == model.config.pad_token_id for token in generated_tokens[0][expectation:]]
token == model.config.pad_token_id for token in generated_tokens[0][expectation:]
)
self.assertTrue(unpadded_correct_condition or padded_correct_condition)
@@ -655,7 +655,7 @@ class GenerationIntegrationTestsMixin:
generated_tokens = model.generate(**tokens, eos_token_id=eos_token_id, **generation_kwargs)
unpadded_correct_condition = expectation == len(generated_tokens[0])
padded_correct_condition = expectation < len(generated_tokens[0]) and all(
[token == model.config.pad_token_id for token in generated_tokens[0][expectation:]]
token == model.config.pad_token_id for token in generated_tokens[0][expectation:]
)
self.assertTrue(unpadded_correct_condition or padded_correct_condition)

2
tests/models/codegen/test_modeling_codegen.py
View File

@@ -521,7 +521,7 @@ class CodeGenModelLanguageGenerationTest(unittest.TestCase):
self.assertEqual(output_str, EXPECTED_OUTPUT_STR)
self.assertTrue(
all([output_seq_strs[idx] != output_seq_tt_strs[idx] for idx in range(len(output_seq_tt_strs))])
all(output_seq_strs[idx] != output_seq_tt_strs[idx] for idx in range(len(output_seq_tt_strs)))
) # token_type_ids should change output
@is_flaky(max_attempts=3, description="measure of timing is somehow flaky.")

2
tests/models/data2vec/test_modeling_data2vec_audio.py
View File

@@ -516,7 +516,7 @@ class Data2VecAudioModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.Tes
"objective.weight",
]
if param.requires_grad:
if any([x in name for x in uniform_init_parms]):
if any(x in name for x in uniform_init_parms):
self.assertTrue(
-1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0,
msg=f"Parameter {name} of model {model_class} seems not properly initialized",

4
tests/models/encodec/test_modeling_encodec.py
View File

@@ -373,12 +373,12 @@ class EncodecModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase)
uniform_init_parms = ["conv"]
ignore_init = ["lstm"]
if param.requires_grad:
if any([x in name for x in uniform_init_parms]):
if any(x in name for x in uniform_init_parms):
self.assertTrue(
-1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0,
msg=f"Parameter {name} of model {model_class} seems not properly initialized",
)
elif not any([x in name for x in ignore_init]):
elif not any(x in name for x in ignore_init):
self.assertIn(
((param.data.mean() * 1e9).round() / 1e9).item(),
[0.0, 1.0],

2
tests/models/gpt2/test_modeling_gpt2.py
View File

@@ -768,7 +768,7 @@ class GPT2ModelLanguageGenerationTest(unittest.TestCase):
)
self.assertEqual(output_str, EXPECTED_OUTPUT_STR)
self.assertTrue(
all([output_seq_strs[idx] != output_seq_tt_strs[idx] for idx in range(len(output_seq_tt_strs))])
all(output_seq_strs[idx] != output_seq_tt_strs[idx] for idx in range(len(output_seq_tt_strs)))
) # token_type_ids should change output
@slow

2
tests/models/gptj/test_modeling_gptj.py
View File

@@ -571,7 +571,7 @@ class GPTJModelLanguageGenerationTest(unittest.TestCase):
self.assertEqual(output_str, EXPECTED_OUTPUT_STR)
self.assertTrue(
all([output_seq_strs[idx] != output_seq_tt_strs[idx] for idx in range(len(output_seq_tt_strs))])
all(output_seq_strs[idx] != output_seq_tt_strs[idx] for idx in range(len(output_seq_tt_strs)))
) # token_type_ids should change output
@slow

4
tests/models/hubert/test_modeling_hubert.py
View File

@@ -423,7 +423,7 @@ class HubertModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase):
"quantizer.weight_proj.weight",
]
if param.requires_grad:
if any([x in name for x in uniform_init_parms]):
if any(x in name for x in uniform_init_parms):
self.assertTrue(
-1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0,
msg=f"Parameter {name} of model {model_class} seems not properly initialized",
@@ -684,7 +684,7 @@ class HubertRobustModelTest(ModelTesterMixin, unittest.TestCase):
"quantizer.weight_proj.weight",
]
if param.requires_grad:
if any([x in name for x in uniform_init_parms]):
if any(x in name for x in uniform_init_parms):
self.assertTrue(
-1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0,
msg=f"Parameter {name} of model {model_class} seems not properly initialized",

4
tests/models/mctct/test_modeling_mctct.py
View File

@@ -386,7 +386,7 @@ class MCTCTModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase):
"objective.weight",
]
if param.requires_grad:
if any([x in name for x in uniform_init_parms]):
if any(x in name for x in uniform_init_parms):
self.assertTrue(
-1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0,
msg=f"Parameter {name} of model {model_class} seems not properly initialized",
@@ -533,7 +533,7 @@ class MCTCTRobustModelTest(ModelTesterMixin, unittest.TestCase):
"objective.weight",
]
if param.requires_grad:
if any([x in name for x in uniform_init_parms]):
if any(x in name for x in uniform_init_parms):
self.assertTrue(
-1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0,
msg=f"Parameter {name} of model {model_class} seems not properly initialized",

2
tests/models/rwkv/test_modeling_rwkv.py
View File

@@ -334,7 +334,7 @@ class RwkvModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin
if param.requires_grad:
# check if it's a ones like
self.assertTrue(torch.allclose(param.data, torch.ones_like(param.data), atol=1e-5, rtol=1e-5))
elif any([x in name for x in ["time_mix_key", "time_mix_receptance"]]):
elif any(x in name for x in ["time_mix_key", "time_mix_receptance"]):
if param.requires_grad:
self.assertInterval(
param.data,

2
tests/models/sew/test_modeling_sew.py
View File

@@ -417,7 +417,7 @@ class SEWModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase):
"quantizer.weight_proj.weight",
]
if param.requires_grad:
if any([x in name for x in uniform_init_parms]):
if any(x in name for x in uniform_init_parms):
self.assertTrue(
-1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0,
msg=f"Parameter {name} of model {model_class} seems not properly initialized",

2
tests/models/sew_d/test_modeling_sew_d.py
View File

@@ -431,7 +431,7 @@ class SEWDModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase):
"quantizer.weight_proj.weight",
]
if param.requires_grad:
if any([x in name for x in uniform_init_parms]):
if any(x in name for x in uniform_init_parms):
self.assertTrue(
-1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0,
msg=f"Parameter {name} of model {model_class} seems not properly initialized",

6
tests/models/speecht5/test_modeling_speecht5.py
View File

@@ -583,7 +583,7 @@ class SpeechT5ForSpeechToTextTest(ModelTesterMixin, unittest.TestCase):
"feature_projection.projection.bias",
]
if param.requires_grad:
if any([x in name for x in uniform_init_parms]):
if any(x in name for x in uniform_init_parms):
self.assertTrue(
-1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0,
msg=f"Parameter {name} of model {model_class} seems not properly initialized",
@@ -927,7 +927,7 @@ class SpeechT5ForTextToSpeechTest(ModelTesterMixin, unittest.TestCase):
"conv.weight",
]
if param.requires_grad:
if any([x in name for x in uniform_init_parms]):
if any(x in name for x in uniform_init_parms):
self.assertTrue(
-1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0,
msg=f"Parameter {name} of model {model_class} seems not properly initialized",
@@ -1337,7 +1337,7 @@ class SpeechT5ForSpeechToSpeechTest(ModelTesterMixin, unittest.TestCase):
"feature_projection.projection.bias",
]
if param.requires_grad:
if any([x in name for x in uniform_init_parms]):
if any(x in name for x in uniform_init_parms):
self.assertTrue(
-1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0,
msg=f"Parameter {name} of model {model_class} seems not properly initialized",

2
tests/models/unispeech/test_modeling_unispeech.py
View File

@@ -432,7 +432,7 @@ class UniSpeechRobustModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.T
"feature_projection.projection.bias",
]
if param.requires_grad:
if any([x in name for x in uniform_init_parms]):
if any(x in name for x in uniform_init_parms):
self.assertTrue(
-1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0,
msg=f"Parameter {name} of model {model_class} seems not properly initialized",

4
tests/models/unispeech_sat/test_modeling_unispeech_sat.py
View File

@@ -484,7 +484,7 @@ class UniSpeechSatModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.Test
"objective.weight",
]
if param.requires_grad:
if any([x in name for x in uniform_init_parms]):
if any(x in name for x in uniform_init_parms):
self.assertTrue(
-1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0,
msg=f"Parameter {name} of model {model_class} seems not properly initialized",
@@ -695,7 +695,7 @@ class UniSpeechSatRobustModelTest(ModelTesterMixin, unittest.TestCase):
"objective.weight",
]
if param.requires_grad:
if any([x in name for x in uniform_init_parms]):
if any(x in name for x in uniform_init_parms):
self.assertTrue(
-1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0,
msg=f"Parameter {name} of model {model_class} seems not properly initialized",

2
tests/models/wav2vec2/test_modeling_flax_wav2vec2.py
View File

@@ -464,7 +464,7 @@ class FlaxWav2Vec2UtilsTest(unittest.TestCase):
negative_indices = _sample_negative_indices(features.shape, num_negatives, attention_mask=attention_mask)
# make sure that no padding tokens are sampled
self.assertTrue(all([idx not in negative_indices for idx in forbidden_indices]))
self.assertTrue(all(idx not in negative_indices for idx in forbidden_indices))
features = features.reshape(-1, hidden_size) # BTC => (BxT)C
# take negative vectors from sampled indices

4
tests/models/wav2vec2/test_modeling_wav2vec2.py
View File

@@ -637,7 +637,7 @@ class Wav2Vec2ModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase
"objective.weight",
]
if param.requires_grad:
if any([x in name for x in uniform_init_parms]):
if any(x in name for x in uniform_init_parms):
self.assertTrue(
-1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0,
msg=f"Parameter {name} of model {model_class} seems not properly initialized",
@@ -971,7 +971,7 @@ class Wav2Vec2RobustModelTest(ModelTesterMixin, unittest.TestCase):
"objective.weight",
]
if param.requires_grad:
if any([x in name for x in uniform_init_parms]):
if any(x in name for x in uniform_init_parms):
self.assertTrue(
-1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0,
msg=f"Parameter {name} of model {model_class} seems not properly initialized",

2
tests/models/wav2vec2_conformer/test_modeling_wav2vec2_conformer.py
View File

@@ -569,7 +569,7 @@ class Wav2Vec2ConformerModelTest(ModelTesterMixin, PipelineTesterMixin, unittest
"objective.weight",
]
if param.requires_grad:
if any([x in name for x in uniform_init_parms]):
if any(x in name for x in uniform_init_parms):
self.assertTrue(
-1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0,
msg=f"Parameter {name} of model {model_class} seems not properly initialized",

2
tests/models/wavlm/test_modeling_wavlm.py
View File

@@ -438,7 +438,7 @@ class WavLMModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase):
"objective.weight",
]
if param.requires_grad:
if any([x in name for x in uniform_init_parms]):
if any(x in name for x in uniform_init_parms):
self.assertTrue(
-1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0,
msg=f"Parameter {name} of model {model_class} seems not properly initialized",

2
tests/models/whisper/test_modeling_whisper.py
View File

@@ -1535,7 +1535,7 @@ class WhisperModelIntegrationTests(unittest.TestCase):
text = processor.decode(output[0])
self.assertTrue(prompt in text)
self.assertTrue(all([token in text for token in expected_tokens]))
self.assertTrue(all(token in text for token in expected_tokens))
@slow
def test_generate_with_prompt_ids_and_no_non_prompt_forced_decoder_ids(self):

2
tests/onnx/test_onnx.py
View File

@@ -145,7 +145,7 @@ class OnnxExportTestCase(unittest.TestCase):
# Assert all variables are present
self.assertEqual(len(shapes), len(variable_names))
self.assertTrue(all([var_name in shapes for var_name in variable_names]))
self.assertTrue(all(var_name in shapes for var_name in variable_names))
self.assertSequenceEqual(variable_names[:3], input_vars)
self.assertSequenceEqual(variable_names[3:], output_vars)

2
tests/test_modeling_tf_common.py
View File

@@ -1566,7 +1566,7 @@ class TFModelTesterMixin:
return_labels=True if "labels" in inspect.signature(model_class.call).parameters.keys() else False,
)
if not any(
[tensor.dtype.is_integer for tensor in prepared_for_class.values() if isinstance(tensor, tf.Tensor)]
tensor.dtype.is_integer for tensor in prepared_for_class.values() if isinstance(tensor, tf.Tensor)
):
return # No integer inputs means no need for this test

2
tests/test_tokenization_common.py
View File

@@ -79,7 +79,7 @@ SMALL_TRAINING_CORPUS = [
def filter_non_english(_, pretrained_name: str):
"""Filter all the model for non-english language"""
return not any([lang in pretrained_name for lang in NON_ENGLISH_TAGS])
return not any(lang in pretrained_name for lang in NON_ENGLISH_TAGS)
def filter_roberta_detectors(_, pretrained_name: str):

4
tests/trainer/test_trainer_seq2seq.py
View File

@@ -56,8 +56,8 @@ class Seq2seqTrainerTester(TestCasePlus):
]
batch["decoder_attention_mask"] = outputs.attention_mask
assert all([len(x) == 512 for x in inputs.input_ids])
assert all([len(x) == 128 for x in outputs.input_ids])
assert all(len(x) == 512 for x in inputs.input_ids)
assert all(len(x) == 128 for x in outputs.input_ids)
return batch