SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Fix TypeError: Object of type int64 is not JSON serializable (#24340)

Browse Source 
* Fix TypeError: Object of type int64 is not JSON serializable

* Convert numpy.float64 and numpy.int64 to float and int for json serialization

* Black reformatted examples/pytorch/token-classification/run_ner_no_trainer.py

* * make style
This commit is contained in:
Xiaoli Wang
2023-06-27 19:15:49 +08:00
committed by GitHub
parent ac19871ce2
commit 239ace152b
44 changed files with 74 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
examples/research_projects/fsner/src/fsner/tokenizer_utils.py
View File

@@ -17,7 +17,7 @@ class FSNERTokenizerUtils(object):
`transformers.tokenization_utils_base.BatchEncoding` dict with additional keys and values for start_token_id, end_token_id and sizes of example lists for each entity type
"""
if isinstance(x, list) and all([isinstance(_x, list) for _x in x]):
if isinstance(x, list) and all(isinstance(_x, list) for _x in x):
d = None
for l in x:
t = self.tokenizer(
@@ -37,7 +37,7 @@ class FSNERTokenizerUtils(object):
d["start_token_id"] = torch.tensor(self.tokenizer.convert_tokens_to_ids("[E]"))
d["end_token_id"] = torch.tensor(self.tokenizer.convert_tokens_to_ids("[/E]"))
elif isinstance(x, list) and all([isinstance(_x, str) for _x in x]):
elif isinstance(x, list) and all(isinstance(_x, str) for _x in x):
d = self.tokenizer(
x,
padding="max_length",