Apply ruff flake8-comprehensions (#21694)
This commit is contained in:
@@ -8,7 +8,7 @@ from transformers import AutoTokenizer, HfArgumentParser
|
||||
|
||||
|
||||
def tokenize(example):
|
||||
output = dict()
|
||||
output = {}
|
||||
output["input_ids"] = tokenizer(example["content"], truncation=False)["input_ids"]
|
||||
output["ratio_char_token"] = len(example["content"]) / len(output["input_ids"])
|
||||
return output
|
||||
|
||||
Reference in New Issue
Block a user