Set cache_dir for evaluate.load() in example scripts (#28422)
While using `run_clm.py`,[^1] I noticed that some files were being added
to my global cache, not the local cache. I set the `cache_dir` parameter
for the one call to `evaluate.load()`, which partially solved the
problem. I figured that while I was fixing the one script upstream, I
might as well fix the problem in all other example scripts that I could.
There are still some files being added to my global cache, but this
appears to be a bug in `evaluate` itself. This commit at least moves
some of the files into the local cache, which is better than before.
To create this PR, I made the following regex-based transformation:
`evaluate\.load\((.*?)\)` -> `evaluate\.load\($1,
cache_dir=model_args.cache_dir\)`. After using that, I manually fixed
all modified files with `ruff` serving as useful guidance. During the
process, I removed one existing usage of the `cache_dir` parameter in a
script that did not have a corresponding `--cache-dir` argument
declared.
[^1]: I specifically used `pytorch/language-modeling/run_clm.py` from
v4.34.1 of the library. For the original code, see the following URL:
acc394c4f5/examples/pytorch/language-modeling/run_clm.py.
This commit is contained in:
@@ -440,7 +440,7 @@ def main():
|
||||
collate_fn = DefaultDataCollator(return_tensors="np")
|
||||
|
||||
# Load the accuracy metric from the datasets package
|
||||
metric = evaluate.load("accuracy")
|
||||
metric = evaluate.load("accuracy", cache_dir=model_args.cache_dir)
|
||||
|
||||
# Define our compute_metrics function. It takes an `EvalPrediction` object (a namedtuple with a
|
||||
# predictions and label_ids field) and has to return a dictionary string to float.
|
||||
|
||||
@@ -631,7 +631,9 @@ def main():
|
||||
references = [{"id": ex["id"], "answers": ex[answer_column_name]} for ex in examples]
|
||||
return EvalPrediction(predictions=formatted_predictions, label_ids=references)
|
||||
|
||||
metric = evaluate.load("squad_v2" if data_args.version_2_with_negative else "squad")
|
||||
metric = evaluate.load(
|
||||
"squad_v2" if data_args.version_2_with_negative else "squad", cache_dir=model_args.cache_dir
|
||||
)
|
||||
|
||||
def compute_metrics(p: EvalPrediction):
|
||||
return metric.compute(predictions=p.predictions, references=p.label_ids)
|
||||
|
||||
@@ -627,7 +627,7 @@ def main():
|
||||
|
||||
# region Metric and KerasMetricCallback
|
||||
if training_args.do_eval:
|
||||
metric = evaluate.load("rouge")
|
||||
metric = evaluate.load("rouge", cache_dir=model_args.cache_dir)
|
||||
|
||||
if data_args.val_max_target_length is None:
|
||||
data_args.val_max_target_length = data_args.max_target_length
|
||||
|
||||
@@ -379,7 +379,7 @@ def main():
|
||||
# endregion
|
||||
|
||||
# region Metric function
|
||||
metric = evaluate.load("glue", data_args.task_name)
|
||||
metric = evaluate.load("glue", data_args.task_name, cache_dir=model_args.cache_dir)
|
||||
|
||||
def compute_metrics(preds, label_ids):
|
||||
preds = preds["logits"]
|
||||
|
||||
@@ -511,7 +511,7 @@ def main():
|
||||
# endregion
|
||||
|
||||
# Metrics
|
||||
metric = evaluate.load("seqeval")
|
||||
metric = evaluate.load("seqeval", cache_dir=model_args.cache_dir)
|
||||
|
||||
def get_labels(y_pred, y_true):
|
||||
# Transform predictions and references tensos to numpy arrays
|
||||
|
||||
@@ -589,7 +589,7 @@ def main():
|
||||
|
||||
# region Metric and postprocessing
|
||||
if training_args.do_eval:
|
||||
metric = evaluate.load("sacrebleu")
|
||||
metric = evaluate.load("sacrebleu", cache_dir=model_args.cache_dir)
|
||||
|
||||
if data_args.val_max_target_length is None:
|
||||
data_args.val_max_target_length = data_args.max_target_length
|
||||
|
||||
Reference in New Issue
Block a user