From 822c5e45b2353e10cd432251215aa61eb982ac05 Mon Sep 17 00:00:00 2001 From: Yuanyuan Chen Date: Mon, 21 Jul 2025 20:38:05 +0800 Subject: [PATCH] Fix pylint warnings (#39477) * Fix pylint warnings Signed-off-by: cyy * Fix variable names Signed-off-by: cyy --------- Signed-off-by: cyy --- src/transformers/commands/add_new_model_like.py | 4 ++-- src/transformers/generation/streamers.py | 14 +++++++------- src/transformers/modeling_gguf_pytorch_utils.py | 3 +-- src/transformers/modeling_utils.py | 6 +++--- src/transformers/models/bark/convert_suno_to_hf.py | 6 +++--- src/transformers/models/bert/tokenization_bert.py | 14 +++++++------- .../bert_japanese/tokenization_bert_japanese.py | 14 +++++++------- src/transformers/models/clip/tokenization_clip.py | 14 +++++++------- .../models/convbert/tokenization_convbert.py | 14 +++++++------- ...t_cvt_original_pytorch_checkpoint_to_pytorch.py | 2 +- .../models/deprecated/realm/tokenization_realm.py | 14 +++++++------- .../deprecated/retribert/tokenization_retribert.py | 14 +++++++------- .../models/distilbert/tokenization_distilbert.py | 14 +++++++------- .../models/electra/tokenization_electra.py | 14 +++++++------- .../models/emu3/convert_emu3_weights_to_hf.py | 2 +- .../models/falcon_h1/configuration_falcon_h1.py | 1 - .../models/funnel/tokenization_funnel.py | 14 +++++++------- .../models/herbert/tokenization_herbert.py | 14 +++++++------- .../models/imagegpt/modeling_imagegpt.py | 4 ++-- .../models/layoutlm/modeling_tf_layoutlm.py | 2 +- .../models/layoutlm/tokenization_layoutlm.py | 14 +++++++------- .../models/layoutlmv2/tokenization_layoutlmv2.py | 14 +++++++------- .../models/lxmert/tokenization_lxmert.py | 14 +++++++------- .../models/mobilebert/tokenization_mobilebert.py | 14 +++++++------- .../models/mpnet/tokenization_mpnet.py | 14 +++++++------- .../models/openai/tokenization_openai.py | 14 +++++++------- .../owlvit/convert_owlvit_original_flax_to_hf.py | 2 +- .../paligemma/convert_paligemma_weights_to_hf.py | 3 --- .../convert_pix2struct_original_pytorch_to_hf.py | 6 +++--- .../models/prophetnet/tokenization_prophetnet.py | 14 +++++++------- .../models/roc_bert/tokenization_roc_bert.py | 14 +++++++------- .../models/roformer/tokenization_roformer.py | 14 +++++++------- .../models/splinter/tokenization_splinter.py | 14 +++++++------- .../models/squeezebert/tokenization_squeezebert.py | 14 +++++++------- .../switch_transformers/convert_big_switch.py | 4 ++-- .../models/tapas/tokenization_tapas.py | 14 +++++++------- .../umt5/convert_umt5_checkpoint_to_pytorch.py | 2 +- src/transformers/processing_utils.py | 14 +++++++------- src/transformers/quantizers/quantizer_hqq.py | 4 ++-- src/transformers/utils/attention_visualizer.py | 6 ++---- tests/models/groupvit/test_modeling_groupvit.py | 2 +- tests/test_tokenization_common.py | 4 +--- utils/check_bad_commit.py | 1 + utils/check_self_hosted_runner.py | 2 +- utils/patch_helper.py | 7 +++++-- 45 files changed, 202 insertions(+), 207 deletions(-) diff --git a/src/transformers/commands/add_new_model_like.py b/src/transformers/commands/add_new_model_like.py index a38f0f317d..cbb751ad75 100644 --- a/src/transformers/commands/add_new_model_like.py +++ b/src/transformers/commands/add_new_model_like.py @@ -1079,10 +1079,10 @@ def add_model_to_auto_classes( new_model_patterns (`ModelPatterns`): The patterns for the new model. model_classes (`dict[str, list[str]]`): A dictionary framework to list of model classes implemented. """ - for filename in AUTO_CLASSES_PATTERNS: + for filename, patterns in AUTO_CLASSES_PATTERNS.items(): # Extend patterns with all model classes if necessary new_patterns = [] - for pattern in AUTO_CLASSES_PATTERNS[filename]: + for pattern in patterns: if re.search("any_([a-z]*)_class", pattern) is not None: framework = re.search("any_([a-z]*)_class", pattern).groups()[0] if framework in model_classes: diff --git a/src/transformers/generation/streamers.py b/src/transformers/generation/streamers.py index ee28e08f2d..eddcfc0f9c 100644 --- a/src/transformers/generation/streamers.py +++ b/src/transformers/generation/streamers.py @@ -146,14 +146,14 @@ class TextStreamer(BaseStreamer): # like the all of the other languages. if ( (cp >= 0x4E00 and cp <= 0x9FFF) - or (cp >= 0x3400 and cp <= 0x4DBF) # - or (cp >= 0x20000 and cp <= 0x2A6DF) # - or (cp >= 0x2A700 and cp <= 0x2B73F) # - or (cp >= 0x2B740 and cp <= 0x2B81F) # - or (cp >= 0x2B820 and cp <= 0x2CEAF) # + or (cp >= 0x3400 and cp <= 0x4DBF) + or (cp >= 0x20000 and cp <= 0x2A6DF) + or (cp >= 0x2A700 and cp <= 0x2B73F) + or (cp >= 0x2B740 and cp <= 0x2B81F) + or (cp >= 0x2B820 and cp <= 0x2CEAF) or (cp >= 0xF900 and cp <= 0xFAFF) - or (cp >= 0x2F800 and cp <= 0x2FA1F) # - ): # + or (cp >= 0x2F800 and cp <= 0x2FA1F) + ): return True return False diff --git a/src/transformers/modeling_gguf_pytorch_utils.py b/src/transformers/modeling_gguf_pytorch_utils.py index 3ce50f8fec..f736e37d4e 100644 --- a/src/transformers/modeling_gguf_pytorch_utils.py +++ b/src/transformers/modeling_gguf_pytorch_utils.py @@ -428,8 +428,7 @@ def load_gguf_checkpoint(gguf_checkpoint_path, return_tensors=False, model_to_lo if isinstance(value, str) and architecture in value: value = value.replace(architecture, updated_architecture) - for parameter in GGUF_TO_TRANSFORMERS_MAPPING: - parameter_renames = GGUF_TO_TRANSFORMERS_MAPPING[parameter] + for parameter, parameter_renames in GGUF_TO_TRANSFORMERS_MAPPING.items(): if prefix in parameter_renames and config_key in parameter_renames[prefix]: renamed_config_key = parameter_renames[prefix][config_key] if renamed_config_key == -1: diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 9d2f70a497..dc6926d43f 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -1572,8 +1572,8 @@ def _find_mismatched_keys( # Fix the key names new_state_dict = {keys_to_rename_mapping[k]: v for k, v in state_dict.items() if k in keys_to_rename_mapping} - for key in new_state_dict.keys(): - if key in model_state_dict and new_state_dict[key].shape != model_state_dict[key].shape: + for key, tensor in new_state_dict.items(): + if key in model_state_dict and tensor.shape != model_state_dict[key].shape: # This skips size mismatches for 4-bit weights. Two 4-bit values share an 8-bit container, causing size differences. # Without matching with module type or parameter type it seems like a practical way to detect valid 4bit weights. if not ( @@ -1582,7 +1582,7 @@ def _find_mismatched_keys( and new_state_dict[key].numel() * 2 == model_state_dict[key].numel() ): mismatched_keys.append(key) - mismatched_shapes.append((new_state_dict[key].shape, model_state_dict[key].shape)) + mismatched_shapes.append((tensor.shape, model_state_dict[key].shape)) return mismatched_keys, mismatched_shapes diff --git a/src/transformers/models/bark/convert_suno_to_hf.py b/src/transformers/models/bark/convert_suno_to_hf.py index 803656b623..af2c4f3e8d 100644 --- a/src/transformers/models/bark/convert_suno_to_hf.py +++ b/src/transformers/models/bark/convert_suno_to_hf.py @@ -130,12 +130,12 @@ def _load_model(ckpt_path, device, use_small=False, model_type="text"): state_dict = checkpoint["model"] # fixup checkpoint unwanted_prefix = "_orig_mod." - for k, v in list(state_dict.items()): + for k in state_dict: if k.startswith(unwanted_prefix): # replace part of the key with corresponding layer name in HF implementation new_k = k[len(unwanted_prefix) :] - for old_layer_name in new_layer_name_dict: - new_k = new_k.replace(old_layer_name, new_layer_name_dict[old_layer_name]) + for old_layer_name, new_layer_name in new_layer_name_dict.items(): + new_k = new_k.replace(old_layer_name, new_layer_name) state_dict[new_k] = state_dict.pop(k) diff --git a/src/transformers/models/bert/tokenization_bert.py b/src/transformers/models/bert/tokenization_bert.py index 160fd99e99..23cda58bfe 100644 --- a/src/transformers/models/bert/tokenization_bert.py +++ b/src/transformers/models/bert/tokenization_bert.py @@ -392,14 +392,14 @@ class BasicTokenizer: # like the all of the other languages. if ( (cp >= 0x4E00 and cp <= 0x9FFF) - or (cp >= 0x3400 and cp <= 0x4DBF) # - or (cp >= 0x20000 and cp <= 0x2A6DF) # - or (cp >= 0x2A700 and cp <= 0x2B73F) # - or (cp >= 0x2B740 and cp <= 0x2B81F) # - or (cp >= 0x2B820 and cp <= 0x2CEAF) # + or (cp >= 0x3400 and cp <= 0x4DBF) + or (cp >= 0x20000 and cp <= 0x2A6DF) + or (cp >= 0x2A700 and cp <= 0x2B73F) + or (cp >= 0x2B740 and cp <= 0x2B81F) + or (cp >= 0x2B820 and cp <= 0x2CEAF) or (cp >= 0xF900 and cp <= 0xFAFF) - or (cp >= 0x2F800 and cp <= 0x2FA1F) # - ): # + or (cp >= 0x2F800 and cp <= 0x2FA1F) + ): return True return False diff --git a/src/transformers/models/bert_japanese/tokenization_bert_japanese.py b/src/transformers/models/bert_japanese/tokenization_bert_japanese.py index 002011795c..cacacd8757 100644 --- a/src/transformers/models/bert_japanese/tokenization_bert_japanese.py +++ b/src/transformers/models/bert_japanese/tokenization_bert_japanese.py @@ -796,14 +796,14 @@ class BasicTokenizer: # like the all of the other languages. if ( (cp >= 0x4E00 and cp <= 0x9FFF) - or (cp >= 0x3400 and cp <= 0x4DBF) # - or (cp >= 0x20000 and cp <= 0x2A6DF) # - or (cp >= 0x2A700 and cp <= 0x2B73F) # - or (cp >= 0x2B740 and cp <= 0x2B81F) # - or (cp >= 0x2B820 and cp <= 0x2CEAF) # + or (cp >= 0x3400 and cp <= 0x4DBF) + or (cp >= 0x20000 and cp <= 0x2A6DF) + or (cp >= 0x2A700 and cp <= 0x2B73F) + or (cp >= 0x2B740 and cp <= 0x2B81F) + or (cp >= 0x2B820 and cp <= 0x2CEAF) or (cp >= 0xF900 and cp <= 0xFAFF) - or (cp >= 0x2F800 and cp <= 0x2FA1F) # - ): # + or (cp >= 0x2F800 and cp <= 0x2FA1F) + ): return True return False diff --git a/src/transformers/models/clip/tokenization_clip.py b/src/transformers/models/clip/tokenization_clip.py index 2db3fd4a9a..625d26dc69 100644 --- a/src/transformers/models/clip/tokenization_clip.py +++ b/src/transformers/models/clip/tokenization_clip.py @@ -225,14 +225,14 @@ class BasicTokenizer: # like the all of the other languages. if ( (cp >= 0x4E00 and cp <= 0x9FFF) - or (cp >= 0x3400 and cp <= 0x4DBF) # - or (cp >= 0x20000 and cp <= 0x2A6DF) # - or (cp >= 0x2A700 and cp <= 0x2B73F) # - or (cp >= 0x2B740 and cp <= 0x2B81F) # - or (cp >= 0x2B820 and cp <= 0x2CEAF) # + or (cp >= 0x3400 and cp <= 0x4DBF) + or (cp >= 0x20000 and cp <= 0x2A6DF) + or (cp >= 0x2A700 and cp <= 0x2B73F) + or (cp >= 0x2B740 and cp <= 0x2B81F) + or (cp >= 0x2B820 and cp <= 0x2CEAF) or (cp >= 0xF900 and cp <= 0xFAFF) - or (cp >= 0x2F800 and cp <= 0x2FA1F) # - ): # + or (cp >= 0x2F800 and cp <= 0x2FA1F) + ): return True return False diff --git a/src/transformers/models/convbert/tokenization_convbert.py b/src/transformers/models/convbert/tokenization_convbert.py index e00bf20b17..b354b0eeae 100644 --- a/src/transformers/models/convbert/tokenization_convbert.py +++ b/src/transformers/models/convbert/tokenization_convbert.py @@ -396,14 +396,14 @@ class BasicTokenizer: # like the all of the other languages. if ( (cp >= 0x4E00 and cp <= 0x9FFF) - or (cp >= 0x3400 and cp <= 0x4DBF) # - or (cp >= 0x20000 and cp <= 0x2A6DF) # - or (cp >= 0x2A700 and cp <= 0x2B73F) # - or (cp >= 0x2B740 and cp <= 0x2B81F) # - or (cp >= 0x2B820 and cp <= 0x2CEAF) # + or (cp >= 0x3400 and cp <= 0x4DBF) + or (cp >= 0x20000 and cp <= 0x2A6DF) + or (cp >= 0x2A700 and cp <= 0x2B73F) + or (cp >= 0x2B740 and cp <= 0x2B81F) + or (cp >= 0x2B820 and cp <= 0x2CEAF) or (cp >= 0xF900 and cp <= 0xFAFF) - or (cp >= 0x2F800 and cp <= 0x2FA1F) # - ): # + or (cp >= 0x2F800 and cp <= 0x2FA1F) + ): return True return False diff --git a/src/transformers/models/cvt/convert_cvt_original_pytorch_checkpoint_to_pytorch.py b/src/transformers/models/cvt/convert_cvt_original_pytorch_checkpoint_to_pytorch.py index 85a1d4b64a..f65389d1d1 100644 --- a/src/transformers/models/cvt/convert_cvt_original_pytorch_checkpoint_to_pytorch.py +++ b/src/transformers/models/cvt/convert_cvt_original_pytorch_checkpoint_to_pytorch.py @@ -290,7 +290,7 @@ def convert_cvt_checkpoint(cvt_model, image_size, cvt_file_name, pytorch_dump_fo id2label = id2label label2id = {v: k for k, v in id2label.items()} - config = config = CvtConfig(num_labels=num_labels, id2label=id2label, label2id=label2id) + config = CvtConfig(num_labels=num_labels, id2label=id2label, label2id=label2id) # For depth size 13 (13 = 1+2+10) if cvt_model.rsplit("/", 1)[-1][4:6] == "13": diff --git a/src/transformers/models/deprecated/realm/tokenization_realm.py b/src/transformers/models/deprecated/realm/tokenization_realm.py index d1c9bcc35b..af7cdc1a5b 100644 --- a/src/transformers/models/deprecated/realm/tokenization_realm.py +++ b/src/transformers/models/deprecated/realm/tokenization_realm.py @@ -448,14 +448,14 @@ class BasicTokenizer: # like the all of the other languages. if ( (cp >= 0x4E00 and cp <= 0x9FFF) - or (cp >= 0x3400 and cp <= 0x4DBF) # - or (cp >= 0x20000 and cp <= 0x2A6DF) # - or (cp >= 0x2A700 and cp <= 0x2B73F) # - or (cp >= 0x2B740 and cp <= 0x2B81F) # - or (cp >= 0x2B820 and cp <= 0x2CEAF) # + or (cp >= 0x3400 and cp <= 0x4DBF) + or (cp >= 0x20000 and cp <= 0x2A6DF) + or (cp >= 0x2A700 and cp <= 0x2B73F) + or (cp >= 0x2B740 and cp <= 0x2B81F) + or (cp >= 0x2B820 and cp <= 0x2CEAF) or (cp >= 0xF900 and cp <= 0xFAFF) - or (cp >= 0x2F800 and cp <= 0x2FA1F) # - ): # + or (cp >= 0x2F800 and cp <= 0x2FA1F) + ): return True return False diff --git a/src/transformers/models/deprecated/retribert/tokenization_retribert.py b/src/transformers/models/deprecated/retribert/tokenization_retribert.py index 3702e7b02f..288b46e267 100644 --- a/src/transformers/models/deprecated/retribert/tokenization_retribert.py +++ b/src/transformers/models/deprecated/retribert/tokenization_retribert.py @@ -389,14 +389,14 @@ class BasicTokenizer: # like the all of the other languages. if ( (cp >= 0x4E00 and cp <= 0x9FFF) - or (cp >= 0x3400 and cp <= 0x4DBF) # - or (cp >= 0x20000 and cp <= 0x2A6DF) # - or (cp >= 0x2A700 and cp <= 0x2B73F) # - or (cp >= 0x2B740 and cp <= 0x2B81F) # - or (cp >= 0x2B820 and cp <= 0x2CEAF) # + or (cp >= 0x3400 and cp <= 0x4DBF) + or (cp >= 0x20000 and cp <= 0x2A6DF) + or (cp >= 0x2A700 and cp <= 0x2B73F) + or (cp >= 0x2B740 and cp <= 0x2B81F) + or (cp >= 0x2B820 and cp <= 0x2CEAF) or (cp >= 0xF900 and cp <= 0xFAFF) - or (cp >= 0x2F800 and cp <= 0x2FA1F) # - ): # + or (cp >= 0x2F800 and cp <= 0x2FA1F) + ): return True return False diff --git a/src/transformers/models/distilbert/tokenization_distilbert.py b/src/transformers/models/distilbert/tokenization_distilbert.py index d2a629052f..4e44468ab1 100644 --- a/src/transformers/models/distilbert/tokenization_distilbert.py +++ b/src/transformers/models/distilbert/tokenization_distilbert.py @@ -405,14 +405,14 @@ class BasicTokenizer: # like the all of the other languages. if ( (cp >= 0x4E00 and cp <= 0x9FFF) - or (cp >= 0x3400 and cp <= 0x4DBF) # - or (cp >= 0x20000 and cp <= 0x2A6DF) # - or (cp >= 0x2A700 and cp <= 0x2B73F) # - or (cp >= 0x2B740 and cp <= 0x2B81F) # - or (cp >= 0x2B820 and cp <= 0x2CEAF) # + or (cp >= 0x3400 and cp <= 0x4DBF) + or (cp >= 0x20000 and cp <= 0x2A6DF) + or (cp >= 0x2A700 and cp <= 0x2B73F) + or (cp >= 0x2B740 and cp <= 0x2B81F) + or (cp >= 0x2B820 and cp <= 0x2CEAF) or (cp >= 0xF900 and cp <= 0xFAFF) - or (cp >= 0x2F800 and cp <= 0x2FA1F) # - ): # + or (cp >= 0x2F800 and cp <= 0x2FA1F) + ): return True return False diff --git a/src/transformers/models/electra/tokenization_electra.py b/src/transformers/models/electra/tokenization_electra.py index e252a19910..d8971dd6f4 100644 --- a/src/transformers/models/electra/tokenization_electra.py +++ b/src/transformers/models/electra/tokenization_electra.py @@ -395,14 +395,14 @@ class BasicTokenizer: # like the all of the other languages. if ( (cp >= 0x4E00 and cp <= 0x9FFF) - or (cp >= 0x3400 and cp <= 0x4DBF) # - or (cp >= 0x20000 and cp <= 0x2A6DF) # - or (cp >= 0x2A700 and cp <= 0x2B73F) # - or (cp >= 0x2B740 and cp <= 0x2B81F) # - or (cp >= 0x2B820 and cp <= 0x2CEAF) # + or (cp >= 0x3400 and cp <= 0x4DBF) + or (cp >= 0x20000 and cp <= 0x2A6DF) + or (cp >= 0x2A700 and cp <= 0x2B73F) + or (cp >= 0x2B740 and cp <= 0x2B81F) + or (cp >= 0x2B820 and cp <= 0x2CEAF) or (cp >= 0xF900 and cp <= 0xFAFF) - or (cp >= 0x2F800 and cp <= 0x2FA1F) # - ): # + or (cp >= 0x2F800 and cp <= 0x2FA1F) + ): return True return False diff --git a/src/transformers/models/emu3/convert_emu3_weights_to_hf.py b/src/transformers/models/emu3/convert_emu3_weights_to_hf.py index 697ccf68f7..e0d0c3c5c5 100644 --- a/src/transformers/models/emu3/convert_emu3_weights_to_hf.py +++ b/src/transformers/models/emu3/convert_emu3_weights_to_hf.py @@ -258,7 +258,7 @@ def convert_model(vq_model_id, llm_model_id, output_dir, hub_model_id=None, test # Convert and save processor tokenizer_tiktoken = AutoTokenizer.from_pretrained(llm_model_id, trust_remote_code=True) convert_tiktoken(tokenizer_tiktoken, output_dir) - extra_special_tokens = extra_special_tokens = { + extra_special_tokens = { "image_token": "", "boi_token": "<|image start|>", "eoi_token": "<|image end|>", diff --git a/src/transformers/models/falcon_h1/configuration_falcon_h1.py b/src/transformers/models/falcon_h1/configuration_falcon_h1.py index 9c41264789..8e9aaaf340 100644 --- a/src/transformers/models/falcon_h1/configuration_falcon_h1.py +++ b/src/transformers/models/falcon_h1/configuration_falcon_h1.py @@ -254,7 +254,6 @@ class FalconH1Config(PretrainedConfig): if ssm_multipliers is not None: self.ssm_multipliers = ssm_multipliers else: - # self.ssm_multipliers = [1.0, 1.0, 1.0, 1.0, 1.0] if ssm_in_multiplier is not None: diff --git a/src/transformers/models/funnel/tokenization_funnel.py b/src/transformers/models/funnel/tokenization_funnel.py index 960e671e4a..e5d44e5e59 100644 --- a/src/transformers/models/funnel/tokenization_funnel.py +++ b/src/transformers/models/funnel/tokenization_funnel.py @@ -455,14 +455,14 @@ class BasicTokenizer: # like the all of the other languages. if ( (cp >= 0x4E00 and cp <= 0x9FFF) - or (cp >= 0x3400 and cp <= 0x4DBF) # - or (cp >= 0x20000 and cp <= 0x2A6DF) # - or (cp >= 0x2A700 and cp <= 0x2B73F) # - or (cp >= 0x2B740 and cp <= 0x2B81F) # - or (cp >= 0x2B820 and cp <= 0x2CEAF) # + or (cp >= 0x3400 and cp <= 0x4DBF) + or (cp >= 0x20000 and cp <= 0x2A6DF) + or (cp >= 0x2A700 and cp <= 0x2B73F) + or (cp >= 0x2B740 and cp <= 0x2B81F) + or (cp >= 0x2B820 and cp <= 0x2CEAF) or (cp >= 0xF900 and cp <= 0xFAFF) - or (cp >= 0x2F800 and cp <= 0x2FA1F) # - ): # + or (cp >= 0x2F800 and cp <= 0x2FA1F) + ): return True return False diff --git a/src/transformers/models/herbert/tokenization_herbert.py b/src/transformers/models/herbert/tokenization_herbert.py index 3025f3c8f6..c1c6bacc87 100644 --- a/src/transformers/models/herbert/tokenization_herbert.py +++ b/src/transformers/models/herbert/tokenization_herbert.py @@ -248,14 +248,14 @@ class BasicTokenizer: # like the all of the other languages. if ( (cp >= 0x4E00 and cp <= 0x9FFF) - or (cp >= 0x3400 and cp <= 0x4DBF) # - or (cp >= 0x20000 and cp <= 0x2A6DF) # - or (cp >= 0x2A700 and cp <= 0x2B73F) # - or (cp >= 0x2B740 and cp <= 0x2B81F) # - or (cp >= 0x2B820 and cp <= 0x2CEAF) # + or (cp >= 0x3400 and cp <= 0x4DBF) + or (cp >= 0x20000 and cp <= 0x2A6DF) + or (cp >= 0x2A700 and cp <= 0x2B73F) + or (cp >= 0x2B740 and cp <= 0x2B81F) + or (cp >= 0x2B820 and cp <= 0x2CEAF) or (cp >= 0xF900 and cp <= 0xFAFF) - or (cp >= 0x2F800 and cp <= 0x2FA1F) # - ): # + or (cp >= 0x2F800 and cp <= 0x2FA1F) + ): return True return False diff --git a/src/transformers/models/imagegpt/modeling_imagegpt.py b/src/transformers/models/imagegpt/modeling_imagegpt.py index cbac5c2792..911fc4d72d 100755 --- a/src/transformers/models/imagegpt/modeling_imagegpt.py +++ b/src/transformers/models/imagegpt/modeling_imagegpt.py @@ -367,11 +367,11 @@ class ImageGPTAttention(nn.Module): if layer_past is not None and is_updated: # reuse k,v, cross_attentions, and compute only q - query = query = self.q_attn(hidden_states) + query = self.q_attn(hidden_states) key = curr_past_key_value.key_cache[self.layer_idx] value = curr_past_key_value.value_cache[self.layer_idx] else: - query = query = self.q_attn(hidden_states) + query = self.q_attn(hidden_states) key, value = self.c_attn(current_states).split(self.split_size, dim=2) key = key.view(bsz, -1, self.num_heads, self.head_dim).transpose(1, 2) value = value.view(bsz, -1, self.num_heads, self.head_dim).transpose(1, 2) diff --git a/src/transformers/models/layoutlm/modeling_tf_layoutlm.py b/src/transformers/models/layoutlm/modeling_tf_layoutlm.py index eecda6d251..f673869384 100644 --- a/src/transformers/models/layoutlm/modeling_tf_layoutlm.py +++ b/src/transformers/models/layoutlm/modeling_tf_layoutlm.py @@ -157,7 +157,7 @@ class TFLayoutLMEmbeddings(keras.layers.Layer): position_ids = tf.expand_dims(tf.range(start=0, limit=input_shape[-1]), axis=0) if bbox is None: - bbox = bbox = tf.fill(input_shape + [4], value=0) + bbox = tf.fill(input_shape + [4], value=0) try: left_position_embeddings = tf.gather(self.x_position_embeddings, bbox[:, :, 0]) upper_position_embeddings = tf.gather(self.y_position_embeddings, bbox[:, :, 1]) diff --git a/src/transformers/models/layoutlm/tokenization_layoutlm.py b/src/transformers/models/layoutlm/tokenization_layoutlm.py index fbb6a92cd4..4caccd691d 100644 --- a/src/transformers/models/layoutlm/tokenization_layoutlm.py +++ b/src/transformers/models/layoutlm/tokenization_layoutlm.py @@ -396,14 +396,14 @@ class BasicTokenizer: # like the all of the other languages. if ( (cp >= 0x4E00 and cp <= 0x9FFF) - or (cp >= 0x3400 and cp <= 0x4DBF) # - or (cp >= 0x20000 and cp <= 0x2A6DF) # - or (cp >= 0x2A700 and cp <= 0x2B73F) # - or (cp >= 0x2B740 and cp <= 0x2B81F) # - or (cp >= 0x2B820 and cp <= 0x2CEAF) # + or (cp >= 0x3400 and cp <= 0x4DBF) + or (cp >= 0x20000 and cp <= 0x2A6DF) + or (cp >= 0x2A700 and cp <= 0x2B73F) + or (cp >= 0x2B740 and cp <= 0x2B81F) + or (cp >= 0x2B820 and cp <= 0x2CEAF) or (cp >= 0xF900 and cp <= 0xFAFF) - or (cp >= 0x2F800 and cp <= 0x2FA1F) # - ): # + or (cp >= 0x2F800 and cp <= 0x2FA1F) + ): return True return False diff --git a/src/transformers/models/layoutlmv2/tokenization_layoutlmv2.py b/src/transformers/models/layoutlmv2/tokenization_layoutlmv2.py index e4a546fc05..7d82b5cf41 100644 --- a/src/transformers/models/layoutlmv2/tokenization_layoutlmv2.py +++ b/src/transformers/models/layoutlmv2/tokenization_layoutlmv2.py @@ -1458,14 +1458,14 @@ class BasicTokenizer: # like the all of the other languages. if ( (cp >= 0x4E00 and cp <= 0x9FFF) - or (cp >= 0x3400 and cp <= 0x4DBF) # - or (cp >= 0x20000 and cp <= 0x2A6DF) # - or (cp >= 0x2A700 and cp <= 0x2B73F) # - or (cp >= 0x2B740 and cp <= 0x2B81F) # - or (cp >= 0x2B820 and cp <= 0x2CEAF) # + or (cp >= 0x3400 and cp <= 0x4DBF) + or (cp >= 0x20000 and cp <= 0x2A6DF) + or (cp >= 0x2A700 and cp <= 0x2B73F) + or (cp >= 0x2B740 and cp <= 0x2B81F) + or (cp >= 0x2B820 and cp <= 0x2CEAF) or (cp >= 0xF900 and cp <= 0xFAFF) - or (cp >= 0x2F800 and cp <= 0x2FA1F) # - ): # + or (cp >= 0x2F800 and cp <= 0x2FA1F) + ): return True return False diff --git a/src/transformers/models/lxmert/tokenization_lxmert.py b/src/transformers/models/lxmert/tokenization_lxmert.py index 5f69ab5e85..dd1d7e205e 100644 --- a/src/transformers/models/lxmert/tokenization_lxmert.py +++ b/src/transformers/models/lxmert/tokenization_lxmert.py @@ -395,14 +395,14 @@ class BasicTokenizer: # like the all of the other languages. if ( (cp >= 0x4E00 and cp <= 0x9FFF) - or (cp >= 0x3400 and cp <= 0x4DBF) # - or (cp >= 0x20000 and cp <= 0x2A6DF) # - or (cp >= 0x2A700 and cp <= 0x2B73F) # - or (cp >= 0x2B740 and cp <= 0x2B81F) # - or (cp >= 0x2B820 and cp <= 0x2CEAF) # + or (cp >= 0x3400 and cp <= 0x4DBF) + or (cp >= 0x20000 and cp <= 0x2A6DF) + or (cp >= 0x2A700 and cp <= 0x2B73F) + or (cp >= 0x2B740 and cp <= 0x2B81F) + or (cp >= 0x2B820 and cp <= 0x2CEAF) or (cp >= 0xF900 and cp <= 0xFAFF) - or (cp >= 0x2F800 and cp <= 0x2FA1F) # - ): # + or (cp >= 0x2F800 and cp <= 0x2FA1F) + ): return True return False diff --git a/src/transformers/models/mobilebert/tokenization_mobilebert.py b/src/transformers/models/mobilebert/tokenization_mobilebert.py index 5845c8e95b..88628400ca 100644 --- a/src/transformers/models/mobilebert/tokenization_mobilebert.py +++ b/src/transformers/models/mobilebert/tokenization_mobilebert.py @@ -397,14 +397,14 @@ class BasicTokenizer: # like the all of the other languages. if ( (cp >= 0x4E00 and cp <= 0x9FFF) - or (cp >= 0x3400 and cp <= 0x4DBF) # - or (cp >= 0x20000 and cp <= 0x2A6DF) # - or (cp >= 0x2A700 and cp <= 0x2B73F) # - or (cp >= 0x2B740 and cp <= 0x2B81F) # - or (cp >= 0x2B820 and cp <= 0x2CEAF) # + or (cp >= 0x3400 and cp <= 0x4DBF) + or (cp >= 0x20000 and cp <= 0x2A6DF) + or (cp >= 0x2A700 and cp <= 0x2B73F) + or (cp >= 0x2B740 and cp <= 0x2B81F) + or (cp >= 0x2B820 and cp <= 0x2CEAF) or (cp >= 0xF900 and cp <= 0xFAFF) - or (cp >= 0x2F800 and cp <= 0x2FA1F) # - ): # + or (cp >= 0x2F800 and cp <= 0x2FA1F) + ): return True return False diff --git a/src/transformers/models/mpnet/tokenization_mpnet.py b/src/transformers/models/mpnet/tokenization_mpnet.py index 42eb7d68e1..bf035cf8e4 100644 --- a/src/transformers/models/mpnet/tokenization_mpnet.py +++ b/src/transformers/models/mpnet/tokenization_mpnet.py @@ -450,14 +450,14 @@ class BasicTokenizer: # like the all of the other languages. if ( (cp >= 0x4E00 and cp <= 0x9FFF) - or (cp >= 0x3400 and cp <= 0x4DBF) # - or (cp >= 0x20000 and cp <= 0x2A6DF) # - or (cp >= 0x2A700 and cp <= 0x2B73F) # - or (cp >= 0x2B740 and cp <= 0x2B81F) # - or (cp >= 0x2B820 and cp <= 0x2CEAF) # + or (cp >= 0x3400 and cp <= 0x4DBF) + or (cp >= 0x20000 and cp <= 0x2A6DF) + or (cp >= 0x2A700 and cp <= 0x2B73F) + or (cp >= 0x2B740 and cp <= 0x2B81F) + or (cp >= 0x2B820 and cp <= 0x2CEAF) or (cp >= 0xF900 and cp <= 0xFAFF) - or (cp >= 0x2F800 and cp <= 0x2FA1F) # - ): # + or (cp >= 0x2F800 and cp <= 0x2FA1F) + ): return True return False diff --git a/src/transformers/models/openai/tokenization_openai.py b/src/transformers/models/openai/tokenization_openai.py index 0617121d13..8a9184cc39 100644 --- a/src/transformers/models/openai/tokenization_openai.py +++ b/src/transformers/models/openai/tokenization_openai.py @@ -178,14 +178,14 @@ class BasicTokenizer: # like the all of the other languages. if ( (cp >= 0x4E00 and cp <= 0x9FFF) - or (cp >= 0x3400 and cp <= 0x4DBF) # - or (cp >= 0x20000 and cp <= 0x2A6DF) # - or (cp >= 0x2A700 and cp <= 0x2B73F) # - or (cp >= 0x2B740 and cp <= 0x2B81F) # - or (cp >= 0x2B820 and cp <= 0x2CEAF) # + or (cp >= 0x3400 and cp <= 0x4DBF) + or (cp >= 0x20000 and cp <= 0x2A6DF) + or (cp >= 0x2A700 and cp <= 0x2B73F) + or (cp >= 0x2B740 and cp <= 0x2B81F) + or (cp >= 0x2B820 and cp <= 0x2CEAF) or (cp >= 0xF900 and cp <= 0xFAFF) - or (cp >= 0x2F800 and cp <= 0x2FA1F) # - ): # + or (cp >= 0x2F800 and cp <= 0x2FA1F) + ): return True return False diff --git a/src/transformers/models/owlvit/convert_owlvit_original_flax_to_hf.py b/src/transformers/models/owlvit/convert_owlvit_original_flax_to_hf.py index 1e9fbb9504..49ce578d00 100644 --- a/src/transformers/models/owlvit/convert_owlvit_original_flax_to_hf.py +++ b/src/transformers/models/owlvit/convert_owlvit_original_flax_to_hf.py @@ -314,7 +314,7 @@ def convert_clip_backbone(flax_params, torch_config): # Copy flax CLIP backbone params to PyTorch params for name, param in new_torch_params.items(): if name in torch_clip_params.keys(): - new_param = torch.from_numpy(new_torch_params[name]) + new_param = torch.from_numpy(param) torch_clip_params[name].copy_(new_param) else: attn_params[name] = param diff --git a/src/transformers/models/paligemma/convert_paligemma_weights_to_hf.py b/src/transformers/models/paligemma/convert_paligemma_weights_to_hf.py index b46300f1bb..32125b1512 100644 --- a/src/transformers/models/paligemma/convert_paligemma_weights_to_hf.py +++ b/src/transformers/models/paligemma/convert_paligemma_weights_to_hf.py @@ -291,9 +291,6 @@ def convert_paligemma_checkpoint( processor.save_pretrained(pytorch_dump_folder_path) -# - - if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( diff --git a/src/transformers/models/pix2struct/convert_pix2struct_original_pytorch_to_hf.py b/src/transformers/models/pix2struct/convert_pix2struct_original_pytorch_to_hf.py index bec4cfa688..f19d864f46 100644 --- a/src/transformers/models/pix2struct/convert_pix2struct_original_pytorch_to_hf.py +++ b/src/transformers/models/pix2struct/convert_pix2struct_original_pytorch_to_hf.py @@ -93,11 +93,11 @@ def rename_and_convert_flax_params(flax_dict): converted_torch_dict = {} # convert converted_dict into torch format - for key in converted_dict.keys(): + for key, value in converted_dict.items(): if ("embed_tokens" not in key) and ("embedder" not in key): - converted_torch_dict[key] = torch.from_numpy(converted_dict[key].T) + converted_torch_dict[key] = torch.from_numpy(value.T) else: - converted_torch_dict[key] = torch.from_numpy(converted_dict[key]) + converted_torch_dict[key] = torch.from_numpy(value) return converted_torch_dict diff --git a/src/transformers/models/prophetnet/tokenization_prophetnet.py b/src/transformers/models/prophetnet/tokenization_prophetnet.py index 4724f3a3b7..24401835c7 100644 --- a/src/transformers/models/prophetnet/tokenization_prophetnet.py +++ b/src/transformers/models/prophetnet/tokenization_prophetnet.py @@ -174,14 +174,14 @@ class BasicTokenizer: # like the all of the other languages. if ( (cp >= 0x4E00 and cp <= 0x9FFF) - or (cp >= 0x3400 and cp <= 0x4DBF) # - or (cp >= 0x20000 and cp <= 0x2A6DF) # - or (cp >= 0x2A700 and cp <= 0x2B73F) # - or (cp >= 0x2B740 and cp <= 0x2B81F) # - or (cp >= 0x2B820 and cp <= 0x2CEAF) # + or (cp >= 0x3400 and cp <= 0x4DBF) + or (cp >= 0x20000 and cp <= 0x2A6DF) + or (cp >= 0x2A700 and cp <= 0x2B73F) + or (cp >= 0x2B740 and cp <= 0x2B81F) + or (cp >= 0x2B820 and cp <= 0x2CEAF) or (cp >= 0xF900 and cp <= 0xFAFF) - or (cp >= 0x2F800 and cp <= 0x2FA1F) # - ): # + or (cp >= 0x2F800 and cp <= 0x2FA1F) + ): return True return False diff --git a/src/transformers/models/roc_bert/tokenization_roc_bert.py b/src/transformers/models/roc_bert/tokenization_roc_bert.py index 2691e620a2..afe4e29920 100644 --- a/src/transformers/models/roc_bert/tokenization_roc_bert.py +++ b/src/transformers/models/roc_bert/tokenization_roc_bert.py @@ -1005,14 +1005,14 @@ class RoCBertBasicTokenizer: # like the all of the other languages. if ( (cp >= 0x4E00 and cp <= 0x9FFF) - or (cp >= 0x3400 and cp <= 0x4DBF) # - or (cp >= 0x20000 and cp <= 0x2A6DF) # - or (cp >= 0x2A700 and cp <= 0x2B73F) # - or (cp >= 0x2B740 and cp <= 0x2B81F) # - or (cp >= 0x2B820 and cp <= 0x2CEAF) # + or (cp >= 0x3400 and cp <= 0x4DBF) + or (cp >= 0x20000 and cp <= 0x2A6DF) + or (cp >= 0x2A700 and cp <= 0x2B73F) + or (cp >= 0x2B740 and cp <= 0x2B81F) + or (cp >= 0x2B820 and cp <= 0x2CEAF) or (cp >= 0xF900 and cp <= 0xFAFF) - or (cp >= 0x2F800 and cp <= 0x2FA1F) # - ): # + or (cp >= 0x2F800 and cp <= 0x2FA1F) + ): return True return False diff --git a/src/transformers/models/roformer/tokenization_roformer.py b/src/transformers/models/roformer/tokenization_roformer.py index df79d02c7c..cf10a1b528 100644 --- a/src/transformers/models/roformer/tokenization_roformer.py +++ b/src/transformers/models/roformer/tokenization_roformer.py @@ -186,14 +186,14 @@ class BasicTokenizer: # like the all of the other languages. if ( (cp >= 0x4E00 and cp <= 0x9FFF) - or (cp >= 0x3400 and cp <= 0x4DBF) # - or (cp >= 0x20000 and cp <= 0x2A6DF) # - or (cp >= 0x2A700 and cp <= 0x2B73F) # - or (cp >= 0x2B740 and cp <= 0x2B81F) # - or (cp >= 0x2B820 and cp <= 0x2CEAF) # + or (cp >= 0x3400 and cp <= 0x4DBF) + or (cp >= 0x20000 and cp <= 0x2A6DF) + or (cp >= 0x2A700 and cp <= 0x2B73F) + or (cp >= 0x2B740 and cp <= 0x2B81F) + or (cp >= 0x2B820 and cp <= 0x2CEAF) or (cp >= 0xF900 and cp <= 0xFAFF) - or (cp >= 0x2F800 and cp <= 0x2FA1F) # - ): # + or (cp >= 0x2F800 and cp <= 0x2FA1F) + ): return True return False diff --git a/src/transformers/models/splinter/tokenization_splinter.py b/src/transformers/models/splinter/tokenization_splinter.py index 926225bbfe..f2ccafe12d 100644 --- a/src/transformers/models/splinter/tokenization_splinter.py +++ b/src/transformers/models/splinter/tokenization_splinter.py @@ -421,14 +421,14 @@ class BasicTokenizer: # like the all of the other languages. if ( (cp >= 0x4E00 and cp <= 0x9FFF) - or (cp >= 0x3400 and cp <= 0x4DBF) # - or (cp >= 0x20000 and cp <= 0x2A6DF) # - or (cp >= 0x2A700 and cp <= 0x2B73F) # - or (cp >= 0x2B740 and cp <= 0x2B81F) # - or (cp >= 0x2B820 and cp <= 0x2CEAF) # + or (cp >= 0x3400 and cp <= 0x4DBF) + or (cp >= 0x20000 and cp <= 0x2A6DF) + or (cp >= 0x2A700 and cp <= 0x2B73F) + or (cp >= 0x2B740 and cp <= 0x2B81F) + or (cp >= 0x2B820 and cp <= 0x2CEAF) or (cp >= 0xF900 and cp <= 0xFAFF) - or (cp >= 0x2F800 and cp <= 0x2FA1F) # - ): # + or (cp >= 0x2F800 and cp <= 0x2FA1F) + ): return True return False diff --git a/src/transformers/models/squeezebert/tokenization_squeezebert.py b/src/transformers/models/squeezebert/tokenization_squeezebert.py index a9bda8bbbf..4834f66816 100644 --- a/src/transformers/models/squeezebert/tokenization_squeezebert.py +++ b/src/transformers/models/squeezebert/tokenization_squeezebert.py @@ -396,14 +396,14 @@ class BasicTokenizer: # like the all of the other languages. if ( (cp >= 0x4E00 and cp <= 0x9FFF) - or (cp >= 0x3400 and cp <= 0x4DBF) # - or (cp >= 0x20000 and cp <= 0x2A6DF) # - or (cp >= 0x2A700 and cp <= 0x2B73F) # - or (cp >= 0x2B740 and cp <= 0x2B81F) # - or (cp >= 0x2B820 and cp <= 0x2CEAF) # + or (cp >= 0x3400 and cp <= 0x4DBF) + or (cp >= 0x20000 and cp <= 0x2A6DF) + or (cp >= 0x2A700 and cp <= 0x2B73F) + or (cp >= 0x2B740 and cp <= 0x2B81F) + or (cp >= 0x2B820 and cp <= 0x2CEAF) or (cp >= 0xF900 and cp <= 0xFAFF) - or (cp >= 0x2F800 and cp <= 0x2FA1F) # - ): # + or (cp >= 0x2F800 and cp <= 0x2FA1F) + ): return True return False diff --git a/src/transformers/models/switch_transformers/convert_big_switch.py b/src/transformers/models/switch_transformers/convert_big_switch.py index 6f422439fc..8d19daf1c1 100644 --- a/src/transformers/models/switch_transformers/convert_big_switch.py +++ b/src/transformers/models/switch_transformers/convert_big_switch.py @@ -89,9 +89,9 @@ def shard_on_the_fly(switch_checkpoint_path, dump_path, max_shard_size, dtype, w else: all_layers[curr_real_layer_name] = {split_layer[-1]: content} - for key in all_layers.keys(): + for key, layer in all_layers.items(): # open tensorstore file - raw_weights = ts.open(unflatten_dict(all_layers[key])).result().read().result() + raw_weights = ts.open(unflatten_dict(layer)).result().read().result() raw_weights = torch.tensor(raw_weights) weight_size = raw_weights.numel() * raw_weights.element_size() diff --git a/src/transformers/models/tapas/tokenization_tapas.py b/src/transformers/models/tapas/tokenization_tapas.py index 87468ceb1a..4d91a1add9 100644 --- a/src/transformers/models/tapas/tokenization_tapas.py +++ b/src/transformers/models/tapas/tokenization_tapas.py @@ -2123,14 +2123,14 @@ class BasicTokenizer: # like the all of the other languages. if ( (cp >= 0x4E00 and cp <= 0x9FFF) - or (cp >= 0x3400 and cp <= 0x4DBF) # - or (cp >= 0x20000 and cp <= 0x2A6DF) # - or (cp >= 0x2A700 and cp <= 0x2B73F) # - or (cp >= 0x2B740 and cp <= 0x2B81F) # - or (cp >= 0x2B820 and cp <= 0x2CEAF) # + or (cp >= 0x3400 and cp <= 0x4DBF) + or (cp >= 0x20000 and cp <= 0x2A6DF) + or (cp >= 0x2A700 and cp <= 0x2B73F) + or (cp >= 0x2B740 and cp <= 0x2B81F) + or (cp >= 0x2B820 and cp <= 0x2CEAF) or (cp >= 0xF900 and cp <= 0xFAFF) - or (cp >= 0x2F800 and cp <= 0x2FA1F) # - ): # + or (cp >= 0x2F800 and cp <= 0x2FA1F) + ): return True return False diff --git a/src/transformers/models/umt5/convert_umt5_checkpoint_to_pytorch.py b/src/transformers/models/umt5/convert_umt5_checkpoint_to_pytorch.py index aaa8c94e2f..1ba7235029 100644 --- a/src/transformers/models/umt5/convert_umt5_checkpoint_to_pytorch.py +++ b/src/transformers/models/umt5/convert_umt5_checkpoint_to_pytorch.py @@ -50,7 +50,7 @@ def t5x_relpos_bias_lookup(params, i, prefix): def t5x_attention_lookup(params, i, prefix, layer_name="attention"): """Returns the KOQV parameters of (self-)attention. Does not transpose.""" - k_tmp = k_tmp = np.ascontiguousarray(params[f"{prefix}/{prefix}/{layer_name}/key/kernel"][:, i, :, :]) + k_tmp = np.ascontiguousarray(params[f"{prefix}/{prefix}/{layer_name}/key/kernel"][:, i, :, :]) k = k_tmp.reshape(k_tmp.shape[0], k_tmp.shape[1] * k_tmp.shape[2]) o_tmp = np.ascontiguousarray(params[f"{prefix}/{prefix}/{layer_name}/out/kernel"][:, i, :, :]) o = o_tmp.reshape(o_tmp.shape[0] * o_tmp.shape[1], o_tmp.shape[2]) diff --git a/src/transformers/processing_utils.py b/src/transformers/processing_utils.py index 0dbac1a08a..a40f2ad7fd 100644 --- a/src/transformers/processing_utils.py +++ b/src/transformers/processing_utils.py @@ -1184,7 +1184,7 @@ class ProcessorMixin(PushToHubMixin): used_keys = set() # get defaults from set model processor kwargs if they exist - for modality in default_kwargs: + for modality in default_kwargs: # noqa: PLC0206 default_kwargs[modality] = ModelProcessorKwargs._defaults.get(modality, {}).copy() # update defaults with arguments from tokenizer init for modality_key in ModelProcessorKwargs.__annotations__[modality].__annotations__.keys(): @@ -1202,7 +1202,7 @@ class ProcessorMixin(PushToHubMixin): # update modality kwargs with passed kwargs non_modality_kwargs = set(kwargs) - set(output_kwargs) - for modality in output_kwargs: + for modality, output_kwarg in output_kwargs.items(): for modality_key in ModelProcessorKwargs.__annotations__[modality].__annotations__.keys(): # check if we received a structured kwarg dict or not to handle it correctly if modality in kwargs: @@ -1220,7 +1220,7 @@ class ProcessorMixin(PushToHubMixin): else: kwarg_value = "__empty__" if not isinstance(kwarg_value, str) or kwarg_value != "__empty__": - output_kwargs[modality][modality_key] = kwarg_value + output_kwarg[modality_key] = kwarg_value used_keys.add(modality_key) # Determine if kwargs is a flat dictionary or contains nested dictionaries @@ -1234,18 +1234,18 @@ class ProcessorMixin(PushToHubMixin): used_keys.add(subkey) else: # kwargs is a flat dictionary - for key in kwargs: + for key, kwarg in kwargs.items(): if key not in used_keys: if key in ModelProcessorKwargs.__annotations__["common_kwargs"].__annotations__.keys(): - output_kwargs["common_kwargs"][key] = kwargs[key] + output_kwargs["common_kwargs"][key] = kwarg elif key not in possible_modality_keywords: logger.warning_once( f"Keyword argument `{key}` is not a valid argument for this processor and will be ignored." ) # all modality-specific kwargs are updated with common kwargs - for modality in output_kwargs: - output_kwargs[modality].update(output_kwargs["common_kwargs"]) + for kwarg in output_kwargs.values(): + kwarg.update(output_kwargs["common_kwargs"]) return output_kwargs @classmethod diff --git a/src/transformers/quantizers/quantizer_hqq.py b/src/transformers/quantizers/quantizer_hqq.py index 896b125b8c..4c6a64705b 100755 --- a/src/transformers/quantizers/quantizer_hqq.py +++ b/src/transformers/quantizers/quantizer_hqq.py @@ -253,8 +253,8 @@ class HqqHfQuantizer(HfQuantizer): return # Step 1: populate module with weight/bias from module state dict - for key in module_state_dict: - setattr(module, key, torch.nn.Parameter(module_state_dict[key])) + for key, tensor in module_state_dict.items(): + setattr(module, key, torch.nn.Parameter(tensor)) # Step 2: Replace module with either HQQLinear or move it to device. We do this via setattr on the parent as doing on it on the module # directly doesn't work. diff --git a/src/transformers/utils/attention_visualizer.py b/src/transformers/utils/attention_visualizer.py index 202c0a1b5a..d9a4041f7f 100644 --- a/src/transformers/utils/attention_visualizer.py +++ b/src/transformers/utils/attention_visualizer.py @@ -118,7 +118,7 @@ def generate_attention_matrix_from_mask( colored_word = f"{YELLOW}{word_repr}{RESET}" if img_token in word else word_repr row_display = " ".join( f"{YELLOW}{BLACK_SQUARE}{RESET}" - if img_token in words[j] and mask[i, j] and img_token in words[i] + if img_token in words[j] and mask[i, j] and img_token in word else f"{GREEN}{BLACK_SQUARE}{RESET}" if i == j else BLACK_SQUARE @@ -130,9 +130,7 @@ def generate_attention_matrix_from_mask( if sliding_window is not None: sliding_window_row = " ".join( f"{YELLOW}{BLACK_SQUARE}{RESET}" - if img_token in words[j] - and img_token in words[i] - and token_type_buckets[0, i] == token_type_buckets[0, j] + if img_token in words[j] and img_token in word and token_type_buckets[0, i] == token_type_buckets[0, j] else f"{GREEN}{BLACK_SQUARE}{RESET}" if i == j else BLACK_SQUARE diff --git a/tests/models/groupvit/test_modeling_groupvit.py b/tests/models/groupvit/test_modeling_groupvit.py index 24e4328ac7..3e438a13b3 100644 --- a/tests/models/groupvit/test_modeling_groupvit.py +++ b/tests/models/groupvit/test_modeling_groupvit.py @@ -247,7 +247,7 @@ class GroupViTVisionModelTest(ModelTesterMixin, unittest.TestCase): continue self.assertListEqual( - list(self_attentions[i].shape[-2:]), + list(self_attn.shape[-2:]), [ self.model_tester.num_output_groups[i], self.model_tester.num_output_groups[i - 1] if i > 0 else seq_len, diff --git a/tests/test_tokenization_common.py b/tests/test_tokenization_common.py index 2b7f8d38c8..3461721808 100644 --- a/tests/test_tokenization_common.py +++ b/tests/test_tokenization_common.py @@ -1375,9 +1375,7 @@ class TokenizerTesterMixin: self.assertEqual(output_pt["assistant_masks"].shape, output_pt["input_ids"].shape) for i, conv in enumerate(conversations): - chat_string = tokenizer_r.apply_chat_template( - conversations[i], tokenize=False, chat_template=dummy_template - ) + chat_string = tokenizer_r.apply_chat_template(conv, tokenize=False, chat_template=dummy_template) assistant_start = output.char_to_token(i, chat_string.index(assistant_prefix_suffix[i][0][0])) assistant_end = output.char_to_token( i, diff --git a/utils/check_bad_commit.py b/utils/check_bad_commit.py index a251b95415..b15c1d6c4f 100644 --- a/utils/check_bad_commit.py +++ b/utils/check_bad_commit.py @@ -97,6 +97,7 @@ git bisect run python3 target_script.py result = subprocess.run( ["bash", "run_git_bisect.sh"], + check=False, capture_output=True, text=True, ) diff --git a/utils/check_self_hosted_runner.py b/utils/check_self_hosted_runner.py index 9c2bfb1f87..0c651d1a12 100644 --- a/utils/check_self_hosted_runner.py +++ b/utils/check_self_hosted_runner.py @@ -15,7 +15,7 @@ def get_runner_status(target_runners, token): "https://api.github.com/repos/huggingface/transformers/actions/runners", ] - output = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE) + output = subprocess.run(cmd, check=False, shell=True, stdout=subprocess.PIPE) o = output.stdout.decode("utf-8") status = json.loads(o) diff --git a/utils/patch_helper.py b/utils/patch_helper.py index cbb39c4913..8aae002518 100644 --- a/utils/patch_helper.py +++ b/utils/patch_helper.py @@ -85,7 +85,7 @@ def get_prs_by_label(label): "--limit", "100", ] - result = subprocess.run(cmd, capture_output=True, text=True) + result = subprocess.run(cmd, check=False, capture_output=True, text=True) result.check_returncode() prs = json.loads(result.stdout) for pr in prs: @@ -97,7 +97,9 @@ def get_prs_by_label(label): def get_commit_timestamp(commit_sha): """Get UNIX timestamp of a commit using git.""" - result = subprocess.run(["git", "show", "-s", "--format=%ct", commit_sha], capture_output=True, text=True) + result = subprocess.run( + ["git", "show", "-s", "--format=%ct", commit_sha], check=False, capture_output=True, text=True + ) result.check_returncode() return int(result.stdout.strip()) @@ -115,6 +117,7 @@ def commit_in_history(commit_sha, base_branch="HEAD"): """Return True if commit is already part of base_branch history.""" result = subprocess.run( ["git", "merge-base", "--is-ancestor", commit_sha, base_branch], + check=False, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, )