Phi3: fix attn for sliding window (#33586)

* fix phi3 attn fir sliding window * fix tests * address most comment * style * update after rebase * add more models * fix tests
2024-10-10 11:50:39 +02:00
parent a265600c60
commit adea67541a
13 changed files with 831 additions and 145 deletions
--- a/tests/models/qwen2_moe/test_modeling_qwen2_moe.py
+++ b/tests/models/qwen2_moe/test_modeling_qwen2_moe.py
@@ -64,7 +64,7 @@ class Qwen2MoeModelTester:
        num_hidden_layers=5,
        max_window_layers=3,
        use_sliding_window=True,
-        sliding_window=2,
+        sliding_window=50,
        num_attention_heads=4,
        num_key_value_heads=2,
        intermediate_size=37,