From 2c66600c3fbbdcfc1610c5862c33bd19cf42859e Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Fri, 12 Apr 2024 10:01:28 +0200 Subject: [PATCH] ENH: [`CI`] Add new workflow to run slow tests of important models on push main if they are modified (#29235) * v1 * v1 * more changes * more models * add more markers * swtich to A10 * use cache * Update .github/workflows/push-important-models.yml * Update .github/workflows/push-important-models.yml * Update modeling_llama.py * test * test * another test * test * test * attempt to fix * fix * try automatic tagging * fix * alternative approach for collecting * fix * fix * fix * test * fix * fix * test * revert some changes * fix * fix * fix * final push * fix * revert * test new slack message * oops * Update send-slack.yml * test * test re-usable workflow in steps * Update action.yml * test * another test * test * another test * test * another test * another test (hopefully last one) * attempt to fix * allez * removing comma * test * another test * attempt * test * test * test push * test * test * another test * test * make it better * fix commas * valid json * test * another test * test * final push * test * final push * more customizable messages * test * push * oops * another test * another test * missing indentation * more tweaks * more tweaks * another test * another test * tests * final push * use global variables instead * Update .github/workflows/push-important-models.yml * Apply suggestions from code review Co-authored-by: Arthur <48595927+ArthurZucker@users.noreply.github.com> * commit to test all models * issue with arrays * another test * attempt to fix failing tests * Update .github/workflows/push-important-models.yml * add ssh * Update .github/workflows/push-important-models.yml * test * test * add install curl * attempt to fix * final fix * test * test * test * fix test * another test * add inherit secrets * push * revert unneeded changes * revert * add env variables * add pip freeze * revert change in gemma * Update .github/workflows/push-important-models.yml * fix mistral and mixtral * add pdb * fix mixtral tesst * fix * fix mistral ? * add fix gemma * fix mistral * fix * test * anoter test * fix * fix * fix mistral tests * fix them again * final fixes for mistral * fix padding right * fix whipser fa2 * fix * fix * fix gemma * test * fix llama * fix * fix * fix llama gemma * add class attribute * fix CI * clarify whisper * compute_capability * rename names in some comments * Add # fmt: skip * make style * Update tests/models/mistral/test_modeling_mistral.py Co-authored-by: Arthur <48595927+ArthurZucker@users.noreply.github.com> * update * update * change branch * correct workflow * modify file * test * works * final test * another fix * install sudo * final fix * add `-y` * set to `main` * Update .github/actions/post-slack/action.yml Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com> * change title * fixup * add upload report * fix * revert to main * add empty lines + add comment --------- Co-authored-by: Arthur <48595927+ArthurZucker@users.noreply.github.com> Co-authored-by: ydshieh Co-authored-by: Yih-Dar <2521628+ydshieh@users.noreply.github.com> Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com> --- .github/actions/post-slack/action.yml | 79 ++++++++++++ .github/workflows/push-important-models.yml | 136 ++++++++++++++++++++ utils/important_models.txt | 4 + 3 files changed, 219 insertions(+) create mode 100644 .github/actions/post-slack/action.yml create mode 100644 .github/workflows/push-important-models.yml create mode 100644 utils/important_models.txt diff --git a/.github/actions/post-slack/action.yml b/.github/actions/post-slack/action.yml new file mode 100644 index 0000000000..74075a4fed --- /dev/null +++ b/.github/actions/post-slack/action.yml @@ -0,0 +1,79 @@ +name: Send message to slack + +description: 'Send results to slack' +author: 'Hugging Face' +inputs: + slack_channel: + required: true + type: string + title: + required: true + type: string + status: + required: true + type: string + slack_token: + required: true + type: string + +runs: + using: "composite" + steps: + - name: Create content to post + id: create-message + run: | + if [ "${{ inputs.status }}" == "success" ]; then + echo STATUS_MESSAGE='🟢 Tests are passing!' >> $GITHUB_ENV + else + echo STATUS_MESSAGE='🔴 Tests failed! Please check the GitHub action link below' >> $GITHUB_ENV + fi + shell: bash + + - name: Post Canceled results Slack channel + id: post-slack + uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 + with: + # Slack channel id, channel name, or user id to post message. + # See also: https://api.slack.com/methods/chat.postMessage#channels + channel-id: ${{ inputs.slack_channel }} + # For posting a rich message using Block Kit + payload: | + { + "text": "${{ inputs.title }}", + "blocks": [ + { + "type": "header", + "text": { + "type": "plain_text", + "text": "${{ inputs.title }}" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "${{ env.STATUS_MESSAGE }}" + } + }, + { + "type": "section", + "text": {"type": "mrkdwn", "text": "*Click the button for more details about the commit*"}, + "accessory": { + "type": "button", + "text": {"type": "plain_text", "text": "Check Commit results"}, + "url": "${{ github.event.pull_request.html_url || github.event.head_commit.url }}" + } + }, + { + "type": "section", + "text": {"type": "mrkdwn", "text": "*Click here for more details about the action ran*"}, + "accessory": { + "type": "button", + "text": {"type": "plain_text", "text": "Check Action results"}, + "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + } + } + ] + } + env: + SLACK_BOT_TOKEN: ${{ inputs.slack_token }} \ No newline at end of file diff --git a/.github/workflows/push-important-models.yml b/.github/workflows/push-important-models.yml new file mode 100644 index 0000000000..5eef6f40f8 --- /dev/null +++ b/.github/workflows/push-important-models.yml @@ -0,0 +1,136 @@ +name: Slow tests on important models (on Push - A10) + +on: + push: + branches: [ main ] + +env: + IS_GITHUB_CI: "1" + OUTPUT_SLACK_CHANNEL_ID: "C06L2SGMEEA" + HF_HUB_READ_TOKEN: ${{ secrets.HF_HUB_READ_TOKEN }} + HF_HOME: /mnt/cache + TRANSFORMERS_IS_CI: yes + OMP_NUM_THREADS: 8 + MKL_NUM_THREADS: 8 + RUN_SLOW: yes # For gated repositories, we still need to agree to share information on the Hub repo. page in order to get access. # This token is created under the bot `hf-transformers-bot`. + SIGOPT_API_TOKEN: ${{ secrets.SIGOPT_API_TOKEN }} + TF_FORCE_GPU_ALLOW_GROWTH: true + RUN_PT_TF_CROSS_TESTS: 1 + +jobs: + get_modified_models: + name: "Get all modified files" + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - name: Check out code + uses: actions/checkout@v3 + + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@3f54ebb830831fc121d3263c1857cfbdc310cdb9 #v42 + with: + files: src/transformers/models/** + + - name: Run step if only the files listed above change + if: steps.changed-files.outputs.any_changed == 'true' + id: set-matrix + env: + ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }} + run: | + model_arrays=() + for file in $ALL_CHANGED_FILES; do + model_path="${file#*models/}" + model_path="models/${model_path%%/*}" + if grep -qFx "$model_path" utils/important_models.txt; then + # Append the file to the matrix string + model_arrays+=("$model_path") + fi + done + matrix_string=$(printf '"%s", ' "${model_arrays[@]}" | sed 's/, $//') + echo "matrix=[$matrix_string]" >> $GITHUB_OUTPUT + test_modified_files: + needs: get_modified_models + name: Slow & FA2 tests + runs-on: [single-gpu, nvidia-gpu, a10, ci] + container: + image: huggingface/transformers-all-latest-gpu + options: --gpus all --privileged --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/ + if: ${{ needs.get_modified_models.outputs.matrix != '[]' && needs.get_modified_models.outputs.matrix != '' }} + strategy: + fail-fast: false + matrix: + model-name: ${{ fromJson(needs.get_modified_models.outputs.matrix) }} + + steps: + - name: Check out code + uses: actions/checkout@v3 + + - name: Install locally transformers & other libs + run: | + apt install sudo + sudo -H pip install --upgrade pip + sudo -H pip uninstall -y transformers + sudo -H pip install -U -e ".[testing]" + MAX_JOBS=4 pip install flash-attn --no-build-isolation + pip install bitsandbytes + + - name: NVIDIA-SMI + run: | + nvidia-smi + + - name: Show installed libraries and their versions + run: pip freeze + + - name: Run FA2 tests + id: run_fa2_tests + run: + pytest -m "flash_attn_test" --make-reports=${{ matrix.model-name }}_fa2_tests/ tests/${{ matrix.model-name }}/test_modeling_* + + - name: "Test suite reports artifacts: ${{ matrix.model-name }}_fa2_tests" + if: ${{ always() }} + uses: actions/upload-artifact@v3 + with: + name: ${{ matrix.model-name }}_fa2_tests + path: /transformers/reports/${{ matrix.model-name }}_fa2_tests + + - name: Post to Slack + if: always() + uses: ./.github/actions/post-slack + with: + slack_channel: ${{ env.OUTPUT_SLACK_CHANNEL_ID }} + title: 🤗 Results of the FA2 tests - ${{ matrix.model-name }} + status: ${{ steps.run_fa2_tests.conclusion}} + slack_token: ${{ secrets.CI_SLACK_BOT_TOKEN }} + + - name: Run integration tests + id: run_integration_tests + if: always() + run: + pytest -k "IntegrationTest" --make-reports=tests_integration_${{ matrix.model-name }} tests/${{ matrix.model-name }}/test_modeling_* + + - name: "Test suite reports artifacts: tests_integration_${{ matrix.model-name }}" + if: ${{ always() }} + uses: actions/upload-artifact@v3 + with: + name: tests_integration_${{ matrix.model-name }} + path: /transformers/reports/tests_integration_${{ matrix.model-name }} + + - name: Post to Slack + if: always() + uses: ./.github/actions/post-slack + with: + slack_channel: ${{ env.OUTPUT_SLACK_CHANNEL_ID }} + title: 🤗 Results of the Integration tests - ${{ matrix.model-name }} + status: ${{ steps.run_integration_tests.conclusion}} + slack_token: ${{ secrets.CI_SLACK_BOT_TOKEN }} + + - name: Tailscale # In order to be able to SSH when a test fails + if: ${{ failure() || runner.debug == '1'}} + uses: huggingface/tailscale-action@ssh-improvments + with: + authkey: ${{ secrets.TAILSCALE_SSH_AUTHKEY }} + slackChannel: ${{ secrets.SLACK_CIFEEDBACK_CHANNEL }} + slackToken: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }} + waitForSSH: true \ No newline at end of file diff --git a/utils/important_models.txt b/utils/important_models.txt new file mode 100644 index 0000000000..6dcd5de8d6 --- /dev/null +++ b/utils/important_models.txt @@ -0,0 +1,4 @@ +models/llama +models/mistral +models/mixtral +models/gemma \ No newline at end of file