From 6589e510fa4e6c442059de2fab84752535de9b23 Mon Sep 17 00:00:00 2001 From: Yih-Dar <2521628+ydshieh@users.noreply.github.com> Date: Sat, 18 Jun 2022 08:35:03 +0200 Subject: [PATCH] Attempt to change Push CI to workflow_run (#17753) * Use workflow_run event for push CI * change to workflow_run * Add comments Co-authored-by: ydshieh --- .github/workflows/self-push-caller.yml | 16 +- .github/workflows/self-push.yml | 214 +++++++++++++++++++++++-- utils/notification_service.py | 95 ++++++----- 3 files changed, 259 insertions(+), 66 deletions(-) diff --git a/.github/workflows/self-push-caller.yml b/.github/workflows/self-push-caller.yml index 394cfb59ff..c6986ddf28 100644 --- a/.github/workflows/self-push-caller.yml +++ b/.github/workflows/self-push-caller.yml @@ -1,3 +1,4 @@ +# Used to trigger self-push CI name: Self-hosted runner (push-caller) on: @@ -13,17 +14,8 @@ on: jobs: run_push_ci: - name: Run Push CI + name: Trigger Push CI runs-on: ubuntu-latest steps: - - name: Checkout transformers - uses: actions/checkout@v2 - with: - fetch-depth: 2 - ssh-key: "${{ secrets.COMMIT_KEY }}" - - - name: Checkout to branch push-ci - # A more strict way to make sure`push-ci` is exactly the same as `main` at the push event commit. - run: | - git checkout -b push-ci - git push -u origin push-ci --force + - name: Trigger push CI via workflow_run + run: echo "Trigger push CI via workflow_run" \ No newline at end of file diff --git a/.github/workflows/self-push.yml b/.github/workflows/self-push.yml index 0251a32a1a..5a53a844c4 100644 --- a/.github/workflows/self-push.yml +++ b/.github/workflows/self-push.yml @@ -1,9 +1,12 @@ name: Self-hosted runner (push) on: + workflow_run: + workflows: ["Self-hosted runner (push-caller)"] + branches: ["main"] + types: [completed] push: branches: - - push-ci - ci_* - ci-* paths: @@ -31,11 +34,47 @@ jobs: matrix: ${{ steps.set-matrix.outputs.matrix }} test_map: ${{ steps.set-matrix.outputs.test_map }} steps: + # Necessary to get the correct branch name and commit SHA for `workflow_run` event + # We also take into account the `push` event (we might want to test some changes in a branch) + - name: Prepare custom environment variables + shell: bash + # `CI_BRANCH_PUSH`: The branch name from the push event + # `CI_BRANCH_WORKFLOW_RUN`: The name of the branch on which this workflow is triggered by `workflow_run` event + # `CI_BRANCH`: The non-empty branch name from the above two (one and only one of them is empty) + # `CI_SHA_PUSH`: The commit SHA from the push event + # `CI_SHA_WORKFLOW_RUN`: The commit SHA that triggers this workflow by `workflow_run` event + # `CI_SHA`: The non-empty commit SHA from the above two (one and only one of them is empty) + run: | + CI_BRANCH_PUSH=${{ github.event.ref }} + CI_BRANCH_PUSH=${CI_BRANCH_PUSH/'refs/heads/'/''} + CI_BRANCH_WORKFLOW_RUN=${{ github.event.workflow_run.head_branch }} + CI_SHA_PUSH=${{ github.event.head_commit.id }} + CI_SHA_WORKFLOW_RUN=${{ github.event.workflow_run.head_sha }} + echo $CI_BRANCH_PUSH + echo $CI_BRANCH_WORKFLOW_RUN + echo $CI_SHA_PUSH + echo $CI_SHA_WORKFLOW_RUN + [[ ! -z "$CI_BRANCH_PUSH" ]] && echo "CI_BRANCH=$CI_BRANCH_PUSH" >> $GITHUB_ENV || echo "CI_BRANCH=$CI_BRANCH_WORKFLOW_RUN" >> $GITHUB_ENV + [[ ! -z "$CI_SHA_PUSH" ]] && echo "CI_SHA=$CI_SHA_PUSH" >> $GITHUB_ENV || echo "CI_SHA=$CI_SHA_WORKFLOW_RUN" >> $GITHUB_ENV + + - name: print environment variables + run: | + echo "env.CI_BRANCH = ${{ env.CI_BRANCH }}" + echo "env.CI_SHA = ${{ env.CI_SHA }}" + - name: Checkout transformers uses: actions/checkout@v2 with: fetch-depth: 2 + - name: Update clone using environment variables + run: | + echo "original branch = $(git branch --show-current)" + git fetch && git checkout ${{ env.CI_BRANCH }} + echo "updated branch = $(git branch --show-current)" + git checkout ${{ env.CI_SHA }} + echo "log = $(git log -n 1)" + - name: Cleanup run: | rm -rf tests/__pycache__ @@ -87,6 +126,38 @@ jobs: image: huggingface/transformers-all-latest-gpu options: --gpus 0 --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/ steps: + # Necessary to get the correct branch name and commit SHA for `workflow_run` event + # We also take into account the `push` event (we might want to test some changes in a branch) + - name: Prepare custom environment variables + shell: bash + # For the meaning of these environment variables, see the job `Setup` + run: | + CI_BRANCH_PUSH=${{ github.event.ref }} + CI_BRANCH_PUSH=${CI_BRANCH_PUSH/'refs/heads/'/''} + CI_BRANCH_WORKFLOW_RUN=${{ github.event.workflow_run.head_branch }} + CI_SHA_PUSH=${{ github.event.head_commit.id }} + CI_SHA_WORKFLOW_RUN=${{ github.event.workflow_run.head_sha }} + echo $CI_BRANCH_PUSH + echo $CI_BRANCH_WORKFLOW_RUN + echo $CI_SHA_PUSH + echo $CI_SHA_WORKFLOW_RUN + [[ ! -z "$CI_BRANCH_PUSH" ]] && echo "CI_BRANCH=$CI_BRANCH_PUSH" >> $GITHUB_ENV || echo "CI_BRANCH=$CI_BRANCH_WORKFLOW_RUN" >> $GITHUB_ENV + [[ ! -z "$CI_SHA_PUSH" ]] && echo "CI_SHA=$CI_SHA_PUSH" >> $GITHUB_ENV || echo "CI_SHA=$CI_SHA_WORKFLOW_RUN" >> $GITHUB_ENV + + - name: print environment variables + run: | + echo "env.CI_BRANCH = ${{ env.CI_BRANCH }}" + echo "env.CI_SHA = ${{ env.CI_SHA }}" + + - name: Update clone using environment variables + working-directory: /transformers + run: | + echo "original branch = $(git branch --show-current)" + git fetch && git checkout ${{ env.CI_BRANCH }} + echo "updated branch = $(git branch --show-current)" + git checkout ${{ env.CI_SHA }} + echo "log = $(git log -n 1)" + - name: Echo folder ${{ matrix.folders }} shell: bash # For folders like `models/bert`, set an env. var. (`matrix_folders`) to `models_bert`, which will be used to @@ -99,10 +170,6 @@ jobs: echo "$matrix_folders" echo "matrix_folders=$matrix_folders" >> $GITHUB_ENV - - name: Update clone - working-directory: /transformers - run: git fetch && git checkout ${{ github.sha }} - - name: NVIDIA-SMI run: | nvidia-smi @@ -144,6 +211,38 @@ jobs: image: huggingface/transformers-all-latest-gpu options: --gpus all --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/ steps: + # Necessary to get the correct branch name and commit SHA for `workflow_run` event + # We also take into account the `push` event (we might want to test some changes in a branch) + - name: Prepare custom environment variables + shell: bash + # For the meaning of these environment variables, see the job `Setup` + run: | + CI_BRANCH_PUSH=${{ github.event.ref }} + CI_BRANCH_PUSH=${CI_BRANCH_PUSH/'refs/heads/'/''} + CI_BRANCH_WORKFLOW_RUN=${{ github.event.workflow_run.head_branch }} + CI_SHA_PUSH=${{ github.event.head_commit.id }} + CI_SHA_WORKFLOW_RUN=${{ github.event.workflow_run.head_sha }} + echo $CI_BRANCH_PUSH + echo $CI_BRANCH_WORKFLOW_RUN + echo $CI_SHA_PUSH + echo $CI_SHA_WORKFLOW_RUN + [[ ! -z "$CI_BRANCH_PUSH" ]] && echo "CI_BRANCH=$CI_BRANCH_PUSH" >> $GITHUB_ENV || echo "CI_BRANCH=$CI_BRANCH_WORKFLOW_RUN" >> $GITHUB_ENV + [[ ! -z "$CI_SHA_PUSH" ]] && echo "CI_SHA=$CI_SHA_PUSH" >> $GITHUB_ENV || echo "CI_SHA=$CI_SHA_WORKFLOW_RUN" >> $GITHUB_ENV + + - name: print environment variables + run: | + echo "env.CI_BRANCH = ${{ env.CI_BRANCH }}" + echo "env.CI_SHA = ${{ env.CI_SHA }}" + + - name: Update clone using environment variables + working-directory: /transformers + run: | + echo "original branch = $(git branch --show-current)" + git fetch && git checkout ${{ env.CI_BRANCH }} + echo "updated branch = $(git branch --show-current)" + git checkout ${{ env.CI_SHA }} + echo "log = $(git log -n 1)" + - name: Echo folder ${{ matrix.folders }} shell: bash # For folders like `models/bert`, set an env. var. (`matrix_folders`) to `models_bert`, which will be used to @@ -156,10 +255,6 @@ jobs: echo "$matrix_folders" echo "matrix_folders=$matrix_folders" >> $GITHUB_ENV - - name: Update clone - working-directory: /transformers - run: git fetch && git checkout ${{ github.sha }} - - name: NVIDIA-SMI run: | nvidia-smi @@ -201,9 +296,37 @@ jobs: image: huggingface/transformers-pytorch-deepspeed-latest-gpu options: --gpus 0 --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/ steps: - - name: Update clone + # Necessary to get the correct branch name and commit SHA for `workflow_run` event + # We also take into account the `push` event (we might want to test some changes in a branch) + - name: Prepare custom environment variables + shell: bash + # For the meaning of these environment variables, see the job `Setup` + run: | + CI_BRANCH_PUSH=${{ github.event.ref }} + CI_BRANCH_PUSH=${CI_BRANCH_PUSH/'refs/heads/'/''} + CI_BRANCH_WORKFLOW_RUN=${{ github.event.workflow_run.head_branch }} + CI_SHA_PUSH=${{ github.event.head_commit.id }} + CI_SHA_WORKFLOW_RUN=${{ github.event.workflow_run.head_sha }} + echo $CI_BRANCH_PUSH + echo $CI_BRANCH_WORKFLOW_RUN + echo $CI_SHA_PUSH + echo $CI_SHA_WORKFLOW_RUN + [[ ! -z "$CI_BRANCH_PUSH" ]] && echo "CI_BRANCH=$CI_BRANCH_PUSH" >> $GITHUB_ENV || echo "CI_BRANCH=$CI_BRANCH_WORKFLOW_RUN" >> $GITHUB_ENV + [[ ! -z "$CI_SHA_PUSH" ]] && echo "CI_SHA=$CI_SHA_PUSH" >> $GITHUB_ENV || echo "CI_SHA=$CI_SHA_WORKFLOW_RUN" >> $GITHUB_ENV + + - name: print environment variables + run: | + echo "env.CI_BRANCH = ${{ env.CI_BRANCH }}" + echo "env.CI_SHA = ${{ env.CI_SHA }}" + + - name: Update clone using environment variables working-directory: /workspace/transformers - run: git fetch && git checkout ${{ github.sha }} + run: | + echo "original branch = $(git branch --show-current)" + git fetch && git checkout ${{ env.CI_BRANCH }} + echo "updated branch = $(git branch --show-current)" + git checkout ${{ env.CI_SHA }} + echo "log = $(git log -n 1)" # To avoid unknown test failures - name: Pre build DeepSpeed *again* @@ -252,9 +375,37 @@ jobs: image: huggingface/transformers-pytorch-deepspeed-latest-gpu options: --gpus all --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/ steps: - - name: Update clone + # Necessary to get the correct branch name and commit SHA for `workflow_run` event + # We also take into account the `push` event (we might want to test some changes in a branch) + - name: Prepare custom environment variables + shell: bash + # For the meaning of these environment variables, see the job `Setup` + run: | + CI_BRANCH_PUSH=${{ github.event.ref }} + CI_BRANCH_PUSH=${CI_BRANCH_PUSH/'refs/heads/'/''} + CI_BRANCH_WORKFLOW_RUN=${{ github.event.workflow_run.head_branch }} + CI_SHA_PUSH=${{ github.event.head_commit.id }} + CI_SHA_WORKFLOW_RUN=${{ github.event.workflow_run.head_sha }} + echo $CI_BRANCH_PUSH + echo $CI_BRANCH_WORKFLOW_RUN + echo $CI_SHA_PUSH + echo $CI_SHA_WORKFLOW_RUN + [[ ! -z "$CI_BRANCH_PUSH" ]] && echo "CI_BRANCH=$CI_BRANCH_PUSH" >> $GITHUB_ENV || echo "CI_BRANCH=$CI_BRANCH_WORKFLOW_RUN" >> $GITHUB_ENV + [[ ! -z "$CI_SHA_PUSH" ]] && echo "CI_SHA=$CI_SHA_PUSH" >> $GITHUB_ENV || echo "CI_SHA=$CI_SHA_WORKFLOW_RUN" >> $GITHUB_ENV + + - name: print environment variables + run: | + echo "env.CI_BRANCH = ${{ env.CI_BRANCH }}" + echo "env.CI_SHA = ${{ env.CI_SHA }}" + + - name: Update clone using environment variables working-directory: /workspace/transformers - run: git fetch && git checkout ${{ github.sha }} + run: | + echo "original branch = $(git branch --show-current)" + git fetch && git checkout ${{ env.CI_BRANCH }} + echo "updated branch = $(git branch --show-current)" + git checkout ${{ env.CI_SHA }} + echo "log = $(git log -n 1)" # To avoid unknown test failures - name: Pre build DeepSpeed *again* @@ -302,7 +453,39 @@ jobs: run_tests_torch_cuda_extensions_multi_gpu ] steps: + # Necessary to get the correct branch name and commit SHA for `workflow_run` event + # We also take into account the `push` event (we might want to test some changes in a branch) + - name: Prepare custom environment variables + shell: bash + # For the meaning of these environment variables, see the job `Setup` + run: | + CI_BRANCH_PUSH=${{ github.event.ref }} + CI_BRANCH_PUSH=${CI_BRANCH_PUSH/'refs/heads/'/''} + CI_BRANCH_WORKFLOW_RUN=${{ github.event.workflow_run.head_branch }} + CI_SHA_PUSH=${{ github.event.head_commit.id }} + CI_SHA_WORKFLOW_RUN=${{ github.event.workflow_run.head_sha }} + echo $CI_BRANCH_PUSH + echo $CI_BRANCH_WORKFLOW_RUN + echo $CI_SHA_PUSH + echo $CI_SHA_WORKFLOW_RUN + [[ ! -z "$CI_BRANCH_PUSH" ]] && echo "CI_BRANCH=$CI_BRANCH_PUSH" >> $GITHUB_ENV || echo "CI_BRANCH=$CI_BRANCH_WORKFLOW_RUN" >> $GITHUB_ENV + [[ ! -z "$CI_SHA_PUSH" ]] && echo "CI_SHA=$CI_SHA_PUSH" >> $GITHUB_ENV || echo "CI_SHA=$CI_SHA_WORKFLOW_RUN" >> $GITHUB_ENV + + - name: print environment variables + run: | + echo "env.CI_BRANCH = ${{ env.CI_BRANCH }}" + echo "env.CI_SHA = ${{ env.CI_SHA }}" + - uses: actions/checkout@v2 + + - name: Update clone using environment variables + run: | + echo "original branch = $(git branch --show-current)" + git fetch && git checkout ${{ env.CI_BRANCH }} + echo "updated branch = $(git branch --show-current)" + git checkout ${{ env.CI_SHA }} + echo "log = $(git log -n 1)" + - uses: actions/download-artifact@v2 - name: Send message to Slack env: @@ -312,8 +495,9 @@ jobs: CI_SLACK_CHANNEL_DUMMY_TESTS: ${{ secrets.CI_SLACK_CHANNEL_DUMMY_TESTS }} CI_SLACK_REPORT_CHANNEL_ID: ${{ secrets.CI_SLACK_CHANNEL_ID }} CI_EVENT: push - CI_TITLE: ${{ github.event.head_commit.message }} - CI_COMMIT_URL: ${{ github.event.head_commit.url }} + CI_TITLE_PUSH: ${{ github.event.head_commit.message }} + CI_TITLE_WORKFLOW_RUN: ${{ github.event.workflow_run.head_commit.message }} + CI_SHA: ${{ env.CI_SHA }} # We pass `needs.setup.outputs.matrix` as the argument. A processing in `notification_service.py` to change # `models/bert` to `models_bert` is required, as the artifact names use `_` instead of `/`. run: | diff --git a/utils/notification_service.py b/utils/notification_service.py index 0c2de5baca..cac4850399 100644 --- a/utils/notification_service.py +++ b/utils/notification_service.py @@ -584,9 +584,65 @@ def retrieve_available_artifacts(): if __name__ == "__main__": + org = "huggingface" + repo = "transformers" + repository_full_name = f"{org}/{repo}" + # This env. variable is set in workflow file (under the job `send_results`). ci_event = os.environ["CI_EVENT"] + # To find the PR number in a commit title, for example, `Add AwesomeFormer model (#99999)` + pr_number_re = re.compile(r"\(#(\d+)\)$") + + title = f"🤗 Results of the {ci_event} tests." + # Add Commit/PR title with a link for push CI + # (check the title in 2 env. variables - depending on the CI is triggered via `push` or `workflow_run` event) + ci_title_push = os.environ.get("CI_TITLE_PUSH") + ci_title_workflow_run = os.environ.get("CI_TITLE_WORKFLOW_RUN") + ci_title = ci_title_push if ci_title_push else ci_title_workflow_run + + ci_sha = os.environ.get("CI_SHA") + + ci_url = None + if ci_sha: + ci_url = f"https://github.com/{repository_full_name}/commit/{ci_sha}" + + if ci_title is not None: + if ci_url is None: + raise ValueError( + "When a title is found (`ci_title`), it means a `push` event or a `workflow_run` even (triggered by " + "another `push` event), and the commit SHA has to be provided in order to create the URL to the " + "commit page." + ) + ci_title = ci_title.strip().split("\n")[0].strip() + + # Retrieve the PR title and author login to complete the report + commit_number = ci_url.split("/")[-1] + ci_detail_url = f"https://api.github.com/repos/{repository_full_name}/commits/{commit_number}" + ci_details = requests.get(ci_detail_url).json() + ci_author = ci_details["author"]["login"] + + merged_by = None + # Find the PR number (if any) and change the url to the actual PR page. + numbers = pr_number_re.findall(ci_title) + if len(numbers) > 0: + pr_number = numbers[0] + ci_detail_url = f"https://api.github.com/repos/{repository_full_name}/pulls/{pr_number}" + ci_details = requests.get(ci_detail_url).json() + + ci_author = ci_details["user"]["login"] + ci_url = f"https://github.com/{repository_full_name}/pull/{pr_number}" + + merged_by = ci_details["merged_by"]["login"] + + if merged_by is None: + ci_title = f"<{ci_url}|{ci_title}>\nAuthor: {ci_author}" + else: + ci_title = f"<{ci_url}|{ci_title}>\nAuthor: {ci_author} | Merged by: {merged_by}" + + else: + ci_title = "" + arguments = sys.argv[1:][0] try: models = ast.literal_eval(arguments) @@ -747,45 +803,6 @@ if __name__ == "__main__": {"line": line, "trace": stacktraces.pop(0)} ) - # To find the PR number in a commit title, for example, `Add AwesomeFormer model (#99999)` - pr_number_re = re.compile(r"\(#(\d+)\)$") - - title = f"🤗 Results of the {ci_event} tests." - # Add PR title with a link for push CI - ci_title = os.environ.get("CI_TITLE") - ci_url = os.environ.get("CI_COMMIT_URL") - - if ci_title is not None: - assert ci_url is not None - ci_title = ci_title.strip().split("\n")[0].strip() - - # Retrieve the PR title and author login to complete the report - commit_number = ci_url.split("/")[-1] - ci_detail_url = f"https://api.github.com/repos/huggingface/transformers/commits/{commit_number}" - ci_details = requests.get(ci_detail_url).json() - ci_author = ci_details["author"]["login"] - - merged_by = None - # Find the PR number (if any) and change the url to the actual PR page. - numbers = pr_number_re.findall(ci_title) - if len(numbers) > 0: - pr_number = numbers[0] - ci_detail_url = f"https://api.github.com/repos/huggingface/transformers/pulls/{pr_number}" - ci_details = requests.get(ci_detail_url).json() - - ci_author = ci_details["user"]["login"] - ci_url = f"https://github.com/huggingface/transformers/pull/{pr_number}" - - merged_by = ci_details["merged_by"]["login"] - - if merged_by is None: - ci_title = f"<{ci_url}|{ci_title}>\nAuthor: {ci_author}" - else: - ci_title = f"<{ci_url}|{ci_title}>\nAuthor: {ci_author} | Merged by: {merged_by}" - - else: - ci_title = "" - message = Message(title, ci_title, model_results, additional_results) # send report only if there is any failure