Collated reports (#40080)

* Add initial collated reports script and job definition

* provide commit hash for this run. Also use hash in generated artifact name. Json formatting

* tidy

* Add option to upload collated reports to hf hub

* Add glob pattern for test report folders

* Fix glob

* Use machine_type as path filter instead of glob. Include machine_type in collated report

.github/workflows/collated-reports.yml

@@ -0,0 +1,49 @@
+name: CI collated reports
+
+on:
+  workflow_call:
+    inputs:
+      job:
+        required: true
+        type: string
+      report_repo_id:
+        required: true
+        type: string
+      machine_type:
+        required: true
+        type: string
+      gpu_name:
+        description: Name of the GPU used for the job. Its enough that the value contains the name of the GPU, e.g. "noise-h100-more-noise". Case insensitive.
+        required: true
+        type: string
+
+jobs:
+  collated_reports:
+    name: Collated reports
+    runs-on: ubuntu-22.04
+    if: always()
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
+
+      - name: Collated reports
+        shell: bash
+        env:
+          ACCESS_REPO_INFO_TOKEN: ${{ secrets.ACCESS_REPO_INFO_TOKEN }}
+          CI_SHA: ${{ github.sha }}
+          TRANSFORMERS_CI_RESULTS_UPLOAD_TOKEN: ${{ secrets.TRANSFORMERS_CI_RESULTS_UPLOAD_TOKEN }}
+        run: |
+          pip install huggingface_hub
+          python3 utils/collated_reports.py                  \
+            --path /transformers/reports/                    \
+            --machine-type ${{ inputs.machine_type }}        \
+            --commit-hash ${{ env.CI_SHA }}                  \
+            --job ${{ inputs.job }}                          \
+            --report-repo-id ${{ inputs.report_repo_id }}    \
+            --gpu-name ${{ inputs.gpu_name }}
+
+      - name: Upload collated reports
+        uses: actions/upload-artifact@v4
+        with:
+          name: collated_reports_${{ env.CI_SHA }}.json
+          path: collated_reports_${{ env.CI_SHA }}.json