From 0fecfd6c4578d17070db91bfa4920888258b987b Mon Sep 17 00:00:00 2001 From: Mauricio Alvarez Leon <65101411+BBBmau@users.noreply.github.com> Date: Wed, 3 Jul 2024 15:27:29 -0700 Subject: [PATCH] CI: test the tests (#32869) --- .github/workflows/ci_weekly_report.yaml | 123 ++++++++++++++++++++++++ .github/workflows/ci_weekly_run.yaml | 21 ++++ .github/workflows/docs.yaml | 9 +- .github/workflows/selfdrive_tests.yaml | 12 ++- .github/workflows/tools_tests.yaml | 9 +- selfdrive/ui/tests/test_ui/run.py | 2 +- 6 files changed, 168 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/ci_weekly_report.yaml create mode 100644 .github/workflows/ci_weekly_run.yaml diff --git a/.github/workflows/ci_weekly_report.yaml b/.github/workflows/ci_weekly_report.yaml new file mode 100644 index 0000000000..c5c98b46b5 --- /dev/null +++ b/.github/workflows/ci_weekly_report.yaml @@ -0,0 +1,123 @@ +name: weekly CI test report +on: + schedule: + - cron: '37 9 * * 1' # 9:37AM UTC -> 2:37AM PST every monday + workflow_dispatch: + inputs: + ci_runs: + description: 'The amount of runs to trigger in CI test report' +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + setup: + if: github.repository == 'commaai/openpilot' + runs-on: ubuntu-latest + outputs: + ci_runs: ${{ steps.ci_runs_setup.outputs.value }} + steps: + - id: ci_runs_setup + run: | + CI_RUNS=${{ inputs.ci_runs || '50' }} + mylist="value=[" + + for i in $(seq 1 $CI_RUNS); + do + if [ $i != $CI_RUNS ]; then + mylist+="\"$i\", " + else + mylist+="\"$i\"]" + fi + done + + echo "$mylist" >> $GITHUB_OUTPUT + echo "Number of CI runs for report: $CI_RUNS" + ci_matrix_run: + needs: [ setup ] + strategy: + fail-fast: false + matrix: + value: ${{fromJSON(needs.setup.outputs.ci_runs)}} + uses: commaai/openpilot/.github/workflows/ci_weekly_run.yaml@master + with: + run_number: ${{ matrix.value }} + + report: + needs: [ci_matrix_run] + runs-on: ubuntu-latest + if: always() + steps: + - name: Get job results + uses: actions/github-script@v7 + id: get-job-results + with: + script: | + const jobs = await github + .paginate("GET /repos/{owner}/{repo}/actions/runs/{run_id}/attempts/{attempt}/jobs", { + owner: "commaai", + repo: "${{ github.event.repository.name }}", + run_id: "${{ github.run_id }}", + attempt: "${{ github.run_attempt }}", + }) + var report = {} + jobs.slice(1, jobs.length-1).forEach(job => { + const jobName = job.name.split('/')[2].trim(); + report[jobName] = report[jobName] || { successes: [], failures: [], cancelled: [] }; + switch (job.conclusion) { + case "success": + report[jobName].successes.push(job.html_url); break; + case "failure": + report[jobName].failures.push(job.html_url); break; + case "cancelled": + report[jobName].cancelled.push(job.html_url); break; + } + }); + return JSON.stringify(report); + + - name: Add job results to summary + env: + JOB_RESULTS: ${{ fromJSON(steps.get-job-results.outputs.result) }} + run: | + echo $JOB_RESULTS > job_results.json + generate_html_table() { + echo "" + echo "" + echo " " + echo " " + echo " " + echo " " + echo " " + echo " " + echo "" + jq -r ' + "", + keys[] as $job | + "", + " ", + " ", + " ", + " ", + "" + ' job_results.json + echo "" + echo "
JobSucceeded ✅Failed ❌Cancelled (timed out) ⏰
\($job)", + "
", + " (\(.[$job].successes | length))", + " \(.[$job].successes[])
", + "
", + " 		", + "
", + " (\(.[$job].failures | length))", + " \(.[$job].failures[])
", + "
", + " 		", + "
", + " (\(.[$job].cancelled | length))", + " \(.[$job].cancelled[])
", + "
", + "
" + } + echo "# CI Job Summary" >> $GITHUB_STEP_SUMMARY + generate_html_table >> $GITHUB_STEP_SUMMARY + diff --git a/.github/workflows/ci_weekly_run.yaml b/.github/workflows/ci_weekly_run.yaml new file mode 100644 index 0000000000..c3ce8c42da --- /dev/null +++ b/.github/workflows/ci_weekly_run.yaml @@ -0,0 +1,21 @@ +name: weekly CI test run +on: + workflow_call: + inputs: + run_number: + required: true + type: string + +concurrency: + group: ci-run-${{ inputs.run_number }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + selfdrive_tests: + uses: commaai/openpilot/.github/workflows/selfdrive_tests.yaml@master + with: + run_number: ${{ inputs.run_number }} + tools_tests: + uses: commaai/openpilot/.github/workflows/tools_tests.yaml@master + with: + run_number: ${{ inputs.run_number }} diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index a32989f7d0..6428729055 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -5,9 +5,14 @@ on: branches: - master pull_request: - + workflow_call: + inputs: + run_number: + default: '1' + required: true + type: string concurrency: - group: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }} + group: docs-tests-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }} cancel-in-progress: true env: diff --git a/.github/workflows/selfdrive_tests.yaml b/.github/workflows/selfdrive_tests.yaml index c2b321525b..db763923b9 100644 --- a/.github/workflows/selfdrive_tests.yaml +++ b/.github/workflows/selfdrive_tests.yaml @@ -6,9 +6,15 @@ on: - master pull_request: workflow_dispatch: + workflow_call: + inputs: + run_number: + default: '1' + required: true + type: string concurrency: - group: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }} + group: selfdrive-tests-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }} cancel-in-progress: true env: @@ -320,5 +326,5 @@ jobs: - name: Upload Test Report uses: actions/upload-artifact@v4 with: - name: report - path: selfdrive/ui/tests/test_ui/report + name: report-${{ inputs.run_number }} + path: selfdrive/ui/tests/test_ui/report_${{ inputs.run_number }} diff --git a/.github/workflows/tools_tests.yaml b/.github/workflows/tools_tests.yaml index a04d5e6b6b..b64963fb60 100644 --- a/.github/workflows/tools_tests.yaml +++ b/.github/workflows/tools_tests.yaml @@ -5,9 +5,14 @@ on: branches: - master pull_request: - + workflow_call: + inputs: + run_number: + default: '1' + required: true + type: string concurrency: - group: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }} + group: tools-tests-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }} cancel-in-progress: true env: diff --git a/selfdrive/ui/tests/test_ui/run.py b/selfdrive/ui/tests/test_ui/run.py index 1d33c103a4..98a600eebc 100644 --- a/selfdrive/ui/tests/test_ui/run.py +++ b/selfdrive/ui/tests/test_ui/run.py @@ -116,7 +116,7 @@ CASES = { TEST_DIR = pathlib.Path(__file__).parent -TEST_OUTPUT_DIR = TEST_DIR / "report" +TEST_OUTPUT_DIR = TEST_DIR / "report_1" SCREENSHOTS_DIR = TEST_OUTPUT_DIR / "screenshots"