From 81c5b2d993deb47c99c893aeb84765d87a0c4645 Mon Sep 17 00:00:00 2001 From: Angela P Wen Date: Mon, 1 Aug 2022 11:09:29 +0200 Subject: [PATCH 1/9] New debug artifacts workflow with separate jobs --- .github/workflows/__debug-artifacts.yml | 96 ------------------ .github/workflows/debug-artifacts.yml | 124 ++++++++++++++++++++++++ pr-checks/checks/debug-artifacts.yml | 36 ------- 3 files changed, 124 insertions(+), 132 deletions(-) delete mode 100644 .github/workflows/__debug-artifacts.yml create mode 100644 .github/workflows/debug-artifacts.yml delete mode 100644 pr-checks/checks/debug-artifacts.yml diff --git a/.github/workflows/__debug-artifacts.yml b/.github/workflows/__debug-artifacts.yml deleted file mode 100644 index e232a7190..000000000 --- a/.github/workflows/__debug-artifacts.yml +++ /dev/null @@ -1,96 +0,0 @@ -# Warning: This file is generated automatically, and should not be modified. -# Instead, please modify the template in the pr-checks directory and run: -# pip install ruamel.yaml && python3 sync.py -# to regenerate this file. - -name: PR Check - Debug artifact upload -env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GO111MODULE: auto -on: - push: - branches: - - main - - releases/v1 - - releases/v2 - pull_request: - types: - - opened - - synchronize - - reopened - - ready_for_review - workflow_dispatch: {} -jobs: - debug-artifacts: - strategy: - matrix: - include: - - os: ubuntu-latest - version: stable-20210308 - - os: macos-latest - version: stable-20210308 - - os: ubuntu-latest - version: stable-20210319 - - os: macos-latest - version: stable-20210319 - - os: ubuntu-latest - version: stable-20210809 - - os: macos-latest - version: stable-20210809 - - os: ubuntu-latest - version: cached - - os: macos-latest - version: cached - - os: ubuntu-latest - version: latest - - os: macos-latest - version: latest - - os: ubuntu-latest - version: nightly-latest - - os: macos-latest - version: nightly-latest - name: Debug artifact upload - timeout-minutes: 45 - runs-on: ${{ matrix.os }} - steps: - - name: Check out repository - uses: actions/checkout@v3 - - name: Prepare test - id: prepare-test - uses: ./.github/prepare-test - with: - version: ${{ matrix.version }} - - uses: ./../action/init - with: - tools: ${{ steps.prepare-test.outputs.tools-url }} - debug: true - debug-artifact-name: my-debug-artifacts - debug-database-name: my-db - - name: Build code - shell: bash - run: ./build.sh - - uses: ./../action/analyze - id: analysis - - uses: actions/download-artifact@v3 - with: - name: my-debug-artifacts-${{ matrix.os }}-${{ matrix.version }} - - shell: bash - run: | - LANGUAGES="cpp csharp go java javascript python" - for language in $LANGUAGES; do - echo "Checking $language" - if [[ ! -f "$language.sarif" ]] ; then - echo "Missing a SARIF file for $language" - exit 1 - fi - if [[ ! -f "my-db-$language.zip" ]] ; then - echo "Missing a database bundle for $language" - exit 1 - fi - if [[ ! -d "$language/log" ]] ; then - echo "Missing logs for $language" - exit 1 - fi - done - env: - INTERNAL_CODEQL_ACTION_DEBUG_LOC: true diff --git a/.github/workflows/debug-artifacts.yml b/.github/workflows/debug-artifacts.yml new file mode 100644 index 000000000..28912a98f --- /dev/null +++ b/.github/workflows/debug-artifacts.yml @@ -0,0 +1,124 @@ +# Checks logs, SARIF, and database bundle debug artifacts exist. +name: PR Check - Debug artifact upload +env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} +on: + push: + branches: + - main + - releases/v1 + - releases/v2 + pull_request: + types: + - opened + - synchronize + - reopened + - ready_for_review + workflow_dispatch: {} +jobs: + upload-artifacts: + strategy: + matrix: + include: + - os: ubuntu-latest + version: stable-20210308 + - os: macos-latest + version: stable-20210308 + - os: ubuntu-latest + version: stable-20210319 + - os: macos-latest + version: stable-20210319 + - os: ubuntu-latest + version: stable-20210809 + - os: macos-latest + version: stable-20210809 + - os: ubuntu-latest + version: cached + - os: macos-latest + version: cached + - os: ubuntu-latest + version: latest + - os: macos-latest + version: latest + - os: ubuntu-latest + version: nightly-latest + - os: macos-latest + version: nightly-latest + name: Upload debug artifacts + timeout-minutes: 45 + runs-on: ${{ matrix.os }} + steps: + - name: Check out repository + uses: actions/checkout@v3 + - name: Prepare test + id: prepare-test + uses: ./.github/prepare-test + with: + version: ${{ matrix.version }} + - uses: ./../action/init + with: + tools: ${{ steps.prepare-test.outputs.tools-url }} + debug: true + debug-artifact-name: my-debug-artifacts + debug-database-name: my-db + - name: Build code + shell: bash + run: ./build.sh + - uses: ./../action/analyze + id: analysis + download-and-check-artifacts: + strategy: + matrix: + include: + - os: ubuntu-latest + version: stable-20210308 + - os: macos-latest + version: stable-20210308 + - os: ubuntu-latest + version: stable-20210319 + - os: macos-latest + version: stable-20210319 + - os: ubuntu-latest + version: stable-20210809 + - os: macos-latest + version: stable-20210809 + - os: ubuntu-latest + version: cached + - os: macos-latest + version: cached + - os: ubuntu-latest + version: latest + - os: macos-latest + version: latest + - os: ubuntu-latest + version: nightly-latest + - os: macos-latest + version: nightly-latest + name: Download and check debug artifacts + needs: upload-artifacts + timeout-minutes: 45 + runs-on: ${{ matrix.os }} + steps: + - uses: actions/download-artifact@v3 + with: + name: my-debug-artifacts-${{ matrix.os }}-${{ matrix.version }} + - shell: bash + run: | + LANGUAGES="cpp csharp go java javascript python" + for language in $LANGUAGES; do + echo "Checking $language" + if [[ ! -f "$language.sarif" ]] ; then + echo "Missing a SARIF file for $language" + exit 1 + fi + if [[ ! -f "my-db-$language.zip" ]] ; then + echo "Missing a database bundle for $language" + exit 1 + fi + if [[ ! -d "$language/log" ]] ; then + echo "Missing logs for $language" + exit 1 + fi + done + env: + INTERNAL_CODEQL_ACTION_DEBUG_LOC: true \ No newline at end of file diff --git a/pr-checks/checks/debug-artifacts.yml b/pr-checks/checks/debug-artifacts.yml deleted file mode 100644 index 3c42dd47b..000000000 --- a/pr-checks/checks/debug-artifacts.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: "Debug artifact upload" -description: "Checks that debugging artifacts are correctly uploaded" -os: ["ubuntu-latest", "macos-latest"] -steps: - - uses: ./../action/init - with: - tools: ${{ steps.prepare-test.outputs.tools-url }} - debug: true - debug-artifact-name: "my-debug-artifacts" - debug-database-name: "my-db" - - name: Build code - shell: bash - run: ./build.sh - - uses: ./../action/analyze - id: analysis - - uses: actions/download-artifact@v3 - with: - name: my-debug-artifacts-${{ matrix.os }}-${{ matrix.version }} - - shell: bash - run: | - LANGUAGES="cpp csharp go java javascript python" - for language in $LANGUAGES; do - echo "Checking $language" - if [[ ! -f "$language.sarif" ]] ; then - echo "Missing a SARIF file for $language" - exit 1 - fi - if [[ ! -f "my-db-$language.zip" ]] ; then - echo "Missing a database bundle for $language" - exit 1 - fi - if [[ ! -d "$language/log" ]] ; then - echo "Missing logs for $language" - exit 1 - fi - done From cc047a34ce921cd0392ace81803c9fade53e0684 Mon Sep 17 00:00:00 2001 From: Angela P Wen Date: Tue, 2 Aug 2022 11:37:18 +0200 Subject: [PATCH 2/9] Condense matrix syntax --- .github/workflows/debug-artifacts.yml | 54 ++------------------------- 1 file changed, 4 insertions(+), 50 deletions(-) diff --git a/.github/workflows/debug-artifacts.yml b/.github/workflows/debug-artifacts.yml index 28912a98f..3de447d8d 100644 --- a/.github/workflows/debug-artifacts.yml +++ b/.github/workflows/debug-artifacts.yml @@ -19,31 +19,8 @@ jobs: upload-artifacts: strategy: matrix: - include: - - os: ubuntu-latest - version: stable-20210308 - - os: macos-latest - version: stable-20210308 - - os: ubuntu-latest - version: stable-20210319 - - os: macos-latest - version: stable-20210319 - - os: ubuntu-latest - version: stable-20210809 - - os: macos-latest - version: stable-20210809 - - os: ubuntu-latest - version: cached - - os: macos-latest - version: cached - - os: ubuntu-latest - version: latest - - os: macos-latest - version: latest - - os: ubuntu-latest - version: nightly-latest - - os: macos-latest - version: nightly-latest + os: [ubuntu-latest, macos-latest] + version: [stable-20210308, stable-20210319, stable-20210809, cached, latest, nightly-latest] name: Upload debug artifacts timeout-minutes: 45 runs-on: ${{ matrix.os }} @@ -69,31 +46,8 @@ jobs: download-and-check-artifacts: strategy: matrix: - include: - - os: ubuntu-latest - version: stable-20210308 - - os: macos-latest - version: stable-20210308 - - os: ubuntu-latest - version: stable-20210319 - - os: macos-latest - version: stable-20210319 - - os: ubuntu-latest - version: stable-20210809 - - os: macos-latest - version: stable-20210809 - - os: ubuntu-latest - version: cached - - os: macos-latest - version: cached - - os: ubuntu-latest - version: latest - - os: macos-latest - version: latest - - os: ubuntu-latest - version: nightly-latest - - os: macos-latest - version: nightly-latest + os: [ubuntu-latest, macos-latest] + version: [stable-20210308, stable-20210319, stable-20210809, cached, latest, nightly-latest] name: Download and check debug artifacts needs: upload-artifacts timeout-minutes: 45 From 13e7a528ebd0d0d5d923f7b91a28b4f12590c1a3 Mon Sep 17 00:00:00 2001 From: Angela P Wen Date: Tue, 2 Aug 2022 11:37:27 +0200 Subject: [PATCH 3/9] Change download job to only run on linux --- .github/workflows/debug-artifacts.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/debug-artifacts.yml b/.github/workflows/debug-artifacts.yml index 3de447d8d..ad7c37c7e 100644 --- a/.github/workflows/debug-artifacts.yml +++ b/.github/workflows/debug-artifacts.yml @@ -51,7 +51,7 @@ jobs: name: Download and check debug artifacts needs: upload-artifacts timeout-minutes: 45 - runs-on: ${{ matrix.os }} + runs-on: ubuntu-latest steps: - uses: actions/download-artifact@v3 with: From 64d83c75f083b74e965692f4d1e73bee121c5c4c Mon Sep 17 00:00:00 2001 From: Angela P Wen Date: Tue, 2 Aug 2022 11:38:33 +0200 Subject: [PATCH 4/9] Fix syntax error --- .github/workflows/debug-artifacts.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/debug-artifacts.yml b/.github/workflows/debug-artifacts.yml index ad7c37c7e..f7af04ff2 100644 --- a/.github/workflows/debug-artifacts.yml +++ b/.github/workflows/debug-artifacts.yml @@ -31,7 +31,7 @@ jobs: id: prepare-test uses: ./.github/prepare-test with: - version: ${{ matrix.version }} + version: ${{ matrix.version }} - uses: ./../action/init with: tools: ${{ steps.prepare-test.outputs.tools-url }} From ef145d688445016133d515516392d41cddac6a29 Mon Sep 17 00:00:00 2001 From: Angela P Wen Date: Wed, 3 Aug 2022 09:35:32 +0200 Subject: [PATCH 5/9] Display downloaded file structure --- .github/workflows/debug-artifacts.yml | 42 +++++++++++++-------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/.github/workflows/debug-artifacts.yml b/.github/workflows/debug-artifacts.yml index f7af04ff2..ce8921012 100644 --- a/.github/workflows/debug-artifacts.yml +++ b/.github/workflows/debug-artifacts.yml @@ -44,35 +44,35 @@ jobs: - uses: ./../action/analyze id: analysis download-and-check-artifacts: - strategy: - matrix: - os: [ubuntu-latest, macos-latest] - version: [stable-20210308, stable-20210319, stable-20210809, cached, latest, nightly-latest] name: Download and check debug artifacts needs: upload-artifacts timeout-minutes: 45 runs-on: ubuntu-latest steps: - uses: actions/download-artifact@v3 - with: - name: my-debug-artifacts-${{ matrix.os }}-${{ matrix.version }} + - name: Display structure of downloaded files + run: ls -R - shell: bash run: | + OPERATING_SYSTEMS="ubuntu-latest macos-latest" + VERSIONS="stable-20210308 stable-20210319 stable-20210809 cached latest nightly-latest" LANGUAGES="cpp csharp go java javascript python" - for language in $LANGUAGES; do - echo "Checking $language" - if [[ ! -f "$language.sarif" ]] ; then - echo "Missing a SARIF file for $language" - exit 1 - fi - if [[ ! -f "my-db-$language.zip" ]] ; then - echo "Missing a database bundle for $language" - exit 1 - fi - if [[ ! -d "$language/log" ]] ; then - echo "Missing logs for $language" - exit 1 - fi - done + for os in $OPERATING_SYSTEMS; do + for version in $VERSIONS; do + for language in $LANGUAGES; do + echo "Checking $language in $version on $os" + if [[ ! -f "$language.sarif" ]] ; then + echo "Missing a SARIF file for $language" + exit 1 + fi + if [[ ! -f "my-db-$language.zip" ]] ; then + echo "Missing a database bundle for $language" + exit 1 + fi + if [[ ! -d "$language/log" ]] ; then + echo "Missing logs for $language" + exit 1 + fi + done env: INTERNAL_CODEQL_ACTION_DEBUG_LOC: true \ No newline at end of file From f7a7ac1799920ee21ef237842427c8680e05ecaf Mon Sep 17 00:00:00 2001 From: Angela P Wen Date: Wed, 3 Aug 2022 10:23:41 +0200 Subject: [PATCH 6/9] bash syntax fixup --- .github/workflows/debug-artifacts.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/debug-artifacts.yml b/.github/workflows/debug-artifacts.yml index ce8921012..9d302a2ca 100644 --- a/.github/workflows/debug-artifacts.yml +++ b/.github/workflows/debug-artifacts.yml @@ -49,10 +49,12 @@ jobs: timeout-minutes: 45 runs-on: ubuntu-latest steps: - - uses: actions/download-artifact@v3 + - name: Download all artifacts + uses: actions/download-artifact@v3 - name: Display structure of downloaded files run: ls -R - - shell: bash + - name: Check expected artifacts exist + shell: bash run: | OPERATING_SYSTEMS="ubuntu-latest macos-latest" VERSIONS="stable-20210308 stable-20210319 stable-20210809 cached latest nightly-latest" @@ -74,5 +76,7 @@ jobs: exit 1 fi done + done + done env: INTERNAL_CODEQL_ACTION_DEBUG_LOC: true \ No newline at end of file From f51da8688a9f2eee73bd63473e7c557c56acded7 Mon Sep 17 00:00:00 2001 From: Angela P Wen Date: Wed, 3 Aug 2022 10:55:10 +0200 Subject: [PATCH 7/9] Add script to check in appropriate directories --- .github/workflows/debug-artifacts.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/debug-artifacts.yml b/.github/workflows/debug-artifacts.yml index 9d302a2ca..e69941c47 100644 --- a/.github/workflows/debug-artifacts.yml +++ b/.github/workflows/debug-artifacts.yml @@ -61,8 +61,10 @@ jobs: LANGUAGES="cpp csharp go java javascript python" for os in $OPERATING_SYSTEMS; do for version in $VERSIONS; do + cd ./my-debug-artifacts-$os-$version + echo "Checking version $version on $os" for language in $LANGUAGES; do - echo "Checking $language in $version on $os" + echo "Checking $language" if [[ ! -f "$language.sarif" ]] ; then echo "Missing a SARIF file for $language" exit 1 @@ -76,6 +78,7 @@ jobs: exit 1 fi done + cd .. done done env: From e16d085a47850ebcd88cf09e955f002f5db803f4 Mon Sep 17 00:00:00 2001 From: Angela P Wen Date: Wed, 3 Aug 2022 11:24:18 +0200 Subject: [PATCH 8/9] Remove ls check, log formatting --- .github/workflows/debug-artifacts.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/debug-artifacts.yml b/.github/workflows/debug-artifacts.yml index e69941c47..c352690ab 100644 --- a/.github/workflows/debug-artifacts.yml +++ b/.github/workflows/debug-artifacts.yml @@ -51,8 +51,6 @@ jobs: steps: - name: Download all artifacts uses: actions/download-artifact@v3 - - name: Display structure of downloaded files - run: ls -R - name: Check expected artifacts exist shell: bash run: | @@ -62,9 +60,9 @@ jobs: for os in $OPERATING_SYSTEMS; do for version in $VERSIONS; do cd ./my-debug-artifacts-$os-$version - echo "Checking version $version on $os" + echo "Artifacts from version $version on $os:" for language in $LANGUAGES; do - echo "Checking $language" + echo "- Checking $language" if [[ ! -f "$language.sarif" ]] ; then echo "Missing a SARIF file for $language" exit 1 From 4ec485d6339328f152e5d6287b8abdc748af8ecf Mon Sep 17 00:00:00 2001 From: Angela P Wen Date: Wed, 3 Aug 2022 17:57:05 +0200 Subject: [PATCH 9/9] bash syntax fixup per review --- .github/workflows/debug-artifacts.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/debug-artifacts.yml b/.github/workflows/debug-artifacts.yml index c352690ab..e10aad0a7 100644 --- a/.github/workflows/debug-artifacts.yml +++ b/.github/workflows/debug-artifacts.yml @@ -59,7 +59,7 @@ jobs: LANGUAGES="cpp csharp go java javascript python" for os in $OPERATING_SYSTEMS; do for version in $VERSIONS; do - cd ./my-debug-artifacts-$os-$version + pushd "./my-debug-artifacts-$os-$version" echo "Artifacts from version $version on $os:" for language in $LANGUAGES; do echo "- Checking $language" @@ -76,7 +76,7 @@ jobs: exit 1 fi done - cd .. + popd done done env: