From f9a19da7bf433d625a6766a0afbca7d853cb4015 Mon Sep 17 00:00:00 2001 From: Aditya Sharad Date: Thu, 8 Apr 2021 13:29:04 -0700 Subject: [PATCH 1/3] PR checks: Run integration tests against both `tools: null` and `tools: latest` Always test against both the default and latest CodeQL bundle. This improves test coverage shortly after a CodeQL bundle release, where the latest bundle may not yet be built into the Actions VM image as the default bundle. It also saves a manual step during bundle release testing, since we no longer need to temporarily change the PR checks to `tools: latest`. There is some redundancy when the latest bundle is the same as the default bundle on the VM image, but this can be considered a test for the `tools: latest` configuration. --- .github/workflows/codeql.yml | 2 ++ .github/workflows/pr-checks.yml | 14 ++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 420c1ce09..1edf53b7b 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -11,6 +11,7 @@ jobs: strategy: matrix: os: [ubuntu-latest,windows-latest,macos-latest] + tools: [~, latest] runs-on: ${{ matrix.os }} steps: @@ -20,6 +21,7 @@ jobs: with: languages: javascript config-file: ./.github/codeql/codeql-config.yml + tools: ${{ matrix.tools }} # confirm steps.init.outputs.codeql-path points to the codeql binary - name: Print CodeQL Version run: ${{steps.init.outputs.codeql-path}} version --format=json diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 5716217c9..634dfc4b7 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -117,6 +117,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, windows-latest, macos-latest] + tools: [~, latest] runs-on: ${{ matrix.os }} steps: @@ -136,6 +137,7 @@ jobs: with: languages: go config-file: ./.github/codeql/custom-queries.yml + tools: ${{ matrix.tools }} - name: Build code shell: bash run: ./build.sh @@ -149,6 +151,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, windows-latest, macos-latest] + tools: [~, latest] runs-on: ${{ matrix.os }} env: CODEQL_EXTRACTOR_GO_BUILD_TRACING: "on" @@ -169,6 +172,7 @@ jobs: - uses: ./../action/init with: languages: go + tools: ${{ matrix.tools }} - name: Build code shell: bash run: go build main.go @@ -178,6 +182,10 @@ jobs: go-custom-tracing-autobuild: needs: [check-js, check-node-modules] + strategy: + fail-fast: false + matrix: + tools: [~, latest] # No need to test Go autobuild on multiple OSes since # we're testing Go custom tracing with a manual build on all OSes. runs-on: ubuntu-latest @@ -196,6 +204,7 @@ jobs: - uses: ./../action/init with: languages: go + tools: ${{ matrix.tools }} - uses: ./../action/autobuild - uses: ./../action/analyze env: @@ -236,6 +245,10 @@ jobs: test-proxy: needs: [check-js, check-node-modules] + strategy: + fail-fast: false + matrix: + tools: [~, latest] runs-on: ubuntu-latest container: image: ubuntu:18.04 @@ -259,6 +272,7 @@ jobs: - uses: ./../action/init with: languages: javascript + tools: ${{ matrix.tools }} - uses: ./../action/analyze env: TEST_MODE: true From 51b1d7d81f882fef7a53cd961b292fe0addf62fb Mon Sep 17 00:00:00 2001 From: Aditya Sharad Date: Fri, 9 Apr 2021 13:13:48 -0700 Subject: [PATCH 2/3] PR checks: Compare the default and latest CodeQL tools bundles Create a prerequisite job that runs the init step twice, with `tools: null` and `tools: latest`. Use the outputs of these steps to compare the two CodeQL versions. Pass the list of distinct tool versions for the integration tests to use in their matrix strategy. This avoids redundant test jobs when the default and latest bundles are actually the same version of CodeQL. `~` is accepted by JSON but not by the Actions context language, so we use `null` to indicate the default version. --- .github/workflows/pr-checks.yml | 72 ++++++++++++++++++++++++++++----- 1 file changed, 62 insertions(+), 10 deletions(-) diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 634dfc4b7..fffec2f2f 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -80,13 +80,65 @@ jobs: exit 1 fi - multi-language-repo_test-custom-queries-and-remote-config: + # Identify the CodeQL tool versions to integration test against. + check-codeql-versions: needs: [check-js, check-node-modules] + runs-on: ubuntu-latest + outputs: + versions: ${{ steps.compare.outputs.versions }} + + steps: + - uses: actions/checkout@v2 + - name: Move codeql-action + shell: bash + run: | + mkdir ../action + mv * .github ../action/ + mv ../action/tests/multi-language-repo/{*,.github} . + mv ../action/.github/workflows .github + - name: Init with default CodeQL bundle from the VM image + id: init-default + uses: ./../action/init + with: + languages: javascript + - name: Remove empty database + # allows us to run init a second time + run: | + rm -rf "$RUNNER_TEMP/codeql_databases" + - name: Init with latest CodeQL bundle + id: init-latest + uses: ./../action/init + with: + tools: latest + languages: javascript + - name: Compare default and latest CodeQL bundle versions + id: compare + env: + CODEQL_DEFAULT: ${{ steps.init-default.outputs.codeql-path }} + CODEQL_LATEST: ${{ steps.init-latest.outputs.codeql-path }} + run: | + CODEQL_VERSION_DEFAULT="$("$CODEQL_DEFAULT" version --format terse)" + CODEQL_VERSION_LATEST="$("$CODEQL_LATEST" version --format terse)" + echo "Default CodeQL bundle version is $CODEQL_VERSION_DEFAULT" + echo "Latest CodeQL bundle version is $CODEQL_VERSION_LATEST" + if [[ "$CODEQL_VERSION_DEFAULT" == "$CODEQL_VERSION_LATEST" ]]; then + # Just use `tools: null` to avoid duplication in the integration tests. + VERSIONS_JSON='[null]' + else + # Use both `tools: null` and `tools: latest` in the integration tests. + VERSIONS_JSON='[null, "latest"]' + fi + # Output a JSON-encoded list with the distinct versions to test against. + echo "Suggested matrix config for integration tests: $VERSIONS_JSON" + echo "::set-output name=versions::${VERSIONS_JSON}" + + multi-language-repo_test-custom-queries-and-remote-config: + needs: [check-js, check-node-modules, check-codeql-versions] strategy: fail-fast: false matrix: os: [ubuntu-latest, windows-latest, macos-latest] - tools: [~, latest] + tools: ${{ fromJson(needs.check-codeql-versions.outputs.versions) }} runs-on: ${{ matrix.os }} steps: @@ -112,12 +164,12 @@ jobs: # Currently is not possible to analyze Go in conjunction with other languages in macos multi-language-repo_test-go-custom-queries: - needs: [check-js, check-node-modules] + needs: [check-js, check-node-modules, check-codeql-versions] strategy: fail-fast: false matrix: os: [ubuntu-latest, windows-latest, macos-latest] - tools: [~, latest] + tools: ${{ fromJson(needs.check-codeql-versions.outputs.versions) }} runs-on: ${{ matrix.os }} steps: @@ -146,12 +198,12 @@ jobs: TEST_MODE: true go-custom-tracing: - needs: [check-js, check-node-modules] + needs: [check-js, check-node-modules, check-codeql-versions] strategy: fail-fast: false matrix: os: [ubuntu-latest, windows-latest, macos-latest] - tools: [~, latest] + tools: ${{ fromJson(needs.check-codeql-versions.outputs.versions) }} runs-on: ${{ matrix.os }} env: CODEQL_EXTRACTOR_GO_BUILD_TRACING: "on" @@ -181,11 +233,11 @@ jobs: TEST_MODE: true go-custom-tracing-autobuild: - needs: [check-js, check-node-modules] + needs: [check-js, check-node-modules, check-codeql-versions] strategy: fail-fast: false matrix: - tools: [~, latest] + tools: ${{ fromJson(needs.check-codeql-versions.outputs.versions) }} # No need to test Go autobuild on multiple OSes since # we're testing Go custom tracing with a manual build on all OSes. runs-on: ubuntu-latest @@ -244,11 +296,11 @@ jobs: TEST_MODE: true test-proxy: - needs: [check-js, check-node-modules] + needs: [check-js, check-node-modules, check-codeql-versions] strategy: fail-fast: false matrix: - tools: [~, latest] + tools: ${{ fromJson(needs.check-codeql-versions.outputs.versions) }} runs-on: ubuntu-latest container: image: ubuntu:18.04 From 64b50fa2a6301f4801faa50d0636e138210f5e3c Mon Sep 17 00:00:00 2001 From: Aditya Sharad Date: Fri, 9 Apr 2021 14:51:17 -0700 Subject: [PATCH 3/3] Code scanning: Compare the default and latest CodeQL tools bundles Create a prerequisite job that runs the init step twice, with `tools: null` and `tools: latest`. Use the outputs of these steps to compare the two CodeQL versions. Pass the list of distinct tool versions for the analysis job to matrix over. This lets us test the analysis against both versions, while avoiding duplication when they are actually the same version. --- .github/workflows/codeql.yml | 47 +++++++++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 1edf53b7b..8d7adfd89 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -7,11 +7,56 @@ on: branches: [main, v1] jobs: + # Identify the CodeQL tool versions to use in the analysis job. + check-codeql-versions: + runs-on: ubuntu-latest + outputs: + versions: ${{ steps.compare.outputs.versions }} + + steps: + - uses: actions/checkout@v2 + - name: Init with default CodeQL bundle from the VM image + id: init-default + uses: ./init + with: + languages: javascript + - name: Remove empty database + # allows us to run init a second time + run: | + rm -rf "$RUNNER_TEMP/codeql_databases" + - name: Init with latest CodeQL bundle + id: init-latest + uses: ./init + with: + tools: latest + languages: javascript + - name: Compare default and latest CodeQL bundle versions + id: compare + env: + CODEQL_DEFAULT: ${{ steps.init-default.outputs.codeql-path }} + CODEQL_LATEST: ${{ steps.init-latest.outputs.codeql-path }} + run: | + CODEQL_VERSION_DEFAULT="$("$CODEQL_DEFAULT" version --format terse)" + CODEQL_VERSION_LATEST="$("$CODEQL_LATEST" version --format terse)" + echo "Default CodeQL bundle version is $CODEQL_VERSION_DEFAULT" + echo "Latest CodeQL bundle version is $CODEQL_VERSION_LATEST" + if [[ "$CODEQL_VERSION_DEFAULT" == "$CODEQL_VERSION_LATEST" ]]; then + # Just use `tools: null` to avoid duplication in the analysis job. + VERSIONS_JSON='[null]' + else + # Use both `tools: null` and `tools: latest` in the analysis job. + VERSIONS_JSON='[null, "latest"]' + fi + # Output a JSON-encoded list with the distinct versions to test against. + echo "Suggested matrix config for analysis job: $VERSIONS_JSON" + echo "::set-output name=versions::${VERSIONS_JSON}" + build: + needs: [check-codeql-versions] strategy: matrix: os: [ubuntu-latest,windows-latest,macos-latest] - tools: [~, latest] + tools: ${{ fromJson(needs.check-codeql-versions.outputs.versions) }} runs-on: ${{ matrix.os }} steps: