Refactor PR checks

.gitattributes

@@ -1,4 +1,5 @@
 lib/*.js linguist-generated=true
+.github/workflows/__* linguist-generated=true
 
 # Reduce incidence of needless merge conflicts on CHANGELOG.md
 # The man page at

.github/prepare-test/action.yml

@@ -0,0 +1,39 @@
+name: "Prepare test"
+description: Performs some preparation to run tests
+inputs:
+  version:
+    required: true
+outputs:
+  tools-url:
+    value: ${{ steps.get-url.outputs.tools-url }}
+runs:
+  using: composite
+  steps:
+    - name: Move codeql-action
+      shell: bash
+      run: |
+        mkdir ../action
+        mv * .github ../action/
+        mv ../action/tests/multi-language-repo/{*,.github} .
+        mv ../action/.github/workflows .github
+    - id: get-url
+      name: Determine URL
+      shell: bash
+      run: |
+        if [[ ${{ inputs.version }} == "nightly-latest" ]]; then
+          export LATEST=`gh release list --repo dsp-testing/codeql-cli-nightlies -L 1 | cut -f 3`
+          echo "::set-output name=tools-url::https://github.com/dsp-testing/codeql-cli-nightlies/releases/download/$LATEST/codeql-bundle.tar.gz"
+        elif [[ ${{ inputs.version }} == *"nightly"* ]]; then
+          export VERSION=`echo ${{ inputs.version }} | sed -e 's/^.*\-//'`
+          echo "::set-output name=tools-url::https://github.com/dsp-testing/codeql-cli-nightlies/releases/download/codeql-bundle-$VERSION-manual/codeql-bundle.tar.gz"
+        elif [[ ${{ inputs.version }} == *"stable"* ]]; then
+          export VERSION=`echo ${{ inputs.version }} | sed -e 's/^.*\-//'`
+          echo "Hello $VERSION"
+          echo "::set-output name=tools-url::https://github.com/github/codeql-action/releases/download/codeql-bundle-$VERSION/codeql-bundle.tar.gz"
+        elif [[ ${{ inputs.version }} == "latest" ]]; then
+          echo "::set-output name=tools-url::latest"
+        elif [[ ${{ inputs.version }} == "cached" ]]; then
+          echo "::set-output name=tools-url::"
+        else
+          echo "::error Unrecognized version specified!"
+        fi

.github/workflows/script/verify-pr-checks.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+set -eu
+
+# Sanity check that repo is clean to start with
+if [ ! -z "$(git status --porcelain)" ]; then
+    # If we get a fail here then this workflow needs attention...
+    >&2 echo "Failed: Repo should be clean before testing!"
+    exit 1
+fi
+
+# Wipe the generated PR checks in case there are extra unnecessary files in there
+rm -rf .github/workflows/__*
+
+# Generate the PR checks
+cd pr-checks && python3 sync.py
+
+# Check that repo is still clean
+if [ ! -z "$(git status --porcelain)" ]; then
+    # If we get a fail here then the PR needs attention
+    git diff
+    git status
+    >&2 echo "Failed: PR checks are not up to date. Run 'cd pr-checks && python3 sync.py' to update"
+    exit 1
+fi
+echo "Success: PR checks are up to date"

pr-checks/checks/go-custom-queries.yml

@@ -0,0 +1,17 @@
+name: "Go: Custom queries"
+description: "Checks that Go works in conjunction with a config file specifying custom queries"
+steps:
+  - uses: actions/setup-go@v2
+    with:
+      go-version: "^1.13.1"
+  - uses: ./../action/init
+    with:
+      languages: go
+      config-file: ./.github/codeql/custom-queries.yml
+      tools: ${{ steps.prepare-test.outputs.tools-url }}
+  - name: Build code
+    shell: bash
+    run: ./build.sh
+  - uses: ./../action/analyze
+    env:
+      TEST_MODE: true

pr-checks/checks/go-custom-tracing-autobuild.yml

@@ -0,0 +1,24 @@
+name: "Go: Autobuild custom tracing"
+description: "Checks that Go tracing works in conjunction with the autobuilder"
+os: ["ubuntu-latest", "macos-latest"]
+env:
+  CODEQL_EXTRACTOR_GO_BUILD_TRACING: "true"
+steps:
+  - uses: actions/setup-go@v2
+    with:
+      go-version: "^1.13.1"
+  - uses: ./../action/init
+    with:
+      languages: go
+      tools: ${{ steps.prepare-test.outputs.tools-url }}
+  - uses: ./../action/autobuild
+  - uses: ./../action/analyze
+    env:
+      TEST_MODE: true
+  - shell: bash
+    run: |
+      cd "$RUNNER_TEMP/codeql_databases"
+      if [[ ! -d go ]]; then
+        echo "Did not find a Go database"
+        exit 1
+      fi

pr-checks/checks/go-custom-tracing.yml

@@ -0,0 +1,18 @@
+name: "Go: Custom tracing"
+description: "Checks that Go tracing works"
+env:
+  CODEQL_EXTRACTOR_GO_BUILD_TRACING: "true"
+steps:
+  - uses: actions/setup-go@v2
+    with:
+      go-version: "^1.13.1"
+  - uses: ./../action/init
+    with:
+      languages: go
+      tools: ${{ steps.prepare-test.outputs.tools-url }}
+  - name: Build code
+    shell: bash
+    run: go build main.go
+  - uses: ./../action/analyze
+    env:
+      TEST_MODE: true

pr-checks/checks/javascript-source-root.yml

@@ -0,0 +1,27 @@
+name: "Custom source root"
+description: "Checks that the argument specifying a non-default source root works"
+versions: ["latest", "cached", "nightly-latest"] # This feature is not compatible with old CLIs
+os: ["ubuntu-latest"]
+steps:
+  - name: Move codeql-action
+    shell: bash
+    run: |
+      mkdir ../new-source-root
+      mv * ../new-source-root
+  - uses: ./../action/init
+    with:
+      languages: javascript
+      source-root: ../new-source-root
+      tools: ${{ steps.prepare-test.outputs.tools-url }}
+  - uses: ./../action/analyze
+    with:
+      skip-queries: true
+      upload: false
+  - name: Assert database exists
+    shell: bash
+    run: |
+      cd "$RUNNER_TEMP/codeql_databases"
+      if [[ ! -d javascript ]]; then
+        echo "Did not find a JavaScript database"
+        exit 1
+      fi

pr-checks/checks/multi-language-autodetect.yml

@@ -0,0 +1,47 @@
+name: "Multi-language repository"
+description: "An end-to-end integration test of a multi-language repository using automatic language detection"
+os: ["ubuntu-latest", "macos-latest"]
+steps:
+  - uses: ./../action/init
+    with:
+      db-location: "${{ runner.temp }}/customDbLocation"
+      tools: ${{ steps.prepare-test.outputs.tools-url }}
+  - name: Build code
+    shell: bash
+    run: ./build.sh
+  - uses: ./../action/analyze
+    id: analysis
+    env:
+      TEST_MODE: true
+  - shell: bash
+    run: |
+      CPP_DB=${{ fromJson(steps.analysis.outputs.db-locations).cpp }}
+      if [[ ! -d $CPP_DB ]] || [[ ! $CPP_DB == ${{ runner.temp }}/customDbLocation/* ]]; then
+        echo "Did not create a database for CPP, or created it in the wrong location."
+        exit 1
+      fi
+      CSHARP_DB=${{ fromJson(steps.analysis.outputs.db-locations).csharp }}
+      if [[ ! -d $CSHARP_DB ]] || [[ ! $CSHARP_DB == ${{ runner.temp }}/customDbLocation/* ]]; then
+        echo "Did not create a database for C Sharp, or created it in the wrong location."
+        exit 1
+      fi
+      GO_DB=${{ fromJson(steps.analysis.outputs.db-locations).go }}
+      if [[ ! -d $GO_DB ]] || [[ ! $GO_DB == ${{ runner.temp }}/customDbLocation/* ]]; then
+        echo "Did not create a database for Go, or created it in the wrong location."
+        exit 1
+      fi
+      JAVA_DB=${{ fromJson(steps.analysis.outputs.db-locations).java }}
+      if [[ ! -d $JAVA_DB ]] || [[ ! $JAVA_DB == ${{ runner.temp }}/customDbLocation/* ]]; then
+        echo "Did not create a database for Java, or created it in the wrong location."
+        exit 1
+      fi
+      JAVASCRIPT_DB=${{ fromJson(steps.analysis.outputs.db-locations).javascript }}
+      if [[ ! -d $JAVASCRIPT_DB ]] || [[ ! $JAVASCRIPT_DB == ${{ runner.temp }}/customDbLocation/* ]]; then
+        echo "Did not create a database for Javascript, or created it in the wrong location."
+        exit 1
+      fi
+      PYTHON_DB=${{ fromJson(steps.analysis.outputs.db-locations).python }}
+      if [[ ! -d $PYTHON_DB ]] || [[ ! $PYTHON_DB == ${{ runner.temp }}/customDbLocation/* ]]; then
+        echo "Did not create a database for Python, or created it in the wrong location."
+        exit 1
+      fi

pr-checks/checks/packaging-config-inputs-js.yml

@@ -0,0 +1,33 @@
+name: "Packaging: Config and input"
+description: "Checks that specifying packages using a combination of a config file and input to the Action works"
+versions: ["nightly-20210831"] # This CLI version is known to work with package used in this test
+os: ["ubuntu-latest", "macos-latest"]
+steps:
+  - uses: ./../action/init
+    with:
+      config-file: ".github/codeql/codeql-config-packaging3.yml"
+      packs: +dsp-testing/codeql-pack1@0.1.0
+      languages: javascript
+      tools: ${{ steps.prepare-test.outputs.tools-url }}
+  - name: Build code
+    shell: bash
+    run: ./build.sh
+  - uses: ./../action/analyze
+    with:
+      output: "${{ runner.temp }}/results"
+    env:
+      TEST_MODE: true
+  - name: Assert Results
+    shell: bash
+    run: |
+      cd "$RUNNER_TEMP/results"
+      # We should have 3 hits from these rules
+      EXPECTED_RULES="javascript/example/empty-or-one-block javascript/example/empty-or-one-block javascript/example/two-block"
+
+      # use tr to replace newlines with spaces and xargs to trim leading and trailing whitespace
+      RULES="$(cat javascript.sarif | jq -r '.runs[0].results[].ruleId' | sort | tr "\n" " " | xargs)"
+      echo "Found matching rules '$RULES'"
+      if [ "$RULES" != "$EXPECTED_RULES" ]; then
+        echo "Did not match expected rules '$EXPECTED_RULES'."
+        exit 1
+      fi

pr-checks/checks/packaging-config-js.yml

@@ -0,0 +1,32 @@
+name: "Packaging: Config file"
+description: "Checks that specifying packages using only a config file works"
+versions: ["nightly-20210831"] # This CLI version is known to work with package used in this test
+os: ["ubuntu-latest", "macos-latest"]
+steps:
+  - uses: ./../action/init
+    with:
+      config-file: ".github/codeql/codeql-config-packaging.yml"
+      languages: javascript
+      tools: ${{ steps.prepare-test.outputs.tools-url }}
+  - name: Build code
+    shell: bash
+    run: ./build.sh
+  - uses: ./../action/analyze
+    with:
+      output: "${{ runner.temp }}/results"
+    env:
+      TEST_MODE: true
+  - name: Assert Results
+    shell: bash
+    run: |
+      cd "$RUNNER_TEMP/results"
+      # We should have 3 hits from these rules
+      EXPECTED_RULES="javascript/example/empty-or-one-block javascript/example/empty-or-one-block javascript/example/two-block"
+
+      # use tr to replace newlines with spaces and xargs to trim leading and trailing whitespace
+      RULES="$(cat javascript.sarif | jq -r '.runs[0].results[].ruleId' | sort | tr "\n" " " | xargs)"
+      echo "Found matching rules '$RULES'"
+      if [ "$RULES" != "$EXPECTED_RULES" ]; then
+        echo "Did not match expected rules '$EXPECTED_RULES'."
+        exit 1
+      fi

pr-checks/checks/packaging-inputs-js.yml

@@ -0,0 +1,33 @@
+name: "Packaging: Action input"
+description: "Checks that specifying packages using the input to the Action works"
+versions: ["nightly-20210831"] # This CLI version is known to work with package used in this test
+os: ["ubuntu-latest", "macos-latest"]
+steps:
+  - uses: ./../action/init
+    with:
+      config-file: ".github/codeql/codeql-config-packaging2.yml"
+      languages: javascript
+      packs: dsp-testing/codeql-pack1@0.1.0, dsp-testing/codeql-pack2
+      tools: ${{ steps.prepare-test.outputs.tools-url }}
+  - name: Build code
+    shell: bash
+    run: ./build.sh
+  - uses: ./../action/analyze
+    with:
+      output: "${{ runner.temp }}/results"
+    env:
+      TEST_MODE: true
+  - name: Assert Results
+    shell: bash
+    run: |
+      cd "$RUNNER_TEMP/results"
+      # We should have 3 hits from these rules
+      EXPECTED_RULES="javascript/example/empty-or-one-block javascript/example/empty-or-one-block javascript/example/two-block"
+
+      # use tr to replace newlines with spaces and xargs to trim leading and trailing whitespace
+      RULES="$(cat javascript.sarif | jq -r '.runs[0].results[].ruleId' | sort | tr "\n" " " | xargs)"
+      echo "Found matching rules '$RULES'"
+      if [ "$RULES" != "$EXPECTED_RULES" ]; then
+        echo "Did not match expected rules '$EXPECTED_RULES'."
+        exit 1
+      fi

pr-checks/checks/remote-config.yml

@@ -0,0 +1,14 @@
+name: "Remote config file"
+description: "Checks that specifying packages using only a config file works"
+steps:
+  - uses: ./../action/init
+    with:
+      tools: ${{ steps.prepare-test.outputs.tools-url }}
+      languages: cpp,csharp,java,javascript,python
+      config-file: github/codeql-action/tests/multi-language-repo/.github/codeql/custom-queries.yml@${{ github.sha }}
+  - name: Build code
+    shell: bash
+    run: ./build.sh
+  - uses: ./../action/analyze
+    env:
+      TEST_MODE: true

pr-checks/checks/rubocop-multi-language.yml

@@ -0,0 +1,26 @@
+name: "RuboCop multi-language"
+description: "Tests using RuboCop to analyze a multi-language repository and then using the CodeQL Action to upload the resulting SARIF"
+os: ["ubuntu-latest"]
+steps:
+  - name: Set up Ruby
+    uses: ruby/setup-ruby@v1
+    with:
+      ruby-version: 2.6
+  - name: Install Code Scanning integration
+    shell: bash
+    run: bundle add code-scanning-rubocop --version 0.3.0 --skip-install
+  - name: Install dependencies
+    shell: bash
+    run: bundle install
+  - name: RuboCop run
+    shell: bash
+    run: |
+      bash -c "
+        bundle exec rubocop --require code_scanning --format CodeScanning::SarifFormatter -o rubocop.sarif
+        [[ $? -ne 2 ]]
+      "
+  - uses: ./../action/upload-sarif
+    with:
+      sarif_file: rubocop.sarif
+    env:
+      TEST_MODE: true

pr-checks/checks/split-workflow.yml

@@ -0,0 +1,46 @@
+name: "Split workflow"
+description: "Tests a split-up workflow in which we first build a database and later analyze it"
+versions: ["nightly-20210831"] # This CLI version is known to work with package used in this test
+os: ["ubuntu-latest", "macos-latest"]
+steps:
+  - uses: ./../action/init
+    with:
+      config-file: ".github/codeql/codeql-config-packaging3.yml"
+      packs: +dsp-testing/codeql-pack1@0.1.0
+      languages: javascript
+      tools: ${{ steps.prepare-test.outputs.tools-url }}
+  - name: Build code
+    shell: bash
+    run: ./build.sh
+  - uses: ./../action/analyze
+    with:
+      skip-queries: true
+      output: "${{ runner.temp }}/results"
+    env:
+      TEST_MODE: true
+  - name: Assert No Results
+    shell: bash
+    run: |
+      if [ "$(ls -A $RUNNER_TEMP/results)" ]; then
+        echo "Expected results directory to be empty after skipping query execution!"
+        exit 1
+      fi
+  - uses: ./../action/analyze
+    with:
+      output: "${{ runner.temp }}/results"
+    env:
+      TEST_MODE: true
+  - name: Assert Results
+    shell: bash
+    run: |
+      cd "$RUNNER_TEMP/results"
+      # We should have 3 hits from these rules
+      EXPECTED_RULES="javascript/example/empty-or-one-block javascript/example/empty-or-one-block javascript/example/two-block"
+
+      # use tr to replace newlines with spaces and xargs to trim leading and trailing whitespace
+      RULES="$(cat javascript.sarif | jq -r '.runs[0].results[].ruleId' | sort | tr "\n" " " | xargs)"
+      echo "Found matching rules '$RULES'"
+      if [ "$RULES" != "$EXPECTED_RULES" ]; then
+        echo "Did not match expected rules '$EXPECTED_RULES'."
+        exit 1
+      fi

pr-checks/checks/test-local-codeql.yml

@@ -0,0 +1,20 @@
+name: "Local CodeQL bundle"
+description: "Tests using a CodeQL bundle from a local file rather than a URL"
+versions: ["nightly-latest"]
+os: ["ubuntu-latest"]
+steps:
+  - name: Fetch a CodeQL bundle
+    shell: bash
+    env:
+      CODEQL_URL: ${{ steps.prepare-test.outputs.tools-url }}
+    run: |
+      wget "$CODEQL_URL"
+  - uses: ./../action/init
+    with:
+      tools: ./codeql-bundle.tar.gz
+  - name: Build code
+    shell: bash
+    run: ./build.sh
+  - uses: ./../action/analyze
+    env:
+      TEST_MODE: true

pr-checks/checks/test-proxy.yml

@@ -0,0 +1,22 @@
+name: "Proxy test"
+description: "Tests using a proxy specified by the https_proxy environment variable"
+versions: ["latest"]
+os: ["ubuntu-latest"]
+container:
+  image: ubuntu:18.04
+  options: --dns 127.0.0.1
+services:
+  squid-proxy:
+    image: datadog/squid:latest
+    ports:
+      - 3128:3128
+env:
+  https_proxy: http://squid-proxy:3128
+steps:
+  - uses: ./../action/init
+    with:
+      languages: javascript
+      tools: ${{ steps.prepare-test.outputs.tools-url }}
+  - uses: ./../action/analyze
+    env:
+      TEST_MODE: true

pr-checks/checks/test-ruby.yml

@@ -0,0 +1,23 @@
+name: "Ruby analysis"
+description: "Tests creation of a Ruby database"
+versions: ["latest", "cached", "nightly-latest"]
+os: ["ubuntu-latest", "macos-latest"]
+env:
+  CODEQL_ENABLE_EXPERIMENTAL_FEATURES: "true"
+steps:
+  - uses: ./../action/init
+    with:
+      languages: ruby
+      tools: ${{ steps.prepare-test.outputs.tools-url }}
+  - uses: ./../action/analyze
+    id: analysis
+    env:
+      TEST_MODE: true
+  - name: Check database
+    shell: bash
+    run: |
+      RUBY_DB="${{ fromJson(steps.analysis.outputs.db-locations).ruby }}"
+      if [[ ! -d "$RUBY_DB" ]]; then
+        echo "Did not create a database for Ruby."
+        exit 1
+      fi

pr-checks/sync.py

@@ -0,0 +1,104 @@
+import ruamel.yaml
+import os
+
+defaultTestVersions = [
+    # The oldest supported CodeQL version: 2.3.1. If bumping, update `CODEQL_MINIMUM_VERSION` in `codeql.ts`
+    "stable-20201028",
+    # The last CodeQL release in the 2.4 series: 2.4.6.
+    "stable-20210319",
+    # The last CodeQL release in the 2.5 series: 2.5.9.
+    "stable-20210809",
+    # The version of CodeQL currently in the toolcache. Typically either the latest release or the one before.
+    "cached",
+    # The latest release of CodeQL.
+    "latest",
+    # A nightly build directly from the our private repo, built in the last 24 hours.
+    "nightly-latest"
+]
+defaultOperatingSystems = ["ubuntu-latest", "macos-latest", "windows-latest"]
+header = """# Warning: This file is generated automatically, and should not be modified.
+# Instead, please modify the template in the pr-checks directory and run:
+#     pip install ruamel.yaml && python3 sync.py
+# to regenerate this file.
+
+"""
+
+
+class NonAliasingRTRepresenter(ruamel.yaml.representer.RoundTripRepresenter):
+    def ignore_aliases(self, data):
+        return True
+
+
+def writeHeader(checkStream):
+    checkStream.write(header)
+
+
+yaml = ruamel.yaml.YAML()
+yaml.Representer = NonAliasingRTRepresenter
+allJobs = {}
+for file in os.listdir('checks'):
+    with open(f"checks/{file}", 'r') as checkStream:
+        checkSpecification = yaml.load(checkStream)
+
+    versions = defaultTestVersions
+    if 'versions' in checkSpecification:
+        versions = checkSpecification['versions']
+    operatingSystems = defaultOperatingSystems
+    if 'os' in checkSpecification:
+        operatingSystems = checkSpecification['os']
+
+    steps = [
+        {
+            'name': 'Check out repository',
+            'uses': 'actions/checkout@v2'
+        },
+        {
+            'name': 'Prepare test',
+            'id': 'prepare-test',
+            'uses': './.github/prepare-test',
+            'with': {
+                'version': '${{ matrix.version }}'
+            }
+        }
+    ]
+    steps.extend(checkSpecification['steps'])
+
+    checkJob = {
+        'strategy': {
+            'matrix': {
+                'version': versions,
+                'os': operatingSystems
+            }
+        },
+        'name': checkSpecification['name'],
+        'runs-on': '${{ matrix.os }}',
+        'steps': steps
+    }
+
+    for key in ["env", "container", "services"]:
+        if key in checkSpecification:
+            checkJob[key] = checkSpecification[key]
+
+    checkName = file[:len(file) - 4]
+
+    with open(f"../.github/workflows/__{checkName}.yml", 'w') as output_stream:
+        writeHeader(output_stream)
+        yaml.dump({
+            'name': f"PR Check - {checkSpecification['name']}",
+            'env': {
+                'GITHUB_TOKEN': '${{ secrets.GITHUB_TOKEN }}',
+                'GO111MODULE': 'auto',
+            },
+            'on': {
+                'push': {
+                    'branches': ['main', 'v1']
+                },
+                'pull_request': {
+                    'types': ["opened", "synchronize", "reopened", "ready_for_review"]
+                },
+                'workflow_dispatch': {}
+            },
+            'jobs': {
+                checkName: checkJob
+            }
+        }, output_stream)