Merge pull request #26 from github/integration-tests

Testing overhaul

.github/codeql/codeql-config.yml

@@ -1,4 +1,6 @@
-me: "CodeQL config"
+name: "CodeQL config"
 queries: 
   - name: Run custom queries
     uses: ./queries
+paths-ignore:
+  - tests

.github/workflows/codeql.yml

@@ -13,5 +13,6 @@ jobs:
     - uses: actions/checkout@v1
     - uses: ./init
       with:
-       config-file: ./.github/codeql/codeql-config.yml
+        languages: javascript
+        config-file: ./.github/codeql/codeql-config.yml
     - uses: ./analyze

.github/workflows/integration-testing.yml

@@ -1,22 +1,117 @@
 name: "Integration Testing"
 
-on: [push]
+on: [push, pull_request]
 
 jobs:
-  dispatch-events:
-    if: github.event.repository.full_name == 'github/codeql-action'
+  multi-language-repo_test-autodetect-languages:
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+    runs-on: ${{ matrix.os }}
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Move codeql-action
+      shell: bash
+      run: |
+        mkdir ../action
+        shopt -s dotglob
+        mv * ../action/
+        mv ../action/tests/multi-language-repo/* .
+    - uses: ./../action/init
+    - name: Build code
+      shell: bash
+      run: ./build.sh
+    - uses: ./../action/analyze
+      env:
+        TEST_MODE: true
+
+  multi-language-repo_test-custom-queries:
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+    runs-on: ${{ matrix.os }}
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Move codeql-action
+      shell: bash
+      run: |
+        mkdir ../action
+        shopt -s dotglob
+        mv * ../action/
+        mv ../action/tests/multi-language-repo/* .
+    - uses: ./../action/init
+      with:
+        languages: cpp,csharp,java,javascript,python
+        config-file: ./.github/codeql/custom-queries.yml
+    - name: Build code
+      shell: bash
+      run: ./build.sh
+    - uses: ./../action/analyze
+      env:
+        TEST_MODE: true
+
+  # Currently is not possible to analyze Go in conjunction with other languages in macos
+  multi-language-repo_test-go-custom-queries:
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+    runs-on: ${{ matrix.os }}
+
+    steps:
+    - uses: actions/setup-go@v2
+      if: ${{ matrix.os ==  'macos-latest' }}
+      with:
+        go-version: '^1.13.1'
+    - uses: actions/checkout@v2
+    - name: Move codeql-action
+      shell: bash
+      run: |
+        mkdir ../action
+        shopt -s dotglob
+        mv * ../action/
+        mv ../action/tests/multi-language-repo/* .
+    - uses: ./../action/init
+      with:
+        languages: go
+        config-file: ./.github/codeql/custom-queries.yml
+    - name: Build code
+      shell: bash
+      run: ./build.sh
+    - uses: ./../action/analyze
+      env:
+        TEST_MODE: true
+
+
+  multi-language-repo_rubocop:
     runs-on: ubuntu-latest
+
     steps:
-    - name: Send repository dispatch events
+    - uses: actions/checkout@v2
+    - name: Move codeql-action
+      shell: bash
+      run: |
+        mkdir ../action
+        shopt -s dotglob
+        mv * ../action/
+        mv ../action/tests/multi-language-repo/* .
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: 2.6
+    - name: Install Code Scanning integration
+      run: bundle add code-scanning-rubocop --version 0.2.0 --skip-install
+    - name: Install dependencies
+      run: bundle install
+    - name: Rubocop run
       run: |
-        curl -X POST \
-          -H "Authorization: Bearer ${{ secrets.CODEQL_TESTING_TOKEN }}" \
-          -H "Accept: application/vnd.github.everest-preview+json" \
-          https://api.github.com/repos/Anthophila/amazon-cognito-js-copy/dispatches \
-          -d '{"event_type":"codeql-integration","client_payload": {"sha": "${{ github.sha }}"}}'
-
-        curl -X POST \
-          -H "Authorization: Bearer ${{ secrets.CODEQL_TESTING_TOKEN }}" \
-          -H "Accept: application/vnd.github.everest-preview+json" \
-          https://api.github.com/repos/Anthophila/electron-test-action/dispatches \
-          -d '{"event_type":"codeql-integration","client_payload": {"sha": "${{ github.sha }}"}}'
+        bash -c "
+          bundle exec rubocop --require code_scanning --format CodeScanning::SarifFormatter -o rubocop.sarif
+          [[ $? -ne 2 ]]
+        "
+    - uses: ./../action/upload-sarif
+      with:
+        sarif_file: rubocop.sarif
+      env:
+        TEST_MODE: true

.github/workflows/js-uptodate-check.yml → .github/workflows/lint-build-test.yml

@@ -1,8 +1,16 @@
-name: "Check generated JavaScript"
+name: "Lint, Build & Test"
 
-on: [pull_request]
+on: [push, pull_request]
 
 jobs:
+  tslint:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v1
+    - name: tslint
+      run: npm run-script lint
+
   check-js:
     runs-on: ubuntu-latest
 
@@ -25,3 +33,11 @@ jobs:
           exit 1
         fi
         echo "Success: JavaScript files are up to date"
+
+  npm-test:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v1
+    - name: npm run-script test
+      run: npm run-script test

src/upload-lib.ts

@@ -52,6 +52,12 @@ export function combineSarifFiles(sarifFiles: string[]): string {
 async function uploadPayload(payload): Promise<boolean> {
     core.info('Uploading results');
 
+    // If in test mode we don't want to upload the results
+    const testMode = process.env['TEST_MODE'] === 'true' || false;
+    if (testMode) {
+        return true;
+    }
+
     const githubToken = core.getInput('token');
     const ph: auth.BearerCredentialHandler = new auth.BearerCredentialHandler(githubToken);
     const client = new http.HttpClient('Code Scanning : Upload SARIF', [ph]);

tests/multi-language-repo/.github/codeql/custom-queries.yml

@@ -0,0 +1,29 @@
+name: Use custom queries
+
+disable-default-queries: true
+
+queries: 
+# Query suites
+  - name: Select a query suite 
+    uses: ./codeql-qlpacks/complex-python-qlpack/rootAndBar.qls
+# QL pack subset
+  - name: Select a ql file 
+    uses: ./codeql-qlpacks/complex-javascript-qlpack/show_ifs.ql
+  - name: Select a subfolder
+    uses: ./codeql-qlpacks/complex-javascript-qlpack/foo
+  - name: Select a folder with two subfolders 
+    uses: ./codeql-qlpacks/complex-javascript-qlpack/foo2
+# Inrepo QL pack
+  - name: Select an inrepo ql pack
+    uses: ./codeql-qlpacks/csharp-qlpack
+  - name: Java queries 
+    uses: ./codeql-qlpacks/java-qlpack
+# External QL packs
+  - name: Go queries 
+    uses: Anthophila/go-querypack@master
+  - name: Cpp queries 
+    uses: Anthophila/cpp-querypack@second-branch
+  - name: Javascript queries 
+    uses: Anthophila/javascript-querypack/show_ifs2.ql@master
+  - name: Python queries 
+    uses: Anthophila/python-querypack/show_ifs2.ql@second-branch 

tests/multi-language-repo/Gemfile

@@ -0,0 +1,2 @@
+source "https://rubygems.org" do
+end

tests/multi-language-repo/Main.java

@@ -0,0 +1,8 @@
+class Main {
+    public static void main(String args[]) {
+        if (true) {
+            System.out.println("Hello, World!");
+        }
+    }
+}
+

tests/multi-language-repo/build.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+gcc -o main main.c
+
+dotnet build
+
+javac Main.java
+

tests/multi-language-repo/codeql-qlpacks/complex-javascript-qlpack/foo/show_ifs.ql

@@ -0,0 +1,11 @@
+/**
+ * @name Foo Show Ifs
+ * @description Foo Show Ifs
+ * @kind problem
+ * @id complex-javascript-querypack/foo-show-ifs
+ */
+
+import javascript
+
+from IfStmt i
+select i, "foo if"

tests/multi-language-repo/codeql-qlpacks/complex-javascript-qlpack/foo2/bar/show_ifs.ql

@@ -0,0 +1,11 @@
+/**
+ * @name Bar Show Ifs
+ * @description Bar Show Ifs
+ * @kind problem
+ * @id complex-javascript-querypack/bar-ifs
+ */
+
+import javascript
+
+from IfStmt i
+select i, "bar if"

tests/multi-language-repo/codeql-qlpacks/complex-javascript-qlpack/foo2/barfoo/barfoobar/show_ifs.ql

@@ -0,0 +1,11 @@
+/**
+ * @name Barfoobar Show Ifs
+ * @description Barfoobar Show Ifs
+ * @kind problem
+ * @id complex-javascript-querypack/barfoobar-ifs
+ */
+
+import javascript
+
+from IfStmt i
+select i, "barfoobar if"

tests/multi-language-repo/codeql-qlpacks/complex-javascript-qlpack/foo2/show_ifs.ql

@@ -0,0 +1,11 @@
+/**
+ * @name Foo2 Show Ifs
+ * @description Foo2 Show Ifs
+ * @kind problem
+ * @id complex-javascript-querypack/foo2-ifs
+ */
+
+import javascript
+
+from IfStmt i
+select i, "foo2 if"

tests/multi-language-repo/codeql-qlpacks/complex-javascript-qlpack/qlpack.yml

@@ -0,0 +1,3 @@
+name: complex-javascript-querypack
+version: 0.0.1
+libraryPathDependencies: codeql-javascript

tests/multi-language-repo/codeql-qlpacks/complex-javascript-qlpack/show_ifs.ql

@@ -0,0 +1,11 @@
+/**
+ * @name Root Show Ifs
+ * @description Root Show Ifs
+ * @kind problem
+ * @id complex-javascript-querypack/root-show-ifs
+ */
+
+import javascript
+
+from IfStmt i
+select i, "root if"

tests/multi-language-repo/codeql-qlpacks/complex-python-qlpack/foo/bar/show_ifs.ql

@@ -0,0 +1,11 @@
+/**
+ * @name Foo/Bar Show Ifs
+ * @description Foo/Bar Show Ifs
+ * @kind problem
+ * @id complex-python-querypack/foo/bar/show-ifs
+ */
+
+import python
+
+from If i
+select i, "foo/bar if"

tests/multi-language-repo/codeql-qlpacks/complex-python-qlpack/foo/show_ifs.ql

@@ -0,0 +1,11 @@
+/**
+ * @name Foo Show Ifs
+ * @description Foo Show Ifs
+ * @kind problem
+ * @id complex-python-querypack/foo/show-ifs
+ */
+
+import python
+
+from If i
+select i, "foo if"

tests/multi-language-repo/codeql-qlpacks/complex-python-qlpack/qlpack.yml

@@ -0,0 +1,3 @@
+name: inrepo-python-querypack
+version: 0.0.1
+libraryPathDependencies: codeql-python

tests/multi-language-repo/codeql-qlpacks/complex-python-qlpack/rootAndBar.qls

@@ -0,0 +1,2 @@
+- query: show_ifs.ql
+- query: foo/bar/show_ifs.ql