From 43de3a9949c52a8cd5d93b2e0e72143ef35d69a6 Mon Sep 17 00:00:00 2001 From: Robert Brignull Date: Thu, 30 Apr 2020 15:02:35 +0100 Subject: [PATCH 1/8] start uploading analysis_key parameter --- lib/shared-environment.js | 1 + lib/upload-lib.js | 2 ++ lib/util.js | 41 +++++++++++++++++++++++++++++++++ src/shared-environment.ts | 1 + src/upload-lib.ts | 2 ++ src/util.ts | 48 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 95 insertions(+) diff --git a/lib/shared-environment.js b/lib/shared-environment.js index 1a6a3d329..fabb260dd 100644 --- a/lib/shared-environment.js +++ b/lib/shared-environment.js @@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true }); exports.CODEQL_ACTION_CMD = 'CODEQL_ACTION_CMD'; exports.CODEQL_ACTION_DATABASE_DIR = 'CODEQL_ACTION_DATABASE_DIR'; exports.CODEQL_ACTION_LANGUAGES = 'CODEQL_ACTION_LANGUAGES'; +exports.CODEQL_ACTION_ANALYSIS_KEY = 'CODEQL_ACTION_ANALYSIS_KEY'; exports.ODASA_TRACER_CONFIGURATION = 'ODASA_TRACER_CONFIGURATION'; exports.CODEQL_ACTION_SCANNED_LANGUAGES = 'CODEQL_ACTION_SCANNED_LANGUAGES'; exports.CODEQL_ACTION_TRACED_LANGUAGES = 'CODEQL_ACTION_TRACED_LANGUAGES'; diff --git a/lib/upload-lib.js b/lib/upload-lib.js index f28f085a4..22d4e226d 100644 --- a/lib/upload-lib.js +++ b/lib/upload-lib.js @@ -83,6 +83,7 @@ async function uploadFiles(sarifFiles) { const commitOid = util.getRequiredEnvParam('GITHUB_SHA'); const workflowRunIDStr = util.getRequiredEnvParam('GITHUB_RUN_ID'); const ref = util.getRequiredEnvParam('GITHUB_REF'); // it's in the form "refs/heads/master" + const analysisKey = await util.getAnalysisKey(); const analysisName = util.getRequiredEnvParam('GITHUB_WORKFLOW'); const startedAt = process.env[sharedEnv.CODEQL_ACTION_STARTED_AT]; core.debug("Uploading sarif files: " + JSON.stringify(sarifFiles)); @@ -104,6 +105,7 @@ async function uploadFiles(sarifFiles) { const payload = JSON.stringify({ "commit_oid": commitOid, "ref": ref, + "analysis_key": analysisKey, "analysis_name": analysisName, "sarif": zipped_sarif, "workflow_run_id": workflowRunID, diff --git a/lib/util.js b/lib/util.js index d12a91044..3a2d90450 100644 --- a/lib/util.js +++ b/lib/util.js @@ -149,6 +149,47 @@ async function getLanguages() { return languages; } exports.getLanguages = getLanguages; +/** + * Get the path of the currently executing workflow. + */ +async function getWorkflowPath() { + const repo_nwo = getRequiredEnvParam('GITHUB_REPOSITORY').split("/"); + const owner = repo_nwo[0]; + const repo = repo_nwo[1]; + const run_id = getRequiredEnvParam('GITHUB_RUN_ID'); + const ok = new octokit.Octokit({ + auth: core.getInput('token'), + userAgent: "CodeQL Action", + log: console_log_level_1.default({ level: 'debug' }) + }); + const runsResponse = await ok.request('GET /repos/:owner/:repo/actions/runs/:run_id', { + owner, + repo, + run_id + }); + const workflowUrl = runsResponse.data.workflow_url; + const workflowResponse = await ok.request('GET ' + workflowUrl); + return workflowResponse.data.path; +} +/** + * Get the analysis key paramter for the current job. + * + * This will combine the workflow path and current job name. + * Computing this the first time requires making requests to + * the github API, but after that the result will be cached. + */ +async function getAnalysisKey() { + let analysisKey = process.env[sharedEnv.CODEQL_ACTION_ANALYSIS_KEY]; + if (analysisKey !== undefined) { + return analysisKey; + } + const workflowPath = await getWorkflowPath(); + const jobName = getRequiredEnvParam('GITHUB_JOB'); + analysisKey = workflowPath + ' - ' + jobName; + core.exportVariable(sharedEnv.CODEQL_ACTION_ANALYSIS_KEY, analysisKey); + return analysisKey; +} +exports.getAnalysisKey = getAnalysisKey; /** * Compose a StatusReport. * diff --git a/src/shared-environment.ts b/src/shared-environment.ts index c9c16e20e..fbc94edb9 100644 --- a/src/shared-environment.ts +++ b/src/shared-environment.ts @@ -1,6 +1,7 @@ export const CODEQL_ACTION_CMD = 'CODEQL_ACTION_CMD'; export const CODEQL_ACTION_DATABASE_DIR = 'CODEQL_ACTION_DATABASE_DIR'; export const CODEQL_ACTION_LANGUAGES = 'CODEQL_ACTION_LANGUAGES'; +export const CODEQL_ACTION_ANALYSIS_KEY = 'CODEQL_ACTION_ANALYSIS_KEY'; export const ODASA_TRACER_CONFIGURATION = 'ODASA_TRACER_CONFIGURATION'; export const CODEQL_ACTION_SCANNED_LANGUAGES = 'CODEQL_ACTION_SCANNED_LANGUAGES'; export const CODEQL_ACTION_TRACED_LANGUAGES = 'CODEQL_ACTION_TRACED_LANGUAGES'; diff --git a/src/upload-lib.ts b/src/upload-lib.ts index 8c6a31e4e..74d4e6dfe 100644 --- a/src/upload-lib.ts +++ b/src/upload-lib.ts @@ -76,6 +76,7 @@ async function uploadFiles(sarifFiles: string[]) { const commitOid = util.getRequiredEnvParam('GITHUB_SHA'); const workflowRunIDStr = util.getRequiredEnvParam('GITHUB_RUN_ID'); const ref = util.getRequiredEnvParam('GITHUB_REF'); // it's in the form "refs/heads/master" + const analysisKey = await util.getAnalysisKey(); const analysisName = util.getRequiredEnvParam('GITHUB_WORKFLOW'); const startedAt = process.env[sharedEnv.CODEQL_ACTION_STARTED_AT]; @@ -103,6 +104,7 @@ async function uploadFiles(sarifFiles: string[]) { const payload = JSON.stringify({ "commit_oid": commitOid, "ref": ref, + "analysis_key": analysisKey, "analysis_name": analysisName, "sarif": zipped_sarif, "workflow_run_id": workflowRunID, diff --git a/src/util.ts b/src/util.ts index cfdd2419c..708aa56c0 100644 --- a/src/util.ts +++ b/src/util.ts @@ -150,6 +150,54 @@ export async function getLanguages(): Promise { return languages; } +/** + * Get the path of the currently executing workflow. + */ +async function getWorkflowPath(): Promise { + const repo_nwo = getRequiredEnvParam('GITHUB_REPOSITORY').split("/"); + const owner = repo_nwo[0]; + const repo = repo_nwo[1]; + const run_id = getRequiredEnvParam('GITHUB_RUN_ID'); + + const ok = new octokit.Octokit({ + auth: core.getInput('token'), + userAgent: "CodeQL Action", + log: consoleLogLevel({ level: 'debug' }) + }); + + const runsResponse = await ok.request('GET /repos/:owner/:repo/actions/runs/:run_id', { + owner, + repo, + run_id + }); + const workflowUrl = runsResponse.data.workflow_url; + + const workflowResponse = await ok.request('GET ' + workflowUrl); + + return workflowResponse.data.path; +} + +/** + * Get the analysis key paramter for the current job. + * + * This will combine the workflow path and current job name. + * Computing this the first time requires making requests to + * the github API, but after that the result will be cached. + */ +export async function getAnalysisKey(): Promise { + let analysisKey = process.env[sharedEnv.CODEQL_ACTION_ANALYSIS_KEY]; + if (analysisKey !== undefined) { + return analysisKey; + } + + const workflowPath = await getWorkflowPath(); + const jobName = getRequiredEnvParam('GITHUB_JOB'); + + analysisKey = workflowPath + ' - ' + jobName; + core.exportVariable(sharedEnv.CODEQL_ACTION_ANALYSIS_KEY, analysisKey); + return analysisKey; +} + interface StatusReport { "workflow_run_id": number; "workflow_name": string; From 546d5a8843825a85bb84e2240128bbde7278c7fc Mon Sep 17 00:00:00 2001 From: Robert Brignull Date: Thu, 30 Apr 2020 16:04:19 +0100 Subject: [PATCH 2/8] URL encode the key --- lib/util.js | 2 +- src/util.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/util.js b/lib/util.js index 3a2d90450..5df9ca205 100644 --- a/lib/util.js +++ b/lib/util.js @@ -185,7 +185,7 @@ async function getAnalysisKey() { } const workflowPath = await getWorkflowPath(); const jobName = getRequiredEnvParam('GITHUB_JOB'); - analysisKey = workflowPath + ' - ' + jobName; + analysisKey = encodeURI(workflowPath) + ':' + encodeURI(jobName); core.exportVariable(sharedEnv.CODEQL_ACTION_ANALYSIS_KEY, analysisKey); return analysisKey; } diff --git a/src/util.ts b/src/util.ts index 708aa56c0..b8c81525c 100644 --- a/src/util.ts +++ b/src/util.ts @@ -193,7 +193,7 @@ export async function getAnalysisKey(): Promise { const workflowPath = await getWorkflowPath(); const jobName = getRequiredEnvParam('GITHUB_JOB'); - analysisKey = workflowPath + ' - ' + jobName; + analysisKey = encodeURI(workflowPath) + ':' + encodeURI(jobName); core.exportVariable(sharedEnv.CODEQL_ACTION_ANALYSIS_KEY, analysisKey); return analysisKey; } From d90fca396a8c2971e72efdb96e26b4b462d098c4 Mon Sep 17 00:00:00 2001 From: Robert Date: Fri, 24 Apr 2020 17:15:22 +0100 Subject: [PATCH 3/8] Create undeclared-action-input.ql --- queries/undeclared-action-input.ql | 63 ++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 queries/undeclared-action-input.ql diff --git a/queries/undeclared-action-input.ql b/queries/undeclared-action-input.ql new file mode 100644 index 000000000..23627205b --- /dev/null +++ b/queries/undeclared-action-input.ql @@ -0,0 +1,63 @@ +/** + * @name Undeclared action input + * @description Code tries to use an input parameter that is not defined for this action. + Perhaps this code is shared by multiple actions. + * @kind problem + * @problem.severity error + * @id javascript/codeql-action/undeclared-action-input + */ + +import javascript + +class ActionDeclaration extends File { + ActionDeclaration() { + getRelativePath().matches("%/action.yml") + } + + string getName() { + result = getRelativePath().regexpCapture("(.*)/action.yml", 1) + } + + YAMLDocument getRootNode() { + result.getFile() = this + } + + string getAnInput() { + result = getRootNode().(YAMLMapping).lookup("inputs").(YAMLMapping).getKey(_).(YAMLString).getValue() + } + + FunctionDeclStmt getEntrypoint() { + result.getFile().getRelativePath() = getRootNode(). + (YAMLMapping).lookup("runs"). + (YAMLMapping).lookup("main"). + (YAMLString).getValue().regexpReplaceAll("\\.\\./lib/(.*)\\.js", "src/$1.ts") and + result.getName() = "run" + } +} + +Expr getAFunctionChildExpr(Function f) { + result = f.getBody().getAChildStmt*().getAChildExpr*() +} + +/* + * Result is a function that is called from the body of the given function `f` + */ +Function calledBy(Function f) { + result = getAFunctionChildExpr(f).(InvokeExpr).getResolvedCallee() +} + +class GetInputMethodCallExpr extends MethodCallExpr { + GetInputMethodCallExpr() { + getMethodName() = "getInput" + } + + string getInputName() { + result = getArgument(0).(StringLiteral).getValue() + } +} + +from ActionDeclaration action, GetInputMethodCallExpr getInputCall, string inputName +where getAFunctionChildExpr(calledBy*(action.getEntrypoint())) = getInputCall and + inputName = getInputCall.getInputName() and + not inputName = action.getAnInput() +select getInputCall, "The $@ input is not defined for the $@ action", inputName, inputName, action, action.getName() From dcd81b5847b81489052bf4eb6e7accdc2d6df8ab Mon Sep 17 00:00:00 2001 From: Robert Brignull Date: Mon, 4 May 2020 15:16:23 +0100 Subject: [PATCH 4/8] Make use of getContainer --- queries/undeclared-action-input.ql | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/queries/undeclared-action-input.ql b/queries/undeclared-action-input.ql index 23627205b..a8ec7c3f4 100644 --- a/queries/undeclared-action-input.ql +++ b/queries/undeclared-action-input.ql @@ -36,7 +36,7 @@ class ActionDeclaration extends File { } Expr getAFunctionChildExpr(Function f) { - result = f.getBody().getAChildStmt*().getAChildExpr*() + result.getContainer() = f } /* @@ -44,6 +44,8 @@ Expr getAFunctionChildExpr(Function f) { */ Function calledBy(Function f) { result = getAFunctionChildExpr(f).(InvokeExpr).getResolvedCallee() + or + result.getEnclosingContainer() = f // assume outer function causes inner function to be called } class GetInputMethodCallExpr extends MethodCallExpr { From 290b34d5dfd896189952af7799c14e04bb6aed67 Mon Sep 17 00:00:00 2001 From: Ana Armas Romero <54946499+anaarmas@users.noreply.github.com> Date: Mon, 4 May 2020 19:55:51 +0200 Subject: [PATCH 5/8] Note in readme about go analysis in macos-latest --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f61fdee5c..a0fade7a0 100644 --- a/README.md +++ b/README.md @@ -137,7 +137,7 @@ env: to `github/codeql-action/analyze`. -### If you do not use a vendor directory +#### If you do not use a vendor directory Dependencies on public repositories should just work. If you have dependencies on private repositories, one option is to use `git config` and a [personal access token](https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line) to authenticate when downloading dependencies. Add a section like @@ -163,6 +163,10 @@ dotnet build /p:UseSharedCompilation=false Version 3 does not require the additional flag. +### Analysing Go together with other languages on `macos-latest` + +This is currently not possible for Java, C/C++, or C#. + ## License This project is released under the [MIT License](LICENSE). From ab918b676bd014a667688f1500e4e1b86ad8cc99 Mon Sep 17 00:00:00 2001 From: Robert Brignull Date: Tue, 5 May 2020 11:59:05 +0100 Subject: [PATCH 6/8] use tmp dir for external queries test --- lib/external-queries.js | 3 ++- lib/util.js | 10 ++++++++++ src/external-queries.test.ts | 12 ++++++++---- src/external-queries.ts | 3 ++- src/util.ts | 10 ++++++++++ 5 files changed, 32 insertions(+), 6 deletions(-) diff --git a/lib/external-queries.js b/lib/external-queries.js index 253cf2762..90e028938 100644 --- a/lib/external-queries.js +++ b/lib/external-queries.js @@ -11,8 +11,9 @@ const core = __importStar(require("@actions/core")); const exec = __importStar(require("@actions/exec")); const fs = __importStar(require("fs")); const path = __importStar(require("path")); +const util = __importStar(require("./util")); async function checkoutExternalQueries(config) { - const folder = process.env['RUNNER_WORKSPACE'] || '/tmp/codeql-action'; + const folder = util.getRequiredEnvParam('RUNNER_WORKSPACE'); for (const externalQuery of config.externalQueries) { core.info('Checking out ' + externalQuery.repository); const checkoutLocation = path.join(folder, externalQuery.repository); diff --git a/lib/util.js b/lib/util.js index d12a91044..a9d79bb41 100644 --- a/lib/util.js +++ b/lib/util.js @@ -15,6 +15,8 @@ const http = __importStar(require("@actions/http-client")); const auth = __importStar(require("@actions/http-client/auth")); const octokit = __importStar(require("@octokit/rest")); const console_log_level_1 = __importDefault(require("console-log-level")); +const fs = __importStar(require("fs")); +const os = __importStar(require("os")); const path = __importStar(require("path")); const sharedEnv = __importStar(require("./shared-environment")); /** @@ -280,3 +282,11 @@ function getToolNames(sarifContents) { return Object.keys(toolNames); } exports.getToolNames = getToolNames; +// Creates a random temporary directory, runs the given body, and then deletes the directory. +// Mostly intended for use within tests. +async function withTmpDir(body) { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'codeql-action-')); + await body(tmpDir); + fs.rmdirSync(tmpDir, { recursive: true }); +} +exports.withTmpDir = withTmpDir; diff --git a/src/external-queries.test.ts b/src/external-queries.test.ts index 088f3a3fe..a79f3f3e5 100644 --- a/src/external-queries.test.ts +++ b/src/external-queries.test.ts @@ -3,15 +3,19 @@ import * as path from "path"; import * as configUtils from "./config-utils"; import * as externalQueries from "./external-queries"; +import * as util from "./util"; test("checkoutExternalQueries", async () => { let config = new configUtils.Config(); config.externalQueries = [ new configUtils.ExternalQuery("github/codeql-go", "df4c6869212341b601005567381944ed90906b6b"), ]; - await externalQueries.checkoutExternalQueries(config); - let destination = process.env["RUNNER_WORKSPACE"] || "/tmp/codeql-action/"; - // COPYRIGHT file existed in df4c6869212341b601005567381944ed90906b6b but not in master - expect(fs.existsSync(path.join(destination, "github", "codeql-go", "COPYRIGHT"))).toBeTruthy(); + await util.withTmpDir(async tmpDir => { + process.env["RUNNER_WORKSPACE"] = tmpDir; + await externalQueries.checkoutExternalQueries(config); + + // COPYRIGHT file existed in df4c6869212341b601005567381944ed90906b6b but not in master + expect(fs.existsSync(path.join(tmpDir, "github", "codeql-go", "COPYRIGHT"))).toBeTruthy(); + }); }); diff --git a/src/external-queries.ts b/src/external-queries.ts index c9724148d..a478f538b 100644 --- a/src/external-queries.ts +++ b/src/external-queries.ts @@ -4,9 +4,10 @@ import * as fs from 'fs'; import * as path from 'path'; import * as configUtils from './config-utils'; +import * as util from './util'; export async function checkoutExternalQueries(config: configUtils.Config) { - const folder = process.env['RUNNER_WORKSPACE'] || '/tmp/codeql-action'; + const folder = util.getRequiredEnvParam('RUNNER_WORKSPACE'); for (const externalQuery of config.externalQueries) { core.info('Checking out ' + externalQuery.repository); diff --git a/src/util.ts b/src/util.ts index cfdd2419c..d17571d5d 100644 --- a/src/util.ts +++ b/src/util.ts @@ -3,6 +3,8 @@ import * as http from '@actions/http-client'; import * as auth from '@actions/http-client/auth'; import * as octokit from '@octokit/rest'; import consoleLogLevel from 'console-log-level'; +import * as fs from "fs"; +import * as os from 'os'; import * as path from 'path'; import * as sharedEnv from './shared-environment'; @@ -313,3 +315,11 @@ export function getToolNames(sarifContents: string): string[] { return Object.keys(toolNames); } + +// Creates a random temporary directory, runs the given body, and then deletes the directory. +// Mostly intended for use within tests. +export async function withTmpDir(body: (tmpDir: string) => Promise) { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'codeql-action-')); + await body(tmpDir); + fs.rmdirSync(tmpDir, { recursive: true }); +} From 4c11b3d9bf7b4658602568713bef2078aa498e66 Mon Sep 17 00:00:00 2001 From: Ana Armas Romero <54946499+anaarmas@users.noreply.github.com> Date: Fri, 8 May 2020 20:16:30 +0200 Subject: [PATCH 7/8] rephrase Go support limitations --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a0fade7a0..c94da7b1d 100644 --- a/README.md +++ b/README.md @@ -165,7 +165,7 @@ Version 3 does not require the additional flag. ### Analysing Go together with other languages on `macos-latest` -This is currently not possible for Java, C/C++, or C#. +When running on macos it is currently not possible to analyze Go in conjunction with any of Java, C/C++, or C#. Each language can still be analyzed separately. ## License From 34557369784dea1ba549a18cf36966ac706856eb Mon Sep 17 00:00:00 2001 From: Robert Brignull Date: Mon, 11 May 2020 16:24:39 +0100 Subject: [PATCH 8/8] remove URL encoding --- lib/util.js | 2 +- src/util.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/util.js b/lib/util.js index 5df9ca205..f44b90fcc 100644 --- a/lib/util.js +++ b/lib/util.js @@ -185,7 +185,7 @@ async function getAnalysisKey() { } const workflowPath = await getWorkflowPath(); const jobName = getRequiredEnvParam('GITHUB_JOB'); - analysisKey = encodeURI(workflowPath) + ':' + encodeURI(jobName); + analysisKey = workflowPath + ':' + jobName; core.exportVariable(sharedEnv.CODEQL_ACTION_ANALYSIS_KEY, analysisKey); return analysisKey; } diff --git a/src/util.ts b/src/util.ts index b8c81525c..c4b514cc4 100644 --- a/src/util.ts +++ b/src/util.ts @@ -193,7 +193,7 @@ export async function getAnalysisKey(): Promise { const workflowPath = await getWorkflowPath(); const jobName = getRequiredEnvParam('GITHUB_JOB'); - analysisKey = encodeURI(workflowPath) + ':' + encodeURI(jobName); + analysisKey = workflowPath + ':' + jobName; core.exportVariable(sharedEnv.CODEQL_ACTION_ANALYSIS_KEY, analysisKey); return analysisKey; }