Permalink
codeql-action/lib/upload-lib.js
Newer
100644
385 lines (385 sloc)
18 KB
2
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
if (k2 === undefined) k2 = k;
4
Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
5
}) : (function(o, m, k, k2) {
6
if (k2 === undefined) k2 = k;
7
o[k2] = m[k];
8
}));
9
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
10
Object.defineProperty(o, "default", { enumerable: true, value: v });
11
}) : function(o, v) {
12
o["default"] = v;
13
});
14
var __importStar = (this && this.__importStar) || function (mod) {
15
if (mod && mod.__esModule) return mod;
16
var result = {};
17
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
18
__setModuleDefault(result, mod);
19
return result;
20
};
21
var __importDefault = (this && this.__importDefault) || function (mod) {
22
return (mod && mod.__esModule) ? mod : { "default": mod };
23
};
24
Object.defineProperty(exports, "__esModule", { value: true });
25
exports.validateUniqueCategory = exports.waitForProcessing = exports.buildPayload = exports.validateSarifFileSchema = exports.countResultsInSarif = exports.uploadFromRunner = exports.uploadFromActions = exports.findSarifFilesInDir = exports.populateRunAutomationDetails = exports.combineSarifFiles = void 0;
26
const fs = __importStar(require("fs"));
27
const path = __importStar(require("path"));
28
const zlib_1 = __importDefault(require("zlib"));
29
const core = __importStar(require("@actions/core"));
30
const file_url_1 = __importDefault(require("file-url"));
31
const jsonschema = __importStar(require("jsonschema"));
32
const semver = __importStar(require("semver"));
33
const actionsUtil = __importStar(require("./actions-util"));
34
const api = __importStar(require("./api-client"));
35
const fingerprints = __importStar(require("./fingerprints"));
36
const repository_1 = require("./repository");
37
const sharedEnv = __importStar(require("./shared-environment"));
38
const util = __importStar(require("./util"));
39
// Takes a list of paths to sarif files and combines them together,
40
// returning the contents of the combined sarif file.
41
function combineSarifFiles(sarifFiles) {
42
const combinedSarif = {
43
version: null,
46
for (const sarifFile of sarifFiles) {
47
const sarifObject = JSON.parse(fs.readFileSync(sarifFile, "utf8"));
48
// Check SARIF version
49
if (combinedSarif.version === null) {
50
combinedSarif.version = sarifObject.version;
51
}
52
else if (combinedSarif.version !== sarifObject.version) {
53
throw new Error(`Different SARIF versions encountered: ${combinedSarif.version} and ${sarifObject.version}`);
54
}
55
combinedSarif.runs.push(...sarifObject.runs);
56
}
57
return combinedSarif;
58
}
59
exports.combineSarifFiles = combineSarifFiles;
60
// Populates the run.automationDetails.id field using the analysis_key and environment
61
// and return an updated sarif file contents.
62
function populateRunAutomationDetails(sarif, category, analysis_key, environment) {
63
const automationID = getAutomationID(category, analysis_key, environment);
64
if (automationID !== undefined) {
65
for (const run of sarif.runs || []) {
66
if (run.automationDetails === undefined) {
67
run.automationDetails = {
68
id: automationID,
69
};
70
}
72
return sarif;
74
return sarif;
75
}
76
exports.populateRunAutomationDetails = populateRunAutomationDetails;
77
function getAutomationID(category, analysis_key, environment) {
78
if (category !== undefined) {
79
let automationID = category;
80
if (!automationID.endsWith("/")) {
81
automationID += "/";
82
}
83
return automationID;
84
}
85
// analysis_key is undefined for the runner.
86
if (analysis_key !== undefined) {
87
return actionsUtil.computeAutomationID(analysis_key, environment);
88
}
89
return undefined;
91
// Upload the given payload.
92
// If the request fails then this will retry a small number of times.
93
async function uploadPayload(payload, repositoryNwo, apiDetails, logger) {
94
logger.info("Uploading results");
95
// If in test mode we don't want to upload the results
96
const testMode = process.env["TEST_MODE"] === "true" || false;
98
const payloadSaveFile = path.join(actionsUtil.getTemporaryDirectory(), "payload.json");
99
logger.info(`In test mode. Results are not uploaded. Saving to ${payloadSaveFile}`);
100
logger.info(`Payload: ${JSON.stringify(payload, null, 2)}`);
101
fs.writeFileSync(payloadSaveFile, JSON.stringify(payload, null, 2));
102
return;
104
const client = api.getApiClient(apiDetails);
105
const reqURL = util.isActions()
106
? "PUT /repos/:owner/:repo/code-scanning/analysis"
107
: "POST /repos/:owner/:repo/code-scanning/sarifs";
108
const response = await client.request(reqURL, {
109
owner: repositoryNwo.owner,
110
repo: repositoryNwo.repo,
111
data: payload,
112
});
113
logger.debug(`response status: ${response.status}`);
114
logger.info("Successfully uploaded results");
115
return response.data.id;
117
// Recursively walks a directory and returns all SARIF files it finds.
118
// Does not follow symlinks.
119
function findSarifFilesInDir(sarifPath) {
120
const sarifFiles = [];
121
const walkSarifFiles = (dir) => {
122
const entries = fs.readdirSync(dir, { withFileTypes: true });
123
for (const entry of entries) {
124
if (entry.isFile() && entry.name.endsWith(".sarif")) {
125
sarifFiles.push(path.resolve(dir, entry.name));
126
}
127
else if (entry.isDirectory()) {
128
walkSarifFiles(path.resolve(dir, entry.name));
129
}
130
}
131
};
132
walkSarifFiles(sarifPath);
133
return sarifFiles;
134
}
135
exports.findSarifFilesInDir = findSarifFilesInDir;
136
// Uploads a single sarif file or a directory of sarif files
137
// depending on what the path happens to refer to.
138
// Returns true iff the upload occurred and succeeded
139
async function uploadFromActions(sarifPath, gitHubVersion, apiDetails, logger) {
140
return await uploadFiles(getSarifFilePaths(sarifPath), (0, repository_1.parseRepositoryNwo)(util.getRequiredEnvParam("GITHUB_REPOSITORY")), await actionsUtil.getCommitOid(actionsUtil.getRequiredInput("checkout_path")), await actionsUtil.getRef(), await actionsUtil.getAnalysisKey(), actionsUtil.getOptionalInput("category"), util.getRequiredEnvParam("GITHUB_WORKFLOW"), actionsUtil.getWorkflowRunID(), actionsUtil.getRequiredInput("checkout_path"), actionsUtil.getRequiredInput("matrix"), gitHubVersion, apiDetails, logger);
141
}
142
exports.uploadFromActions = uploadFromActions;
143
// Uploads a single sarif file or a directory of sarif files
144
// depending on what the path happens to refer to.
145
// Returns true iff the upload occurred and succeeded
146
async function uploadFromRunner(sarifPath, repositoryNwo, commitOid, ref, category, sourceRoot, gitHubVersion, apiDetails, logger) {
147
return await uploadFiles(getSarifFilePaths(sarifPath), repositoryNwo, commitOid, ref, undefined, category, undefined, undefined, sourceRoot, undefined, gitHubVersion, apiDetails, logger);
148
}
149
exports.uploadFromRunner = uploadFromRunner;
150
function getSarifFilePaths(sarifPath) {
151
if (!fs.existsSync(sarifPath)) {
152
throw new Error(`Path does not exist: ${sarifPath}`);
153
}
154
let sarifFiles;
155
if (fs.lstatSync(sarifPath).isDirectory()) {
156
sarifFiles = findSarifFilesInDir(sarifPath);
157
if (sarifFiles.length === 0) {
158
throw new Error(`No SARIF files found to upload in "${sarifPath}".`);
159
}
160
}
161
else {
162
sarifFiles = [sarifPath];
164
return sarifFiles;
166
// Counts the number of results in the given SARIF file
167
function countResultsInSarif(sarif) {
168
let numResults = 0;
169
let parsedSarif;
170
try {
171
parsedSarif = JSON.parse(sarif);
172
}
173
catch (e) {
174
throw new Error(`Invalid SARIF. JSON syntax error: ${e instanceof Error ? e.message : String(e)}`);
175
}
176
if (!Array.isArray(parsedSarif.runs)) {
177
throw new Error("Invalid SARIF. Missing 'runs' array.");
178
}
179
for (const run of parsedSarif.runs) {
180
if (!Array.isArray(run.results)) {
181
throw new Error("Invalid SARIF. Missing 'results' array in run.");
182
}
183
numResults += run.results.length;
184
}
185
return numResults;
186
}
187
exports.countResultsInSarif = countResultsInSarif;
188
// Validates that the given file path refers to a valid SARIF file.
189
// Throws an error if the file is invalid.
190
function validateSarifFileSchema(sarifFilePath, logger) {
191
const sarif = JSON.parse(fs.readFileSync(sarifFilePath, "utf8"));
192
const schema = require("../src/sarif_v2.1.0_schema.json");
193
const result = new jsonschema.Validator().validate(sarif, schema);
194
if (!result.valid) {
195
// Output the more verbose error messages in groups as these may be very large.
196
for (const error of result.errors) {
197
logger.startGroup(`Error details: ${error.stack}`);
198
logger.info(JSON.stringify(error, null, 2));
199
logger.endGroup();
201
// Set the main error message to the stacks of all the errors.
202
// This should be of a manageable size and may even give enough to fix the error.
203
const sarifErrors = result.errors.map((e) => `- ${e.stack}`);
204
throw new Error(`Unable to upload "${sarifFilePath}" as it is not valid SARIF:\n${sarifErrors.join("\n")}`);
205
}
206
}
207
exports.validateSarifFileSchema = validateSarifFileSchema;
208
// buildPayload constructs a map ready to be uploaded to the API from the given
209
// parameters, respecting the current mode and target GitHub instance version.
210
function buildPayload(commitOid, ref, analysisKey, analysisName, zippedSarif, workflowRunID, checkoutURI, environment, toolNames, gitHubVersion, mergeBaseCommitOid) {
211
if (util.isActions()) {
212
const payloadObj = {
213
commit_oid: commitOid,
214
ref,
215
analysis_key: analysisKey,
216
analysis_name: analysisName,
217
sarif: zippedSarif,
218
workflow_run_id: workflowRunID,
219
checkout_uri: checkoutURI,
220
environment,
221
started_at: process.env[sharedEnv.CODEQL_WORKFLOW_STARTED_AT],
222
tool_names: toolNames,
223
base_ref: undefined,
224
base_sha: undefined,
225
};
226
// This behaviour can be made the default when support for GHES 3.0 is discontinued.
227
if (gitHubVersion.type !== util.GitHubVariant.GHES ||
228
semver.satisfies(gitHubVersion.version, `>=3.1`)) {
229
if (process.env.GITHUB_EVENT_NAME === "pull_request") {
230
if (commitOid === util.getRequiredEnvParam("GITHUB_SHA") &&
231
mergeBaseCommitOid) {
232
// We're uploading results for the merge commit
233
// and were able to determine the merge base.
234
// So we use that as the most accurate base.
235
payloadObj.base_ref = `refs/heads/${util.getRequiredEnvParam("GITHUB_BASE_REF")}`;
236
payloadObj.base_sha = mergeBaseCommitOid;
237
}
238
else if (process.env.GITHUB_EVENT_PATH) {
239
// Either we're not uploading results for the merge commit
240
// or we could not determine the merge base.
241
// Using the PR base is the only option here
242
const githubEvent = JSON.parse(fs.readFileSync(process.env.GITHUB_EVENT_PATH, "utf8"));
243
payloadObj.base_ref = `refs/heads/${githubEvent.pull_request.base.ref}`;
244
payloadObj.base_sha = githubEvent.pull_request.base.sha;
245
}
246
}
247
}
248
return payloadObj;
250
else {
251
return {
252
commit_sha: commitOid,
253
ref,
254
sarif: zippedSarif,
255
checkout_uri: checkoutURI,
256
tool_name: toolNames[0],
257
};
259
}
260
exports.buildPayload = buildPayload;
261
// Uploads the given set of sarif files.
262
// Returns true iff the upload occurred and succeeded
263
async function uploadFiles(sarifFiles, repositoryNwo, commitOid, ref, analysisKey, category, analysisName, workflowRunID, sourceRoot, environment, gitHubVersion, apiDetails, logger) {
264
logger.startGroup("Uploading results");
265
logger.info(`Processing sarif files: ${JSON.stringify(sarifFiles)}`);
266
// Validate that the files we were asked to upload are all valid SARIF files
267
for (const file of sarifFiles) {
268
validateSarifFileSchema(file, logger);
270
let sarif = combineSarifFiles(sarifFiles);
271
sarif = await fingerprints.addFingerprints(sarif, sourceRoot, logger);
272
sarif = populateRunAutomationDetails(sarif, category, analysisKey, environment);
273
const toolNames = util.getToolNames(sarif);
274
validateUniqueCategory(sarif);
275
const sarifPayload = JSON.stringify(sarif);
276
const zippedSarif = zlib_1.default.gzipSync(sarifPayload).toString("base64");
277
const checkoutURI = (0, file_url_1.default)(sourceRoot);
278
const payload = buildPayload(commitOid, ref, analysisKey, analysisName, zippedSarif, workflowRunID, checkoutURI, environment, toolNames, gitHubVersion, await actionsUtil.determineMergeBaseCommitOid());
279
// Log some useful debug info about the info
280
const rawUploadSizeBytes = sarifPayload.length;
281
logger.debug(`Raw upload size: ${rawUploadSizeBytes} bytes`);
282
const zippedUploadSizeBytes = zippedSarif.length;
283
logger.debug(`Base64 zipped upload size: ${zippedUploadSizeBytes} bytes`);
284
const numResultInSarif = countResultsInSarif(sarifPayload);
285
logger.debug(`Number of results in upload: ${numResultInSarif}`);
286
// Make the upload
287
const sarifID = await uploadPayload(payload, repositoryNwo, apiDetails, logger);
288
logger.endGroup();
289
return {
290
statusReport: {
291
raw_upload_size_bytes: rawUploadSizeBytes,
292
zipped_upload_size_bytes: zippedUploadSizeBytes,
293
num_results_in_sarif: numResultInSarif,
294
},
295
sarifID,
296
};
297
}
298
const STATUS_CHECK_FREQUENCY_MILLISECONDS = 5 * 1000;
299
const STATUS_CHECK_TIMEOUT_MILLISECONDS = 2 * 60 * 1000;
300
// Waits until either the analysis is successfully processed, a processing error is reported, or STATUS_CHECK_TIMEOUT_MILLISECONDS elapses.
301
async function waitForProcessing(repositoryNwo, sarifID, apiDetails, logger) {
302
logger.startGroup("Waiting for processing to finish");
303
const client = api.getApiClient(apiDetails);
304
const statusCheckingStarted = Date.now();
305
// eslint-disable-next-line no-constant-condition
306
while (true) {
307
if (Date.now() >
308
statusCheckingStarted + STATUS_CHECK_TIMEOUT_MILLISECONDS) {
309
// If the analysis hasn't finished processing in the allotted time, we continue anyway rather than failing.
310
// It's possible the analysis will eventually finish processing, but it's not worth spending more Actions time waiting.
311
logger.warning("Timed out waiting for analysis to finish processing. Continuing.");
312
break;
313
}
314
try {
315
const response = await client.request("GET /repos/:owner/:repo/code-scanning/sarifs/:sarif_id", {
316
owner: repositoryNwo.owner,
317
repo: repositoryNwo.repo,
318
sarif_id: sarifID,
319
});
320
const status = response.data.processing_status;
321
logger.info(`Analysis upload status is ${status}.`);
322
if (status === "complete") {
323
break;
324
}
325
else if (status === "failed") {
326
throw new Error(`Code Scanning could not process the submitted SARIF file:\n${response.data.errors}`);
327
}
328
}
329
catch (e) {
330
if (util.isHTTPError(e)) {
331
switch (e.status) {
332
case 404:
333
logger.debug("Analysis is not found yet...");
334
break; // Note this breaks from the case statement, not the outer loop.
335
default:
336
throw e;
337
}
338
}
339
else {
340
throw e;
341
}
342
}
343
await util.delay(STATUS_CHECK_FREQUENCY_MILLISECONDS);
344
}
345
logger.endGroup();
347
exports.waitForProcessing = waitForProcessing;
348
function validateUniqueCategory(sarif) {
349
var _a, _b, _c;
350
// This check only works on actions as env vars don't persist between calls to the runner
351
if (util.isActions()) {
352
// duplicate categories are allowed in the same sarif file
353
// but not across multiple sarif files
354
const categories = {};
355
for (const run of sarif.runs) {
356
const id = (_a = run === null || run === void 0 ? void 0 : run.automationDetails) === null || _a === void 0 ? void 0 : _a.id;
357
const tool = (_c = (_b = run.tool) === null || _b === void 0 ? void 0 : _b.driver) === null || _c === void 0 ? void 0 : _c.name;
358
const category = `${sanitize(id)}_${sanitize(tool)}`;
359
categories[category] = { id, tool };
360
}
361
for (const [category, { id, tool }] of Object.entries(categories)) {
362
const sentinelEnvVar = `CODEQL_UPLOAD_SARIF_${category}`;
363
if (process.env[sentinelEnvVar]) {
364
throw new Error("Aborting upload: only one run of the codeql/analyze or codeql/upload-sarif actions is allowed per job per tool/category. " +
365
"The easiest fix is to specify a unique value for the `category` input. " +
366
`Category: (${id ? id : "none"}) Tool: (${tool ? tool : "none"})`);
367
}
368
core.exportVariable(sentinelEnvVar, sentinelEnvVar);
369
}
370
}
371
}
372
exports.validateUniqueCategory = validateUniqueCategory;
373
/**
374
* Santizes a string to be used as an environment variable name.
375
* This will replace all non-alphanumeric characters with underscores.
376
* There could still be some false category clashes if two uploads
377
* occur that differ only in their non-alphanumeric characters. This is
378
* unlikely.
379
*
380
* @param str the initial value to sanitize
381
*/
382
function sanitize(str) {
383
return (str !== null && str !== void 0 ? str : "_").replace(/[^a-zA-Z0-9_]/g, "_").toLocaleUpperCase();
385
//# sourceMappingURL=upload-lib.js.map