Merge branch 'master' into config_utils_tests

analyze/action.yml

@@ -12,6 +12,9 @@ inputs:
     description: Upload the SARIF file
     required: false
     default: true
+  ram:
+    description: Override the amount of memory in MB to be used by CodeQL. By default, almost all the memory of the machine is used.
+    required: false
   token:
     default: ${{ github.token }}
   matrix:

src/finalize-db.ts

@@ -2,6 +2,7 @@ import * as core from '@actions/core';
 import * as exec from '@actions/exec';
 import * as io from '@actions/io';
 import * as fs from 'fs';
+import * as os from 'os';
 import * as path from 'path';
 
 import * as configUtils from './config-utils';
@@ -10,6 +11,23 @@ import * as sharedEnv from './shared-environment';
 import * as upload_lib from './upload-lib';
 import * as util from './util';
 
+function getMemoryFlag(): string {
+  let memoryToUseMegaBytes: number;
+  const memoryToUseString = core.getInput("ram");
+  if (memoryToUseString) {
+    memoryToUseMegaBytes = Number(memoryToUseString);
+    if (Number.isNaN(memoryToUseMegaBytes) || memoryToUseMegaBytes <= 0) {
+      throw new Error("Invalid RAM setting \"" + memoryToUseString + "\", specified.");
+    }
+  } else {
+    const totalMemoryBytes = os.totalmem();
+    const totalMemoryMegaBytes = totalMemoryBytes / (1024 * 1024);
+    const systemReservedMemoryMegaBytes = 256;
+    memoryToUseMegaBytes = totalMemoryMegaBytes - systemReservedMemoryMegaBytes;
+  }
+  return "--ram=" + Math.floor(memoryToUseMegaBytes);
+}
+
 async function createdDBForScannedLanguages(codeqlCmd: string, databaseFolder: string) {
   const scannedLanguages = process.env[sharedEnv.CODEQL_ACTION_SCANNED_LANGUAGES];
   if (scannedLanguages) {
@@ -113,6 +131,7 @@ async function runQueries(codeqlCmd: string, databaseFolder: string, sarifFolder
     await exec.exec(codeqlCmd, [
       'database',
       'analyze',
+      getMemoryFlag(),
       path.join(databaseFolder, database),
       '--format=sarif-latest',
       '--output=' + sarifFile,

src/upload-lib.ts

@@ -1,7 +1,6 @@
 import * as core from '@actions/core';
 import * as http from '@actions/http-client';
 import * as auth from '@actions/http-client/auth';
-import * as io from '@actions/io';
 import fileUrl from 'file-url';
 import * as fs from 'fs';
 import * as path from 'path';
@@ -11,19 +10,6 @@ import * as fingerprints from './fingerprints';
 import * as sharedEnv from './shared-environment';
 import * as util from './util';
 
-// Construct the location of the sentinel file for detecting multiple uploads.
-// The returned location should be writable.
-async function getSentinelFilePath(): Promise<string> {
-    // Use the temp dir instead of placing next to the sarif file because of
-    // issues with docker actions. The directory containing the sarif file
-    // may not be writable by us.
-    const uploadsTmpDir = path.join(process.env['RUNNER_TEMP'] || '/tmp/codeql-action', 'uploads');
-    await io.mkdirP(uploadsTmpDir);
-    // Hash the absolute path so we'll behave correctly in the unlikely
-    // scenario a file is referenced twice with different paths.
-    return path.join(uploadsTmpDir, 'codeql-action-upload-sentinel');
-}
-
 // Takes a list of paths to sarif files and combines them together,
 // returning the contents of the combined sarif file.
 export function combineSarifFiles(sarifFiles: string[]): string {
@@ -128,20 +114,27 @@ export async function upload(input: string): Promise<boolean> {
     }
 }
 
+// Counts the number of results in the given SARIF file
+export function countResultsInSarif(sarif: string): number {
+    let numResults = 0;
+    for (const run of JSON.parse(sarif).runs) {
+        numResults += run.results.length;
+    }
+    return numResults;
+}
+
 // Uploads the given set of sarif files.
 // Returns true iff the upload occurred and succeeded
 async function uploadFiles(sarifFiles: string[]): Promise<boolean> {
     core.startGroup("Uploading results");
     let succeeded = false;
     try {
-        // Check if an upload has happened before. If so then abort.
-        // This is intended to catch when the finish and upload-sarif actions
-        // are used together, and then the upload-sarif action is invoked twice.
-        const sentinelFile = await getSentinelFilePath();
-        if (fs.existsSync(sentinelFile)) {
-            core.info("Aborting as an upload has already happened from this job");
+        const sentinelEnvVar = "CODEQL_UPLOAD_SARIF";
+        if (process.env[sentinelEnvVar]) {
+            core.error("Aborting upload: only one run of the codeql/analyze or codeql/upload-sarif actions is allowed per job");
             return false;
         }
+        core.exportVariable(sentinelEnvVar, sentinelEnvVar);
 
         const commitOid = util.getRequiredEnvParam('GITHUB_SHA');
         const workflowRunIDStr = util.getRequiredEnvParam('GITHUB_RUN_ID');
@@ -184,12 +177,14 @@ async function uploadFiles(sarifFiles: string[]): Promise<boolean> {
             "tool_names": toolNames,
         });
 
+        // Log some useful debug info about the info
+        core.debug("Raw upload size: " + sarifPayload.length + " bytes");
+        core.debug("Base64 zipped upload size: " + zipped_sarif.length + " bytes");
+        core.debug("Number of results in upload: " + countResultsInSarif(sarifPayload));
+
         // Make the upload
         succeeded = await uploadPayload(payload);
 
-        // Mark that we have made an upload
-        fs.writeFileSync(sentinelFile, '');
-
     } catch (error) {
         core.setFailed(error.message);
     }