Merge branch 'main' into dbartol/bundle-20230105

src/codeql.ts

@@ -23,7 +23,7 @@ import {
   getTrapCachingExtractorConfigArgsForLang,
 } from "./trap-caching";
 import * as util from "./util";
-import { isGoodVersion } from "./util";
+import { assertNever, isGoodVersion } from "./util";
 
 type Options = Array<string | number | boolean>;
 
@@ -403,21 +403,161 @@ async function getCodeQLBundleDownloadURL(
   return `https://github.com/${CODEQL_DEFAULT_ACTION_REPOSITORY}/releases/download/${CODEQL_BUNDLE_VERSION}/${codeQLBundleName}`;
 }
 
+type CodeQLToolsSource =
+  | { codeqlTarPath: string; sourceType: "local"; toolsVersion: "local" }
+  | {
+      codeqlFolder: string;
+      sourceType: "toolcache";
+      toolsVersion: string;
+    }
+  | {
+      codeqlURL: string;
+      semanticVersion: string;
+      sourceType: "download";
+      toolsVersion: string;
+    };
+
+async function getCodeQLSource(
+  toolsInput: string | undefined,
+  bypassToolcache: boolean,
+  apiDetails: api.GitHubApiDetails,
+  variant: util.GitHubVariant,
+  logger: Logger
+): Promise<CodeQLToolsSource> {
+  if (toolsInput && toolsInput !== "latest" && !toolsInput.startsWith("http")) {
+    return {
+      codeqlTarPath: toolsInput,
+      sourceType: "local",
+      toolsVersion: "local",
+    };
+  }
+
+  const forceLatestReason =
+    // We use the special value of 'latest' to prioritize the version in the
+    // defaults over any pinned cached version.
+    toolsInput === "latest"
+      ? '"tools: latest" was requested'
+      : // If the user hasn't requested a particular CodeQL version, then bypass
+      // the toolcache when the appropriate feature is enabled. This
+      // allows us to quickly rollback a broken bundle that has made its way
+      // into the toolcache.
+      toolsInput === undefined && bypassToolcache
+      ? "a specific version of CodeQL was not requested and the bypass toolcache feature is enabled"
+      : undefined;
+  const forceLatest = forceLatestReason !== undefined;
+  if (forceLatest) {
+    logger.debug(
+      `Forcing the latest version of the CodeQL tools since ${forceLatestReason}.`
+    );
+  }
+
+  const codeqlURL = forceLatest ? undefined : toolsInput;
+  const requestedSemVer = convertToSemVer(
+    getCodeQLURLVersion(codeqlURL || `/${CODEQL_BUNDLE_VERSION}/`),
+    logger
+  );
+
+  // If we find the specified version, we always use that.
+  const codeqlFolder = toolcache.find("CodeQL", requestedSemVer);
+  if (codeqlFolder) {
+    return {
+      codeqlFolder,
+      sourceType: "toolcache",
+      toolsVersion: requestedSemVer,
+    };
+  }
+
+  // If we don't find the requested version, in some cases we may allow a
+  // different version to save download time if the version hasn't been
+  // specified explicitly (in which case we always honor it).
+  if (!codeqlURL && !forceLatest) {
+    const codeqlVersions = toolcache.findAllVersions("CodeQL");
+    if (codeqlVersions.length === 1 && isGoodVersion(codeqlVersions[0])) {
+      const tmpCodeqlFolder = toolcache.find("CodeQL", codeqlVersions[0]);
+      if (fs.existsSync(path.join(tmpCodeqlFolder, "pinned-version"))) {
+        logger.debug(
+          `CodeQL in cache overriding the default ${CODEQL_BUNDLE_VERSION}`
+        );
+        return {
+          codeqlFolder: tmpCodeqlFolder,
+          sourceType: "toolcache",
+          toolsVersion: codeqlVersions[0],
+        };
+      }
+    }
+  }
+
+  return {
+    codeqlURL:
+      codeqlURL ||
+      (await getCodeQLBundleDownloadURL(apiDetails, variant, logger)),
+    semanticVersion: requestedSemVer,
+    sourceType: "download",
+    toolsVersion:
+      semver.prerelease(requestedSemVer)?.join(".") || requestedSemVer,
+  };
+}
+
+async function downloadCodeQL(
+  codeqlURL: string,
+  semanticVersion: string,
+  apiDetails: api.GitHubApiDetails,
+  tempDir: string,
+  logger: Logger
+): Promise<string> {
+  const parsedCodeQLURL = new URL(codeqlURL);
+  const searchParams = new URLSearchParams(parsedCodeQLURL.search);
+  const headers: OutgoingHttpHeaders = {
+    accept: "application/octet-stream",
+  };
+  // We only want to provide an authorization header if we are downloading
+  // from the same GitHub instance the Action is running on.
+  // This avoids leaking Enterprise tokens to dotcom.
+  // We also don't want to send an authorization header if there's already a token provided in the URL.
+  if (searchParams.has("token")) {
+    logger.debug("CodeQL tools URL contains an authorization token.");
+  } else if (codeqlURL.startsWith(`${apiDetails.url}/`)) {
+    logger.debug("Providing an authorization token to download CodeQL tools.");
+    headers.authorization = `token ${apiDetails.auth}`;
+  } else {
+    logger.debug("Downloading CodeQL tools without an authorization token.");
+  }
+  logger.info(
+    `Downloading CodeQL tools from ${codeqlURL}. This may take a while.`
+  );
+
+  const dest = path.join(tempDir, uuidV4());
+  const finalHeaders = Object.assign(
+    { "User-Agent": "CodeQL Action" },
+    headers
+  );
+  const codeqlPath = await toolcache.downloadTool(
+    codeqlURL,
+    dest,
+    undefined,
+    finalHeaders
+  );
+  logger.debug(`CodeQL bundle download to ${codeqlPath} complete.`);
+
+  const codeqlExtracted = await toolcache.extractTar(codeqlPath);
+  return await toolcache.cacheDir(codeqlExtracted, "CodeQL", semanticVersion);
+}
+
 /**
  * Set up CodeQL CLI access.
  *
- * @param codeqlURL
+ * @param toolsInput
  * @param apiDetails
  * @param tempDir
  * @param variant
- * @param features
+ * @param bypassToolcache
  * @param logger
  * @param checkVersion Whether to check that CodeQL CLI meets the minimum
  *        version requirement. Must be set to true outside tests.
  * @returns a { CodeQL, toolsVersion } object.
  */
 export async function setupCodeQL(
-  codeqlURL: string | undefined,
+  toolsInput: string | undefined,
   apiDetails: api.GitHubApiDetails,
   tempDir: string,
   variant: util.GitHubVariant,
@@ -426,110 +566,37 @@ export async function setupCodeQL(
   checkVersion: boolean
 ): Promise<{ codeql: CodeQL; toolsVersion: string }> {
   try {
-    const forceLatestReason =
-      // We use the special value of 'latest' to prioritize the version in the
-      // defaults over any pinned cached version.
-      codeqlURL === "latest"
-        ? '"tools: latest" was requested'
-        : // If the user hasn't requested a particular CodeQL version, then bypass
-        // the toolcache when the appropriate feature is enabled. This
-        // allows us to quickly rollback a broken bundle that has made its way
-        // into the toolcache.
-        codeqlURL === undefined && bypassToolcache
-        ? "a specific version of CodeQL was not requested and the bypass toolcache feature is enabled"
-        : undefined;
-    const forceLatest = forceLatestReason !== undefined;
-    if (forceLatest) {
-      logger.debug(
-        `Forcing the latest version of the CodeQL tools since ${forceLatestReason}.`
-      );
-      codeqlURL = undefined;
-    }
+    const source = await getCodeQLSource(
+      toolsInput,
+      bypassToolcache,
+      apiDetails,
+      variant,
+      logger
+    );
+
     let codeqlFolder: string;
-    let codeqlURLVersion: string;
-    if (codeqlURL && !codeqlURL.startsWith("http")) {
-      codeqlFolder = await toolcache.extractTar(codeqlURL);
-      codeqlURLVersion = "local";
-    } else {
-      codeqlURLVersion = getCodeQLURLVersion(
-        codeqlURL || `/${CODEQL_BUNDLE_VERSION}/`
-      );
-      const codeqlURLSemVer = convertToSemVer(codeqlURLVersion, logger);
-
-      // If we find the specified version, we always use that.
-      codeqlFolder = toolcache.find("CodeQL", codeqlURLSemVer);
-
-      // If we don't find the requested version, in some cases we may allow a
-      // different version to save download time if the version hasn't been
-      // specified explicitly (in which case we always honor it).
-      if (!codeqlFolder && !codeqlURL && !forceLatest) {
-        const codeqlVersions = toolcache.findAllVersions("CodeQL");
-        if (codeqlVersions.length === 1 && isGoodVersion(codeqlVersions[0])) {
-          const tmpCodeqlFolder = toolcache.find("CodeQL", codeqlVersions[0]);
-          if (fs.existsSync(path.join(tmpCodeqlFolder, "pinned-version"))) {
-            logger.debug(
-              `CodeQL in cache overriding the default ${CODEQL_BUNDLE_VERSION}`
-            );
-            codeqlFolder = tmpCodeqlFolder;
-            codeqlURLVersion = codeqlVersions[0];
-          }
-        }
-      }
 
-      if (codeqlFolder) {
+    switch (source.sourceType) {
+      case "local":
+        codeqlFolder = await toolcache.extractTar(source.codeqlTarPath);
+        break;
+      case "toolcache":
+        codeqlFolder = source.codeqlFolder;
         logger.debug(`CodeQL found in cache ${codeqlFolder}`);
-      } else {
-        if (!codeqlURL) {
-          codeqlURL = await getCodeQLBundleDownloadURL(
-            apiDetails,
-            variant,
-            logger
-          );
-        }
-
-        const parsedCodeQLURL = new URL(codeqlURL);
-        const searchParams = new URLSearchParams(parsedCodeQLURL.search);
-        const headers: OutgoingHttpHeaders = {
-          accept: "application/octet-stream",
-        };
-        // We only want to provide an authorization header if we are downloading
-        // from the same GitHub instance the Action is running on.
-        // This avoids leaking Enterprise tokens to dotcom.
-        // We also don't want to send an authorization header if there's already a token provided in the URL.
-        if (
-          codeqlURL.startsWith(`${apiDetails.url}/`) &&
-          !searchParams.has("token")
-        ) {
-          logger.debug("Downloading CodeQL bundle with token.");
-          headers.authorization = `token ${apiDetails.auth}`;
-        } else {
-          logger.debug("Downloading CodeQL bundle without token.");
-        }
-        logger.info(
-          `Downloading CodeQL tools from ${codeqlURL}. This may take a while.`
-        );
-
-        const dest = path.join(tempDir, uuidV4());
-        const finalHeaders = Object.assign(
-          { "User-Agent": "CodeQL Action" },
-          headers
-        );
-        const codeqlPath = await toolcache.downloadTool(
-          codeqlURL,
-          dest,
-          undefined,
-          finalHeaders
+        break;
+      case "download":
+        codeqlFolder = await downloadCodeQL(
+          source.codeqlURL,
+          source.semanticVersion,
+          apiDetails,
+          tempDir,
+          logger
         );
-        logger.debug(`CodeQL bundle download to ${codeqlPath} complete.`);
-
-        const codeqlExtracted = await toolcache.extractTar(codeqlPath);
-        codeqlFolder = await toolcache.cacheDir(
-          codeqlExtracted,
-          "CodeQL",
-          codeqlURLSemVer
-        );
-      }
+        break;
+      default:
+        assertNever(source);
     }
+
     let codeqlCmd = path.join(codeqlFolder, "codeql", "codeql");
     if (process.platform === "win32") {
       codeqlCmd += ".exe";
@@ -538,7 +605,7 @@ export async function setupCodeQL(
     }
 
     cachedCodeQL = await getCodeQLForCmd(codeqlCmd, checkVersion);
-    return { codeql: cachedCodeQL, toolsVersion: codeqlURLVersion };
+    return { codeql: cachedCodeQL, toolsVersion: source.toolsVersion };
   } catch (e) {
     logger.error(e instanceof Error ? e : new Error(String(e)));
     throw new Error("Unable to download and extract CodeQL CLI");

src/database-upload.ts

@@ -36,29 +36,34 @@ export async function uploadDatabases(
   const codeql = await getCodeQL(config.codeQLCmd);
 
   for (const language of config.languages) {
-    // Upload the database bundle.
-    // Although we are uploading arbitrary file contents to the API, it's worth
-    // noting that it's the API's job to validate that the contents is acceptable.
-    // This API method is available to anyone with write access to the repo.
-    const payload = fs.readFileSync(
-      await bundleDb(config, language, codeql, language)
-    );
     try {
-      await client.request(
-        `POST https://uploads.github.com/repos/:owner/:repo/code-scanning/codeql/databases/:language?name=:name`,
-        {
-          owner: repositoryNwo.owner,
-          repo: repositoryNwo.repo,
-          language,
-          name: `${language}-database`,
-          data: payload,
-          headers: {
-            authorization: `token ${apiDetails.auth}`,
-            "Content-Type": "application/zip",
-          },
-        }
-      );
-      logger.debug(`Successfully uploaded database for ${language}`);
+      // Upload the database bundle.
+      // Although we are uploading arbitrary file contents to the API, it's worth
+      // noting that it's the API's job to validate that the contents is acceptable.
+      // This API method is available to anyone with write access to the repo.
+      const bundledDb = await bundleDb(config, language, codeql, language);
+      const bundledDbSize = fs.statSync(bundledDb).size;
+      const bundledDbReadStream = fs.createReadStream(bundledDb);
+      try {
+        await client.request(
+          `POST https://uploads.github.com/repos/:owner/:repo/code-scanning/codeql/databases/:language?name=:name`,
+          {
+            owner: repositoryNwo.owner,
+            repo: repositoryNwo.repo,
+            language,
+            name: `${language}-database`,
+            data: bundledDbReadStream,
+            headers: {
+              authorization: `token ${apiDetails.auth}`,
+              "Content-Type": "application/zip",
+              "Content-Length": bundledDbSize,
+            },
+          }
+        );
+        logger.debug(`Successfully uploaded database for ${language}`);
+      } finally {
+        bundledDbReadStream.close();
+      }
     } catch (e) {
       console.log(e);
       // Log a warning but don't fail the workflow

src/init.ts

@@ -16,7 +16,7 @@ import * as util from "./util";
 import { codeQlVersionAbove } from "./util";
 
 export async function initCodeQL(
-  codeqlURL: string | undefined,
+  toolsInput: string | undefined,
   apiDetails: GitHubApiDetails,
   tempDir: string,
   variant: util.GitHubVariant,
@@ -25,7 +25,7 @@ export async function initCodeQL(
 ): Promise<{ codeql: CodeQL; toolsVersion: string }> {
   logger.startGroup("Setup CodeQL tools");
   const { codeql, toolsVersion } = await setupCodeQL(
-    codeqlURL,
+    toolsInput,
     apiDetails,
     tempDir,
     variant,