From 2b7af49a3d3fb95d8bb2f2fa743988441baa6382 Mon Sep 17 00:00:00 2001 From: "Michael B. Gale" Date: Mon, 22 Jul 2024 15:29:56 +0100 Subject: [PATCH] Add bare-bones dependency caching functions --- lib/dependency-caching.js | 125 +++++++++++++++++++++++++++++ lib/dependency-caching.js.map | 1 + src/dependency-caching.ts | 145 ++++++++++++++++++++++++++++++++++ 3 files changed, 271 insertions(+) create mode 100644 lib/dependency-caching.js create mode 100644 lib/dependency-caching.js.map create mode 100644 src/dependency-caching.ts diff --git a/lib/dependency-caching.js b/lib/dependency-caching.js new file mode 100644 index 000000000..4b0feaa5a --- /dev/null +++ b/lib/dependency-caching.js @@ -0,0 +1,125 @@ +"use strict"; +var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + var desc = Object.getOwnPropertyDescriptor(m, k); + if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) { + desc = { enumerable: true, get: function() { return m[k]; } }; + } + Object.defineProperty(o, k2, desc); +}) : (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + o[k2] = m[k]; +})); +var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) { + Object.defineProperty(o, "default", { enumerable: true, value: v }); +}) : function(o, v) { + o["default"] = v; +}); +var __importStar = (this && this.__importStar) || function (mod) { + if (mod && mod.__esModule) return mod; + var result = {}; + if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k); + __setModuleDefault(result, mod); + return result; +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.downloadDependencyCaches = downloadDependencyCaches; +exports.uploadDependencyCaches = uploadDependencyCaches; +const os = __importStar(require("os")); +const path_1 = require("path"); +const actionsCache = __importStar(require("@actions/cache")); +const glob = __importStar(require("@actions/glob")); +const caching_utils_1 = require("./caching-utils"); +const util_1 = require("./util"); +const CODEQL_DEPENDENCY_CACHE_PREFIX = "codeql-dependencies"; +const CODEQL_DEPENDENCY_CACHE_VERSION = 1; +const CODEQL_DEFAULT_CACHE_CONFIG = { + java: { + paths: [ + // Maven + (0, path_1.join)(os.homedir(), ".m2", "repository"), + // Gradle + (0, path_1.join)(os.homedir(), ".gradle", "caches"), + ], + hash: [ + // Maven + "**/pom.xml", + // Gradle + "**/*.gradle*", + "**/gradle-wrapper.properties", + "buildSrc/**/Versions.kt", + "buildSrc/**/Dependencies.kt", + "gradle/*.versions.toml", + "**/versions.properties", + ], + }, +}; +/** + * Attempts to restore dependency caches for the languages being analyzed. + * + * @param languages The languages being analyzed. + * @param logger A logger to record some informational messages to. + * @returns A list of languages for which dependency caches were restored. + */ +async function downloadDependencyCaches(languages, logger) { + const restoredCaches = []; + for (const language of languages) { + const cacheConfig = CODEQL_DEFAULT_CACHE_CONFIG[language]; + if (cacheConfig === undefined) { + logger.info(`Skipping download of dependency cache for ${language} as we have no caching configuration for it.`); + continue; + } + const primaryKey = await cacheKey(language, cacheConfig); + const restoreKeys = [await cachePrefix(language)]; + logger.info(`Downloading cache for ${language} with key ${primaryKey} and restore keys ${restoreKeys.join(", ")}`); + const hitKey = await actionsCache.restoreCache(cacheConfig.paths, primaryKey, restoreKeys); + if (hitKey !== undefined) { + logger.info(`Cache hit on key ${hitKey} for ${language}.`); + restoredCaches.push(language); + } + } + return restoredCaches; +} +/** + * Attempts to store caches for the languages that were analyzed. + * + * @param config The configuration for this workflow. + * @param logger A logger to record some informational messages to. + */ +async function uploadDependencyCaches(config, logger) { + for (const language of config.languages) { + const cacheConfig = CODEQL_DEFAULT_CACHE_CONFIG[language]; + if (cacheConfig === undefined) { + logger.info(`Skipping upload of dependency cache for ${language} as we have no caching configuration for it.`); + continue; + } + const globber = await glob.create(cacheConfig.hash.join("\n")); + const size = await (0, caching_utils_1.getTotalCacheSize)(await globber.glob(), logger); + const key = await cacheKey(language, cacheConfig); + logger.info(`Uploading cache of size ${size} for ${language} with key ${key}`); + await actionsCache.saveCache(cacheConfig.paths, key); + } +} +/** + * Computes a cache key for the specified language. + * + * @param language The language being analyzed. + * @param cacheConfig The cache configuration for the language. + * @returns A cache key capturing information about the project(s) being analyzed in the specified language. + */ +async function cacheKey(language, cacheConfig) { + const hash = await glob.hashFiles(cacheConfig.hash.join("\n")); + return `${await cachePrefix(language)}${hash}`; +} +/** + * Constructs a prefix for the cache key, comprised of a CodeQL-specific prefix, a version number that + * can be changed to invalidate old caches, the runner's operating system, and the specified language name. + * + * @param language The language being analyzed. + * @returns The prefix that identifies what a cache is for. + */ +async function cachePrefix(language) { + const runnerOs = (0, util_1.getRequiredEnvParam)("RUNNER_OS"); + return `${CODEQL_DEPENDENCY_CACHE_PREFIX}-${CODEQL_DEPENDENCY_CACHE_VERSION}-${runnerOs}-${language}-`; +} +//# sourceMappingURL=dependency-caching.js.map \ No newline at end of file diff --git a/lib/dependency-caching.js.map b/lib/dependency-caching.js.map new file mode 100644 index 000000000..b6228351b --- /dev/null +++ b/lib/dependency-caching.js.map @@ -0,0 +1 @@ +{"version":3,"file":"dependency-caching.js","sourceRoot":"","sources":["../src/dependency-caching.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AAiDA,4DAsCC;AAQD,wDAsBC;AArHD,uCAAyB;AACzB,+BAA4B;AAE5B,6DAA+C;AAC/C,oDAAsC;AAEtC,mDAAoD;AAIpD,iCAA6C;AAO7C,MAAM,8BAA8B,GAAG,qBAAqB,CAAC;AAC7D,MAAM,+BAA+B,GAAG,CAAC,CAAC;AAE1C,MAAM,2BAA2B,GAAwC;IACvE,IAAI,EAAE;QACJ,KAAK,EAAE;YACL,QAAQ;YACR,IAAA,WAAI,EAAC,EAAE,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,YAAY,CAAC;YACvC,SAAS;YACT,IAAA,WAAI,EAAC,EAAE,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,QAAQ,CAAC;SACxC;QACD,IAAI,EAAE;YACJ,QAAQ;YACR,YAAY;YACZ,SAAS;YACT,cAAc;YACd,8BAA8B;YAC9B,yBAAyB;YACzB,6BAA6B;YAC7B,wBAAwB;YACxB,wBAAwB;SACzB;KACF;CACF,CAAC;AAEF;;;;;;GAMG;AACI,KAAK,UAAU,wBAAwB,CAC5C,SAAqB,EACrB,MAAc;IAEd,MAAM,cAAc,GAAe,EAAE,CAAC;IAEtC,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,MAAM,WAAW,GAAG,2BAA2B,CAAC,QAAQ,CAAC,CAAC;QAE1D,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;YAC9B,MAAM,CAAC,IAAI,CACT,6CAA6C,QAAQ,8CAA8C,CACpG,CAAC;YACF,SAAS;QACX,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;QACzD,MAAM,WAAW,GAAa,CAAC,MAAM,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC;QAE5D,MAAM,CAAC,IAAI,CACT,yBAAyB,QAAQ,aAAa,UAAU,qBAAqB,WAAW,CAAC,IAAI,CAC3F,IAAI,CACL,EAAE,CACJ,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,YAAY,CAC5C,WAAW,CAAC,KAAK,EACjB,UAAU,EACV,WAAW,CACZ,CAAC;QAEF,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,MAAM,CAAC,IAAI,CAAC,oBAAoB,MAAM,QAAQ,QAAQ,GAAG,CAAC,CAAC;YAC3D,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;IAED,OAAO,cAAc,CAAC;AACxB,CAAC;AAED;;;;;GAKG;AACI,KAAK,UAAU,sBAAsB,CAAC,MAAc,EAAE,MAAc;IACzE,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;QACxC,MAAM,WAAW,GAAG,2BAA2B,CAAC,QAAQ,CAAC,CAAC;QAE1D,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;YAC9B,MAAM,CAAC,IAAI,CACT,2CAA2C,QAAQ,8CAA8C,CAClG,CAAC;YACF,SAAS;QACX,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QAC/D,MAAM,IAAI,GAAG,MAAM,IAAA,iCAAiB,EAAC,MAAM,OAAO,CAAC,IAAI,EAAE,EAAE,MAAM,CAAC,CAAC;QAEnE,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;QAElD,MAAM,CAAC,IAAI,CACT,2BAA2B,IAAI,QAAQ,QAAQ,aAAa,GAAG,EAAE,CAClE,CAAC;QAEF,MAAM,YAAY,CAAC,SAAS,CAAC,WAAW,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IACvD,CAAC;AACH,CAAC;AAED;;;;;;GAMG;AACH,KAAK,UAAU,QAAQ,CACrB,QAAkB,EAClB,WAAwB;IAExB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IAC/D,OAAO,GAAG,MAAM,WAAW,CAAC,QAAQ,CAAC,GAAG,IAAI,EAAE,CAAC;AACjD,CAAC;AAED;;;;;;GAMG;AACH,KAAK,UAAU,WAAW,CAAC,QAAkB;IAC3C,MAAM,QAAQ,GAAG,IAAA,0BAAmB,EAAC,WAAW,CAAC,CAAC;IAClD,OAAO,GAAG,8BAA8B,IAAI,+BAA+B,IAAI,QAAQ,IAAI,QAAQ,GAAG,CAAC;AACzG,CAAC"} \ No newline at end of file diff --git a/src/dependency-caching.ts b/src/dependency-caching.ts new file mode 100644 index 000000000..35dcd42a9 --- /dev/null +++ b/src/dependency-caching.ts @@ -0,0 +1,145 @@ +import * as os from "os"; +import { join } from "path"; + +import * as actionsCache from "@actions/cache"; +import * as glob from "@actions/glob"; + +import { getTotalCacheSize } from "./caching-utils"; +import { Config } from "./config-utils"; +import { Language } from "./languages"; +import { Logger } from "./logging"; +import { getRequiredEnvParam } from "./util"; + +interface CacheConfig { + paths: string[]; + hash: string[]; +} + +const CODEQL_DEPENDENCY_CACHE_PREFIX = "codeql-dependencies"; +const CODEQL_DEPENDENCY_CACHE_VERSION = 1; + +const CODEQL_DEFAULT_CACHE_CONFIG: { [language: string]: CacheConfig } = { + java: { + paths: [ + // Maven + join(os.homedir(), ".m2", "repository"), + // Gradle + join(os.homedir(), ".gradle", "caches"), + ], + hash: [ + // Maven + "**/pom.xml", + // Gradle + "**/*.gradle*", + "**/gradle-wrapper.properties", + "buildSrc/**/Versions.kt", + "buildSrc/**/Dependencies.kt", + "gradle/*.versions.toml", + "**/versions.properties", + ], + }, +}; + +/** + * Attempts to restore dependency caches for the languages being analyzed. + * + * @param languages The languages being analyzed. + * @param logger A logger to record some informational messages to. + * @returns A list of languages for which dependency caches were restored. + */ +export async function downloadDependencyCaches( + languages: Language[], + logger: Logger, +): Promise { + const restoredCaches: Language[] = []; + + for (const language of languages) { + const cacheConfig = CODEQL_DEFAULT_CACHE_CONFIG[language]; + + if (cacheConfig === undefined) { + logger.info( + `Skipping download of dependency cache for ${language} as we have no caching configuration for it.`, + ); + continue; + } + + const primaryKey = await cacheKey(language, cacheConfig); + const restoreKeys: string[] = [await cachePrefix(language)]; + + logger.info( + `Downloading cache for ${language} with key ${primaryKey} and restore keys ${restoreKeys.join( + ", ", + )}`, + ); + + const hitKey = await actionsCache.restoreCache( + cacheConfig.paths, + primaryKey, + restoreKeys, + ); + + if (hitKey !== undefined) { + logger.info(`Cache hit on key ${hitKey} for ${language}.`); + restoredCaches.push(language); + } + } + + return restoredCaches; +} + +/** + * Attempts to store caches for the languages that were analyzed. + * + * @param config The configuration for this workflow. + * @param logger A logger to record some informational messages to. + */ +export async function uploadDependencyCaches(config: Config, logger: Logger) { + for (const language of config.languages) { + const cacheConfig = CODEQL_DEFAULT_CACHE_CONFIG[language]; + + if (cacheConfig === undefined) { + logger.info( + `Skipping upload of dependency cache for ${language} as we have no caching configuration for it.`, + ); + continue; + } + + const globber = await glob.create(cacheConfig.hash.join("\n")); + const size = await getTotalCacheSize(await globber.glob(), logger); + + const key = await cacheKey(language, cacheConfig); + + logger.info( + `Uploading cache of size ${size} for ${language} with key ${key}`, + ); + + await actionsCache.saveCache(cacheConfig.paths, key); + } +} + +/** + * Computes a cache key for the specified language. + * + * @param language The language being analyzed. + * @param cacheConfig The cache configuration for the language. + * @returns A cache key capturing information about the project(s) being analyzed in the specified language. + */ +async function cacheKey( + language: Language, + cacheConfig: CacheConfig, +): Promise { + const hash = await glob.hashFiles(cacheConfig.hash.join("\n")); + return `${await cachePrefix(language)}${hash}`; +} + +/** + * Constructs a prefix for the cache key, comprised of a CodeQL-specific prefix, a version number that + * can be changed to invalidate old caches, the runner's operating system, and the specified language name. + * + * @param language The language being analyzed. + * @returns The prefix that identifies what a cache is for. + */ +async function cachePrefix(language: Language): Promise { + const runnerOs = getRequiredEnvParam("RUNNER_OS"); + return `${CODEQL_DEPENDENCY_CACHE_PREFIX}-${CODEQL_DEPENDENCY_CACHE_VERSION}-${runnerOs}-${language}-`; +}