Skip to content

Commit

Permalink
Cache multiple paths and add glob pattern support (#212)
Browse files Browse the repository at this point in the history 
* Allow for multiple line-delimited paths to cache

* Add initial minimatch support

* Use @actions/glob for pattern matching

* Cache multiple entries using --files-from tar input

remove known failing test

Quote tar paths

Add salt to test cache

Try reading input files from manifest

bump salt

Run test on macos

more testing

Run caching tests on 3 platforms

Run tests on self-hosted

Apparently cant reference hosted runners by name

Bump salt

wait for some time after save

more timing out

smarter waiting

Cache in tmp dir that won't be deleted

Use child_process instead of actions/exec

Revert tempDir hack

bump salt

more logging

More console logging

Use filepath to with cacheHttpClient

Test cache restoration

Revert temp dir hack

debug logging

clean up cache.yml testing

Bump salt

change debug output

build actions

* unit test coverage for caching multiple dirs

* Ensure there's a locateable test folder at homedir

* Clean up code

* Version cache with all inputs

* Unit test getCacheVersion

* Include keys in getCacheEntry request

* Clean import orders

* Use fs promises in actionUtils tests

* Update import order for to fix linter errors

* Fix remaining linter error

* Remove platform-specific test code

* Add lerna example for caching multiple dirs

* Lerna example updated to v2

Co-Authored-By: Josh Gross <joshmgross@github.com>

Co-authored-by: Josh Gross <joshmgross@github.com>
  • Loading branch information
2 people authored and GitHub committed Mar 20, 2020
1 parent 22d71e3 commit eb78578
Show file tree
Hide file tree
Showing 16 changed files with 4,842 additions and 182 deletions.
144 changes: 126 additions & 18 deletions __tests__/actionUtils.test.ts
View file
Original file line number Diff line number Diff line change
@@ -1,19 +1,31 @@
import * as core from "@actions/core";
import * as fs from "fs";
import * as io from "@actions/io";
import { promises as fs } from "fs";
import * as os from "os";
import * as path from "path";

import { Events, Outputs, State } from "../src/constants";
import { ArtifactCacheEntry } from "../src/contracts";
import * as actionUtils from "../src/utils/actionUtils";

import uuid = require("uuid");

jest.mock("@actions/core");
jest.mock("os");

function getTempDir(): string {
return path.join(__dirname, "_temp", "actionUtils");
}

afterEach(() => {
delete process.env[Events.Key];
});

afterAll(async () => {
delete process.env["GITHUB_WORKSPACE"];
await io.rmRF(getTempDir());
});

test("getArchiveFileSize returns file size", () => {
const filePath = path.join(__dirname, "__fixtures__", "helloWorld.txt");

Expand Down Expand Up @@ -182,42 +194,137 @@ test("isValidEvent returns false for unknown event", () => {
expect(isValidEvent).toBe(false);
});

test("resolvePath with no ~ in path", () => {
const filePath = ".cache/yarn";
test("resolvePaths with no ~ in path", async () => {
const filePath = ".cache";

// Create the following layout:
// cwd
// cwd/.cache
// cwd/.cache/file.txt

const root = path.join(getTempDir(), "no-tilde");
// tarball entries will be relative to workspace
process.env["GITHUB_WORKSPACE"] = root;

await fs.mkdir(root, { recursive: true });
const cache = path.join(root, ".cache");
await fs.mkdir(cache, { recursive: true });
await fs.writeFile(path.join(cache, "file.txt"), "cached");

const resolvedPath = actionUtils.resolvePath(filePath);
const originalCwd = process.cwd();

const expectedPath = path.resolve(filePath);
expect(resolvedPath).toBe(expectedPath);
try {
process.chdir(root);

const resolvedPath = await actionUtils.resolvePaths([filePath]);

const expectedPath = [filePath];
expect(resolvedPath).toStrictEqual(expectedPath);
} finally {
process.chdir(originalCwd);
}
});

test("resolvePath with ~ in path", () => {
const filePath = "~/.cache/yarn";
test("resolvePaths with ~ in path", async () => {
const cacheDir = uuid();
const filePath = `~/${cacheDir}`;
// Create the following layout:
// ~/uuid
// ~/uuid/file.txt

const homedir = jest.requireActual("os").homedir();
const homedirMock = jest.spyOn(os, "homedir");
homedirMock.mockImplementation(() => {
return homedir;
});

const resolvedPath = actionUtils.resolvePath(filePath);
const target = path.join(homedir, cacheDir);
await fs.mkdir(target, { recursive: true });
await fs.writeFile(path.join(target, "file.txt"), "cached");

const root = getTempDir();
process.env["GITHUB_WORKSPACE"] = root;

const expectedPath = path.join(homedir, ".cache/yarn");
expect(resolvedPath).toBe(expectedPath);
try {
const resolvedPath = await actionUtils.resolvePaths([filePath]);

const expectedPath = [path.relative(root, target)];
expect(resolvedPath).toStrictEqual(expectedPath);
} finally {
await io.rmRF(target);
}
});

test("resolvePath with home not found", () => {
test("resolvePaths with home not found", async () => {
const filePath = "~/.cache/yarn";
const homedirMock = jest.spyOn(os, "homedir");
homedirMock.mockImplementation(() => {
return "";
});

expect(() => actionUtils.resolvePath(filePath)).toThrow(
"Unable to resolve `~` to HOME"
await expect(actionUtils.resolvePaths([filePath])).rejects.toThrow(
"Unable to determine HOME directory"
);
});

test("resolvePaths inclusion pattern returns found", async () => {
const pattern = "*.ts";
// Create the following layout:
// inclusion-patterns
// inclusion-patterns/miss.txt
// inclusion-patterns/test.ts

const root = path.join(getTempDir(), "inclusion-patterns");
// tarball entries will be relative to workspace
process.env["GITHUB_WORKSPACE"] = root;

await fs.mkdir(root, { recursive: true });
await fs.writeFile(path.join(root, "miss.txt"), "no match");
await fs.writeFile(path.join(root, "test.ts"), "match");

const originalCwd = process.cwd();

try {
process.chdir(root);

const resolvedPath = await actionUtils.resolvePaths([pattern]);

const expectedPath = ["test.ts"];
expect(resolvedPath).toStrictEqual(expectedPath);
} finally {
process.chdir(originalCwd);
}
});

test("resolvePaths exclusion pattern returns not found", async () => {
const patterns = ["*.ts", "!test.ts"];
// Create the following layout:
// exclusion-patterns
// exclusion-patterns/miss.txt
// exclusion-patterns/test.ts

const root = path.join(getTempDir(), "exclusion-patterns");
// tarball entries will be relative to workspace
process.env["GITHUB_WORKSPACE"] = root;

await fs.mkdir(root, { recursive: true });
await fs.writeFile(path.join(root, "miss.txt"), "no match");
await fs.writeFile(path.join(root, "test.ts"), "no match");

const originalCwd = process.cwd();

try {
process.chdir(root);

const resolvedPath = await actionUtils.resolvePaths(patterns);

const expectedPath = [];
expect(resolvedPath).toStrictEqual(expectedPath);
} finally {
process.chdir(originalCwd);
}
});

test("isValidEvent returns true for push event", () => {
const event = Events.Push;
process.env[Events.Key] = event;
Expand All @@ -237,13 +344,14 @@ test("isValidEvent returns true for pull request event", () => {
});

test("unlinkFile unlinks file", async () => {
const testDirectory = fs.mkdtempSync("unlinkFileTest");
const testDirectory = await fs.mkdtemp("unlinkFileTest");
const testFile = path.join(testDirectory, "test.txt");
fs.writeFileSync(testFile, "hello world");
await fs.writeFile(testFile, "hello world");

await actionUtils.unlinkFile(testFile);

expect(fs.existsSync(testFile)).toBe(false);
// This should throw as testFile should not exist
await expect(fs.stat(testFile)).rejects.toThrow();

fs.rmdirSync(testDirectory);
await fs.rmdir(testDirectory);
});
21 changes: 21 additions & 0 deletions __tests__/cacheHttpsClient.test.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
import { getCacheVersion } from "../src/cacheHttpClient";
import { Inputs } from "../src/constants";
import * as testUtils from "../src/utils/testUtils";

afterEach(() => {
testUtils.clearInputs();
});

test("getCacheVersion with path input returns version", async () => {
testUtils.setInput(Inputs.Path, "node_modules");

const result = getCacheVersion();

expect(result).toEqual(
"b3e0c6cb5ecf32614eeb2997d905b9c297046d7cbf69062698f25b14b4cb0985"
);
});

test("getCacheVersion with no input throws", async () => {
expect(() => getCacheVersion()).toThrow();
});
16 changes: 5 additions & 11 deletions __tests__/restore.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,6 @@ jest.mock("../src/tar");
jest.mock("../src/utils/actionUtils");

beforeAll(() => {
jest.spyOn(actionUtils, "resolvePath").mockImplementation(filePath => {
return path.resolve(filePath);
});

jest.spyOn(actionUtils, "isExactKeyMatch").mockImplementation(
(key, cacheResult) => {
const actualUtils = jest.requireActual("../src/utils/actionUtils");
Expand Down Expand Up @@ -60,7 +56,8 @@ test("restore with invalid event outputs warning", async () => {
test("restore with no path should fail", async () => {
const failedMock = jest.spyOn(core, "setFailed");
await run();
expect(failedMock).toHaveBeenCalledWith(
// this input isn't necessary for restore b/c tarball contains entries relative to workspace
expect(failedMock).not.toHaveBeenCalledWith(
"Input required and not supplied: path"
);
});
Expand Down Expand Up @@ -202,7 +199,6 @@ test("restore with restore keys and no cache found", async () => {

test("restore with cache found", async () => {
const key = "node-test";
const cachePath = path.resolve("node_modules");
testUtils.setInputs({
path: "node_modules",
key
Expand Down Expand Up @@ -257,7 +253,7 @@ test("restore with cache found", async () => {
expect(getArchiveFileSizeMock).toHaveBeenCalledWith(archivePath);

expect(extractTarMock).toHaveBeenCalledTimes(1);
expect(extractTarMock).toHaveBeenCalledWith(archivePath, cachePath);
expect(extractTarMock).toHaveBeenCalledWith(archivePath);

expect(unlinkFileMock).toHaveBeenCalledTimes(1);
expect(unlinkFileMock).toHaveBeenCalledWith(archivePath);
Expand All @@ -271,7 +267,6 @@ test("restore with cache found", async () => {

test("restore with a pull request event and cache found", async () => {
const key = "node-test";
const cachePath = path.resolve("node_modules");
testUtils.setInputs({
path: "node_modules",
key
Expand Down Expand Up @@ -328,7 +323,7 @@ test("restore with a pull request event and cache found", async () => {
expect(infoMock).toHaveBeenCalledWith(`Cache Size: ~60 MB (62915000 B)`);

expect(extractTarMock).toHaveBeenCalledTimes(1);
expect(extractTarMock).toHaveBeenCalledWith(archivePath, cachePath);
expect(extractTarMock).toHaveBeenCalledWith(archivePath);

expect(setCacheHitOutputMock).toHaveBeenCalledTimes(1);
expect(setCacheHitOutputMock).toHaveBeenCalledWith(true);
Expand All @@ -340,7 +335,6 @@ test("restore with a pull request event and cache found", async () => {
test("restore with cache found for restore key", async () => {
const key = "node-test";
const restoreKey = "node-";
const cachePath = path.resolve("node_modules");
testUtils.setInputs({
path: "node_modules",
key,
Expand Down Expand Up @@ -396,7 +390,7 @@ test("restore with cache found for restore key", async () => {
expect(infoMock).toHaveBeenCalledWith(`Cache Size: ~0 MB (142 B)`);

expect(extractTarMock).toHaveBeenCalledTimes(1);
expect(extractTarMock).toHaveBeenCalledWith(archivePath, cachePath);
expect(extractTarMock).toHaveBeenCalledWith(archivePath);

expect(setCacheHitOutputMock).toHaveBeenCalledTimes(1);
expect(setCacheHitOutputMock).toHaveBeenCalledWith(false);
Expand Down
34 changes: 19 additions & 15 deletions __tests__/save.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import * as core from "@actions/core";
import * as path from "path";

import * as cacheHttpClient from "../src/cacheHttpClient";
import { Events, Inputs } from "../src/constants";
import { CacheFilename, Events, Inputs } from "../src/constants";
import { ArtifactCacheEntry } from "../src/contracts";
import run from "../src/save";
import * as tar from "../src/tar";
Expand Down Expand Up @@ -41,9 +41,11 @@ beforeAll(() => {
return actualUtils.getSupportedEvents();
});

jest.spyOn(actionUtils, "resolvePath").mockImplementation(filePath => {
return path.resolve(filePath);
});
jest.spyOn(actionUtils, "resolvePaths").mockImplementation(
async filePaths => {
return filePaths.map(x => path.resolve(x));
}
);

jest.spyOn(actionUtils, "createTempDirectory").mockImplementation(() => {
return Promise.resolve("/foo/bar");
Expand Down Expand Up @@ -190,7 +192,7 @@ test("save with large cache outputs warning", async () => {
});

const inputPath = "node_modules";
const cachePath = path.resolve(inputPath);
const cachePaths = [path.resolve(inputPath)];
testUtils.setInput(Inputs.Path, inputPath);

const createTarMock = jest.spyOn(tar, "createTar");
Expand All @@ -202,10 +204,10 @@ test("save with large cache outputs warning", async () => {

await run();

const archivePath = path.join("/foo/bar", "cache.tgz");
const archiveFolder = "/foo/bar";

expect(createTarMock).toHaveBeenCalledTimes(1);
expect(createTarMock).toHaveBeenCalledWith(archivePath, cachePath);
expect(createTarMock).toHaveBeenCalledWith(archiveFolder, cachePaths);

expect(logWarningMock).toHaveBeenCalledTimes(1);
expect(logWarningMock).toHaveBeenCalledWith(
Expand Down Expand Up @@ -289,7 +291,7 @@ test("save with server error outputs warning", async () => {
});

const inputPath = "node_modules";
const cachePath = path.resolve(inputPath);
const cachePaths = [path.resolve(inputPath)];
testUtils.setInput(Inputs.Path, inputPath);

const cacheId = 4;
Expand All @@ -312,13 +314,14 @@ test("save with server error outputs warning", async () => {
expect(reserveCacheMock).toHaveBeenCalledTimes(1);
expect(reserveCacheMock).toHaveBeenCalledWith(primaryKey);

const archivePath = path.join("/foo/bar", "cache.tgz");
const archiveFolder = "/foo/bar";
const archiveFile = path.join(archiveFolder, CacheFilename);

expect(createTarMock).toHaveBeenCalledTimes(1);
expect(createTarMock).toHaveBeenCalledWith(archivePath, cachePath);
expect(createTarMock).toHaveBeenCalledWith(archiveFolder, cachePaths);

expect(saveCacheMock).toHaveBeenCalledTimes(1);
expect(saveCacheMock).toHaveBeenCalledWith(cacheId, archivePath);
expect(saveCacheMock).toHaveBeenCalledWith(cacheId, archiveFile);

expect(logWarningMock).toHaveBeenCalledTimes(1);
expect(logWarningMock).toHaveBeenCalledWith("HTTP Error Occurred");
Expand Down Expand Up @@ -348,7 +351,7 @@ test("save with valid inputs uploads a cache", async () => {
});

const inputPath = "node_modules";
const cachePath = path.resolve(inputPath);
const cachePaths = [path.resolve(inputPath)];
testUtils.setInput(Inputs.Path, inputPath);

const cacheId = 4;
Expand All @@ -367,13 +370,14 @@ test("save with valid inputs uploads a cache", async () => {
expect(reserveCacheMock).toHaveBeenCalledTimes(1);
expect(reserveCacheMock).toHaveBeenCalledWith(primaryKey);

const archivePath = path.join("/foo/bar", "cache.tgz");
const archiveFolder = "/foo/bar";
const archiveFile = path.join(archiveFolder, CacheFilename);

expect(createTarMock).toHaveBeenCalledTimes(1);
expect(createTarMock).toHaveBeenCalledWith(archivePath, cachePath);
expect(createTarMock).toHaveBeenCalledWith(archiveFolder, cachePaths);

expect(saveCacheMock).toHaveBeenCalledTimes(1);
expect(saveCacheMock).toHaveBeenCalledWith(cacheId, archivePath);
expect(saveCacheMock).toHaveBeenCalledWith(cacheId, archiveFile);

expect(failedMock).toHaveBeenCalledTimes(0);
});
Loading

0 comments on commit eb78578

Please sign in to comment.