From a2206945da5483036d24bf2c1a466d5d22b3e610 Mon Sep 17 00:00:00 2001 From: Jurre Stender Date: Tue, 10 Aug 2021 12:20:58 +0200 Subject: [PATCH] [WIP] Set up proxy --- __tests__/helpers.ts | 7 +- __tests__/server/server.js | 4 +- __tests__/updater-integration.test.ts | 5 +- __tests__/updater.test.ts | 8 +- package-lock.json | 35 ++++++ package.json | 2 + src/container-service.ts | 16 ++- src/file-types.ts | 16 +++ src/main.ts | 5 +- src/proxy.ts | 153 ++++++++++++++++++++++++++ src/updater.ts | 43 +++++++- 11 files changed, 280 insertions(+), 14 deletions(-) create mode 100644 src/proxy.ts diff --git a/__tests__/helpers.ts b/__tests__/helpers.ts index 0fceeba..4186e0e 100644 --- a/__tests__/helpers.ts +++ b/__tests__/helpers.ts @@ -1,5 +1,5 @@ import Docker from 'dockerode' -import {UPDATER_IMAGE_NAME} from '../src/main' +import {UPDATER_IMAGE_NAME, PROXY_IMAGE_NAME} from '../src/main' import waitPort from 'wait-port' import path from 'path' import {spawn} from 'child_process' @@ -9,7 +9,10 @@ export const removeDanglingUpdaterContainers = async (): Promise => { const containers = (await docker.listContainers()) || [] for (const container of containers) { - if (container.Image.includes(UPDATER_IMAGE_NAME)) { + if ( + container.Image.includes(UPDATER_IMAGE_NAME) || + container.Image.includes(PROXY_IMAGE_NAME) + ) { try { await docker.getContainer(container.Id).remove({v: true, force: true}) } catch (e) { diff --git a/__tests__/server/server.js b/__tests__/server/server.js index bcb43ef..155200b 100755 --- a/__tests__/server/server.js +++ b/__tests__/server/server.js @@ -45,12 +45,12 @@ server.get('/update_jobs/:id/credentials', (_, res) => { res.jsonp({ data: { attributes: { - credentials: { + credentials: [{ type: 'git_source', host: 'github.com', username: 'x-access-token', password: process.env.GITHUB_TOKEN - } + }] } } }) diff --git a/__tests__/updater-integration.test.ts b/__tests__/updater-integration.test.ts index d437a9d..6d58394 100644 --- a/__tests__/updater-integration.test.ts +++ b/__tests__/updater-integration.test.ts @@ -2,7 +2,7 @@ import axios from 'axios' import {APIClient, JobParameters} from '../src/api-client' import {ImageService} from '../src/image-service' -import {UPDATER_IMAGE_NAME} from '../src/main' +import {UPDATER_IMAGE_NAME, PROXY_IMAGE_NAME} from '../src/main' import {Updater} from '../src/updater' import {removeDanglingUpdaterContainers, runFakeDependabotApi} from './helpers' @@ -40,7 +40,7 @@ describe('Updater', () => { const client = axios.create({baseURL: externalDependabotApiUrl}) const apiClient = new APIClient(client, params) - const updater = new Updater(UPDATER_IMAGE_NAME, apiClient) + const updater = new Updater(UPDATER_IMAGE_NAME, PROXY_IMAGE_NAME, apiClient) beforeAll(async () => { // Skip the test when we haven't preloaded the updater image @@ -49,6 +49,7 @@ describe('Updater', () => { } await ImageService.pull(UPDATER_IMAGE_NAME) + await ImageService.pull(PROXY_IMAGE_NAME) if (externalDependabotApiUrl === fakeDependabotApiUrl) { server = await runFakeDependabotApi(FAKE_SERVER_PORT) diff --git a/__tests__/updater.test.ts b/__tests__/updater.test.ts index 829a15d..addde06 100644 --- a/__tests__/updater.test.ts +++ b/__tests__/updater.test.ts @@ -1,4 +1,4 @@ -import {UPDATER_IMAGE_NAME} from '../src/main' +import {UPDATER_IMAGE_NAME, PROXY_IMAGE_NAME} from '../src/main' import {Updater} from '../src/updater' describe('Updater', () => { @@ -12,7 +12,11 @@ describe('Updater', () => { dependabotAPIURL: 'http://host.docker.internal:3001' } } - const updater = new Updater(UPDATER_IMAGE_NAME, mockAPIClient) + const updater = new Updater( + UPDATER_IMAGE_NAME, + PROXY_IMAGE_NAME, + mockAPIClient + ) it('should fetch job details', async () => { mockAPIClient.getJobDetails.mockImplementation(() => { diff --git a/package-lock.json b/package-lock.json index f2662d4..2562bf8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,6 +20,7 @@ "@types/dockerode": "^3.2.6", "@types/jest": "^26.0.24", "@types/node": "^16.4.6", + "@types/node-forge": "^0.10.0", "@types/tar-stream": "^2.2.1", "@typescript-eslint/parser": "^4.28.5", "@vercel/ncc": "^0.29.0", @@ -33,6 +34,7 @@ "js-yaml": "^4.1.0", "json-server": "^0.16.3", "lint-staged": "^11.1.1", + "node-forge": "^0.10.0", "prettier": "2.3.2", "ts-jest": "^27.0.4", "ts-node": "^10.1.0", @@ -1375,6 +1377,15 @@ "integrity": "sha512-FKyawK3o5KL16AwbeFajen8G4K3mmqUrQsehn5wNKs8IzlKHE8TfnSmILXVMVziAEcnB23u1RCFU1NT6hSyr7Q==", "dev": true }, + "node_modules/@types/node-forge": { + "version": "0.10.2", + "resolved": "https://registry.npmjs.org/@types/node-forge/-/node-forge-0.10.2.tgz", + "integrity": "sha512-nEWO3mkJ1j7eGxGUu32jaGFJj+YSvUt/zG4sEAXbUDbjkQMf9u98Bf3peC4oGFR3zA1n3M3KaXcw6xQyZpl5jg==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/parse-json": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.0.tgz", @@ -6759,6 +6770,15 @@ "node": "4.x || >=6.0.0" } }, + "node_modules/node-forge": { + "version": "0.10.0", + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.10.0.tgz", + "integrity": "sha512-PPmu8eEeG9saEUvI97fm4OYxXVB6bFvyNTyiUOBichBpFG8A1Ljw3bY62+5oOjDEMHRnd0Y7HQ+x7uzxOzC6JA==", + "dev": true, + "engines": { + "node": ">= 6.0.0" + } + }, "node_modules/node-int64": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", @@ -10288,6 +10308,15 @@ "integrity": "sha512-FKyawK3o5KL16AwbeFajen8G4K3mmqUrQsehn5wNKs8IzlKHE8TfnSmILXVMVziAEcnB23u1RCFU1NT6hSyr7Q==", "dev": true }, + "@types/node-forge": { + "version": "0.10.2", + "resolved": "https://registry.npmjs.org/@types/node-forge/-/node-forge-0.10.2.tgz", + "integrity": "sha512-nEWO3mkJ1j7eGxGUu32jaGFJj+YSvUt/zG4sEAXbUDbjkQMf9u98Bf3peC4oGFR3zA1n3M3KaXcw6xQyZpl5jg==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "@types/parse-json": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.0.tgz", @@ -14360,6 +14389,12 @@ "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==" }, + "node-forge": { + "version": "0.10.0", + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.10.0.tgz", + "integrity": "sha512-PPmu8eEeG9saEUvI97fm4OYxXVB6bFvyNTyiUOBichBpFG8A1Ljw3bY62+5oOjDEMHRnd0Y7HQ+x7uzxOzC6JA==", + "dev": true + }, "node-int64": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", diff --git a/package.json b/package.json index bee9e90..20280d3 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "@types/dockerode": "^3.2.6", "@types/jest": "^26.0.24", "@types/node": "^16.4.6", + "@types/node-forge": "^0.10.0", "@types/tar-stream": "^2.2.1", "@typescript-eslint/parser": "^4.28.5", "@vercel/ncc": "^0.29.0", @@ -49,6 +50,7 @@ "js-yaml": "^4.1.0", "json-server": "^0.16.3", "lint-staged": "^11.1.1", + "node-forge": "^0.10.0", "prettier": "2.3.2", "ts-jest": "^27.0.4", "ts-node": "^10.1.0", diff --git a/src/container-service.ts b/src/container-service.ts index c0e04b5..e025fd9 100644 --- a/src/container-service.ts +++ b/src/container-service.ts @@ -1,14 +1,14 @@ import * as core from '@actions/core' import {Container} from 'dockerode' import {pack} from 'tar-stream' -import {FileFetcherInput, FileUpdaterInput} from './file-types' +import {FileFetcherInput, FileUpdaterInput, ProxyConfig} from './file-types' export const ContainerService = { async storeInput( name: string, path: string, container: Container, - input: FileFetcherInput | FileUpdaterInput + input: FileFetcherInput | FileUpdaterInput | ProxyConfig ): Promise { const tar = pack() tar.entry({name}, JSON.stringify(input)) @@ -16,6 +16,18 @@ export const ContainerService = { await container.putArchive(tar, {path}) }, + async storeCert( + name: string, + path: string, + container: Container, + cert: string + ): Promise { + const tar = pack() + tar.entry({name}, cert) + tar.finalize() + await container.putArchive(tar, {path}) + }, + async run(container: Container): Promise { try { const stream = await container.attach({ diff --git a/src/file-types.ts b/src/file-types.ts index d40d15b..0bd32c4 100644 --- a/src/file-types.ts +++ b/src/file-types.ts @@ -25,3 +25,19 @@ export type DependencyFile = { export type FileUpdaterInput = FetchedFiles & { job: JobDetails } + +export type CertificateAuthority = { + cert: string + key: string +} + +export type BasicAuthCredentials = { + username: string + password: string +} + +export type ProxyConfig = { + all_credentials: Credential[] + ca: CertificateAuthority + proxy_auth: BasicAuthCredentials +} diff --git a/src/main.ts b/src/main.ts index b97cb1c..af92ca8 100644 --- a/src/main.ts +++ b/src/main.ts @@ -8,6 +8,8 @@ import axios from 'axios' export const UPDATER_IMAGE_NAME = 'docker.pkg.github.com/dependabot/dependabot-updater:latest' +export const PROXY_IMAGE_NAME = + 'docker.pkg.github.com/github/dependabot-update-job-proxy:latest' async function run(): Promise { try { @@ -24,8 +26,9 @@ async function run(): Promise { const client = axios.create({baseURL: params.dependabotAPIURL}) const apiClient = new APIClient(client, params) - const updater = new Updater(UPDATER_IMAGE_NAME, apiClient) + const updater = new Updater(UPDATER_IMAGE_NAME, PROXY_IMAGE_NAME, apiClient) await ImageService.pull(UPDATER_IMAGE_NAME) + await ImageService.pull(PROXY_IMAGE_NAME) await updater.runUpdater() } catch (error) { diff --git a/src/proxy.ts b/src/proxy.ts new file mode 100644 index 0000000..857250f --- /dev/null +++ b/src/proxy.ts @@ -0,0 +1,153 @@ +import * as core from '@actions/core' +import Docker, {Container} from 'dockerode' +import crypto from 'crypto' +import { + BasicAuthCredentials, + CertificateAuthority, + ProxyConfig +} from './file-types' +import {ContainerService} from './container-service' +import {Credential, JobDetails} from './api-client' +import {pki} from 'node-forge' + +const KEY_SIZE = 2048 +const KEY_EXPIRY_YEARS = 2 +const CONFIG_FILE_PATH = '/' +const CONFIG_FILE_NAME = 'config.json' +const CERT_SUBJECT = [ + { + name: 'commonName', + value: 'Dependabot Internal CA' + }, + { + name: 'organizationName', + value: 'GitHub ic.' + }, + { + shortName: 'OU', + value: 'Dependabot' + }, + { + name: 'countryName', + value: 'US' + }, + { + shortName: 'ST', + value: 'California' + }, + { + name: 'localityName', + value: 'San Francisco' + } +] + +export class Proxy { + container?: Container + url: string + cert: string + + constructor( + private readonly docker: Docker, + private readonly proxyImage: string + ) { + // TODO: this is obviously gnarly, decouple some things so we don't need to + // initialize these as empty strings + this.url = '' + this.cert = '' + } + + async run(details: JobDetails, credentials: Credential[]): Promise { + const name = `job-${details.id}-proxy` + const config = this.buildProxyConfig(credentials, details.id) + this.cert = config.ca.cert + + core.info(JSON.stringify(config)) // TODO: remove! + this.container = await this.createContainer(details.id, name) + await ContainerService.storeInput( + CONFIG_FILE_NAME, + CONFIG_FILE_PATH, + this.container, + config + ) + + const stream = await this.container.attach({ + stream: true, + stdout: true, + stderr: true + }) + this.container.modem.demuxStream(stream, process.stdout, process.stderr) + + this.container.start() + this.url = `http://${config.proxy_auth.username}:${config.proxy_auth.password}@${name}:1080` + core.info(this.url) + } + + private buildProxyConfig( + credentials: Credential[], + jobID: string + ): ProxyConfig { + const ca = this.generateCertificateAuthority() + const password = crypto.randomBytes(20).toString('hex') + const proxy_auth: BasicAuthCredentials = { + username: `${jobID}`, + password + } + + const config: ProxyConfig = {all_credentials: credentials, ca, proxy_auth} + + return config + } + + private generateCertificateAuthority(): CertificateAuthority { + const keys = crypto.generateKeyPairSync('rsa', { + modulusLength: KEY_SIZE, + publicKeyEncoding: { + type: 'spki', + format: 'pem' + }, + privateKeyEncoding: { + type: 'pkcs8', + format: 'pem' + } + }) + + const prKey = pki.privateKeyFromPem(keys.privateKey) + const pubKey = pki.publicKeyFromPem(keys.publicKey) + + const cert = pki.createCertificate() + + cert.publicKey = pubKey + cert.serialNumber = '01' + cert.validity.notBefore = new Date() + cert.validity.notAfter = new Date() + cert.validity.notAfter.setFullYear( + cert.validity.notBefore.getFullYear() + KEY_EXPIRY_YEARS + ) + + cert.setSubject(CERT_SUBJECT) + cert.setIssuer(CERT_SUBJECT) + cert.sign(prKey) + + const pemCert = pki.certificateToPem(cert) + return {cert: pemCert, key: keys.privateKey} + } + + private async createContainer( + jobID: string, + containerName: string + ): Promise { + const container = await this.docker.createContainer({ + Image: this.proxyImage, + name: containerName, + AttachStdout: true, + AttachStderr: true, + Env: [`DEPENDABOT_JOB_ID=${jobID}`], + HostConfig: { + NetworkMode: `job-test-network` // TODO: Dynamically generate network + } + }) + + core.info(`Created proxy container: ${container.id}`) + return container + } +} diff --git a/src/updater.ts b/src/updater.ts index 5af72f6..e623a71 100644 --- a/src/updater.ts +++ b/src/updater.ts @@ -6,20 +6,27 @@ import {Credential, JobDetails, APIClient} from './api-client' import {ContainerService} from './container-service' import {base64DecodeDependencyFile} from './utils' import {DependencyFile, FetchedFiles, FileUpdaterInput} from './file-types' +import {Proxy} from './proxy' const JOB_INPUT_FILENAME = 'job.json' const JOB_INPUT_PATH = `/home/dependabot/dependabot-updater` const JOB_OUTPUT_FILENAME = 'output.json' const JOB_OUTPUT_PATH = '/home/dependabot/dependabot-updater/output' const REPO_CONTENTS_PATH = '/home/dependabot/dependabot-updater/repo' +const CA_CERT_INPUT_PATH = '/usr/local/share/ca-certificates' +const CA_CERT_FILENAME = 'dbot-ca.crt' export class Updater { docker: Docker + proxy: Proxy + constructor( private readonly updaterImage: string, + private readonly proxyImage: string, private readonly apiClient: APIClient ) { this.docker = new Docker() + this.proxy = new Proxy(this.docker, proxyImage) } /** @@ -32,6 +39,8 @@ export class Updater { // TODO: once the proxy is set up, remove credentials from the job details details.credentials = credentials + await this.proxy.run(details, credentials) + const files = await this.runFileFetcher(details, credentials) if (!files) { core.error(`failed during fetch, skipping updater`) @@ -43,6 +52,9 @@ export class Updater { } catch (e) { // TODO: report job runner_error? core.error(`Error ${e}`) + } finally { + this.proxy.container?.stop() + this.proxy.container?.remove() } } @@ -60,6 +72,13 @@ export class Updater { credentials } ) + await ContainerService.storeCert( + CA_CERT_FILENAME, + CA_CERT_INPUT_PATH, + container, + this.proxy.cert + ) + await ContainerService.run(container) const outputPath = path.join(__dirname, '../output/output.json') @@ -99,10 +118,18 @@ export class Updater { container, containerInput ) + await ContainerService.storeCert( + CA_CERT_FILENAME, + CA_CERT_INPUT_PATH, + container, + this.proxy.cert + ) + await ContainerService.run(container) } private async createContainer(updaterCommand: string): Promise { + core.info(`Proxy: ${this.proxy.url}`) const container = await this.docker.createContainer({ Image: this.updaterImage, AttachStdout: true, @@ -113,11 +140,21 @@ export class Updater { `DEPENDABOT_JOB_PATH=${JOB_INPUT_PATH}/${JOB_INPUT_FILENAME}`, `DEPENDABOT_OUTPUT_PATH=${JOB_OUTPUT_PATH}/${JOB_OUTPUT_FILENAME}`, `DEPENDABOT_REPO_CONTENTS_PATH=${REPO_CONTENTS_PATH}`, - `DEPENDABOT_API_URL=${this.apiClient.params.dependabotAPIURL}` + `DEPENDABOT_API_URL=${this.apiClient.params.dependabotAPIURL}`, + `SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt`, + `SSL_CERT_DIR=/etc/ssl/certs`, + `http_proxy=${this.proxy.url}`, + `HTTP_PROXY=${this.proxy.url}`, + `https_proxy=${this.proxy.url}`, + `HTTPS_PROXY=${this.proxy.url}` + ], + Cmd: [ + 'sh', + '-c', + `/usr/sbin/update-ca-certificates && $DEPENDABOT_HOME/dependabot-updater/bin/run ${updaterCommand}` ], - Cmd: ['bin/run', updaterCommand], HostConfig: { - NetworkMode: 'host', + NetworkMode: 'job-test-network', Binds: [ `${path.join(__dirname, '../output')}:${JOB_OUTPUT_PATH}:rw`, `${path.join(__dirname, '../repo')}:${REPO_CONTENTS_PATH}:rw`