Merge pull request #75 from github/jurre/proxy

Set up proxy and configure file fetcher and updater traffic to route through it

__tests__/helpers.ts

@@ -1,5 +1,5 @@
 import Docker from 'dockerode'
-import {UPDATER_IMAGE_NAME} from '../src/main'
+import {UPDATER_IMAGE_NAME, PROXY_IMAGE_NAME} from '../src/main'
 import waitPort from 'wait-port'
 import path from 'path'
 import {spawn} from 'child_process'
@@ -9,14 +9,20 @@ export const removeDanglingUpdaterContainers = async (): Promise<void> => {
   const containers = (await docker.listContainers()) || []
 
   for (const container of containers) {
-    if (container.Image.includes(UPDATER_IMAGE_NAME)) {
+    if (
+      container.Image.includes(UPDATER_IMAGE_NAME) ||
+      container.Image.includes(PROXY_IMAGE_NAME)
+    ) {
       try {
         await docker.getContainer(container.Id).remove({v: true, force: true})
       } catch (e) {
         // ignore
       }
     }
   }
+
+  await docker.pruneNetworks()
+  await docker.pruneContainers()
 }
 
 export const runFakeDependabotApi = async (port: number): Promise<Function> => {

__tests__/proxy-integration.test.ts

@@ -0,0 +1,63 @@
+import Docker from 'dockerode'
+import {Credential, JobDetails, PackageManager} from '../src/api-client'
+import {ImageService} from '../src/image-service'
+import {PROXY_IMAGE_NAME} from '../src/main'
+import {ProxyBuilder} from '../src/proxy'
+import {removeDanglingUpdaterContainers} from './helpers'
+
+describe('ProxyBuilder', () => {
+  const docker = new Docker()
+  const details: JobDetails = {
+    id: '1',
+    'allowed-updates': [
+      {
+        'dependency-type': 'all'
+      }
+    ],
+    'package-manager': PackageManager.NpmAndYarn
+  }
+  const credentials: Credential[] = [
+    {
+      type: 'git_source',
+      host: 'github.com',
+      username: 'x-access-token',
+      password: 'ghp_some_token'
+    }
+  ]
+
+  const builder = new ProxyBuilder(docker, PROXY_IMAGE_NAME)
+
+  beforeAll(async () => {
+    // Skip the test when we haven't preloaded the updater image
+    if (process.env.SKIP_INTEGRATION_TESTS) {
+      return
+    }
+    await ImageService.pull(PROXY_IMAGE_NAME)
+  })
+
+  afterEach(async () => {
+    await removeDanglingUpdaterContainers()
+  })
+
+  it('should create a proxy container with the right details', async () => {
+    // Skip the test when we haven't preloaded the updater image
+    if (process.env.SKIP_INTEGRATION_TESTS) {
+      return
+    }
+
+    const proxy = await builder.run(details, credentials)
+
+    expect(proxy.networkName).toBe('job-1-network')
+    expect(proxy.url).toMatch(/^http:\/\/1:.+job-1-proxy:1080$/)
+
+    const containerInfo = await proxy.container.inspect()
+    expect(containerInfo.Name).toBe('/job-1-proxy')
+    expect(containerInfo.HostConfig.NetworkMode).toBe('job-1-network')
+
+    const networkInfo = await proxy.network.inspect()
+    expect(networkInfo.Name).toBe('job-1-network')
+    expect(networkInfo.Internal).toBe(false)
+
+    await proxy.shutdown()
+  })
+})

__tests__/server/server.js

@@ -45,12 +45,12 @@ server.get('/update_jobs/:id/credentials', (_, res) => {
   res.jsonp({
     data: {
       attributes: {
-        credentials: {
+        credentials: [{
           type: 'git_source',
           host: 'github.com',
           username: 'x-access-token',
           password: process.env.GITHUB_TOKEN
-        }
+        }]
       }
     }
   })

__tests__/updater-integration.test.ts

@@ -2,7 +2,7 @@ import axios from 'axios'
 
 import {APIClient, JobParameters} from '../src/api-client'
 import {ImageService} from '../src/image-service'
-import {UPDATER_IMAGE_NAME} from '../src/main'
+import {UPDATER_IMAGE_NAME, PROXY_IMAGE_NAME} from '../src/main'
 import {Updater} from '../src/updater'
 
 import {removeDanglingUpdaterContainers, runFakeDependabotApi} from './helpers'
@@ -27,7 +27,7 @@ describe('Updater', () => {
     process.env.DEPENDABOT_API_URL || fakeDependabotApiUrl
   // Used from within the updater container to update the job state and create prs
   const internalDockerHost =
-    process.platform === 'darwin' ? 'host.docker.internal' : 'localhost'
+    process.platform === 'darwin' ? 'host.docker.internal' : '172.17.0.1'
   const internalDependabotApiUrl =
     process.env.DEPENDABOT_API_URL ||
     `http://${internalDockerHost}:${FAKE_SERVER_PORT}`
@@ -40,7 +40,6 @@ describe('Updater', () => {
 
   const client = axios.create({baseURL: externalDependabotApiUrl})
   const apiClient = new APIClient(client, params)
-  const updater = new Updater(UPDATER_IMAGE_NAME, apiClient)
 
   beforeAll(async () => {
     // Skip the test when we haven't preloaded the updater image
@@ -49,6 +48,7 @@ describe('Updater', () => {
     }
 
     await ImageService.pull(UPDATER_IMAGE_NAME)
+    await ImageService.pull(PROXY_IMAGE_NAME)
 
     if (externalDependabotApiUrl === fakeDependabotApiUrl) {
       server = await runFakeDependabotApi(FAKE_SERVER_PORT)
@@ -60,13 +60,24 @@ describe('Updater', () => {
     await removeDanglingUpdaterContainers()
   })
 
-  jest.setTimeout(25000)
+  jest.setTimeout(120000)
   it('should run the updater and create a pull request', async () => {
     // Skip the test when we haven't preloaded the updater image
     if (process.env.SKIP_INTEGRATION_TESTS) {
       return
     }
 
+    const details = await apiClient.getJobDetails()
+    const credentials = await apiClient.getCredentials()
+
+    const updater = new Updater(
+      UPDATER_IMAGE_NAME,
+      PROXY_IMAGE_NAME,
+      apiClient,
+      details,
+      credentials
+    )
+
     await updater.runUpdater()
 
     // NOTE: This will not work when running against the actual dependabot-api

__tests__/updater.test.ts

@@ -1,5 +1,6 @@
-import {UPDATER_IMAGE_NAME} from '../src/main'
+import {UPDATER_IMAGE_NAME, PROXY_IMAGE_NAME} from '../src/main'
 import {Updater} from '../src/updater'
+import {PackageManager} from '../src/api-client'
 
 describe('Updater', () => {
   const mockAPIClient: any = {
@@ -12,7 +13,23 @@ describe('Updater', () => {
       dependabotAPIURL: 'http://host.docker.internal:3001'
     }
   }
-  const updater = new Updater(UPDATER_IMAGE_NAME, mockAPIClient)
+  const mockJobDetails: any = {
+    id: '1',
+    'allowed-updates': [
+      {
+        'dependency-type': 'all'
+      }
+    ],
+    'package-manage': PackageManager.NpmAndYarn
+  }
+
+  const updater = new Updater(
+    UPDATER_IMAGE_NAME,
+    PROXY_IMAGE_NAME,
+    mockAPIClient,
+    mockJobDetails,
+    []
+  )
 
   it('should fetch job details', async () => {
     mockAPIClient.getJobDetails.mockImplementation(() => {

package.json

@@ -36,6 +36,7 @@
     "@types/dockerode": "^3.2.6",
     "@types/jest": "^26.0.24",
     "@types/node": "^16.4.6",
+    "@types/node-forge": "^0.10.0",
     "@types/tar-stream": "^2.2.1",
     "@typescript-eslint/parser": "^4.28.5",
     "@vercel/ncc": "^0.29.0",
@@ -49,6 +50,7 @@
     "js-yaml": "^4.1.0",
     "json-server": "^0.16.3",
     "lint-staged": "^11.1.1",
+    "node-forge": "^0.10.0",
     "prettier": "2.3.2",
     "ts-jest": "^27.0.4",
     "ts-node": "^10.1.0",

src/api-client.ts

@@ -21,7 +21,6 @@ export type JobDetails = {
   }[]
   id: string
   'package-manager': PackageManager
-  credentials: Credential[] // TODO: Remove these once the proxy is set up
 }
 
 export type Credential = {

src/file-types.ts → src/config-types.ts

@@ -8,7 +8,6 @@ export type FetchedFiles = {
 
 export type FileFetcherInput = {
   job: JobDetails
-  credentials: Credential[]
 }
 
 export type DependencyFile = {
@@ -25,3 +24,19 @@ export type DependencyFile = {
 export type FileUpdaterInput = FetchedFiles & {
   job: JobDetails
 }
+
+export type CertificateAuthority = {
+  cert: string
+  key: string
+}
+
+export type BasicAuthCredentials = {
+  username: string
+  password: string
+}
+
+export type ProxyConfig = {
+  all_credentials: Credential[]
+  ca: CertificateAuthority
+  proxy_auth: BasicAuthCredentials
+}

src/container-service.ts

@@ -1,29 +1,46 @@
 import * as core from '@actions/core'
 import {Container} from 'dockerode'
 import {pack} from 'tar-stream'
-import {FileFetcherInput, FileUpdaterInput} from './file-types'
+import {FileFetcherInput, FileUpdaterInput, ProxyConfig} from './config-types'
+import {outStream, errStream} from './utils'
 
 export const ContainerService = {
   async storeInput(
     name: string,
     path: string,
     container: Container,
-    input: FileFetcherInput | FileUpdaterInput
+    input: FileFetcherInput | FileUpdaterInput | ProxyConfig
   ): Promise<void> {
     const tar = pack()
     tar.entry({name}, JSON.stringify(input))
     tar.finalize()
     await container.putArchive(tar, {path})
   },
 
+  async storeCert(
+    name: string,
+    path: string,
+    container: Container,
+    cert: string
+  ): Promise<void> {
+    const tar = pack()
+    tar.entry({name}, cert)
+    tar.finalize()
+    await container.putArchive(tar, {path})
+  },
+
   async run(container: Container): Promise<void> {
     try {
       const stream = await container.attach({
         stream: true,
         stdout: true,
         stderr: true
       })
-      container.modem.demuxStream(stream, process.stdout, process.stderr)
+      container.modem.demuxStream(
+        stream,
+        outStream('updater'),
+        errStream('updater')
+      )
 
       await container.start()
       await container.wait()