From 7da583bcb301b9ea6c02f2a73b0bca93c8cb025e Mon Sep 17 00:00:00 2001 From: Marco Gario Date: Thu, 17 Sep 2020 13:36:26 +0200 Subject: [PATCH 1/6] Workflow to split the bundle into components --- .github/workflows/split.yml | 85 +++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 .github/workflows/split.yml diff --git a/.github/workflows/split.yml b/.github/workflows/split.yml new file mode 100644 index 000000000..6f84b45c0 --- /dev/null +++ b/.github/workflows/split.yml @@ -0,0 +1,85 @@ +name: Split Bundle + +on: + workflow_dispatch: + inputs: + cli-release: + description: 'CodeQL CLI Release (e.g., "v2.2.5")' + required: true + bundle-tag: + description: 'Tag of the bundle release (e.g., "codeql-bundle-20200826")' + required: true + +jobs: + build: + runs-on: ubuntu-latest + env: + CLI_RELEASE: "${{ github.event.inputs.cli-release }}" + RELEASE_TAG: "${{ github.event.inputs.bundle-tag }}" + + strategy: + matrix: + platform: ["linux64", "osx64", "win64"] + language: ["cpp", "csharp", "go", "java", "javascript", "python"] + + steps: + - name: Resolve Upload URL for the release + id: save_url + run: | + UPLOAD_URL=$(curl -sS \ + "https://api.github.com/repos/${GITHUB_REPOSITORY}/releases/tags/${RELEASE_TAG}" \ + -H "Accept: application/json" \ + -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" | jq .upload_url | sed s/\"//g) + echo ${UPLOAD_URL} + echo "::set-output name=upload_url::${UPLOAD_URL}" + + - name: Download CodeQL CLI and Bundle + run: | + wget --no-verbose "https://github.com/${GITHUB_REPOSITORY}/releases/download/${RELEASE_TAG}/codeql-bundle.tar.gz" + wget --no-verbose "https://github.com/github/codeql-cli-binaries/releases/download/${CLI_RELEASE}/codeql-${{matrix.platform}}.zip" + + - name: Create Platform Package + # Replace the codeql-binaries with the platform specific ones + run: | + gunzip codeql-bundle.tar.gz + tar -f codeql-bundle.tar --delete codeql + unzip -q codeql-${{matrix.platform}}.zip + tar -f codeql-bundle.tar --append codeql + gzip codeql-bundle.tar + mv codeql-bundle.tar.gz codeql-bundle-${{matrix.platform}}.tar.gz + du -sh codeql-bundle-${{matrix.platform}}.tar.gz + + - name: Upload Platform Package + uses: actions/upload-release-asset@v1 + if: matrix.language == 'cpp' # Only once per platform, cpp is arbitrary + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.save_url.outputs.upload_url }} + asset_path: ./codeql-bundle-${{matrix.platform}}.tar.gz + asset_name: codeql-bundle-${{matrix.platform}}.tar.gz + asset_content_type: application/tar+gzip + + - name: Create Platform-Language Package + # Replace the codeql-binaries with the platform specific ones + run: | + gunzip codeql-bundle-${{matrix.platform}}.tar.gz + [ "${{matrix.language}}" != "cpp" ] && tar -f codeql-bundle-${{matrix.platform}}.tar --delete codeql/cpp --delete ql/cpp + [ "${{matrix.language}}" != "csharp" ] && tar -f codeql-bundle-${{matrix.platform}}.tar --delete codeql/csharp --delete ql/csharp + [ "${{matrix.language}}" != "go" ] && tar -f codeql-bundle-${{matrix.platform}}.tar --delete codeql/go --delete codeql-go + [ "${{matrix.language}}" != "java" ] && tar -f codeql-bundle-${{matrix.platform}}.tar --delete codeql/java --delete ql/java + [ "${{matrix.language}}" != "javascript" ] && tar -f codeql-bundle-${{matrix.platform}}.tar --delete codeql/javascript --delete ql/javascript + [ "${{matrix.language}}" != "python" ] && tar -f codeql-bundle-${{matrix.platform}}.tar --delete codeql/python --delete ql/python + gzip codeql-bundle-${{matrix.platform}}.tar + mv codeql-bundle-${{matrix.platform}}.tar.gz codeql-bundle-${{matrix.platform}}-${{matrix.language}}.tar.gz + du -sh codeql-bundle-${{matrix.platform}}-${{matrix.language}}.tar.gz + + - name: Upload Language Package + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.save_url.outputs.upload_url }} + asset_path: ./codeql-bundle-${{matrix.platform}}-${{matrix.language}}.tar.gz + asset_name: codeql-bundle-${{matrix.platform}}-${{matrix.language}}.tar.gz + asset_content_type: application/tar+gzip \ No newline at end of file From b4a8cfa05c4721a2ab1d069f3e1dfea5aa746ab0 Mon Sep 17 00:00:00 2001 From: Marco Gario Date: Thu, 17 Sep 2020 13:42:19 +0200 Subject: [PATCH 2/6] Add instructions --- .github/workflows/split.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/split.yml b/.github/workflows/split.yml index 6f84b45c0..23cb95eee 100644 --- a/.github/workflows/split.yml +++ b/.github/workflows/split.yml @@ -1,3 +1,16 @@ +# +# Split the CodeQL Bundle into language-platform bundles +# +# Instructions: +# 1. Upload the new codeql-bundle (codeql-bundle.tar.gz) as an asset of the +# release (codeql-bundle-20200826) +# 2. Take note of the CLI Release used by the bundle (e.g., v2.2.5) +# 3. Manually launch this workflow file (via the Actions UI) specifying +# - The CLI Release (e.g., v2.2.5) +# - The release tag (codeql-bundle-20200826) +# 4. If everything succeeds you should see 21 new assets. +# + name: Split Bundle on: From 1154bf6df96f00197c08375606677f1b45d24ba8 Mon Sep 17 00:00:00 2001 From: Marco Gario Date: Mon, 21 Sep 2020 10:13:23 +0200 Subject: [PATCH 3/6] Update split.yml --- .github/workflows/split.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/split.yml b/.github/workflows/split.yml index 23cb95eee..1e3388506 100644 --- a/.github/workflows/split.yml +++ b/.github/workflows/split.yml @@ -31,6 +31,7 @@ jobs: RELEASE_TAG: "${{ github.event.inputs.bundle-tag }}" strategy: + fail-fast: false matrix: platform: ["linux64", "osx64", "win64"] language: ["cpp", "csharp", "go", "java", "javascript", "python"] @@ -95,4 +96,4 @@ jobs: upload_url: ${{ steps.save_url.outputs.upload_url }} asset_path: ./codeql-bundle-${{matrix.platform}}-${{matrix.language}}.tar.gz asset_name: codeql-bundle-${{matrix.platform}}-${{matrix.language}}.tar.gz - asset_content_type: application/tar+gzip \ No newline at end of file + asset_content_type: application/tar+gzip From 47eb66815539f02dd4bfc2ef6e74debc2b66ff0a Mon Sep 17 00:00:00 2001 From: Marco Gario Date: Mon, 21 Sep 2020 12:00:00 +0200 Subject: [PATCH 4/6] Update split.yml --- .github/workflows/split.yml | 28 +--------------------------- 1 file changed, 1 insertion(+), 27 deletions(-) diff --git a/.github/workflows/split.yml b/.github/workflows/split.yml index 1e3388506..61795c4b6 100644 --- a/.github/workflows/split.yml +++ b/.github/workflows/split.yml @@ -8,7 +8,7 @@ # 3. Manually launch this workflow file (via the Actions UI) specifying # - The CLI Release (e.g., v2.2.5) # - The release tag (codeql-bundle-20200826) -# 4. If everything succeeds you should see 21 new assets. +# 4. If everything succeeds you should 3 new assets. # name: Split Bundle @@ -34,7 +34,6 @@ jobs: fail-fast: false matrix: platform: ["linux64", "osx64", "win64"] - language: ["cpp", "csharp", "go", "java", "javascript", "python"] steps: - name: Resolve Upload URL for the release @@ -65,7 +64,6 @@ jobs: - name: Upload Platform Package uses: actions/upload-release-asset@v1 - if: matrix.language == 'cpp' # Only once per platform, cpp is arbitrary env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: @@ -73,27 +71,3 @@ jobs: asset_path: ./codeql-bundle-${{matrix.platform}}.tar.gz asset_name: codeql-bundle-${{matrix.platform}}.tar.gz asset_content_type: application/tar+gzip - - - name: Create Platform-Language Package - # Replace the codeql-binaries with the platform specific ones - run: | - gunzip codeql-bundle-${{matrix.platform}}.tar.gz - [ "${{matrix.language}}" != "cpp" ] && tar -f codeql-bundle-${{matrix.platform}}.tar --delete codeql/cpp --delete ql/cpp - [ "${{matrix.language}}" != "csharp" ] && tar -f codeql-bundle-${{matrix.platform}}.tar --delete codeql/csharp --delete ql/csharp - [ "${{matrix.language}}" != "go" ] && tar -f codeql-bundle-${{matrix.platform}}.tar --delete codeql/go --delete codeql-go - [ "${{matrix.language}}" != "java" ] && tar -f codeql-bundle-${{matrix.platform}}.tar --delete codeql/java --delete ql/java - [ "${{matrix.language}}" != "javascript" ] && tar -f codeql-bundle-${{matrix.platform}}.tar --delete codeql/javascript --delete ql/javascript - [ "${{matrix.language}}" != "python" ] && tar -f codeql-bundle-${{matrix.platform}}.tar --delete codeql/python --delete ql/python - gzip codeql-bundle-${{matrix.platform}}.tar - mv codeql-bundle-${{matrix.platform}}.tar.gz codeql-bundle-${{matrix.platform}}-${{matrix.language}}.tar.gz - du -sh codeql-bundle-${{matrix.platform}}-${{matrix.language}}.tar.gz - - - name: Upload Language Package - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ steps.save_url.outputs.upload_url }} - asset_path: ./codeql-bundle-${{matrix.platform}}-${{matrix.language}}.tar.gz - asset_name: codeql-bundle-${{matrix.platform}}-${{matrix.language}}.tar.gz - asset_content_type: application/tar+gzip From d265935d24d797381b16d2999b4b029ad2d40060 Mon Sep 17 00:00:00 2001 From: Marco Gario Date: Mon, 21 Sep 2020 12:00:29 +0200 Subject: [PATCH 5/6] Update split.yml --- .github/workflows/split.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/split.yml b/.github/workflows/split.yml index 61795c4b6..6ee8d56a9 100644 --- a/.github/workflows/split.yml +++ b/.github/workflows/split.yml @@ -1,5 +1,5 @@ # -# Split the CodeQL Bundle into language-platform bundles +# Split the CodeQL Bundle into platform bundles # # Instructions: # 1. Upload the new codeql-bundle (codeql-bundle.tar.gz) as an asset of the From 590fdcd891632efcd4c8384f191147dc7beb3cc1 Mon Sep 17 00:00:00 2001 From: Marco Gario Date: Mon, 21 Sep 2020 18:12:23 +0200 Subject: [PATCH 6/6] Apply suggestions from code review Co-authored-by: Robert --- .github/workflows/split.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/split.yml b/.github/workflows/split.yml index 6ee8d56a9..547ac1fba 100644 --- a/.github/workflows/split.yml +++ b/.github/workflows/split.yml @@ -7,8 +7,8 @@ # 2. Take note of the CLI Release used by the bundle (e.g., v2.2.5) # 3. Manually launch this workflow file (via the Actions UI) specifying # - The CLI Release (e.g., v2.2.5) -# - The release tag (codeql-bundle-20200826) -# 4. If everything succeeds you should 3 new assets. +# - The release tag (e.g., codeql-bundle-20200826) +# 4. If everything succeeds you should see 3 new assets. # name: Split Bundle