From 603c47cb50a545877c567584abcd68622ae0b9f3 Mon Sep 17 00:00:00 2001 From: Robert Date: Fri, 7 May 2021 09:49:11 +0100 Subject: [PATCH 1/3] make matrix run all cases --- .github/workflows/python-deps.yml | 105 ++++++++++++++++-------------- 1 file changed, 57 insertions(+), 48 deletions(-) diff --git a/.github/workflows/python-deps.yml b/.github/workflows/python-deps.yml index 4bc27abef..545f3ebd8 100644 --- a/.github/workflows/python-deps.yml +++ b/.github/workflows/python-deps.yml @@ -6,37 +6,14 @@ on: pull_request: jobs: - test-setup-python-scripts: runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: os: [ubuntu-latest, macos-latest] - include: - - test_dir: python-setup/tests/pipenv/requests-2 - test_script: $GITHUB_WORKSPACE/python-setup/tests/check_requests_123.sh 2 - - test_dir: python-setup/tests/pipenv/requests-3 - test_script: $GITHUB_WORKSPACE/python-setup/tests/check_requests_123.sh 3 - - - test_dir: python-setup/tests/poetry/requests-2 - test_script: $GITHUB_WORKSPACE/python-setup/tests/check_requests_123.sh 2 - - test_dir: python-setup/tests/poetry/requests-3 - test_script: $GITHUB_WORKSPACE/python-setup/tests/check_requests_123.sh 3 - - - test_dir: python-setup/tests/requirements/requests-2 - test_script: $GITHUB_WORKSPACE/python-setup/tests/check_requests_123.sh 2 - - test_dir: python-setup/tests/requirements/requests-3 - test_script: $GITHUB_WORKSPACE/python-setup/tests/check_requests_123.sh 3 - - - test_dir: python-setup/tests/setup_py/requests-2 - test_script: $GITHUB_WORKSPACE/python-setup/tests/check_requests_123.sh 2 - - test_dir: python-setup/tests/setup_py/requests-3 - test_script: $GITHUB_WORKSPACE/python-setup/tests/check_requests_123.sh 3 - - # This one shouldn't fail, but also won't install packages - - test_dir: python-setup/tests/requirements/non-standard-location - test_script: test -z $LGTM_INDEX_IMPORT_PATH + python_deps_type: [pipenv, poetry, requirements, setup_py] + python_version: [2, 3] steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it @@ -55,7 +32,7 @@ jobs: set -x $GITHUB_WORKSPACE/python-setup/install_tools.sh - cd $GITHUB_WORKSPACE/${{ matrix.test_dir }} + cd $GITHUB_WORKSPACE/python-setup/tests/${{ matrix.python_deps_type }}/requests-${{ matrix.python_version }} case ${{ matrix.os }} in ubuntu-latest*) basePath="/opt";; @@ -71,35 +48,65 @@ jobs: if [ ! -z $CODEQL_PYTHON ]; then $GITHUB_WORKSPACE/python-setup/tests/from_python_exe.py $CODEQL_PYTHON; fi + - name: Verify packages installed run: | - ${{ matrix.test_script }} + $GITHUB_WORKSPACE/python-setup/tests/check_requests_123.sh ${{ matrix.python_version }} + + # This one shouldn't fail, but also won't install packages + test-setup-python-scripts-non-standard-location: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v2 + + - name: Initialize CodeQL + uses: ./init + id: init + with: + tools: latest + languages: python + setup-python-dependencies: false + + - name: Test Auto Package Installation + run: | + set -x + $GITHUB_WORKSPACE/python-setup/install_tools.sh + + cd $GITHUB_WORKSPACE/python-setup/tests/requirements/non-standard-location + + case ${{ matrix.os }} in + ubuntu-latest*) basePath="/opt";; + macos-latest*) basePath="/Users/runner";; + esac + echo ${basePath} + + $GITHUB_WORKSPACE/python-setup/auto_install_packages.py "$(dirname ${{steps.init.outputs.codeql-path}})" + + - name: Setup for extractor + run: | + echo $CODEQL_PYTHON + # only run if $CODEQL_PYTHON is set + if [ ! -z $CODEQL_PYTHON ]; then + $GITHUB_WORKSPACE/python-setup/tests/from_python_exe.py $CODEQL_PYTHON; + fi + + - name: Verify packages installed + run: | + test -z $LGTM_INDEX_IMPORT_PATH test-setup-python-scripts-windows: runs-on: windows-latest strategy: fail-fast: false matrix: - include: - - test_dir: python-setup/tests/pipenv/requests-2 - python_version: 2 - - test_dir: python-setup/tests/pipenv/requests-3 - python_version: 3 - - - test_dir: python-setup/tests/poetry/requests-2 - python_version: 2 - - test_dir: python-setup/tests/poetry/requests-3 - python_version: 3 - - - test_dir: python-setup/tests/requirements/requests-2 - python_version: 2 - - test_dir: python-setup/tests/requirements/requests-3 - python_version: 3 - - - test_dir: python-setup/tests/setup_py/requests-2 - python_version: 2 - - test_dir: python-setup/tests/setup_py/requests-3 - python_version: 3 + python_deps_type: [pipenv, poetry, requirements, setup_py] + python_version: [2, 3] steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it @@ -117,16 +124,18 @@ jobs: $cmd = $Env:GITHUB_WORKSPACE + "\\python-setup\\install_tools.ps1" powershell -File $cmd - cd $Env:GITHUB_WORKSPACE\\${{ matrix.test_dir }} + cd $Env:GITHUB_WORKSPACE\\python-setup/tests/${{ matrix.python_deps_type }}/requests-${{ matrix.python_version }} $DefaultsPath = Join-Path (Join-Path $Env:GITHUB_WORKSPACE "src") "defaults.json" $CodeQLBundleName = (Get-Content -Raw -Path $DefaultsPath | ConvertFrom-Json).bundleVersion $CodeQLVersion = "0.0.0-" + $CodeQLBundleName.split("-")[-1] py -3 $Env:GITHUB_WORKSPACE\\python-setup\\auto_install_packages.py C:\\hostedtoolcache\\windows\\CodeQL\\$CodeQLVersion\\x64\\codeql + - name: Setup for extractor run: | echo $Env:CODEQL_PYTHON py -3 $Env:GITHUB_WORKSPACE\\python-setup\\tests\\from_python_exe.py $Env:CODEQL_PYTHON + - name: Verify packages installed run: | $cmd = $Env:GITHUB_WORKSPACE + "\\python-setup\\tests\\check_requests_123.ps1" From f742f910aa0a7916f033b2dee06eba3658e2054c Mon Sep 17 00:00:00 2001 From: Robert Date: Mon, 10 May 2021 16:19:38 +0100 Subject: [PATCH 2/3] put matrix vars in env vars for whitespace safety --- .github/workflows/python-deps.yml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python-deps.yml b/.github/workflows/python-deps.yml index 545f3ebd8..2f9e0c990 100644 --- a/.github/workflows/python-deps.yml +++ b/.github/workflows/python-deps.yml @@ -15,6 +15,10 @@ jobs: python_deps_type: [pipenv, poetry, requirements, setup_py] python_version: [2, 3] + env: + PYTHON_DEPS_TYPE: ${{ matrix.python_deps_type }} + PYTHON_VERSION: ${{ matrix.python_version }} + steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v2 @@ -32,7 +36,7 @@ jobs: set -x $GITHUB_WORKSPACE/python-setup/install_tools.sh - cd $GITHUB_WORKSPACE/python-setup/tests/${{ matrix.python_deps_type }}/requests-${{ matrix.python_version }} + cd $GITHUB_WORKSPACE/python-setup/tests/${PYTHON_DEPS_TYPE}/requests-${PYTHON_VERSION} case ${{ matrix.os }} in ubuntu-latest*) basePath="/opt";; @@ -51,7 +55,7 @@ jobs: - name: Verify packages installed run: | - $GITHUB_WORKSPACE/python-setup/tests/check_requests_123.sh ${{ matrix.python_version }} + $GITHUB_WORKSPACE/python-setup/tests/check_requests_123.sh ${PYTHON_VERSION} # This one shouldn't fail, but also won't install packages test-setup-python-scripts-non-standard-location: @@ -108,6 +112,10 @@ jobs: python_deps_type: [pipenv, poetry, requirements, setup_py] python_version: [2, 3] + env: + PYTHON_DEPS_TYPE: ${{ matrix.python_deps_type }} + PYTHON_VERSION: ${{ matrix.python_version }} + steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v2 @@ -124,7 +132,7 @@ jobs: $cmd = $Env:GITHUB_WORKSPACE + "\\python-setup\\install_tools.ps1" powershell -File $cmd - cd $Env:GITHUB_WORKSPACE\\python-setup/tests/${{ matrix.python_deps_type }}/requests-${{ matrix.python_version }} + cd $Env:GITHUB_WORKSPACE\\python-setup/tests/${PYTHON_DEPS_TYPE}/requests-${PYTHON_VERSION} $DefaultsPath = Join-Path (Join-Path $Env:GITHUB_WORKSPACE "src") "defaults.json" $CodeQLBundleName = (Get-Content -Raw -Path $DefaultsPath | ConvertFrom-Json).bundleVersion $CodeQLVersion = "0.0.0-" + $CodeQLBundleName.split("-")[-1] @@ -139,4 +147,4 @@ jobs: - name: Verify packages installed run: | $cmd = $Env:GITHUB_WORKSPACE + "\\python-setup\\tests\\check_requests_123.ps1" - powershell -File $cmd ${{ matrix.python_version }} + powershell -File $cmd ${PYTHON_VERSION} From 3c960195561770de43b9979f4c6ca3a0dc7cf6d0 Mon Sep 17 00:00:00 2001 From: Robert Date: Mon, 10 May 2021 16:24:52 +0100 Subject: [PATCH 3/3] fix windows hopefully --- .github/workflows/python-deps.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-deps.yml b/.github/workflows/python-deps.yml index 2f9e0c990..63fbafb89 100644 --- a/.github/workflows/python-deps.yml +++ b/.github/workflows/python-deps.yml @@ -132,7 +132,7 @@ jobs: $cmd = $Env:GITHUB_WORKSPACE + "\\python-setup\\install_tools.ps1" powershell -File $cmd - cd $Env:GITHUB_WORKSPACE\\python-setup/tests/${PYTHON_DEPS_TYPE}/requests-${PYTHON_VERSION} + cd $Env:GITHUB_WORKSPACE\\python-setup/tests/$Env:PYTHON_DEPS_TYPE/requests-$Env:PYTHON_VERSION $DefaultsPath = Join-Path (Join-Path $Env:GITHUB_WORKSPACE "src") "defaults.json" $CodeQLBundleName = (Get-Content -Raw -Path $DefaultsPath | ConvertFrom-Json).bundleVersion $CodeQLVersion = "0.0.0-" + $CodeQLBundleName.split("-")[-1] @@ -147,4 +147,4 @@ jobs: - name: Verify packages installed run: | $cmd = $Env:GITHUB_WORKSPACE + "\\python-setup\\tests\\check_requests_123.ps1" - powershell -File $cmd ${PYTHON_VERSION} + powershell -File $cmd $Env:PYTHON_VERSION