From 74afd3c373a0ea4d5580cf14a2fab5e9eae05800 Mon Sep 17 00:00:00 2001 From: David Verdeguer Date: Tue, 6 Oct 2020 14:14:12 +0200 Subject: [PATCH 01/16] Port python deps setup to windows --- ...{python-deps.yml => python-deps-linux.yml} | 2 +- .github/workflows/python-deps-windows.yml | 60 +++++++ python-setup/auto_install_packages_windows.py | 150 ++++++++++++++++++ python-setup/install_tools.ps1 | 13 ++ python-setup/tests/check_requests_123.ps1 | 28 ++++ 5 files changed, 252 insertions(+), 1 deletion(-) rename .github/workflows/{python-deps.yml => python-deps-linux.yml} (98%) create mode 100644 .github/workflows/python-deps-windows.yml create mode 100644 python-setup/auto_install_packages_windows.py create mode 100644 python-setup/install_tools.ps1 create mode 100644 python-setup/tests/check_requests_123.ps1 diff --git a/.github/workflows/python-deps.yml b/.github/workflows/python-deps-linux.yml similarity index 98% rename from .github/workflows/python-deps.yml rename to .github/workflows/python-deps-linux.yml index 75be193b3..0ec27f7e7 100644 --- a/.github/workflows/python-deps.yml +++ b/.github/workflows/python-deps-linux.yml @@ -1,4 +1,4 @@ -name: Test Python Package Installation +name: Test Python Package Installation on Linux on: push: diff --git a/.github/workflows/python-deps-windows.yml b/.github/workflows/python-deps-windows.yml new file mode 100644 index 000000000..1df21a52b --- /dev/null +++ b/.github/workflows/python-deps-windows.yml @@ -0,0 +1,60 @@ +name: Test Python Package Installation on Windows + + +on: + push: + pull_request: + +jobs: + + test-setup-python-scripts: + runs-on: windows-latest + strategy: + fail-fast: false + matrix: + include: + - test_dir: examples/pipenv/requests-2 + python_version: 2 + - test_dir: examples/pipenv/requests-3 + python_version: 3 + + - test_dir: examples/poetry/requests-2 + python_version: 2 + - test_dir: examples/poetry/requests-3 + python_version: 3 + + - test_dir: examples/requirements/requests-2 + python_version: 2 + - test_dir: examples/requirements/requests-3 + python_version: 3 + + - test_dir: examples/setup_py/requests-2 + python_version: 2 + - test_dir: examples/setup_py/requests-3 + python_version: 3 + + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v2 + + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: python + + - name: Test Auto Package Installation + run: | + $cmd = $Env:GITHUB_WORKSPACE + "\\python-setup\\install_tools.ps1" + powershell -File $cmd + + cd $Env:GITHUB_WORKSPACE\\${{ matrix.test_dir }} + py -3 $Env:GITHUB_WORKSPACE\\python-setup\\auto_install_packages_windows.py C:\\hostedtoolcache\\windows\\CodeQL\\0.0.0-20200826\\x64\\codeql + - name: Setup for extractor + run: | + echo $Env:CODEQL_PYTHON + + py -3 $Env:GITHUB_WORKSPACE\\python-setup\\tests\\from_python_exe.py $Env:CODEQL_PYTHON + - name: Verify packages installed + run: | + $cmd = $Env:GITHUB_WORKSPACE + "\\python-setup\\tests\\check_requests_123.ps1" + powershell -File $cmd ${{ matrix.python_version }} \ No newline at end of file diff --git a/python-setup/auto_install_packages_windows.py b/python-setup/auto_install_packages_windows.py new file mode 100644 index 000000000..a9252f8c6 --- /dev/null +++ b/python-setup/auto_install_packages_windows.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 + +import sys +import os +import subprocess +from tempfile import mkdtemp + +import extractor_version + + +def _check_call(command): + print('+ {}'.format(' '.join(command)), flush=True) + subprocess.check_call(command, stdin=subprocess.DEVNULL) + + +def _check_output(command): + print('+ {}'.format(' '.join(command)), flush=True) + out = subprocess.check_output(command, stdin=subprocess.DEVNULL) + print(out, flush=True) + sys.stderr.flush() + return out + + +def install_packages_with_poetry(): + os.environ['POETRY_VIRTUALENVS_PATH'] = os.environ['RUNNER_WORKSPACE'] + '\\virtualenvs' + try: + _check_call(['poetry', 'install', '--no-root']) + except subprocess.CalledProcessError: + sys.exit('package installation with poetry failed, see error above') + + # poetry is super annoying with `poetry run`, since it will put lots of output on + # STDOUT if the current global python interpreter is not matching the one in the + # virtualenv for the package, which was the case for using poetry for Python 2 when + # default system interpreter was Python 3 :/ + + poetry_out = _check_output(['poetry', 'run', 'which', 'python']) + python_executable_path = poetry_out.decode('utf-8').splitlines()[-1] + + return python_executable_path[2:] + + +def install_packages_with_pipenv(): + os.environ['WORKON_HOME'] = os.environ['RUNNER_WORKSPACE'] + '\\virtualenvs' + try: + _check_call(['pipenv', 'install', '--keep-outdated', '--ignore-pipfile']) + except subprocess.CalledProcessError: + sys.exit('package installation with pipenv failed, see error above') + + pipenv_out = _check_output(['pipenv', 'run', 'which', 'python']) + python_executable_path = pipenv_out.decode('utf-8').splitlines()[-1] + + return python_executable_path[2:] + + +def _create_venv(version: int): + # create temporary directory ... that just lives "forever" + venv_path = os.environ['RUNNER_WORKSPACE']+'/codeql-action-python-autoinstall' + print ("Creating venv in "+venv_path, flush = True) + + # virtualenv is a bit nicer for setting up virtual environment, since it will provide + # up-to-date versions of pip/setuptools/wheel which basic `python3 -m venv venv` won't + + if version == 2: + _check_call(['py', '-2', '-m', 'virtualenv', venv_path]) + elif version == 3: + _check_call(['py', '-3', '-m', 'virtualenv', venv_path]) + + return venv_path + + +def install_requirements_txt_packages(version: int): + venv_path = _create_venv(version) + venv_pip = os.path.join(venv_path, 'Scripts', 'pip') + venv_python = os.path.join(venv_path, 'Scripts', 'python') + + try: + _check_call([venv_pip, 'install', '-r', 'requirements.txt']) + except subprocess.CalledProcessError: + sys.exit('package installation with `pip install -r requirements.txt` failed, see error above') + + return venv_python + + +def install_with_setup_py(version: int): + venv_path = _create_venv(version) + venv_pip = os.path.join(venv_path, 'Scripts', 'pip') + venv_python = os.path.join(venv_path, 'Scripts', 'python') + + try: + # We have to choose between `python setup.py develop` and `pip install -e .`. + # Modern projects use `pip install -e .` and I wasn't able to see any downsides + # to doing so. However, `python setup.py develop` has some downsides -- from + # https://stackoverflow.com/a/19048754 : + # > Note that it is highly recommended to use pip install . (install) and pip + # > install -e . (developer install) to install packages, as invoking setup.py + # > directly will do the wrong things for many dependencies, such as pull + # > prereleases and incompatible package versions, or make the package hard to + # > uninstall with pip. + + _check_call([venv_pip, 'install', '-e', '.']) + except subprocess.CalledProcessError: + sys.exit('package installation with `pip install -e .` failed, see error above') + + return venv_python + + +def install_packages() -> str: + if os.path.exists('poetry.lock'): + print('Found poetry.lock, will install packages with poetry', flush=True) + return install_packages_with_poetry() + + if os.path.exists('Pipfile') or os.path.exists('Pipfile.lock'): + if os.path.exists('Pipfile.lock'): + print('Found Pipfile.lock, will install packages with Pipenv', flush=True) + else: + print('Found Pipfile, will install packages with Pipenv', flush=True) + return install_packages_with_pipenv() + + version = extractor_version.get_extractor_version(sys.argv[1], quiet=False) + + if os.path.exists('requirements.txt'): + print('Found requirements.txt, will install packages with pip', flush=True) + return install_requirements_txt_packages(version) + + if os.path.exists('setup.py'): + print('Found setup.py, will install package with pip in editable mode', flush=True) + return install_with_setup_py(version) + + print("was not able to install packages automatically", flush=True) + + +if __name__ == "__main__": + if len(sys.argv) != 2: + sys.exit('Must provide base directory for codeql tool as only argument') + + # The binaries for packages installed with `pip install --user` are not available on + # PATH by default, so we need to manually add them. + os.environ['PATH'] = os.path.expandvars('%APPDATA%\Python\\Python38\\scripts') + os.pathsep + os.environ['PATH'] + + python_executable_path = install_packages() + + if python_executable_path is not None: + print("Setting CODEQL_PYTHON={}".format(python_executable_path)) + print("::set-env name=CODEQL_PYTHON::{}".format(python_executable_path)) + +# TODO: +# - no packages +# - poetry without version +# - pipenv without version +# - pipenv without lockfile diff --git a/python-setup/install_tools.ps1 b/python-setup/install_tools.ps1 new file mode 100644 index 000000000..6f03b435b --- /dev/null +++ b/python-setup/install_tools.ps1 @@ -0,0 +1,13 @@ +#! /usr/bin/pwsh + +py -2 -m pip install --user --upgrade pip setuptools wheel +py -3 -m pip install --user --upgrade pip setuptools wheel + +# virtualenv is a bit nicer for setting up virtual environment, since it will provide up-to-date versions of +# pip/setuptools/wheel which basic `python3 -m venv venv` won't +py -2 -m pip install --user virtualenv +py -3 -m pip install --user virtualenv + +# poetry 1.0.10 has error (https://github.com/python-poetry/poetry/issues/2711) +py -3 -m pip install --user poetry!=1.0.10 +py -3 -m pip install --user pipenv \ No newline at end of file diff --git a/python-setup/tests/check_requests_123.ps1 b/python-setup/tests/check_requests_123.ps1 new file mode 100644 index 000000000..5e4ef458b --- /dev/null +++ b/python-setup/tests/check_requests_123.ps1 @@ -0,0 +1,28 @@ +#! /usr/bin/pwsh + +$EXPECTED_VERSION=$args[0] + +$FOUND_VERSION="$Env:LGTM_PYTHON_SETUP_VERSION" +$FOUND_PYTHONPATH="$Env:LGTM_INDEX_IMPORT_PATH" + +write-host "FOUND_VERSION=$FOUND_VERSION FOUND_PYTHONPATH=$FOUND_PYTHONPATH " + +if ($FOUND_VERSION -ne $EXPECTED_VERSION) { + write-host "Script told us to use Python $FOUND_VERSION, but expected $EXPECTED_VERSION" + exit 1 +} else { + write-host "Script told us to use Python $FOUND_VERSION, which was expected" +} + +$env:PYTHONPATH=$FOUND_PYTHONPATH + +$INSTALLED_REQUESTS_VERSION = (py -3 -c "import requests; print(requests.__version__)") + +$EXPECTED_REQUESTS="1.2.3" + +if ($INSTALLED_REQUESTS_VERSION -ne $EXPECTED_REQUESTS) { + write-host "Using $FOUND_PYTHONPATH as PYTHONPATH, we found version $INSTALLED_REQUESTS_VERSION of requests, but expected $EXPECTED_REQUESTS" + exit 1 +} else { + write-host "Using $FOUND_PYTHONPATH as PYTHONPATH, we found version $INSTALLED_REQUESTS_VERSION of requests, which was expected" +} \ No newline at end of file From 2748e6c5c13b6d4228b28721e0ae307df0b68f91 Mon Sep 17 00:00:00 2001 From: David Verdeguer Date: Tue, 6 Oct 2020 14:21:10 +0200 Subject: [PATCH 02/16] Update python-deps-windows test_dir paths --- .github/workflows/python-deps-windows.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/python-deps-windows.yml b/.github/workflows/python-deps-windows.yml index 1df21a52b..fd1b5034a 100644 --- a/.github/workflows/python-deps-windows.yml +++ b/.github/workflows/python-deps-windows.yml @@ -13,24 +13,24 @@ jobs: fail-fast: false matrix: include: - - test_dir: examples/pipenv/requests-2 + - test_dir: python-setup/tests/pipenv/requests-2 python_version: 2 - - test_dir: examples/pipenv/requests-3 + - test_dir: python-setup/tests/pipenv/requests-3 python_version: 3 - - test_dir: examples/poetry/requests-2 + - test_dir: python-setup/tests/poetry/requests-2 python_version: 2 - - test_dir: examples/poetry/requests-3 + - test_dir: python-setup/tests/poetry/requests-3 python_version: 3 - - test_dir: examples/requirements/requests-2 + - test_dir: python-setup/tests/requirements/requests-2 python_version: 2 - - test_dir: examples/requirements/requests-3 + - test_dir: python-setup/tests/requirements/requests-3 python_version: 3 - - test_dir: examples/setup_py/requests-2 + - test_dir: python-setup/tests/setup_py/requests-2 python_version: 2 - - test_dir: examples/setup_py/requests-3 + - test_dir: python-setup/tests/setup_py/requests-3 python_version: 3 steps: From 0f2fa46e3c2828aeed492227c931752a43b62c30 Mon Sep 17 00:00:00 2001 From: David Verdeguer Date: Tue, 6 Oct 2020 14:37:35 +0200 Subject: [PATCH 03/16] Update installPythonDeps for windows --- lib/init.js | 14 ++++++++++---- lib/init.js.map | 2 +- src/init.ts | 16 ++++++++++++---- 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/lib/init.js b/lib/init.js index bb0e00503..d4b13fe31 100644 --- a/lib/init.js +++ b/lib/init.js @@ -123,16 +123,22 @@ async function injectWindowsTracer(processName, processLevel, config, codeql, tr exports.injectWindowsTracer = injectWindowsTracer; async function installPythonDeps(codeql, logger) { logger.startGroup("Setup Python dependencies"); - if (process.platform !== "linux") { - logger.info("Currently, auto-installing python dependancies is only supported on linux"); + if (process.platform === "darwin") { + logger.info("Currently, auto-installing python dependancies is not supported on MacOS"); logger.endGroup(); return; } + let install_tools_script = "install_tools.sh"; + let auto_install_packages_script = "auto_install_packages.py"; + if (process.platform === "win32") { + install_tools_script = "install_tools.ps1"; + auto_install_packages_script = "auto_install_packages_windows.py"; + } const scriptsFolder = path.resolve(__dirname, "../python-setup"); // Setup tools on the Github hosted runners if (process.env["ImageOS"] !== undefined) { try { - await new toolrunnner.ToolRunner(path.join(scriptsFolder, "install_tools.sh")).exec(); + await new toolrunnner.ToolRunner(path.join(scriptsFolder, install_tools_script)).exec(); } catch (e) { // This script tries to install some needed tools in the runner. It should not fail, but if it does @@ -143,7 +149,7 @@ async function installPythonDeps(codeql, logger) { } // Install dependencies try { - await new toolrunnner.ToolRunner(path.join(scriptsFolder, "auto_install_packages.py"), [path.dirname(codeql.getPath())]).exec(); + await new toolrunnner.ToolRunner(path.join(scriptsFolder, auto_install_packages_script), [path.dirname(codeql.getPath())]).exec(); } catch (e) { logger.endGroup(); diff --git a/lib/init.js.map b/lib/init.js.map index dd4458cb4..68aaa5736 100644 --- a/lib/init.js.map +++ b/lib/init.js.map @@ -1 +1 @@ -{"version":3,"file":"init.js","sourceRoot":"","sources":["../src/init.ts"],"names":[],"mappings":";;;;;;;;;AAAA,uCAAyB;AACzB,2CAA6B;AAE7B,0EAA4D;AAE5D,gEAAkD;AAClD,qCAA+C;AAC/C,4DAA8C;AAG9C,mDAAwE;AACxE,6CAA+B;AAExB,KAAK,UAAU,UAAU,CAC9B,SAA6B,EAC7B,UAAkB,EAClB,SAAiB,EACjB,OAAe,EACf,QAAgB,EAChB,IAAe,EACf,MAAc;IAEd,MAAM,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,MAAM,oBAAW,CAC9B,SAAS,EACT,UAAU,EACV,SAAS,EACT,OAAO,EACP,QAAQ,EACR,IAAI,EACJ,MAAM,CACP,CAAC;IACF,MAAM,MAAM,CAAC,YAAY,EAAE,CAAC;IAC5B,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO,MAAM,CAAC;AAChB,CAAC;AAtBD,gCAsBC;AAEM,KAAK,UAAU,UAAU,CAC9B,cAAkC,EAClC,YAAgC,EAChC,UAA8B,EAC9B,UAAyB,EACzB,OAAe,EACf,YAAoB,EACpB,MAAc,EACd,YAAoB,EACpB,UAAkB,EAClB,SAAiB,EACjB,MAAc;IAEd,MAAM,CAAC,UAAU,CAAC,6BAA6B,CAAC,CAAC;IACjD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,UAAU,CACzC,cAAc,EACd,YAAY,EACZ,UAAU,EACV,UAAU,EACV,OAAO,EACP,YAAY,EACZ,MAAM,EACN,YAAY,EACZ,UAAU,EACV,SAAS,EACT,MAAM,CACP,CAAC;IACF,aAAa,CAAC,uBAAuB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtD,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO,MAAM,CAAC;AAChB,CAAC;AA9BD,gCA8BC;AAEM,KAAK,UAAU,OAAO,CAC3B,MAAc,EACd,MAA0B;IAE1B,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;IAElC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE9E,sEAAsE;IACtE,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACvC,yBAAyB;QACzB,MAAM,MAAM,CAAC,YAAY,CACvB,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,EACpD,QAAQ,EACR,UAAU,CACX,CAAC;KACH;IAED,OAAO,MAAM,uCAAuB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AACvD,CAAC;AAnBD,0BAmBC;AAED,sEAAsE;AACtE,4EAA4E;AAC5E,4EAA4E;AAC5E,6EAA6E;AAC7E,+CAA+C;AACxC,KAAK,UAAU,mBAAmB,CACvC,WAA+B,EAC/B,YAAgC,EAChC,MAA0B,EAC1B,MAAc,EACd,YAA0B;IAE1B,IAAI,MAAc,CAAC;IACnB,IAAI,WAAW,KAAK,SAAS,EAAE;QAC7B,MAAM,GAAG;;;;;;;;;;;;uCAY0B,WAAW;;8BAEpB,WAAW;;;;;;;;gDAQO,CAAC;KAC9C;SAAM;QACL,oEAAoE;QACpE,mFAAmF;QACnF,+EAA+E;QAC/E,kFAAkF;QAClF,6EAA6E;QAC7E,oFAAoF;QACpF,6CAA6C;QAC7C,YAAY,GAAG,YAAY,IAAI,CAAC,CAAC;QACjC,MAAM,GAAG;;;;;;;;4BAQe,YAAY;;;;;;;;;;;;;;;;;gDAiBQ,CAAC;KAC9C;IAED,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,mBAAmB,CAAC,CAAC;IACxE,EAAE,CAAC,aAAa,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC;IAE3C,MAAM,IAAI,WAAW,CAAC,UAAU,CAC9B,YAAY,EACZ;QACE,kBAAkB;QAClB,QAAQ;QACR,OAAO;QACP,gBAAgB;QAChB,IAAI,CAAC,OAAO,CACV,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,EAC9B,OAAO,EACP,OAAO,EACP,YAAY,CACb;KACF,EACD,EAAE,GAAG,EAAE,EAAE,0BAA0B,EAAE,YAAY,CAAC,IAAI,EAAE,EAAE,CAC3D,CAAC,IAAI,EAAE,CAAC;AACX,CAAC;AAxFD,kDAwFC;AAEM,KAAK,UAAU,iBAAiB,CAAC,MAAc,EAAE,MAAc;IACpE,MAAM,CAAC,UAAU,CAAC,2BAA2B,CAAC,CAAC;IAE/C,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE;QAChC,MAAM,CAAC,IAAI,CACT,2EAA2E,CAC5E,CAAC;QACF,MAAM,CAAC,QAAQ,EAAE,CAAC;QAClB,OAAO;KACR;IAED,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,iBAAiB,CAAC,CAAC;IAEjE,2CAA2C;IAC3C,IAAI,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,SAAS,EAAE;QACxC,IAAI;YACF,MAAM,IAAI,WAAW,CAAC,UAAU,CAC9B,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,kBAAkB,CAAC,CAC7C,CAAC,IAAI,EAAE,CAAC;SACV;QAAC,OAAO,CAAC,EAAE;YACV,mGAAmG;YACnG,uDAAuD;YACvD,MAAM,CAAC,QAAQ,EAAE,CAAC;YAClB,MAAM,CAAC,OAAO,CACZ,kLAAkL,CACnL,CAAC;SACH;KACF;IAED,uBAAuB;IACvB,IAAI;QACF,MAAM,IAAI,WAAW,CAAC,UAAU,CAC9B,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,0BAA0B,CAAC,EACpD,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,CACjC,CAAC,IAAI,EAAE,CAAC;KACV;IAAC,OAAO,CAAC,EAAE;QACV,MAAM,CAAC,QAAQ,EAAE,CAAC;QAClB,MAAM,CAAC,OAAO,CACZ,+IAA+I,CAChJ,CAAC;KACH;IACD,MAAM,CAAC,QAAQ,EAAE,CAAC;AACpB,CAAC;AA1CD,8CA0CC"} \ No newline at end of file +{"version":3,"file":"init.js","sourceRoot":"","sources":["../src/init.ts"],"names":[],"mappings":";;;;;;;;;AAAA,uCAAyB;AACzB,2CAA6B;AAE7B,0EAA4D;AAE5D,gEAAkD;AAClD,qCAA+C;AAC/C,4DAA8C;AAG9C,mDAAwE;AACxE,6CAA+B;AAExB,KAAK,UAAU,UAAU,CAC9B,SAA6B,EAC7B,UAAkB,EAClB,SAAiB,EACjB,OAAe,EACf,QAAgB,EAChB,IAAe,EACf,MAAc;IAEd,MAAM,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,MAAM,oBAAW,CAC9B,SAAS,EACT,UAAU,EACV,SAAS,EACT,OAAO,EACP,QAAQ,EACR,IAAI,EACJ,MAAM,CACP,CAAC;IACF,MAAM,MAAM,CAAC,YAAY,EAAE,CAAC;IAC5B,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO,MAAM,CAAC;AAChB,CAAC;AAtBD,gCAsBC;AAEM,KAAK,UAAU,UAAU,CAC9B,cAAkC,EAClC,YAAgC,EAChC,UAA8B,EAC9B,UAAyB,EACzB,OAAe,EACf,YAAoB,EACpB,MAAc,EACd,YAAoB,EACpB,UAAkB,EAClB,SAAiB,EACjB,MAAc;IAEd,MAAM,CAAC,UAAU,CAAC,6BAA6B,CAAC,CAAC;IACjD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,UAAU,CACzC,cAAc,EACd,YAAY,EACZ,UAAU,EACV,UAAU,EACV,OAAO,EACP,YAAY,EACZ,MAAM,EACN,YAAY,EACZ,UAAU,EACV,SAAS,EACT,MAAM,CACP,CAAC;IACF,aAAa,CAAC,uBAAuB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtD,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO,MAAM,CAAC;AAChB,CAAC;AA9BD,gCA8BC;AAEM,KAAK,UAAU,OAAO,CAC3B,MAAc,EACd,MAA0B;IAE1B,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;IAElC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE9E,sEAAsE;IACtE,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACvC,yBAAyB;QACzB,MAAM,MAAM,CAAC,YAAY,CACvB,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,EACpD,QAAQ,EACR,UAAU,CACX,CAAC;KACH;IAED,OAAO,MAAM,uCAAuB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AACvD,CAAC;AAnBD,0BAmBC;AAED,sEAAsE;AACtE,4EAA4E;AAC5E,4EAA4E;AAC5E,6EAA6E;AAC7E,+CAA+C;AACxC,KAAK,UAAU,mBAAmB,CACvC,WAA+B,EAC/B,YAAgC,EAChC,MAA0B,EAC1B,MAAc,EACd,YAA0B;IAE1B,IAAI,MAAc,CAAC;IACnB,IAAI,WAAW,KAAK,SAAS,EAAE;QAC7B,MAAM,GAAG;;;;;;;;;;;;uCAY0B,WAAW;;8BAEpB,WAAW;;;;;;;;gDAQO,CAAC;KAC9C;SAAM;QACL,oEAAoE;QACpE,mFAAmF;QACnF,+EAA+E;QAC/E,kFAAkF;QAClF,6EAA6E;QAC7E,oFAAoF;QACpF,6CAA6C;QAC7C,YAAY,GAAG,YAAY,IAAI,CAAC,CAAC;QACjC,MAAM,GAAG;;;;;;;;4BAQe,YAAY;;;;;;;;;;;;;;;;;gDAiBQ,CAAC;KAC9C;IAED,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,mBAAmB,CAAC,CAAC;IACxE,EAAE,CAAC,aAAa,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC;IAE3C,MAAM,IAAI,WAAW,CAAC,UAAU,CAC9B,YAAY,EACZ;QACE,kBAAkB;QAClB,QAAQ;QACR,OAAO;QACP,gBAAgB;QAChB,IAAI,CAAC,OAAO,CACV,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,EAC9B,OAAO,EACP,OAAO,EACP,YAAY,CACb;KACF,EACD,EAAE,GAAG,EAAE,EAAE,0BAA0B,EAAE,YAAY,CAAC,IAAI,EAAE,EAAE,CAC3D,CAAC,IAAI,EAAE,CAAC;AACX,CAAC;AAxFD,kDAwFC;AAEM,KAAK,UAAU,iBAAiB,CAAC,MAAc,EAAE,MAAc;IACpE,MAAM,CAAC,UAAU,CAAC,2BAA2B,CAAC,CAAC;IAE/C,IAAI,OAAO,CAAC,QAAQ,KAAK,QAAQ,EAAE;QACjC,MAAM,CAAC,IAAI,CACT,0EAA0E,CAC3E,CAAC;QACF,MAAM,CAAC,QAAQ,EAAE,CAAC;QAClB,OAAO;KACR;IAED,IAAI,oBAAoB,GAAG,kBAAkB,CAAC;IAC9C,IAAI,4BAA4B,GAAG,0BAA0B,CAAC;IAE9D,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE;QAChC,oBAAoB,GAAG,mBAAmB,CAAC;QAC3C,4BAA4B,GAAG,kCAAkC,CAAC;KACnE;IAED,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,iBAAiB,CAAC,CAAC;IAEjE,2CAA2C;IAC3C,IAAI,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,SAAS,EAAE;QACxC,IAAI;YACF,MAAM,IAAI,WAAW,CAAC,UAAU,CAC9B,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,oBAAoB,CAAC,CAC/C,CAAC,IAAI,EAAE,CAAC;SACV;QAAC,OAAO,CAAC,EAAE;YACV,mGAAmG;YACnG,uDAAuD;YACvD,MAAM,CAAC,QAAQ,EAAE,CAAC;YAClB,MAAM,CAAC,OAAO,CACZ,kLAAkL,CACnL,CAAC;SACH;KACF;IAED,uBAAuB;IACvB,IAAI;QACF,MAAM,IAAI,WAAW,CAAC,UAAU,CAC9B,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,4BAA4B,CAAC,EACtD,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,CACjC,CAAC,IAAI,EAAE,CAAC;KACV;IAAC,OAAO,CAAC,EAAE;QACV,MAAM,CAAC,QAAQ,EAAE,CAAC;QAClB,MAAM,CAAC,OAAO,CACZ,+IAA+I,CAChJ,CAAC;KACH;IACD,MAAM,CAAC,QAAQ,EAAE,CAAC;AACpB,CAAC;AAlDD,8CAkDC"} \ No newline at end of file diff --git a/src/init.ts b/src/init.ts index 6a738a090..e39c7dcd7 100644 --- a/src/init.ts +++ b/src/init.ts @@ -186,21 +186,29 @@ export async function injectWindowsTracer( export async function installPythonDeps(codeql: CodeQL, logger: Logger) { logger.startGroup("Setup Python dependencies"); - if (process.platform !== "linux") { + if (process.platform === "darwin") { logger.info( - "Currently, auto-installing python dependancies is only supported on linux" + "Currently, auto-installing python dependancies is not supported on MacOS" ); logger.endGroup(); return; } + let install_tools_script = "install_tools.sh"; + let auto_install_packages_script = "auto_install_packages.py"; + + if (process.platform === "win32") { + install_tools_script = "install_tools.ps1"; + auto_install_packages_script = "auto_install_packages_windows.py"; + } + const scriptsFolder = path.resolve(__dirname, "../python-setup"); // Setup tools on the Github hosted runners if (process.env["ImageOS"] !== undefined) { try { await new toolrunnner.ToolRunner( - path.join(scriptsFolder, "install_tools.sh") + path.join(scriptsFolder, install_tools_script) ).exec(); } catch (e) { // This script tries to install some needed tools in the runner. It should not fail, but if it does @@ -215,7 +223,7 @@ export async function installPythonDeps(codeql: CodeQL, logger: Logger) { // Install dependencies try { await new toolrunnner.ToolRunner( - path.join(scriptsFolder, "auto_install_packages.py"), + path.join(scriptsFolder, auto_install_packages_script), [path.dirname(codeql.getPath())] ).exec(); } catch (e) { From 526dac0f911deacba4f3b3b733c42f475450493b Mon Sep 17 00:00:00 2001 From: David Verdeguer Date: Tue, 6 Oct 2020 14:47:20 +0200 Subject: [PATCH 04/16] Only run tests on main and v1 --- .github/workflows/python-deps-windows.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python-deps-windows.yml b/.github/workflows/python-deps-windows.yml index fd1b5034a..00b373aa8 100644 --- a/.github/workflows/python-deps-windows.yml +++ b/.github/workflows/python-deps-windows.yml @@ -3,6 +3,7 @@ name: Test Python Package Installation on Windows on: push: + branches: [main, v1] pull_request: jobs: From e97bdbdfac13cf3e912fdc3c0a9db5ade8e55a9b Mon Sep 17 00:00:00 2001 From: David Verdeguer Date: Wed, 7 Oct 2020 11:58:55 +0200 Subject: [PATCH 05/16] Combine auto_install scripts --- .github/workflows/python-deps-windows.yml | 2 +- lib/init.js | 3 +- lib/init.js.map | 2 +- python-setup/auto_install_packages.py | 40 ++++- python-setup/auto_install_packages_windows.py | 150 ------------------ src/init.ts | 3 +- 6 files changed, 38 insertions(+), 162 deletions(-) delete mode 100644 python-setup/auto_install_packages_windows.py diff --git a/.github/workflows/python-deps-windows.yml b/.github/workflows/python-deps-windows.yml index 00b373aa8..5de085adb 100644 --- a/.github/workflows/python-deps-windows.yml +++ b/.github/workflows/python-deps-windows.yml @@ -49,7 +49,7 @@ jobs: powershell -File $cmd cd $Env:GITHUB_WORKSPACE\\${{ matrix.test_dir }} - py -3 $Env:GITHUB_WORKSPACE\\python-setup\\auto_install_packages_windows.py C:\\hostedtoolcache\\windows\\CodeQL\\0.0.0-20200826\\x64\\codeql + py -3 $Env:GITHUB_WORKSPACE\\python-setup\\auto_install_packages.py C:\\hostedtoolcache\\windows\\CodeQL\\0.0.0-20200826\\x64\\codeql - name: Setup for extractor run: | echo $Env:CODEQL_PYTHON diff --git a/lib/init.js b/lib/init.js index d4b13fe31..b63f7eaa2 100644 --- a/lib/init.js +++ b/lib/init.js @@ -129,10 +129,9 @@ async function installPythonDeps(codeql, logger) { return; } let install_tools_script = "install_tools.sh"; - let auto_install_packages_script = "auto_install_packages.py"; + const auto_install_packages_script = "auto_install_packages.py"; if (process.platform === "win32") { install_tools_script = "install_tools.ps1"; - auto_install_packages_script = "auto_install_packages_windows.py"; } const scriptsFolder = path.resolve(__dirname, "../python-setup"); // Setup tools on the Github hosted runners diff --git a/lib/init.js.map b/lib/init.js.map index 68aaa5736..8c055778b 100644 --- a/lib/init.js.map +++ b/lib/init.js.map @@ -1 +1 @@ -{"version":3,"file":"init.js","sourceRoot":"","sources":["../src/init.ts"],"names":[],"mappings":";;;;;;;;;AAAA,uCAAyB;AACzB,2CAA6B;AAE7B,0EAA4D;AAE5D,gEAAkD;AAClD,qCAA+C;AAC/C,4DAA8C;AAG9C,mDAAwE;AACxE,6CAA+B;AAExB,KAAK,UAAU,UAAU,CAC9B,SAA6B,EAC7B,UAAkB,EAClB,SAAiB,EACjB,OAAe,EACf,QAAgB,EAChB,IAAe,EACf,MAAc;IAEd,MAAM,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,MAAM,oBAAW,CAC9B,SAAS,EACT,UAAU,EACV,SAAS,EACT,OAAO,EACP,QAAQ,EACR,IAAI,EACJ,MAAM,CACP,CAAC;IACF,MAAM,MAAM,CAAC,YAAY,EAAE,CAAC;IAC5B,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO,MAAM,CAAC;AAChB,CAAC;AAtBD,gCAsBC;AAEM,KAAK,UAAU,UAAU,CAC9B,cAAkC,EAClC,YAAgC,EAChC,UAA8B,EAC9B,UAAyB,EACzB,OAAe,EACf,YAAoB,EACpB,MAAc,EACd,YAAoB,EACpB,UAAkB,EAClB,SAAiB,EACjB,MAAc;IAEd,MAAM,CAAC,UAAU,CAAC,6BAA6B,CAAC,CAAC;IACjD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,UAAU,CACzC,cAAc,EACd,YAAY,EACZ,UAAU,EACV,UAAU,EACV,OAAO,EACP,YAAY,EACZ,MAAM,EACN,YAAY,EACZ,UAAU,EACV,SAAS,EACT,MAAM,CACP,CAAC;IACF,aAAa,CAAC,uBAAuB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtD,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO,MAAM,CAAC;AAChB,CAAC;AA9BD,gCA8BC;AAEM,KAAK,UAAU,OAAO,CAC3B,MAAc,EACd,MAA0B;IAE1B,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;IAElC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE9E,sEAAsE;IACtE,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACvC,yBAAyB;QACzB,MAAM,MAAM,CAAC,YAAY,CACvB,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,EACpD,QAAQ,EACR,UAAU,CACX,CAAC;KACH;IAED,OAAO,MAAM,uCAAuB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AACvD,CAAC;AAnBD,0BAmBC;AAED,sEAAsE;AACtE,4EAA4E;AAC5E,4EAA4E;AAC5E,6EAA6E;AAC7E,+CAA+C;AACxC,KAAK,UAAU,mBAAmB,CACvC,WAA+B,EAC/B,YAAgC,EAChC,MAA0B,EAC1B,MAAc,EACd,YAA0B;IAE1B,IAAI,MAAc,CAAC;IACnB,IAAI,WAAW,KAAK,SAAS,EAAE;QAC7B,MAAM,GAAG;;;;;;;;;;;;uCAY0B,WAAW;;8BAEpB,WAAW;;;;;;;;gDAQO,CAAC;KAC9C;SAAM;QACL,oEAAoE;QACpE,mFAAmF;QACnF,+EAA+E;QAC/E,kFAAkF;QAClF,6EAA6E;QAC7E,oFAAoF;QACpF,6CAA6C;QAC7C,YAAY,GAAG,YAAY,IAAI,CAAC,CAAC;QACjC,MAAM,GAAG;;;;;;;;4BAQe,YAAY;;;;;;;;;;;;;;;;;gDAiBQ,CAAC;KAC9C;IAED,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,mBAAmB,CAAC,CAAC;IACxE,EAAE,CAAC,aAAa,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC;IAE3C,MAAM,IAAI,WAAW,CAAC,UAAU,CAC9B,YAAY,EACZ;QACE,kBAAkB;QAClB,QAAQ;QACR,OAAO;QACP,gBAAgB;QAChB,IAAI,CAAC,OAAO,CACV,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,EAC9B,OAAO,EACP,OAAO,EACP,YAAY,CACb;KACF,EACD,EAAE,GAAG,EAAE,EAAE,0BAA0B,EAAE,YAAY,CAAC,IAAI,EAAE,EAAE,CAC3D,CAAC,IAAI,EAAE,CAAC;AACX,CAAC;AAxFD,kDAwFC;AAEM,KAAK,UAAU,iBAAiB,CAAC,MAAc,EAAE,MAAc;IACpE,MAAM,CAAC,UAAU,CAAC,2BAA2B,CAAC,CAAC;IAE/C,IAAI,OAAO,CAAC,QAAQ,KAAK,QAAQ,EAAE;QACjC,MAAM,CAAC,IAAI,CACT,0EAA0E,CAC3E,CAAC;QACF,MAAM,CAAC,QAAQ,EAAE,CAAC;QAClB,OAAO;KACR;IAED,IAAI,oBAAoB,GAAG,kBAAkB,CAAC;IAC9C,IAAI,4BAA4B,GAAG,0BAA0B,CAAC;IAE9D,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE;QAChC,oBAAoB,GAAG,mBAAmB,CAAC;QAC3C,4BAA4B,GAAG,kCAAkC,CAAC;KACnE;IAED,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,iBAAiB,CAAC,CAAC;IAEjE,2CAA2C;IAC3C,IAAI,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,SAAS,EAAE;QACxC,IAAI;YACF,MAAM,IAAI,WAAW,CAAC,UAAU,CAC9B,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,oBAAoB,CAAC,CAC/C,CAAC,IAAI,EAAE,CAAC;SACV;QAAC,OAAO,CAAC,EAAE;YACV,mGAAmG;YACnG,uDAAuD;YACvD,MAAM,CAAC,QAAQ,EAAE,CAAC;YAClB,MAAM,CAAC,OAAO,CACZ,kLAAkL,CACnL,CAAC;SACH;KACF;IAED,uBAAuB;IACvB,IAAI;QACF,MAAM,IAAI,WAAW,CAAC,UAAU,CAC9B,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,4BAA4B,CAAC,EACtD,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,CACjC,CAAC,IAAI,EAAE,CAAC;KACV;IAAC,OAAO,CAAC,EAAE;QACV,MAAM,CAAC,QAAQ,EAAE,CAAC;QAClB,MAAM,CAAC,OAAO,CACZ,+IAA+I,CAChJ,CAAC;KACH;IACD,MAAM,CAAC,QAAQ,EAAE,CAAC;AACpB,CAAC;AAlDD,8CAkDC"} \ No newline at end of file +{"version":3,"file":"init.js","sourceRoot":"","sources":["../src/init.ts"],"names":[],"mappings":";;;;;;;;;AAAA,uCAAyB;AACzB,2CAA6B;AAE7B,0EAA4D;AAE5D,gEAAkD;AAClD,qCAA+C;AAC/C,4DAA8C;AAG9C,mDAAwE;AACxE,6CAA+B;AAExB,KAAK,UAAU,UAAU,CAC9B,SAA6B,EAC7B,UAAkB,EAClB,SAAiB,EACjB,OAAe,EACf,QAAgB,EAChB,IAAe,EACf,MAAc;IAEd,MAAM,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,MAAM,oBAAW,CAC9B,SAAS,EACT,UAAU,EACV,SAAS,EACT,OAAO,EACP,QAAQ,EACR,IAAI,EACJ,MAAM,CACP,CAAC;IACF,MAAM,MAAM,CAAC,YAAY,EAAE,CAAC;IAC5B,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO,MAAM,CAAC;AAChB,CAAC;AAtBD,gCAsBC;AAEM,KAAK,UAAU,UAAU,CAC9B,cAAkC,EAClC,YAAgC,EAChC,UAA8B,EAC9B,UAAyB,EACzB,OAAe,EACf,YAAoB,EACpB,MAAc,EACd,YAAoB,EACpB,UAAkB,EAClB,SAAiB,EACjB,MAAc;IAEd,MAAM,CAAC,UAAU,CAAC,6BAA6B,CAAC,CAAC;IACjD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,UAAU,CACzC,cAAc,EACd,YAAY,EACZ,UAAU,EACV,UAAU,EACV,OAAO,EACP,YAAY,EACZ,MAAM,EACN,YAAY,EACZ,UAAU,EACV,SAAS,EACT,MAAM,CACP,CAAC;IACF,aAAa,CAAC,uBAAuB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtD,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO,MAAM,CAAC;AAChB,CAAC;AA9BD,gCA8BC;AAEM,KAAK,UAAU,OAAO,CAC3B,MAAc,EACd,MAA0B;IAE1B,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;IAElC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE9E,sEAAsE;IACtE,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACvC,yBAAyB;QACzB,MAAM,MAAM,CAAC,YAAY,CACvB,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,EACpD,QAAQ,EACR,UAAU,CACX,CAAC;KACH;IAED,OAAO,MAAM,uCAAuB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AACvD,CAAC;AAnBD,0BAmBC;AAED,sEAAsE;AACtE,4EAA4E;AAC5E,4EAA4E;AAC5E,6EAA6E;AAC7E,+CAA+C;AACxC,KAAK,UAAU,mBAAmB,CACvC,WAA+B,EAC/B,YAAgC,EAChC,MAA0B,EAC1B,MAAc,EACd,YAA0B;IAE1B,IAAI,MAAc,CAAC;IACnB,IAAI,WAAW,KAAK,SAAS,EAAE;QAC7B,MAAM,GAAG;;;;;;;;;;;;uCAY0B,WAAW;;8BAEpB,WAAW;;;;;;;;gDAQO,CAAC;KAC9C;SAAM;QACL,oEAAoE;QACpE,mFAAmF;QACnF,+EAA+E;QAC/E,kFAAkF;QAClF,6EAA6E;QAC7E,oFAAoF;QACpF,6CAA6C;QAC7C,YAAY,GAAG,YAAY,IAAI,CAAC,CAAC;QACjC,MAAM,GAAG;;;;;;;;4BAQe,YAAY;;;;;;;;;;;;;;;;;gDAiBQ,CAAC;KAC9C;IAED,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,mBAAmB,CAAC,CAAC;IACxE,EAAE,CAAC,aAAa,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC;IAE3C,MAAM,IAAI,WAAW,CAAC,UAAU,CAC9B,YAAY,EACZ;QACE,kBAAkB;QAClB,QAAQ;QACR,OAAO;QACP,gBAAgB;QAChB,IAAI,CAAC,OAAO,CACV,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,EAC9B,OAAO,EACP,OAAO,EACP,YAAY,CACb;KACF,EACD,EAAE,GAAG,EAAE,EAAE,0BAA0B,EAAE,YAAY,CAAC,IAAI,EAAE,EAAE,CAC3D,CAAC,IAAI,EAAE,CAAC;AACX,CAAC;AAxFD,kDAwFC;AAEM,KAAK,UAAU,iBAAiB,CAAC,MAAc,EAAE,MAAc;IACpE,MAAM,CAAC,UAAU,CAAC,2BAA2B,CAAC,CAAC;IAE/C,IAAI,OAAO,CAAC,QAAQ,KAAK,QAAQ,EAAE;QACjC,MAAM,CAAC,IAAI,CACT,0EAA0E,CAC3E,CAAC;QACF,MAAM,CAAC,QAAQ,EAAE,CAAC;QAClB,OAAO;KACR;IAED,IAAI,oBAAoB,GAAG,kBAAkB,CAAC;IAC9C,MAAM,4BAA4B,GAAG,0BAA0B,CAAC;IAEhE,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE;QAChC,oBAAoB,GAAG,mBAAmB,CAAC;KAC5C;IAED,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,iBAAiB,CAAC,CAAC;IAEjE,2CAA2C;IAC3C,IAAI,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,SAAS,EAAE;QACxC,IAAI;YACF,MAAM,IAAI,WAAW,CAAC,UAAU,CAC9B,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,oBAAoB,CAAC,CAC/C,CAAC,IAAI,EAAE,CAAC;SACV;QAAC,OAAO,CAAC,EAAE;YACV,mGAAmG;YACnG,uDAAuD;YACvD,MAAM,CAAC,QAAQ,EAAE,CAAC;YAClB,MAAM,CAAC,OAAO,CACZ,kLAAkL,CACnL,CAAC;SACH;KACF;IAED,uBAAuB;IACvB,IAAI;QACF,MAAM,IAAI,WAAW,CAAC,UAAU,CAC9B,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,4BAA4B,CAAC,EACtD,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,CACjC,CAAC,IAAI,EAAE,CAAC;KACV;IAAC,OAAO,CAAC,EAAE;QACV,MAAM,CAAC,QAAQ,EAAE,CAAC;QAClB,MAAM,CAAC,OAAO,CACZ,+IAA+I,CAChJ,CAAC;KACH;IACD,MAAM,CAAC,QAAQ,EAAE,CAAC;AACpB,CAAC;AAjDD,8CAiDC"} \ No newline at end of file diff --git a/python-setup/auto_install_packages.py b/python-setup/auto_install_packages.py index cdb70c6d6..ffc795592 100755 --- a/python-setup/auto_install_packages.py +++ b/python-setup/auto_install_packages.py @@ -23,6 +23,8 @@ def _check_output(command): def install_packages_with_poetry(): + if sys.platform.startswith('win32'): + os.environ['POETRY_VIRTUALENVS_PATH'] = os.environ['RUNNER_WORKSPACE'] + '\\virtualenvs' try: _check_call(['poetry', 'install', '--no-root']) except subprocess.CalledProcessError: @@ -36,10 +38,14 @@ def install_packages_with_poetry(): poetry_out = _check_output(['poetry', 'run', 'which', 'python']) python_executable_path = poetry_out.decode('utf-8').splitlines()[-1] + if sys.platform.startswith('win32'): + python_executable_path = python_executable_path[2:] return python_executable_path def install_packages_with_pipenv(): + if sys.platform.startswith('win32'): + os.environ['WORKON_HOME'] = os.environ['RUNNER_WORKSPACE'] + '\\virtualenvs' try: _check_call(['pipenv', 'install', '--keep-outdated', '--ignore-pipfile']) except subprocess.CalledProcessError: @@ -48,29 +54,43 @@ def install_packages_with_pipenv(): pipenv_out = _check_output(['pipenv', 'run', 'which', 'python']) python_executable_path = pipenv_out.decode('utf-8').splitlines()[-1] + if sys.platform.startswith('win32'): + python_executable_path = python_executable_path[2:] return python_executable_path def _create_venv(version: int): # create temporary directory ... that just lives "forever" - venv_path = mkdtemp(prefix='codeql-action-python-autoinstall-') + venv_path = os.environ['RUNNER_WORKSPACE']+'/codeql-action-python-autoinstall' + print ("Creating venv in "+venv_path, flush = True) # virtualenv is a bit nicer for setting up virtual environment, since it will provide # up-to-date versions of pip/setuptools/wheel which basic `python3 -m venv venv` won't - if version == 2: - _check_call(['python2', '-m', 'virtualenv', venv_path]) - elif version == 3: - _check_call(['python3', '-m', 'virtualenv', venv_path]) + if sys.platform.startswith('win32'): + if version == 2: + _check_call(['py', '-2', '-m', 'virtualenv', venv_path]) + elif version == 3: + _check_call(['py', '-3', '-m', 'virtualenv', venv_path]) + else: + if version == 2: + _check_call(['python2', '-m', 'virtualenv', venv_path]) + elif version == 3: + _check_call(['python3', '-m', 'virtualenv', venv_path]) return venv_path def install_requirements_txt_packages(version: int): venv_path = _create_venv(version) + venv_pip = os.path.join(venv_path, 'bin', 'pip') venv_python = os.path.join(venv_path, 'bin', 'python') + if sys.platform.startswith('win32'): + venv_pip = os.path.join(venv_path, 'Scripts', 'pip') + venv_python = os.path.join(venv_path, 'Scripts', 'python') + try: _check_call([venv_pip, 'install', '-r', 'requirements.txt']) except subprocess.CalledProcessError: @@ -81,9 +101,14 @@ def install_requirements_txt_packages(version: int): def install_with_setup_py(version: int): venv_path = _create_venv(version) + venv_pip = os.path.join(venv_path, 'bin', 'pip') venv_python = os.path.join(venv_path, 'bin', 'python') + if sys.platform.startswith('win32'): + venv_pip = os.path.join(venv_path, 'Scripts', 'pip') + venv_python = os.path.join(venv_path, 'Scripts', 'python') + try: # We have to choose between `python setup.py develop` and `pip install -e .`. # Modern projects use `pip install -e .` and I wasn't able to see any downsides @@ -137,7 +162,10 @@ def install_packages(codeql_base_dir) -> Optional[str]: # The binaries for packages installed with `pip install --user` are not available on # PATH by default, so we need to manually add them. - os.environ['PATH'] = os.path.expanduser('~/.local/bin') + os.pathsep + os.environ['PATH'] + if sys.platform.startswith('win32'): + os.environ['PATH'] = os.path.expandvars('%APPDATA%\Python\\Python38\\scripts') + os.pathsep + os.environ['PATH'] + else: + os.environ['PATH'] = os.path.expanduser('~/.local/bin') + os.pathsep + os.environ['PATH'] python_executable_path = install_packages(codeql_base_dir) diff --git a/python-setup/auto_install_packages_windows.py b/python-setup/auto_install_packages_windows.py deleted file mode 100644 index a9252f8c6..000000000 --- a/python-setup/auto_install_packages_windows.py +++ /dev/null @@ -1,150 +0,0 @@ -#!/usr/bin/env python3 - -import sys -import os -import subprocess -from tempfile import mkdtemp - -import extractor_version - - -def _check_call(command): - print('+ {}'.format(' '.join(command)), flush=True) - subprocess.check_call(command, stdin=subprocess.DEVNULL) - - -def _check_output(command): - print('+ {}'.format(' '.join(command)), flush=True) - out = subprocess.check_output(command, stdin=subprocess.DEVNULL) - print(out, flush=True) - sys.stderr.flush() - return out - - -def install_packages_with_poetry(): - os.environ['POETRY_VIRTUALENVS_PATH'] = os.environ['RUNNER_WORKSPACE'] + '\\virtualenvs' - try: - _check_call(['poetry', 'install', '--no-root']) - except subprocess.CalledProcessError: - sys.exit('package installation with poetry failed, see error above') - - # poetry is super annoying with `poetry run`, since it will put lots of output on - # STDOUT if the current global python interpreter is not matching the one in the - # virtualenv for the package, which was the case for using poetry for Python 2 when - # default system interpreter was Python 3 :/ - - poetry_out = _check_output(['poetry', 'run', 'which', 'python']) - python_executable_path = poetry_out.decode('utf-8').splitlines()[-1] - - return python_executable_path[2:] - - -def install_packages_with_pipenv(): - os.environ['WORKON_HOME'] = os.environ['RUNNER_WORKSPACE'] + '\\virtualenvs' - try: - _check_call(['pipenv', 'install', '--keep-outdated', '--ignore-pipfile']) - except subprocess.CalledProcessError: - sys.exit('package installation with pipenv failed, see error above') - - pipenv_out = _check_output(['pipenv', 'run', 'which', 'python']) - python_executable_path = pipenv_out.decode('utf-8').splitlines()[-1] - - return python_executable_path[2:] - - -def _create_venv(version: int): - # create temporary directory ... that just lives "forever" - venv_path = os.environ['RUNNER_WORKSPACE']+'/codeql-action-python-autoinstall' - print ("Creating venv in "+venv_path, flush = True) - - # virtualenv is a bit nicer for setting up virtual environment, since it will provide - # up-to-date versions of pip/setuptools/wheel which basic `python3 -m venv venv` won't - - if version == 2: - _check_call(['py', '-2', '-m', 'virtualenv', venv_path]) - elif version == 3: - _check_call(['py', '-3', '-m', 'virtualenv', venv_path]) - - return venv_path - - -def install_requirements_txt_packages(version: int): - venv_path = _create_venv(version) - venv_pip = os.path.join(venv_path, 'Scripts', 'pip') - venv_python = os.path.join(venv_path, 'Scripts', 'python') - - try: - _check_call([venv_pip, 'install', '-r', 'requirements.txt']) - except subprocess.CalledProcessError: - sys.exit('package installation with `pip install -r requirements.txt` failed, see error above') - - return venv_python - - -def install_with_setup_py(version: int): - venv_path = _create_venv(version) - venv_pip = os.path.join(venv_path, 'Scripts', 'pip') - venv_python = os.path.join(venv_path, 'Scripts', 'python') - - try: - # We have to choose between `python setup.py develop` and `pip install -e .`. - # Modern projects use `pip install -e .` and I wasn't able to see any downsides - # to doing so. However, `python setup.py develop` has some downsides -- from - # https://stackoverflow.com/a/19048754 : - # > Note that it is highly recommended to use pip install . (install) and pip - # > install -e . (developer install) to install packages, as invoking setup.py - # > directly will do the wrong things for many dependencies, such as pull - # > prereleases and incompatible package versions, or make the package hard to - # > uninstall with pip. - - _check_call([venv_pip, 'install', '-e', '.']) - except subprocess.CalledProcessError: - sys.exit('package installation with `pip install -e .` failed, see error above') - - return venv_python - - -def install_packages() -> str: - if os.path.exists('poetry.lock'): - print('Found poetry.lock, will install packages with poetry', flush=True) - return install_packages_with_poetry() - - if os.path.exists('Pipfile') or os.path.exists('Pipfile.lock'): - if os.path.exists('Pipfile.lock'): - print('Found Pipfile.lock, will install packages with Pipenv', flush=True) - else: - print('Found Pipfile, will install packages with Pipenv', flush=True) - return install_packages_with_pipenv() - - version = extractor_version.get_extractor_version(sys.argv[1], quiet=False) - - if os.path.exists('requirements.txt'): - print('Found requirements.txt, will install packages with pip', flush=True) - return install_requirements_txt_packages(version) - - if os.path.exists('setup.py'): - print('Found setup.py, will install package with pip in editable mode', flush=True) - return install_with_setup_py(version) - - print("was not able to install packages automatically", flush=True) - - -if __name__ == "__main__": - if len(sys.argv) != 2: - sys.exit('Must provide base directory for codeql tool as only argument') - - # The binaries for packages installed with `pip install --user` are not available on - # PATH by default, so we need to manually add them. - os.environ['PATH'] = os.path.expandvars('%APPDATA%\Python\\Python38\\scripts') + os.pathsep + os.environ['PATH'] - - python_executable_path = install_packages() - - if python_executable_path is not None: - print("Setting CODEQL_PYTHON={}".format(python_executable_path)) - print("::set-env name=CODEQL_PYTHON::{}".format(python_executable_path)) - -# TODO: -# - no packages -# - poetry without version -# - pipenv without version -# - pipenv without lockfile diff --git a/src/init.ts b/src/init.ts index e39c7dcd7..d9af96d6c 100644 --- a/src/init.ts +++ b/src/init.ts @@ -195,11 +195,10 @@ export async function installPythonDeps(codeql: CodeQL, logger: Logger) { } let install_tools_script = "install_tools.sh"; - let auto_install_packages_script = "auto_install_packages.py"; + const auto_install_packages_script = "auto_install_packages.py"; if (process.platform === "win32") { install_tools_script = "install_tools.ps1"; - auto_install_packages_script = "auto_install_packages_windows.py"; } const scriptsFolder = path.resolve(__dirname, "../python-setup"); From da5edaf1b4c9ab90c21aeff3b45bd9fcd3ac82e9 Mon Sep 17 00:00:00 2001 From: Andy McKay Date: Wed, 7 Oct 2020 09:59:53 -0700 Subject: [PATCH 06/16] Change time in template This template proposes a time which is our peak load for Actions, would this time work better for our customers? Fixes https://github.com/github/codeql-action/issues/258 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f7e91d629..e517dfa76 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ on: # │ │ │ │ │ # │ │ │ │ │ # * * * * * - - cron: '0 0 * * 0' + - cron: '30 1 * * 0' jobs: CodeQL-Build: From 424a9cfa1caa6b7dd592aae0290a168eaec152cf Mon Sep 17 00:00:00 2001 From: David Verdeguer <47184891+Daverlo@users.noreply.github.com> Date: Thu, 8 Oct 2020 11:59:35 +0200 Subject: [PATCH 07/16] Apply suggestions from code review Co-authored-by: Arthur Baars --- python-setup/auto_install_packages.py | 8 ++++---- src/init.ts | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/python-setup/auto_install_packages.py b/python-setup/auto_install_packages.py index ffc795592..fc276431d 100755 --- a/python-setup/auto_install_packages.py +++ b/python-setup/auto_install_packages.py @@ -24,7 +24,7 @@ def _check_output(command): def install_packages_with_poetry(): if sys.platform.startswith('win32'): - os.environ['POETRY_VIRTUALENVS_PATH'] = os.environ['RUNNER_WORKSPACE'] + '\\virtualenvs' + os.environ['POETRY_VIRTUALENVS_PATH'] = os.path.join(os.environ['RUNNER_WORKSPACE', 'virtualenvs') try: _check_call(['poetry', 'install', '--no-root']) except subprocess.CalledProcessError: @@ -45,7 +45,7 @@ def install_packages_with_poetry(): def install_packages_with_pipenv(): if sys.platform.startswith('win32'): - os.environ['WORKON_HOME'] = os.environ['RUNNER_WORKSPACE'] + '\\virtualenvs' + os.environ['WORKON_HOME'] = os.path.join(os.environ['RUNNER_WORKSPACE'], 'virtualenvs') try: _check_call(['pipenv', 'install', '--keep-outdated', '--ignore-pipfile']) except subprocess.CalledProcessError: @@ -61,8 +61,8 @@ def install_packages_with_pipenv(): def _create_venv(version: int): # create temporary directory ... that just lives "forever" - venv_path = os.environ['RUNNER_WORKSPACE']+'/codeql-action-python-autoinstall' - print ("Creating venv in "+venv_path, flush = True) + venv_path = os.path.join(os.environ['RUNNER_WORKSPACE'], 'codeql-action-python-autoinstall') + print ("Creating venv in " + venv_path, flush = True) # virtualenv is a bit nicer for setting up virtual environment, since it will provide # up-to-date versions of pip/setuptools/wheel which basic `python3 -m venv venv` won't diff --git a/src/init.ts b/src/init.ts index d9af96d6c..1379e7336 100644 --- a/src/init.ts +++ b/src/init.ts @@ -188,7 +188,7 @@ export async function installPythonDeps(codeql: CodeQL, logger: Logger) { if (process.platform === "darwin") { logger.info( - "Currently, auto-installing python dependancies is not supported on MacOS" + "Currently, auto-installing python dependencies is not supported on MacOS" ); logger.endGroup(); return; From 735ec7d414bbbeecb942ac4e9f22afd8cf730050 Mon Sep 17 00:00:00 2001 From: David Verdeguer Date: Thu, 8 Oct 2020 12:06:29 +0200 Subject: [PATCH 08/16] Add powershell to call --- lib/init.js | 20 ++++++++++++-------- lib/init.js.map | 2 +- src/init.ts | 23 ++++++++++++----------- 3 files changed, 25 insertions(+), 20 deletions(-) diff --git a/lib/init.js b/lib/init.js index b63f7eaa2..e09e33c83 100644 --- a/lib/init.js +++ b/lib/init.js @@ -124,35 +124,39 @@ exports.injectWindowsTracer = injectWindowsTracer; async function installPythonDeps(codeql, logger) { logger.startGroup("Setup Python dependencies"); if (process.platform === "darwin") { - logger.info("Currently, auto-installing python dependancies is not supported on MacOS"); + logger.info("Currently, auto-installing python dependencies is not supported on MacOS"); logger.endGroup(); return; } - let install_tools_script = "install_tools.sh"; - const auto_install_packages_script = "auto_install_packages.py"; - if (process.platform === "win32") { - install_tools_script = "install_tools.ps1"; - } const scriptsFolder = path.resolve(__dirname, "../python-setup"); // Setup tools on the Github hosted runners if (process.env["ImageOS"] !== undefined) { try { - await new toolrunnner.ToolRunner(path.join(scriptsFolder, install_tools_script)).exec(); + if (process.platform === "win32") { + await new toolrunnner.ToolRunner("powershell", [ + path.join(scriptsFolder, "install_tools.ps1"), + ]).exec(); + } + else { + await new toolrunnner.ToolRunner(path.join(scriptsFolder, "install_tools.sh")).exec(); + } } catch (e) { // This script tries to install some needed tools in the runner. It should not fail, but if it does // we just abort the process without failing the action logger.endGroup(); logger.warning("Unable to download and extract the tools needed for installing the python dependecies. You can call this action with 'setup-python-dependencies: false' to disable this process."); + return; } } // Install dependencies try { - await new toolrunnner.ToolRunner(path.join(scriptsFolder, auto_install_packages_script), [path.dirname(codeql.getPath())]).exec(); + await new toolrunnner.ToolRunner(path.join(scriptsFolder, "auto_install_packages.py"), [path.dirname(codeql.getPath())]).exec(); } catch (e) { logger.endGroup(); logger.warning("We were unable to install your python dependencies. You can call this action with 'setup-python-dependencies: false' to disable this process."); + return; } logger.endGroup(); } diff --git a/lib/init.js.map b/lib/init.js.map index 8c055778b..44c628021 100644 --- a/lib/init.js.map +++ b/lib/init.js.map @@ -1 +1 @@ -{"version":3,"file":"init.js","sourceRoot":"","sources":["../src/init.ts"],"names":[],"mappings":";;;;;;;;;AAAA,uCAAyB;AACzB,2CAA6B;AAE7B,0EAA4D;AAE5D,gEAAkD;AAClD,qCAA+C;AAC/C,4DAA8C;AAG9C,mDAAwE;AACxE,6CAA+B;AAExB,KAAK,UAAU,UAAU,CAC9B,SAA6B,EAC7B,UAAkB,EAClB,SAAiB,EACjB,OAAe,EACf,QAAgB,EAChB,IAAe,EACf,MAAc;IAEd,MAAM,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,MAAM,oBAAW,CAC9B,SAAS,EACT,UAAU,EACV,SAAS,EACT,OAAO,EACP,QAAQ,EACR,IAAI,EACJ,MAAM,CACP,CAAC;IACF,MAAM,MAAM,CAAC,YAAY,EAAE,CAAC;IAC5B,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO,MAAM,CAAC;AAChB,CAAC;AAtBD,gCAsBC;AAEM,KAAK,UAAU,UAAU,CAC9B,cAAkC,EAClC,YAAgC,EAChC,UAA8B,EAC9B,UAAyB,EACzB,OAAe,EACf,YAAoB,EACpB,MAAc,EACd,YAAoB,EACpB,UAAkB,EAClB,SAAiB,EACjB,MAAc;IAEd,MAAM,CAAC,UAAU,CAAC,6BAA6B,CAAC,CAAC;IACjD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,UAAU,CACzC,cAAc,EACd,YAAY,EACZ,UAAU,EACV,UAAU,EACV,OAAO,EACP,YAAY,EACZ,MAAM,EACN,YAAY,EACZ,UAAU,EACV,SAAS,EACT,MAAM,CACP,CAAC;IACF,aAAa,CAAC,uBAAuB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtD,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO,MAAM,CAAC;AAChB,CAAC;AA9BD,gCA8BC;AAEM,KAAK,UAAU,OAAO,CAC3B,MAAc,EACd,MAA0B;IAE1B,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;IAElC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE9E,sEAAsE;IACtE,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACvC,yBAAyB;QACzB,MAAM,MAAM,CAAC,YAAY,CACvB,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,EACpD,QAAQ,EACR,UAAU,CACX,CAAC;KACH;IAED,OAAO,MAAM,uCAAuB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AACvD,CAAC;AAnBD,0BAmBC;AAED,sEAAsE;AACtE,4EAA4E;AAC5E,4EAA4E;AAC5E,6EAA6E;AAC7E,+CAA+C;AACxC,KAAK,UAAU,mBAAmB,CACvC,WAA+B,EAC/B,YAAgC,EAChC,MAA0B,EAC1B,MAAc,EACd,YAA0B;IAE1B,IAAI,MAAc,CAAC;IACnB,IAAI,WAAW,KAAK,SAAS,EAAE;QAC7B,MAAM,GAAG;;;;;;;;;;;;uCAY0B,WAAW;;8BAEpB,WAAW;;;;;;;;gDAQO,CAAC;KAC9C;SAAM;QACL,oEAAoE;QACpE,mFAAmF;QACnF,+EAA+E;QAC/E,kFAAkF;QAClF,6EAA6E;QAC7E,oFAAoF;QACpF,6CAA6C;QAC7C,YAAY,GAAG,YAAY,IAAI,CAAC,CAAC;QACjC,MAAM,GAAG;;;;;;;;4BAQe,YAAY;;;;;;;;;;;;;;;;;gDAiBQ,CAAC;KAC9C;IAED,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,mBAAmB,CAAC,CAAC;IACxE,EAAE,CAAC,aAAa,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC;IAE3C,MAAM,IAAI,WAAW,CAAC,UAAU,CAC9B,YAAY,EACZ;QACE,kBAAkB;QAClB,QAAQ;QACR,OAAO;QACP,gBAAgB;QAChB,IAAI,CAAC,OAAO,CACV,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,EAC9B,OAAO,EACP,OAAO,EACP,YAAY,CACb;KACF,EACD,EAAE,GAAG,EAAE,EAAE,0BAA0B,EAAE,YAAY,CAAC,IAAI,EAAE,EAAE,CAC3D,CAAC,IAAI,EAAE,CAAC;AACX,CAAC;AAxFD,kDAwFC;AAEM,KAAK,UAAU,iBAAiB,CAAC,MAAc,EAAE,MAAc;IACpE,MAAM,CAAC,UAAU,CAAC,2BAA2B,CAAC,CAAC;IAE/C,IAAI,OAAO,CAAC,QAAQ,KAAK,QAAQ,EAAE;QACjC,MAAM,CAAC,IAAI,CACT,0EAA0E,CAC3E,CAAC;QACF,MAAM,CAAC,QAAQ,EAAE,CAAC;QAClB,OAAO;KACR;IAED,IAAI,oBAAoB,GAAG,kBAAkB,CAAC;IAC9C,MAAM,4BAA4B,GAAG,0BAA0B,CAAC;IAEhE,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE;QAChC,oBAAoB,GAAG,mBAAmB,CAAC;KAC5C;IAED,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,iBAAiB,CAAC,CAAC;IAEjE,2CAA2C;IAC3C,IAAI,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,SAAS,EAAE;QACxC,IAAI;YACF,MAAM,IAAI,WAAW,CAAC,UAAU,CAC9B,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,oBAAoB,CAAC,CAC/C,CAAC,IAAI,EAAE,CAAC;SACV;QAAC,OAAO,CAAC,EAAE;YACV,mGAAmG;YACnG,uDAAuD;YACvD,MAAM,CAAC,QAAQ,EAAE,CAAC;YAClB,MAAM,CAAC,OAAO,CACZ,kLAAkL,CACnL,CAAC;SACH;KACF;IAED,uBAAuB;IACvB,IAAI;QACF,MAAM,IAAI,WAAW,CAAC,UAAU,CAC9B,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,4BAA4B,CAAC,EACtD,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,CACjC,CAAC,IAAI,EAAE,CAAC;KACV;IAAC,OAAO,CAAC,EAAE;QACV,MAAM,CAAC,QAAQ,EAAE,CAAC;QAClB,MAAM,CAAC,OAAO,CACZ,+IAA+I,CAChJ,CAAC;KACH;IACD,MAAM,CAAC,QAAQ,EAAE,CAAC;AACpB,CAAC;AAjDD,8CAiDC"} \ No newline at end of file +{"version":3,"file":"init.js","sourceRoot":"","sources":["../src/init.ts"],"names":[],"mappings":";;;;;;;;;AAAA,uCAAyB;AACzB,2CAA6B;AAE7B,0EAA4D;AAE5D,gEAAkD;AAClD,qCAA+C;AAC/C,4DAA8C;AAG9C,mDAAwE;AACxE,6CAA+B;AAExB,KAAK,UAAU,UAAU,CAC9B,SAA6B,EAC7B,UAAkB,EAClB,SAAiB,EACjB,OAAe,EACf,QAAgB,EAChB,IAAe,EACf,MAAc;IAEd,MAAM,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,MAAM,oBAAW,CAC9B,SAAS,EACT,UAAU,EACV,SAAS,EACT,OAAO,EACP,QAAQ,EACR,IAAI,EACJ,MAAM,CACP,CAAC;IACF,MAAM,MAAM,CAAC,YAAY,EAAE,CAAC;IAC5B,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO,MAAM,CAAC;AAChB,CAAC;AAtBD,gCAsBC;AAEM,KAAK,UAAU,UAAU,CAC9B,cAAkC,EAClC,YAAgC,EAChC,UAA8B,EAC9B,UAAyB,EACzB,OAAe,EACf,YAAoB,EACpB,MAAc,EACd,YAAoB,EACpB,UAAkB,EAClB,SAAiB,EACjB,MAAc;IAEd,MAAM,CAAC,UAAU,CAAC,6BAA6B,CAAC,CAAC;IACjD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,UAAU,CACzC,cAAc,EACd,YAAY,EACZ,UAAU,EACV,UAAU,EACV,OAAO,EACP,YAAY,EACZ,MAAM,EACN,YAAY,EACZ,UAAU,EACV,SAAS,EACT,MAAM,CACP,CAAC;IACF,aAAa,CAAC,uBAAuB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtD,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO,MAAM,CAAC;AAChB,CAAC;AA9BD,gCA8BC;AAEM,KAAK,UAAU,OAAO,CAC3B,MAAc,EACd,MAA0B;IAE1B,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;IAElC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE9E,sEAAsE;IACtE,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACvC,yBAAyB;QACzB,MAAM,MAAM,CAAC,YAAY,CACvB,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,EACpD,QAAQ,EACR,UAAU,CACX,CAAC;KACH;IAED,OAAO,MAAM,uCAAuB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AACvD,CAAC;AAnBD,0BAmBC;AAED,sEAAsE;AACtE,4EAA4E;AAC5E,4EAA4E;AAC5E,6EAA6E;AAC7E,+CAA+C;AACxC,KAAK,UAAU,mBAAmB,CACvC,WAA+B,EAC/B,YAAgC,EAChC,MAA0B,EAC1B,MAAc,EACd,YAA0B;IAE1B,IAAI,MAAc,CAAC;IACnB,IAAI,WAAW,KAAK,SAAS,EAAE;QAC7B,MAAM,GAAG;;;;;;;;;;;;uCAY0B,WAAW;;8BAEpB,WAAW;;;;;;;;gDAQO,CAAC;KAC9C;SAAM;QACL,oEAAoE;QACpE,mFAAmF;QACnF,+EAA+E;QAC/E,kFAAkF;QAClF,6EAA6E;QAC7E,oFAAoF;QACpF,6CAA6C;QAC7C,YAAY,GAAG,YAAY,IAAI,CAAC,CAAC;QACjC,MAAM,GAAG;;;;;;;;4BAQe,YAAY;;;;;;;;;;;;;;;;;gDAiBQ,CAAC;KAC9C;IAED,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,mBAAmB,CAAC,CAAC;IACxE,EAAE,CAAC,aAAa,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC;IAE3C,MAAM,IAAI,WAAW,CAAC,UAAU,CAC9B,YAAY,EACZ;QACE,kBAAkB;QAClB,QAAQ;QACR,OAAO;QACP,gBAAgB;QAChB,IAAI,CAAC,OAAO,CACV,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,EAC9B,OAAO,EACP,OAAO,EACP,YAAY,CACb;KACF,EACD,EAAE,GAAG,EAAE,EAAE,0BAA0B,EAAE,YAAY,CAAC,IAAI,EAAE,EAAE,CAC3D,CAAC,IAAI,EAAE,CAAC;AACX,CAAC;AAxFD,kDAwFC;AAEM,KAAK,UAAU,iBAAiB,CAAC,MAAc,EAAE,MAAc;IACpE,MAAM,CAAC,UAAU,CAAC,2BAA2B,CAAC,CAAC;IAE/C,IAAI,OAAO,CAAC,QAAQ,KAAK,QAAQ,EAAE;QACjC,MAAM,CAAC,IAAI,CACT,0EAA0E,CAC3E,CAAC;QACF,MAAM,CAAC,QAAQ,EAAE,CAAC;QAClB,OAAO;KACR;IAED,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,iBAAiB,CAAC,CAAC;IAEjE,2CAA2C;IAC3C,IAAI,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,SAAS,EAAE;QACxC,IAAI;YACF,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE;gBAChC,MAAM,IAAI,WAAW,CAAC,UAAU,CAAC,YAAY,EAAE;oBAC7C,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,mBAAmB,CAAC;iBAC9C,CAAC,CAAC,IAAI,EAAE,CAAC;aACX;iBAAM;gBACL,MAAM,IAAI,WAAW,CAAC,UAAU,CAC9B,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,kBAAkB,CAAC,CAC7C,CAAC,IAAI,EAAE,CAAC;aACV;SACF;QAAC,OAAO,CAAC,EAAE;YACV,mGAAmG;YACnG,uDAAuD;YACvD,MAAM,CAAC,QAAQ,EAAE,CAAC;YAClB,MAAM,CAAC,OAAO,CACZ,kLAAkL,CACnL,CAAC;YACF,OAAO;SACR;KACF;IAED,uBAAuB;IACvB,IAAI;QACF,MAAM,IAAI,WAAW,CAAC,UAAU,CAC9B,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,0BAA0B,CAAC,EACpD,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,CACjC,CAAC,IAAI,EAAE,CAAC;KACV;IAAC,OAAO,CAAC,EAAE;QACV,MAAM,CAAC,QAAQ,EAAE,CAAC;QAClB,MAAM,CAAC,OAAO,CACZ,+IAA+I,CAChJ,CAAC;QACF,OAAO;KACR;IACD,MAAM,CAAC,QAAQ,EAAE,CAAC;AACpB,CAAC;AAlDD,8CAkDC"} \ No newline at end of file diff --git a/src/init.ts b/src/init.ts index 1379e7336..d8f7d3ba2 100644 --- a/src/init.ts +++ b/src/init.ts @@ -194,21 +194,20 @@ export async function installPythonDeps(codeql: CodeQL, logger: Logger) { return; } - let install_tools_script = "install_tools.sh"; - const auto_install_packages_script = "auto_install_packages.py"; - - if (process.platform === "win32") { - install_tools_script = "install_tools.ps1"; - } - const scriptsFolder = path.resolve(__dirname, "../python-setup"); // Setup tools on the Github hosted runners if (process.env["ImageOS"] !== undefined) { try { - await new toolrunnner.ToolRunner( - path.join(scriptsFolder, install_tools_script) - ).exec(); + if (process.platform === "win32") { + await new toolrunnner.ToolRunner("powershell", [ + path.join(scriptsFolder, "install_tools.ps1"), + ]).exec(); + } else { + await new toolrunnner.ToolRunner( + path.join(scriptsFolder, "install_tools.sh") + ).exec(); + } } catch (e) { // This script tries to install some needed tools in the runner. It should not fail, but if it does // we just abort the process without failing the action @@ -216,13 +215,14 @@ export async function installPythonDeps(codeql: CodeQL, logger: Logger) { logger.warning( "Unable to download and extract the tools needed for installing the python dependecies. You can call this action with 'setup-python-dependencies: false' to disable this process." ); + return; } } // Install dependencies try { await new toolrunnner.ToolRunner( - path.join(scriptsFolder, auto_install_packages_script), + path.join(scriptsFolder, "auto_install_packages.py"), [path.dirname(codeql.getPath())] ).exec(); } catch (e) { @@ -230,6 +230,7 @@ export async function installPythonDeps(codeql: CodeQL, logger: Logger) { logger.warning( "We were unable to install your python dependencies. You can call this action with 'setup-python-dependencies: false' to disable this process." ); + return; } logger.endGroup(); } From e3d2d4afc4385556244327c93b9eac325af15f69 Mon Sep 17 00:00:00 2001 From: David Verdeguer Date: Thu, 8 Oct 2020 12:14:41 +0200 Subject: [PATCH 09/16] Add call to py --- lib/init.js | 14 +++++++++++++- lib/init.js.map | 2 +- src/init.ts | 16 ++++++++++++---- 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/lib/init.js b/lib/init.js index e09e33c83..4b1d15632 100644 --- a/lib/init.js +++ b/lib/init.js @@ -151,7 +151,19 @@ async function installPythonDeps(codeql, logger) { } // Install dependencies try { - await new toolrunnner.ToolRunner(path.join(scriptsFolder, "auto_install_packages.py"), [path.dirname(codeql.getPath())]).exec(); + const script = "auto_install_packages.py"; + if (process.platform === "win32") { + await new toolrunnner.ToolRunner("py", [ + "-3", + path.join(scriptsFolder, script), + path.dirname(codeql.getPath()), + ]).exec(); + } + else { + await new toolrunnner.ToolRunner(path.join(scriptsFolder, script), [ + path.dirname(codeql.getPath()), + ]).exec(); + } } catch (e) { logger.endGroup(); diff --git a/lib/init.js.map b/lib/init.js.map index 44c628021..277b31665 100644 --- a/lib/init.js.map +++ b/lib/init.js.map @@ -1 +1 @@ -{"version":3,"file":"init.js","sourceRoot":"","sources":["../src/init.ts"],"names":[],"mappings":";;;;;;;;;AAAA,uCAAyB;AACzB,2CAA6B;AAE7B,0EAA4D;AAE5D,gEAAkD;AAClD,qCAA+C;AAC/C,4DAA8C;AAG9C,mDAAwE;AACxE,6CAA+B;AAExB,KAAK,UAAU,UAAU,CAC9B,SAA6B,EAC7B,UAAkB,EAClB,SAAiB,EACjB,OAAe,EACf,QAAgB,EAChB,IAAe,EACf,MAAc;IAEd,MAAM,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,MAAM,oBAAW,CAC9B,SAAS,EACT,UAAU,EACV,SAAS,EACT,OAAO,EACP,QAAQ,EACR,IAAI,EACJ,MAAM,CACP,CAAC;IACF,MAAM,MAAM,CAAC,YAAY,EAAE,CAAC;IAC5B,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO,MAAM,CAAC;AAChB,CAAC;AAtBD,gCAsBC;AAEM,KAAK,UAAU,UAAU,CAC9B,cAAkC,EAClC,YAAgC,EAChC,UAA8B,EAC9B,UAAyB,EACzB,OAAe,EACf,YAAoB,EACpB,MAAc,EACd,YAAoB,EACpB,UAAkB,EAClB,SAAiB,EACjB,MAAc;IAEd,MAAM,CAAC,UAAU,CAAC,6BAA6B,CAAC,CAAC;IACjD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,UAAU,CACzC,cAAc,EACd,YAAY,EACZ,UAAU,EACV,UAAU,EACV,OAAO,EACP,YAAY,EACZ,MAAM,EACN,YAAY,EACZ,UAAU,EACV,SAAS,EACT,MAAM,CACP,CAAC;IACF,aAAa,CAAC,uBAAuB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtD,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO,MAAM,CAAC;AAChB,CAAC;AA9BD,gCA8BC;AAEM,KAAK,UAAU,OAAO,CAC3B,MAAc,EACd,MAA0B;IAE1B,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;IAElC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE9E,sEAAsE;IACtE,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACvC,yBAAyB;QACzB,MAAM,MAAM,CAAC,YAAY,CACvB,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,EACpD,QAAQ,EACR,UAAU,CACX,CAAC;KACH;IAED,OAAO,MAAM,uCAAuB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AACvD,CAAC;AAnBD,0BAmBC;AAED,sEAAsE;AACtE,4EAA4E;AAC5E,4EAA4E;AAC5E,6EAA6E;AAC7E,+CAA+C;AACxC,KAAK,UAAU,mBAAmB,CACvC,WAA+B,EAC/B,YAAgC,EAChC,MAA0B,EAC1B,MAAc,EACd,YAA0B;IAE1B,IAAI,MAAc,CAAC;IACnB,IAAI,WAAW,KAAK,SAAS,EAAE;QAC7B,MAAM,GAAG;;;;;;;;;;;;uCAY0B,WAAW;;8BAEpB,WAAW;;;;;;;;gDAQO,CAAC;KAC9C;SAAM;QACL,oEAAoE;QACpE,mFAAmF;QACnF,+EAA+E;QAC/E,kFAAkF;QAClF,6EAA6E;QAC7E,oFAAoF;QACpF,6CAA6C;QAC7C,YAAY,GAAG,YAAY,IAAI,CAAC,CAAC;QACjC,MAAM,GAAG;;;;;;;;4BAQe,YAAY;;;;;;;;;;;;;;;;;gDAiBQ,CAAC;KAC9C;IAED,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,mBAAmB,CAAC,CAAC;IACxE,EAAE,CAAC,aAAa,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC;IAE3C,MAAM,IAAI,WAAW,CAAC,UAAU,CAC9B,YAAY,EACZ;QACE,kBAAkB;QAClB,QAAQ;QACR,OAAO;QACP,gBAAgB;QAChB,IAAI,CAAC,OAAO,CACV,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,EAC9B,OAAO,EACP,OAAO,EACP,YAAY,CACb;KACF,EACD,EAAE,GAAG,EAAE,EAAE,0BAA0B,EAAE,YAAY,CAAC,IAAI,EAAE,EAAE,CAC3D,CAAC,IAAI,EAAE,CAAC;AACX,CAAC;AAxFD,kDAwFC;AAEM,KAAK,UAAU,iBAAiB,CAAC,MAAc,EAAE,MAAc;IACpE,MAAM,CAAC,UAAU,CAAC,2BAA2B,CAAC,CAAC;IAE/C,IAAI,OAAO,CAAC,QAAQ,KAAK,QAAQ,EAAE;QACjC,MAAM,CAAC,IAAI,CACT,0EAA0E,CAC3E,CAAC;QACF,MAAM,CAAC,QAAQ,EAAE,CAAC;QAClB,OAAO;KACR;IAED,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,iBAAiB,CAAC,CAAC;IAEjE,2CAA2C;IAC3C,IAAI,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,SAAS,EAAE;QACxC,IAAI;YACF,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE;gBAChC,MAAM,IAAI,WAAW,CAAC,UAAU,CAAC,YAAY,EAAE;oBAC7C,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,mBAAmB,CAAC;iBAC9C,CAAC,CAAC,IAAI,EAAE,CAAC;aACX;iBAAM;gBACL,MAAM,IAAI,WAAW,CAAC,UAAU,CAC9B,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,kBAAkB,CAAC,CAC7C,CAAC,IAAI,EAAE,CAAC;aACV;SACF;QAAC,OAAO,CAAC,EAAE;YACV,mGAAmG;YACnG,uDAAuD;YACvD,MAAM,CAAC,QAAQ,EAAE,CAAC;YAClB,MAAM,CAAC,OAAO,CACZ,kLAAkL,CACnL,CAAC;YACF,OAAO;SACR;KACF;IAED,uBAAuB;IACvB,IAAI;QACF,MAAM,IAAI,WAAW,CAAC,UAAU,CAC9B,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,0BAA0B,CAAC,EACpD,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,CACjC,CAAC,IAAI,EAAE,CAAC;KACV;IAAC,OAAO,CAAC,EAAE;QACV,MAAM,CAAC,QAAQ,EAAE,CAAC;QAClB,MAAM,CAAC,OAAO,CACZ,+IAA+I,CAChJ,CAAC;QACF,OAAO;KACR;IACD,MAAM,CAAC,QAAQ,EAAE,CAAC;AACpB,CAAC;AAlDD,8CAkDC"} \ No newline at end of file +{"version":3,"file":"init.js","sourceRoot":"","sources":["../src/init.ts"],"names":[],"mappings":";;;;;;;;;AAAA,uCAAyB;AACzB,2CAA6B;AAE7B,0EAA4D;AAE5D,gEAAkD;AAClD,qCAA+C;AAC/C,4DAA8C;AAG9C,mDAAwE;AACxE,6CAA+B;AAExB,KAAK,UAAU,UAAU,CAC9B,SAA6B,EAC7B,UAAkB,EAClB,SAAiB,EACjB,OAAe,EACf,QAAgB,EAChB,IAAe,EACf,MAAc;IAEd,MAAM,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,MAAM,oBAAW,CAC9B,SAAS,EACT,UAAU,EACV,SAAS,EACT,OAAO,EACP,QAAQ,EACR,IAAI,EACJ,MAAM,CACP,CAAC;IACF,MAAM,MAAM,CAAC,YAAY,EAAE,CAAC;IAC5B,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO,MAAM,CAAC;AAChB,CAAC;AAtBD,gCAsBC;AAEM,KAAK,UAAU,UAAU,CAC9B,cAAkC,EAClC,YAAgC,EAChC,UAA8B,EAC9B,UAAyB,EACzB,OAAe,EACf,YAAoB,EACpB,MAAc,EACd,YAAoB,EACpB,UAAkB,EAClB,SAAiB,EACjB,MAAc;IAEd,MAAM,CAAC,UAAU,CAAC,6BAA6B,CAAC,CAAC;IACjD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,UAAU,CACzC,cAAc,EACd,YAAY,EACZ,UAAU,EACV,UAAU,EACV,OAAO,EACP,YAAY,EACZ,MAAM,EACN,YAAY,EACZ,UAAU,EACV,SAAS,EACT,MAAM,CACP,CAAC;IACF,aAAa,CAAC,uBAAuB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtD,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO,MAAM,CAAC;AAChB,CAAC;AA9BD,gCA8BC;AAEM,KAAK,UAAU,OAAO,CAC3B,MAAc,EACd,MAA0B;IAE1B,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;IAElC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE9E,sEAAsE;IACtE,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACvC,yBAAyB;QACzB,MAAM,MAAM,CAAC,YAAY,CACvB,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,EACpD,QAAQ,EACR,UAAU,CACX,CAAC;KACH;IAED,OAAO,MAAM,uCAAuB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AACvD,CAAC;AAnBD,0BAmBC;AAED,sEAAsE;AACtE,4EAA4E;AAC5E,4EAA4E;AAC5E,6EAA6E;AAC7E,+CAA+C;AACxC,KAAK,UAAU,mBAAmB,CACvC,WAA+B,EAC/B,YAAgC,EAChC,MAA0B,EAC1B,MAAc,EACd,YAA0B;IAE1B,IAAI,MAAc,CAAC;IACnB,IAAI,WAAW,KAAK,SAAS,EAAE;QAC7B,MAAM,GAAG;;;;;;;;;;;;uCAY0B,WAAW;;8BAEpB,WAAW;;;;;;;;gDAQO,CAAC;KAC9C;SAAM;QACL,oEAAoE;QACpE,mFAAmF;QACnF,+EAA+E;QAC/E,kFAAkF;QAClF,6EAA6E;QAC7E,oFAAoF;QACpF,6CAA6C;QAC7C,YAAY,GAAG,YAAY,IAAI,CAAC,CAAC;QACjC,MAAM,GAAG;;;;;;;;4BAQe,YAAY;;;;;;;;;;;;;;;;;gDAiBQ,CAAC;KAC9C;IAED,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,mBAAmB,CAAC,CAAC;IACxE,EAAE,CAAC,aAAa,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC;IAE3C,MAAM,IAAI,WAAW,CAAC,UAAU,CAC9B,YAAY,EACZ;QACE,kBAAkB;QAClB,QAAQ;QACR,OAAO;QACP,gBAAgB;QAChB,IAAI,CAAC,OAAO,CACV,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,EAC9B,OAAO,EACP,OAAO,EACP,YAAY,CACb;KACF,EACD,EAAE,GAAG,EAAE,EAAE,0BAA0B,EAAE,YAAY,CAAC,IAAI,EAAE,EAAE,CAC3D,CAAC,IAAI,EAAE,CAAC;AACX,CAAC;AAxFD,kDAwFC;AAEM,KAAK,UAAU,iBAAiB,CAAC,MAAc,EAAE,MAAc;IACpE,MAAM,CAAC,UAAU,CAAC,2BAA2B,CAAC,CAAC;IAE/C,IAAI,OAAO,CAAC,QAAQ,KAAK,QAAQ,EAAE;QACjC,MAAM,CAAC,IAAI,CACT,0EAA0E,CAC3E,CAAC;QACF,MAAM,CAAC,QAAQ,EAAE,CAAC;QAClB,OAAO;KACR;IAED,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,iBAAiB,CAAC,CAAC;IAEjE,2CAA2C;IAC3C,IAAI,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,SAAS,EAAE;QACxC,IAAI;YACF,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE;gBAChC,MAAM,IAAI,WAAW,CAAC,UAAU,CAAC,YAAY,EAAE;oBAC7C,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,mBAAmB,CAAC;iBAC9C,CAAC,CAAC,IAAI,EAAE,CAAC;aACX;iBAAM;gBACL,MAAM,IAAI,WAAW,CAAC,UAAU,CAC9B,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,kBAAkB,CAAC,CAC7C,CAAC,IAAI,EAAE,CAAC;aACV;SACF;QAAC,OAAO,CAAC,EAAE;YACV,mGAAmG;YACnG,uDAAuD;YACvD,MAAM,CAAC,QAAQ,EAAE,CAAC;YAClB,MAAM,CAAC,OAAO,CACZ,kLAAkL,CACnL,CAAC;YACF,OAAO;SACR;KACF;IAED,uBAAuB;IACvB,IAAI;QACF,MAAM,MAAM,GAAG,0BAA0B,CAAC;QAC1C,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE;YAChC,MAAM,IAAI,WAAW,CAAC,UAAU,CAAC,IAAI,EAAE;gBACrC,IAAI;gBACJ,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,MAAM,CAAC;gBAChC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;aAC/B,CAAC,CAAC,IAAI,EAAE,CAAC;SACX;aAAM;YACL,MAAM,IAAI,WAAW,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,MAAM,CAAC,EAAE;gBACjE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;aAC/B,CAAC,CAAC,IAAI,EAAE,CAAC;SACX;KACF;IAAC,OAAO,CAAC,EAAE;QACV,MAAM,CAAC,QAAQ,EAAE,CAAC;QAClB,MAAM,CAAC,OAAO,CACZ,+IAA+I,CAChJ,CAAC;QACF,OAAO;KACR;IACD,MAAM,CAAC,QAAQ,EAAE,CAAC;AACpB,CAAC;AA1DD,8CA0DC"} \ No newline at end of file diff --git a/src/init.ts b/src/init.ts index d8f7d3ba2..0fbd8a404 100644 --- a/src/init.ts +++ b/src/init.ts @@ -221,10 +221,18 @@ export async function installPythonDeps(codeql: CodeQL, logger: Logger) { // Install dependencies try { - await new toolrunnner.ToolRunner( - path.join(scriptsFolder, "auto_install_packages.py"), - [path.dirname(codeql.getPath())] - ).exec(); + const script = "auto_install_packages.py"; + if (process.platform === "win32") { + await new toolrunnner.ToolRunner("py", [ + "-3", + path.join(scriptsFolder, script), + path.dirname(codeql.getPath()), + ]).exec(); + } else { + await new toolrunnner.ToolRunner(path.join(scriptsFolder, script), [ + path.dirname(codeql.getPath()), + ]).exec(); + } } catch (e) { logger.endGroup(); logger.warning( From 9bc8c56ef03e88271a0c73bcca9070022a38ec50 Mon Sep 17 00:00:00 2001 From: David Verdeguer Date: Thu, 8 Oct 2020 12:17:28 +0200 Subject: [PATCH 10/16] Add missing [ --- python-setup/auto_install_packages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-setup/auto_install_packages.py b/python-setup/auto_install_packages.py index fc276431d..63904111b 100755 --- a/python-setup/auto_install_packages.py +++ b/python-setup/auto_install_packages.py @@ -24,7 +24,7 @@ def _check_output(command): def install_packages_with_poetry(): if sys.platform.startswith('win32'): - os.environ['POETRY_VIRTUALENVS_PATH'] = os.path.join(os.environ['RUNNER_WORKSPACE', 'virtualenvs') + os.environ['POETRY_VIRTUALENVS_PATH'] = os.path.join(os.environ['RUNNER_WORKSPACE'], 'virtualenvs') try: _check_call(['poetry', 'install', '--no-root']) except subprocess.CalledProcessError: From 2da5fbf0d45ad52ccce2d5573d6a5e0730f8d80d Mon Sep 17 00:00:00 2001 From: Robin Neatherway Date: Thu, 8 Oct 2020 14:48:20 +0100 Subject: [PATCH 11/16] Update default CodeQL version to codeql-bundle-20201008 --- lib/defaults.json | 2 +- src/defaults.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/defaults.json b/lib/defaults.json index f5e364e85..a19748cfb 100644 --- a/lib/defaults.json +++ b/lib/defaults.json @@ -1,3 +1,3 @@ { - "bundleVersion": "codeql-bundle-20200826" + "bundleVersion": "codeql-bundle-20201008" } diff --git a/src/defaults.json b/src/defaults.json index 097a83fcf..0ba345843 100644 --- a/src/defaults.json +++ b/src/defaults.json @@ -1,3 +1,3 @@ { - "bundleVersion": "codeql-bundle-20200826" + "bundleVersion": "codeql-bundle-20201008" } From 64ebf10c1d6e5d668d4847ca688216e43c0c52dc Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Thu, 8 Oct 2020 16:12:40 +0200 Subject: [PATCH 12/16] Don't use ::set-env in python-setup Is now deprecated as described in https://github.blog/changelog/2020-10-01-github-actions-deprecating-set-env-and-add-path-commands/ --- python-setup/auto_install_packages.py | 5 ++++- python-setup/tests/from_python_exe.py | 8 ++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/python-setup/auto_install_packages.py b/python-setup/auto_install_packages.py index cdb70c6d6..7b768bfd9 100755 --- a/python-setup/auto_install_packages.py +++ b/python-setup/auto_install_packages.py @@ -142,5 +142,8 @@ def install_packages(codeql_base_dir) -> Optional[str]: python_executable_path = install_packages(codeql_base_dir) if python_executable_path is not None: + # see https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable + env_file = open(os.environ["GITHUB_ENV"], mode="at") + print("Setting CODEQL_PYTHON={}".format(python_executable_path)) - print("::set-env name=CODEQL_PYTHON::{}".format(python_executable_path)) + print("CODEQL_PYTHON={}".format(python_executable_path), file=env_file) diff --git a/python-setup/tests/from_python_exe.py b/python-setup/tests/from_python_exe.py index 69e2e873e..19702cf37 100755 --- a/python-setup/tests/from_python_exe.py +++ b/python-setup/tests/from_python_exe.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import sys +import os import subprocess from typing import Tuple @@ -24,8 +25,11 @@ def get_details(path_to_python_exe: str) -> Tuple[str, str]: if __name__ == "__main__": version, import_path = get_details(sys.argv[1]) + # see https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable + env_file = open(os.environ["GITHUB_ENV"], mode="at") + print("Setting LGTM_PYTHON_SETUP_VERSION={}".format(version)) - print("::set-env name=LGTM_PYTHON_SETUP_VERSION::{}".format(version)) + print("LGTM_PYTHON_SETUP_VERSION={}".format(version), file=env_file) print("Setting LGTM_INDEX_IMPORT_PATH={}".format(import_path)) - print("::set-env name=LGTM_INDEX_IMPORT_PATH::{}".format(import_path)) \ No newline at end of file + print("LGTM_INDEX_IMPORT_PATH={}".format(import_path), file=env_file) From f79717f3c33c75201c2f643119b813672c45dd9e Mon Sep 17 00:00:00 2001 From: Robin Neatherway Date: Tue, 13 Oct 2020 10:19:15 +0100 Subject: [PATCH 13/16] Start analysing merge commit for PRs --- .github/workflows/codeql.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index d3dda5dc6..e5e35000a 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -14,16 +14,6 @@ jobs: steps: - uses: actions/checkout@v2 - with: - # Must fetch at least the immediate parents so that if this is - # a pull request then we can checkout the head of the pull request. - fetch-depth: 2 - - # If this run was triggered by a pull request event then checkout - # the head of the pull request instead of the merge commit. - - run: git checkout HEAD^2 - if: ${{ github.event_name == 'pull_request' }} - - uses: ./init with: languages: javascript From 0d97ea8f149d6e3ddcc940750604ae92a9027eab Mon Sep 17 00:00:00 2001 From: David Verdeguer Date: Tue, 13 Oct 2020 17:38:53 +0200 Subject: [PATCH 14/16] Call pipenv and poetry through python --- python-setup/auto_install_packages.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/python-setup/auto_install_packages.py b/python-setup/auto_install_packages.py index 63904111b..5bd6bd96c 100755 --- a/python-setup/auto_install_packages.py +++ b/python-setup/auto_install_packages.py @@ -23,10 +23,12 @@ def _check_output(command): def install_packages_with_poetry(): + command = ['poetry'] if sys.platform.startswith('win32'): + command = ['py', '-3' , '-m', 'poetry'] os.environ['POETRY_VIRTUALENVS_PATH'] = os.path.join(os.environ['RUNNER_WORKSPACE'], 'virtualenvs') try: - _check_call(['poetry', 'install', '--no-root']) + _check_call(command + ['install', '--no-root']) except subprocess.CalledProcessError: sys.exit('package installation with poetry failed, see error above') @@ -35,7 +37,7 @@ def install_packages_with_poetry(): # virtualenv for the package, which was the case for using poetry for Python 2 when # default system interpreter was Python 3 :/ - poetry_out = _check_output(['poetry', 'run', 'which', 'python']) + poetry_out = _check_output(command + ['run', 'which', 'python']) python_executable_path = poetry_out.decode('utf-8').splitlines()[-1] if sys.platform.startswith('win32'): @@ -44,14 +46,16 @@ def install_packages_with_poetry(): def install_packages_with_pipenv(): + command = ['pipenv'] if sys.platform.startswith('win32'): + command = ['py', '-3' , '-m', 'pipenv'] os.environ['WORKON_HOME'] = os.path.join(os.environ['RUNNER_WORKSPACE'], 'virtualenvs') try: - _check_call(['pipenv', 'install', '--keep-outdated', '--ignore-pipfile']) + _check_call(command + ['install', '--keep-outdated', '--ignore-pipfile']) except subprocess.CalledProcessError: sys.exit('package installation with pipenv failed, see error above') - pipenv_out = _check_output(['pipenv', 'run', 'which', 'python']) + pipenv_out = _check_output(command + ['run', 'which', 'python']) python_executable_path = pipenv_out.decode('utf-8').splitlines()[-1] if sys.platform.startswith('win32'): @@ -162,9 +166,7 @@ def install_packages(codeql_base_dir) -> Optional[str]: # The binaries for packages installed with `pip install --user` are not available on # PATH by default, so we need to manually add them. - if sys.platform.startswith('win32'): - os.environ['PATH'] = os.path.expandvars('%APPDATA%\Python\\Python38\\scripts') + os.pathsep + os.environ['PATH'] - else: + if sys.platform.startswith('linux'): os.environ['PATH'] = os.path.expanduser('~/.local/bin') + os.pathsep + os.environ['PATH'] python_executable_path = install_packages(codeql_base_dir) From 2c3dafc1626d535f942a28623415b279a5f2accb Mon Sep 17 00:00:00 2001 From: David Verdeguer Date: Wed, 14 Oct 2020 10:54:41 +0200 Subject: [PATCH 15/16] Use sys.executable --- python-setup/auto_install_packages.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/python-setup/auto_install_packages.py b/python-setup/auto_install_packages.py index 5bd6bd96c..1a7f4358a 100755 --- a/python-setup/auto_install_packages.py +++ b/python-setup/auto_install_packages.py @@ -23,9 +23,8 @@ def _check_output(command): def install_packages_with_poetry(): - command = ['poetry'] + command = [sys.executable, '-m', 'poetry'] if sys.platform.startswith('win32'): - command = ['py', '-3' , '-m', 'poetry'] os.environ['POETRY_VIRTUALENVS_PATH'] = os.path.join(os.environ['RUNNER_WORKSPACE'], 'virtualenvs') try: _check_call(command + ['install', '--no-root']) @@ -46,9 +45,8 @@ def install_packages_with_poetry(): def install_packages_with_pipenv(): - command = ['pipenv'] + command = [sys.executable, '-m', 'pipenv'] if sys.platform.startswith('win32'): - command = ['py', '-3' , '-m', 'pipenv'] os.environ['WORKON_HOME'] = os.path.join(os.environ['RUNNER_WORKSPACE'], 'virtualenvs') try: _check_call(command + ['install', '--keep-outdated', '--ignore-pipfile']) From 7d1f309c391fd18a896acf8cb206e6b853a7853f Mon Sep 17 00:00:00 2001 From: David Verdeguer Date: Thu, 15 Oct 2020 11:56:40 +0200 Subject: [PATCH 16/16] Address comments --- python-setup/auto_install_packages.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/python-setup/auto_install_packages.py b/python-setup/auto_install_packages.py index 1a7f4358a..f0c102056 100755 --- a/python-setup/auto_install_packages.py +++ b/python-setup/auto_install_packages.py @@ -25,6 +25,8 @@ def _check_output(command): def install_packages_with_poetry(): command = [sys.executable, '-m', 'poetry'] if sys.platform.startswith('win32'): + # In windows the default path were the deps are installed gets wiped out between steps, + # so we have to set it up to a folder that will be kept os.environ['POETRY_VIRTUALENVS_PATH'] = os.path.join(os.environ['RUNNER_WORKSPACE'], 'virtualenvs') try: _check_call(command + ['install', '--no-root']) @@ -40,6 +42,8 @@ def install_packages_with_poetry(): python_executable_path = poetry_out.decode('utf-8').splitlines()[-1] if sys.platform.startswith('win32'): + # Poetry produces a path that starts by /d instead of D:\ and Windows doesn't like that way of specifying the drive letter. + # We completely remove it because it is not needed as everything is in the same drive (We are installing the dependencies in the RUNNER_WORKSPACE) python_executable_path = python_executable_path[2:] return python_executable_path @@ -47,6 +51,8 @@ def install_packages_with_poetry(): def install_packages_with_pipenv(): command = [sys.executable, '-m', 'pipenv'] if sys.platform.startswith('win32'): + # In windows the default path were the deps are installed gets wiped out between steps, + # so we have to set it up to a folder that will be kept os.environ['WORKON_HOME'] = os.path.join(os.environ['RUNNER_WORKSPACE'], 'virtualenvs') try: _check_call(command + ['install', '--keep-outdated', '--ignore-pipfile']) @@ -57,6 +63,8 @@ def install_packages_with_pipenv(): python_executable_path = pipenv_out.decode('utf-8').splitlines()[-1] if sys.platform.startswith('win32'): + # Pipenv produces a path that starts by /d instead of D:\ and Windows doesn't like that way of specifying the drive letter. + # We completely remove it because it is not needed as everything is in the same drive (We are installing the dependencies in the RUNNER_WORKSPACE) python_executable_path = python_executable_path[2:] return python_executable_path @@ -162,11 +170,6 @@ def install_packages(codeql_base_dir) -> Optional[str]: codeql_base_dir = sys.argv[1] - # The binaries for packages installed with `pip install --user` are not available on - # PATH by default, so we need to manually add them. - if sys.platform.startswith('linux'): - os.environ['PATH'] = os.path.expanduser('~/.local/bin') + os.pathsep + os.environ['PATH'] - python_executable_path = install_packages(codeql_base_dir) if python_executable_path is not None: