Merge pull request #155 from github/daverlo/python

Python deps setup

.github/workflows/python-deps.yml

@@ -0,0 +1,63 @@
+name: Test Python Package Installation
+
+on:
+  push:
+    branches: [main, v1]
+  pull_request:
+
+jobs:
+
+  test-setup-python-scripts:
+    runs-on: ubuntu-latest
+    strategy:
+        fail-fast: false
+        matrix:
+          include:
+            - test_dir: python-setup/tests/pipenv/requests-2
+              test_script: $GITHUB_WORKSPACE/python-setup/tests/check_requests_123.sh 2
+            - test_dir: python-setup/tests/pipenv/requests-3
+              test_script: $GITHUB_WORKSPACE/python-setup/tests/check_requests_123.sh 3
+
+            - test_dir: python-setup/tests/poetry/requests-2
+              test_script: $GITHUB_WORKSPACE/python-setup/tests/check_requests_123.sh 2
+            - test_dir: python-setup/tests/poetry/requests-3
+              test_script: $GITHUB_WORKSPACE/python-setup/tests/check_requests_123.sh 3
+
+            - test_dir: python-setup/tests/requirements/requests-2
+              test_script: $GITHUB_WORKSPACE/python-setup/tests/check_requests_123.sh 2
+            - test_dir: python-setup/tests/requirements/requests-3
+              test_script: $GITHUB_WORKSPACE/python-setup/tests/check_requests_123.sh 3
+
+            - test_dir: python-setup/tests/setup_py/requests-2
+              test_script: $GITHUB_WORKSPACE/python-setup/tests/check_requests_123.sh 2
+            - test_dir: python-setup/tests/setup_py/requests-3
+              test_script: $GITHUB_WORKSPACE/python-setup/tests/check_requests_123.sh 3
+
+            # This one shouldn't fail, but also won't install packages
+            - test_dir: python-setup/tests/requirements/non-standard-location
+              test_script: test -z $LGTM_INDEX_IMPORT_PATH
+
+    steps:
+    # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+    - uses: actions/checkout@v2
+
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: python
+
+    - name: Test Auto Package Installation
+      run: |
+        set -x
+        $GITHUB_WORKSPACE/python-setup/install_tools.sh
+        echo -e '\n\n\n\n\n' && sleep 0.5
+        cd $GITHUB_WORKSPACE/${{ matrix.test_dir }}
+        find /opt/hostedtoolcache/CodeQL -path "*x64/codeql" -exec $GITHUB_WORKSPACE/python-setup/auto_install_packages.py {} \;
+    - name: Setup for extractor
+      run: |
+        echo $CODEQL_PYTHON
+        # only run if $CODEQL_PYTHON is set
+        test ! -z $CODEQL_PYTHON && $GITHUB_WORKSPACE/python-setup/tests/from_python_exe.py $CODEQL_PYTHON || /bin/true
+    - name: Verify packages installed
+      run: |
+        ${{ matrix.test_script }}

init/action.yml

@@ -19,6 +19,10 @@ inputs:
   queries:
     description: Comma-separated list of additional queries to run. By default, this overrides the same setting in a configuration file; prefix with "+" to use both sets of queries.
     required: false
+  setup-python-dependencies:
+    description: Try to auto-install your python dependencies
+    required: true
+    default: 'true'
 runs:
   using: 'node12'
   main: '../lib/init-action.js'

python-setup/auto_install_packages.py

@@ -0,0 +1,146 @@
+#!/usr/bin/env python3
+
+import sys
+import os
+import subprocess
+from tempfile import mkdtemp
+from typing import Optional
+
+import extractor_version
+
+
+def _check_call(command):
+    print('+ {}'.format(' '.join(command)), flush=True)
+    subprocess.check_call(command, stdin=subprocess.DEVNULL)
+
+
+def _check_output(command):
+    print('+ {}'.format(' '.join(command)), flush=True)
+    out = subprocess.check_output(command, stdin=subprocess.DEVNULL)
+    print(out, flush=True)
+    sys.stderr.flush()
+    return out
+
+
+def install_packages_with_poetry():
+    try:
+        _check_call(['poetry', 'install', '--no-root'])
+    except subprocess.CalledProcessError:
+        sys.exit('package installation with poetry failed, see error above')
+
+    # poetry is super annoying with `poetry run`, since it will put lots of output on
+    # STDOUT if the current global python interpreter is not matching the one in the
+    # virtualenv for the package, which was the case for using poetry for Python 2 when
+    # default system interpreter was Python 3 :/
+
+    poetry_out = _check_output(['poetry', 'run', 'which', 'python'])
+    python_executable_path = poetry_out.decode('utf-8').splitlines()[-1]
+
+    return python_executable_path
+
+
+def install_packages_with_pipenv():
+    try:
+        _check_call(['pipenv', 'install', '--keep-outdated', '--ignore-pipfile'])
+    except subprocess.CalledProcessError:
+        sys.exit('package installation with pipenv failed, see error above')
+
+    pipenv_out = _check_output(['pipenv', 'run', 'which', 'python'])
+    python_executable_path = pipenv_out.decode('utf-8').splitlines()[-1]
+
+    return python_executable_path
+
+
+def _create_venv(version: int):
+    # create temporary directory ... that just lives "forever"
+    venv_path = mkdtemp(prefix='codeql-action-python-autoinstall-')
+
+    # virtualenv is a bit nicer for setting up virtual environment, since it will provide
+    # up-to-date versions of pip/setuptools/wheel which basic `python3 -m venv venv` won't
+
+    if version == 2:
+        _check_call(['python2', '-m', 'virtualenv', venv_path])
+    elif version == 3:
+        _check_call(['python3', '-m', 'virtualenv', venv_path])
+
+    return venv_path
+
+
+def install_requirements_txt_packages(version: int):
+    venv_path = _create_venv(version)
+    venv_pip = os.path.join(venv_path, 'bin', 'pip')
+    venv_python = os.path.join(venv_path, 'bin', 'python')
+
+    try:
+        _check_call([venv_pip, 'install', '-r', 'requirements.txt'])
+    except subprocess.CalledProcessError:
+        sys.exit('package installation with `pip install -r requirements.txt` failed, see error above')
+
+    return venv_python
+
+
+def install_with_setup_py(version: int):
+    venv_path = _create_venv(version)
+    venv_pip = os.path.join(venv_path, 'bin', 'pip')
+    venv_python = os.path.join(venv_path, 'bin', 'python')
+
+    try:
+        # We have to choose between `python setup.py develop` and `pip install -e .`.
+        # Modern projects use `pip install -e .` and I wasn't able to see any downsides
+        # to doing so. However, `python setup.py develop` has some downsides -- from
+        # https://stackoverflow.com/a/19048754 :
+        # > Note that it is highly recommended to use pip install . (install) and pip
+        # > install -e . (developer install) to install packages, as invoking setup.py
+        # > directly will do the wrong things for many dependencies, such as pull
+        # > prereleases and incompatible package versions, or make the package hard to
+        # > uninstall with pip.
+
+        _check_call([venv_pip, 'install', '-e', '.'])
+    except subprocess.CalledProcessError:
+        sys.exit('package installation with `pip install -e .` failed, see error above')
+
+    return venv_python
+
+
+def install_packages(codeql_base_dir) -> Optional[str]:
+    if os.path.exists('poetry.lock'):
+        print('Found poetry.lock, will install packages with poetry', flush=True)
+        return install_packages_with_poetry()
+
+    if os.path.exists('Pipfile') or os.path.exists('Pipfile.lock'):
+        if os.path.exists('Pipfile.lock'):
+            print('Found Pipfile.lock, will install packages with Pipenv', flush=True)
+        else:
+            print('Found Pipfile, will install packages with Pipenv', flush=True)
+        return install_packages_with_pipenv()
+
+    # get_extractor_version returns the Python version the extractor thinks this repo is using
+    version = extractor_version.get_extractor_version(codeql_base_dir, quiet=False)
+
+    if os.path.exists('requirements.txt'):
+        print('Found requirements.txt, will install packages with pip', flush=True)
+        return install_requirements_txt_packages(version)
+
+    if os.path.exists('setup.py'):
+        print('Found setup.py, will install package with pip in editable mode', flush=True)
+        return install_with_setup_py(version)
+
+    print("was not able to install packages automatically", flush=True)
+    return None
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 2:
+        sys.exit('Must provide base directory for codeql tool as only argument')
+
+    codeql_base_dir = sys.argv[1]
+
+    # The binaries for packages installed with `pip install --user` are not available on
+    # PATH by default, so we need to manually add them.
+    os.environ['PATH'] = os.path.expanduser('~/.local/bin') + os.pathsep + os.environ['PATH']
+
+    python_executable_path = install_packages(codeql_base_dir)
+
+    if python_executable_path is not None:
+        print("Setting CODEQL_PYTHON={}".format(python_executable_path))
+        print("::set-env name=CODEQL_PYTHON::{}".format(python_executable_path))

python-setup/extractor_version.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+
+# A quick hack to get package installation for Code Scanning to work,
+# since it needs to know which version we're going to analyze the project as.
+
+# This file needs to be placed next to `python_tracer.py`, so in
+# `<codeql-path>/python/tools/`
+
+from __future__ import print_function, division
+
+import os
+import sys
+from contextlib import contextmanager
+
+
+@contextmanager
+def suppress_stdout_stderr():
+    # taken from
+    # https://thesmithfam.org/blog/2012/10/25/temporarily-suppress-console-output-in-python/
+    with open(os.devnull, "w") as devnull:
+        old_stdout = sys.stdout
+        old_stderr = sys.stderr
+        sys.stdout = devnull
+        sys.stderr = devnull
+        try:
+            yield
+        finally:
+            sys.stdout = old_stdout
+            sys.stderr = old_stderr
+
+
+def get_extractor_version(codeql_base_dir: str, quiet: bool = True) -> int:
+    extractor_dir = os.path.join(codeql_base_dir, 'python', 'tools')
+    sys.path = [extractor_dir] + sys.path
+
+    from python_tracer import getzipfilename
+
+    zippath = os.path.join(extractor_dir, getzipfilename())
+    sys.path = [zippath] + sys.path
+    import buildtools.discover
+
+    if quiet:
+        with suppress_stdout_stderr():
+            return buildtools.discover.get_version()
+    else:
+        return buildtools.discover.get_version()
+
+
+if __name__ == "__main__":
+    codeql_base_dir = sys.argv[1]
+    version = get_extractor_version(codeql_base_dir)
+    print('{!r}'.format(version))

python-setup/install_tools.sh

@@ -0,0 +1,33 @@
+#!/bin/sh
+set -x
+set -e
+
+# The binaries for packages installed with `pip install --user` are not available on PATH
+# by default, so we fix up PATH to suppress warnings by pip. This also needs to be done by
+# any script that needs to access poetry/pipenv.
+#
+# Using `::add-path::` from the actions toolkit is not enough, since that only affects
+# subsequent actions in the current job, and not the current action.
+export PATH="$HOME/.local/bin:$PATH"
+
+python2 -m pip install --user --upgrade pip setuptools wheel
+python3 -m pip install --user --upgrade pip setuptools wheel
+
+# virtualenv is a bit nicer for setting up virtual environment, since it will provide up-to-date versions of
+# pip/setuptools/wheel which basic `python3 -m venv venv` won't
+python2 -m pip install --user virtualenv
+python3 -m pip install --user virtualenv
+
+# venv is required for installation of poetry or pipenv (I forgot which)
+sudo apt-get install -y python3-venv
+
+# We install poetry with pip instead of the recommended way, since the recommended way
+# caused some problem since `poetry run` gives output like:
+#
+#     /root/.poetry/lib/poetry/_vendor/py2.7/subprocess32.py:149: RuntimeWarning: The _posixsubprocess module is not being used. Child process reliability may suffer if your program uses threads.
+#       "program uses threads.", RuntimeWarning)
+#     LGTM_PYTHON_SETUP_VERSION=The currently activated Python version 2.7.18 is not supported by the project (^3.5). Trying to find and use a compatible version. Using python3 (3.8.2) 3
+
+# poetry 1.0.10 has error (https://github.com/python-poetry/poetry/issues/2711)
+python3 -m pip install --user poetry!=1.0.10
+python3 -m pip install --user pipenv

python-setup/tests/check_requests_123.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+set -e
+
+SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+EXPECTED_VERSION=$1
+
+FOUND_VERSION="$LGTM_PYTHON_SETUP_VERSION"
+FOUND_PYTHONPATH="$LGTM_INDEX_IMPORT_PATH"
+
+echo "FOUND_VERSION=${FOUND_VERSION} FOUND_PYTHONPATH=${FOUND_PYTHONPATH} "
+
+if [[ $FOUND_VERSION != $EXPECTED_VERSION ]]; then
+    echo "Script told us to use Python ${FOUND_VERSION}, but expected ${EXPECTED_VERSION}"
+    exit 1
+else
+    echo "Script told us to use Python ${FOUND_VERSION}, which was expected"
+fi
+
+PYTHON_EXE="python${EXPECTED_VERSION}"
+
+INSTALLED_REQUESTS_VERSION=$(PYTHONPATH="${FOUND_PYTHONPATH}" "${PYTHON_EXE}" -c 'import requests; print(requests.__version__)')
+
+EXPECTED_REQUESTS="1.2.3"
+
+if [[ "$INSTALLED_REQUESTS_VERSION" != "$EXPECTED_REQUESTS" ]]; then
+    echo "Using ${FOUND_PYTHONPATH} as PYTHONPATH, we found version $INSTALLED_REQUESTS_VERSION of requests, but expected $EXPECTED_REQUESTS"
+    exit 1
+else
+    echo "Using ${FOUND_PYTHONPATH} as PYTHONPATH, we found version $INSTALLED_REQUESTS_VERSION of requests, which was expected"
+fi

python-setup/tests/from_python_exe.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+
+import sys
+import subprocess
+from typing import Tuple
+
+def get_details(path_to_python_exe: str) -> Tuple[str, str]:
+    import_path = subprocess.check_output(
+        [
+            path_to_python_exe,
+            "-c",
+            "import os; import pip; print(os.path.dirname(os.path.dirname(pip.__file__)))",
+        ],
+        stdin=subprocess.DEVNULL,
+    )
+    version = subprocess.check_output(
+        [path_to_python_exe, "-c", "import sys; print(sys.version_info[0])"],
+        stdin=subprocess.DEVNULL,
+    )
+
+    return version.decode("utf-8").strip(), import_path.decode("utf-8").strip()
+
+
+if __name__ == "__main__":
+    version, import_path = get_details(sys.argv[1])
+
+    print("Setting LGTM_PYTHON_SETUP_VERSION={}".format(version))
+    print("::set-env name=LGTM_PYTHON_SETUP_VERSION::{}".format(version))
+
+    print("Setting LGTM_INDEX_IMPORT_PATH={}".format(import_path))
+    print("::set-env name=LGTM_INDEX_IMPORT_PATH::{}".format(import_path))

python-setup/tests/pipenv/python-3.8/Pipfile

@@ -0,0 +1,12 @@
+[[source]]
+name = "pypi"
+url = "https://pypi.org/simple"
+verify_ssl = true
+
+[dev-packages]
+
+[packages]
+requests = "*"
+
+[requires]
+python_version = "3.8"

python-setup/tests/pipenv/requests-2/Pipfile

@@ -0,0 +1,12 @@
+[[source]]
+name = "pypi"
+url = "https://pypi.org/simple"
+verify_ssl = true
+
+[dev-packages]
+
+[packages]
+requests = "*"
+
+[requires]
+python_version = "2.7"

python-setup/tests/pipenv/requests-3/Pipfile

@@ -0,0 +1,11 @@
+[[source]]
+name = "pypi"
+url = "https://pypi.org/simple"
+verify_ssl = true
+
+[dev-packages]
+
+[packages]
+requests = "*"
+
+[requires]

python-setup/tests/poetry/python-3.8/pyproject.toml

@@ -0,0 +1,15 @@
+[tool.poetry]
+name = "autoinstall-test"
+version = "0.1.0"
+description = ""
+authors = ["Your Name <you@example.com>"]
+
+[tool.poetry.dependencies]
+python = "^3.8"
+requests = "*"
+
+[tool.poetry.dev-dependencies]
+
+[build-system]
+requires = ["poetry>=0.12"]
+build-backend = "poetry.masonry.api"

python-setup/tests/poetry/requests-2/pyproject.toml

@@ -0,0 +1,15 @@
+[tool.poetry]
+name = "autoinstall-test"
+version = "0.1.0"
+description = ""
+authors = ["Your Name <you@example.com>"]
+
+[tool.poetry.dependencies]
+python = "^2.7"
+requests = "*"
+
+[tool.poetry.dev-dependencies]
+
+[build-system]
+requires = ["poetry>=0.12"]
+build-backend = "poetry.masonry.api"

python-setup/tests/poetry/requests-3/pyproject.toml

@@ -0,0 +1,15 @@
+[tool.poetry]
+name = "autoinstall-test"
+version = "0.1.0"
+description = ""
+authors = ["Your Name <you@example.com>"]
+
+[tool.poetry.dependencies]
+python = "^3.5"
+requests = "*"
+
+[tool.poetry.dev-dependencies]
+
+[build-system]
+requires = ["poetry>=0.12"]
+build-backend = "poetry.masonry.api"

python-setup/tests/requirements/non-standard-location/non-standard/requirements.txt

@@ -0,0 +1 @@
+requests==1.2.3

python-setup/tests/requirements/non-standard-location/wat.py

@@ -0,0 +1 @@
+print('hello')

python-setup/tests/requirements/requests-2/requirements.txt

@@ -0,0 +1 @@
+requests==1.2.3

python-setup/tests/requirements/requests-2/setup.py

@@ -0,0 +1,3 @@
+# fake setup.py with Trove classifier to fool Python extractor to believe this is Python 2 for sure
+
+# Programming Language :: Python :: 2.7

python-setup/tests/requirements/requests-3/requirements.txt

@@ -0,0 +1 @@
+requests==1.2.3

python-setup/tests/requirements/requests-3/setup.py

@@ -0,0 +1,3 @@
+# fake setup.py with Trove classifier to fool Python extractor to believe this is Python 3 for sure
+
+# Programming Language :: Python :: 3.7

python-setup/tests/setup_py/requests-2/setup.py

@@ -0,0 +1,12 @@
+from setuptools import setup
+
+# has fake Trove classifier to fool Python extractor to believe this is Python 2 for sure
+
+# Programming Language :: Python :: 2.7
+
+
+setup(
+    name="example-setup.py",
+    install_requires=["requests==1.2.3"],
+    python_requires=">=2.7, <3",
+)

python-setup/tests/setup_py/requests-3/setup.py

@@ -0,0 +1,12 @@
+from setuptools import setup
+
+# has fake Trove classifier to fool Python extractor to believe this is Python 3 for sure
+
+# Programming Language :: Python :: 3.7
+
+
+setup(
+    name="example-setup.py",
+    install_requires=["requests==1.2.3"],
+    python_requires='>=3.5',
+)

src/analyze.ts

@@ -1,10 +1,12 @@
 import * as fs from "fs";
 import * as path from "path";
 
+import * as toolrunnner from "@actions/exec/lib/toolrunner";
+
 import * as analysisPaths from "./analysis-paths";
 import { getCodeQL } from "./codeql";
 import * as configUtils from "./config-utils";
-import { isScannedLanguage } from "./languages";
+import { isScannedLanguage, Language } from "./languages";
 import { Logger } from "./logging";
 import { RepositoryNwo } from "./repository";
 import * as sharedEnv from "./shared-environment";
@@ -44,6 +46,43 @@ export interface AnalysisStatusReport
   extends upload_lib.UploadStatusReport,
     QueriesStatusReport {}
 
+async function setupPythonExtractor(logger: Logger) {
+  const codeqlPython = process.env["CODEQL_PYTHON"];
+  if (codeqlPython === undefined || codeqlPython.length === 0) {
+    // If CODEQL_PYTHON is not set, no dependencies were installed, so we don't need to do anything
+    return;
+  }
+
+  let output = "";
+  const options = {
+    listeners: {
+      stdout: (data: Buffer) => {
+        output += data.toString();
+      },
+    },
+  };
+
+  await new toolrunnner.ToolRunner(
+    codeqlPython,
+    [
+      "-c",
+      "import os; import pip; print(os.path.dirname(os.path.dirname(pip.__file__)))",
+    ],
+    options
+  ).exec();
+  logger.info(`Setting LGTM_INDEX_IMPORT_PATH=${output}`);
+  process.env["LGTM_INDEX_IMPORT_PATH"] = output;
+
+  output = "";
+  await new toolrunnner.ToolRunner(
+    codeqlPython,
+    ["-c", "import sys; print(sys.version_info[0])"],
+    options
+  ).exec();
+  logger.info(`Setting LGTM_PYTHON_SETUP_VERSION=${output}`);
+  process.env["LGTM_PYTHON_SETUP_VERSION"] = output;
+}
+
 async function createdDBForScannedLanguages(
   config: configUtils.Config,
   logger: Logger
@@ -56,6 +95,11 @@ async function createdDBForScannedLanguages(
   for (const language of config.languages) {
     if (isScannedLanguage(language)) {
       logger.startGroup(`Extracting ${language}`);
+
+      if (language === Language.python) {
+        await setupPythonExtractor(logger);
+      }
+
       await codeql.extractScannedLanguage(
         util.getCodeQLDatabasePath(config.tempDir, language),
         language

src/init-action.ts

@@ -3,7 +3,13 @@ import * as core from "@actions/core";
 import * as actionsUtil from "./actions-util";
 import { CodeQL } from "./codeql";
 import * as configUtils from "./config-utils";
-import { initCodeQL, initConfig, injectWindowsTracer, runInit } from "./init";
+import {
+  initCodeQL,
+  initConfig,
+  injectWindowsTracer,
+  installPythonDeps,
+  runInit,
+} from "./init";
 import { getActionsLogger } from "./logging";
 import { parseRepositoryNwo } from "./repository";
 
@@ -111,6 +117,14 @@ async function run() {
       actionsUtil.getRequiredEnvParam("GITHUB_SERVER_URL"),
       logger
     );
+
+    try {
+      await installPythonDeps(codeql, logger);
+    } catch (err) {
+      logger.warning(
+        `${err.message} You can call this action with 'setup-python-dependencies: false' to disable this process`
+      );
+    }
   } catch (e) {
     core.setFailed(e.message);
     console.log(e);

src/init.ts

@@ -182,3 +182,47 @@ export async function injectWindowsTracer(
     { env: { ODASA_TRACER_CONFIGURATION: tracerConfig.spec } }
   ).exec();
 }
+
+export async function installPythonDeps(codeql: CodeQL, logger: Logger) {
+  logger.startGroup("Setup Python dependencies");
+
+  if (process.platform !== "linux") {
+    logger.info(
+      "Currently, auto-installing python dependancies is only supported on linux"
+    );
+    logger.endGroup();
+    return;
+  }
+
+  const scriptsFolder = path.resolve(__dirname, "../python-setup");
+
+  // Setup tools on the Github hosted runners
+  if (process.env["ImageOS"] !== undefined) {
+    try {
+      await new toolrunnner.ToolRunner(
+        path.join(scriptsFolder, "install_tools.sh")
+      ).exec();
+    } catch (e) {
+      // This script tries to install some needed tools in the runner. It should not fail, but if it does
+      // we just abort the process without failing the action
+      logger.endGroup();
+      logger.warning(
+        "Unable to download and extract the tools needed for installing the python dependecies. You can call this action with 'setup-python-dependencies: false' to disable this process."
+      );
+    }
+  }
+
+  // Install dependencies
+  try {
+    await new toolrunnner.ToolRunner(
+      path.join(scriptsFolder, "auto_install_packages.py"),
+      [path.dirname(codeql.getPath())]
+    ).exec();
+  } catch (e) {
+    logger.endGroup();
+    logger.warning(
+      "We were unable to install your python dependencies. You can call this action with 'setup-python-dependencies: false' to disable this process."
+    );
+  }
+  logger.endGroup();
+}