diff --git a/.github/workflows/python-deps.yml b/.github/workflows/python-deps.yml new file mode 100644 index 000000000..75be193b3 --- /dev/null +++ b/.github/workflows/python-deps.yml @@ -0,0 +1,63 @@ +name: Test Python Package Installation + +on: + push: + branches: [main, v1] + pull_request: + +jobs: + + test-setup-python-scripts: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - test_dir: python-setup/tests/pipenv/requests-2 + test_script: $GITHUB_WORKSPACE/python-setup/tests/check_requests_123.sh 2 + - test_dir: python-setup/tests/pipenv/requests-3 + test_script: $GITHUB_WORKSPACE/python-setup/tests/check_requests_123.sh 3 + + - test_dir: python-setup/tests/poetry/requests-2 + test_script: $GITHUB_WORKSPACE/python-setup/tests/check_requests_123.sh 2 + - test_dir: python-setup/tests/poetry/requests-3 + test_script: $GITHUB_WORKSPACE/python-setup/tests/check_requests_123.sh 3 + + - test_dir: python-setup/tests/requirements/requests-2 + test_script: $GITHUB_WORKSPACE/python-setup/tests/check_requests_123.sh 2 + - test_dir: python-setup/tests/requirements/requests-3 + test_script: $GITHUB_WORKSPACE/python-setup/tests/check_requests_123.sh 3 + + - test_dir: python-setup/tests/setup_py/requests-2 + test_script: $GITHUB_WORKSPACE/python-setup/tests/check_requests_123.sh 2 + - test_dir: python-setup/tests/setup_py/requests-3 + test_script: $GITHUB_WORKSPACE/python-setup/tests/check_requests_123.sh 3 + + # This one shouldn't fail, but also won't install packages + - test_dir: python-setup/tests/requirements/non-standard-location + test_script: test -z $LGTM_INDEX_IMPORT_PATH + + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v2 + + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: python + + - name: Test Auto Package Installation + run: | + set -x + $GITHUB_WORKSPACE/python-setup/install_tools.sh + echo -e '\n\n\n\n\n' && sleep 0.5 + cd $GITHUB_WORKSPACE/${{ matrix.test_dir }} + find /opt/hostedtoolcache/CodeQL -path "*x64/codeql" -exec $GITHUB_WORKSPACE/python-setup/auto_install_packages.py {} \; + - name: Setup for extractor + run: | + echo $CODEQL_PYTHON + # only run if $CODEQL_PYTHON is set + test ! -z $CODEQL_PYTHON && $GITHUB_WORKSPACE/python-setup/tests/from_python_exe.py $CODEQL_PYTHON || /bin/true + - name: Verify packages installed + run: | + ${{ matrix.test_script }} \ No newline at end of file diff --git a/init/action.yml b/init/action.yml index 0b3ea9da8..f2c1519c7 100644 --- a/init/action.yml +++ b/init/action.yml @@ -19,6 +19,10 @@ inputs: queries: description: Comma-separated list of additional queries to run. By default, this overrides the same setting in a configuration file; prefix with "+" to use both sets of queries. required: false + setup-python-dependencies: + description: Try to auto-install your python dependencies + required: true + default: 'true' runs: using: 'node12' main: '../lib/init-action.js' diff --git a/lib/analyze.js b/lib/analyze.js index a8b186514..f765756c8 100644 --- a/lib/analyze.js +++ b/lib/analyze.js @@ -9,12 +9,38 @@ var __importStar = (this && this.__importStar) || function (mod) { Object.defineProperty(exports, "__esModule", { value: true }); const fs = __importStar(require("fs")); const path = __importStar(require("path")); +const toolrunnner = __importStar(require("@actions/exec/lib/toolrunner")); const analysisPaths = __importStar(require("./analysis-paths")); const codeql_1 = require("./codeql"); const languages_1 = require("./languages"); const sharedEnv = __importStar(require("./shared-environment")); const upload_lib = __importStar(require("./upload-lib")); const util = __importStar(require("./util")); +async function setupPythonExtractor(logger) { + const codeqlPython = process.env["CODEQL_PYTHON"]; + if (codeqlPython === undefined || codeqlPython.length === 0) { + // If CODEQL_PYTHON is not set, no dependencies were installed, so we don't need to do anything + return; + } + let output = ""; + const options = { + listeners: { + stdout: (data) => { + output += data.toString(); + }, + }, + }; + await new toolrunnner.ToolRunner(codeqlPython, [ + "-c", + "import os; import pip; print(os.path.dirname(os.path.dirname(pip.__file__)))", + ], options).exec(); + logger.info(`Setting LGTM_INDEX_IMPORT_PATH=${output}`); + process.env["LGTM_INDEX_IMPORT_PATH"] = output; + output = ""; + await new toolrunnner.ToolRunner(codeqlPython, ["-c", "import sys; print(sys.version_info[0])"], options).exec(); + logger.info(`Setting LGTM_PYTHON_SETUP_VERSION=${output}`); + process.env["LGTM_PYTHON_SETUP_VERSION"] = output; +} async function createdDBForScannedLanguages(config, logger) { // Insert the LGTM_INDEX_X env vars at this point so they are set when // we extract any scanned languages. @@ -23,6 +49,9 @@ async function createdDBForScannedLanguages(config, logger) { for (const language of config.languages) { if (languages_1.isScannedLanguage(language)) { logger.startGroup(`Extracting ${language}`); + if (language === languages_1.Language.python) { + await setupPythonExtractor(logger); + } await codeql.extractScannedLanguage(util.getCodeQLDatabasePath(config.tempDir, language), language); logger.endGroup(); } diff --git a/lib/analyze.js.map b/lib/analyze.js.map index 6dbfba9ce..702372ad8 100644 --- a/lib/analyze.js.map +++ b/lib/analyze.js.map @@ -1 +1 @@ -{"version":3,"file":"analyze.js","sourceRoot":"","sources":["../src/analyze.ts"],"names":[],"mappings":";;;;;;;;;AAAA,uCAAyB;AACzB,2CAA6B;AAE7B,gEAAkD;AAClD,qCAAqC;AAErC,2CAAgD;AAGhD,gEAAkD;AAClD,yDAA2C;AAC3C,6CAA+B;AAmC/B,KAAK,UAAU,4BAA4B,CACzC,MAA0B,EAC1B,MAAc;IAEd,sEAAsE;IACtE,oCAAoC;IACpC,aAAa,CAAC,8BAA8B,CAAC,MAAM,CAAC,CAAC;IAErD,MAAM,MAAM,GAAG,kBAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAC3C,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACvC,IAAI,6BAAiB,CAAC,QAAQ,CAAC,EAAE;YAC/B,MAAM,CAAC,UAAU,CAAC,cAAc,QAAQ,EAAE,CAAC,CAAC;YAC5C,MAAM,MAAM,CAAC,sBAAsB,CACjC,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,EACpD,QAAQ,CACT,CAAC;YACF,MAAM,CAAC,QAAQ,EAAE,CAAC;SACnB;KACF;AACH,CAAC;AAED,KAAK,UAAU,wBAAwB,CACrC,MAA0B,EAC1B,MAAc;IAEd,MAAM,4BAA4B,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEnD,MAAM,MAAM,GAAG,kBAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAC3C,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACvC,MAAM,CAAC,UAAU,CAAC,cAAc,QAAQ,EAAE,CAAC,CAAC;QAC5C,MAAM,MAAM,CAAC,gBAAgB,CAC3B,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,CACrD,CAAC;QACF,MAAM,CAAC,QAAQ,EAAE,CAAC;KACnB;AACH,CAAC;AAED,2DAA2D;AACpD,KAAK,UAAU,UAAU,CAC9B,WAAmB,EACnB,UAAkB,EAClB,eAAuB,EACvB,WAAmB,EACnB,MAA0B,EAC1B,MAAc;IAEd,MAAM,YAAY,GAAwB,EAAE,CAAC;IAE7C,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACvC,MAAM,CAAC,UAAU,CAAC,aAAa,QAAQ,EAAE,CAAC,CAAC;QAE3C,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACzC,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE;YAC/D,MAAM,IAAI,KAAK,CACb,qBAAqB,QAAQ,gDAAgD,CAC9E,CAAC;SACH;QAED,IAAI;YACF,KAAK,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,EAAE;gBACxC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE;oBAC5B,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;oBAEvC,MAAM,YAAY,GAAG,IAAI,CAAC,qBAAqB,CAC7C,MAAM,CAAC,OAAO,EACd,QAAQ,CACT,CAAC;oBACF,uEAAuE;oBACvE,2EAA2E;oBAC3E,MAAM,cAAc,GAAG,GAAG,YAAY,YAAY,IAAI,MAAM,CAAC;oBAC7D,MAAM,kBAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;yBACrC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,YAAY,CAAC,EAAE,CAAC;yBACnC,IAAI,CAAC,IAAI,CAAC,CAAC;oBACd,EAAE,CAAC,aAAa,CAAC,cAAc,EAAE,kBAAkB,CAAC,CAAC;oBACrD,MAAM,CAAC,KAAK,CACV,wBAAwB,QAAQ,QAAQ,kBAAkB,EAAE,CAC7D,CAAC;oBAEF,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,QAAQ,IAAI,IAAI,QAAQ,CAAC,CAAC;oBAEtE,MAAM,MAAM,GAAG,kBAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;oBAC3C,MAAM,MAAM,CAAC,eAAe,CAC1B,YAAY,EACZ,SAAS,EACT,cAAc,EACd,UAAU,EACV,eAAe,EACf,WAAW,CACZ,CAAC;oBAEF,MAAM,CAAC,KAAK,CACV,8BAA8B,QAAQ,gBAAgB,SAAS,GAAG,CACnE,CAAC;oBACF,MAAM,CAAC,QAAQ,EAAE,CAAC;oBAElB,yBAAyB;oBACzB,MAAM,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;oBACrC,YAAY,CAAC,WAAW,IAAI,YAAY,QAAQ,cAAc,CAAC;wBAC7D,OAAO,GAAG,SAAS,CAAC;iBACvB;aACF;SACF;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,KAAK,CAAC,8BAA8B,QAAQ,KAAK,CAAC,EAAE,CAAC,CAAC;YAC7D,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACf,YAAY,CAAC,wBAAwB,GAAG,QAAQ,CAAC;YACjD,OAAO,YAAY,CAAC;SACrB;KACF;IAED,OAAO,YAAY,CAAC;AACtB,CAAC;AAxED,gCAwEC;AAEM,KAAK,UAAU,UAAU,CAC9B,aAA4B,EAC5B,SAAiB,EACjB,GAAW,EACX,WAA+B,EAC/B,YAAgC,EAChC,aAAiC,EACjC,YAAoB,EACpB,WAA+B,EAC/B,UAAkB,EAClB,SAAiB,EACjB,QAAiB,EACjB,IAAe,EACf,SAAiB,EACjB,UAAkB,EAClB,eAAuB,EACvB,WAAmB,EACnB,MAA0B,EAC1B,MAAc;IAEd,8DAA8D;IAC9D,OAAO,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,0BAA0B,CAAC,CAAC;IAEzD,EAAE,CAAC,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE7C,MAAM,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;IAC5C,MAAM,wBAAwB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAE/C,MAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IAClC,MAAM,YAAY,GAAG,MAAM,UAAU,CACnC,SAAS,EACT,UAAU,EACV,eAAe,EACf,WAAW,EACX,MAAM,EACN,MAAM,CACP,CAAC;IAEF,IAAI,CAAC,QAAQ,EAAE;QACb,MAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;QACrC,OAAO,EAAE,GAAG,YAAY,EAAE,CAAC;KAC5B;IAED,MAAM,WAAW,GAAG,MAAM,UAAU,CAAC,MAAM,CACzC,SAAS,EACT,aAAa,EACb,SAAS,EACT,GAAG,EACH,WAAW,EACX,YAAY,EACZ,aAAa,EACb,YAAY,EACZ,WAAW,EACX,UAAU,EACV,SAAS,EACT,IAAI,EACJ,MAAM,CACP,CAAC;IAEF,OAAO,EAAE,GAAG,YAAY,EAAE,GAAG,WAAW,EAAE,CAAC;AAC7C,CAAC;AA5DD,gCA4DC"} \ No newline at end of file +{"version":3,"file":"analyze.js","sourceRoot":"","sources":["../src/analyze.ts"],"names":[],"mappings":";;;;;;;;;AAAA,uCAAyB;AACzB,2CAA6B;AAE7B,0EAA4D;AAE5D,gEAAkD;AAClD,qCAAqC;AAErC,2CAA0D;AAG1D,gEAAkD;AAClD,yDAA2C;AAC3C,6CAA+B;AAmC/B,KAAK,UAAU,oBAAoB,CAAC,MAAc;IAChD,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;IAClD,IAAI,YAAY,KAAK,SAAS,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE;QAC3D,+FAA+F;QAC/F,OAAO;KACR;IAED,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,MAAM,OAAO,GAAG;QACd,SAAS,EAAE;YACT,MAAM,EAAE,CAAC,IAAY,EAAE,EAAE;gBACvB,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC5B,CAAC;SACF;KACF,CAAC;IAEF,MAAM,IAAI,WAAW,CAAC,UAAU,CAC9B,YAAY,EACZ;QACE,IAAI;QACJ,8EAA8E;KAC/E,EACD,OAAO,CACR,CAAC,IAAI,EAAE,CAAC;IACT,MAAM,CAAC,IAAI,CAAC,kCAAkC,MAAM,EAAE,CAAC,CAAC;IACxD,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,GAAG,MAAM,CAAC;IAE/C,MAAM,GAAG,EAAE,CAAC;IACZ,MAAM,IAAI,WAAW,CAAC,UAAU,CAC9B,YAAY,EACZ,CAAC,IAAI,EAAE,wCAAwC,CAAC,EAChD,OAAO,CACR,CAAC,IAAI,EAAE,CAAC;IACT,MAAM,CAAC,IAAI,CAAC,qCAAqC,MAAM,EAAE,CAAC,CAAC;IAC3D,OAAO,CAAC,GAAG,CAAC,2BAA2B,CAAC,GAAG,MAAM,CAAC;AACpD,CAAC;AAED,KAAK,UAAU,4BAA4B,CACzC,MAA0B,EAC1B,MAAc;IAEd,sEAAsE;IACtE,oCAAoC;IACpC,aAAa,CAAC,8BAA8B,CAAC,MAAM,CAAC,CAAC;IAErD,MAAM,MAAM,GAAG,kBAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAC3C,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACvC,IAAI,6BAAiB,CAAC,QAAQ,CAAC,EAAE;YAC/B,MAAM,CAAC,UAAU,CAAC,cAAc,QAAQ,EAAE,CAAC,CAAC;YAE5C,IAAI,QAAQ,KAAK,oBAAQ,CAAC,MAAM,EAAE;gBAChC,MAAM,oBAAoB,CAAC,MAAM,CAAC,CAAC;aACpC;YAED,MAAM,MAAM,CAAC,sBAAsB,CACjC,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,EACpD,QAAQ,CACT,CAAC;YACF,MAAM,CAAC,QAAQ,EAAE,CAAC;SACnB;KACF;AACH,CAAC;AAED,KAAK,UAAU,wBAAwB,CACrC,MAA0B,EAC1B,MAAc;IAEd,MAAM,4BAA4B,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEnD,MAAM,MAAM,GAAG,kBAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAC3C,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACvC,MAAM,CAAC,UAAU,CAAC,cAAc,QAAQ,EAAE,CAAC,CAAC;QAC5C,MAAM,MAAM,CAAC,gBAAgB,CAC3B,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,CACrD,CAAC;QACF,MAAM,CAAC,QAAQ,EAAE,CAAC;KACnB;AACH,CAAC;AAED,2DAA2D;AACpD,KAAK,UAAU,UAAU,CAC9B,WAAmB,EACnB,UAAkB,EAClB,eAAuB,EACvB,WAAmB,EACnB,MAA0B,EAC1B,MAAc;IAEd,MAAM,YAAY,GAAwB,EAAE,CAAC;IAE7C,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACvC,MAAM,CAAC,UAAU,CAAC,aAAa,QAAQ,EAAE,CAAC,CAAC;QAE3C,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACzC,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE;YAC/D,MAAM,IAAI,KAAK,CACb,qBAAqB,QAAQ,gDAAgD,CAC9E,CAAC;SACH;QAED,IAAI;YACF,KAAK,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,EAAE;gBACxC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE;oBAC5B,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;oBAEvC,MAAM,YAAY,GAAG,IAAI,CAAC,qBAAqB,CAC7C,MAAM,CAAC,OAAO,EACd,QAAQ,CACT,CAAC;oBACF,uEAAuE;oBACvE,2EAA2E;oBAC3E,MAAM,cAAc,GAAG,GAAG,YAAY,YAAY,IAAI,MAAM,CAAC;oBAC7D,MAAM,kBAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;yBACrC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,YAAY,CAAC,EAAE,CAAC;yBACnC,IAAI,CAAC,IAAI,CAAC,CAAC;oBACd,EAAE,CAAC,aAAa,CAAC,cAAc,EAAE,kBAAkB,CAAC,CAAC;oBACrD,MAAM,CAAC,KAAK,CACV,wBAAwB,QAAQ,QAAQ,kBAAkB,EAAE,CAC7D,CAAC;oBAEF,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,QAAQ,IAAI,IAAI,QAAQ,CAAC,CAAC;oBAEtE,MAAM,MAAM,GAAG,kBAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;oBAC3C,MAAM,MAAM,CAAC,eAAe,CAC1B,YAAY,EACZ,SAAS,EACT,cAAc,EACd,UAAU,EACV,eAAe,EACf,WAAW,CACZ,CAAC;oBAEF,MAAM,CAAC,KAAK,CACV,8BAA8B,QAAQ,gBAAgB,SAAS,GAAG,CACnE,CAAC;oBACF,MAAM,CAAC,QAAQ,EAAE,CAAC;oBAElB,yBAAyB;oBACzB,MAAM,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;oBACrC,YAAY,CAAC,WAAW,IAAI,YAAY,QAAQ,cAAc,CAAC;wBAC7D,OAAO,GAAG,SAAS,CAAC;iBACvB;aACF;SACF;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,KAAK,CAAC,8BAA8B,QAAQ,KAAK,CAAC,EAAE,CAAC,CAAC;YAC7D,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACf,YAAY,CAAC,wBAAwB,GAAG,QAAQ,CAAC;YACjD,OAAO,YAAY,CAAC;SACrB;KACF;IAED,OAAO,YAAY,CAAC;AACtB,CAAC;AAxED,gCAwEC;AAEM,KAAK,UAAU,UAAU,CAC9B,aAA4B,EAC5B,SAAiB,EACjB,GAAW,EACX,WAA+B,EAC/B,YAAgC,EAChC,aAAiC,EACjC,YAAoB,EACpB,WAA+B,EAC/B,UAAkB,EAClB,SAAiB,EACjB,QAAiB,EACjB,IAAe,EACf,SAAiB,EACjB,UAAkB,EAClB,eAAuB,EACvB,WAAmB,EACnB,MAA0B,EAC1B,MAAc;IAEd,8DAA8D;IAC9D,OAAO,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,0BAA0B,CAAC,CAAC;IAEzD,EAAE,CAAC,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE7C,MAAM,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;IAC5C,MAAM,wBAAwB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAE/C,MAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IAClC,MAAM,YAAY,GAAG,MAAM,UAAU,CACnC,SAAS,EACT,UAAU,EACV,eAAe,EACf,WAAW,EACX,MAAM,EACN,MAAM,CACP,CAAC;IAEF,IAAI,CAAC,QAAQ,EAAE;QACb,MAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;QACrC,OAAO,EAAE,GAAG,YAAY,EAAE,CAAC;KAC5B;IAED,MAAM,WAAW,GAAG,MAAM,UAAU,CAAC,MAAM,CACzC,SAAS,EACT,aAAa,EACb,SAAS,EACT,GAAG,EACH,WAAW,EACX,YAAY,EACZ,aAAa,EACb,YAAY,EACZ,WAAW,EACX,UAAU,EACV,SAAS,EACT,IAAI,EACJ,MAAM,CACP,CAAC;IAEF,OAAO,EAAE,GAAG,YAAY,EAAE,GAAG,WAAW,EAAE,CAAC;AAC7C,CAAC;AA5DD,gCA4DC"} \ No newline at end of file diff --git a/lib/init-action.js b/lib/init-action.js index 8a83b7ca7..6c35a3eac 100644 --- a/lib/init-action.js +++ b/lib/init-action.js @@ -56,6 +56,12 @@ async function run() { } codeql = await init_1.initCodeQL(actionsUtil.getOptionalInput("tools"), actionsUtil.getRequiredInput("token"), actionsUtil.getRequiredEnvParam("GITHUB_SERVER_URL"), actionsUtil.getRequiredEnvParam("RUNNER_TEMP"), actionsUtil.getRequiredEnvParam("RUNNER_TOOL_CACHE"), "actions", logger); config = await init_1.initConfig(actionsUtil.getOptionalInput("languages"), actionsUtil.getOptionalInput("queries"), actionsUtil.getOptionalInput("config-file"), repository_1.parseRepositoryNwo(actionsUtil.getRequiredEnvParam("GITHUB_REPOSITORY")), actionsUtil.getRequiredEnvParam("RUNNER_TEMP"), actionsUtil.getRequiredEnvParam("RUNNER_TOOL_CACHE"), codeql, actionsUtil.getRequiredEnvParam("GITHUB_WORKSPACE"), actionsUtil.getRequiredInput("token"), actionsUtil.getRequiredEnvParam("GITHUB_SERVER_URL"), logger); + try { + await init_1.installPythonDeps(codeql, logger); + } + catch (err) { + logger.warning(`${err.message} You can call this action with 'setup-python-dependencies: false' to disable this process`); + } } catch (e) { core.setFailed(e.message); diff --git a/lib/init-action.js.map b/lib/init-action.js.map index 2cd0cb2c1..e9ef20a12 100644 --- a/lib/init-action.js.map +++ b/lib/init-action.js.map @@ -1 +1 @@ -{"version":3,"file":"init-action.js","sourceRoot":"","sources":["../src/init-action.ts"],"names":[],"mappings":";;;;;;;;;AAAA,oDAAsC;AAEtC,4DAA8C;AAG9C,iCAA8E;AAC9E,uCAA6C;AAC7C,6CAAkD;AAkBlD,KAAK,UAAU,uBAAuB,CACpC,SAAe,EACf,MAA0B;;IAE1B,MAAM,gBAAgB,GAAG,MAAM,WAAW,CAAC,sBAAsB,CAC/D,MAAM,EACN,SAAS,EACT,SAAS,CACV,CAAC;IAEF,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC7C,MAAM,iBAAiB,GAAG,WAAW,CAAC,gBAAgB,CAAC,WAAW,CAAC,CAAC;IACpE,MAAM,KAAK,GAAG,CAAC,MAAM,CAAC,iBAAiB,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC/D,MAAM,WAAW,GAAG,CAAC,MAAM,CAAC,iBAAiB,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CACvE,GAAG,CACJ,CAAC;IACF,MAAM,qBAAqB,GAAG,MAAM,CAAC,iBAAiB,CACpD,yBAAyB,CAC1B;QACC,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,EAAE,CAAC;IAEP,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,YAAY,SAAG,WAAW,CAAC,gBAAgB,CAAC,SAAS,CAAC,0CAAE,IAAI,EAAE,CAAC;IACnE,IAAI,YAAY,KAAK,SAAS,IAAI,YAAY,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE;QAC9D,OAAO,CAAC,IAAI,CACV,GAAG,CAAC,MAAM,CAAC,iBAAiB,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAC/D,CAAC;KACH;IACD,IAAI,YAAY,KAAK,SAAS,EAAE;QAC9B,YAAY,GAAG,YAAY,CAAC,UAAU,CAAC,GAAG,CAAC;YACzC,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC;YACxB,CAAC,CAAC,YAAY,CAAC;QACjB,OAAO,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;KAC1C;IAED,MAAM,YAAY,GAA4B;QAC5C,GAAG,gBAAgB;QACnB,SAAS;QACT,kBAAkB,EAAE,iBAAiB,IAAI,EAAE;QAC3C,KAAK;QACL,YAAY,EAAE,WAAW;QACzB,uBAAuB,EAAE,qBAAqB;QAC9C,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC;KAC3B,CAAC;IAEF,MAAM,WAAW,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAC;AACnD,CAAC;AAED,KAAK,UAAU,GAAG;IAChB,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;IAC7B,MAAM,MAAM,GAAG,0BAAgB,EAAE,CAAC;IAClC,IAAI,MAA0B,CAAC;IAC/B,IAAI,MAAc,CAAC;IAEnB,IAAI;QACF,WAAW,CAAC,0BAA0B,EAAE,CAAC;QACzC,IACE,CAAC,CAAC,MAAM,WAAW,CAAC,gBAAgB,CAClC,MAAM,WAAW,CAAC,sBAAsB,CAAC,MAAM,EAAE,UAAU,EAAE,SAAS,CAAC,EACvE,IAAI,CACL,CAAC,EACF;YACA,OAAO;SACR;QAED,MAAM,GAAG,MAAM,iBAAU,CACvB,WAAW,CAAC,gBAAgB,CAAC,OAAO,CAAC,EACrC,WAAW,CAAC,gBAAgB,CAAC,OAAO,CAAC,EACrC,WAAW,CAAC,mBAAmB,CAAC,mBAAmB,CAAC,EACpD,WAAW,CAAC,mBAAmB,CAAC,aAAa,CAAC,EAC9C,WAAW,CAAC,mBAAmB,CAAC,mBAAmB,CAAC,EACpD,SAAS,EACT,MAAM,CACP,CAAC;QACF,MAAM,GAAG,MAAM,iBAAU,CACvB,WAAW,CAAC,gBAAgB,CAAC,WAAW,CAAC,EACzC,WAAW,CAAC,gBAAgB,CAAC,SAAS,CAAC,EACvC,WAAW,CAAC,gBAAgB,CAAC,aAAa,CAAC,EAC3C,+BAAkB,CAAC,WAAW,CAAC,mBAAmB,CAAC,mBAAmB,CAAC,CAAC,EACxE,WAAW,CAAC,mBAAmB,CAAC,aAAa,CAAC,EAC9C,WAAW,CAAC,mBAAmB,CAAC,mBAAmB,CAAC,EACpD,MAAM,EACN,WAAW,CAAC,mBAAmB,CAAC,kBAAkB,CAAC,EACnD,WAAW,CAAC,gBAAgB,CAAC,OAAO,CAAC,EACrC,WAAW,CAAC,mBAAmB,CAAC,mBAAmB,CAAC,EACpD,MAAM,CACP,CAAC;KACH;IAAC,OAAO,CAAC,EAAE;QACV,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAC1B,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACf,MAAM,WAAW,CAAC,gBAAgB,CAChC,MAAM,WAAW,CAAC,sBAAsB,CACtC,MAAM,EACN,SAAS,EACT,SAAS,EACT,CAAC,CAAC,OAAO,CACV,CACF,CAAC;QACF,OAAO;KACR;IAED,IAAI;QACF,mBAAmB;QACnB,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACvC,IAAI,OAAO,EAAE;YACX,IAAI,CAAC,cAAc,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;YACxC,IAAI,CAAC,OAAO,CACV,6GAA6G,CAC9G,CAAC;SACH;QAED,mGAAmG;QACnG,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,MAAM,CAAC;QACtD,IAAI,CAAC,cAAc,CAAC,YAAY,EAAE,SAAS,CAAC,CAAC;QAE7C,MAAM,YAAY,GAAG,MAAM,cAAO,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACnD,IAAI,YAAY,KAAK,SAAS,EAAE;YAC9B,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE;gBAC3D,IAAI,CAAC,cAAc,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;aACjC;YAED,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE;gBAChC,MAAM,0BAAmB,CACvB,mBAAmB,EACnB,SAAS,EACT,MAAM,EACN,MAAM,EACN,YAAY,CACb,CAAC;aACH;SACF;KACF;IAAC,OAAO,KAAK,EAAE;QACd,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC9B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACnB,MAAM,WAAW,CAAC,gBAAgB,CAChC,MAAM,WAAW,CAAC,sBAAsB,CACtC,MAAM,EACN,SAAS,EACT,SAAS,EACT,KAAK,CAAC,OAAO,EACb,KAAK,CAAC,KAAK,CACZ,CACF,CAAC;QACF,OAAO;KACR;IACD,MAAM,uBAAuB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;AACnD,CAAC;AAED,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;IAChB,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC,EAAE,CAAC,CAAC;IAC3C,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AACjB,CAAC,CAAC,CAAC"} \ No newline at end of file +{"version":3,"file":"init-action.js","sourceRoot":"","sources":["../src/init-action.ts"],"names":[],"mappings":";;;;;;;;;AAAA,oDAAsC;AAEtC,4DAA8C;AAG9C,iCAMgB;AAChB,uCAA6C;AAC7C,6CAAkD;AAkBlD,KAAK,UAAU,uBAAuB,CACpC,SAAe,EACf,MAA0B;;IAE1B,MAAM,gBAAgB,GAAG,MAAM,WAAW,CAAC,sBAAsB,CAC/D,MAAM,EACN,SAAS,EACT,SAAS,CACV,CAAC;IAEF,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC7C,MAAM,iBAAiB,GAAG,WAAW,CAAC,gBAAgB,CAAC,WAAW,CAAC,CAAC;IACpE,MAAM,KAAK,GAAG,CAAC,MAAM,CAAC,iBAAiB,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC/D,MAAM,WAAW,GAAG,CAAC,MAAM,CAAC,iBAAiB,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CACvE,GAAG,CACJ,CAAC;IACF,MAAM,qBAAqB,GAAG,MAAM,CAAC,iBAAiB,CACpD,yBAAyB,CAC1B;QACC,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,EAAE,CAAC;IAEP,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,YAAY,SAAG,WAAW,CAAC,gBAAgB,CAAC,SAAS,CAAC,0CAAE,IAAI,EAAE,CAAC;IACnE,IAAI,YAAY,KAAK,SAAS,IAAI,YAAY,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE;QAC9D,OAAO,CAAC,IAAI,CACV,GAAG,CAAC,MAAM,CAAC,iBAAiB,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAC/D,CAAC;KACH;IACD,IAAI,YAAY,KAAK,SAAS,EAAE;QAC9B,YAAY,GAAG,YAAY,CAAC,UAAU,CAAC,GAAG,CAAC;YACzC,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC;YACxB,CAAC,CAAC,YAAY,CAAC;QACjB,OAAO,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;KAC1C;IAED,MAAM,YAAY,GAA4B;QAC5C,GAAG,gBAAgB;QACnB,SAAS;QACT,kBAAkB,EAAE,iBAAiB,IAAI,EAAE;QAC3C,KAAK;QACL,YAAY,EAAE,WAAW;QACzB,uBAAuB,EAAE,qBAAqB;QAC9C,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC;KAC3B,CAAC;IAEF,MAAM,WAAW,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAC;AACnD,CAAC;AAED,KAAK,UAAU,GAAG;IAChB,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;IAC7B,MAAM,MAAM,GAAG,0BAAgB,EAAE,CAAC;IAClC,IAAI,MAA0B,CAAC;IAC/B,IAAI,MAAc,CAAC;IAEnB,IAAI;QACF,WAAW,CAAC,0BAA0B,EAAE,CAAC;QACzC,IACE,CAAC,CAAC,MAAM,WAAW,CAAC,gBAAgB,CAClC,MAAM,WAAW,CAAC,sBAAsB,CAAC,MAAM,EAAE,UAAU,EAAE,SAAS,CAAC,EACvE,IAAI,CACL,CAAC,EACF;YACA,OAAO;SACR;QAED,MAAM,GAAG,MAAM,iBAAU,CACvB,WAAW,CAAC,gBAAgB,CAAC,OAAO,CAAC,EACrC,WAAW,CAAC,gBAAgB,CAAC,OAAO,CAAC,EACrC,WAAW,CAAC,mBAAmB,CAAC,mBAAmB,CAAC,EACpD,WAAW,CAAC,mBAAmB,CAAC,aAAa,CAAC,EAC9C,WAAW,CAAC,mBAAmB,CAAC,mBAAmB,CAAC,EACpD,SAAS,EACT,MAAM,CACP,CAAC;QACF,MAAM,GAAG,MAAM,iBAAU,CACvB,WAAW,CAAC,gBAAgB,CAAC,WAAW,CAAC,EACzC,WAAW,CAAC,gBAAgB,CAAC,SAAS,CAAC,EACvC,WAAW,CAAC,gBAAgB,CAAC,aAAa,CAAC,EAC3C,+BAAkB,CAAC,WAAW,CAAC,mBAAmB,CAAC,mBAAmB,CAAC,CAAC,EACxE,WAAW,CAAC,mBAAmB,CAAC,aAAa,CAAC,EAC9C,WAAW,CAAC,mBAAmB,CAAC,mBAAmB,CAAC,EACpD,MAAM,EACN,WAAW,CAAC,mBAAmB,CAAC,kBAAkB,CAAC,EACnD,WAAW,CAAC,gBAAgB,CAAC,OAAO,CAAC,EACrC,WAAW,CAAC,mBAAmB,CAAC,mBAAmB,CAAC,EACpD,MAAM,CACP,CAAC;QAEF,IAAI;YACF,MAAM,wBAAiB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;SACzC;QAAC,OAAO,GAAG,EAAE;YACZ,MAAM,CAAC,OAAO,CACZ,GAAG,GAAG,CAAC,OAAO,2FAA2F,CAC1G,CAAC;SACH;KACF;IAAC,OAAO,CAAC,EAAE;QACV,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAC1B,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACf,MAAM,WAAW,CAAC,gBAAgB,CAChC,MAAM,WAAW,CAAC,sBAAsB,CACtC,MAAM,EACN,SAAS,EACT,SAAS,EACT,CAAC,CAAC,OAAO,CACV,CACF,CAAC;QACF,OAAO;KACR;IAED,IAAI;QACF,mBAAmB;QACnB,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACvC,IAAI,OAAO,EAAE;YACX,IAAI,CAAC,cAAc,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;YACxC,IAAI,CAAC,OAAO,CACV,6GAA6G,CAC9G,CAAC;SACH;QAED,mGAAmG;QACnG,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,MAAM,CAAC;QACtD,IAAI,CAAC,cAAc,CAAC,YAAY,EAAE,SAAS,CAAC,CAAC;QAE7C,MAAM,YAAY,GAAG,MAAM,cAAO,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACnD,IAAI,YAAY,KAAK,SAAS,EAAE;YAC9B,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE;gBAC3D,IAAI,CAAC,cAAc,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;aACjC;YAED,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE;gBAChC,MAAM,0BAAmB,CACvB,mBAAmB,EACnB,SAAS,EACT,MAAM,EACN,MAAM,EACN,YAAY,CACb,CAAC;aACH;SACF;KACF;IAAC,OAAO,KAAK,EAAE;QACd,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC9B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACnB,MAAM,WAAW,CAAC,gBAAgB,CAChC,MAAM,WAAW,CAAC,sBAAsB,CACtC,MAAM,EACN,SAAS,EACT,SAAS,EACT,KAAK,CAAC,OAAO,EACb,KAAK,CAAC,KAAK,CACZ,CACF,CAAC;QACF,OAAO;KACR;IACD,MAAM,uBAAuB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;AACnD,CAAC;AAED,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;IAChB,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC,EAAE,CAAC,CAAC;IAC3C,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AACjB,CAAC,CAAC,CAAC"} \ No newline at end of file diff --git a/lib/init.js b/lib/init.js index 1817f8026..bb0e00503 100644 --- a/lib/init.js +++ b/lib/init.js @@ -121,4 +121,35 @@ async function injectWindowsTracer(processName, processLevel, config, codeql, tr ], { env: { ODASA_TRACER_CONFIGURATION: tracerConfig.spec } }).exec(); } exports.injectWindowsTracer = injectWindowsTracer; +async function installPythonDeps(codeql, logger) { + logger.startGroup("Setup Python dependencies"); + if (process.platform !== "linux") { + logger.info("Currently, auto-installing python dependancies is only supported on linux"); + logger.endGroup(); + return; + } + const scriptsFolder = path.resolve(__dirname, "../python-setup"); + // Setup tools on the Github hosted runners + if (process.env["ImageOS"] !== undefined) { + try { + await new toolrunnner.ToolRunner(path.join(scriptsFolder, "install_tools.sh")).exec(); + } + catch (e) { + // This script tries to install some needed tools in the runner. It should not fail, but if it does + // we just abort the process without failing the action + logger.endGroup(); + logger.warning("Unable to download and extract the tools needed for installing the python dependecies. You can call this action with 'setup-python-dependencies: false' to disable this process."); + } + } + // Install dependencies + try { + await new toolrunnner.ToolRunner(path.join(scriptsFolder, "auto_install_packages.py"), [path.dirname(codeql.getPath())]).exec(); + } + catch (e) { + logger.endGroup(); + logger.warning("We were unable to install your python dependencies. You can call this action with 'setup-python-dependencies: false' to disable this process."); + } + logger.endGroup(); +} +exports.installPythonDeps = installPythonDeps; //# sourceMappingURL=init.js.map \ No newline at end of file diff --git a/lib/init.js.map b/lib/init.js.map index ba376d035..dd4458cb4 100644 --- a/lib/init.js.map +++ b/lib/init.js.map @@ -1 +1 @@ -{"version":3,"file":"init.js","sourceRoot":"","sources":["../src/init.ts"],"names":[],"mappings":";;;;;;;;;AAAA,uCAAyB;AACzB,2CAA6B;AAE7B,0EAA4D;AAE5D,gEAAkD;AAClD,qCAA+C;AAC/C,4DAA8C;AAG9C,mDAAwE;AACxE,6CAA+B;AAExB,KAAK,UAAU,UAAU,CAC9B,SAA6B,EAC7B,UAAkB,EAClB,SAAiB,EACjB,OAAe,EACf,QAAgB,EAChB,IAAe,EACf,MAAc;IAEd,MAAM,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,MAAM,oBAAW,CAC9B,SAAS,EACT,UAAU,EACV,SAAS,EACT,OAAO,EACP,QAAQ,EACR,IAAI,EACJ,MAAM,CACP,CAAC;IACF,MAAM,MAAM,CAAC,YAAY,EAAE,CAAC;IAC5B,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO,MAAM,CAAC;AAChB,CAAC;AAtBD,gCAsBC;AAEM,KAAK,UAAU,UAAU,CAC9B,cAAkC,EAClC,YAAgC,EAChC,UAA8B,EAC9B,UAAyB,EACzB,OAAe,EACf,YAAoB,EACpB,MAAc,EACd,YAAoB,EACpB,UAAkB,EAClB,SAAiB,EACjB,MAAc;IAEd,MAAM,CAAC,UAAU,CAAC,6BAA6B,CAAC,CAAC;IACjD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,UAAU,CACzC,cAAc,EACd,YAAY,EACZ,UAAU,EACV,UAAU,EACV,OAAO,EACP,YAAY,EACZ,MAAM,EACN,YAAY,EACZ,UAAU,EACV,SAAS,EACT,MAAM,CACP,CAAC;IACF,aAAa,CAAC,uBAAuB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtD,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO,MAAM,CAAC;AAChB,CAAC;AA9BD,gCA8BC;AAEM,KAAK,UAAU,OAAO,CAC3B,MAAc,EACd,MAA0B;IAE1B,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;IAElC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE9E,sEAAsE;IACtE,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACvC,yBAAyB;QACzB,MAAM,MAAM,CAAC,YAAY,CACvB,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,EACpD,QAAQ,EACR,UAAU,CACX,CAAC;KACH;IAED,OAAO,MAAM,uCAAuB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AACvD,CAAC;AAnBD,0BAmBC;AAED,sEAAsE;AACtE,4EAA4E;AAC5E,4EAA4E;AAC5E,6EAA6E;AAC7E,+CAA+C;AACxC,KAAK,UAAU,mBAAmB,CACvC,WAA+B,EAC/B,YAAgC,EAChC,MAA0B,EAC1B,MAAc,EACd,YAA0B;IAE1B,IAAI,MAAc,CAAC;IACnB,IAAI,WAAW,KAAK,SAAS,EAAE;QAC7B,MAAM,GAAG;;;;;;;;;;;;uCAY0B,WAAW;;8BAEpB,WAAW;;;;;;;;gDAQO,CAAC;KAC9C;SAAM;QACL,oEAAoE;QACpE,mFAAmF;QACnF,+EAA+E;QAC/E,kFAAkF;QAClF,6EAA6E;QAC7E,oFAAoF;QACpF,6CAA6C;QAC7C,YAAY,GAAG,YAAY,IAAI,CAAC,CAAC;QACjC,MAAM,GAAG;;;;;;;;4BAQe,YAAY;;;;;;;;;;;;;;;;;gDAiBQ,CAAC;KAC9C;IAED,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,mBAAmB,CAAC,CAAC;IACxE,EAAE,CAAC,aAAa,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC;IAE3C,MAAM,IAAI,WAAW,CAAC,UAAU,CAC9B,YAAY,EACZ;QACE,kBAAkB;QAClB,QAAQ;QACR,OAAO;QACP,gBAAgB;QAChB,IAAI,CAAC,OAAO,CACV,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,EAC9B,OAAO,EACP,OAAO,EACP,YAAY,CACb;KACF,EACD,EAAE,GAAG,EAAE,EAAE,0BAA0B,EAAE,YAAY,CAAC,IAAI,EAAE,EAAE,CAC3D,CAAC,IAAI,EAAE,CAAC;AACX,CAAC;AAxFD,kDAwFC"} \ No newline at end of file +{"version":3,"file":"init.js","sourceRoot":"","sources":["../src/init.ts"],"names":[],"mappings":";;;;;;;;;AAAA,uCAAyB;AACzB,2CAA6B;AAE7B,0EAA4D;AAE5D,gEAAkD;AAClD,qCAA+C;AAC/C,4DAA8C;AAG9C,mDAAwE;AACxE,6CAA+B;AAExB,KAAK,UAAU,UAAU,CAC9B,SAA6B,EAC7B,UAAkB,EAClB,SAAiB,EACjB,OAAe,EACf,QAAgB,EAChB,IAAe,EACf,MAAc;IAEd,MAAM,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,MAAM,oBAAW,CAC9B,SAAS,EACT,UAAU,EACV,SAAS,EACT,OAAO,EACP,QAAQ,EACR,IAAI,EACJ,MAAM,CACP,CAAC;IACF,MAAM,MAAM,CAAC,YAAY,EAAE,CAAC;IAC5B,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO,MAAM,CAAC;AAChB,CAAC;AAtBD,gCAsBC;AAEM,KAAK,UAAU,UAAU,CAC9B,cAAkC,EAClC,YAAgC,EAChC,UAA8B,EAC9B,UAAyB,EACzB,OAAe,EACf,YAAoB,EACpB,MAAc,EACd,YAAoB,EACpB,UAAkB,EAClB,SAAiB,EACjB,MAAc;IAEd,MAAM,CAAC,UAAU,CAAC,6BAA6B,CAAC,CAAC;IACjD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,UAAU,CACzC,cAAc,EACd,YAAY,EACZ,UAAU,EACV,UAAU,EACV,OAAO,EACP,YAAY,EACZ,MAAM,EACN,YAAY,EACZ,UAAU,EACV,SAAS,EACT,MAAM,CACP,CAAC;IACF,aAAa,CAAC,uBAAuB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtD,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO,MAAM,CAAC;AAChB,CAAC;AA9BD,gCA8BC;AAEM,KAAK,UAAU,OAAO,CAC3B,MAAc,EACd,MAA0B;IAE1B,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;IAElC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE9E,sEAAsE;IACtE,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACvC,yBAAyB;QACzB,MAAM,MAAM,CAAC,YAAY,CACvB,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,EACpD,QAAQ,EACR,UAAU,CACX,CAAC;KACH;IAED,OAAO,MAAM,uCAAuB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AACvD,CAAC;AAnBD,0BAmBC;AAED,sEAAsE;AACtE,4EAA4E;AAC5E,4EAA4E;AAC5E,6EAA6E;AAC7E,+CAA+C;AACxC,KAAK,UAAU,mBAAmB,CACvC,WAA+B,EAC/B,YAAgC,EAChC,MAA0B,EAC1B,MAAc,EACd,YAA0B;IAE1B,IAAI,MAAc,CAAC;IACnB,IAAI,WAAW,KAAK,SAAS,EAAE;QAC7B,MAAM,GAAG;;;;;;;;;;;;uCAY0B,WAAW;;8BAEpB,WAAW;;;;;;;;gDAQO,CAAC;KAC9C;SAAM;QACL,oEAAoE;QACpE,mFAAmF;QACnF,+EAA+E;QAC/E,kFAAkF;QAClF,6EAA6E;QAC7E,oFAAoF;QACpF,6CAA6C;QAC7C,YAAY,GAAG,YAAY,IAAI,CAAC,CAAC;QACjC,MAAM,GAAG;;;;;;;;4BAQe,YAAY;;;;;;;;;;;;;;;;;gDAiBQ,CAAC;KAC9C;IAED,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,mBAAmB,CAAC,CAAC;IACxE,EAAE,CAAC,aAAa,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC;IAE3C,MAAM,IAAI,WAAW,CAAC,UAAU,CAC9B,YAAY,EACZ;QACE,kBAAkB;QAClB,QAAQ;QACR,OAAO;QACP,gBAAgB;QAChB,IAAI,CAAC,OAAO,CACV,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,EAC9B,OAAO,EACP,OAAO,EACP,YAAY,CACb;KACF,EACD,EAAE,GAAG,EAAE,EAAE,0BAA0B,EAAE,YAAY,CAAC,IAAI,EAAE,EAAE,CAC3D,CAAC,IAAI,EAAE,CAAC;AACX,CAAC;AAxFD,kDAwFC;AAEM,KAAK,UAAU,iBAAiB,CAAC,MAAc,EAAE,MAAc;IACpE,MAAM,CAAC,UAAU,CAAC,2BAA2B,CAAC,CAAC;IAE/C,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE;QAChC,MAAM,CAAC,IAAI,CACT,2EAA2E,CAC5E,CAAC;QACF,MAAM,CAAC,QAAQ,EAAE,CAAC;QAClB,OAAO;KACR;IAED,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,iBAAiB,CAAC,CAAC;IAEjE,2CAA2C;IAC3C,IAAI,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,SAAS,EAAE;QACxC,IAAI;YACF,MAAM,IAAI,WAAW,CAAC,UAAU,CAC9B,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,kBAAkB,CAAC,CAC7C,CAAC,IAAI,EAAE,CAAC;SACV;QAAC,OAAO,CAAC,EAAE;YACV,mGAAmG;YACnG,uDAAuD;YACvD,MAAM,CAAC,QAAQ,EAAE,CAAC;YAClB,MAAM,CAAC,OAAO,CACZ,kLAAkL,CACnL,CAAC;SACH;KACF;IAED,uBAAuB;IACvB,IAAI;QACF,MAAM,IAAI,WAAW,CAAC,UAAU,CAC9B,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,0BAA0B,CAAC,EACpD,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,CACjC,CAAC,IAAI,EAAE,CAAC;KACV;IAAC,OAAO,CAAC,EAAE;QACV,MAAM,CAAC,QAAQ,EAAE,CAAC;QAClB,MAAM,CAAC,OAAO,CACZ,+IAA+I,CAChJ,CAAC;KACH;IACD,MAAM,CAAC,QAAQ,EAAE,CAAC;AACpB,CAAC;AA1CD,8CA0CC"} \ No newline at end of file diff --git a/python-setup/auto_install_packages.py b/python-setup/auto_install_packages.py new file mode 100755 index 000000000..cdb70c6d6 --- /dev/null +++ b/python-setup/auto_install_packages.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 + +import sys +import os +import subprocess +from tempfile import mkdtemp +from typing import Optional + +import extractor_version + + +def _check_call(command): + print('+ {}'.format(' '.join(command)), flush=True) + subprocess.check_call(command, stdin=subprocess.DEVNULL) + + +def _check_output(command): + print('+ {}'.format(' '.join(command)), flush=True) + out = subprocess.check_output(command, stdin=subprocess.DEVNULL) + print(out, flush=True) + sys.stderr.flush() + return out + + +def install_packages_with_poetry(): + try: + _check_call(['poetry', 'install', '--no-root']) + except subprocess.CalledProcessError: + sys.exit('package installation with poetry failed, see error above') + + # poetry is super annoying with `poetry run`, since it will put lots of output on + # STDOUT if the current global python interpreter is not matching the one in the + # virtualenv for the package, which was the case for using poetry for Python 2 when + # default system interpreter was Python 3 :/ + + poetry_out = _check_output(['poetry', 'run', 'which', 'python']) + python_executable_path = poetry_out.decode('utf-8').splitlines()[-1] + + return python_executable_path + + +def install_packages_with_pipenv(): + try: + _check_call(['pipenv', 'install', '--keep-outdated', '--ignore-pipfile']) + except subprocess.CalledProcessError: + sys.exit('package installation with pipenv failed, see error above') + + pipenv_out = _check_output(['pipenv', 'run', 'which', 'python']) + python_executable_path = pipenv_out.decode('utf-8').splitlines()[-1] + + return python_executable_path + + +def _create_venv(version: int): + # create temporary directory ... that just lives "forever" + venv_path = mkdtemp(prefix='codeql-action-python-autoinstall-') + + # virtualenv is a bit nicer for setting up virtual environment, since it will provide + # up-to-date versions of pip/setuptools/wheel which basic `python3 -m venv venv` won't + + if version == 2: + _check_call(['python2', '-m', 'virtualenv', venv_path]) + elif version == 3: + _check_call(['python3', '-m', 'virtualenv', venv_path]) + + return venv_path + + +def install_requirements_txt_packages(version: int): + venv_path = _create_venv(version) + venv_pip = os.path.join(venv_path, 'bin', 'pip') + venv_python = os.path.join(venv_path, 'bin', 'python') + + try: + _check_call([venv_pip, 'install', '-r', 'requirements.txt']) + except subprocess.CalledProcessError: + sys.exit('package installation with `pip install -r requirements.txt` failed, see error above') + + return venv_python + + +def install_with_setup_py(version: int): + venv_path = _create_venv(version) + venv_pip = os.path.join(venv_path, 'bin', 'pip') + venv_python = os.path.join(venv_path, 'bin', 'python') + + try: + # We have to choose between `python setup.py develop` and `pip install -e .`. + # Modern projects use `pip install -e .` and I wasn't able to see any downsides + # to doing so. However, `python setup.py develop` has some downsides -- from + # https://stackoverflow.com/a/19048754 : + # > Note that it is highly recommended to use pip install . (install) and pip + # > install -e . (developer install) to install packages, as invoking setup.py + # > directly will do the wrong things for many dependencies, such as pull + # > prereleases and incompatible package versions, or make the package hard to + # > uninstall with pip. + + _check_call([venv_pip, 'install', '-e', '.']) + except subprocess.CalledProcessError: + sys.exit('package installation with `pip install -e .` failed, see error above') + + return venv_python + + +def install_packages(codeql_base_dir) -> Optional[str]: + if os.path.exists('poetry.lock'): + print('Found poetry.lock, will install packages with poetry', flush=True) + return install_packages_with_poetry() + + if os.path.exists('Pipfile') or os.path.exists('Pipfile.lock'): + if os.path.exists('Pipfile.lock'): + print('Found Pipfile.lock, will install packages with Pipenv', flush=True) + else: + print('Found Pipfile, will install packages with Pipenv', flush=True) + return install_packages_with_pipenv() + + # get_extractor_version returns the Python version the extractor thinks this repo is using + version = extractor_version.get_extractor_version(codeql_base_dir, quiet=False) + + if os.path.exists('requirements.txt'): + print('Found requirements.txt, will install packages with pip', flush=True) + return install_requirements_txt_packages(version) + + if os.path.exists('setup.py'): + print('Found setup.py, will install package with pip in editable mode', flush=True) + return install_with_setup_py(version) + + print("was not able to install packages automatically", flush=True) + return None + + +if __name__ == "__main__": + if len(sys.argv) != 2: + sys.exit('Must provide base directory for codeql tool as only argument') + + codeql_base_dir = sys.argv[1] + + # The binaries for packages installed with `pip install --user` are not available on + # PATH by default, so we need to manually add them. + os.environ['PATH'] = os.path.expanduser('~/.local/bin') + os.pathsep + os.environ['PATH'] + + python_executable_path = install_packages(codeql_base_dir) + + if python_executable_path is not None: + print("Setting CODEQL_PYTHON={}".format(python_executable_path)) + print("::set-env name=CODEQL_PYTHON::{}".format(python_executable_path)) diff --git a/python-setup/extractor_version.py b/python-setup/extractor_version.py new file mode 100755 index 000000000..63e1c4211 --- /dev/null +++ b/python-setup/extractor_version.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python + +# A quick hack to get package installation for Code Scanning to work, +# since it needs to know which version we're going to analyze the project as. + +# This file needs to be placed next to `python_tracer.py`, so in +# `/python/tools/` + +from __future__ import print_function, division + +import os +import sys +from contextlib import contextmanager + + +@contextmanager +def suppress_stdout_stderr(): + # taken from + # https://thesmithfam.org/blog/2012/10/25/temporarily-suppress-console-output-in-python/ + with open(os.devnull, "w") as devnull: + old_stdout = sys.stdout + old_stderr = sys.stderr + sys.stdout = devnull + sys.stderr = devnull + try: + yield + finally: + sys.stdout = old_stdout + sys.stderr = old_stderr + + +def get_extractor_version(codeql_base_dir: str, quiet: bool = True) -> int: + extractor_dir = os.path.join(codeql_base_dir, 'python', 'tools') + sys.path = [extractor_dir] + sys.path + + from python_tracer import getzipfilename + + zippath = os.path.join(extractor_dir, getzipfilename()) + sys.path = [zippath] + sys.path + import buildtools.discover + + if quiet: + with suppress_stdout_stderr(): + return buildtools.discover.get_version() + else: + return buildtools.discover.get_version() + + +if __name__ == "__main__": + codeql_base_dir = sys.argv[1] + version = get_extractor_version(codeql_base_dir) + print('{!r}'.format(version)) diff --git a/python-setup/install_tools.sh b/python-setup/install_tools.sh new file mode 100755 index 000000000..5179a5c24 --- /dev/null +++ b/python-setup/install_tools.sh @@ -0,0 +1,33 @@ +#!/bin/sh +set -x +set -e + +# The binaries for packages installed with `pip install --user` are not available on PATH +# by default, so we fix up PATH to suppress warnings by pip. This also needs to be done by +# any script that needs to access poetry/pipenv. +# +# Using `::add-path::` from the actions toolkit is not enough, since that only affects +# subsequent actions in the current job, and not the current action. +export PATH="$HOME/.local/bin:$PATH" + +python2 -m pip install --user --upgrade pip setuptools wheel +python3 -m pip install --user --upgrade pip setuptools wheel + +# virtualenv is a bit nicer for setting up virtual environment, since it will provide up-to-date versions of +# pip/setuptools/wheel which basic `python3 -m venv venv` won't +python2 -m pip install --user virtualenv +python3 -m pip install --user virtualenv + +# venv is required for installation of poetry or pipenv (I forgot which) +sudo apt-get install -y python3-venv + +# We install poetry with pip instead of the recommended way, since the recommended way +# caused some problem since `poetry run` gives output like: +# +# /root/.poetry/lib/poetry/_vendor/py2.7/subprocess32.py:149: RuntimeWarning: The _posixsubprocess module is not being used. Child process reliability may suffer if your program uses threads. +# "program uses threads.", RuntimeWarning) +# LGTM_PYTHON_SETUP_VERSION=The currently activated Python version 2.7.18 is not supported by the project (^3.5). Trying to find and use a compatible version. Using python3 (3.8.2) 3 + +# poetry 1.0.10 has error (https://github.com/python-poetry/poetry/issues/2711) +python3 -m pip install --user poetry!=1.0.10 +python3 -m pip install --user pipenv diff --git a/python-setup/tests/check_requests_123.sh b/python-setup/tests/check_requests_123.sh new file mode 100755 index 000000000..f44eb8093 --- /dev/null +++ b/python-setup/tests/check_requests_123.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +set -e + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +EXPECTED_VERSION=$1 + +FOUND_VERSION="$LGTM_PYTHON_SETUP_VERSION" +FOUND_PYTHONPATH="$LGTM_INDEX_IMPORT_PATH" + +echo "FOUND_VERSION=${FOUND_VERSION} FOUND_PYTHONPATH=${FOUND_PYTHONPATH} " + +if [[ $FOUND_VERSION != $EXPECTED_VERSION ]]; then + echo "Script told us to use Python ${FOUND_VERSION}, but expected ${EXPECTED_VERSION}" + exit 1 +else + echo "Script told us to use Python ${FOUND_VERSION}, which was expected" +fi + +PYTHON_EXE="python${EXPECTED_VERSION}" + +INSTALLED_REQUESTS_VERSION=$(PYTHONPATH="${FOUND_PYTHONPATH}" "${PYTHON_EXE}" -c 'import requests; print(requests.__version__)') + +EXPECTED_REQUESTS="1.2.3" + +if [[ "$INSTALLED_REQUESTS_VERSION" != "$EXPECTED_REQUESTS" ]]; then + echo "Using ${FOUND_PYTHONPATH} as PYTHONPATH, we found version $INSTALLED_REQUESTS_VERSION of requests, but expected $EXPECTED_REQUESTS" + exit 1 +else + echo "Using ${FOUND_PYTHONPATH} as PYTHONPATH, we found version $INSTALLED_REQUESTS_VERSION of requests, which was expected" +fi \ No newline at end of file diff --git a/python-setup/tests/from_python_exe.py b/python-setup/tests/from_python_exe.py new file mode 100755 index 000000000..69e2e873e --- /dev/null +++ b/python-setup/tests/from_python_exe.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 + +import sys +import subprocess +from typing import Tuple + +def get_details(path_to_python_exe: str) -> Tuple[str, str]: + import_path = subprocess.check_output( + [ + path_to_python_exe, + "-c", + "import os; import pip; print(os.path.dirname(os.path.dirname(pip.__file__)))", + ], + stdin=subprocess.DEVNULL, + ) + version = subprocess.check_output( + [path_to_python_exe, "-c", "import sys; print(sys.version_info[0])"], + stdin=subprocess.DEVNULL, + ) + + return version.decode("utf-8").strip(), import_path.decode("utf-8").strip() + + +if __name__ == "__main__": + version, import_path = get_details(sys.argv[1]) + + print("Setting LGTM_PYTHON_SETUP_VERSION={}".format(version)) + print("::set-env name=LGTM_PYTHON_SETUP_VERSION::{}".format(version)) + + print("Setting LGTM_INDEX_IMPORT_PATH={}".format(import_path)) + print("::set-env name=LGTM_INDEX_IMPORT_PATH::{}".format(import_path)) \ No newline at end of file diff --git a/python-setup/tests/pipenv/python-3.8/Pipfile b/python-setup/tests/pipenv/python-3.8/Pipfile new file mode 100644 index 000000000..b799f0f56 --- /dev/null +++ b/python-setup/tests/pipenv/python-3.8/Pipfile @@ -0,0 +1,12 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[dev-packages] + +[packages] +requests = "*" + +[requires] +python_version = "3.8" diff --git a/python-setup/tests/pipenv/python-3.8/Pipfile.lock b/python-setup/tests/pipenv/python-3.8/Pipfile.lock new file mode 100644 index 000000000..519591de8 --- /dev/null +++ b/python-setup/tests/pipenv/python-3.8/Pipfile.lock @@ -0,0 +1,28 @@ +{ + "_meta": { + "hash": { + "sha256": "2296b1fab66570fa3660c5e6a4e923332dcd7785814624c584be6623bc62c0e7" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.8" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "requests": { + "hashes": [ + "sha256:156bf3ec27ba9ec7e0cf8fbe02808718099d218de403eb64a714d73ba1a29ab1" + ], + "index": "pypi", + "version": "==1.2.3" + } + }, + "develop": {} +} diff --git a/python-setup/tests/pipenv/requests-2/Pipfile b/python-setup/tests/pipenv/requests-2/Pipfile new file mode 100644 index 000000000..c9450bcf7 --- /dev/null +++ b/python-setup/tests/pipenv/requests-2/Pipfile @@ -0,0 +1,12 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[dev-packages] + +[packages] +requests = "*" + +[requires] +python_version = "2.7" diff --git a/python-setup/tests/pipenv/requests-2/Pipfile.lock b/python-setup/tests/pipenv/requests-2/Pipfile.lock new file mode 100644 index 000000000..789382967 --- /dev/null +++ b/python-setup/tests/pipenv/requests-2/Pipfile.lock @@ -0,0 +1,28 @@ +{ + "_meta": { + "hash": { + "sha256": "fba50410545d38a2f88b8d6964eb292055389e363b5c75a56b01ec80e7de14c1" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "2.7" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "requests": { + "hashes": [ + "sha256:156bf3ec27ba9ec7e0cf8fbe02808718099d218de403eb64a714d73ba1a29ab1" + ], + "index": "pypi", + "version": "==1.2.3" + } + }, + "develop": {} +} diff --git a/python-setup/tests/pipenv/requests-3/Pipfile b/python-setup/tests/pipenv/requests-3/Pipfile new file mode 100644 index 000000000..fcab73720 --- /dev/null +++ b/python-setup/tests/pipenv/requests-3/Pipfile @@ -0,0 +1,11 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[dev-packages] + +[packages] +requests = "*" + +[requires] diff --git a/python-setup/tests/pipenv/requests-3/Pipfile.lock b/python-setup/tests/pipenv/requests-3/Pipfile.lock new file mode 100644 index 000000000..519591de8 --- /dev/null +++ b/python-setup/tests/pipenv/requests-3/Pipfile.lock @@ -0,0 +1,28 @@ +{ + "_meta": { + "hash": { + "sha256": "2296b1fab66570fa3660c5e6a4e923332dcd7785814624c584be6623bc62c0e7" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.8" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "requests": { + "hashes": [ + "sha256:156bf3ec27ba9ec7e0cf8fbe02808718099d218de403eb64a714d73ba1a29ab1" + ], + "index": "pypi", + "version": "==1.2.3" + } + }, + "develop": {} +} diff --git a/python-setup/tests/poetry/python-3.8/poetry.lock b/python-setup/tests/poetry/python-3.8/poetry.lock new file mode 100644 index 000000000..cda1a33c2 --- /dev/null +++ b/python-setup/tests/poetry/python-3.8/poetry.lock @@ -0,0 +1,16 @@ +[[package]] +category = "main" +description = "Python HTTP for Humans." +name = "requests" +optional = false +python-versions = "^3.8" +version = "1.2.3" + +[metadata] +content-hash = "b1abdce30bbdad3ac786dafe6840129df522cae3c5988f9ba8061320e3c1a34e" +python-versions = "^3.8" + +[metadata.files] +requests = [ + {file = "requests-1.2.3.tar.gz", hash = "sha256:156bf3ec27ba9ec7e0cf8fbe02808718099d218de403eb64a714d73ba1a29ab1"}, +] diff --git a/python-setup/tests/poetry/python-3.8/pyproject.toml b/python-setup/tests/poetry/python-3.8/pyproject.toml new file mode 100644 index 000000000..8d82c19db --- /dev/null +++ b/python-setup/tests/poetry/python-3.8/pyproject.toml @@ -0,0 +1,15 @@ +[tool.poetry] +name = "autoinstall-test" +version = "0.1.0" +description = "" +authors = ["Your Name "] + +[tool.poetry.dependencies] +python = "^3.8" +requests = "*" + +[tool.poetry.dev-dependencies] + +[build-system] +requires = ["poetry>=0.12"] +build-backend = "poetry.masonry.api" diff --git a/python-setup/tests/poetry/requests-2/poetry.lock b/python-setup/tests/poetry/requests-2/poetry.lock new file mode 100644 index 000000000..c16b09b4c --- /dev/null +++ b/python-setup/tests/poetry/requests-2/poetry.lock @@ -0,0 +1,16 @@ +[[package]] +category = "main" +description = "Python HTTP for Humans." +name = "requests" +optional = false +python-versions = "*" +version = "1.2.3" + +[metadata] +content-hash = "b8c27a00aece20cb2ff6b3dcd394b1b5ef5e4acc3d7a059cfeb9f9cc11c9eff4" +python-versions = "^2.7" + +[metadata.files] +requests = [ + {file = "requests-1.2.3.tar.gz", hash = "sha256:156bf3ec27ba9ec7e0cf8fbe02808718099d218de403eb64a714d73ba1a29ab1"}, +] diff --git a/python-setup/tests/poetry/requests-2/pyproject.toml b/python-setup/tests/poetry/requests-2/pyproject.toml new file mode 100644 index 000000000..890d59147 --- /dev/null +++ b/python-setup/tests/poetry/requests-2/pyproject.toml @@ -0,0 +1,15 @@ +[tool.poetry] +name = "autoinstall-test" +version = "0.1.0" +description = "" +authors = ["Your Name "] + +[tool.poetry.dependencies] +python = "^2.7" +requests = "*" + +[tool.poetry.dev-dependencies] + +[build-system] +requires = ["poetry>=0.12"] +build-backend = "poetry.masonry.api" diff --git a/python-setup/tests/poetry/requests-3/poetry.lock b/python-setup/tests/poetry/requests-3/poetry.lock new file mode 100644 index 000000000..ecc2ca3b6 --- /dev/null +++ b/python-setup/tests/poetry/requests-3/poetry.lock @@ -0,0 +1,16 @@ +[[package]] +category = "main" +description = "Python HTTP for Humans." +name = "requests" +optional = false +python-versions = "^3.5" +version = "1.2.3" + +[metadata] +content-hash = "b1abdce30bbdad3ac786dafe6840129df522cae3c5988f9ba8061320e3c1a34e" +python-versions = "^3.5" + +[metadata.files] +requests = [ + {file = "requests-1.2.3.tar.gz", hash = "sha256:156bf3ec27ba9ec7e0cf8fbe02808718099d218de403eb64a714d73ba1a29ab1"}, +] diff --git a/python-setup/tests/poetry/requests-3/pyproject.toml b/python-setup/tests/poetry/requests-3/pyproject.toml new file mode 100644 index 000000000..155de4618 --- /dev/null +++ b/python-setup/tests/poetry/requests-3/pyproject.toml @@ -0,0 +1,15 @@ +[tool.poetry] +name = "autoinstall-test" +version = "0.1.0" +description = "" +authors = ["Your Name "] + +[tool.poetry.dependencies] +python = "^3.5" +requests = "*" + +[tool.poetry.dev-dependencies] + +[build-system] +requires = ["poetry>=0.12"] +build-backend = "poetry.masonry.api" diff --git a/python-setup/tests/requirements/non-standard-location/non-standard/requirements.txt b/python-setup/tests/requirements/non-standard-location/non-standard/requirements.txt new file mode 100644 index 000000000..134975b53 --- /dev/null +++ b/python-setup/tests/requirements/non-standard-location/non-standard/requirements.txt @@ -0,0 +1 @@ +requests==1.2.3 diff --git a/python-setup/tests/requirements/non-standard-location/wat.py b/python-setup/tests/requirements/non-standard-location/wat.py new file mode 100644 index 000000000..b376c9941 --- /dev/null +++ b/python-setup/tests/requirements/non-standard-location/wat.py @@ -0,0 +1 @@ +print('hello') diff --git a/python-setup/tests/requirements/requests-2/requirements.txt b/python-setup/tests/requirements/requests-2/requirements.txt new file mode 100644 index 000000000..134975b53 --- /dev/null +++ b/python-setup/tests/requirements/requests-2/requirements.txt @@ -0,0 +1 @@ +requests==1.2.3 diff --git a/python-setup/tests/requirements/requests-2/setup.py b/python-setup/tests/requirements/requests-2/setup.py new file mode 100644 index 000000000..ea6a81184 --- /dev/null +++ b/python-setup/tests/requirements/requests-2/setup.py @@ -0,0 +1,3 @@ +# fake setup.py with Trove classifier to fool Python extractor to believe this is Python 2 for sure + +# Programming Language :: Python :: 2.7 diff --git a/python-setup/tests/requirements/requests-3/requirements.txt b/python-setup/tests/requirements/requests-3/requirements.txt new file mode 100644 index 000000000..134975b53 --- /dev/null +++ b/python-setup/tests/requirements/requests-3/requirements.txt @@ -0,0 +1 @@ +requests==1.2.3 diff --git a/python-setup/tests/requirements/requests-3/setup.py b/python-setup/tests/requirements/requests-3/setup.py new file mode 100644 index 000000000..f2eba6c01 --- /dev/null +++ b/python-setup/tests/requirements/requests-3/setup.py @@ -0,0 +1,3 @@ +# fake setup.py with Trove classifier to fool Python extractor to believe this is Python 3 for sure + +# Programming Language :: Python :: 3.7 diff --git a/python-setup/tests/setup_py/requests-2/setup.py b/python-setup/tests/setup_py/requests-2/setup.py new file mode 100644 index 000000000..26e0d9753 --- /dev/null +++ b/python-setup/tests/setup_py/requests-2/setup.py @@ -0,0 +1,12 @@ +from setuptools import setup + +# has fake Trove classifier to fool Python extractor to believe this is Python 2 for sure + +# Programming Language :: Python :: 2.7 + + +setup( + name="example-setup.py", + install_requires=["requests==1.2.3"], + python_requires=">=2.7, <3", +) diff --git a/python-setup/tests/setup_py/requests-3/setup.py b/python-setup/tests/setup_py/requests-3/setup.py new file mode 100644 index 000000000..b401073da --- /dev/null +++ b/python-setup/tests/setup_py/requests-3/setup.py @@ -0,0 +1,12 @@ +from setuptools import setup + +# has fake Trove classifier to fool Python extractor to believe this is Python 3 for sure + +# Programming Language :: Python :: 3.7 + + +setup( + name="example-setup.py", + install_requires=["requests==1.2.3"], + python_requires='>=3.5', +) diff --git a/src/analyze.ts b/src/analyze.ts index 8b9e80ab6..451861049 100644 --- a/src/analyze.ts +++ b/src/analyze.ts @@ -1,10 +1,12 @@ import * as fs from "fs"; import * as path from "path"; +import * as toolrunnner from "@actions/exec/lib/toolrunner"; + import * as analysisPaths from "./analysis-paths"; import { getCodeQL } from "./codeql"; import * as configUtils from "./config-utils"; -import { isScannedLanguage } from "./languages"; +import { isScannedLanguage, Language } from "./languages"; import { Logger } from "./logging"; import { RepositoryNwo } from "./repository"; import * as sharedEnv from "./shared-environment"; @@ -44,6 +46,43 @@ export interface AnalysisStatusReport extends upload_lib.UploadStatusReport, QueriesStatusReport {} +async function setupPythonExtractor(logger: Logger) { + const codeqlPython = process.env["CODEQL_PYTHON"]; + if (codeqlPython === undefined || codeqlPython.length === 0) { + // If CODEQL_PYTHON is not set, no dependencies were installed, so we don't need to do anything + return; + } + + let output = ""; + const options = { + listeners: { + stdout: (data: Buffer) => { + output += data.toString(); + }, + }, + }; + + await new toolrunnner.ToolRunner( + codeqlPython, + [ + "-c", + "import os; import pip; print(os.path.dirname(os.path.dirname(pip.__file__)))", + ], + options + ).exec(); + logger.info(`Setting LGTM_INDEX_IMPORT_PATH=${output}`); + process.env["LGTM_INDEX_IMPORT_PATH"] = output; + + output = ""; + await new toolrunnner.ToolRunner( + codeqlPython, + ["-c", "import sys; print(sys.version_info[0])"], + options + ).exec(); + logger.info(`Setting LGTM_PYTHON_SETUP_VERSION=${output}`); + process.env["LGTM_PYTHON_SETUP_VERSION"] = output; +} + async function createdDBForScannedLanguages( config: configUtils.Config, logger: Logger @@ -56,6 +95,11 @@ async function createdDBForScannedLanguages( for (const language of config.languages) { if (isScannedLanguage(language)) { logger.startGroup(`Extracting ${language}`); + + if (language === Language.python) { + await setupPythonExtractor(logger); + } + await codeql.extractScannedLanguage( util.getCodeQLDatabasePath(config.tempDir, language), language diff --git a/src/init-action.ts b/src/init-action.ts index 8fd5cf8d6..5d0a0d7a9 100644 --- a/src/init-action.ts +++ b/src/init-action.ts @@ -3,7 +3,13 @@ import * as core from "@actions/core"; import * as actionsUtil from "./actions-util"; import { CodeQL } from "./codeql"; import * as configUtils from "./config-utils"; -import { initCodeQL, initConfig, injectWindowsTracer, runInit } from "./init"; +import { + initCodeQL, + initConfig, + injectWindowsTracer, + installPythonDeps, + runInit, +} from "./init"; import { getActionsLogger } from "./logging"; import { parseRepositoryNwo } from "./repository"; @@ -111,6 +117,14 @@ async function run() { actionsUtil.getRequiredEnvParam("GITHUB_SERVER_URL"), logger ); + + try { + await installPythonDeps(codeql, logger); + } catch (err) { + logger.warning( + `${err.message} You can call this action with 'setup-python-dependencies: false' to disable this process` + ); + } } catch (e) { core.setFailed(e.message); console.log(e); diff --git a/src/init.ts b/src/init.ts index 31c6167e0..6a738a090 100644 --- a/src/init.ts +++ b/src/init.ts @@ -182,3 +182,47 @@ export async function injectWindowsTracer( { env: { ODASA_TRACER_CONFIGURATION: tracerConfig.spec } } ).exec(); } + +export async function installPythonDeps(codeql: CodeQL, logger: Logger) { + logger.startGroup("Setup Python dependencies"); + + if (process.platform !== "linux") { + logger.info( + "Currently, auto-installing python dependancies is only supported on linux" + ); + logger.endGroup(); + return; + } + + const scriptsFolder = path.resolve(__dirname, "../python-setup"); + + // Setup tools on the Github hosted runners + if (process.env["ImageOS"] !== undefined) { + try { + await new toolrunnner.ToolRunner( + path.join(scriptsFolder, "install_tools.sh") + ).exec(); + } catch (e) { + // This script tries to install some needed tools in the runner. It should not fail, but if it does + // we just abort the process without failing the action + logger.endGroup(); + logger.warning( + "Unable to download and extract the tools needed for installing the python dependecies. You can call this action with 'setup-python-dependencies: false' to disable this process." + ); + } + } + + // Install dependencies + try { + await new toolrunnner.ToolRunner( + path.join(scriptsFolder, "auto_install_packages.py"), + [path.dirname(codeql.getPath())] + ).exec(); + } catch (e) { + logger.endGroup(); + logger.warning( + "We were unable to install your python dependencies. You can call this action with 'setup-python-dependencies: false' to disable this process." + ); + } + logger.endGroup(); +}