From 8cea21575cda72abcc74243dad4d11cdebd13601 Mon Sep 17 00:00:00 2001 From: David Verdeguer Date: Fri, 11 Sep 2020 10:53:41 +0200 Subject: [PATCH 01/15] Install python deps on init --- init/action.yml | 4 ++++ lib/init-action.js | 6 ++++++ lib/init-action.js.map | 2 +- lib/init.js | 24 ++++++++++++++++++++++++ lib/init.js.map | 2 +- src/init-action.ts | 9 ++++++++- src/init.ts | 26 ++++++++++++++++++++++++++ 7 files changed, 70 insertions(+), 3 deletions(-) diff --git a/init/action.yml b/init/action.yml index 0b3ea9da8..f2c1519c7 100644 --- a/init/action.yml +++ b/init/action.yml @@ -19,6 +19,10 @@ inputs: queries: description: Comma-separated list of additional queries to run. By default, this overrides the same setting in a configuration file; prefix with "+" to use both sets of queries. required: false + setup-python-dependencies: + description: Try to auto-install your python dependencies + required: true + default: 'true' runs: using: 'node12' main: '../lib/init-action.js' diff --git a/lib/init-action.js b/lib/init-action.js index ff1b3206b..e4a1c87fe 100644 --- a/lib/init-action.js +++ b/lib/init-action.js @@ -42,6 +42,12 @@ async function run() { return; } codeql = await init_1.initCodeQL(core.getInput('tools'), core.getInput('token'), util.getRequiredEnvParam('GITHUB_SERVER_URL'), util.getRequiredEnvParam('RUNNER_TEMP'), util.getRequiredEnvParam('RUNNER_TOOL_CACHE'), 'actions', logger); + try { + await init_1.installPythonDeps(codeql, logger); + } + catch (err) { + logger.warning(err.message + 'You can call this action with "setup-python-dependencies: false" to disable this process'); + } config = await init_1.initConfig(core.getInput('languages'), core.getInput('queries'), core.getInput('config-file'), repository_1.parseRepositoryNwo(util.getRequiredEnvParam('GITHUB_REPOSITORY')), util.getRequiredEnvParam('RUNNER_TEMP'), util.getRequiredEnvParam('RUNNER_TOOL_CACHE'), codeql, util.getRequiredEnvParam('GITHUB_WORKSPACE'), core.getInput('token'), util.getRequiredEnvParam('GITHUB_SERVER_URL'), logger); } catch (e) { diff --git a/lib/init-action.js.map b/lib/init-action.js.map index d9a715298..5edb9af5c 100644 --- a/lib/init-action.js.map +++ b/lib/init-action.js.map @@ -1 +1 @@ -{"version":3,"file":"init-action.js","sourceRoot":"","sources":["../src/init-action.ts"],"names":[],"mappings":";;;;;;;;;AAAA,oDAAsC;AAItC,iCAA8E;AAC9E,uCAA6C;AAC7C,6CAAkD;AAClD,6CAA+B;AAkB/B,KAAK,UAAU,uBAAuB,CAAC,SAAe,EAAE,MAA0B;IAChF,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,sBAAsB,CAAC,MAAM,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IAEzF,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC7C,MAAM,iBAAiB,GAAG,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAC1E,MAAM,KAAK,GAAG,CAAC,MAAM,CAAC,iBAAiB,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC/D,MAAM,WAAW,GAAG,CAAC,MAAM,CAAC,iBAAiB,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC/E,MAAM,qBAAqB,GAAG,MAAM,CAAC,iBAAiB,CAAC,yBAAyB,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;IACnG,MAAM,OAAO,GAAG,CAAC,MAAM,CAAC,iBAAiB,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAEpF,MAAM,YAAY,GAA4B;QAC5C,GAAG,gBAAgB;QACnB,SAAS,EAAE,SAAS;QACpB,kBAAkB,EAAE,iBAAiB;QACrC,KAAK,EAAE,KAAK;QACZ,YAAY,EAAE,WAAW;QACzB,uBAAuB,EAAE,qBAAqB;QAC9C,OAAO,EAAE,OAAO;KACjB,CAAC;IAEF,MAAM,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAC;AAC5C,CAAC;AAED,KAAK,UAAU,GAAG;IAChB,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;IAC7B,MAAM,MAAM,GAAG,0BAAgB,EAAE,CAAC;IAClC,IAAI,MAA0B,CAAC;IAC/B,IAAI,MAAc,CAAC;IAEnB,IAAI;QACF,IAAI,CAAC,0BAA0B,EAAE,CAAC;QAClC,IAAI,CAAC,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,IAAI,CAAC,sBAAsB,CAAC,MAAM,EAAE,UAAU,EAAE,SAAS,CAAC,EAAE,IAAI,CAAC,EAAE;YACxG,OAAO;SACR;QAED,MAAM,GAAG,MAAM,iBAAU,CACvB,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EACtB,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EACtB,IAAI,CAAC,mBAAmB,CAAC,mBAAmB,CAAC,EAC7C,IAAI,CAAC,mBAAmB,CAAC,aAAa,CAAC,EACvC,IAAI,CAAC,mBAAmB,CAAC,mBAAmB,CAAC,EAC7C,SAAS,EACT,MAAM,CAAC,CAAC;QACV,MAAM,GAAG,MAAM,iBAAU,CACvB,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,EAC1B,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,EACxB,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,EAC5B,+BAAkB,CAAC,IAAI,CAAC,mBAAmB,CAAC,mBAAmB,CAAC,CAAC,EACjE,IAAI,CAAC,mBAAmB,CAAC,aAAa,CAAC,EACvC,IAAI,CAAC,mBAAmB,CAAC,mBAAmB,CAAC,EAC7C,MAAM,EACN,IAAI,CAAC,mBAAmB,CAAC,kBAAkB,CAAC,EAC5C,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EACtB,IAAI,CAAC,mBAAmB,CAAC,mBAAmB,CAAC,EAC7C,MAAM,CAAC,CAAC;KAEX;IAAC,OAAO,CAAC,EAAE;QACV,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAC1B,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACf,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,IAAI,CAAC,sBAAsB,CAAC,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;QACxG,OAAO;KACR;IAED,IAAI;QAEF,mBAAmB;QACnB,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACvC,IAAI,OAAO,EAAE;YACX,IAAI,CAAC,cAAc,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;YACxC,IAAI,CAAC,OAAO,CAAC,6GAA6G,CAAC,CAAC;SAC7H;QAED,mGAAmG;QACnG,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,MAAM,CAAC;QACtD,IAAI,CAAC,cAAc,CAAC,YAAY,EAAE,SAAS,CAAC,CAAC;QAE7C,MAAM,YAAY,GAAG,MAAM,cAAO,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACnD,IAAI,YAAY,KAAK,SAAS,EAAE;YAC9B,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC;YAE5F,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE;gBAChC,MAAM,0BAAmB,CAAC,mBAAmB,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,CAAC,CAAC;aACzF;SACF;KAEF;IAAC,OAAO,KAAK,EAAE;QACd,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC9B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACnB,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,IAAI,CAAC,sBAAsB,CAC3D,MAAM,EACN,SAAS,EACT,SAAS,EACT,KAAK,CAAC,OAAO,EACb,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;QAChB,OAAO;KACR;IACD,MAAM,uBAAuB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;AACnD,CAAC;AAED,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;IACd,IAAI,CAAC,SAAS,CAAC,sBAAsB,GAAG,CAAC,CAAC,CAAC;IAC3C,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AACjB,CAAC,CAAC,CAAC"} \ No newline at end of file +{"version":3,"file":"init-action.js","sourceRoot":"","sources":["../src/init-action.ts"],"names":[],"mappings":";;;;;;;;;AAAA,oDAAsC;AAItC,iCAAgG;AAChG,uCAA6C;AAC7C,6CAAkD;AAClD,6CAA+B;AAkB/B,KAAK,UAAU,uBAAuB,CAAC,SAAe,EAAE,MAA0B;IAChF,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,sBAAsB,CAAC,MAAM,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IAEzF,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC7C,MAAM,iBAAiB,GAAG,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAC1E,MAAM,KAAK,GAAG,CAAC,MAAM,CAAC,iBAAiB,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC/D,MAAM,WAAW,GAAG,CAAC,MAAM,CAAC,iBAAiB,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC/E,MAAM,qBAAqB,GAAG,MAAM,CAAC,iBAAiB,CAAC,yBAAyB,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;IACnG,MAAM,OAAO,GAAG,CAAC,MAAM,CAAC,iBAAiB,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAEpF,MAAM,YAAY,GAA4B;QAC5C,GAAG,gBAAgB;QACnB,SAAS,EAAE,SAAS;QACpB,kBAAkB,EAAE,iBAAiB;QACrC,KAAK,EAAE,KAAK;QACZ,YAAY,EAAE,WAAW;QACzB,uBAAuB,EAAE,qBAAqB;QAC9C,OAAO,EAAE,OAAO;KACjB,CAAC;IAEF,MAAM,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAC;AAC5C,CAAC;AAED,KAAK,UAAU,GAAG;IAChB,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;IAC7B,MAAM,MAAM,GAAG,0BAAgB,EAAE,CAAC;IAClC,IAAI,MAA0B,CAAC;IAC/B,IAAI,MAAc,CAAC;IAEnB,IAAI;QACF,IAAI,CAAC,0BAA0B,EAAE,CAAC;QAClC,IAAI,CAAC,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,IAAI,CAAC,sBAAsB,CAAC,MAAM,EAAE,UAAU,EAAE,SAAS,CAAC,EAAE,IAAI,CAAC,EAAE;YACxG,OAAO;SACR;QAED,MAAM,GAAG,MAAM,iBAAU,CACvB,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EACtB,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EACtB,IAAI,CAAC,mBAAmB,CAAC,mBAAmB,CAAC,EAC7C,IAAI,CAAC,mBAAmB,CAAC,aAAa,CAAC,EACvC,IAAI,CAAC,mBAAmB,CAAC,mBAAmB,CAAC,EAC7C,SAAS,EACT,MAAM,CAAC,CAAC;QAEV,IAAI;YACF,MAAM,wBAAiB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;SACzC;QAAC,OAAO,GAAG,EAAE;YACZ,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,GAAG,0FAA0F,CAAC,CAAC;SAC1H;QAED,MAAM,GAAG,MAAM,iBAAU,CACvB,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,EAC1B,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,EACxB,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,EAC5B,+BAAkB,CAAC,IAAI,CAAC,mBAAmB,CAAC,mBAAmB,CAAC,CAAC,EACjE,IAAI,CAAC,mBAAmB,CAAC,aAAa,CAAC,EACvC,IAAI,CAAC,mBAAmB,CAAC,mBAAmB,CAAC,EAC7C,MAAM,EACN,IAAI,CAAC,mBAAmB,CAAC,kBAAkB,CAAC,EAC5C,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EACtB,IAAI,CAAC,mBAAmB,CAAC,mBAAmB,CAAC,EAC7C,MAAM,CAAC,CAAC;KAEX;IAAC,OAAO,CAAC,EAAE;QACV,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAC1B,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACf,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,IAAI,CAAC,sBAAsB,CAAC,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;QACxG,OAAO;KACR;IAED,IAAI;QAEF,mBAAmB;QACnB,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACvC,IAAI,OAAO,EAAE;YACX,IAAI,CAAC,cAAc,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;YACxC,IAAI,CAAC,OAAO,CAAC,6GAA6G,CAAC,CAAC;SAC7H;QAED,mGAAmG;QACnG,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,MAAM,CAAC;QACtD,IAAI,CAAC,cAAc,CAAC,YAAY,EAAE,SAAS,CAAC,CAAC;QAE7C,MAAM,YAAY,GAAG,MAAM,cAAO,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACnD,IAAI,YAAY,KAAK,SAAS,EAAE;YAC9B,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC;YAE5F,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE;gBAChC,MAAM,0BAAmB,CAAC,mBAAmB,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,CAAC,CAAC;aACzF;SACF;KAEF;IAAC,OAAO,KAAK,EAAE;QACd,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC9B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACnB,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,IAAI,CAAC,sBAAsB,CAC3D,MAAM,EACN,SAAS,EACT,SAAS,EACT,KAAK,CAAC,OAAO,EACb,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;QAChB,OAAO;KACR;IACD,MAAM,uBAAuB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;AACnD,CAAC;AAED,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;IACd,IAAI,CAAC,SAAS,CAAC,sBAAsB,GAAG,CAAC,CAAC,CAAC;IAC3C,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AACjB,CAAC,CAAC,CAAC"} \ No newline at end of file diff --git a/lib/init.js b/lib/init.js index 3a476381f..a45fe1032 100644 --- a/lib/init.js +++ b/lib/init.js @@ -119,4 +119,28 @@ async function injectWindowsTracer(processName, processLevel, config, codeql, tr ], { env: { 'ODASA_TRACER_CONFIGURATION': tracerConfig.spec } }).exec(); } exports.injectWindowsTracer = injectWindowsTracer; +async function installPythonDeps(codeql, logger) { + logger.startGroup('Setup Python dependencies'); + const scriptsFolder = path.resolve(__dirname, '../python-setup'); + // Setup tools + try { + await new toolrunnner.ToolRunner(path.join(scriptsFolder, 'install_tools.sh')).exec(); + } + catch (e) { + // This script tries to install some needed tools in the runner. It should not fail, but if it does + // we just abort the process without failing the action + logger.endGroup(); + throw new Error('Unable to download and extract the scripts needed for installing the python dependecies'); + } + // Install dependencies + try { + await new toolrunnner.ToolRunner(path.join(scriptsFolder, 'auto_install_packages.py'), [path.dirname(codeql.getPath())]).exec(); + } + catch (e) { + logger.endGroup(); + throw new Error('We were unable to install your python dependencies.'); + } + logger.endGroup(); +} +exports.installPythonDeps = installPythonDeps; //# sourceMappingURL=init.js.map \ No newline at end of file diff --git a/lib/init.js.map b/lib/init.js.map index 39c99f3aa..1f7ff9fcb 100644 --- a/lib/init.js.map +++ b/lib/init.js.map @@ -1 +1 @@ -{"version":3,"file":"init.js","sourceRoot":"","sources":["../src/init.ts"],"names":[],"mappings":";;;;;;;;;AAAA,0EAA4D;AAC5D,uCAAyB;AACzB,2CAA6B;AAE7B,gEAAkD;AAClD,qCAA+C;AAC/C,4DAA8C;AAG9C,mDAAwE;AACxE,6CAA+B;AAExB,KAAK,UAAU,UAAU,CAC9B,SAA6B,EAC7B,UAAkB,EAClB,SAAiB,EACjB,OAAe,EACf,QAAgB,EAChB,IAAe,EACf,MAAc;IAEd,MAAM,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,MAAM,oBAAW,CAC9B,SAAS,EACT,UAAU,EACV,SAAS,EACT,OAAO,EACP,QAAQ,EACR,IAAI,EACJ,MAAM,CAAC,CAAC;IACV,MAAM,MAAM,CAAC,YAAY,EAAE,CAAC;IAC5B,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO,MAAM,CAAC;AAChB,CAAC;AArBD,gCAqBC;AAEM,KAAK,UAAU,UAAU,CAC9B,cAAkC,EAClC,YAAgC,EAChC,UAA8B,EAC9B,UAAyB,EACzB,OAAe,EACf,YAAoB,EACpB,MAAc,EACd,YAAoB,EACpB,UAAkB,EAClB,SAAiB,EACjB,MAAc;IAEd,MAAM,CAAC,UAAU,CAAC,6BAA6B,CAAC,CAAC;IACjD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,UAAU,CACzC,cAAc,EACd,YAAY,EACZ,UAAU,EACV,UAAU,EACV,OAAO,EACP,YAAY,EACZ,MAAM,EACN,YAAY,EACZ,UAAU,EACV,SAAS,EACT,MAAM,CAAC,CAAC;IACV,aAAa,CAAC,uBAAuB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtD,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO,MAAM,CAAC;AAChB,CAAC;AA7BD,gCA6BC;AAEM,KAAK,UAAU,OAAO,CAC3B,MAAc,EACd,MAA0B;IAE1B,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;IAElC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE9E,sEAAsE;IACtE,KAAK,IAAI,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACrC,yBAAyB;QACzB,MAAM,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;KACvG;IAED,OAAO,MAAM,uCAAuB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AACvD,CAAC;AAfD,0BAeC;AAED,sEAAsE;AACtE,4EAA4E;AAC5E,4EAA4E;AAC5E,6EAA6E;AAC7E,+CAA+C;AACxC,KAAK,UAAU,mBAAmB,CACvC,WAA+B,EAC/B,YAAgC,EAChC,MAA0B,EAC1B,MAAc,EACd,YAA0B;IAE1B,IAAI,MAAc,CAAC;IACnB,IAAI,WAAW,KAAK,SAAS,EAAE;QAC7B,MAAM,GAAG;;;;;;;;;;;;uCAY0B,WAAW;;8BAEpB,WAAW;;;;;;;;gDAQO,CAAC;KAC9C;SAAM;QACL,oEAAoE;QACpE,mFAAmF;QACnF,+EAA+E;QAC/E,kFAAkF;QAClF,6EAA6E;QAC7E,oFAAoF;QACpF,6CAA6C;QAC7C,YAAY,GAAG,YAAY,IAAI,CAAC,CAAC;QACjC,MAAM,GAAG;;;;;;;;4BAQe,YAAY;;;;;;;;;;;;;;;;;gDAiBQ,CAAC;KAC9C;IAED,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,mBAAmB,CAAC,CAAC;IACxE,EAAE,CAAC,aAAa,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC;IAE3C,MAAM,IAAI,WAAW,CAAC,UAAU,CAC9B,YAAY,EACZ;QACE,kBAAkB,EAAE,QAAQ;QAC5B,OAAO,EAAE,gBAAgB;QACzB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,YAAY,CAAC;KAC7E,EACD,EAAE,GAAG,EAAE,EAAE,4BAA4B,EAAE,YAAY,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;AACzE,CAAC;AAhFD,kDAgFC"} \ No newline at end of file +{"version":3,"file":"init.js","sourceRoot":"","sources":["../src/init.ts"],"names":[],"mappings":";;;;;;;;;AAAA,0EAA4D;AAC5D,uCAAyB;AACzB,2CAA6B;AAE7B,gEAAkD;AAClD,qCAA+C;AAC/C,4DAA8C;AAG9C,mDAAwE;AACxE,6CAA+B;AAExB,KAAK,UAAU,UAAU,CAC9B,SAA6B,EAC7B,UAAkB,EAClB,SAAiB,EACjB,OAAe,EACf,QAAgB,EAChB,IAAe,EACf,MAAc;IAEd,MAAM,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,MAAM,oBAAW,CAC9B,SAAS,EACT,UAAU,EACV,SAAS,EACT,OAAO,EACP,QAAQ,EACR,IAAI,EACJ,MAAM,CAAC,CAAC;IACV,MAAM,MAAM,CAAC,YAAY,EAAE,CAAC;IAC5B,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO,MAAM,CAAC;AAChB,CAAC;AArBD,gCAqBC;AAEM,KAAK,UAAU,UAAU,CAC9B,cAAkC,EAClC,YAAgC,EAChC,UAA8B,EAC9B,UAAyB,EACzB,OAAe,EACf,YAAoB,EACpB,MAAc,EACd,YAAoB,EACpB,UAAkB,EAClB,SAAiB,EACjB,MAAc;IAEd,MAAM,CAAC,UAAU,CAAC,6BAA6B,CAAC,CAAC;IACjD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,UAAU,CACzC,cAAc,EACd,YAAY,EACZ,UAAU,EACV,UAAU,EACV,OAAO,EACP,YAAY,EACZ,MAAM,EACN,YAAY,EACZ,UAAU,EACV,SAAS,EACT,MAAM,CAAC,CAAC;IACV,aAAa,CAAC,uBAAuB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtD,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO,MAAM,CAAC;AAChB,CAAC;AA7BD,gCA6BC;AAEM,KAAK,UAAU,OAAO,CAC3B,MAAc,EACd,MAA0B;IAE1B,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;IAElC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE9E,sEAAsE;IACtE,KAAK,IAAI,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACrC,yBAAyB;QACzB,MAAM,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;KACvG;IAED,OAAO,MAAM,uCAAuB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AACvD,CAAC;AAfD,0BAeC;AAED,sEAAsE;AACtE,4EAA4E;AAC5E,4EAA4E;AAC5E,6EAA6E;AAC7E,+CAA+C;AACxC,KAAK,UAAU,mBAAmB,CACvC,WAA+B,EAC/B,YAAgC,EAChC,MAA0B,EAC1B,MAAc,EACd,YAA0B;IAE1B,IAAI,MAAc,CAAC;IACnB,IAAI,WAAW,KAAK,SAAS,EAAE;QAC7B,MAAM,GAAG;;;;;;;;;;;;uCAY0B,WAAW;;8BAEpB,WAAW;;;;;;;;gDAQO,CAAC;KAC9C;SAAM;QACL,oEAAoE;QACpE,mFAAmF;QACnF,+EAA+E;QAC/E,kFAAkF;QAClF,6EAA6E;QAC7E,oFAAoF;QACpF,6CAA6C;QAC7C,YAAY,GAAG,YAAY,IAAI,CAAC,CAAC;QACjC,MAAM,GAAG;;;;;;;;4BAQe,YAAY;;;;;;;;;;;;;;;;;gDAiBQ,CAAC;KAC9C;IAED,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,mBAAmB,CAAC,CAAC;IACxE,EAAE,CAAC,aAAa,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC;IAE3C,MAAM,IAAI,WAAW,CAAC,UAAU,CAC9B,YAAY,EACZ;QACE,kBAAkB,EAAE,QAAQ;QAC5B,OAAO,EAAE,gBAAgB;QACzB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,YAAY,CAAC;KAC7E,EACD,EAAE,GAAG,EAAE,EAAE,4BAA4B,EAAE,YAAY,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;AACzE,CAAC;AAhFD,kDAgFC;AAEM,KAAK,UAAU,iBAAiB,CAAC,MAAc,EAAE,MAAc;IACpE,MAAM,CAAC,UAAU,CAAC,2BAA2B,CAAC,CAAC;IAE/C,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,iBAAiB,CAAC,CAAC;IAEjE,cAAc;IACd,IAAI;QACF,MAAM,IAAI,WAAW,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,kBAAkB,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;KACvF;IAAC,OAAO,CAAC,EAAE;QACV,mGAAmG;QACnG,uDAAuD;QACvD,MAAM,CAAC,QAAQ,EAAE,CAAC;QAClB,MAAM,IAAI,KAAK,CAAC,yFAAyF,CAAC,CAAC;KAC5G;IACD,uBAAuB;IACvB,IAAI;QACF,MAAM,IAAI,WAAW,CAAC,UAAU,CAC9B,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,0BAA0B,CAAC,EACpD,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;KAC5C;IAAC,OAAO,CAAC,EAAE;QACV,MAAM,CAAC,QAAQ,EAAE,CAAC;QAClB,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;KACxE;IACD,MAAM,CAAC,QAAQ,EAAE,CAAC;AACpB,CAAC;AAxBD,8CAwBC"} \ No newline at end of file diff --git a/src/init-action.ts b/src/init-action.ts index 504136a2f..6ca76bcc1 100644 --- a/src/init-action.ts +++ b/src/init-action.ts @@ -2,7 +2,7 @@ import * as core from '@actions/core'; import { CodeQL } from './codeql'; import * as configUtils from './config-utils'; -import { initCodeQL, initConfig, injectWindowsTracer, runInit } from './init'; +import { initCodeQL, initConfig, injectWindowsTracer, installPythonDeps, runInit} from './init'; import { getActionsLogger } from './logging'; import { parseRepositoryNwo } from './repository'; import * as util from './util'; @@ -66,6 +66,13 @@ async function run() { util.getRequiredEnvParam('RUNNER_TOOL_CACHE'), 'actions', logger); + + try { + await installPythonDeps(codeql, logger); + } catch (err) { + logger.warning(err.message + 'You can call this action with "setup-python-dependencies: false" to disable this process'); + } + config = await initConfig( core.getInput('languages'), core.getInput('queries'), diff --git a/src/init.ts b/src/init.ts index 8379e99f5..3a0d230d9 100644 --- a/src/init.ts +++ b/src/init.ts @@ -167,3 +167,29 @@ export async function injectWindowsTracer( ], { env: { 'ODASA_TRACER_CONFIGURATION': tracerConfig.spec } }).exec(); } + +export async function installPythonDeps(codeql: CodeQL, logger: Logger) { + logger.startGroup('Setup Python dependencies'); + + const scriptsFolder = path.resolve(__dirname, '../python-setup'); + + // Setup tools + try { + await new toolrunnner.ToolRunner(path.join(scriptsFolder, 'install_tools.sh')).exec(); + } catch (e) { + // This script tries to install some needed tools in the runner. It should not fail, but if it does + // we just abort the process without failing the action + logger.endGroup(); + throw new Error('Unable to download and extract the scripts needed for installing the python dependecies'); + } + // Install dependencies + try { + await new toolrunnner.ToolRunner( + path.join(scriptsFolder, 'auto_install_packages.py'), + [path.dirname(codeql.getPath())]).exec(); + } catch (e) { + logger.endGroup(); + throw new Error('We were unable to install your python dependencies.'); + } + logger.endGroup(); +} From 52274f18152908609f2ffa75f108edc04497d089 Mon Sep 17 00:00:00 2001 From: David Verdeguer Date: Fri, 11 Sep 2020 11:07:27 +0200 Subject: [PATCH 02/15] Setup python extractor for installed deps --- lib/analyze.js | 26 ++++++++++++++++++++++++++ lib/analyze.js.map | 2 +- src/analyze.ts | 40 +++++++++++++++++++++++++++++++++++++++- 3 files changed, 66 insertions(+), 2 deletions(-) diff --git a/lib/analyze.js b/lib/analyze.js index 2fc552fb1..851a30ab2 100644 --- a/lib/analyze.js +++ b/lib/analyze.js @@ -7,6 +7,7 @@ var __importStar = (this && this.__importStar) || function (mod) { return result; }; Object.defineProperty(exports, "__esModule", { value: true }); +const toolrunnner = __importStar(require("@actions/exec/lib/toolrunner")); const fs = __importStar(require("fs")); const path = __importStar(require("path")); const analysisPaths = __importStar(require("./analysis-paths")); @@ -15,6 +16,28 @@ const languages_1 = require("./languages"); const sharedEnv = __importStar(require("./shared-environment")); const upload_lib = __importStar(require("./upload-lib")); const util = __importStar(require("./util")); +async function setupPythonExtractor(logger) { + const codeqlPython = process.env["CODEQL_PYTHON"]; + if (codeqlPython === undefined || codeqlPython.length === 0) { + // If CODEQL_PYTHON is not set, no dependencies were installed, so we don't need to do anything + return; + } + let output = ''; + const options = { + listeners: { + stdout: (data) => { + output += data.toString(); + } + } + }; + await new toolrunnner.ToolRunner(codeqlPython, ['-c', 'import os; import pip; print(os.path.dirname(os.path.dirname(pip.__file__)))'], options).exec(); + logger.info('Setting LGTM_INDEX_IMPORT_PATH=' + output); + process.env['LGTM_INDEX_IMPORT_PATH'] = output; + output = ''; + await new toolrunnner.ToolRunner(codeqlPython, ['-c', 'import sys; print(sys.version_info[0])'], options).exec(); + logger.info('Setting LGTM_PYTHON_SETUP_VERSION=' + output); + process.env['LGTM_PYTHON_SETUP_VERSION'] = output; +} async function createdDBForScannedLanguages(config, logger) { // Insert the LGTM_INDEX_X env vars at this point so they are set when // we extract any scanned languages. @@ -23,6 +46,9 @@ async function createdDBForScannedLanguages(config, logger) { for (const language of config.languages) { if (languages_1.isScannedLanguage(language)) { logger.startGroup('Extracting ' + language); + if (language === languages_1.Language.python) { + await setupPythonExtractor(logger); + } await codeql.extractScannedLanguage(util.getCodeQLDatabasePath(config.tempDir, language), language); logger.endGroup(); } diff --git a/lib/analyze.js.map b/lib/analyze.js.map index 3907718ac..4625e8999 100644 --- a/lib/analyze.js.map +++ b/lib/analyze.js.map @@ -1 +1 @@ -{"version":3,"file":"analyze.js","sourceRoot":"","sources":["../src/analyze.ts"],"names":[],"mappings":";;;;;;;;;AAAA,uCAAyB;AACzB,2CAA6B;AAE7B,gEAAkD;AAClD,qCAAqC;AAErC,2CAAgD;AAGhD,gEAAkD;AAClD,yDAA2C;AAC3C,6CAA+B;AAiC/B,KAAK,UAAU,4BAA4B,CACzC,MAA0B,EAC1B,MAAc;IAEd,sEAAsE;IACtE,oCAAoC;IACpC,aAAa,CAAC,8BAA8B,CAAC,MAAM,CAAC,CAAC;IAErD,MAAM,MAAM,GAAG,kBAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAC3C,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACvC,IAAI,6BAAiB,CAAC,QAAQ,CAAC,EAAE;YAC/B,MAAM,CAAC,UAAU,CAAC,aAAa,GAAG,QAAQ,CAAC,CAAC;YAC5C,MAAM,MAAM,CAAC,sBAAsB,CAAC,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,EAAE,QAAQ,CAAC,CAAC;YACpG,MAAM,CAAC,QAAQ,EAAE,CAAC;SACnB;KACF;AACH,CAAC;AAED,KAAK,UAAU,wBAAwB,CACrC,MAA0B,EAC1B,MAAc;IAEd,MAAM,4BAA4B,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEnD,MAAM,MAAM,GAAG,kBAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAC3C,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACvC,MAAM,CAAC,UAAU,CAAC,aAAa,GAAG,QAAQ,CAAC,CAAC;QAC5C,MAAM,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;QACpF,MAAM,CAAC,QAAQ,EAAE,CAAC;KACnB;AACH,CAAC;AAED,2DAA2D;AAC3D,KAAK,UAAU,UAAU,CACvB,WAAmB,EACnB,UAAkB,EAClB,WAAmB,EACnB,MAA0B,EAC1B,MAAc;IAEd,MAAM,MAAM,GAAG,kBAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAC3C,KAAK,IAAI,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACrC,MAAM,CAAC,UAAU,CAAC,YAAY,GAAG,QAAQ,CAAC,CAAC;QAE3C,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC/C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE;YACxB,MAAM,IAAI,KAAK,CAAC,oBAAoB,GAAG,QAAQ,GAAG,gDAAgD,CAAC,CAAC;SACrG;QAED,IAAI;YACF,MAAM,YAAY,GAAG,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;YAC1E,uEAAuE;YACvE,2EAA2E;YAC3E,MAAM,UAAU,GAAG,YAAY,GAAG,cAAc,CAAC;YACjD,MAAM,kBAAkB,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACxE,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,kBAAkB,CAAC,CAAC;YACjD,MAAM,CAAC,KAAK,CAAC,uBAAuB,GAAG,QAAQ,GAAG,OAAO,GAAG,kBAAkB,CAAC,CAAC;YAEhF,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,QAAQ,GAAG,QAAQ,CAAC,CAAC;YAE9D,MAAM,MAAM,CAAC,eAAe,CAAC,YAAY,EAAE,SAAS,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW,CAAC,CAAC;YAE3F,MAAM,CAAC,KAAK,CAAC,6BAA6B,GAAG,QAAQ,GAAG,eAAe,GAAG,SAAS,GAAG,GAAG,CAAC,CAAC;YAC3F,MAAM,CAAC,QAAQ,EAAE,CAAC;SAEnB;QAAC,OAAO,CAAC,EAAE;YACV,+DAA+D;YAC/D,OAAO;gBACL,wBAAwB,EAAE,QAAQ;aACnC,CAAC;SACH;KACF;IAED,OAAO,EAAE,CAAC;AACZ,CAAC;AAEM,KAAK,UAAU,UAAU,CAC9B,aAA4B,EAC5B,SAAiB,EACjB,GAAW,EACX,WAA+B,EAC/B,YAAgC,EAChC,aAAiC,EACjC,YAAoB,EACpB,WAA+B,EAC/B,UAAkB,EAClB,SAAiB,EACjB,QAAiB,EACjB,IAAe,EACf,SAAiB,EACjB,UAAkB,EAClB,WAAmB,EACnB,MAA0B,EAC1B,MAAc;IAEd,8DAA8D;IAC9D,OAAO,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,0BAA0B,CAAC,CAAC;IAEzD,EAAE,CAAC,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE7C,MAAM,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;IAC5C,MAAM,wBAAwB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAE/C,MAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IAClC,MAAM,YAAY,GAAG,MAAM,UAAU,CAAC,SAAS,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IAE1F,IAAI,CAAC,QAAQ,EAAE;QACb,MAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;QACrC,OAAO,EAAE,GAAG,YAAY,EAAE,CAAC;KAC5B;IAED,MAAM,WAAW,GAAG,MAAM,UAAU,CAAC,MAAM,CACzC,SAAS,EACT,aAAa,EACb,SAAS,EACT,GAAG,EACH,WAAW,EACX,YAAY,EACZ,aAAa,EACb,YAAY,EACZ,WAAW,EACX,UAAU,EACV,SAAS,EACT,IAAI,EACJ,MAAM,CAAC,CAAC;IAEV,OAAO,EAAE,GAAG,YAAY,EAAE,GAAG,WAAW,EAAE,CAAC;AAC7C,CAAC;AAnDD,gCAmDC"} \ No newline at end of file +{"version":3,"file":"analyze.js","sourceRoot":"","sources":["../src/analyze.ts"],"names":[],"mappings":";;;;;;;;;AAAA,0EAA4D;AAC5D,uCAAyB;AACzB,2CAA6B;AAE7B,gEAAkD;AAClD,qCAAqC;AAErC,2CAA0D;AAG1D,gEAAkD;AAClD,yDAA2C;AAC3C,6CAA+B;AAiC/B,KAAK,UAAU,oBAAoB,CAAC,MAAc;IAChD,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;IAClD,IAAI,YAAY,KAAK,SAAS,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE;QAC3D,+FAA+F;QAC/F,OAAO;KACR;IAED,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,MAAM,OAAO,GAAG;QACd,SAAS,EAAE;YACT,MAAM,EAAE,CAAC,IAAY,EAAE,EAAE;gBACvB,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC5B,CAAC;SACF;KACF,CAAC;IAEF,MAAM,IAAI,WAAW,CAAC,UAAU,CAC9B,YAAY,EACZ,CAAC,IAAI,EAAE,8EAA8E,CAAC,EACtF,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;IAClB,MAAM,CAAC,IAAI,CAAC,iCAAiC,GAAG,MAAM,CAAC,CAAC;IACxD,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,GAAG,MAAM,CAAC;IAE/C,MAAM,GAAG,EAAE,CAAC;IACZ,MAAM,IAAI,WAAW,CAAC,UAAU,CAC9B,YAAY,EACZ,CAAC,IAAI,EAAE,wCAAwC,CAAC,EAChD,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;IAClB,MAAM,CAAC,IAAI,CAAC,oCAAoC,GAAG,MAAM,CAAC,CAAC;IAC3D,OAAO,CAAC,GAAG,CAAC,2BAA2B,CAAC,GAAG,MAAM,CAAC;AACpD,CAAC;AAED,KAAK,UAAU,4BAA4B,CACzC,MAA0B,EAC1B,MAAc;IAEd,sEAAsE;IACtE,oCAAoC;IACpC,aAAa,CAAC,8BAA8B,CAAC,MAAM,CAAC,CAAC;IAErD,MAAM,MAAM,GAAG,kBAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAC3C,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACvC,IAAI,6BAAiB,CAAC,QAAQ,CAAC,EAAE;YAC/B,MAAM,CAAC,UAAU,CAAC,aAAa,GAAG,QAAQ,CAAC,CAAC;YAE5C,IAAI,QAAQ,KAAK,oBAAQ,CAAC,MAAM,EAAE;gBAChC,MAAM,oBAAoB,CAAC,MAAM,CAAC,CAAC;aACpC;YAED,MAAM,MAAM,CAAC,sBAAsB,CAAC,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,EAAE,QAAQ,CAAC,CAAC;YACpG,MAAM,CAAC,QAAQ,EAAE,CAAC;SACnB;KACF;AACH,CAAC;AAED,KAAK,UAAU,wBAAwB,CACrC,MAA0B,EAC1B,MAAc;IAEd,MAAM,4BAA4B,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEnD,MAAM,MAAM,GAAG,kBAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAC3C,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACvC,MAAM,CAAC,UAAU,CAAC,aAAa,GAAG,QAAQ,CAAC,CAAC;QAC5C,MAAM,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;QACpF,MAAM,CAAC,QAAQ,EAAE,CAAC;KACnB;AACH,CAAC;AAED,2DAA2D;AAC3D,KAAK,UAAU,UAAU,CACvB,WAAmB,EACnB,UAAkB,EAClB,WAAmB,EACnB,MAA0B,EAC1B,MAAc;IAEd,MAAM,MAAM,GAAG,kBAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAC3C,KAAK,IAAI,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACrC,MAAM,CAAC,UAAU,CAAC,YAAY,GAAG,QAAQ,CAAC,CAAC;QAE3C,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC/C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE;YACxB,MAAM,IAAI,KAAK,CAAC,oBAAoB,GAAG,QAAQ,GAAG,gDAAgD,CAAC,CAAC;SACrG;QAED,IAAI;YACF,MAAM,YAAY,GAAG,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;YAC1E,uEAAuE;YACvE,2EAA2E;YAC3E,MAAM,UAAU,GAAG,YAAY,GAAG,cAAc,CAAC;YACjD,MAAM,kBAAkB,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACxE,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,kBAAkB,CAAC,CAAC;YACjD,MAAM,CAAC,KAAK,CAAC,uBAAuB,GAAG,QAAQ,GAAG,OAAO,GAAG,kBAAkB,CAAC,CAAC;YAEhF,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,QAAQ,GAAG,QAAQ,CAAC,CAAC;YAE9D,MAAM,MAAM,CAAC,eAAe,CAAC,YAAY,EAAE,SAAS,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW,CAAC,CAAC;YAE3F,MAAM,CAAC,KAAK,CAAC,6BAA6B,GAAG,QAAQ,GAAG,eAAe,GAAG,SAAS,GAAG,GAAG,CAAC,CAAC;YAC3F,MAAM,CAAC,QAAQ,EAAE,CAAC;SAEnB;QAAC,OAAO,CAAC,EAAE;YACV,+DAA+D;YAC/D,OAAO;gBACL,wBAAwB,EAAE,QAAQ;aACnC,CAAC;SACH;KACF;IAED,OAAO,EAAE,CAAC;AACZ,CAAC;AAEM,KAAK,UAAU,UAAU,CAC9B,aAA4B,EAC5B,SAAiB,EACjB,GAAW,EACX,WAA+B,EAC/B,YAAgC,EAChC,aAAiC,EACjC,YAAoB,EACpB,WAA+B,EAC/B,UAAkB,EAClB,SAAiB,EACjB,QAAiB,EACjB,IAAe,EACf,SAAiB,EACjB,UAAkB,EAClB,WAAmB,EACnB,MAA0B,EAC1B,MAAc;IAEd,8DAA8D;IAC9D,OAAO,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,0BAA0B,CAAC,CAAC;IAEzD,EAAE,CAAC,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE7C,MAAM,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;IAC5C,MAAM,wBAAwB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAE/C,MAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IAClC,MAAM,YAAY,GAAG,MAAM,UAAU,CAAC,SAAS,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IAE1F,IAAI,CAAC,QAAQ,EAAE;QACb,MAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;QACrC,OAAO,EAAE,GAAG,YAAY,EAAE,CAAC;KAC5B;IAED,MAAM,WAAW,GAAG,MAAM,UAAU,CAAC,MAAM,CACzC,SAAS,EACT,aAAa,EACb,SAAS,EACT,GAAG,EACH,WAAW,EACX,YAAY,EACZ,aAAa,EACb,YAAY,EACZ,WAAW,EACX,UAAU,EACV,SAAS,EACT,IAAI,EACJ,MAAM,CAAC,CAAC;IAEV,OAAO,EAAE,GAAG,YAAY,EAAE,GAAG,WAAW,EAAE,CAAC;AAC7C,CAAC;AAnDD,gCAmDC"} \ No newline at end of file diff --git a/src/analyze.ts b/src/analyze.ts index 588238793..e03744d3c 100644 --- a/src/analyze.ts +++ b/src/analyze.ts @@ -1,10 +1,11 @@ +import * as toolrunnner from '@actions/exec/lib/toolrunner'; import * as fs from 'fs'; import * as path from 'path'; import * as analysisPaths from './analysis-paths'; import { getCodeQL } from './codeql'; import * as configUtils from './config-utils'; -import { isScannedLanguage } from './languages'; +import { isScannedLanguage, Language } from './languages'; import { Logger } from './logging'; import { RepositoryNwo } from './repository'; import * as sharedEnv from './shared-environment'; @@ -42,6 +43,38 @@ export interface QueriesStatusReport { export interface AnalysisStatusReport extends upload_lib.UploadStatusReport, QueriesStatusReport {} +async function setupPythonExtractor(logger: Logger) { + const codeqlPython = process.env["CODEQL_PYTHON"]; + if (codeqlPython === undefined || codeqlPython.length === 0) { + // If CODEQL_PYTHON is not set, no dependencies were installed, so we don't need to do anything + return; + } + + let output = ''; + const options = { + listeners: { + stdout: (data: Buffer) => { + output += data.toString(); + } + } + }; + + await new toolrunnner.ToolRunner( + codeqlPython, + ['-c', 'import os; import pip; print(os.path.dirname(os.path.dirname(pip.__file__)))'], + options).exec(); + logger.info('Setting LGTM_INDEX_IMPORT_PATH=' + output); + process.env['LGTM_INDEX_IMPORT_PATH'] = output; + + output = ''; + await new toolrunnner.ToolRunner( + codeqlPython, + ['-c', 'import sys; print(sys.version_info[0])'], + options).exec(); + logger.info('Setting LGTM_PYTHON_SETUP_VERSION=' + output); + process.env['LGTM_PYTHON_SETUP_VERSION'] = output; +} + async function createdDBForScannedLanguages( config: configUtils.Config, logger: Logger) { @@ -54,6 +87,11 @@ async function createdDBForScannedLanguages( for (const language of config.languages) { if (isScannedLanguage(language)) { logger.startGroup('Extracting ' + language); + + if (language === Language.python) { + await setupPythonExtractor(logger); + } + await codeql.extractScannedLanguage(util.getCodeQLDatabasePath(config.tempDir, language), language); logger.endGroup(); } From 57010378502fc80386d9ee28c33236eca955df5c Mon Sep 17 00:00:00 2001 From: David Verdeguer Date: Fri, 11 Sep 2020 11:11:21 +0200 Subject: [PATCH 03/15] Add python scripts --- python-setup/auto_install_packages.py | 115 ++++++++++++++++++++++++++ python-setup/extractor_version.py | 53 ++++++++++++ python-setup/install_tools.sh | 31 +++++++ 3 files changed, 199 insertions(+) create mode 100755 python-setup/auto_install_packages.py create mode 100755 python-setup/extractor_version.py create mode 100755 python-setup/install_tools.sh diff --git a/python-setup/auto_install_packages.py b/python-setup/auto_install_packages.py new file mode 100755 index 000000000..dd456ed7c --- /dev/null +++ b/python-setup/auto_install_packages.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 + +import sys +import os +import subprocess +from tempfile import mkdtemp + +import extractor_version + + +def _check_call(command): + print('+ {}'.format(' '.join(command)), flush=True) + subprocess.check_call(command, stdin=subprocess.DEVNULL) + + +def _check_output(command): + print('+ {}'.format(' '.join(command)), flush=True) + out = subprocess.check_output(command, stdin=subprocess.DEVNULL) + print(out, flush=True) + sys.stderr.flush() + return out + + +def install_packages_with_poetry(): + try: + _check_call(['poetry', 'install', '--no-root']) + except subprocess.CalledProcessError: + sys.exit('package installation with poetry failed, see error above') + + # poetry is super annoying with `poetry run`, since it will put lots of output on + # STDOUT if the current global python interpreter is not matching the one in the + # virtualenv for the package, which was the case for using poetry for Python 2 when + # default system interpreter was Python 3 :/ + + poetry_out = _check_output(['poetry', 'run', 'which', 'python']) + python_executable_path = poetry_out.decode('utf-8').splitlines()[-1] + + return python_executable_path + + +def install_packages_with_pipenv(): + try: + _check_call(['pipenv', 'install', '--keep-outdated', '--ignore-pipfile']) + except subprocess.CalledProcessError: + sys.exit('package installation with pipenv failed, see error above') + + pipenv_out = _check_output(['pipenv', 'run', 'which', 'python']) + python_executable_path = pipenv_out.decode('utf-8').splitlines()[-1] + + return python_executable_path + + +def install_requirements_txt_packages(version: int, requirements_txt_path: str): + # create temporary directory ... that just lives "forever" + venv_path = mkdtemp(prefix='codeql-action-python-autoinstall-') + + # virtualenv is a bit nicer for setting up virtual environment, since it will provide + # up-to-date versions of pip/setuptools/wheel which basic `python3 -m venv venv` won't + + if version == 2: + _check_call(['python2', '-m', 'virtualenv', venv_path]) + elif version == 3: + _check_call(['python3', '-m', 'virtualenv', venv_path]) + + venv_pip = os.path.join(venv_path, 'bin', 'pip') + try: + _check_call([venv_pip, 'install', '-r', requirements_txt_path]) + except subprocess.CalledProcessError: + sys.exit('package installation with pip failed, see error above') + + venv_python = os.path.join(venv_path, 'bin', 'python') + + return venv_python + + +def install_packages() -> str: + if os.path.exists('poetry.lock'): + print('Found poetry.lock, will install packages with poetry', flush=True) + return install_packages_with_poetry() + + if os.path.exists('Pipfile') or os.path.exists('Pipfile.lock'): + if os.path.exists('Pipfile.lock'): + print('Found Pipfile.lock, will install packages with Pipenv', flush=True) + else: + print('Found Pipfile, will install packages with Pipenv', flush=True) + return install_packages_with_pipenv() + + version = extractor_version.get_extractor_version(sys.argv[1], quiet=False) + + if os.path.exists('requirements.txt'): + print('Found requirements.txt, will install packages with pip', flush=True) + return install_requirements_txt_packages(version, 'requirements.txt') + + print("was not able to install packages automatically", flush=True) + + +if __name__ == "__main__": + if len(sys.argv) != 2: + sys.exit('Must provide base directory for codeql tool as only argument') + + # The binaries for packages installed with `pip install --user` are not available on + # PATH by default, so we need to manually add them. + os.environ['PATH'] = os.path.expanduser('~/.local/bin') + os.pathsep + os.environ['PATH'] + + python_executable_path = install_packages() + + if python_executable_path is not None: + print("Setting CODEQL_PYTHON={}".format(python_executable_path)) + print("::set-env name=CODEQL_PYTHON::{}".format(python_executable_path)) + +# TODO: +# - no packages +# - poetry without version +# - pipenv without version +# - pipenv without lockfile \ No newline at end of file diff --git a/python-setup/extractor_version.py b/python-setup/extractor_version.py new file mode 100755 index 000000000..0465c32ae --- /dev/null +++ b/python-setup/extractor_version.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python + +# A quick hack to get package installation for Code Scanning to work, +# since it needs to know which version we're going to analyze the project as. + +# This file needs to be placed next to `python_tracer.py`, so in +# `/python/tools/` + +from __future__ import print_function, division + +import os +import sys +from contextlib import contextmanager + + +@contextmanager +def suppress_stdout_stderr(): + # taken from + # https://thesmithfam.org/blog/2012/10/25/temporarily-suppress-console-output-in-python/ + with open(os.devnull, "w") as devnull: + old_stdout = sys.stdout + old_stderr = sys.stderr + sys.stdout = devnull + sys.stderr = devnull + try: + yield + finally: + sys.stdout = old_stdout + sys.stderr = old_stderr + + +def get_extractor_version(codeql_base_dir: str, quiet: bool = True) -> int: + + extractor_dir = os.path.join(codeql_base_dir, 'python', 'tools') + sys.path = [extractor_dir] + sys.path + + from python_tracer import getzipfilename + + zippath = os.path.join(extractor_dir, getzipfilename()) + sys.path = [zippath] + sys.path + import buildtools.discover + + if quiet: + with suppress_stdout_stderr(): + return buildtools.discover.get_version() + else: + return buildtools.discover.get_version() + + +if __name__ == "__main__": + codeql_base_dir = sys.argv[1] + version = get_extractor_version(codeql_base_dir) + print('{!r}'.format(version)) \ No newline at end of file diff --git a/python-setup/install_tools.sh b/python-setup/install_tools.sh new file mode 100755 index 000000000..cf38509f1 --- /dev/null +++ b/python-setup/install_tools.sh @@ -0,0 +1,31 @@ +#!/bin/sh +set -x +set -e + +# The binaries for packages installed with `pip install --user` are not available on PATH +# by default, so we fix up PATH to suppress warnings by pip. This also needs to be done by +# any script that needs to access poetry/pipenv. +# +# Using `::add-path::` from the actions toolkit is not enough, since that only affects +# subsequent actions in the current job, and not the current action. +export PATH="$HOME/.local/bin:$PATH" + +python2 -m pip install --user --upgrade pip setuptools wheel +python3 -m pip install --user --upgrade pip setuptools wheel + +# virtualenv is a bit nicer for setting up virtual environment, since it will provide up-to-date versions of +# pip/setuptools/wheel which basic `python3 -m venv venv` won't +python2 -m pip install --user virtualenv +python3 -m pip install --user virtualenv + +# venv is required for installation of poetry or pipenv (I forgot which) +sudo apt-get install -y python3-venv + +# We're install poetry with pip instead of the recommended way, since the recommended way +# caused some problem since `poetry run` gives output like: +# +# /root/.poetry/lib/poetry/_vendor/py2.7/subprocess32.py:149: RuntimeWarning: The _posixsubprocess module is not being used. Child process reliability may suffer if your program uses threads. +# "program uses threads.", RuntimeWarning) +# LGTM_PYTHON_SETUP_VERSION=The currently activated Python version 2.7.18 is not supported by the project (^3.5). Trying to find and use a compatible version. Using python3 (3.8.2) 3 + +python3 -m pip install --user poetry pipenv \ No newline at end of file From 7fc41c62bede1ccc3ebd653ca9557bdf42113ada Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Mon, 7 Sep 2020 18:14:36 +0200 Subject: [PATCH 04/15] Update python scripts to handle setup.py --- python-setup/auto_install_packages.py | 46 +++++++++++++++++++++------ 1 file changed, 36 insertions(+), 10 deletions(-) diff --git a/python-setup/auto_install_packages.py b/python-setup/auto_install_packages.py index dd456ed7c..f9db92520 100755 --- a/python-setup/auto_install_packages.py +++ b/python-setup/auto_install_packages.py @@ -50,7 +50,7 @@ def install_packages_with_pipenv(): return python_executable_path -def install_requirements_txt_packages(version: int, requirements_txt_path: str): +def _create_venv(version: int): # create temporary directory ... that just lives "forever" venv_path = mkdtemp(prefix='codeql-action-python-autoinstall-') @@ -62,14 +62,42 @@ def install_requirements_txt_packages(version: int, requirements_txt_path: str): elif version == 3: _check_call(['python3', '-m', 'virtualenv', venv_path]) + return venv_path + + +def install_requirements_txt_packages(version: int): + venv_path = _create_venv(version) venv_pip = os.path.join(venv_path, 'bin', 'pip') + venv_python = os.path.join(venv_path, 'bin', 'python') + try: - _check_call([venv_pip, 'install', '-r', requirements_txt_path]) + _check_call([venv_pip, 'install', '-r', 'requirements.txt']) except subprocess.CalledProcessError: - sys.exit('package installation with pip failed, see error above') + sys.exit('package installation with `pip install -r requirements.txt` failed, see error above') + + return venv_python + +def install_with_setup_py(version: int): + venv_path = _create_venv(version) + venv_pip = os.path.join(venv_path, 'bin', 'pip') venv_python = os.path.join(venv_path, 'bin', 'python') + try: + # We have to choose between `python setup.py develop` and `pip install -e .`. + # Modern projects use `pip install -e .` and I wasn't able to see any downsides + # to doing so. However, `python setup.py develop` has some downsides -- from + # https://stackoverflow.com/a/19048754 : + # > Note that it is highly recommended to use pip install . (install) and pip + # > install -e . (developer install) to install packages, as invoking setup.py + # > directly will do the wrong things for many dependencies, such as pull + # > prereleases and incompatible package versions, or make the package hard to + # > uninstall with pip. + + _check_call([venv_pip, 'install', '-e', '.']) + except subprocess.CalledProcessError: + sys.exit('package installation with `pip install -e .` failed, see error above') + return venv_python @@ -89,7 +117,11 @@ def install_packages() -> str: if os.path.exists('requirements.txt'): print('Found requirements.txt, will install packages with pip', flush=True) - return install_requirements_txt_packages(version, 'requirements.txt') + return install_requirements_txt_packages(version) + + if os.path.exists('setup.py'): + print('Found setup.py, will install package with pip in editable mode', flush=True) + return install_with_setup_py(version) print("was not able to install packages automatically", flush=True) @@ -107,9 +139,3 @@ def install_packages() -> str: if python_executable_path is not None: print("Setting CODEQL_PYTHON={}".format(python_executable_path)) print("::set-env name=CODEQL_PYTHON::{}".format(python_executable_path)) - -# TODO: -# - no packages -# - poetry without version -# - pipenv without version -# - pipenv without lockfile \ No newline at end of file From 1b20fa78be4a91f178eae29d381de3a8f7e08671 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Mon, 7 Sep 2020 18:14:44 +0200 Subject: [PATCH 05/15] Handle error in poetry 1.0.10 --- python-setup/install_tools.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python-setup/install_tools.sh b/python-setup/install_tools.sh index cf38509f1..8d6ac75d8 100755 --- a/python-setup/install_tools.sh +++ b/python-setup/install_tools.sh @@ -28,4 +28,6 @@ sudo apt-get install -y python3-venv # "program uses threads.", RuntimeWarning) # LGTM_PYTHON_SETUP_VERSION=The currently activated Python version 2.7.18 is not supported by the project (^3.5). Trying to find and use a compatible version. Using python3 (3.8.2) 3 -python3 -m pip install --user poetry pipenv \ No newline at end of file +# poetry 1.0.10 has error (https://github.com/python-poetry/poetry/issues/2711) +python3 -m pip install --user poetry!=1.0.10 +python3 -m pip install --user pipenv From 147734591c9314cbe42722000208af95de3846c0 Mon Sep 17 00:00:00 2001 From: David Verdeguer Date: Wed, 23 Sep 2020 17:49:39 +0200 Subject: [PATCH 06/15] Check platform before installing python tools --- lib/init.js | 27 +++++++++++++++++---------- lib/init.js.map | 2 +- src/init.ts | 37 ++++++++++++++++++++++++------------- 3 files changed, 42 insertions(+), 24 deletions(-) diff --git a/lib/init.js b/lib/init.js index eec0b749a..1ad956db2 100644 --- a/lib/init.js +++ b/lib/init.js @@ -123,16 +123,23 @@ async function injectWindowsTracer(processName, processLevel, config, codeql, tr exports.injectWindowsTracer = injectWindowsTracer; async function installPythonDeps(codeql, logger) { logger.startGroup("Setup Python dependencies"); - const scriptsFolder = path.resolve(__dirname, "../python-setup"); - // Setup tools - try { - await new toolrunnner.ToolRunner(path.join(scriptsFolder, "install_tools.sh")).exec(); - } - catch (e) { - // This script tries to install some needed tools in the runner. It should not fail, but if it does - // we just abort the process without failing the action + if (process.platform !== "linux") { + logger.info("Currently, auto-installing python dependancies is only supported on linux"); logger.endGroup(); - throw new Error("Unable to download and extract the scripts needed for installing the python dependecies"); + return; + } + const scriptsFolder = path.resolve(__dirname, "../python-setup"); + // Setup tools on the Github hosted runners + if (process.env["ImageOS"] !== undefined) { + try { + await new toolrunnner.ToolRunner(path.join(scriptsFolder, "install_tools.sh")).exec(); + } + catch (e) { + // This script tries to install some needed tools in the runner. It should not fail, but if it does + // we just abort the process without failing the action + logger.endGroup(); + logger.warning("Unable to download and extract the tools needed for installing the python dependecies"); + } } // Install dependencies try { @@ -140,7 +147,7 @@ async function installPythonDeps(codeql, logger) { } catch (e) { logger.endGroup(); - throw new Error("We were unable to install your python dependencies."); + logger.warning("We were unable to install your python dependencies."); } logger.endGroup(); } diff --git a/lib/init.js.map b/lib/init.js.map index 76ae1d647..af3e3bc76 100644 --- a/lib/init.js.map +++ b/lib/init.js.map @@ -1 +1 @@ -{"version":3,"file":"init.js","sourceRoot":"","sources":["../src/init.ts"],"names":[],"mappings":";;;;;;;;;AAAA,0EAA4D;AAC5D,uCAAyB;AACzB,2CAA6B;AAE7B,gEAAkD;AAClD,qCAA+C;AAC/C,4DAA8C;AAG9C,mDAAwE;AACxE,6CAA+B;AAExB,KAAK,UAAU,UAAU,CAC9B,SAA6B,EAC7B,UAAkB,EAClB,SAAiB,EACjB,OAAe,EACf,QAAgB,EAChB,IAAe,EACf,MAAc;IAEd,MAAM,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,MAAM,oBAAW,CAC9B,SAAS,EACT,UAAU,EACV,SAAS,EACT,OAAO,EACP,QAAQ,EACR,IAAI,EACJ,MAAM,CACP,CAAC;IACF,MAAM,MAAM,CAAC,YAAY,EAAE,CAAC;IAC5B,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO,MAAM,CAAC;AAChB,CAAC;AAtBD,gCAsBC;AAEM,KAAK,UAAU,UAAU,CAC9B,cAAkC,EAClC,YAAgC,EAChC,UAA8B,EAC9B,UAAyB,EACzB,OAAe,EACf,YAAoB,EACpB,MAAc,EACd,YAAoB,EACpB,UAAkB,EAClB,SAAiB,EACjB,MAAc;IAEd,MAAM,CAAC,UAAU,CAAC,6BAA6B,CAAC,CAAC;IACjD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,UAAU,CACzC,cAAc,EACd,YAAY,EACZ,UAAU,EACV,UAAU,EACV,OAAO,EACP,YAAY,EACZ,MAAM,EACN,YAAY,EACZ,UAAU,EACV,SAAS,EACT,MAAM,CACP,CAAC;IACF,aAAa,CAAC,uBAAuB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtD,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO,MAAM,CAAC;AAChB,CAAC;AA9BD,gCA8BC;AAEM,KAAK,UAAU,OAAO,CAC3B,MAAc,EACd,MAA0B;IAE1B,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;IAElC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE9E,sEAAsE;IACtE,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACvC,yBAAyB;QACzB,MAAM,MAAM,CAAC,YAAY,CACvB,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,EACpD,QAAQ,EACR,UAAU,CACX,CAAC;KACH;IAED,OAAO,MAAM,uCAAuB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AACvD,CAAC;AAnBD,0BAmBC;AAED,sEAAsE;AACtE,4EAA4E;AAC5E,4EAA4E;AAC5E,6EAA6E;AAC7E,+CAA+C;AACxC,KAAK,UAAU,mBAAmB,CACvC,WAA+B,EAC/B,YAAgC,EAChC,MAA0B,EAC1B,MAAc,EACd,YAA0B;IAE1B,IAAI,MAAc,CAAC;IACnB,IAAI,WAAW,KAAK,SAAS,EAAE;QAC7B,MAAM,GAAG;;;;;;;;;;;;uCAY0B,WAAW;;8BAEpB,WAAW;;;;;;;;gDAQO,CAAC;KAC9C;SAAM;QACL,oEAAoE;QACpE,mFAAmF;QACnF,+EAA+E;QAC/E,kFAAkF;QAClF,6EAA6E;QAC7E,oFAAoF;QACpF,6CAA6C;QAC7C,YAAY,GAAG,YAAY,IAAI,CAAC,CAAC;QACjC,MAAM,GAAG;;;;;;;;4BAQe,YAAY;;;;;;;;;;;;;;;;;gDAiBQ,CAAC;KAC9C;IAED,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,mBAAmB,CAAC,CAAC;IACxE,EAAE,CAAC,aAAa,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC;IAE3C,MAAM,IAAI,WAAW,CAAC,UAAU,CAC9B,YAAY,EACZ;QACE,kBAAkB;QAClB,QAAQ;QACR,OAAO;QACP,gBAAgB;QAChB,IAAI,CAAC,OAAO,CACV,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,EAC9B,OAAO,EACP,OAAO,EACP,YAAY,CACb;KACF,EACD,EAAE,GAAG,EAAE,EAAE,0BAA0B,EAAE,YAAY,CAAC,IAAI,EAAE,EAAE,CAC3D,CAAC,IAAI,EAAE,CAAC;AACX,CAAC;AAxFD,kDAwFC;AAEM,KAAK,UAAU,iBAAiB,CAAC,MAAc,EAAE,MAAc;IACpE,MAAM,CAAC,UAAU,CAAC,2BAA2B,CAAC,CAAC;IAE/C,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,iBAAiB,CAAC,CAAC;IAEjE,cAAc;IACd,IAAI;QACF,MAAM,IAAI,WAAW,CAAC,UAAU,CAC9B,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,kBAAkB,CAAC,CAC7C,CAAC,IAAI,EAAE,CAAC;KACV;IAAC,OAAO,CAAC,EAAE;QACV,mGAAmG;QACnG,uDAAuD;QACvD,MAAM,CAAC,QAAQ,EAAE,CAAC;QAClB,MAAM,IAAI,KAAK,CACb,yFAAyF,CAC1F,CAAC;KACH;IACD,uBAAuB;IACvB,IAAI;QACF,MAAM,IAAI,WAAW,CAAC,UAAU,CAC9B,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,0BAA0B,CAAC,EACpD,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,CACjC,CAAC,IAAI,EAAE,CAAC;KACV;IAAC,OAAO,CAAC,EAAE;QACV,MAAM,CAAC,QAAQ,EAAE,CAAC;QAClB,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;KACxE;IACD,MAAM,CAAC,QAAQ,EAAE,CAAC;AACpB,CAAC;AA7BD,8CA6BC"} \ No newline at end of file +{"version":3,"file":"init.js","sourceRoot":"","sources":["../src/init.ts"],"names":[],"mappings":";;;;;;;;;AAAA,0EAA4D;AAC5D,uCAAyB;AACzB,2CAA6B;AAE7B,gEAAkD;AAClD,qCAA+C;AAC/C,4DAA8C;AAG9C,mDAAwE;AACxE,6CAA+B;AAExB,KAAK,UAAU,UAAU,CAC9B,SAA6B,EAC7B,UAAkB,EAClB,SAAiB,EACjB,OAAe,EACf,QAAgB,EAChB,IAAe,EACf,MAAc;IAEd,MAAM,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,MAAM,oBAAW,CAC9B,SAAS,EACT,UAAU,EACV,SAAS,EACT,OAAO,EACP,QAAQ,EACR,IAAI,EACJ,MAAM,CACP,CAAC;IACF,MAAM,MAAM,CAAC,YAAY,EAAE,CAAC;IAC5B,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO,MAAM,CAAC;AAChB,CAAC;AAtBD,gCAsBC;AAEM,KAAK,UAAU,UAAU,CAC9B,cAAkC,EAClC,YAAgC,EAChC,UAA8B,EAC9B,UAAyB,EACzB,OAAe,EACf,YAAoB,EACpB,MAAc,EACd,YAAoB,EACpB,UAAkB,EAClB,SAAiB,EACjB,MAAc;IAEd,MAAM,CAAC,UAAU,CAAC,6BAA6B,CAAC,CAAC;IACjD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,UAAU,CACzC,cAAc,EACd,YAAY,EACZ,UAAU,EACV,UAAU,EACV,OAAO,EACP,YAAY,EACZ,MAAM,EACN,YAAY,EACZ,UAAU,EACV,SAAS,EACT,MAAM,CACP,CAAC;IACF,aAAa,CAAC,uBAAuB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtD,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO,MAAM,CAAC;AAChB,CAAC;AA9BD,gCA8BC;AAEM,KAAK,UAAU,OAAO,CAC3B,MAAc,EACd,MAA0B;IAE1B,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;IAElC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE9E,sEAAsE;IACtE,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACvC,yBAAyB;QACzB,MAAM,MAAM,CAAC,YAAY,CACvB,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,EACpD,QAAQ,EACR,UAAU,CACX,CAAC;KACH;IAED,OAAO,MAAM,uCAAuB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AACvD,CAAC;AAnBD,0BAmBC;AAED,sEAAsE;AACtE,4EAA4E;AAC5E,4EAA4E;AAC5E,6EAA6E;AAC7E,+CAA+C;AACxC,KAAK,UAAU,mBAAmB,CACvC,WAA+B,EAC/B,YAAgC,EAChC,MAA0B,EAC1B,MAAc,EACd,YAA0B;IAE1B,IAAI,MAAc,CAAC;IACnB,IAAI,WAAW,KAAK,SAAS,EAAE;QAC7B,MAAM,GAAG;;;;;;;;;;;;uCAY0B,WAAW;;8BAEpB,WAAW;;;;;;;;gDAQO,CAAC;KAC9C;SAAM;QACL,oEAAoE;QACpE,mFAAmF;QACnF,+EAA+E;QAC/E,kFAAkF;QAClF,6EAA6E;QAC7E,oFAAoF;QACpF,6CAA6C;QAC7C,YAAY,GAAG,YAAY,IAAI,CAAC,CAAC;QACjC,MAAM,GAAG;;;;;;;;4BAQe,YAAY;;;;;;;;;;;;;;;;;gDAiBQ,CAAC;KAC9C;IAED,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,mBAAmB,CAAC,CAAC;IACxE,EAAE,CAAC,aAAa,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC;IAE3C,MAAM,IAAI,WAAW,CAAC,UAAU,CAC9B,YAAY,EACZ;QACE,kBAAkB;QAClB,QAAQ;QACR,OAAO;QACP,gBAAgB;QAChB,IAAI,CAAC,OAAO,CACV,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,EAC9B,OAAO,EACP,OAAO,EACP,YAAY,CACb;KACF,EACD,EAAE,GAAG,EAAE,EAAE,0BAA0B,EAAE,YAAY,CAAC,IAAI,EAAE,EAAE,CAC3D,CAAC,IAAI,EAAE,CAAC;AACX,CAAC;AAxFD,kDAwFC;AAEM,KAAK,UAAU,iBAAiB,CAAC,MAAc,EAAE,MAAc;IACpE,MAAM,CAAC,UAAU,CAAC,2BAA2B,CAAC,CAAC;IAE/C,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE;QAChC,MAAM,CAAC,IAAI,CACT,2EAA2E,CAC5E,CAAC;QACF,MAAM,CAAC,QAAQ,EAAE,CAAC;QAClB,OAAO;KACR;IAED,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,iBAAiB,CAAC,CAAC;IAEjE,2CAA2C;IAC3C,IAAI,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,SAAS,EAAE;QACxC,IAAI;YACF,MAAM,IAAI,WAAW,CAAC,UAAU,CAC9B,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,kBAAkB,CAAC,CAC7C,CAAC,IAAI,EAAE,CAAC;SACV;QAAC,OAAO,CAAC,EAAE;YACV,mGAAmG;YACnG,uDAAuD;YACvD,MAAM,CAAC,QAAQ,EAAE,CAAC;YAClB,MAAM,CAAC,OAAO,CACZ,uFAAuF,CACxF,CAAC;SACH;KACF;IAED,uBAAuB;IACvB,IAAI;QACF,MAAM,IAAI,WAAW,CAAC,UAAU,CAC9B,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,0BAA0B,CAAC,EACpD,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,CACjC,CAAC,IAAI,EAAE,CAAC;KACV;IAAC,OAAO,CAAC,EAAE;QACV,MAAM,CAAC,QAAQ,EAAE,CAAC;QAClB,MAAM,CAAC,OAAO,CAAC,qDAAqD,CAAC,CAAC;KACvE;IACD,MAAM,CAAC,QAAQ,EAAE,CAAC;AACpB,CAAC;AAxCD,8CAwCC"} \ No newline at end of file diff --git a/src/init.ts b/src/init.ts index d33f5348a..8913802d1 100644 --- a/src/init.ts +++ b/src/init.ts @@ -185,21 +185,32 @@ export async function injectWindowsTracer( export async function installPythonDeps(codeql: CodeQL, logger: Logger) { logger.startGroup("Setup Python dependencies"); + if (process.platform !== "linux") { + logger.info( + "Currently, auto-installing python dependancies is only supported on linux" + ); + logger.endGroup(); + return; + } + const scriptsFolder = path.resolve(__dirname, "../python-setup"); - // Setup tools - try { - await new toolrunnner.ToolRunner( - path.join(scriptsFolder, "install_tools.sh") - ).exec(); - } catch (e) { - // This script tries to install some needed tools in the runner. It should not fail, but if it does - // we just abort the process without failing the action - logger.endGroup(); - throw new Error( - "Unable to download and extract the scripts needed for installing the python dependecies" - ); + // Setup tools on the Github hosted runners + if (process.env["ImageOS"] !== undefined) { + try { + await new toolrunnner.ToolRunner( + path.join(scriptsFolder, "install_tools.sh") + ).exec(); + } catch (e) { + // This script tries to install some needed tools in the runner. It should not fail, but if it does + // we just abort the process without failing the action + logger.endGroup(); + logger.warning( + "Unable to download and extract the tools needed for installing the python dependecies" + ); + } } + // Install dependencies try { await new toolrunnner.ToolRunner( @@ -208,7 +219,7 @@ export async function installPythonDeps(codeql: CodeQL, logger: Logger) { ).exec(); } catch (e) { logger.endGroup(); - throw new Error("We were unable to install your python dependencies."); + logger.warning("We were unable to install your python dependencies."); } logger.endGroup(); } From 34955967b7554dee0db1533ad41d1b44479d251c Mon Sep 17 00:00:00 2001 From: David Verdeguer Date: Wed, 23 Sep 2020 18:06:08 +0200 Subject: [PATCH 07/15] Fix python string --- lib/analyze.js | 2 +- src/analyze.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/analyze.js b/lib/analyze.js index a00232e42..5c03ddd34 100644 --- a/lib/analyze.js +++ b/lib/analyze.js @@ -32,7 +32,7 @@ async function setupPythonExtractor(logger) { }; await new toolrunnner.ToolRunner(codeqlPython, [ "-c", - "import·os;·import·pip;·print(os.path.dirname(os.path.dirname(pip.__file__)))", + "import os; import pip; print(os.path.dirname(os.path.dirname(pip.__file__)))", ], options).exec(); logger.info(`Setting LGTM_INDEX_IMPORT_PATH=${output}`); process.env["LGTM_INDEX_IMPORT_PATH"] = output; diff --git a/src/analyze.ts b/src/analyze.ts index f52fe04c9..eaa23a918 100644 --- a/src/analyze.ts +++ b/src/analyze.ts @@ -65,7 +65,7 @@ async function setupPythonExtractor(logger: Logger) { codeqlPython, [ "-c", - "import·os;·import·pip;·print(os.path.dirname(os.path.dirname(pip.__file__)))", + "import os; import pip; print(os.path.dirname(os.path.dirname(pip.__file__)))", ], options ).exec(); From 6645c550ef99d5cb1f0e75bf12dd320072feb5a7 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Thu, 24 Sep 2020 15:36:00 +0200 Subject: [PATCH 08/15] Apply suggestions from code review Co-authored-by: Marco Gario --- python-setup/extractor_version.py | 3 +-- python-setup/install_tools.sh | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/python-setup/extractor_version.py b/python-setup/extractor_version.py index 0465c32ae..63e1c4211 100755 --- a/python-setup/extractor_version.py +++ b/python-setup/extractor_version.py @@ -30,7 +30,6 @@ def suppress_stdout_stderr(): def get_extractor_version(codeql_base_dir: str, quiet: bool = True) -> int: - extractor_dir = os.path.join(codeql_base_dir, 'python', 'tools') sys.path = [extractor_dir] + sys.path @@ -50,4 +49,4 @@ def get_extractor_version(codeql_base_dir: str, quiet: bool = True) -> int: if __name__ == "__main__": codeql_base_dir = sys.argv[1] version = get_extractor_version(codeql_base_dir) - print('{!r}'.format(version)) \ No newline at end of file + print('{!r}'.format(version)) diff --git a/python-setup/install_tools.sh b/python-setup/install_tools.sh index 8d6ac75d8..5179a5c24 100755 --- a/python-setup/install_tools.sh +++ b/python-setup/install_tools.sh @@ -21,7 +21,7 @@ python3 -m pip install --user virtualenv # venv is required for installation of poetry or pipenv (I forgot which) sudo apt-get install -y python3-venv -# We're install poetry with pip instead of the recommended way, since the recommended way +# We install poetry with pip instead of the recommended way, since the recommended way # caused some problem since `poetry run` gives output like: # # /root/.poetry/lib/poetry/_vendor/py2.7/subprocess32.py:149: RuntimeWarning: The _posixsubprocess module is not being used. Child process reliability may suffer if your program uses threads. From dffce9945cccc9e617037d91935bdd2d73362fa9 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Thu, 24 Sep 2020 15:43:34 +0200 Subject: [PATCH 09/15] Minor fixes to python installation scripts based on code review --- python-setup/auto_install_packages.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/python-setup/auto_install_packages.py b/python-setup/auto_install_packages.py index f9db92520..54a2d9669 100755 --- a/python-setup/auto_install_packages.py +++ b/python-setup/auto_install_packages.py @@ -4,6 +4,7 @@ import os import subprocess from tempfile import mkdtemp +from typing import Optional import extractor_version @@ -101,7 +102,7 @@ def install_with_setup_py(version: int): return venv_python -def install_packages() -> str: +def install_packages(codeql_base_dir) -> Optional[str]: if os.path.exists('poetry.lock'): print('Found poetry.lock, will install packages with poetry', flush=True) return install_packages_with_poetry() @@ -113,7 +114,7 @@ def install_packages() -> str: print('Found Pipfile, will install packages with Pipenv', flush=True) return install_packages_with_pipenv() - version = extractor_version.get_extractor_version(sys.argv[1], quiet=False) + version = extractor_version.get_extractor_version(codeql_base_dir, quiet=False) if os.path.exists('requirements.txt'): print('Found requirements.txt, will install packages with pip', flush=True) @@ -124,17 +125,20 @@ def install_packages() -> str: return install_with_setup_py(version) print("was not able to install packages automatically", flush=True) + return None if __name__ == "__main__": if len(sys.argv) != 2: sys.exit('Must provide base directory for codeql tool as only argument') + codeql_base_dir = sys.argv[1] + # The binaries for packages installed with `pip install --user` are not available on # PATH by default, so we need to manually add them. os.environ['PATH'] = os.path.expanduser('~/.local/bin') + os.pathsep + os.environ['PATH'] - python_executable_path = install_packages() + python_executable_path = install_packages(codeql_base_dir) if python_executable_path is not None: print("Setting CODEQL_PYTHON={}".format(python_executable_path)) From 5419fcd735d49504a441061f09a2d45974f6f4a4 Mon Sep 17 00:00:00 2001 From: David Verdeguer Date: Fri, 25 Sep 2020 11:28:57 +0200 Subject: [PATCH 10/15] Improve warning message --- lib/init.js | 4 ++-- lib/init.js.map | 2 +- src/init.ts | 6 ++++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/init.js b/lib/init.js index 1ad956db2..840a68a48 100644 --- a/lib/init.js +++ b/lib/init.js @@ -138,7 +138,7 @@ async function installPythonDeps(codeql, logger) { // This script tries to install some needed tools in the runner. It should not fail, but if it does // we just abort the process without failing the action logger.endGroup(); - logger.warning("Unable to download and extract the tools needed for installing the python dependecies"); + logger.warning("Unable to download and extract the tools needed for installing the python dependecies. You can call this action with 'setup-python-dependencies: false' to disable this process."); } } // Install dependencies @@ -147,7 +147,7 @@ async function installPythonDeps(codeql, logger) { } catch (e) { logger.endGroup(); - logger.warning("We were unable to install your python dependencies."); + logger.warning("We were unable to install your python dependencies. You can call this action with 'setup-python-dependencies: false' to disable this process."); } logger.endGroup(); } diff --git a/lib/init.js.map b/lib/init.js.map index af3e3bc76..2bc906642 100644 --- a/lib/init.js.map +++ b/lib/init.js.map @@ -1 +1 @@ -{"version":3,"file":"init.js","sourceRoot":"","sources":["../src/init.ts"],"names":[],"mappings":";;;;;;;;;AAAA,0EAA4D;AAC5D,uCAAyB;AACzB,2CAA6B;AAE7B,gEAAkD;AAClD,qCAA+C;AAC/C,4DAA8C;AAG9C,mDAAwE;AACxE,6CAA+B;AAExB,KAAK,UAAU,UAAU,CAC9B,SAA6B,EAC7B,UAAkB,EAClB,SAAiB,EACjB,OAAe,EACf,QAAgB,EAChB,IAAe,EACf,MAAc;IAEd,MAAM,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,MAAM,oBAAW,CAC9B,SAAS,EACT,UAAU,EACV,SAAS,EACT,OAAO,EACP,QAAQ,EACR,IAAI,EACJ,MAAM,CACP,CAAC;IACF,MAAM,MAAM,CAAC,YAAY,EAAE,CAAC;IAC5B,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO,MAAM,CAAC;AAChB,CAAC;AAtBD,gCAsBC;AAEM,KAAK,UAAU,UAAU,CAC9B,cAAkC,EAClC,YAAgC,EAChC,UAA8B,EAC9B,UAAyB,EACzB,OAAe,EACf,YAAoB,EACpB,MAAc,EACd,YAAoB,EACpB,UAAkB,EAClB,SAAiB,EACjB,MAAc;IAEd,MAAM,CAAC,UAAU,CAAC,6BAA6B,CAAC,CAAC;IACjD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,UAAU,CACzC,cAAc,EACd,YAAY,EACZ,UAAU,EACV,UAAU,EACV,OAAO,EACP,YAAY,EACZ,MAAM,EACN,YAAY,EACZ,UAAU,EACV,SAAS,EACT,MAAM,CACP,CAAC;IACF,aAAa,CAAC,uBAAuB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtD,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO,MAAM,CAAC;AAChB,CAAC;AA9BD,gCA8BC;AAEM,KAAK,UAAU,OAAO,CAC3B,MAAc,EACd,MAA0B;IAE1B,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;IAElC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE9E,sEAAsE;IACtE,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACvC,yBAAyB;QACzB,MAAM,MAAM,CAAC,YAAY,CACvB,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,EACpD,QAAQ,EACR,UAAU,CACX,CAAC;KACH;IAED,OAAO,MAAM,uCAAuB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AACvD,CAAC;AAnBD,0BAmBC;AAED,sEAAsE;AACtE,4EAA4E;AAC5E,4EAA4E;AAC5E,6EAA6E;AAC7E,+CAA+C;AACxC,KAAK,UAAU,mBAAmB,CACvC,WAA+B,EAC/B,YAAgC,EAChC,MAA0B,EAC1B,MAAc,EACd,YAA0B;IAE1B,IAAI,MAAc,CAAC;IACnB,IAAI,WAAW,KAAK,SAAS,EAAE;QAC7B,MAAM,GAAG;;;;;;;;;;;;uCAY0B,WAAW;;8BAEpB,WAAW;;;;;;;;gDAQO,CAAC;KAC9C;SAAM;QACL,oEAAoE;QACpE,mFAAmF;QACnF,+EAA+E;QAC/E,kFAAkF;QAClF,6EAA6E;QAC7E,oFAAoF;QACpF,6CAA6C;QAC7C,YAAY,GAAG,YAAY,IAAI,CAAC,CAAC;QACjC,MAAM,GAAG;;;;;;;;4BAQe,YAAY;;;;;;;;;;;;;;;;;gDAiBQ,CAAC;KAC9C;IAED,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,mBAAmB,CAAC,CAAC;IACxE,EAAE,CAAC,aAAa,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC;IAE3C,MAAM,IAAI,WAAW,CAAC,UAAU,CAC9B,YAAY,EACZ;QACE,kBAAkB;QAClB,QAAQ;QACR,OAAO;QACP,gBAAgB;QAChB,IAAI,CAAC,OAAO,CACV,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,EAC9B,OAAO,EACP,OAAO,EACP,YAAY,CACb;KACF,EACD,EAAE,GAAG,EAAE,EAAE,0BAA0B,EAAE,YAAY,CAAC,IAAI,EAAE,EAAE,CAC3D,CAAC,IAAI,EAAE,CAAC;AACX,CAAC;AAxFD,kDAwFC;AAEM,KAAK,UAAU,iBAAiB,CAAC,MAAc,EAAE,MAAc;IACpE,MAAM,CAAC,UAAU,CAAC,2BAA2B,CAAC,CAAC;IAE/C,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE;QAChC,MAAM,CAAC,IAAI,CACT,2EAA2E,CAC5E,CAAC;QACF,MAAM,CAAC,QAAQ,EAAE,CAAC;QAClB,OAAO;KACR;IAED,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,iBAAiB,CAAC,CAAC;IAEjE,2CAA2C;IAC3C,IAAI,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,SAAS,EAAE;QACxC,IAAI;YACF,MAAM,IAAI,WAAW,CAAC,UAAU,CAC9B,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,kBAAkB,CAAC,CAC7C,CAAC,IAAI,EAAE,CAAC;SACV;QAAC,OAAO,CAAC,EAAE;YACV,mGAAmG;YACnG,uDAAuD;YACvD,MAAM,CAAC,QAAQ,EAAE,CAAC;YAClB,MAAM,CAAC,OAAO,CACZ,uFAAuF,CACxF,CAAC;SACH;KACF;IAED,uBAAuB;IACvB,IAAI;QACF,MAAM,IAAI,WAAW,CAAC,UAAU,CAC9B,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,0BAA0B,CAAC,EACpD,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,CACjC,CAAC,IAAI,EAAE,CAAC;KACV;IAAC,OAAO,CAAC,EAAE;QACV,MAAM,CAAC,QAAQ,EAAE,CAAC;QAClB,MAAM,CAAC,OAAO,CAAC,qDAAqD,CAAC,CAAC;KACvE;IACD,MAAM,CAAC,QAAQ,EAAE,CAAC;AACpB,CAAC;AAxCD,8CAwCC"} \ No newline at end of file +{"version":3,"file":"init.js","sourceRoot":"","sources":["../src/init.ts"],"names":[],"mappings":";;;;;;;;;AAAA,0EAA4D;AAC5D,uCAAyB;AACzB,2CAA6B;AAE7B,gEAAkD;AAClD,qCAA+C;AAC/C,4DAA8C;AAG9C,mDAAwE;AACxE,6CAA+B;AAExB,KAAK,UAAU,UAAU,CAC9B,SAA6B,EAC7B,UAAkB,EAClB,SAAiB,EACjB,OAAe,EACf,QAAgB,EAChB,IAAe,EACf,MAAc;IAEd,MAAM,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,MAAM,oBAAW,CAC9B,SAAS,EACT,UAAU,EACV,SAAS,EACT,OAAO,EACP,QAAQ,EACR,IAAI,EACJ,MAAM,CACP,CAAC;IACF,MAAM,MAAM,CAAC,YAAY,EAAE,CAAC;IAC5B,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO,MAAM,CAAC;AAChB,CAAC;AAtBD,gCAsBC;AAEM,KAAK,UAAU,UAAU,CAC9B,cAAkC,EAClC,YAAgC,EAChC,UAA8B,EAC9B,UAAyB,EACzB,OAAe,EACf,YAAoB,EACpB,MAAc,EACd,YAAoB,EACpB,UAAkB,EAClB,SAAiB,EACjB,MAAc;IAEd,MAAM,CAAC,UAAU,CAAC,6BAA6B,CAAC,CAAC;IACjD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,UAAU,CACzC,cAAc,EACd,YAAY,EACZ,UAAU,EACV,UAAU,EACV,OAAO,EACP,YAAY,EACZ,MAAM,EACN,YAAY,EACZ,UAAU,EACV,SAAS,EACT,MAAM,CACP,CAAC;IACF,aAAa,CAAC,uBAAuB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtD,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO,MAAM,CAAC;AAChB,CAAC;AA9BD,gCA8BC;AAEM,KAAK,UAAU,OAAO,CAC3B,MAAc,EACd,MAA0B;IAE1B,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;IAElC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE9E,sEAAsE;IACtE,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACvC,yBAAyB;QACzB,MAAM,MAAM,CAAC,YAAY,CACvB,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,EACpD,QAAQ,EACR,UAAU,CACX,CAAC;KACH;IAED,OAAO,MAAM,uCAAuB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AACvD,CAAC;AAnBD,0BAmBC;AAED,sEAAsE;AACtE,4EAA4E;AAC5E,4EAA4E;AAC5E,6EAA6E;AAC7E,+CAA+C;AACxC,KAAK,UAAU,mBAAmB,CACvC,WAA+B,EAC/B,YAAgC,EAChC,MAA0B,EAC1B,MAAc,EACd,YAA0B;IAE1B,IAAI,MAAc,CAAC;IACnB,IAAI,WAAW,KAAK,SAAS,EAAE;QAC7B,MAAM,GAAG;;;;;;;;;;;;uCAY0B,WAAW;;8BAEpB,WAAW;;;;;;;;gDAQO,CAAC;KAC9C;SAAM;QACL,oEAAoE;QACpE,mFAAmF;QACnF,+EAA+E;QAC/E,kFAAkF;QAClF,6EAA6E;QAC7E,oFAAoF;QACpF,6CAA6C;QAC7C,YAAY,GAAG,YAAY,IAAI,CAAC,CAAC;QACjC,MAAM,GAAG;;;;;;;;4BAQe,YAAY;;;;;;;;;;;;;;;;;gDAiBQ,CAAC;KAC9C;IAED,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,mBAAmB,CAAC,CAAC;IACxE,EAAE,CAAC,aAAa,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC;IAE3C,MAAM,IAAI,WAAW,CAAC,UAAU,CAC9B,YAAY,EACZ;QACE,kBAAkB;QAClB,QAAQ;QACR,OAAO;QACP,gBAAgB;QAChB,IAAI,CAAC,OAAO,CACV,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,EAC9B,OAAO,EACP,OAAO,EACP,YAAY,CACb;KACF,EACD,EAAE,GAAG,EAAE,EAAE,0BAA0B,EAAE,YAAY,CAAC,IAAI,EAAE,EAAE,CAC3D,CAAC,IAAI,EAAE,CAAC;AACX,CAAC;AAxFD,kDAwFC;AAEM,KAAK,UAAU,iBAAiB,CAAC,MAAc,EAAE,MAAc;IACpE,MAAM,CAAC,UAAU,CAAC,2BAA2B,CAAC,CAAC;IAE/C,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE;QAChC,MAAM,CAAC,IAAI,CACT,2EAA2E,CAC5E,CAAC;QACF,MAAM,CAAC,QAAQ,EAAE,CAAC;QAClB,OAAO;KACR;IAED,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,iBAAiB,CAAC,CAAC;IAEjE,2CAA2C;IAC3C,IAAI,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,SAAS,EAAE;QACxC,IAAI;YACF,MAAM,IAAI,WAAW,CAAC,UAAU,CAC9B,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,kBAAkB,CAAC,CAC7C,CAAC,IAAI,EAAE,CAAC;SACV;QAAC,OAAO,CAAC,EAAE;YACV,mGAAmG;YACnG,uDAAuD;YACvD,MAAM,CAAC,QAAQ,EAAE,CAAC;YAClB,MAAM,CAAC,OAAO,CACZ,kLAAkL,CACnL,CAAC;SACH;KACF;IAED,uBAAuB;IACvB,IAAI;QACF,MAAM,IAAI,WAAW,CAAC,UAAU,CAC9B,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,0BAA0B,CAAC,EACpD,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,CACjC,CAAC,IAAI,EAAE,CAAC;KACV;IAAC,OAAO,CAAC,EAAE;QACV,MAAM,CAAC,QAAQ,EAAE,CAAC;QAClB,MAAM,CAAC,OAAO,CACZ,+IAA+I,CAChJ,CAAC;KACH;IACD,MAAM,CAAC,QAAQ,EAAE,CAAC;AACpB,CAAC;AA1CD,8CA0CC"} \ No newline at end of file diff --git a/src/init.ts b/src/init.ts index 8913802d1..30539d41f 100644 --- a/src/init.ts +++ b/src/init.ts @@ -206,7 +206,7 @@ export async function installPythonDeps(codeql: CodeQL, logger: Logger) { // we just abort the process without failing the action logger.endGroup(); logger.warning( - "Unable to download and extract the tools needed for installing the python dependecies" + "Unable to download and extract the tools needed for installing the python dependecies. You can call this action with 'setup-python-dependencies: false' to disable this process." ); } } @@ -219,7 +219,9 @@ export async function installPythonDeps(codeql: CodeQL, logger: Logger) { ).exec(); } catch (e) { logger.endGroup(); - logger.warning("We were unable to install your python dependencies."); + logger.warning( + "We were unable to install your python dependencies. You can call this action with 'setup-python-dependencies: false' to disable this process." + ); } logger.endGroup(); } From 18312707fe090d5a25b837386abd584f21d8a119 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Fri, 25 Sep 2020 12:07:22 +0200 Subject: [PATCH 11/15] Explain get_extractor_version for python setup scripts --- python-setup/auto_install_packages.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python-setup/auto_install_packages.py b/python-setup/auto_install_packages.py index 54a2d9669..cdb70c6d6 100755 --- a/python-setup/auto_install_packages.py +++ b/python-setup/auto_install_packages.py @@ -114,6 +114,7 @@ def install_packages(codeql_base_dir) -> Optional[str]: print('Found Pipfile, will install packages with Pipenv', flush=True) return install_packages_with_pipenv() + # get_extractor_version returns the Python version the extractor thinks this repo is using version = extractor_version.get_extractor_version(codeql_base_dir, quiet=False) if os.path.exists('requirements.txt'): From 67ddca1d9c5cb1f2042457ce0494d1a3cd1ec7bf Mon Sep 17 00:00:00 2001 From: David Verdeguer Date: Tue, 29 Sep 2020 09:57:29 +0200 Subject: [PATCH 12/15] Add workflow for testing python setup --- .github/workflows/python-deps.yml | 81 +++++++++++++++++++ python-setup/tests/check_requests_123.sh | 32 ++++++++ python-setup/tests/from_python_exe.py | 31 +++++++ python-setup/tests/pipenv/python-3.8/Pipfile | 12 +++ .../tests/pipenv/python-3.8/Pipfile.lock | 28 +++++++ .../python-version-not-available/Pipfile | 12 +++ .../python-version-not-available/Pipfile.lock | 28 +++++++ python-setup/tests/pipenv/requests-2/Pipfile | 12 +++ .../tests/pipenv/requests-2/Pipfile.lock | 28 +++++++ python-setup/tests/pipenv/requests-3/Pipfile | 11 +++ .../tests/pipenv/requests-3/Pipfile.lock | 28 +++++++ .../tests/poetry/python-3.8/poetry.lock | 16 ++++ .../tests/poetry/python-3.8/pyproject.toml | 15 ++++ .../python-version-not-available/poetry.lock | 16 ++++ .../pyproject.toml | 15 ++++ .../tests/poetry/requests-2/poetry.lock | 16 ++++ .../tests/poetry/requests-2/pyproject.toml | 15 ++++ .../tests/poetry/requests-3/poetry.lock | 16 ++++ .../tests/poetry/requests-3/pyproject.toml | 15 ++++ .../invalid-package/requirements.txt | 2 + .../invalid-version/requirements.txt | 2 + .../non-standard/requirements.txt | 1 + .../requirements/non-standard-location/wat.py | 1 + .../requirements/requests-2/requirements.txt | 1 + .../tests/requirements/requests-2/setup.py | 3 + .../requirements/requests-3/requirements.txt | 1 + .../tests/requirements/requests-3/setup.py | 3 + .../extra-require-not-installed/setup.py | 11 +++ .../tests/setup_py/invalid-file/setup.py | 1 + .../tests/setup_py/invalid-version/setup.py | 6 ++ .../tests/setup_py/requests-2/setup.py | 12 +++ .../tests/setup_py/requests-3/setup.py | 12 +++ .../setup_py/wrong-python-version/setup.py | 12 +++ 33 files changed, 495 insertions(+) create mode 100644 .github/workflows/python-deps.yml create mode 100644 python-setup/tests/check_requests_123.sh create mode 100644 python-setup/tests/from_python_exe.py create mode 100644 python-setup/tests/pipenv/python-3.8/Pipfile create mode 100644 python-setup/tests/pipenv/python-3.8/Pipfile.lock create mode 100644 python-setup/tests/pipenv/python-version-not-available/Pipfile create mode 100644 python-setup/tests/pipenv/python-version-not-available/Pipfile.lock create mode 100644 python-setup/tests/pipenv/requests-2/Pipfile create mode 100644 python-setup/tests/pipenv/requests-2/Pipfile.lock create mode 100644 python-setup/tests/pipenv/requests-3/Pipfile create mode 100644 python-setup/tests/pipenv/requests-3/Pipfile.lock create mode 100644 python-setup/tests/poetry/python-3.8/poetry.lock create mode 100644 python-setup/tests/poetry/python-3.8/pyproject.toml create mode 100644 python-setup/tests/poetry/python-version-not-available/poetry.lock create mode 100644 python-setup/tests/poetry/python-version-not-available/pyproject.toml create mode 100644 python-setup/tests/poetry/requests-2/poetry.lock create mode 100644 python-setup/tests/poetry/requests-2/pyproject.toml create mode 100644 python-setup/tests/poetry/requests-3/poetry.lock create mode 100644 python-setup/tests/poetry/requests-3/pyproject.toml create mode 100644 python-setup/tests/requirements/invalid-package/requirements.txt create mode 100644 python-setup/tests/requirements/invalid-version/requirements.txt create mode 100644 python-setup/tests/requirements/non-standard-location/non-standard/requirements.txt create mode 100644 python-setup/tests/requirements/non-standard-location/wat.py create mode 100644 python-setup/tests/requirements/requests-2/requirements.txt create mode 100644 python-setup/tests/requirements/requests-2/setup.py create mode 100644 python-setup/tests/requirements/requests-3/requirements.txt create mode 100644 python-setup/tests/requirements/requests-3/setup.py create mode 100644 python-setup/tests/setup_py/extra-require-not-installed/setup.py create mode 100644 python-setup/tests/setup_py/invalid-file/setup.py create mode 100644 python-setup/tests/setup_py/invalid-version/setup.py create mode 100644 python-setup/tests/setup_py/requests-2/setup.py create mode 100644 python-setup/tests/setup_py/requests-3/setup.py create mode 100644 python-setup/tests/setup_py/wrong-python-version/setup.py diff --git a/.github/workflows/python-deps.yml b/.github/workflows/python-deps.yml new file mode 100644 index 000000000..dba136bf8 --- /dev/null +++ b/.github/workflows/python-deps.yml @@ -0,0 +1,81 @@ +name: Test Python Package Installation + +on: + push: + branches: [main, v1] + pull_request: + +jobs: + + test-setup-python-scripts: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - test_dir: python-setup/tests/pipenv/requests-2 + test_script: $GITHUB_WORKSPACE/python-setup/tests/check_requests_123.sh 2 + - test_dir: python-setup/tests/pipenv/requests-3 + test_script: $GITHUB_WORKSPACE/python-setup/tests/check_requests_123.sh 3 + + - test_dir: python-setup/tests/poetry/requests-2 + test_script: $GITHUB_WORKSPACE/python-setup/tests/check_requests_123.sh 2 + - test_dir: python-setup/tests/poetry/requests-3 + test_script: $GITHUB_WORKSPACE/python-setup/tests/check_requests_123.sh 3 + + - test_dir: python-setup/tests/requirements/requests-2 + test_script: $GITHUB_WORKSPACE/python-setup/tests/check_requests_123.sh 2 + - test_dir: python-setup/tests/requirements/requests-3 + test_script: $GITHUB_WORKSPACE/python-setup/tests/check_requests_123.sh 3 + + - test_dir: python-setup/tests/setup_py/requests-2 + test_script: $GITHUB_WORKSPACE/python-setup/tests/check_requests_123.sh 2 + - test_dir: python-setup/tests/setup_py/requests-3 + test_script: $GITHUB_WORKSPACE/python-setup/tests/check_requests_123.sh 3 + + # This one shouldn't fail, but also won't install packages + - test_dir: python-setup/tests/requirements/non-standard-location + test_script: test -z $LGTM_INDEX_IMPORT_PATH + + # All of these should fail + - test_dir: python-setup/tests/pipenv/python-version-not-available + test_script: /bin/false + - test_dir: python-setup/tests/poetry/python-version-not-available + test_script: /bin/false + - test_dir: python-setup/tests/requirements/invalid-package + test_script: /bin/false + - test_dir: python-setup/tests/requirements/invalid-version + test_script: /bin/false + - test_dir: python-setup/tests/setup_py/invalid-version + test_script: /bin/false + - test_dir: python-setup/tests/setup_py/invalid-file + test_script: /bin/false + - test_dir: python-setup/tests/setup_py/extra-require-not-installed + test_script: $GITHUB_WORKSPACE/python-setup/tests/check_requests_123.sh 3 + - test_dir: python-setup/tests/setup_py/wrong-python-version + test_script: /bin/false + + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v2 + + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: python + + - name: Test Auto Package Installation + run: | + set -x + $GITHUB_WORKSPACE/python-setup/install_tools.sh + echo -e '\n\n\n\n\n' && sleep 0.5 + cd $GITHUB_WORKSPACE/${{ matrix.test_dir }} + $GITHUB_WORKSPACE/python-setup/auto_install_packages.py /opt/hostedtoolcache/CodeQL/0.0.0-20200826/x64/codeql/ + - name: Setup for extractor + run: | + echo $CODEQL_PYTHON + # only run if $CODEQL_PYTHON is set + test ! -z $CODEQL_PYTHON && $GITHUB_WORKSPACE/python-setup/tests/from_python_exe.py $CODEQL_PYTHON || /bin/true + - name: Verify packages installed + run: | + ${{ matrix.test_script }} \ No newline at end of file diff --git a/python-setup/tests/check_requests_123.sh b/python-setup/tests/check_requests_123.sh new file mode 100644 index 000000000..f44eb8093 --- /dev/null +++ b/python-setup/tests/check_requests_123.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +set -e + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +EXPECTED_VERSION=$1 + +FOUND_VERSION="$LGTM_PYTHON_SETUP_VERSION" +FOUND_PYTHONPATH="$LGTM_INDEX_IMPORT_PATH" + +echo "FOUND_VERSION=${FOUND_VERSION} FOUND_PYTHONPATH=${FOUND_PYTHONPATH} " + +if [[ $FOUND_VERSION != $EXPECTED_VERSION ]]; then + echo "Script told us to use Python ${FOUND_VERSION}, but expected ${EXPECTED_VERSION}" + exit 1 +else + echo "Script told us to use Python ${FOUND_VERSION}, which was expected" +fi + +PYTHON_EXE="python${EXPECTED_VERSION}" + +INSTALLED_REQUESTS_VERSION=$(PYTHONPATH="${FOUND_PYTHONPATH}" "${PYTHON_EXE}" -c 'import requests; print(requests.__version__)') + +EXPECTED_REQUESTS="1.2.3" + +if [[ "$INSTALLED_REQUESTS_VERSION" != "$EXPECTED_REQUESTS" ]]; then + echo "Using ${FOUND_PYTHONPATH} as PYTHONPATH, we found version $INSTALLED_REQUESTS_VERSION of requests, but expected $EXPECTED_REQUESTS" + exit 1 +else + echo "Using ${FOUND_PYTHONPATH} as PYTHONPATH, we found version $INSTALLED_REQUESTS_VERSION of requests, which was expected" +fi \ No newline at end of file diff --git a/python-setup/tests/from_python_exe.py b/python-setup/tests/from_python_exe.py new file mode 100644 index 000000000..69e2e873e --- /dev/null +++ b/python-setup/tests/from_python_exe.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 + +import sys +import subprocess +from typing import Tuple + +def get_details(path_to_python_exe: str) -> Tuple[str, str]: + import_path = subprocess.check_output( + [ + path_to_python_exe, + "-c", + "import os; import pip; print(os.path.dirname(os.path.dirname(pip.__file__)))", + ], + stdin=subprocess.DEVNULL, + ) + version = subprocess.check_output( + [path_to_python_exe, "-c", "import sys; print(sys.version_info[0])"], + stdin=subprocess.DEVNULL, + ) + + return version.decode("utf-8").strip(), import_path.decode("utf-8").strip() + + +if __name__ == "__main__": + version, import_path = get_details(sys.argv[1]) + + print("Setting LGTM_PYTHON_SETUP_VERSION={}".format(version)) + print("::set-env name=LGTM_PYTHON_SETUP_VERSION::{}".format(version)) + + print("Setting LGTM_INDEX_IMPORT_PATH={}".format(import_path)) + print("::set-env name=LGTM_INDEX_IMPORT_PATH::{}".format(import_path)) \ No newline at end of file diff --git a/python-setup/tests/pipenv/python-3.8/Pipfile b/python-setup/tests/pipenv/python-3.8/Pipfile new file mode 100644 index 000000000..b799f0f56 --- /dev/null +++ b/python-setup/tests/pipenv/python-3.8/Pipfile @@ -0,0 +1,12 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[dev-packages] + +[packages] +requests = "*" + +[requires] +python_version = "3.8" diff --git a/python-setup/tests/pipenv/python-3.8/Pipfile.lock b/python-setup/tests/pipenv/python-3.8/Pipfile.lock new file mode 100644 index 000000000..519591de8 --- /dev/null +++ b/python-setup/tests/pipenv/python-3.8/Pipfile.lock @@ -0,0 +1,28 @@ +{ + "_meta": { + "hash": { + "sha256": "2296b1fab66570fa3660c5e6a4e923332dcd7785814624c584be6623bc62c0e7" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.8" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "requests": { + "hashes": [ + "sha256:156bf3ec27ba9ec7e0cf8fbe02808718099d218de403eb64a714d73ba1a29ab1" + ], + "index": "pypi", + "version": "==1.2.3" + } + }, + "develop": {} +} diff --git a/python-setup/tests/pipenv/python-version-not-available/Pipfile b/python-setup/tests/pipenv/python-version-not-available/Pipfile new file mode 100644 index 000000000..336c2301b --- /dev/null +++ b/python-setup/tests/pipenv/python-version-not-available/Pipfile @@ -0,0 +1,12 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[dev-packages] + +[packages] +requests = "*" + +[requires] +python_version = "3.100" diff --git a/python-setup/tests/pipenv/python-version-not-available/Pipfile.lock b/python-setup/tests/pipenv/python-version-not-available/Pipfile.lock new file mode 100644 index 000000000..519591de8 --- /dev/null +++ b/python-setup/tests/pipenv/python-version-not-available/Pipfile.lock @@ -0,0 +1,28 @@ +{ + "_meta": { + "hash": { + "sha256": "2296b1fab66570fa3660c5e6a4e923332dcd7785814624c584be6623bc62c0e7" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.8" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "requests": { + "hashes": [ + "sha256:156bf3ec27ba9ec7e0cf8fbe02808718099d218de403eb64a714d73ba1a29ab1" + ], + "index": "pypi", + "version": "==1.2.3" + } + }, + "develop": {} +} diff --git a/python-setup/tests/pipenv/requests-2/Pipfile b/python-setup/tests/pipenv/requests-2/Pipfile new file mode 100644 index 000000000..c9450bcf7 --- /dev/null +++ b/python-setup/tests/pipenv/requests-2/Pipfile @@ -0,0 +1,12 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[dev-packages] + +[packages] +requests = "*" + +[requires] +python_version = "2.7" diff --git a/python-setup/tests/pipenv/requests-2/Pipfile.lock b/python-setup/tests/pipenv/requests-2/Pipfile.lock new file mode 100644 index 000000000..789382967 --- /dev/null +++ b/python-setup/tests/pipenv/requests-2/Pipfile.lock @@ -0,0 +1,28 @@ +{ + "_meta": { + "hash": { + "sha256": "fba50410545d38a2f88b8d6964eb292055389e363b5c75a56b01ec80e7de14c1" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "2.7" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "requests": { + "hashes": [ + "sha256:156bf3ec27ba9ec7e0cf8fbe02808718099d218de403eb64a714d73ba1a29ab1" + ], + "index": "pypi", + "version": "==1.2.3" + } + }, + "develop": {} +} diff --git a/python-setup/tests/pipenv/requests-3/Pipfile b/python-setup/tests/pipenv/requests-3/Pipfile new file mode 100644 index 000000000..fcab73720 --- /dev/null +++ b/python-setup/tests/pipenv/requests-3/Pipfile @@ -0,0 +1,11 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[dev-packages] + +[packages] +requests = "*" + +[requires] diff --git a/python-setup/tests/pipenv/requests-3/Pipfile.lock b/python-setup/tests/pipenv/requests-3/Pipfile.lock new file mode 100644 index 000000000..519591de8 --- /dev/null +++ b/python-setup/tests/pipenv/requests-3/Pipfile.lock @@ -0,0 +1,28 @@ +{ + "_meta": { + "hash": { + "sha256": "2296b1fab66570fa3660c5e6a4e923332dcd7785814624c584be6623bc62c0e7" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.8" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "requests": { + "hashes": [ + "sha256:156bf3ec27ba9ec7e0cf8fbe02808718099d218de403eb64a714d73ba1a29ab1" + ], + "index": "pypi", + "version": "==1.2.3" + } + }, + "develop": {} +} diff --git a/python-setup/tests/poetry/python-3.8/poetry.lock b/python-setup/tests/poetry/python-3.8/poetry.lock new file mode 100644 index 000000000..cda1a33c2 --- /dev/null +++ b/python-setup/tests/poetry/python-3.8/poetry.lock @@ -0,0 +1,16 @@ +[[package]] +category = "main" +description = "Python HTTP for Humans." +name = "requests" +optional = false +python-versions = "^3.8" +version = "1.2.3" + +[metadata] +content-hash = "b1abdce30bbdad3ac786dafe6840129df522cae3c5988f9ba8061320e3c1a34e" +python-versions = "^3.8" + +[metadata.files] +requests = [ + {file = "requests-1.2.3.tar.gz", hash = "sha256:156bf3ec27ba9ec7e0cf8fbe02808718099d218de403eb64a714d73ba1a29ab1"}, +] diff --git a/python-setup/tests/poetry/python-3.8/pyproject.toml b/python-setup/tests/poetry/python-3.8/pyproject.toml new file mode 100644 index 000000000..8d82c19db --- /dev/null +++ b/python-setup/tests/poetry/python-3.8/pyproject.toml @@ -0,0 +1,15 @@ +[tool.poetry] +name = "autoinstall-test" +version = "0.1.0" +description = "" +authors = ["Your Name "] + +[tool.poetry.dependencies] +python = "^3.8" +requests = "*" + +[tool.poetry.dev-dependencies] + +[build-system] +requires = ["poetry>=0.12"] +build-backend = "poetry.masonry.api" diff --git a/python-setup/tests/poetry/python-version-not-available/poetry.lock b/python-setup/tests/poetry/python-version-not-available/poetry.lock new file mode 100644 index 000000000..6db68ee77 --- /dev/null +++ b/python-setup/tests/poetry/python-version-not-available/poetry.lock @@ -0,0 +1,16 @@ +[[package]] +category = "main" +description = "Python HTTP for Humans." +name = "requests" +optional = false +python-versions = "^3.100" +version = "1.2.3" + +[metadata] +content-hash = "b1abdce30bbdad3ac786dafe6840129df522cae3c5988f9ba8061320e3c1a34e" +python-versions = "^3.100" + +[metadata.files] +requests = [ + {file = "requests-1.2.3.tar.gz", hash = "sha256:156bf3ec27ba9ec7e0cf8fbe02808718099d218de403eb64a714d73ba1a29ab1"}, +] diff --git a/python-setup/tests/poetry/python-version-not-available/pyproject.toml b/python-setup/tests/poetry/python-version-not-available/pyproject.toml new file mode 100644 index 000000000..86007afea --- /dev/null +++ b/python-setup/tests/poetry/python-version-not-available/pyproject.toml @@ -0,0 +1,15 @@ +[tool.poetry] +name = "autoinstall-test" +version = "0.1.0" +description = "" +authors = ["Your Name "] + +[tool.poetry.dependencies] +python = "^3.100" +requests = "*" + +[tool.poetry.dev-dependencies] + +[build-system] +requires = ["poetry>=0.12"] +build-backend = "poetry.masonry.api" diff --git a/python-setup/tests/poetry/requests-2/poetry.lock b/python-setup/tests/poetry/requests-2/poetry.lock new file mode 100644 index 000000000..c16b09b4c --- /dev/null +++ b/python-setup/tests/poetry/requests-2/poetry.lock @@ -0,0 +1,16 @@ +[[package]] +category = "main" +description = "Python HTTP for Humans." +name = "requests" +optional = false +python-versions = "*" +version = "1.2.3" + +[metadata] +content-hash = "b8c27a00aece20cb2ff6b3dcd394b1b5ef5e4acc3d7a059cfeb9f9cc11c9eff4" +python-versions = "^2.7" + +[metadata.files] +requests = [ + {file = "requests-1.2.3.tar.gz", hash = "sha256:156bf3ec27ba9ec7e0cf8fbe02808718099d218de403eb64a714d73ba1a29ab1"}, +] diff --git a/python-setup/tests/poetry/requests-2/pyproject.toml b/python-setup/tests/poetry/requests-2/pyproject.toml new file mode 100644 index 000000000..890d59147 --- /dev/null +++ b/python-setup/tests/poetry/requests-2/pyproject.toml @@ -0,0 +1,15 @@ +[tool.poetry] +name = "autoinstall-test" +version = "0.1.0" +description = "" +authors = ["Your Name "] + +[tool.poetry.dependencies] +python = "^2.7" +requests = "*" + +[tool.poetry.dev-dependencies] + +[build-system] +requires = ["poetry>=0.12"] +build-backend = "poetry.masonry.api" diff --git a/python-setup/tests/poetry/requests-3/poetry.lock b/python-setup/tests/poetry/requests-3/poetry.lock new file mode 100644 index 000000000..ecc2ca3b6 --- /dev/null +++ b/python-setup/tests/poetry/requests-3/poetry.lock @@ -0,0 +1,16 @@ +[[package]] +category = "main" +description = "Python HTTP for Humans." +name = "requests" +optional = false +python-versions = "^3.5" +version = "1.2.3" + +[metadata] +content-hash = "b1abdce30bbdad3ac786dafe6840129df522cae3c5988f9ba8061320e3c1a34e" +python-versions = "^3.5" + +[metadata.files] +requests = [ + {file = "requests-1.2.3.tar.gz", hash = "sha256:156bf3ec27ba9ec7e0cf8fbe02808718099d218de403eb64a714d73ba1a29ab1"}, +] diff --git a/python-setup/tests/poetry/requests-3/pyproject.toml b/python-setup/tests/poetry/requests-3/pyproject.toml new file mode 100644 index 000000000..155de4618 --- /dev/null +++ b/python-setup/tests/poetry/requests-3/pyproject.toml @@ -0,0 +1,15 @@ +[tool.poetry] +name = "autoinstall-test" +version = "0.1.0" +description = "" +authors = ["Your Name "] + +[tool.poetry.dependencies] +python = "^3.5" +requests = "*" + +[tool.poetry.dev-dependencies] + +[build-system] +requires = ["poetry>=0.12"] +build-backend = "poetry.masonry.api" diff --git a/python-setup/tests/requirements/invalid-package/requirements.txt b/python-setup/tests/requirements/invalid-package/requirements.txt new file mode 100644 index 000000000..4acb6327e --- /dev/null +++ b/python-setup/tests/requirements/invalid-package/requirements.txt @@ -0,0 +1,2 @@ +# non-existing package +ajsdiofjasodinfkajsndfoiuqwhefoanisdfojnasdofas diff --git a/python-setup/tests/requirements/invalid-version/requirements.txt b/python-setup/tests/requirements/invalid-version/requirements.txt new file mode 100644 index 000000000..915977da0 --- /dev/null +++ b/python-setup/tests/requirements/invalid-version/requirements.txt @@ -0,0 +1,2 @@ +# valid package with invalid version +requests==10000 diff --git a/python-setup/tests/requirements/non-standard-location/non-standard/requirements.txt b/python-setup/tests/requirements/non-standard-location/non-standard/requirements.txt new file mode 100644 index 000000000..134975b53 --- /dev/null +++ b/python-setup/tests/requirements/non-standard-location/non-standard/requirements.txt @@ -0,0 +1 @@ +requests==1.2.3 diff --git a/python-setup/tests/requirements/non-standard-location/wat.py b/python-setup/tests/requirements/non-standard-location/wat.py new file mode 100644 index 000000000..b376c9941 --- /dev/null +++ b/python-setup/tests/requirements/non-standard-location/wat.py @@ -0,0 +1 @@ +print('hello') diff --git a/python-setup/tests/requirements/requests-2/requirements.txt b/python-setup/tests/requirements/requests-2/requirements.txt new file mode 100644 index 000000000..134975b53 --- /dev/null +++ b/python-setup/tests/requirements/requests-2/requirements.txt @@ -0,0 +1 @@ +requests==1.2.3 diff --git a/python-setup/tests/requirements/requests-2/setup.py b/python-setup/tests/requirements/requests-2/setup.py new file mode 100644 index 000000000..ea6a81184 --- /dev/null +++ b/python-setup/tests/requirements/requests-2/setup.py @@ -0,0 +1,3 @@ +# fake setup.py with Trove classifier to fool Python extractor to believe this is Python 2 for sure + +# Programming Language :: Python :: 2.7 diff --git a/python-setup/tests/requirements/requests-3/requirements.txt b/python-setup/tests/requirements/requests-3/requirements.txt new file mode 100644 index 000000000..134975b53 --- /dev/null +++ b/python-setup/tests/requirements/requests-3/requirements.txt @@ -0,0 +1 @@ +requests==1.2.3 diff --git a/python-setup/tests/requirements/requests-3/setup.py b/python-setup/tests/requirements/requests-3/setup.py new file mode 100644 index 000000000..f2eba6c01 --- /dev/null +++ b/python-setup/tests/requirements/requests-3/setup.py @@ -0,0 +1,3 @@ +# fake setup.py with Trove classifier to fool Python extractor to believe this is Python 3 for sure + +# Programming Language :: Python :: 3.7 diff --git a/python-setup/tests/setup_py/extra-require-not-installed/setup.py b/python-setup/tests/setup_py/extra-require-not-installed/setup.py new file mode 100644 index 000000000..f14e3597d --- /dev/null +++ b/python-setup/tests/setup_py/extra-require-not-installed/setup.py @@ -0,0 +1,11 @@ +from setuptools import setup + +# has fake Trove classifier to fool Python extractor to believe this is Python 3 for sure + +# Programming Language :: Python :: 3.7 + +setup( + name="example-setup.py", + install_requires=[], + extras_require={"all": ["requests==1.2.3"]} +) diff --git a/python-setup/tests/setup_py/invalid-file/setup.py b/python-setup/tests/setup_py/invalid-file/setup.py new file mode 100644 index 000000000..39f67eb3f --- /dev/null +++ b/python-setup/tests/setup_py/invalid-file/setup.py @@ -0,0 +1 @@ +raise Exception("some kind of error") diff --git a/python-setup/tests/setup_py/invalid-version/setup.py b/python-setup/tests/setup_py/invalid-version/setup.py new file mode 100644 index 000000000..f549eac22 --- /dev/null +++ b/python-setup/tests/setup_py/invalid-version/setup.py @@ -0,0 +1,6 @@ +from setuptools import setup + +setup( + name="example-setup.py", + install_requires=["requests==10000"], +) diff --git a/python-setup/tests/setup_py/requests-2/setup.py b/python-setup/tests/setup_py/requests-2/setup.py new file mode 100644 index 000000000..26e0d9753 --- /dev/null +++ b/python-setup/tests/setup_py/requests-2/setup.py @@ -0,0 +1,12 @@ +from setuptools import setup + +# has fake Trove classifier to fool Python extractor to believe this is Python 2 for sure + +# Programming Language :: Python :: 2.7 + + +setup( + name="example-setup.py", + install_requires=["requests==1.2.3"], + python_requires=">=2.7, <3", +) diff --git a/python-setup/tests/setup_py/requests-3/setup.py b/python-setup/tests/setup_py/requests-3/setup.py new file mode 100644 index 000000000..b401073da --- /dev/null +++ b/python-setup/tests/setup_py/requests-3/setup.py @@ -0,0 +1,12 @@ +from setuptools import setup + +# has fake Trove classifier to fool Python extractor to believe this is Python 3 for sure + +# Programming Language :: Python :: 3.7 + + +setup( + name="example-setup.py", + install_requires=["requests==1.2.3"], + python_requires='>=3.5', +) diff --git a/python-setup/tests/setup_py/wrong-python-version/setup.py b/python-setup/tests/setup_py/wrong-python-version/setup.py new file mode 100644 index 000000000..4b2f68ae5 --- /dev/null +++ b/python-setup/tests/setup_py/wrong-python-version/setup.py @@ -0,0 +1,12 @@ +from setuptools import setup + +# has fake Trove classifier to fool Python extractor to believe this is Python 3 for sure + +# Programming Language :: Python :: 3.7 + + +setup( + name="example-setup.py", + install_requires=["requests==1.2.3"], + python_requires=">=2.7, <3", +) From 5e2245cc23068ab64f9345cadd977fdbe6c2f673 Mon Sep 17 00:00:00 2001 From: David Verdeguer Date: Tue, 29 Sep 2020 10:06:06 +0200 Subject: [PATCH 13/15] Add x permission to the test scripts --- python-setup/tests/check_requests_123.sh | 0 python-setup/tests/from_python_exe.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 python-setup/tests/check_requests_123.sh mode change 100644 => 100755 python-setup/tests/from_python_exe.py diff --git a/python-setup/tests/check_requests_123.sh b/python-setup/tests/check_requests_123.sh old mode 100644 new mode 100755 diff --git a/python-setup/tests/from_python_exe.py b/python-setup/tests/from_python_exe.py old mode 100644 new mode 100755 From 4e39b768c7a37712f347efb8b553b5ea04fa9729 Mon Sep 17 00:00:00 2001 From: David Verdeguer Date: Tue, 29 Sep 2020 10:33:25 +0200 Subject: [PATCH 14/15] Remove tests that should fail --- .github/workflows/python-deps.yml | 18 ------------ .../python-version-not-available/Pipfile | 12 -------- .../python-version-not-available/Pipfile.lock | 28 ------------------- .../python-version-not-available/poetry.lock | 16 ----------- .../pyproject.toml | 15 ---------- .../invalid-package/requirements.txt | 2 -- .../invalid-version/requirements.txt | 2 -- .../extra-require-not-installed/setup.py | 11 -------- .../tests/setup_py/invalid-file/setup.py | 1 - .../tests/setup_py/invalid-version/setup.py | 6 ---- .../setup_py/wrong-python-version/setup.py | 12 -------- 11 files changed, 123 deletions(-) delete mode 100644 python-setup/tests/pipenv/python-version-not-available/Pipfile delete mode 100644 python-setup/tests/pipenv/python-version-not-available/Pipfile.lock delete mode 100644 python-setup/tests/poetry/python-version-not-available/poetry.lock delete mode 100644 python-setup/tests/poetry/python-version-not-available/pyproject.toml delete mode 100644 python-setup/tests/requirements/invalid-package/requirements.txt delete mode 100644 python-setup/tests/requirements/invalid-version/requirements.txt delete mode 100644 python-setup/tests/setup_py/extra-require-not-installed/setup.py delete mode 100644 python-setup/tests/setup_py/invalid-file/setup.py delete mode 100644 python-setup/tests/setup_py/invalid-version/setup.py delete mode 100644 python-setup/tests/setup_py/wrong-python-version/setup.py diff --git a/.github/workflows/python-deps.yml b/.github/workflows/python-deps.yml index dba136bf8..291be5735 100644 --- a/.github/workflows/python-deps.yml +++ b/.github/workflows/python-deps.yml @@ -37,24 +37,6 @@ jobs: - test_dir: python-setup/tests/requirements/non-standard-location test_script: test -z $LGTM_INDEX_IMPORT_PATH - # All of these should fail - - test_dir: python-setup/tests/pipenv/python-version-not-available - test_script: /bin/false - - test_dir: python-setup/tests/poetry/python-version-not-available - test_script: /bin/false - - test_dir: python-setup/tests/requirements/invalid-package - test_script: /bin/false - - test_dir: python-setup/tests/requirements/invalid-version - test_script: /bin/false - - test_dir: python-setup/tests/setup_py/invalid-version - test_script: /bin/false - - test_dir: python-setup/tests/setup_py/invalid-file - test_script: /bin/false - - test_dir: python-setup/tests/setup_py/extra-require-not-installed - test_script: $GITHUB_WORKSPACE/python-setup/tests/check_requests_123.sh 3 - - test_dir: python-setup/tests/setup_py/wrong-python-version - test_script: /bin/false - steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v2 diff --git a/python-setup/tests/pipenv/python-version-not-available/Pipfile b/python-setup/tests/pipenv/python-version-not-available/Pipfile deleted file mode 100644 index 336c2301b..000000000 --- a/python-setup/tests/pipenv/python-version-not-available/Pipfile +++ /dev/null @@ -1,12 +0,0 @@ -[[source]] -name = "pypi" -url = "https://pypi.org/simple" -verify_ssl = true - -[dev-packages] - -[packages] -requests = "*" - -[requires] -python_version = "3.100" diff --git a/python-setup/tests/pipenv/python-version-not-available/Pipfile.lock b/python-setup/tests/pipenv/python-version-not-available/Pipfile.lock deleted file mode 100644 index 519591de8..000000000 --- a/python-setup/tests/pipenv/python-version-not-available/Pipfile.lock +++ /dev/null @@ -1,28 +0,0 @@ -{ - "_meta": { - "hash": { - "sha256": "2296b1fab66570fa3660c5e6a4e923332dcd7785814624c584be6623bc62c0e7" - }, - "pipfile-spec": 6, - "requires": { - "python_version": "3.8" - }, - "sources": [ - { - "name": "pypi", - "url": "https://pypi.org/simple", - "verify_ssl": true - } - ] - }, - "default": { - "requests": { - "hashes": [ - "sha256:156bf3ec27ba9ec7e0cf8fbe02808718099d218de403eb64a714d73ba1a29ab1" - ], - "index": "pypi", - "version": "==1.2.3" - } - }, - "develop": {} -} diff --git a/python-setup/tests/poetry/python-version-not-available/poetry.lock b/python-setup/tests/poetry/python-version-not-available/poetry.lock deleted file mode 100644 index 6db68ee77..000000000 --- a/python-setup/tests/poetry/python-version-not-available/poetry.lock +++ /dev/null @@ -1,16 +0,0 @@ -[[package]] -category = "main" -description = "Python HTTP for Humans." -name = "requests" -optional = false -python-versions = "^3.100" -version = "1.2.3" - -[metadata] -content-hash = "b1abdce30bbdad3ac786dafe6840129df522cae3c5988f9ba8061320e3c1a34e" -python-versions = "^3.100" - -[metadata.files] -requests = [ - {file = "requests-1.2.3.tar.gz", hash = "sha256:156bf3ec27ba9ec7e0cf8fbe02808718099d218de403eb64a714d73ba1a29ab1"}, -] diff --git a/python-setup/tests/poetry/python-version-not-available/pyproject.toml b/python-setup/tests/poetry/python-version-not-available/pyproject.toml deleted file mode 100644 index 86007afea..000000000 --- a/python-setup/tests/poetry/python-version-not-available/pyproject.toml +++ /dev/null @@ -1,15 +0,0 @@ -[tool.poetry] -name = "autoinstall-test" -version = "0.1.0" -description = "" -authors = ["Your Name "] - -[tool.poetry.dependencies] -python = "^3.100" -requests = "*" - -[tool.poetry.dev-dependencies] - -[build-system] -requires = ["poetry>=0.12"] -build-backend = "poetry.masonry.api" diff --git a/python-setup/tests/requirements/invalid-package/requirements.txt b/python-setup/tests/requirements/invalid-package/requirements.txt deleted file mode 100644 index 4acb6327e..000000000 --- a/python-setup/tests/requirements/invalid-package/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -# non-existing package -ajsdiofjasodinfkajsndfoiuqwhefoanisdfojnasdofas diff --git a/python-setup/tests/requirements/invalid-version/requirements.txt b/python-setup/tests/requirements/invalid-version/requirements.txt deleted file mode 100644 index 915977da0..000000000 --- a/python-setup/tests/requirements/invalid-version/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -# valid package with invalid version -requests==10000 diff --git a/python-setup/tests/setup_py/extra-require-not-installed/setup.py b/python-setup/tests/setup_py/extra-require-not-installed/setup.py deleted file mode 100644 index f14e3597d..000000000 --- a/python-setup/tests/setup_py/extra-require-not-installed/setup.py +++ /dev/null @@ -1,11 +0,0 @@ -from setuptools import setup - -# has fake Trove classifier to fool Python extractor to believe this is Python 3 for sure - -# Programming Language :: Python :: 3.7 - -setup( - name="example-setup.py", - install_requires=[], - extras_require={"all": ["requests==1.2.3"]} -) diff --git a/python-setup/tests/setup_py/invalid-file/setup.py b/python-setup/tests/setup_py/invalid-file/setup.py deleted file mode 100644 index 39f67eb3f..000000000 --- a/python-setup/tests/setup_py/invalid-file/setup.py +++ /dev/null @@ -1 +0,0 @@ -raise Exception("some kind of error") diff --git a/python-setup/tests/setup_py/invalid-version/setup.py b/python-setup/tests/setup_py/invalid-version/setup.py deleted file mode 100644 index f549eac22..000000000 --- a/python-setup/tests/setup_py/invalid-version/setup.py +++ /dev/null @@ -1,6 +0,0 @@ -from setuptools import setup - -setup( - name="example-setup.py", - install_requires=["requests==10000"], -) diff --git a/python-setup/tests/setup_py/wrong-python-version/setup.py b/python-setup/tests/setup_py/wrong-python-version/setup.py deleted file mode 100644 index 4b2f68ae5..000000000 --- a/python-setup/tests/setup_py/wrong-python-version/setup.py +++ /dev/null @@ -1,12 +0,0 @@ -from setuptools import setup - -# has fake Trove classifier to fool Python extractor to believe this is Python 3 for sure - -# Programming Language :: Python :: 3.7 - - -setup( - name="example-setup.py", - install_requires=["requests==1.2.3"], - python_requires=">=2.7, <3", -) From 4ce302bdb93728340daa064ac21baf231c8cb241 Mon Sep 17 00:00:00 2001 From: David Verdeguer Date: Sun, 4 Oct 2020 23:24:22 +0200 Subject: [PATCH 15/15] Use find in python-des tests --- .github/workflows/python-deps.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-deps.yml b/.github/workflows/python-deps.yml index 291be5735..75be193b3 100644 --- a/.github/workflows/python-deps.yml +++ b/.github/workflows/python-deps.yml @@ -52,7 +52,7 @@ jobs: $GITHUB_WORKSPACE/python-setup/install_tools.sh echo -e '\n\n\n\n\n' && sleep 0.5 cd $GITHUB_WORKSPACE/${{ matrix.test_dir }} - $GITHUB_WORKSPACE/python-setup/auto_install_packages.py /opt/hostedtoolcache/CodeQL/0.0.0-20200826/x64/codeql/ + find /opt/hostedtoolcache/CodeQL -path "*x64/codeql" -exec $GITHUB_WORKSPACE/python-setup/auto_install_packages.py {} \; - name: Setup for extractor run: | echo $CODEQL_PYTHON