Skip to content

Commit

Permalink
Use CLI's own baseline LOC counting
Browse files Browse the repository at this point in the history
  • Loading branch information
Edoardo Pirovano committed Sep 29, 2021
1 parent cd1b9df commit f04acbb
Show file tree
Hide file tree
Showing 21 changed files with 124 additions and 25 deletions.
2 changes: 2 additions & 0 deletions .github/workflows/__go-custom-queries.yml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions .github/workflows/__go-custom-tracing-autobuild.yml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions .github/workflows/__go-custom-tracing.yml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions .github/workflows/__javascript-source-root.yml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions .github/workflows/__multi-language-autodetect.yml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions .github/workflows/__packaging-config-inputs-js.yml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions .github/workflows/__packaging-config-js.yml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions .github/workflows/__packaging-inputs-js.yml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions .github/workflows/__remote-config.yml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions .github/workflows/__rubocop-multi-language.yml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions .github/workflows/__split-workflow.yml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions .github/workflows/__test-local-codeql.yml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions .github/workflows/__test-proxy.yml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions .github/workflows/__test-ruby.yml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

35 changes: 27 additions & 8 deletions lib/analyze.js

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion lib/analyze.js.map

Large diffs are not rendered by default.

13 changes: 12 additions & 1 deletion lib/codeql.js

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion lib/codeql.js.map

Large diffs are not rendered by default.

2 changes: 2 additions & 0 deletions pr-checks/sync.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,8 @@ def writeHeader(checkStream):
if key in checkSpecification:
checkJob[key] = checkSpecification[key]

checkJob['env'] = checkJob.get('env', {})
checkJob['env']['INTERNAL_CODEQL_ACTION_DEBUG_LOC'] = True
checkName = file[:len(file) - 4]

with open(f"../.github/workflows/__{checkName}.yml", 'w') as output_stream:
Expand Down
53 changes: 39 additions & 14 deletions src/analyze.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import * as toolrunner from "@actions/exec/lib/toolrunner";
import * as yaml from "js-yaml";

import * as analysisPaths from "./analysis-paths";
import { getCodeQL } from "./codeql";
import { CODEQL_VERSION_COUNTS_LINES, getCodeQL } from "./codeql";
import * as configUtils from "./config-utils";
import { countLoc } from "./count-loc";
import { isScannedLanguage, Language } from "./languages";
Expand Down Expand Up @@ -194,17 +194,27 @@ export async function runQueries(
): Promise<QueriesStatusReport> {
const statusReport: QueriesStatusReport = {};

// count the number of lines in the background
const locPromise = countLoc(
path.resolve(),
// config.paths specifies external directories. the current
// directory is included in the analysis by default. Replicate
// that here.
config.paths,
config.pathsIgnore,
config.languages,
logger
let locPromise: Promise<Partial<Record<Language, number>>> = Promise.resolve(
{}
);
const cliCanCountBaseline = await cliCanCountLoC();
const debugMode =
process.env["INTERNAL_CODEQL_ACTION_DEBUG_LOC"] ||
process.env["ACTIONS_RUNNER_DEBUG"] ||
process.env["ACTIONS_STEP_DEBUG"];
if (!cliCanCountBaseline || debugMode) {
// count the number of lines in the background
locPromise = countLoc(
path.resolve(),
// config.paths specifies external directories. the current
// directory is included in the analysis by default. Replicate
// that here.
config.paths,
config.pathsIgnore,
config.languages,
logger
);
}

for (const language of config.languages) {
const queries = config.queries[language];
Expand Down Expand Up @@ -295,12 +305,15 @@ export async function runQueries(
querySuitePaths,
sarifFile
);
await injectLinesOfCode(sarifFile, language, locPromise);
if (!cliCanCountBaseline)
await injectLinesOfCode(sarifFile, language, locPromise);
statusReport[`interpret_results_${language}_duration_ms`] =
new Date().getTime() - startTimeInterpretResults;
logger.endGroup();
logger.info(analysisSummary);
printLinesOfCodeSummary(logger, language, await locPromise);
if (!cliCanCountBaseline || debugMode)
printLinesOfCodeSummary(logger, language, await locPromise);
if (cliCanCountBaseline) logger.info(await runPrintLinesOfCode(language));
} catch (e) {
logger.info(String(e));
if (e instanceof Error) {
Expand Down Expand Up @@ -333,6 +346,19 @@ export async function runQueries(
);
}

async function cliCanCountLoC() {
return await util.codeQlVersionAbove(
await getCodeQL(config.codeQLCmd),
CODEQL_VERSION_COUNTS_LINES
);
}

async function runPrintLinesOfCode(language: Language): Promise<string> {
const databasePath = util.getCodeQLDatabasePath(config, language);
const codeql = await getCodeQL(config.codeQLCmd);
return await codeql.databasePrintBaseline(databasePath);
}

async function runQueryGroup(
language: Language,
type: string,
Expand Down Expand Up @@ -361,7 +387,6 @@ export async function runQueries(
return querySuitePath;
}
}

function createQuerySuiteContents(queries: string[]) {
return queries.map((q: string) => `- query: ${q}`).join("\n");
}
Expand Down
18 changes: 18 additions & 0 deletions src/codeql.ts
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,10 @@ export interface CodeQL {
threadsFlag: string,
automationDetailsId: string | undefined
): Promise<string>;
/**
* Run 'codeql database print-baseline'.
*/
databasePrintBaseline(databasePath: string): Promise<string>;
}

export interface ResolveLanguagesOutput {
Expand Down Expand Up @@ -209,6 +213,7 @@ const CODEQL_VERSION_METRICS = "2.5.5";
const CODEQL_VERSION_GROUP_RULES = "2.5.5";
const CODEQL_VERSION_SARIF_GROUP = "2.5.3";
export const CODEQL_VERSION_NEW_TRACING = "2.6.0"; // Use multi-language (>= 2.5.6) and indirect (>= 2.6.0) tracing.
export const CODEQL_VERSION_COUNTS_LINES = "2.6.2";

function getCodeQLBundleName(): string {
let platform: string;
Expand Down Expand Up @@ -556,6 +561,10 @@ export function setCodeQL(partialCodeql: Partial<CodeQL>): CodeQL {
partialCodeql,
"databaseInterpretResults"
),
databasePrintBaseline: resolveFunction(
partialCodeql,
"databasePrintBaseline"
),
};
return cachedCodeQL;
}
Expand Down Expand Up @@ -860,6 +869,15 @@ async function getCodeQLForCmd(
// capture stdout, which contains analysis summaries
return await runTool(cmd, codeqlArgs);
},
async databasePrintBaseline(databasePath: string): Promise<string> {
const codeqlArgs = [
"database",
"print-baseline",
...getExtraOptionsFromEnv(["database", "print-baseline"]),
databasePath,
];
return await runTool(cmd, codeqlArgs);
},

/**
* Download specified packs into the package cache. If the specified
Expand Down

0 comments on commit f04acbb

Please sign in to comment.