diff --git a/README.md b/README.md index c9bb541..6eba5f6 100644 --- a/README.md +++ b/README.md @@ -39,8 +39,9 @@ report_generator from-folder --artifacts ./sample [--path ] whose metadata is gone. The inputs are one SARIF file (`merged.sarif`, with one run per tool) and the vulnerability report -(`vdr.json`); either is enough on its own. `metadata.json` is optional, and without it the project -page is left out. The report has these parts: +(`vdr.json`); either is enough on its own. `metadata.json` and `cloc.txt` (cloc's plain +table, for the lines of code and the three to five main languages) are optional; without either the +project page is left out. The report has these parts: 1. the cover and a table of contents, 2. a summary and the most critical issues across all sources, @@ -98,8 +99,8 @@ vectors by `persistent_storage.utils.cvss_utils`. ### What it cannot fill in A field the artifacts do not answer is written as ``, never guessed or quietly dropped. Today -that is the manufacturer (unless given with `--manufacturer`), the analysed lines of code and the language -breakdown. +that is the manufacturer (unless given with `--manufacturer`), and, when there is no `cloc.txt`, the +analysed lines of code and the language breakdown. A measured zero survives: `0 filtered` is a reading, and a blank is a gap. diff --git a/report_generator/cloc.py b/report_generator/cloc.py new file mode 100644 index 0000000..d189102 --- /dev/null +++ b/report_generator/cloc.py @@ -0,0 +1,90 @@ +"""Reads the project's size and languages out of the table cloc prints. + +cloc writes one row per language -- files, blank, comment and code lines -- and a ``SUM:`` row under +them. Only the code lines are reported: that is what "lines of code" means to a reader, and blanks +and comments say nothing about how much there is to analyse. +""" + +from __future__ import annotations + +import re +from typing import Any + +# How many languages the report names at most, and how many it names at least where there are that +# many. Between the two, a language has to carry :data:`MINIMUM_SHARE` of the code to be named. +MOST_LANGUAGES = 5 +FEWEST_LANGUAGES = 3 +MINIMUM_SHARE = 0.01 + +# What cloc counts that is not code anyone analyses: prose, and data files. Configuration and build +# files -- YAML, Dockerfiles, TOML -- are scanned like code and so stay in. +NOT_CODE = frozenset( + { + "AsciiDoc", + "CSV", + "JSON", + "JSON5", + "Markdown", + "reStructuredText", + "SVG", + "Text", + "TSV", + } +) + +# A row of the table: a name, which may contain spaces, then the four counts. +_ROW = re.compile(r"^(?P\S.*?)\s+(?P\d+)\s+\d+\s+\d+\s+(?P\d+)\s*$") + + +def parse_cloc(content: bytes | None) -> dict[str, Any] | None: + """Read cloc's table. + + Args: + content: The table as cloc printed it, or nothing where the scan stored none. + + Returns: + The ``files`` and ``code`` lines of the whole project, and its ``languages`` + with the same two counts each, largest first; ``None`` where there is no + table to read. + """ + if content is None: + return None + + languages = [] + total = None + + for line in content.decode("utf-8", errors="replace").splitlines(): + row = _ROW.match(line.strip()) + + if row is None: + continue + + counts = {"files": int(row["files"]), "code": int(row["code"])} + + if row["name"] == "SUM:": + total = counts + else: + languages.append({"name": row["name"], **counts}) + + if not languages: + return None + + # cloc leaves the sum out when it counted a single language. + total = total or { + "files": sum(one["files"] for one in languages), + "code": sum(one["code"] for one in languages), + } + + return {**total, "languages": sorted(languages, key=lambda one: -one["code"])} + + +def main_languages(cloc: dict[str, Any]) -> list[dict[str, Any]]: + """The languages worth naming: the largest few that are code, leaving out the negligible ones.""" + code = [one for one in cloc["languages"] if one["name"] not in NOT_CODE] + counted = sum(one["code"] for one in code) or 1 + + return [ + one + for rank, one in enumerate(code[:MOST_LANGUAGES]) + if rank < FEWEST_LANGUAGES or one["code"] / counted >= MINIMUM_SHARE + ] diff --git a/report_generator/context.py b/report_generator/context.py index 2fc90f2..6577cac 100644 --- a/report_generator/context.py +++ b/report_generator/context.py @@ -25,6 +25,7 @@ from typing import TYPE_CHECKING, Any from .vendor.cipherverifier.core.services.sarif import _build_finding, _build_report, _rules_by_id from .vendor.cipherverifier.core.services.vdr import summarize_vdr +from .cloc import main_languages from .layout import TITLE from .severity import finding_severity @@ -79,9 +80,9 @@ def build_context( "sections": [_tool_section(name, report) for name, report in tools], } - # Without the pipeline's metadata this page would be nothing but gaps. - if metadata: - context["project"] = _project(metadata, project) + # Without the pipeline's metadata or cloc's count this page would be nothing but gaps. + if metadata or documents.get("cloc"): + context["project"] = _project(metadata, project, documents.get("cloc")) critical = _critical(tools, sca) @@ -240,24 +241,36 @@ def _version(metadata: dict[str, Any]) -> str | None: return commit if isinstance(commit, str) and _COMMIT.fullmatch(commit) else None -def _project(metadata: dict[str, Any], project: str | None) -> dict[str, Any]: - """What was scanned.""" +def _project(metadata: dict[str, Any], project: str | None, cloc: dict[str, Any] | None) -> dict[str, Any]: + """What was scanned, and how much of it there is, as cloc counted it.""" return { "id": "project", "title": "Project Information", "overview": [ _fact("Project", project), _fact("Version", _version(metadata), kind="mono"), - # Nothing in the pipeline counts lines, so nothing here can report them. - _fact("Analysed lines of code", None), + _fact("Analysed lines of code", f"{cloc['code']:,} in {cloc['files']:,} files" if cloc else None), ], - # Left as one unstated row rather than a table of zeroes: no stage reports a language - # breakdown, and a share invented from the file extensions of the findings would be a - # measurement of where the bugs are, not of what the project is written in. - "languages": {"title": "Languages", "rows": [{"label": UNKNOWN, "share": UNKNOWN}]}, + "languages": {"title": "Main languages", "rows": _languages(cloc)}, } +def _languages(cloc: dict[str, Any] | None) -> list[dict[str, Any]]: + """The main languages with their lines of code and share of the whole project. + + Left as one unstated row rather than a table of zeroes where cloc did not run: a share invented + from the file extensions of the findings would measure where the bugs are, not what the project + is written in. + """ + if not cloc: + return [{"label": UNKNOWN, "share": UNKNOWN}] + + return [ + {"label": one["name"], "share": f"{one['code']:,} lines ({one['code'] / cloc['code'] * 100:.1f} %)"} + for one in main_languages(cloc) + ] + + def _summary( tools: list[tuple[str, SarifReport]], sca: VdrReport | None, metrics: dict[str, Any] | None ) -> dict[str, Any]: diff --git a/report_generator/rendering.py b/report_generator/rendering.py index c3c1cd8..e37d30f 100644 --- a/report_generator/rendering.py +++ b/report_generator/rendering.py @@ -16,6 +16,7 @@ from typing import TYPE_CHECKING, Any from .vendor.cc_common.api import PipelineError from .vendor.cc_common.api import context as logging_context +from .cloc import parse_cloc from .context import build_context from .layout import render_pdf @@ -72,6 +73,7 @@ def render( """ documents = {name: _parsed(sources, name, candidates) for name, candidates in INPUTS.items()} documents["metrics"] = metrics + documents["cloc"] = parse_cloc(sources.first("cloc.txt")) if all(documents.get(name) is None for name in REQUIRED_ANY): message = f"Scan {sources.scan_id} carries no {' and no '.join(REQUIRED_ANY)}, so no report can be built."