Report lines of code and the main languages from cloc.txt

The project page takes the size from cloc's sum and names the three to
five largest languages, leaving out prose and data files such as Markdown,
text, CSV and JSON, and anything under 1% of the code beyond the third.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

README.md

@@ -39,8 +39,9 @@ report_generator from-folder --artifacts ./sample [--path <file-or-directory>]
 whose metadata is gone.
 
 The inputs are one SARIF file (`merged.sarif`, with one run per tool) and the vulnerability report
-(`vdr.json`); either is enough on its own. `metadata.json` is optional, and without it the project
-page is left out. The report has these parts:
+(`vdr.json`); either is enough on its own. `metadata.json` and `cloc.txt` (cloc's plain
+table, for the lines of code and the three to five main languages) are optional; without either the
+project page is left out. The report has these parts:
 
 1. the cover and a table of contents,
 2. a summary and the most critical issues across all sources,
@@ -98,8 +99,8 @@ vectors by `persistent_storage.utils.cvss_utils`.
 ### What it cannot fill in
 
 A field the artifacts do not answer is written as `<xxx>`, never guessed or quietly dropped. Today
-that is the manufacturer (unless given with `--manufacturer`), the analysed lines of code and the language
-breakdown.
+that is the manufacturer (unless given with `--manufacturer`), and, when there is no `cloc.txt`, the
+analysed lines of code and the language breakdown.
 
 A measured zero survives: `0 filtered` is a reading, and a blank is a gap.
 

report_generator/cloc.py

@@ -0,0 +1,90 @@
+"""Reads the project's size and languages out of the table cloc prints.
+
+cloc writes one row per language -- files, blank, comment and code lines -- and a ``SUM:`` row under
+them. Only the code lines are reported: that is what "lines of code" means to a reader, and blanks
+and comments say nothing about how much there is to analyse.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+# How many languages the report names at most, and how many it names at least where there are that
+# many. Between the two, a language has to carry :data:`MINIMUM_SHARE` of the code to be named.
+MOST_LANGUAGES = 5
+FEWEST_LANGUAGES = 3
+MINIMUM_SHARE = 0.01
+
+# What cloc counts that is not code anyone analyses: prose, and data files. Configuration and build
+# files -- YAML, Dockerfiles, TOML -- are scanned like code and so stay in.
+NOT_CODE = frozenset(
+    {
+        "AsciiDoc",
+        "CSV",
+        "JSON",
+        "JSON5",
+        "Markdown",
+        "reStructuredText",
+        "SVG",
+        "Text",
+        "TSV",
+    }
+)
+
+# A row of the table: a name, which may contain spaces, then the four counts.
+_ROW = re.compile(r"^(?P<name>\S.*?)\s+(?P<files>\d+)\s+\d+\s+\d+\s+(?P<code>\d+)\s*$")
+
+
+def parse_cloc(content: bytes | None) -> dict[str, Any] | None:
+    """Read cloc's table.
+
+    Args:
+        content: The table as cloc printed it, or nothing where the scan stored none.
+
+    Returns:
+        The ``files`` and ``code`` lines of the whole project, and its ``languages``
+        with the same two counts each, largest first; ``None`` where there is no
+        table to read.
+    """
+    if content is None:
+        return None
+
+    languages = []
+    total = None
+
+    for line in content.decode("utf-8", errors="replace").splitlines():
+        row = _ROW.match(line.strip())
+
+        if row is None:
+            continue
+
+        counts = {"files": int(row["files"]), "code": int(row["code"])}
+
+        if row["name"] == "SUM:":
+            total = counts
+        else:
+            languages.append({"name": row["name"], **counts})
+
+    if not languages:
+        return None
+
+    # cloc leaves the sum out when it counted a single language.
+    total = total or {
+        "files": sum(one["files"] for one in languages),
+        "code": sum(one["code"] for one in languages),
+    }
+
+    return {**total, "languages": sorted(languages, key=lambda one: -one["code"])}
+
+
+def main_languages(cloc: dict[str, Any]) -> list[dict[str, Any]]:
+    """The languages worth naming: the largest few that are code, leaving out the negligible ones."""
+    code = [one for one in cloc["languages"] if one["name"] not in NOT_CODE]
+    counted = sum(one["code"] for one in code) or 1
+
+    return [
+        one
+        for rank, one in enumerate(code[:MOST_LANGUAGES])
+        if rank < FEWEST_LANGUAGES or one["code"] / counted >= MINIMUM_SHARE
+    ]

report_generator/context.py

@@ -25,6 +25,7 @@ from typing import TYPE_CHECKING, Any
 from .vendor.cipherverifier.core.services.sarif import _build_finding, _build_report, _rules_by_id
 from .vendor.cipherverifier.core.services.vdr import summarize_vdr
 
+from .cloc import main_languages
 from .layout import TITLE
 from .severity import finding_severity
 
@@ -79,9 +80,9 @@ def build_context(
         "sections": [_tool_section(name, report) for name, report in tools],
     }
 
-    # Without the pipeline's metadata this page would be nothing but gaps.
-    if metadata:
-        context["project"] = _project(metadata, project)
+    # Without the pipeline's metadata or cloc's count this page would be nothing but gaps.
+    if metadata or documents.get("cloc"):
+        context["project"] = _project(metadata, project, documents.get("cloc"))
 
     critical = _critical(tools, sca)
 
@@ -240,24 +241,36 @@ def _version(metadata: dict[str, Any]) -> str | None:
     return commit if isinstance(commit, str) and _COMMIT.fullmatch(commit) else None
 
 
-def _project(metadata: dict[str, Any], project: str | None) -> dict[str, Any]:
-    """What was scanned."""
+def _project(metadata: dict[str, Any], project: str | None, cloc: dict[str, Any] | None) -> dict[str, Any]:
+    """What was scanned, and how much of it there is, as cloc counted it."""
     return {
         "id": "project",
         "title": "Project Information",
         "overview": [
             _fact("Project", project),
             _fact("Version", _version(metadata), kind="mono"),
-            # Nothing in the pipeline counts lines, so nothing here can report them.
-            _fact("Analysed lines of code", None),
+            _fact("Analysed lines of code", f"{cloc['code']:,} in {cloc['files']:,} files" if cloc else None),
         ],
-        # Left as one unstated row rather than a table of zeroes: no stage reports a language
-        # breakdown, and a share invented from the file extensions of the findings would be a
-        # measurement of where the bugs are, not of what the project is written in.
-        "languages": {"title": "Languages", "rows": [{"label": UNKNOWN, "share": UNKNOWN}]},
+        "languages": {"title": "Main languages", "rows": _languages(cloc)},
     }
 
 
+def _languages(cloc: dict[str, Any] | None) -> list[dict[str, Any]]:
+    """The main languages with their lines of code and share of the whole project.
+
+    Left as one unstated row rather than a table of zeroes where cloc did not run: a share invented
+    from the file extensions of the findings would measure where the bugs are, not what the project
+    is written in.
+    """
+    if not cloc:
+        return [{"label": UNKNOWN, "share": UNKNOWN}]
+
+    return [
+        {"label": one["name"], "share": f"{one['code']:,} lines ({one['code'] / cloc['code'] * 100:.1f} %)"}
+        for one in main_languages(cloc)
+    ]
+
+
 def _summary(
     tools: list[tuple[str, SarifReport]], sca: VdrReport | None, metrics: dict[str, Any] | None
 ) -> dict[str, Any]:

report_generator/rendering.py

@@ -16,6 +16,7 @@ from typing import TYPE_CHECKING, Any
 from .vendor.cc_common.api import PipelineError
 from .vendor.cc_common.api import context as logging_context
 
+from .cloc import parse_cloc
 from .context import build_context
 from .layout import render_pdf
 
@@ -72,6 +73,7 @@ def render(
     """
     documents = {name: _parsed(sources, name, candidates) for name, candidates in INPUTS.items()}
     documents["metrics"] = metrics
+    documents["cloc"] = parse_cloc(sources.first("cloc.txt"))
     if all(documents.get(name) is None for name in REQUIRED_ANY):
         message = f"Scan {sources.scan_id} carries no {' and no '.join(REQUIRED_ANY)}, so no report can be built."