florian/pdf-reporting
1
0 Fork 1
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Initial commit. monorepo deps are copied in a dirty way to the vendor folder

Browse Source
This commit is contained in:
Florian Frank committed 2026-09-18 16:58:29 +02:00
commit 06593ac488
53 files changed
+7692
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

report_generator/commands/__init__.py
+1
View File
@@ -0,0 +1 @@
"""The commands the report generator exposes."""
report_generator/commands/generate.py
+88
View File
@@ -0,0 +1,88 @@
"""report_generator generate command."""
from pathlib import Path
from uuid import UUID
from cc_common.api import PipelineError, context
from cc_common.utils import print
from pydantic import BaseModel, Field
from persistent_storage.shared import StorageConfiguration
from ..rendering import render
from ..sources import collect
# Where the report goes when the caller names no path.
DEFAULT_DIRECTORY = Path("reports")
class GenerateProperties(BaseModel):
"""Configuration properties for the generate CLI command.
Attributes:
scan_id: The scan to report on.
path: Where to write the PDF. A directory is written into under the name
the renderer chose; leaving it out writes into ``reports/``.
storage_config: The store the scan's artifacts are read from.
"""
scan_id: UUID = Field(description="The scan the report should be generated for.")
path: Path | None = Field(default=None, description="Where to write the PDF report.")
storage_config: StorageConfiguration
class GeneratedReport(BaseModel):
"""The report that was written.
Attributes:
scan_id: The scan it is about.
path: The file written.
size: How large it came out, in bytes.
"""
scan_id: UUID
path: Path
size: int
def print_as_cc_common_result(self) -> None:
"""Say what was written and where."""
print(f"[bold]Report for scan {self.scan_id}[/bold]")
print(f" {self.size / 1024:.0f} kB written to {self.path}")
def generate(properties: GenerateProperties) -> GeneratedReport:
"""Build the PDF report of one scan and write it.
Args:
properties: Parsed command properties naming the scan and the destination.
Returns:
The report that was written.
Raises:
PipelineError: The scan cannot be read, is missing an input the report
needs, or the destination cannot be written.
"""
sources = collect(properties.storage_config, properties.scan_id)
name, content = render(sources)
destination = _destination(properties.path, name)
try:
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_bytes(content)
except OSError as error:
message = f"Cannot write the report to '{destination}': {error}"
raise PipelineError(message) from error
context.logger.info(f"Report written to {destination} ✔")
return GeneratedReport(scan_id=properties.scan_id, path=destination, size=len(content))
def _destination(path: Path | None, name: str) -> Path:
"""Where the report goes, treating a directory as a place to put it rather than as the file."""
if path is None:
return DEFAULT_DIRECTORY / name
return path / name if path.is_dir() or not path.suffix else path
report_generator/context.py
+595
View File
@@ -0,0 +1,595 @@
"""Turns a scan's stored artifacts into the context the report template draws.
No parsing of its own. The SARIF and the vulnerability report are read by the summarisers in
:mod:`cipherverifier.core.services.sarif` and :mod:`cipherverifier.core.services.vdr` -- both of
which are deliberately free of Django and of any store -- and the pipeline metadata by the helpers
in :mod:`persistent_storage.utils.rep_utils`. What is left here is the mapping: which summarised
number goes in which row, and what to say where the artifacts do not say anything.
That last part is the rule this module is built around. A field the artifacts cannot answer is
written as :data:`UNKNOWN` rather than guessed, left blank or quietly dropped. A reader who sees
``<xxx>`` knows the pipeline did not report it; a reader who sees a plausible zero does not.
Moving this to archon is a change of what :mod:`sources` reads
from. The two summarisers travel with it -- they belong beside whatever holds the artifacts, not
inside the web application that happens to import them today.
"""
from __future__ import annotations
from datetime import UTC, datetime
from typing import TYPE_CHECKING, Any
from cipherverifier.core.services.sarif import summarize_sarif
from cipherverifier.core.services.vdr import summarize_vdr
if TYPE_CHECKING:
from cipherverifier.core.services.sarif import Finding, SarifReport
from cipherverifier.core.services.vdr import VdrReport, Vulnerability
from .sources import ReportSources
# What stands in for a field the stored artifacts do not answer.
UNKNOWN = "<xxx>"
# How many of each list the report carries. A report is read, not audited; the full set is in the
# artifacts it was built from.
MOST_CRITICAL = 6
CARDS_PER_GROUP = 8
BARS = 10
# The severity a SARIF level is reported under. SARIF names four levels and the report bands five,
# so the mapping is written down rather than inferred from the spelling.
SEVERITY_OF_LEVEL = {
"error": "Critical",
"warning": "High",
"note": "Medium",
"none": "Low",
"info": "Low",
}
SEVERITY_ORDER = ("Critical", "High", "Medium", "Low")
def build_context(sources: ReportSources, documents: dict[str, Any]) -> dict[str, Any]:
"""Build the report context from a scan's parsed artifacts.
Args:
sources: The scan and its stored files.
documents: The parsed artifacts by role -- ``metadata``, ``findings``,
``vulnerabilities`` -- each absent where the scan stored none.
Returns:
The context :func:`layout.render_pdf`
draws. Sections whose artifact is missing are left out entirely, which
the template renders as a shorter report.
"""
metadata = documents.get("metadata") or {}
sast = summarize_sarif(documents["findings"]) if documents.get("findings") else None
sca = summarize_vdr(documents["vulnerabilities"]) if documents.get("vulnerabilities") else None
context: dict[str, Any] = {
"cover": _cover(sources, metadata, sast, sca),
"scan": _scan(sources, metadata),
"project": _project(metadata),
"summary": _summary(sast, sca),
"sections": [],
}
critical = _critical(sast, sca)
if critical["table"]["rows"]:
context["critical"] = critical
if sast is not None:
context["sections"].append(_sast_section(sast))
if sca is not None:
context["sections"].append(_sca_section(sca))
return context
# --- the page ------------------------------------------------------------------------------
def _fact(label: str, value: Any, *, gap: bool = False, kind: str | None = None) -> dict[str, Any]: # noqa: ANN401
"""One label/value row, with anything unstated written as :data:`UNKNOWN`."""
return {"label": label, "value": _stated(value), "gap": gap, "kind": kind}
def _row(label: str, value: Any, **marks: Any) -> dict[str, Any]: # noqa: ANN401
"""One row of a card."""
return {"label": label, "value": _stated(value), **marks}
def _stated(value: Any) -> str: # noqa: ANN401
"""What a value reads as, or :data:`UNKNOWN` where there is nothing to read.
Zero is a reading and survives; an empty string and ``None`` are not, and do not. The difference
matters on a page where both appear: "0 filtered" is a measurement, and a blank is a gap.
"""
if value is None or (isinstance(value, str) and not value.strip()):
return UNKNOWN
return str(value)
def _cover(
sources: ReportSources,
metadata: dict[str, Any],
sast: SarifReport | None,
sca: VdrReport | None,
) -> dict[str, Any]:
"""The first page."""
configuration = metadata.get("configuration") or {}
technologies = [name for name, present in (("SAST", sast is not None), ("SCA", sca is not None)) if present]
return {
"scan_id": str(sources.scan_id),
"subject": _stated(configuration.get("repository_name")),
"footer_date": datetime.now(UTC).strftime("%d %B %Y"),
"logo": "cc_logo.png",
"title": "SAST/SCA Report",
"identity": [
_fact("Scan Identifier", sources.scan_id, kind="mono"),
# Neither exists in the store today: a scan has artifacts, and a report of it is not
# itself recorded as one.
_fact("Report ID", None, kind="mono"),
_fact("Artifact ID", None, kind="mono"),
_fact("Technology", ", ".join(technologies) or None),
_fact("Project", configuration.get("repository_name"), gap=True),
_fact("Version", configuration.get("repository_version")),
_fact("Report Created", datetime.now(UTC).strftime("%A, %B %d, %Y")),
_fact("Manufacturer", None),
],
"severity_ledger": _severity_ledger(sast, sca),
}
def _severity_ledger(sast: SarifReport | None, sca: VdrReport | None) -> dict[str, Any] | None:
"""How many issues of each severity survived filtering, over both halves of the pipeline."""
counts = dict.fromkeys(SEVERITY_ORDER, 0)
for finding in _true_findings(sast):
counts[SEVERITY_OF_LEVEL.get(finding.level, "Low")] += 1
for vulnerability in sca.vulnerabilities if sca else []:
counts[_vdr_severity(vulnerability)] = counts.get(_vdr_severity(vulnerability), 0) + 1
if not any(counts.values()):
return None
rows: list[dict[str, Any]] = [
{"label": f"{name} severity issues", "count": counts.get(name, 0)} for name in SEVERITY_ORDER
]
rows.append({"label": "Total true issues", "count": sum(counts.values()), "total": True})
return {"title": "True issues by severity", "rows": rows}
def _scan(sources: ReportSources, metadata: dict[str, Any]) -> dict[str, Any]:
"""What the pipeline did, stage by stage."""
stages = [stage for stage in metadata.get("stages") or [] if isinstance(stage, dict)]
failed = [stage["label"] for stage in stages if stage.get("status") not in ("success", None)]
return {
"title": "CipherCell Scan Information",
"overview": [
_fact("Scan started at", _moment(metadata.get("pipeline_start_iso"))),
_fact("Scan duration", _duration(metadata.get("pipeline_duration_seconds"))),
_fact("Scan completed at", _moment(metadata.get("pipeline_end_iso"))),
_fact(
"Pipeline Steps", ", ".join(_stated(stage.get("label") or stage.get("key")) for stage in stages) or None
),
_fact("Errors Detected", ", ".join(failed) if failed else "None" if stages else None),
],
"stages": [_stage(stage, sources) for stage in stages],
}
def _stage(stage: dict[str, Any], sources: ReportSources) -> dict[str, Any]:
"""One stage of the pipeline.
The tools a stage ran and the profile it ran them under are not recorded in the metadata, so
they are left unstated. What is recorded is when it ran, how it ended, and -- indirectly -- what
it produced, which is read off the names of the files the scan stored.
"""
return {
"title": stage.get("label") or stage.get("key") or UNKNOWN,
"rows": [
_fact("Selected Tools", None),
_fact("Profile", None),
_fact("Duration", _duration(stage.get("duration_seconds"))),
_fact("Created Artifacts", ", ".join(sorted(sources.files)) or None, kind="mono"),
_fact("Errors Detected", "None" if stage.get("status") == "success" else stage.get("status")),
],
}
def _project(metadata: dict[str, Any]) -> dict[str, Any]:
"""What was scanned."""
configuration = metadata.get("configuration") or {}
return {
"title": "Project Information",
"overview": [
_fact("Project", configuration.get("repository_name")),
_fact("Version", configuration.get("repository_version")),
# Nothing in the pipeline counts lines, so nothing here can report them.
_fact("Analysed lines of code", None),
],
# Left as one unstated row rather than a table of zeroes: no stage reports a language
# breakdown, and a share invented from the file extensions of the findings would be a
# measurement of where the bugs are, not of what the project is written in.
"languages": {"title": "Languages", "rows": [{"label": UNKNOWN, "share": UNKNOWN}]},
}
def _summary(sast: SarifReport | None, sca: VdrReport | None) -> dict[str, Any]:
"""The counts, and where they concentrate."""
true_sast = _true_findings(sast)
sca_total = sca.total if sca else 0
raw = (sast.total if sast else 0) + sca_total
filtered = sast.false_positive_count if sast else 0
return {
"title": "Summary",
"overview": [
_fact("Raw findings", raw or None),
# Only SAST findings carry a filtering verdict. A vulnerability suppressed in
# Dependency-Track never reaches the report at all, so the two cannot be added up.
_fact("Filtered", f"{filtered} (SAST)" if sast else None),
_fact("True issues", f"{len(true_sast)} SAST, {sca_total} SCA" if sast or sca else None),
# Each half stated only where its artifact was there. A scan that ran no SCA stage
# reports its SAST rules and says nothing about components, rather than nothing at all.
_fact("Distinct issue types (true)", ", ".join(_issue_types(sast, sca)) or None),
],
"distributions": _distributions(true_sast, sca),
}
def _issue_types(sast: SarifReport | None, sca: VdrReport | None) -> list[str]:
"""How many distinct kinds of issue each half of the pipeline found."""
counted = []
if sast is not None:
counted.append(f"{sast.rule_count} SAST rules")
if sca is not None:
counted.append(f"{sca.component_count} affected components")
return counted
def _distributions(findings: list[Finding], sca: VdrReport | None) -> list[dict[str, Any]]:
"""Where the issues concentrate: by rule for SAST, by package for SCA."""
blocks = []
if findings:
blocks.append(
{
"title": f"True SAST issues by rule ({len(findings)})",
**_bars(_group(findings, lambda one: one.rule_id or UNKNOWN, _finding_severity)),
}
)
if sca and sca.vulnerabilities:
blocks.append(
{
"title": f"Most affected components, true SCA issues ({sca.component_count})",
**_bars(
_group(
[(one, name) for one in sca.vulnerabilities for name in one.component_names],
lambda pair: pair[1],
lambda pair: _vdr_severity(pair[0]),
),
mono=True,
),
}
)
return blocks
def _group[T](
items: list[T],
label_of: Any, # noqa: ANN401
severity_of: Any, # noqa: ANN401
) -> list[tuple[str, dict[str, int]]]:
"""Count items by label, and within each label by severity."""
grouped: dict[str, dict[str, int]] = {}
for item in items:
counts = grouped.setdefault(label_of(item), dict.fromkeys(SEVERITY_ORDER, 0))
counts[severity_of(item)] += 1
return sorted(grouped.items(), key=lambda pair: -sum(pair[1].values()))
def _bars(grouped: list[tuple[str, dict[str, int]]], *, mono: bool = False) -> dict[str, Any]:
"""Turn grouped counts into bars scaled against the busiest row.
Only the busiest :data:`BARS` are drawn, and the caveat says so where the rest were left off.
Without that the legend counts what is on the page while the heading counts the report, and the
two quietly disagree by however much the tail held.
"""
shown = grouped[:BARS]
largest = max((sum(counts.values()) for _, counts in shown), default=0) or 1
rows: list[dict[str, Any]] = []
totals = dict.fromkeys(SEVERITY_ORDER, 0)
for label, counts in shown:
rows.append(
{
"label": label,
"count": sum(counts.values()),
"mono": mono,
"parts": [
{"severity": name, "share": round(counts[name] / largest * 100, 1)}
for name in SEVERITY_ORDER
if counts[name]
],
}
)
for name, count in counts.items():
totals[name] += count
return {
"rows": rows,
"legend": {
"entries": [
{"severity": name, "label": f"{name} ({totals[name]})"}
for name in reversed(SEVERITY_ORDER)
if totals[name]
],
"caveat": _caveat(shown, grouped, largest),
},
}
def _caveat(shown: list[Any], grouped: list[Any], largest: int) -> str:
"""What the bars leave out, and what their length is measured against."""
scale = f"bar length is proportional to the issue count (100 % = {largest})."
if len(shown) == len(grouped):
return scale
return f"{len(shown)} of {len(grouped)} shown; {scale}"
def _critical(sast: SarifReport | None, sca: VdrReport | None) -> dict[str, Any]:
"""The handful worth reading first, whichever half of the pipeline found them."""
rows: list[dict[str, Any]] = []
for finding in sorted(_true_findings(sast), key=lambda one: SEVERITY_ORDER.index(_finding_severity(one))):
rows.append(
{
"cells": [
"SAST",
{"value": _finding_severity(finding), "kind": "severity"},
_stated(finding.rule_id),
{"value": _stated(_where(finding)), "kind": "mono"},
_stated(finding.message),
]
}
)
for vulnerability in sorted(
sca.vulnerabilities if sca else [],
key=lambda one: SEVERITY_ORDER.index(_vdr_severity(one)),
):
rows.append(
{
"cells": [
"SCA",
{"value": _vdr_severity(vulnerability), "kind": "severity"},
_stated(vulnerability.id),
{"value": _stated(", ".join(vulnerability.component_names)), "kind": "mono"},
_stated(vulnerability.description),
]
}
)
return {
"title": "Most Critical Vulnerabilities",
"table": {
"columns": ["Type", "Severity", "CVE/CWE", "Affected File / Component", "Description"],
"rows": rows[:MOST_CRITICAL],
},
}
def _sast_section(sast: SarifReport) -> dict[str, Any]:
"""The SAST findings, split into what was kept and what was set aside."""
true_findings = _true_findings(sast)
filtered = [one for one in sast.findings if one.false_positive]
groups: list[dict[str, Any]] = []
if true_findings:
groups.append(
{
"title": f"True Issues ({len(true_findings)})",
"cards": [_finding_card(one) for one in true_findings[:CARDS_PER_GROUP]],
}
)
if filtered:
groups.append(
{
"title": f"Filtered Issues ({len(filtered)})",
"note": (
"The following findings were reported by the scanners but assessed as not exploitable in "
"this artifact. Each entry carries the verdict that filtered it."
),
"cards": [_finding_card(one, filtered_out=True) for one in filtered[:CARDS_PER_GROUP]],
"table": _filtered_table(filtered),
}
)
return {"title": "SAST Vulnerabilities", "groups": groups}
def _finding_card(finding: Finding, *, filtered_out: bool = False) -> dict[str, Any]:
"""One SARIF finding."""
rows = [
_row("Severity", _finding_severity(finding), kind="severity"),
_row("Location", _where(finding), kind="mono"),
_row("Description", finding.message),
_row("Reported By", ", ".join(finding.tools) or None),
]
if filtered_out:
rows.append(
_row(
"Filtering Reason",
finding.polygraph_reasoning,
aside=True,
token=finding.polygraph_verdict,
)
)
else:
# Polygraph's verdict where it looked at this one, and nothing where it did not: silence is
# not a clean bill of health, and must not be written as one.
rows.append(_row("Evidence", finding.verdict_label or None))
return {"title": f"{_stated(finding.rule_id)} — {_stated(finding.rule_name)}", "rows": rows}
def _filtered_table(findings: list[Finding]) -> dict[str, Any]:
"""How many of each rule were filtered, and on what grounds."""
grouped: dict[str, list[Finding]] = {}
for finding in findings:
grouped.setdefault(finding.rule_id or UNKNOWN, []).append(finding)
rows: list[dict[str, Any]] = [
{
"cells": [
rule,
len(group),
{"value": _stated(group[0].polygraph_verdict), "kind": "mono"},
]
}
for rule, group in sorted(grouped.items(), key=lambda pair: -len(pair[1]))
]
rows.append({"cells": ["Total", len(findings), ""], "total": True})
return {
"title": f"Filtered SAST issues by rule ({len(findings)})",
"columns": ["Rule", "Filtered", "Dominant verdict"],
"rows": rows,
}
def _sca_section(sca: VdrReport) -> dict[str, Any]:
"""The dependency findings."""
return {
"title": "SBOM / SCA Vulnerabilities",
"new_page": True,
"groups": [
{
"title": f"True Issues by CVEs ({sca.total})",
"cards": [_vulnerability_card(one) for one in sca.vulnerabilities[:CARDS_PER_GROUP]],
}
],
}
def _vulnerability_card(vulnerability: Vulnerability) -> dict[str, Any]:
"""One dependency finding, as much of it as the report stated."""
metrics = vulnerability.metrics
impact = [
f"{name} ({value})"
for name, value in (
("Confidentiality", metrics.confidentiality),
("Integrity", metrics.integrity),
("Availability", metrics.availability),
)
if value
]
return {
"title": _stated(vulnerability.id),
"rows": [
_row("Affected component", ", ".join(vulnerability.component_names) or None, kind="mono"),
_row("Severity", _vdr_severity(vulnerability), kind="severity"),
_row("Description", vulnerability.description),
_row("CWE", ", ".join(f"{one}: {vulnerability.cwe_names[one]}" for one in vulnerability.cwes) or None),
_row("CVSS", f"{vulnerability.score} / 10" if vulnerability.score is not None else None),
_row("Impact", ", ".join(impact) or None),
_row("Exploitation likelihood (EPSS)", _percent(metrics.epss)),
_row(
"Fixed in",
# The report names the newest version of the package, which is not the same as the
# version that fixes this advisory -- so it is labelled as what it is.
next(
(one.latest_version for one in vulnerability.components if one.is_outdated),
None,
),
kind="mono",
),
_row("Known for", f"{metrics.published_days_ago} days" if metrics.published_days_ago else None),
_row("Recommendation", vulnerability.recommendation),
],
}
# --- readings ------------------------------------------------------------------------------
def _true_findings(sast: SarifReport | None) -> list[Finding]:
"""The findings that survived filtering."""
return [one for one in sast.findings if not one.false_positive] if sast else []
def _finding_severity(finding: Finding) -> str:
"""The severity band a SARIF level falls in."""
return SEVERITY_OF_LEVEL.get(finding.level, "Low")
def _vdr_severity(vulnerability: Vulnerability) -> str:
"""The severity band a vulnerability reports, capitalised the way the report bands them."""
named = (vulnerability.severity or "").capitalize()
return named if named in SEVERITY_ORDER else "Low"
def _where(finding: Finding) -> str | None:
"""Where a finding points, as ``file:line``."""
if finding.location is None:
return None
line = finding.location.start_line
return f"{finding.location.file}:{line}" if line else finding.location.file
def _percent(share: float | None) -> str | None:
"""A probability as a percentage, at the precision it is worth stating."""
if share is None:
return None
return f"{share * 100:.2f} %" if share < 0.01 else f"{share * 100:.1f} %"
def _moment(stated: Any) -> str | None: # noqa: ANN401
"""An ISO timestamp as the report writes it, or nothing where it cannot be read."""
if not isinstance(stated, str):
return None
try:
return datetime.fromisoformat(stated).strftime("%A, %B %d, %Y %H:%M:%S (%Z)").replace("()", "").strip()
except ValueError:
return None
def _duration(seconds: Any) -> str | None: # noqa: ANN401
"""A number of seconds as ``HH:MM:SS``."""
if not isinstance(seconds, (int, float)) or isinstance(seconds, bool):
return None
whole = int(seconds)
return f"{whole // 3600:02d}:{whole % 3600 // 60:02d}:{whole % 60:02d}"
report_generator/layout.py
+90
View File
@@ -0,0 +1,90 @@
"""Renders the SAST/SCA report template.
The template and its stylesheet are the design; this module is only what turns them into a
document. It takes a context that is already shaped for the page -- every number formatted, every
bar already a percentage -- so that the template places strings and never computes.
Nothing here reads an artifact. Filling a context from a scan is :mod:`context`, which works from
what :mod:`sources` collected; :func:`sample.sample_context` builds one by hand, which is what the
template tests render.
"""
from __future__ import annotations
from pathlib import Path
from typing import Any
from jinja2 import Environment, FileSystemLoader, select_autoescape
TEMPLATE_DIR = Path(__file__).resolve().parent / "template"
TEMPLATE_NAME = "report.html.j2"
STYLESHEET = TEMPLATE_DIR / "report.css"
# What the document calls itself, in the PDF's own metadata rather than only on its first page.
TITLE = "SAST/SCA Report"
AUTHOR = "CipherCell"
# The shape :func:`render_html` expects. Written down because the template guards on it: a key that
# is absent leaves its section out, which is how a scan that ran half the pipeline produces a
# shorter report rather than one full of empty tables.
CONTEXT = {
"cover": "scan_id, footer_date, logo, title, identity[], severity_ledger{title, rows[]}",
"scan": "title, overview[], stages[{title, rows[], table?}]",
"project": "title, overview[], languages{title, rows[]}",
"summary": "title, overview[], distributions[{title, rows[], legend}]",
"critical": "title, table{columns[], rows[]}",
"sections": "[{title, new_page?, groups[{title, note?, cards[], table?}]}]",
}
def render_html(context: dict[str, Any]) -> str:
"""Render the report as an HTML document.
Args:
context: The page-ready context; see :data:`CONTEXT`.
Returns:
The rendered HTML.
"""
environment = Environment(
loader=FileSystemLoader(TEMPLATE_DIR),
autoescape=select_autoescape(default_for_string=True, default=True),
trim_blocks=True,
lstrip_blocks=True,
)
return environment.get_template(TEMPLATE_NAME).render(**context)
def render_pdf(context: dict[str, Any]) -> bytes:
"""Render the report as a PDF.
Args:
context: The page-ready context; see :data:`CONTEXT`.
Returns:
The PDF.
"""
# WeasyPrint pulls in native cairo and pango bindings. Imported here so that importing this
# module -- to render the HTML, or merely to read CONTEXT -- does not pay for them.
from weasyprint import CSS, HTML
from weasyprint.text.fonts import FontConfiguration
# Without this the @font-face rules are ignored and Space Grotesk is silently replaced by
# whatever the host happens to have, which is how the design sample came to be set in Carlito.
fonts = FontConfiguration()
# Relative URLs in the template and the stylesheet -- the mark, the font files -- resolve
# against the template directory.
document = HTML(string=render_html(context), base_url=str(TEMPLATE_DIR)).render(
stylesheets=[CSS(filename=str(STYLESHEET), font_config=fonts)],
font_config=fonts,
)
document.metadata.title = f"{TITLE} - {context.get('cover', {}).get('subject', '')}".strip(" -")
document.metadata.authors = [AUTHOR]
return bytes(document.write_pdf())
report_generator/rendering.py
+100
View File
@@ -0,0 +1,100 @@
"""Turns collected sources into a PDF.
Three steps, each somewhere else: this module decides which stored file feeds which part of the
report, :mod:`context` maps the parsed artifacts onto the page,
and :mod:`layout` draws it.
Adding a section to the report is an entry in :data:`INPUTS` and a block in the template -- not
another renderer.
"""
from __future__ import annotations
import json
from typing import TYPE_CHECKING, Any
from cc_common.api import PipelineError
from cc_common.api import context as logging_context
from .context import build_context
from .layout import render_pdf
if TYPE_CHECKING:
from .sources import ReportSources
# Which stored file each input of the report is taken from, under the names the pipeline stages
# write. Several per input because the same document travels under more than one name depending on
# which stage produced it; the first one a scan carries wins.
INPUTS = {
"metadata": ("metadata.json", "meta.json"),
"findings": ("sast_scanner_ranked.sarif", "polygraph.sarif", "sast_scanner_combined.sarif", "prism.sarif"),
"vulnerabilities": ("vdr.json", "vdr-2.json", "vdr-3.json"),
}
# What a report cannot be built without. The rest is reported as far as it was found: a scan that
# ran only half the pipeline is a shorter report, not a refusal.
REQUIRED = ("metadata",)
# What the file is called. The scan it is about is in the name, because a directory of these is
# otherwise unreadable.
NAME = "report_{scan}.pdf"
def render(sources: ReportSources) -> tuple[str, bytes]:
"""Render the report of one scan.
Args:
sources: What :func:`sources.collect` found.
Returns:
The file name the report should be saved under, and its bytes.
Raises:
PipelineError: The scan is missing an input the report cannot be built
without, or one of its documents cannot be parsed. Both are reported
by name: "no PDF was produced" leaves whoever ran it guessing which
stage of their pipeline did not finish.
"""
documents = {name: _parsed(sources, name, candidates) for name, candidates in INPUTS.items()}
missing = [name for name in REQUIRED if documents.get(name) is None]
if missing:
message = f"Scan {sources.scan_id} carries no {' and no '.join(missing)}, so no report can be built."
raise PipelineError(message)
found = [name for name, document in documents.items() if document is not None]
logging_context.logger.info(f"Rendering the report from {', '.join(found)}...")
return NAME.format(scan=sources.scan_id), render_pdf(build_context(sources, documents))
def _parsed(sources: ReportSources, name: str, candidates: tuple[str, ...]) -> Any | None: # noqa: ANN401
"""Read one of the scan's documents, or nothing where it stored none.
Args:
sources: The scan and its files.
name: What the document is called in :data:`INPUTS`.
candidates: The file names it may have been stored under.
Returns:
The parsed JSON, or ``None`` when the scan carries none.
Raises:
PipelineError: The file is there and is not readable. Kept apart from an
absent one on purpose: one is a pipeline that did not run a stage,
the other is a stage that produced something broken, and a report
that treated them alike would hide the second.
"""
content = sources.first(*candidates)
if content is None:
return None
try:
return json.loads(content)
except ValueError as error:
message = f"The {name} of scan {sources.scan_id} is not readable JSON: {error}"
raise PipelineError(message) from error
report_generator/sample.py
+525
View File
@@ -0,0 +1,525 @@
"""A complete report context, filled by hand.
Every section the template can draw appears here at least once, with the content of the design
sample in ``sample_report_template/``. It is what the tests render, and what to compare a real
context against once one is built from a scan: whatever the sample shows, the template supports.
Written out rather than generated so that it stays readable as a specification of the shape. When a
section gains a field, this file is where it is seen first.
"""
from __future__ import annotations
from typing import Any
SCAN_ID = "3f7a1c9e-8b42-4d61-9c05-27ae1f6b3d80"
def _fact(label: str, value: str, *, gap: bool = False, kind: str | None = None) -> dict[str, Any]:
"""One label/value row, optionally opening a gap above itself."""
return {"label": label, "value": value, "gap": gap, "kind": kind}
def _row(
label: str,
value: str,
*,
aside: bool = False,
kind: str | None = None,
token: str | None = None,
) -> dict[str, Any]:
"""One row of a card.
Args:
label: What the row is called.
value: What it says.
aside: Tints the row, for the one that argues rather than states.
kind: ``severity`` to colour it, ``mono`` for anything a reader might
have to type or compare character by character.
token: A VEX justification or similar, set as the token it is and placed
before the prose that explains it.
"""
return {"label": label, "value": value, "aside": aside, "kind": kind, "token": token}
def _mono(value: str) -> dict[str, Any]:
"""A grid cell set in the monospace face."""
return {"value": value, "kind": "mono"}
def _sev(value: str) -> dict[str, Any]:
"""A grid cell carrying a severity's colour."""
return {"value": value, "kind": "severity"}
def _bar(label: str, count: int, parts: list[tuple[str, float]], *, mono: bool = False) -> dict[str, Any]:
"""One distribution row: a label, its count, and the severity segments of its bar.
``mono`` sets the label in the monospace face, for the rows labelled by a package rather than by
a sentence.
"""
return {
"label": label,
"count": count,
"mono": mono,
"parts": [{"severity": severity, "share": share} for severity, share in parts],
}
def _cover() -> dict[str, Any]:
"""The first page: who it is about, and the one table that answers "how bad"."""
return {
"scan_id": SCAN_ID,
"subject": "SampleRepository v2.4.0",
"footer_date": "16 September 2026",
"logo": "cc_logo.png",
"title": "SAST/SCA Report",
"identity": [
_fact("Scan Identifier", SCAN_ID, kind="mono"),
_fact("Report ID", "b64d05f2-1ac7-4e9a-8f13-6d20c9b47e55", kind="mono"),
_fact("Artifact ID", "e1c8a730-5f96-42bb-b0d4-9a7351ec2f18", kind="mono"),
_fact("Technology", "SAST, SCA"),
_fact("Project", "SampleRepository", gap=True),
_fact("Version", "v2.4.0 (commit 9f3c2ab1d4e7)"),
_fact("Report Created", "Wednesday, September 16, 2026"),
_fact("Manufacturer", "Firma XY welche den Code besitzt"),
],
"severity_ledger": {
"title": "True issues by severity",
"rows": [
{"label": "Critical severity issues", "count": 3},
{"label": "High severity issues", "count": 73},
{"label": "Medium severity issues", "count": 28},
{"label": "Low severity issues", "count": 11},
{"label": "Total true issues", "count": 115, "total": True},
],
},
}
def _scan() -> dict[str, Any]:
"""What the pipeline did, stage by stage, and what each stage was configured with."""
return {
"title": "CipherCell Scan Information",
"overview": [
_fact("Scan started at", "Wednesday, September 16, 2026 05:43:14 (UTC)"),
_fact("Scan duration", "07:30:38"),
_fact("Scan completed at", "Wednesday, September 16, 2026 13:13:52 (UTC)"),
_fact(
"Pipeline Steps",
"Preanalysis, SAST, SBOM Dependency Tracking, SBOM Reachability, AI FP Detection, AI Ranking",
),
_fact("Errors Detected", "None (2 warnings, see SBOM Reachability)"),
],
"stages": [
{
"title": "Pre-Analysis Settings",
"rows": [
_fact("Selected Tools", "Propylon v0.9.4"),
_fact("Profile", "deterministic"),
_fact("Errors Detected", "None"),
],
},
{
"title": "SAST Settings",
"rows": [
_fact("Selected Tools", "ruff v0.14.4, bandit v1.8.6, cppcheck v2.18.0, semgrep v1.145.0"),
_fact("Language Coverage", "C (cppcheck, semgrep), Python (ruff, bandit, semgrep)"),
_fact("Rule Profile", "security — ruleset ciphercell-sec-2026.03, sha256 8f2b41c6…d97a"),
_fact("Deduplication Method", "Deterministic"),
_fact("Created Artifacts", "sast_combined.sarif", kind="mono"),
_fact("Errors Detected", "None"),
],
},
{
"title": "SBOM Dependency Tracking Settings",
"rows": [
_fact("Selected Tools", "syft v1.33.0, cdxgen v11.10.0, deptrace v0.7.2"),
_fact("Profile", "Standard SBOM"),
_fact("SBOM Format", "CycloneDX 1.6"),
_fact("Advisory Sources", "NVD, OSV.dev, GitHub Advisory Database, Sonatype OSS Index"),
_fact("Created Artifacts", "sbom.cdx.json, sbom.vdr.json", kind="mono"),
_fact("Errors Detected", "None"),
],
},
{
"title": "SBOM Reachability",
"rows": [
_fact("Selected Tools", "CodeQL v2.24.1, Joern v4.0.312"),
_fact("Profile", "Default Taint Analysis"),
_fact("Created Artifacts", "sbom_annotated.vdr.json", kind="mono"),
_fact("Errors Detected", "2 warnings"),
],
"table": {
"columns": ["ID", "Stage", "Warning"],
"rows": [
{
"cells": [
"W-01",
"Joern, JavaScript",
"2 of 148 entry points could not be resolved (dynamic require() in "
"server/plugins/loader.js:41). The 6 findings depending on these entry points "
"were escalated and are reported conservatively as true issues.",
]
},
{
"cells": [
"W-02",
"CodeQL, C",
"Database build skipped for vendored sources under third_party/; excluded by "
"scan scope, no reachability verdict is claimed for those paths.",
]
},
],
},
},
{
"title": "AI FP Detection",
"rows": [
_fact("Selected Tools", "Polygraph v1.6.2"),
_fact("Profile", "static and escalation agent"),
_fact("AI Model", "Qwen3-32B, digest sha256:4ab1…c70f"),
_fact("Inference", "On-premise, CipherCell inference cluster"),
_fact("Reasoning Effort", "Default (classification), Default (escalation)"),
_fact("Response Cache", "Enabled"),
_fact("Context Enrichment", "Static (50 lines before, 50 lines after)"),
_fact("Errors Detected", "None"),
],
},
{
"title": "Ranking",
"rows": [
_fact("Selected Tools", "Prism v0.4.1"),
_fact("Effort", "Standard"),
_fact("Context Size", "Minimal"),
_fact("Errors Detected", "None"),
],
},
],
}
def _project() -> dict[str, Any]:
"""What was scanned, and what it is written in."""
return {
"title": "Project Information",
"overview": [
_fact("Project", "SampleRepository"),
_fact("Version", "v2.4.0 (commit 9f3c2ab1d4e7)"),
_fact("Analysed lines of code", "184,920"),
],
"languages": {
"title": "Languages",
"rows": [
{"label": "C", "share": "40.2 %"},
{"label": "Python", "share": "39.6 %"},
{"label": "JavaScript", "share": "20.2 %"},
{"label": "Total", "share": "100.0 %", "total": True},
],
},
}
def _summary() -> dict[str, Any]:
"""The counts, and where they concentrate."""
return {
"title": "Summary",
"overview": [
_fact("Raw findings", "210"),
_fact("Filtered", "95 (82 SAST, 13 SCA)"),
_fact("True issues", "115 (86 SAST, 29 SCA)"),
_fact("Distinct issue types (true)", "8 SAST CWEs, 12 affected components"),
],
"distributions": [
{
"title": "True SAST issues by CWE (86)",
"rows": [
_bar("CWE-22: Improper Limitation of a Pathname to a Restricted Directory", 3, [("High", 6)]),
_bar(
"CWE-78: Improper Neutralization of Special Elements used in an OS Command",
2,
[("Critical", 4)],
),
_bar("CWE-327: Use of a Broken or Risky Cryptographic Algorithm", 3, [("Medium", 6)]),
_bar("CWE-674: Uncontrolled Recursion", 50, [("High", 84), ("Medium", 16)]),
_bar(
"CWE-770: Allocation of Resources Without Limits or Throttling",
10,
[("High", 14), ("Medium", 6)],
),
_bar("CWE-798: Use of Hard-coded Credentials", 4, [("High", 5), ("Medium", 3)]),
_bar("CWE-918: Server-Side Request Forgery", 2, [("High", 4)]),
_bar("CWE-1333: Inefficient Regular Expression Complexity", 12, [("Medium", 5), ("Low", 19)]),
],
"legend": {
"entries": [
{"severity": "Low", "label": "Low (9)"},
{"severity": "Medium", "label": "Medium (20)"},
{"severity": "High", "label": "High (55)"},
{"severity": "Critical", "label": "Critical (2)"},
],
"caveat": "bar length is proportional to the issue count (100 % = 50).",
},
},
{
"title": "Most affected components, true SCA issues (top 4 of 12)",
"rows": [
_bar("next@16.3.2", 4, [("Critical", 25), ("High", 50), ("Medium", 25)], mono=True),
_bar("glibc@2.36-9+deb12u7", 3, [("High", 75)], mono=True),
_bar("nanoid@3.3.14", 2, [("Medium", 50)], mono=True),
_bar("lodash@4.17.21", 2, [("High", 25), ("Medium", 25)], mono=True),
],
"legend": {
"entries": [
{"severity": "Low", "label": "Low"},
{"severity": "Medium", "label": "Medium"},
{"severity": "High", "label": "High"},
{"severity": "Critical", "label": "Critical"},
],
"caveat": "11 of 29 true SCA issues shown; 100 % = 4.",
},
},
],
}
def _critical() -> dict[str, Any]:
"""The handful worth reading first, whichever half of the pipeline found them."""
return {
"title": "Most Critical Vulnerabilities",
"table": {
"columns": ["Type", "Severity", "CVE/CWE", "Affected File / Component", "Description"],
"rows": [
{
"cells": [
"SAST",
_sev("Critical"),
"CWE-78",
_mono("src/tools/exporter.py:57"),
"User-controlled argument passed to subprocess.run(…, shell=True).",
]
},
{
"cells": [
"SCA",
_sev("Critical"),
"GHSA-2xp9-vwfh-vxw4",
_mono("next@16.3.2"),
"Heap buffer overflow in libheif, reached through sharp during AVIF image "
"optimization; allows unauthenticated remote code execution.",
]
},
{
"cells": [
"SAST",
_sev("High"),
"CWE-22",
_mono("src/storage/paths.py:114"),
"Non-literal path component passed to os.path.join().",
]
},
],
},
}
def _sast_section() -> dict[str, Any]:
"""The SAST findings: what was kept, and what was set aside and why."""
return {
"title": "SAST Vulnerabilities",
"groups": [
{
"title": "True Issues by CWEs",
"cards": [
{
"title": "CWE-22: Improper Limitation of a Pathname to a Restricted Directory (3 issues)",
"rows": [
_row("Severity", "High", kind="severity"),
_row(
"Description",
"A path component that is not a string literal is passed to a file system API. "
"Where the component originates from a request parameter, an attacker can "
"traverse outside the intended directory.",
),
],
},
{
"title": "CWE-22 #1 — sast-9f3c2ab-0001",
"rows": [
_row("Location", "src/storage/paths.py:114", kind="mono"),
_row(
"Description",
"Non-literal path component passed to os.path.join(). The value originates "
'from request.args["file"] and is not normalised before use.',
),
_row("Reported By", "bandit v1.8.6 (B108), semgrep v1.145.0"),
_row(
"Reachability",
"Reachable — taint path from entry point api/upload.py:31 (upload_handler), "
"4 hops (Joern)",
),
_row("Evidence", "Deterministic, no LLM escalation"),
],
},
],
},
{
"title": "Filtered Issues by CWEs",
"note": (
"The following findings were reported by the scanners but assessed as not exploitable in "
"this artifact. Each entry carries the filtering reason together with the VEX justification "
"that is written to sbom_annotated.vdr.json. Filtered issues are not counted in the "
"severity summary."
),
"cards": [
{
"title": "CWE-1333 #1 — sast-9f3c2ab-0113",
"rows": [
_row("Location", "server/lib/llm.js:68", kind="mono"),
_row(
"Description", "Backtracking regular expression in the log line parser parseLogLine()."
),
_row("Reported By", "semgrep v1.145.0"),
_row(
"Filtering Reason",
"no call path from any of the 148 resolved entry points to parseLogLine(); the "
"only caller is the debug CLI, which is excluded from the production build. "
"Deterministic (Joern), no LLM escalation. Decided 2026-09-16 by Polygraph "
"v1.6.2.",
aside=True,
token="code_not_reachable",
),
],
},
],
"table": {
"title": "Filtered SAST issues by CWE (82)",
"columns": ["CWE", "Filtered", "Dominant VEX justification"],
"rows": [
{"cells": ["CWE-674: Uncontrolled Recursion", "41", _mono("protected_by_mitigating_control")]},
{
"cells": [
"CWE-1333: Inefficient Regular Expression Complexity",
"18",
_mono("code_not_reachable"),
]
},
{
"cells": [
"CWE-327: Use of a Broken or Risky Cryptographic Algorithm",
"9",
"code_not_reachable",
]
},
{"cells": ["CWE-798: Use of Hard-coded Credentials", "8", _mono("code_not_present")]},
{"cells": ["Total", "82", ""], "total": True},
],
},
},
],
}
def _sca_section() -> dict[str, Any]:
"""The dependency findings, in the same two halves."""
return {
"title": "SBOM / SCA Vulnerabilities",
"new_page": True,
"groups": [
{
"title": "True Issues by CVEs",
"cards": [
{
"title": "GHSA-2xp9-vwfh-vxw4",
"rows": [
_row("Affected component", "next@16.3.2 (via sharp → libheif@1.23.1)", kind="mono"),
_row("CVE", "None assigned"),
_row("Severity", "Critical", kind="severity"),
_row(
"Description",
"A heap buffer overflow in the libheif library used by sharp, which Next.js uses "
"for image optimization, can lead to remote code execution when AVIF files are "
"optimized.",
),
_row("CVSS", "9.5 / 10 (CVSS v4.0) — CVSS:4.0/AV:N/AC:L/AT:P/PR:N/UI:N/VC:H/VI:H/VA:H"),
_row("Impact", "Confidentiality (High), Integrity (High), Availability (High)"),
_row("Fixed in", "next 16.3.3 / 15.5.24", kind="mono"),
_row(
"Reachability",
"Reachable — AVIF output is enabled in next.config.js:22 "
"(images.formats contains image/avif)",
),
_row("Known for", "22 days (advisory published 2026-08-25)"),
],
},
],
},
{
"title": "Filtered Issues by CVEs",
"note": (
"Vulnerabilities present in the SBOM whose vulnerable code path is not exploitable in this "
"artifact. The filtering reason is carried into sbom.vdr.json as a CycloneDX VEX statement."
),
"cards": [
{
"title": "CVE-2026-75604 (GHSA-p293-qw3h-jr36)",
"rows": [
_row("Affected component", "next@16.3.2", kind="mono"),
_row("Reported Severity", "Critical", kind="severity"),
_row(
"Description",
"Path traversal (CWE-22) in Next.js applications served from a Windows "
"filesystem, leading to unauthenticated remote code execution.",
),
_row("CVSS", "9.0 / 10 (CVSS v3.1) — CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:C/C:H/I:H/A:H"),
_row("Fixed in", "next 16.3.3 / 15.5.24", kind="mono"),
_row(
"Filtering Reason",
"the vulnerable code path only executes on a Windows filesystem. The deployment "
"target of this artifact is linux/amd64 (base image debian:12-slim, see "
"sbom.cdx.json). Deterministic, no LLM escalation.",
aside=True,
token="requires_environment",
),
_row("Known for", "22 days (advisory published 2026-08-25)"),
],
},
],
"table": {
"title": "Filtered SCA issues by justification (13)",
"columns": ["VEX justification", "Filtered", "Typical case in this artifact"],
"rows": [
{
"cells": [
_mono("code_not_reachable"),
"7",
"Development-scoped dependencies and unused packages",
]
},
{"cells": [_mono("requires_environment"), "3", "Windows-only or macOS-only code paths"]},
{
"cells": [
_mono("requires_configuration"),
"2",
"Feature flags not enabled in the shipped config",
]
},
{"cells": [_mono("code_not_present"), "1", "Vulnerable function removed by the tree shaker"]},
{"cells": ["Total", "13", ""], "total": True},
],
},
},
],
}
def sample_context() -> dict[str, Any]:
"""A complete context, covering every section the template can draw."""
return {
"cover": _cover(),
"scan": _scan(),
"project": _project(),
"summary": _summary(),
"critical": _critical(),
"sections": [_sast_section(), _sca_section()],
}
report_generator/sources.py
+137
View File
@@ -0,0 +1,137 @@
"""Collects everything a scan's report is built from.
The one place that knows where the bytes come from. Swapping the store -- persistent_storage today,
archon tomorrow -- is a change to :func:`collect` and to nothing else: what it returns is files by
name, and a renderer cannot tell which store handed them over.
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import TYPE_CHECKING, cast
from uuid import UUID
from cc_common.api import PipelineError, context
from persistent_storage.shared import StorageConfiguration
from persistent_storage.utils.db_utils import get_scan
from persistent_storage.utils.s3_utils import fetch_artifact_files
if TYPE_CHECKING:
from collections.abc import Sequence
from datetime import datetime
from mypy_boto3_s3 import S3Client
from persistent_storage.wrapper import Artifact
@dataclass(frozen=True)
class ReportSources:
"""One scan and every stored file belonging to it.
Attributes:
scan_id: The scan the report is about.
run_at: When it ran, which is what the report is dated by.
artifacts: The artifact rows, for the inventory the report lists.
files: The stored bytes by file name. Held in memory because a report is
built once and every section wants a different one of them; the
largest scan in the sample set comes to a few megabytes.
"""
scan_id: UUID
run_at: datetime
artifacts: Sequence[Artifact]
files: dict[str, bytes]
def first(self, *names: str) -> bytes | None:
"""Return the first of ``names`` the scan stored, or nothing.
Matched by suffix rather than equality: a stored name carries whatever the producing stage
called the file, and the same document travels under `meta.json` and `metadata.json`
depending on which stage wrote it.
"""
for name in names:
for stored, content in self.files.items():
if stored.endswith(name):
return content
return None
def _why(configuration: StorageConfiguration, artifacts: Sequence[Artifact], error: Exception) -> str:
"""Say which artifacts the store has a row for but no object.
The bucket reports a missing key without naming it, which on a scan of six artifacts leaves
whoever ran this with nothing to act on. Rows outliving their objects is the ordinary way this
fails -- a bucket restored from nothing, a lifecycle rule, a bucket that was never persistent --
so the answer worth giving is which files are gone, not that one is.
Args:
configuration: The store being read.
artifacts: The artifact rows the scan claims.
error: What the fetch raised.
Returns:
The original message, followed by the missing file names when they can be
established. Diagnosing must never raise on top of the failure it is
explaining, so anything going wrong here falls back to the message alone.
"""
try:
client = cast("S3Client", configuration.s3.s3_client)
missing = []
for artifact in artifacts:
try:
client.head_object(Bucket=configuration.s3.bucket, Key=artifact.s3_key)
except Exception: # noqa: BLE001, PERF203
missing.append(artifact.file_name or artifact.s3_key)
if not missing:
return str(error)
return (
f"the bucket holds no object for {len(missing)} of {len(artifacts)} artifact(s) -- "
f"{', '.join(missing)}. The rows outlived the files; re-upload the scan with "
f"'persistent_storage put'."
)
except Exception: # noqa: BLE001
return str(error)
def collect(configuration: StorageConfiguration, scan_id: UUID) -> ReportSources:
"""Read one scan and its files out of the store.
Args:
configuration: The store to read from.
scan_id: The scan to collect.
Returns:
The scan's rows and files.
Raises:
PipelineError: The scan is unknown, or its files cannot be fetched.
"""
context.logger.info(f"Collecting the artifacts of scan {scan_id}...")
try:
scan = get_scan(configuration.database, scan_id)
except Exception as error:
message = f"Cannot read scan {scan_id}: {error}"
raise PipelineError(message) from error
try:
contents, names = fetch_artifact_files(configuration, [one.artifact_id for one in scan.artifacts])
except Exception as error:
message = f"Cannot collect the artifacts of scan {scan_id}: {_why(configuration, scan.artifacts, error)}"
raise PipelineError(message) from error
context.logger.info(f"Collected {len(names)} file(s) ✔")
return ReportSources(
scan_id=scan_id,
run_at=scan.run_at,
artifacts=scan.artifacts,
files=dict(zip(names, contents, strict=True)),
)
report_generator/template/cc_logo.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 59 KiB

report_generator/template/fonts/OFL.txt
+93
View File
@@ -0,0 +1,93 @@
Copyright 2020 The Space Grotesk Project Authors (https://github.com/floriankarsten/space-grotesk)
This Font Software is licensed under the SIL Open Font License, Version 1.1.
This license is copied below, and is also available with a FAQ at:
http://scripts.sil.org/OFL
-----------------------------------------------------------
SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007
-----------------------------------------------------------
PREAMBLE
The goals of the Open Font License (OFL) are to stimulate worldwide
development of collaborative font projects, to support the font creation
efforts of academic and linguistic communities, and to provide a free and
open framework in which fonts may be shared and improved in partnership
with others.
The OFL allows the licensed fonts to be used, studied, modified and
redistributed freely as long as they are not sold by themselves. The
fonts, including any derivative works, can be bundled, embedded,
redistributed and/or sold with any software provided that any reserved
names are not used by derivative works. The fonts and derivatives,
however, cannot be released under any other type of license. The
requirement for fonts to remain under this license does not apply
to any document created using the fonts or their derivatives.
DEFINITIONS
"Font Software" refers to the set of files released by the Copyright
Holder(s) under this license and clearly marked as such. This may
include source files, build scripts and documentation.
"Reserved Font Name" refers to any names specified as such after the
copyright statement(s).
"Original Version" refers to the collection of Font Software components as
distributed by the Copyright Holder(s).
"Modified Version" refers to any derivative made by adding to, deleting,
or substituting -- in part or in whole -- any of the components of the
Original Version, by changing formats or by porting the Font Software to a
new environment.
"Author" refers to any designer, engineer, programmer, technical
writer or other person who contributed to the Font Software.
PERMISSION & CONDITIONS
Permission is hereby granted, free of charge, to any person obtaining
a copy of the Font Software, to use, study, copy, merge, embed, modify,
redistribute, and sell modified and unmodified copies of the Font
Software, subject to the following conditions:
1) Neither the Font Software nor any of its individual components,
in Original or Modified Versions, may be sold by itself.
2) Original or Modified Versions of the Font Software may be bundled,
redistributed and/or sold with any software, provided that each copy
contains the above copyright notice and this license. These can be
included either as stand-alone text files, human-readable headers or
in the appropriate machine-readable metadata fields within text or
binary files as long as those fields can be easily viewed by the user.
3) No Modified Version of the Font Software may use the Reserved Font
Name(s) unless explicit written permission is granted by the corresponding
Copyright Holder. This restriction only applies to the primary font name as
presented to the users.
4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font
Software shall not be used to promote, endorse or advertise any
Modified Version, except to acknowledge the contribution(s) of the
Copyright Holder(s) and the Author(s) or with their explicit written
permission.
5) The Font Software, modified or unmodified, in part or in whole,
must be distributed entirely under this license, and must not be
distributed under any other license. The requirement for fonts to
remain under this license does not apply to any document created
using the Font Software.
TERMINATION
This license becomes null and void if any of the above conditions are
not met.
DISCLAIMER
THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE
COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL
DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM
OTHER DEALINGS IN THE FONT SOFTWARE.
report_generator/template/fonts/SpaceGrotesk-Bold.ttf
BIN
View File
Binary file not shown.
report_generator/template/fonts/SpaceGrotesk-Medium.ttf
BIN
View File
Binary file not shown.
report_generator/template/fonts/SpaceGrotesk-Regular.ttf
BIN
View File
Binary file not shown.
report_generator/template/fonts/SpaceMono-Bold.ttf
BIN
View File
Binary file not shown.
report_generator/template/fonts/SpaceMono-Regular.ttf
BIN
View File
Binary file not shown.
report_generator/template/report.css
+422
View File
@@ -0,0 +1,422 @@
/* The print stylesheet of the SAST/SCA report.
*
* Follows the design sample in `sample_report_template/`. Everything is expressed here rather than
* in the template: the template places content, this decides what it looks like, and the two are
* changed for different reasons.
*
* Space Grotesk throughout, with a monospace face for anything a reader might have to type or
* compare character by character -- identifiers, paths, package URLs, CVSS vectors, VEX
* justifications. That distinction is the one typographic rule the whole report rests on.
*/
@font-face {
font-family: "Space Grotesk";
font-weight: 400;
src: url("fonts/SpaceGrotesk-Regular.ttf");
}
@font-face {
font-family: "Space Grotesk";
font-weight: 500;
src: url("fonts/SpaceGrotesk-Medium.ttf");
}
@font-face {
font-family: "Space Grotesk";
font-weight: 700;
src: url("fonts/SpaceGrotesk-Bold.ttf");
}
/* Shipped rather than asked of the host. `monospace` resolves to whatever a machine happens to
have -- DejaVu on one, Andale on another -- and a report whose identifiers change shape between
two machines is a report nobody can compare. Space Mono is the companion face to Space Grotesk. */
@font-face {
font-family: "Space Mono";
font-weight: 400;
src: url("fonts/SpaceMono-Regular.ttf");
}
@font-face {
font-family: "Space Mono";
font-weight: 700;
src: url("fonts/SpaceMono-Bold.ttf");
}
:root {
--ink: #1a1a1a;
--muted: #55585c;
--rule: #1a1a1a;
--border: #c9cbcd;
--header-fill: #e7e8ea;
--label-fill: #f4f5f6;
/* The filtering reason is the one row that argues rather than states, and it is tinted so that
it reads as an aside even when the page is skimmed. */
--aside-fill: #fdf5ea;
--critical: #d0021b;
--high: #e8710a;
--medium: #f0ad20;
--low: #1f9d55;
}
@page {
size: A4;
margin: 22mm 18mm 18mm 18mm;
@bottom-left {
content: string(scan-footer);
font-family: "Space Grotesk", sans-serif;
font-size: 6.5pt;
color: var(--muted);
}
@bottom-center {
content: counter(page);
font-family: "Space Grotesk", sans-serif;
font-size: 6.5pt;
color: var(--muted);
}
@bottom-right {
content: string(date-footer);
font-family: "Space Grotesk", sans-serif;
font-size: 6.5pt;
color: var(--muted);
}
}
html {
font-family: "Space Grotesk", sans-serif;
font-size: 8.5pt;
line-height: 1.45;
color: var(--ink);
}
body {
margin: 0;
}
code,
.mono {
font-family: "Space Mono", monospace;
font-size: 0.93em;
}
/* Carried into the running footer on every page. Set once, on the cover. */
.footer-scan {
string-set: scan-footer content();
}
.footer-date {
string-set: date-footer content();
}
.footer-scan,
.footer-date {
position: absolute;
visibility: hidden;
}
/* --- Cover ------------------------------------------------------------------------------- */
.cover-mark {
text-align: right;
margin-bottom: 42mm;
}
.cover-mark img {
height: 15mm;
}
.cover-title {
font-size: 19pt;
font-weight: 400;
margin: 0 0 9mm 0;
}
/* --- Headings ---------------------------------------------------------------------------- */
/* A section heading and the rule under it are one thing, so the rule belongs to the heading and
cannot be separated from it by a page break. */
h1 {
font-size: 12.5pt;
font-weight: 700;
margin: 12mm 0 4mm 0;
padding-bottom: 1.6mm;
border-bottom: 0.8pt solid var(--rule);
break-after: avoid;
}
h1:first-of-type {
margin-top: 0;
}
h2 {
font-size: 9pt;
font-weight: 700;
margin: 7mm 0 2.5mm 0;
break-after: avoid;
}
h3 {
font-size: 8.5pt;
font-weight: 700;
margin: 5mm 0 2mm 0;
break-after: avoid;
}
p {
margin: 0 0 2.5mm 0;
}
p.note {
color: var(--ink);
max-width: 155mm;
}
/* --- Definition rows --------------------------------------------------------------------- */
/* Label left, value right, on one grid so that every block of them lines up down the page even
when they are separated by headings. */
.facts {
width: 100%;
border-collapse: collapse;
margin-bottom: 3mm;
}
.facts td {
padding: 0.7mm 0;
vertical-align: top;
}
.facts td:first-child {
width: 46mm;
padding-right: 4mm;
}
.facts.wide td:first-child {
width: 62mm;
}
.facts tr.gap td {
padding-top: 4mm;
}
/* --- Ledger tables (severity counts, languages) ------------------------------------------- */
/* Ruled above and below with hairlines between, and a heavier line before the total: the shape a
reader already knows from an invoice, used for the same reason -- the last row is a sum. */
.ledger {
width: 62%;
border-collapse: collapse;
margin-bottom: 4mm;
}
.ledger th,
.ledger td {
padding: 1.4mm 1mm;
border-bottom: 0.4pt solid var(--border);
}
.ledger thead th {
border-bottom: 0.8pt solid var(--rule);
text-align: left;
font-weight: 700;
}
.ledger td:last-child {
text-align: right;
}
.ledger tr.total td {
border-top: 0.8pt solid var(--rule);
border-bottom: none;
font-weight: 700;
}
/* --- Grid tables -------------------------------------------------------------------------- */
.grid {
width: 100%;
border-collapse: collapse;
margin-bottom: 4mm;
/* A table that splits across a page keeps its header on the second half. */
break-inside: auto;
}
.grid th,
.grid td {
border: 0.4pt solid var(--border);
padding: 1.6mm 2mm;
vertical-align: top;
text-align: left;
}
.grid thead th {
background: var(--header-fill);
font-weight: 700;
}
.grid tr.total td {
font-weight: 700;
}
.grid tr,
.facts tr {
break-inside: avoid;
}
/* --- Cards -------------------------------------------------------------------------------- */
/* One finding, as a titled block of label/value rows. Kept on one page where it fits: a card split
down the middle loses the thing that makes it a card. */
.card {
width: 100%;
border-collapse: collapse;
margin-bottom: 3.5mm;
break-inside: avoid;
}
.card caption {
background: var(--header-fill);
border: 0.4pt solid var(--border);
padding: 1.6mm 2mm;
text-align: left;
font-weight: 700;
caption-side: top;
}
.card th,
.card td {
border: 0.4pt solid var(--border);
padding: 1.6mm 2mm;
vertical-align: top;
text-align: left;
}
.card th {
width: 34mm;
background: var(--label-fill);
font-weight: 400;
}
.card tr.aside th,
.card tr.aside td {
background: var(--aside-fill);
}
/* The justification itself, set as the token it is rather than as prose about a token. */
.token {
font-family: "Space Mono", monospace;
font-size: 0.93em;
background: #ffffff;
border: 0.4pt solid var(--border);
border-radius: 1mm;
padding: 0.3mm 1.2mm;
}
/* --- Severity ----------------------------------------------------------------------------- */
.sev {
font-weight: 700;
}
.sev-critical {
color: var(--critical);
}
.sev-high {
color: var(--high);
}
.sev-medium {
color: var(--medium);
}
.sev-low {
color: var(--low);
}
/* --- Distribution bars --------------------------------------------------------------------- */
/* Label, count, then one bar made of stacked severity segments. The bar is scaled against the
largest row rather than against the total, so the shape of the distribution is what is read and
the busiest row fills the width. */
.bars {
width: 100%;
border-collapse: collapse;
margin-bottom: 1.5mm;
}
.bars td {
border: 0.4pt solid var(--border);
padding: 1.2mm 2mm;
vertical-align: middle;
}
.bars td.count {
width: 12mm;
text-align: right;
}
.bars td.track {
width: 50%;
padding: 1.2mm 2mm;
}
.bar {
height: 3mm;
width: 100%;
/* The segments are inline-block, so the newlines between them in the template would be
rendered as spaces and push a full-width bar onto a second line. */
font-size: 0;
white-space: nowrap;
}
.bar span {
display: inline-block;
height: 3mm;
vertical-align: top;
}
.seg-critical {
background: var(--critical);
}
.seg-high {
background: var(--high);
}
.seg-medium {
background: var(--medium);
}
.seg-low {
background: var(--low);
}
/* A distribution and its legend travel together. Where the pair does not fit, it moves to the
next page whole rather than leaving a stranded row and an orphaned key behind. */
.block {
break-inside: avoid;
}
.legend {
font-size: 7.5pt;
color: var(--ink);
margin-bottom: 5mm;
}
.legend .key {
display: inline-block;
width: 2.4mm;
height: 2.4mm;
margin-right: 1.2mm;
}
.legend .item {
margin-right: 4mm;
}
.legend .caveat {
color: var(--muted);
}
report_generator/template/report.html.j2
+182
View File
@@ -0,0 +1,182 @@
{#
The SAST/SCA report, after the design sample in `sample_report_template/`.
This file places content and nothing else: every dimension, colour and rule lives in `report.css`.
Each section is guarded by the data it needs, so a report of a scan that ran half the pipeline is
the same document with fewer sections rather than a document full of empty tables.
The context this expects is described by `layout.CONTEXT`, and `sample.sample_context()` builds a
complete one.
#}
{# A value is drawn by what it is, not by where it sits: an identifier, a path or a CVSS vector is
set in the monospace face wherever it appears, and a severity carries its colour. #}
{% macro value(row) %}
{% if row.token %}<span class="token">{{ row.token }}</span> — {% endif %}
{% if row.kind == "severity" %}
<span class="sev sev-{{ row.value|lower }}">{{ row.value }}</span>
{% elif row.kind == "mono" %}
<span class="mono">{{ row.value }}</span>
{% else %}
{{ row.value }}
{% endif %}
{% endmacro %}
{% macro facts(rows, wide=false) %}
<table class="facts{% if wide %} wide{% endif %}">
{% for row in rows %}
<tr{% if row.gap %} class="gap"{% endif %}>
<td>{{ row.label }}:</td>
<td>{{ value(row) }}</td>
</tr>
{% endfor %}
</table>
{% endmacro %}
{% macro card(title, rows) %}
{# Wrapped, because break-inside on the table does not hold its caption: the title would stay on
one page while the rows it titles moved to the next. #}
<div class="block">
<table class="card">
<caption>{{ title }}</caption>
{% for row in rows %}
<tr{% if row.aside %} class="aside"{% endif %}>
<th>{{ row.label }}</th>
<td>{{ value(row) }}</td>
</tr>
{% endfor %}
</table>
</div>
{% endmacro %}
{# Scaled against the busiest row, not against the total: what is read off a row of these is which
one is the outlier, and a scale set by the sum flattens all of them. #}
{% macro distribution(rows, legend) %}
{# The bars and the legend that decodes them are one thing; a break between them leaves a reader
with coloured segments and no key. #}
<div class="block">
<table class="bars">
{% for row in rows %}
<tr>
<td{% if row.mono %} class="mono"{% endif %}>{{ row.label }}</td>
<td class="count">{{ row.count }}</td>
<td class="track">
<div class="bar">
{% for part in row.parts %}
<span class="seg-{{ part.severity|lower }}" style="width: {{ part.share }}%"></span>
{% endfor %}
</div>
</td>
</tr>
{% endfor %}
</table>
<div class="legend">
{% for item in legend.entries %}
<span class="item"><span class="key seg-{{ item.severity|lower }}"></span>{{ item.label }}</span>
{% endfor %}
{% if legend.caveat %}<span class="caveat">— {{ legend.caveat }}</span>{% endif %}
</div>
</div>
{% endmacro %}
{% macro grid(table) %}
<table class="grid">
<thead>
<tr>{% for column in table.columns %}<th>{{ column }}</th>{% endfor %}</tr>
</thead>
<tbody>
{% for row in table.rows %}
<tr{% if row.total %} class="total"{% endif %}>
{% for cell in row.cells %}
<td>{% if cell is mapping %}{{ value(cell) }}{% else %}{{ cell }}{% endif %}</td>
{% endfor %}
</tr>
{% endfor %}
</tbody>
</table>
{% endmacro %}
{# ------------------------------------------------------------------------------------------ #}
<span class="footer-scan">Scan: {{ cover.scan_id }}</span>
<span class="footer-date">{{ cover.footer_date }}</span>
<div class="cover-mark">
<img src="{{ cover.logo }}" alt="CipherCell" />
</div>
<div class="cover-title">{{ cover.title }}</div>
{{ facts(cover.identity) }}
{% if cover.severity_ledger %}
<h2>{{ cover.severity_ledger.title }}</h2>
<table class="ledger">
{% for row in cover.severity_ledger.rows %}
<tr{% if row.total %} class="total"{% endif %}>
<td>{{ row.label }}</td>
<td>{{ row.count }}</td>
</tr>
{% endfor %}
</table>
{% endif %}
{% if scan %}
<h1 style="break-before: page">{{ scan.title }}</h1>
{{ facts(scan.overview, wide=true) }}
{% for stage in scan.stages %}
<h3>{{ stage.title }}</h3>
{{ facts(stage.rows, wide=true) }}
{% if stage.table %}{{ grid(stage.table) }}{% endif %}
{% endfor %}
{% endif %}
{% if project %}
<h1 style="break-before: page">{{ project.title }}</h1>
{{ facts(project.overview, wide=true) }}
{% if project.languages %}
<h2>{{ project.languages.title }}</h2>
<table class="ledger">
{% for row in project.languages.rows %}
<tr{% if row.total %} class="total"{% endif %}>
<td>{{ row.label }}</td>
<td>{{ row.share }}</td>
</tr>
{% endfor %}
</table>
{% endif %}
{% endif %}
{% if summary %}
<h1>{{ summary.title }}</h1>
{{ facts(summary.overview, wide=true) }}
{% for block in summary.distributions %}
<h2>{{ block.title }}</h2>
{{ distribution(block.rows, block.legend) }}
{% endfor %}
{% endif %}
{% if critical %}
<h2 style="break-before: page">{{ critical.title }}</h2>
{{ grid(critical.table) }}
{% endif %}
{% for section in sections %}
<h1{% if section.new_page %} style="break-before: page"{% endif %}>{{ section.title }}</h1>
{% for group in section.groups %}
<h2>{{ group.title }}</h2>
{% if group.note %}<p class="note">{{ group.note }}</p>{% endif %}
{% for entry in group.cards %}
{{ card(entry.title, entry.rows) }}
{% endfor %}
{% if group.table %}
{% if group.table.title %}<h2>{{ group.table.title }}</h2>{% endif %}
{{ grid(group.table) }}
{% endif %}
{% endfor %}
{% endfor %}
report_generator/vendor/README.md
Vendored
+42
View File
@@ -0,0 +1,42 @@
# vendor
Copies of everything `report_generator` borrows from the monorepo, so the package can be lifted into
a repository of its own.
Nothing in here is edited. `sync.py` writes it, under the same import paths the monorepo uses:
```
python -m persistent_storage.report_generator.vendor.sync
```
Re-run it after changing anything it lists; the diff then shows what drifted. The copied packages
are removed whole first, so dropping an entry from the list actually drops it from the tree.
`__init__.py` decides whether these copies are used. It asks whether the real packages can be
imported — not a flag, not an environment variable — so the same code runs in the monorepo and
outside it, and the copies can never shadow the real packages while those are present.
## What is copied, and what for
| Copied | Used for |
| --- | --- |
| `cc_common` | The CLI framework, the logging context, `PipelineError`. |
| `persistent_storage/shared.py` | The storage configuration and its secrets. |
| `persistent_storage/wrapper.py` | The `Scan` and `Artifact` rows. |
| `persistent_storage/utils/db_utils.py` | Reading a scan and its artifacts. |
| `persistent_storage/utils/s3_utils.py` | Fetching the stored files. |
| `persistent_storage/utils/cvss_utils.py` | Reading a CVSS vector of any generation. |
| `persistent_storage/helper.py` | What `db_utils` prints its tables with. |
| `cipherverifier/core/services/sarif.py` | Summarising a SARIF report. |
| `cipherverifier/core/services/vdr.py` | Summarising a vulnerability report. |
The two summarisers live in the web application today and are written to be free of it — no Django,
no store. That is what makes copying them honest rather than a workaround: wherever this tool ends
up, they belong beside it rather than inside a web application it does not otherwise use.
Not copied: the report's own template, stylesheet and fonts, which are this package's own and live
in `template/`.
## Third-party dependencies
`pyproject.toml` lists them. WeasyPrint additionally needs the native Pango and cairo libraries.
report_generator/vendor/__init__.py
Vendored
+53
View File
@@ -0,0 +1,53 @@
"""Copies of what this tool borrows, and the switch that makes them the ones it uses.
In the monorepo the real packages are importable and nothing here is touched. In a repository of
its own they are not, and :func:`activate` puts these copies on the path instead.
The check is "can it be imported", not a flag or an environment variable, so the same code runs in
both places without being told which one it is in -- and so the copies can never shadow the real
packages while those are present.
"""
from __future__ import annotations
import importlib.util
import sys
from pathlib import Path
VENDOR_ROOT = Path(__file__).resolve().parent
# One module out of each copied package. If these resolve, the real thing is installed.
PROBES = ("cc_common", "persistent_storage.shared", "cipherverifier.core.services.sarif")
def activate() -> bool:
"""Put the vendored copies on the path where the real packages are missing.
Returns:
Whether the copies were used. ``False`` in the monorepo, where the real
packages are installed and are left to win.
"""
if all(_installed(name) for name in PROBES):
return False
if str(VENDOR_ROOT) not in sys.path:
sys.path.insert(0, str(VENDOR_ROOT))
return True
def _installed(name: str) -> bool:
"""Whether a module can be imported without importing it.
Args:
name: A dotted module name.
Returns:
Whether the import system can find it. Wrapped because ``find_spec``
raises rather than returning ``None`` when a parent package is itself
missing, which is exactly the case being tested for.
"""
try:
return importlib.util.find_spec(name) is not None
except (ImportError, ValueError):
return False
report_generator/vendor/cc_common/__init__.py
Vendored
+1
View File
@@ -0,0 +1 @@
"""Collection of CLI helper functions."""
report_generator/vendor/cc_common/api/__init__.py
Vendored
+11
View File
@@ -0,0 +1,11 @@
"""API for interacting with CipherCell tools.
Exports core exception classes and property protocols used throughout
the cc_common package.
"""
from .context import Context, clear_context, set_context
from .context import _context as context
from .pipeline_error import ExceptionSeverity, PipelineError
__all__ = ["Context", "context", "set_context", "clear_context", "ExceptionSeverity", "PipelineError"]
report_generator/vendor/cc_common/api/cli/__init__.py
Vendored
+5
View File
@@ -0,0 +1,5 @@
"""CLI utilities and command management for CipherCell tools."""
from .cli_manager import CLIManager
__all__ = ["CLIManager"]
report_generator/vendor/cc_common/api/cli/_utils.py
Vendored
+203
View File
@@ -0,0 +1,203 @@
"""Internal utilities for CLI argument flattening and config loading."""
import json
from annotationlib import Format
from inspect import isclass
from pathlib import Path
from types import UnionType
from typing import Any, ForwardRef, Union, get_args, get_origin
import yaml
from pydantic import BaseModel
from pydantic.fields import FieldInfo
from yaml.parser import ParserError as YAMLDecodeError
FLATTENING_SEPERATOR = "__CC__COMMON__SEPARATOR__"
def _unwrap_optional(annotation: Any) -> Any: # noqa: ANN401
"""Return the non-``None`` member of an ``Optional``/union annotation, else the annotation.
A ``SomeModel | None`` field still has to be recursed into like a plain nested model: Typer
cannot build a CLI option for a union type.
"""
if get_origin(annotation) in (Union, UnionType):
args = [arg for arg in get_args(annotation) if arg is not type(None)]
if len(args) == 1:
return args[0]
return annotation
def flatten_model_fields(
model: type[BaseModel],
parent_path: str = "",
owner_model: type[BaseModel] | None = None,
) -> list[tuple[str, FieldInfo, type[BaseModel] | None]]:
"""Flatten nested Pydantic model fields into dot-separated key paths.
Recursively processes a Pydantic model and produces a flat list of
(field_path, field_info, owner_model) tuples. Nested models are
expanded with a separator between levels.
Args:
model: The Pydantic model to flatten.
parent_path: Prefix for nested field paths.
owner_model: The top-level model that owns this field.
Returns:
List of tuples containing the flattened field path, field info,
and the owner model.
Raises:
ValueError: If the model contains forward references.
"""
result: list[tuple[str, FieldInfo, type[BaseModel] | None]] = []
for field_name, field_info in model.model_fields.items():
if isinstance(field_info.annotation, ForwardRef):
raise ValueError(
f"Class {model.__name__} contains a forward reference to "
f"{field_info.annotation.evaluate(format=Format.STRING)}. "
f"This is not supported. Restructure the model."
)
annotation = _unwrap_optional(field_info.annotation)
if isclass(annotation) and issubclass(annotation, BaseModel):
result.extend(flatten_model_fields(annotation, f"{parent_path}{field_name}{FLATTENING_SEPERATOR}", model))
else:
result.append((f"{parent_path}{field_name}", field_info, owner_model))
return result
def build_arg_structure(args: dict[str, Any]) -> dict[str, Any]:
"""Rebuild nested dicts from flattened key-value pairs.
Splits keys by the flattening separator and inserts values into
a nested dictionary structure.
Args:
args: Flattened key-value pairs with separator-containing keys.
Returns:
Nested dictionary matching the original model structure.
"""
result: dict[str, Any] = {}
for key in args:
key_parts = key.split(FLATTENING_SEPERATOR)
_insert_value(result, key_parts, args[key])
return result
def _insert_value(result: dict[str, Any], key_list: list[str], value: str) -> None:
"""Recursively insert a value into a nested dict using a key path.
Args:
result: The dictionary to insert into.
key_list: Split key path parts.
value: Value to insert at the leaf position.
"""
if len(key_list) == 1:
result[key_list[0]] = value
else:
if key_list[0] not in result:
result[key_list[0]] = {}
_insert_value(result[key_list[0]], key_list[1:], value)
def drop_none_leaves(data: dict[str, Any]) -> dict[str, Any]:
"""Recursively drop ``None`` values and the empty dicts they leave behind.
For a command registered with ``validate=False``, whose options all default to ``None``: only
the options someone actually typed may be merged as a config overlay, since a ``None`` would
blank out whatever an earlier layer set for that path.
"""
result: dict[str, Any] = {}
for key, value in data.items():
if isinstance(value, dict):
nested = drop_none_leaves(value)
if nested:
result[key] = nested
elif value is not None:
result[key] = value
return result
def flatten_config(config: dict[str, Any], parent_key: str = "") -> dict[str, Any]:
"""Recursively flatten a nested dict using the flattening separator.
Keys in nested dictionaries are joined with the separator to produce
a single-level dictionary.
Args:
config: The dictionary to flatten.
parent_key: Prefix for nested keys.
Returns:
Flattened single-level dictionary.
"""
items: list[tuple[str, Any]] = []
for key, value in config.items():
new_key = parent_key + FLATTENING_SEPERATOR + key if parent_key else key
if isinstance(value, dict):
items.extend(flatten_config(value, new_key).items())
else:
items.append((new_key, value))
return dict(items)
def load_config(path: Path) -> dict[str, Any] | None:
"""Load a configuration file (JSON or YAML) from disk.
Attempts to parse the file as JSON first, then YAML. Returns None
if neither parser succeeds or the file is invalid.
Args:
path: Path to the configuration file.
Returns:
Parsed configuration dict, or None if loading failed.
"""
if json_config := _load_json_config(path):
return json_config
if yaml_config := _load_yaml_config(path):
return yaml_config
return None
def _load_json_config(path: Path) -> dict[str, Any] | None:
"""Attempt to load a JSON configuration file.
Args:
path: Path to the JSON file.
Returns:
Parsed dict if valid JSON, otherwise None.
"""
with path.open("r") as file:
try:
config = json.load(file)
if isinstance(config, dict):
return config
except json.JSONDecodeError:
return None
return None
def _load_yaml_config(path: Path) -> dict[str, Any] | None:
"""Attempt to load a YAML configuration file.
Args:
path: Path to the YAML file.
Returns:
Parsed dict if valid YAML, otherwise None.
"""
with path.open("r") as file:
try:
config = yaml.safe_load(file)
if isinstance(config, dict):
return config
except YAMLDecodeError:
return None
return None
report_generator/vendor/cc_common/api/cli/cli_manager.py
Vendored
+216
View File
@@ -0,0 +1,216 @@
"""CLI command registration and execution using Typer."""
import inspect
import logging
from collections.abc import Callable, Sequence
from inspect import signature
from pathlib import Path
from typing import Any, TypeVar
import typer
import typer_config
from pydantic import BaseModel
from pydantic.fields import FieldInfo
from rich import print
from rich.console import Console
from rich.logging import RichHandler
from cc_common.api import Context, PipelineError, set_context
from ._utils import (
FLATTENING_SEPERATOR,
_unwrap_optional,
build_arg_structure,
drop_none_leaves,
flatten_config,
flatten_model_fields,
load_config,
)
def _result_callback(data: Any | None) -> None: # noqa: ANN401
"""Print the result data if it is not None."""
if data is not None:
if hasattr(data, "print_as_cc_common_result") and callable(data.print_as_cc_common_result):
data.print_as_cc_common_result()
else:
print(data)
class CLIManager:
"""Manages Typer-based CLI applications with automatic argument generation.
Wraps a Typer app and provides a decorator to register commands whose
arguments are derived from Pydantic models. Configuration files (JSON/YAML)
are automatically loaded and flattened to match nested model fields.
Args:
program_name: Name of the CLI program.
help: Help text displayed in the CLI help.
"""
def __init__(self, program_name: str, help: str = "") -> None:
"""Initialize the CLIManager with a Typer app.
Args:
program_name: Name of the CLI program.
help: Help text displayed in the CLI help.
"""
self._app = typer.Typer(
name=program_name,
help=help,
add_completion=True,
rich_markup_mode="rich",
no_args_is_help=True,
pretty_exceptions_show_locals=False,
result_callback=_result_callback,
)
self._logger = logging.getLogger("cc_common_cli")
self._logger.setLevel(logging.INFO)
self._logger.addHandler(RichHandler(show_time=False, show_level=False))
# _SOME_MODEL is a generic
_SOME_MODEL = TypeVar("_SOME_MODEL", bound=BaseModel)
def command(
self,
name: str,
model: type[_SOME_MODEL] | None = None,
help: str = "",
) -> Callable[[Callable[[_SOME_MODEL], Any]], None] | None:
"""Decorator to register a CLI command.
Args:
name: The command name.
model: Optional Pydantic model whose fields become CLI options.
help: Help text for the command.
Returns:
Decorator function that registers the command.
"""
def decorator(fn: Callable[[CLIManager._SOME_MODEL], Any] | Callable[[], Any]) -> None:
self.register_command(name, fn, model, help)
return decorator
def register_command(
self,
name: str,
callback: Callable[[_SOME_MODEL], Any] | Callable[[dict[str, Any]], Any] | Callable[[], Any],
model: type[_SOME_MODEL] | None = None,
help: str = "",
*,
validate: bool = True,
) -> None:
"""Register a callback as a CLI command with auto-generated arguments.
If a Pydantic model is provided, its fields (including nested ones)
are flattened into CLI ``--option`` flags. A ``--config`` flag is
added to load JSON/YAML configuration files.
Args:
name: The command name.
callback: The function to call when the command is invoked.
model: Optional Pydantic model whose fields become CLI options.
help: Help text for the command.
validate: When ``True`` (the default), every required field becomes a required CLI
option and the callback receives a validated ``model`` instance. When ``False``,
every option defaults to unset and the callback receives the raw ``dict`` of
whichever ones were typed -- for a command whose model is only complete once
merged with layers the CLI does not know about, such as ``orchestrator run``.
Raises:
ValueError: If the model contains a field named 'config'.
"""
params: list[inspect.Parameter] = []
if model is not None:
for field_name, field_info, _ in flatten_model_fields(model):
if field_name == "config":
raise ValueError(
f"You must not have a parameter named 'config' in "
f"your top level Properties Model '{model.__name__}'."
)
cli_name = field_name.replace(FLATTENING_SEPERATOR, "-")
# A bool gets a paired --x/--no-x declaration so it can be turned off from the CLI
# even when its model default is True -- a single "--x" name could only ever set it
# to True, which is why every boolean field used to have to default to False.
is_bool = _unwrap_optional(field_info.annotation) is bool
param_names = [f"--{cli_name}/--no-{cli_name}"] if is_bool else [f"--{cli_name}"]
field_description = field_info.description or field_info.title or ""
# Every field defaults to unset, the optional ones included: their defaults would
# otherwise override what another layer set for the same path once merged in.
default = None if not validate else _extract_default_value(field_info)
params.append(
inspect.Parameter(
field_name,
inspect.Parameter.POSITIONAL_OR_KEYWORD,
default=typer.Option(
default,
*param_names,
help=field_description,
),
annotation=field_info.annotation,
)
)
def cmd_wrapper(**kwargs: Any) -> Any: # noqa: ANN401
logger = self._logger.getChild(name)
context = Context(logger, True, Console())
set_context(context)
if model is not None:
unflattened_kwargs = build_arg_structure(kwargs)
if not validate:
return callback(drop_none_leaves(unflattened_kwargs)) # type: ignore[call-arg,arg-type]
instance = model.model_validate(unflattened_kwargs)
return callback(instance) # type: ignore[call-arg,arg-type]
else:
return callback() # type: ignore[call-arg]
cmd_wrapper.__signature__ = inspect.Signature(params) # type: ignore[attr-defined]
cmd_wrapper.__name__ = name
cmd_wrapper.__doc__ = help
def load_and_flatten_config(path: str) -> dict[str, Any]:
if path:
config = load_config(Path(path))
if config is None:
raise ValueError(f"Could not parse file: '{path}'")
return flatten_config(config)
return {}
load_config_callback = typer_config.conf_callback_factory(load_and_flatten_config)
wrapped = typer_config.use_config(load_config_callback)(cmd_wrapper)
self._app.command(name=name)(wrapped)
def run(self, args: Sequence[str] | list[str] | None = None) -> None:
"""Execute the CLI application.
Parses arguments and runs registered commands. PipelineErrors are
caught and printed in red.
Args:
args: Optional list of CLI arguments. Defaults to sys.argv.
"""
if args is not None:
args = list(args)
try:
self._app(args=args)
except PipelineError as e:
print(f"[red]{e}[/]")
def _extract_default_value(field_info: FieldInfo) -> Any: # noqa: ANN401
if field_info.is_required():
default = ...
elif field_info.default_factory is not None:
# TODO: low prio, if possible evaluate this right before the callback is called
default_factory_signature = signature(field_info.default_factory)
if len(default_factory_signature.parameters) != 0:
raise ValueError("Only default factories with zero arguments are supported.")
default = field_info.default_factory() # type: ignore[call-arg]
else:
default = field_info.default
return default
report_generator/vendor/cc_common/api/context.py
Vendored
+70
View File
@@ -0,0 +1,70 @@
"""Module handling the application wide context."""
import logging
import sys
import threading
from logging import Logger
from typing import Any
from rich.console import Console
class Context:
"""Context of the application, containing global settings, state, etc."""
def __init__(self, logger: Logger, is_cli: bool, console: Console) -> None:
"""Create a new Context Object."""
self.logger = logger
self.is_cli = is_cli
self.console = console
class _ContextWrapper(Context):
def __init__(self, context: Context) -> None:
super().__init__(None, None, None) # type: ignore[arg-type]
self._default_context = context
self._contexts_for_thread: dict[int, Context] = {}
def __getattribute__(self, attr: str) -> Any: # noqa: ANN401
if attr in ["_contexts_for_thread", "_default_context", "set_context", "clear_context"]:
return super().__getattribute__(attr)
thread_id = threading.current_thread().ident
if thread_id is not None and thread_id in self._contexts_for_thread:
context_to_use = self._contexts_for_thread[thread_id]
else:
context_to_use = self._default_context
return getattr(context_to_use, attr)
def set_context(self, context: Context) -> None:
"""Set the context for the current thread."""
thread_id = threading.current_thread().ident
if thread_id is not None:
self._contexts_for_thread[thread_id] = context
def clear_context(self) -> None:
"""Clear the custom set context for the current thread."""
thread_id = threading.current_thread().ident
if thread_id is not None and thread_id in self._contexts_for_thread:
del self._contexts_for_thread[thread_id]
_default_logger = logging.getLogger("cc_common_default")
_default_logger.setLevel(logging.INFO)
_default_logger.addHandler(logging.StreamHandler(sys.stdout))
_context: Context = _ContextWrapper(Context(logger=_default_logger, is_cli=True, console=Console()))
def set_context(context: Context) -> None:
"""Overwrite the application context."""
global _context
if not isinstance(_context, _ContextWrapper):
raise TypeError(f"_context must be a _ContextWrapper, got {type(_context)}")
_context.set_context(context)
def clear_context() -> None:
"""Clear the custom set application context."""
global _context
if not isinstance(_context, _ContextWrapper):
raise TypeError(f"_context must be a _ContextWrapper, got {type(_context)}")
_context.clear_context()
report_generator/vendor/cc_common/api/pipeline_error.py
Vendored
+70
View File
@@ -0,0 +1,70 @@
"""Custom exception classes for uniform error handling across CipherCell applications."""
from enum import Enum, auto
class ExceptionSeverity(Enum):
"""Severity levels for pipeline exceptions.
Determines whether a pipeline can continue execution after an error
or must stop.
"""
RECOVERABLE = auto() # Failure in a stage; pipeline can continue
FATAL = auto() # Critical failure; pipeline cannot continue
class PipelineError(Exception):
"""Base exception for components within the CipherCell pipeline.
Attributes:
message: A human-readable description of the error.
severity: The severity level of the exception.
tool: The name of the tool or component where the exception originated.
exit_code: Optional exit code associated with the failure.
"""
def __init__(
self,
message: str,
severity: ExceptionSeverity = ExceptionSeverity.FATAL,
tool: str | None = None,
exit_code: int | None = None,
) -> None:
"""Initialize a PipelineError instance.
Args:
message: A human-readable description of the error.
severity: The severity level of the exception.
tool: The name of the tool or component where the exception originated.
exit_code: Optional exit code associated with the failure.
"""
self._severity = severity
self._message = message
self._tool = tool
self._exit_code = exit_code
def can_continue(self) -> bool:
"""Check whether the pipeline can continue after this exception.
Returns:
True if the severity is RECOVERABLE, False otherwise.
"""
return self._severity == ExceptionSeverity.RECOVERABLE
def __str__(self) -> str:
"""Return a human-readable string representation of the exception.
Returns:
Formatted string with severity, tool, message, exit code, and
cause information.
"""
parts = [f"[{self._severity.name}]"]
if self._tool:
parts.append(f"[{self._tool}]")
parts.append(self._message)
if self._exit_code:
parts.append(f"(exit code: {self._exit_code})")
if self.__cause__:
parts.append(f"(caused by: {type(self.__cause__).__name__}: {self.__cause__})")
return " ".join(parts)
report_generator/vendor/cc_common/config.py
Vendored
+190
View File
@@ -0,0 +1,190 @@
"""Shared, model-agnostic config-file and environment-override primitives.
Works on plain nested ``dict``s only, so every layer of a pipeline configuration can be merged with
:func:`merge` before the caller validates the result once, at the end.
"""
from __future__ import annotations
import copy
import functools
import json
import os
from collections.abc import Mapping
from pathlib import Path
from types import UnionType
from typing import Any, Union, get_args, get_origin
import yaml
from dotenv import load_dotenv
from pydantic import BaseModel
from cc_common.api import PipelineError
CONFIG_PATH_ENV_VAR = "MONOCIPHER_CONFIG"
DEFAULT_CONFIG_FILENAME = "config.yaml"
def load_environment() -> None:
"""Load ``.env`` into the process environment.
Never overrides an already-set variable, so every entrypoint can call it in any order.
"""
load_dotenv()
def _default_config_path() -> Path:
"""Where the config file is read from: ``$MONOCIPHER_CONFIG``, else ``./config.yaml``."""
configured = os.environ.get(CONFIG_PATH_ENV_VAR)
return Path(configured) if configured else Path.cwd() / DEFAULT_CONFIG_FILENAME
@functools.cache
def _parse_config_file(target: Path, _mtime_ns: int, _size: int) -> dict[str, Any]:
"""Parse one config file, cached on its own stamp so an edit is still picked up.
Callers must not mutate the result -- :func:`load_config_file` hands out a deep copy.
"""
text = target.read_text()
try:
parsed: Any = json.loads(text)
except json.JSONDecodeError as json_error:
try:
parsed = yaml.safe_load(text)
except yaml.YAMLError as yaml_error:
raise PipelineError(
f"Could not parse config file '{target}' as JSON ({json_error}) or YAML ({yaml_error}).",
tool="config",
) from yaml_error
if not isinstance(parsed, dict):
raise PipelineError(f"Config file '{target}' must contain a mapping at its top level.", tool="config")
return parsed
#: Forget every parsed config file. For tests that write a config file per case.
reset_config_cache = _parse_config_file.cache_clear
def load_config_file(path: Path | None = None) -> dict[str, Any]:
"""Load the config file as plain data, or ``{}`` if it does not exist.
A file that exists but fails to parse raises, rather than falling back to the schema defaults.
Raises:
PipelineError: The file is neither valid JSON nor valid YAML, or is not a mapping.
"""
target = path or _default_config_path()
if not target.is_file():
return {}
stat = target.stat()
return copy.deepcopy(_parse_config_file(target.resolve(), stat.st_mtime_ns, stat.st_size))
def merge(base: Mapping[str, Any], overlay: Mapping[str, Any]) -> dict[str, Any]:
"""Recursively merge ``overlay`` onto ``base``, ``overlay`` winning. Neither is mutated."""
result = copy.deepcopy(dict(base))
for key, value in overlay.items():
existing = result.get(key)
if isinstance(existing, dict) and isinstance(value, Mapping):
result[key] = merge(existing, value)
else:
result[key] = copy.deepcopy(value)
return result
def set_path(data: dict[str, Any], dotted: str, value: Any) -> None: # noqa: ANN401
"""Set one dotted path (e.g. ``"polygraph.llm.model"``) in ``data``, creating levels as needed."""
parts = dotted.split(".")
target = data
for part in parts[:-1]:
target = target.setdefault(part, {})
target[parts[-1]] = value
def get_path(data: Mapping[str, Any], dotted: str) -> Any: # noqa: ANN401
"""Read one dotted path out of ``data``, or ``None`` if any part of it is missing."""
current: Any = data
for part in dotted.split("."):
if not isinstance(current, Mapping):
return None
current = current.get(part)
return current
def env_var_name(dotted: str) -> str:
"""The environment variable that overrides one dotted config path: dots to underscores, upper-cased.
The whole naming convention. No config entry has a hardcoded env var name anywhere else -- a
setting's variable is always derivable from its path alone.
"""
return dotted.replace(".", "_").upper()
def _unwrap_optional(annotation: Any) -> Any: # noqa: ANN401
"""Return the non-``None`` member of an ``Optional``/union annotation, else the annotation."""
if get_origin(annotation) in (Union, UnionType):
args = [arg for arg in get_args(annotation) if arg is not type(None)]
if len(args) == 1:
return args[0]
return annotation
@functools.cache
def overridable_paths(model_cls: type[BaseModel], prefix: str = "") -> tuple[str, ...]:
"""Every dotted leaf path of ``model_cls`` that can be set from the environment.
Descends into nested models; skips ``list``/``dict``/mapping-valued leaves, since a raw string
from the environment can never validate against one of those, so such a field must not claim an
env var name.
"""
paths: list[str] = []
for name, field in model_cls.model_fields.items():
path = f"{prefix}.{name}" if prefix else name
annotation = _unwrap_optional(field.annotation)
if isinstance(annotation, type) and issubclass(annotation, BaseModel):
paths.extend(overridable_paths(annotation, path))
elif get_origin(annotation) in (list, dict, Mapping):
continue
else:
paths.append(path)
return tuple(paths)
def env_overlay(model_cls: type[BaseModel], prefix: str = "") -> dict[str, Any]:
"""A :func:`merge`-able overlay of every setting of ``model_cls`` that is set in the environment.
Each of :func:`overridable_paths` is looked up under its derived :func:`env_var_name`; an unset
or blank variable contributes nothing, so a blank placeholder (as ``.env.sample`` ships) never
overrides with an empty string.
"""
overlay: dict[str, Any] = {}
for path in overridable_paths(model_cls, prefix):
value = os.environ.get(env_var_name(path))
if value:
set_path(overlay, path, value)
return overlay
def apply_env_defaults(instance: BaseModel, prefix: str = "", model_cls: type[BaseModel] | None = None) -> None:
"""Fill in every field of ``instance`` still unset from its derived environment variable.
For a standalone tool's own config model, once it is already validated: only ``None`` fields are
touched, so a value already supplied (by CLI flag or config file) is never overridden.
``model_cls`` defaults to ``type(instance)``; pass the pipeline-side model explicitly when
``instance`` is a CLI-only subclass with extra fields that have no place under ``prefix`` in
:class:`orchestrator.commands.run.OrchestratorConfig`.
"""
for path in overridable_paths(model_cls or type(instance), prefix):
relative = path[len(prefix) + 1 :] if prefix else path
target = instance
parts = relative.split(".")
for part in parts[:-1]:
target = getattr(target, part)
leaf = parts[-1]
if getattr(target, leaf) is None:
value = os.environ.get(env_var_name(path))
if value:
setattr(target, leaf, value)
report_generator/vendor/cc_common/defines/__init__.py
Vendored
+5
View File
@@ -0,0 +1,5 @@
"""Collection of CLI helper functions."""
from .general_defines import BuildType, CPUArchitecture, FileFormat, OperatingSystem, ProgrammingLanguage
__all__ = ["BuildType", "CPUArchitecture", "FileFormat", "OperatingSystem", "ProgrammingLanguage"]
report_generator/vendor/cc_common/defines/general_defines.py
Vendored
+49
View File
@@ -0,0 +1,49 @@
"""Collection of general definitions used across all CipherCell products."""
from enum import Enum, auto
class FileFormat(Enum):
"""Definitions of file formats supported by CipherCell products."""
SARIF = auto()
SPDX = auto()
CYCLONEDX = auto()
VEX = auto()
class ProgrammingLanguage(Enum):
"""Supported programming languages for code-related operations."""
PYTHON = "python"
C = "c"
CPP = "cpp"
JAVA = "java"
RUST = "rust"
JAVASCRIPT = "javascript"
TYPESCRIPT = "typescript"
UNKNOWN = "unknown"
class OperatingSystem(Enum):
"""Supported operating systems for code-related operations."""
LINUX = "linux"
MACOS = "macos"
WINDOWS = "windows"
class CPUArchitecture(Enum):
"""Supported CPU architectures."""
AMD64 = "amd64"
ARM64 = "arm64"
X86 = "386"
UNKNOWN = "unknown"
class BuildType(Enum):
"""Supported build types for code-related operations."""
NATIVE = "native"
DOCKER = "docker"
report_generator/vendor/cc_common/docker/__init__.py
Vendored
+26
View File
@@ -0,0 +1,26 @@
"""Collection of CLI helper functions."""
from .compose_utils import compose_down, compose_up
from .defs import DockerArchitecture
from .docker_utils import (
abort_container,
check_image_exist,
create_docker_image,
exec_command,
is_docker_running,
run_command,
run_command_with_input,
)
__all__ = [
"DockerArchitecture",
"abort_container",
"check_image_exist",
"compose_down",
"compose_up",
"create_docker_image",
"exec_command",
"is_docker_running",
"run_command",
"run_command_with_input",
]
report_generator/vendor/cc_common/docker/compose_utils.py
Vendored
+112
View File
@@ -0,0 +1,112 @@
"""Provide utilities for driving Docker Compose stacks.
Unlike the rest of this package, Compose is not reachable through the docker-py
SDK. It is a separate binary. These helpers are therefore implemented separately,
while keeping the same logging and error conventions as the SDK-based utilities next to them.
"""
from pathlib import Path
from shutil import which
from cc_common.api import PipelineError, context
from cc_common.processes import run_subprocess
def compose_up(compose_file: Path, project_name: str, *arguments: str) -> None:
"""Start the services of a Compose file.
Nothing is implied about how they start: pass ``"--detach"`` to return once
the containers exist, ``"--build"`` to rebuild images first, and so on. A
call without arguments runs in the foreground, exactly as Compose would.
Args:
compose_file: Path to the Compose file describing the stack.
project_name: Name of the Compose project to act on.
arguments: Additional options forwarded to ``compose up``.
Raises:
PipelineError: If Docker is unavailable, the Compose file does not
exist, or Compose exits with a non-zero status.
"""
run_compose(compose_file, project_name, "up", *arguments)
def compose_down(compose_file: Path, project_name: str, *arguments: str) -> None:
"""Stop the services of a Compose file and remove their containers.
Named volumes survive unless ``"--volumes"`` is passed, which is the one
argument here that destroys data — Compose's own default is to keep it, and
this function does not second-guess that either way.
Args:
compose_file: Path to the Compose file describing the stack.
project_name: Name of the Compose project to act on.
arguments: Additional options forwarded to ``compose down``.
Raises:
PipelineError: If Docker is unavailable, the Compose file does not
exist, or Compose exits with a non-zero status.
"""
run_compose(compose_file, project_name, "down", *arguments)
def run_compose(compose_file: Path, project_name: str, *arguments: str) -> None:
"""Run any Docker Compose subcommand against a Compose file.
Args:
compose_file: Path to the Compose file describing the stack.
project_name: Name of the Compose project to act on. Passing it
explicitly keeps containers and volumes stable no matter which
directory the caller is invoked from — Compose would otherwise
derive the name from the directory holding the Compose file.
arguments: The Compose subcommand and its options, e.g. ``"up",
"--detach"``.
Raises:
PipelineError: If Docker is unavailable, the Compose file does not
exist, or Compose exits with a non-zero status.
"""
if not compose_file.is_file():
raise PipelineError(f"Compose file not found at {compose_file}")
command = compose_command()
subcommand = " ".join(arguments)
context.logger.info(f"Running 'compose {subcommand}' on {compose_file} ...")
# Currently, no solution without using subprocesses available.
exit_code = run_subprocess(
[*command, "--project-name", project_name, "--file", str(compose_file), *arguments],
Path.cwd(),
stream_output=True,
on_stdout=context.logger.info,
on_stderr=context.logger.info,
fill_stream_threaded=True,
)
if exit_code != 0:
raise PipelineError(f"'compose {subcommand}' failed with exit code {exit_code}")
def compose_command() -> list[str]:
"""Determine how Docker Compose can be invoked on this machine.
Returns:
The executable and subcommand to prefix the Compose arguments with,
preferring the ``docker compose`` plugin over the standalone binary.
Raises:
PipelineError: If neither ``docker`` nor the legacy ``docker-compose``
executable is on the PATH.
"""
docker_path = which("docker")
if docker_path is not None:
return [docker_path, "compose"]
legacy_compose_path = which("docker-compose")
if legacy_compose_path is None:
raise PipelineError("Docker executable not found")
return [legacy_compose_path]
report_generator/vendor/cc_common/docker/defs.py
Vendored
+19
View File
@@ -0,0 +1,19 @@
"""Docker-specific architecture and platform definitions."""
from enum import Enum
from cc_common.defines import CPUArchitecture, OperatingSystem
CONTAINER_DEFAULT_ABORT_TIMEOUT_IN_S = 5
class DockerArchitecture(Enum):
"""Enumeration of supported Docker platforms combining OS and CPU architecture."""
LINUX_AMD64 = (OperatingSystem.LINUX, CPUArchitecture.AMD64)
LINUX_ARM64 = (OperatingSystem.LINUX, CPUArchitecture.ARM64)
@property
def arch_str(self) -> str:
"""Return the os/cpu string which can be passed to docker command as plattform."""
return f"{self.value[0].value}/{self.value[1].value}"
report_generator/vendor/cc_common/docker/docker_utils.py
Vendored
+266
View File
@@ -0,0 +1,266 @@
"""Provide utility functions for interacting with Docker via the Docker SDK.
Supports image management, container execution, and environment validation
through the official docker-py client with logging integration.
"""
import os
import shlex
import socket
from pathlib import Path
from typing import Any
import docker
import docker.errors
from docker import DockerClient
from cc_common.api import context
from .defs import CONTAINER_DEFAULT_ABORT_TIMEOUT_IN_S, DockerArchitecture
def _get_client() -> DockerClient:
"""Create and return a Docker client from the local environment.
Raises:
docker.errors.DockerException: If the Docker daemon is unreachable.
"""
return docker.from_env()
def _stop_container(container_id: str, timeout_in_s: int) -> None:
"""Stops a specific container by ID."""
client = _get_client()
try:
container = client.containers.get(container_id)
context.logger.info(f"Aborting and removing container: {container_id}")
container.stop(timeout=timeout_in_s)
container.remove(force=True)
except docker.errors.NotFound:
context.logger.debug(f"Container {container_id} already removed.")
except Exception as e:
context.logger.error(f"Failed to abort container {container_id}: {e}")
def is_docker_running() -> bool:
"""Check whether the Docker daemon is running."""
context.logger.info("Checking if Docker is running")
try:
_get_client().ping()
return True
except docker.errors.DockerException:
return False
def check_image_exist(image_name: str) -> bool:
"""Check whether a Docker image exists locally."""
context.logger.info(f"Checking if image '{image_name}' exists")
try:
_get_client().images.get(image_name)
return True
except docker.errors.ImageNotFound:
return False
def create_docker_image(
image_name: str,
project_folder: Path,
docker_architecture: DockerArchitecture | None = None,
build_dargs: dict[str, str] | None = None,
) -> None:
"""Build a Docker image from the specified project directory."""
context.logger.info(f"Creating docker image '{image_name}' from '{project_folder}'")
client = _get_client()
try:
for chunk in client.api.build(
path=str(project_folder),
tag=image_name,
buildargs=build_dargs,
decode=True, # Low-level API handles this correctly
rm=True,
platform=docker_architecture.arch_str if docker_architecture else None,
):
if not isinstance(chunk, dict):
continue
if "stream" in chunk:
message = str(chunk["stream"]).strip()
if message:
context.logger.debug(message)
if "error" in chunk:
error_data = chunk.get("errorDetail", {})
error_msg = error_data.get("message", chunk["error"])
raise docker.errors.BuildError(error_msg, iter([chunk]))
except docker.errors.BuildError as e:
context.logger.error(f"Docker build failed: {e}")
raise e
except Exception as e:
context.logger.error(f"An unexpected error occurred during build: {e}")
raise e
def _rewrite_path_in_cmd(cmd: list[str], old_path: str, new_path: str) -> list[str]:
"""Replace `old_path` with `new_path` in every argument that points at it."""
rewritten = []
for arg in cmd:
if arg == old_path:
rewritten.append(new_path)
elif arg.startswith(f"{old_path}/"):
rewritten.append(f"{new_path}{arg[len(old_path) :]}")
else:
rewritten.append(arg)
return rewritten
def _wrap_cmd_with_copy(cmd: list[str], mount_folders: list[tuple[Path, str]], target_folder: str) -> list[str]:
"""Wrap `cmd` so the mounted folders are copied into `target_folder` first.
The copy is created by the container's own user, so it is writable without any
privilege escalation, and the read-only mounts stay untouched. Arguments in `cmd`
that point into a mounted folder are rewritten to the copy. Requires `sh`, `mkdir`
and `cp` in the image.
"""
copy_cmds = []
for _, container_path in mount_folders:
target_path = f"{target_folder.rstrip('/')}/{Path(container_path).name}"
quoted_target = shlex.quote(target_path)
copy_cmds.append(f"mkdir -p {quoted_target} && cp -a {shlex.quote(container_path)}/. {quoted_target}/")
cmd = _rewrite_path_in_cmd(cmd, container_path, target_path)
script = " && ".join([*copy_cmds, f"exec {shlex.join(cmd)}"])
return ["sh", "-c", script]
def run_command(
image_name: str,
cmd: list[str],
mount_folders: list[tuple[Path, str]],
remove_after_execution: bool,
copy_mounts_to: str | None = None,
) -> str:
"""Run a command inside a Docker container with optional volume mounts.
Args:
image_name: Image to run the command in.
cmd: Command and arguments to execute.
mount_folders: Pairs of (host path, container path) to mount.
remove_after_execution: Whether the container is removed after it exits.
copy_mounts_to: Optional container folder (for example the user's home) the
mounted folders are copied into before the command runs. The mounts are
then read-only and `cmd` arguments pointing at them are rewritten to the
copy, so the command works on data owned by the container's own user.
"""
mount_mode = "ro" if copy_mounts_to else "rw"
volumes = {
str(local_path.resolve()): {"bind": container_path, "mode": mount_mode}
for local_path, container_path in mount_folders
}
context.logger.info(f"Running command {cmd} in image '{image_name}'")
run_cmd = _wrap_cmd_with_copy(cmd, mount_folders, copy_mounts_to) if copy_mounts_to else cmd
output = _get_client().containers.run(
image=image_name, command=run_cmd, volumes=volumes, remove=remove_after_execution, stdout=True, stderr=True
)
return output.decode("utf-8")
def run_command_with_input(
image_name: str, cmd: list[str], input_data: str, environment: dict[str, Any] | None = None
) -> str:
"""Run a command inside a Docker container, streaming a string directly to its stdin."""
context.logger.info(f"Running command {cmd} in image '{image_name}'")
client = _get_client()
container = client.containers.create(
image=image_name, command=cmd, stdin_open=True, tty=False, environment=environment
)
try:
container.start()
docker_sock = container.attach_socket(params={"stdin": 1, "stream": 1})
try:
fd = docker_sock.fileno() # type: ignore[union-attr]
clean_fd = os.dup(fd)
if hasattr(docker_sock, "_response"):
try: # noqa: SIM105
docker_sock._response.close()
except Exception as e:
context.logger.error(f"hasattr returns {e}")
docker_sock._response = None # drop the reference entirely
family = socket.AF_UNIX if fd < 100 else socket.AF_INET
clean_sock = socket.socket(family=family, fileno=clean_fd)
finally:
docker_sock.close()
try:
clean_sock.sendall(input_data.encode("utf-8"))
clean_sock.shutdown(socket.SHUT_WR)
finally:
clean_sock.close()
exit_result = container.wait()
output = container.logs(stdout=True, stderr=True).decode("utf-8")
if exit_result["StatusCode"] != 0:
raise RuntimeError(f"Container failed ({exit_result['StatusCode']}):\n{output}")
return output
finally:
container.remove(force=True)
def exec_command(container_id: str, cmd: list[str]) -> str:
"""Execute a command inside a running container.
Returns:
The stdout output of the command as a decoded string.
"""
context.logger.info(f"Executing command {cmd} in container '{container_id}'")
container = _get_client().containers.get(container_id)
exit_code, output = container.exec_run(cmd)
if exit_code != 0:
raise docker.errors.APIError(f"Command {cmd} failed in container '{container_id}' with exit code {exit_code}")
if isinstance(output, bytes):
return output.decode("utf-8")
return b"".join(output).decode("utf-8")
def abort_container(
container_id: str | None = None,
image_name: str | None = None,
timeout_in_s: int = CONTAINER_DEFAULT_ABORT_TIMEOUT_IN_S,
) -> None:
"""Forcefully stop and remove a specific container by ID."""
if not container_id and not image_name:
context.logger.error("Neither container nor image specified.")
return
client = _get_client()
if container_id:
_stop_container(container_id, timeout_in_s)
else:
running_containers = client.containers.list()
for container in running_containers:
if (
image_name
and container.id
and container.image
and container.image.tags
and any(image_name in tag for tag in container.image.tags)
):
_stop_container(container.id, timeout_in_s)
report_generator/vendor/cc_common/processes/__init__.py
Vendored
+5
View File
@@ -0,0 +1,5 @@
"""Collection of CLI helper functions."""
from .subprocess_utils import run_subprocess # nosec B404
__all__ = ["run_subprocess"]
report_generator/vendor/cc_common/processes/subprocess_utils.py
Vendored
+72
View File
@@ -0,0 +1,72 @@
"""Provide utilities for running and managing subprocess execution."""
import subprocess # nosec B404
import threading
from collections.abc import Callable
from pathlib import Path
from typing import IO
from cc_common.api import context
def run_subprocess(
command: list[str],
cwd: Path,
stream_output: bool = False,
on_stdout: Callable[[str], None] | None = None,
on_stderr: Callable[[str], None] | None = None,
fill_stream_threaded: bool = False,
env: dict[str, str] | None = None,
) -> int:
"""Execute a subprocess and return its exit code.
When stream_output=True stdout and stderr are either forwarded to the on_stdout and on_stderr output.
If these are not set, the results are logged using the logging singleton.
When stream_output=False the output of the subprocess is not printed.
Args:
command: Command and arguments to execute.
cwd: Working directory for the subprocess.
stream_output: If True, stream output line-by-line instead of capturing.
on_stdout: Callback for each stdout line; falls back to DEBUG logging.
on_stderr: Callback for each stderr line; falls back to DEBUG logging.
fill_stream_threaded: Triggers the stdout and stderr callback in two seperate threads to prevent blocking.
env: Environment variables for the subprocess; inherits current env if None.
Returns:
Exit code of the subprocess.
"""
try:
if stream_output:
process = subprocess.Popen( # nosec B603
command, cwd=cwd, text=True, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
def drain(pipe: IO[str] | None, callback: Callable[[str], None] | None) -> None:
if pipe:
for line in pipe:
if callback:
callback(line.rstrip())
else:
context.logger.debug(line.rstrip())
if fill_stream_threaded:
t_out = threading.Thread(target=drain, args=(process.stdout, on_stdout))
t_err = threading.Thread(target=drain, args=(process.stderr, on_stderr))
t_out.start()
t_err.start()
t_out.join()
t_err.join()
else:
drain(process.stdout, on_stdout)
drain(process.stderr, on_stderr)
process.wait()
return process.returncode
else:
result = subprocess.run(command, cwd=cwd, capture_output=True, text=True, check=True, env=env) # nosec B603
return result.returncode
except subprocess.CalledProcessError as e:
context.logger.warning(f"Command caused an exception the accepted return codes. Message:\n{e.stdout} {e}")
return e.returncode
report_generator/vendor/cc_common/py.typed
Vendored
View File
Whitespace-only changes.
report_generator/vendor/cc_common/utils/__init__.py
Vendored
+14
View File
@@ -0,0 +1,14 @@
"""Collection of CLI helper functions."""
from .progress_manager import linear_progress, spinner
from .rich_utils import print, rich_print_to_string
from .system_detection import get_cpu_architecture, get_operating_system
__all__ = [
"linear_progress",
"spinner",
"print",
"rich_print_to_string",
"get_cpu_architecture",
"get_operating_system",
]
report_generator/vendor/cc_common/utils/progress_manager.py
Vendored
+57
View File
@@ -0,0 +1,57 @@
"""Provide utilities for displaying progress bars and spinners in CLI applications."""
from collections.abc import Generator
from contextlib import contextmanager, nullcontext
from rich.progress import (
BarColumn,
MofNCompleteColumn,
Progress,
TaskID,
TextColumn,
TimeElapsedColumn,
TimeRemainingColumn,
)
from cc_common.api import context
def _create_progress() -> Progress:
"""Creates a configured Rich Progress instance used for linear processes."""
return Progress(
TextColumn("[progress.description]{task.description}"),
TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
BarColumn(),
MofNCompleteColumn(),
TextColumn("•"),
TimeElapsedColumn(),
TextColumn("•"),
TimeRemainingColumn(),
console=context.console,
)
@contextmanager
def spinner(message: str, show_progress: bool = True) -> Generator[None]:
"""Optional spinner context manager.
Args:
message: Text shown next to the spinner.
show_progress: If False, disables spinner output.
"""
ctx = context.console.status(message, spinner="dots") if show_progress else nullcontext()
with ctx:
yield
@contextmanager
def linear_progress(description: str, total: int) -> Generator[tuple[Progress, TaskID]]:
"""Linear progress bar context manager with a single task.
Args:
description: Task description shown in the progress bar.
total: Total number of steps for completion tracking.
"""
with _create_progress() as p:
task = p.add_task(description, total=total)
yield p, task
report_generator/vendor/cc_common/utils/rich_utils.py
Vendored
+32
View File
@@ -0,0 +1,32 @@
"""Utilities for interacting with rich."""
from typing import IO, Any
from rich import print as rich_print
from rich.console import Console
from rich.text import Text
from cc_common.api import context
def rich_print_to_string(*objects: Any, width: int = 150) -> str: # noqa: ANN401
"""Render rich output into a string."""
console = Console(width=width)
with console.capture() as capture:
console.print(objects)
return str(Text.from_ansi(capture.get()))
def print(
*objects: Any, # noqa: ANN401
sep: str = " ",
end: str = "\n",
file: IO[str] | None = None,
flush: bool = False,
) -> None:
"""Wrapper for the rich.print function.
This function will do nothing if the application is not running in cli mode.
"""
if context.is_cli:
rich_print(*objects, sep=sep, end=end, file=file, flush=flush)
report_generator/vendor/cc_common/utils/system_detection.py
Vendored
+41
View File
@@ -0,0 +1,41 @@
"""Collection of helper functions, e.g. to detect the operating system or architecture of a system."""
import platform
from cc_common.defines import CPUArchitecture, OperatingSystem
def get_operating_system() -> OperatingSystem:
"""Returns the OS of the current system.
:raises NotImplementedError: when the operating system is not supported.
"""
os_name = platform.system().lower()
match os_name:
case "darwin":
return OperatingSystem.MACOS
case "linux":
return OperatingSystem.LINUX
case "windows":
return OperatingSystem.WINDOWS
case _:
raise NotImplementedError(f"Unknown operating system {os_name}")
def get_cpu_architecture() -> CPUArchitecture:
"""Return the CPU architecture of the current system."""
machine = platform.machine().lower()
match machine:
case "x86_64" | "amd64":
return CPUArchitecture.AMD64
# 64-bit ARM (Apple M1/M2/M3, modern Raspberry Pis, etc.
case "arm64" | "aarch64":
return CPUArchitecture.ARM64
case "i386" | "i686" | "x86" | "x86_32":
return CPUArchitecture.X86
case _:
return CPUArchitecture.UNKNOWN
report_generator/vendor/cipherverifier/__init__.py
Vendored
+1
View File
@@ -0,0 +1 @@
"""Top-level package for the CipherVerifier Django project."""
report_generator/vendor/cipherverifier/core/__init__.py
Vendored
+1
View File
@@ -0,0 +1 @@
"""Core Django app for CipherVerifier."""
report_generator/vendor/cipherverifier/core/services/__init__.py
Vendored
+1
View File
@@ -0,0 +1 @@
"""Service layer for pipeline orchestration."""
report_generator/vendor/cipherverifier/core/services/sarif.py
Vendored
+677
View File
@@ -0,0 +1,677 @@
"""Summarising of SARIF reports for presentation in the UI.
The functions in here are deliberately free of any storage or Django dependency: they take an
already parsed SARIF document and turn it into a plain summary that a template can render.
"""
from __future__ import annotations
from collections import Counter
from dataclasses import dataclass, field
from typing import Any
TOP_LIST_SIZE = 8
"""How many entries of a top list are shown before the "See all" button has to be used."""
LEVEL_ORDER = ["error", "warning", "note", "none"]
# Kept in sync with the --scan-chart-color-* palette in tailwind/input.css so the charts on the
# scan page look like the ones on the dashboard.
LEVEL_COLORS = {
"error": "oklch(0.5 0.199 28.536)",
"warning": "oklch(0.747 0.155 103.227)",
"note": "oklch(0.451 0.251 265.29)",
"none": "oklch(0.708 0 0)",
}
# The verdicts polygraph can hand down, with the wording the UI uses for them.
POLYGRAPH_VERDICT_LABELS = {
"false_positive": "Likely false positive",
"true_positive": "Likely true positive",
"uncertain": "Uncertain",
}
@dataclass
class Region:
"""A span within a file. Carries no path of its own - the location around it holds that."""
start_line: int | None
end_line: int | None = None
start_column: int | None = None
end_column: int | None = None
@property
def display(self) -> str:
"""The span as ``3``, ``3-5`` or ``3:13-26``, empty when it does not name a line."""
if self.start_line is None:
return ""
# A span across lines reads as a line range; one within a single line adds its columns.
if self.end_line and self.end_line != self.start_line:
return f"{self.start_line}-{self.end_line}"
if self.start_column is None:
return str(self.start_line)
columns = str(self.start_column)
if self.end_column and self.end_column != self.start_column:
columns = f"{columns}-{self.end_column}"
return f"{self.start_line}:{columns}"
@dataclass
class Occurrence:
"""A single tool hit that was merged into a finding by the sarifcombiner."""
found_by: str
tool_identifier: str
message: str
start_line: int | None
end_line: int | None = None
start_column: int | None = None
end_column: int | None = None
related_positions: list[Region] = field(default_factory=list)
@property
def region(self) -> Region:
"""The span this tool reported the finding at."""
return Region(self.start_line, self.end_line, self.start_column, self.end_column)
@dataclass
class Location:
"""A source location a finding points at."""
file: str
start_line: int | None
end_line: int | None
start_column: int | None = None
end_column: int | None = None
@property
def region(self) -> Region:
"""The span within the file."""
return Region(self.start_line, self.end_line, self.start_column, self.end_column)
@property
def display(self) -> str:
"""The location as ``app/views.py:3:13-26``, degrading to just the path without a line."""
region = self.region.display
return f"{self.file}:{region}" if region else self.file
@dataclass
class Suppression:
"""A reason the finding was suppressed. Polygraph adds one for every false positive it finds."""
kind: str
justification: str
@dataclass
class Finding:
"""A single SARIF result, flattened into the fields the template needs."""
rule_id: str
rule_name: str
rule_description: str
level: str
message: str
location: Location | None
extra_locations: list[Location]
tools: list[str]
occurrences: list[Occurrence]
rank: int | None
polygraph_verdict: str | None = None
polygraph_reasoning: str | None = None
polygraph_certainty: str = ""
polygraph_priority: str = ""
polygraph_prompt_hash: str = ""
rule_full_description: str = ""
rule_help: str = ""
rule_help_uri: str = ""
rule_tags: list[str] = field(default_factory=list)
rule_precision: str = ""
rule_security_severity: str = ""
rule_default_level: str = ""
suppressions: list[Suppression] = field(default_factory=list)
@property
def tool_count(self) -> int:
"""Number of distinct tools that reported this finding."""
return len(self.tools)
@property
def downgraded(self) -> bool:
"""Whether this result is reported below the severity its rule defaults to.
Tools do this per hit, so the gap is worth showing: a rule that defaults to ``error`` and
lands as a ``note`` says the tool itself was unsure about this particular match. The
reverse (a hit reported *above* the rule default) is not flagged here.
"""
if self.rule_default_level not in LEVEL_ORDER:
return False
return LEVEL_ORDER.index(self.level) > LEVEL_ORDER.index(self.rule_default_level)
@property
def false_positive(self) -> bool:
"""Whether polygraph classified this finding as a likely false positive."""
return self.polygraph_verdict == "false_positive"
@property
def verdict_label(self) -> str:
"""The polygraph verdict in words, empty when the finding carries no known verdict."""
return POLYGRAPH_VERDICT_LABELS.get(self.polygraph_verdict or "", "")
@dataclass
class Stat:
"""A labelled count with its share of the largest count in the same list, in percent."""
label: str
sublabel: str
count: int
percent: int
@dataclass
class SarifReport:
"""Everything the scan page shows about one SARIF report."""
total: int
findings: list[Finding]
level_counts: list[Stat]
tool_counts: list[Stat]
top_rules: list[Stat]
top_files: list[Stat]
file_count: int
rule_count: int
multi_tool_count: int
false_positive_count: int
ranked: bool
chart_data: dict[str, Any] = field(default_factory=dict)
@property
def levels(self) -> dict[str, int]:
"""Finding count per severity level, including the levels without any finding."""
counts = dict.fromkeys(LEVEL_ORDER, 0)
for stat in self.level_counts:
counts[stat.label] = stat.count
return counts
@property
def top_list_size(self) -> int:
"""How many entries of ``top_rules`` and ``top_files`` the page shows without unfolding."""
return TOP_LIST_SIZE
def _rules_by_id(runs: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
"""Collect the rule definitions of every run, keyed by rule id."""
rules: dict[str, dict[str, Any]] = {}
for run in runs:
for rule in run.get("tool", {}).get("driver", {}).get("rules", []):
if isinstance(rule, dict) and "id" in rule:
rules.setdefault(rule["id"], rule)
return rules
def _driver_name(run: dict[str, Any]) -> str:
"""Return the display name of the tool that produced a run."""
driver = run.get("tool", {}).get("driver", {})
return str(driver.get("name") or driver.get("fullName") or "unknown")
def _level_of(result: dict[str, Any], rule: dict[str, Any]) -> str:
"""Return the severity of a result.
Some tools (semgrep for instance) omit ``level`` on the result and only declare it on the rule,
so the rule's default configuration is used as a fallback.
Args:
result: A SARIF result.
rule: The rule definition belonging to the result, or an empty dict.
Returns:
One of the levels in ``LEVEL_ORDER``.
"""
level = result.get("level") or rule.get("defaultConfiguration", {}).get("level")
return level if level in LEVEL_ORDER else "none"
def _region_of(region: dict[str, Any]) -> Region:
"""Read a SARIF region, or a sarifcombiner position - both spell the keys the same way."""
return Region(
start_line=region.get("startLine"),
end_line=region.get("endLine"),
start_column=region.get("startColumn"),
end_column=region.get("endColumn"),
)
def _locations_of(result: dict[str, Any]) -> list[Location]:
"""Extract all physical locations of a result."""
locations = []
for location in result.get("locations", []):
physical = location.get("physicalLocation", {})
uri = physical.get("artifactLocation", {}).get("uri")
if not uri:
continue
region = _region_of(physical.get("region", {}))
locations.append(
Location(
file=uri,
start_line=region.start_line,
end_line=region.end_line,
start_column=region.start_column,
end_column=region.end_column,
)
)
return locations
def _occurrences_of(result: dict[str, Any]) -> list[Occurrence]:
"""Extract the per-tool occurrences the sarifcombiner merged into a result."""
occurrences = []
for occurrence in result.get("properties", {}).get("sarifgrouper", {}).get("occurrences", []):
found_by = occurrence.get("foundBy")
if not found_by:
continue
position = _region_of(occurrence.get("position", {}) or {})
related = [_region_of(related) for related in occurrence.get("relatedPositions", []) or []]
occurrences.append(
Occurrence(
found_by=found_by,
tool_identifier=occurrence.get("toolIdentifier", ""),
message=occurrence.get("message", ""),
start_line=position.start_line,
end_line=position.end_line,
start_column=position.start_column,
end_column=position.end_column,
related_positions=[region for region in related if region.start_line is not None],
)
)
return occurrences
def _suppressions_of(result: dict[str, Any]) -> list[Suppression]:
"""Extract the suppressions of a result. Polygraph adds one for each false positive it finds."""
suppressions = []
for suppression in result.get("suppressions", []) or []:
if not isinstance(suppression, dict):
continue
suppressions.append(
Suppression(
kind=str(suppression.get("kind") or ""),
justification=str(suppression.get("justification") or ""),
)
)
return suppressions
def _rank_of(result: dict[str, Any]) -> int | None:
"""Return the prism rank of a result, if the report was ranked."""
rank = result.get("properties", {}).get("prism", {}).get("rank")
if isinstance(rank, bool) or rank is None:
return None
if isinstance(rank, int | float):
return int(rank)
if isinstance(rank, str) and rank.isdigit():
return int(rank)
return None
POLYGRAPH_VERDICTS = set(POLYGRAPH_VERDICT_LABELS)
EMPTY_POLYGRAPH: dict[str, Any] = {"verdict": None, "reasoning": None, "certainty": "", "priority": "", "hash": ""}
def _polygraph_of(result: dict[str, Any]) -> dict[str, Any]:
"""Return what polygraph concluded about a result, if it was classified.
Polygraph annotates each classified SARIF result with a ``polygraph`` property block. Findings
that were not classified (polygraph disabled, below the severity threshold, or errored) carry no
such block, so an empty verdict is returned and the finding reads as "not classified".
Args:
result: A SARIF result.
Returns:
A dict with the ``verdict`` (one of ``POLYGRAPH_VERDICTS`` or ``None``), the ``reasoning``,
and the ``certainty``, ``priority`` and prompt ``hash`` of the classification. Certainty and
priority are lowercased, so the templates can capitalise them like they do severities.
"""
polygraph = result.get("properties", {}).get("polygraph")
if not isinstance(polygraph, dict):
return dict(EMPTY_POLYGRAPH)
verdict = str(polygraph.get("verdict", "")).lower()
if verdict not in POLYGRAPH_VERDICTS:
# Tolerate an older shape that only carries the boolean flag.
verdict = "false_positive" if polygraph.get("isFalsePositive") else ""
reasoning = polygraph.get("reasoning")
return {
"verdict": verdict or None,
"reasoning": str(reasoning) if reasoning else None,
"certainty": str(polygraph.get("certainty") or "").lower(),
"priority": str(polygraph.get("priority") or "").lower(),
"hash": str(polygraph.get("promptHash") or ""),
}
def _rule_metadata(rule: dict[str, Any]) -> dict[str, Any]:
"""Pull everything a rule definition says about itself.
Which of these a rule carries is entirely up to the tool that declared it: the sarifcombiner
keeps descriptions and tags, while raw tool reports (rayfusion's, for instance) also come with
a ``helpUri`` and a precision. Anything absent comes back empty and the template leaves it out.
Args:
rule: A SARIF rule definition, or an empty dict when the rule was not declared.
Returns:
The rule's descriptions, help text and link, tags, precision, security severity and the
severity it defaults to.
"""
properties = rule.get("properties") or {}
tags = properties.get("tags")
security_severity = properties.get("security-severity", properties.get("securitySeverity"))
return {
"short_description": rule.get("shortDescription", {}).get("text") or "",
"full_description": rule.get("fullDescription", {}).get("text") or "",
"help": (rule.get("help") or {}).get("text") or "",
"help_uri": str(rule.get("helpUri") or ""),
"tags": [str(tag) for tag in tags] if isinstance(tags, list) else [],
"precision": str(properties.get("precision") or ""),
"security_severity": str(security_severity) if security_severity is not None else "",
"default_level": str(rule.get("defaultConfiguration", {}).get("level") or ""),
}
def _build_finding(result: dict[str, Any], run: dict[str, Any], rules: dict[str, dict[str, Any]]) -> Finding:
"""Flatten a single SARIF result into a ``Finding``.
Args:
result: The SARIF result to flatten.
run: The run the result belongs to, used as fallback for the tool name.
rules: All rule definitions of the report, keyed by rule id.
Returns:
The flattened finding.
"""
rule_id = str(result.get("ruleId", ""))
rule = rules.get(rule_id, {})
meta = _rule_metadata(rule)
occurrences = _occurrences_of(result)
locations = _locations_of(result)
polygraph = _polygraph_of(result)
# The full description repeats the short one when a tool declares only one of the two, so it is
# dropped rather than shown twice underneath it.
description = meta["short_description"] or meta["full_description"]
full_description = meta["full_description"] if meta["full_description"] != description else ""
return Finding(
rule_id=rule_id,
rule_name=rule.get("name", ""),
rule_description=description,
level=_level_of(result, rule),
message=result.get("message", {}).get("text", ""),
location=locations[0] if locations else None,
extra_locations=locations[1:],
tools=sorted({occurrence.found_by for occurrence in occurrences}) or [_driver_name(run)],
occurrences=occurrences,
rank=_rank_of(result),
polygraph_verdict=polygraph["verdict"],
polygraph_reasoning=polygraph["reasoning"],
polygraph_certainty=polygraph["certainty"],
polygraph_priority=polygraph["priority"],
polygraph_prompt_hash=polygraph["hash"],
rule_full_description=full_description,
rule_help=meta["help"] if meta["help"] not in (description, full_description) else "",
rule_help_uri=meta["help_uri"],
rule_tags=meta["tags"],
rule_precision=meta["precision"],
rule_security_severity=meta["security_severity"],
rule_default_level=meta["default_level"],
suppressions=_suppressions_of(result),
)
def _sort_findings(findings: list[Finding], ranked: bool) -> list[Finding]:
"""Sort findings, keeping the ones polygraph flagged as false positives at the bottom.
The likely false positives sink below every real, uncertain or unclassified finding; within
each of those two groups the usual order is kept - by rank when the report is ranked, by
severity and location otherwise.
"""
if ranked:
return sorted(findings, key=lambda f: (f.false_positive, f.rank is None, f.rank or 0))
return sorted(
findings,
key=lambda f: (
f.false_positive,
LEVEL_ORDER.index(f.level),
f.location.file if f.location else "",
f.location.start_line or 0 if f.location else 0,
),
)
def _to_stats(counts: list[tuple[str, int]], sublabels: dict[str, str] | None = None) -> list[Stat]:
"""Turn ``(label, count)`` pairs into ``Stat`` entries with a percentage of the largest count."""
if not counts:
return []
largest = max(count for _, count in counts)
return [
Stat(
label=label,
sublabel=(sublabels or {}).get(label, ""),
count=count,
percent=round(count / largest * 100) if largest else 0,
)
for label, count in counts
]
def _level_stats(findings: list[Finding]) -> list[Stat]:
"""Count findings per severity level, in severity order and without empty buckets."""
counter = Counter(finding.level for finding in findings)
return _to_stats([(level, counter[level]) for level in LEVEL_ORDER if counter[level]])
def _tool_stats(findings: list[Finding]) -> list[Stat]:
"""Count findings per reporting tool, most findings first."""
counter: Counter[str] = Counter()
for finding in findings:
counter.update(finding.tools)
return _to_stats(counter.most_common())
def _rule_stats(findings: list[Finding]) -> list[Stat]:
"""Rank every triggered rule by how often it was triggered, most triggered first.
The scan page renders the whole list: it shows the first ``TOP_LIST_SIZE`` entries and reveals
the rest behind a "See all" button.
"""
counter = Counter(finding.rule_id for finding in findings if finding.rule_id)
names = {finding.rule_id: finding.rule_name for finding in findings}
return _to_stats(counter.most_common(), sublabels=names)
def _file_stats(findings: list[Finding]) -> list[Stat]:
"""Rank every affected file by its number of findings, most findings first."""
counter = Counter(finding.location.file for finding in findings if finding.location)
return _to_stats(counter.most_common())
def _chart_data(level_counts: list[Stat], tool_counts: list[Stat]) -> dict[str, Any]:
"""Build the data blob the Chart.js charts on the scan page are fed with."""
return {
"severity": {
"labels": [stat.label.capitalize() for stat in level_counts],
"values": [stat.count for stat in level_counts],
"colors": [LEVEL_COLORS[stat.label] for stat in level_counts],
},
"tools": {
"labels": [stat.label for stat in tool_counts],
"values": [stat.count for stat in tool_counts],
},
}
def _build_report(findings: list[Finding], ranked: bool) -> SarifReport:
"""Sort a list of findings and derive every statistic the templates render from it."""
findings = _sort_findings(findings, ranked)
level_counts = _level_stats(findings)
tool_counts = _tool_stats(findings)
return SarifReport(
total=len(findings),
findings=findings,
level_counts=level_counts,
tool_counts=tool_counts,
top_rules=_rule_stats(findings),
top_files=_file_stats(findings),
file_count=len({finding.location.file for finding in findings if finding.location}),
rule_count=len({finding.rule_id for finding in findings if finding.rule_id}),
multi_tool_count=sum(1 for finding in findings if finding.tool_count > 1),
false_positive_count=sum(1 for finding in findings if finding.false_positive),
ranked=ranked,
chart_data=_chart_data(level_counts, tool_counts),
)
def summarize_sarif(sarif_data: dict[str, Any]) -> SarifReport:
"""Summarise a parsed SARIF document for rendering on the scan page.
All runs of the document are merged, so reports that keep one run per tool are summarised just
as well as the combined report the pipeline usually produces. Every finding of the report is
included; the scan page renders the complete list.
Args:
sarif_data: A parsed SARIF document.
Returns:
The summarised report.
"""
runs = [run for run in sarif_data.get("runs", []) if isinstance(run, dict)]
rules = _rules_by_id(runs)
findings = [_build_finding(result, run, rules) for run in runs for result in run.get("results", [])]
return _build_report(findings, ranked=any(finding.rank is not None for finding in findings))
@dataclass
class FindingsSummary:
"""The numbers of one or more SARIF reports, in a form that can be stored and added up.
The listing pages only show counts, never individual findings, so they work off this instead of
the full report: it is small enough to cache per scan and can be merged across repositories
without having the findings themselves at hand.
"""
total: int = 0
file_count: int = 0
multi_tool_count: int = 0
false_positive_count: int = 0
level_totals: dict[str, int] = field(default_factory=dict)
rule_totals: dict[str, int] = field(default_factory=dict)
rule_names: dict[str, str] = field(default_factory=dict)
@property
def levels(self) -> dict[str, int]:
"""Finding count per severity level, including the levels without any finding."""
counts = dict.fromkeys(LEVEL_ORDER, 0)
for level, count in self.level_totals.items():
if level in counts:
counts[level] = count
return counts
@property
def level_counts(self) -> list[Stat]:
"""Findings per severity level, in severity order and without empty buckets."""
levels = self.levels
return _to_stats([(level, levels[level]) for level in LEVEL_ORDER if levels[level]])
@property
def rule_count(self) -> int:
"""Number of distinct rules that were triggered."""
return len(self.rule_totals)
@property
def top_rules(self) -> list[Stat]:
"""The most frequently triggered rules."""
return _to_stats(Counter(self.rule_totals).most_common(TOP_LIST_SIZE), sublabels=self.rule_names)
@classmethod
def of(cls, report: SarifReport) -> FindingsSummary:
"""Reduce a full report to the numbers the listing pages need."""
return cls(
total=report.total,
file_count=report.file_count,
multi_tool_count=report.multi_tool_count,
false_positive_count=report.false_positive_count,
level_totals=report.levels,
rule_totals=dict(Counter(finding.rule_id for finding in report.findings if finding.rule_id)),
rule_names={
finding.rule_id: finding.rule_name
for finding in report.findings
if finding.rule_id and finding.rule_name
},
)
def to_dict(self) -> dict[str, Any]:
"""Serialise the summary so it can be stored as JSON."""
return {
"total": self.total,
"file_count": self.file_count,
"multi_tool_count": self.multi_tool_count,
"false_positive_count": self.false_positive_count,
"level_totals": self.level_totals,
"rule_totals": self.rule_totals,
"rule_names": self.rule_names,
}
@classmethod
def from_dict(cls, data: dict[str, Any]) -> FindingsSummary:
"""Rebuild a summary from stored JSON, tolerating entries that miss a key."""
return cls(
total=int(data.get("total", 0)),
file_count=int(data.get("file_count", 0)),
multi_tool_count=int(data.get("multi_tool_count", 0)),
false_positive_count=int(data.get("false_positive_count", 0)),
level_totals=dict(data.get("level_totals") or {}),
rule_totals=dict(data.get("rule_totals") or {}),
rule_names=dict(data.get("rule_names") or {}),
)
def merge_summaries(summaries: list[FindingsSummary]) -> FindingsSummary:
"""Add up the summaries of several scans.
Files are counted per scan, so the same path in two repositories counts twice - which is what
the listing pages want, as those are two different files.
Args:
summaries: The summaries to add up.
Returns:
One summary covering all of them.
"""
levels: Counter[str] = Counter()
rules: Counter[str] = Counter()
names: dict[str, str] = {}
for summary in summaries:
levels.update(summary.level_totals)
rules.update(summary.rule_totals)
names.update(summary.rule_names)
return FindingsSummary(
total=sum(summary.total for summary in summaries),
file_count=sum(summary.file_count for summary in summaries),
multi_tool_count=sum(summary.multi_tool_count for summary in summaries),
false_positive_count=sum(summary.false_positive_count for summary in summaries),
level_totals=dict(levels),
rule_totals=dict(rules),
rule_names=names,
)
report_generator/vendor/cipherverifier/core/services/vdr.py
Vendored
+813
View File
@@ -0,0 +1,813 @@
"""Summarising of vulnerability reports for presentation in the UI.
Two shapes end up in the ``vdr.json`` artifact of a scan: the Finding Packaging Format that
Dependency-Track's findings export produces, and a plain CycloneDX VDR document. Both are accepted
here and reduced to the same summary.
The functions in here are deliberately free of any storage or Django dependency: they take an
already parsed document and turn it into a plain summary that a template can render.
"""
from __future__ import annotations
import json
import re
from collections import Counter
from collections.abc import Mapping
from dataclasses import dataclass, field
from datetime import datetime
from typing import Any
from cipherverifier.core.services.sarif import TOP_LIST_SIZE, Stat
from persistent_storage.utils.cvss_utils import IMPACT_KEYS, describe, parse_vector
# CycloneDX severity values, most severe first. ``unknown`` covers reports that carry no rating.
SEVERITY_ORDER = ["critical", "high", "medium", "low", "info", "unknown"]
# Higher rank means more severe, so the highest rated severity of a vulnerability wins.
SEVERITY_RANK: dict[str, int] = {
"critical": 5,
"high": 4,
"medium": 3,
"low": 2,
"info": 1,
"unknown": 0,
}
# Kept as OKLCH so the scan page charts match the style of the SARIF charts.
SEVERITY_COLORS: dict[str, str] = {
"critical": "oklch(0.45 0.22 27)",
"high": "oklch(0.62 0.20 38)",
"medium": "oklch(0.75 0.15 80)",
"low": "oklch(0.65 0.12 250)",
"info": "oklch(0.708 0 0)",
"unknown": "oklch(0.708 0 0)",
}
# How CycloneDX names each CVSS generation in a rating's ``method``, by which one is newer.
CYCLONEDX_METHODS = {"cvssv2": 2, "cvssv3": 3, "cvssv31": 3, "cvssv4": 4}
@dataclass
class Component:
"""A dependency a vulnerability affects.
Attributes:
name: What the package is called.
version: The version this scan depends on, empty when the report named none.
latest_version: The newest version the reporting tool knew of. What makes
a finding answerable -- a newer version existing is not proof that it
fixes this advisory, only that there is somewhere to upgrade to.
"""
name: str
version: str
latest_version: str = ""
@property
def is_outdated(self) -> bool:
"""Whether the package has been released again since the version in use."""
return bool(self.latest_version) and self.latest_version != self.version
@dataclass
class VulnerabilityMetrics:
"""What one advisory's CVSS vector and scoring say, in the words a reader uses.
Attributes:
attack_vector: Where an attacker has to be.
attack_complexity: Whether anything outside their control has to line up.
privileges_required: What account they need. v2 states how often they log
in instead, which is the same question.
user_interaction: Who they have to fool.
confidentiality: What they get to read.
integrity: What they get to change.
availability: How far they can take the service down.
epss: The probability of exploitation in the next 30 days, 0 to 1, or
``None`` where nobody scored it -- which is not a probability of zero.
epss_percentile: Where that probability ranks among everything scored.
published_days_ago: How long the advisory has been public.
"""
attack_vector: str = ""
attack_complexity: str = ""
privileges_required: str = ""
user_interaction: str = ""
confidentiality: str = ""
integrity: str = ""
availability: str = ""
epss: float | None = None
epss_percentile: float | None = None
published_days_ago: int | None = None
@dataclass
class Vulnerability:
"""A single CycloneDX vulnerability, flattened into the fields the template needs."""
id: str
severity: str
description: str
cwes: list[str]
components: list[Component]
score: float | None
recommendation: str
url: str
metrics: VulnerabilityMetrics = field(default_factory=VulnerabilityMetrics)
cwe_names: dict[str, str] = field(default_factory=dict)
@property
def cwes_json(self) -> str:
"""The weakness classes as one JSON array of identifier and name.
One attribute rather than two parallel pipe-joined lists, which would have to be kept in
step by whoever writes the template and would silently mispair if they ever were not.
A class the report named without describing carries an empty name, and the panel shows the
identifier alone rather than inventing a description for it.
"""
return json.dumps(
[{"id": one, "name": self.cwe_names.get(one, "")} for one in self.cwes],
separators=(",", ":"),
)
@property
def fix_available(self) -> bool:
"""Whether any affected package has been released again since."""
return any(component.is_outdated for component in self.components)
@property
def metrics_json(self) -> str:
"""The metrics as one JSON object, for the row to carry into the panel.
One attribute rather than ten: the panel needs all of them or none, and a single blob
cannot end up half-updated by a template that forgot a field. Keyed the way the script
reads them, which is the same shape its sample values have -- so the panel does not know
or care which of the two it was handed.
"""
return json.dumps(
{
"score": self.score,
"band": self.severity.capitalize(),
"epss": self.metrics.epss,
"epssPercentile": self.metrics.epss_percentile,
"confidentiality": self.metrics.confidentiality,
"integrity": self.metrics.integrity,
"availability": self.metrics.availability,
"attackVector": self.metrics.attack_vector,
"attackComplexity": self.metrics.attack_complexity,
"privilegesRequired": self.metrics.privileges_required,
"userInteraction": self.metrics.user_interaction,
"publishedDaysAgo": self.metrics.published_days_ago,
"fixAvailable": self.fix_available,
},
separators=(",", ":"),
)
@property
def component_names(self) -> list[str]:
"""Display names of the affected components, name plus version when the version is known."""
return [
f"{component.name}@{component.version}" if component.version else component.name
for component in self.components
]
@dataclass
class VdrReport:
"""Everything the scan page shows about one CycloneDX VDR report."""
total: int
vulnerabilities: list[Vulnerability]
severity_counts: list[Stat]
top_components: list[Stat]
component_count: int
chart_data: dict[str, Any] = field(default_factory=dict)
@property
def critical_high_count(self) -> int:
"""Number of vulnerabilities rated critical or high."""
return sum(1 for vuln in self.vulnerabilities if vuln.severity in ("critical", "high"))
def _components_by_ref(data: dict[str, Any]) -> dict[str, Component]:
"""Map the bom-ref of every component to a ``Component``, for resolving ``affects``.
Args:
data: A parsed CycloneDX VDR document.
Returns:
The components keyed by their bom-ref.
"""
components: dict[str, Component] = {}
for component in data.get("components", []):
if not isinstance(component, dict):
continue
bom_ref = component.get("bom-ref") or component.get("purl")
if not bom_ref:
continue
components[bom_ref] = Component(
name=str(component.get("name") or bom_ref),
version=str(component.get("version") or ""),
)
return components
def _severity_of(vulnerability: dict[str, Any]) -> str:
"""Return the most severe rating of a vulnerability, ``unknown`` when none is given.
Args:
vulnerability: A single CycloneDX vulnerability.
Returns:
One of the severities in ``SEVERITY_ORDER``.
"""
best = "unknown"
for rating in vulnerability.get("ratings", []):
if not isinstance(rating, dict):
continue
severity = rating.get("severity")
if isinstance(severity, str) and severity in SEVERITY_RANK and SEVERITY_RANK[severity] > SEVERITY_RANK[best]:
best = severity
return best
def _score_of(vulnerability: dict[str, Any]) -> float | None:
"""Return the highest CVSS score carried by any rating of a vulnerability.
Args:
vulnerability: A single CycloneDX vulnerability.
Returns:
The highest score, or ``None`` when no rating has a numeric score.
"""
scores = [
float(rating["score"])
for rating in vulnerability.get("ratings", [])
if isinstance(rating, dict) and isinstance(rating.get("score"), int | float)
]
return max(scores) if scores else None
def _cwes_of(vulnerability: dict[str, Any]) -> list[str]:
"""Return the CWE ids of a vulnerability as ``CWE-<n>`` strings.
Args:
vulnerability: A single CycloneDX vulnerability.
Returns:
The CWE ids, normalised to the ``CWE-<n>`` form.
"""
cwes: list[str] = []
for value in vulnerability.get("cwes", []):
if isinstance(value, int):
cwes.append(f"CWE-{value}")
elif isinstance(value, str) and value:
cwes.append(value if value.startswith("CWE-") else f"CWE-{value}")
return cwes
def _components_of(vulnerability: dict[str, Any], components: dict[str, Component]) -> list[Component]:
"""Resolve the affected components of a vulnerability via their bom-ref.
Affects whose reference is unknown are still listed, using the reference itself as the name so
the vulnerability is never left without a target.
Args:
vulnerability: A single CycloneDX vulnerability.
components: All components of the document, keyed by bom-ref.
Returns:
The resolved components the vulnerability affects.
"""
resolved: list[Component] = []
for affect in vulnerability.get("affects", []):
ref = affect.get("ref") if isinstance(affect, dict) else None
if not ref:
continue
resolved.append(components.get(ref, Component(name=str(ref), version="")))
return resolved
def _url_of(vulnerability: dict[str, Any]) -> str:
"""Return a reference url for a vulnerability: its source url or the first advisory.
Args:
vulnerability: A single CycloneDX vulnerability.
Returns:
A url, or an empty string when the vulnerability carries none.
"""
source_url = vulnerability.get("source", {})
if isinstance(source_url, dict) and source_url.get("url"):
return str(source_url["url"])
for advisory in vulnerability.get("advisories", []):
url = advisory.get("url") if isinstance(advisory, dict) else None
if url:
return str(url)
return ""
def _build_vulnerability(vulnerability: dict[str, Any], components: dict[str, Component]) -> Vulnerability:
"""Flatten a single CycloneDX vulnerability into a ``Vulnerability``.
Args:
vulnerability: A single CycloneDX vulnerability.
components: All components of the document, keyed by bom-ref.
Returns:
The flattened vulnerability.
"""
return Vulnerability(
id=str(vulnerability.get("id") or "Unknown"),
severity=_severity_of(vulnerability),
description=str(vulnerability.get("description") or vulnerability.get("detail") or ""),
cwes=_cwes_of(vulnerability),
components=_components_of(vulnerability, components),
score=_score_of(vulnerability),
recommendation=str(vulnerability.get("recommendation") or ""),
url=_url_of(vulnerability),
metrics=_metrics_of(vulnerability, _vector_of(vulnerability)),
)
def _vector_of(vulnerability: dict[str, Any]) -> str | None:
"""Return the newest CVSS vector a CycloneDX vulnerability rates itself with.
Args:
vulnerability: A single CycloneDX vulnerability.
Returns:
The vector, or ``None`` when no rating carries one. CycloneDX names the
method rather than versioning the vector, so the ratings are ordered by
the method they name and the last generation wins.
"""
rated = [
(str(rating.get("method") or ""), str(rating["vector"]))
for rating in vulnerability.get("ratings", [])
if isinstance(rating, dict) and rating.get("vector")
]
if not rated:
return None
return max(rated, key=lambda pair: CYCLONEDX_METHODS.get(pair[0].lower(), 0))[1]
def _sort_vulnerabilities(vulnerabilities: list[Vulnerability]) -> list[Vulnerability]:
"""Sort vulnerabilities by severity, most severe first."""
return sorted(vulnerabilities, key=lambda vuln: -SEVERITY_RANK.get(vuln.severity, 0))
def _to_stats(counts: list[tuple[str, int]], sublabels: dict[str, str] | None = None) -> list[Stat]:
"""Turn ``(label, count)`` pairs into ``Stat`` entries with a percentage of the largest count.
Args:
counts: The ``(label, count)`` pairs to turn into stats.
sublabels: Optional extra labels keyed by the stat label.
Returns:
The stats, each carrying its share of the largest count in percent.
"""
if not counts:
return []
largest = max(count for _, count in counts)
return [
Stat(
label=label,
sublabel=(sublabels or {}).get(label, ""),
count=count,
percent=round(count / largest * 100) if largest else 0,
)
for label, count in counts
]
def _severity_stats(severity_totals: Mapping[str, int]) -> list[Stat]:
"""Turn vulnerability counts per severity into stats, in severity order and without empty buckets.
The percentage is the share of the total number of vulnerabilities, so the bars in the overview
show each severity's proportion rather than just its size relative to the largest bucket.
Args:
severity_totals: The number of vulnerabilities per severity.
Returns:
The per-severity stats.
"""
total = sum(severity_totals.values())
return [
Stat(
label=severity,
sublabel="",
count=severity_totals[severity],
percent=round(severity_totals[severity] / total * 100) if total else 0,
)
for severity in SEVERITY_ORDER
if severity_totals.get(severity)
]
def _component_stats(vulnerabilities: list[Vulnerability]) -> list[Stat]:
"""Count the components with the most vulnerabilities.
Args:
vulnerabilities: The vulnerabilities to count components from.
Returns:
The most frequently affected components.
"""
counter: Counter[str] = Counter()
for vuln in vulnerabilities:
counter.update(vuln.component_names)
return _to_stats(counter.most_common(TOP_LIST_SIZE))
def _chart_data(severity_counts: list[Stat], component_counts: list[Stat]) -> dict[str, Any]:
"""Build the data blob the Chart.js charts on the scan page are fed with.
Args:
severity_counts: The per-severity stats.
component_counts: The per-component stats.
Returns:
The chart data for the severity doughnut and the components bar chart.
"""
return {
"severity": {
"labels": [stat.label.capitalize() for stat in severity_counts],
"values": [stat.count for stat in severity_counts],
"colors": [SEVERITY_COLORS[stat.label] for stat in severity_counts],
},
"components": {
"labels": [stat.label for stat in component_counts],
"values": [stat.count for stat in component_counts],
},
}
def _build_report(vulnerabilities: list[Vulnerability]) -> VdrReport:
"""Sort a list of vulnerabilities and derive every statistic the templates render from it.
Args:
vulnerabilities: The vulnerabilities to summarise.
Returns:
The summarised report.
"""
vulnerabilities = _sort_vulnerabilities(vulnerabilities)
severity_counts = _severity_stats(Counter(vuln.severity for vuln in vulnerabilities))
component_counts = _component_stats(vulnerabilities)
return VdrReport(
total=len(vulnerabilities),
vulnerabilities=vulnerabilities,
severity_counts=severity_counts,
top_components=component_counts,
component_count=len({name for vuln in vulnerabilities for name in vuln.component_names}),
chart_data=_chart_data(severity_counts, component_counts),
)
# ── Dependency-Track findings export (Finding Packaging Format) ─────────────
# Dependency-Track spells its severities in upper case and uses ``UNASSIGNED`` for unrated findings.
FPF_SEVERITIES: dict[str, str] = {
"CRITICAL": "critical",
"HIGH": "high",
"MEDIUM": "medium",
"LOW": "low",
"INFO": "info",
"UNASSIGNED": "unknown",
}
# The ``references`` field is markdown, so the urls are pulled back out of the link targets.
_MARKDOWN_LINK = re.compile(r"\((https?://[^)\s]+)\)")
def _fpf_severity_of(vulnerability: dict[str, Any]) -> str:
"""Return the severity of a Dependency-Track finding, ``unknown`` when it carries none."""
severity = vulnerability.get("severity")
return FPF_SEVERITIES.get(severity.upper(), "unknown") if isinstance(severity, str) else "unknown"
def _fpf_score_of(vulnerability: dict[str, Any]) -> float | None:
"""Return the highest CVSS base score a Dependency-Track finding carries.
Args:
vulnerability: The ``vulnerability`` part of a single finding.
Returns:
The highest score of any CVSS version, or ``None`` when the finding carries none.
"""
scores = [
float(vulnerability[key])
for key in ("cvssV2BaseScore", "cvssV3BaseScore", "cvssV4Score")
if isinstance(vulnerability.get(key), int | float)
]
return max(scores) if scores else None
def _fpf_cwes_of(vulnerability: dict[str, Any]) -> list[str]:
"""Return the CWE ids of a Dependency-Track finding as ``CWE-<n>`` strings.
Newer reports carry a ``cwes`` list, older ones only a single ``cweId``.
Args:
vulnerability: The ``vulnerability`` part of a single finding.
Returns:
The CWE ids, normalised to the ``CWE-<n>`` form.
"""
cwes = [
f"CWE-{cwe['cweId']}"
for cwe in vulnerability.get("cwes", [])
if isinstance(cwe, dict) and cwe.get("cweId") is not None
]
if not cwes and vulnerability.get("cweId") is not None:
cwes.append(f"CWE-{vulnerability['cweId']}")
return cwes
def _fpf_cwe_names_of(vulnerability: dict[str, Any]) -> dict[str, str]:
"""Return what each weakness class of a Dependency-Track finding is called.
Args:
vulnerability: The ``vulnerability`` part of a single finding.
Returns:
The names by identifier. Older reports name only the single ``cweId``
they carry, under ``cweName``; newer ones name each entry of ``cwes``.
Both are read, and a report that names none simply yields nothing.
"""
names = {
f"CWE-{cwe['cweId']}": str(cwe["name"])
for cwe in vulnerability.get("cwes", [])
if isinstance(cwe, dict) and cwe.get("cweId") is not None and cwe.get("name")
}
if not names and vulnerability.get("cweId") is not None and vulnerability.get("cweName"):
names[f"CWE-{vulnerability['cweId']}"] = str(vulnerability["cweName"])
return names
def _fpf_url_of(vulnerability: dict[str, Any]) -> str:
"""Return the first reference url of a Dependency-Track finding.
Args:
vulnerability: The ``vulnerability`` part of a single finding.
Returns:
A url, or an empty string when the finding references none.
"""
references = vulnerability.get("references")
if not isinstance(references, str):
return ""
match = _MARKDOWN_LINK.search(references)
return match.group(1) if match else ""
def _fpf_component_of(finding: dict[str, Any]) -> Component | None:
"""Return the component a Dependency-Track finding was raised against."""
component = finding.get("component")
if not isinstance(component, dict) or not component.get("name"):
return None
return Component(
name=str(component["name"]),
version=str(component.get("version") or ""),
latest_version=str(component.get("latestVersion") or ""),
)
# The vectors a Dependency-Track export can state, newest generation first.
FPF_VECTORS = ("cvssV4Vector", "cvssV3Vector", "cvssV2Vector")
def _fpf_vector_of(vulnerability: dict[str, Any]) -> str | None:
"""Return the newest CVSS vector a Dependency-Track finding states, if any."""
for key in FPF_VECTORS:
if isinstance(vulnerability.get(key), str) and vulnerability[key]:
return str(vulnerability[key])
return None
def _metrics_of(vulnerability: dict[str, Any], vector: str | None) -> VulnerabilityMetrics:
"""Read one advisory's vector and scoring into the words the panel shows.
Args:
vulnerability: The advisory, in either document's shape.
vector: The CVSS vector to read, if the report stated one.
Returns:
The metrics. Everything the report did not state stays empty rather than
being filled with a default: a metric nobody stated is not one stated as
harmless, and the panel greys out what it was not told.
"""
reading = parse_vector(vector)
spoken = {}
if reading is not None:
confidentiality, integrity, availability = IMPACT_KEYS[reading.generation]
spoken = {
"attack_vector": _spoken("AV", reading.attack_vector, reading.generation),
"attack_complexity": _spoken("AC", reading.attack_complexity, reading.generation),
"privileges_required": _spoken(
"Au" if reading.generation == 2 else "PR",
reading.authentication if reading.generation == 2 else reading.privileges_required,
reading.generation,
),
"user_interaction": _spoken("UI", reading.user_interaction, reading.generation),
"confidentiality": _spoken(confidentiality, reading.confidentiality_impact, reading.generation),
"integrity": _spoken(integrity, reading.integrity_impact, reading.generation),
"availability": _spoken(availability, reading.availability_impact, reading.generation),
}
return VulnerabilityMetrics(
epss=_probability(vulnerability.get("epssScore")),
epss_percentile=_probability(vulnerability.get("epssPercentile")),
published_days_ago=_days_since(vulnerability.get("published")),
**spoken,
)
def _spoken(metric: str, value: str | None, generation: int) -> str:
"""Return the word behind one abbreviation, or nothing where the vector carried none."""
return "" if value is None else describe(metric, value, generation)
def _probability(stated: Any) -> float | None: # noqa: ANN401
"""Return a probability the report stated, refusing anything that is not one."""
return float(stated) if isinstance(stated, int | float) and not isinstance(stated, bool) else None
def _days_since(stated: Any) -> int | None: # noqa: ANN401
"""Return how many days ago a timestamp was, or nothing where it cannot be read."""
if not isinstance(stated, str):
return None
try:
published = datetime.fromisoformat(stated)
except ValueError:
return None
now = datetime.now(published.tzinfo) if published.tzinfo else datetime.now() # noqa: DTZ005
return max(0, (now - published).days)
def _is_suppressed(finding: dict[str, Any]) -> bool:
"""Tell whether a finding was suppressed in Dependency-Track."""
analysis = finding.get("analysis")
return isinstance(analysis, dict) and bool(analysis.get("isSuppressed"))
def _summarize_findings(findings: list[Any]) -> VdrReport:
"""Summarise a Dependency-Track findings export.
Every finding pairs one component with one vulnerability, so a vulnerability affecting several
dependencies arrives as several findings. They are merged back into a single vulnerability that
lists all of its components, which is what the scan page shows.
Args:
findings: The ``findings`` list of the export.
Returns:
The summarised report.
"""
merged: dict[str, Vulnerability] = {}
for finding in findings:
if not isinstance(finding, dict) or _is_suppressed(finding):
continue
vulnerability = finding.get("vulnerability")
if not isinstance(vulnerability, dict):
continue
key = str(vulnerability.get("uuid") or vulnerability.get("vulnId") or "")
component = _fpf_component_of(finding)
if key in merged:
if component is not None:
merged[key].components.append(component)
continue
merged[key] = Vulnerability(
id=str(vulnerability.get("vulnId") or "Unknown"),
severity=_fpf_severity_of(vulnerability),
description=str(vulnerability.get("description") or vulnerability.get("title") or ""),
cwes=_fpf_cwes_of(vulnerability),
components=[component] if component is not None else [],
score=_fpf_score_of(vulnerability),
recommendation=str(vulnerability.get("recommendation") or ""),
url=_fpf_url_of(vulnerability),
metrics=_metrics_of(vulnerability, _fpf_vector_of(vulnerability)),
cwe_names=_fpf_cwe_names_of(vulnerability),
)
return _build_report(list(merged.values()))
def summarize_vdr(vdr_data: dict[str, Any]) -> VdrReport:
"""Summarise a parsed vulnerability report for rendering on the scan page.
Both the Dependency-Track findings export and a CycloneDX VDR document are accepted; the
``findings`` key is what tells them apart. Every vulnerability of the document is included; the
scan page renders the complete list.
Args:
vdr_data: A parsed findings export or CycloneDX VDR document.
Returns:
The summarised report.
"""
findings = vdr_data.get("findings")
if isinstance(findings, list):
return _summarize_findings(findings)
components = _components_by_ref(vdr_data)
vulnerabilities = [
_build_vulnerability(vuln, components) for vuln in vdr_data.get("vulnerabilities", []) if isinstance(vuln, dict)
]
return _build_report(vulnerabilities)
@dataclass
class VulnerabilitySummary:
"""The numbers of one or more vulnerability reports, in a form that can be stored and added up.
The counterpart of ``FindingsSummary``: the listing pages only show counts, never individual
vulnerabilities, so they work off this instead of the full report.
"""
total: int = 0
component_count: int = 0
severity_totals: dict[str, int] = field(default_factory=dict)
@property
def severities(self) -> dict[str, int]:
"""Vulnerability count per severity, including the severities without any vulnerability."""
counts = dict.fromkeys(SEVERITY_ORDER, 0)
for severity, count in self.severity_totals.items():
if severity in counts:
counts[severity] = count
return counts
@property
def severity_counts(self) -> list[Stat]:
"""Vulnerabilities per severity, in severity order and without empty buckets."""
return _severity_stats(self.severities)
@property
def critical_high_count(self) -> int:
"""Number of vulnerabilities rated critical or high."""
severities = self.severities
return severities["critical"] + severities["high"]
@classmethod
def of(cls, report: VdrReport) -> VulnerabilitySummary:
"""Reduce a full report to the numbers the listing pages need."""
return cls(
total=report.total,
component_count=report.component_count,
severity_totals={stat.label: stat.count for stat in report.severity_counts},
)
def to_dict(self) -> dict[str, Any]:
"""Serialise the summary so it can be stored as JSON."""
return {
"total": self.total,
"component_count": self.component_count,
"severity_totals": self.severity_totals,
}
@classmethod
def from_dict(cls, data: dict[str, Any]) -> VulnerabilitySummary:
"""Rebuild a summary from stored JSON, tolerating entries that miss a key."""
return cls(
total=int(data.get("total", 0)),
component_count=int(data.get("component_count", 0)),
severity_totals=dict(data.get("severity_totals") or {}),
)
def merge_vulnerability_summaries(summaries: list[VulnerabilitySummary]) -> VulnerabilitySummary:
"""Add up the summaries of several scans.
Components are counted per scan, so the same dependency in two repositories counts twice - which
is what the listing pages want, as those are two different components to update.
Args:
summaries: The summaries to add up.
Returns:
One summary covering all of them.
"""
severities: Counter[str] = Counter()
for summary in summaries:
severities.update(summary.severity_totals)
return VulnerabilitySummary(
total=sum(summary.total for summary in summaries),
component_count=sum(summary.component_count for summary in summaries),
severity_totals=dict(severities),
)
report_generator/vendor/persistent_storage/__init__.py
Vendored
+1
View File
@@ -0,0 +1 @@
"""persistent_storage - Persistent Metadata and SBOM/SARIF file storage."""
report_generator/vendor/persistent_storage/helper.py
Vendored
+471
View File
@@ -0,0 +1,471 @@
"""Helper functions for persistent storage."""
import json
from pathlib import Path
from shutil import which
from typing import TYPE_CHECKING, Any
from rich.table import Table
from sqlalchemy import inspect
from cc_common.api import context
from cc_common.processes import run_subprocess
from cc_common.utils import print
from persistent_storage.wrapper import Artifact, Component, CvssMetric, DependencyFinding, Scan
if TYPE_CHECKING:
# pandas (~3s) is only needed by the DataFrame-building calls below; keep it off the import
# path (this module loads on every Django boot via orchestrator -> persistent_storage.commands)
# and import it lazily where it is actually used.
import pandas as pd
from pandas import DataFrame
type JsonData = str | int | float | bool | None | list["JsonData"] | dict[str, "JsonData"]
TOOL_NAME_KEYS = {
"tool_name",
"toolName",
"name",
"scanner",
"scanner_name",
"generator",
"generated_by",
"generatedBy",
"created_by",
"createdBy",
"producer",
"application",
}
TOOL_VERSION_KEYS = {
"tool_version",
"toolVersion",
"version",
"scanner_version",
"generator_version",
"generatedByVersion",
"createdByVersion",
}
TOOL_CONTAINER_KEYS = {
"tool",
"tools",
"scanner",
"generator",
"metadata",
"producer",
"createdBy",
"generatedBy",
}
def start_docker_containers() -> None:
"""Starts the docker containers for development.
Raises:
FileNotFoundError:
Raised if the docker-compose.yml file does not exist.
RuntimeError:
Raised if the docker executable cannot be found.
"""
compose_file = Path("src/persistent_storage/docker-compose.yml")
if not compose_file.exists():
context.logger.error(f"Compose file not found at {compose_file} ❌")
raise FileNotFoundError(f"Compose file not found at {compose_file}")
docker_path = which("docker")
if docker_path is not None:
command = [docker_path, "compose"]
else:
legacy_compose_path = which("docker-compose")
if legacy_compose_path is None:
context.logger.error("Docker executable not found ❌")
raise RuntimeError("Docker executable not found ❌")
command = [legacy_compose_path]
command += ["-f", str(compose_file), "up", "-d", "--build"]
run_subprocess(
command,
Path.cwd(),
)
def extract_tool_information_from_sarif(file_path: Path) -> tuple[str | None, str | None]:
"""Extract tool information from Sarif file."""
with file_path.open(encoding="utf-8") as file:
data = json.load(file)
runs = data.get("runs", [])
if not runs:
return "Unknown", "Unknown"
driver = runs[0].get("tool", {}).get("driver", {})
tool_name = driver.get("name")
tool_version = driver.get("version") or driver.get("semanticVersion") or driver.get("dottedQuadFileVersion")
return tool_name, tool_version
def extract_tool_information_from_cyclonedx(file_path: Path) -> tuple[str | None, str | None]:
"""Extract tool information from CyclonedX file."""
with file_path.open(encoding="utf-8") as file:
data = json.load(file)
tools = data.get("metadata", {}).get("tools")
if isinstance(tools, list) and tools:
tool = tools[0]
return tool.get("name"), tool.get("version")
if isinstance(tools, dict):
components = tools.get("components", [])
if components:
tool = components[0]
return tool.get("name"), tool.get("version")
return "Unknown", "Unknown"
def extract_tool_information_from_spdx(file_path: Path) -> tuple[str | None, str | None]:
"""Extract tool information from SPDX file."""
with file_path.open(encoding="utf-8") as file:
data = json.load(file)
creators = data.get("creationInfo", {}).get("creators", [])
for creator in creators:
if isinstance(creator, str) and creator.startswith("Tool:"):
value = creator.removeprefix("Tool:").strip()
if "-" in value:
name, version = value.rsplit("-", 1)
return name, version
return value, "Unknown"
return "Unknown", "Unknown"
def _first_string_value(data: dict[str, Any], keys: set[str]) -> str | None:
"""Return the first non-empty string value for any matching key."""
for key in keys:
value = data.get(key)
if isinstance(value, str) and value.strip():
return value.strip()
return None
def extract_tool_information_from_json(data: JsonData) -> tuple[str | None, str | None]:
"""Recursively search generic JSON data for tool name and version."""
if isinstance(data, dict):
direct_name = _first_string_value(data, TOOL_NAME_KEYS)
direct_version = _first_string_value(data, TOOL_VERSION_KEYS)
if direct_name or direct_version:
return direct_name, direct_version
for key in TOOL_CONTAINER_KEYS:
value = data.get(key)
tool_name, tool_version = extract_tool_information_from_json(value)
if tool_name or tool_version:
return tool_name, tool_version
for value in data.values():
tool_name, tool_version = extract_tool_information_from_json(value)
if tool_name or tool_version:
return tool_name, tool_version
elif isinstance(data, list):
for item in data:
tool_name, tool_version = extract_tool_information_from_json(item)
if tool_name or tool_version:
return tool_name, tool_version
return None, None
def print_rich_table(df: pd.DataFrame, table_type: str) -> None:
"""Print a pandas DataFrame as a rich table."""
str_df = df.fillna("").astype(str)
table = Table(
show_header=True,
header_style="bold",
)
if table_type == "scan":
_print_rich_table_scan(str_df, table)
elif table_type == "artifact":
_print_rich_table_artifact(str_df, table)
elif table_type in ("component", "dependency_finding", "cvss"):
_print_rich_table_plain(str_df, table, table_type)
else:
raise ValueError(f"Invalid table type: {table_type}")
print(table)
def _print_rich_table_scan(str_df: DataFrame, table: Table) -> None:
for column in str_df.columns:
if column == "meta_file_content":
table.add_column(
str(column),
no_wrap=False,
overflow="fold",
)
else:
table.add_column(
str(column),
no_wrap=True,
overflow="ignore",
)
for row in str_df.itertuples(index=False):
values = []
for col, value in zip(str_df.columns, row, strict=False):
text = value
if col == "meta_file_content":
text = text.replace("\\n", "\n")
values.append(text)
table.add_row(*values)
print("\n[bold]----------Stored scan metadata entries----------[/bold]")
def _print_rich_table_artifact(str_df: DataFrame, table: Table) -> None:
for column in str_df.columns:
if column in ("file_name", "created_at", "artifact_metadata"):
table.add_column(
str(column),
max_width=20,
overflow="fold",
)
else:
table.add_column(str(column))
for artifact_row in str_df.itertuples(
index=False,
name=None,
):
table.add_row(*artifact_row)
print("\n[bold]----------Stored artifacts metadata entry----------[/bold]")
# What each of the dependency tables is called above its rows.
PLAIN_TABLE_HEADINGS = {
"component": "Dependencies found by the scan",
"dependency_finding": "Advisories against those dependencies",
"cvss": "CVSS scorings of those advisories",
}
# Columns long enough to need folding rather than a column of their own width.
WIDE_COLUMNS = ("purl", "description", "vector", "metrics", "cwes")
def _print_rich_table_plain(str_df: DataFrame, table: Table, table_type: str) -> None:
for column in str_df.columns:
if column in WIDE_COLUMNS:
table.add_column(str(column), max_width=32, overflow="fold")
else:
table.add_column(str(column))
for row in str_df.itertuples(index=False, name=None):
table.add_row(*row)
print(f"\n[bold]----------{PLAIN_TABLE_HEADINGS[table_type]}----------[/bold]")
def print_artifact_table(artifacts: list[Artifact]) -> None:
"""Create a pandas DataFrame containing artifact information for each artifact and call a print function."""
import pandas as pd
columns = [column.key for column in Artifact.__mapper__.columns if column.key != "s3_key"]
df = pd.DataFrame(
[
{
column: getattr(
artifact,
column,
)
for column in columns
}
for artifact in artifacts
]
)
print_rich_table(df, table_type="artifact")
def print_scan_table(scans: list[Scan] | Scan) -> None:
"""Create a pandas DataFrame containing scan information for each scan and call a print function."""
import pandas as pd
# exclude meta_file_s3_key from output
columns = [column.key for column in Scan.__mapper__.columns if column.key != "meta_file_s3_key"]
if isinstance(scans, list):
df = pd.DataFrame([{column: getattr(scan, column) for column in columns} for scan in scans])
else:
df = pd.DataFrame([{column: getattr(scans, column) for column in columns}])
print_rich_table(df, table_type="scan")
if isinstance(scans, list):
for scan in scans:
print_artifact_table(scan.artifacts)
else:
print_artifact_table(scans.artifacts)
def find_values_in_file(file_content: bytes, search_key: str | tuple[str, str]) -> list[str] | str:
"""Searches a JSON or SARIF byte object recursively for specific keys.
Returns the values associated with the search key.
"""
data: JsonData = json.loads(file_content)
found, values = _find_values(data, search_key)
return values if found else "None"
def _find_values(data: JsonData, search_key: str | tuple[str, str]) -> tuple[bool, list[str]]:
"""Recursively search JSON data for matching values.
Args:
data: JSON data to search.
search_key: Either a single key or a ``(parent_key, key)`` pair.
Returns:
A tuple containing:
- ``True`` if at least one matching value was found, otherwise ``False``.
- A list of all matching values as strings.
"""
results = (
_find_values_by_tuple(data, search_key)
if isinstance(search_key, tuple)
else _find_values_by_key(data, search_key)
)
return bool(results), results
def _find_values_by_key(data: JsonData, search_key: str) -> list[str]:
"""Recursively collect values matching a single key."""
results: list[str] = []
def _search(node: JsonData) -> None:
if isinstance(node, dict):
value = node.get(search_key)
if isinstance(value, (str, int, float)):
results.append(str(value))
for child in node.values():
_search(child)
elif isinstance(node, list):
for item in node:
_search(item)
_search(data)
return results
def _find_values_by_tuple(data: JsonData, search_key: tuple[str, str]) -> list[str]:
"""Recursively collect values from nested parent/key pairs."""
results: list[str] = []
parent_key, key = search_key
def _search(node: JsonData) -> None:
if isinstance(node, dict):
parent = node.get(parent_key)
if parent is not None:
_append_tuple_matches(parent, key, results)
for child in node.values():
_search(child)
return
if isinstance(node, list):
for item in node:
_search(item)
_search(data)
return results
def _append_tuple_matches(parent: JsonData, key: str, results: list[str]) -> None:
"""Append matching values from a parent object or list of objects."""
if isinstance(parent, dict):
value = parent.get(key)
if isinstance(value, (str, int, float)):
results.append(str(value))
return
if isinstance(parent, list):
for item in parent:
if not isinstance(item, dict):
continue
value = item.get(key)
if isinstance(value, (str, int, float)):
results.append(str(value))
def print_component_table(components: list[Component]) -> None:
"""Create a pandas DataFrame of components and call a print function."""
_print_plain_table(components, Component, "component", skipped=("id",))
def print_dependency_finding_table(findings: list[DependencyFinding]) -> None:
"""Create a pandas DataFrame of dependency findings and call a print function."""
_print_plain_table(findings, DependencyFinding, "dependency_finding", skipped=("id",))
def print_cvss_table(scorings: list[CvssMetric]) -> None:
"""Create a pandas DataFrame of CVSS scorings and call a print function.
The columns a scoring leaves empty are dropped rather than printed as a row of blanks: a v3
vector has no ``authentication`` and a v2 one has no ``scope``, and showing every column of
every version at once makes the one in hand harder to read, not easier.
"""
columns = [column.key for column in inspect(CvssMetric).columns if column.key not in ("id", "cvss_id")]
columns = [column for column in columns if any(getattr(scoring, column) is not None for scoring in scorings)]
_print_dataframe(scorings, columns, "cvss")
def _print_plain_table(rows: list[Any], model: type[Any], table_type: str, skipped: tuple[str, ...]) -> None:
"""Render every column of a model except the ones a reader has no use for."""
columns = [column.key for column in inspect(model).columns if column.key not in skipped]
_print_dataframe(rows, columns, table_type)
def _print_dataframe(rows: list[Any], columns: list[str], table_type: str) -> None:
"""Build the frame and hand it to the renderer."""
import pandas as pd
df = pd.DataFrame([{column: getattr(row, column) for column in columns} for row in rows])
print_rich_table(df, table_type=table_type)
report_generator/vendor/persistent_storage/shared.py
Vendored
+101
View File
@@ -0,0 +1,101 @@
"""Shared configuration Classes."""
from typing import TYPE_CHECKING
from pydantic import BaseModel, Field
from cc_common.api import PipelineError
from cc_common.config import apply_env_defaults, load_environment
if TYPE_CHECKING:
from botocore.client import BaseClient
def __getattr__(name: str) -> object:
"""Lazily resolve ``boto3`` as a module attribute (PEP 562).
boto3 costs ~2.4s to import and is only needed once an S3 client is actually built
(:attr:`S3Configuration.s3_client`). Resolving it here, rather than importing it inline in the
property, keeps ``persistent_storage.shared.boto3`` patchable the same way it always was
(``monkeypatch.setattr("persistent_storage.shared.boto3.client", ...)``) while every other
consumer of this module avoids the cost.
"""
if name == "boto3":
import boto3
return boto3
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
class DatabaseConfiguration(BaseModel):
"""Configuration for the SQL database."""
host: str = Field(description="")
port: int = Field(description="")
user: str = Field(description="")
name: str = Field(description="")
password: str | None = Field(
default=None,
description="Database password.",
exclude=True,
)
@property
def db_url(self) -> str:
"""Construct the database url that should be used to connect to the database."""
if self.password is None:
raise PipelineError("Database password is not set.")
return f"postgresql+psycopg2://{self.user}:{self.password}@{self.host}:{self.port}/{self.name}"
class S3Configuration(BaseModel):
"""Configuration for the S3 database."""
url: str = Field(description="")
bucket: str = Field(description="")
access_key_id: str = Field(description="")
region: str = Field(description="")
secret_access_key: str | None = Field(
default=None,
description="S3 secret access key.",
exclude=True,
)
@property
def s3_client(self) -> BaseClient:
"""Construct a S3 client that should be used to connect to the database."""
import boto3
from botocore.config import Config
if self.secret_access_key is None:
raise PipelineError("S3 secret access key is not set.")
return boto3.client(
"s3",
endpoint_url=self.url,
aws_access_key_id=self.access_key_id,
aws_secret_access_key=self.secret_access_key,
region_name=self.region,
# we need when_required, because garage does not support checksums for multipart files (see https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/787)
config=Config(request_checksum_calculation="when_required", response_checksum_validation="when_required"),
)
def __hash__(self) -> int:
"""Basic hash implementation on self."""
return hash(repr(self))
class StorageConfiguration(BaseModel):
"""Container for both database configurations."""
database: DatabaseConfiguration
s3: S3Configuration
def inject_secrets(config: StorageConfiguration) -> None:
"""Fill in the database password and S3 secret key from their environment variables.
Only fills in what is unset. For the standalone CLI; the orchestrator injects the same
overrides while assembling the pipeline config.
"""
load_environment()
apply_env_defaults(config, "persistent_storage.storage_config")
report_generator/vendor/persistent_storage/utils/__init__.py
Vendored
+1
View File
@@ -0,0 +1 @@
"""DB, S3 and Put utils for persistent storage."""
report_generator/vendor/persistent_storage/utils/cvss_utils.py
Vendored
+492
View File
@@ -0,0 +1,492 @@
"""Reads a CVSS vector of any generation and says what it carries.
Three generations are in circulation at once and a single advisory regularly states two of them, so
nothing here assumes a version -- the vector names its own, and everything after that follows from
the version it named.
What changed between them, and why one parser cannot pretend they are the same:
* **v2 (2007)** scores the whole system. It asks how often an attacker has to authenticate (``Au``)
and grades impact as none, partial or complete.
* **v3 (2015, 3.1 in 2019)** scores the vulnerable component instead. ``Au`` becomes ``PR``, which
asks what privileges are needed rather than how many logins; ``UI`` and ``S`` are new; impact
becomes none, low or high; ``AC`` loses its middle value.
* **v4 (2023)** splits impact in two -- the vulnerable system (``VC``/``VI``/``VA``) and whatever
sits behind it (``SC``/``SI``/``SA``) -- which makes ``S`` unnecessary. ``AT`` is new, ``UI`` gains
a passive value, and the temporal group becomes the threat group.
This module holds no database and no framework, so that the same reading serves persistent_storage
today and archon when the tables move there. Everything that touches a session lives next to the
model instead.
>>> reading = parse_vector("CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H")
>>> reading.version, reading.reachability, reading.remotely_reachable
('3.1', 4, True)
"""
from __future__ import annotations
from typing import Any, NamedTuple
# A vector that names no version is a v2 one: v2 is the only generation that did not prefix itself.
DEFAULT_VERSION = "2.0"
# Which generation a version string belongs to, which is what decides how its metrics are read.
GENERATIONS = {"2.0": 2, "3.0": 3, "3.1": 3, "4.0": 4}
# The metric that says where an attacker has to be, and the value that puts them anywhere.
FROM_ANYWHERE = "N"
# What each generation calls "no obstacle at all" for the four metrics an attacker has to get past.
# Answering all four with these is an attack that needs nothing: no position, no effort, no account
# and nobody to fool. v2 has no user interaction metric, so it is scored over three.
# TODO das sollte ich noch refaktoren das ist unübersichtlich
# AV: Access Vector, AC: Access Complexity, Au: Authentication
EASIEST: dict[int, dict[str, str]] = {
2: {"AV": "N", "AC": "L", "Au": "N"},
3: {"AV": "N", "AC": "L", "PR": "N", "UI": "N"},
4: {"AV": "N", "AC": "L", "PR": "N", "UI": "N"},
}
# What every abbreviation means, per generation, for anything that puts a reading before a person.
# Taken from the specifications; the weights behind them are the scoring formula's business and not
# this module's, which reads a vector rather than recomputing the score from it.
VOCABULARY: dict[int, dict[str, dict[str, str]]] = {
2: {
"AV": {"L": "Local", "A": "Adjacent network", "N": "Network"},
"AC": {"H": "High", "M": "Medium", "L": "Low"},
"Au": {"M": "Multiple", "S": "Single", "N": "None"},
"C": {"N": "None", "P": "Partial", "C": "Complete"},
"I": {"N": "None", "P": "Partial", "C": "Complete"},
"A": {"N": "None", "P": "Partial", "C": "Complete"},
"E": {"U": "Unproven", "POC": "Proof of concept", "F": "Functional", "H": "High", "ND": "Not defined"},
"RL": {"OF": "Official fix", "TF": "Temporary fix", "W": "Workaround", "U": "Unavailable", "ND": "Not defined"},
"RC": {"UC": "Unconfirmed", "UR": "Uncorroborated", "C": "Confirmed", "ND": "Not defined"},
},
3: {
"AV": {"N": "Network", "A": "Adjacent network", "L": "Local", "P": "Physical"},
"AC": {"L": "Low", "H": "High"},
"PR": {"N": "None", "L": "Low", "H": "High"},
"UI": {"N": "None", "R": "Required"},
"S": {"U": "Unchanged", "C": "Changed"},
"C": {"N": "None", "L": "Low", "H": "High"},
"I": {"N": "None", "L": "Low", "H": "High"},
"A": {"N": "None", "L": "Low", "H": "High"},
"E": {"X": "Not defined", "U": "Unproven", "P": "Proof of concept", "F": "Functional", "H": "High"},
"RL": {"X": "Not defined", "O": "Official fix", "T": "Temporary fix", "W": "Workaround", "U": "Unavailable"},
"RC": {"X": "Not defined", "U": "Unknown", "R": "Reasonable", "C": "Confirmed"},
},
4: {
"AV": {"N": "Network", "A": "Adjacent network", "L": "Local", "P": "Physical"},
"AC": {"L": "Low", "H": "High"},
"AT": {"N": "None", "P": "Present"},
"PR": {"N": "None", "L": "Low", "H": "High"},
"UI": {"N": "None", "P": "Passive", "A": "Active"},
"VC": {"H": "High", "L": "Low", "N": "None"},
"VI": {"H": "High", "L": "Low", "N": "None"},
"VA": {"H": "High", "L": "Low", "N": "None"},
"SC": {"H": "High", "L": "Low", "N": "None"},
"SI": {"H": "High", "L": "Low", "N": "None"},
"SA": {"H": "High", "L": "Low", "N": "None"},
"E": {"X": "Not defined", "A": "Attacked", "P": "Proof of concept", "U": "Unreported"},
},
}
# Which key carries the impact on the vulnerable system, per generation. v4 moved it aside to make
# room for a second set covering whatever sits behind that system.
IMPACT_KEYS = {2: ("C", "I", "A"), 3: ("C", "I", "A"), 4: ("VC", "VI", "VA")}
# The second set, which only v4 has.
SUBSEQUENT_KEYS = ("SC", "SI", "SA")
# What the base score is called once it is a word. v2 stops at high; v3 added critical, and pushed
# the same vulnerability further up the scale than v2 would have put it.
SEVERITY_BANDS = {
2: ((0.0, "None"), (4.0, "Low"), (7.0, "Medium"), (10.0, "High")),
3: ((0.0, "None"), (4.0, "Low"), (7.0, "Medium"), (9.0, "High"), (10.0, "Critical")),
4: ((0.0, "None"), (4.0, "Low"), (7.0, "Medium"), (9.0, "High"), (10.0, "Critical")),
}
# What each value is worth when a set of findings has to be averaged into one number per axis.
#
# The base metrics carry the specification's own constants. They are not comparable between
# generations as written -- v2 runs its access vector to 1.0 where v3 stops at 0.85 -- so
# :func:`weight` divides by the largest value of that metric in that generation. What comes back is
# "how far towards the worst case this value sits", 0 to 1, which does compare.
#
# The threat metrics do not carry them. ``E``, ``RL`` and ``RC`` are multipliers in the
# specification, deliberately clustered between 0.91 and 1.0 so that they can only nudge a base
# score. Averaging those would produce a number that is always 0.95 and says nothing, so they are
# ranked by what they actually mean instead: an unproven exploit at 0, one under active attack at 1;
# an official fix at 0, nothing available at 1. Both directions agree with the base metrics --
# larger is always worse.
WEIGHTS: dict[int, dict[str, dict[str, float]]] = {
2: {
"AV": {"L": 0.395, "A": 0.646, "N": 1.0},
"AC": {"H": 0.35, "M": 0.61, "L": 0.71},
"Au": {"M": 0.45, "S": 0.56, "N": 0.704},
"C": {"N": 0.0, "P": 0.275, "C": 0.660},
"I": {"N": 0.0, "P": 0.275, "C": 0.660},
"A": {"N": 0.0, "P": 0.275, "C": 0.660},
"E": {"U": 0.0, "POC": 0.5, "F": 0.85, "H": 1.0},
"RL": {"OF": 0.0, "TF": 0.33, "W": 0.67, "U": 1.0},
"RC": {"UC": 1.0, "UR": 0.5, "C": 0.0},
},
3: {
"AV": {"P": 0.2, "L": 0.55, "A": 0.62, "N": 0.85},
"AC": {"H": 0.44, "L": 0.77},
"PR": {"H": 0.27, "L": 0.62, "N": 0.85},
"UI": {"R": 0.62, "N": 0.85},
"C": {"N": 0.0, "L": 0.22, "H": 0.56},
"I": {"N": 0.0, "L": 0.22, "H": 0.56},
"A": {"N": 0.0, "L": 0.22, "H": 0.56},
"E": {"U": 0.0, "P": 0.5, "F": 0.85, "H": 1.0},
"RL": {"O": 0.0, "T": 0.33, "W": 0.67, "U": 1.0},
"RC": {"U": 1.0, "R": 0.5, "C": 0.0},
},
# v4 scores by lookup table rather than per-metric constants, so there are none to quote. The
# metrics it shares with v3 keep v3's, which preserves their ordering and their spacing; the two
# values v4 added are placed by where the specification ranks them -- ``AT:P`` is an extra
# condition to meet, like a high complexity, and a passive interaction sits between none and an
# active one.
4: {
"AV": {"P": 0.2, "L": 0.55, "A": 0.62, "N": 0.85},
"AC": {"H": 0.44, "L": 0.77},
"AT": {"P": 0.44, "N": 0.85},
"PR": {"H": 0.27, "L": 0.62, "N": 0.85},
"UI": {"A": 0.62, "P": 0.73, "N": 0.85},
"VC": {"N": 0.0, "L": 0.22, "H": 0.56},
"VI": {"N": 0.0, "L": 0.22, "H": 0.56},
"VA": {"N": 0.0, "L": 0.22, "H": 0.56},
"SC": {"N": 0.0, "L": 0.22, "H": 0.56},
"SI": {"N": 0.0, "L": 0.22, "H": 0.56},
"SA": {"N": 0.0, "L": 0.22, "H": 0.56},
"E": {"U": 0.0, "P": 0.5, "A": 1.0},
},
}
# The value every generation uses for "not stated", which is not a value and must not be averaged.
NOT_DEFINED = ("X", "ND")
# Widths a reading is stored in, so a malformed vector cannot fail an insert.
MAX_VECTOR_LENGTH = 512
MAX_METRIC_LENGTH = 16
MAX_VERSION_LENGTH = 8
class CvssReading(NamedTuple):
"""What one CVSS vector states, and the two things that follow from it.
Only the metrics a generation actually has are filled; the rest stay ``None``, because a metric
a version does not define is not the same as one it defines as absent.
Attributes:
version: The version the vector named, e.g. ``"3.1"``.
generation: ``2``, ``3`` or ``4`` -- which set of metrics applies.
vector: The vector as the report wrote it.
metrics: Every metric of the vector by its abbreviation, including the
environmental and supplemental ones this class has no field for.
attack_vector: ``AV``, in every generation.
attack_complexity: ``AC``, in every generation.
attack_requirements: ``AT``, v4 only.
authentication: ``Au``, v2 only -- how often an attacker has to log in.
privileges_required: ``PR``, v3 and v4 -- what account they need instead.
user_interaction: ``UI``, v3 and v4.
scope: ``S``, v3 only; v4 replaced it with a second impact set.
confidentiality_impact: ``C`` in v2 and v3, ``VC`` in v4.
integrity_impact: ``I`` in v2 and v3, ``VI`` in v4.
availability_impact: ``A`` in v2 and v3, ``VA`` in v4.
subsequent_confidentiality_impact: ``SC``, v4 only.
subsequent_integrity_impact: ``SI``, v4 only.
subsequent_availability_impact: ``SA``, v4 only.
exploit_maturity: ``E`` -- temporal in v2 and v3, threat in v4.
remediation_level: ``RL``, v2 and v3.
report_confidence: ``RC``, v2 and v3.
reachability: How many of the metrics an attacker has to get past sit at
the value that costs them nothing, out of :func:`max_reachability`.
remotely_reachable: Over the network and without an account, which is the
pair that decides whether the internet is in a position to try.
"""
version: str
generation: int
vector: str
metrics: dict[str, str]
attack_vector: str | None
attack_complexity: str | None
attack_requirements: str | None
authentication: str | None
privileges_required: str | None
user_interaction: str | None
scope: str | None
confidentiality_impact: str | None
integrity_impact: str | None
availability_impact: str | None
subsequent_confidentiality_impact: str | None
subsequent_integrity_impact: str | None
subsequent_availability_impact: str | None
exploit_maturity: str | None
remediation_level: str | None
report_confidence: str | None
reachability: int
remotely_reachable: bool
def parse_vector(vector: str | None) -> CvssReading | None:
"""Read a CVSS vector of any generation.
Args:
vector: A vector string, e.g. ``CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H``
or a v2 one, which carries no prefix: ``AV:N/AC:L/Au:N/C:P/I:P/A:P``.
Returns:
What the vector states, or ``None`` when there is no vector or it names
none of the metrics its generation defines. ``None`` means "not stated",
which is not the same as "poses no risk", so the two are kept apart.
"""
if not vector:
return None
version = _version(vector)
generation = GENERATIONS.get(version, 3)
metrics = _metrics(vector, generation)
if not metrics:
return None
confidentiality, integrity, availability = IMPACT_KEYS[generation]
return CvssReading(
version=version,
generation=generation,
vector=vector[:MAX_VECTOR_LENGTH],
metrics=metrics,
attack_vector=metrics.get("AV"),
attack_complexity=metrics.get("AC"),
attack_requirements=metrics.get("AT"),
authentication=metrics.get("Au"),
privileges_required=metrics.get("PR"),
user_interaction=metrics.get("UI"),
scope=metrics.get("S") if generation == 3 else None,
confidentiality_impact=metrics.get(confidentiality),
integrity_impact=metrics.get(integrity),
availability_impact=metrics.get(availability),
subsequent_confidentiality_impact=metrics.get("SC"),
subsequent_integrity_impact=metrics.get("SI"),
subsequent_availability_impact=metrics.get("SA"),
exploit_maturity=metrics.get("E"),
remediation_level=metrics.get("RL"),
report_confidence=metrics.get("RC"),
reachability=sum(metrics.get(key) == value for key, value in EASIEST[generation].items()),
# A metric the vector does not state is not counted as an obstacle either, so an absent one
# leaves the claim unmade rather than making it falsely.
remotely_reachable=_remotely_reachable(metrics, generation),
)
def max_reachability(generation: int) -> int:
"""How many obstacles a generation knows about.
Args:
generation: ``2``, ``3`` or ``4``.
Returns:
Three for v2, which has no user interaction metric, four otherwise. A
reading is only comparable with another of the same generation, which is
why this is asked for rather than assumed.
"""
return len(EASIEST.get(generation, EASIEST[3]))
def severity_band(score: float | None, generation: int) -> str | None:
"""Return the word a base score falls under.
Args:
score: The base score, if the report stated one.
generation: Which scale to read it against -- v2 stops at high, v3 and v4
added critical, so the same number is not the same word.
Returns:
The band, or ``None`` when there is no score.
"""
if score is None:
return None
for limit, name in SEVERITY_BANDS.get(generation, SEVERITY_BANDS[3]):
if score <= limit:
return name
return SEVERITY_BANDS[generation][-1][1]
def weight(metric: str, value: str | None, generation: int = 3) -> float | None:
"""Return how far towards the worst case one metric value sits, 0 to 1.
# TODO hier nochmal das Gewichten explizit prüfen.
This is what makes a set of findings averageable along one axis, and what makes a v2 reading
averageable together with a v4 one: each value is placed between the best and the worst its own
generation allows for that metric, so the result says "how far towards the worst case" rather
than quoting a constant that only means something inside its own formula.
Stretched across the axis rather than divided by its top, because the specification's constants
are multipliers and do not start at zero: an attack that needs physical access is worth 0.2 of
the 0.85 a network one is worth, and reported as 0.24 it would say a quarter of the worst case
for what is in fact the best case the metric has. Zero here means "as hard as this metric can
make it", one means "as easy".
Args:
metric: An abbreviation such as ``AV``, ``PR`` or ``VC``.
value: The value the vector carried, if any.
generation: Which generation's scale to read it against.
Returns:
The position between best and worst case, or ``None`` when the vector
stated nothing, stated "not defined", or stated something the
specification does not list. All three mean the same thing here -- there
is nothing to average -- and are kept apart from a stated zero, which is
a measurement.
"""
if value is None or value in NOT_DEFINED:
return None
scale = WEIGHTS.get(generation, {}).get(metric, {})
stated = scale.get(value)
if stated is None or not scale:
return None
best, worst = min(scale.values()), max(scale.values())
# A metric with a single value says nothing about degree, so it reads as the worst case it has.
return (stated - best) / (worst - best) if worst > best else 1.0
def describe(metric: str, value: str | None, generation: int = 3) -> str:
"""Return the word behind one abbreviation.
Args:
metric: An abbreviation such as ``AV``, ``PR`` or ``VC``.
value: The value the vector carried, if any.
generation: Which vocabulary to read it in. The same letter means
different things across generations -- ``C`` under v2 is "complete",
under v3 it is a metric name -- so this is not optional in practice.
Returns:
The name the specification gives it, the raw value when the vocabulary
has no entry, or ``"Not stated"`` when the vector did not carry it.
"""
if value is None:
return "Not stated"
return VOCABULARY.get(generation, {}).get(metric, {}).get(value, value)
def _version(vector: str) -> str:
"""Return the version a vector names.
Args:
vector: The vector string. v3 and v4 begin with ``CVSS:<version>``; v2
begins with its first metric and is recognised by that absence.
Returns:
The version, defaulting to :data:`DEFAULT_VERSION` for an unprefixed
vector and to the string as written for one this module does not know --
a version nobody has implemented yet is still worth recording.
"""
head = vector.split("/", 1)[0]
if not head.upper().startswith("CVSS:"):
return DEFAULT_VERSION
return head.split(":", 1)[1].strip()[:MAX_VERSION_LENGTH] or DEFAULT_VERSION
def _metrics(vector: str, generation: int) -> dict[str, str]:
"""Split a vector into its metrics.
Args:
vector: The vector string.
generation: Which generation it belongs to, which decides whether the
first segment is a version prefix to skip.
Returns:
The metrics by their abbreviation, spelled as the specification spells
them -- ``Au`` keeps its lower-case second letter, which is the one
abbreviation that is not upper case. A segment that is not a
``key:value`` pair is ignored rather than raising: a report this store
cannot fully read is still a report worth keeping.
"""
segments = vector.split("/")
metrics: dict[str, str] = {}
for segment in segments if generation == 2 else segments[1:]:
key, separator, value = segment.partition(":")
if not separator or not key or not value:
continue
name = key.strip()[:MAX_METRIC_LENGTH]
metrics["Au" if name.upper() == "AU" and generation == 2 else name.upper()] = value.strip().upper()[
:MAX_METRIC_LENGTH
]
return metrics
def _remotely_reachable(metrics: dict[str, str], generation: int) -> bool:
"""Whether the vector says an attacker needs neither a position nor an account.
Args:
metrics: The metrics of the vector.
generation: Which generation states the account requirement, since v2
asks how often an attacker logs in and v3 onwards what they log in
as.
Returns:
``True`` only when the vector states both and both are open.
"""
account = "Au" if generation == 2 else "PR"
return metrics.get("AV") == FROM_ANYWHERE and metrics.get(account) == "N"
def as_row(reading: CvssReading, base_score: float | None = None) -> dict[str, Any]:
"""Return a reading shaped like the columns it is stored in.
Kept here rather than next to the model so that the move to archon is a change of table and not
a change of mapping: whatever holds these columns can be filled from this.
Args:
reading: What a vector stated.
base_score: The score the report stated alongside the vector, if any.
Not recomputed from the vector -- the report's own number is the one
that has to come back out of the store.
Returns:
The reading by column name, ready to be handed to a model.
"""
return {
"version": reading.version,
"vector": reading.vector,
"base_score": base_score,
"base_severity": severity_band(base_score, reading.generation),
"attack_vector": reading.attack_vector,
"attack_complexity": reading.attack_complexity,
"attack_requirements": reading.attack_requirements,
"authentication": reading.authentication,
"privileges_required": reading.privileges_required,
"user_interaction": reading.user_interaction,
"scope": reading.scope,
"confidentiality_impact": reading.confidentiality_impact,
"integrity_impact": reading.integrity_impact,
"availability_impact": reading.availability_impact,
"subsequent_confidentiality_impact": reading.subsequent_confidentiality_impact,
"subsequent_integrity_impact": reading.subsequent_integrity_impact,
"subsequent_availability_impact": reading.subsequent_availability_impact,
"exploit_maturity": reading.exploit_maturity,
"remediation_level": reading.remediation_level,
"report_confidence": reading.report_confidence,
"metrics": reading.metrics,
"reachability": reading.reachability,
"remotely_reachable": reading.remotely_reachable,
}
report_generator/vendor/persistent_storage/utils/db_utils.py
Vendored
+324
View File
@@ -0,0 +1,324 @@
"""Utilities for configuring and interacting with the artifact metadata database.
This module defines the database schema for artifact records, initializes the
database connection and tables, and provides functionality for storing and
retrieving artifact metadata.
"""
from uuid import UUID
import sqlalchemy
from sqlalchemy import create_engine, delete, select
from sqlalchemy.orm import Session, selectinload, sessionmaker
from cc_common.api import context
from persistent_storage.helper import print_scan_table
from persistent_storage.shared import DatabaseConfiguration
from persistent_storage.wrapper import Artifact, Base, Scan
def get_session(configuration: DatabaseConfiguration) -> sqlalchemy.orm.Session:
"""Create and return a new SQLAlchemy session bound to the application's engine.
The session is configured using the engine returned by
`create_engine()` and can be used to interact with the database.
Returns:
sqlalchemy.orm.Session: A new SQLAlchemy session instance.
"""
engine = create_engine(configuration.db_url)
session = sessionmaker(bind=engine, expire_on_commit=False)
return session()
def init_db(configuration: DatabaseConfiguration) -> None:
"""Initialize the application's database schema.
Creates all tables defined in the SQLAlchemy metadata using the
engine returned by `create_engine()`. Existing tables are
left unchanged.
This function should typically be called during application startup
or initial setup.
"""
engine = create_engine(configuration.db_url)
Base.metadata.create_all(engine)
def save_object_to_db(instance: Base, configuration: DatabaseConfiguration) -> None:
"""Persist artifact metadata to the database.
Creates a new Artifact record from validated metadata,
stores it in the database, and commits the transaction.
Args:
instance (Base):
Generic Base Object for Scan and Artifact Records.
configuration:
The configuration parameters of the db.
Side Effects:
- Inserts a new artifact entry into the database.
- Commits transaction.
"""
session: Session = get_session(configuration)
try:
session.add(instance)
session.commit()
except Exception as e:
session.rollback()
context.logger.error(f"Failed to save artifact: {e}")
raise
finally:
session.close()
def get_scan(configuration: DatabaseConfiguration, scan_id: UUID) -> Scan:
"""Retrieve a scan and its associated artifacts from the database.
Args:
configuration: Database configuration containing the connection URL.
scan_id: UUID of the scan to retrieve.
Returns:
The matching ``Scan`` instance with its artifacts eagerly loaded.
Raises:
sqlalchemy.exc.NoResultFound: If no scan with the given UUID exists.
sqlalchemy.exc.MultipleResultsFound: If multiple scans with the given
UUID are found.
"""
engine = create_engine(configuration.db_url)
try:
with Session(engine) as session:
stmt = select(Scan).where(Scan.scan_id == scan_id).options(selectinload(Scan.artifacts))
return session.scalars(stmt).one()
finally:
engine.dispose()
class ScanList(list[Scan]):
"""A list of ``Scan`` objects with convenience methods for presentation."""
def print_as_cc_common_result(self) -> None:
"""Print the scans as a rich table."""
print_scan_table(self)
def get_all_scans(configuration: DatabaseConfiguration) -> list[Scan]:
"""Retrieve all scans and their associated artifacts from the database.
Args:
configuration: Database configuration containing the connection URL.
Returns:
A ``ScanList`` containing all scans with their artifacts eagerly loaded.
"""
engine = create_engine(configuration.db_url)
try:
with Session(engine) as session:
stmt = select(Scan).options(selectinload(Scan.artifacts))
scans = session.scalars(stmt).all()
return ScanList(scans)
finally:
engine.dispose()
# DROP DB ENTRY
def drop_database_entry(configuration: DatabaseConfiguration, database_entry_uuid: UUID) -> bool:
"""Delete a specific database entry by UUID.
Removes only the matching database entry from the artifact table.
Args:
database_entry_uuid:
UUID of the database entry to delete.
configuration:
The configuration parameters of the db.
Returns:
bool:
The value shows if the deletion was successful.
Side Effects:
- Permanently deletes a single database row
- Prints deletion progress information to stdout
"""
engine = create_engine(configuration.db_url)
success = False
try:
with engine.begin() as conn:
stmt = delete(Scan).where(Scan.scan_id == database_entry_uuid)
result = conn.execute(stmt)
if result.rowcount == 0:
context.logger.warning(
f"No entry found for UUID {database_entry_uuid}",
)
else:
context.logger.warning(
f"Scan Entry {database_entry_uuid} deleted ⚠",
)
success = True
finally:
engine.dispose()
return success
# WAIT FOR POSTGRES
def wait_for_postgres(configuration: DatabaseConfiguration, max_retries: int = 10, delay: int = 1) -> None:
"""Wait until the PostgreSQL server becomes available.
Repeatedly attempts to establish a connection to the PostgreSQL
instance using the admin credentials. If the connection succeeds,
the function returns immediately. Otherwise, retries are performed
until the maximum retry limit is reached.
Args:
max_retries (int):
Maximum number of connection attempts before failing.
delay (int):
Delay in seconds between retry attempts.
configuration:
The configuration parameters of the db.
Raises:
RuntimeError:
Raised if PostgreSQL does not become available within the
configured retry limit.
"""
import time
from sqlalchemy import create_engine
from sqlalchemy.exc import OperationalError
context.logger.info("Waiting for PostgreSQL to become available...")
engine = create_engine(
configuration.db_url,
isolation_level="AUTOCOMMIT",
)
try:
for attempt in range(max_retries):
try:
with engine.connect():
context.logger.info("PostgreSQL is ready ✔")
return
except OperationalError:
context.logger.warning(
f"PostgreSQL not ready (attempt {attempt + 1}/{max_retries})",
)
time.sleep(delay)
finally:
engine.dispose()
context.logger.error(
"PostgreSQL did not become ready in time",
)
raise RuntimeError(
"PostgreSQL did not become ready in time",
)
def get_s3_key(configuration: DatabaseConfiguration, target_artifact_id: UUID) -> str:
"""Retrieves the S3 object key for a specific artifact.
Queries the database for the artifact identified by the given UUID and
returns the corresponding S3 key used to locate the artifact file in
the configured S3 bucket.
Args:
target_artifact_id: UUID of the artifact whose S3 key should be
retrieved.
configuration: The configuration parameters of the db.
Returns:
The S3 object key associated with the artifact.
Raises:
ValueError: If no artifact with the specified UUID exists or no
valid S3 key is found.
"""
from sqlalchemy import create_engine, select
from sqlalchemy.orm import Session
engine = create_engine(configuration.db_url)
try:
with Session(engine) as session:
s3_key = session.scalar(select(Artifact.s3_key).where(Artifact.artifact_id == target_artifact_id))
if s3_key is None or not isinstance(s3_key, str):
context.logger.error(f"Artifact {target_artifact_id} not found ❌")
raise ValueError(f"Artifact {target_artifact_id} not found ❌")
else:
return str(s3_key)
finally:
engine.dispose()
def get_s3_keys(configuration: DatabaseConfiguration, target_scan_id: UUID) -> list[str]:
"""Retrieves the S3 object keys for all artifacts belonging to a scan.
Queries the database for all artifacts associated with the specified
scan UUID and returns their corresponding S3 object keys.
Args:
target_scan_id: UUID of the scan whose artifact S3 keys should be
retrieved.
configuration: The configuration parameters of the db.
Returns:
A list containing the S3 object keys of all artifacts associated
with the scan.
Raises:
ValueError: If no artifact files are associated with the specified
scan UUID or if an invalid S3 key is encountered.
"""
from sqlalchemy import create_engine, select
from sqlalchemy.orm import Session
engine = create_engine(configuration.db_url)
try:
with Session(engine) as session:
s3_keys: list[str] = list(
session.scalars(select(Artifact.s3_key).where(Artifact.scan_id == target_scan_id))
)
if len(s3_keys) == 0:
context.logger.error(f"No artifact files for scan UUID '{target_scan_id}' found ❌")
raise ValueError(f"No artifact files for scan UUID '{target_scan_id}' found ❌")
for s3_key in s3_keys:
if not isinstance(s3_key, str):
context.logger.error(f"Invalid S3 key for scan UUID '{target_scan_id}' ❌")
raise ValueError(f"Invalid S3 key for scan UUID '{target_scan_id}' ❌")
return s3_keys
finally:
engine.dispose()
report_generator/vendor/persistent_storage/utils/s3_utils.py
Vendored
+170
View File
@@ -0,0 +1,170 @@
"""Utilities for managing artifact storage in an S3-compatible object store.
This module provides helper functions for:
- Ensuring the configured S3 bucket exists
- Uploading artifacts to object storage
"""
from typing import TYPE_CHECKING, cast
from uuid import UUID
from botocore.exceptions import ClientError
from cc_common.api import context
from persistent_storage.shared import S3Configuration, StorageConfiguration
from persistent_storage.utils.db_utils import get_s3_key
if TYPE_CHECKING:
# mypy_boto3_s3 drags in boto3 (~1.2s) purely for these type stubs; the runtime object it
# names is never used (typing.cast() is a no-op that just returns its second argument), so
# the callers below reference it as a string forward-ref instead.
from mypy_boto3_s3 import S3Client
from mypy_boto3_s3.type_defs import ObjectIdentifierTypeDef
def ensure_bucket_exists(configuration: S3Configuration) -> None:
"""Ensure that the configured S3 bucket exists.
Checks whether the configured bucket is available in the connected
S3-compatible storage service. If the bucket does not exist, it is
created automatically.
Side Effects:
- May create a new S3 bucket
- Prints bucket status information to stdout
"""
try:
s3_client = cast("S3Client", configuration.s3_client)
s3_client.create_bucket(Bucket=configuration.bucket)
context.logger.warning(f"Bucket '{configuration.bucket}' created ✔")
except ClientError as e:
code = e.response["Error"].get("Code")
if code in ("BucketAlreadyOwnedByYou", "BucketAlreadyExists"):
context.logger.warning(f"Bucket '{configuration.bucket}' already exists")
return
raise
def upload_file(configuration: S3Configuration, path: str, s3_key: str) -> None:
"""Upload a local artifact file to the configured S3 bucket.
Args:
path (str):
Local filesystem path to the artifact or meta file.
s3_key (str):
Object key under which the artifact or meta file will be stored
in the S3 bucket.
configuration (StorageConfiguration):
The configuration parameters for the S3 storage.
Side Effects:
- Uploads a file to S3-compatible object storage
- Prints upload status information to Logger
"""
s3_client = cast("S3Client", configuration.s3_client)
s3_client.upload_file(str(path), configuration.bucket, s3_key)
context.logger.info(f"Uploaded {path} -> s3://{configuration.bucket}/{s3_key}")
def open_rustfs_bucket(configuration: S3Configuration) -> None:
"""Open the RustFS bucket in the RustFS web console."""
import webbrowser
webbrowser.open(configuration.url[:-1] + "1") # Port 9001
def fetch_artifact_file(configuration: StorageConfiguration, artifact_id: UUID) -> bytes:
"""Retrieves one artifact files from the configured S3 bucket.
The corresponding artifact, associated with artifact_id is fetched and
returned as a single bytes object.
Args:
artifact_id: Unique identifier of the artifact to retrieve.
configuration: The configuration parameters for the S3 storage.
Returns:
The artifact as a bytes-like object.
"""
s3_client = cast("S3Client", configuration.s3.s3_client)
response = s3_client.get_object(
Bucket=configuration.s3.bucket,
Key=get_s3_key(configuration=configuration.database, target_artifact_id=artifact_id),
)
return response["Body"].read()
def fetch_artifact_files(
configuration: StorageConfiguration, artifact_ids: list[UUID]
) -> tuple[list[bytes], list[str]]: # noqa: E501
"""Retrieves multiple artifact files from the configured S3 bucket.
All files associated with the scan are fetched and returned as a list of byte
objects and a list of file names.
Args:
artifact_ids: All unique UUIDs of the scan to retrieve.
configuration: The configuration parameters for the S3 storage.
Returns:
A list[bytes] with all artifacts of a scan as bytes-like objects.
A list[str] with all artifact file names.
"""
s3_client = cast("S3Client", configuration.s3.s3_client)
files = []
file_names = []
for artifact_id in artifact_ids:
s3_key = get_s3_key(configuration=configuration.database, target_artifact_id=artifact_id)
response = s3_client.get_object(
Bucket=configuration.s3.bucket,
Key=s3_key,
)
file = response["Body"].read()
files.append(file)
file_name = s3_key.split("/")[-1]
file_names.append(file_name)
return files, file_names
def drop_storage_files(configuration: S3Configuration, target_scan_id: UUID) -> bool:
"""Delete all S3 objects associated with a scan.
Deletes every object in the configured S3 bucket whose key has the prefix
``<target_scan_id>/``.
Args:
configuration: S3 configuration containing the bucket name and client.
target_scan_id: Identifier of the scan whose stored files should be removed.
Returns:
``True`` if all matching objects were deleted successfully, otherwise
``False`` if an AWS ``ClientError`` was raised.
"""
s3_client = cast("S3Client", configuration.s3_client)
prefix = f"{target_scan_id}/"
paginator = s3_client.get_paginator("list_objects_v2")
try:
for page in paginator.paginate(Bucket=configuration.bucket, Prefix=prefix):
if "Contents" not in page:
continue
objects: list[ObjectIdentifierTypeDef] = [{"Key": obj["Key"]} for obj in page["Contents"]]
s3_client.delete_objects(Bucket=configuration.bucket, Delete={"Objects": objects})
return True
except ClientError as e:
context.logger.error(e)
return False
report_generator/vendor/persistent_storage/wrapper.py
Vendored
+744
View File
@@ -0,0 +1,744 @@
"""Wrapper for database storage."""
from datetime import UTC, datetime
from uuid import UUID
from sqlalchemy import (
JSON,
TIMESTAMP,
Boolean,
Float,
ForeignKey,
Index,
Integer,
String,
UniqueConstraint,
text,
)
from sqlalchemy.dialects.postgresql import UUID as PG_UUID
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship
class Base(DeclarativeBase):
"""Base model for SQLAlchemy models."""
pass
class Artifact(Base):
"""SQLAlchemy model representing a stored artifact.
This table stores metadata about files that are generated or uploaded
as part of a project. Each artifact is linked to a scan and stored
in S3-compatible object storage.
Attributes:
id (int):
Primary key of the artifact record.
artifact_id (UUID):
Unique identifier of the artifact.
artifact_type (str):
Type/category of the artifact (e.g. SARIF, SBOM).
s3_key (str):
Object storage key used to locate the file in S3-compatible storage.
file_name (str | None):
Original filename of the uploaded artifact, if available.
created_at (datetime):
Timestamp when the artifact record was created (UTC).
artifact_metadata (dict | None):
Optional JSON metadata associated with the artifact,
such as file size or additional computed attributes.
scan_id (UUID):
Foreign key linked to the unique identifier of the scan record.
"""
__tablename__ = "artifacts"
id: Mapped[int] = mapped_column(
Integer,
primary_key=True,
)
artifact_id: Mapped[UUID] = mapped_column(
PG_UUID(as_uuid=True),
unique=True,
nullable=False,
)
artifact_type: Mapped[str] = mapped_column(
String,
nullable=False,
)
s3_key: Mapped[str] = mapped_column(
String,
nullable=False,
)
file_name: Mapped[str | None] = mapped_column(
String,
nullable=True,
)
created_at: Mapped[datetime] = mapped_column(
TIMESTAMP,
nullable=False,
)
artifact_metadata: Mapped[str | None] = mapped_column(
JSON,
nullable=True,
)
# On delete cascade
scan_id: Mapped[UUID] = mapped_column(
PG_UUID(as_uuid=True),
ForeignKey("scans.scan_id", ondelete="CASCADE"),
nullable=False,
)
# ORM relationship to scans
scan: Mapped[Scan] = relationship(
"Scan",
back_populates="artifacts",
)
def print_as_cc_common_result(self) -> None:
"""Prints the artifact objects content."""
from .helper import print_artifact_table
print_artifact_table([self])
class Scan(Base):
"""SQLAlchemy model representing a scan execution.
This table stores metadata about a single scan run, including
information about the tool that produced the scan, execution time,
and the associated meta file stored in S3-compatible
object storage.
A scan can be associated with one or more artifacts, which reference
the scan through its unique scan identifier.
Attributes:
id (int):
Internal database primary key.
scan_id (UUID):
Unique identifier of the scan execution. Used as a stable
reference for related artifacts.
run_at (datetime):
Timestamp when the scan was executed (UTC).
tool_name (str):
Name of the tool that generated the scan.
tool_version (int | None):
Version of the tool that generated the scan, if available.
meta_file_s3_key (str):
Object storage key of the scan meta file stored in
S3-compatible storage.
meta_file_content (dict[str, Any]):
Parsed content of the scan meta file stored as JSON.
"""
__tablename__ = "scans"
id: Mapped[int] = mapped_column(Integer, primary_key=True)
scan_id: Mapped[UUID] = mapped_column(
PG_UUID(as_uuid=True),
unique=True,
nullable=False,
)
run_at: Mapped[datetime] = mapped_column(TIMESTAMP, default=lambda: datetime.now(UTC), nullable=False)
tool_name: Mapped[str] = mapped_column(String, nullable=False)
tool_version: Mapped[str] = mapped_column(String, nullable=True)
meta_file_s3_key: Mapped[str] = mapped_column(String, unique=True, nullable=False)
meta_file_content: Mapped[str] = mapped_column(String, nullable=False)
# ORM relationship to artifacts
artifacts: Mapped[list[Artifact]] = relationship(
"Artifact",
back_populates="scan",
)
# ORM relationships to what this scan found
component_scans: Mapped[list[ComponentScan]] = relationship(
"ComponentScan",
back_populates="scan",
cascade="all, delete-orphan",
)
dep_finding_scans: Mapped[list[DepFindingScan]] = relationship(
"DepFindingScan",
back_populates="scan",
cascade="all, delete-orphan",
)
def print_as_cc_common_result(self) -> None:
"""Prints the scan objects content."""
from .helper import print_scan_table
print_scan_table([self])
class Component(Base):
"""A third-party package, identified by what it is rather than by who found it.
``mypy 1.0`` is the same package in every scan that depends on it, so it is one row. Which scans
depend on it is a different fact, and it gets its own table -- archon calls that one
``component_scans`` and this package will call it the same, so that the two line up when these
tables move. Until it exists, a component says what it is and nothing about where it came from.
Mirrors archon's ``components`` table without the three columns that belong to a document rather
than to the package: ``scope`` and ``bom_ref`` are what one SBOM said about it in one place, and
the timestamp is when the row was written.
Attributes:
id (int):
Internal database primary key, as everything in this package has.
component_id (UUID):
Stable handle of the component. A UUID so that archon, whose tables
are keyed by one, can adopt it as the primary key unchanged.
component_name (str):
Name of the package.
version (str):
Version of the package as the document stated it, empty when it
stated none -- which CycloneDX allows for a range nobody pinned.
latest_version (str | None):
The newest version the reporting tool knew of, refreshed by every
export that states one. What makes a finding answerable: without it a
report says what is wrong and never what to do about it. A newer
version existing is not proof that it fixes this advisory, so
anything read off it is "there is somewhere to upgrade to", never
"this is fixed".
purl (str | None):
Package URL, which already encodes ecosystem, name and version
(``pkg:pypi/requests@2.31.0``) and is therefore the identity where it
exists. NULL for something no package manager knows -- an installed
application rather than a library -- which name and version identify
instead.
"""
__tablename__ = "components"
__table_args__ = (
# The package URL is the identity where there is one: it carries the ecosystem, which is
# what keeps pkg:npm/requests apart from pkg:pypi/requests. Both are called "requests" at
# the same version and are unrelated packages, so name and version alone cannot be unique.
UniqueConstraint("purl", name="uq_components_purl"),
# Where there is no package URL there is no ecosystem either, so name and version are all
# there is to go on. Partial, so it constrains only those rows and leaves the others to the
# constraint above.
Index(
"uq_components_unregistered",
"component_name",
"version",
unique=True,
postgresql_where=text("purl IS NULL"),
sqlite_where=text("purl IS NULL"),
),
Index("ix_components_name_version", "component_name", "version"),
)
id: Mapped[int] = mapped_column(Integer, primary_key=True)
component_id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), unique=True, nullable=False)
component_name: Mapped[str] = mapped_column(String, nullable=False)
version: Mapped[str] = mapped_column(String, nullable=False)
latest_version: Mapped[str | None] = mapped_column(String, nullable=True)
purl: Mapped[str | None] = mapped_column(String, nullable=True)
# ORM relationship to the advisories recorded against this component
findings: Mapped[list[DependencyFinding]] = relationship(
"DependencyFinding",
back_populates="component",
cascade="all, delete-orphan",
)
# ORM relationship to the scans that found it
scans: Mapped[list[ComponentScan]] = relationship(
"ComponentScan",
back_populates="component",
cascade="all, delete-orphan",
)
def print_as_cc_common_result(self) -> None:
"""Prints the component objects content."""
from .helper import print_component_table
print_component_table([self])
class DependencyFinding(Base):
"""A known vulnerability affecting one component.
Attributes:
id (int):
Internal database primary key.
finding_dep_id (UUID):
Stable handle of the finding, and archon's primary key once moved.
component_id (UUID):
The component the advisory affects.
cve (str | None):
The identifier the advisory carries, e.g. ``CVE-2024-35195``. Not
every advisory has a CVE -- GitHub and OSV assign their own, and
those are stored here too.
cwes (list[str] | None):
The weakness classes the advisory names, as ``CWE-79`` strings.
description (str):
What the advisory says about it.
default_level (str):
The severity the report headlined: ``critical``, ``high``,
``medium``, ``low``, ``info`` or ``unassigned``. Kept as the report
wrote it rather than derived from a score, because the report is what
is being stored.
cwe_name (str | None):
What the weakness class is called, e.g. ``Protection Mechanism
Failure``. The identifier alone means nothing to a reader.
source (str | None):
Who published the advisory: ``NVD``, ``GITHUB``, ``OSV``.
published (datetime | None):
When the advisory was made public. The **age** of a finding is read
off this rather than stored: an age is a number that is wrong again
tomorrow, a date is not.
references (str | None):
Links to the advisory and its fix, as the export wrote them -- one
markdown list rather than a structured field, which is what
Dependency-Track hands over.
"""
__tablename__ = "dependency_findings"
__table_args__ = (
# One advisory per component. A second report naming it again finds this row rather than
# writing another one.
UniqueConstraint("component_id", "cve", name="uq_dependency_findings_advisory"),
)
id: Mapped[int] = mapped_column(Integer, primary_key=True)
finding_dep_id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), unique=True, nullable=False)
component_id: Mapped[UUID] = mapped_column(
PG_UUID(as_uuid=True),
ForeignKey("components.component_id", ondelete="CASCADE"),
nullable=False,
)
cve: Mapped[str | None] = mapped_column(String, nullable=True)
cwes: Mapped[list[str] | None] = mapped_column(JSON, nullable=True)
description: Mapped[str] = mapped_column(String, nullable=False)
default_level: Mapped[str] = mapped_column(String, nullable=False)
cwe_name: Mapped[str | None] = mapped_column(String, nullable=True)
source: Mapped[str | None] = mapped_column(String, nullable=True)
published: Mapped[datetime | None] = mapped_column(TIMESTAMP, nullable=True)
references: Mapped[str | None] = mapped_column(String, nullable=True)
# ORM relationship to components
component: Mapped[Component] = relationship("Component", back_populates="findings")
# ORM relationship to the CVSS scorings of this advisory, one per version of the standard
cvss: Mapped[list[CvssMetric]] = relationship(
"CvssMetric",
back_populates="finding",
cascade="all, delete-orphan",
)
# ORM relationship to the scans that reported it
scans: Mapped[list[DepFindingScan]] = relationship(
"DepFindingScan",
back_populates="finding",
cascade="all, delete-orphan",
)
def print_as_cc_common_result(self) -> None:
"""Prints the dependency finding objects content."""
from .helper import print_dependency_finding_table
print_dependency_finding_table([self])
class CvssMetric(Base):
"""One CVSS scoring of one advisory, in one version of the standard.
An advisory is regularly scored more than once -- a Dependency-Track export routinely carries a
v3 vector and a v4 one for the same CVE, and occasionally a v2 one as well -- and the versions
do not agree with each other. So this is one row per version rather than one per advisory, and
nothing is folded together: the columns hold what the vector stated, ``metrics`` holds all of it
verbatim, and the score is the report's own rather than one recomputed here.
Not every column applies to every version, which is why nearly all of them are nullable:
* ``authentication`` is v2 only -- how often an attacker has to log in.
* ``privileges_required`` and ``user_interaction`` are v3 and v4, which ask what account is
needed rather than how many logins.
* ``scope`` is v3 only; v4 replaced it with the ``subsequent_*`` impact columns.
* ``attack_requirements`` and the ``subsequent_*`` columns are v4 only.
A NULL therefore means one of two things -- the version has no such metric, or the report did
not state it -- and ``version`` is what tells the two apart.
**Built to be moved.** The row points at ``dependency_findings`` and at nothing else of this
package, and that table is archon's under the same name with the same handle. The reading comes
from :mod:`persistent_storage.utils.cvss_utils`, which imports nothing from here.
Attributes:
id (int):
Internal database primary key.
cvss_id (UUID):
Stable handle of the scoring.
finding_dep_id (UUID):
The advisory this scores.
version (str):
The version of the standard: ``2.0``, ``3.0``, ``3.1`` or ``4.0``.
vector (str):
The vector as the report wrote it, kept verbatim so the columns
beside it can be checked or re-derived without the report at hand.
base_score (float | None):
The base score the report stated, 0 to 10.
base_severity (str | None):
The word that score falls under, read against the scale of its own
version: v2 stops at high, v3 and v4 added critical, so the same
number is not the same word.
attack_vector (str | None):
``AV`` -- where an attacker has to be.
attack_complexity (str | None):
``AC`` -- how much has to fall into place.
attack_requirements (str | None):
``AT`` -- what the target environment has to look like. v4 only.
authentication (str | None):
``Au`` -- how often an attacker has to authenticate. v2 only.
privileges_required (str | None):
``PR`` -- what account is needed. v3 and v4.
user_interaction (str | None):
``UI`` -- whether somebody has to be fooled into helping. v3 and v4.
scope (str | None):
``S`` -- whether the blast reaches past the component. v3 only.
confidentiality_impact (str | None):
What an attacker gets to read. ``C`` in v2 and v3, ``VC`` in v4.
integrity_impact (str | None):
What an attacker gets to change. ``I`` in v2 and v3, ``VI`` in v4.
availability_impact (str | None):
How far the service can be taken down. ``A`` in v2 and v3, ``VA`` in
v4.
subsequent_confidentiality_impact (str | None):
``SC`` -- the same, for whatever sits behind the vulnerable system.
v4 only, and the reason v4 needs no scope metric.
subsequent_integrity_impact (str | None):
``SI``. v4 only.
subsequent_availability_impact (str | None):
``SA``. v4 only.
exploit_maturity (str | None):
``E`` -- how far along a working exploit is. Temporal in v2 and v3,
threat in v4, and spelled differently in each.
remediation_level (str | None):
``RL`` -- whether there is a fix. v2 and v3.
report_confidence (str | None):
``RC`` -- how sure anyone is that this is real. v2 and v3.
metrics (dict[str, str] | None):
Every metric of the vector by its abbreviation, environmental and
supplemental ones included. Those are filled in by whoever operates
the system rather than by a scanner, so they get no columns of their
own -- but a report that does state them does not lose them here.
reachability (int):
How many of the metrics an attacker has to get past sit at the value
that costs them nothing. Out of three under v2, which has no user
interaction metric, and four otherwise -- so it compares only within
one version.
remotely_reachable (bool):
Over the network and without an account. The pair that decides
whether the internet is in a position to try at all.
"""
__tablename__ = "cvss_metrics"
__table_args__ = (
# One scoring per advisory per version of the standard.
UniqueConstraint("finding_dep_id", "version", name="uq_cvss_metrics_version"),
# What the portfolio question runs on: how much of this can be reached from outside.
Index("ix_cvss_metrics_reach", "remotely_reachable", "reachability"),
)
id: Mapped[int] = mapped_column(Integer, primary_key=True)
cvss_id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), unique=True, nullable=False)
finding_dep_id: Mapped[UUID] = mapped_column(
PG_UUID(as_uuid=True),
ForeignKey("dependency_findings.finding_dep_id", ondelete="CASCADE"),
nullable=False,
)
version: Mapped[str] = mapped_column(String, nullable=False)
vector: Mapped[str] = mapped_column(String, nullable=False)
base_score: Mapped[float | None] = mapped_column(Float, nullable=True)
base_severity: Mapped[str | None] = mapped_column(String, nullable=True)
attack_vector: Mapped[str | None] = mapped_column(String, nullable=True)
attack_complexity: Mapped[str | None] = mapped_column(String, nullable=True)
attack_requirements: Mapped[str | None] = mapped_column(String, nullable=True)
authentication: Mapped[str | None] = mapped_column(String, nullable=True)
privileges_required: Mapped[str | None] = mapped_column(String, nullable=True)
user_interaction: Mapped[str | None] = mapped_column(String, nullable=True)
scope: Mapped[str | None] = mapped_column(String, nullable=True)
confidentiality_impact: Mapped[str | None] = mapped_column(String, nullable=True)
integrity_impact: Mapped[str | None] = mapped_column(String, nullable=True)
availability_impact: Mapped[str | None] = mapped_column(String, nullable=True)
subsequent_confidentiality_impact: Mapped[str | None] = mapped_column(String, nullable=True)
subsequent_integrity_impact: Mapped[str | None] = mapped_column(String, nullable=True)
subsequent_availability_impact: Mapped[str | None] = mapped_column(String, nullable=True)
exploit_maturity: Mapped[str | None] = mapped_column(String, nullable=True)
remediation_level: Mapped[str | None] = mapped_column(String, nullable=True)
report_confidence: Mapped[str | None] = mapped_column(String, nullable=True)
metrics: Mapped[dict[str, str] | None] = mapped_column(JSON, nullable=True)
reachability: Mapped[int] = mapped_column(Integer, nullable=False)
remotely_reachable: Mapped[bool] = mapped_column(Boolean, nullable=False)
# ORM relationship to dependency findings
finding: Mapped[DependencyFinding] = relationship("DependencyFinding", back_populates="cvss")
def print_as_cc_common_result(self) -> None:
"""Prints the CVSS scoring objects content."""
from .helper import print_cvss_table
print_cvss_table([self])
class ComponentScan(Base):
"""Records that one scan found the project depending on one package.
The table the component table stopped carrying when a package became a package rather than a
per-scan row. A component says what it is; this says who ran into it and when.
Attributes:
id (int):
Internal database primary key.
component_scan_id (UUID):
Stable handle of the record.
scan_id (UUID):
The scan that found the dependency.
component_id (UUID):
The package it found.
artifact_id (UUID | None):
The document it read it out of, when there was one.
"""
__tablename__ = "component_scans"
__table_args__ = (
# A scan depends on a package once, however many documents of that scan mention it.
UniqueConstraint("scan_id", "component_id", name="uq_component_scans_dependency"),
)
id: Mapped[int] = mapped_column(Integer, primary_key=True)
component_scan_id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), unique=True, nullable=False)
scan_id: Mapped[UUID] = mapped_column(
PG_UUID(as_uuid=True),
ForeignKey("scans.scan_id", ondelete="CASCADE"),
nullable=False,
)
component_id: Mapped[UUID] = mapped_column(
PG_UUID(as_uuid=True),
ForeignKey("components.component_id", ondelete="CASCADE"),
nullable=False,
)
artifact_id: Mapped[UUID | None] = mapped_column(
PG_UUID(as_uuid=True),
ForeignKey("artifacts.artifact_id", ondelete="SET NULL"),
nullable=True,
)
# ORM relationships
scan: Mapped[Scan] = relationship("Scan", back_populates="component_scans")
component: Mapped[Component] = relationship("Component", back_populates="scans")
class DepFindingScan(Base):
"""Records that one scan reported one advisory.
Separate from :class:`ComponentScan` because the two say different things, and a later scan can
say one without the other: it still depends on the package, and the advisory against it has
since been withdrawn or fixed. Depending on something is not the same as being told about it.
Attributes:
id (int):
Internal database primary key.
dep_finding_scan_id (UUID):
Stable handle of the record.
scan_id (UUID):
The scan that reported the advisory.
finding_dep_id (UUID):
The advisory it reported.
artifact_id (UUID | None):
The document it read it out of, when there was one.
dependency_track_id (UUID | None):
The Dependency-Track project the export was taken of. A notification
arriving weeks later names that project and nothing else, so this is
what matches it back to a scan.
epss_score (float | None):
The probability that this advisory is exploited somewhere in the next
thirty days, 0 to 1, **as it stood when this scan ran**. EPSS is
recomputed daily, so it belongs to the occurrence and not to the
advisory -- and keeping it here means a series of scans carries the
history of that probability for free.
epss_percentile (float | None):
Where that probability ranked among all scored vulnerabilities at the
same moment. Uniform by construction, which is what makes it the
comparable half of the pair.
analyzer_identity (str | None):
Which Dependency-Track analyzer reported it, e.g.
``TRIVY_ANALYZER``. Two analyzers disagree about the same project, so
this belongs to the run rather than to the advisory.
is_suppressed (bool | None):
Whether Dependency-Track was told to suppress it at the time of the
export. Somebody's judgement, recorded as what it is: a statement of
the exporting system, not of this store.
"""
__tablename__ = "dep_finding_scans"
__table_args__ = (
# A scan reports an advisory once, however many documents of that scan carry it.
UniqueConstraint("scan_id", "finding_dep_id", name="uq_dep_finding_scans_occurrence"),
Index("ix_dep_finding_scans_project", "dependency_track_id"),
# What "show me what is actually likely to be exploited" runs on.
Index("ix_dep_finding_scans_epss", "epss_score"),
)
id: Mapped[int] = mapped_column(Integer, primary_key=True)
dep_finding_scan_id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), unique=True, nullable=False)
scan_id: Mapped[UUID] = mapped_column(
PG_UUID(as_uuid=True),
ForeignKey("scans.scan_id", ondelete="CASCADE"),
nullable=False,
)
finding_dep_id: Mapped[UUID] = mapped_column(
PG_UUID(as_uuid=True),
ForeignKey("dependency_findings.finding_dep_id", ondelete="CASCADE"),
nullable=False,
)
artifact_id: Mapped[UUID | None] = mapped_column(
PG_UUID(as_uuid=True),
ForeignKey("artifacts.artifact_id", ondelete="SET NULL"),
nullable=True,
)
dependency_track_id: Mapped[UUID | None] = mapped_column(PG_UUID(as_uuid=True), nullable=True)
epss_score: Mapped[float | None] = mapped_column(Float, nullable=True)
epss_percentile: Mapped[float | None] = mapped_column(Float, nullable=True)
analyzer_identity: Mapped[str | None] = mapped_column(String, nullable=True)
is_suppressed: Mapped[bool | None] = mapped_column(Boolean, nullable=True)
# ORM relationships
scan: Mapped[Scan] = relationship("Scan", back_populates="dep_finding_scans")
finding: Mapped[DependencyFinding] = relationship("DependencyFinding", back_populates="scans")
report_generator/vendor/sync.py
Vendored
+103
View File
@@ -0,0 +1,103 @@
"""Copies everything this tool borrows from the monorepo into ``vendor/``.
The package is meant to be liftable into a repository of its own, where none of ``cc_common`` or
``persistent_storage`` exists. The copies here are what it would need to take along.
They are produced by a script rather than by hand for one reason: a copy made once goes stale
without saying so. Re-run this after touching anything it lists and the diff shows what drifted.
python -m persistent_storage.report_generator.vendor.sync
Copied under their own import paths, so ``vendor/`` on ``sys.path`` makes the package import
unchanged in the new repository.
"""
from __future__ import annotations
import shutil
import sys
from pathlib import Path
# Where the monorepo's sources sit, relative to this file.
SOURCE_ROOT = Path(__file__).resolve().parents[4]
VENDOR_ROOT = Path(__file__).resolve().parent
# What the tool reaches for, and nothing beyond it. A whole package is copied where the tool uses
# enough of it that picking modules would only mean chasing imports on the next change.
COPIED = (
# The CLI framework, the logging context and the error type every command raises.
"cc_common",
# The scan and artifact rows, the storage configuration, and the readers over them.
"persistent_storage/__init__.py",
"persistent_storage/shared.py",
"persistent_storage/wrapper.py",
"persistent_storage/helper.py",
"persistent_storage/utils/__init__.py",
"persistent_storage/utils/db_utils.py",
"persistent_storage/utils/s3_utils.py",
# The CVSS reader the vulnerability summariser works through.
"persistent_storage/utils/cvss_utils.py",
# The two summarisers the report is built from. They live in the web application today and are
# written to be free of it -- no Django, no store -- which is what makes copying them honest
# rather than a workaround. Wherever this package ends up, they belong beside it.
"cipherverifier/__init__.py",
"cipherverifier/core/__init__.py",
"cipherverifier/core/services/__init__.py",
"cipherverifier/core/services/sarif.py",
"cipherverifier/core/services/vdr.py",
)
# Never copied: caches and the compiled leftovers of whoever ran a tool over the source tree last.
# They live inside the packages being copied, so leaving any of them out puts megabytes of someone
# else's machine into the vendor tree.
SKIPPED = shutil.ignore_patterns(
"__pycache__", "*.pyc", ".DS_Store", ".mypy_cache", ".ruff_cache", ".pytest_cache", "*.egg-info"
)
def sync() -> list[Path]:
"""Refresh every copy, replacing what is there.
The copied packages are removed whole before anything is written, so that dropping an entry
from :data:`COPIED` actually drops it from the tree. Replacing only what is listed leaves what
used to be listed behind, and the copy then carries files nothing refers to any more -- which is
the same staleness this script exists to prevent, one level up.
Returns:
What was written, relative to ``vendor/``.
"""
for root in {entry.split("/")[0] for entry in COPIED}:
stale = VENDOR_ROOT / root
if stale.is_dir():
shutil.rmtree(stale)
elif stale.exists():
stale.unlink()
written = []
for entry in COPIED:
source = SOURCE_ROOT / "src" / entry
target = VENDOR_ROOT / entry
if not source.exists():
message = f"{source} does not exist -- the monorepo moved it, and this list has not followed."
raise FileNotFoundError(message)
target.parent.mkdir(parents=True, exist_ok=True)
if source.is_dir():
shutil.copytree(source, target, ignore=SKIPPED)
else:
shutil.copy2(source, target)
written.append(Path(entry))
return written
if __name__ == "__main__":
for path in sync():
sys.stdout.write(f"{path}\n")