commit 06593ac488d131a63b8e431a456ed7f807abaac1 Author: Florian Frank Date: Fri Sep 18 16:58:29 2026 +0200 Initial commit. monorepo deps are copied in a dirty way to the vendor folder diff --git a/report_generator/commands/__init__.py b/report_generator/commands/__init__.py new file mode 100644 index 0000000..7675758 --- /dev/null +++ b/report_generator/commands/__init__.py @@ -0,0 +1 @@ +"""The commands the report generator exposes.""" diff --git a/report_generator/commands/generate.py b/report_generator/commands/generate.py new file mode 100644 index 0000000..463fb02 --- /dev/null +++ b/report_generator/commands/generate.py @@ -0,0 +1,88 @@ +"""report_generator generate command.""" + +from pathlib import Path +from uuid import UUID + +from cc_common.api import PipelineError, context +from cc_common.utils import print +from pydantic import BaseModel, Field + +from persistent_storage.shared import StorageConfiguration + +from ..rendering import render +from ..sources import collect + +# Where the report goes when the caller names no path. +DEFAULT_DIRECTORY = Path("reports") + + +class GenerateProperties(BaseModel): + """Configuration properties for the generate CLI command. + + Attributes: + scan_id: The scan to report on. + path: Where to write the PDF. A directory is written into under the name + the renderer chose; leaving it out writes into ``reports/``. + storage_config: The store the scan's artifacts are read from. + """ + + scan_id: UUID = Field(description="The scan the report should be generated for.") + path: Path | None = Field(default=None, description="Where to write the PDF report.") + storage_config: StorageConfiguration + + +class GeneratedReport(BaseModel): + """The report that was written. + + Attributes: + scan_id: The scan it is about. + path: The file written. + size: How large it came out, in bytes. + """ + + scan_id: UUID + path: Path + size: int + + def print_as_cc_common_result(self) -> None: + """Say what was written and where.""" + print(f"[bold]Report for scan {self.scan_id}[/bold]") + print(f" {self.size / 1024:.0f} kB written to {self.path}") + + +def generate(properties: GenerateProperties) -> GeneratedReport: + """Build the PDF report of one scan and write it. + + Args: + properties: Parsed command properties naming the scan and the destination. + + Returns: + The report that was written. + + Raises: + PipelineError: The scan cannot be read, is missing an input the report + needs, or the destination cannot be written. + """ + sources = collect(properties.storage_config, properties.scan_id) + name, content = render(sources) + destination = _destination(properties.path, name) + + try: + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_bytes(content) + except OSError as error: + message = f"Cannot write the report to '{destination}': {error}" + + raise PipelineError(message) from error + + context.logger.info(f"Report written to {destination} ✔") + + return GeneratedReport(scan_id=properties.scan_id, path=destination, size=len(content)) + + +def _destination(path: Path | None, name: str) -> Path: + """Where the report goes, treating a directory as a place to put it rather than as the file.""" + if path is None: + return DEFAULT_DIRECTORY / name + + return path / name if path.is_dir() or not path.suffix else path diff --git a/report_generator/context.py b/report_generator/context.py new file mode 100644 index 0000000..1369900 --- /dev/null +++ b/report_generator/context.py @@ -0,0 +1,595 @@ +"""Turns a scan's stored artifacts into the context the report template draws. + +No parsing of its own. The SARIF and the vulnerability report are read by the summarisers in +:mod:`cipherverifier.core.services.sarif` and :mod:`cipherverifier.core.services.vdr` -- both of +which are deliberately free of Django and of any store -- and the pipeline metadata by the helpers +in :mod:`persistent_storage.utils.rep_utils`. What is left here is the mapping: which summarised +number goes in which row, and what to say where the artifacts do not say anything. + +That last part is the rule this module is built around. A field the artifacts cannot answer is +written as :data:`UNKNOWN` rather than guessed, left blank or quietly dropped. A reader who sees +```` knows the pipeline did not report it; a reader who sees a plausible zero does not. + +Moving this to archon is a change of what :mod:`sources` reads +from. The two summarisers travel with it -- they belong beside whatever holds the artifacts, not +inside the web application that happens to import them today. +""" + +from __future__ import annotations + +from datetime import UTC, datetime +from typing import TYPE_CHECKING, Any + +from cipherverifier.core.services.sarif import summarize_sarif +from cipherverifier.core.services.vdr import summarize_vdr + +if TYPE_CHECKING: + from cipherverifier.core.services.sarif import Finding, SarifReport + from cipherverifier.core.services.vdr import VdrReport, Vulnerability + + from .sources import ReportSources + +# What stands in for a field the stored artifacts do not answer. +UNKNOWN = "" + +# How many of each list the report carries. A report is read, not audited; the full set is in the +# artifacts it was built from. +MOST_CRITICAL = 6 +CARDS_PER_GROUP = 8 +BARS = 10 + +# The severity a SARIF level is reported under. SARIF names four levels and the report bands five, +# so the mapping is written down rather than inferred from the spelling. +SEVERITY_OF_LEVEL = { + "error": "Critical", + "warning": "High", + "note": "Medium", + "none": "Low", + "info": "Low", +} + +SEVERITY_ORDER = ("Critical", "High", "Medium", "Low") + + +def build_context(sources: ReportSources, documents: dict[str, Any]) -> dict[str, Any]: + """Build the report context from a scan's parsed artifacts. + + Args: + sources: The scan and its stored files. + documents: The parsed artifacts by role -- ``metadata``, ``findings``, + ``vulnerabilities`` -- each absent where the scan stored none. + + Returns: + The context :func:`layout.render_pdf` + draws. Sections whose artifact is missing are left out entirely, which + the template renders as a shorter report. + """ + metadata = documents.get("metadata") or {} + sast = summarize_sarif(documents["findings"]) if documents.get("findings") else None + sca = summarize_vdr(documents["vulnerabilities"]) if documents.get("vulnerabilities") else None + + context: dict[str, Any] = { + "cover": _cover(sources, metadata, sast, sca), + "scan": _scan(sources, metadata), + "project": _project(metadata), + "summary": _summary(sast, sca), + "sections": [], + } + + critical = _critical(sast, sca) + + if critical["table"]["rows"]: + context["critical"] = critical + + if sast is not None: + context["sections"].append(_sast_section(sast)) + + if sca is not None: + context["sections"].append(_sca_section(sca)) + + return context + + +# --- the page ------------------------------------------------------------------------------ + + +def _fact(label: str, value: Any, *, gap: bool = False, kind: str | None = None) -> dict[str, Any]: # noqa: ANN401 + """One label/value row, with anything unstated written as :data:`UNKNOWN`.""" + return {"label": label, "value": _stated(value), "gap": gap, "kind": kind} + + +def _row(label: str, value: Any, **marks: Any) -> dict[str, Any]: # noqa: ANN401 + """One row of a card.""" + return {"label": label, "value": _stated(value), **marks} + + +def _stated(value: Any) -> str: # noqa: ANN401 + """What a value reads as, or :data:`UNKNOWN` where there is nothing to read. + + Zero is a reading and survives; an empty string and ``None`` are not, and do not. The difference + matters on a page where both appear: "0 filtered" is a measurement, and a blank is a gap. + """ + if value is None or (isinstance(value, str) and not value.strip()): + return UNKNOWN + + return str(value) + + +def _cover( + sources: ReportSources, + metadata: dict[str, Any], + sast: SarifReport | None, + sca: VdrReport | None, +) -> dict[str, Any]: + """The first page.""" + configuration = metadata.get("configuration") or {} + technologies = [name for name, present in (("SAST", sast is not None), ("SCA", sca is not None)) if present] + + return { + "scan_id": str(sources.scan_id), + "subject": _stated(configuration.get("repository_name")), + "footer_date": datetime.now(UTC).strftime("%d %B %Y"), + "logo": "cc_logo.png", + "title": "SAST/SCA Report", + "identity": [ + _fact("Scan Identifier", sources.scan_id, kind="mono"), + # Neither exists in the store today: a scan has artifacts, and a report of it is not + # itself recorded as one. + _fact("Report ID", None, kind="mono"), + _fact("Artifact ID", None, kind="mono"), + _fact("Technology", ", ".join(technologies) or None), + _fact("Project", configuration.get("repository_name"), gap=True), + _fact("Version", configuration.get("repository_version")), + _fact("Report Created", datetime.now(UTC).strftime("%A, %B %d, %Y")), + _fact("Manufacturer", None), + ], + "severity_ledger": _severity_ledger(sast, sca), + } + + +def _severity_ledger(sast: SarifReport | None, sca: VdrReport | None) -> dict[str, Any] | None: + """How many issues of each severity survived filtering, over both halves of the pipeline.""" + counts = dict.fromkeys(SEVERITY_ORDER, 0) + + for finding in _true_findings(sast): + counts[SEVERITY_OF_LEVEL.get(finding.level, "Low")] += 1 + + for vulnerability in sca.vulnerabilities if sca else []: + counts[_vdr_severity(vulnerability)] = counts.get(_vdr_severity(vulnerability), 0) + 1 + + if not any(counts.values()): + return None + + rows: list[dict[str, Any]] = [ + {"label": f"{name} severity issues", "count": counts.get(name, 0)} for name in SEVERITY_ORDER + ] + rows.append({"label": "Total true issues", "count": sum(counts.values()), "total": True}) + + return {"title": "True issues by severity", "rows": rows} + + +def _scan(sources: ReportSources, metadata: dict[str, Any]) -> dict[str, Any]: + """What the pipeline did, stage by stage.""" + stages = [stage for stage in metadata.get("stages") or [] if isinstance(stage, dict)] + failed = [stage["label"] for stage in stages if stage.get("status") not in ("success", None)] + + return { + "title": "CipherCell Scan Information", + "overview": [ + _fact("Scan started at", _moment(metadata.get("pipeline_start_iso"))), + _fact("Scan duration", _duration(metadata.get("pipeline_duration_seconds"))), + _fact("Scan completed at", _moment(metadata.get("pipeline_end_iso"))), + _fact( + "Pipeline Steps", ", ".join(_stated(stage.get("label") or stage.get("key")) for stage in stages) or None + ), + _fact("Errors Detected", ", ".join(failed) if failed else "None" if stages else None), + ], + "stages": [_stage(stage, sources) for stage in stages], + } + + +def _stage(stage: dict[str, Any], sources: ReportSources) -> dict[str, Any]: + """One stage of the pipeline. + + The tools a stage ran and the profile it ran them under are not recorded in the metadata, so + they are left unstated. What is recorded is when it ran, how it ended, and -- indirectly -- what + it produced, which is read off the names of the files the scan stored. + """ + return { + "title": stage.get("label") or stage.get("key") or UNKNOWN, + "rows": [ + _fact("Selected Tools", None), + _fact("Profile", None), + _fact("Duration", _duration(stage.get("duration_seconds"))), + _fact("Created Artifacts", ", ".join(sorted(sources.files)) or None, kind="mono"), + _fact("Errors Detected", "None" if stage.get("status") == "success" else stage.get("status")), + ], + } + + +def _project(metadata: dict[str, Any]) -> dict[str, Any]: + """What was scanned.""" + configuration = metadata.get("configuration") or {} + + return { + "title": "Project Information", + "overview": [ + _fact("Project", configuration.get("repository_name")), + _fact("Version", configuration.get("repository_version")), + # Nothing in the pipeline counts lines, so nothing here can report them. + _fact("Analysed lines of code", None), + ], + # Left as one unstated row rather than a table of zeroes: no stage reports a language + # breakdown, and a share invented from the file extensions of the findings would be a + # measurement of where the bugs are, not of what the project is written in. + "languages": {"title": "Languages", "rows": [{"label": UNKNOWN, "share": UNKNOWN}]}, + } + + +def _summary(sast: SarifReport | None, sca: VdrReport | None) -> dict[str, Any]: + """The counts, and where they concentrate.""" + true_sast = _true_findings(sast) + sca_total = sca.total if sca else 0 + raw = (sast.total if sast else 0) + sca_total + filtered = sast.false_positive_count if sast else 0 + + return { + "title": "Summary", + "overview": [ + _fact("Raw findings", raw or None), + # Only SAST findings carry a filtering verdict. A vulnerability suppressed in + # Dependency-Track never reaches the report at all, so the two cannot be added up. + _fact("Filtered", f"{filtered} (SAST)" if sast else None), + _fact("True issues", f"{len(true_sast)} SAST, {sca_total} SCA" if sast or sca else None), + # Each half stated only where its artifact was there. A scan that ran no SCA stage + # reports its SAST rules and says nothing about components, rather than nothing at all. + _fact("Distinct issue types (true)", ", ".join(_issue_types(sast, sca)) or None), + ], + "distributions": _distributions(true_sast, sca), + } + + +def _issue_types(sast: SarifReport | None, sca: VdrReport | None) -> list[str]: + """How many distinct kinds of issue each half of the pipeline found.""" + counted = [] + + if sast is not None: + counted.append(f"{sast.rule_count} SAST rules") + + if sca is not None: + counted.append(f"{sca.component_count} affected components") + + return counted + + +def _distributions(findings: list[Finding], sca: VdrReport | None) -> list[dict[str, Any]]: + """Where the issues concentrate: by rule for SAST, by package for SCA.""" + blocks = [] + + if findings: + blocks.append( + { + "title": f"True SAST issues by rule ({len(findings)})", + **_bars(_group(findings, lambda one: one.rule_id or UNKNOWN, _finding_severity)), + } + ) + + if sca and sca.vulnerabilities: + blocks.append( + { + "title": f"Most affected components, true SCA issues ({sca.component_count})", + **_bars( + _group( + [(one, name) for one in sca.vulnerabilities for name in one.component_names], + lambda pair: pair[1], + lambda pair: _vdr_severity(pair[0]), + ), + mono=True, + ), + } + ) + + return blocks + + +def _group[T]( + items: list[T], + label_of: Any, # noqa: ANN401 + severity_of: Any, # noqa: ANN401 +) -> list[tuple[str, dict[str, int]]]: + """Count items by label, and within each label by severity.""" + grouped: dict[str, dict[str, int]] = {} + + for item in items: + counts = grouped.setdefault(label_of(item), dict.fromkeys(SEVERITY_ORDER, 0)) + counts[severity_of(item)] += 1 + + return sorted(grouped.items(), key=lambda pair: -sum(pair[1].values())) + + +def _bars(grouped: list[tuple[str, dict[str, int]]], *, mono: bool = False) -> dict[str, Any]: + """Turn grouped counts into bars scaled against the busiest row. + + Only the busiest :data:`BARS` are drawn, and the caveat says so where the rest were left off. + Without that the legend counts what is on the page while the heading counts the report, and the + two quietly disagree by however much the tail held. + """ + shown = grouped[:BARS] + largest = max((sum(counts.values()) for _, counts in shown), default=0) or 1 + rows: list[dict[str, Any]] = [] + totals = dict.fromkeys(SEVERITY_ORDER, 0) + + for label, counts in shown: + rows.append( + { + "label": label, + "count": sum(counts.values()), + "mono": mono, + "parts": [ + {"severity": name, "share": round(counts[name] / largest * 100, 1)} + for name in SEVERITY_ORDER + if counts[name] + ], + } + ) + + for name, count in counts.items(): + totals[name] += count + + return { + "rows": rows, + "legend": { + "entries": [ + {"severity": name, "label": f"{name} ({totals[name]})"} + for name in reversed(SEVERITY_ORDER) + if totals[name] + ], + "caveat": _caveat(shown, grouped, largest), + }, + } + + +def _caveat(shown: list[Any], grouped: list[Any], largest: int) -> str: + """What the bars leave out, and what their length is measured against.""" + scale = f"bar length is proportional to the issue count (100 % = {largest})." + + if len(shown) == len(grouped): + return scale + + return f"{len(shown)} of {len(grouped)} shown; {scale}" + + +def _critical(sast: SarifReport | None, sca: VdrReport | None) -> dict[str, Any]: + """The handful worth reading first, whichever half of the pipeline found them.""" + rows: list[dict[str, Any]] = [] + + for finding in sorted(_true_findings(sast), key=lambda one: SEVERITY_ORDER.index(_finding_severity(one))): + rows.append( + { + "cells": [ + "SAST", + {"value": _finding_severity(finding), "kind": "severity"}, + _stated(finding.rule_id), + {"value": _stated(_where(finding)), "kind": "mono"}, + _stated(finding.message), + ] + } + ) + + for vulnerability in sorted( + sca.vulnerabilities if sca else [], + key=lambda one: SEVERITY_ORDER.index(_vdr_severity(one)), + ): + rows.append( + { + "cells": [ + "SCA", + {"value": _vdr_severity(vulnerability), "kind": "severity"}, + _stated(vulnerability.id), + {"value": _stated(", ".join(vulnerability.component_names)), "kind": "mono"}, + _stated(vulnerability.description), + ] + } + ) + + return { + "title": "Most Critical Vulnerabilities", + "table": { + "columns": ["Type", "Severity", "CVE/CWE", "Affected File / Component", "Description"], + "rows": rows[:MOST_CRITICAL], + }, + } + + +def _sast_section(sast: SarifReport) -> dict[str, Any]: + """The SAST findings, split into what was kept and what was set aside.""" + true_findings = _true_findings(sast) + filtered = [one for one in sast.findings if one.false_positive] + groups: list[dict[str, Any]] = [] + + if true_findings: + groups.append( + { + "title": f"True Issues ({len(true_findings)})", + "cards": [_finding_card(one) for one in true_findings[:CARDS_PER_GROUP]], + } + ) + + if filtered: + groups.append( + { + "title": f"Filtered Issues ({len(filtered)})", + "note": ( + "The following findings were reported by the scanners but assessed as not exploitable in " + "this artifact. Each entry carries the verdict that filtered it." + ), + "cards": [_finding_card(one, filtered_out=True) for one in filtered[:CARDS_PER_GROUP]], + "table": _filtered_table(filtered), + } + ) + + return {"title": "SAST Vulnerabilities", "groups": groups} + + +def _finding_card(finding: Finding, *, filtered_out: bool = False) -> dict[str, Any]: + """One SARIF finding.""" + rows = [ + _row("Severity", _finding_severity(finding), kind="severity"), + _row("Location", _where(finding), kind="mono"), + _row("Description", finding.message), + _row("Reported By", ", ".join(finding.tools) or None), + ] + + if filtered_out: + rows.append( + _row( + "Filtering Reason", + finding.polygraph_reasoning, + aside=True, + token=finding.polygraph_verdict, + ) + ) + else: + # Polygraph's verdict where it looked at this one, and nothing where it did not: silence is + # not a clean bill of health, and must not be written as one. + rows.append(_row("Evidence", finding.verdict_label or None)) + + return {"title": f"{_stated(finding.rule_id)} — {_stated(finding.rule_name)}", "rows": rows} + + +def _filtered_table(findings: list[Finding]) -> dict[str, Any]: + """How many of each rule were filtered, and on what grounds.""" + grouped: dict[str, list[Finding]] = {} + + for finding in findings: + grouped.setdefault(finding.rule_id or UNKNOWN, []).append(finding) + + rows: list[dict[str, Any]] = [ + { + "cells": [ + rule, + len(group), + {"value": _stated(group[0].polygraph_verdict), "kind": "mono"}, + ] + } + for rule, group in sorted(grouped.items(), key=lambda pair: -len(pair[1])) + ] + rows.append({"cells": ["Total", len(findings), ""], "total": True}) + + return { + "title": f"Filtered SAST issues by rule ({len(findings)})", + "columns": ["Rule", "Filtered", "Dominant verdict"], + "rows": rows, + } + + +def _sca_section(sca: VdrReport) -> dict[str, Any]: + """The dependency findings.""" + return { + "title": "SBOM / SCA Vulnerabilities", + "new_page": True, + "groups": [ + { + "title": f"True Issues by CVEs ({sca.total})", + "cards": [_vulnerability_card(one) for one in sca.vulnerabilities[:CARDS_PER_GROUP]], + } + ], + } + + +def _vulnerability_card(vulnerability: Vulnerability) -> dict[str, Any]: + """One dependency finding, as much of it as the report stated.""" + metrics = vulnerability.metrics + impact = [ + f"{name} ({value})" + for name, value in ( + ("Confidentiality", metrics.confidentiality), + ("Integrity", metrics.integrity), + ("Availability", metrics.availability), + ) + if value + ] + + return { + "title": _stated(vulnerability.id), + "rows": [ + _row("Affected component", ", ".join(vulnerability.component_names) or None, kind="mono"), + _row("Severity", _vdr_severity(vulnerability), kind="severity"), + _row("Description", vulnerability.description), + _row("CWE", ", ".join(f"{one}: {vulnerability.cwe_names[one]}" for one in vulnerability.cwes) or None), + _row("CVSS", f"{vulnerability.score} / 10" if vulnerability.score is not None else None), + _row("Impact", ", ".join(impact) or None), + _row("Exploitation likelihood (EPSS)", _percent(metrics.epss)), + _row( + "Fixed in", + # The report names the newest version of the package, which is not the same as the + # version that fixes this advisory -- so it is labelled as what it is. + next( + (one.latest_version for one in vulnerability.components if one.is_outdated), + None, + ), + kind="mono", + ), + _row("Known for", f"{metrics.published_days_ago} days" if metrics.published_days_ago else None), + _row("Recommendation", vulnerability.recommendation), + ], + } + + +# --- readings ------------------------------------------------------------------------------ + + +def _true_findings(sast: SarifReport | None) -> list[Finding]: + """The findings that survived filtering.""" + return [one for one in sast.findings if not one.false_positive] if sast else [] + + +def _finding_severity(finding: Finding) -> str: + """The severity band a SARIF level falls in.""" + return SEVERITY_OF_LEVEL.get(finding.level, "Low") + + +def _vdr_severity(vulnerability: Vulnerability) -> str: + """The severity band a vulnerability reports, capitalised the way the report bands them.""" + named = (vulnerability.severity or "").capitalize() + + return named if named in SEVERITY_ORDER else "Low" + + +def _where(finding: Finding) -> str | None: + """Where a finding points, as ``file:line``.""" + if finding.location is None: + return None + + line = finding.location.start_line + + return f"{finding.location.file}:{line}" if line else finding.location.file + + +def _percent(share: float | None) -> str | None: + """A probability as a percentage, at the precision it is worth stating.""" + if share is None: + return None + + return f"{share * 100:.2f} %" if share < 0.01 else f"{share * 100:.1f} %" + + +def _moment(stated: Any) -> str | None: # noqa: ANN401 + """An ISO timestamp as the report writes it, or nothing where it cannot be read.""" + if not isinstance(stated, str): + return None + + try: + return datetime.fromisoformat(stated).strftime("%A, %B %d, %Y %H:%M:%S (%Z)").replace("()", "").strip() + except ValueError: + return None + + +def _duration(seconds: Any) -> str | None: # noqa: ANN401 + """A number of seconds as ``HH:MM:SS``.""" + if not isinstance(seconds, (int, float)) or isinstance(seconds, bool): + return None + + whole = int(seconds) + + return f"{whole // 3600:02d}:{whole % 3600 // 60:02d}:{whole % 60:02d}" diff --git a/report_generator/layout.py b/report_generator/layout.py new file mode 100644 index 0000000..ee268b4 --- /dev/null +++ b/report_generator/layout.py @@ -0,0 +1,90 @@ +"""Renders the SAST/SCA report template. + +The template and its stylesheet are the design; this module is only what turns them into a +document. It takes a context that is already shaped for the page -- every number formatted, every +bar already a percentage -- so that the template places strings and never computes. + +Nothing here reads an artifact. Filling a context from a scan is :mod:`context`, which works from +what :mod:`sources` collected; :func:`sample.sample_context` builds one by hand, which is what the +template tests render. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from jinja2 import Environment, FileSystemLoader, select_autoescape + +TEMPLATE_DIR = Path(__file__).resolve().parent / "template" + +TEMPLATE_NAME = "report.html.j2" + +STYLESHEET = TEMPLATE_DIR / "report.css" + +# What the document calls itself, in the PDF's own metadata rather than only on its first page. +TITLE = "SAST/SCA Report" + +AUTHOR = "CipherCell" + +# The shape :func:`render_html` expects. Written down because the template guards on it: a key that +# is absent leaves its section out, which is how a scan that ran half the pipeline produces a +# shorter report rather than one full of empty tables. +CONTEXT = { + "cover": "scan_id, footer_date, logo, title, identity[], severity_ledger{title, rows[]}", + "scan": "title, overview[], stages[{title, rows[], table?}]", + "project": "title, overview[], languages{title, rows[]}", + "summary": "title, overview[], distributions[{title, rows[], legend}]", + "critical": "title, table{columns[], rows[]}", + "sections": "[{title, new_page?, groups[{title, note?, cards[], table?}]}]", +} + + +def render_html(context: dict[str, Any]) -> str: + """Render the report as an HTML document. + + Args: + context: The page-ready context; see :data:`CONTEXT`. + + Returns: + The rendered HTML. + """ + environment = Environment( + loader=FileSystemLoader(TEMPLATE_DIR), + autoescape=select_autoescape(default_for_string=True, default=True), + trim_blocks=True, + lstrip_blocks=True, + ) + + return environment.get_template(TEMPLATE_NAME).render(**context) + + +def render_pdf(context: dict[str, Any]) -> bytes: + """Render the report as a PDF. + + Args: + context: The page-ready context; see :data:`CONTEXT`. + + Returns: + The PDF. + """ + # WeasyPrint pulls in native cairo and pango bindings. Imported here so that importing this + # module -- to render the HTML, or merely to read CONTEXT -- does not pay for them. + from weasyprint import CSS, HTML + from weasyprint.text.fonts import FontConfiguration + + # Without this the @font-face rules are ignored and Space Grotesk is silently replaced by + # whatever the host happens to have, which is how the design sample came to be set in Carlito. + fonts = FontConfiguration() + + # Relative URLs in the template and the stylesheet -- the mark, the font files -- resolve + # against the template directory. + document = HTML(string=render_html(context), base_url=str(TEMPLATE_DIR)).render( + stylesheets=[CSS(filename=str(STYLESHEET), font_config=fonts)], + font_config=fonts, + ) + + document.metadata.title = f"{TITLE} - {context.get('cover', {}).get('subject', '')}".strip(" -") + document.metadata.authors = [AUTHOR] + + return bytes(document.write_pdf()) diff --git a/report_generator/rendering.py b/report_generator/rendering.py new file mode 100644 index 0000000..eaa3d93 --- /dev/null +++ b/report_generator/rendering.py @@ -0,0 +1,100 @@ +"""Turns collected sources into a PDF. + +Three steps, each somewhere else: this module decides which stored file feeds which part of the +report, :mod:`context` maps the parsed artifacts onto the page, +and :mod:`layout` draws it. + +Adding a section to the report is an entry in :data:`INPUTS` and a block in the template -- not +another renderer. +""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING, Any + +from cc_common.api import PipelineError +from cc_common.api import context as logging_context + +from .context import build_context +from .layout import render_pdf + +if TYPE_CHECKING: + from .sources import ReportSources + +# Which stored file each input of the report is taken from, under the names the pipeline stages +# write. Several per input because the same document travels under more than one name depending on +# which stage produced it; the first one a scan carries wins. +INPUTS = { + "metadata": ("metadata.json", "meta.json"), + "findings": ("sast_scanner_ranked.sarif", "polygraph.sarif", "sast_scanner_combined.sarif", "prism.sarif"), + "vulnerabilities": ("vdr.json", "vdr-2.json", "vdr-3.json"), +} + +# What a report cannot be built without. The rest is reported as far as it was found: a scan that +# ran only half the pipeline is a shorter report, not a refusal. +REQUIRED = ("metadata",) + +# What the file is called. The scan it is about is in the name, because a directory of these is +# otherwise unreadable. +NAME = "report_{scan}.pdf" + + +def render(sources: ReportSources) -> tuple[str, bytes]: + """Render the report of one scan. + + Args: + sources: What :func:`sources.collect` found. + + Returns: + The file name the report should be saved under, and its bytes. + + Raises: + PipelineError: The scan is missing an input the report cannot be built + without, or one of its documents cannot be parsed. Both are reported + by name: "no PDF was produced" leaves whoever ran it guessing which + stage of their pipeline did not finish. + """ + documents = {name: _parsed(sources, name, candidates) for name, candidates in INPUTS.items()} + missing = [name for name in REQUIRED if documents.get(name) is None] + + if missing: + message = f"Scan {sources.scan_id} carries no {' and no '.join(missing)}, so no report can be built." + + raise PipelineError(message) + + found = [name for name, document in documents.items() if document is not None] + + logging_context.logger.info(f"Rendering the report from {', '.join(found)}...") + + return NAME.format(scan=sources.scan_id), render_pdf(build_context(sources, documents)) + + +def _parsed(sources: ReportSources, name: str, candidates: tuple[str, ...]) -> Any | None: # noqa: ANN401 + """Read one of the scan's documents, or nothing where it stored none. + + Args: + sources: The scan and its files. + name: What the document is called in :data:`INPUTS`. + candidates: The file names it may have been stored under. + + Returns: + The parsed JSON, or ``None`` when the scan carries none. + + Raises: + PipelineError: The file is there and is not readable. Kept apart from an + absent one on purpose: one is a pipeline that did not run a stage, + the other is a stage that produced something broken, and a report + that treated them alike would hide the second. + """ + content = sources.first(*candidates) + + if content is None: + return None + + try: + return json.loads(content) + except ValueError as error: + message = f"The {name} of scan {sources.scan_id} is not readable JSON: {error}" + + raise PipelineError(message) from error diff --git a/report_generator/sample.py b/report_generator/sample.py new file mode 100644 index 0000000..35ea060 --- /dev/null +++ b/report_generator/sample.py @@ -0,0 +1,525 @@ +"""A complete report context, filled by hand. + +Every section the template can draw appears here at least once, with the content of the design +sample in ``sample_report_template/``. It is what the tests render, and what to compare a real +context against once one is built from a scan: whatever the sample shows, the template supports. + +Written out rather than generated so that it stays readable as a specification of the shape. When a +section gains a field, this file is where it is seen first. +""" + +from __future__ import annotations + +from typing import Any + +SCAN_ID = "3f7a1c9e-8b42-4d61-9c05-27ae1f6b3d80" + + +def _fact(label: str, value: str, *, gap: bool = False, kind: str | None = None) -> dict[str, Any]: + """One label/value row, optionally opening a gap above itself.""" + return {"label": label, "value": value, "gap": gap, "kind": kind} + + +def _row( + label: str, + value: str, + *, + aside: bool = False, + kind: str | None = None, + token: str | None = None, +) -> dict[str, Any]: + """One row of a card. + + Args: + label: What the row is called. + value: What it says. + aside: Tints the row, for the one that argues rather than states. + kind: ``severity`` to colour it, ``mono`` for anything a reader might + have to type or compare character by character. + token: A VEX justification or similar, set as the token it is and placed + before the prose that explains it. + """ + return {"label": label, "value": value, "aside": aside, "kind": kind, "token": token} + + +def _mono(value: str) -> dict[str, Any]: + """A grid cell set in the monospace face.""" + return {"value": value, "kind": "mono"} + + +def _sev(value: str) -> dict[str, Any]: + """A grid cell carrying a severity's colour.""" + return {"value": value, "kind": "severity"} + + +def _bar(label: str, count: int, parts: list[tuple[str, float]], *, mono: bool = False) -> dict[str, Any]: + """One distribution row: a label, its count, and the severity segments of its bar. + + ``mono`` sets the label in the monospace face, for the rows labelled by a package rather than by + a sentence. + """ + return { + "label": label, + "count": count, + "mono": mono, + "parts": [{"severity": severity, "share": share} for severity, share in parts], + } + + +def _cover() -> dict[str, Any]: + """The first page: who it is about, and the one table that answers "how bad".""" + return { + "scan_id": SCAN_ID, + "subject": "SampleRepository v2.4.0", + "footer_date": "16 September 2026", + "logo": "cc_logo.png", + "title": "SAST/SCA Report", + "identity": [ + _fact("Scan Identifier", SCAN_ID, kind="mono"), + _fact("Report ID", "b64d05f2-1ac7-4e9a-8f13-6d20c9b47e55", kind="mono"), + _fact("Artifact ID", "e1c8a730-5f96-42bb-b0d4-9a7351ec2f18", kind="mono"), + _fact("Technology", "SAST, SCA"), + _fact("Project", "SampleRepository", gap=True), + _fact("Version", "v2.4.0 (commit 9f3c2ab1d4e7)"), + _fact("Report Created", "Wednesday, September 16, 2026"), + _fact("Manufacturer", "Firma XY welche den Code besitzt"), + ], + "severity_ledger": { + "title": "True issues by severity", + "rows": [ + {"label": "Critical severity issues", "count": 3}, + {"label": "High severity issues", "count": 73}, + {"label": "Medium severity issues", "count": 28}, + {"label": "Low severity issues", "count": 11}, + {"label": "Total true issues", "count": 115, "total": True}, + ], + }, + } + + +def _scan() -> dict[str, Any]: + """What the pipeline did, stage by stage, and what each stage was configured with.""" + return { + "title": "CipherCell Scan Information", + "overview": [ + _fact("Scan started at", "Wednesday, September 16, 2026 05:43:14 (UTC)"), + _fact("Scan duration", "07:30:38"), + _fact("Scan completed at", "Wednesday, September 16, 2026 13:13:52 (UTC)"), + _fact( + "Pipeline Steps", + "Preanalysis, SAST, SBOM Dependency Tracking, SBOM Reachability, AI FP Detection, AI Ranking", + ), + _fact("Errors Detected", "None (2 warnings, see SBOM Reachability)"), + ], + "stages": [ + { + "title": "Pre-Analysis Settings", + "rows": [ + _fact("Selected Tools", "Propylon v0.9.4"), + _fact("Profile", "deterministic"), + _fact("Errors Detected", "None"), + ], + }, + { + "title": "SAST Settings", + "rows": [ + _fact("Selected Tools", "ruff v0.14.4, bandit v1.8.6, cppcheck v2.18.0, semgrep v1.145.0"), + _fact("Language Coverage", "C (cppcheck, semgrep), Python (ruff, bandit, semgrep)"), + _fact("Rule Profile", "security — ruleset ciphercell-sec-2026.03, sha256 8f2b41c6…d97a"), + _fact("Deduplication Method", "Deterministic"), + _fact("Created Artifacts", "sast_combined.sarif", kind="mono"), + _fact("Errors Detected", "None"), + ], + }, + { + "title": "SBOM Dependency Tracking Settings", + "rows": [ + _fact("Selected Tools", "syft v1.33.0, cdxgen v11.10.0, deptrace v0.7.2"), + _fact("Profile", "Standard SBOM"), + _fact("SBOM Format", "CycloneDX 1.6"), + _fact("Advisory Sources", "NVD, OSV.dev, GitHub Advisory Database, Sonatype OSS Index"), + _fact("Created Artifacts", "sbom.cdx.json, sbom.vdr.json", kind="mono"), + _fact("Errors Detected", "None"), + ], + }, + { + "title": "SBOM Reachability", + "rows": [ + _fact("Selected Tools", "CodeQL v2.24.1, Joern v4.0.312"), + _fact("Profile", "Default Taint Analysis"), + _fact("Created Artifacts", "sbom_annotated.vdr.json", kind="mono"), + _fact("Errors Detected", "2 warnings"), + ], + "table": { + "columns": ["ID", "Stage", "Warning"], + "rows": [ + { + "cells": [ + "W-01", + "Joern, JavaScript", + "2 of 148 entry points could not be resolved (dynamic require() in " + "server/plugins/loader.js:41). The 6 findings depending on these entry points " + "were escalated and are reported conservatively as true issues.", + ] + }, + { + "cells": [ + "W-02", + "CodeQL, C", + "Database build skipped for vendored sources under third_party/; excluded by " + "scan scope, no reachability verdict is claimed for those paths.", + ] + }, + ], + }, + }, + { + "title": "AI FP Detection", + "rows": [ + _fact("Selected Tools", "Polygraph v1.6.2"), + _fact("Profile", "static and escalation agent"), + _fact("AI Model", "Qwen3-32B, digest sha256:4ab1…c70f"), + _fact("Inference", "On-premise, CipherCell inference cluster"), + _fact("Reasoning Effort", "Default (classification), Default (escalation)"), + _fact("Response Cache", "Enabled"), + _fact("Context Enrichment", "Static (50 lines before, 50 lines after)"), + _fact("Errors Detected", "None"), + ], + }, + { + "title": "Ranking", + "rows": [ + _fact("Selected Tools", "Prism v0.4.1"), + _fact("Effort", "Standard"), + _fact("Context Size", "Minimal"), + _fact("Errors Detected", "None"), + ], + }, + ], + } + + +def _project() -> dict[str, Any]: + """What was scanned, and what it is written in.""" + return { + "title": "Project Information", + "overview": [ + _fact("Project", "SampleRepository"), + _fact("Version", "v2.4.0 (commit 9f3c2ab1d4e7)"), + _fact("Analysed lines of code", "184,920"), + ], + "languages": { + "title": "Languages", + "rows": [ + {"label": "C", "share": "40.2 %"}, + {"label": "Python", "share": "39.6 %"}, + {"label": "JavaScript", "share": "20.2 %"}, + {"label": "Total", "share": "100.0 %", "total": True}, + ], + }, + } + + +def _summary() -> dict[str, Any]: + """The counts, and where they concentrate.""" + return { + "title": "Summary", + "overview": [ + _fact("Raw findings", "210"), + _fact("Filtered", "95 (82 SAST, 13 SCA)"), + _fact("True issues", "115 (86 SAST, 29 SCA)"), + _fact("Distinct issue types (true)", "8 SAST CWEs, 12 affected components"), + ], + "distributions": [ + { + "title": "True SAST issues by CWE (86)", + "rows": [ + _bar("CWE-22: Improper Limitation of a Pathname to a Restricted Directory", 3, [("High", 6)]), + _bar( + "CWE-78: Improper Neutralization of Special Elements used in an OS Command", + 2, + [("Critical", 4)], + ), + _bar("CWE-327: Use of a Broken or Risky Cryptographic Algorithm", 3, [("Medium", 6)]), + _bar("CWE-674: Uncontrolled Recursion", 50, [("High", 84), ("Medium", 16)]), + _bar( + "CWE-770: Allocation of Resources Without Limits or Throttling", + 10, + [("High", 14), ("Medium", 6)], + ), + _bar("CWE-798: Use of Hard-coded Credentials", 4, [("High", 5), ("Medium", 3)]), + _bar("CWE-918: Server-Side Request Forgery", 2, [("High", 4)]), + _bar("CWE-1333: Inefficient Regular Expression Complexity", 12, [("Medium", 5), ("Low", 19)]), + ], + "legend": { + "entries": [ + {"severity": "Low", "label": "Low (9)"}, + {"severity": "Medium", "label": "Medium (20)"}, + {"severity": "High", "label": "High (55)"}, + {"severity": "Critical", "label": "Critical (2)"}, + ], + "caveat": "bar length is proportional to the issue count (100 % = 50).", + }, + }, + { + "title": "Most affected components, true SCA issues (top 4 of 12)", + "rows": [ + _bar("next@16.3.2", 4, [("Critical", 25), ("High", 50), ("Medium", 25)], mono=True), + _bar("glibc@2.36-9+deb12u7", 3, [("High", 75)], mono=True), + _bar("nanoid@3.3.14", 2, [("Medium", 50)], mono=True), + _bar("lodash@4.17.21", 2, [("High", 25), ("Medium", 25)], mono=True), + ], + "legend": { + "entries": [ + {"severity": "Low", "label": "Low"}, + {"severity": "Medium", "label": "Medium"}, + {"severity": "High", "label": "High"}, + {"severity": "Critical", "label": "Critical"}, + ], + "caveat": "11 of 29 true SCA issues shown; 100 % = 4.", + }, + }, + ], + } + + +def _critical() -> dict[str, Any]: + """The handful worth reading first, whichever half of the pipeline found them.""" + return { + "title": "Most Critical Vulnerabilities", + "table": { + "columns": ["Type", "Severity", "CVE/CWE", "Affected File / Component", "Description"], + "rows": [ + { + "cells": [ + "SAST", + _sev("Critical"), + "CWE-78", + _mono("src/tools/exporter.py:57"), + "User-controlled argument passed to subprocess.run(…, shell=True).", + ] + }, + { + "cells": [ + "SCA", + _sev("Critical"), + "GHSA-2xp9-vwfh-vxw4", + _mono("next@16.3.2"), + "Heap buffer overflow in libheif, reached through sharp during AVIF image " + "optimization; allows unauthenticated remote code execution.", + ] + }, + { + "cells": [ + "SAST", + _sev("High"), + "CWE-22", + _mono("src/storage/paths.py:114"), + "Non-literal path component passed to os.path.join().", + ] + }, + ], + }, + } + + +def _sast_section() -> dict[str, Any]: + """The SAST findings: what was kept, and what was set aside and why.""" + return { + "title": "SAST Vulnerabilities", + "groups": [ + { + "title": "True Issues by CWEs", + "cards": [ + { + "title": "CWE-22: Improper Limitation of a Pathname to a Restricted Directory (3 issues)", + "rows": [ + _row("Severity", "High", kind="severity"), + _row( + "Description", + "A path component that is not a string literal is passed to a file system API. " + "Where the component originates from a request parameter, an attacker can " + "traverse outside the intended directory.", + ), + ], + }, + { + "title": "CWE-22 #1 — sast-9f3c2ab-0001", + "rows": [ + _row("Location", "src/storage/paths.py:114", kind="mono"), + _row( + "Description", + "Non-literal path component passed to os.path.join(). The value originates " + 'from request.args["file"] and is not normalised before use.', + ), + _row("Reported By", "bandit v1.8.6 (B108), semgrep v1.145.0"), + _row( + "Reachability", + "Reachable — taint path from entry point api/upload.py:31 (upload_handler), " + "4 hops (Joern)", + ), + _row("Evidence", "Deterministic, no LLM escalation"), + ], + }, + ], + }, + { + "title": "Filtered Issues by CWEs", + "note": ( + "The following findings were reported by the scanners but assessed as not exploitable in " + "this artifact. Each entry carries the filtering reason together with the VEX justification " + "that is written to sbom_annotated.vdr.json. Filtered issues are not counted in the " + "severity summary." + ), + "cards": [ + { + "title": "CWE-1333 #1 — sast-9f3c2ab-0113", + "rows": [ + _row("Location", "server/lib/llm.js:68", kind="mono"), + _row( + "Description", "Backtracking regular expression in the log line parser parseLogLine()." + ), + _row("Reported By", "semgrep v1.145.0"), + _row( + "Filtering Reason", + "no call path from any of the 148 resolved entry points to parseLogLine(); the " + "only caller is the debug CLI, which is excluded from the production build. " + "Deterministic (Joern), no LLM escalation. Decided 2026-09-16 by Polygraph " + "v1.6.2.", + aside=True, + token="code_not_reachable", + ), + ], + }, + ], + "table": { + "title": "Filtered SAST issues by CWE (82)", + "columns": ["CWE", "Filtered", "Dominant VEX justification"], + "rows": [ + {"cells": ["CWE-674: Uncontrolled Recursion", "41", _mono("protected_by_mitigating_control")]}, + { + "cells": [ + "CWE-1333: Inefficient Regular Expression Complexity", + "18", + _mono("code_not_reachable"), + ] + }, + { + "cells": [ + "CWE-327: Use of a Broken or Risky Cryptographic Algorithm", + "9", + "code_not_reachable", + ] + }, + {"cells": ["CWE-798: Use of Hard-coded Credentials", "8", _mono("code_not_present")]}, + {"cells": ["Total", "82", ""], "total": True}, + ], + }, + }, + ], + } + + +def _sca_section() -> dict[str, Any]: + """The dependency findings, in the same two halves.""" + return { + "title": "SBOM / SCA Vulnerabilities", + "new_page": True, + "groups": [ + { + "title": "True Issues by CVEs", + "cards": [ + { + "title": "GHSA-2xp9-vwfh-vxw4", + "rows": [ + _row("Affected component", "next@16.3.2 (via sharp → libheif@1.23.1)", kind="mono"), + _row("CVE", "None assigned"), + _row("Severity", "Critical", kind="severity"), + _row( + "Description", + "A heap buffer overflow in the libheif library used by sharp, which Next.js uses " + "for image optimization, can lead to remote code execution when AVIF files are " + "optimized.", + ), + _row("CVSS", "9.5 / 10 (CVSS v4.0) — CVSS:4.0/AV:N/AC:L/AT:P/PR:N/UI:N/VC:H/VI:H/VA:H"), + _row("Impact", "Confidentiality (High), Integrity (High), Availability (High)"), + _row("Fixed in", "next 16.3.3 / 15.5.24", kind="mono"), + _row( + "Reachability", + "Reachable — AVIF output is enabled in next.config.js:22 " + "(images.formats contains image/avif)", + ), + _row("Known for", "22 days (advisory published 2026-08-25)"), + ], + }, + ], + }, + { + "title": "Filtered Issues by CVEs", + "note": ( + "Vulnerabilities present in the SBOM whose vulnerable code path is not exploitable in this " + "artifact. The filtering reason is carried into sbom.vdr.json as a CycloneDX VEX statement." + ), + "cards": [ + { + "title": "CVE-2026-75604 (GHSA-p293-qw3h-jr36)", + "rows": [ + _row("Affected component", "next@16.3.2", kind="mono"), + _row("Reported Severity", "Critical", kind="severity"), + _row( + "Description", + "Path traversal (CWE-22) in Next.js applications served from a Windows " + "filesystem, leading to unauthenticated remote code execution.", + ), + _row("CVSS", "9.0 / 10 (CVSS v3.1) — CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:C/C:H/I:H/A:H"), + _row("Fixed in", "next 16.3.3 / 15.5.24", kind="mono"), + _row( + "Filtering Reason", + "the vulnerable code path only executes on a Windows filesystem. The deployment " + "target of this artifact is linux/amd64 (base image debian:12-slim, see " + "sbom.cdx.json). Deterministic, no LLM escalation.", + aside=True, + token="requires_environment", + ), + _row("Known for", "22 days (advisory published 2026-08-25)"), + ], + }, + ], + "table": { + "title": "Filtered SCA issues by justification (13)", + "columns": ["VEX justification", "Filtered", "Typical case in this artifact"], + "rows": [ + { + "cells": [ + _mono("code_not_reachable"), + "7", + "Development-scoped dependencies and unused packages", + ] + }, + {"cells": [_mono("requires_environment"), "3", "Windows-only or macOS-only code paths"]}, + { + "cells": [ + _mono("requires_configuration"), + "2", + "Feature flags not enabled in the shipped config", + ] + }, + {"cells": [_mono("code_not_present"), "1", "Vulnerable function removed by the tree shaker"]}, + {"cells": ["Total", "13", ""], "total": True}, + ], + }, + }, + ], + } + + +def sample_context() -> dict[str, Any]: + """A complete context, covering every section the template can draw.""" + return { + "cover": _cover(), + "scan": _scan(), + "project": _project(), + "summary": _summary(), + "critical": _critical(), + "sections": [_sast_section(), _sca_section()], + } diff --git a/report_generator/sources.py b/report_generator/sources.py new file mode 100644 index 0000000..7dab0e9 --- /dev/null +++ b/report_generator/sources.py @@ -0,0 +1,137 @@ +"""Collects everything a scan's report is built from. + +The one place that knows where the bytes come from. Swapping the store -- persistent_storage today, +archon tomorrow -- is a change to :func:`collect` and to nothing else: what it returns is files by +name, and a renderer cannot tell which store handed them over. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, cast +from uuid import UUID + +from cc_common.api import PipelineError, context +from persistent_storage.shared import StorageConfiguration +from persistent_storage.utils.db_utils import get_scan +from persistent_storage.utils.s3_utils import fetch_artifact_files + +if TYPE_CHECKING: + from collections.abc import Sequence + from datetime import datetime + + from mypy_boto3_s3 import S3Client + + from persistent_storage.wrapper import Artifact + + +@dataclass(frozen=True) +class ReportSources: + """One scan and every stored file belonging to it. + + Attributes: + scan_id: The scan the report is about. + run_at: When it ran, which is what the report is dated by. + artifacts: The artifact rows, for the inventory the report lists. + files: The stored bytes by file name. Held in memory because a report is + built once and every section wants a different one of them; the + largest scan in the sample set comes to a few megabytes. + """ + + scan_id: UUID + run_at: datetime + artifacts: Sequence[Artifact] + files: dict[str, bytes] + + def first(self, *names: str) -> bytes | None: + """Return the first of ``names`` the scan stored, or nothing. + + Matched by suffix rather than equality: a stored name carries whatever the producing stage + called the file, and the same document travels under `meta.json` and `metadata.json` + depending on which stage wrote it. + """ + for name in names: + for stored, content in self.files.items(): + if stored.endswith(name): + return content + + return None + + +def _why(configuration: StorageConfiguration, artifacts: Sequence[Artifact], error: Exception) -> str: + """Say which artifacts the store has a row for but no object. + + The bucket reports a missing key without naming it, which on a scan of six artifacts leaves + whoever ran this with nothing to act on. Rows outliving their objects is the ordinary way this + fails -- a bucket restored from nothing, a lifecycle rule, a bucket that was never persistent -- + so the answer worth giving is which files are gone, not that one is. + + Args: + configuration: The store being read. + artifacts: The artifact rows the scan claims. + error: What the fetch raised. + + Returns: + The original message, followed by the missing file names when they can be + established. Diagnosing must never raise on top of the failure it is + explaining, so anything going wrong here falls back to the message alone. + """ + try: + client = cast("S3Client", configuration.s3.s3_client) + missing = [] + + for artifact in artifacts: + try: + client.head_object(Bucket=configuration.s3.bucket, Key=artifact.s3_key) + except Exception: # noqa: BLE001, PERF203 + missing.append(artifact.file_name or artifact.s3_key) + + if not missing: + return str(error) + + return ( + f"the bucket holds no object for {len(missing)} of {len(artifacts)} artifact(s) -- " + f"{', '.join(missing)}. The rows outlived the files; re-upload the scan with " + f"'persistent_storage put'." + ) + except Exception: # noqa: BLE001 + return str(error) + + +def collect(configuration: StorageConfiguration, scan_id: UUID) -> ReportSources: + """Read one scan and its files out of the store. + + Args: + configuration: The store to read from. + scan_id: The scan to collect. + + Returns: + The scan's rows and files. + + Raises: + PipelineError: The scan is unknown, or its files cannot be fetched. + """ + context.logger.info(f"Collecting the artifacts of scan {scan_id}...") + + try: + scan = get_scan(configuration.database, scan_id) + except Exception as error: + message = f"Cannot read scan {scan_id}: {error}" + + raise PipelineError(message) from error + + try: + contents, names = fetch_artifact_files(configuration, [one.artifact_id for one in scan.artifacts]) + except Exception as error: + message = f"Cannot collect the artifacts of scan {scan_id}: {_why(configuration, scan.artifacts, error)}" + + raise PipelineError(message) from error + + context.logger.info(f"Collected {len(names)} file(s) ✔") + + return ReportSources( + scan_id=scan_id, + run_at=scan.run_at, + artifacts=scan.artifacts, + files=dict(zip(names, contents, strict=True)), + ) diff --git a/report_generator/template/cc_logo.png b/report_generator/template/cc_logo.png new file mode 100644 index 0000000..7b7542a Binary files /dev/null and b/report_generator/template/cc_logo.png differ diff --git a/report_generator/template/fonts/OFL.txt b/report_generator/template/fonts/OFL.txt new file mode 100644 index 0000000..f831dbc --- /dev/null +++ b/report_generator/template/fonts/OFL.txt @@ -0,0 +1,93 @@ +Copyright 2020 The Space Grotesk Project Authors (https://github.com/floriankarsten/space-grotesk) + +This Font Software is licensed under the SIL Open Font License, Version 1.1. +This license is copied below, and is also available with a FAQ at: +http://scripts.sil.org/OFL + + +----------------------------------------------------------- +SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007 +----------------------------------------------------------- + +PREAMBLE +The goals of the Open Font License (OFL) are to stimulate worldwide +development of collaborative font projects, to support the font creation +efforts of academic and linguistic communities, and to provide a free and +open framework in which fonts may be shared and improved in partnership +with others. + +The OFL allows the licensed fonts to be used, studied, modified and +redistributed freely as long as they are not sold by themselves. The +fonts, including any derivative works, can be bundled, embedded, +redistributed and/or sold with any software provided that any reserved +names are not used by derivative works. The fonts and derivatives, +however, cannot be released under any other type of license. The +requirement for fonts to remain under this license does not apply +to any document created using the fonts or their derivatives. + +DEFINITIONS +"Font Software" refers to the set of files released by the Copyright +Holder(s) under this license and clearly marked as such. This may +include source files, build scripts and documentation. + +"Reserved Font Name" refers to any names specified as such after the +copyright statement(s). + +"Original Version" refers to the collection of Font Software components as +distributed by the Copyright Holder(s). + +"Modified Version" refers to any derivative made by adding to, deleting, +or substituting -- in part or in whole -- any of the components of the +Original Version, by changing formats or by porting the Font Software to a +new environment. + +"Author" refers to any designer, engineer, programmer, technical +writer or other person who contributed to the Font Software. + +PERMISSION & CONDITIONS +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Font Software, to use, study, copy, merge, embed, modify, +redistribute, and sell modified and unmodified copies of the Font +Software, subject to the following conditions: + +1) Neither the Font Software nor any of its individual components, +in Original or Modified Versions, may be sold by itself. + +2) Original or Modified Versions of the Font Software may be bundled, +redistributed and/or sold with any software, provided that each copy +contains the above copyright notice and this license. These can be +included either as stand-alone text files, human-readable headers or +in the appropriate machine-readable metadata fields within text or +binary files as long as those fields can be easily viewed by the user. + +3) No Modified Version of the Font Software may use the Reserved Font +Name(s) unless explicit written permission is granted by the corresponding +Copyright Holder. This restriction only applies to the primary font name as +presented to the users. + +4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font +Software shall not be used to promote, endorse or advertise any +Modified Version, except to acknowledge the contribution(s) of the +Copyright Holder(s) and the Author(s) or with their explicit written +permission. + +5) The Font Software, modified or unmodified, in part or in whole, +must be distributed entirely under this license, and must not be +distributed under any other license. The requirement for fonts to +remain under this license does not apply to any document created +using the Font Software. + +TERMINATION +This license becomes null and void if any of the above conditions are +not met. + +DISCLAIMER +THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE +COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL +DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM +OTHER DEALINGS IN THE FONT SOFTWARE. diff --git a/report_generator/template/fonts/SpaceGrotesk-Bold.ttf b/report_generator/template/fonts/SpaceGrotesk-Bold.ttf new file mode 100644 index 0000000..15f9ac2 Binary files /dev/null and b/report_generator/template/fonts/SpaceGrotesk-Bold.ttf differ diff --git a/report_generator/template/fonts/SpaceGrotesk-Medium.ttf b/report_generator/template/fonts/SpaceGrotesk-Medium.ttf new file mode 100644 index 0000000..1b235f5 Binary files /dev/null and b/report_generator/template/fonts/SpaceGrotesk-Medium.ttf differ diff --git a/report_generator/template/fonts/SpaceGrotesk-Regular.ttf b/report_generator/template/fonts/SpaceGrotesk-Regular.ttf new file mode 100644 index 0000000..c7ebdd0 Binary files /dev/null and b/report_generator/template/fonts/SpaceGrotesk-Regular.ttf differ diff --git a/report_generator/template/fonts/SpaceMono-Bold.ttf b/report_generator/template/fonts/SpaceMono-Bold.ttf new file mode 100644 index 0000000..2c4f268 Binary files /dev/null and b/report_generator/template/fonts/SpaceMono-Bold.ttf differ diff --git a/report_generator/template/fonts/SpaceMono-Regular.ttf b/report_generator/template/fonts/SpaceMono-Regular.ttf new file mode 100644 index 0000000..1cfa365 Binary files /dev/null and b/report_generator/template/fonts/SpaceMono-Regular.ttf differ diff --git a/report_generator/template/report.css b/report_generator/template/report.css new file mode 100644 index 0000000..9db88c7 --- /dev/null +++ b/report_generator/template/report.css @@ -0,0 +1,422 @@ +/* The print stylesheet of the SAST/SCA report. + * + * Follows the design sample in `sample_report_template/`. Everything is expressed here rather than + * in the template: the template places content, this decides what it looks like, and the two are + * changed for different reasons. + * + * Space Grotesk throughout, with a monospace face for anything a reader might have to type or + * compare character by character -- identifiers, paths, package URLs, CVSS vectors, VEX + * justifications. That distinction is the one typographic rule the whole report rests on. + */ + +@font-face { + font-family: "Space Grotesk"; + font-weight: 400; + src: url("fonts/SpaceGrotesk-Regular.ttf"); +} + +@font-face { + font-family: "Space Grotesk"; + font-weight: 500; + src: url("fonts/SpaceGrotesk-Medium.ttf"); +} + +@font-face { + font-family: "Space Grotesk"; + font-weight: 700; + src: url("fonts/SpaceGrotesk-Bold.ttf"); +} + +/* Shipped rather than asked of the host. `monospace` resolves to whatever a machine happens to + have -- DejaVu on one, Andale on another -- and a report whose identifiers change shape between + two machines is a report nobody can compare. Space Mono is the companion face to Space Grotesk. */ +@font-face { + font-family: "Space Mono"; + font-weight: 400; + src: url("fonts/SpaceMono-Regular.ttf"); +} + +@font-face { + font-family: "Space Mono"; + font-weight: 700; + src: url("fonts/SpaceMono-Bold.ttf"); +} + +:root { + --ink: #1a1a1a; + --muted: #55585c; + --rule: #1a1a1a; + --border: #c9cbcd; + --header-fill: #e7e8ea; + --label-fill: #f4f5f6; + /* The filtering reason is the one row that argues rather than states, and it is tinted so that + it reads as an aside even when the page is skimmed. */ + --aside-fill: #fdf5ea; + --critical: #d0021b; + --high: #e8710a; + --medium: #f0ad20; + --low: #1f9d55; +} + +@page { + size: A4; + margin: 22mm 18mm 18mm 18mm; + + @bottom-left { + content: string(scan-footer); + font-family: "Space Grotesk", sans-serif; + font-size: 6.5pt; + color: var(--muted); + } + + @bottom-center { + content: counter(page); + font-family: "Space Grotesk", sans-serif; + font-size: 6.5pt; + color: var(--muted); + } + + @bottom-right { + content: string(date-footer); + font-family: "Space Grotesk", sans-serif; + font-size: 6.5pt; + color: var(--muted); + } +} + +html { + font-family: "Space Grotesk", sans-serif; + font-size: 8.5pt; + line-height: 1.45; + color: var(--ink); +} + +body { + margin: 0; +} + +code, +.mono { + font-family: "Space Mono", monospace; + font-size: 0.93em; +} + +/* Carried into the running footer on every page. Set once, on the cover. */ +.footer-scan { + string-set: scan-footer content(); +} + +.footer-date { + string-set: date-footer content(); +} + +.footer-scan, +.footer-date { + position: absolute; + visibility: hidden; +} + +/* --- Cover ------------------------------------------------------------------------------- */ + +.cover-mark { + text-align: right; + margin-bottom: 42mm; +} + +.cover-mark img { + height: 15mm; +} + +.cover-title { + font-size: 19pt; + font-weight: 400; + margin: 0 0 9mm 0; +} + +/* --- Headings ---------------------------------------------------------------------------- */ + +/* A section heading and the rule under it are one thing, so the rule belongs to the heading and + cannot be separated from it by a page break. */ +h1 { + font-size: 12.5pt; + font-weight: 700; + margin: 12mm 0 4mm 0; + padding-bottom: 1.6mm; + border-bottom: 0.8pt solid var(--rule); + break-after: avoid; +} + +h1:first-of-type { + margin-top: 0; +} + +h2 { + font-size: 9pt; + font-weight: 700; + margin: 7mm 0 2.5mm 0; + break-after: avoid; +} + +h3 { + font-size: 8.5pt; + font-weight: 700; + margin: 5mm 0 2mm 0; + break-after: avoid; +} + +p { + margin: 0 0 2.5mm 0; +} + +p.note { + color: var(--ink); + max-width: 155mm; +} + +/* --- Definition rows --------------------------------------------------------------------- */ + +/* Label left, value right, on one grid so that every block of them lines up down the page even + when they are separated by headings. */ +.facts { + width: 100%; + border-collapse: collapse; + margin-bottom: 3mm; +} + +.facts td { + padding: 0.7mm 0; + vertical-align: top; +} + +.facts td:first-child { + width: 46mm; + padding-right: 4mm; +} + +.facts.wide td:first-child { + width: 62mm; +} + +.facts tr.gap td { + padding-top: 4mm; +} + +/* --- Ledger tables (severity counts, languages) ------------------------------------------- */ + +/* Ruled above and below with hairlines between, and a heavier line before the total: the shape a + reader already knows from an invoice, used for the same reason -- the last row is a sum. */ +.ledger { + width: 62%; + border-collapse: collapse; + margin-bottom: 4mm; +} + +.ledger th, +.ledger td { + padding: 1.4mm 1mm; + border-bottom: 0.4pt solid var(--border); +} + +.ledger thead th { + border-bottom: 0.8pt solid var(--rule); + text-align: left; + font-weight: 700; +} + +.ledger td:last-child { + text-align: right; +} + +.ledger tr.total td { + border-top: 0.8pt solid var(--rule); + border-bottom: none; + font-weight: 700; +} + +/* --- Grid tables -------------------------------------------------------------------------- */ + +.grid { + width: 100%; + border-collapse: collapse; + margin-bottom: 4mm; + /* A table that splits across a page keeps its header on the second half. */ + break-inside: auto; +} + +.grid th, +.grid td { + border: 0.4pt solid var(--border); + padding: 1.6mm 2mm; + vertical-align: top; + text-align: left; +} + +.grid thead th { + background: var(--header-fill); + font-weight: 700; +} + +.grid tr.total td { + font-weight: 700; +} + +.grid tr, +.facts tr { + break-inside: avoid; +} + +/* --- Cards -------------------------------------------------------------------------------- */ + +/* One finding, as a titled block of label/value rows. Kept on one page where it fits: a card split + down the middle loses the thing that makes it a card. */ +.card { + width: 100%; + border-collapse: collapse; + margin-bottom: 3.5mm; + break-inside: avoid; +} + +.card caption { + background: var(--header-fill); + border: 0.4pt solid var(--border); + padding: 1.6mm 2mm; + text-align: left; + font-weight: 700; + caption-side: top; +} + +.card th, +.card td { + border: 0.4pt solid var(--border); + padding: 1.6mm 2mm; + vertical-align: top; + text-align: left; +} + +.card th { + width: 34mm; + background: var(--label-fill); + font-weight: 400; +} + +.card tr.aside th, +.card tr.aside td { + background: var(--aside-fill); +} + +/* The justification itself, set as the token it is rather than as prose about a token. */ +.token { + font-family: "Space Mono", monospace; + font-size: 0.93em; + background: #ffffff; + border: 0.4pt solid var(--border); + border-radius: 1mm; + padding: 0.3mm 1.2mm; +} + +/* --- Severity ----------------------------------------------------------------------------- */ + +.sev { + font-weight: 700; +} + +.sev-critical { + color: var(--critical); +} + +.sev-high { + color: var(--high); +} + +.sev-medium { + color: var(--medium); +} + +.sev-low { + color: var(--low); +} + +/* --- Distribution bars --------------------------------------------------------------------- */ + +/* Label, count, then one bar made of stacked severity segments. The bar is scaled against the + largest row rather than against the total, so the shape of the distribution is what is read and + the busiest row fills the width. */ +.bars { + width: 100%; + border-collapse: collapse; + margin-bottom: 1.5mm; +} + +.bars td { + border: 0.4pt solid var(--border); + padding: 1.2mm 2mm; + vertical-align: middle; +} + +.bars td.count { + width: 12mm; + text-align: right; +} + +.bars td.track { + width: 50%; + padding: 1.2mm 2mm; +} + +.bar { + height: 3mm; + width: 100%; + /* The segments are inline-block, so the newlines between them in the template would be + rendered as spaces and push a full-width bar onto a second line. */ + font-size: 0; + white-space: nowrap; +} + +.bar span { + display: inline-block; + height: 3mm; + vertical-align: top; +} + +.seg-critical { + background: var(--critical); +} + +.seg-high { + background: var(--high); +} + +.seg-medium { + background: var(--medium); +} + +.seg-low { + background: var(--low); +} + +/* A distribution and its legend travel together. Where the pair does not fit, it moves to the + next page whole rather than leaving a stranded row and an orphaned key behind. */ +.block { + break-inside: avoid; +} + +.legend { + font-size: 7.5pt; + color: var(--ink); + margin-bottom: 5mm; +} + +.legend .key { + display: inline-block; + width: 2.4mm; + height: 2.4mm; + margin-right: 1.2mm; +} + +.legend .item { + margin-right: 4mm; +} + +.legend .caveat { + color: var(--muted); +} diff --git a/report_generator/template/report.html.j2 b/report_generator/template/report.html.j2 new file mode 100644 index 0000000..a636cf1 --- /dev/null +++ b/report_generator/template/report.html.j2 @@ -0,0 +1,182 @@ +{# + The SAST/SCA report, after the design sample in `sample_report_template/`. + + This file places content and nothing else: every dimension, colour and rule lives in `report.css`. + Each section is guarded by the data it needs, so a report of a scan that ran half the pipeline is + the same document with fewer sections rather than a document full of empty tables. + + The context this expects is described by `layout.CONTEXT`, and `sample.sample_context()` builds a + complete one. +#} +{# A value is drawn by what it is, not by where it sits: an identifier, a path or a CVSS vector is + set in the monospace face wherever it appears, and a severity carries its colour. #} +{% macro value(row) %} + {% if row.token %}{{ row.token }} — {% endif %} + {% if row.kind == "severity" %} + {{ row.value }} + {% elif row.kind == "mono" %} + {{ row.value }} + {% else %} + {{ row.value }} + {% endif %} +{% endmacro %} + +{% macro facts(rows, wide=false) %} + + {% for row in rows %} + + + + + {% endfor %} +
{{ row.label }}:{{ value(row) }}
+{% endmacro %} + +{% macro card(title, rows) %} + {# Wrapped, because break-inside on the table does not hold its caption: the title would stay on + one page while the rows it titles moved to the next. #} +
+ + + {% for row in rows %} + + + + + {% endfor %} +
{{ title }}
{{ row.label }}{{ value(row) }}
+
+{% endmacro %} + +{# Scaled against the busiest row, not against the total: what is read off a row of these is which + one is the outlier, and a scale set by the sum flattens all of them. #} +{% macro distribution(rows, legend) %} + {# The bars and the legend that decodes them are one thing; a break between them leaves a reader + with coloured segments and no key. #} +
+ + {% for row in rows %} + + {{ row.label }} + + + + {% endfor %} +
{{ row.count }} +
+ {% for part in row.parts %} + + {% endfor %} +
+
+
+ {% for item in legend.entries %} + {{ item.label }} + {% endfor %} + {% if legend.caveat %}— {{ legend.caveat }}{% endif %} +
+
+{% endmacro %} + +{% macro grid(table) %} + + + {% for column in table.columns %}{% endfor %} + + + {% for row in table.rows %} + + {% for cell in row.cells %} + + {% endfor %} + + {% endfor %} + +
{{ column }}
{% if cell is mapping %}{{ value(cell) }}{% else %}{{ cell }}{% endif %}
+{% endmacro %} + +{# ------------------------------------------------------------------------------------------ #} + +Scan: {{ cover.scan_id }} +{{ cover.footer_date }} + +
+ CipherCell +
+ +
{{ cover.title }}
+ +{{ facts(cover.identity) }} + +{% if cover.severity_ledger %} +

{{ cover.severity_ledger.title }}

+ + {% for row in cover.severity_ledger.rows %} + + + + + {% endfor %} +
{{ row.label }}{{ row.count }}
+{% endif %} + +{% if scan %} +

{{ scan.title }}

+ {{ facts(scan.overview, wide=true) }} + + {% for stage in scan.stages %} +

{{ stage.title }}

+ {{ facts(stage.rows, wide=true) }} + {% if stage.table %}{{ grid(stage.table) }}{% endif %} + {% endfor %} +{% endif %} + +{% if project %} +

{{ project.title }}

+ {{ facts(project.overview, wide=true) }} + + {% if project.languages %} +

{{ project.languages.title }}

+ + {% for row in project.languages.rows %} + + + + + {% endfor %} +
{{ row.label }}{{ row.share }}
+ {% endif %} +{% endif %} + +{% if summary %} +

{{ summary.title }}

+ {{ facts(summary.overview, wide=true) }} + + {% for block in summary.distributions %} +

{{ block.title }}

+ {{ distribution(block.rows, block.legend) }} + {% endfor %} +{% endif %} + +{% if critical %} +

{{ critical.title }}

+ {{ grid(critical.table) }} +{% endif %} + +{% for section in sections %} + {{ section.title }} + + {% for group in section.groups %} +

{{ group.title }}

+ {% if group.note %}

{{ group.note }}

{% endif %} + + {% for entry in group.cards %} + {{ card(entry.title, entry.rows) }} + {% endfor %} + + {% if group.table %} + {% if group.table.title %}

{{ group.table.title }}

{% endif %} + {{ grid(group.table) }} + {% endif %} + {% endfor %} +{% endfor %} diff --git a/report_generator/vendor/README.md b/report_generator/vendor/README.md new file mode 100644 index 0000000..e92c3ee --- /dev/null +++ b/report_generator/vendor/README.md @@ -0,0 +1,42 @@ +# vendor + +Copies of everything `report_generator` borrows from the monorepo, so the package can be lifted into +a repository of its own. + +Nothing in here is edited. `sync.py` writes it, under the same import paths the monorepo uses: + +``` +python -m persistent_storage.report_generator.vendor.sync +``` + +Re-run it after changing anything it lists; the diff then shows what drifted. The copied packages +are removed whole first, so dropping an entry from the list actually drops it from the tree. + +`__init__.py` decides whether these copies are used. It asks whether the real packages can be +imported — not a flag, not an environment variable — so the same code runs in the monorepo and +outside it, and the copies can never shadow the real packages while those are present. + +## What is copied, and what for + +| Copied | Used for | +| --- | --- | +| `cc_common` | The CLI framework, the logging context, `PipelineError`. | +| `persistent_storage/shared.py` | The storage configuration and its secrets. | +| `persistent_storage/wrapper.py` | The `Scan` and `Artifact` rows. | +| `persistent_storage/utils/db_utils.py` | Reading a scan and its artifacts. | +| `persistent_storage/utils/s3_utils.py` | Fetching the stored files. | +| `persistent_storage/utils/cvss_utils.py` | Reading a CVSS vector of any generation. | +| `persistent_storage/helper.py` | What `db_utils` prints its tables with. | +| `cipherverifier/core/services/sarif.py` | Summarising a SARIF report. | +| `cipherverifier/core/services/vdr.py` | Summarising a vulnerability report. | + +The two summarisers live in the web application today and are written to be free of it — no Django, +no store. That is what makes copying them honest rather than a workaround: wherever this tool ends +up, they belong beside it rather than inside a web application it does not otherwise use. + +Not copied: the report's own template, stylesheet and fonts, which are this package's own and live +in `template/`. + +## Third-party dependencies + +`pyproject.toml` lists them. WeasyPrint additionally needs the native Pango and cairo libraries. diff --git a/report_generator/vendor/__init__.py b/report_generator/vendor/__init__.py new file mode 100644 index 0000000..e5e3e71 --- /dev/null +++ b/report_generator/vendor/__init__.py @@ -0,0 +1,53 @@ +"""Copies of what this tool borrows, and the switch that makes them the ones it uses. + +In the monorepo the real packages are importable and nothing here is touched. In a repository of +its own they are not, and :func:`activate` puts these copies on the path instead. + +The check is "can it be imported", not a flag or an environment variable, so the same code runs in +both places without being told which one it is in -- and so the copies can never shadow the real +packages while those are present. +""" + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + +VENDOR_ROOT = Path(__file__).resolve().parent + +# One module out of each copied package. If these resolve, the real thing is installed. +PROBES = ("cc_common", "persistent_storage.shared", "cipherverifier.core.services.sarif") + + +def activate() -> bool: + """Put the vendored copies on the path where the real packages are missing. + + Returns: + Whether the copies were used. ``False`` in the monorepo, where the real + packages are installed and are left to win. + """ + if all(_installed(name) for name in PROBES): + return False + + if str(VENDOR_ROOT) not in sys.path: + sys.path.insert(0, str(VENDOR_ROOT)) + + return True + + +def _installed(name: str) -> bool: + """Whether a module can be imported without importing it. + + Args: + name: A dotted module name. + + Returns: + Whether the import system can find it. Wrapped because ``find_spec`` + raises rather than returning ``None`` when a parent package is itself + missing, which is exactly the case being tested for. + """ + try: + return importlib.util.find_spec(name) is not None + except (ImportError, ValueError): + return False diff --git a/report_generator/vendor/cc_common/__init__.py b/report_generator/vendor/cc_common/__init__.py new file mode 100644 index 0000000..4f1e78e --- /dev/null +++ b/report_generator/vendor/cc_common/__init__.py @@ -0,0 +1 @@ +"""Collection of CLI helper functions.""" diff --git a/report_generator/vendor/cc_common/api/__init__.py b/report_generator/vendor/cc_common/api/__init__.py new file mode 100644 index 0000000..9aac80d --- /dev/null +++ b/report_generator/vendor/cc_common/api/__init__.py @@ -0,0 +1,11 @@ +"""API for interacting with CipherCell tools. + +Exports core exception classes and property protocols used throughout +the cc_common package. +""" + +from .context import Context, clear_context, set_context +from .context import _context as context +from .pipeline_error import ExceptionSeverity, PipelineError + +__all__ = ["Context", "context", "set_context", "clear_context", "ExceptionSeverity", "PipelineError"] diff --git a/report_generator/vendor/cc_common/api/cli/__init__.py b/report_generator/vendor/cc_common/api/cli/__init__.py new file mode 100644 index 0000000..fcff0b5 --- /dev/null +++ b/report_generator/vendor/cc_common/api/cli/__init__.py @@ -0,0 +1,5 @@ +"""CLI utilities and command management for CipherCell tools.""" + +from .cli_manager import CLIManager + +__all__ = ["CLIManager"] diff --git a/report_generator/vendor/cc_common/api/cli/_utils.py b/report_generator/vendor/cc_common/api/cli/_utils.py new file mode 100644 index 0000000..b3ea093 --- /dev/null +++ b/report_generator/vendor/cc_common/api/cli/_utils.py @@ -0,0 +1,203 @@ +"""Internal utilities for CLI argument flattening and config loading.""" + +import json +from annotationlib import Format +from inspect import isclass +from pathlib import Path +from types import UnionType +from typing import Any, ForwardRef, Union, get_args, get_origin + +import yaml +from pydantic import BaseModel +from pydantic.fields import FieldInfo +from yaml.parser import ParserError as YAMLDecodeError + +FLATTENING_SEPERATOR = "__CC__COMMON__SEPARATOR__" + + +def _unwrap_optional(annotation: Any) -> Any: # noqa: ANN401 + """Return the non-``None`` member of an ``Optional``/union annotation, else the annotation. + + A ``SomeModel | None`` field still has to be recursed into like a plain nested model: Typer + cannot build a CLI option for a union type. + """ + if get_origin(annotation) in (Union, UnionType): + args = [arg for arg in get_args(annotation) if arg is not type(None)] + if len(args) == 1: + return args[0] + return annotation + + +def flatten_model_fields( + model: type[BaseModel], + parent_path: str = "", + owner_model: type[BaseModel] | None = None, +) -> list[tuple[str, FieldInfo, type[BaseModel] | None]]: + """Flatten nested Pydantic model fields into dot-separated key paths. + + Recursively processes a Pydantic model and produces a flat list of + (field_path, field_info, owner_model) tuples. Nested models are + expanded with a separator between levels. + + Args: + model: The Pydantic model to flatten. + parent_path: Prefix for nested field paths. + owner_model: The top-level model that owns this field. + + Returns: + List of tuples containing the flattened field path, field info, + and the owner model. + + Raises: + ValueError: If the model contains forward references. + """ + result: list[tuple[str, FieldInfo, type[BaseModel] | None]] = [] + + for field_name, field_info in model.model_fields.items(): + if isinstance(field_info.annotation, ForwardRef): + raise ValueError( + f"Class {model.__name__} contains a forward reference to " + f"{field_info.annotation.evaluate(format=Format.STRING)}. " + f"This is not supported. Restructure the model." + ) + annotation = _unwrap_optional(field_info.annotation) + if isclass(annotation) and issubclass(annotation, BaseModel): + result.extend(flatten_model_fields(annotation, f"{parent_path}{field_name}{FLATTENING_SEPERATOR}", model)) + else: + result.append((f"{parent_path}{field_name}", field_info, owner_model)) + return result + + +def build_arg_structure(args: dict[str, Any]) -> dict[str, Any]: + """Rebuild nested dicts from flattened key-value pairs. + + Splits keys by the flattening separator and inserts values into + a nested dictionary structure. + + Args: + args: Flattened key-value pairs with separator-containing keys. + + Returns: + Nested dictionary matching the original model structure. + """ + result: dict[str, Any] = {} + + for key in args: + key_parts = key.split(FLATTENING_SEPERATOR) + _insert_value(result, key_parts, args[key]) + + return result + + +def _insert_value(result: dict[str, Any], key_list: list[str], value: str) -> None: + """Recursively insert a value into a nested dict using a key path. + + Args: + result: The dictionary to insert into. + key_list: Split key path parts. + value: Value to insert at the leaf position. + """ + if len(key_list) == 1: + result[key_list[0]] = value + else: + if key_list[0] not in result: + result[key_list[0]] = {} + _insert_value(result[key_list[0]], key_list[1:], value) + + +def drop_none_leaves(data: dict[str, Any]) -> dict[str, Any]: + """Recursively drop ``None`` values and the empty dicts they leave behind. + + For a command registered with ``validate=False``, whose options all default to ``None``: only + the options someone actually typed may be merged as a config overlay, since a ``None`` would + blank out whatever an earlier layer set for that path. + """ + result: dict[str, Any] = {} + for key, value in data.items(): + if isinstance(value, dict): + nested = drop_none_leaves(value) + if nested: + result[key] = nested + elif value is not None: + result[key] = value + return result + + +def flatten_config(config: dict[str, Any], parent_key: str = "") -> dict[str, Any]: + """Recursively flatten a nested dict using the flattening separator. + + Keys in nested dictionaries are joined with the separator to produce + a single-level dictionary. + + Args: + config: The dictionary to flatten. + parent_key: Prefix for nested keys. + + Returns: + Flattened single-level dictionary. + """ + items: list[tuple[str, Any]] = [] + for key, value in config.items(): + new_key = parent_key + FLATTENING_SEPERATOR + key if parent_key else key + if isinstance(value, dict): + items.extend(flatten_config(value, new_key).items()) + else: + items.append((new_key, value)) + return dict(items) + + +def load_config(path: Path) -> dict[str, Any] | None: + """Load a configuration file (JSON or YAML) from disk. + + Attempts to parse the file as JSON first, then YAML. Returns None + if neither parser succeeds or the file is invalid. + + Args: + path: Path to the configuration file. + + Returns: + Parsed configuration dict, or None if loading failed. + """ + if json_config := _load_json_config(path): + return json_config + if yaml_config := _load_yaml_config(path): + return yaml_config + return None + + +def _load_json_config(path: Path) -> dict[str, Any] | None: + """Attempt to load a JSON configuration file. + + Args: + path: Path to the JSON file. + + Returns: + Parsed dict if valid JSON, otherwise None. + """ + with path.open("r") as file: + try: + config = json.load(file) + if isinstance(config, dict): + return config + except json.JSONDecodeError: + return None + return None + + +def _load_yaml_config(path: Path) -> dict[str, Any] | None: + """Attempt to load a YAML configuration file. + + Args: + path: Path to the YAML file. + + Returns: + Parsed dict if valid YAML, otherwise None. + """ + with path.open("r") as file: + try: + config = yaml.safe_load(file) + if isinstance(config, dict): + return config + except YAMLDecodeError: + return None + return None diff --git a/report_generator/vendor/cc_common/api/cli/cli_manager.py b/report_generator/vendor/cc_common/api/cli/cli_manager.py new file mode 100644 index 0000000..c0ca2a6 --- /dev/null +++ b/report_generator/vendor/cc_common/api/cli/cli_manager.py @@ -0,0 +1,216 @@ +"""CLI command registration and execution using Typer.""" + +import inspect +import logging +from collections.abc import Callable, Sequence +from inspect import signature +from pathlib import Path +from typing import Any, TypeVar + +import typer +import typer_config +from pydantic import BaseModel +from pydantic.fields import FieldInfo +from rich import print +from rich.console import Console +from rich.logging import RichHandler + +from cc_common.api import Context, PipelineError, set_context + +from ._utils import ( + FLATTENING_SEPERATOR, + _unwrap_optional, + build_arg_structure, + drop_none_leaves, + flatten_config, + flatten_model_fields, + load_config, +) + + +def _result_callback(data: Any | None) -> None: # noqa: ANN401 + """Print the result data if it is not None.""" + if data is not None: + if hasattr(data, "print_as_cc_common_result") and callable(data.print_as_cc_common_result): + data.print_as_cc_common_result() + else: + print(data) + + +class CLIManager: + """Manages Typer-based CLI applications with automatic argument generation. + + Wraps a Typer app and provides a decorator to register commands whose + arguments are derived from Pydantic models. Configuration files (JSON/YAML) + are automatically loaded and flattened to match nested model fields. + + Args: + program_name: Name of the CLI program. + help: Help text displayed in the CLI help. + """ + + def __init__(self, program_name: str, help: str = "") -> None: + """Initialize the CLIManager with a Typer app. + + Args: + program_name: Name of the CLI program. + help: Help text displayed in the CLI help. + """ + self._app = typer.Typer( + name=program_name, + help=help, + add_completion=True, + rich_markup_mode="rich", + no_args_is_help=True, + pretty_exceptions_show_locals=False, + result_callback=_result_callback, + ) + self._logger = logging.getLogger("cc_common_cli") + self._logger.setLevel(logging.INFO) + self._logger.addHandler(RichHandler(show_time=False, show_level=False)) + + # _SOME_MODEL is a generic + _SOME_MODEL = TypeVar("_SOME_MODEL", bound=BaseModel) + + def command( + self, + name: str, + model: type[_SOME_MODEL] | None = None, + help: str = "", + ) -> Callable[[Callable[[_SOME_MODEL], Any]], None] | None: + """Decorator to register a CLI command. + + Args: + name: The command name. + model: Optional Pydantic model whose fields become CLI options. + help: Help text for the command. + + Returns: + Decorator function that registers the command. + """ + + def decorator(fn: Callable[[CLIManager._SOME_MODEL], Any] | Callable[[], Any]) -> None: + self.register_command(name, fn, model, help) + + return decorator + + def register_command( + self, + name: str, + callback: Callable[[_SOME_MODEL], Any] | Callable[[dict[str, Any]], Any] | Callable[[], Any], + model: type[_SOME_MODEL] | None = None, + help: str = "", + *, + validate: bool = True, + ) -> None: + """Register a callback as a CLI command with auto-generated arguments. + + If a Pydantic model is provided, its fields (including nested ones) + are flattened into CLI ``--option`` flags. A ``--config`` flag is + added to load JSON/YAML configuration files. + + Args: + name: The command name. + callback: The function to call when the command is invoked. + model: Optional Pydantic model whose fields become CLI options. + help: Help text for the command. + validate: When ``True`` (the default), every required field becomes a required CLI + option and the callback receives a validated ``model`` instance. When ``False``, + every option defaults to unset and the callback receives the raw ``dict`` of + whichever ones were typed -- for a command whose model is only complete once + merged with layers the CLI does not know about, such as ``orchestrator run``. + + Raises: + ValueError: If the model contains a field named 'config'. + """ + params: list[inspect.Parameter] = [] + + if model is not None: + for field_name, field_info, _ in flatten_model_fields(model): + if field_name == "config": + raise ValueError( + f"You must not have a parameter named 'config' in " + f"your top level Properties Model '{model.__name__}'." + ) + cli_name = field_name.replace(FLATTENING_SEPERATOR, "-") + # A bool gets a paired --x/--no-x declaration so it can be turned off from the CLI + # even when its model default is True -- a single "--x" name could only ever set it + # to True, which is why every boolean field used to have to default to False. + is_bool = _unwrap_optional(field_info.annotation) is bool + param_names = [f"--{cli_name}/--no-{cli_name}"] if is_bool else [f"--{cli_name}"] + field_description = field_info.description or field_info.title or "" + # Every field defaults to unset, the optional ones included: their defaults would + # otherwise override what another layer set for the same path once merged in. + default = None if not validate else _extract_default_value(field_info) + + params.append( + inspect.Parameter( + field_name, + inspect.Parameter.POSITIONAL_OR_KEYWORD, + default=typer.Option( + default, + *param_names, + help=field_description, + ), + annotation=field_info.annotation, + ) + ) + + def cmd_wrapper(**kwargs: Any) -> Any: # noqa: ANN401 + logger = self._logger.getChild(name) + context = Context(logger, True, Console()) + set_context(context) + if model is not None: + unflattened_kwargs = build_arg_structure(kwargs) + if not validate: + return callback(drop_none_leaves(unflattened_kwargs)) # type: ignore[call-arg,arg-type] + instance = model.model_validate(unflattened_kwargs) + return callback(instance) # type: ignore[call-arg,arg-type] + else: + return callback() # type: ignore[call-arg] + + cmd_wrapper.__signature__ = inspect.Signature(params) # type: ignore[attr-defined] + cmd_wrapper.__name__ = name + cmd_wrapper.__doc__ = help + + def load_and_flatten_config(path: str) -> dict[str, Any]: + if path: + config = load_config(Path(path)) + if config is None: + raise ValueError(f"Could not parse file: '{path}'") + return flatten_config(config) + return {} + + load_config_callback = typer_config.conf_callback_factory(load_and_flatten_config) + wrapped = typer_config.use_config(load_config_callback)(cmd_wrapper) + self._app.command(name=name)(wrapped) + + def run(self, args: Sequence[str] | list[str] | None = None) -> None: + """Execute the CLI application. + + Parses arguments and runs registered commands. PipelineErrors are + caught and printed in red. + + Args: + args: Optional list of CLI arguments. Defaults to sys.argv. + """ + if args is not None: + args = list(args) + try: + self._app(args=args) + except PipelineError as e: + print(f"[red]{e}[/]") + + +def _extract_default_value(field_info: FieldInfo) -> Any: # noqa: ANN401 + if field_info.is_required(): + default = ... + elif field_info.default_factory is not None: + # TODO: low prio, if possible evaluate this right before the callback is called + default_factory_signature = signature(field_info.default_factory) + if len(default_factory_signature.parameters) != 0: + raise ValueError("Only default factories with zero arguments are supported.") + default = field_info.default_factory() # type: ignore[call-arg] + else: + default = field_info.default + return default diff --git a/report_generator/vendor/cc_common/api/context.py b/report_generator/vendor/cc_common/api/context.py new file mode 100644 index 0000000..10918c3 --- /dev/null +++ b/report_generator/vendor/cc_common/api/context.py @@ -0,0 +1,70 @@ +"""Module handling the application wide context.""" + +import logging +import sys +import threading +from logging import Logger +from typing import Any + +from rich.console import Console + + +class Context: + """Context of the application, containing global settings, state, etc.""" + + def __init__(self, logger: Logger, is_cli: bool, console: Console) -> None: + """Create a new Context Object.""" + self.logger = logger + self.is_cli = is_cli + self.console = console + + +class _ContextWrapper(Context): + def __init__(self, context: Context) -> None: + super().__init__(None, None, None) # type: ignore[arg-type] + self._default_context = context + self._contexts_for_thread: dict[int, Context] = {} + + def __getattribute__(self, attr: str) -> Any: # noqa: ANN401 + if attr in ["_contexts_for_thread", "_default_context", "set_context", "clear_context"]: + return super().__getattribute__(attr) + thread_id = threading.current_thread().ident + if thread_id is not None and thread_id in self._contexts_for_thread: + context_to_use = self._contexts_for_thread[thread_id] + else: + context_to_use = self._default_context + return getattr(context_to_use, attr) + + def set_context(self, context: Context) -> None: + """Set the context for the current thread.""" + thread_id = threading.current_thread().ident + if thread_id is not None: + self._contexts_for_thread[thread_id] = context + + def clear_context(self) -> None: + """Clear the custom set context for the current thread.""" + thread_id = threading.current_thread().ident + if thread_id is not None and thread_id in self._contexts_for_thread: + del self._contexts_for_thread[thread_id] + + +_default_logger = logging.getLogger("cc_common_default") +_default_logger.setLevel(logging.INFO) +_default_logger.addHandler(logging.StreamHandler(sys.stdout)) +_context: Context = _ContextWrapper(Context(logger=_default_logger, is_cli=True, console=Console())) + + +def set_context(context: Context) -> None: + """Overwrite the application context.""" + global _context + if not isinstance(_context, _ContextWrapper): + raise TypeError(f"_context must be a _ContextWrapper, got {type(_context)}") + _context.set_context(context) + + +def clear_context() -> None: + """Clear the custom set application context.""" + global _context + if not isinstance(_context, _ContextWrapper): + raise TypeError(f"_context must be a _ContextWrapper, got {type(_context)}") + _context.clear_context() diff --git a/report_generator/vendor/cc_common/api/pipeline_error.py b/report_generator/vendor/cc_common/api/pipeline_error.py new file mode 100644 index 0000000..ef7fe80 --- /dev/null +++ b/report_generator/vendor/cc_common/api/pipeline_error.py @@ -0,0 +1,70 @@ +"""Custom exception classes for uniform error handling across CipherCell applications.""" + +from enum import Enum, auto + + +class ExceptionSeverity(Enum): + """Severity levels for pipeline exceptions. + + Determines whether a pipeline can continue execution after an error + or must stop. + """ + + RECOVERABLE = auto() # Failure in a stage; pipeline can continue + FATAL = auto() # Critical failure; pipeline cannot continue + + +class PipelineError(Exception): + """Base exception for components within the CipherCell pipeline. + + Attributes: + message: A human-readable description of the error. + severity: The severity level of the exception. + tool: The name of the tool or component where the exception originated. + exit_code: Optional exit code associated with the failure. + """ + + def __init__( + self, + message: str, + severity: ExceptionSeverity = ExceptionSeverity.FATAL, + tool: str | None = None, + exit_code: int | None = None, + ) -> None: + """Initialize a PipelineError instance. + + Args: + message: A human-readable description of the error. + severity: The severity level of the exception. + tool: The name of the tool or component where the exception originated. + exit_code: Optional exit code associated with the failure. + """ + self._severity = severity + self._message = message + self._tool = tool + self._exit_code = exit_code + + def can_continue(self) -> bool: + """Check whether the pipeline can continue after this exception. + + Returns: + True if the severity is RECOVERABLE, False otherwise. + """ + return self._severity == ExceptionSeverity.RECOVERABLE + + def __str__(self) -> str: + """Return a human-readable string representation of the exception. + + Returns: + Formatted string with severity, tool, message, exit code, and + cause information. + """ + parts = [f"[{self._severity.name}]"] + if self._tool: + parts.append(f"[{self._tool}]") + parts.append(self._message) + if self._exit_code: + parts.append(f"(exit code: {self._exit_code})") + if self.__cause__: + parts.append(f"(caused by: {type(self.__cause__).__name__}: {self.__cause__})") + return " ".join(parts) diff --git a/report_generator/vendor/cc_common/config.py b/report_generator/vendor/cc_common/config.py new file mode 100644 index 0000000..80028cf --- /dev/null +++ b/report_generator/vendor/cc_common/config.py @@ -0,0 +1,190 @@ +"""Shared, model-agnostic config-file and environment-override primitives. + +Works on plain nested ``dict``s only, so every layer of a pipeline configuration can be merged with +:func:`merge` before the caller validates the result once, at the end. +""" + +from __future__ import annotations + +import copy +import functools +import json +import os +from collections.abc import Mapping +from pathlib import Path +from types import UnionType +from typing import Any, Union, get_args, get_origin + +import yaml +from dotenv import load_dotenv +from pydantic import BaseModel + +from cc_common.api import PipelineError + +CONFIG_PATH_ENV_VAR = "MONOCIPHER_CONFIG" +DEFAULT_CONFIG_FILENAME = "config.yaml" + + +def load_environment() -> None: + """Load ``.env`` into the process environment. + + Never overrides an already-set variable, so every entrypoint can call it in any order. + """ + load_dotenv() + + +def _default_config_path() -> Path: + """Where the config file is read from: ``$MONOCIPHER_CONFIG``, else ``./config.yaml``.""" + configured = os.environ.get(CONFIG_PATH_ENV_VAR) + return Path(configured) if configured else Path.cwd() / DEFAULT_CONFIG_FILENAME + + +@functools.cache +def _parse_config_file(target: Path, _mtime_ns: int, _size: int) -> dict[str, Any]: + """Parse one config file, cached on its own stamp so an edit is still picked up. + + Callers must not mutate the result -- :func:`load_config_file` hands out a deep copy. + """ + text = target.read_text() + try: + parsed: Any = json.loads(text) + except json.JSONDecodeError as json_error: + try: + parsed = yaml.safe_load(text) + except yaml.YAMLError as yaml_error: + raise PipelineError( + f"Could not parse config file '{target}' as JSON ({json_error}) or YAML ({yaml_error}).", + tool="config", + ) from yaml_error + + if not isinstance(parsed, dict): + raise PipelineError(f"Config file '{target}' must contain a mapping at its top level.", tool="config") + return parsed + + +#: Forget every parsed config file. For tests that write a config file per case. +reset_config_cache = _parse_config_file.cache_clear + + +def load_config_file(path: Path | None = None) -> dict[str, Any]: + """Load the config file as plain data, or ``{}`` if it does not exist. + + A file that exists but fails to parse raises, rather than falling back to the schema defaults. + + Raises: + PipelineError: The file is neither valid JSON nor valid YAML, or is not a mapping. + """ + target = path or _default_config_path() + if not target.is_file(): + return {} + + stat = target.stat() + return copy.deepcopy(_parse_config_file(target.resolve(), stat.st_mtime_ns, stat.st_size)) + + +def merge(base: Mapping[str, Any], overlay: Mapping[str, Any]) -> dict[str, Any]: + """Recursively merge ``overlay`` onto ``base``, ``overlay`` winning. Neither is mutated.""" + result = copy.deepcopy(dict(base)) + for key, value in overlay.items(): + existing = result.get(key) + if isinstance(existing, dict) and isinstance(value, Mapping): + result[key] = merge(existing, value) + else: + result[key] = copy.deepcopy(value) + return result + + +def set_path(data: dict[str, Any], dotted: str, value: Any) -> None: # noqa: ANN401 + """Set one dotted path (e.g. ``"polygraph.llm.model"``) in ``data``, creating levels as needed.""" + parts = dotted.split(".") + target = data + for part in parts[:-1]: + target = target.setdefault(part, {}) + target[parts[-1]] = value + + +def get_path(data: Mapping[str, Any], dotted: str) -> Any: # noqa: ANN401 + """Read one dotted path out of ``data``, or ``None`` if any part of it is missing.""" + current: Any = data + for part in dotted.split("."): + if not isinstance(current, Mapping): + return None + current = current.get(part) + return current + + +def env_var_name(dotted: str) -> str: + """The environment variable that overrides one dotted config path: dots to underscores, upper-cased. + + The whole naming convention. No config entry has a hardcoded env var name anywhere else -- a + setting's variable is always derivable from its path alone. + """ + return dotted.replace(".", "_").upper() + + +def _unwrap_optional(annotation: Any) -> Any: # noqa: ANN401 + """Return the non-``None`` member of an ``Optional``/union annotation, else the annotation.""" + if get_origin(annotation) in (Union, UnionType): + args = [arg for arg in get_args(annotation) if arg is not type(None)] + if len(args) == 1: + return args[0] + return annotation + + +@functools.cache +def overridable_paths(model_cls: type[BaseModel], prefix: str = "") -> tuple[str, ...]: + """Every dotted leaf path of ``model_cls`` that can be set from the environment. + + Descends into nested models; skips ``list``/``dict``/mapping-valued leaves, since a raw string + from the environment can never validate against one of those, so such a field must not claim an + env var name. + """ + paths: list[str] = [] + for name, field in model_cls.model_fields.items(): + path = f"{prefix}.{name}" if prefix else name + annotation = _unwrap_optional(field.annotation) + if isinstance(annotation, type) and issubclass(annotation, BaseModel): + paths.extend(overridable_paths(annotation, path)) + elif get_origin(annotation) in (list, dict, Mapping): + continue + else: + paths.append(path) + return tuple(paths) + + +def env_overlay(model_cls: type[BaseModel], prefix: str = "") -> dict[str, Any]: + """A :func:`merge`-able overlay of every setting of ``model_cls`` that is set in the environment. + + Each of :func:`overridable_paths` is looked up under its derived :func:`env_var_name`; an unset + or blank variable contributes nothing, so a blank placeholder (as ``.env.sample`` ships) never + overrides with an empty string. + """ + overlay: dict[str, Any] = {} + for path in overridable_paths(model_cls, prefix): + value = os.environ.get(env_var_name(path)) + if value: + set_path(overlay, path, value) + return overlay + + +def apply_env_defaults(instance: BaseModel, prefix: str = "", model_cls: type[BaseModel] | None = None) -> None: + """Fill in every field of ``instance`` still unset from its derived environment variable. + + For a standalone tool's own config model, once it is already validated: only ``None`` fields are + touched, so a value already supplied (by CLI flag or config file) is never overridden. + + ``model_cls`` defaults to ``type(instance)``; pass the pipeline-side model explicitly when + ``instance`` is a CLI-only subclass with extra fields that have no place under ``prefix`` in + :class:`orchestrator.commands.run.OrchestratorConfig`. + """ + for path in overridable_paths(model_cls or type(instance), prefix): + relative = path[len(prefix) + 1 :] if prefix else path + target = instance + parts = relative.split(".") + for part in parts[:-1]: + target = getattr(target, part) + leaf = parts[-1] + if getattr(target, leaf) is None: + value = os.environ.get(env_var_name(path)) + if value: + setattr(target, leaf, value) diff --git a/report_generator/vendor/cc_common/defines/__init__.py b/report_generator/vendor/cc_common/defines/__init__.py new file mode 100644 index 0000000..c3bb7c1 --- /dev/null +++ b/report_generator/vendor/cc_common/defines/__init__.py @@ -0,0 +1,5 @@ +"""Collection of CLI helper functions.""" + +from .general_defines import BuildType, CPUArchitecture, FileFormat, OperatingSystem, ProgrammingLanguage + +__all__ = ["BuildType", "CPUArchitecture", "FileFormat", "OperatingSystem", "ProgrammingLanguage"] diff --git a/report_generator/vendor/cc_common/defines/general_defines.py b/report_generator/vendor/cc_common/defines/general_defines.py new file mode 100644 index 0000000..9704019 --- /dev/null +++ b/report_generator/vendor/cc_common/defines/general_defines.py @@ -0,0 +1,49 @@ +"""Collection of general definitions used across all CipherCell products.""" + +from enum import Enum, auto + + +class FileFormat(Enum): + """Definitions of file formats supported by CipherCell products.""" + + SARIF = auto() + SPDX = auto() + CYCLONEDX = auto() + VEX = auto() + + +class ProgrammingLanguage(Enum): + """Supported programming languages for code-related operations.""" + + PYTHON = "python" + C = "c" + CPP = "cpp" + JAVA = "java" + RUST = "rust" + JAVASCRIPT = "javascript" + TYPESCRIPT = "typescript" + UNKNOWN = "unknown" + + +class OperatingSystem(Enum): + """Supported operating systems for code-related operations.""" + + LINUX = "linux" + MACOS = "macos" + WINDOWS = "windows" + + +class CPUArchitecture(Enum): + """Supported CPU architectures.""" + + AMD64 = "amd64" + ARM64 = "arm64" + X86 = "386" + UNKNOWN = "unknown" + + +class BuildType(Enum): + """Supported build types for code-related operations.""" + + NATIVE = "native" + DOCKER = "docker" diff --git a/report_generator/vendor/cc_common/docker/__init__.py b/report_generator/vendor/cc_common/docker/__init__.py new file mode 100644 index 0000000..dea4a24 --- /dev/null +++ b/report_generator/vendor/cc_common/docker/__init__.py @@ -0,0 +1,26 @@ +"""Collection of CLI helper functions.""" + +from .compose_utils import compose_down, compose_up +from .defs import DockerArchitecture +from .docker_utils import ( + abort_container, + check_image_exist, + create_docker_image, + exec_command, + is_docker_running, + run_command, + run_command_with_input, +) + +__all__ = [ + "DockerArchitecture", + "abort_container", + "check_image_exist", + "compose_down", + "compose_up", + "create_docker_image", + "exec_command", + "is_docker_running", + "run_command", + "run_command_with_input", +] diff --git a/report_generator/vendor/cc_common/docker/compose_utils.py b/report_generator/vendor/cc_common/docker/compose_utils.py new file mode 100644 index 0000000..b1aa9f4 --- /dev/null +++ b/report_generator/vendor/cc_common/docker/compose_utils.py @@ -0,0 +1,112 @@ +"""Provide utilities for driving Docker Compose stacks. + +Unlike the rest of this package, Compose is not reachable through the docker-py +SDK. It is a separate binary. These helpers are therefore implemented separately, +while keeping the same logging and error conventions as the SDK-based utilities next to them. +""" + +from pathlib import Path +from shutil import which + +from cc_common.api import PipelineError, context +from cc_common.processes import run_subprocess + + +def compose_up(compose_file: Path, project_name: str, *arguments: str) -> None: + """Start the services of a Compose file. + + Nothing is implied about how they start: pass ``"--detach"`` to return once + the containers exist, ``"--build"`` to rebuild images first, and so on. A + call without arguments runs in the foreground, exactly as Compose would. + + Args: + compose_file: Path to the Compose file describing the stack. + project_name: Name of the Compose project to act on. + arguments: Additional options forwarded to ``compose up``. + + Raises: + PipelineError: If Docker is unavailable, the Compose file does not + exist, or Compose exits with a non-zero status. + """ + run_compose(compose_file, project_name, "up", *arguments) + + +def compose_down(compose_file: Path, project_name: str, *arguments: str) -> None: + """Stop the services of a Compose file and remove their containers. + + Named volumes survive unless ``"--volumes"`` is passed, which is the one + argument here that destroys data — Compose's own default is to keep it, and + this function does not second-guess that either way. + + Args: + compose_file: Path to the Compose file describing the stack. + project_name: Name of the Compose project to act on. + arguments: Additional options forwarded to ``compose down``. + + Raises: + PipelineError: If Docker is unavailable, the Compose file does not + exist, or Compose exits with a non-zero status. + """ + run_compose(compose_file, project_name, "down", *arguments) + + +def run_compose(compose_file: Path, project_name: str, *arguments: str) -> None: + """Run any Docker Compose subcommand against a Compose file. + + Args: + compose_file: Path to the Compose file describing the stack. + project_name: Name of the Compose project to act on. Passing it + explicitly keeps containers and volumes stable no matter which + directory the caller is invoked from — Compose would otherwise + derive the name from the directory holding the Compose file. + arguments: The Compose subcommand and its options, e.g. ``"up", + "--detach"``. + + Raises: + PipelineError: If Docker is unavailable, the Compose file does not + exist, or Compose exits with a non-zero status. + """ + if not compose_file.is_file(): + raise PipelineError(f"Compose file not found at {compose_file}") + + command = compose_command() + subcommand = " ".join(arguments) + + context.logger.info(f"Running 'compose {subcommand}' on {compose_file} ...") + + # Currently, no solution without using subprocesses available. + exit_code = run_subprocess( + [*command, "--project-name", project_name, "--file", str(compose_file), *arguments], + Path.cwd(), + stream_output=True, + on_stdout=context.logger.info, + on_stderr=context.logger.info, + fill_stream_threaded=True, + ) + + if exit_code != 0: + raise PipelineError(f"'compose {subcommand}' failed with exit code {exit_code}") + + +def compose_command() -> list[str]: + """Determine how Docker Compose can be invoked on this machine. + + Returns: + The executable and subcommand to prefix the Compose arguments with, + preferring the ``docker compose`` plugin over the standalone binary. + + Raises: + PipelineError: If neither ``docker`` nor the legacy ``docker-compose`` + executable is on the PATH. + """ + docker_path = which("docker") + + if docker_path is not None: + return [docker_path, "compose"] + + legacy_compose_path = which("docker-compose") + + if legacy_compose_path is None: + raise PipelineError("Docker executable not found") + + return [legacy_compose_path] diff --git a/report_generator/vendor/cc_common/docker/defs.py b/report_generator/vendor/cc_common/docker/defs.py new file mode 100644 index 0000000..4f094a9 --- /dev/null +++ b/report_generator/vendor/cc_common/docker/defs.py @@ -0,0 +1,19 @@ +"""Docker-specific architecture and platform definitions.""" + +from enum import Enum + +from cc_common.defines import CPUArchitecture, OperatingSystem + +CONTAINER_DEFAULT_ABORT_TIMEOUT_IN_S = 5 + + +class DockerArchitecture(Enum): + """Enumeration of supported Docker platforms combining OS and CPU architecture.""" + + LINUX_AMD64 = (OperatingSystem.LINUX, CPUArchitecture.AMD64) + LINUX_ARM64 = (OperatingSystem.LINUX, CPUArchitecture.ARM64) + + @property + def arch_str(self) -> str: + """Return the os/cpu string which can be passed to docker command as plattform.""" + return f"{self.value[0].value}/{self.value[1].value}" diff --git a/report_generator/vendor/cc_common/docker/docker_utils.py b/report_generator/vendor/cc_common/docker/docker_utils.py new file mode 100644 index 0000000..79ccb1e --- /dev/null +++ b/report_generator/vendor/cc_common/docker/docker_utils.py @@ -0,0 +1,266 @@ +"""Provide utility functions for interacting with Docker via the Docker SDK. + +Supports image management, container execution, and environment validation +through the official docker-py client with logging integration. +""" + +import os +import shlex +import socket +from pathlib import Path +from typing import Any + +import docker +import docker.errors +from docker import DockerClient + +from cc_common.api import context + +from .defs import CONTAINER_DEFAULT_ABORT_TIMEOUT_IN_S, DockerArchitecture + + +def _get_client() -> DockerClient: + """Create and return a Docker client from the local environment. + + Raises: + docker.errors.DockerException: If the Docker daemon is unreachable. + """ + return docker.from_env() + + +def _stop_container(container_id: str, timeout_in_s: int) -> None: + """Stops a specific container by ID.""" + client = _get_client() + try: + container = client.containers.get(container_id) + context.logger.info(f"Aborting and removing container: {container_id}") + container.stop(timeout=timeout_in_s) + container.remove(force=True) + except docker.errors.NotFound: + context.logger.debug(f"Container {container_id} already removed.") + except Exception as e: + context.logger.error(f"Failed to abort container {container_id}: {e}") + + +def is_docker_running() -> bool: + """Check whether the Docker daemon is running.""" + context.logger.info("Checking if Docker is running") + try: + _get_client().ping() + return True + except docker.errors.DockerException: + return False + + +def check_image_exist(image_name: str) -> bool: + """Check whether a Docker image exists locally.""" + context.logger.info(f"Checking if image '{image_name}' exists") + try: + _get_client().images.get(image_name) + return True + except docker.errors.ImageNotFound: + return False + + +def create_docker_image( + image_name: str, + project_folder: Path, + docker_architecture: DockerArchitecture | None = None, + build_dargs: dict[str, str] | None = None, +) -> None: + """Build a Docker image from the specified project directory.""" + context.logger.info(f"Creating docker image '{image_name}' from '{project_folder}'") + + client = _get_client() + + try: + for chunk in client.api.build( + path=str(project_folder), + tag=image_name, + buildargs=build_dargs, + decode=True, # Low-level API handles this correctly + rm=True, + platform=docker_architecture.arch_str if docker_architecture else None, + ): + if not isinstance(chunk, dict): + continue + + if "stream" in chunk: + message = str(chunk["stream"]).strip() + if message: + context.logger.debug(message) + + if "error" in chunk: + error_data = chunk.get("errorDetail", {}) + error_msg = error_data.get("message", chunk["error"]) + + raise docker.errors.BuildError(error_msg, iter([chunk])) + + except docker.errors.BuildError as e: + context.logger.error(f"Docker build failed: {e}") + raise e + except Exception as e: + context.logger.error(f"An unexpected error occurred during build: {e}") + raise e + + +def _rewrite_path_in_cmd(cmd: list[str], old_path: str, new_path: str) -> list[str]: + """Replace `old_path` with `new_path` in every argument that points at it.""" + rewritten = [] + for arg in cmd: + if arg == old_path: + rewritten.append(new_path) + elif arg.startswith(f"{old_path}/"): + rewritten.append(f"{new_path}{arg[len(old_path) :]}") + else: + rewritten.append(arg) + return rewritten + + +def _wrap_cmd_with_copy(cmd: list[str], mount_folders: list[tuple[Path, str]], target_folder: str) -> list[str]: + """Wrap `cmd` so the mounted folders are copied into `target_folder` first. + + The copy is created by the container's own user, so it is writable without any + privilege escalation, and the read-only mounts stay untouched. Arguments in `cmd` + that point into a mounted folder are rewritten to the copy. Requires `sh`, `mkdir` + and `cp` in the image. + """ + copy_cmds = [] + for _, container_path in mount_folders: + target_path = f"{target_folder.rstrip('/')}/{Path(container_path).name}" + quoted_target = shlex.quote(target_path) + copy_cmds.append(f"mkdir -p {quoted_target} && cp -a {shlex.quote(container_path)}/. {quoted_target}/") + cmd = _rewrite_path_in_cmd(cmd, container_path, target_path) + + script = " && ".join([*copy_cmds, f"exec {shlex.join(cmd)}"]) + return ["sh", "-c", script] + + +def run_command( + image_name: str, + cmd: list[str], + mount_folders: list[tuple[Path, str]], + remove_after_execution: bool, + copy_mounts_to: str | None = None, +) -> str: + """Run a command inside a Docker container with optional volume mounts. + + Args: + image_name: Image to run the command in. + cmd: Command and arguments to execute. + mount_folders: Pairs of (host path, container path) to mount. + remove_after_execution: Whether the container is removed after it exits. + copy_mounts_to: Optional container folder (for example the user's home) the + mounted folders are copied into before the command runs. The mounts are + then read-only and `cmd` arguments pointing at them are rewritten to the + copy, so the command works on data owned by the container's own user. + """ + mount_mode = "ro" if copy_mounts_to else "rw" + volumes = { + str(local_path.resolve()): {"bind": container_path, "mode": mount_mode} + for local_path, container_path in mount_folders + } + + context.logger.info(f"Running command {cmd} in image '{image_name}'") + + run_cmd = _wrap_cmd_with_copy(cmd, mount_folders, copy_mounts_to) if copy_mounts_to else cmd + + output = _get_client().containers.run( + image=image_name, command=run_cmd, volumes=volumes, remove=remove_after_execution, stdout=True, stderr=True + ) + return output.decode("utf-8") + + +def run_command_with_input( + image_name: str, cmd: list[str], input_data: str, environment: dict[str, Any] | None = None +) -> str: + """Run a command inside a Docker container, streaming a string directly to its stdin.""" + context.logger.info(f"Running command {cmd} in image '{image_name}'") + client = _get_client() + + container = client.containers.create( + image=image_name, command=cmd, stdin_open=True, tty=False, environment=environment + ) + + try: + container.start() + + docker_sock = container.attach_socket(params={"stdin": 1, "stream": 1}) + + try: + fd = docker_sock.fileno() # type: ignore[union-attr] + clean_fd = os.dup(fd) + + if hasattr(docker_sock, "_response"): + try: # noqa: SIM105 + docker_sock._response.close() + except Exception as e: + context.logger.error(f"hasattr returns {e}") + docker_sock._response = None # drop the reference entirely + + family = socket.AF_UNIX if fd < 100 else socket.AF_INET + clean_sock = socket.socket(family=family, fileno=clean_fd) + + finally: + docker_sock.close() + try: + clean_sock.sendall(input_data.encode("utf-8")) + clean_sock.shutdown(socket.SHUT_WR) + finally: + clean_sock.close() + + exit_result = container.wait() + output = container.logs(stdout=True, stderr=True).decode("utf-8") + + if exit_result["StatusCode"] != 0: + raise RuntimeError(f"Container failed ({exit_result['StatusCode']}):\n{output}") + + return output + finally: + container.remove(force=True) + + +def exec_command(container_id: str, cmd: list[str]) -> str: + """Execute a command inside a running container. + + Returns: + The stdout output of the command as a decoded string. + """ + context.logger.info(f"Executing command {cmd} in container '{container_id}'") + + container = _get_client().containers.get(container_id) + exit_code, output = container.exec_run(cmd) + + if exit_code != 0: + raise docker.errors.APIError(f"Command {cmd} failed in container '{container_id}' with exit code {exit_code}") + + if isinstance(output, bytes): + return output.decode("utf-8") + + return b"".join(output).decode("utf-8") + + +def abort_container( + container_id: str | None = None, + image_name: str | None = None, + timeout_in_s: int = CONTAINER_DEFAULT_ABORT_TIMEOUT_IN_S, +) -> None: + """Forcefully stop and remove a specific container by ID.""" + if not container_id and not image_name: + context.logger.error("Neither container nor image specified.") + return + + client = _get_client() + if container_id: + _stop_container(container_id, timeout_in_s) + else: + running_containers = client.containers.list() + for container in running_containers: + if ( + image_name + and container.id + and container.image + and container.image.tags + and any(image_name in tag for tag in container.image.tags) + ): + _stop_container(container.id, timeout_in_s) diff --git a/report_generator/vendor/cc_common/processes/__init__.py b/report_generator/vendor/cc_common/processes/__init__.py new file mode 100644 index 0000000..24aae6f --- /dev/null +++ b/report_generator/vendor/cc_common/processes/__init__.py @@ -0,0 +1,5 @@ +"""Collection of CLI helper functions.""" + +from .subprocess_utils import run_subprocess # nosec B404 + +__all__ = ["run_subprocess"] diff --git a/report_generator/vendor/cc_common/processes/subprocess_utils.py b/report_generator/vendor/cc_common/processes/subprocess_utils.py new file mode 100644 index 0000000..17c9203 --- /dev/null +++ b/report_generator/vendor/cc_common/processes/subprocess_utils.py @@ -0,0 +1,72 @@ +"""Provide utilities for running and managing subprocess execution.""" + +import subprocess # nosec B404 +import threading +from collections.abc import Callable +from pathlib import Path +from typing import IO + +from cc_common.api import context + + +def run_subprocess( + command: list[str], + cwd: Path, + stream_output: bool = False, + on_stdout: Callable[[str], None] | None = None, + on_stderr: Callable[[str], None] | None = None, + fill_stream_threaded: bool = False, + env: dict[str, str] | None = None, +) -> int: + """Execute a subprocess and return its exit code. + + When stream_output=True stdout and stderr are either forwarded to the on_stdout and on_stderr output. + If these are not set, the results are logged using the logging singleton. + When stream_output=False the output of the subprocess is not printed. + + Args: + command: Command and arguments to execute. + cwd: Working directory for the subprocess. + stream_output: If True, stream output line-by-line instead of capturing. + on_stdout: Callback for each stdout line; falls back to DEBUG logging. + on_stderr: Callback for each stderr line; falls back to DEBUG logging. + fill_stream_threaded: Triggers the stdout and stderr callback in two seperate threads to prevent blocking. + env: Environment variables for the subprocess; inherits current env if None. + + Returns: + Exit code of the subprocess. + """ + try: + if stream_output: + process = subprocess.Popen( # nosec B603 + command, cwd=cwd, text=True, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + + def drain(pipe: IO[str] | None, callback: Callable[[str], None] | None) -> None: + if pipe: + for line in pipe: + if callback: + callback(line.rstrip()) + else: + context.logger.debug(line.rstrip()) + + if fill_stream_threaded: + t_out = threading.Thread(target=drain, args=(process.stdout, on_stdout)) + t_err = threading.Thread(target=drain, args=(process.stderr, on_stderr)) + t_out.start() + t_err.start() + t_out.join() + t_err.join() + else: + drain(process.stdout, on_stdout) + drain(process.stderr, on_stderr) + + process.wait() + return process.returncode + else: + result = subprocess.run(command, cwd=cwd, capture_output=True, text=True, check=True, env=env) # nosec B603 + return result.returncode + + except subprocess.CalledProcessError as e: + context.logger.warning(f"Command caused an exception the accepted return codes. Message:\n{e.stdout} {e}") + return e.returncode diff --git a/report_generator/vendor/cc_common/py.typed b/report_generator/vendor/cc_common/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/report_generator/vendor/cc_common/utils/__init__.py b/report_generator/vendor/cc_common/utils/__init__.py new file mode 100644 index 0000000..6c487c3 --- /dev/null +++ b/report_generator/vendor/cc_common/utils/__init__.py @@ -0,0 +1,14 @@ +"""Collection of CLI helper functions.""" + +from .progress_manager import linear_progress, spinner +from .rich_utils import print, rich_print_to_string +from .system_detection import get_cpu_architecture, get_operating_system + +__all__ = [ + "linear_progress", + "spinner", + "print", + "rich_print_to_string", + "get_cpu_architecture", + "get_operating_system", +] diff --git a/report_generator/vendor/cc_common/utils/progress_manager.py b/report_generator/vendor/cc_common/utils/progress_manager.py new file mode 100644 index 0000000..6ce5f5b --- /dev/null +++ b/report_generator/vendor/cc_common/utils/progress_manager.py @@ -0,0 +1,57 @@ +"""Provide utilities for displaying progress bars and spinners in CLI applications.""" + +from collections.abc import Generator +from contextlib import contextmanager, nullcontext + +from rich.progress import ( + BarColumn, + MofNCompleteColumn, + Progress, + TaskID, + TextColumn, + TimeElapsedColumn, + TimeRemainingColumn, +) + +from cc_common.api import context + + +def _create_progress() -> Progress: + """Creates a configured Rich Progress instance used for linear processes.""" + return Progress( + TextColumn("[progress.description]{task.description}"), + TextColumn("[progress.percentage]{task.percentage:>3.0f}%"), + BarColumn(), + MofNCompleteColumn(), + TextColumn("•"), + TimeElapsedColumn(), + TextColumn("•"), + TimeRemainingColumn(), + console=context.console, + ) + + +@contextmanager +def spinner(message: str, show_progress: bool = True) -> Generator[None]: + """Optional spinner context manager. + + Args: + message: Text shown next to the spinner. + show_progress: If False, disables spinner output. + """ + ctx = context.console.status(message, spinner="dots") if show_progress else nullcontext() + with ctx: + yield + + +@contextmanager +def linear_progress(description: str, total: int) -> Generator[tuple[Progress, TaskID]]: + """Linear progress bar context manager with a single task. + + Args: + description: Task description shown in the progress bar. + total: Total number of steps for completion tracking. + """ + with _create_progress() as p: + task = p.add_task(description, total=total) + yield p, task diff --git a/report_generator/vendor/cc_common/utils/rich_utils.py b/report_generator/vendor/cc_common/utils/rich_utils.py new file mode 100644 index 0000000..2862f2c --- /dev/null +++ b/report_generator/vendor/cc_common/utils/rich_utils.py @@ -0,0 +1,32 @@ +"""Utilities for interacting with rich.""" + +from typing import IO, Any + +from rich import print as rich_print +from rich.console import Console +from rich.text import Text + +from cc_common.api import context + + +def rich_print_to_string(*objects: Any, width: int = 150) -> str: # noqa: ANN401 + """Render rich output into a string.""" + console = Console(width=width) + with console.capture() as capture: + console.print(objects) + return str(Text.from_ansi(capture.get())) + + +def print( + *objects: Any, # noqa: ANN401 + sep: str = " ", + end: str = "\n", + file: IO[str] | None = None, + flush: bool = False, +) -> None: + """Wrapper for the rich.print function. + + This function will do nothing if the application is not running in cli mode. + """ + if context.is_cli: + rich_print(*objects, sep=sep, end=end, file=file, flush=flush) diff --git a/report_generator/vendor/cc_common/utils/system_detection.py b/report_generator/vendor/cc_common/utils/system_detection.py new file mode 100644 index 0000000..7ba9051 --- /dev/null +++ b/report_generator/vendor/cc_common/utils/system_detection.py @@ -0,0 +1,41 @@ +"""Collection of helper functions, e.g. to detect the operating system or architecture of a system.""" + +import platform + +from cc_common.defines import CPUArchitecture, OperatingSystem + + +def get_operating_system() -> OperatingSystem: + """Returns the OS of the current system. + + :raises NotImplementedError: when the operating system is not supported. + """ + os_name = platform.system().lower() + match os_name: + case "darwin": + return OperatingSystem.MACOS + case "linux": + return OperatingSystem.LINUX + case "windows": + return OperatingSystem.WINDOWS + case _: + raise NotImplementedError(f"Unknown operating system {os_name}") + + +def get_cpu_architecture() -> CPUArchitecture: + """Return the CPU architecture of the current system.""" + machine = platform.machine().lower() + + match machine: + case "x86_64" | "amd64": + return CPUArchitecture.AMD64 + + # 64-bit ARM (Apple M1/M2/M3, modern Raspberry Pis, etc. + case "arm64" | "aarch64": + return CPUArchitecture.ARM64 + + case "i386" | "i686" | "x86" | "x86_32": + return CPUArchitecture.X86 + + case _: + return CPUArchitecture.UNKNOWN diff --git a/report_generator/vendor/cipherverifier/__init__.py b/report_generator/vendor/cipherverifier/__init__.py new file mode 100644 index 0000000..6341923 --- /dev/null +++ b/report_generator/vendor/cipherverifier/__init__.py @@ -0,0 +1 @@ +"""Top-level package for the CipherVerifier Django project.""" diff --git a/report_generator/vendor/cipherverifier/core/__init__.py b/report_generator/vendor/cipherverifier/core/__init__.py new file mode 100644 index 0000000..5a88867 --- /dev/null +++ b/report_generator/vendor/cipherverifier/core/__init__.py @@ -0,0 +1 @@ +"""Core Django app for CipherVerifier.""" diff --git a/report_generator/vendor/cipherverifier/core/services/__init__.py b/report_generator/vendor/cipherverifier/core/services/__init__.py new file mode 100644 index 0000000..9890b06 --- /dev/null +++ b/report_generator/vendor/cipherverifier/core/services/__init__.py @@ -0,0 +1 @@ +"""Service layer for pipeline orchestration.""" diff --git a/report_generator/vendor/cipherverifier/core/services/sarif.py b/report_generator/vendor/cipherverifier/core/services/sarif.py new file mode 100644 index 0000000..645832b --- /dev/null +++ b/report_generator/vendor/cipherverifier/core/services/sarif.py @@ -0,0 +1,677 @@ +"""Summarising of SARIF reports for presentation in the UI. + +The functions in here are deliberately free of any storage or Django dependency: they take an +already parsed SARIF document and turn it into a plain summary that a template can render. +""" + +from __future__ import annotations + +from collections import Counter +from dataclasses import dataclass, field +from typing import Any + +TOP_LIST_SIZE = 8 +"""How many entries of a top list are shown before the "See all" button has to be used.""" + +LEVEL_ORDER = ["error", "warning", "note", "none"] + +# Kept in sync with the --scan-chart-color-* palette in tailwind/input.css so the charts on the +# scan page look like the ones on the dashboard. +LEVEL_COLORS = { + "error": "oklch(0.5 0.199 28.536)", + "warning": "oklch(0.747 0.155 103.227)", + "note": "oklch(0.451 0.251 265.29)", + "none": "oklch(0.708 0 0)", +} + +# The verdicts polygraph can hand down, with the wording the UI uses for them. +POLYGRAPH_VERDICT_LABELS = { + "false_positive": "Likely false positive", + "true_positive": "Likely true positive", + "uncertain": "Uncertain", +} + + +@dataclass +class Region: + """A span within a file. Carries no path of its own - the location around it holds that.""" + + start_line: int | None + end_line: int | None = None + start_column: int | None = None + end_column: int | None = None + + @property + def display(self) -> str: + """The span as ``3``, ``3-5`` or ``3:13-26``, empty when it does not name a line.""" + if self.start_line is None: + return "" + # A span across lines reads as a line range; one within a single line adds its columns. + if self.end_line and self.end_line != self.start_line: + return f"{self.start_line}-{self.end_line}" + if self.start_column is None: + return str(self.start_line) + columns = str(self.start_column) + if self.end_column and self.end_column != self.start_column: + columns = f"{columns}-{self.end_column}" + return f"{self.start_line}:{columns}" + + +@dataclass +class Occurrence: + """A single tool hit that was merged into a finding by the sarifcombiner.""" + + found_by: str + tool_identifier: str + message: str + start_line: int | None + end_line: int | None = None + start_column: int | None = None + end_column: int | None = None + related_positions: list[Region] = field(default_factory=list) + + @property + def region(self) -> Region: + """The span this tool reported the finding at.""" + return Region(self.start_line, self.end_line, self.start_column, self.end_column) + + +@dataclass +class Location: + """A source location a finding points at.""" + + file: str + start_line: int | None + end_line: int | None + start_column: int | None = None + end_column: int | None = None + + @property + def region(self) -> Region: + """The span within the file.""" + return Region(self.start_line, self.end_line, self.start_column, self.end_column) + + @property + def display(self) -> str: + """The location as ``app/views.py:3:13-26``, degrading to just the path without a line.""" + region = self.region.display + return f"{self.file}:{region}" if region else self.file + + +@dataclass +class Suppression: + """A reason the finding was suppressed. Polygraph adds one for every false positive it finds.""" + + kind: str + justification: str + + +@dataclass +class Finding: + """A single SARIF result, flattened into the fields the template needs.""" + + rule_id: str + rule_name: str + rule_description: str + level: str + message: str + location: Location | None + extra_locations: list[Location] + tools: list[str] + occurrences: list[Occurrence] + rank: int | None + polygraph_verdict: str | None = None + polygraph_reasoning: str | None = None + polygraph_certainty: str = "" + polygraph_priority: str = "" + polygraph_prompt_hash: str = "" + rule_full_description: str = "" + rule_help: str = "" + rule_help_uri: str = "" + rule_tags: list[str] = field(default_factory=list) + rule_precision: str = "" + rule_security_severity: str = "" + rule_default_level: str = "" + suppressions: list[Suppression] = field(default_factory=list) + + @property + def tool_count(self) -> int: + """Number of distinct tools that reported this finding.""" + return len(self.tools) + + @property + def downgraded(self) -> bool: + """Whether this result is reported below the severity its rule defaults to. + + Tools do this per hit, so the gap is worth showing: a rule that defaults to ``error`` and + lands as a ``note`` says the tool itself was unsure about this particular match. The + reverse (a hit reported *above* the rule default) is not flagged here. + """ + if self.rule_default_level not in LEVEL_ORDER: + return False + return LEVEL_ORDER.index(self.level) > LEVEL_ORDER.index(self.rule_default_level) + + @property + def false_positive(self) -> bool: + """Whether polygraph classified this finding as a likely false positive.""" + return self.polygraph_verdict == "false_positive" + + @property + def verdict_label(self) -> str: + """The polygraph verdict in words, empty when the finding carries no known verdict.""" + return POLYGRAPH_VERDICT_LABELS.get(self.polygraph_verdict or "", "") + + +@dataclass +class Stat: + """A labelled count with its share of the largest count in the same list, in percent.""" + + label: str + sublabel: str + count: int + percent: int + + +@dataclass +class SarifReport: + """Everything the scan page shows about one SARIF report.""" + + total: int + findings: list[Finding] + level_counts: list[Stat] + tool_counts: list[Stat] + top_rules: list[Stat] + top_files: list[Stat] + file_count: int + rule_count: int + multi_tool_count: int + false_positive_count: int + ranked: bool + chart_data: dict[str, Any] = field(default_factory=dict) + + @property + def levels(self) -> dict[str, int]: + """Finding count per severity level, including the levels without any finding.""" + counts = dict.fromkeys(LEVEL_ORDER, 0) + for stat in self.level_counts: + counts[stat.label] = stat.count + return counts + + @property + def top_list_size(self) -> int: + """How many entries of ``top_rules`` and ``top_files`` the page shows without unfolding.""" + return TOP_LIST_SIZE + + +def _rules_by_id(runs: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: + """Collect the rule definitions of every run, keyed by rule id.""" + rules: dict[str, dict[str, Any]] = {} + for run in runs: + for rule in run.get("tool", {}).get("driver", {}).get("rules", []): + if isinstance(rule, dict) and "id" in rule: + rules.setdefault(rule["id"], rule) + return rules + + +def _driver_name(run: dict[str, Any]) -> str: + """Return the display name of the tool that produced a run.""" + driver = run.get("tool", {}).get("driver", {}) + return str(driver.get("name") or driver.get("fullName") or "unknown") + + +def _level_of(result: dict[str, Any], rule: dict[str, Any]) -> str: + """Return the severity of a result. + + Some tools (semgrep for instance) omit ``level`` on the result and only declare it on the rule, + so the rule's default configuration is used as a fallback. + + Args: + result: A SARIF result. + rule: The rule definition belonging to the result, or an empty dict. + + Returns: + One of the levels in ``LEVEL_ORDER``. + """ + level = result.get("level") or rule.get("defaultConfiguration", {}).get("level") + return level if level in LEVEL_ORDER else "none" + + +def _region_of(region: dict[str, Any]) -> Region: + """Read a SARIF region, or a sarifcombiner position - both spell the keys the same way.""" + return Region( + start_line=region.get("startLine"), + end_line=region.get("endLine"), + start_column=region.get("startColumn"), + end_column=region.get("endColumn"), + ) + + +def _locations_of(result: dict[str, Any]) -> list[Location]: + """Extract all physical locations of a result.""" + locations = [] + for location in result.get("locations", []): + physical = location.get("physicalLocation", {}) + uri = physical.get("artifactLocation", {}).get("uri") + if not uri: + continue + region = _region_of(physical.get("region", {})) + locations.append( + Location( + file=uri, + start_line=region.start_line, + end_line=region.end_line, + start_column=region.start_column, + end_column=region.end_column, + ) + ) + return locations + + +def _occurrences_of(result: dict[str, Any]) -> list[Occurrence]: + """Extract the per-tool occurrences the sarifcombiner merged into a result.""" + occurrences = [] + for occurrence in result.get("properties", {}).get("sarifgrouper", {}).get("occurrences", []): + found_by = occurrence.get("foundBy") + if not found_by: + continue + position = _region_of(occurrence.get("position", {}) or {}) + related = [_region_of(related) for related in occurrence.get("relatedPositions", []) or []] + occurrences.append( + Occurrence( + found_by=found_by, + tool_identifier=occurrence.get("toolIdentifier", ""), + message=occurrence.get("message", ""), + start_line=position.start_line, + end_line=position.end_line, + start_column=position.start_column, + end_column=position.end_column, + related_positions=[region for region in related if region.start_line is not None], + ) + ) + return occurrences + + +def _suppressions_of(result: dict[str, Any]) -> list[Suppression]: + """Extract the suppressions of a result. Polygraph adds one for each false positive it finds.""" + suppressions = [] + for suppression in result.get("suppressions", []) or []: + if not isinstance(suppression, dict): + continue + suppressions.append( + Suppression( + kind=str(suppression.get("kind") or ""), + justification=str(suppression.get("justification") or ""), + ) + ) + return suppressions + + +def _rank_of(result: dict[str, Any]) -> int | None: + """Return the prism rank of a result, if the report was ranked.""" + rank = result.get("properties", {}).get("prism", {}).get("rank") + if isinstance(rank, bool) or rank is None: + return None + if isinstance(rank, int | float): + return int(rank) + if isinstance(rank, str) and rank.isdigit(): + return int(rank) + return None + + +POLYGRAPH_VERDICTS = set(POLYGRAPH_VERDICT_LABELS) + + +EMPTY_POLYGRAPH: dict[str, Any] = {"verdict": None, "reasoning": None, "certainty": "", "priority": "", "hash": ""} + + +def _polygraph_of(result: dict[str, Any]) -> dict[str, Any]: + """Return what polygraph concluded about a result, if it was classified. + + Polygraph annotates each classified SARIF result with a ``polygraph`` property block. Findings + that were not classified (polygraph disabled, below the severity threshold, or errored) carry no + such block, so an empty verdict is returned and the finding reads as "not classified". + + Args: + result: A SARIF result. + + Returns: + A dict with the ``verdict`` (one of ``POLYGRAPH_VERDICTS`` or ``None``), the ``reasoning``, + and the ``certainty``, ``priority`` and prompt ``hash`` of the classification. Certainty and + priority are lowercased, so the templates can capitalise them like they do severities. + """ + polygraph = result.get("properties", {}).get("polygraph") + if not isinstance(polygraph, dict): + return dict(EMPTY_POLYGRAPH) + verdict = str(polygraph.get("verdict", "")).lower() + if verdict not in POLYGRAPH_VERDICTS: + # Tolerate an older shape that only carries the boolean flag. + verdict = "false_positive" if polygraph.get("isFalsePositive") else "" + reasoning = polygraph.get("reasoning") + return { + "verdict": verdict or None, + "reasoning": str(reasoning) if reasoning else None, + "certainty": str(polygraph.get("certainty") or "").lower(), + "priority": str(polygraph.get("priority") or "").lower(), + "hash": str(polygraph.get("promptHash") or ""), + } + + +def _rule_metadata(rule: dict[str, Any]) -> dict[str, Any]: + """Pull everything a rule definition says about itself. + + Which of these a rule carries is entirely up to the tool that declared it: the sarifcombiner + keeps descriptions and tags, while raw tool reports (rayfusion's, for instance) also come with + a ``helpUri`` and a precision. Anything absent comes back empty and the template leaves it out. + + Args: + rule: A SARIF rule definition, or an empty dict when the rule was not declared. + + Returns: + The rule's descriptions, help text and link, tags, precision, security severity and the + severity it defaults to. + """ + properties = rule.get("properties") or {} + tags = properties.get("tags") + security_severity = properties.get("security-severity", properties.get("securitySeverity")) + return { + "short_description": rule.get("shortDescription", {}).get("text") or "", + "full_description": rule.get("fullDescription", {}).get("text") or "", + "help": (rule.get("help") or {}).get("text") or "", + "help_uri": str(rule.get("helpUri") or ""), + "tags": [str(tag) for tag in tags] if isinstance(tags, list) else [], + "precision": str(properties.get("precision") or ""), + "security_severity": str(security_severity) if security_severity is not None else "", + "default_level": str(rule.get("defaultConfiguration", {}).get("level") or ""), + } + + +def _build_finding(result: dict[str, Any], run: dict[str, Any], rules: dict[str, dict[str, Any]]) -> Finding: + """Flatten a single SARIF result into a ``Finding``. + + Args: + result: The SARIF result to flatten. + run: The run the result belongs to, used as fallback for the tool name. + rules: All rule definitions of the report, keyed by rule id. + + Returns: + The flattened finding. + """ + rule_id = str(result.get("ruleId", "")) + rule = rules.get(rule_id, {}) + meta = _rule_metadata(rule) + occurrences = _occurrences_of(result) + locations = _locations_of(result) + polygraph = _polygraph_of(result) + + # The full description repeats the short one when a tool declares only one of the two, so it is + # dropped rather than shown twice underneath it. + description = meta["short_description"] or meta["full_description"] + full_description = meta["full_description"] if meta["full_description"] != description else "" + + return Finding( + rule_id=rule_id, + rule_name=rule.get("name", ""), + rule_description=description, + level=_level_of(result, rule), + message=result.get("message", {}).get("text", ""), + location=locations[0] if locations else None, + extra_locations=locations[1:], + tools=sorted({occurrence.found_by for occurrence in occurrences}) or [_driver_name(run)], + occurrences=occurrences, + rank=_rank_of(result), + polygraph_verdict=polygraph["verdict"], + polygraph_reasoning=polygraph["reasoning"], + polygraph_certainty=polygraph["certainty"], + polygraph_priority=polygraph["priority"], + polygraph_prompt_hash=polygraph["hash"], + rule_full_description=full_description, + rule_help=meta["help"] if meta["help"] not in (description, full_description) else "", + rule_help_uri=meta["help_uri"], + rule_tags=meta["tags"], + rule_precision=meta["precision"], + rule_security_severity=meta["security_severity"], + rule_default_level=meta["default_level"], + suppressions=_suppressions_of(result), + ) + + +def _sort_findings(findings: list[Finding], ranked: bool) -> list[Finding]: + """Sort findings, keeping the ones polygraph flagged as false positives at the bottom. + + The likely false positives sink below every real, uncertain or unclassified finding; within + each of those two groups the usual order is kept - by rank when the report is ranked, by + severity and location otherwise. + """ + if ranked: + return sorted(findings, key=lambda f: (f.false_positive, f.rank is None, f.rank or 0)) + return sorted( + findings, + key=lambda f: ( + f.false_positive, + LEVEL_ORDER.index(f.level), + f.location.file if f.location else "", + f.location.start_line or 0 if f.location else 0, + ), + ) + + +def _to_stats(counts: list[tuple[str, int]], sublabels: dict[str, str] | None = None) -> list[Stat]: + """Turn ``(label, count)`` pairs into ``Stat`` entries with a percentage of the largest count.""" + if not counts: + return [] + largest = max(count for _, count in counts) + return [ + Stat( + label=label, + sublabel=(sublabels or {}).get(label, ""), + count=count, + percent=round(count / largest * 100) if largest else 0, + ) + for label, count in counts + ] + + +def _level_stats(findings: list[Finding]) -> list[Stat]: + """Count findings per severity level, in severity order and without empty buckets.""" + counter = Counter(finding.level for finding in findings) + return _to_stats([(level, counter[level]) for level in LEVEL_ORDER if counter[level]]) + + +def _tool_stats(findings: list[Finding]) -> list[Stat]: + """Count findings per reporting tool, most findings first.""" + counter: Counter[str] = Counter() + for finding in findings: + counter.update(finding.tools) + return _to_stats(counter.most_common()) + + +def _rule_stats(findings: list[Finding]) -> list[Stat]: + """Rank every triggered rule by how often it was triggered, most triggered first. + + The scan page renders the whole list: it shows the first ``TOP_LIST_SIZE`` entries and reveals + the rest behind a "See all" button. + """ + counter = Counter(finding.rule_id for finding in findings if finding.rule_id) + names = {finding.rule_id: finding.rule_name for finding in findings} + return _to_stats(counter.most_common(), sublabels=names) + + +def _file_stats(findings: list[Finding]) -> list[Stat]: + """Rank every affected file by its number of findings, most findings first.""" + counter = Counter(finding.location.file for finding in findings if finding.location) + return _to_stats(counter.most_common()) + + +def _chart_data(level_counts: list[Stat], tool_counts: list[Stat]) -> dict[str, Any]: + """Build the data blob the Chart.js charts on the scan page are fed with.""" + return { + "severity": { + "labels": [stat.label.capitalize() for stat in level_counts], + "values": [stat.count for stat in level_counts], + "colors": [LEVEL_COLORS[stat.label] for stat in level_counts], + }, + "tools": { + "labels": [stat.label for stat in tool_counts], + "values": [stat.count for stat in tool_counts], + }, + } + + +def _build_report(findings: list[Finding], ranked: bool) -> SarifReport: + """Sort a list of findings and derive every statistic the templates render from it.""" + findings = _sort_findings(findings, ranked) + + level_counts = _level_stats(findings) + tool_counts = _tool_stats(findings) + + return SarifReport( + total=len(findings), + findings=findings, + level_counts=level_counts, + tool_counts=tool_counts, + top_rules=_rule_stats(findings), + top_files=_file_stats(findings), + file_count=len({finding.location.file for finding in findings if finding.location}), + rule_count=len({finding.rule_id for finding in findings if finding.rule_id}), + multi_tool_count=sum(1 for finding in findings if finding.tool_count > 1), + false_positive_count=sum(1 for finding in findings if finding.false_positive), + ranked=ranked, + chart_data=_chart_data(level_counts, tool_counts), + ) + + +def summarize_sarif(sarif_data: dict[str, Any]) -> SarifReport: + """Summarise a parsed SARIF document for rendering on the scan page. + + All runs of the document are merged, so reports that keep one run per tool are summarised just + as well as the combined report the pipeline usually produces. Every finding of the report is + included; the scan page renders the complete list. + + Args: + sarif_data: A parsed SARIF document. + + Returns: + The summarised report. + """ + runs = [run for run in sarif_data.get("runs", []) if isinstance(run, dict)] + rules = _rules_by_id(runs) + + findings = [_build_finding(result, run, rules) for run in runs for result in run.get("results", [])] + + return _build_report(findings, ranked=any(finding.rank is not None for finding in findings)) + + +@dataclass +class FindingsSummary: + """The numbers of one or more SARIF reports, in a form that can be stored and added up. + + The listing pages only show counts, never individual findings, so they work off this instead of + the full report: it is small enough to cache per scan and can be merged across repositories + without having the findings themselves at hand. + """ + + total: int = 0 + file_count: int = 0 + multi_tool_count: int = 0 + false_positive_count: int = 0 + level_totals: dict[str, int] = field(default_factory=dict) + rule_totals: dict[str, int] = field(default_factory=dict) + rule_names: dict[str, str] = field(default_factory=dict) + + @property + def levels(self) -> dict[str, int]: + """Finding count per severity level, including the levels without any finding.""" + counts = dict.fromkeys(LEVEL_ORDER, 0) + for level, count in self.level_totals.items(): + if level in counts: + counts[level] = count + return counts + + @property + def level_counts(self) -> list[Stat]: + """Findings per severity level, in severity order and without empty buckets.""" + levels = self.levels + return _to_stats([(level, levels[level]) for level in LEVEL_ORDER if levels[level]]) + + @property + def rule_count(self) -> int: + """Number of distinct rules that were triggered.""" + return len(self.rule_totals) + + @property + def top_rules(self) -> list[Stat]: + """The most frequently triggered rules.""" + return _to_stats(Counter(self.rule_totals).most_common(TOP_LIST_SIZE), sublabels=self.rule_names) + + @classmethod + def of(cls, report: SarifReport) -> FindingsSummary: + """Reduce a full report to the numbers the listing pages need.""" + return cls( + total=report.total, + file_count=report.file_count, + multi_tool_count=report.multi_tool_count, + false_positive_count=report.false_positive_count, + level_totals=report.levels, + rule_totals=dict(Counter(finding.rule_id for finding in report.findings if finding.rule_id)), + rule_names={ + finding.rule_id: finding.rule_name + for finding in report.findings + if finding.rule_id and finding.rule_name + }, + ) + + def to_dict(self) -> dict[str, Any]: + """Serialise the summary so it can be stored as JSON.""" + return { + "total": self.total, + "file_count": self.file_count, + "multi_tool_count": self.multi_tool_count, + "false_positive_count": self.false_positive_count, + "level_totals": self.level_totals, + "rule_totals": self.rule_totals, + "rule_names": self.rule_names, + } + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> FindingsSummary: + """Rebuild a summary from stored JSON, tolerating entries that miss a key.""" + return cls( + total=int(data.get("total", 0)), + file_count=int(data.get("file_count", 0)), + multi_tool_count=int(data.get("multi_tool_count", 0)), + false_positive_count=int(data.get("false_positive_count", 0)), + level_totals=dict(data.get("level_totals") or {}), + rule_totals=dict(data.get("rule_totals") or {}), + rule_names=dict(data.get("rule_names") or {}), + ) + + +def merge_summaries(summaries: list[FindingsSummary]) -> FindingsSummary: + """Add up the summaries of several scans. + + Files are counted per scan, so the same path in two repositories counts twice - which is what + the listing pages want, as those are two different files. + + Args: + summaries: The summaries to add up. + + Returns: + One summary covering all of them. + """ + levels: Counter[str] = Counter() + rules: Counter[str] = Counter() + names: dict[str, str] = {} + for summary in summaries: + levels.update(summary.level_totals) + rules.update(summary.rule_totals) + names.update(summary.rule_names) + + return FindingsSummary( + total=sum(summary.total for summary in summaries), + file_count=sum(summary.file_count for summary in summaries), + multi_tool_count=sum(summary.multi_tool_count for summary in summaries), + false_positive_count=sum(summary.false_positive_count for summary in summaries), + level_totals=dict(levels), + rule_totals=dict(rules), + rule_names=names, + ) diff --git a/report_generator/vendor/cipherverifier/core/services/vdr.py b/report_generator/vendor/cipherverifier/core/services/vdr.py new file mode 100644 index 0000000..dda6691 --- /dev/null +++ b/report_generator/vendor/cipherverifier/core/services/vdr.py @@ -0,0 +1,813 @@ +"""Summarising of vulnerability reports for presentation in the UI. + +Two shapes end up in the ``vdr.json`` artifact of a scan: the Finding Packaging Format that +Dependency-Track's findings export produces, and a plain CycloneDX VDR document. Both are accepted +here and reduced to the same summary. + +The functions in here are deliberately free of any storage or Django dependency: they take an +already parsed document and turn it into a plain summary that a template can render. +""" + +from __future__ import annotations + +import json +import re +from collections import Counter +from collections.abc import Mapping +from dataclasses import dataclass, field +from datetime import datetime +from typing import Any + +from cipherverifier.core.services.sarif import TOP_LIST_SIZE, Stat +from persistent_storage.utils.cvss_utils import IMPACT_KEYS, describe, parse_vector + +# CycloneDX severity values, most severe first. ``unknown`` covers reports that carry no rating. +SEVERITY_ORDER = ["critical", "high", "medium", "low", "info", "unknown"] + +# Higher rank means more severe, so the highest rated severity of a vulnerability wins. +SEVERITY_RANK: dict[str, int] = { + "critical": 5, + "high": 4, + "medium": 3, + "low": 2, + "info": 1, + "unknown": 0, +} + +# Kept as OKLCH so the scan page charts match the style of the SARIF charts. +SEVERITY_COLORS: dict[str, str] = { + "critical": "oklch(0.45 0.22 27)", + "high": "oklch(0.62 0.20 38)", + "medium": "oklch(0.75 0.15 80)", + "low": "oklch(0.65 0.12 250)", + "info": "oklch(0.708 0 0)", + "unknown": "oklch(0.708 0 0)", +} + + +# How CycloneDX names each CVSS generation in a rating's ``method``, by which one is newer. +CYCLONEDX_METHODS = {"cvssv2": 2, "cvssv3": 3, "cvssv31": 3, "cvssv4": 4} + + +@dataclass +class Component: + """A dependency a vulnerability affects. + + Attributes: + name: What the package is called. + version: The version this scan depends on, empty when the report named none. + latest_version: The newest version the reporting tool knew of. What makes + a finding answerable -- a newer version existing is not proof that it + fixes this advisory, only that there is somewhere to upgrade to. + """ + + name: str + version: str + latest_version: str = "" + + @property + def is_outdated(self) -> bool: + """Whether the package has been released again since the version in use.""" + return bool(self.latest_version) and self.latest_version != self.version + + +@dataclass +class VulnerabilityMetrics: + """What one advisory's CVSS vector and scoring say, in the words a reader uses. + + Attributes: + attack_vector: Where an attacker has to be. + attack_complexity: Whether anything outside their control has to line up. + privileges_required: What account they need. v2 states how often they log + in instead, which is the same question. + user_interaction: Who they have to fool. + confidentiality: What they get to read. + integrity: What they get to change. + availability: How far they can take the service down. + epss: The probability of exploitation in the next 30 days, 0 to 1, or + ``None`` where nobody scored it -- which is not a probability of zero. + epss_percentile: Where that probability ranks among everything scored. + published_days_ago: How long the advisory has been public. + """ + + attack_vector: str = "" + attack_complexity: str = "" + privileges_required: str = "" + user_interaction: str = "" + confidentiality: str = "" + integrity: str = "" + availability: str = "" + epss: float | None = None + epss_percentile: float | None = None + published_days_ago: int | None = None + + +@dataclass +class Vulnerability: + """A single CycloneDX vulnerability, flattened into the fields the template needs.""" + + id: str + severity: str + description: str + cwes: list[str] + components: list[Component] + score: float | None + recommendation: str + url: str + metrics: VulnerabilityMetrics = field(default_factory=VulnerabilityMetrics) + cwe_names: dict[str, str] = field(default_factory=dict) + + @property + def cwes_json(self) -> str: + """The weakness classes as one JSON array of identifier and name. + + One attribute rather than two parallel pipe-joined lists, which would have to be kept in + step by whoever writes the template and would silently mispair if they ever were not. + A class the report named without describing carries an empty name, and the panel shows the + identifier alone rather than inventing a description for it. + """ + return json.dumps( + [{"id": one, "name": self.cwe_names.get(one, "")} for one in self.cwes], + separators=(",", ":"), + ) + + @property + def fix_available(self) -> bool: + """Whether any affected package has been released again since.""" + return any(component.is_outdated for component in self.components) + + @property + def metrics_json(self) -> str: + """The metrics as one JSON object, for the row to carry into the panel. + + One attribute rather than ten: the panel needs all of them or none, and a single blob + cannot end up half-updated by a template that forgot a field. Keyed the way the script + reads them, which is the same shape its sample values have -- so the panel does not know + or care which of the two it was handed. + """ + return json.dumps( + { + "score": self.score, + "band": self.severity.capitalize(), + "epss": self.metrics.epss, + "epssPercentile": self.metrics.epss_percentile, + "confidentiality": self.metrics.confidentiality, + "integrity": self.metrics.integrity, + "availability": self.metrics.availability, + "attackVector": self.metrics.attack_vector, + "attackComplexity": self.metrics.attack_complexity, + "privilegesRequired": self.metrics.privileges_required, + "userInteraction": self.metrics.user_interaction, + "publishedDaysAgo": self.metrics.published_days_ago, + "fixAvailable": self.fix_available, + }, + separators=(",", ":"), + ) + + @property + def component_names(self) -> list[str]: + """Display names of the affected components, name plus version when the version is known.""" + return [ + f"{component.name}@{component.version}" if component.version else component.name + for component in self.components + ] + + +@dataclass +class VdrReport: + """Everything the scan page shows about one CycloneDX VDR report.""" + + total: int + vulnerabilities: list[Vulnerability] + severity_counts: list[Stat] + top_components: list[Stat] + component_count: int + chart_data: dict[str, Any] = field(default_factory=dict) + + @property + def critical_high_count(self) -> int: + """Number of vulnerabilities rated critical or high.""" + return sum(1 for vuln in self.vulnerabilities if vuln.severity in ("critical", "high")) + + +def _components_by_ref(data: dict[str, Any]) -> dict[str, Component]: + """Map the bom-ref of every component to a ``Component``, for resolving ``affects``. + + Args: + data: A parsed CycloneDX VDR document. + + Returns: + The components keyed by their bom-ref. + """ + components: dict[str, Component] = {} + for component in data.get("components", []): + if not isinstance(component, dict): + continue + bom_ref = component.get("bom-ref") or component.get("purl") + if not bom_ref: + continue + components[bom_ref] = Component( + name=str(component.get("name") or bom_ref), + version=str(component.get("version") or ""), + ) + return components + + +def _severity_of(vulnerability: dict[str, Any]) -> str: + """Return the most severe rating of a vulnerability, ``unknown`` when none is given. + + Args: + vulnerability: A single CycloneDX vulnerability. + + Returns: + One of the severities in ``SEVERITY_ORDER``. + """ + best = "unknown" + for rating in vulnerability.get("ratings", []): + if not isinstance(rating, dict): + continue + severity = rating.get("severity") + if isinstance(severity, str) and severity in SEVERITY_RANK and SEVERITY_RANK[severity] > SEVERITY_RANK[best]: + best = severity + return best + + +def _score_of(vulnerability: dict[str, Any]) -> float | None: + """Return the highest CVSS score carried by any rating of a vulnerability. + + Args: + vulnerability: A single CycloneDX vulnerability. + + Returns: + The highest score, or ``None`` when no rating has a numeric score. + """ + scores = [ + float(rating["score"]) + for rating in vulnerability.get("ratings", []) + if isinstance(rating, dict) and isinstance(rating.get("score"), int | float) + ] + return max(scores) if scores else None + + +def _cwes_of(vulnerability: dict[str, Any]) -> list[str]: + """Return the CWE ids of a vulnerability as ``CWE-`` strings. + + Args: + vulnerability: A single CycloneDX vulnerability. + + Returns: + The CWE ids, normalised to the ``CWE-`` form. + """ + cwes: list[str] = [] + for value in vulnerability.get("cwes", []): + if isinstance(value, int): + cwes.append(f"CWE-{value}") + elif isinstance(value, str) and value: + cwes.append(value if value.startswith("CWE-") else f"CWE-{value}") + return cwes + + +def _components_of(vulnerability: dict[str, Any], components: dict[str, Component]) -> list[Component]: + """Resolve the affected components of a vulnerability via their bom-ref. + + Affects whose reference is unknown are still listed, using the reference itself as the name so + the vulnerability is never left without a target. + + Args: + vulnerability: A single CycloneDX vulnerability. + components: All components of the document, keyed by bom-ref. + + Returns: + The resolved components the vulnerability affects. + """ + resolved: list[Component] = [] + for affect in vulnerability.get("affects", []): + ref = affect.get("ref") if isinstance(affect, dict) else None + if not ref: + continue + resolved.append(components.get(ref, Component(name=str(ref), version=""))) + return resolved + + +def _url_of(vulnerability: dict[str, Any]) -> str: + """Return a reference url for a vulnerability: its source url or the first advisory. + + Args: + vulnerability: A single CycloneDX vulnerability. + + Returns: + A url, or an empty string when the vulnerability carries none. + """ + source_url = vulnerability.get("source", {}) + if isinstance(source_url, dict) and source_url.get("url"): + return str(source_url["url"]) + for advisory in vulnerability.get("advisories", []): + url = advisory.get("url") if isinstance(advisory, dict) else None + if url: + return str(url) + return "" + + +def _build_vulnerability(vulnerability: dict[str, Any], components: dict[str, Component]) -> Vulnerability: + """Flatten a single CycloneDX vulnerability into a ``Vulnerability``. + + Args: + vulnerability: A single CycloneDX vulnerability. + components: All components of the document, keyed by bom-ref. + + Returns: + The flattened vulnerability. + """ + return Vulnerability( + id=str(vulnerability.get("id") or "Unknown"), + severity=_severity_of(vulnerability), + description=str(vulnerability.get("description") or vulnerability.get("detail") or ""), + cwes=_cwes_of(vulnerability), + components=_components_of(vulnerability, components), + score=_score_of(vulnerability), + recommendation=str(vulnerability.get("recommendation") or ""), + url=_url_of(vulnerability), + metrics=_metrics_of(vulnerability, _vector_of(vulnerability)), + ) + + +def _vector_of(vulnerability: dict[str, Any]) -> str | None: + """Return the newest CVSS vector a CycloneDX vulnerability rates itself with. + + Args: + vulnerability: A single CycloneDX vulnerability. + + Returns: + The vector, or ``None`` when no rating carries one. CycloneDX names the + method rather than versioning the vector, so the ratings are ordered by + the method they name and the last generation wins. + """ + rated = [ + (str(rating.get("method") or ""), str(rating["vector"])) + for rating in vulnerability.get("ratings", []) + if isinstance(rating, dict) and rating.get("vector") + ] + + if not rated: + return None + + return max(rated, key=lambda pair: CYCLONEDX_METHODS.get(pair[0].lower(), 0))[1] + + +def _sort_vulnerabilities(vulnerabilities: list[Vulnerability]) -> list[Vulnerability]: + """Sort vulnerabilities by severity, most severe first.""" + return sorted(vulnerabilities, key=lambda vuln: -SEVERITY_RANK.get(vuln.severity, 0)) + + +def _to_stats(counts: list[tuple[str, int]], sublabels: dict[str, str] | None = None) -> list[Stat]: + """Turn ``(label, count)`` pairs into ``Stat`` entries with a percentage of the largest count. + + Args: + counts: The ``(label, count)`` pairs to turn into stats. + sublabels: Optional extra labels keyed by the stat label. + + Returns: + The stats, each carrying its share of the largest count in percent. + """ + if not counts: + return [] + largest = max(count for _, count in counts) + return [ + Stat( + label=label, + sublabel=(sublabels or {}).get(label, ""), + count=count, + percent=round(count / largest * 100) if largest else 0, + ) + for label, count in counts + ] + + +def _severity_stats(severity_totals: Mapping[str, int]) -> list[Stat]: + """Turn vulnerability counts per severity into stats, in severity order and without empty buckets. + + The percentage is the share of the total number of vulnerabilities, so the bars in the overview + show each severity's proportion rather than just its size relative to the largest bucket. + + Args: + severity_totals: The number of vulnerabilities per severity. + + Returns: + The per-severity stats. + """ + total = sum(severity_totals.values()) + return [ + Stat( + label=severity, + sublabel="", + count=severity_totals[severity], + percent=round(severity_totals[severity] / total * 100) if total else 0, + ) + for severity in SEVERITY_ORDER + if severity_totals.get(severity) + ] + + +def _component_stats(vulnerabilities: list[Vulnerability]) -> list[Stat]: + """Count the components with the most vulnerabilities. + + Args: + vulnerabilities: The vulnerabilities to count components from. + + Returns: + The most frequently affected components. + """ + counter: Counter[str] = Counter() + for vuln in vulnerabilities: + counter.update(vuln.component_names) + return _to_stats(counter.most_common(TOP_LIST_SIZE)) + + +def _chart_data(severity_counts: list[Stat], component_counts: list[Stat]) -> dict[str, Any]: + """Build the data blob the Chart.js charts on the scan page are fed with. + + Args: + severity_counts: The per-severity stats. + component_counts: The per-component stats. + + Returns: + The chart data for the severity doughnut and the components bar chart. + """ + return { + "severity": { + "labels": [stat.label.capitalize() for stat in severity_counts], + "values": [stat.count for stat in severity_counts], + "colors": [SEVERITY_COLORS[stat.label] for stat in severity_counts], + }, + "components": { + "labels": [stat.label for stat in component_counts], + "values": [stat.count for stat in component_counts], + }, + } + + +def _build_report(vulnerabilities: list[Vulnerability]) -> VdrReport: + """Sort a list of vulnerabilities and derive every statistic the templates render from it. + + Args: + vulnerabilities: The vulnerabilities to summarise. + + Returns: + The summarised report. + """ + vulnerabilities = _sort_vulnerabilities(vulnerabilities) + severity_counts = _severity_stats(Counter(vuln.severity for vuln in vulnerabilities)) + component_counts = _component_stats(vulnerabilities) + + return VdrReport( + total=len(vulnerabilities), + vulnerabilities=vulnerabilities, + severity_counts=severity_counts, + top_components=component_counts, + component_count=len({name for vuln in vulnerabilities for name in vuln.component_names}), + chart_data=_chart_data(severity_counts, component_counts), + ) + + +# ── Dependency-Track findings export (Finding Packaging Format) ───────────── + +# Dependency-Track spells its severities in upper case and uses ``UNASSIGNED`` for unrated findings. +FPF_SEVERITIES: dict[str, str] = { + "CRITICAL": "critical", + "HIGH": "high", + "MEDIUM": "medium", + "LOW": "low", + "INFO": "info", + "UNASSIGNED": "unknown", +} + +# The ``references`` field is markdown, so the urls are pulled back out of the link targets. +_MARKDOWN_LINK = re.compile(r"\((https?://[^)\s]+)\)") + + +def _fpf_severity_of(vulnerability: dict[str, Any]) -> str: + """Return the severity of a Dependency-Track finding, ``unknown`` when it carries none.""" + severity = vulnerability.get("severity") + return FPF_SEVERITIES.get(severity.upper(), "unknown") if isinstance(severity, str) else "unknown" + + +def _fpf_score_of(vulnerability: dict[str, Any]) -> float | None: + """Return the highest CVSS base score a Dependency-Track finding carries. + + Args: + vulnerability: The ``vulnerability`` part of a single finding. + + Returns: + The highest score of any CVSS version, or ``None`` when the finding carries none. + """ + scores = [ + float(vulnerability[key]) + for key in ("cvssV2BaseScore", "cvssV3BaseScore", "cvssV4Score") + if isinstance(vulnerability.get(key), int | float) + ] + return max(scores) if scores else None + + +def _fpf_cwes_of(vulnerability: dict[str, Any]) -> list[str]: + """Return the CWE ids of a Dependency-Track finding as ``CWE-`` strings. + + Newer reports carry a ``cwes`` list, older ones only a single ``cweId``. + + Args: + vulnerability: The ``vulnerability`` part of a single finding. + + Returns: + The CWE ids, normalised to the ``CWE-`` form. + """ + cwes = [ + f"CWE-{cwe['cweId']}" + for cwe in vulnerability.get("cwes", []) + if isinstance(cwe, dict) and cwe.get("cweId") is not None + ] + if not cwes and vulnerability.get("cweId") is not None: + cwes.append(f"CWE-{vulnerability['cweId']}") + return cwes + + +def _fpf_cwe_names_of(vulnerability: dict[str, Any]) -> dict[str, str]: + """Return what each weakness class of a Dependency-Track finding is called. + + Args: + vulnerability: The ``vulnerability`` part of a single finding. + + Returns: + The names by identifier. Older reports name only the single ``cweId`` + they carry, under ``cweName``; newer ones name each entry of ``cwes``. + Both are read, and a report that names none simply yields nothing. + """ + names = { + f"CWE-{cwe['cweId']}": str(cwe["name"]) + for cwe in vulnerability.get("cwes", []) + if isinstance(cwe, dict) and cwe.get("cweId") is not None and cwe.get("name") + } + + if not names and vulnerability.get("cweId") is not None and vulnerability.get("cweName"): + names[f"CWE-{vulnerability['cweId']}"] = str(vulnerability["cweName"]) + + return names + + +def _fpf_url_of(vulnerability: dict[str, Any]) -> str: + """Return the first reference url of a Dependency-Track finding. + + Args: + vulnerability: The ``vulnerability`` part of a single finding. + + Returns: + A url, or an empty string when the finding references none. + """ + references = vulnerability.get("references") + if not isinstance(references, str): + return "" + match = _MARKDOWN_LINK.search(references) + return match.group(1) if match else "" + + +def _fpf_component_of(finding: dict[str, Any]) -> Component | None: + """Return the component a Dependency-Track finding was raised against.""" + component = finding.get("component") + if not isinstance(component, dict) or not component.get("name"): + return None + return Component( + name=str(component["name"]), + version=str(component.get("version") or ""), + latest_version=str(component.get("latestVersion") or ""), + ) + + +# The vectors a Dependency-Track export can state, newest generation first. +FPF_VECTORS = ("cvssV4Vector", "cvssV3Vector", "cvssV2Vector") + + +def _fpf_vector_of(vulnerability: dict[str, Any]) -> str | None: + """Return the newest CVSS vector a Dependency-Track finding states, if any.""" + for key in FPF_VECTORS: + if isinstance(vulnerability.get(key), str) and vulnerability[key]: + return str(vulnerability[key]) + + return None + + +def _metrics_of(vulnerability: dict[str, Any], vector: str | None) -> VulnerabilityMetrics: + """Read one advisory's vector and scoring into the words the panel shows. + + Args: + vulnerability: The advisory, in either document's shape. + vector: The CVSS vector to read, if the report stated one. + + Returns: + The metrics. Everything the report did not state stays empty rather than + being filled with a default: a metric nobody stated is not one stated as + harmless, and the panel greys out what it was not told. + """ + reading = parse_vector(vector) + spoken = {} + + if reading is not None: + confidentiality, integrity, availability = IMPACT_KEYS[reading.generation] + spoken = { + "attack_vector": _spoken("AV", reading.attack_vector, reading.generation), + "attack_complexity": _spoken("AC", reading.attack_complexity, reading.generation), + "privileges_required": _spoken( + "Au" if reading.generation == 2 else "PR", + reading.authentication if reading.generation == 2 else reading.privileges_required, + reading.generation, + ), + "user_interaction": _spoken("UI", reading.user_interaction, reading.generation), + "confidentiality": _spoken(confidentiality, reading.confidentiality_impact, reading.generation), + "integrity": _spoken(integrity, reading.integrity_impact, reading.generation), + "availability": _spoken(availability, reading.availability_impact, reading.generation), + } + + return VulnerabilityMetrics( + epss=_probability(vulnerability.get("epssScore")), + epss_percentile=_probability(vulnerability.get("epssPercentile")), + published_days_ago=_days_since(vulnerability.get("published")), + **spoken, + ) + + +def _spoken(metric: str, value: str | None, generation: int) -> str: + """Return the word behind one abbreviation, or nothing where the vector carried none.""" + return "" if value is None else describe(metric, value, generation) + + +def _probability(stated: Any) -> float | None: # noqa: ANN401 + """Return a probability the report stated, refusing anything that is not one.""" + return float(stated) if isinstance(stated, int | float) and not isinstance(stated, bool) else None + + +def _days_since(stated: Any) -> int | None: # noqa: ANN401 + """Return how many days ago a timestamp was, or nothing where it cannot be read.""" + if not isinstance(stated, str): + return None + + try: + published = datetime.fromisoformat(stated) + except ValueError: + return None + + now = datetime.now(published.tzinfo) if published.tzinfo else datetime.now() # noqa: DTZ005 + + return max(0, (now - published).days) + + +def _is_suppressed(finding: dict[str, Any]) -> bool: + """Tell whether a finding was suppressed in Dependency-Track.""" + analysis = finding.get("analysis") + return isinstance(analysis, dict) and bool(analysis.get("isSuppressed")) + + +def _summarize_findings(findings: list[Any]) -> VdrReport: + """Summarise a Dependency-Track findings export. + + Every finding pairs one component with one vulnerability, so a vulnerability affecting several + dependencies arrives as several findings. They are merged back into a single vulnerability that + lists all of its components, which is what the scan page shows. + + Args: + findings: The ``findings`` list of the export. + + Returns: + The summarised report. + """ + merged: dict[str, Vulnerability] = {} + for finding in findings: + if not isinstance(finding, dict) or _is_suppressed(finding): + continue + vulnerability = finding.get("vulnerability") + if not isinstance(vulnerability, dict): + continue + + key = str(vulnerability.get("uuid") or vulnerability.get("vulnId") or "") + component = _fpf_component_of(finding) + if key in merged: + if component is not None: + merged[key].components.append(component) + continue + + merged[key] = Vulnerability( + id=str(vulnerability.get("vulnId") or "Unknown"), + severity=_fpf_severity_of(vulnerability), + description=str(vulnerability.get("description") or vulnerability.get("title") or ""), + cwes=_fpf_cwes_of(vulnerability), + components=[component] if component is not None else [], + score=_fpf_score_of(vulnerability), + recommendation=str(vulnerability.get("recommendation") or ""), + url=_fpf_url_of(vulnerability), + metrics=_metrics_of(vulnerability, _fpf_vector_of(vulnerability)), + cwe_names=_fpf_cwe_names_of(vulnerability), + ) + + return _build_report(list(merged.values())) + + +def summarize_vdr(vdr_data: dict[str, Any]) -> VdrReport: + """Summarise a parsed vulnerability report for rendering on the scan page. + + Both the Dependency-Track findings export and a CycloneDX VDR document are accepted; the + ``findings`` key is what tells them apart. Every vulnerability of the document is included; the + scan page renders the complete list. + + Args: + vdr_data: A parsed findings export or CycloneDX VDR document. + + Returns: + The summarised report. + """ + findings = vdr_data.get("findings") + if isinstance(findings, list): + return _summarize_findings(findings) + + components = _components_by_ref(vdr_data) + vulnerabilities = [ + _build_vulnerability(vuln, components) for vuln in vdr_data.get("vulnerabilities", []) if isinstance(vuln, dict) + ] + return _build_report(vulnerabilities) + + +@dataclass +class VulnerabilitySummary: + """The numbers of one or more vulnerability reports, in a form that can be stored and added up. + + The counterpart of ``FindingsSummary``: the listing pages only show counts, never individual + vulnerabilities, so they work off this instead of the full report. + """ + + total: int = 0 + component_count: int = 0 + severity_totals: dict[str, int] = field(default_factory=dict) + + @property + def severities(self) -> dict[str, int]: + """Vulnerability count per severity, including the severities without any vulnerability.""" + counts = dict.fromkeys(SEVERITY_ORDER, 0) + for severity, count in self.severity_totals.items(): + if severity in counts: + counts[severity] = count + return counts + + @property + def severity_counts(self) -> list[Stat]: + """Vulnerabilities per severity, in severity order and without empty buckets.""" + return _severity_stats(self.severities) + + @property + def critical_high_count(self) -> int: + """Number of vulnerabilities rated critical or high.""" + severities = self.severities + return severities["critical"] + severities["high"] + + @classmethod + def of(cls, report: VdrReport) -> VulnerabilitySummary: + """Reduce a full report to the numbers the listing pages need.""" + return cls( + total=report.total, + component_count=report.component_count, + severity_totals={stat.label: stat.count for stat in report.severity_counts}, + ) + + def to_dict(self) -> dict[str, Any]: + """Serialise the summary so it can be stored as JSON.""" + return { + "total": self.total, + "component_count": self.component_count, + "severity_totals": self.severity_totals, + } + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> VulnerabilitySummary: + """Rebuild a summary from stored JSON, tolerating entries that miss a key.""" + return cls( + total=int(data.get("total", 0)), + component_count=int(data.get("component_count", 0)), + severity_totals=dict(data.get("severity_totals") or {}), + ) + + +def merge_vulnerability_summaries(summaries: list[VulnerabilitySummary]) -> VulnerabilitySummary: + """Add up the summaries of several scans. + + Components are counted per scan, so the same dependency in two repositories counts twice - which + is what the listing pages want, as those are two different components to update. + + Args: + summaries: The summaries to add up. + + Returns: + One summary covering all of them. + """ + severities: Counter[str] = Counter() + for summary in summaries: + severities.update(summary.severity_totals) + + return VulnerabilitySummary( + total=sum(summary.total for summary in summaries), + component_count=sum(summary.component_count for summary in summaries), + severity_totals=dict(severities), + ) diff --git a/report_generator/vendor/persistent_storage/__init__.py b/report_generator/vendor/persistent_storage/__init__.py new file mode 100644 index 0000000..f93fde0 --- /dev/null +++ b/report_generator/vendor/persistent_storage/__init__.py @@ -0,0 +1 @@ +"""persistent_storage - Persistent Metadata and SBOM/SARIF file storage.""" diff --git a/report_generator/vendor/persistent_storage/helper.py b/report_generator/vendor/persistent_storage/helper.py new file mode 100644 index 0000000..4d3ed47 --- /dev/null +++ b/report_generator/vendor/persistent_storage/helper.py @@ -0,0 +1,471 @@ +"""Helper functions for persistent storage.""" + +import json +from pathlib import Path +from shutil import which +from typing import TYPE_CHECKING, Any + +from rich.table import Table +from sqlalchemy import inspect + +from cc_common.api import context +from cc_common.processes import run_subprocess +from cc_common.utils import print +from persistent_storage.wrapper import Artifact, Component, CvssMetric, DependencyFinding, Scan + +if TYPE_CHECKING: + # pandas (~3s) is only needed by the DataFrame-building calls below; keep it off the import + # path (this module loads on every Django boot via orchestrator -> persistent_storage.commands) + # and import it lazily where it is actually used. + import pandas as pd + from pandas import DataFrame + +type JsonData = str | int | float | bool | None | list["JsonData"] | dict[str, "JsonData"] + +TOOL_NAME_KEYS = { + "tool_name", + "toolName", + "name", + "scanner", + "scanner_name", + "generator", + "generated_by", + "generatedBy", + "created_by", + "createdBy", + "producer", + "application", +} + +TOOL_VERSION_KEYS = { + "tool_version", + "toolVersion", + "version", + "scanner_version", + "generator_version", + "generatedByVersion", + "createdByVersion", +} + +TOOL_CONTAINER_KEYS = { + "tool", + "tools", + "scanner", + "generator", + "metadata", + "producer", + "createdBy", + "generatedBy", +} + + +def start_docker_containers() -> None: + """Starts the docker containers for development. + + Raises: + FileNotFoundError: + Raised if the docker-compose.yml file does not exist. + RuntimeError: + Raised if the docker executable cannot be found. + """ + compose_file = Path("src/persistent_storage/docker-compose.yml") + + if not compose_file.exists(): + context.logger.error(f"Compose file not found at {compose_file} ❌") + raise FileNotFoundError(f"Compose file not found at {compose_file}") + + docker_path = which("docker") + + if docker_path is not None: + command = [docker_path, "compose"] + else: + legacy_compose_path = which("docker-compose") + + if legacy_compose_path is None: + context.logger.error("Docker executable not found ❌") + raise RuntimeError("Docker executable not found ❌") + + command = [legacy_compose_path] + + command += ["-f", str(compose_file), "up", "-d", "--build"] + + run_subprocess( + command, + Path.cwd(), + ) + + +def extract_tool_information_from_sarif(file_path: Path) -> tuple[str | None, str | None]: + """Extract tool information from Sarif file.""" + with file_path.open(encoding="utf-8") as file: + data = json.load(file) + + runs = data.get("runs", []) + if not runs: + return "Unknown", "Unknown" + + driver = runs[0].get("tool", {}).get("driver", {}) + + tool_name = driver.get("name") + tool_version = driver.get("version") or driver.get("semanticVersion") or driver.get("dottedQuadFileVersion") + + return tool_name, tool_version + + +def extract_tool_information_from_cyclonedx(file_path: Path) -> tuple[str | None, str | None]: + """Extract tool information from CyclonedX file.""" + with file_path.open(encoding="utf-8") as file: + data = json.load(file) + + tools = data.get("metadata", {}).get("tools") + + if isinstance(tools, list) and tools: + tool = tools[0] + return tool.get("name"), tool.get("version") + + if isinstance(tools, dict): + components = tools.get("components", []) + if components: + tool = components[0] + return tool.get("name"), tool.get("version") + + return "Unknown", "Unknown" + + +def extract_tool_information_from_spdx(file_path: Path) -> tuple[str | None, str | None]: + """Extract tool information from SPDX file.""" + with file_path.open(encoding="utf-8") as file: + data = json.load(file) + + creators = data.get("creationInfo", {}).get("creators", []) + + for creator in creators: + if isinstance(creator, str) and creator.startswith("Tool:"): + value = creator.removeprefix("Tool:").strip() + + if "-" in value: + name, version = value.rsplit("-", 1) + return name, version + + return value, "Unknown" + + return "Unknown", "Unknown" + + +def _first_string_value(data: dict[str, Any], keys: set[str]) -> str | None: + """Return the first non-empty string value for any matching key.""" + for key in keys: + value = data.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + + return None + + +def extract_tool_information_from_json(data: JsonData) -> tuple[str | None, str | None]: + """Recursively search generic JSON data for tool name and version.""" + if isinstance(data, dict): + direct_name = _first_string_value(data, TOOL_NAME_KEYS) + direct_version = _first_string_value(data, TOOL_VERSION_KEYS) + + if direct_name or direct_version: + return direct_name, direct_version + + for key in TOOL_CONTAINER_KEYS: + value = data.get(key) + tool_name, tool_version = extract_tool_information_from_json(value) + if tool_name or tool_version: + return tool_name, tool_version + + for value in data.values(): + tool_name, tool_version = extract_tool_information_from_json(value) + if tool_name or tool_version: + return tool_name, tool_version + + elif isinstance(data, list): + for item in data: + tool_name, tool_version = extract_tool_information_from_json(item) + if tool_name or tool_version: + return tool_name, tool_version + + return None, None + + +def print_rich_table(df: pd.DataFrame, table_type: str) -> None: + """Print a pandas DataFrame as a rich table.""" + str_df = df.fillna("").astype(str) + + table = Table( + show_header=True, + header_style="bold", + ) + + if table_type == "scan": + _print_rich_table_scan(str_df, table) + elif table_type == "artifact": + _print_rich_table_artifact(str_df, table) + elif table_type in ("component", "dependency_finding", "cvss"): + _print_rich_table_plain(str_df, table, table_type) + else: + raise ValueError(f"Invalid table type: {table_type}") + + print(table) + + +def _print_rich_table_scan(str_df: DataFrame, table: Table) -> None: + for column in str_df.columns: + if column == "meta_file_content": + table.add_column( + str(column), + no_wrap=False, + overflow="fold", + ) + else: + table.add_column( + str(column), + no_wrap=True, + overflow="ignore", + ) + + for row in str_df.itertuples(index=False): + values = [] + + for col, value in zip(str_df.columns, row, strict=False): + text = value + + if col == "meta_file_content": + text = text.replace("\\n", "\n") + + values.append(text) + + table.add_row(*values) + + print("\n[bold]----------Stored scan metadata entries----------[/bold]") + + +def _print_rich_table_artifact(str_df: DataFrame, table: Table) -> None: + for column in str_df.columns: + if column in ("file_name", "created_at", "artifact_metadata"): + table.add_column( + str(column), + max_width=20, + overflow="fold", + ) + else: + table.add_column(str(column)) + + for artifact_row in str_df.itertuples( + index=False, + name=None, + ): + table.add_row(*artifact_row) + + print("\n[bold]----------Stored artifacts metadata entry----------[/bold]") + + +# What each of the dependency tables is called above its rows. +PLAIN_TABLE_HEADINGS = { + "component": "Dependencies found by the scan", + "dependency_finding": "Advisories against those dependencies", + "cvss": "CVSS scorings of those advisories", +} + +# Columns long enough to need folding rather than a column of their own width. +WIDE_COLUMNS = ("purl", "description", "vector", "metrics", "cwes") + + +def _print_rich_table_plain(str_df: DataFrame, table: Table, table_type: str) -> None: + for column in str_df.columns: + if column in WIDE_COLUMNS: + table.add_column(str(column), max_width=32, overflow="fold") + else: + table.add_column(str(column)) + + for row in str_df.itertuples(index=False, name=None): + table.add_row(*row) + + print(f"\n[bold]----------{PLAIN_TABLE_HEADINGS[table_type]}----------[/bold]") + + +def print_artifact_table(artifacts: list[Artifact]) -> None: + """Create a pandas DataFrame containing artifact information for each artifact and call a print function.""" + import pandas as pd + + columns = [column.key for column in Artifact.__mapper__.columns if column.key != "s3_key"] + + df = pd.DataFrame( + [ + { + column: getattr( + artifact, + column, + ) + for column in columns + } + for artifact in artifacts + ] + ) + print_rich_table(df, table_type="artifact") + + +def print_scan_table(scans: list[Scan] | Scan) -> None: + """Create a pandas DataFrame containing scan information for each scan and call a print function.""" + import pandas as pd + + # exclude meta_file_s3_key from output + columns = [column.key for column in Scan.__mapper__.columns if column.key != "meta_file_s3_key"] + + if isinstance(scans, list): + df = pd.DataFrame([{column: getattr(scan, column) for column in columns} for scan in scans]) + else: + df = pd.DataFrame([{column: getattr(scans, column) for column in columns}]) + + print_rich_table(df, table_type="scan") + + if isinstance(scans, list): + for scan in scans: + print_artifact_table(scan.artifacts) + else: + print_artifact_table(scans.artifacts) + + +def find_values_in_file(file_content: bytes, search_key: str | tuple[str, str]) -> list[str] | str: + """Searches a JSON or SARIF byte object recursively for specific keys. + + Returns the values associated with the search key. + """ + data: JsonData = json.loads(file_content) + + found, values = _find_values(data, search_key) + + return values if found else "None" + + +def _find_values(data: JsonData, search_key: str | tuple[str, str]) -> tuple[bool, list[str]]: + """Recursively search JSON data for matching values. + + Args: + data: JSON data to search. + search_key: Either a single key or a ``(parent_key, key)`` pair. + + Returns: + A tuple containing: + - ``True`` if at least one matching value was found, otherwise ``False``. + - A list of all matching values as strings. + """ + results = ( + _find_values_by_tuple(data, search_key) + if isinstance(search_key, tuple) + else _find_values_by_key(data, search_key) + ) + + return bool(results), results + + +def _find_values_by_key(data: JsonData, search_key: str) -> list[str]: + """Recursively collect values matching a single key.""" + results: list[str] = [] + + def _search(node: JsonData) -> None: + if isinstance(node, dict): + value = node.get(search_key) + + if isinstance(value, (str, int, float)): + results.append(str(value)) + + for child in node.values(): + _search(child) + + elif isinstance(node, list): + for item in node: + _search(item) + + _search(data) + + return results + + +def _find_values_by_tuple(data: JsonData, search_key: tuple[str, str]) -> list[str]: + """Recursively collect values from nested parent/key pairs.""" + results: list[str] = [] + parent_key, key = search_key + + def _search(node: JsonData) -> None: + if isinstance(node, dict): + parent = node.get(parent_key) + + if parent is not None: + _append_tuple_matches(parent, key, results) + + for child in node.values(): + _search(child) + + return + + if isinstance(node, list): + for item in node: + _search(item) + + _search(data) + + return results + + +def _append_tuple_matches(parent: JsonData, key: str, results: list[str]) -> None: + """Append matching values from a parent object or list of objects.""" + if isinstance(parent, dict): + value = parent.get(key) + + if isinstance(value, (str, int, float)): + results.append(str(value)) + + return + + if isinstance(parent, list): + for item in parent: + if not isinstance(item, dict): + continue + + value = item.get(key) + + if isinstance(value, (str, int, float)): + results.append(str(value)) + + +def print_component_table(components: list[Component]) -> None: + """Create a pandas DataFrame of components and call a print function.""" + _print_plain_table(components, Component, "component", skipped=("id",)) + + +def print_dependency_finding_table(findings: list[DependencyFinding]) -> None: + """Create a pandas DataFrame of dependency findings and call a print function.""" + _print_plain_table(findings, DependencyFinding, "dependency_finding", skipped=("id",)) + + +def print_cvss_table(scorings: list[CvssMetric]) -> None: + """Create a pandas DataFrame of CVSS scorings and call a print function. + + The columns a scoring leaves empty are dropped rather than printed as a row of blanks: a v3 + vector has no ``authentication`` and a v2 one has no ``scope``, and showing every column of + every version at once makes the one in hand harder to read, not easier. + """ + columns = [column.key for column in inspect(CvssMetric).columns if column.key not in ("id", "cvss_id")] + columns = [column for column in columns if any(getattr(scoring, column) is not None for scoring in scorings)] + + _print_dataframe(scorings, columns, "cvss") + + +def _print_plain_table(rows: list[Any], model: type[Any], table_type: str, skipped: tuple[str, ...]) -> None: + """Render every column of a model except the ones a reader has no use for.""" + columns = [column.key for column in inspect(model).columns if column.key not in skipped] + + _print_dataframe(rows, columns, table_type) + + +def _print_dataframe(rows: list[Any], columns: list[str], table_type: str) -> None: + """Build the frame and hand it to the renderer.""" + import pandas as pd + + df = pd.DataFrame([{column: getattr(row, column) for column in columns} for row in rows]) + + print_rich_table(df, table_type=table_type) diff --git a/report_generator/vendor/persistent_storage/shared.py b/report_generator/vendor/persistent_storage/shared.py new file mode 100644 index 0000000..1435dd9 --- /dev/null +++ b/report_generator/vendor/persistent_storage/shared.py @@ -0,0 +1,101 @@ +"""Shared configuration Classes.""" + +from typing import TYPE_CHECKING + +from pydantic import BaseModel, Field + +from cc_common.api import PipelineError +from cc_common.config import apply_env_defaults, load_environment + +if TYPE_CHECKING: + from botocore.client import BaseClient + + +def __getattr__(name: str) -> object: + """Lazily resolve ``boto3`` as a module attribute (PEP 562). + + boto3 costs ~2.4s to import and is only needed once an S3 client is actually built + (:attr:`S3Configuration.s3_client`). Resolving it here, rather than importing it inline in the + property, keeps ``persistent_storage.shared.boto3`` patchable the same way it always was + (``monkeypatch.setattr("persistent_storage.shared.boto3.client", ...)``) while every other + consumer of this module avoids the cost. + """ + if name == "boto3": + import boto3 + + return boto3 + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +class DatabaseConfiguration(BaseModel): + """Configuration for the SQL database.""" + + host: str = Field(description="") + port: int = Field(description="") + user: str = Field(description="") + name: str = Field(description="") + password: str | None = Field( + default=None, + description="Database password.", + exclude=True, + ) + + @property + def db_url(self) -> str: + """Construct the database url that should be used to connect to the database.""" + if self.password is None: + raise PipelineError("Database password is not set.") + return f"postgresql+psycopg2://{self.user}:{self.password}@{self.host}:{self.port}/{self.name}" + + +class S3Configuration(BaseModel): + """Configuration for the S3 database.""" + + url: str = Field(description="") + bucket: str = Field(description="") + access_key_id: str = Field(description="") + region: str = Field(description="") + secret_access_key: str | None = Field( + default=None, + description="S3 secret access key.", + exclude=True, + ) + + @property + def s3_client(self) -> BaseClient: + """Construct a S3 client that should be used to connect to the database.""" + import boto3 + from botocore.config import Config + + if self.secret_access_key is None: + raise PipelineError("S3 secret access key is not set.") + return boto3.client( + "s3", + endpoint_url=self.url, + aws_access_key_id=self.access_key_id, + aws_secret_access_key=self.secret_access_key, + region_name=self.region, + # we need when_required, because garage does not support checksums for multipart files (see https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/787) + config=Config(request_checksum_calculation="when_required", response_checksum_validation="when_required"), + ) + + def __hash__(self) -> int: + """Basic hash implementation on self.""" + return hash(repr(self)) + + +class StorageConfiguration(BaseModel): + """Container for both database configurations.""" + + database: DatabaseConfiguration + s3: S3Configuration + + +def inject_secrets(config: StorageConfiguration) -> None: + """Fill in the database password and S3 secret key from their environment variables. + + Only fills in what is unset. For the standalone CLI; the orchestrator injects the same + overrides while assembling the pipeline config. + """ + load_environment() + apply_env_defaults(config, "persistent_storage.storage_config") diff --git a/report_generator/vendor/persistent_storage/utils/__init__.py b/report_generator/vendor/persistent_storage/utils/__init__.py new file mode 100644 index 0000000..59e17b2 --- /dev/null +++ b/report_generator/vendor/persistent_storage/utils/__init__.py @@ -0,0 +1 @@ +"""DB, S3 and Put utils for persistent storage.""" diff --git a/report_generator/vendor/persistent_storage/utils/cvss_utils.py b/report_generator/vendor/persistent_storage/utils/cvss_utils.py new file mode 100644 index 0000000..16d798d --- /dev/null +++ b/report_generator/vendor/persistent_storage/utils/cvss_utils.py @@ -0,0 +1,492 @@ +"""Reads a CVSS vector of any generation and says what it carries. + +Three generations are in circulation at once and a single advisory regularly states two of them, so +nothing here assumes a version -- the vector names its own, and everything after that follows from +the version it named. + +What changed between them, and why one parser cannot pretend they are the same: + +* **v2 (2007)** scores the whole system. It asks how often an attacker has to authenticate (``Au``) + and grades impact as none, partial or complete. +* **v3 (2015, 3.1 in 2019)** scores the vulnerable component instead. ``Au`` becomes ``PR``, which + asks what privileges are needed rather than how many logins; ``UI`` and ``S`` are new; impact + becomes none, low or high; ``AC`` loses its middle value. +* **v4 (2023)** splits impact in two -- the vulnerable system (``VC``/``VI``/``VA``) and whatever + sits behind it (``SC``/``SI``/``SA``) -- which makes ``S`` unnecessary. ``AT`` is new, ``UI`` gains + a passive value, and the temporal group becomes the threat group. + +This module holds no database and no framework, so that the same reading serves persistent_storage +today and archon when the tables move there. Everything that touches a session lives next to the +model instead. + + >>> reading = parse_vector("CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H") + >>> reading.version, reading.reachability, reading.remotely_reachable + ('3.1', 4, True) +""" + +from __future__ import annotations + +from typing import Any, NamedTuple + +# A vector that names no version is a v2 one: v2 is the only generation that did not prefix itself. +DEFAULT_VERSION = "2.0" + +# Which generation a version string belongs to, which is what decides how its metrics are read. +GENERATIONS = {"2.0": 2, "3.0": 3, "3.1": 3, "4.0": 4} + +# The metric that says where an attacker has to be, and the value that puts them anywhere. +FROM_ANYWHERE = "N" + +# What each generation calls "no obstacle at all" for the four metrics an attacker has to get past. +# Answering all four with these is an attack that needs nothing: no position, no effort, no account +# and nobody to fool. v2 has no user interaction metric, so it is scored over three. +# TODO das sollte ich noch refaktoren das ist unübersichtlich +# AV: Access Vector, AC: Access Complexity, Au: Authentication +EASIEST: dict[int, dict[str, str]] = { + 2: {"AV": "N", "AC": "L", "Au": "N"}, + 3: {"AV": "N", "AC": "L", "PR": "N", "UI": "N"}, + 4: {"AV": "N", "AC": "L", "PR": "N", "UI": "N"}, +} + +# What every abbreviation means, per generation, for anything that puts a reading before a person. +# Taken from the specifications; the weights behind them are the scoring formula's business and not +# this module's, which reads a vector rather than recomputing the score from it. +VOCABULARY: dict[int, dict[str, dict[str, str]]] = { + 2: { + "AV": {"L": "Local", "A": "Adjacent network", "N": "Network"}, + "AC": {"H": "High", "M": "Medium", "L": "Low"}, + "Au": {"M": "Multiple", "S": "Single", "N": "None"}, + "C": {"N": "None", "P": "Partial", "C": "Complete"}, + "I": {"N": "None", "P": "Partial", "C": "Complete"}, + "A": {"N": "None", "P": "Partial", "C": "Complete"}, + "E": {"U": "Unproven", "POC": "Proof of concept", "F": "Functional", "H": "High", "ND": "Not defined"}, + "RL": {"OF": "Official fix", "TF": "Temporary fix", "W": "Workaround", "U": "Unavailable", "ND": "Not defined"}, + "RC": {"UC": "Unconfirmed", "UR": "Uncorroborated", "C": "Confirmed", "ND": "Not defined"}, + }, + 3: { + "AV": {"N": "Network", "A": "Adjacent network", "L": "Local", "P": "Physical"}, + "AC": {"L": "Low", "H": "High"}, + "PR": {"N": "None", "L": "Low", "H": "High"}, + "UI": {"N": "None", "R": "Required"}, + "S": {"U": "Unchanged", "C": "Changed"}, + "C": {"N": "None", "L": "Low", "H": "High"}, + "I": {"N": "None", "L": "Low", "H": "High"}, + "A": {"N": "None", "L": "Low", "H": "High"}, + "E": {"X": "Not defined", "U": "Unproven", "P": "Proof of concept", "F": "Functional", "H": "High"}, + "RL": {"X": "Not defined", "O": "Official fix", "T": "Temporary fix", "W": "Workaround", "U": "Unavailable"}, + "RC": {"X": "Not defined", "U": "Unknown", "R": "Reasonable", "C": "Confirmed"}, + }, + 4: { + "AV": {"N": "Network", "A": "Adjacent network", "L": "Local", "P": "Physical"}, + "AC": {"L": "Low", "H": "High"}, + "AT": {"N": "None", "P": "Present"}, + "PR": {"N": "None", "L": "Low", "H": "High"}, + "UI": {"N": "None", "P": "Passive", "A": "Active"}, + "VC": {"H": "High", "L": "Low", "N": "None"}, + "VI": {"H": "High", "L": "Low", "N": "None"}, + "VA": {"H": "High", "L": "Low", "N": "None"}, + "SC": {"H": "High", "L": "Low", "N": "None"}, + "SI": {"H": "High", "L": "Low", "N": "None"}, + "SA": {"H": "High", "L": "Low", "N": "None"}, + "E": {"X": "Not defined", "A": "Attacked", "P": "Proof of concept", "U": "Unreported"}, + }, +} + +# Which key carries the impact on the vulnerable system, per generation. v4 moved it aside to make +# room for a second set covering whatever sits behind that system. +IMPACT_KEYS = {2: ("C", "I", "A"), 3: ("C", "I", "A"), 4: ("VC", "VI", "VA")} + +# The second set, which only v4 has. +SUBSEQUENT_KEYS = ("SC", "SI", "SA") + +# What the base score is called once it is a word. v2 stops at high; v3 added critical, and pushed +# the same vulnerability further up the scale than v2 would have put it. +SEVERITY_BANDS = { + 2: ((0.0, "None"), (4.0, "Low"), (7.0, "Medium"), (10.0, "High")), + 3: ((0.0, "None"), (4.0, "Low"), (7.0, "Medium"), (9.0, "High"), (10.0, "Critical")), + 4: ((0.0, "None"), (4.0, "Low"), (7.0, "Medium"), (9.0, "High"), (10.0, "Critical")), +} + +# What each value is worth when a set of findings has to be averaged into one number per axis. +# +# The base metrics carry the specification's own constants. They are not comparable between +# generations as written -- v2 runs its access vector to 1.0 where v3 stops at 0.85 -- so +# :func:`weight` divides by the largest value of that metric in that generation. What comes back is +# "how far towards the worst case this value sits", 0 to 1, which does compare. +# +# The threat metrics do not carry them. ``E``, ``RL`` and ``RC`` are multipliers in the +# specification, deliberately clustered between 0.91 and 1.0 so that they can only nudge a base +# score. Averaging those would produce a number that is always 0.95 and says nothing, so they are +# ranked by what they actually mean instead: an unproven exploit at 0, one under active attack at 1; +# an official fix at 0, nothing available at 1. Both directions agree with the base metrics -- +# larger is always worse. +WEIGHTS: dict[int, dict[str, dict[str, float]]] = { + 2: { + "AV": {"L": 0.395, "A": 0.646, "N": 1.0}, + "AC": {"H": 0.35, "M": 0.61, "L": 0.71}, + "Au": {"M": 0.45, "S": 0.56, "N": 0.704}, + "C": {"N": 0.0, "P": 0.275, "C": 0.660}, + "I": {"N": 0.0, "P": 0.275, "C": 0.660}, + "A": {"N": 0.0, "P": 0.275, "C": 0.660}, + "E": {"U": 0.0, "POC": 0.5, "F": 0.85, "H": 1.0}, + "RL": {"OF": 0.0, "TF": 0.33, "W": 0.67, "U": 1.0}, + "RC": {"UC": 1.0, "UR": 0.5, "C": 0.0}, + }, + 3: { + "AV": {"P": 0.2, "L": 0.55, "A": 0.62, "N": 0.85}, + "AC": {"H": 0.44, "L": 0.77}, + "PR": {"H": 0.27, "L": 0.62, "N": 0.85}, + "UI": {"R": 0.62, "N": 0.85}, + "C": {"N": 0.0, "L": 0.22, "H": 0.56}, + "I": {"N": 0.0, "L": 0.22, "H": 0.56}, + "A": {"N": 0.0, "L": 0.22, "H": 0.56}, + "E": {"U": 0.0, "P": 0.5, "F": 0.85, "H": 1.0}, + "RL": {"O": 0.0, "T": 0.33, "W": 0.67, "U": 1.0}, + "RC": {"U": 1.0, "R": 0.5, "C": 0.0}, + }, + # v4 scores by lookup table rather than per-metric constants, so there are none to quote. The + # metrics it shares with v3 keep v3's, which preserves their ordering and their spacing; the two + # values v4 added are placed by where the specification ranks them -- ``AT:P`` is an extra + # condition to meet, like a high complexity, and a passive interaction sits between none and an + # active one. + 4: { + "AV": {"P": 0.2, "L": 0.55, "A": 0.62, "N": 0.85}, + "AC": {"H": 0.44, "L": 0.77}, + "AT": {"P": 0.44, "N": 0.85}, + "PR": {"H": 0.27, "L": 0.62, "N": 0.85}, + "UI": {"A": 0.62, "P": 0.73, "N": 0.85}, + "VC": {"N": 0.0, "L": 0.22, "H": 0.56}, + "VI": {"N": 0.0, "L": 0.22, "H": 0.56}, + "VA": {"N": 0.0, "L": 0.22, "H": 0.56}, + "SC": {"N": 0.0, "L": 0.22, "H": 0.56}, + "SI": {"N": 0.0, "L": 0.22, "H": 0.56}, + "SA": {"N": 0.0, "L": 0.22, "H": 0.56}, + "E": {"U": 0.0, "P": 0.5, "A": 1.0}, + }, +} + +# The value every generation uses for "not stated", which is not a value and must not be averaged. +NOT_DEFINED = ("X", "ND") + +# Widths a reading is stored in, so a malformed vector cannot fail an insert. +MAX_VECTOR_LENGTH = 512 +MAX_METRIC_LENGTH = 16 +MAX_VERSION_LENGTH = 8 + + +class CvssReading(NamedTuple): + """What one CVSS vector states, and the two things that follow from it. + + Only the metrics a generation actually has are filled; the rest stay ``None``, because a metric + a version does not define is not the same as one it defines as absent. + + Attributes: + version: The version the vector named, e.g. ``"3.1"``. + generation: ``2``, ``3`` or ``4`` -- which set of metrics applies. + vector: The vector as the report wrote it. + metrics: Every metric of the vector by its abbreviation, including the + environmental and supplemental ones this class has no field for. + attack_vector: ``AV``, in every generation. + attack_complexity: ``AC``, in every generation. + attack_requirements: ``AT``, v4 only. + authentication: ``Au``, v2 only -- how often an attacker has to log in. + privileges_required: ``PR``, v3 and v4 -- what account they need instead. + user_interaction: ``UI``, v3 and v4. + scope: ``S``, v3 only; v4 replaced it with a second impact set. + confidentiality_impact: ``C`` in v2 and v3, ``VC`` in v4. + integrity_impact: ``I`` in v2 and v3, ``VI`` in v4. + availability_impact: ``A`` in v2 and v3, ``VA`` in v4. + subsequent_confidentiality_impact: ``SC``, v4 only. + subsequent_integrity_impact: ``SI``, v4 only. + subsequent_availability_impact: ``SA``, v4 only. + exploit_maturity: ``E`` -- temporal in v2 and v3, threat in v4. + remediation_level: ``RL``, v2 and v3. + report_confidence: ``RC``, v2 and v3. + reachability: How many of the metrics an attacker has to get past sit at + the value that costs them nothing, out of :func:`max_reachability`. + remotely_reachable: Over the network and without an account, which is the + pair that decides whether the internet is in a position to try. + """ + + version: str + generation: int + vector: str + metrics: dict[str, str] + attack_vector: str | None + attack_complexity: str | None + attack_requirements: str | None + authentication: str | None + privileges_required: str | None + user_interaction: str | None + scope: str | None + confidentiality_impact: str | None + integrity_impact: str | None + availability_impact: str | None + subsequent_confidentiality_impact: str | None + subsequent_integrity_impact: str | None + subsequent_availability_impact: str | None + exploit_maturity: str | None + remediation_level: str | None + report_confidence: str | None + reachability: int + remotely_reachable: bool + + +def parse_vector(vector: str | None) -> CvssReading | None: + """Read a CVSS vector of any generation. + + Args: + vector: A vector string, e.g. ``CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H`` + or a v2 one, which carries no prefix: ``AV:N/AC:L/Au:N/C:P/I:P/A:P``. + + Returns: + What the vector states, or ``None`` when there is no vector or it names + none of the metrics its generation defines. ``None`` means "not stated", + which is not the same as "poses no risk", so the two are kept apart. + """ + if not vector: + return None + + version = _version(vector) + generation = GENERATIONS.get(version, 3) + metrics = _metrics(vector, generation) + + if not metrics: + return None + + confidentiality, integrity, availability = IMPACT_KEYS[generation] + + return CvssReading( + version=version, + generation=generation, + vector=vector[:MAX_VECTOR_LENGTH], + metrics=metrics, + attack_vector=metrics.get("AV"), + attack_complexity=metrics.get("AC"), + attack_requirements=metrics.get("AT"), + authentication=metrics.get("Au"), + privileges_required=metrics.get("PR"), + user_interaction=metrics.get("UI"), + scope=metrics.get("S") if generation == 3 else None, + confidentiality_impact=metrics.get(confidentiality), + integrity_impact=metrics.get(integrity), + availability_impact=metrics.get(availability), + subsequent_confidentiality_impact=metrics.get("SC"), + subsequent_integrity_impact=metrics.get("SI"), + subsequent_availability_impact=metrics.get("SA"), + exploit_maturity=metrics.get("E"), + remediation_level=metrics.get("RL"), + report_confidence=metrics.get("RC"), + reachability=sum(metrics.get(key) == value for key, value in EASIEST[generation].items()), + # A metric the vector does not state is not counted as an obstacle either, so an absent one + # leaves the claim unmade rather than making it falsely. + remotely_reachable=_remotely_reachable(metrics, generation), + ) + + +def max_reachability(generation: int) -> int: + """How many obstacles a generation knows about. + + Args: + generation: ``2``, ``3`` or ``4``. + + Returns: + Three for v2, which has no user interaction metric, four otherwise. A + reading is only comparable with another of the same generation, which is + why this is asked for rather than assumed. + """ + return len(EASIEST.get(generation, EASIEST[3])) + + +def severity_band(score: float | None, generation: int) -> str | None: + """Return the word a base score falls under. + + Args: + score: The base score, if the report stated one. + generation: Which scale to read it against -- v2 stops at high, v3 and v4 + added critical, so the same number is not the same word. + + Returns: + The band, or ``None`` when there is no score. + """ + if score is None: + return None + + for limit, name in SEVERITY_BANDS.get(generation, SEVERITY_BANDS[3]): + if score <= limit: + return name + + return SEVERITY_BANDS[generation][-1][1] + + +def weight(metric: str, value: str | None, generation: int = 3) -> float | None: + """Return how far towards the worst case one metric value sits, 0 to 1. + + # TODO hier nochmal das Gewichten explizit prüfen. + This is what makes a set of findings averageable along one axis, and what makes a v2 reading + averageable together with a v4 one: each value is placed between the best and the worst its own + generation allows for that metric, so the result says "how far towards the worst case" rather + than quoting a constant that only means something inside its own formula. + + Stretched across the axis rather than divided by its top, because the specification's constants + are multipliers and do not start at zero: an attack that needs physical access is worth 0.2 of + the 0.85 a network one is worth, and reported as 0.24 it would say a quarter of the worst case + for what is in fact the best case the metric has. Zero here means "as hard as this metric can + make it", one means "as easy". + + Args: + metric: An abbreviation such as ``AV``, ``PR`` or ``VC``. + value: The value the vector carried, if any. + generation: Which generation's scale to read it against. + + Returns: + The position between best and worst case, or ``None`` when the vector + stated nothing, stated "not defined", or stated something the + specification does not list. All three mean the same thing here -- there + is nothing to average -- and are kept apart from a stated zero, which is + a measurement. + """ + if value is None or value in NOT_DEFINED: + return None + + scale = WEIGHTS.get(generation, {}).get(metric, {}) + stated = scale.get(value) + + if stated is None or not scale: + return None + + best, worst = min(scale.values()), max(scale.values()) + + # A metric with a single value says nothing about degree, so it reads as the worst case it has. + return (stated - best) / (worst - best) if worst > best else 1.0 + + +def describe(metric: str, value: str | None, generation: int = 3) -> str: + """Return the word behind one abbreviation. + + Args: + metric: An abbreviation such as ``AV``, ``PR`` or ``VC``. + value: The value the vector carried, if any. + generation: Which vocabulary to read it in. The same letter means + different things across generations -- ``C`` under v2 is "complete", + under v3 it is a metric name -- so this is not optional in practice. + + Returns: + The name the specification gives it, the raw value when the vocabulary + has no entry, or ``"Not stated"`` when the vector did not carry it. + """ + if value is None: + return "Not stated" + + return VOCABULARY.get(generation, {}).get(metric, {}).get(value, value) + + +def _version(vector: str) -> str: + """Return the version a vector names. + + Args: + vector: The vector string. v3 and v4 begin with ``CVSS:``; v2 + begins with its first metric and is recognised by that absence. + + Returns: + The version, defaulting to :data:`DEFAULT_VERSION` for an unprefixed + vector and to the string as written for one this module does not know -- + a version nobody has implemented yet is still worth recording. + """ + head = vector.split("/", 1)[0] + + if not head.upper().startswith("CVSS:"): + return DEFAULT_VERSION + + return head.split(":", 1)[1].strip()[:MAX_VERSION_LENGTH] or DEFAULT_VERSION + + +def _metrics(vector: str, generation: int) -> dict[str, str]: + """Split a vector into its metrics. + + Args: + vector: The vector string. + generation: Which generation it belongs to, which decides whether the + first segment is a version prefix to skip. + + Returns: + The metrics by their abbreviation, spelled as the specification spells + them -- ``Au`` keeps its lower-case second letter, which is the one + abbreviation that is not upper case. A segment that is not a + ``key:value`` pair is ignored rather than raising: a report this store + cannot fully read is still a report worth keeping. + """ + segments = vector.split("/") + metrics: dict[str, str] = {} + + for segment in segments if generation == 2 else segments[1:]: + key, separator, value = segment.partition(":") + + if not separator or not key or not value: + continue + + name = key.strip()[:MAX_METRIC_LENGTH] + metrics["Au" if name.upper() == "AU" and generation == 2 else name.upper()] = value.strip().upper()[ + :MAX_METRIC_LENGTH + ] + + return metrics + + +def _remotely_reachable(metrics: dict[str, str], generation: int) -> bool: + """Whether the vector says an attacker needs neither a position nor an account. + + Args: + metrics: The metrics of the vector. + generation: Which generation states the account requirement, since v2 + asks how often an attacker logs in and v3 onwards what they log in + as. + + Returns: + ``True`` only when the vector states both and both are open. + """ + account = "Au" if generation == 2 else "PR" + + return metrics.get("AV") == FROM_ANYWHERE and metrics.get(account) == "N" + + +def as_row(reading: CvssReading, base_score: float | None = None) -> dict[str, Any]: + """Return a reading shaped like the columns it is stored in. + + Kept here rather than next to the model so that the move to archon is a change of table and not + a change of mapping: whatever holds these columns can be filled from this. + + Args: + reading: What a vector stated. + base_score: The score the report stated alongside the vector, if any. + Not recomputed from the vector -- the report's own number is the one + that has to come back out of the store. + + Returns: + The reading by column name, ready to be handed to a model. + """ + return { + "version": reading.version, + "vector": reading.vector, + "base_score": base_score, + "base_severity": severity_band(base_score, reading.generation), + "attack_vector": reading.attack_vector, + "attack_complexity": reading.attack_complexity, + "attack_requirements": reading.attack_requirements, + "authentication": reading.authentication, + "privileges_required": reading.privileges_required, + "user_interaction": reading.user_interaction, + "scope": reading.scope, + "confidentiality_impact": reading.confidentiality_impact, + "integrity_impact": reading.integrity_impact, + "availability_impact": reading.availability_impact, + "subsequent_confidentiality_impact": reading.subsequent_confidentiality_impact, + "subsequent_integrity_impact": reading.subsequent_integrity_impact, + "subsequent_availability_impact": reading.subsequent_availability_impact, + "exploit_maturity": reading.exploit_maturity, + "remediation_level": reading.remediation_level, + "report_confidence": reading.report_confidence, + "metrics": reading.metrics, + "reachability": reading.reachability, + "remotely_reachable": reading.remotely_reachable, + } diff --git a/report_generator/vendor/persistent_storage/utils/db_utils.py b/report_generator/vendor/persistent_storage/utils/db_utils.py new file mode 100644 index 0000000..1693e5b --- /dev/null +++ b/report_generator/vendor/persistent_storage/utils/db_utils.py @@ -0,0 +1,324 @@ +"""Utilities for configuring and interacting with the artifact metadata database. + +This module defines the database schema for artifact records, initializes the +database connection and tables, and provides functionality for storing and +retrieving artifact metadata. +""" + +from uuid import UUID + +import sqlalchemy +from sqlalchemy import create_engine, delete, select +from sqlalchemy.orm import Session, selectinload, sessionmaker + +from cc_common.api import context +from persistent_storage.helper import print_scan_table +from persistent_storage.shared import DatabaseConfiguration +from persistent_storage.wrapper import Artifact, Base, Scan + + +def get_session(configuration: DatabaseConfiguration) -> sqlalchemy.orm.Session: + """Create and return a new SQLAlchemy session bound to the application's engine. + + The session is configured using the engine returned by + `create_engine()` and can be used to interact with the database. + + Returns: + sqlalchemy.orm.Session: A new SQLAlchemy session instance. + """ + engine = create_engine(configuration.db_url) + session = sessionmaker(bind=engine, expire_on_commit=False) + + return session() + + +def init_db(configuration: DatabaseConfiguration) -> None: + """Initialize the application's database schema. + + Creates all tables defined in the SQLAlchemy metadata using the + engine returned by `create_engine()`. Existing tables are + left unchanged. + + This function should typically be called during application startup + or initial setup. + """ + engine = create_engine(configuration.db_url) + Base.metadata.create_all(engine) + + +def save_object_to_db(instance: Base, configuration: DatabaseConfiguration) -> None: + """Persist artifact metadata to the database. + + Creates a new Artifact record from validated metadata, + stores it in the database, and commits the transaction. + + Args: + instance (Base): + Generic Base Object for Scan and Artifact Records. + + configuration: + The configuration parameters of the db. + + Side Effects: + - Inserts a new artifact entry into the database. + - Commits transaction. + """ + session: Session = get_session(configuration) + + try: + session.add(instance) + session.commit() + + except Exception as e: + session.rollback() + context.logger.error(f"Failed to save artifact: {e}") + raise + + finally: + session.close() + + +def get_scan(configuration: DatabaseConfiguration, scan_id: UUID) -> Scan: + """Retrieve a scan and its associated artifacts from the database. + + Args: + configuration: Database configuration containing the connection URL. + scan_id: UUID of the scan to retrieve. + + Returns: + The matching ``Scan`` instance with its artifacts eagerly loaded. + + Raises: + sqlalchemy.exc.NoResultFound: If no scan with the given UUID exists. + sqlalchemy.exc.MultipleResultsFound: If multiple scans with the given + UUID are found. + """ + engine = create_engine(configuration.db_url) + + try: + with Session(engine) as session: + stmt = select(Scan).where(Scan.scan_id == scan_id).options(selectinload(Scan.artifacts)) + return session.scalars(stmt).one() + finally: + engine.dispose() + + +class ScanList(list[Scan]): + """A list of ``Scan`` objects with convenience methods for presentation.""" + + def print_as_cc_common_result(self) -> None: + """Print the scans as a rich table.""" + print_scan_table(self) + + +def get_all_scans(configuration: DatabaseConfiguration) -> list[Scan]: + """Retrieve all scans and their associated artifacts from the database. + + Args: + configuration: Database configuration containing the connection URL. + + Returns: + A ``ScanList`` containing all scans with their artifacts eagerly loaded. + """ + engine = create_engine(configuration.db_url) + + try: + with Session(engine) as session: + stmt = select(Scan).options(selectinload(Scan.artifacts)) + scans = session.scalars(stmt).all() + return ScanList(scans) + finally: + engine.dispose() + + +# DROP DB ENTRY +def drop_database_entry(configuration: DatabaseConfiguration, database_entry_uuid: UUID) -> bool: + """Delete a specific database entry by UUID. + + Removes only the matching database entry from the artifact table. + + Args: + database_entry_uuid: + UUID of the database entry to delete. + + configuration: + The configuration parameters of the db. + + Returns: + bool: + The value shows if the deletion was successful. + + Side Effects: + - Permanently deletes a single database row + - Prints deletion progress information to stdout + """ + engine = create_engine(configuration.db_url) + success = False + + try: + with engine.begin() as conn: + stmt = delete(Scan).where(Scan.scan_id == database_entry_uuid) + + result = conn.execute(stmt) + + if result.rowcount == 0: + context.logger.warning( + f"No entry found for UUID {database_entry_uuid}", + ) + + else: + context.logger.warning( + f"Scan Entry {database_entry_uuid} deleted ⚠", + ) + + success = True + + finally: + engine.dispose() + + return success + + +# WAIT FOR POSTGRES +def wait_for_postgres(configuration: DatabaseConfiguration, max_retries: int = 10, delay: int = 1) -> None: + """Wait until the PostgreSQL server becomes available. + + Repeatedly attempts to establish a connection to the PostgreSQL + instance using the admin credentials. If the connection succeeds, + the function returns immediately. Otherwise, retries are performed + until the maximum retry limit is reached. + + Args: + max_retries (int): + Maximum number of connection attempts before failing. + + delay (int): + Delay in seconds between retry attempts. + + configuration: + The configuration parameters of the db. + + Raises: + RuntimeError: + Raised if PostgreSQL does not become available within the + configured retry limit. + """ + import time + + from sqlalchemy import create_engine + from sqlalchemy.exc import OperationalError + + context.logger.info("Waiting for PostgreSQL to become available...") + + engine = create_engine( + configuration.db_url, + isolation_level="AUTOCOMMIT", + ) + + try: + for attempt in range(max_retries): + try: + with engine.connect(): + context.logger.info("PostgreSQL is ready ✔") + + return + + except OperationalError: + context.logger.warning( + f"PostgreSQL not ready (attempt {attempt + 1}/{max_retries})", + ) + + time.sleep(delay) + + finally: + engine.dispose() + + context.logger.error( + "PostgreSQL did not become ready in time", + ) + + raise RuntimeError( + "PostgreSQL did not become ready in time", + ) + + +def get_s3_key(configuration: DatabaseConfiguration, target_artifact_id: UUID) -> str: + """Retrieves the S3 object key for a specific artifact. + + Queries the database for the artifact identified by the given UUID and + returns the corresponding S3 key used to locate the artifact file in + the configured S3 bucket. + + Args: + target_artifact_id: UUID of the artifact whose S3 key should be + retrieved. + + configuration: The configuration parameters of the db. + + Returns: + The S3 object key associated with the artifact. + + Raises: + ValueError: If no artifact with the specified UUID exists or no + valid S3 key is found. + """ + from sqlalchemy import create_engine, select + from sqlalchemy.orm import Session + + engine = create_engine(configuration.db_url) + + try: + with Session(engine) as session: + s3_key = session.scalar(select(Artifact.s3_key).where(Artifact.artifact_id == target_artifact_id)) + + if s3_key is None or not isinstance(s3_key, str): + context.logger.error(f"Artifact {target_artifact_id} not found ❌") + raise ValueError(f"Artifact {target_artifact_id} not found ❌") + else: + return str(s3_key) + finally: + engine.dispose() + + +def get_s3_keys(configuration: DatabaseConfiguration, target_scan_id: UUID) -> list[str]: + """Retrieves the S3 object keys for all artifacts belonging to a scan. + + Queries the database for all artifacts associated with the specified + scan UUID and returns their corresponding S3 object keys. + + Args: + target_scan_id: UUID of the scan whose artifact S3 keys should be + retrieved. + + configuration: The configuration parameters of the db. + + Returns: + A list containing the S3 object keys of all artifacts associated + with the scan. + + Raises: + ValueError: If no artifact files are associated with the specified + scan UUID or if an invalid S3 key is encountered. + """ + from sqlalchemy import create_engine, select + from sqlalchemy.orm import Session + + engine = create_engine(configuration.db_url) + + try: + with Session(engine) as session: + s3_keys: list[str] = list( + session.scalars(select(Artifact.s3_key).where(Artifact.scan_id == target_scan_id)) + ) + if len(s3_keys) == 0: + context.logger.error(f"No artifact files for scan UUID '{target_scan_id}' found ❌") + raise ValueError(f"No artifact files for scan UUID '{target_scan_id}' found ❌") + + for s3_key in s3_keys: + if not isinstance(s3_key, str): + context.logger.error(f"Invalid S3 key for scan UUID '{target_scan_id}' ❌") + raise ValueError(f"Invalid S3 key for scan UUID '{target_scan_id}' ❌") + + return s3_keys + finally: + engine.dispose() diff --git a/report_generator/vendor/persistent_storage/utils/s3_utils.py b/report_generator/vendor/persistent_storage/utils/s3_utils.py new file mode 100644 index 0000000..93866d3 --- /dev/null +++ b/report_generator/vendor/persistent_storage/utils/s3_utils.py @@ -0,0 +1,170 @@ +"""Utilities for managing artifact storage in an S3-compatible object store. + +This module provides helper functions for: + +- Ensuring the configured S3 bucket exists +- Uploading artifacts to object storage +""" + +from typing import TYPE_CHECKING, cast +from uuid import UUID + +from botocore.exceptions import ClientError + +from cc_common.api import context +from persistent_storage.shared import S3Configuration, StorageConfiguration +from persistent_storage.utils.db_utils import get_s3_key + +if TYPE_CHECKING: + # mypy_boto3_s3 drags in boto3 (~1.2s) purely for these type stubs; the runtime object it + # names is never used (typing.cast() is a no-op that just returns its second argument), so + # the callers below reference it as a string forward-ref instead. + from mypy_boto3_s3 import S3Client + from mypy_boto3_s3.type_defs import ObjectIdentifierTypeDef + + +def ensure_bucket_exists(configuration: S3Configuration) -> None: + """Ensure that the configured S3 bucket exists. + + Checks whether the configured bucket is available in the connected + S3-compatible storage service. If the bucket does not exist, it is + created automatically. + + Side Effects: + - May create a new S3 bucket + - Prints bucket status information to stdout + """ + try: + s3_client = cast("S3Client", configuration.s3_client) + s3_client.create_bucket(Bucket=configuration.bucket) + + context.logger.warning(f"Bucket '{configuration.bucket}' created ✔") + + except ClientError as e: + code = e.response["Error"].get("Code") + + if code in ("BucketAlreadyOwnedByYou", "BucketAlreadyExists"): + context.logger.warning(f"Bucket '{configuration.bucket}' already exists") + return + + raise + + +def upload_file(configuration: S3Configuration, path: str, s3_key: str) -> None: + """Upload a local artifact file to the configured S3 bucket. + + Args: + path (str): + Local filesystem path to the artifact or meta file. + + s3_key (str): + Object key under which the artifact or meta file will be stored + in the S3 bucket. + + configuration (StorageConfiguration): + The configuration parameters for the S3 storage. + + Side Effects: + - Uploads a file to S3-compatible object storage + - Prints upload status information to Logger + """ + s3_client = cast("S3Client", configuration.s3_client) + s3_client.upload_file(str(path), configuration.bucket, s3_key) + + context.logger.info(f"Uploaded {path} -> s3://{configuration.bucket}/{s3_key}") + + +def open_rustfs_bucket(configuration: S3Configuration) -> None: + """Open the RustFS bucket in the RustFS web console.""" + import webbrowser + + webbrowser.open(configuration.url[:-1] + "1") # Port 9001 + + +def fetch_artifact_file(configuration: StorageConfiguration, artifact_id: UUID) -> bytes: + """Retrieves one artifact files from the configured S3 bucket. + + The corresponding artifact, associated with artifact_id is fetched and + returned as a single bytes object. + + Args: + artifact_id: Unique identifier of the artifact to retrieve. + configuration: The configuration parameters for the S3 storage. + + Returns: + The artifact as a bytes-like object. + """ + s3_client = cast("S3Client", configuration.s3.s3_client) + + response = s3_client.get_object( + Bucket=configuration.s3.bucket, + Key=get_s3_key(configuration=configuration.database, target_artifact_id=artifact_id), + ) + return response["Body"].read() + + +def fetch_artifact_files( + configuration: StorageConfiguration, artifact_ids: list[UUID] +) -> tuple[list[bytes], list[str]]: # noqa: E501 + """Retrieves multiple artifact files from the configured S3 bucket. + + All files associated with the scan are fetched and returned as a list of byte + objects and a list of file names. + + Args: + artifact_ids: All unique UUIDs of the scan to retrieve. + configuration: The configuration parameters for the S3 storage. + + Returns: + A list[bytes] with all artifacts of a scan as bytes-like objects. + A list[str] with all artifact file names. + """ + s3_client = cast("S3Client", configuration.s3.s3_client) + files = [] + file_names = [] + + for artifact_id in artifact_ids: + s3_key = get_s3_key(configuration=configuration.database, target_artifact_id=artifact_id) + response = s3_client.get_object( + Bucket=configuration.s3.bucket, + Key=s3_key, + ) + file = response["Body"].read() + files.append(file) + file_name = s3_key.split("/")[-1] + file_names.append(file_name) + + return files, file_names + + +def drop_storage_files(configuration: S3Configuration, target_scan_id: UUID) -> bool: + """Delete all S3 objects associated with a scan. + + Deletes every object in the configured S3 bucket whose key has the prefix + ``/``. + + Args: + configuration: S3 configuration containing the bucket name and client. + target_scan_id: Identifier of the scan whose stored files should be removed. + + Returns: + ``True`` if all matching objects were deleted successfully, otherwise + ``False`` if an AWS ``ClientError`` was raised. + """ + s3_client = cast("S3Client", configuration.s3_client) + prefix = f"{target_scan_id}/" + paginator = s3_client.get_paginator("list_objects_v2") + + try: + for page in paginator.paginate(Bucket=configuration.bucket, Prefix=prefix): + if "Contents" not in page: + continue + + objects: list[ObjectIdentifierTypeDef] = [{"Key": obj["Key"]} for obj in page["Contents"]] + + s3_client.delete_objects(Bucket=configuration.bucket, Delete={"Objects": objects}) + return True + + except ClientError as e: + context.logger.error(e) + return False diff --git a/report_generator/vendor/persistent_storage/wrapper.py b/report_generator/vendor/persistent_storage/wrapper.py new file mode 100644 index 0000000..a8726d1 --- /dev/null +++ b/report_generator/vendor/persistent_storage/wrapper.py @@ -0,0 +1,744 @@ +"""Wrapper for database storage.""" + +from datetime import UTC, datetime +from uuid import UUID + +from sqlalchemy import ( + JSON, + TIMESTAMP, + Boolean, + Float, + ForeignKey, + Index, + Integer, + String, + UniqueConstraint, + text, +) +from sqlalchemy.dialects.postgresql import UUID as PG_UUID +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship + + +class Base(DeclarativeBase): + """Base model for SQLAlchemy models.""" + + pass + + +class Artifact(Base): + """SQLAlchemy model representing a stored artifact. + + This table stores metadata about files that are generated or uploaded + as part of a project. Each artifact is linked to a scan and stored + in S3-compatible object storage. + + Attributes: + id (int): + Primary key of the artifact record. + + artifact_id (UUID): + Unique identifier of the artifact. + + artifact_type (str): + Type/category of the artifact (e.g. SARIF, SBOM). + + s3_key (str): + Object storage key used to locate the file in S3-compatible storage. + + file_name (str | None): + Original filename of the uploaded artifact, if available. + + created_at (datetime): + Timestamp when the artifact record was created (UTC). + + artifact_metadata (dict | None): + Optional JSON metadata associated with the artifact, + such as file size or additional computed attributes. + + scan_id (UUID): + Foreign key linked to the unique identifier of the scan record. + """ + + __tablename__ = "artifacts" + + id: Mapped[int] = mapped_column( + Integer, + primary_key=True, + ) + + artifact_id: Mapped[UUID] = mapped_column( + PG_UUID(as_uuid=True), + unique=True, + nullable=False, + ) + + artifact_type: Mapped[str] = mapped_column( + String, + nullable=False, + ) + + s3_key: Mapped[str] = mapped_column( + String, + nullable=False, + ) + + file_name: Mapped[str | None] = mapped_column( + String, + nullable=True, + ) + + created_at: Mapped[datetime] = mapped_column( + TIMESTAMP, + nullable=False, + ) + + artifact_metadata: Mapped[str | None] = mapped_column( + JSON, + nullable=True, + ) + + # On delete cascade + scan_id: Mapped[UUID] = mapped_column( + PG_UUID(as_uuid=True), + ForeignKey("scans.scan_id", ondelete="CASCADE"), + nullable=False, + ) + + # ORM relationship to scans + scan: Mapped[Scan] = relationship( + "Scan", + back_populates="artifacts", + ) + + def print_as_cc_common_result(self) -> None: + """Prints the artifact objects content.""" + from .helper import print_artifact_table + + print_artifact_table([self]) + + +class Scan(Base): + """SQLAlchemy model representing a scan execution. + + This table stores metadata about a single scan run, including + information about the tool that produced the scan, execution time, + and the associated meta file stored in S3-compatible + object storage. + + A scan can be associated with one or more artifacts, which reference + the scan through its unique scan identifier. + + Attributes: + id (int): + Internal database primary key. + + scan_id (UUID): + Unique identifier of the scan execution. Used as a stable + reference for related artifacts. + + run_at (datetime): + Timestamp when the scan was executed (UTC). + + tool_name (str): + Name of the tool that generated the scan. + + tool_version (int | None): + Version of the tool that generated the scan, if available. + + meta_file_s3_key (str): + Object storage key of the scan meta file stored in + S3-compatible storage. + + meta_file_content (dict[str, Any]): + Parsed content of the scan meta file stored as JSON. + """ + + __tablename__ = "scans" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + + scan_id: Mapped[UUID] = mapped_column( + PG_UUID(as_uuid=True), + unique=True, + nullable=False, + ) + + run_at: Mapped[datetime] = mapped_column(TIMESTAMP, default=lambda: datetime.now(UTC), nullable=False) + + tool_name: Mapped[str] = mapped_column(String, nullable=False) + + tool_version: Mapped[str] = mapped_column(String, nullable=True) + + meta_file_s3_key: Mapped[str] = mapped_column(String, unique=True, nullable=False) + + meta_file_content: Mapped[str] = mapped_column(String, nullable=False) + + # ORM relationship to artifacts + artifacts: Mapped[list[Artifact]] = relationship( + "Artifact", + back_populates="scan", + ) + + # ORM relationships to what this scan found + component_scans: Mapped[list[ComponentScan]] = relationship( + "ComponentScan", + back_populates="scan", + cascade="all, delete-orphan", + ) + + dep_finding_scans: Mapped[list[DepFindingScan]] = relationship( + "DepFindingScan", + back_populates="scan", + cascade="all, delete-orphan", + ) + + def print_as_cc_common_result(self) -> None: + """Prints the scan objects content.""" + from .helper import print_scan_table + + print_scan_table([self]) + + +class Component(Base): + """A third-party package, identified by what it is rather than by who found it. + + ``mypy 1.0`` is the same package in every scan that depends on it, so it is one row. Which scans + depend on it is a different fact, and it gets its own table -- archon calls that one + ``component_scans`` and this package will call it the same, so that the two line up when these + tables move. Until it exists, a component says what it is and nothing about where it came from. + + Mirrors archon's ``components`` table without the three columns that belong to a document rather + than to the package: ``scope`` and ``bom_ref`` are what one SBOM said about it in one place, and + the timestamp is when the row was written. + + Attributes: + id (int): + Internal database primary key, as everything in this package has. + + component_id (UUID): + Stable handle of the component. A UUID so that archon, whose tables + are keyed by one, can adopt it as the primary key unchanged. + + component_name (str): + Name of the package. + + version (str): + Version of the package as the document stated it, empty when it + stated none -- which CycloneDX allows for a range nobody pinned. + + latest_version (str | None): + The newest version the reporting tool knew of, refreshed by every + export that states one. What makes a finding answerable: without it a + report says what is wrong and never what to do about it. A newer + version existing is not proof that it fixes this advisory, so + anything read off it is "there is somewhere to upgrade to", never + "this is fixed". + + purl (str | None): + Package URL, which already encodes ecosystem, name and version + (``pkg:pypi/requests@2.31.0``) and is therefore the identity where it + exists. NULL for something no package manager knows -- an installed + application rather than a library -- which name and version identify + instead. + """ + + __tablename__ = "components" + __table_args__ = ( + # The package URL is the identity where there is one: it carries the ecosystem, which is + # what keeps pkg:npm/requests apart from pkg:pypi/requests. Both are called "requests" at + # the same version and are unrelated packages, so name and version alone cannot be unique. + UniqueConstraint("purl", name="uq_components_purl"), + # Where there is no package URL there is no ecosystem either, so name and version are all + # there is to go on. Partial, so it constrains only those rows and leaves the others to the + # constraint above. + Index( + "uq_components_unregistered", + "component_name", + "version", + unique=True, + postgresql_where=text("purl IS NULL"), + sqlite_where=text("purl IS NULL"), + ), + Index("ix_components_name_version", "component_name", "version"), + ) + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + + component_id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), unique=True, nullable=False) + + component_name: Mapped[str] = mapped_column(String, nullable=False) + + version: Mapped[str] = mapped_column(String, nullable=False) + + latest_version: Mapped[str | None] = mapped_column(String, nullable=True) + + purl: Mapped[str | None] = mapped_column(String, nullable=True) + + # ORM relationship to the advisories recorded against this component + findings: Mapped[list[DependencyFinding]] = relationship( + "DependencyFinding", + back_populates="component", + cascade="all, delete-orphan", + ) + + # ORM relationship to the scans that found it + scans: Mapped[list[ComponentScan]] = relationship( + "ComponentScan", + back_populates="component", + cascade="all, delete-orphan", + ) + + def print_as_cc_common_result(self) -> None: + """Prints the component objects content.""" + from .helper import print_component_table + + print_component_table([self]) + + +class DependencyFinding(Base): + """A known vulnerability affecting one component. + + Attributes: + id (int): + Internal database primary key. + + finding_dep_id (UUID): + Stable handle of the finding, and archon's primary key once moved. + + component_id (UUID): + The component the advisory affects. + + cve (str | None): + The identifier the advisory carries, e.g. ``CVE-2024-35195``. Not + every advisory has a CVE -- GitHub and OSV assign their own, and + those are stored here too. + + cwes (list[str] | None): + The weakness classes the advisory names, as ``CWE-79`` strings. + + description (str): + What the advisory says about it. + + default_level (str): + The severity the report headlined: ``critical``, ``high``, + ``medium``, ``low``, ``info`` or ``unassigned``. Kept as the report + wrote it rather than derived from a score, because the report is what + is being stored. + + cwe_name (str | None): + What the weakness class is called, e.g. ``Protection Mechanism + Failure``. The identifier alone means nothing to a reader. + + source (str | None): + Who published the advisory: ``NVD``, ``GITHUB``, ``OSV``. + + published (datetime | None): + When the advisory was made public. The **age** of a finding is read + off this rather than stored: an age is a number that is wrong again + tomorrow, a date is not. + + references (str | None): + Links to the advisory and its fix, as the export wrote them -- one + markdown list rather than a structured field, which is what + Dependency-Track hands over. + """ + + __tablename__ = "dependency_findings" + __table_args__ = ( + # One advisory per component. A second report naming it again finds this row rather than + # writing another one. + UniqueConstraint("component_id", "cve", name="uq_dependency_findings_advisory"), + ) + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + + finding_dep_id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), unique=True, nullable=False) + + component_id: Mapped[UUID] = mapped_column( + PG_UUID(as_uuid=True), + ForeignKey("components.component_id", ondelete="CASCADE"), + nullable=False, + ) + + cve: Mapped[str | None] = mapped_column(String, nullable=True) + + cwes: Mapped[list[str] | None] = mapped_column(JSON, nullable=True) + + description: Mapped[str] = mapped_column(String, nullable=False) + + default_level: Mapped[str] = mapped_column(String, nullable=False) + + cwe_name: Mapped[str | None] = mapped_column(String, nullable=True) + + source: Mapped[str | None] = mapped_column(String, nullable=True) + + published: Mapped[datetime | None] = mapped_column(TIMESTAMP, nullable=True) + + references: Mapped[str | None] = mapped_column(String, nullable=True) + + # ORM relationship to components + component: Mapped[Component] = relationship("Component", back_populates="findings") + + # ORM relationship to the CVSS scorings of this advisory, one per version of the standard + cvss: Mapped[list[CvssMetric]] = relationship( + "CvssMetric", + back_populates="finding", + cascade="all, delete-orphan", + ) + + # ORM relationship to the scans that reported it + scans: Mapped[list[DepFindingScan]] = relationship( + "DepFindingScan", + back_populates="finding", + cascade="all, delete-orphan", + ) + + def print_as_cc_common_result(self) -> None: + """Prints the dependency finding objects content.""" + from .helper import print_dependency_finding_table + + print_dependency_finding_table([self]) + + +class CvssMetric(Base): + """One CVSS scoring of one advisory, in one version of the standard. + + An advisory is regularly scored more than once -- a Dependency-Track export routinely carries a + v3 vector and a v4 one for the same CVE, and occasionally a v2 one as well -- and the versions + do not agree with each other. So this is one row per version rather than one per advisory, and + nothing is folded together: the columns hold what the vector stated, ``metrics`` holds all of it + verbatim, and the score is the report's own rather than one recomputed here. + + Not every column applies to every version, which is why nearly all of them are nullable: + + * ``authentication`` is v2 only -- how often an attacker has to log in. + * ``privileges_required`` and ``user_interaction`` are v3 and v4, which ask what account is + needed rather than how many logins. + * ``scope`` is v3 only; v4 replaced it with the ``subsequent_*`` impact columns. + * ``attack_requirements`` and the ``subsequent_*`` columns are v4 only. + + A NULL therefore means one of two things -- the version has no such metric, or the report did + not state it -- and ``version`` is what tells the two apart. + + **Built to be moved.** The row points at ``dependency_findings`` and at nothing else of this + package, and that table is archon's under the same name with the same handle. The reading comes + from :mod:`persistent_storage.utils.cvss_utils`, which imports nothing from here. + + Attributes: + id (int): + Internal database primary key. + + cvss_id (UUID): + Stable handle of the scoring. + + finding_dep_id (UUID): + The advisory this scores. + + version (str): + The version of the standard: ``2.0``, ``3.0``, ``3.1`` or ``4.0``. + + vector (str): + The vector as the report wrote it, kept verbatim so the columns + beside it can be checked or re-derived without the report at hand. + + base_score (float | None): + The base score the report stated, 0 to 10. + + base_severity (str | None): + The word that score falls under, read against the scale of its own + version: v2 stops at high, v3 and v4 added critical, so the same + number is not the same word. + + attack_vector (str | None): + ``AV`` -- where an attacker has to be. + + attack_complexity (str | None): + ``AC`` -- how much has to fall into place. + + attack_requirements (str | None): + ``AT`` -- what the target environment has to look like. v4 only. + + authentication (str | None): + ``Au`` -- how often an attacker has to authenticate. v2 only. + + privileges_required (str | None): + ``PR`` -- what account is needed. v3 and v4. + + user_interaction (str | None): + ``UI`` -- whether somebody has to be fooled into helping. v3 and v4. + + scope (str | None): + ``S`` -- whether the blast reaches past the component. v3 only. + + confidentiality_impact (str | None): + What an attacker gets to read. ``C`` in v2 and v3, ``VC`` in v4. + + integrity_impact (str | None): + What an attacker gets to change. ``I`` in v2 and v3, ``VI`` in v4. + + availability_impact (str | None): + How far the service can be taken down. ``A`` in v2 and v3, ``VA`` in + v4. + + subsequent_confidentiality_impact (str | None): + ``SC`` -- the same, for whatever sits behind the vulnerable system. + v4 only, and the reason v4 needs no scope metric. + + subsequent_integrity_impact (str | None): + ``SI``. v4 only. + + subsequent_availability_impact (str | None): + ``SA``. v4 only. + + exploit_maturity (str | None): + ``E`` -- how far along a working exploit is. Temporal in v2 and v3, + threat in v4, and spelled differently in each. + + remediation_level (str | None): + ``RL`` -- whether there is a fix. v2 and v3. + + report_confidence (str | None): + ``RC`` -- how sure anyone is that this is real. v2 and v3. + + metrics (dict[str, str] | None): + Every metric of the vector by its abbreviation, environmental and + supplemental ones included. Those are filled in by whoever operates + the system rather than by a scanner, so they get no columns of their + own -- but a report that does state them does not lose them here. + + reachability (int): + How many of the metrics an attacker has to get past sit at the value + that costs them nothing. Out of three under v2, which has no user + interaction metric, and four otherwise -- so it compares only within + one version. + + remotely_reachable (bool): + Over the network and without an account. The pair that decides + whether the internet is in a position to try at all. + """ + + __tablename__ = "cvss_metrics" + __table_args__ = ( + # One scoring per advisory per version of the standard. + UniqueConstraint("finding_dep_id", "version", name="uq_cvss_metrics_version"), + # What the portfolio question runs on: how much of this can be reached from outside. + Index("ix_cvss_metrics_reach", "remotely_reachable", "reachability"), + ) + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + + cvss_id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), unique=True, nullable=False) + + finding_dep_id: Mapped[UUID] = mapped_column( + PG_UUID(as_uuid=True), + ForeignKey("dependency_findings.finding_dep_id", ondelete="CASCADE"), + nullable=False, + ) + + version: Mapped[str] = mapped_column(String, nullable=False) + + vector: Mapped[str] = mapped_column(String, nullable=False) + + base_score: Mapped[float | None] = mapped_column(Float, nullable=True) + + base_severity: Mapped[str | None] = mapped_column(String, nullable=True) + + attack_vector: Mapped[str | None] = mapped_column(String, nullable=True) + + attack_complexity: Mapped[str | None] = mapped_column(String, nullable=True) + + attack_requirements: Mapped[str | None] = mapped_column(String, nullable=True) + + authentication: Mapped[str | None] = mapped_column(String, nullable=True) + + privileges_required: Mapped[str | None] = mapped_column(String, nullable=True) + + user_interaction: Mapped[str | None] = mapped_column(String, nullable=True) + + scope: Mapped[str | None] = mapped_column(String, nullable=True) + + confidentiality_impact: Mapped[str | None] = mapped_column(String, nullable=True) + + integrity_impact: Mapped[str | None] = mapped_column(String, nullable=True) + + availability_impact: Mapped[str | None] = mapped_column(String, nullable=True) + + subsequent_confidentiality_impact: Mapped[str | None] = mapped_column(String, nullable=True) + + subsequent_integrity_impact: Mapped[str | None] = mapped_column(String, nullable=True) + + subsequent_availability_impact: Mapped[str | None] = mapped_column(String, nullable=True) + + exploit_maturity: Mapped[str | None] = mapped_column(String, nullable=True) + + remediation_level: Mapped[str | None] = mapped_column(String, nullable=True) + + report_confidence: Mapped[str | None] = mapped_column(String, nullable=True) + + metrics: Mapped[dict[str, str] | None] = mapped_column(JSON, nullable=True) + + reachability: Mapped[int] = mapped_column(Integer, nullable=False) + + remotely_reachable: Mapped[bool] = mapped_column(Boolean, nullable=False) + + # ORM relationship to dependency findings + finding: Mapped[DependencyFinding] = relationship("DependencyFinding", back_populates="cvss") + + def print_as_cc_common_result(self) -> None: + """Prints the CVSS scoring objects content.""" + from .helper import print_cvss_table + + print_cvss_table([self]) + + +class ComponentScan(Base): + """Records that one scan found the project depending on one package. + + The table the component table stopped carrying when a package became a package rather than a + per-scan row. A component says what it is; this says who ran into it and when. + + Attributes: + id (int): + Internal database primary key. + + component_scan_id (UUID): + Stable handle of the record. + + scan_id (UUID): + The scan that found the dependency. + + component_id (UUID): + The package it found. + + artifact_id (UUID | None): + The document it read it out of, when there was one. + """ + + __tablename__ = "component_scans" + __table_args__ = ( + # A scan depends on a package once, however many documents of that scan mention it. + UniqueConstraint("scan_id", "component_id", name="uq_component_scans_dependency"), + ) + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + + component_scan_id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), unique=True, nullable=False) + + scan_id: Mapped[UUID] = mapped_column( + PG_UUID(as_uuid=True), + ForeignKey("scans.scan_id", ondelete="CASCADE"), + nullable=False, + ) + + component_id: Mapped[UUID] = mapped_column( + PG_UUID(as_uuid=True), + ForeignKey("components.component_id", ondelete="CASCADE"), + nullable=False, + ) + + artifact_id: Mapped[UUID | None] = mapped_column( + PG_UUID(as_uuid=True), + ForeignKey("artifacts.artifact_id", ondelete="SET NULL"), + nullable=True, + ) + + # ORM relationships + scan: Mapped[Scan] = relationship("Scan", back_populates="component_scans") + component: Mapped[Component] = relationship("Component", back_populates="scans") + + +class DepFindingScan(Base): + """Records that one scan reported one advisory. + + Separate from :class:`ComponentScan` because the two say different things, and a later scan can + say one without the other: it still depends on the package, and the advisory against it has + since been withdrawn or fixed. Depending on something is not the same as being told about it. + + Attributes: + id (int): + Internal database primary key. + + dep_finding_scan_id (UUID): + Stable handle of the record. + + scan_id (UUID): + The scan that reported the advisory. + + finding_dep_id (UUID): + The advisory it reported. + + artifact_id (UUID | None): + The document it read it out of, when there was one. + + dependency_track_id (UUID | None): + The Dependency-Track project the export was taken of. A notification + arriving weeks later names that project and nothing else, so this is + what matches it back to a scan. + + epss_score (float | None): + The probability that this advisory is exploited somewhere in the next + thirty days, 0 to 1, **as it stood when this scan ran**. EPSS is + recomputed daily, so it belongs to the occurrence and not to the + advisory -- and keeping it here means a series of scans carries the + history of that probability for free. + + epss_percentile (float | None): + Where that probability ranked among all scored vulnerabilities at the + same moment. Uniform by construction, which is what makes it the + comparable half of the pair. + + analyzer_identity (str | None): + Which Dependency-Track analyzer reported it, e.g. + ``TRIVY_ANALYZER``. Two analyzers disagree about the same project, so + this belongs to the run rather than to the advisory. + + is_suppressed (bool | None): + Whether Dependency-Track was told to suppress it at the time of the + export. Somebody's judgement, recorded as what it is: a statement of + the exporting system, not of this store. + """ + + __tablename__ = "dep_finding_scans" + __table_args__ = ( + # A scan reports an advisory once, however many documents of that scan carry it. + UniqueConstraint("scan_id", "finding_dep_id", name="uq_dep_finding_scans_occurrence"), + Index("ix_dep_finding_scans_project", "dependency_track_id"), + # What "show me what is actually likely to be exploited" runs on. + Index("ix_dep_finding_scans_epss", "epss_score"), + ) + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + + dep_finding_scan_id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), unique=True, nullable=False) + + scan_id: Mapped[UUID] = mapped_column( + PG_UUID(as_uuid=True), + ForeignKey("scans.scan_id", ondelete="CASCADE"), + nullable=False, + ) + + finding_dep_id: Mapped[UUID] = mapped_column( + PG_UUID(as_uuid=True), + ForeignKey("dependency_findings.finding_dep_id", ondelete="CASCADE"), + nullable=False, + ) + + artifact_id: Mapped[UUID | None] = mapped_column( + PG_UUID(as_uuid=True), + ForeignKey("artifacts.artifact_id", ondelete="SET NULL"), + nullable=True, + ) + + dependency_track_id: Mapped[UUID | None] = mapped_column(PG_UUID(as_uuid=True), nullable=True) + + epss_score: Mapped[float | None] = mapped_column(Float, nullable=True) + + epss_percentile: Mapped[float | None] = mapped_column(Float, nullable=True) + + analyzer_identity: Mapped[str | None] = mapped_column(String, nullable=True) + + is_suppressed: Mapped[bool | None] = mapped_column(Boolean, nullable=True) + + # ORM relationships + scan: Mapped[Scan] = relationship("Scan", back_populates="dep_finding_scans") + finding: Mapped[DependencyFinding] = relationship("DependencyFinding", back_populates="scans") diff --git a/report_generator/vendor/sync.py b/report_generator/vendor/sync.py new file mode 100644 index 0000000..b24b78f --- /dev/null +++ b/report_generator/vendor/sync.py @@ -0,0 +1,103 @@ +"""Copies everything this tool borrows from the monorepo into ``vendor/``. + +The package is meant to be liftable into a repository of its own, where none of ``cc_common`` or +``persistent_storage`` exists. The copies here are what it would need to take along. + +They are produced by a script rather than by hand for one reason: a copy made once goes stale +without saying so. Re-run this after touching anything it lists and the diff shows what drifted. + + python -m persistent_storage.report_generator.vendor.sync + +Copied under their own import paths, so ``vendor/`` on ``sys.path`` makes the package import +unchanged in the new repository. +""" + +from __future__ import annotations + +import shutil +import sys +from pathlib import Path + +# Where the monorepo's sources sit, relative to this file. +SOURCE_ROOT = Path(__file__).resolve().parents[4] + +VENDOR_ROOT = Path(__file__).resolve().parent + +# What the tool reaches for, and nothing beyond it. A whole package is copied where the tool uses +# enough of it that picking modules would only mean chasing imports on the next change. +COPIED = ( + # The CLI framework, the logging context and the error type every command raises. + "cc_common", + # The scan and artifact rows, the storage configuration, and the readers over them. + "persistent_storage/__init__.py", + "persistent_storage/shared.py", + "persistent_storage/wrapper.py", + "persistent_storage/helper.py", + "persistent_storage/utils/__init__.py", + "persistent_storage/utils/db_utils.py", + "persistent_storage/utils/s3_utils.py", + # The CVSS reader the vulnerability summariser works through. + "persistent_storage/utils/cvss_utils.py", + # The two summarisers the report is built from. They live in the web application today and are + # written to be free of it -- no Django, no store -- which is what makes copying them honest + # rather than a workaround. Wherever this package ends up, they belong beside it. + "cipherverifier/__init__.py", + "cipherverifier/core/__init__.py", + "cipherverifier/core/services/__init__.py", + "cipherverifier/core/services/sarif.py", + "cipherverifier/core/services/vdr.py", +) + +# Never copied: caches and the compiled leftovers of whoever ran a tool over the source tree last. +# They live inside the packages being copied, so leaving any of them out puts megabytes of someone +# else's machine into the vendor tree. +SKIPPED = shutil.ignore_patterns( + "__pycache__", "*.pyc", ".DS_Store", ".mypy_cache", ".ruff_cache", ".pytest_cache", "*.egg-info" +) + + +def sync() -> list[Path]: + """Refresh every copy, replacing what is there. + + The copied packages are removed whole before anything is written, so that dropping an entry + from :data:`COPIED` actually drops it from the tree. Replacing only what is listed leaves what + used to be listed behind, and the copy then carries files nothing refers to any more -- which is + the same staleness this script exists to prevent, one level up. + + Returns: + What was written, relative to ``vendor/``. + """ + for root in {entry.split("/")[0] for entry in COPIED}: + stale = VENDOR_ROOT / root + + if stale.is_dir(): + shutil.rmtree(stale) + elif stale.exists(): + stale.unlink() + + written = [] + + for entry in COPIED: + source = SOURCE_ROOT / "src" / entry + target = VENDOR_ROOT / entry + + if not source.exists(): + message = f"{source} does not exist -- the monorepo moved it, and this list has not followed." + + raise FileNotFoundError(message) + + target.parent.mkdir(parents=True, exist_ok=True) + + if source.is_dir(): + shutil.copytree(source, target, ignore=SKIPPED) + else: + shutil.copy2(source, target) + + written.append(Path(entry)) + + return written + + +if __name__ == "__main__": + for path in sync(): + sys.stdout.write(f"{path}\n")