"""Collects everything a scan's report is built from. The one place that knows where the bytes come from. Swapping the store -- persistent_storage today, archon tomorrow -- is a change to :func:`collect` and to nothing else: what it returns is files by name, and a renderer cannot tell which store handed them over. """ from __future__ import annotations from dataclasses import dataclass from typing import TYPE_CHECKING, cast from uuid import UUID from sqlalchemy import create_engine, text from cc_common.api import PipelineError, context from persistent_storage.shared import StorageConfiguration from persistent_storage.utils.db_utils import get_scan from persistent_storage.utils.s3_utils import fetch_artifact_files if TYPE_CHECKING: from collections.abc import Sequence from datetime import datetime from mypy_boto3_s3 import S3Client from sqlalchemy.sql.elements import TextClause from persistent_storage.wrapper import Artifact @dataclass(frozen=True) class ReportSources: """One scan and every stored file belonging to it. Attributes: scan_id: The scan the report is about, and is named by -- the pipeline id whenever one is recorded, because that is the id the GUI shows and so the only one a reader of the report can match against anything. Falls back to ``stored_scan_id`` when the run has no pipeline row. stored_scan_id: The scan row the files were actually read from. Kept apart from :attr:`scan_id` because the store is keyed by this one and the report is identified by the other; collapsing them makes one of the two wrong. run_at: When it ran, which is what the report is dated by. artifacts: The artifact rows, for the inventory the report lists. files: The stored bytes by file name. Held in memory because a report is built once and every section wants a different one of them; the largest scan in the sample set comes to a few megabytes. """ scan_id: UUID stored_scan_id: UUID run_at: datetime artifacts: Sequence[Artifact] files: dict[str, bytes] def first(self, *names: str) -> bytes | None: """Return the first of ``names`` the scan stored, or nothing. Matched by suffix rather than equality: a stored name carries whatever the producing stage called the file, and the same document travels under `meta.json` and `metadata.json` depending on which stage wrote it. """ for name in names: for stored, content in self.files.items(): if stored.endswith(name): return content return None def _why(configuration: StorageConfiguration, artifacts: Sequence[Artifact], error: Exception) -> str: """Say which artifacts the store has a row for but no object. The bucket reports a missing key without naming it, which on a scan of six artifacts leaves whoever ran this with nothing to act on. Rows outliving their objects is the ordinary way this fails -- a bucket restored from nothing, a lifecycle rule, a bucket that was never persistent -- so the answer worth giving is which files are gone, not that one is. Args: configuration: The store being read. artifacts: The artifact rows the scan claims. error: What the fetch raised. Returns: The original message, followed by the missing file names when they can be established. Diagnosing must never raise on top of the failure it is explaining, so anything going wrong here falls back to the message alone. """ try: client = cast("S3Client", configuration.s3.s3_client) missing = [] for artifact in artifacts: try: client.head_object(Bucket=configuration.s3.bucket, Key=artifact.s3_key) except Exception: # noqa: BLE001, PERF203 missing.append(artifact.file_name or artifact.s3_key) if not missing: return str(error) return ( f"the bucket holds no object for {len(missing)} of {len(artifacts)} artifact(s) -- " f"{', '.join(missing)}. The rows outlived the files; re-upload the scan with " f"'persistent_storage put'." ) except Exception: # noqa: BLE001 return str(error) _STORAGE_SCAN_OF_PIPELINE = text( """ SELECT result.id FROM core_scanresult AS result JOIN scans AS stored ON stored.scan_id = result.id WHERE result.scan_id = :pipeline ORDER BY stored.run_at DESC LIMIT 1 """ ) _PIPELINE_OF_STORAGE_SCAN = text("SELECT result.scan_id FROM core_scanresult AS result WHERE result.id = :stored") def _lookup(configuration: StorageConfiguration, statement: TextClause, parameter: dict[str, str]) -> UUID | None: """Run one id translation against ``core_scanresult``, or give up. Every reason it can fail means the same thing to the caller -- no translation is available -- and none of them is worth failing a report over: this deployment may have no archon tables at all, and the ids still have to be reported under whichever name is known. """ engine = create_engine(configuration.database.db_url) try: with engine.connect() as connection: found = connection.execute(statement, parameter).scalar_one_or_none() except Exception: # noqa: BLE001 return None finally: engine.dispose() return UUID(str(found)) if found is not None else None def _storage_scan_id(configuration: StorageConfiguration, pipeline_id: UUID) -> UUID | None: """Return the stored scan holding ``pipeline_id``'s artifacts, if that is what was passed. A run is recorded under two different ids. The pipeline mints one -- what cipherverifier names its working directories after, what archon's ``core_scan`` row is keyed by, and so the one a human has to hand; uploading the artifacts afterwards mints a second, which is what ``scans`` and ``artifacts`` are keyed by and the only one :func:`get_scan` can look up. ``core_scanresult`` is where the two meet: its own id is the stored scan, its ``scan_id`` the pipeline that produced it. Joining ``scans`` is not decoration -- it keeps a result row whose upload never landed from being offered as an answer, and picks the newest upload when a pipeline was stored more than once. Args: configuration: The store to read from. pipeline_id: The id that was passed and that ``scans`` has no row for. Returns: The stored scan id, or ``None`` -- the id is not a pipeline's either, or this deployment runs without archon's tables, which is not an error: it just means there is nothing to translate and the original lookup failure stands. """ return _lookup(configuration, _STORAGE_SCAN_OF_PIPELINE, {"pipeline": str(pipeline_id)}) def _pipeline_scan_id(configuration: StorageConfiguration, stored_id: UUID) -> UUID | None: """Return the pipeline that produced ``stored_id``, which is the id the report is named by. The translation :func:`_storage_scan_id` performs, read the other way. Needed even though the store was reached without it: a report identifies itself by the id the GUI shows, so passing the stored id has to produce the same report as passing the pipeline id, not one filed under a number nobody can look up. Args: configuration: The store to read from. stored_id: The scan the files were read from. Returns: The pipeline id, or ``None`` when the run has no pipeline row and the stored id is therefore the only name it has. """ return _lookup(configuration, _PIPELINE_OF_STORAGE_SCAN, {"stored": str(stored_id)}) def collect(configuration: StorageConfiguration, scan_id: UUID) -> ReportSources: """Read one scan and its files out of the store. Accepts either id a run is known by -- the pipeline id the GUI shows, or the stored scan id -- and reports under the pipeline id whichever was given, so that the same run yields the same report either way. See :func:`_storage_scan_id` and :func:`_pipeline_scan_id`. Args: configuration: The store to read from. scan_id: The scan to collect, by either of its ids. Returns: The scan's rows and files, identified by the pipeline id where there is one. Raises: PipelineError: The scan is unknown, or its files cannot be fetched. """ context.logger.info(f"Collecting the artifacts of scan {scan_id}...") reported_id = scan_id try: scan = get_scan(configuration.database, scan_id) except Exception as error: stored_id = _storage_scan_id(configuration, scan_id) if stored_id is None: message = ( f"Cannot read scan {scan_id}: {error}. No stored scan has that id, and no pipeline " f"either -- 'persistent_storage list' shows the scans that can be reported on." ) raise PipelineError(message) from error # A pipeline id was passed, so it is already the one to report under; only the row to read # has to be looked up. context.logger.info(f"{scan_id} is the pipeline; its artifacts are stored under scan {stored_id}") scan_id = stored_id try: scan = get_scan(configuration.database, scan_id) except Exception as retried: message = f"Cannot read scan {scan_id}: {retried}" raise PipelineError(message) from retried try: contents, names = fetch_artifact_files(configuration, [one.artifact_id for one in scan.artifacts]) except Exception as error: message = f"Cannot collect the artifacts of scan {scan_id}: {_why(configuration, scan.artifacts, error)}" raise PipelineError(message) from error context.logger.info(f"Collected {len(names)} file(s) ✔") if reported_id == scan_id: # The stored id was passed. The report still has to name the run the way the GUI does, so # the pipeline is looked up even though nothing needed it to reach the files. reported_id = _pipeline_scan_id(configuration, scan_id) or scan_id return ReportSources( scan_id=reported_id, stored_scan_id=scan_id, run_at=scan.run_at, artifacts=scan.artifacts, files=dict(zip(names, contents, strict=True)), )