"""Collects everything a scan's report is built from. The one place that knows where the bytes come from. Swapping the store -- persistent_storage today, archon tomorrow -- is a change to :func:`collect` and to nothing else: what it returns is files by name, and a renderer cannot tell which store handed them over. """ from __future__ import annotations from dataclasses import dataclass from typing import TYPE_CHECKING, cast from uuid import UUID from cc_common.api import PipelineError, context from persistent_storage.shared import StorageConfiguration from persistent_storage.utils.db_utils import get_scan from persistent_storage.utils.s3_utils import fetch_artifact_files if TYPE_CHECKING: from collections.abc import Sequence from datetime import datetime from mypy_boto3_s3 import S3Client from persistent_storage.wrapper import Artifact @dataclass(frozen=True) class ReportSources: """One scan and every stored file belonging to it. Attributes: scan_id: The scan the report is about. run_at: When it ran, which is what the report is dated by. artifacts: The artifact rows, for the inventory the report lists. files: The stored bytes by file name. Held in memory because a report is built once and every section wants a different one of them; the largest scan in the sample set comes to a few megabytes. """ scan_id: UUID run_at: datetime artifacts: Sequence[Artifact] files: dict[str, bytes] def first(self, *names: str) -> bytes | None: """Return the first of ``names`` the scan stored, or nothing. Matched by suffix rather than equality: a stored name carries whatever the producing stage called the file, and the same document travels under `meta.json` and `metadata.json` depending on which stage wrote it. """ for name in names: for stored, content in self.files.items(): if stored.endswith(name): return content return None def _why(configuration: StorageConfiguration, artifacts: Sequence[Artifact], error: Exception) -> str: """Say which artifacts the store has a row for but no object. The bucket reports a missing key without naming it, which on a scan of six artifacts leaves whoever ran this with nothing to act on. Rows outliving their objects is the ordinary way this fails -- a bucket restored from nothing, a lifecycle rule, a bucket that was never persistent -- so the answer worth giving is which files are gone, not that one is. Args: configuration: The store being read. artifacts: The artifact rows the scan claims. error: What the fetch raised. Returns: The original message, followed by the missing file names when they can be established. Diagnosing must never raise on top of the failure it is explaining, so anything going wrong here falls back to the message alone. """ try: client = cast("S3Client", configuration.s3.s3_client) missing = [] for artifact in artifacts: try: client.head_object(Bucket=configuration.s3.bucket, Key=artifact.s3_key) except Exception: # noqa: BLE001, PERF203 missing.append(artifact.file_name or artifact.s3_key) if not missing: return str(error) return ( f"the bucket holds no object for {len(missing)} of {len(artifacts)} artifact(s) -- " f"{', '.join(missing)}. The rows outlived the files; re-upload the scan with " f"'persistent_storage put'." ) except Exception: # noqa: BLE001 return str(error) def collect(configuration: StorageConfiguration, scan_id: UUID) -> ReportSources: """Read one scan and its files out of the store. Args: configuration: The store to read from. scan_id: The scan to collect. Returns: The scan's rows and files. Raises: PipelineError: The scan is unknown, or its files cannot be fetched. """ context.logger.info(f"Collecting the artifacts of scan {scan_id}...") try: scan = get_scan(configuration.database, scan_id) except Exception as error: message = f"Cannot read scan {scan_id}: {error}" raise PipelineError(message) from error try: contents, names = fetch_artifact_files(configuration, [one.artifact_id for one in scan.artifacts]) except Exception as error: message = f"Cannot collect the artifacts of scan {scan_id}: {_why(configuration, scan.artifacts, error)}" raise PipelineError(message) from error context.logger.info(f"Collected {len(names)} file(s) ✔") return ReportSources( scan_id=scan_id, run_at=scan.run_at, artifacts=scan.artifacts, files=dict(zip(names, contents, strict=True)), )