Initial commit. monorepo deps are copied in a dirty way to the vendor folder

report_generator/sources.py

@@ -0,0 +1,137 @@
+"""Collects everything a scan's report is built from.
+
+The one place that knows where the bytes come from. Swapping the store -- persistent_storage today,
+archon tomorrow -- is a change to :func:`collect` and to nothing else: what it returns is files by
+name, and a renderer cannot tell which store handed them over.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, cast
+from uuid import UUID
+
+from cc_common.api import PipelineError, context
+from persistent_storage.shared import StorageConfiguration
+from persistent_storage.utils.db_utils import get_scan
+from persistent_storage.utils.s3_utils import fetch_artifact_files
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+    from datetime import datetime
+
+    from mypy_boto3_s3 import S3Client
+
+    from persistent_storage.wrapper import Artifact
+
+
+@dataclass(frozen=True)
+class ReportSources:
+    """One scan and every stored file belonging to it.
+
+    Attributes:
+        scan_id: The scan the report is about.
+        run_at: When it ran, which is what the report is dated by.
+        artifacts: The artifact rows, for the inventory the report lists.
+        files: The stored bytes by file name. Held in memory because a report is
+            built once and every section wants a different one of them; the
+            largest scan in the sample set comes to a few megabytes.
+    """
+
+    scan_id: UUID
+    run_at: datetime
+    artifacts: Sequence[Artifact]
+    files: dict[str, bytes]
+
+    def first(self, *names: str) -> bytes | None:
+        """Return the first of ``names`` the scan stored, or nothing.
+
+        Matched by suffix rather than equality: a stored name carries whatever the producing stage
+        called the file, and the same document travels under `meta.json` and `metadata.json`
+        depending on which stage wrote it.
+        """
+        for name in names:
+            for stored, content in self.files.items():
+                if stored.endswith(name):
+                    return content
+
+        return None
+
+
+def _why(configuration: StorageConfiguration, artifacts: Sequence[Artifact], error: Exception) -> str:
+    """Say which artifacts the store has a row for but no object.
+
+    The bucket reports a missing key without naming it, which on a scan of six artifacts leaves
+    whoever ran this with nothing to act on. Rows outliving their objects is the ordinary way this
+    fails -- a bucket restored from nothing, a lifecycle rule, a bucket that was never persistent --
+    so the answer worth giving is which files are gone, not that one is.
+
+    Args:
+        configuration: The store being read.
+        artifacts: The artifact rows the scan claims.
+        error: What the fetch raised.
+
+    Returns:
+        The original message, followed by the missing file names when they can be
+        established. Diagnosing must never raise on top of the failure it is
+        explaining, so anything going wrong here falls back to the message alone.
+    """
+    try:
+        client = cast("S3Client", configuration.s3.s3_client)
+        missing = []
+
+        for artifact in artifacts:
+            try:
+                client.head_object(Bucket=configuration.s3.bucket, Key=artifact.s3_key)
+            except Exception:  # noqa: BLE001, PERF203
+                missing.append(artifact.file_name or artifact.s3_key)
+
+        if not missing:
+            return str(error)
+
+        return (
+            f"the bucket holds no object for {len(missing)} of {len(artifacts)} artifact(s) -- "
+            f"{', '.join(missing)}. The rows outlived the files; re-upload the scan with "
+            f"'persistent_storage put'."
+        )
+    except Exception:  # noqa: BLE001
+        return str(error)
+
+
+def collect(configuration: StorageConfiguration, scan_id: UUID) -> ReportSources:
+    """Read one scan and its files out of the store.
+
+    Args:
+        configuration: The store to read from.
+        scan_id: The scan to collect.
+
+    Returns:
+        The scan's rows and files.
+
+    Raises:
+        PipelineError: The scan is unknown, or its files cannot be fetched.
+    """
+    context.logger.info(f"Collecting the artifacts of scan {scan_id}...")
+
+    try:
+        scan = get_scan(configuration.database, scan_id)
+    except Exception as error:
+        message = f"Cannot read scan {scan_id}: {error}"
+
+        raise PipelineError(message) from error
+
+    try:
+        contents, names = fetch_artifact_files(configuration, [one.artifact_id for one in scan.artifacts])
+    except Exception as error:
+        message = f"Cannot collect the artifacts of scan {scan_id}: {_why(configuration, scan.artifacts, error)}"
+
+        raise PipelineError(message) from error
+
+    context.logger.info(f"Collected {len(names)} file(s) ✔")
+
+    return ReportSources(
+        scan_id=scan_id,
+        run_at=scan.run_at,
+        artifacts=scan.artifacts,
+        files=dict(zip(names, contents, strict=True)),
+    )