138 lines
4.8 KiB
Python
138 lines
4.8 KiB
Python
"""Collects everything a scan's report is built from.
|
|
|
|
The one place that knows where the bytes come from. Swapping the store -- persistent_storage today,
|
|
archon tomorrow -- is a change to :func:`collect` and to nothing else: what it returns is files by
|
|
name, and a renderer cannot tell which store handed them over.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
from typing import TYPE_CHECKING, cast
|
|
from uuid import UUID
|
|
|
|
from cc_common.api import PipelineError, context
|
|
from persistent_storage.shared import StorageConfiguration
|
|
from persistent_storage.utils.db_utils import get_scan
|
|
from persistent_storage.utils.s3_utils import fetch_artifact_files
|
|
|
|
if TYPE_CHECKING:
|
|
from collections.abc import Sequence
|
|
from datetime import datetime
|
|
|
|
from mypy_boto3_s3 import S3Client
|
|
|
|
from persistent_storage.wrapper import Artifact
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ReportSources:
|
|
"""One scan and every stored file belonging to it.
|
|
|
|
Attributes:
|
|
scan_id: The scan the report is about.
|
|
run_at: When it ran, which is what the report is dated by.
|
|
artifacts: The artifact rows, for the inventory the report lists.
|
|
files: The stored bytes by file name. Held in memory because a report is
|
|
built once and every section wants a different one of them; the
|
|
largest scan in the sample set comes to a few megabytes.
|
|
"""
|
|
|
|
scan_id: UUID
|
|
run_at: datetime
|
|
artifacts: Sequence[Artifact]
|
|
files: dict[str, bytes]
|
|
|
|
def first(self, *names: str) -> bytes | None:
|
|
"""Return the first of ``names`` the scan stored, or nothing.
|
|
|
|
Matched by suffix rather than equality: a stored name carries whatever the producing stage
|
|
called the file, and the same document travels under `meta.json` and `metadata.json`
|
|
depending on which stage wrote it.
|
|
"""
|
|
for name in names:
|
|
for stored, content in self.files.items():
|
|
if stored.endswith(name):
|
|
return content
|
|
|
|
return None
|
|
|
|
|
|
def _why(configuration: StorageConfiguration, artifacts: Sequence[Artifact], error: Exception) -> str:
|
|
"""Say which artifacts the store has a row for but no object.
|
|
|
|
The bucket reports a missing key without naming it, which on a scan of six artifacts leaves
|
|
whoever ran this with nothing to act on. Rows outliving their objects is the ordinary way this
|
|
fails -- a bucket restored from nothing, a lifecycle rule, a bucket that was never persistent --
|
|
so the answer worth giving is which files are gone, not that one is.
|
|
|
|
Args:
|
|
configuration: The store being read.
|
|
artifacts: The artifact rows the scan claims.
|
|
error: What the fetch raised.
|
|
|
|
Returns:
|
|
The original message, followed by the missing file names when they can be
|
|
established. Diagnosing must never raise on top of the failure it is
|
|
explaining, so anything going wrong here falls back to the message alone.
|
|
"""
|
|
try:
|
|
client = cast("S3Client", configuration.s3.s3_client)
|
|
missing = []
|
|
|
|
for artifact in artifacts:
|
|
try:
|
|
client.head_object(Bucket=configuration.s3.bucket, Key=artifact.s3_key)
|
|
except Exception: # noqa: BLE001, PERF203
|
|
missing.append(artifact.file_name or artifact.s3_key)
|
|
|
|
if not missing:
|
|
return str(error)
|
|
|
|
return (
|
|
f"the bucket holds no object for {len(missing)} of {len(artifacts)} artifact(s) -- "
|
|
f"{', '.join(missing)}. The rows outlived the files; re-upload the scan with "
|
|
f"'persistent_storage put'."
|
|
)
|
|
except Exception: # noqa: BLE001
|
|
return str(error)
|
|
|
|
|
|
def collect(configuration: StorageConfiguration, scan_id: UUID) -> ReportSources:
|
|
"""Read one scan and its files out of the store.
|
|
|
|
Args:
|
|
configuration: The store to read from.
|
|
scan_id: The scan to collect.
|
|
|
|
Returns:
|
|
The scan's rows and files.
|
|
|
|
Raises:
|
|
PipelineError: The scan is unknown, or its files cannot be fetched.
|
|
"""
|
|
context.logger.info(f"Collecting the artifacts of scan {scan_id}...")
|
|
|
|
try:
|
|
scan = get_scan(configuration.database, scan_id)
|
|
except Exception as error:
|
|
message = f"Cannot read scan {scan_id}: {error}"
|
|
|
|
raise PipelineError(message) from error
|
|
|
|
try:
|
|
contents, names = fetch_artifact_files(configuration, [one.artifact_id for one in scan.artifacts])
|
|
except Exception as error:
|
|
message = f"Cannot collect the artifacts of scan {scan_id}: {_why(configuration, scan.artifacts, error)}"
|
|
|
|
raise PipelineError(message) from error
|
|
|
|
context.logger.info(f"Collected {len(names)} file(s) ✔")
|
|
|
|
return ReportSources(
|
|
scan_id=scan_id,
|
|
run_at=scan.run_at,
|
|
artifacts=scan.artifacts,
|
|
files=dict(zip(names, contents, strict=True)),
|
|
)
|