394 lines
15 KiB
Python
394 lines
15 KiB
Python
"""Collects everything a scan's report is built from.
|
|
|
|
The one place that knows where the bytes come from. Two ways in:
|
|
|
|
* :func:`collect` reads a scan out of the store -- Postgres for the rows, the bucket for the files.
|
|
* :func:`collect_from_directory` reads a folder of exported artifacts, and needs neither.
|
|
|
|
Both return the same thing: files by name. Nothing downstream can tell which of the two handed them
|
|
over, which is what makes the second one worth having rather than a second pipeline -- a report of
|
|
an exported run is the same report, built by the same code.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
from dataclasses import dataclass
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from typing import TYPE_CHECKING, Any, cast
|
|
from uuid import UUID
|
|
|
|
from sqlalchemy import create_engine, text
|
|
|
|
from cc_common.api import PipelineError, context
|
|
from persistent_storage.shared import StorageConfiguration
|
|
from persistent_storage.utils.db_utils import get_scan
|
|
from persistent_storage.utils.s3_utils import fetch_artifact_files
|
|
|
|
if TYPE_CHECKING:
|
|
from collections.abc import Sequence
|
|
|
|
from mypy_boto3_s3 import S3Client
|
|
from sqlalchemy.sql.elements import TextClause
|
|
|
|
from persistent_storage.wrapper import Artifact
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ReportSources:
|
|
"""One scan and every stored file belonging to it.
|
|
|
|
Attributes:
|
|
scan_id: The scan the report is about, and is named by -- the pipeline id
|
|
whenever one is recorded, because that is the id the GUI shows and so
|
|
the only one a reader of the report can match against anything. Falls
|
|
back to ``stored_scan_id`` when the run has no pipeline row.
|
|
stored_scan_id: The scan row the files were actually read from. Kept apart
|
|
from :attr:`scan_id` because the store is keyed by this one and the
|
|
report is identified by the other; collapsing them makes one of the two
|
|
wrong.
|
|
run_at: When it ran, which is what the report is dated by.
|
|
artifacts: The artifact rows, for the inventory the report lists.
|
|
files: The stored bytes by file name. Held in memory because a report is
|
|
built once and every section wants a different one of them; the
|
|
largest scan in the sample set comes to a few megabytes.
|
|
"""
|
|
|
|
scan_id: UUID
|
|
stored_scan_id: UUID
|
|
run_at: datetime
|
|
artifacts: Sequence[Artifact]
|
|
files: dict[str, bytes]
|
|
|
|
def first(self, *names: str) -> bytes | None:
|
|
"""Return the first of ``names`` the scan stored, or nothing.
|
|
|
|
Matched by suffix rather than equality: a stored name carries whatever the producing stage
|
|
called the file, and the same document travels under `meta.json` and `metadata.json`
|
|
depending on which stage wrote it.
|
|
"""
|
|
for name in names:
|
|
for stored, content in self.files.items():
|
|
if stored.endswith(name):
|
|
return content
|
|
|
|
return None
|
|
|
|
|
|
def _why(configuration: StorageConfiguration, artifacts: Sequence[Artifact], error: Exception) -> str:
|
|
"""Say which artifacts the store has a row for but no object.
|
|
|
|
The bucket reports a missing key without naming it, which on a scan of six artifacts leaves
|
|
whoever ran this with nothing to act on. Rows outliving their objects is the ordinary way this
|
|
fails -- a bucket restored from nothing, a lifecycle rule, a bucket that was never persistent --
|
|
so the answer worth giving is which files are gone, not that one is.
|
|
|
|
Args:
|
|
configuration: The store being read.
|
|
artifacts: The artifact rows the scan claims.
|
|
error: What the fetch raised.
|
|
|
|
Returns:
|
|
The original message, followed by the missing file names when they can be
|
|
established. Diagnosing must never raise on top of the failure it is
|
|
explaining, so anything going wrong here falls back to the message alone.
|
|
"""
|
|
try:
|
|
client = cast("S3Client", configuration.s3.s3_client)
|
|
missing = []
|
|
|
|
for artifact in artifacts:
|
|
try:
|
|
client.head_object(Bucket=configuration.s3.bucket, Key=artifact.s3_key)
|
|
except Exception: # noqa: BLE001, PERF203
|
|
missing.append(artifact.file_name or artifact.s3_key)
|
|
|
|
if not missing:
|
|
return str(error)
|
|
|
|
return (
|
|
f"the bucket holds no object for {len(missing)} of {len(artifacts)} artifact(s) -- "
|
|
f"{', '.join(missing)}. The rows outlived the files; re-upload the scan with "
|
|
f"'persistent_storage put'."
|
|
)
|
|
except Exception: # noqa: BLE001
|
|
return str(error)
|
|
|
|
|
|
_STORAGE_SCAN_OF_PIPELINE = text(
|
|
"""
|
|
SELECT result.id
|
|
FROM core_scanresult AS result
|
|
JOIN scans AS stored ON stored.scan_id = result.id
|
|
WHERE result.scan_id = :pipeline
|
|
ORDER BY stored.run_at DESC
|
|
LIMIT 1
|
|
"""
|
|
)
|
|
|
|
_PIPELINE_OF_STORAGE_SCAN = text("SELECT result.scan_id FROM core_scanresult AS result WHERE result.id = :stored")
|
|
|
|
|
|
def _lookup(configuration: StorageConfiguration, statement: TextClause, parameter: dict[str, str]) -> UUID | None:
|
|
"""Run one id translation against ``core_scanresult``, or give up.
|
|
|
|
Every reason it can fail means the same thing to the caller -- no translation is available --
|
|
and none of them is worth failing a report over: this deployment may have no archon tables at
|
|
all, and the ids still have to be reported under whichever name is known.
|
|
"""
|
|
engine = create_engine(configuration.database.db_url)
|
|
|
|
try:
|
|
with engine.connect() as connection:
|
|
found = connection.execute(statement, parameter).scalar_one_or_none()
|
|
except Exception: # noqa: BLE001
|
|
return None
|
|
finally:
|
|
engine.dispose()
|
|
|
|
return UUID(str(found)) if found is not None else None
|
|
|
|
|
|
def _storage_scan_id(configuration: StorageConfiguration, pipeline_id: UUID) -> UUID | None:
|
|
"""Return the stored scan holding ``pipeline_id``'s artifacts, if that is what was passed.
|
|
|
|
A run is recorded under two different ids. The pipeline mints one -- what cipherverifier names
|
|
its working directories after, what archon's ``core_scan`` row is keyed by, and so the one a
|
|
human has to hand; uploading the artifacts afterwards mints a second, which is what ``scans``
|
|
and ``artifacts`` are keyed by and the only one :func:`get_scan` can look up. ``core_scanresult``
|
|
is where the two meet: its own id is the stored scan, its ``scan_id`` the pipeline that produced
|
|
it.
|
|
|
|
Joining ``scans`` is not decoration -- it keeps a result row whose upload never landed from
|
|
being offered as an answer, and picks the newest upload when a pipeline was stored more than
|
|
once.
|
|
|
|
Args:
|
|
configuration: The store to read from.
|
|
pipeline_id: The id that was passed and that ``scans`` has no row for.
|
|
|
|
Returns:
|
|
The stored scan id, or ``None`` -- the id is not a pipeline's either, or this deployment
|
|
runs without archon's tables, which is not an error: it just means there is nothing to
|
|
translate and the original lookup failure stands.
|
|
"""
|
|
return _lookup(configuration, _STORAGE_SCAN_OF_PIPELINE, {"pipeline": str(pipeline_id)})
|
|
|
|
|
|
def _pipeline_scan_id(configuration: StorageConfiguration, stored_id: UUID) -> UUID | None:
|
|
"""Return the pipeline that produced ``stored_id``, which is the id the report is named by.
|
|
|
|
The translation :func:`_storage_scan_id` performs, read the other way. Needed even though the
|
|
store was reached without it: a report identifies itself by the id the GUI shows, so passing
|
|
the stored id has to produce the same report as passing the pipeline id, not one filed under a
|
|
number nobody can look up.
|
|
|
|
Args:
|
|
configuration: The store to read from.
|
|
stored_id: The scan the files were read from.
|
|
|
|
Returns:
|
|
The pipeline id, or ``None`` when the run has no pipeline row and the stored id is
|
|
therefore the only name it has.
|
|
"""
|
|
return _lookup(configuration, _PIPELINE_OF_STORAGE_SCAN, {"stored": str(stored_id)})
|
|
|
|
|
|
def collect(configuration: StorageConfiguration, scan_id: UUID) -> ReportSources:
|
|
"""Read one scan and its files out of the store.
|
|
|
|
Accepts either id a run is known by -- the pipeline id the GUI shows, or the stored scan id --
|
|
and reports under the pipeline id whichever was given, so that the same run yields the same
|
|
report either way. See :func:`_storage_scan_id` and :func:`_pipeline_scan_id`.
|
|
|
|
Args:
|
|
configuration: The store to read from.
|
|
scan_id: The scan to collect, by either of its ids.
|
|
|
|
Returns:
|
|
The scan's rows and files, identified by the pipeline id where there is one.
|
|
|
|
Raises:
|
|
PipelineError: The scan is unknown, or its files cannot be fetched.
|
|
"""
|
|
context.logger.info(f"Collecting the artifacts of scan {scan_id}...")
|
|
reported_id = scan_id
|
|
|
|
try:
|
|
scan = get_scan(configuration.database, scan_id)
|
|
except Exception as error:
|
|
stored_id = _storage_scan_id(configuration, scan_id)
|
|
|
|
if stored_id is None:
|
|
message = (
|
|
f"Cannot read scan {scan_id}: {error}. No stored scan has that id, and no pipeline "
|
|
f"either -- 'persistent_storage list' shows the scans that can be reported on."
|
|
)
|
|
|
|
raise PipelineError(message) from error
|
|
|
|
# A pipeline id was passed, so it is already the one to report under; only the row to read
|
|
# has to be looked up.
|
|
context.logger.info(f"{scan_id} is the pipeline; its artifacts are stored under scan {stored_id}")
|
|
scan_id = stored_id
|
|
|
|
try:
|
|
scan = get_scan(configuration.database, scan_id)
|
|
except Exception as retried:
|
|
message = f"Cannot read scan {scan_id}: {retried}"
|
|
|
|
raise PipelineError(message) from retried
|
|
|
|
try:
|
|
contents, names = fetch_artifact_files(configuration, [one.artifact_id for one in scan.artifacts])
|
|
except Exception as error:
|
|
message = f"Cannot collect the artifacts of scan {scan_id}: {_why(configuration, scan.artifacts, error)}"
|
|
|
|
raise PipelineError(message) from error
|
|
|
|
context.logger.info(f"Collected {len(names)} file(s) ✔")
|
|
|
|
if reported_id == scan_id:
|
|
# The stored id was passed. The report still has to name the run the way the GUI does, so
|
|
# the pipeline is looked up even though nothing needed it to reach the files.
|
|
reported_id = _pipeline_scan_id(configuration, scan_id) or scan_id
|
|
|
|
return ReportSources(
|
|
scan_id=reported_id,
|
|
stored_scan_id=scan_id,
|
|
run_at=scan.run_at,
|
|
artifacts=scan.artifacts,
|
|
files=dict(zip(names, contents, strict=True)),
|
|
)
|
|
|
|
|
|
# A UUID wherever it appears inside a longer string, which is how the pipeline writes its own id
|
|
# into the paths it records.
|
|
_UUID = re.compile(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", re.IGNORECASE)
|
|
|
|
# The metadata fields that carry a path ending in the run's own id. Both are written by the
|
|
# pipeline itself, so a folder exported from a run identifies itself without being told.
|
|
_ID_BEARING = ("export_folder", "repository_path")
|
|
|
|
# Read whole rather than streamed: the report needs every file at once, and the largest export in
|
|
# the sample set comes to a few megabytes. A folder that does not fit in memory is not an export.
|
|
_METADATA = ("metadata.json", "meta.json")
|
|
|
|
|
|
def collect_from_directory(directory: Path, scan_id: UUID | None = None) -> ReportSources:
|
|
"""Read a folder of exported artifacts, without touching Postgres or the bucket.
|
|
|
|
The same report from a directory somebody was sent: a scan that was exported, a pipeline run on
|
|
a machine with no store attached, a bug report that arrived as a zip. Nothing downstream knows
|
|
the difference.
|
|
|
|
Args:
|
|
directory: The folder holding the artifacts, as the pipeline exported them.
|
|
scan_id: The scan to report under. Read out of the artifacts when not
|
|
given -- the pipeline writes its own id into the paths it records in
|
|
``metadata.json``, and failing that the folder may be named by it.
|
|
|
|
Returns:
|
|
The files, under the id the run is known by.
|
|
|
|
Raises:
|
|
PipelineError: The folder does not exist, holds no files, or holds nothing
|
|
that names the run and none was given.
|
|
"""
|
|
context.logger.info(f"Collecting the artifacts in {directory}...")
|
|
|
|
if not directory.is_dir():
|
|
message = f"'{directory}' is not a directory."
|
|
|
|
raise PipelineError(message)
|
|
|
|
files = {entry.name: entry.read_bytes() for entry in sorted(directory.iterdir()) if entry.is_file()}
|
|
|
|
if not files:
|
|
message = f"'{directory}' holds no files."
|
|
|
|
raise PipelineError(message)
|
|
|
|
context.logger.info(f"Collected {len(files)} file(s) ✔")
|
|
|
|
metadata = _metadata_of(files)
|
|
reported = scan_id or _scan_id_of(metadata, directory)
|
|
|
|
if reported is None:
|
|
message = (
|
|
f"Nothing in '{directory}' names the scan it belongs to. Pass --scan_id, or keep the "
|
|
f"metadata.json the pipeline exported alongside the other artifacts."
|
|
)
|
|
|
|
raise PipelineError(message)
|
|
|
|
return ReportSources(
|
|
scan_id=reported,
|
|
stored_scan_id=reported,
|
|
run_at=_run_at_of(metadata, directory),
|
|
artifacts=[],
|
|
files=files,
|
|
)
|
|
|
|
|
|
def _metadata_of(files: dict[str, bytes]) -> dict[str, Any]:
|
|
"""The pipeline metadata among the collected files, or nothing.
|
|
|
|
Read here as well as in the renderer because the id and the date are wanted before a context is
|
|
built, and a folder without metadata still has to produce a report.
|
|
"""
|
|
for name in _METADATA:
|
|
content = files.get(name)
|
|
|
|
if content is None:
|
|
continue
|
|
|
|
try:
|
|
parsed = json.loads(content)
|
|
except ValueError:
|
|
return {}
|
|
|
|
return parsed if isinstance(parsed, dict) else {}
|
|
|
|
return {}
|
|
|
|
|
|
def _scan_id_of(metadata: dict[str, Any], directory: Path) -> UUID | None:
|
|
"""Work out which run a folder of artifacts belongs to.
|
|
|
|
The pipeline records where it worked and where it exported to, and both paths end in its own
|
|
id, so an export identifies itself. Where the metadata is gone, the folder may still be named
|
|
by the run -- which is how these are usually kept.
|
|
"""
|
|
for field in _ID_BEARING:
|
|
stated = metadata.get(field)
|
|
|
|
if isinstance(stated, str):
|
|
found = _UUID.search(stated)
|
|
|
|
if found:
|
|
return UUID(found.group(0))
|
|
|
|
found = _UUID.fullmatch(directory.resolve().name)
|
|
|
|
return UUID(found.group(0)) if found else None
|
|
|
|
|
|
def _run_at_of(metadata: dict[str, Any], directory: Path) -> datetime:
|
|
"""When the run happened.
|
|
|
|
Its own timestamp where the metadata kept one; otherwise the folder's, which is when it was
|
|
exported rather than when it ran -- close enough to date a report by, and better than now.
|
|
"""
|
|
stated = metadata.get("pipeline_start_iso")
|
|
|
|
if isinstance(stated, str):
|
|
try:
|
|
return datetime.fromisoformat(stated)
|
|
except ValueError:
|
|
pass
|
|
|
|
return datetime.fromtimestamp(directory.stat().st_mtime, tz=UTC)
|