Allow generation of metrics.json simply by accessing the artifacts folder.
This commit is contained in:
Florian Frank committed
1 parent
7b6c19d8d9
commit
9d7df77923
12 files changed
+802
-486
No files matched your search
Vendored
+3
-12
@@ -3,18 +3,9 @@
|
||||
Copies of everything `report_generator` borrows from the monorepo, so the package can be lifted into
|
||||
a repository of its own.
|
||||
|
||||
Nothing in here is edited. `sync.py` writes it, under the same import paths the monorepo uses:
|
||||
|
||||
```
|
||||
python -m persistent_storage.report_generator.vendor.sync
|
||||
```
|
||||
|
||||
Re-run it after changing anything it lists; the diff then shows what drifted. The copied packages
|
||||
are removed whole first, so dropping an entry from the list actually drops it from the tree.
|
||||
|
||||
`__init__.py` decides whether these copies are used. It asks whether the real packages can be
|
||||
imported — not a flag, not an environment variable — so the same code runs in the monorepo and
|
||||
outside it, and the copies can never shadow the real packages while those are present.
|
||||
The files in here are committed with the standalone package. `__init__.py` puts
|
||||
these copies first on `sys.path`, so the package does not accidentally use an
|
||||
unrelated installed copy of one of these packages.
|
||||
|
||||
## What is copied, and what for
|
||||
|
||||
|
||||
Vendored
+7
-34
@@ -1,53 +1,26 @@
|
||||
"""Copies of what this tool borrows, and the switch that makes them the ones it uses.
|
||||
"""Make the report generator's bundled support packages importable.
|
||||
|
||||
In the monorepo the real packages are importable and nothing here is touched. In a repository of
|
||||
its own they are not, and :func:`activate` puts these copies on the path instead.
|
||||
|
||||
The check is "can it be imported", not a flag or an environment variable, so the same code runs in
|
||||
both places without being told which one it is in -- and so the copies can never shadow the real
|
||||
packages while those are present.
|
||||
The report generator is a standalone repository. Its copies of ``cc_common``,
|
||||
``persistent_storage`` and ``cipherverifier`` are part of this package and must
|
||||
win even when similarly named packages happen to be installed on the machine.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
VENDOR_ROOT = Path(__file__).resolve().parent
|
||||
|
||||
# One module out of each copied package. If these resolve, the real thing is installed.
|
||||
PROBES = ("cc_common", "persistent_storage.shared", "cipherverifier.core.services.sarif")
|
||||
|
||||
|
||||
def activate() -> bool:
|
||||
"""Put the vendored copies on the path where the real packages are missing.
|
||||
"""Put the bundled support packages first on the import path.
|
||||
|
||||
Returns:
|
||||
Whether the copies were used. ``False`` in the monorepo, where the real
|
||||
packages are installed and are left to win.
|
||||
Always ``True``. The return value is retained for callers that used the
|
||||
previous activation API.
|
||||
"""
|
||||
if all(_installed(name) for name in PROBES):
|
||||
return False
|
||||
|
||||
if str(VENDOR_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(VENDOR_ROOT))
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def _installed(name: str) -> bool:
|
||||
"""Whether a module can be imported without importing it.
|
||||
|
||||
Args:
|
||||
name: A dotted module name.
|
||||
|
||||
Returns:
|
||||
Whether the import system can find it. Wrapped because ``find_spec``
|
||||
raises rather than returning ``None`` when a parent package is itself
|
||||
missing, which is exactly the case being tested for.
|
||||
"""
|
||||
try:
|
||||
return importlib.util.find_spec(name) is not None
|
||||
except (ImportError, ValueError):
|
||||
return False
|
||||
@@ -1,26 +0,0 @@
|
||||
"""Collection of CLI helper functions."""
|
||||
|
||||
from .compose_utils import compose_down, compose_up
|
||||
from .defs import DockerArchitecture
|
||||
from .docker_utils import (
|
||||
abort_container,
|
||||
check_image_exist,
|
||||
create_docker_image,
|
||||
exec_command,
|
||||
is_docker_running,
|
||||
run_command,
|
||||
run_command_with_input,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"DockerArchitecture",
|
||||
"abort_container",
|
||||
"check_image_exist",
|
||||
"compose_down",
|
||||
"compose_up",
|
||||
"create_docker_image",
|
||||
"exec_command",
|
||||
"is_docker_running",
|
||||
"run_command",
|
||||
"run_command_with_input",
|
||||
]
|
||||
@@ -1,112 +0,0 @@
|
||||
"""Provide utilities for driving Docker Compose stacks.
|
||||
|
||||
Unlike the rest of this package, Compose is not reachable through the docker-py
|
||||
SDK. It is a separate binary. These helpers are therefore implemented separately,
|
||||
while keeping the same logging and error conventions as the SDK-based utilities next to them.
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
from shutil import which
|
||||
|
||||
from cc_common.api import PipelineError, context
|
||||
from cc_common.processes import run_subprocess
|
||||
|
||||
|
||||
def compose_up(compose_file: Path, project_name: str, *arguments: str) -> None:
|
||||
"""Start the services of a Compose file.
|
||||
|
||||
Nothing is implied about how they start: pass ``"--detach"`` to return once
|
||||
the containers exist, ``"--build"`` to rebuild images first, and so on. A
|
||||
call without arguments runs in the foreground, exactly as Compose would.
|
||||
|
||||
Args:
|
||||
compose_file: Path to the Compose file describing the stack.
|
||||
project_name: Name of the Compose project to act on.
|
||||
arguments: Additional options forwarded to ``compose up``.
|
||||
|
||||
Raises:
|
||||
PipelineError: If Docker is unavailable, the Compose file does not
|
||||
exist, or Compose exits with a non-zero status.
|
||||
"""
|
||||
run_compose(compose_file, project_name, "up", *arguments)
|
||||
|
||||
|
||||
def compose_down(compose_file: Path, project_name: str, *arguments: str) -> None:
|
||||
"""Stop the services of a Compose file and remove their containers.
|
||||
|
||||
Named volumes survive unless ``"--volumes"`` is passed, which is the one
|
||||
argument here that destroys data — Compose's own default is to keep it, and
|
||||
this function does not second-guess that either way.
|
||||
|
||||
Args:
|
||||
compose_file: Path to the Compose file describing the stack.
|
||||
project_name: Name of the Compose project to act on.
|
||||
arguments: Additional options forwarded to ``compose down``.
|
||||
|
||||
Raises:
|
||||
PipelineError: If Docker is unavailable, the Compose file does not
|
||||
exist, or Compose exits with a non-zero status.
|
||||
"""
|
||||
run_compose(compose_file, project_name, "down", *arguments)
|
||||
|
||||
|
||||
def run_compose(compose_file: Path, project_name: str, *arguments: str) -> None:
|
||||
"""Run any Docker Compose subcommand against a Compose file.
|
||||
|
||||
Args:
|
||||
compose_file: Path to the Compose file describing the stack.
|
||||
project_name: Name of the Compose project to act on. Passing it
|
||||
explicitly keeps containers and volumes stable no matter which
|
||||
directory the caller is invoked from — Compose would otherwise
|
||||
derive the name from the directory holding the Compose file.
|
||||
arguments: The Compose subcommand and its options, e.g. ``"up",
|
||||
"--detach"``.
|
||||
|
||||
Raises:
|
||||
PipelineError: If Docker is unavailable, the Compose file does not
|
||||
exist, or Compose exits with a non-zero status.
|
||||
"""
|
||||
if not compose_file.is_file():
|
||||
raise PipelineError(f"Compose file not found at {compose_file}")
|
||||
|
||||
command = compose_command()
|
||||
subcommand = " ".join(arguments)
|
||||
|
||||
context.logger.info(f"Running 'compose {subcommand}' on {compose_file} ...")
|
||||
|
||||
# Currently, no solution without using subprocesses available.
|
||||
exit_code = run_subprocess(
|
||||
[*command, "--project-name", project_name, "--file", str(compose_file), *arguments],
|
||||
Path.cwd(),
|
||||
stream_output=True,
|
||||
on_stdout=context.logger.info,
|
||||
on_stderr=context.logger.info,
|
||||
fill_stream_threaded=True,
|
||||
)
|
||||
|
||||
if exit_code != 0:
|
||||
raise PipelineError(f"'compose {subcommand}' failed with exit code {exit_code}")
|
||||
|
||||
|
||||
def compose_command() -> list[str]:
|
||||
"""Determine how Docker Compose can be invoked on this machine.
|
||||
|
||||
Returns:
|
||||
The executable and subcommand to prefix the Compose arguments with,
|
||||
preferring the ``docker compose`` plugin over the standalone binary.
|
||||
|
||||
Raises:
|
||||
PipelineError: If neither ``docker`` nor the legacy ``docker-compose``
|
||||
executable is on the PATH.
|
||||
"""
|
||||
docker_path = which("docker")
|
||||
|
||||
if docker_path is not None:
|
||||
return [docker_path, "compose"]
|
||||
|
||||
legacy_compose_path = which("docker-compose")
|
||||
|
||||
if legacy_compose_path is None:
|
||||
raise PipelineError("Docker executable not found")
|
||||
|
||||
return [legacy_compose_path]
|
||||
@@ -1,19 +0,0 @@
|
||||
"""Docker-specific architecture and platform definitions."""
|
||||
|
||||
from enum import Enum
|
||||
|
||||
from cc_common.defines import CPUArchitecture, OperatingSystem
|
||||
|
||||
CONTAINER_DEFAULT_ABORT_TIMEOUT_IN_S = 5
|
||||
|
||||
|
||||
class DockerArchitecture(Enum):
|
||||
"""Enumeration of supported Docker platforms combining OS and CPU architecture."""
|
||||
|
||||
LINUX_AMD64 = (OperatingSystem.LINUX, CPUArchitecture.AMD64)
|
||||
LINUX_ARM64 = (OperatingSystem.LINUX, CPUArchitecture.ARM64)
|
||||
|
||||
@property
|
||||
def arch_str(self) -> str:
|
||||
"""Return the os/cpu string which can be passed to docker command as plattform."""
|
||||
return f"{self.value[0].value}/{self.value[1].value}"
|
||||
@@ -1,266 +0,0 @@
|
||||
"""Provide utility functions for interacting with Docker via the Docker SDK.
|
||||
|
||||
Supports image management, container execution, and environment validation
|
||||
through the official docker-py client with logging integration.
|
||||
"""
|
||||
|
||||
import os
|
||||
import shlex
|
||||
import socket
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import docker
|
||||
import docker.errors
|
||||
from docker import DockerClient
|
||||
|
||||
from cc_common.api import context
|
||||
|
||||
from .defs import CONTAINER_DEFAULT_ABORT_TIMEOUT_IN_S, DockerArchitecture
|
||||
|
||||
|
||||
def _get_client() -> DockerClient:
|
||||
"""Create and return a Docker client from the local environment.
|
||||
|
||||
Raises:
|
||||
docker.errors.DockerException: If the Docker daemon is unreachable.
|
||||
"""
|
||||
return docker.from_env()
|
||||
|
||||
|
||||
def _stop_container(container_id: str, timeout_in_s: int) -> None:
|
||||
"""Stops a specific container by ID."""
|
||||
client = _get_client()
|
||||
try:
|
||||
container = client.containers.get(container_id)
|
||||
context.logger.info(f"Aborting and removing container: {container_id}")
|
||||
container.stop(timeout=timeout_in_s)
|
||||
container.remove(force=True)
|
||||
except docker.errors.NotFound:
|
||||
context.logger.debug(f"Container {container_id} already removed.")
|
||||
except Exception as e:
|
||||
context.logger.error(f"Failed to abort container {container_id}: {e}")
|
||||
|
||||
|
||||
def is_docker_running() -> bool:
|
||||
"""Check whether the Docker daemon is running."""
|
||||
context.logger.info("Checking if Docker is running")
|
||||
try:
|
||||
_get_client().ping()
|
||||
return True
|
||||
except docker.errors.DockerException:
|
||||
return False
|
||||
|
||||
|
||||
def check_image_exist(image_name: str) -> bool:
|
||||
"""Check whether a Docker image exists locally."""
|
||||
context.logger.info(f"Checking if image '{image_name}' exists")
|
||||
try:
|
||||
_get_client().images.get(image_name)
|
||||
return True
|
||||
except docker.errors.ImageNotFound:
|
||||
return False
|
||||
|
||||
|
||||
def create_docker_image(
|
||||
image_name: str,
|
||||
project_folder: Path,
|
||||
docker_architecture: DockerArchitecture | None = None,
|
||||
build_dargs: dict[str, str] | None = None,
|
||||
) -> None:
|
||||
"""Build a Docker image from the specified project directory."""
|
||||
context.logger.info(f"Creating docker image '{image_name}' from '{project_folder}'")
|
||||
|
||||
client = _get_client()
|
||||
|
||||
try:
|
||||
for chunk in client.api.build(
|
||||
path=str(project_folder),
|
||||
tag=image_name,
|
||||
buildargs=build_dargs,
|
||||
decode=True, # Low-level API handles this correctly
|
||||
rm=True,
|
||||
platform=docker_architecture.arch_str if docker_architecture else None,
|
||||
):
|
||||
if not isinstance(chunk, dict):
|
||||
continue
|
||||
|
||||
if "stream" in chunk:
|
||||
message = str(chunk["stream"]).strip()
|
||||
if message:
|
||||
context.logger.debug(message)
|
||||
|
||||
if "error" in chunk:
|
||||
error_data = chunk.get("errorDetail", {})
|
||||
error_msg = error_data.get("message", chunk["error"])
|
||||
|
||||
raise docker.errors.BuildError(error_msg, iter([chunk]))
|
||||
|
||||
except docker.errors.BuildError as e:
|
||||
context.logger.error(f"Docker build failed: {e}")
|
||||
raise e
|
||||
except Exception as e:
|
||||
context.logger.error(f"An unexpected error occurred during build: {e}")
|
||||
raise e
|
||||
|
||||
|
||||
def _rewrite_path_in_cmd(cmd: list[str], old_path: str, new_path: str) -> list[str]:
|
||||
"""Replace `old_path` with `new_path` in every argument that points at it."""
|
||||
rewritten = []
|
||||
for arg in cmd:
|
||||
if arg == old_path:
|
||||
rewritten.append(new_path)
|
||||
elif arg.startswith(f"{old_path}/"):
|
||||
rewritten.append(f"{new_path}{arg[len(old_path) :]}")
|
||||
else:
|
||||
rewritten.append(arg)
|
||||
return rewritten
|
||||
|
||||
|
||||
def _wrap_cmd_with_copy(cmd: list[str], mount_folders: list[tuple[Path, str]], target_folder: str) -> list[str]:
|
||||
"""Wrap `cmd` so the mounted folders are copied into `target_folder` first.
|
||||
|
||||
The copy is created by the container's own user, so it is writable without any
|
||||
privilege escalation, and the read-only mounts stay untouched. Arguments in `cmd`
|
||||
that point into a mounted folder are rewritten to the copy. Requires `sh`, `mkdir`
|
||||
and `cp` in the image.
|
||||
"""
|
||||
copy_cmds = []
|
||||
for _, container_path in mount_folders:
|
||||
target_path = f"{target_folder.rstrip('/')}/{Path(container_path).name}"
|
||||
quoted_target = shlex.quote(target_path)
|
||||
copy_cmds.append(f"mkdir -p {quoted_target} && cp -a {shlex.quote(container_path)}/. {quoted_target}/")
|
||||
cmd = _rewrite_path_in_cmd(cmd, container_path, target_path)
|
||||
|
||||
script = " && ".join([*copy_cmds, f"exec {shlex.join(cmd)}"])
|
||||
return ["sh", "-c", script]
|
||||
|
||||
|
||||
def run_command(
|
||||
image_name: str,
|
||||
cmd: list[str],
|
||||
mount_folders: list[tuple[Path, str]],
|
||||
remove_after_execution: bool,
|
||||
copy_mounts_to: str | None = None,
|
||||
) -> str:
|
||||
"""Run a command inside a Docker container with optional volume mounts.
|
||||
|
||||
Args:
|
||||
image_name: Image to run the command in.
|
||||
cmd: Command and arguments to execute.
|
||||
mount_folders: Pairs of (host path, container path) to mount.
|
||||
remove_after_execution: Whether the container is removed after it exits.
|
||||
copy_mounts_to: Optional container folder (for example the user's home) the
|
||||
mounted folders are copied into before the command runs. The mounts are
|
||||
then read-only and `cmd` arguments pointing at them are rewritten to the
|
||||
copy, so the command works on data owned by the container's own user.
|
||||
"""
|
||||
mount_mode = "ro" if copy_mounts_to else "rw"
|
||||
volumes = {
|
||||
str(local_path.resolve()): {"bind": container_path, "mode": mount_mode}
|
||||
for local_path, container_path in mount_folders
|
||||
}
|
||||
|
||||
context.logger.info(f"Running command {cmd} in image '{image_name}'")
|
||||
|
||||
run_cmd = _wrap_cmd_with_copy(cmd, mount_folders, copy_mounts_to) if copy_mounts_to else cmd
|
||||
|
||||
output = _get_client().containers.run(
|
||||
image=image_name, command=run_cmd, volumes=volumes, remove=remove_after_execution, stdout=True, stderr=True
|
||||
)
|
||||
return output.decode("utf-8")
|
||||
|
||||
|
||||
def run_command_with_input(
|
||||
image_name: str, cmd: list[str], input_data: str, environment: dict[str, Any] | None = None
|
||||
) -> str:
|
||||
"""Run a command inside a Docker container, streaming a string directly to its stdin."""
|
||||
context.logger.info(f"Running command {cmd} in image '{image_name}'")
|
||||
client = _get_client()
|
||||
|
||||
container = client.containers.create(
|
||||
image=image_name, command=cmd, stdin_open=True, tty=False, environment=environment
|
||||
)
|
||||
|
||||
try:
|
||||
container.start()
|
||||
|
||||
docker_sock = container.attach_socket(params={"stdin": 1, "stream": 1})
|
||||
|
||||
try:
|
||||
fd = docker_sock.fileno() # type: ignore[union-attr]
|
||||
clean_fd = os.dup(fd)
|
||||
|
||||
if hasattr(docker_sock, "_response"):
|
||||
try: # noqa: SIM105
|
||||
docker_sock._response.close()
|
||||
except Exception as e:
|
||||
context.logger.error(f"hasattr returns {e}")
|
||||
docker_sock._response = None # drop the reference entirely
|
||||
|
||||
family = socket.AF_UNIX if fd < 100 else socket.AF_INET
|
||||
clean_sock = socket.socket(family=family, fileno=clean_fd)
|
||||
|
||||
finally:
|
||||
docker_sock.close()
|
||||
try:
|
||||
clean_sock.sendall(input_data.encode("utf-8"))
|
||||
clean_sock.shutdown(socket.SHUT_WR)
|
||||
finally:
|
||||
clean_sock.close()
|
||||
|
||||
exit_result = container.wait()
|
||||
output = container.logs(stdout=True, stderr=True).decode("utf-8")
|
||||
|
||||
if exit_result["StatusCode"] != 0:
|
||||
raise RuntimeError(f"Container failed ({exit_result['StatusCode']}):\n{output}")
|
||||
|
||||
return output
|
||||
finally:
|
||||
container.remove(force=True)
|
||||
|
||||
|
||||
def exec_command(container_id: str, cmd: list[str]) -> str:
|
||||
"""Execute a command inside a running container.
|
||||
|
||||
Returns:
|
||||
The stdout output of the command as a decoded string.
|
||||
"""
|
||||
context.logger.info(f"Executing command {cmd} in container '{container_id}'")
|
||||
|
||||
container = _get_client().containers.get(container_id)
|
||||
exit_code, output = container.exec_run(cmd)
|
||||
|
||||
if exit_code != 0:
|
||||
raise docker.errors.APIError(f"Command {cmd} failed in container '{container_id}' with exit code {exit_code}")
|
||||
|
||||
if isinstance(output, bytes):
|
||||
return output.decode("utf-8")
|
||||
|
||||
return b"".join(output).decode("utf-8")
|
||||
|
||||
|
||||
def abort_container(
|
||||
container_id: str | None = None,
|
||||
image_name: str | None = None,
|
||||
timeout_in_s: int = CONTAINER_DEFAULT_ABORT_TIMEOUT_IN_S,
|
||||
) -> None:
|
||||
"""Forcefully stop and remove a specific container by ID."""
|
||||
if not container_id and not image_name:
|
||||
context.logger.error("Neither container nor image specified.")
|
||||
return
|
||||
|
||||
client = _get_client()
|
||||
if container_id:
|
||||
_stop_container(container_id, timeout_in_s)
|
||||
else:
|
||||
running_containers = client.containers.list()
|
||||
for container in running_containers:
|
||||
if (
|
||||
image_name
|
||||
and container.id
|
||||
and container.image
|
||||
and container.image.tags
|
||||
and any(image_name in tag for tag in container.image.tags)
|
||||
):
|
||||
_stop_container(container.id, timeout_in_s)
|
||||
Reference in new issue
Block a user