diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..76bf638 --- /dev/null +++ b/.gitignore @@ -0,0 +1,28 @@ +# Bytecode and caches. +__pycache__/ +*.py[cod] +.mypy_cache/ +.ruff_cache/ +.pytest_cache/ + +# Virtual environments. +.venv/ +venv/ + +# Build and editable-install artefacts. `pyproject.toml` is tracked; what it produces is not. +build/ +dist/ +*.egg-info/ +log.txt + +# Rendered output. The report is built from a scan, never committed. +*.pdf +!sample_report_template/*.pdf + +# Local configuration and credentials the CLI reads. +.env +.env.* +!.env.example + +# macOS. +.DS_Store diff --git a/README.md b/README.md new file mode 100644 index 0000000..6671a5f --- /dev/null +++ b/README.md @@ -0,0 +1,89 @@ +# report_generator + +Builds the PDF report of one stored scan, from the artifacts that scan put in the bucket. + +## Installing + +The package lives in `report_generator/`, beside the `pyproject.toml` that builds it. Install it +from the repository root — the directory holding this file, not the package directory: + +``` +pip install -e . # add [dev] for the test and lint tooling +``` + +Installing from inside `report_generator/` does not work: `packages.find` looks for the package as a +subdirectory, so it finds nothing, and the `report_generator` console script is then installed +pointing at a module that was never put on the path. + +## Running + +``` +report_generator --scan_id [--path ] --config config.json +``` + +Without `--path` the report is written into `reports/`. The S3 secret comes from the environment: +`PERSISTENT_STORAGE_STORAGE_CONFIG_S3_SECRET_ACCESS_KEY` (and the database password from +`PERSISTENT_STORAGE_STORAGE_CONFIG_DATABASE_PASSWORD`). The name is not this tool's to choose: it is +derived from the config path `inject_secrets` fills in under — `persistent_storage.storage_config` — +by the one naming convention in `cc_common.config.env_var_name`, so it stays whatever the monorepo +calls it. A `.env` beside the `pyproject.toml` is read too. `--config` is resolved relative to the +working directory. + +## How it is put together + +Four steps, each its own module, so that the one that will change is the one that is separate: + +| Module | Does | +| --- | --- | +| `report_generator/sources.py` | Reads the scan and its files out of the store. **The only module that knows which store.** | +| `report_generator/rendering.py` | Decides which stored file feeds which part of the report, and parses it. | +| `report_generator/context.py` | Maps the parsed artifacts onto the page, and writes `` where they say nothing. | +| `report_generator/layout.py` | Draws `template/report.html.j2` through WeasyPrint. | + +No parsing of its own. SARIF and the vulnerability report are read by the summarisers in +`cipherverifier.core.services`, which are written to be free of Django and of any store; the CVSS +vectors by `persistent_storage.utils.cvss_utils`. + +### What it cannot fill in + +A field the artifacts do not answer is written as ``, never guessed or quietly dropped. Today +that is the report and artifact ids, the manufacturer, the analysed lines of code, the language +breakdown, and the tools and profile of each pipeline stage — the metadata records that a stage ran +and how long it took, not what it ran. + +A measured zero survives: `0 filtered` is a reading, and a blank is a gap. + +## Moving it to archon + +`sources.collect()` is the whole of it. It returns files by name and the rows a report lists; +everything after that point works on bytes and cannot tell which store handed them over. Pointing it +at archon's tables is a change to that function and to nothing else. + +## Its relationship to the monorepo + +This repository is the split of `src/persistent_storage/report_generator/` out of the monorepo. The +package names nothing about itself — every import inside it is relative — so the split was a move +and not a rewrite: the sources sit unchanged under `report_generator/`, with `pyproject.toml` beside +them rather than inside them. + +`report_generator/vendor/` supplies everything the package borrows from the monorepo — `cc_common`, +`persistent_storage.shared`, and the two summarisers. `vendor/__init__.py` puts the copies on the +path when the real packages are not installed, and stays out of the way when they are, so the same +sources run in both places. See `report_generator/vendor/README.md`. + +Still to come across: `tests/persistent_storage/test_report_*.py` from the monorepo, with +`persistent_storage.report_generator` changed to `report_generator` in their imports. Until they +land, `testpaths = ["tests"]` in `pyproject.toml` points at a directory that does not exist yet. + +On the monorepo side, `tests/persistent_storage/test_report_vendor.py` performs this same split into +a temporary directory on every run and renders a report from it, so the copies cannot quietly stop +being sufficient. + +### Native dependencies + +WeasyPrint needs Pango and cairo, which pip does not install: + +``` +brew install pango # macOS +apt-get install libpango-1.0-0 libpangoft2-1.0-0 # Debian/Ubuntu +``` diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..ec75bf6 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,116 @@ +# Packaging for report_generator as a repository of its own. +# +# The split out of the monorepo has happened: the package sits in `report_generator/` beside this +# file, which is what `packages.find` below looks for. In the monorepo the same sources live at +# `src/persistent_storage/report_generator/` and are built by the monorepo's own pyproject; this +# one is not read there. +# +# The dependency pins mirror the monorepo's, so that a tool split out of it does not silently start +# resolving different versions of the libraries it was tested against. + +[build-system] +requires = ["setuptools>=77.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "report-generator" +version = "0.1.0" +description = "Builds the PDF report of a stored scan from the artifacts it produced." +readme = "README.md" +requires-python = ">=3.14" +# PEP 639: an SPDX expression. "Proprietary" is not one; a private licence is spelled LicenseRef-*. +license = "LicenseRef-Proprietary" +authors = [{ name = "CipherCell" }] + +dependencies = [ + # The CLI, its configuration files and the properties models every command is described by. + "typer>=0.24.1", + "typer-config>=1.5.1", + "pydantic>=2.13.4", + "python-dotenv>=1.2.2", + "rich>=15.0.0", + "pyyaml>=6.0.3", + + # The store: scan and artifact rows out of Postgres, artifact files out of the bucket. + "SQLAlchemy>=2.0.49", + "psycopg2-binary>=2.9.9", + "boto3>=1.43.0", + + # The report: the template, and the engine that turns it into a PDF. WeasyPrint additionally + # needs the native Pango and cairo libraries, which pip does not install -- see the README. + "jinja2>=3.1.6", + "weasyprint>=69.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=9.0.0", + "mypy>=1.19.0", + "ruff>=0.15.16", + "mypy-boto3-s3>=1.43.31", + # Reads the rendered PDF back, so the tests can check what actually came out. + "pymupdf>=1.24.0", +] + +[project.scripts] +report_generator = "report_generator.__main__:run" + +[tool.setuptools.packages.find] +# `vendor` is deliberately included: outside the monorepo it is what supplies cc_common, +# persistent_storage and the two summarisers, and `vendor/__init__.py` puts it on the path. +include = ["report_generator*"] + +[tool.setuptools.package-data] +# Loaded at runtime by __file__, so it has to be installed beside the module: the Jinja template, +# the print stylesheet, the mark, and the font files the stylesheet references by relative URL. +"report_generator" = ["template/**/*"] +# The copies are data as far as packaging is concerned -- they are put on the path, not imported +# as submodules of this package. +"report_generator.vendor" = ["**/*.py", "**/py.typed"] + +[tool.ruff] +line-length = 120 +target-version = "py314" +# The copies are checked where they are written, not where they are copied to. Left in, every class +# they define is read twice under two import paths. +exclude = ["report_generator/vendor"] + +[tool.ruff.lint] +select = ["E", "W", "F", "I", "N", "D", "ANN", "B", "C4", "SIM", "RUF"] +# RUF100: the sources carry `# noqa` for rules the monorepo's own ruff config does not select +# either -- they are stale there too, just invisible, because the monorepo does not select RUF. +# Flagging them here would mean editing the copies, and the copies have to stay copies. +ignore = ["D203", "D213", "RUF100"] + +[tool.ruff.lint.pydocstyle] +convention = "google" + +[tool.ruff.lint.isort] +# What the monorepo sorts these as: it lints from a root where all three live under `src/`, which +# ruff's default `src = [".", "src"]` makes first-party. Here they arrive through `vendor/` and +# would be sorted as third-party instead. Declared so that this repository reports the same import +# findings the monorepo does -- no more and no fewer -- and the copies can stay copies. +known-first-party = ["cc_common", "persistent_storage", "cipherverifier"] + +[tool.mypy] +strict = true +python_version = "3.14" +# The copies are on the path at runtime only because `vendor/__init__.py` puts them there, which +# mypy does not execute. Named here so the borrowed modules resolve while still being excluded +# from being checked themselves. +mypy_path = ["report_generator/vendor"] +exclude = ["(^|/)report_generator/vendor/"] + +[[tool.mypy.overrides]] +module = ["weasyprint.*", "pymupdf.*"] +ignore_missing_imports = true + +# `exclude` keeps mypy from crawling into the copies; it does not stop it following an import into +# them. They are checked where they are written, so their own errors are not this repository's to +# report -- their types still are, which is why they are followed at all rather than ignored. +[[tool.mypy.overrides]] +module = ["cc_common.*", "persistent_storage.*", "cipherverifier.*"] +follow_imports = "silent" + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/report_generator/__init__.py b/report_generator/__init__.py new file mode 100644 index 0000000..ec64252 --- /dev/null +++ b/report_generator/__init__.py @@ -0,0 +1,16 @@ +"""Builds the PDF report of one stored scan, from the artifacts that scan put in the bucket. + +A standalone tool rather than another persistent_storage command: it reads the store and writes a +file, and nothing else in the pipeline depends on it. The seams are drawn so that the whole package +moves to archon by changing what :mod:`sources` reads from -- +everything after that point works on bytes and rows, not on a particular store. + +The parts it does not own are reused rather than reimplemented, and copied into ``vendor/`` so the +package can also be lifted into a repository of its own. See ``vendor/README.md``. +""" + +from .vendor import activate + +# Before anything else imports what this borrows. In the monorepo it finds the real packages and +# does nothing; in a repository of its own it puts the copies on the path. +activate() diff --git a/report_generator/__main__.py b/report_generator/__main__.py new file mode 100644 index 0000000..1675840 --- /dev/null +++ b/report_generator/__main__.py @@ -0,0 +1,33 @@ +"""Entrypoint for the report generator CLI.""" + +from cc_common.api.cli import CLIManager +from cc_common.config import load_environment +from persistent_storage.shared import inject_secrets + +from .commands.generate import GenerateProperties, generate + + +def run() -> None: + """Entrypoint for CLI.""" + load_environment() + + cli = CLIManager("report_generator", "Generates the PDF report of a stored scan.") + + cli.register_command( + "generate", + _with_storage_secrets, + GenerateProperties, + help="Builds the PDF report of one scan from the artifacts it stored.", + ) + cli.run() + + +def _with_storage_secrets(properties: GenerateProperties) -> object: + """Fill the storage configuration's secrets from the environment before the command runs.""" + inject_secrets(properties.storage_config) + + return generate(properties) + + +if __name__ == "__main__": + run() diff --git a/report_generator/commands/generate.py b/report_generator/commands/generate.py index 463fb02..9efaeab 100644 --- a/report_generator/commands/generate.py +++ b/report_generator/commands/generate.py @@ -3,10 +3,10 @@ from pathlib import Path from uuid import UUID -from cc_common.api import PipelineError, context -from cc_common.utils import print from pydantic import BaseModel, Field +from cc_common.api import PipelineError, context +from cc_common.utils import print from persistent_storage.shared import StorageConfiguration from ..rendering import render diff --git a/sample_report_template/sast_report_sample_fixed.pdf b/sample_report_template/sast_report_sample_fixed.pdf new file mode 100644 index 0000000..5d9276c Binary files /dev/null and b/sample_report_template/sast_report_sample_fixed.pdf differ