Files
ModelForge/backend/tests/test_packaging_m17.py

681 lines
28 KiB
Python

"""Packaging and configuration guarantees for the v1 release.
The release artefact is what an operator receives. These tests hold it to the things that are only
noticeable once it is too late: a secret that shipped, a production overlay that quietly accepts a
development default, an image without the identity to trace it back to a commit.
"""
from __future__ import annotations
import json
import re
import subprocess
import sys
import tomllib
from pathlib import Path, PurePosixPath
import pytest
from modelforge_api.domain.configuration_reference import (
DEPLOYMENT_DOCS,
SETTING_DOCS,
Sensitivity,
)
from modelforge_api.domain.release import PRODUCT_VERSION
from modelforge_api.settings import Settings
ROOT = Path(__file__).resolve().parents[2]
DOCKERFILES = (
"backend/Dockerfile",
"frontend/Dockerfile",
"node-agent/Dockerfile",
"runtime-worker/Dockerfile",
)
# --------------------------------------------------------------------------- configuration
def test_every_setting_is_documented() -> None:
"""A setting added without documentation fails the build rather than shipping quietly."""
undocumented = sorted(set(Settings.model_fields) - set(SETTING_DOCS))
assert not undocumented, f"undocumented settings: {undocumented}"
def test_the_reference_documents_no_setting_that_no_longer_exists() -> None:
stale = sorted(set(SETTING_DOCS) - set(Settings.model_fields))
assert not stale, f"documented settings that no longer exist: {stale}"
def test_the_generated_configuration_files_are_current() -> None:
"""`.env.example` and the configuration reference are generated, never hand-edited."""
completed = subprocess.run( # noqa: S603 - fixed argv, no shell
[sys.executable, str(ROOT / "scripts" / "generate_configuration_docs.py"), "--check"],
capture_output=True,
text=True,
encoding="utf-8",
errors="replace",
cwd=ROOT,
check=False,
)
assert completed.returncode == 0, (
"configuration documentation is stale; run "
f"scripts/generate_configuration_docs.py\n{completed.stdout}{completed.stderr}"
)
def test_the_example_configuration_carries_no_secret_values() -> None:
"""The example is committed, so a real value in it would be published with the release."""
text = (ROOT / ".env.example").read_text("utf-8")
secret_names = [
f"MODELFORGE_{name.upper()}"
for name, doc in SETTING_DOCS.items()
if doc.sensitivity is Sensitivity.SECRET
]
for name in secret_names:
for line in text.splitlines():
if line.startswith(f"{name}="):
assert line == f"{name}=", f"{name} carries a value in .env.example"
def test_every_required_production_setting_appears_in_the_example() -> None:
text = (ROOT / ".env.example").read_text("utf-8")
for name, doc in SETTING_DOCS.items():
if doc.required_in_production:
assert f"MODELFORGE_{name.upper()}=" in text, f"{name} is required but not offered"
def test_deployment_variables_are_documented_too() -> None:
"""Variables Compose and the agents read are still variables an operator has to set."""
text = (ROOT / ".env.example").read_text("utf-8")
for name in DEPLOYMENT_DOCS:
assert f"{name}=" in text, f"{name} is undocumented in .env.example"
# --------------------------------------------------------------------------- production overlay
def test_the_production_overlay_exists_and_sets_the_production_profile() -> None:
"""Without MODELFORGE_ENV=production every fail-closed startup rule stays switched off."""
text = (ROOT / "docker-compose.production.yml").read_text("utf-8")
assert "MODELFORGE_ENV: production" in text
@pytest.mark.parametrize(
"variable",
[
"MODELFORGE_POSTGRES_ADMIN_PASSWORD",
"MODELFORGE_MIGRATION_DB_PASSWORD",
"MODELFORGE_RUNTIME_DB_PASSWORD",
"MODELFORGE_MIGRATION_DATABASE_URL",
"MODELFORGE_RUNTIME_DATABASE_URL",
"MODELFORGE_OPERATOR_API_KEY",
"MODELFORGE_BACKUP_ENCRYPTION_KEY",
"MODELFORGE_CORS_ORIGINS",
],
)
def test_the_production_overlay_refuses_to_render_without_its_secrets(variable: str) -> None:
"""`${VAR:?message}` makes Compose fail before a single container starts."""
text = (ROOT / "docker-compose.production.yml").read_text("utf-8")
assert re.search(rf"\$\{{{variable}:\?[^}}]+\}}", text), (
f"{variable} must use the ${{VAR:?message}} form so a missing value fails the render"
)
def test_the_production_overlay_never_permits_remote_code() -> None:
text = (ROOT / "docker-compose.production.yml").read_text("utf-8")
assert 'MODELFORGE_ALLOW_REMOTE_CODE: "false"' in text
def test_the_base_compose_file_is_not_mistakable_for_production() -> None:
"""The base file is development-only; role secrets are still required and never embedded."""
text = (ROOT / "docker-compose.yml").read_text("utf-8")
assert "MODELFORGE_ENV: production" not in text
# --------------------------------------------------------------------------- images
@pytest.mark.parametrize("dockerfile", DOCKERFILES)
def test_every_image_carries_oci_identity_labels(dockerfile: str) -> None:
text = (ROOT / dockerfile).read_text("utf-8")
for label in (
"org.opencontainers.image.version",
"org.opencontainers.image.revision",
"org.opencontainers.image.created",
"org.opencontainers.image.source",
):
assert label in text, f"{dockerfile} does not declare {label}"
@pytest.mark.parametrize("dockerfile", DOCKERFILES)
def test_every_image_accepts_build_identity_arguments(dockerfile: str) -> None:
text = (ROOT / dockerfile).read_text("utf-8")
for argument in ("MODELFORGE_VERSION", "MODELFORGE_COMMIT", "MODELFORGE_BUILT_AT"):
assert f"ARG {argument}" in text, f"{dockerfile} does not accept {argument}"
@pytest.mark.parametrize("dockerfile", DOCKERFILES)
def test_every_image_declares_the_product_license(dockerfile: str) -> None:
text = (ROOT / dockerfile).read_text("utf-8")
assert 'org.opencontainers.image.licenses="AGPL-3.0-or-later"' in text
assert 'org.opencontainers.image.licenses="Proprietary"' not in text
def test_every_component_declares_the_product_license() -> None:
for relative_path in (
"backend/pyproject.toml",
"node-agent/pyproject.toml",
"runtime-worker/pyproject.toml",
):
metadata = tomllib.loads((ROOT / relative_path).read_text("utf-8"))
assert metadata["project"]["license"] == "AGPL-3.0-or-later"
package = json.loads((ROOT / "frontend/package.json").read_text("utf-8"))
lock = json.loads((ROOT / "frontend/package-lock.json").read_text("utf-8"))
assert package["license"] == "AGPL-3.0-or-later"
assert lock["packages"][""]["license"] == "AGPL-3.0-or-later"
def test_the_console_image_serves_a_build_not_a_development_server() -> None:
text = (ROOT / "frontend" / "Dockerfile").read_text("utf-8")
assert "npm run build" in text
assert "nginx-unprivileged" in text
assert "npm run dev" not in text
def test_the_node_agent_compose_projection_accepts_an_exact_release_image() -> None:
text = (ROOT / "docker-compose.node-agent.yml").read_text("utf-8")
assert "MODELFORGE_NODE_AGENT_IMAGE" in text
assert "image:" in text
assert "build:" in text, "the optional local source-build workflow must remain available"
assert "latest" not in next(
line for line in text.splitlines() if line.strip().startswith("image:")
)
def test_the_production_control_plane_never_falls_back_to_latest() -> None:
text = (ROOT / "docker-compose.production.yml").read_text("utf-8")
image_lines = [line for line in text.splitlines() if line.strip().startswith("image:")]
assert len(image_lines) == 3
assert all("latest" not in line for line in image_lines)
assert all("MODELFORGE_VERSION:?" in line for line in image_lines)
assert text.count("build:") == 2, "the explicit local source-build workflow must remain"
def test_the_release_builder_never_mutates_a_floating_latest_tag() -> None:
source = (ROOT / "scripts" / "release_build.py").read_text("utf-8")
assert 'f"{image}:latest"' not in source
def test_upgrade_runbook_starts_prebuilt_release_images() -> None:
runbook = (ROOT / "docs" / "UPGRADE.md").read_text("utf-8")
start_step = runbook.split("## 5. Start the new version", 1)[1].split("## 6. Verify", 1)[0]
version = (ROOT / "VERSION").read_text("utf-8").strip()
assert f"MODELFORGE_VERSION={version}" in start_step
assert f"MODELFORGE_API_IMAGE=modelforge-api:{version}" in start_step
assert f"MODELFORGE_WEB_IMAGE=modelforge-web:{version}" in start_step
assert "up -d --no-build api web" in start_step
assert "up -d --build api web" not in start_step
# --------------------------------------------------------------------------- release build
def test_the_release_package_carries_no_secret_material() -> None:
"""Whatever else changes, the tarball must never contain a credential or a database dump."""
from importlib.util import module_from_spec, spec_from_file_location
spec = spec_from_file_location("release_build", ROOT / "scripts" / "release_build.py")
assert spec and spec.loader
module = module_from_spec(spec)
sys.modules["release_build"] = module
spec.loader.exec_module(module)
# Match on what a file *is*, not on what its name mentions: docker-compose.backup.yml is a
# deployment manifest for taking backups, not a backup, and a substring rule that cannot tell
# those apart is a rule nobody will trust the next time it fires.
secret_suffixes = {".key", ".pem", ".p12", ".pfx", ".crt", ".sql", ".dump", ".tar", ".gz"}
secret_names = {".env", "secrets.yaml", "secrets.yml", "secrets.json", "credentials.json"}
for path in module.ARTIFACT_PATHS:
name = PurePosixPath(path).name.lower()
assert name not in secret_names, f"{path} is secret material and must not be packaged"
assert PurePosixPath(name).suffix not in secret_suffixes, (
f"{path} has a {PurePosixPath(name).suffix} extension and must not be packaged"
)
assert ".env" not in module.ARTIFACT_PATHS
assert ".env.example" in module.ARTIFACT_PATHS
def test_the_release_package_carries_no_model_weights() -> None:
from importlib.util import module_from_spec, spec_from_file_location
spec = spec_from_file_location("release_build2", ROOT / "scripts" / "release_build.py")
assert spec and spec.loader
module = module_from_spec(spec)
spec.loader.exec_module(module)
for path in module.ARTIFACT_PATHS:
assert "artifact" not in path.lower()
assert "model-registry" not in path.lower()
def test_the_release_package_includes_what_an_operator_needs_to_install() -> None:
from importlib.util import module_from_spec, spec_from_file_location
spec = spec_from_file_location("release_build3", ROOT / "scripts" / "release_build.py")
assert spec and spec.loader
module = module_from_spec(spec)
spec.loader.exec_module(module)
paths = set(module.ARTIFACT_PATHS)
for required in (
"docker-compose.yml",
"docker-compose.production.yml",
".env.example",
"VERSION",
"docs",
"config",
"scripts/bootstrap.py",
"scripts/preflight.py",
):
assert required in paths, f"a release without {required} cannot be installed from"
def test_the_version_file_matches_the_product_version() -> None:
assert (ROOT / "VERSION").read_text("utf-8").strip() == PRODUCT_VERSION
# --------------------------------------------------------------------------- migration targeting
def test_the_migration_environment_honours_an_explicitly_supplied_url() -> None:
"""A migration must run where the caller aimed it, not where the settings point.
env.py used to overwrite `sqlalchemy.url` with the settings default unconditionally, so both
`-x db_url=...` and a programmatic `set_main_option` were silently discarded. A bootstrap or
upgrade rehearsal aimed at an isolated copy would have migrated the deployment's own database
while reporting success against the copy.
"""
text = (ROOT / "backend" / "alembic" / "env.py").read_text("utf-8")
assert "get_x_argument" in text, "-x db_url must be honoured"
assert 'config.get_main_option("sqlalchemy.url", None)' in text, (
"a programmatically supplied URL must be honoured"
)
unconditional = 'config.set_main_option("sqlalchemy.url", get_settings().database_url)'
assert unconditional not in text, (
"the settings URL must be a fallback, never an unconditional override"
)
def test_the_migration_configuration_hardcodes_no_database_url() -> None:
"""A URL baked into alembic.ini is a URL an operator can migrate the wrong database with."""
for line in (ROOT / "backend" / "alembic.ini").read_text("utf-8").splitlines():
if line.strip().startswith("sqlalchemy.url"):
_, _, value = line.partition("=")
assert not value.strip(), f"alembic.ini pins a database URL: {value.strip()!r}"
# --------------------------------------------------------------------------- clean checkout
def _tracked_files() -> set[str]:
completed = subprocess.run(
["git", "ls-files"], # noqa: S607 - git resolves from PATH, as it must
capture_output=True,
text=True,
encoding="utf-8",
errors="replace",
cwd=ROOT,
check=True,
)
return set(completed.stdout.split())
def test_every_manifest_the_control_plane_needs_at_startup_is_tracked() -> None:
"""A clean clone must be able to start. Ours could not.
`.gitignore` carried a bare `models/` rule intended for model weights. It also matched
`config/models/`, so the candidate registry manifest was silently excluded from the repository,
and the API failed at startup with FileNotFoundError whenever registry seeding was enabled. The
fresh-install rehearsal only passed because it mounted the developer's untracked copy — which is
exactly the developer-state dependency a release is supposed to rule out.
"""
tracked = _tracked_files()
required = [
"config/models/initial-candidates.yaml",
"config/policies/defaults.yaml",
]
missing = [path for path in required if path not in tracked]
assert not missing, f"a clean checkout would be missing: {missing}"
@pytest.mark.parametrize(
"directory",
["config", "backend/src", "backend/tests", "node-agent/src", "runtime-worker/src", "docs"],
)
def test_no_source_file_is_excluded_from_the_repository(directory: str) -> None:
"""Whatever these directories grow, none of it may be invisible to a clean clone.
Two over-broad ignore rules each swallowed something that was reported as delivered. `models/`
hid the candidate registry manifest the control plane needs at startup, so a clean clone could
not start. `*credential*` hid backend/tests/test_credential_security_m16.py — the whole M16
credential security suite, 35 tests — and its documentation. Both existed on the machine that
wrote them and nowhere else.
"""
tracked = _tracked_files()
suffixes = {".py", ".ts", ".tsx", ".md", ".yaml", ".yml", ".json", ".inc", ".template"}
on_disk = {
str(path.relative_to(ROOT)).replace("\\", "/")
for path in (ROOT / directory).rglob("*")
if path.is_file()
and path.suffix in suffixes
and not any(
part in {"__pycache__", "node_modules", ".pytest_cache", ".ruff_cache", ".mypy_cache"}
or part.endswith(".egg-info")
for part in path.parts
)
}
untracked = sorted(on_disk - tracked)
assert not untracked, (
f"these files exist locally but not in the repository, so a clean clone would not have "
f"them: {untracked}"
)
# --------------------------------------------------------------------------- readiness cost
def test_the_manifest_registry_reads_each_manifest_once() -> None:
"""Readiness must not re-validate the whole manifest set on every probe.
M16 measured a p50 of 638 ms on /api/v1/health/ready under load and reported it as unexplained.
The cause was here: one readiness pass performed 77 YAML reads across 14 files — one file ten
times — because every accessor re-read from disk and projects() called capabilities() inside a
nested loop. Manifests are read-only configuration that cannot change without restarting the
process, so they are parsed once.
"""
from modelforge_api.services.manifest_registry import ManifestRegistry
reads: list[str] = []
original = ManifestRegistry._read_yaml
def counting(path: Path) -> dict[str, object]:
reads.append(str(path))
return original(path)
registry = ManifestRegistry(ROOT / "config")
ManifestRegistry._read_yaml = staticmethod(counting) # type: ignore[method-assign]
try:
for _ in range(3):
registry.capabilities()
registry.projects()
registry.candidates()
registry.benchmarks()
registry.policies()
finally:
ManifestRegistry._read_yaml = staticmethod(original) # type: ignore[method-assign]
assert len(reads) == len(set(reads)), (
f"a manifest was read more than once: {sorted(reads)}"
)
# --------------------------------------------------------------------------- content security
def _csp_template() -> str:
return (ROOT / "frontend" / "security-headers.inc.template").read_text("utf-8")
def test_the_console_declares_a_content_security_policy() -> None:
"""M16 shipped six security headers and no CSP, and said so. v1 closes that."""
assert "Content-Security-Policy" in _csp_template()
@pytest.mark.parametrize(
"directive",
[
"default-src 'none'",
"script-src 'self'",
"base-uri 'none'",
"frame-ancestors 'none'",
"object-src 'none'",
],
)
def test_the_policy_is_restrictive_where_it_can_be(directive: str) -> None:
assert directive in _csp_template()
def test_the_policy_never_allows_inline_or_evaluated_script() -> None:
"""The two directives that would make the rest of the policy decorative."""
policy = _csp_template()
script_directive = policy.split("script-src", 1)[1].split(";", 1)[0]
assert "unsafe-inline" not in script_directive
assert "unsafe-eval" not in script_directive
def test_inline_style_is_permitted_only_as_an_attribute() -> None:
"""The console sets five dynamic widths through style attributes and nothing else.
`style-src-attr 'unsafe-inline'` allows exactly those while still blocking an injected
`<style>` element, which a blanket `style-src 'unsafe-inline'` would not.
"""
policy = _csp_template()
style_directive = policy.split("style-src ", 1)[1].split(";", 1)[0]
assert "unsafe-inline" not in style_directive
assert "style-src-attr 'unsafe-inline'" in policy
def test_the_built_console_contains_nothing_the_policy_forbids() -> None:
"""A policy the application violates is a policy someone will switch off.
Checked against the built bundle rather than asserted: no inline <script>, no inline <style>,
and no javascript: URL. The favicon is a data: URI, which img-src permits.
"""
index = ROOT / "frontend" / "dist" / "index.html"
if not index.is_file():
pytest.skip("the console has not been built in this working tree")
html = index.read_text("utf-8")
assert not re.search(r"<script(?![^>]*\ssrc=)[^>]*>", html), "an inline <script> would be blocked"
assert "<style" not in html, "an inline <style> element would be blocked"
assert "javascript:" not in html
for match in re.findall(r'src="([^"]+)"|href="([^"]+)"', html):
value = match[0] or match[1]
assert value.startswith(("/", "./", "data:")), (
f"{value} is an external reference the policy does not allow"
)
# --------------------------------------------------------------------------- documentation
def test_the_operator_documentation_a_release_promises_is_present() -> None:
"""An operator should not need a milestone report to install or run ModelForge."""
required = [
"README.md",
"CHANGELOG.md",
"docs/INSTALLATION.md",
"docs/FIRST_RUN.md",
"docs/UPGRADE.md",
"docs/CONFIGURATION.md",
"docs/COMPATIBILITY.md",
"docs/NODE_AGENT.md",
"docs/UNRAID_DEPLOYMENT.md",
"docs/CAPABILITIES.md",
"docs/PROJECT_INTEGRATION.md",
"docs/OPERATIONS.md",
"docs/TROUBLESHOOTING.md",
"docs/SECURITY.md",
f"docs/RELEASE_NOTES_v{PRODUCT_VERSION}.md",
]
missing = [path for path in required if not (ROOT / path).is_file()]
assert not missing, f"missing operator documentation: {missing}"
def test_no_documentation_link_is_broken() -> None:
"""Documentation that points at a file which is not there is worse than none."""
broken: list[str] = []
documents = list(ROOT.glob("*.md")) + list((ROOT / "docs").rglob("*.md"))
for document in documents:
for target in re.findall(r"\]\(([^)#]+?)(?:#[^)]*)?\)", document.read_text("utf-8")):
if target.startswith(("http://", "https://", "mailto:")):
continue
if not (document.parent / target).resolve().exists():
broken.append(f"{document.relative_to(ROOT).as_posix()} -> {target}")
assert not broken, f"broken documentation links: {broken}"
def test_the_release_notes_name_this_version() -> None:
notes = (ROOT / "docs" / f"RELEASE_NOTES_v{PRODUCT_VERSION}.md").read_text("utf-8")
assert PRODUCT_VERSION in notes
assert "Known limitations" in notes, "a release that lists no limitations has not looked"
def test_the_release_archive_is_built_reproducibly() -> None:
"""Two builds of the same tree must produce the same bytes, or a published checksum means little.
Two things break this and both were found by building twice and comparing rather than by
reasoning about it: tar records uid, gid and mtime per entry, and gzip writes the current time
into its own header.
"""
source = (ROOT / "scripts" / "release_build.py").read_text("utf-8")
assert "mtime=0" in source, "the gzip header must not carry a build timestamp"
assert "info.mtime = 0" in source, "entry mtimes must be normalised"
assert 'info.uname = info.gname = "root"' in source, "entry ownership must be normalised"
assert "recursive=False" in source, "tar.add must not recurse and duplicate every entry"
def test_build_identity_is_never_overridden_at_deployment_time() -> None:
"""An image knows what it was built from; whoever runs `up` does not.
The Compose files used to pass MODELFORGE_BUILD_COMMIT and MODELFORGE_BUILD_TIMESTAMP as runtime
environment, which override the image's own values. The release candidate reported the commit
of whatever was checked out when it was started — not the commit its image was built from — and
a null build time, because the unset variable blanked what the image already carried.
"""
for name in ("docker-compose.yml", "docker-compose.production.yml"):
text = (ROOT / name).read_text("utf-8")
for variable in ("MODELFORGE_BUILD_COMMIT", "MODELFORGE_BUILD_TIMESTAMP"):
for line in text.splitlines():
stripped = line.strip()
if stripped.startswith(f"{variable}:"):
raise AssertionError(
f"{name} sets {variable} at runtime; build identity must come from the "
f"image's own build arguments"
)
# The build arguments themselves must still be accepted.
assert "MODELFORGE_COMMIT:" in text, f"{name} must pass the commit as a build argument"
# --------------------------------------------------------------------------- reproducible install
@pytest.mark.parametrize(
"manifest",
["backend/pyproject.toml", "node-agent/pyproject.toml", "runtime-worker/pyproject.toml"],
)
def test_every_dependency_is_pinned_to_an_exact_version(manifest: str) -> None:
"""A release must resolve to the same versions on any day, on any machine.
It did not. `sqlalchemy>=2.0,<3` resolved to 2.0.35 in the development environment and 2.0.52
in a clean install — and 2.0.52 narrowed `Session.execute`'s return type, so the type check
passed locally and failed in CI on identical code. The gate depended on when you ran it.
"""
import tomllib
data = tomllib.loads((ROOT / manifest).read_text("utf-8"))
project = data["project"]
specs = list(project.get("dependencies", []))
for extra in project.get("optional-dependencies", {}).values():
specs.extend(extra)
unpinned = [
spec
for spec in specs
if "==" not in spec or any(operator in spec for operator in (">=", "<=", ">", "<", "~="))
]
assert not unpinned, f"{manifest} declares unpinned dependencies: {unpinned}"
def test_the_type_checker_version_is_pinned_everywhere_it_runs() -> None:
"""The gate must not change its answer because a tool released a new version overnight."""
import tomllib
seen: set[str] = set()
for manifest in (
"backend/pyproject.toml",
"node-agent/pyproject.toml",
"runtime-worker/pyproject.toml",
):
data = tomllib.loads((ROOT / manifest).read_text("utf-8"))
for extra in data["project"].get("optional-dependencies", {}).values():
for spec in extra:
if spec.startswith(("mypy", "ruff")):
seen.add(spec)
mypy_pins = {spec for spec in seen if spec.startswith("mypy")}
ruff_pins = {spec for spec in seen if spec.startswith("ruff")}
assert len(mypy_pins) == 1, f"components disagree on the mypy version: {sorted(mypy_pins)}"
assert len(ruff_pins) == 1, f"components disagree on the ruff version: {sorted(ruff_pins)}"
def test_the_example_configuration_uses_container_paths_not_host_paths() -> None:
"""The defaults are paths inside a Linux container, whatever platform generated the file.
`str(Path("/data/backups"))` yields `\\data\backups` on Windows, and the committed
.env.example shipped exactly that — telling an operator to point a Linux container at a Windows
path, and making the generated files differ by platform so the freshness check passed on one and
failed on the other.
"""
text = (ROOT / ".env.example").read_text("utf-8")
offenders = [
line
for line in text.splitlines()
if line.startswith("MODELFORGE_") and "\\" in line.partition("=")[2]
]
assert not offenders, f"host-style paths in .env.example: {offenders}"
def test_no_test_silently_requires_a_live_database() -> None:
"""A suite that passes only where a database happens to be running is not a suite.
test_api.py called an endpoint without overriding `get_session`, so it used the real engine and
quietly required PostgreSQL on localhost. It passed on the development machine and failed
everywhere else.
"""
source = (ROOT / "backend" / "tests" / "test_api.py").read_text("utf-8")
# Every test that reaches a session-backed route must install an override first.
uses_client = source.count("client.get(")
overrides = source.count("app.dependency_overrides[get_session]")
assert overrides >= 1, "test_api.py must override get_session rather than use the real engine"
# A DSN, not the word: the docstring above mentions PostgreSQL precisely because that is what
# went wrong, and a rule that cannot tell prose from a connection string is a rule that will be
# switched off the first time it fires wrongly.
assert not re.search(r"postgresql(\+\w+)?://", source), (
"no test may name a PostgreSQL DSN; the suite runs against SQLite"
)
assert uses_client > 0