Initial public ModelForge release
This commit is contained in:
Jens
@@ -0,0 +1,274 @@
|
||||
"""Generate `.env.example` and `docs/CONFIGURATION.md` from the typed settings.
|
||||
|
||||
Hand-maintained configuration documentation drifts, and the operator discovers the drift when a
|
||||
production deployment does something the manual said it would not. Both files are generated from
|
||||
`Settings` joined with `SETTING_DOCS`, and a test fails when the committed files no longer match —
|
||||
so adding a setting without documenting it breaks the build rather than shipping quietly.
|
||||
|
||||
python scripts/generate_configuration_docs.py # write both files
|
||||
python scripts/generate_configuration_docs.py --check # fail if they are stale
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from pathlib import Path, PurePath
|
||||
from types import UnionType
|
||||
from typing import Any, Literal, Union, get_args, get_origin
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "backend" / "src"))
|
||||
|
||||
from modelforge_api.domain.configuration_reference import ( # noqa: E402
|
||||
DEPLOYMENT_DOCS,
|
||||
SETTING_DOCS,
|
||||
Sensitivity,
|
||||
)
|
||||
from modelforge_api.domain.release import ( # noqa: E402
|
||||
MINIMUM_POSTGRES_MAJOR,
|
||||
PRODUCT_NAME,
|
||||
PRODUCT_VERSION,
|
||||
)
|
||||
from modelforge_api.settings import Settings # noqa: E402
|
||||
|
||||
ENV_PREFIX = "MODELFORGE_"
|
||||
|
||||
#: Values a generated example must never contain. The example is committed to the repository, so a
|
||||
#: real secret placed here would be published with it.
|
||||
EXAMPLE_SECRET_PLACEHOLDER = ""
|
||||
|
||||
#: Settings a container sets for itself. Listing them in the example invites an operator to override
|
||||
#: a path that only makes sense inside the image.
|
||||
CONTAINER_MANAGED = frozenset(
|
||||
{
|
||||
"config_root",
|
||||
"alembic_directory",
|
||||
"build_commit",
|
||||
"build_timestamp",
|
||||
"build_image_digest",
|
||||
"agent_protocol_version",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _default(name: str) -> str:
|
||||
field = Settings.model_fields[name]
|
||||
default: Any = field.default
|
||||
if default is None:
|
||||
return ""
|
||||
if isinstance(default, bool):
|
||||
return "true" if default else "false"
|
||||
if isinstance(default, PurePath):
|
||||
# These are paths *inside a Linux container*, so they must render with forward slashes
|
||||
# whatever platform generates the file. `str(Path("/data/backups"))` gives `\dataackups`
|
||||
# on Windows, and the committed .env.example shipped exactly that — telling operators to
|
||||
# point a Linux container at a Windows path. It also made the generated files differ by
|
||||
# platform, so the freshness check passed on one and failed on the other.
|
||||
return default.as_posix()
|
||||
if callable(default): # pragma: no cover - default_factory fields
|
||||
return ""
|
||||
return str(default)
|
||||
|
||||
|
||||
def _render_annotation(annotation: Any) -> str:
|
||||
"""A stable, human-readable name for a setting's type.
|
||||
|
||||
Deliberately not `str(annotation)`: Python 3.13 moved Path into `pathlib._local`, so the same
|
||||
field rendered as `pathlib.Path` on 3.12 and `pathlib._local.Path` on 3.13. The generated
|
||||
reference then differed by interpreter version and the freshness check failed on whichever
|
||||
machine had not produced it.
|
||||
"""
|
||||
|
||||
simple = {
|
||||
"str": "string",
|
||||
"int": "integer",
|
||||
"float": "number",
|
||||
"bool": "boolean",
|
||||
"Path": "path",
|
||||
"PosixPath": "path",
|
||||
"WindowsPath": "path",
|
||||
"SecretStr": "secret",
|
||||
"NoneType": "none",
|
||||
}
|
||||
origin = get_origin(annotation)
|
||||
if origin is Literal:
|
||||
# Comma-separated, not pipe-separated: these land in a markdown table cell, and a pipe
|
||||
# there silently splits the row into extra columns.
|
||||
return ", ".join(f"`{value}`" for value in get_args(annotation))
|
||||
if origin in (Union, UnionType):
|
||||
parts = [
|
||||
_render_annotation(argument)
|
||||
for argument in get_args(annotation)
|
||||
if argument is not type(None)
|
||||
]
|
||||
rendered = " or ".join(dict.fromkeys(parts))
|
||||
return f"{rendered}, optional"
|
||||
if isinstance(annotation, type):
|
||||
return simple.get(annotation.__name__, annotation.__name__)
|
||||
return simple.get(str(annotation), str(annotation))
|
||||
|
||||
|
||||
def _type_name(name: str) -> str:
|
||||
return _render_annotation(Settings.model_fields[name].annotation)
|
||||
|
||||
|
||||
def render_env_example() -> str:
|
||||
lines = [
|
||||
f"# {PRODUCT_NAME} {PRODUCT_VERSION} — configuration example",
|
||||
"#",
|
||||
"# Generated by scripts/generate_configuration_docs.py. Do not edit by hand.",
|
||||
"# Copy to .env and fill in the values marked REQUIRED. See docs/CONFIGURATION.md.",
|
||||
"#",
|
||||
"# Secrets are intentionally empty here. This file is committed to the repository, so a",
|
||||
"# real value placed in it would be published with the release.",
|
||||
"",
|
||||
]
|
||||
for name, doc in SETTING_DOCS.items():
|
||||
if name in CONTAINER_MANAGED:
|
||||
continue
|
||||
marker = " (REQUIRED in production)" if doc.required_in_production else ""
|
||||
secret = doc.sensitivity is Sensitivity.SECRET
|
||||
lines.append(f"# {doc.description}{marker}")
|
||||
value = EXAMPLE_SECRET_PLACEHOLDER if secret else _default(name)
|
||||
lines.append(f"{ENV_PREFIX}{name.upper()}={value}")
|
||||
lines.append("")
|
||||
|
||||
lines += [
|
||||
"# --------------------------------------------------------------------------",
|
||||
"# Deployment variables. Read by Compose, the Node Agent and the Runtime Worker",
|
||||
"# rather than by the control-plane process - an operator still has to set them.",
|
||||
"# --------------------------------------------------------------------------",
|
||||
"",
|
||||
]
|
||||
for name, doc in DEPLOYMENT_DOCS.items():
|
||||
marker = " (REQUIRED in production)" if doc.required_in_production else ""
|
||||
lines.append(f"# {doc.description}{marker}")
|
||||
lines.append(f"{name}=")
|
||||
lines.append("")
|
||||
return "\n".join(lines).rstrip("\n") + "\n"
|
||||
|
||||
|
||||
def render_configuration_doc() -> str:
|
||||
required = [name for name, doc in SETTING_DOCS.items() if doc.required_in_production]
|
||||
secrets = [
|
||||
name for name, doc in SETTING_DOCS.items() if doc.sensitivity is Sensitivity.SECRET
|
||||
]
|
||||
lines = [
|
||||
"# Configuration reference",
|
||||
"",
|
||||
f"Generated from the typed settings by `scripts/generate_configuration_docs.py` for "
|
||||
f"{PRODUCT_NAME} {PRODUCT_VERSION}. Every setting the control plane reads appears here; a "
|
||||
"setting added without documentation fails the build.",
|
||||
"",
|
||||
"All settings are environment variables with the `MODELFORGE_` prefix, read from the "
|
||||
"process environment or from `.env`.",
|
||||
"",
|
||||
"## Required in production",
|
||||
"",
|
||||
"`MODELFORGE_ENV=production` turns on fail-closed startup validation. With it set, the "
|
||||
"control plane refuses to start unless each of these is present and sound:",
|
||||
"",
|
||||
]
|
||||
for name in required:
|
||||
lines.append(f"- `{ENV_PREFIX}{name.upper()}` — {SETTING_DOCS[name].description}")
|
||||
lines += [
|
||||
"",
|
||||
"Production additionally refuses: a well-known development database password, a wildcard "
|
||||
"CORS origin, remote model code execution, an unwritable storage root, an unsupported "
|
||||
f"schema revision, a PostgreSQL major below {MINIMUM_POSTGRES_MAJOR}, and three policy "
|
||||
"combinations that cannot all hold at once.",
|
||||
"",
|
||||
"## Generating secrets",
|
||||
"",
|
||||
"ModelForge never mints its own credentials — a platform that generates its own admin "
|
||||
"secret has no way to tell you it did. Generate them yourself and store them outside the "
|
||||
"deployment:",
|
||||
"",
|
||||
"```bash",
|
||||
"# Operator API key (at least 32 characters)",
|
||||
"python -c \"import secrets; print(secrets.token_urlsafe(48))\"",
|
||||
"",
|
||||
"# Backup encryption key (base64 AES-256)",
|
||||
"python -c \"import base64, os; print(base64.b64encode(os.urandom(32)).decode())\"",
|
||||
"",
|
||||
"# Database password",
|
||||
"python -c \"import secrets; print(secrets.token_urlsafe(32))\"",
|
||||
"```",
|
||||
"",
|
||||
"Losing the backup encryption key makes every existing backup unrecoverable. It is the one "
|
||||
"value that must be stored somewhere the deployment cannot take down with it.",
|
||||
"",
|
||||
"## Sensitivity",
|
||||
"",
|
||||
f"{len(secrets)} settings are credentials. They are never logged, never written to a "
|
||||
"release artefact and never echoed in an error response:",
|
||||
"",
|
||||
]
|
||||
for name in secrets:
|
||||
lines.append(f"- `{ENV_PREFIX}{name.upper()}`")
|
||||
lines += ["", "## Every setting", "", "| Variable | Type | Default | Required | Sensitivity | Description |", "| --- | --- | --- | --- | --- | --- |"]
|
||||
for name, doc in SETTING_DOCS.items():
|
||||
default = _default(name)
|
||||
rendered = f"`{default}`" if default else "—"
|
||||
if doc.sensitivity is Sensitivity.SECRET:
|
||||
rendered = "—"
|
||||
lines.append(
|
||||
f"| `{ENV_PREFIX}{name.upper()}` | {_type_name(name)} | {rendered} | "
|
||||
f"{'yes' if doc.required_in_production else 'no'} | {doc.sensitivity} | "
|
||||
f"{doc.description} |"
|
||||
)
|
||||
lines += [
|
||||
"",
|
||||
"## Deployment variables",
|
||||
"",
|
||||
"Read by Compose, the Node Agent and the Runtime Worker rather than by the control-plane "
|
||||
"process. An operator still has to set them, so they are documented here too.",
|
||||
"",
|
||||
"| Variable | Required | Sensitivity | Description |",
|
||||
"| --- | --- | --- | --- |",
|
||||
]
|
||||
for name, doc in DEPLOYMENT_DOCS.items():
|
||||
lines.append(
|
||||
f"| `{name}` | {'yes' if doc.required_in_production else 'no'} | "
|
||||
f"{doc.sensitivity} | {doc.description} |"
|
||||
)
|
||||
lines += [
|
||||
"",
|
||||
"## Restarts",
|
||||
"",
|
||||
"Every setting is read at process start. Changing any of them requires restarting the "
|
||||
"control plane; none is re-read from the environment while the process is running.",
|
||||
"",
|
||||
]
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--check", action="store_true")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
targets = {
|
||||
ROOT / ".env.example": render_env_example(),
|
||||
ROOT / "docs" / "CONFIGURATION.md": render_configuration_doc(),
|
||||
}
|
||||
stale: list[str] = []
|
||||
for path, content in targets.items():
|
||||
current = path.read_text("utf-8") if path.is_file() else None
|
||||
if current == content:
|
||||
print(f" current {path.relative_to(ROOT)}")
|
||||
continue
|
||||
stale.append(str(path.relative_to(ROOT)))
|
||||
if not args.check:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(content, encoding="utf-8")
|
||||
print(f" written {path.relative_to(ROOT)}")
|
||||
if args.check and stale:
|
||||
print(f"stale: {', '.join(stale)} — run scripts/generate_configuration_docs.py")
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user