127 lines
4.8 KiB
Python
127 lines
4.8 KiB
Python
#!/usr/bin/env python3
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
from statistics import mean
|
|
from typing import Any
|
|
from urllib.parse import urlparse
|
|
|
|
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings")
|
|
|
|
import django
|
|
|
|
django.setup()
|
|
|
|
from django.conf import settings # noqa: E402
|
|
from django.core.management import call_command # noqa: E402
|
|
|
|
from scripts.benchmark import DEFAULT_DATASET, run_benchmark_report # noqa: E402
|
|
|
|
SAFE_DATABASE_PREFIX = "vacatureradar_bench_"
|
|
SAFE_PUBLIC_HOSTS = {"", "benchmark.invalid", "localhost", "127.0.0.1"}
|
|
|
|
|
|
class UnsafeBenchmarkEnvironment(RuntimeError):
|
|
pass
|
|
|
|
|
|
def validate_benchmark_identity(database: dict[str, Any], public_base_url: str) -> None:
|
|
engine = str(database.get("ENGINE", ""))
|
|
name = str(database.get("NAME", ""))
|
|
host = str(database.get("HOST", ""))
|
|
public_host = (urlparse(public_base_url).hostname or "").casefold()
|
|
if engine != "django.db.backends.postgresql":
|
|
raise UnsafeBenchmarkEnvironment("De importbenchmark vereist PostgreSQL.")
|
|
if not name.startswith(SAFE_DATABASE_PREFIX):
|
|
raise UnsafeBenchmarkEnvironment(
|
|
f"Onveilige databasenaam {name!r}; verwacht prefix {SAFE_DATABASE_PREFIX!r}."
|
|
)
|
|
if host not in {"127.0.0.1", "localhost", "postgres"}:
|
|
raise UnsafeBenchmarkEnvironment(f"Onveilige benchmarkdatabasehost {host!r}.")
|
|
if public_host not in SAFE_PUBLIC_HOSTS:
|
|
raise UnsafeBenchmarkEnvironment(
|
|
f"Publieke host {public_host!r} is niet toegestaan voor een benchmark."
|
|
)
|
|
|
|
|
|
def _summary(reports: list[dict[str, Any]]) -> dict[str, float]:
|
|
performance = [report["performance"] for report in reports]
|
|
imports = [item["import"] for item in performance]
|
|
return {
|
|
"query_count_mean": round(mean(item["query_count"] for item in imports), 3),
|
|
"query_count_total_mean": round(mean(item["query_count"] for item in performance), 3),
|
|
"duration_ms_mean": round(mean(item["duration_ms_total"] for item in imports), 3),
|
|
"p50_ms_mean": round(mean(item["p50_ms"] for item in imports), 3),
|
|
"p95_ms_mean": round(mean(item["p95_ms"] for item in imports), 3),
|
|
"throughput_per_second": round(
|
|
mean(
|
|
report["jobs_processed"] / (item["duration_ms_total"] / 1000)
|
|
for report, item in zip(reports, imports, strict=True)
|
|
),
|
|
3,
|
|
),
|
|
"memory_peak_mb_mean": round(
|
|
mean(item["performance"]["memory_peak_mb"] for item in performance), 3
|
|
),
|
|
}
|
|
|
|
|
|
def run_isolated_benchmark(
|
|
*, dataset: Path, jobs: int, iterations: int, output: Path
|
|
) -> dict[str, Any]:
|
|
validate_benchmark_identity(settings.DATABASES["default"], settings.PUBLIC_BASE_URL)
|
|
if iterations < 1:
|
|
raise ValueError("iterations moet minstens 1 zijn.")
|
|
call_command("migrate", interactive=False, verbosity=0)
|
|
call_command("flush", interactive=False, verbosity=0)
|
|
warmup = run_benchmark_report(dataset, quick=True, jobs=jobs)
|
|
measured: list[dict[str, Any]] = []
|
|
for _ in range(iterations):
|
|
call_command("flush", interactive=False, verbosity=0)
|
|
measured.append(run_benchmark_report(dataset, quick=True, jobs=jobs))
|
|
report = {
|
|
"schema_version": "1.0",
|
|
"database_identity": {
|
|
"engine": settings.DATABASES["default"]["ENGINE"],
|
|
"name_prefix": SAFE_DATABASE_PREFIX,
|
|
"host": settings.DATABASES["default"]["HOST"],
|
|
},
|
|
"configuration": {
|
|
"jobs": jobs,
|
|
"iterations": iterations,
|
|
"warmup_runs": 1,
|
|
"concurrency": 1,
|
|
"dataset": str(dataset),
|
|
},
|
|
"warmup_status": warmup["status"],
|
|
"summary": _summary(measured),
|
|
"iterations": measured,
|
|
"status": "passed" if all(item["status"] == "passed" for item in measured) else "failed",
|
|
}
|
|
output.parent.mkdir(parents=True, exist_ok=True)
|
|
output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
return report
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description="Fail-closed geïsoleerde importbenchmark")
|
|
parser.add_argument("--dataset", type=Path, default=DEFAULT_DATASET)
|
|
parser.add_argument("--jobs", type=int, default=250)
|
|
parser.add_argument("--iterations", type=int, default=3)
|
|
parser.add_argument(
|
|
"--output", type=Path, default=Path("artifacts/import-performance-report.json")
|
|
)
|
|
args = parser.parse_args()
|
|
report = run_isolated_benchmark(
|
|
dataset=args.dataset, jobs=args.jobs, iterations=args.iterations, output=args.output
|
|
)
|
|
print(json.dumps(report, indent=2, sort_keys=True))
|
|
return 0 if report["status"] == "passed" else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|