Jens
@@ -0,0 +1,349 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from django.utils import timezone
|
||||
|
||||
from apps.sources.models import Source, SourcePolicyReview, SourceRun
|
||||
|
||||
from .policy import create_policy_review
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SourceHealth:
|
||||
source_id: int
|
||||
source_name: str
|
||||
source_type: str
|
||||
source_status: str
|
||||
monitored_runs: int
|
||||
success_ratio: float
|
||||
avg_latency_ms: float | None
|
||||
error_counts: dict[str, int]
|
||||
http_status_counts: dict[str, int]
|
||||
extracted_count: int
|
||||
updated_count: int
|
||||
duplicate_count: int
|
||||
last_parser: str | None
|
||||
last_parser_warnings: int
|
||||
last_health_action: str | None
|
||||
last_health_reason: str | None
|
||||
|
||||
|
||||
SOURCE_HEALTH_RUN_WINDOW = 30
|
||||
SOURCE_HEALTH_MIN_RUNS_FOR_TEMPORARY_QUARANTINE = 6
|
||||
SOURCE_HEALTH_TEMPORARY_ERROR_RATIO = 0.7
|
||||
SOURCE_HEALTH_PARSER_DRIFT_SUCCESS_STREAK = 3
|
||||
SOURCE_HEALTH_PARSER_MIN_WARNINGS = 2
|
||||
SOURCE_HEALTH_PARSER_DRIFT_EXTRACTED_MAX = 0
|
||||
SOURCE_HEALTH_RECOVERY_COOLDOWN_HOURS = 12
|
||||
|
||||
TEMPORARY_ERROR_CATEGORIES = {
|
||||
"timeout",
|
||||
"rate_limited",
|
||||
"FetchTimeoutError",
|
||||
"FetchError",
|
||||
"unexpected",
|
||||
"NetworkError",
|
||||
}
|
||||
POLICY_ERROR_CATEGORIES = {"policy"}
|
||||
|
||||
|
||||
def _normalize_metrics(raw: object) -> dict[str, object]:
|
||||
if isinstance(raw, dict):
|
||||
return raw
|
||||
return {}
|
||||
|
||||
|
||||
def _warnings_from_run(run: SourceRun) -> list[str]:
|
||||
metrics = _normalize_metrics(run.metrics)
|
||||
warnings = metrics.get("warnings", [])
|
||||
if not isinstance(warnings, list):
|
||||
return []
|
||||
return [str(item) for item in warnings if isinstance(item, str)]
|
||||
|
||||
|
||||
def _parser_from_run(run: SourceRun) -> str | None:
|
||||
metrics = _normalize_metrics(run.metrics)
|
||||
parser = metrics.get("parser")
|
||||
if isinstance(parser, str) and parser:
|
||||
return parser
|
||||
return None
|
||||
|
||||
|
||||
def _int(value: object) -> int:
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
|
||||
|
||||
def _float(value: object) -> float:
|
||||
try:
|
||||
return float(value)
|
||||
except (TypeError, ValueError):
|
||||
return 0.0
|
||||
|
||||
|
||||
def _to_iso(dt: datetime | None) -> str | None:
|
||||
if dt is None:
|
||||
return None
|
||||
return dt.isoformat()
|
||||
|
||||
|
||||
def _from_iso(value: str | None) -> datetime | None:
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
dt = datetime.fromisoformat(value)
|
||||
except ValueError:
|
||||
return None
|
||||
if timezone.is_naive(dt):
|
||||
return timezone.make_aware(dt)
|
||||
return dt
|
||||
|
||||
|
||||
def _health_metadata(source: Source) -> dict[str, object]:
|
||||
metadata = source.metadata
|
||||
if not isinstance(metadata, dict):
|
||||
return {}
|
||||
health = metadata.get("source_health")
|
||||
return health if isinstance(health, dict) else {}
|
||||
|
||||
|
||||
def _set_health_metadata(source: Source, health_data: dict[str, object]) -> None:
|
||||
metadata = source.metadata
|
||||
if not isinstance(metadata, dict):
|
||||
metadata = {}
|
||||
metadata["source_health"] = health_data
|
||||
source.metadata = metadata
|
||||
source.save(update_fields=["metadata", "updated_at"])
|
||||
|
||||
|
||||
def _run_counts(runs: Iterable[SourceRun]) -> tuple[dict[str, int], dict[str, int]]:
|
||||
error_counts: dict[str, int] = {}
|
||||
http_status_counts: dict[str, int] = {}
|
||||
for run in runs:
|
||||
if run.status != SourceRun.Status.SUCCESS:
|
||||
category = run.error_category or "unknown"
|
||||
error_counts[category] = error_counts.get(category, 0) + 1
|
||||
if run.http_status:
|
||||
status = str(run.http_status)
|
||||
http_status_counts[status] = http_status_counts.get(status, 0) + 1
|
||||
return error_counts, http_status_counts
|
||||
|
||||
|
||||
def _last_parser_output(runs: list[SourceRun]) -> tuple[str | None, int]:
|
||||
for run in runs:
|
||||
if run.status != SourceRun.Status.SUCCESS:
|
||||
continue
|
||||
parser = _parser_from_run(run)
|
||||
if parser:
|
||||
warnings = _warnings_from_run(run)
|
||||
return parser, len(warnings)
|
||||
return None, 0
|
||||
|
||||
|
||||
def _latency_ms(runs: list[SourceRun]) -> float | None:
|
||||
latencies = []
|
||||
for run in runs:
|
||||
if run.finished_at is None or run.started_at is None:
|
||||
continue
|
||||
latencies.append(max(0.0, (run.finished_at - run.started_at).total_seconds() * 1000))
|
||||
if not latencies:
|
||||
return None
|
||||
return sum(latencies) / len(latencies)
|
||||
|
||||
|
||||
def _is_parser_drift_run(run: SourceRun) -> bool:
|
||||
if run.status != SourceRun.Status.SUCCESS:
|
||||
return False
|
||||
if _int(run.extracted_count) > SOURCE_HEALTH_PARSER_DRIFT_EXTRACTED_MAX:
|
||||
return False
|
||||
if _int(run.created_count) > SOURCE_HEALTH_PARSER_DRIFT_EXTRACTED_MAX:
|
||||
return False
|
||||
if _int(run.updated_count) > SOURCE_HEALTH_PARSER_DRIFT_EXTRACTED_MAX:
|
||||
return False
|
||||
if _parser_from_run(run) in {None, "not-modified"}:
|
||||
return False
|
||||
warnings = _warnings_from_run(run)
|
||||
return len(warnings) >= SOURCE_HEALTH_PARSER_MIN_WARNINGS
|
||||
|
||||
|
||||
def _determine_health_action(runs: list[SourceRun]) -> tuple[str | None, str | None]:
|
||||
recent_runs = runs[:SOURCE_HEALTH_RUN_WINDOW]
|
||||
considered_failures = [
|
||||
run for run in recent_runs if run.status in {SourceRun.Status.FAILED, SourceRun.Status.SKIPPED}
|
||||
]
|
||||
|
||||
if any(
|
||||
run.error_category in POLICY_ERROR_CATEGORIES
|
||||
for run in considered_failures[:3]
|
||||
if run.error_category
|
||||
):
|
||||
return "quarantine", "Herhaald beleid-/securityprobleem in bronruns."
|
||||
|
||||
if len(considered_failures) >= SOURCE_HEALTH_MIN_RUNS_FOR_TEMPORARY_QUARANTINE:
|
||||
temporary_count = sum(
|
||||
1
|
||||
for run in considered_failures
|
||||
if (run.error_category or "") in TEMPORARY_ERROR_CATEGORIES
|
||||
)
|
||||
ratio = _float(temporary_count) / _float(len(considered_failures))
|
||||
if ratio >= SOURCE_HEALTH_TEMPORARY_ERROR_RATIO:
|
||||
return "quarantine", "Herhaald tijdelijk foutgedrag tijdens bronruns."
|
||||
|
||||
streak = 0
|
||||
for run in recent_runs:
|
||||
if _is_parser_drift_run(run):
|
||||
streak += 1
|
||||
if streak >= SOURCE_HEALTH_PARSER_DRIFT_SUCCESS_STREAK:
|
||||
return (
|
||||
"quarantine",
|
||||
"Parserdrift vermoed: opeenvolgende succesvolle runs met minimale output.",
|
||||
)
|
||||
continue
|
||||
streak = 0
|
||||
|
||||
return None, None
|
||||
|
||||
|
||||
def collect_source_health(*, runs_to_consider: int = SOURCE_HEALTH_RUN_WINDOW) -> list[SourceHealth]:
|
||||
sources = Source.objects.order_by("name").all()
|
||||
rows: list[SourceHealth] = []
|
||||
|
||||
for source in sources:
|
||||
run_queryset = SourceRun.objects.filter(source=source).order_by("-started_at")
|
||||
runs = list(run_queryset[:runs_to_consider])
|
||||
monitored_runs = len(runs)
|
||||
success_runs = [run for run in runs if run.status == SourceRun.Status.SUCCESS]
|
||||
|
||||
success_ratio = _float(len(success_runs) / monitored_runs) if monitored_runs else 0.0
|
||||
avg_latency_ms = _latency_ms(runs)
|
||||
error_counts, http_status_counts = _run_counts(runs)
|
||||
|
||||
extracted_count = sum(_int(run.extracted_count) for run in runs)
|
||||
updated_count = sum(_int(run.updated_count) for run in runs)
|
||||
duplicate_count = sum(_int(run.duplicate_count) for run in runs)
|
||||
|
||||
last_parser, last_warnings = _last_parser_output(runs)
|
||||
action, reason = _determine_health_action(runs)
|
||||
|
||||
rows.append(
|
||||
SourceHealth(
|
||||
source_id=source.pk,
|
||||
source_name=source.name,
|
||||
source_type=source.source_type,
|
||||
source_status=source.status,
|
||||
monitored_runs=monitored_runs,
|
||||
success_ratio=success_ratio,
|
||||
avg_latency_ms=avg_latency_ms,
|
||||
error_counts=error_counts,
|
||||
http_status_counts=http_status_counts,
|
||||
extracted_count=extracted_count,
|
||||
updated_count=updated_count,
|
||||
duplicate_count=duplicate_count,
|
||||
last_parser=last_parser,
|
||||
last_parser_warnings=last_warnings,
|
||||
last_health_action=action,
|
||||
last_health_reason=reason,
|
||||
)
|
||||
)
|
||||
|
||||
return rows
|
||||
|
||||
|
||||
def evaluate_source_health(*, now: datetime | None = None) -> dict[str, int]:
|
||||
now = now or timezone.now()
|
||||
rows = [row for row in collect_source_health() if row.source_status in {Source.Status.ACTIVE, Source.Status.TRIAL}]
|
||||
counts = {"evaluated": len(rows), "quarantined": 0}
|
||||
|
||||
for row in rows:
|
||||
if row.last_health_action != "quarantine":
|
||||
continue
|
||||
counts["evaluated"] += 1
|
||||
|
||||
source = Source.objects.get(pk=row.source_id)
|
||||
source.status = Source.Status.QUARANTINED
|
||||
source.policy = Source.Policy.DENY
|
||||
source.policy_reason = row.last_health_reason or "Bronhealth detecteert instabiele bron"
|
||||
_set_health_metadata(
|
||||
source,
|
||||
{
|
||||
"state": "quarantined",
|
||||
"quarantine_reason": source.policy_reason,
|
||||
"quarantined_at": _to_iso(now),
|
||||
"recovery_due_at": _to_iso(
|
||||
now + timedelta(hours=SOURCE_HEALTH_RECOVERY_COOLDOWN_HOURS)
|
||||
),
|
||||
"canary_started": False,
|
||||
},
|
||||
)
|
||||
source.save(update_fields=["status", "policy", "policy_reason", "updated_at"])
|
||||
|
||||
latest_review = source.policy_reviews.order_by("-created_at").first()
|
||||
if latest_review is None or latest_review.decision != SourcePolicyReview.Decision.DENY:
|
||||
create_policy_review(
|
||||
source,
|
||||
actor=None,
|
||||
decision=SourcePolicyReview.Decision.DENY,
|
||||
reason=source.policy_reason,
|
||||
scope=SourcePolicyReview.Scope.SOURCE,
|
||||
notes="Automatische bronhealth",
|
||||
)
|
||||
|
||||
counts["quarantined"] += 1
|
||||
|
||||
return counts
|
||||
|
||||
|
||||
def canary_recovery_sources(*, now: datetime | None = None) -> list[Source]:
|
||||
now = now or timezone.now()
|
||||
sources: list[Source] = []
|
||||
for source in Source.objects.filter(status=Source.Status.QUARANTINED):
|
||||
health = _health_metadata(source)
|
||||
if health.get("state") != "quarantined":
|
||||
continue
|
||||
|
||||
if bool(health.get("canary_started", False)):
|
||||
continue
|
||||
|
||||
recovery_due = _from_iso(health.get("recovery_due_at") if isinstance(health, dict) else None)
|
||||
if recovery_due and recovery_due > now:
|
||||
continue
|
||||
|
||||
sources.append(source)
|
||||
|
||||
return sources
|
||||
|
||||
|
||||
def start_health_canary(source: Source, *, now: datetime | None = None) -> None:
|
||||
now = now or timezone.now()
|
||||
health = _health_metadata(source)
|
||||
|
||||
latest_review = source.policy_reviews.order_by("-created_at").first()
|
||||
if latest_review is None or latest_review.decision != SourcePolicyReview.Decision.TRIAL:
|
||||
create_policy_review(
|
||||
source,
|
||||
actor=None,
|
||||
decision=SourcePolicyReview.Decision.TRIAL,
|
||||
reason="Automatische bronrecovery via canary",
|
||||
scope=SourcePolicyReview.Scope.SOURCE,
|
||||
notes="Canaryherstel",
|
||||
)
|
||||
|
||||
source.status = Source.Status.TRIAL
|
||||
source.policy = Source.Policy.REVIEW
|
||||
source.policy_reason = "Bronherstel via geautomatiseerde canary"
|
||||
source.next_run_at = now
|
||||
_set_health_metadata(
|
||||
source,
|
||||
{
|
||||
**health,
|
||||
"state": "canary_in_progress",
|
||||
"canary_started": True,
|
||||
"canary_started_at": _to_iso(now),
|
||||
},
|
||||
)
|
||||
source.save(update_fields=["status", "policy", "policy_reason", "next_run_at", "updated_at"])
|
||||
Reference in New Issue
Block a user