from __future__ import annotations from collections.abc import Iterable from dataclasses import dataclass from datetime import datetime, timedelta from django.db.models import Prefetch from django.utils import timezone from apps.sources.models import Source, SourcePolicyReview, SourceRun from .policy import create_policy_review @dataclass(frozen=True) class SourceHealth: source_id: int source_name: str source_type: str source_status: str monitored_runs: int success_ratio: float avg_latency_ms: float | None error_counts: dict[str, int] http_status_counts: dict[str, int] extracted_count: int updated_count: int duplicate_count: int last_parser: str | None last_parser_warnings: int last_health_action: str | None last_health_reason: str | None SOURCE_HEALTH_RUN_WINDOW = 30 SOURCE_HEALTH_MIN_RUNS_FOR_TEMPORARY_QUARANTINE = 6 SOURCE_HEALTH_TEMPORARY_ERROR_RATIO = 0.7 SOURCE_HEALTH_PARSER_DRIFT_SUCCESS_STREAK = 3 SOURCE_HEALTH_PARSER_MIN_WARNINGS = 2 SOURCE_HEALTH_PARSER_DRIFT_EXTRACTED_MAX = 0 SOURCE_HEALTH_RECOVERY_COOLDOWN_HOURS = 12 TEMPORARY_ERROR_CATEGORIES = { "timeout", "rate_limited", "FetchTimeoutError", "FetchError", "unexpected", "NetworkError", } POLICY_ERROR_CATEGORIES = {"policy"} def _normalize_metrics(raw: object) -> dict[str, object]: if isinstance(raw, dict): return raw return {} def _warnings_from_run(run: SourceRun) -> list[str]: metrics = _normalize_metrics(run.metrics) warnings = metrics.get("warnings", []) if not isinstance(warnings, list): return [] return [str(item) for item in warnings if isinstance(item, str)] def _parser_from_run(run: SourceRun) -> str | None: metrics = _normalize_metrics(run.metrics) parser = metrics.get("parser") if isinstance(parser, str) and parser: return parser return None def _int(value: object) -> int: try: return int(value) except (TypeError, ValueError): return 0 def _float(value: object) -> float: try: return float(value) except (TypeError, ValueError): return 0.0 def _to_iso(dt: datetime | None) -> str | None: if dt is None: return None return dt.isoformat() def _from_iso(value: str | None) -> datetime | None: if not value: return None try: dt = datetime.fromisoformat(value) except ValueError: return None if timezone.is_naive(dt): return timezone.make_aware(dt) return dt def _health_metadata(source: Source) -> dict[str, object]: metadata = source.metadata if not isinstance(metadata, dict): return {} health = metadata.get("source_health") return health if isinstance(health, dict) else {} def _set_health_metadata(source: Source, health_data: dict[str, object]) -> None: metadata = source.metadata if not isinstance(metadata, dict): metadata = {} metadata["source_health"] = health_data source.metadata = metadata source.save(update_fields=["metadata", "updated_at"]) def _run_counts(runs: Iterable[SourceRun]) -> tuple[dict[str, int], dict[str, int]]: error_counts: dict[str, int] = {} http_status_counts: dict[str, int] = {} for run in runs: if run.status != SourceRun.Status.SUCCESS: category = run.error_category or "unknown" error_counts[category] = error_counts.get(category, 0) + 1 if run.http_status: status = str(run.http_status) http_status_counts[status] = http_status_counts.get(status, 0) + 1 return error_counts, http_status_counts def _last_parser_output(runs: list[SourceRun]) -> tuple[str | None, int]: for run in runs: if run.status != SourceRun.Status.SUCCESS: continue parser = _parser_from_run(run) if parser: warnings = _warnings_from_run(run) return parser, len(warnings) return None, 0 def _latency_ms(runs: list[SourceRun]) -> float | None: latencies = [] for run in runs: if run.finished_at is None or run.started_at is None: continue latencies.append(max(0.0, (run.finished_at - run.started_at).total_seconds() * 1000)) if not latencies: return None return sum(latencies) / len(latencies) def _is_parser_drift_run(run: SourceRun) -> bool: if run.status != SourceRun.Status.SUCCESS: return False if _int(run.extracted_count) > SOURCE_HEALTH_PARSER_DRIFT_EXTRACTED_MAX: return False if _int(run.created_count) > SOURCE_HEALTH_PARSER_DRIFT_EXTRACTED_MAX: return False if _int(run.updated_count) > SOURCE_HEALTH_PARSER_DRIFT_EXTRACTED_MAX: return False if _parser_from_run(run) in {None, "not-modified"}: return False warnings = _warnings_from_run(run) return len(warnings) >= SOURCE_HEALTH_PARSER_MIN_WARNINGS def _determine_health_action(runs: list[SourceRun]) -> tuple[str | None, str | None]: recent_runs = runs[:SOURCE_HEALTH_RUN_WINDOW] considered_failures = [ run for run in recent_runs if run.status in {SourceRun.Status.FAILED, SourceRun.Status.SKIPPED} ] if any( run.error_category in POLICY_ERROR_CATEGORIES for run in considered_failures[:3] if run.error_category ): return "quarantine", "Herhaald beleid-/securityprobleem in bronruns." if len(considered_failures) >= SOURCE_HEALTH_MIN_RUNS_FOR_TEMPORARY_QUARANTINE: temporary_count = sum( 1 for run in considered_failures if (run.error_category or "") in TEMPORARY_ERROR_CATEGORIES ) ratio = _float(temporary_count) / _float(len(considered_failures)) if ratio >= SOURCE_HEALTH_TEMPORARY_ERROR_RATIO: return "quarantine", "Herhaald tijdelijk foutgedrag tijdens bronruns." streak = 0 for run in recent_runs: if _is_parser_drift_run(run): streak += 1 if streak >= SOURCE_HEALTH_PARSER_DRIFT_SUCCESS_STREAK: return ( "quarantine", "Parserdrift vermoed: opeenvolgende succesvolle runs met minimale output.", ) continue streak = 0 return None, None def collect_source_health( *, runs_to_consider: int = SOURCE_HEALTH_RUN_WINDOW ) -> list[SourceHealth]: sources = Source.objects.prefetch_related( Prefetch( "runs", queryset=SourceRun.objects.order_by("-started_at")[:runs_to_consider], to_attr="health_runs", ) ).order_by("name") rows: list[SourceHealth] = [] for source in sources: runs = source.health_runs monitored_runs = len(runs) success_runs = [run for run in runs if run.status == SourceRun.Status.SUCCESS] success_ratio = _float(len(success_runs) / monitored_runs) if monitored_runs else 0.0 avg_latency_ms = _latency_ms(runs) error_counts, http_status_counts = _run_counts(runs) extracted_count = sum(_int(run.extracted_count) for run in runs) updated_count = sum(_int(run.updated_count) for run in runs) duplicate_count = sum(_int(run.duplicate_count) for run in runs) last_parser, last_warnings = _last_parser_output(runs) action, reason = _determine_health_action(runs) rows.append( SourceHealth( source_id=source.pk, source_name=source.name, source_type=source.source_type, source_status=source.status, monitored_runs=monitored_runs, success_ratio=success_ratio, avg_latency_ms=avg_latency_ms, error_counts=error_counts, http_status_counts=http_status_counts, extracted_count=extracted_count, updated_count=updated_count, duplicate_count=duplicate_count, last_parser=last_parser, last_parser_warnings=last_warnings, last_health_action=action, last_health_reason=reason, ) ) return rows def evaluate_source_health(*, now: datetime | None = None) -> dict[str, int]: now = now or timezone.now() rows = [ row for row in collect_source_health() if row.source_status in {Source.Status.ACTIVE, Source.Status.TRIAL} ] counts = {"evaluated": len(rows), "quarantined": 0} for row in rows: if row.last_health_action != "quarantine": continue counts["evaluated"] += 1 source = Source.objects.get(pk=row.source_id) source.status = Source.Status.QUARANTINED source.policy = Source.Policy.DENY source.policy_reason = row.last_health_reason or "Bronhealth detecteert instabiele bron" _set_health_metadata( source, { "state": "quarantined", "quarantine_reason": source.policy_reason, "quarantined_at": _to_iso(now), "recovery_due_at": _to_iso( now + timedelta(hours=SOURCE_HEALTH_RECOVERY_COOLDOWN_HOURS) ), "canary_started": False, }, ) source.save(update_fields=["status", "policy", "policy_reason", "updated_at"]) latest_review = source.policy_reviews.order_by("-created_at").first() if latest_review is None or latest_review.decision != SourcePolicyReview.Decision.DENY: create_policy_review( source, actor=None, decision=SourcePolicyReview.Decision.DENY, reason=source.policy_reason, scope=SourcePolicyReview.Scope.SOURCE, notes="Automatische bronhealth", ) counts["quarantined"] += 1 return counts def canary_recovery_sources(*, now: datetime | None = None) -> list[Source]: now = now or timezone.now() sources: list[Source] = [] for source in Source.objects.filter(status=Source.Status.QUARANTINED): health = _health_metadata(source) if health.get("state") != "quarantined": continue if bool(health.get("canary_started", False)): continue recovery_due = _from_iso( health.get("recovery_due_at") if isinstance(health, dict) else None ) if recovery_due and recovery_due > now: continue sources.append(source) return sources def start_health_canary(source: Source, *, now: datetime | None = None) -> None: now = now or timezone.now() health = _health_metadata(source) latest_review = source.policy_reviews.order_by("-created_at").first() if latest_review is None or latest_review.decision != SourcePolicyReview.Decision.TRIAL: create_policy_review( source, actor=None, decision=SourcePolicyReview.Decision.TRIAL, reason="Automatische bronrecovery via canary", scope=SourcePolicyReview.Scope.SOURCE, notes="Canaryherstel", ) source.status = Source.Status.TRIAL source.policy = Source.Policy.REVIEW source.policy_reason = "Bronherstel via geautomatiseerde canary" source.next_run_at = now _set_health_metadata( source, { **health, "state": "canary_in_progress", "canary_started": True, "canary_started_at": _to_iso(now), }, ) source.save(update_fields=["status", "policy", "policy_reason", "next_run_at", "updated_at"])