366 lines
12 KiB
Python
366 lines
12 KiB
Python
from __future__ import annotations
|
|
|
|
from collections.abc import Iterable
|
|
from dataclasses import dataclass
|
|
from datetime import datetime, timedelta
|
|
|
|
from django.db.models import Prefetch
|
|
from django.utils import timezone
|
|
|
|
from apps.sources.models import Source, SourcePolicyReview, SourceRun
|
|
|
|
from .policy import create_policy_review
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class SourceHealth:
|
|
source_id: int
|
|
source_name: str
|
|
source_type: str
|
|
source_status: str
|
|
monitored_runs: int
|
|
success_ratio: float
|
|
avg_latency_ms: float | None
|
|
error_counts: dict[str, int]
|
|
http_status_counts: dict[str, int]
|
|
extracted_count: int
|
|
updated_count: int
|
|
duplicate_count: int
|
|
last_parser: str | None
|
|
last_parser_warnings: int
|
|
last_health_action: str | None
|
|
last_health_reason: str | None
|
|
|
|
|
|
SOURCE_HEALTH_RUN_WINDOW = 30
|
|
SOURCE_HEALTH_MIN_RUNS_FOR_TEMPORARY_QUARANTINE = 6
|
|
SOURCE_HEALTH_TEMPORARY_ERROR_RATIO = 0.7
|
|
SOURCE_HEALTH_PARSER_DRIFT_SUCCESS_STREAK = 3
|
|
SOURCE_HEALTH_PARSER_MIN_WARNINGS = 2
|
|
SOURCE_HEALTH_PARSER_DRIFT_EXTRACTED_MAX = 0
|
|
SOURCE_HEALTH_RECOVERY_COOLDOWN_HOURS = 12
|
|
|
|
TEMPORARY_ERROR_CATEGORIES = {
|
|
"timeout",
|
|
"rate_limited",
|
|
"FetchTimeoutError",
|
|
"FetchError",
|
|
"unexpected",
|
|
"NetworkError",
|
|
}
|
|
POLICY_ERROR_CATEGORIES = {"policy"}
|
|
|
|
|
|
def _normalize_metrics(raw: object) -> dict[str, object]:
|
|
if isinstance(raw, dict):
|
|
return raw
|
|
return {}
|
|
|
|
|
|
def _warnings_from_run(run: SourceRun) -> list[str]:
|
|
metrics = _normalize_metrics(run.metrics)
|
|
warnings = metrics.get("warnings", [])
|
|
if not isinstance(warnings, list):
|
|
return []
|
|
return [str(item) for item in warnings if isinstance(item, str)]
|
|
|
|
|
|
def _parser_from_run(run: SourceRun) -> str | None:
|
|
metrics = _normalize_metrics(run.metrics)
|
|
parser = metrics.get("parser")
|
|
if isinstance(parser, str) and parser:
|
|
return parser
|
|
return None
|
|
|
|
|
|
def _int(value: object) -> int:
|
|
try:
|
|
return int(value)
|
|
except (TypeError, ValueError):
|
|
return 0
|
|
|
|
|
|
def _float(value: object) -> float:
|
|
try:
|
|
return float(value)
|
|
except (TypeError, ValueError):
|
|
return 0.0
|
|
|
|
|
|
def _to_iso(dt: datetime | None) -> str | None:
|
|
if dt is None:
|
|
return None
|
|
return dt.isoformat()
|
|
|
|
|
|
def _from_iso(value: str | None) -> datetime | None:
|
|
if not value:
|
|
return None
|
|
try:
|
|
dt = datetime.fromisoformat(value)
|
|
except ValueError:
|
|
return None
|
|
if timezone.is_naive(dt):
|
|
return timezone.make_aware(dt)
|
|
return dt
|
|
|
|
|
|
def _health_metadata(source: Source) -> dict[str, object]:
|
|
metadata = source.metadata
|
|
if not isinstance(metadata, dict):
|
|
return {}
|
|
health = metadata.get("source_health")
|
|
return health if isinstance(health, dict) else {}
|
|
|
|
|
|
def _set_health_metadata(source: Source, health_data: dict[str, object]) -> None:
|
|
metadata = source.metadata
|
|
if not isinstance(metadata, dict):
|
|
metadata = {}
|
|
metadata["source_health"] = health_data
|
|
source.metadata = metadata
|
|
source.save(update_fields=["metadata", "updated_at"])
|
|
|
|
|
|
def _run_counts(runs: Iterable[SourceRun]) -> tuple[dict[str, int], dict[str, int]]:
|
|
error_counts: dict[str, int] = {}
|
|
http_status_counts: dict[str, int] = {}
|
|
for run in runs:
|
|
if run.status != SourceRun.Status.SUCCESS:
|
|
category = run.error_category or "unknown"
|
|
error_counts[category] = error_counts.get(category, 0) + 1
|
|
if run.http_status:
|
|
status = str(run.http_status)
|
|
http_status_counts[status] = http_status_counts.get(status, 0) + 1
|
|
return error_counts, http_status_counts
|
|
|
|
|
|
def _last_parser_output(runs: list[SourceRun]) -> tuple[str | None, int]:
|
|
for run in runs:
|
|
if run.status != SourceRun.Status.SUCCESS:
|
|
continue
|
|
parser = _parser_from_run(run)
|
|
if parser:
|
|
warnings = _warnings_from_run(run)
|
|
return parser, len(warnings)
|
|
return None, 0
|
|
|
|
|
|
def _latency_ms(runs: list[SourceRun]) -> float | None:
|
|
latencies = []
|
|
for run in runs:
|
|
if run.finished_at is None or run.started_at is None:
|
|
continue
|
|
latencies.append(max(0.0, (run.finished_at - run.started_at).total_seconds() * 1000))
|
|
if not latencies:
|
|
return None
|
|
return sum(latencies) / len(latencies)
|
|
|
|
|
|
def _is_parser_drift_run(run: SourceRun) -> bool:
|
|
if run.status != SourceRun.Status.SUCCESS:
|
|
return False
|
|
if _int(run.extracted_count) > SOURCE_HEALTH_PARSER_DRIFT_EXTRACTED_MAX:
|
|
return False
|
|
if _int(run.created_count) > SOURCE_HEALTH_PARSER_DRIFT_EXTRACTED_MAX:
|
|
return False
|
|
if _int(run.updated_count) > SOURCE_HEALTH_PARSER_DRIFT_EXTRACTED_MAX:
|
|
return False
|
|
if _parser_from_run(run) in {None, "not-modified"}:
|
|
return False
|
|
warnings = _warnings_from_run(run)
|
|
return len(warnings) >= SOURCE_HEALTH_PARSER_MIN_WARNINGS
|
|
|
|
|
|
def _determine_health_action(runs: list[SourceRun]) -> tuple[str | None, str | None]:
|
|
recent_runs = runs[:SOURCE_HEALTH_RUN_WINDOW]
|
|
considered_failures = [
|
|
run
|
|
for run in recent_runs
|
|
if run.status in {SourceRun.Status.FAILED, SourceRun.Status.SKIPPED}
|
|
]
|
|
|
|
if any(
|
|
run.error_category in POLICY_ERROR_CATEGORIES
|
|
for run in considered_failures[:3]
|
|
if run.error_category
|
|
):
|
|
return "quarantine", "Herhaald beleid-/securityprobleem in bronruns."
|
|
|
|
if len(considered_failures) >= SOURCE_HEALTH_MIN_RUNS_FOR_TEMPORARY_QUARANTINE:
|
|
temporary_count = sum(
|
|
1
|
|
for run in considered_failures
|
|
if (run.error_category or "") in TEMPORARY_ERROR_CATEGORIES
|
|
)
|
|
ratio = _float(temporary_count) / _float(len(considered_failures))
|
|
if ratio >= SOURCE_HEALTH_TEMPORARY_ERROR_RATIO:
|
|
return "quarantine", "Herhaald tijdelijk foutgedrag tijdens bronruns."
|
|
|
|
streak = 0
|
|
for run in recent_runs:
|
|
if _is_parser_drift_run(run):
|
|
streak += 1
|
|
if streak >= SOURCE_HEALTH_PARSER_DRIFT_SUCCESS_STREAK:
|
|
return (
|
|
"quarantine",
|
|
"Parserdrift vermoed: opeenvolgende succesvolle runs met minimale output.",
|
|
)
|
|
continue
|
|
streak = 0
|
|
|
|
return None, None
|
|
|
|
|
|
def collect_source_health(
|
|
*, runs_to_consider: int = SOURCE_HEALTH_RUN_WINDOW
|
|
) -> list[SourceHealth]:
|
|
sources = Source.objects.prefetch_related(
|
|
Prefetch(
|
|
"runs",
|
|
queryset=SourceRun.objects.order_by("-started_at")[:runs_to_consider],
|
|
to_attr="health_runs",
|
|
)
|
|
).order_by("name")
|
|
rows: list[SourceHealth] = []
|
|
|
|
for source in sources:
|
|
runs = source.health_runs
|
|
monitored_runs = len(runs)
|
|
success_runs = [run for run in runs if run.status == SourceRun.Status.SUCCESS]
|
|
|
|
success_ratio = _float(len(success_runs) / monitored_runs) if monitored_runs else 0.0
|
|
avg_latency_ms = _latency_ms(runs)
|
|
error_counts, http_status_counts = _run_counts(runs)
|
|
|
|
extracted_count = sum(_int(run.extracted_count) for run in runs)
|
|
updated_count = sum(_int(run.updated_count) for run in runs)
|
|
duplicate_count = sum(_int(run.duplicate_count) for run in runs)
|
|
|
|
last_parser, last_warnings = _last_parser_output(runs)
|
|
action, reason = _determine_health_action(runs)
|
|
|
|
rows.append(
|
|
SourceHealth(
|
|
source_id=source.pk,
|
|
source_name=source.name,
|
|
source_type=source.source_type,
|
|
source_status=source.status,
|
|
monitored_runs=monitored_runs,
|
|
success_ratio=success_ratio,
|
|
avg_latency_ms=avg_latency_ms,
|
|
error_counts=error_counts,
|
|
http_status_counts=http_status_counts,
|
|
extracted_count=extracted_count,
|
|
updated_count=updated_count,
|
|
duplicate_count=duplicate_count,
|
|
last_parser=last_parser,
|
|
last_parser_warnings=last_warnings,
|
|
last_health_action=action,
|
|
last_health_reason=reason,
|
|
)
|
|
)
|
|
|
|
return rows
|
|
|
|
|
|
def evaluate_source_health(*, now: datetime | None = None) -> dict[str, int]:
|
|
now = now or timezone.now()
|
|
rows = [
|
|
row
|
|
for row in collect_source_health()
|
|
if row.source_status in {Source.Status.ACTIVE, Source.Status.TRIAL}
|
|
]
|
|
counts = {"evaluated": len(rows), "quarantined": 0}
|
|
|
|
for row in rows:
|
|
if row.last_health_action != "quarantine":
|
|
continue
|
|
counts["evaluated"] += 1
|
|
|
|
source = Source.objects.get(pk=row.source_id)
|
|
source.status = Source.Status.QUARANTINED
|
|
source.policy = Source.Policy.DENY
|
|
source.policy_reason = row.last_health_reason or "Bronhealth detecteert instabiele bron"
|
|
_set_health_metadata(
|
|
source,
|
|
{
|
|
"state": "quarantined",
|
|
"quarantine_reason": source.policy_reason,
|
|
"quarantined_at": _to_iso(now),
|
|
"recovery_due_at": _to_iso(
|
|
now + timedelta(hours=SOURCE_HEALTH_RECOVERY_COOLDOWN_HOURS)
|
|
),
|
|
"canary_started": False,
|
|
},
|
|
)
|
|
source.save(update_fields=["status", "policy", "policy_reason", "updated_at"])
|
|
|
|
latest_review = source.policy_reviews.order_by("-created_at").first()
|
|
if latest_review is None or latest_review.decision != SourcePolicyReview.Decision.DENY:
|
|
create_policy_review(
|
|
source,
|
|
actor=None,
|
|
decision=SourcePolicyReview.Decision.DENY,
|
|
reason=source.policy_reason,
|
|
scope=SourcePolicyReview.Scope.SOURCE,
|
|
notes="Automatische bronhealth",
|
|
)
|
|
|
|
counts["quarantined"] += 1
|
|
|
|
return counts
|
|
|
|
|
|
def canary_recovery_sources(*, now: datetime | None = None) -> list[Source]:
|
|
now = now or timezone.now()
|
|
sources: list[Source] = []
|
|
for source in Source.objects.filter(status=Source.Status.QUARANTINED):
|
|
health = _health_metadata(source)
|
|
if health.get("state") != "quarantined":
|
|
continue
|
|
|
|
if bool(health.get("canary_started", False)):
|
|
continue
|
|
|
|
recovery_due = _from_iso(
|
|
health.get("recovery_due_at") if isinstance(health, dict) else None
|
|
)
|
|
if recovery_due and recovery_due > now:
|
|
continue
|
|
|
|
sources.append(source)
|
|
|
|
return sources
|
|
|
|
|
|
def start_health_canary(source: Source, *, now: datetime | None = None) -> None:
|
|
now = now or timezone.now()
|
|
health = _health_metadata(source)
|
|
|
|
latest_review = source.policy_reviews.order_by("-created_at").first()
|
|
if latest_review is None or latest_review.decision != SourcePolicyReview.Decision.TRIAL:
|
|
create_policy_review(
|
|
source,
|
|
actor=None,
|
|
decision=SourcePolicyReview.Decision.TRIAL,
|
|
reason="Automatische bronrecovery via canary",
|
|
scope=SourcePolicyReview.Scope.SOURCE,
|
|
notes="Canaryherstel",
|
|
)
|
|
|
|
source.status = Source.Status.TRIAL
|
|
source.policy = Source.Policy.REVIEW
|
|
source.policy_reason = "Bronherstel via geautomatiseerde canary"
|
|
source.next_run_at = now
|
|
_set_health_metadata(
|
|
source,
|
|
{
|
|
**health,
|
|
"state": "canary_in_progress",
|
|
"canary_started": True,
|
|
"canary_started_at": _to_iso(now),
|
|
},
|
|
)
|
|
source.save(update_fields=["status", "policy", "policy_reason", "next_run_at", "updated_at"])
|