import hashlib from collections.abc import Callable from dataclasses import dataclass from datetime import UTC, datetime, timedelta from .models import Campaign, Category, IncidentCluster, NormalizedEvent, Severity from .normalize import ( is_application_error, is_dns_client, is_event_log, is_kernel_general, is_kernel_power, is_log_cleared, is_security_audit, is_service_control_manager, is_windows_error_reporting, ) TitleBuilder = Callable[[list[NormalizedEvent]], str] ReasonBuilder = Callable[[list[NormalizedEvent]], str] GroupFilter = Callable[[list[NormalizedEvent]], bool] Group = tuple[str | None, list[NormalizedEvent]] MAX_TITLE_CHARS = 256 FALLBACK_RULE_ID = "windows.generic.attention" FALLBACK_WINDOW_SECONDS = 300 SPRAY_MIN_DISTINCT_ACCOUNTS = 3 SEVERITY_RANK: dict[str, int] = {"info": 0, "warning": 1, "error": 2, "critical": 3} LEVEL_SEVERITY: dict[str, Severity] = {"critical": "critical", "error": "error", "warning": "warning"} @dataclass(frozen=True) class Rule: id: str category: Category severity: Severity window_seconds: int predicate: Callable[[NormalizedEvent], bool] key: Callable[[NormalizedEvent], str | None] title_for: TitleBuilder reason_for: ReasonBuilder accept: GroupFilter | None = None def _field(event: NormalizedEvent, *names: str) -> str: for name in names: value = event.fields.get(name, "").strip() if value: return value return "" def _dimension(value: str) -> str | None: normalized = value.strip() if not normalized or normalized.casefold() in { "unknown", "unknown-account", "unknown-host", "unknown-process", "unknown-service", "n/a", "-", }: return None return normalized def _host(event: NormalizedEvent) -> str | None: return _dimension(event.computer) def _process(event: NormalizedEvent) -> str | None: return _dimension(_field(event, "process", "EventData.AppName", "EventData.Data[0]")) def _service(event: NormalizedEvent) -> str | None: return _dimension(_field(event, "service", "EventData.param1", "UserData.param1")) def _new_service(event: NormalizedEvent) -> str | None: return _dimension(_field(event, "EventData.ServiceName", "service")) def _account(event: NormalizedEvent) -> str | None: return _dimension(_field(event, "EventData.TargetUserName", "TargetUserName")) def _source_ip(event: NormalizedEvent) -> str | None: return _dimension(_field(event, "EventData.IpAddress", "IpAddress")) def _group_key(*dimensions: str | None) -> str | None: if any(value is None for value in dimensions): return None return "|".join(value.casefold() for value in dimensions if value is not None) def _bounded_title(value: str, fallback: str) -> str: text = " ".join(value.split()) or fallback return text if len(text) <= MAX_TITLE_CHARS else f"{text[: MAX_TITLE_CHARS - 1]}…" def _distinct_accounts(evidence: list[NormalizedEvent]) -> int: return len({account.casefold() for event in evidence if (account := _account(event))}) # --- predicates ------------------------------------------------------------- def _crash(event: NormalizedEvent) -> bool: return is_application_error(event.provider, event.channel, event.event_id) or is_windows_error_reporting( event.provider, event.channel, event.event_id ) def _service_failure(event: NormalizedEvent) -> bool: return is_service_control_manager(event.provider, event.channel) and event.event_id in {7023, 7024, 7031, 7034} def _service_installed(event: NormalizedEvent) -> bool: return is_service_control_manager(event.provider, event.channel) and event.event_id == 7045 def _dns_timeout(event: NormalizedEvent) -> bool: return is_dns_client(event.provider, event.channel) and event.event_id == 1014 def _failed_logon(event: NormalizedEvent) -> bool: return is_security_audit(event.provider, event.channel) and event.event_id == 4625 def _account_lockout(event: NormalizedEvent) -> bool: return is_security_audit(event.provider, event.channel) and event.event_id == 4740 def _account_change(event: NormalizedEvent) -> bool: return is_security_audit(event.provider, event.channel) and event.event_id in {4720, 4728, 4732, 4756} def _log_cleared(event: NormalizedEvent) -> bool: return is_log_cleared(event.provider, event.channel, event.event_id) def _normal_startup(event: NormalizedEvent) -> bool: return (is_kernel_general(event.provider, event.channel) and event.event_id == 12) or ( is_event_log(event.provider, event.channel) and event.event_id == 6005 ) def _normal_shutdown(event: NormalizedEvent) -> bool: return (is_kernel_general(event.provider, event.channel) and event.event_id == 13) or ( is_event_log(event.provider, event.channel) and event.event_id == 6006 ) def _unexpected_shutdown(event: NormalizedEvent) -> bool: return (is_kernel_power(event.provider, event.channel) and event.event_id == 41) or ( is_event_log(event.provider, event.channel) and event.event_id == 6008 ) # --- titles and reasons ----------------------------------------------------- def _crash_title(evidence: list[NormalizedEvent]) -> str: ids = {event.event_id for event in evidence} if {1000, 1001}.issubset(ids): return "Application crash and reporting sequence" if 1000 in ids: return "Application crash recorded" return "Crash report recorded" def _crash_reason(evidence: list[NormalizedEvent]) -> str: ids = {event.event_id for event in evidence} if {1000, 1001}.issubset(ids): return "Application Error and Windows Error Reporting records were temporally related; no causal claim is made." return "A crash-related provider record was observed; a reporting sequence is not established by this evidence." def _service_title(evidence: list[NormalizedEvent]) -> str: return "Service failure window" if len(evidence) > 1 else "Service failure recorded" def _service_reason(evidence: list[NormalizedEvent]) -> str: service = _service(evidence[0]) if service is None or _host(evidence[0]) is None: return "Service Control Manager reported a service failure; no service/host grouping was applied because a grouping dimension was unavailable." return "Service Control Manager reported an abnormal service termination for the same known service and host." def _service_installed_key(event: NormalizedEvent) -> str | None: return _group_key(_host(event), _new_service(event)) def _service_installed_title(evidence: list[NormalizedEvent]) -> str: return "New service installed" if len(evidence) == 1 else "New service installation activity" def _service_installed_reason(evidence: list[NormalizedEvent]) -> str: if _service_installed_key(evidence[0]) is None: return "Service Control Manager recorded a service installation (7045); no service/host grouping was applied because a grouping dimension was unavailable." return "Service Control Manager recorded installation of the same known service on the same host (7045); installation alone is not evidence of malice." def _dns_title(evidence: list[NormalizedEvent]) -> str: return "DNS timeout activity" if len(evidence) > 1 else "DNS timeout recorded" def _dns_reason(evidence: list[NormalizedEvent]) -> str: if _host(evidence[0]) is None: return "DNS Client reported a timeout; no host grouping was applied because the computer identity was unavailable." return "DNS Client timeout events were observed in the same bounded window on the same known host." def _auth_title(evidence: list[NormalizedEvent]) -> str: if len(evidence) >= 3: return "Failed sign-in burst" if len(evidence) == 2: return "Repeated failed sign-ins" return "Failed sign-in recorded" def _auth_reason(evidence: list[NormalizedEvent]) -> str: if _group_key(_host(evidence[0]), _account(evidence[0])) is None: return "Security Auditing reported a failed sign-in; no account/host grouping was applied because a grouping dimension was unavailable." return "Security Auditing reported failed sign-ins for the same known account and host; this is evidence of activity, not proof of cause." def _spray_title(evidence: list[NormalizedEvent]) -> str: return "Failed sign-ins across multiple accounts from one source" def _spray_reason(evidence: list[NormalizedEvent]) -> str: return ( f"Security Auditing reported failed sign-ins (4625) for {_distinct_accounts(evidence)} distinct accounts " "from the same known source address on the same host; this pattern is consistent with, but does not prove, password spraying." ) def _lockout_title(evidence: list[NormalizedEvent]) -> str: return "Account lockout" if len(evidence) == 1 else "Repeated account lockouts" def _lockout_reason(evidence: list[NormalizedEvent]) -> str: if _group_key(_host(evidence[0]), _account(evidence[0])) is None: return "Security Auditing recorded an account lockout (4740); no account/host grouping was applied because a grouping dimension was unavailable." return "Security Auditing recorded lockouts (4740) for the same known account on the same host." def _account_change_title(evidence: list[NormalizedEvent]) -> str: ids = {event.event_id for event in evidence} if 4720 in ids and ids & {4728, 4732, 4756}: return "Account created and added to a security group" if 4720 in ids: return "User account created" return "Security group membership changed" def _account_change_reason(evidence: list[NormalizedEvent]) -> str: if _host(evidence[0]) is None: return "Security Auditing recorded an account or security-group change; no host grouping was applied because the computer identity was unavailable." return "Security Auditing recorded account creation or security-group membership changes (4720/4728/4732/4756) on the same known host in one bounded window." def _log_cleared_title(evidence: list[NormalizedEvent]) -> str: return "Event log cleared" if len(evidence) == 1 else "Event logs cleared" def _log_cleared_reason(evidence: list[NormalizedEvent]) -> str: return ( "EventLog recorded that an event log was cleared (Security 1102 / System 104); records from before this point " "may be missing. The record identifies the clearing account only as far as its own fields state." ) def _startup_title(evidence: list[NormalizedEvent]) -> str: return "Normal startup sequence" if len(evidence) > 1 else "Normal startup evidence" def _startup_reason(evidence: list[NormalizedEvent]) -> str: return "Kernel-General/EventLog startup records were temporally related; no cause is inferred." def _shutdown_title(evidence: list[NormalizedEvent]) -> str: return "Normal shutdown evidence" def _shutdown_reason(evidence: list[NormalizedEvent]) -> str: return "Kernel-General/EventLog shutdown evidence was observed; no cause is inferred." def _unexpected_title(evidence: list[NormalizedEvent]) -> str: return "Unexpected shutdown/restart evidence" def _unexpected_reason(evidence: list[NormalizedEvent]) -> str: return "Kernel-Power/EventLog records indicate an unexpected shutdown or restart; the logs do not establish why it occurred." RULES = ( Rule( "windows.audit.log-cleared", "security", "critical", 300, _log_cleared, lambda event: _group_key(_host(event)), _log_cleared_title, _log_cleared_reason, ), Rule( "windows.crash.application-report", "crash", "critical", 120, _crash, lambda event: _group_key(_host(event), _process(event)), _crash_title, _crash_reason, ), Rule( "windows.service.failure", "service", "warning", 120, _service_failure, lambda event: _group_key(_host(event), _service(event)), _service_title, _service_reason, ), Rule( "windows.service.installed", "service", "warning", 300, _service_installed, _service_installed_key, _service_installed_title, _service_installed_reason, ), Rule( "windows.network.dns-timeout", "network", "warning", 120, _dns_timeout, lambda event: _group_key(_host(event)), _dns_title, _dns_reason, ), Rule( "windows.auth.failed-logon", "authentication", "warning", 120, _failed_logon, lambda event: _group_key(_host(event), _account(event)), _auth_title, _auth_reason, ), Rule( "windows.auth.password-spray", "authentication", "warning", 300, _failed_logon, lambda event: _group_key(_host(event), _source_ip(event)), _spray_title, _spray_reason, accept=lambda evidence: _distinct_accounts(evidence) >= SPRAY_MIN_DISTINCT_ACCOUNTS, ), Rule( "windows.auth.account-lockout", "authentication", "warning", 300, _account_lockout, lambda event: _group_key(_host(event), _account(event)), _lockout_title, _lockout_reason, ), Rule( "windows.account.change", "security", "warning", 300, _account_change, lambda event: _group_key(_host(event)), _account_change_title, _account_change_reason, ), Rule( "windows.system.startup", "system", "info", 300, _normal_startup, lambda event: _group_key(_host(event)), _startup_title, _startup_reason, ), Rule( "windows.system.shutdown.normal", "system", "info", 300, _normal_shutdown, lambda event: _group_key(_host(event)), _shutdown_title, _shutdown_reason, ), Rule( "windows.system.shutdown.unexpected", "system", "warning", 300, _unexpected_shutdown, lambda event: _group_key(_host(event)), _unexpected_title, _unexpected_reason, ), ) # --- grouping --------------------------------------------------------------- def _event_sort_key(event: NormalizedEvent) -> tuple[bool, datetime, str]: reliable_timestamp = event.timestamp is not None and event.timestamp_valid return ( not reliable_timestamp, event.timestamp if reliable_timestamp else datetime.max.replace(tzinfo=UTC), event.id, ) Signature = tuple[str, str, int | None] def _signature(event: NormalizedEvent) -> Signature: return (event.provider, event.channel, event.event_id) def _matching(events: list[NormalizedEvent], predicate: Callable[[NormalizedEvent], bool]) -> list[NormalizedEvent]: """Filter events by a rule predicate, evaluating it once per distinct signature. Every rule predicate depends only on provider, channel and event ID, so a 100,000-event log with a few hundred signatures needs a few hundred predicate calls per rule instead of 100,000. """ verdicts: dict[Signature, bool] = {} matched: list[NormalizedEvent] = [] for event in events: signature = _signature(event) verdict = verdicts.get(signature) if verdict is None: verdict = verdicts[signature] = predicate(event) if verdict: matched.append(event) return matched def _windowed_groups( events: list[NormalizedEvent], predicate: Callable[[NormalizedEvent], bool], key_for: Callable[[NormalizedEvent], str | None], window_seconds: int, ) -> list[Group]: """Bucket timed events by key and split each bucket into bounded windows. A window is anchored on its first event, so a group can never chain beyond ``window_seconds``. Events whose key is unknown become isolated groups (key ``None``) so that missing dimensions never merge unrelated evidence. """ buckets: dict[str, list[NormalizedEvent]] = {} isolated: list[NormalizedEvent] = [] for event in events: if not predicate(event) or event.timestamp is None or not event.timestamp_valid: continue key = key_for(event) if key is None: isolated.append(event) else: buckets.setdefault(key, []).append(event) groups: list[Group] = [] window = timedelta(seconds=window_seconds) for key in sorted(buckets): ordered = sorted(buckets[key], key=_event_sort_key) current: list[NormalizedEvent] = [] for event in ordered: if not current or event.timestamp - current[0].timestamp <= window: current.append(event) else: groups.append((key, current)) current = [event] if current: groups.append((key, current)) groups.extend((None, [event]) for event in sorted(isolated, key=_event_sort_key)) return groups def _always(_event: NormalizedEvent) -> bool: return True def _groups_for_rule(events: list[NormalizedEvent], rule: Rule) -> list[Group]: groups = _windowed_groups(_matching(events, rule.predicate), _always, rule.key, rule.window_seconds) if rule.accept is None: return groups return [group for group in groups if rule.accept(group[1])] def _incident_id(rule_id: str, evidence: list[NormalizedEvent]) -> str: seed = f"{rule_id}|{'|'.join(event.id for event in evidence)}".encode() return f"incident-{hashlib.sha256(seed).hexdigest()[:16]}" def _signatures(evidence: list[NormalizedEvent]) -> set[tuple[str, int | None, str]]: return {(event.provider, event.event_id, event.summary) for event in evidence} def _compression_reasons(evidence: list[NormalizedEvent]) -> list[str]: reasons: list[str] = [] timed = all(event.timestamp is not None and event.timestamp_valid for event in evidence) if len(evidence) > 1 and timed: seconds = int((evidence[-1].timestamp - evidence[0].timestamp).total_seconds()) reasons.append(f"{len(evidence)} evidence events occurred within {seconds} second(s).") signatures = _signatures(evidence) if len(signatures) < len(evidence): reasons.append(f"Repeated signals compress from {len(evidence)} records to {len(signatures)} distinct signature(s).") return reasons def _build(rule: Rule, evidence: list[NormalizedEvent]) -> IncidentCluster: return IncidentCluster.model_construct( id=_incident_id(rule.id, evidence), rule_id=rule.id, start=evidence[0].timestamp, end=evidence[-1].timestamp, title=_bounded_title(rule.title_for(evidence), "Correlated evidence"), category=rule.category, severity=rule.severity, reasons=[rule.reason_for(evidence), *_compression_reasons(evidence)], evidence_event_ids=[event.id for event in evidence], original_count=len(evidence), compressed_count=max(1, len(_signatures(evidence))), ) def _attention(event: NormalizedEvent) -> bool: return event.level.casefold() in LEVEL_SEVERITY def _fallback_key(event: NormalizedEvent) -> str | None: provider = _dimension(event.provider) if provider is None or provider.casefold() == "unknown provider" or event.event_id is None: return None return _group_key(_host(event), provider, str(event.event_id)) def _max_severity(levels: list[Severity]) -> Severity: return max(levels, key=lambda level: SEVERITY_RANK[level]) def _build_fallback(evidence: list[NormalizedEvent]) -> IncidentCluster: first = evidence[0] timed = first.timestamp is not None and first.timestamp_valid if len(evidence) == 1: reason = "An error or warning was not matched by a more specific correlation rule." else: reason = ( "Errors or warnings with the same provider, event ID and known host were not matched by a more specific " "rule and were compressed into one bounded window; repetition is not a causal claim." ) return IncidentCluster.model_construct( id=_incident_id(FALLBACK_RULE_ID, evidence), rule_id=FALLBACK_RULE_ID, start=first.timestamp if timed else None, end=evidence[-1].timestamp if timed else None, title=_bounded_title(first.summary, "Unclassified attention event"), category=first.category, severity=_max_severity([LEVEL_SEVERITY.get(event.level.casefold(), "warning") for event in evidence]), reasons=[reason, *_compression_reasons(evidence)], evidence_event_ids=[event.id for event in evidence], original_count=len(evidence), compressed_count=max(1, len(_signatures(evidence))), ) # --- campaigns -------------------------------------------------------------- def _campaigns_for( rule_id: str, window_seconds: int, keyed: list[tuple[str | None, IncidentCluster]] ) -> list[Campaign]: """Link adjacent windows of one rule and one known key into a summary campaign. ``keyed`` is ordered by key and then by time, as produced by :func:`_windowed_groups`. Incidents keep their bounded windows; a campaign only records that the same rule recurred for the same known dimensions. """ campaigns: list[Campaign] = [] window = timedelta(seconds=window_seconds) run: list[IncidentCluster] = [] run_key: str | None = None def flush() -> None: if len(run) < 2: return first, last = run[0], run[-1] incident_ids = [incident.id for incident in run] seed = f"{rule_id}|{'|'.join(incident_ids)}".encode() campaign = Campaign.model_construct( id=f"campaign-{hashlib.sha256(seed).hexdigest()[:16]}", rule_id=rule_id, title=_bounded_title(f"Recurring: {first.title}", "Recurring activity"), category=first.category, severity=_max_severity([incident.severity for incident in run]), start=first.start, end=last.end, incident_ids=incident_ids, event_count=sum(incident.original_count for incident in run), reasons=[ ( f"The same rule fired for the same known grouping dimensions in {len(run)} adjacent windows " f"(gaps of at most {window_seconds} second(s))." ), "Each incident stays bounded by its rule span; the campaign only summarizes recurrence and makes no causal claim.", ], ) for incident in run: incident.campaign_id = campaign.id campaigns.append(campaign) for key, incident in keyed: previous = run[-1] if run else None contiguous = ( key is not None and key == run_key and previous is not None and previous.end is not None and incident.start is not None and incident.start - previous.end <= window ) if not contiguous: flush() run = [] run_key = key if key is not None and incident.start is not None: run.append(incident) flush() return campaigns def _incident_sort_key(incident: IncidentCluster) -> tuple[bool, datetime, str, str]: return ( incident.start is None, incident.start or datetime.max.replace(tzinfo=UTC), incident.rule_id, incident.id, ) def correlate_with_campaigns(events: list[NormalizedEvent]) -> tuple[list[IncidentCluster], list[Campaign]]: ordered = sorted(events, key=_event_sort_key) incidents: list[IncidentCluster] = [] campaigns: list[Campaign] = [] consumed: set[str] = set() for rule in RULES: keyed = [(key, _build(rule, group)) for key, group in _groups_for_rule(ordered, rule)] for _, incident in keyed: consumed.update(incident.evidence_event_ids) incidents.extend(incident for _, incident in keyed) campaigns.extend(_campaigns_for(rule.id, rule.window_seconds, keyed)) remaining = [event for event in ordered if event.id not in consumed and _attention(event)] keyed_fallback = [ (key, _build_fallback(group)) for key, group in _windowed_groups(remaining, _always, _fallback_key, FALLBACK_WINDOW_SECONDS) ] incidents.extend(incident for _, incident in keyed_fallback) campaigns.extend(_campaigns_for(FALLBACK_RULE_ID, FALLBACK_WINDOW_SECONDS, keyed_fallback)) # Untimed attention events stay visible as evidence but never join a temporal group. incidents.extend( _build_fallback([event]) for event in remaining if event.timestamp is None or not event.timestamp_valid ) campaigns.sort(key=lambda campaign: (campaign.start, campaign.rule_id, campaign.id)) return sorted(incidents, key=_incident_sort_key), campaigns def correlate(events: list[NormalizedEvent]) -> list[IncidentCluster]: return correlate_with_campaigns(events)[0]