Files
2026-11-20 12:20:18 +00:00

694 lines
25 KiB
Python

import hashlib
from collections.abc import Callable
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from .models import Campaign, Category, IncidentCluster, NormalizedEvent, Severity
from .normalize import (
is_application_error,
is_dns_client,
is_event_log,
is_kernel_general,
is_kernel_power,
is_log_cleared,
is_security_audit,
is_service_control_manager,
is_windows_error_reporting,
)
TitleBuilder = Callable[[list[NormalizedEvent]], str]
ReasonBuilder = Callable[[list[NormalizedEvent]], str]
GroupFilter = Callable[[list[NormalizedEvent]], bool]
Group = tuple[str | None, list[NormalizedEvent]]
MAX_TITLE_CHARS = 256
FALLBACK_RULE_ID = "windows.generic.attention"
FALLBACK_WINDOW_SECONDS = 300
SPRAY_MIN_DISTINCT_ACCOUNTS = 3
SEVERITY_RANK: dict[str, int] = {"info": 0, "warning": 1, "error": 2, "critical": 3}
LEVEL_SEVERITY: dict[str, Severity] = {"critical": "critical", "error": "error", "warning": "warning"}
@dataclass(frozen=True)
class Rule:
id: str
category: Category
severity: Severity
window_seconds: int
predicate: Callable[[NormalizedEvent], bool]
key: Callable[[NormalizedEvent], str | None]
title_for: TitleBuilder
reason_for: ReasonBuilder
accept: GroupFilter | None = None
def _field(event: NormalizedEvent, *names: str) -> str:
for name in names:
value = event.fields.get(name, "").strip()
if value:
return value
return ""
def _dimension(value: str) -> str | None:
normalized = value.strip()
if not normalized or normalized.casefold() in {
"unknown",
"unknown-account",
"unknown-host",
"unknown-process",
"unknown-service",
"n/a",
"-",
}:
return None
return normalized
def _host(event: NormalizedEvent) -> str | None:
return _dimension(event.computer)
def _process(event: NormalizedEvent) -> str | None:
return _dimension(_field(event, "process", "EventData.AppName", "EventData.Data[0]"))
def _service(event: NormalizedEvent) -> str | None:
return _dimension(_field(event, "service", "EventData.param1", "UserData.param1"))
def _new_service(event: NormalizedEvent) -> str | None:
return _dimension(_field(event, "EventData.ServiceName", "service"))
def _account(event: NormalizedEvent) -> str | None:
return _dimension(_field(event, "EventData.TargetUserName", "TargetUserName"))
def _source_ip(event: NormalizedEvent) -> str | None:
return _dimension(_field(event, "EventData.IpAddress", "IpAddress"))
def _group_key(*dimensions: str | None) -> str | None:
if any(value is None for value in dimensions):
return None
return "|".join(value.casefold() for value in dimensions if value is not None)
def _bounded_title(value: str, fallback: str) -> str:
text = " ".join(value.split()) or fallback
return text if len(text) <= MAX_TITLE_CHARS else f"{text[: MAX_TITLE_CHARS - 1]}…"
def _distinct_accounts(evidence: list[NormalizedEvent]) -> int:
return len({account.casefold() for event in evidence if (account := _account(event))})
# --- predicates -------------------------------------------------------------
def _crash(event: NormalizedEvent) -> bool:
return is_application_error(event.provider, event.channel, event.event_id) or is_windows_error_reporting(
event.provider, event.channel, event.event_id
)
def _service_failure(event: NormalizedEvent) -> bool:
return is_service_control_manager(event.provider, event.channel) and event.event_id in {7023, 7024, 7031, 7034}
def _service_installed(event: NormalizedEvent) -> bool:
return is_service_control_manager(event.provider, event.channel) and event.event_id == 7045
def _dns_timeout(event: NormalizedEvent) -> bool:
return is_dns_client(event.provider, event.channel) and event.event_id == 1014
def _failed_logon(event: NormalizedEvent) -> bool:
return is_security_audit(event.provider, event.channel) and event.event_id == 4625
def _account_lockout(event: NormalizedEvent) -> bool:
return is_security_audit(event.provider, event.channel) and event.event_id == 4740
def _account_change(event: NormalizedEvent) -> bool:
return is_security_audit(event.provider, event.channel) and event.event_id in {4720, 4728, 4732, 4756}
def _log_cleared(event: NormalizedEvent) -> bool:
return is_log_cleared(event.provider, event.channel, event.event_id)
def _normal_startup(event: NormalizedEvent) -> bool:
return (is_kernel_general(event.provider, event.channel) and event.event_id == 12) or (
is_event_log(event.provider, event.channel) and event.event_id == 6005
)
def _normal_shutdown(event: NormalizedEvent) -> bool:
return (is_kernel_general(event.provider, event.channel) and event.event_id == 13) or (
is_event_log(event.provider, event.channel) and event.event_id == 6006
)
def _unexpected_shutdown(event: NormalizedEvent) -> bool:
return (is_kernel_power(event.provider, event.channel) and event.event_id == 41) or (
is_event_log(event.provider, event.channel) and event.event_id == 6008
)
# --- titles and reasons -----------------------------------------------------
def _crash_title(evidence: list[NormalizedEvent]) -> str:
ids = {event.event_id for event in evidence}
if {1000, 1001}.issubset(ids):
return "Application crash and reporting sequence"
if 1000 in ids:
return "Application crash recorded"
return "Crash report recorded"
def _crash_reason(evidence: list[NormalizedEvent]) -> str:
ids = {event.event_id for event in evidence}
if {1000, 1001}.issubset(ids):
return "Application Error and Windows Error Reporting records were temporally related; no causal claim is made."
return "A crash-related provider record was observed; a reporting sequence is not established by this evidence."
def _service_title(evidence: list[NormalizedEvent]) -> str:
return "Service failure window" if len(evidence) > 1 else "Service failure recorded"
def _service_reason(evidence: list[NormalizedEvent]) -> str:
service = _service(evidence[0])
if service is None or _host(evidence[0]) is None:
return "Service Control Manager reported a service failure; no service/host grouping was applied because a grouping dimension was unavailable."
return "Service Control Manager reported an abnormal service termination for the same known service and host."
def _service_installed_key(event: NormalizedEvent) -> str | None:
return _group_key(_host(event), _new_service(event))
def _service_installed_title(evidence: list[NormalizedEvent]) -> str:
return "New service installed" if len(evidence) == 1 else "New service installation activity"
def _service_installed_reason(evidence: list[NormalizedEvent]) -> str:
if _service_installed_key(evidence[0]) is None:
return "Service Control Manager recorded a service installation (7045); no service/host grouping was applied because a grouping dimension was unavailable."
return "Service Control Manager recorded installation of the same known service on the same host (7045); installation alone is not evidence of malice."
def _dns_title(evidence: list[NormalizedEvent]) -> str:
return "DNS timeout activity" if len(evidence) > 1 else "DNS timeout recorded"
def _dns_reason(evidence: list[NormalizedEvent]) -> str:
if _host(evidence[0]) is None:
return "DNS Client reported a timeout; no host grouping was applied because the computer identity was unavailable."
return "DNS Client timeout events were observed in the same bounded window on the same known host."
def _auth_title(evidence: list[NormalizedEvent]) -> str:
if len(evidence) >= 3:
return "Failed sign-in burst"
if len(evidence) == 2:
return "Repeated failed sign-ins"
return "Failed sign-in recorded"
def _auth_reason(evidence: list[NormalizedEvent]) -> str:
if _group_key(_host(evidence[0]), _account(evidence[0])) is None:
return "Security Auditing reported a failed sign-in; no account/host grouping was applied because a grouping dimension was unavailable."
return "Security Auditing reported failed sign-ins for the same known account and host; this is evidence of activity, not proof of cause."
def _spray_title(evidence: list[NormalizedEvent]) -> str:
return "Failed sign-ins across multiple accounts from one source"
def _spray_reason(evidence: list[NormalizedEvent]) -> str:
return (
f"Security Auditing reported failed sign-ins (4625) for {_distinct_accounts(evidence)} distinct accounts "
"from the same known source address on the same host; this pattern is consistent with, but does not prove, password spraying."
)
def _lockout_title(evidence: list[NormalizedEvent]) -> str:
return "Account lockout" if len(evidence) == 1 else "Repeated account lockouts"
def _lockout_reason(evidence: list[NormalizedEvent]) -> str:
if _group_key(_host(evidence[0]), _account(evidence[0])) is None:
return "Security Auditing recorded an account lockout (4740); no account/host grouping was applied because a grouping dimension was unavailable."
return "Security Auditing recorded lockouts (4740) for the same known account on the same host."
def _account_change_title(evidence: list[NormalizedEvent]) -> str:
ids = {event.event_id for event in evidence}
if 4720 in ids and ids & {4728, 4732, 4756}:
return "Account created and added to a security group"
if 4720 in ids:
return "User account created"
return "Security group membership changed"
def _account_change_reason(evidence: list[NormalizedEvent]) -> str:
if _host(evidence[0]) is None:
return "Security Auditing recorded an account or security-group change; no host grouping was applied because the computer identity was unavailable."
return "Security Auditing recorded account creation or security-group membership changes (4720/4728/4732/4756) on the same known host in one bounded window."
def _log_cleared_title(evidence: list[NormalizedEvent]) -> str:
return "Event log cleared" if len(evidence) == 1 else "Event logs cleared"
def _log_cleared_reason(evidence: list[NormalizedEvent]) -> str:
return (
"EventLog recorded that an event log was cleared (Security 1102 / System 104); records from before this point "
"may be missing. The record identifies the clearing account only as far as its own fields state."
)
def _startup_title(evidence: list[NormalizedEvent]) -> str:
return "Normal startup sequence" if len(evidence) > 1 else "Normal startup evidence"
def _startup_reason(evidence: list[NormalizedEvent]) -> str:
return "Kernel-General/EventLog startup records were temporally related; no cause is inferred."
def _shutdown_title(evidence: list[NormalizedEvent]) -> str:
return "Normal shutdown evidence"
def _shutdown_reason(evidence: list[NormalizedEvent]) -> str:
return "Kernel-General/EventLog shutdown evidence was observed; no cause is inferred."
def _unexpected_title(evidence: list[NormalizedEvent]) -> str:
return "Unexpected shutdown/restart evidence"
def _unexpected_reason(evidence: list[NormalizedEvent]) -> str:
return "Kernel-Power/EventLog records indicate an unexpected shutdown or restart; the logs do not establish why it occurred."
RULES = (
Rule(
"windows.audit.log-cleared",
"security",
"critical",
300,
_log_cleared,
lambda event: _group_key(_host(event)),
_log_cleared_title,
_log_cleared_reason,
),
Rule(
"windows.crash.application-report",
"crash",
"critical",
120,
_crash,
lambda event: _group_key(_host(event), _process(event)),
_crash_title,
_crash_reason,
),
Rule(
"windows.service.failure",
"service",
"warning",
120,
_service_failure,
lambda event: _group_key(_host(event), _service(event)),
_service_title,
_service_reason,
),
Rule(
"windows.service.installed",
"service",
"warning",
300,
_service_installed,
_service_installed_key,
_service_installed_title,
_service_installed_reason,
),
Rule(
"windows.network.dns-timeout",
"network",
"warning",
120,
_dns_timeout,
lambda event: _group_key(_host(event)),
_dns_title,
_dns_reason,
),
Rule(
"windows.auth.failed-logon",
"authentication",
"warning",
120,
_failed_logon,
lambda event: _group_key(_host(event), _account(event)),
_auth_title,
_auth_reason,
),
Rule(
"windows.auth.password-spray",
"authentication",
"warning",
300,
_failed_logon,
lambda event: _group_key(_host(event), _source_ip(event)),
_spray_title,
_spray_reason,
accept=lambda evidence: _distinct_accounts(evidence) >= SPRAY_MIN_DISTINCT_ACCOUNTS,
),
Rule(
"windows.auth.account-lockout",
"authentication",
"warning",
300,
_account_lockout,
lambda event: _group_key(_host(event), _account(event)),
_lockout_title,
_lockout_reason,
),
Rule(
"windows.account.change",
"security",
"warning",
300,
_account_change,
lambda event: _group_key(_host(event)),
_account_change_title,
_account_change_reason,
),
Rule(
"windows.system.startup",
"system",
"info",
300,
_normal_startup,
lambda event: _group_key(_host(event)),
_startup_title,
_startup_reason,
),
Rule(
"windows.system.shutdown.normal",
"system",
"info",
300,
_normal_shutdown,
lambda event: _group_key(_host(event)),
_shutdown_title,
_shutdown_reason,
),
Rule(
"windows.system.shutdown.unexpected",
"system",
"warning",
300,
_unexpected_shutdown,
lambda event: _group_key(_host(event)),
_unexpected_title,
_unexpected_reason,
),
)
# --- grouping ---------------------------------------------------------------
def _event_sort_key(event: NormalizedEvent) -> tuple[bool, datetime, str]:
reliable_timestamp = event.timestamp is not None and event.timestamp_valid
return (
not reliable_timestamp,
event.timestamp if reliable_timestamp else datetime.max.replace(tzinfo=UTC),
event.id,
)
Signature = tuple[str, str, int | None]
def _signature(event: NormalizedEvent) -> Signature:
return (event.provider, event.channel, event.event_id)
def _matching(events: list[NormalizedEvent], predicate: Callable[[NormalizedEvent], bool]) -> list[NormalizedEvent]:
"""Filter events by a rule predicate, evaluating it once per distinct signature.
Every rule predicate depends only on provider, channel and event ID, so a
100,000-event log with a few hundred signatures needs a few hundred
predicate calls per rule instead of 100,000.
"""
verdicts: dict[Signature, bool] = {}
matched: list[NormalizedEvent] = []
for event in events:
signature = _signature(event)
verdict = verdicts.get(signature)
if verdict is None:
verdict = verdicts[signature] = predicate(event)
if verdict:
matched.append(event)
return matched
def _windowed_groups(
events: list[NormalizedEvent],
predicate: Callable[[NormalizedEvent], bool],
key_for: Callable[[NormalizedEvent], str | None],
window_seconds: int,
) -> list[Group]:
"""Bucket timed events by key and split each bucket into bounded windows.
A window is anchored on its first event, so a group can never chain beyond
``window_seconds``. Events whose key is unknown become isolated groups
(key ``None``) so that missing dimensions never merge unrelated evidence.
"""
buckets: dict[str, list[NormalizedEvent]] = {}
isolated: list[NormalizedEvent] = []
for event in events:
if not predicate(event) or event.timestamp is None or not event.timestamp_valid:
continue
key = key_for(event)
if key is None:
isolated.append(event)
else:
buckets.setdefault(key, []).append(event)
groups: list[Group] = []
window = timedelta(seconds=window_seconds)
for key in sorted(buckets):
ordered = sorted(buckets[key], key=_event_sort_key)
current: list[NormalizedEvent] = []
for event in ordered:
if not current or event.timestamp - current[0].timestamp <= window:
current.append(event)
else:
groups.append((key, current))
current = [event]
if current:
groups.append((key, current))
groups.extend((None, [event]) for event in sorted(isolated, key=_event_sort_key))
return groups
def _always(_event: NormalizedEvent) -> bool:
return True
def _groups_for_rule(events: list[NormalizedEvent], rule: Rule) -> list[Group]:
groups = _windowed_groups(_matching(events, rule.predicate), _always, rule.key, rule.window_seconds)
if rule.accept is None:
return groups
return [group for group in groups if rule.accept(group[1])]
def _incident_id(rule_id: str, evidence: list[NormalizedEvent]) -> str:
seed = f"{rule_id}|{'|'.join(event.id for event in evidence)}".encode()
return f"incident-{hashlib.sha256(seed).hexdigest()[:16]}"
def _signatures(evidence: list[NormalizedEvent]) -> set[tuple[str, int | None, str]]:
return {(event.provider, event.event_id, event.summary) for event in evidence}
def _compression_reasons(evidence: list[NormalizedEvent]) -> list[str]:
reasons: list[str] = []
timed = all(event.timestamp is not None and event.timestamp_valid for event in evidence)
if len(evidence) > 1 and timed:
seconds = int((evidence[-1].timestamp - evidence[0].timestamp).total_seconds())
reasons.append(f"{len(evidence)} evidence events occurred within {seconds} second(s).")
signatures = _signatures(evidence)
if len(signatures) < len(evidence):
reasons.append(f"Repeated signals compress from {len(evidence)} records to {len(signatures)} distinct signature(s).")
return reasons
def _build(rule: Rule, evidence: list[NormalizedEvent]) -> IncidentCluster:
return IncidentCluster.model_construct(
id=_incident_id(rule.id, evidence),
rule_id=rule.id,
start=evidence[0].timestamp,
end=evidence[-1].timestamp,
title=_bounded_title(rule.title_for(evidence), "Correlated evidence"),
category=rule.category,
severity=rule.severity,
reasons=[rule.reason_for(evidence), *_compression_reasons(evidence)],
evidence_event_ids=[event.id for event in evidence],
original_count=len(evidence),
compressed_count=max(1, len(_signatures(evidence))),
)
def _attention(event: NormalizedEvent) -> bool:
return event.level.casefold() in LEVEL_SEVERITY
def _fallback_key(event: NormalizedEvent) -> str | None:
provider = _dimension(event.provider)
if provider is None or provider.casefold() == "unknown provider" or event.event_id is None:
return None
return _group_key(_host(event), provider, str(event.event_id))
def _max_severity(levels: list[Severity]) -> Severity:
return max(levels, key=lambda level: SEVERITY_RANK[level])
def _build_fallback(evidence: list[NormalizedEvent]) -> IncidentCluster:
first = evidence[0]
timed = first.timestamp is not None and first.timestamp_valid
if len(evidence) == 1:
reason = "An error or warning was not matched by a more specific correlation rule."
else:
reason = (
"Errors or warnings with the same provider, event ID and known host were not matched by a more specific "
"rule and were compressed into one bounded window; repetition is not a causal claim."
)
return IncidentCluster.model_construct(
id=_incident_id(FALLBACK_RULE_ID, evidence),
rule_id=FALLBACK_RULE_ID,
start=first.timestamp if timed else None,
end=evidence[-1].timestamp if timed else None,
title=_bounded_title(first.summary, "Unclassified attention event"),
category=first.category,
severity=_max_severity([LEVEL_SEVERITY.get(event.level.casefold(), "warning") for event in evidence]),
reasons=[reason, *_compression_reasons(evidence)],
evidence_event_ids=[event.id for event in evidence],
original_count=len(evidence),
compressed_count=max(1, len(_signatures(evidence))),
)
# --- campaigns --------------------------------------------------------------
def _campaigns_for(
rule_id: str, window_seconds: int, keyed: list[tuple[str | None, IncidentCluster]]
) -> list[Campaign]:
"""Link adjacent windows of one rule and one known key into a summary campaign.
``keyed`` is ordered by key and then by time, as produced by
:func:`_windowed_groups`. Incidents keep their bounded windows; a campaign
only records that the same rule recurred for the same known dimensions.
"""
campaigns: list[Campaign] = []
window = timedelta(seconds=window_seconds)
run: list[IncidentCluster] = []
run_key: str | None = None
def flush() -> None:
if len(run) < 2:
return
first, last = run[0], run[-1]
incident_ids = [incident.id for incident in run]
seed = f"{rule_id}|{'|'.join(incident_ids)}".encode()
campaign = Campaign.model_construct(
id=f"campaign-{hashlib.sha256(seed).hexdigest()[:16]}",
rule_id=rule_id,
title=_bounded_title(f"Recurring: {first.title}", "Recurring activity"),
category=first.category,
severity=_max_severity([incident.severity for incident in run]),
start=first.start,
end=last.end,
incident_ids=incident_ids,
event_count=sum(incident.original_count for incident in run),
reasons=[
(
f"The same rule fired for the same known grouping dimensions in {len(run)} adjacent windows "
f"(gaps of at most {window_seconds} second(s))."
),
"Each incident stays bounded by its rule span; the campaign only summarizes recurrence and makes no causal claim.",
],
)
for incident in run:
incident.campaign_id = campaign.id
campaigns.append(campaign)
for key, incident in keyed:
previous = run[-1] if run else None
contiguous = (
key is not None
and key == run_key
and previous is not None
and previous.end is not None
and incident.start is not None
and incident.start - previous.end <= window
)
if not contiguous:
flush()
run = []
run_key = key
if key is not None and incident.start is not None:
run.append(incident)
flush()
return campaigns
def _incident_sort_key(incident: IncidentCluster) -> tuple[bool, datetime, str, str]:
return (
incident.start is None,
incident.start or datetime.max.replace(tzinfo=UTC),
incident.rule_id,
incident.id,
)
def correlate_with_campaigns(events: list[NormalizedEvent]) -> tuple[list[IncidentCluster], list[Campaign]]:
ordered = sorted(events, key=_event_sort_key)
incidents: list[IncidentCluster] = []
campaigns: list[Campaign] = []
consumed: set[str] = set()
for rule in RULES:
keyed = [(key, _build(rule, group)) for key, group in _groups_for_rule(ordered, rule)]
for _, incident in keyed:
consumed.update(incident.evidence_event_ids)
incidents.extend(incident for _, incident in keyed)
campaigns.extend(_campaigns_for(rule.id, rule.window_seconds, keyed))
remaining = [event for event in ordered if event.id not in consumed and _attention(event)]
keyed_fallback = [
(key, _build_fallback(group))
for key, group in _windowed_groups(remaining, _always, _fallback_key, FALLBACK_WINDOW_SECONDS)
]
incidents.extend(incident for _, incident in keyed_fallback)
campaigns.extend(_campaigns_for(FALLBACK_RULE_ID, FALLBACK_WINDOW_SECONDS, keyed_fallback))
# Untimed attention events stay visible as evidence but never join a temporal group.
incidents.extend(
_build_fallback([event]) for event in remaining if event.timestamp is None or not event.timestamp_valid
)
campaigns.sort(key=lambda campaign: (campaign.start, campaign.rule_id, campaign.id))
return sorted(incidents, key=_incident_sort_key), campaigns
def correlate(events: list[NormalizedEvent]) -> list[IncidentCluster]:
return correlate_with_campaigns(events)[0]