Includes campaign summaries, expanded Windows event correlation, worker-based redaction, updated demo assets and a local-only Compose example. Publish curated EventLens source from 6a2ca12f81a2dcf816c61a86bd8811fcc95e8d8d
694 lines
25 KiB
Python
694 lines
25 KiB
Python
import hashlib
|
|
from collections.abc import Callable
|
|
from dataclasses import dataclass
|
|
from datetime import UTC, datetime, timedelta
|
|
|
|
from .models import Campaign, Category, IncidentCluster, NormalizedEvent, Severity
|
|
from .normalize import (
|
|
is_application_error,
|
|
is_dns_client,
|
|
is_event_log,
|
|
is_kernel_general,
|
|
is_kernel_power,
|
|
is_log_cleared,
|
|
is_security_audit,
|
|
is_service_control_manager,
|
|
is_windows_error_reporting,
|
|
)
|
|
|
|
TitleBuilder = Callable[[list[NormalizedEvent]], str]
|
|
ReasonBuilder = Callable[[list[NormalizedEvent]], str]
|
|
GroupFilter = Callable[[list[NormalizedEvent]], bool]
|
|
Group = tuple[str | None, list[NormalizedEvent]]
|
|
|
|
MAX_TITLE_CHARS = 256
|
|
FALLBACK_RULE_ID = "windows.generic.attention"
|
|
FALLBACK_WINDOW_SECONDS = 300
|
|
SPRAY_MIN_DISTINCT_ACCOUNTS = 3
|
|
SEVERITY_RANK: dict[str, int] = {"info": 0, "warning": 1, "error": 2, "critical": 3}
|
|
LEVEL_SEVERITY: dict[str, Severity] = {"critical": "critical", "error": "error", "warning": "warning"}
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Rule:
|
|
id: str
|
|
category: Category
|
|
severity: Severity
|
|
window_seconds: int
|
|
predicate: Callable[[NormalizedEvent], bool]
|
|
key: Callable[[NormalizedEvent], str | None]
|
|
title_for: TitleBuilder
|
|
reason_for: ReasonBuilder
|
|
accept: GroupFilter | None = None
|
|
|
|
|
|
def _field(event: NormalizedEvent, *names: str) -> str:
|
|
for name in names:
|
|
value = event.fields.get(name, "").strip()
|
|
if value:
|
|
return value
|
|
return ""
|
|
|
|
|
|
def _dimension(value: str) -> str | None:
|
|
normalized = value.strip()
|
|
if not normalized or normalized.casefold() in {
|
|
"unknown",
|
|
"unknown-account",
|
|
"unknown-host",
|
|
"unknown-process",
|
|
"unknown-service",
|
|
"n/a",
|
|
"-",
|
|
}:
|
|
return None
|
|
return normalized
|
|
|
|
|
|
def _host(event: NormalizedEvent) -> str | None:
|
|
return _dimension(event.computer)
|
|
|
|
|
|
def _process(event: NormalizedEvent) -> str | None:
|
|
return _dimension(_field(event, "process", "EventData.AppName", "EventData.Data[0]"))
|
|
|
|
|
|
def _service(event: NormalizedEvent) -> str | None:
|
|
return _dimension(_field(event, "service", "EventData.param1", "UserData.param1"))
|
|
|
|
|
|
def _new_service(event: NormalizedEvent) -> str | None:
|
|
return _dimension(_field(event, "EventData.ServiceName", "service"))
|
|
|
|
|
|
def _account(event: NormalizedEvent) -> str | None:
|
|
return _dimension(_field(event, "EventData.TargetUserName", "TargetUserName"))
|
|
|
|
|
|
def _source_ip(event: NormalizedEvent) -> str | None:
|
|
return _dimension(_field(event, "EventData.IpAddress", "IpAddress"))
|
|
|
|
|
|
def _group_key(*dimensions: str | None) -> str | None:
|
|
if any(value is None for value in dimensions):
|
|
return None
|
|
return "|".join(value.casefold() for value in dimensions if value is not None)
|
|
|
|
|
|
def _bounded_title(value: str, fallback: str) -> str:
|
|
text = " ".join(value.split()) or fallback
|
|
return text if len(text) <= MAX_TITLE_CHARS else f"{text[: MAX_TITLE_CHARS - 1]}…"
|
|
|
|
|
|
def _distinct_accounts(evidence: list[NormalizedEvent]) -> int:
|
|
return len({account.casefold() for event in evidence if (account := _account(event))})
|
|
|
|
|
|
# --- predicates -------------------------------------------------------------
|
|
|
|
|
|
def _crash(event: NormalizedEvent) -> bool:
|
|
return is_application_error(event.provider, event.channel, event.event_id) or is_windows_error_reporting(
|
|
event.provider, event.channel, event.event_id
|
|
)
|
|
|
|
|
|
def _service_failure(event: NormalizedEvent) -> bool:
|
|
return is_service_control_manager(event.provider, event.channel) and event.event_id in {7023, 7024, 7031, 7034}
|
|
|
|
|
|
def _service_installed(event: NormalizedEvent) -> bool:
|
|
return is_service_control_manager(event.provider, event.channel) and event.event_id == 7045
|
|
|
|
|
|
def _dns_timeout(event: NormalizedEvent) -> bool:
|
|
return is_dns_client(event.provider, event.channel) and event.event_id == 1014
|
|
|
|
|
|
def _failed_logon(event: NormalizedEvent) -> bool:
|
|
return is_security_audit(event.provider, event.channel) and event.event_id == 4625
|
|
|
|
|
|
def _account_lockout(event: NormalizedEvent) -> bool:
|
|
return is_security_audit(event.provider, event.channel) and event.event_id == 4740
|
|
|
|
|
|
def _account_change(event: NormalizedEvent) -> bool:
|
|
return is_security_audit(event.provider, event.channel) and event.event_id in {4720, 4728, 4732, 4756}
|
|
|
|
|
|
def _log_cleared(event: NormalizedEvent) -> bool:
|
|
return is_log_cleared(event.provider, event.channel, event.event_id)
|
|
|
|
|
|
def _normal_startup(event: NormalizedEvent) -> bool:
|
|
return (is_kernel_general(event.provider, event.channel) and event.event_id == 12) or (
|
|
is_event_log(event.provider, event.channel) and event.event_id == 6005
|
|
)
|
|
|
|
|
|
def _normal_shutdown(event: NormalizedEvent) -> bool:
|
|
return (is_kernel_general(event.provider, event.channel) and event.event_id == 13) or (
|
|
is_event_log(event.provider, event.channel) and event.event_id == 6006
|
|
)
|
|
|
|
|
|
def _unexpected_shutdown(event: NormalizedEvent) -> bool:
|
|
return (is_kernel_power(event.provider, event.channel) and event.event_id == 41) or (
|
|
is_event_log(event.provider, event.channel) and event.event_id == 6008
|
|
)
|
|
|
|
|
|
# --- titles and reasons -----------------------------------------------------
|
|
|
|
|
|
def _crash_title(evidence: list[NormalizedEvent]) -> str:
|
|
ids = {event.event_id for event in evidence}
|
|
if {1000, 1001}.issubset(ids):
|
|
return "Application crash and reporting sequence"
|
|
if 1000 in ids:
|
|
return "Application crash recorded"
|
|
return "Crash report recorded"
|
|
|
|
|
|
def _crash_reason(evidence: list[NormalizedEvent]) -> str:
|
|
ids = {event.event_id for event in evidence}
|
|
if {1000, 1001}.issubset(ids):
|
|
return "Application Error and Windows Error Reporting records were temporally related; no causal claim is made."
|
|
return "A crash-related provider record was observed; a reporting sequence is not established by this evidence."
|
|
|
|
|
|
def _service_title(evidence: list[NormalizedEvent]) -> str:
|
|
return "Service failure window" if len(evidence) > 1 else "Service failure recorded"
|
|
|
|
|
|
def _service_reason(evidence: list[NormalizedEvent]) -> str:
|
|
service = _service(evidence[0])
|
|
if service is None or _host(evidence[0]) is None:
|
|
return "Service Control Manager reported a service failure; no service/host grouping was applied because a grouping dimension was unavailable."
|
|
return "Service Control Manager reported an abnormal service termination for the same known service and host."
|
|
|
|
|
|
def _service_installed_key(event: NormalizedEvent) -> str | None:
|
|
return _group_key(_host(event), _new_service(event))
|
|
|
|
|
|
def _service_installed_title(evidence: list[NormalizedEvent]) -> str:
|
|
return "New service installed" if len(evidence) == 1 else "New service installation activity"
|
|
|
|
|
|
def _service_installed_reason(evidence: list[NormalizedEvent]) -> str:
|
|
if _service_installed_key(evidence[0]) is None:
|
|
return "Service Control Manager recorded a service installation (7045); no service/host grouping was applied because a grouping dimension was unavailable."
|
|
return "Service Control Manager recorded installation of the same known service on the same host (7045); installation alone is not evidence of malice."
|
|
|
|
|
|
def _dns_title(evidence: list[NormalizedEvent]) -> str:
|
|
return "DNS timeout activity" if len(evidence) > 1 else "DNS timeout recorded"
|
|
|
|
|
|
def _dns_reason(evidence: list[NormalizedEvent]) -> str:
|
|
if _host(evidence[0]) is None:
|
|
return "DNS Client reported a timeout; no host grouping was applied because the computer identity was unavailable."
|
|
return "DNS Client timeout events were observed in the same bounded window on the same known host."
|
|
|
|
|
|
def _auth_title(evidence: list[NormalizedEvent]) -> str:
|
|
if len(evidence) >= 3:
|
|
return "Failed sign-in burst"
|
|
if len(evidence) == 2:
|
|
return "Repeated failed sign-ins"
|
|
return "Failed sign-in recorded"
|
|
|
|
|
|
def _auth_reason(evidence: list[NormalizedEvent]) -> str:
|
|
if _group_key(_host(evidence[0]), _account(evidence[0])) is None:
|
|
return "Security Auditing reported a failed sign-in; no account/host grouping was applied because a grouping dimension was unavailable."
|
|
return "Security Auditing reported failed sign-ins for the same known account and host; this is evidence of activity, not proof of cause."
|
|
|
|
|
|
def _spray_title(evidence: list[NormalizedEvent]) -> str:
|
|
return "Failed sign-ins across multiple accounts from one source"
|
|
|
|
|
|
def _spray_reason(evidence: list[NormalizedEvent]) -> str:
|
|
return (
|
|
f"Security Auditing reported failed sign-ins (4625) for {_distinct_accounts(evidence)} distinct accounts "
|
|
"from the same known source address on the same host; this pattern is consistent with, but does not prove, password spraying."
|
|
)
|
|
|
|
|
|
def _lockout_title(evidence: list[NormalizedEvent]) -> str:
|
|
return "Account lockout" if len(evidence) == 1 else "Repeated account lockouts"
|
|
|
|
|
|
def _lockout_reason(evidence: list[NormalizedEvent]) -> str:
|
|
if _group_key(_host(evidence[0]), _account(evidence[0])) is None:
|
|
return "Security Auditing recorded an account lockout (4740); no account/host grouping was applied because a grouping dimension was unavailable."
|
|
return "Security Auditing recorded lockouts (4740) for the same known account on the same host."
|
|
|
|
|
|
def _account_change_title(evidence: list[NormalizedEvent]) -> str:
|
|
ids = {event.event_id for event in evidence}
|
|
if 4720 in ids and ids & {4728, 4732, 4756}:
|
|
return "Account created and added to a security group"
|
|
if 4720 in ids:
|
|
return "User account created"
|
|
return "Security group membership changed"
|
|
|
|
|
|
def _account_change_reason(evidence: list[NormalizedEvent]) -> str:
|
|
if _host(evidence[0]) is None:
|
|
return "Security Auditing recorded an account or security-group change; no host grouping was applied because the computer identity was unavailable."
|
|
return "Security Auditing recorded account creation or security-group membership changes (4720/4728/4732/4756) on the same known host in one bounded window."
|
|
|
|
|
|
def _log_cleared_title(evidence: list[NormalizedEvent]) -> str:
|
|
return "Event log cleared" if len(evidence) == 1 else "Event logs cleared"
|
|
|
|
|
|
def _log_cleared_reason(evidence: list[NormalizedEvent]) -> str:
|
|
return (
|
|
"EventLog recorded that an event log was cleared (Security 1102 / System 104); records from before this point "
|
|
"may be missing. The record identifies the clearing account only as far as its own fields state."
|
|
)
|
|
|
|
|
|
def _startup_title(evidence: list[NormalizedEvent]) -> str:
|
|
return "Normal startup sequence" if len(evidence) > 1 else "Normal startup evidence"
|
|
|
|
|
|
def _startup_reason(evidence: list[NormalizedEvent]) -> str:
|
|
return "Kernel-General/EventLog startup records were temporally related; no cause is inferred."
|
|
|
|
|
|
def _shutdown_title(evidence: list[NormalizedEvent]) -> str:
|
|
return "Normal shutdown evidence"
|
|
|
|
|
|
def _shutdown_reason(evidence: list[NormalizedEvent]) -> str:
|
|
return "Kernel-General/EventLog shutdown evidence was observed; no cause is inferred."
|
|
|
|
|
|
def _unexpected_title(evidence: list[NormalizedEvent]) -> str:
|
|
return "Unexpected shutdown/restart evidence"
|
|
|
|
|
|
def _unexpected_reason(evidence: list[NormalizedEvent]) -> str:
|
|
return "Kernel-Power/EventLog records indicate an unexpected shutdown or restart; the logs do not establish why it occurred."
|
|
|
|
|
|
RULES = (
|
|
Rule(
|
|
"windows.audit.log-cleared",
|
|
"security",
|
|
"critical",
|
|
300,
|
|
_log_cleared,
|
|
lambda event: _group_key(_host(event)),
|
|
_log_cleared_title,
|
|
_log_cleared_reason,
|
|
),
|
|
Rule(
|
|
"windows.crash.application-report",
|
|
"crash",
|
|
"critical",
|
|
120,
|
|
_crash,
|
|
lambda event: _group_key(_host(event), _process(event)),
|
|
_crash_title,
|
|
_crash_reason,
|
|
),
|
|
Rule(
|
|
"windows.service.failure",
|
|
"service",
|
|
"warning",
|
|
120,
|
|
_service_failure,
|
|
lambda event: _group_key(_host(event), _service(event)),
|
|
_service_title,
|
|
_service_reason,
|
|
),
|
|
Rule(
|
|
"windows.service.installed",
|
|
"service",
|
|
"warning",
|
|
300,
|
|
_service_installed,
|
|
_service_installed_key,
|
|
_service_installed_title,
|
|
_service_installed_reason,
|
|
),
|
|
Rule(
|
|
"windows.network.dns-timeout",
|
|
"network",
|
|
"warning",
|
|
120,
|
|
_dns_timeout,
|
|
lambda event: _group_key(_host(event)),
|
|
_dns_title,
|
|
_dns_reason,
|
|
),
|
|
Rule(
|
|
"windows.auth.failed-logon",
|
|
"authentication",
|
|
"warning",
|
|
120,
|
|
_failed_logon,
|
|
lambda event: _group_key(_host(event), _account(event)),
|
|
_auth_title,
|
|
_auth_reason,
|
|
),
|
|
Rule(
|
|
"windows.auth.password-spray",
|
|
"authentication",
|
|
"warning",
|
|
300,
|
|
_failed_logon,
|
|
lambda event: _group_key(_host(event), _source_ip(event)),
|
|
_spray_title,
|
|
_spray_reason,
|
|
accept=lambda evidence: _distinct_accounts(evidence) >= SPRAY_MIN_DISTINCT_ACCOUNTS,
|
|
),
|
|
Rule(
|
|
"windows.auth.account-lockout",
|
|
"authentication",
|
|
"warning",
|
|
300,
|
|
_account_lockout,
|
|
lambda event: _group_key(_host(event), _account(event)),
|
|
_lockout_title,
|
|
_lockout_reason,
|
|
),
|
|
Rule(
|
|
"windows.account.change",
|
|
"security",
|
|
"warning",
|
|
300,
|
|
_account_change,
|
|
lambda event: _group_key(_host(event)),
|
|
_account_change_title,
|
|
_account_change_reason,
|
|
),
|
|
Rule(
|
|
"windows.system.startup",
|
|
"system",
|
|
"info",
|
|
300,
|
|
_normal_startup,
|
|
lambda event: _group_key(_host(event)),
|
|
_startup_title,
|
|
_startup_reason,
|
|
),
|
|
Rule(
|
|
"windows.system.shutdown.normal",
|
|
"system",
|
|
"info",
|
|
300,
|
|
_normal_shutdown,
|
|
lambda event: _group_key(_host(event)),
|
|
_shutdown_title,
|
|
_shutdown_reason,
|
|
),
|
|
Rule(
|
|
"windows.system.shutdown.unexpected",
|
|
"system",
|
|
"warning",
|
|
300,
|
|
_unexpected_shutdown,
|
|
lambda event: _group_key(_host(event)),
|
|
_unexpected_title,
|
|
_unexpected_reason,
|
|
),
|
|
)
|
|
|
|
|
|
# --- grouping ---------------------------------------------------------------
|
|
|
|
|
|
def _event_sort_key(event: NormalizedEvent) -> tuple[bool, datetime, str]:
|
|
reliable_timestamp = event.timestamp is not None and event.timestamp_valid
|
|
return (
|
|
not reliable_timestamp,
|
|
event.timestamp if reliable_timestamp else datetime.max.replace(tzinfo=UTC),
|
|
event.id,
|
|
)
|
|
|
|
|
|
Signature = tuple[str, str, int | None]
|
|
|
|
|
|
def _signature(event: NormalizedEvent) -> Signature:
|
|
return (event.provider, event.channel, event.event_id)
|
|
|
|
|
|
def _matching(events: list[NormalizedEvent], predicate: Callable[[NormalizedEvent], bool]) -> list[NormalizedEvent]:
|
|
"""Filter events by a rule predicate, evaluating it once per distinct signature.
|
|
|
|
Every rule predicate depends only on provider, channel and event ID, so a
|
|
100,000-event log with a few hundred signatures needs a few hundred
|
|
predicate calls per rule instead of 100,000.
|
|
"""
|
|
verdicts: dict[Signature, bool] = {}
|
|
matched: list[NormalizedEvent] = []
|
|
for event in events:
|
|
signature = _signature(event)
|
|
verdict = verdicts.get(signature)
|
|
if verdict is None:
|
|
verdict = verdicts[signature] = predicate(event)
|
|
if verdict:
|
|
matched.append(event)
|
|
return matched
|
|
|
|
|
|
def _windowed_groups(
|
|
events: list[NormalizedEvent],
|
|
predicate: Callable[[NormalizedEvent], bool],
|
|
key_for: Callable[[NormalizedEvent], str | None],
|
|
window_seconds: int,
|
|
) -> list[Group]:
|
|
"""Bucket timed events by key and split each bucket into bounded windows.
|
|
|
|
A window is anchored on its first event, so a group can never chain beyond
|
|
``window_seconds``. Events whose key is unknown become isolated groups
|
|
(key ``None``) so that missing dimensions never merge unrelated evidence.
|
|
"""
|
|
buckets: dict[str, list[NormalizedEvent]] = {}
|
|
isolated: list[NormalizedEvent] = []
|
|
for event in events:
|
|
if not predicate(event) or event.timestamp is None or not event.timestamp_valid:
|
|
continue
|
|
key = key_for(event)
|
|
if key is None:
|
|
isolated.append(event)
|
|
else:
|
|
buckets.setdefault(key, []).append(event)
|
|
groups: list[Group] = []
|
|
window = timedelta(seconds=window_seconds)
|
|
for key in sorted(buckets):
|
|
ordered = sorted(buckets[key], key=_event_sort_key)
|
|
current: list[NormalizedEvent] = []
|
|
for event in ordered:
|
|
if not current or event.timestamp - current[0].timestamp <= window:
|
|
current.append(event)
|
|
else:
|
|
groups.append((key, current))
|
|
current = [event]
|
|
if current:
|
|
groups.append((key, current))
|
|
groups.extend((None, [event]) for event in sorted(isolated, key=_event_sort_key))
|
|
return groups
|
|
|
|
|
|
def _always(_event: NormalizedEvent) -> bool:
|
|
return True
|
|
|
|
|
|
def _groups_for_rule(events: list[NormalizedEvent], rule: Rule) -> list[Group]:
|
|
groups = _windowed_groups(_matching(events, rule.predicate), _always, rule.key, rule.window_seconds)
|
|
if rule.accept is None:
|
|
return groups
|
|
return [group for group in groups if rule.accept(group[1])]
|
|
|
|
|
|
def _incident_id(rule_id: str, evidence: list[NormalizedEvent]) -> str:
|
|
seed = f"{rule_id}|{'|'.join(event.id for event in evidence)}".encode()
|
|
return f"incident-{hashlib.sha256(seed).hexdigest()[:16]}"
|
|
|
|
|
|
def _signatures(evidence: list[NormalizedEvent]) -> set[tuple[str, int | None, str]]:
|
|
return {(event.provider, event.event_id, event.summary) for event in evidence}
|
|
|
|
|
|
def _compression_reasons(evidence: list[NormalizedEvent]) -> list[str]:
|
|
reasons: list[str] = []
|
|
timed = all(event.timestamp is not None and event.timestamp_valid for event in evidence)
|
|
if len(evidence) > 1 and timed:
|
|
seconds = int((evidence[-1].timestamp - evidence[0].timestamp).total_seconds())
|
|
reasons.append(f"{len(evidence)} evidence events occurred within {seconds} second(s).")
|
|
signatures = _signatures(evidence)
|
|
if len(signatures) < len(evidence):
|
|
reasons.append(f"Repeated signals compress from {len(evidence)} records to {len(signatures)} distinct signature(s).")
|
|
return reasons
|
|
|
|
|
|
def _build(rule: Rule, evidence: list[NormalizedEvent]) -> IncidentCluster:
|
|
return IncidentCluster.model_construct(
|
|
id=_incident_id(rule.id, evidence),
|
|
rule_id=rule.id,
|
|
start=evidence[0].timestamp,
|
|
end=evidence[-1].timestamp,
|
|
title=_bounded_title(rule.title_for(evidence), "Correlated evidence"),
|
|
category=rule.category,
|
|
severity=rule.severity,
|
|
reasons=[rule.reason_for(evidence), *_compression_reasons(evidence)],
|
|
evidence_event_ids=[event.id for event in evidence],
|
|
original_count=len(evidence),
|
|
compressed_count=max(1, len(_signatures(evidence))),
|
|
)
|
|
|
|
|
|
def _attention(event: NormalizedEvent) -> bool:
|
|
return event.level.casefold() in LEVEL_SEVERITY
|
|
|
|
|
|
def _fallback_key(event: NormalizedEvent) -> str | None:
|
|
provider = _dimension(event.provider)
|
|
if provider is None or provider.casefold() == "unknown provider" or event.event_id is None:
|
|
return None
|
|
return _group_key(_host(event), provider, str(event.event_id))
|
|
|
|
|
|
def _max_severity(levels: list[Severity]) -> Severity:
|
|
return max(levels, key=lambda level: SEVERITY_RANK[level])
|
|
|
|
|
|
def _build_fallback(evidence: list[NormalizedEvent]) -> IncidentCluster:
|
|
first = evidence[0]
|
|
timed = first.timestamp is not None and first.timestamp_valid
|
|
if len(evidence) == 1:
|
|
reason = "An error or warning was not matched by a more specific correlation rule."
|
|
else:
|
|
reason = (
|
|
"Errors or warnings with the same provider, event ID and known host were not matched by a more specific "
|
|
"rule and were compressed into one bounded window; repetition is not a causal claim."
|
|
)
|
|
return IncidentCluster.model_construct(
|
|
id=_incident_id(FALLBACK_RULE_ID, evidence),
|
|
rule_id=FALLBACK_RULE_ID,
|
|
start=first.timestamp if timed else None,
|
|
end=evidence[-1].timestamp if timed else None,
|
|
title=_bounded_title(first.summary, "Unclassified attention event"),
|
|
category=first.category,
|
|
severity=_max_severity([LEVEL_SEVERITY.get(event.level.casefold(), "warning") for event in evidence]),
|
|
reasons=[reason, *_compression_reasons(evidence)],
|
|
evidence_event_ids=[event.id for event in evidence],
|
|
original_count=len(evidence),
|
|
compressed_count=max(1, len(_signatures(evidence))),
|
|
)
|
|
|
|
|
|
# --- campaigns --------------------------------------------------------------
|
|
|
|
|
|
def _campaigns_for(
|
|
rule_id: str, window_seconds: int, keyed: list[tuple[str | None, IncidentCluster]]
|
|
) -> list[Campaign]:
|
|
"""Link adjacent windows of one rule and one known key into a summary campaign.
|
|
|
|
``keyed`` is ordered by key and then by time, as produced by
|
|
:func:`_windowed_groups`. Incidents keep their bounded windows; a campaign
|
|
only records that the same rule recurred for the same known dimensions.
|
|
"""
|
|
campaigns: list[Campaign] = []
|
|
window = timedelta(seconds=window_seconds)
|
|
run: list[IncidentCluster] = []
|
|
run_key: str | None = None
|
|
|
|
def flush() -> None:
|
|
if len(run) < 2:
|
|
return
|
|
first, last = run[0], run[-1]
|
|
incident_ids = [incident.id for incident in run]
|
|
seed = f"{rule_id}|{'|'.join(incident_ids)}".encode()
|
|
campaign = Campaign.model_construct(
|
|
id=f"campaign-{hashlib.sha256(seed).hexdigest()[:16]}",
|
|
rule_id=rule_id,
|
|
title=_bounded_title(f"Recurring: {first.title}", "Recurring activity"),
|
|
category=first.category,
|
|
severity=_max_severity([incident.severity for incident in run]),
|
|
start=first.start,
|
|
end=last.end,
|
|
incident_ids=incident_ids,
|
|
event_count=sum(incident.original_count for incident in run),
|
|
reasons=[
|
|
(
|
|
f"The same rule fired for the same known grouping dimensions in {len(run)} adjacent windows "
|
|
f"(gaps of at most {window_seconds} second(s))."
|
|
),
|
|
"Each incident stays bounded by its rule span; the campaign only summarizes recurrence and makes no causal claim.",
|
|
],
|
|
)
|
|
for incident in run:
|
|
incident.campaign_id = campaign.id
|
|
campaigns.append(campaign)
|
|
|
|
for key, incident in keyed:
|
|
previous = run[-1] if run else None
|
|
contiguous = (
|
|
key is not None
|
|
and key == run_key
|
|
and previous is not None
|
|
and previous.end is not None
|
|
and incident.start is not None
|
|
and incident.start - previous.end <= window
|
|
)
|
|
if not contiguous:
|
|
flush()
|
|
run = []
|
|
run_key = key
|
|
if key is not None and incident.start is not None:
|
|
run.append(incident)
|
|
flush()
|
|
return campaigns
|
|
|
|
|
|
def _incident_sort_key(incident: IncidentCluster) -> tuple[bool, datetime, str, str]:
|
|
return (
|
|
incident.start is None,
|
|
incident.start or datetime.max.replace(tzinfo=UTC),
|
|
incident.rule_id,
|
|
incident.id,
|
|
)
|
|
|
|
|
|
def correlate_with_campaigns(events: list[NormalizedEvent]) -> tuple[list[IncidentCluster], list[Campaign]]:
|
|
ordered = sorted(events, key=_event_sort_key)
|
|
incidents: list[IncidentCluster] = []
|
|
campaigns: list[Campaign] = []
|
|
consumed: set[str] = set()
|
|
for rule in RULES:
|
|
keyed = [(key, _build(rule, group)) for key, group in _groups_for_rule(ordered, rule)]
|
|
for _, incident in keyed:
|
|
consumed.update(incident.evidence_event_ids)
|
|
incidents.extend(incident for _, incident in keyed)
|
|
campaigns.extend(_campaigns_for(rule.id, rule.window_seconds, keyed))
|
|
|
|
remaining = [event for event in ordered if event.id not in consumed and _attention(event)]
|
|
keyed_fallback = [
|
|
(key, _build_fallback(group))
|
|
for key, group in _windowed_groups(remaining, _always, _fallback_key, FALLBACK_WINDOW_SECONDS)
|
|
]
|
|
incidents.extend(incident for _, incident in keyed_fallback)
|
|
campaigns.extend(_campaigns_for(FALLBACK_RULE_ID, FALLBACK_WINDOW_SECONDS, keyed_fallback))
|
|
# Untimed attention events stay visible as evidence but never join a temporal group.
|
|
incidents.extend(
|
|
_build_fallback([event]) for event in remaining if event.timestamp is None or not event.timestamp_valid
|
|
)
|
|
campaigns.sort(key=lambda campaign: (campaign.start, campaign.rule_id, campaign.id))
|
|
return sorted(incidents, key=_incident_sort_key), campaigns
|
|
|
|
|
|
def correlate(events: list[NormalizedEvent]) -> list[IncidentCluster]:
|
|
return correlate_with_campaigns(events)[0]
|