Publish EventLens v1.1.0 (#1)
This commit was merged in pull request #1.
This commit is contained in:
1 parent
c0ade21029
commit
87432f3283
63 files changed
+7381
-1420
No files matched your search
+387
-56
@@ -3,13 +3,14 @@ from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from .models import Category, IncidentCluster, NormalizedEvent
|
||||
from .models import Campaign, Category, IncidentCluster, NormalizedEvent, Severity
|
||||
from .normalize import (
|
||||
is_application_error,
|
||||
is_dns_client,
|
||||
is_event_log,
|
||||
is_kernel_general,
|
||||
is_kernel_power,
|
||||
is_log_cleared,
|
||||
is_security_audit,
|
||||
is_service_control_manager,
|
||||
is_windows_error_reporting,
|
||||
@@ -17,18 +18,28 @@ from .normalize import (
|
||||
|
||||
TitleBuilder = Callable[[list[NormalizedEvent]], str]
|
||||
ReasonBuilder = Callable[[list[NormalizedEvent]], str]
|
||||
GroupFilter = Callable[[list[NormalizedEvent]], bool]
|
||||
Group = tuple[str | None, list[NormalizedEvent]]
|
||||
|
||||
MAX_TITLE_CHARS = 256
|
||||
FALLBACK_RULE_ID = "windows.generic.attention"
|
||||
FALLBACK_WINDOW_SECONDS = 300
|
||||
SPRAY_MIN_DISTINCT_ACCOUNTS = 3
|
||||
SEVERITY_RANK: dict[str, int] = {"info": 0, "warning": 1, "error": 2, "critical": 3}
|
||||
LEVEL_SEVERITY: dict[str, Severity] = {"critical": "critical", "error": "error", "warning": "warning"}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Rule:
|
||||
id: str
|
||||
category: Category
|
||||
severity: str
|
||||
severity: Severity
|
||||
window_seconds: int
|
||||
predicate: Callable[[NormalizedEvent], bool]
|
||||
key: Callable[[NormalizedEvent], str | None]
|
||||
title_for: TitleBuilder
|
||||
reason_for: ReasonBuilder
|
||||
accept: GroupFilter | None = None
|
||||
|
||||
|
||||
def _field(event: NormalizedEvent, *names: str) -> str:
|
||||
@@ -66,16 +77,36 @@ def _service(event: NormalizedEvent) -> str | None:
|
||||
return _dimension(_field(event, "service", "EventData.param1", "UserData.param1"))
|
||||
|
||||
|
||||
def _new_service(event: NormalizedEvent) -> str | None:
|
||||
return _dimension(_field(event, "EventData.ServiceName", "service"))
|
||||
|
||||
|
||||
def _account(event: NormalizedEvent) -> str | None:
|
||||
return _dimension(_field(event, "EventData.TargetUserName", "TargetUserName"))
|
||||
|
||||
|
||||
def _source_ip(event: NormalizedEvent) -> str | None:
|
||||
return _dimension(_field(event, "EventData.IpAddress", "IpAddress"))
|
||||
|
||||
|
||||
def _group_key(*dimensions: str | None) -> str | None:
|
||||
if any(value is None for value in dimensions):
|
||||
return None
|
||||
return "|".join(value.casefold() for value in dimensions if value is not None)
|
||||
|
||||
|
||||
def _bounded_title(value: str, fallback: str) -> str:
|
||||
text = " ".join(value.split()) or fallback
|
||||
return text if len(text) <= MAX_TITLE_CHARS else f"{text[: MAX_TITLE_CHARS - 1]}…"
|
||||
|
||||
|
||||
def _distinct_accounts(evidence: list[NormalizedEvent]) -> int:
|
||||
return len({account.casefold() for event in evidence if (account := _account(event))})
|
||||
|
||||
|
||||
# --- predicates -------------------------------------------------------------
|
||||
|
||||
|
||||
def _crash(event: NormalizedEvent) -> bool:
|
||||
return is_application_error(event.provider, event.channel, event.event_id) or is_windows_error_reporting(
|
||||
event.provider, event.channel, event.event_id
|
||||
@@ -86,6 +117,10 @@ def _service_failure(event: NormalizedEvent) -> bool:
|
||||
return is_service_control_manager(event.provider, event.channel) and event.event_id in {7023, 7024, 7031, 7034}
|
||||
|
||||
|
||||
def _service_installed(event: NormalizedEvent) -> bool:
|
||||
return is_service_control_manager(event.provider, event.channel) and event.event_id == 7045
|
||||
|
||||
|
||||
def _dns_timeout(event: NormalizedEvent) -> bool:
|
||||
return is_dns_client(event.provider, event.channel) and event.event_id == 1014
|
||||
|
||||
@@ -94,6 +129,18 @@ def _failed_logon(event: NormalizedEvent) -> bool:
|
||||
return is_security_audit(event.provider, event.channel) and event.event_id == 4625
|
||||
|
||||
|
||||
def _account_lockout(event: NormalizedEvent) -> bool:
|
||||
return is_security_audit(event.provider, event.channel) and event.event_id == 4740
|
||||
|
||||
|
||||
def _account_change(event: NormalizedEvent) -> bool:
|
||||
return is_security_audit(event.provider, event.channel) and event.event_id in {4720, 4728, 4732, 4756}
|
||||
|
||||
|
||||
def _log_cleared(event: NormalizedEvent) -> bool:
|
||||
return is_log_cleared(event.provider, event.channel, event.event_id)
|
||||
|
||||
|
||||
def _normal_startup(event: NormalizedEvent) -> bool:
|
||||
return (is_kernel_general(event.provider, event.channel) and event.event_id == 12) or (
|
||||
is_event_log(event.provider, event.channel) and event.event_id == 6005
|
||||
@@ -112,6 +159,9 @@ def _unexpected_shutdown(event: NormalizedEvent) -> bool:
|
||||
)
|
||||
|
||||
|
||||
# --- titles and reasons -----------------------------------------------------
|
||||
|
||||
|
||||
def _crash_title(evidence: list[NormalizedEvent]) -> str:
|
||||
ids = {event.event_id for event in evidence}
|
||||
if {1000, 1001}.issubset(ids):
|
||||
@@ -139,6 +189,20 @@ def _service_reason(evidence: list[NormalizedEvent]) -> str:
|
||||
return "Service Control Manager reported an abnormal service termination for the same known service and host."
|
||||
|
||||
|
||||
def _service_installed_key(event: NormalizedEvent) -> str | None:
|
||||
return _group_key(_host(event), _new_service(event))
|
||||
|
||||
|
||||
def _service_installed_title(evidence: list[NormalizedEvent]) -> str:
|
||||
return "New service installed" if len(evidence) == 1 else "New service installation activity"
|
||||
|
||||
|
||||
def _service_installed_reason(evidence: list[NormalizedEvent]) -> str:
|
||||
if _service_installed_key(evidence[0]) is None:
|
||||
return "Service Control Manager recorded a service installation (7045); no service/host grouping was applied because a grouping dimension was unavailable."
|
||||
return "Service Control Manager recorded installation of the same known service on the same host (7045); installation alone is not evidence of malice."
|
||||
|
||||
|
||||
def _dns_title(evidence: list[NormalizedEvent]) -> str:
|
||||
return "DNS timeout activity" if len(evidence) > 1 else "DNS timeout recorded"
|
||||
|
||||
@@ -163,6 +227,53 @@ def _auth_reason(evidence: list[NormalizedEvent]) -> str:
|
||||
return "Security Auditing reported failed sign-ins for the same known account and host; this is evidence of activity, not proof of cause."
|
||||
|
||||
|
||||
def _spray_title(evidence: list[NormalizedEvent]) -> str:
|
||||
return "Failed sign-ins across multiple accounts from one source"
|
||||
|
||||
|
||||
def _spray_reason(evidence: list[NormalizedEvent]) -> str:
|
||||
return (
|
||||
f"Security Auditing reported failed sign-ins (4625) for {_distinct_accounts(evidence)} distinct accounts "
|
||||
"from the same known source address on the same host; this pattern is consistent with, but does not prove, password spraying."
|
||||
)
|
||||
|
||||
|
||||
def _lockout_title(evidence: list[NormalizedEvent]) -> str:
|
||||
return "Account lockout" if len(evidence) == 1 else "Repeated account lockouts"
|
||||
|
||||
|
||||
def _lockout_reason(evidence: list[NormalizedEvent]) -> str:
|
||||
if _group_key(_host(evidence[0]), _account(evidence[0])) is None:
|
||||
return "Security Auditing recorded an account lockout (4740); no account/host grouping was applied because a grouping dimension was unavailable."
|
||||
return "Security Auditing recorded lockouts (4740) for the same known account on the same host."
|
||||
|
||||
|
||||
def _account_change_title(evidence: list[NormalizedEvent]) -> str:
|
||||
ids = {event.event_id for event in evidence}
|
||||
if 4720 in ids and ids & {4728, 4732, 4756}:
|
||||
return "Account created and added to a security group"
|
||||
if 4720 in ids:
|
||||
return "User account created"
|
||||
return "Security group membership changed"
|
||||
|
||||
|
||||
def _account_change_reason(evidence: list[NormalizedEvent]) -> str:
|
||||
if _host(evidence[0]) is None:
|
||||
return "Security Auditing recorded an account or security-group change; no host grouping was applied because the computer identity was unavailable."
|
||||
return "Security Auditing recorded account creation or security-group membership changes (4720/4728/4732/4756) on the same known host in one bounded window."
|
||||
|
||||
|
||||
def _log_cleared_title(evidence: list[NormalizedEvent]) -> str:
|
||||
return "Event log cleared" if len(evidence) == 1 else "Event logs cleared"
|
||||
|
||||
|
||||
def _log_cleared_reason(evidence: list[NormalizedEvent]) -> str:
|
||||
return (
|
||||
"EventLog recorded that an event log was cleared (Security 1102 / System 104); records from before this point "
|
||||
"may be missing. The record identifies the clearing account only as far as its own fields state."
|
||||
)
|
||||
|
||||
|
||||
def _startup_title(evidence: list[NormalizedEvent]) -> str:
|
||||
return "Normal startup sequence" if len(evidence) > 1 else "Normal startup evidence"
|
||||
|
||||
@@ -188,6 +299,16 @@ def _unexpected_reason(evidence: list[NormalizedEvent]) -> str:
|
||||
|
||||
|
||||
RULES = (
|
||||
Rule(
|
||||
"windows.audit.log-cleared",
|
||||
"security",
|
||||
"critical",
|
||||
300,
|
||||
_log_cleared,
|
||||
lambda event: _group_key(_host(event)),
|
||||
_log_cleared_title,
|
||||
_log_cleared_reason,
|
||||
),
|
||||
Rule(
|
||||
"windows.crash.application-report",
|
||||
"crash",
|
||||
@@ -208,6 +329,16 @@ RULES = (
|
||||
_service_title,
|
||||
_service_reason,
|
||||
),
|
||||
Rule(
|
||||
"windows.service.installed",
|
||||
"service",
|
||||
"warning",
|
||||
300,
|
||||
_service_installed,
|
||||
_service_installed_key,
|
||||
_service_installed_title,
|
||||
_service_installed_reason,
|
||||
),
|
||||
Rule(
|
||||
"windows.network.dns-timeout",
|
||||
"network",
|
||||
@@ -228,6 +359,37 @@ RULES = (
|
||||
_auth_title,
|
||||
_auth_reason,
|
||||
),
|
||||
Rule(
|
||||
"windows.auth.password-spray",
|
||||
"authentication",
|
||||
"warning",
|
||||
300,
|
||||
_failed_logon,
|
||||
lambda event: _group_key(_host(event), _source_ip(event)),
|
||||
_spray_title,
|
||||
_spray_reason,
|
||||
accept=lambda evidence: _distinct_accounts(evidence) >= SPRAY_MIN_DISTINCT_ACCOUNTS,
|
||||
),
|
||||
Rule(
|
||||
"windows.auth.account-lockout",
|
||||
"authentication",
|
||||
"warning",
|
||||
300,
|
||||
_account_lockout,
|
||||
lambda event: _group_key(_host(event), _account(event)),
|
||||
_lockout_title,
|
||||
_lockout_reason,
|
||||
),
|
||||
Rule(
|
||||
"windows.account.change",
|
||||
"security",
|
||||
"warning",
|
||||
300,
|
||||
_account_change,
|
||||
lambda event: _group_key(_host(event)),
|
||||
_account_change_title,
|
||||
_account_change_reason,
|
||||
),
|
||||
Rule(
|
||||
"windows.system.startup",
|
||||
"system",
|
||||
@@ -261,6 +423,9 @@ RULES = (
|
||||
)
|
||||
|
||||
|
||||
# --- grouping ---------------------------------------------------------------
|
||||
|
||||
|
||||
def _event_sort_key(event: NormalizedEvent) -> tuple[bool, datetime, str]:
|
||||
reliable_timestamp = event.timestamp is not None and event.timestamp_valid
|
||||
return (
|
||||
@@ -270,19 +435,56 @@ def _event_sort_key(event: NormalizedEvent) -> tuple[bool, datetime, str]:
|
||||
)
|
||||
|
||||
|
||||
def _groups_for_rule(events: list[NormalizedEvent], rule: Rule) -> list[list[NormalizedEvent]]:
|
||||
buckets: dict[str, list[NormalizedEvent]] = {}
|
||||
isolated_number = 0
|
||||
Signature = tuple[str, str, int | None]
|
||||
|
||||
|
||||
def _signature(event: NormalizedEvent) -> Signature:
|
||||
return (event.provider, event.channel, event.event_id)
|
||||
|
||||
|
||||
def _matching(events: list[NormalizedEvent], predicate: Callable[[NormalizedEvent], bool]) -> list[NormalizedEvent]:
|
||||
"""Filter events by a rule predicate, evaluating it once per distinct signature.
|
||||
|
||||
Every rule predicate depends only on provider, channel and event ID, so a
|
||||
100,000-event log with a few hundred signatures needs a few hundred
|
||||
predicate calls per rule instead of 100,000.
|
||||
"""
|
||||
verdicts: dict[Signature, bool] = {}
|
||||
matched: list[NormalizedEvent] = []
|
||||
for event in events:
|
||||
if not rule.predicate(event) or event.timestamp is None or not event.timestamp_valid:
|
||||
signature = _signature(event)
|
||||
verdict = verdicts.get(signature)
|
||||
if verdict is None:
|
||||
verdict = verdicts[signature] = predicate(event)
|
||||
if verdict:
|
||||
matched.append(event)
|
||||
return matched
|
||||
|
||||
|
||||
def _windowed_groups(
|
||||
events: list[NormalizedEvent],
|
||||
predicate: Callable[[NormalizedEvent], bool],
|
||||
key_for: Callable[[NormalizedEvent], str | None],
|
||||
window_seconds: int,
|
||||
) -> list[Group]:
|
||||
"""Bucket timed events by key and split each bucket into bounded windows.
|
||||
|
||||
A window is anchored on its first event, so a group can never chain beyond
|
||||
``window_seconds``. Events whose key is unknown become isolated groups
|
||||
(key ``None``) so that missing dimensions never merge unrelated evidence.
|
||||
"""
|
||||
buckets: dict[str, list[NormalizedEvent]] = {}
|
||||
isolated: list[NormalizedEvent] = []
|
||||
for event in events:
|
||||
if not predicate(event) or event.timestamp is None or not event.timestamp_valid:
|
||||
continue
|
||||
key = rule.key(event)
|
||||
key = key_for(event)
|
||||
if key is None:
|
||||
isolated_number += 1
|
||||
key = f"__isolated__{isolated_number}:{event.id}"
|
||||
buckets.setdefault(key, []).append(event)
|
||||
groups: list[list[NormalizedEvent]] = []
|
||||
window = timedelta(seconds=rule.window_seconds)
|
||||
isolated.append(event)
|
||||
else:
|
||||
buckets.setdefault(key, []).append(event)
|
||||
groups: list[Group] = []
|
||||
window = timedelta(seconds=window_seconds)
|
||||
for key in sorted(buckets):
|
||||
ordered = sorted(buckets[key], key=_event_sort_key)
|
||||
current: list[NormalizedEvent] = []
|
||||
@@ -290,73 +492,202 @@ def _groups_for_rule(events: list[NormalizedEvent], rule: Rule) -> list[list[Nor
|
||||
if not current or event.timestamp - current[0].timestamp <= window:
|
||||
current.append(event)
|
||||
else:
|
||||
groups.append(current)
|
||||
groups.append((key, current))
|
||||
current = [event]
|
||||
if current:
|
||||
groups.append(current)
|
||||
groups.append((key, current))
|
||||
groups.extend((None, [event]) for event in sorted(isolated, key=_event_sort_key))
|
||||
return groups
|
||||
|
||||
|
||||
def _always(_event: NormalizedEvent) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
def _groups_for_rule(events: list[NormalizedEvent], rule: Rule) -> list[Group]:
|
||||
groups = _windowed_groups(_matching(events, rule.predicate), _always, rule.key, rule.window_seconds)
|
||||
if rule.accept is None:
|
||||
return groups
|
||||
return [group for group in groups if rule.accept(group[1])]
|
||||
|
||||
|
||||
def _incident_id(rule_id: str, evidence: list[NormalizedEvent]) -> str:
|
||||
seed = f"{rule_id}|{'|'.join(event.id for event in evidence)}".encode()
|
||||
return f"incident-{hashlib.sha256(seed).hexdigest()[:16]}"
|
||||
|
||||
|
||||
def _build(rule: Rule, evidence: list[NormalizedEvent]) -> IncidentCluster:
|
||||
signatures = {(event.provider, event.event_id, event.summary) for event in evidence}
|
||||
reasons = [rule.reason_for(evidence)]
|
||||
if len(evidence) > 1:
|
||||
def _signatures(evidence: list[NormalizedEvent]) -> set[tuple[str, int | None, str]]:
|
||||
return {(event.provider, event.event_id, event.summary) for event in evidence}
|
||||
|
||||
|
||||
def _compression_reasons(evidence: list[NormalizedEvent]) -> list[str]:
|
||||
reasons: list[str] = []
|
||||
timed = all(event.timestamp is not None and event.timestamp_valid for event in evidence)
|
||||
if len(evidence) > 1 and timed:
|
||||
seconds = int((evidence[-1].timestamp - evidence[0].timestamp).total_seconds())
|
||||
reasons.append(f"{len(evidence)} evidence events occurred within {seconds} second(s).")
|
||||
signatures = _signatures(evidence)
|
||||
if len(signatures) < len(evidence):
|
||||
reasons.append(f"Repeated signals compress from {len(evidence)} records to {len(signatures)} distinct signature(s).")
|
||||
return IncidentCluster(
|
||||
return reasons
|
||||
|
||||
|
||||
def _build(rule: Rule, evidence: list[NormalizedEvent]) -> IncidentCluster:
|
||||
return IncidentCluster.model_construct(
|
||||
id=_incident_id(rule.id, evidence),
|
||||
rule_id=rule.id,
|
||||
start=evidence[0].timestamp,
|
||||
end=evidence[-1].timestamp,
|
||||
title=rule.title_for(evidence),
|
||||
title=_bounded_title(rule.title_for(evidence), "Correlated evidence"),
|
||||
category=rule.category,
|
||||
severity=rule.severity,
|
||||
reasons=reasons,
|
||||
reasons=[rule.reason_for(evidence), *_compression_reasons(evidence)],
|
||||
evidence_event_ids=[event.id for event in evidence],
|
||||
original_count=len(evidence),
|
||||
compressed_count=max(1, len(signatures)),
|
||||
compressed_count=max(1, len(_signatures(evidence))),
|
||||
)
|
||||
|
||||
|
||||
def _attention(event: NormalizedEvent) -> bool:
|
||||
return event.level.casefold() in LEVEL_SEVERITY
|
||||
|
||||
|
||||
def _fallback_key(event: NormalizedEvent) -> str | None:
|
||||
provider = _dimension(event.provider)
|
||||
if provider is None or provider.casefold() == "unknown provider" or event.event_id is None:
|
||||
return None
|
||||
return _group_key(_host(event), provider, str(event.event_id))
|
||||
|
||||
|
||||
def _max_severity(levels: list[Severity]) -> Severity:
|
||||
return max(levels, key=lambda level: SEVERITY_RANK[level])
|
||||
|
||||
|
||||
def _build_fallback(evidence: list[NormalizedEvent]) -> IncidentCluster:
|
||||
first = evidence[0]
|
||||
timed = first.timestamp is not None and first.timestamp_valid
|
||||
if len(evidence) == 1:
|
||||
reason = "An error or warning was not matched by a more specific correlation rule."
|
||||
else:
|
||||
reason = (
|
||||
"Errors or warnings with the same provider, event ID and known host were not matched by a more specific "
|
||||
"rule and were compressed into one bounded window; repetition is not a causal claim."
|
||||
)
|
||||
return IncidentCluster.model_construct(
|
||||
id=_incident_id(FALLBACK_RULE_ID, evidence),
|
||||
rule_id=FALLBACK_RULE_ID,
|
||||
start=first.timestamp if timed else None,
|
||||
end=evidence[-1].timestamp if timed else None,
|
||||
title=_bounded_title(first.summary, "Unclassified attention event"),
|
||||
category=first.category,
|
||||
severity=_max_severity([LEVEL_SEVERITY.get(event.level.casefold(), "warning") for event in evidence]),
|
||||
reasons=[reason, *_compression_reasons(evidence)],
|
||||
evidence_event_ids=[event.id for event in evidence],
|
||||
original_count=len(evidence),
|
||||
compressed_count=max(1, len(_signatures(evidence))),
|
||||
)
|
||||
|
||||
|
||||
# --- campaigns --------------------------------------------------------------
|
||||
|
||||
|
||||
def _campaigns_for(
|
||||
rule_id: str, window_seconds: int, keyed: list[tuple[str | None, IncidentCluster]]
|
||||
) -> list[Campaign]:
|
||||
"""Link adjacent windows of one rule and one known key into a summary campaign.
|
||||
|
||||
``keyed`` is ordered by key and then by time, as produced by
|
||||
:func:`_windowed_groups`. Incidents keep their bounded windows; a campaign
|
||||
only records that the same rule recurred for the same known dimensions.
|
||||
"""
|
||||
campaigns: list[Campaign] = []
|
||||
window = timedelta(seconds=window_seconds)
|
||||
run: list[IncidentCluster] = []
|
||||
run_key: str | None = None
|
||||
|
||||
def flush() -> None:
|
||||
if len(run) < 2:
|
||||
return
|
||||
first, last = run[0], run[-1]
|
||||
incident_ids = [incident.id for incident in run]
|
||||
seed = f"{rule_id}|{'|'.join(incident_ids)}".encode()
|
||||
campaign = Campaign.model_construct(
|
||||
id=f"campaign-{hashlib.sha256(seed).hexdigest()[:16]}",
|
||||
rule_id=rule_id,
|
||||
title=_bounded_title(f"Recurring: {first.title}", "Recurring activity"),
|
||||
category=first.category,
|
||||
severity=_max_severity([incident.severity for incident in run]),
|
||||
start=first.start,
|
||||
end=last.end,
|
||||
incident_ids=incident_ids,
|
||||
event_count=sum(incident.original_count for incident in run),
|
||||
reasons=[
|
||||
(
|
||||
f"The same rule fired for the same known grouping dimensions in {len(run)} adjacent windows "
|
||||
f"(gaps of at most {window_seconds} second(s))."
|
||||
),
|
||||
"Each incident stays bounded by its rule span; the campaign only summarizes recurrence and makes no causal claim.",
|
||||
],
|
||||
)
|
||||
for incident in run:
|
||||
incident.campaign_id = campaign.id
|
||||
campaigns.append(campaign)
|
||||
|
||||
for key, incident in keyed:
|
||||
previous = run[-1] if run else None
|
||||
contiguous = (
|
||||
key is not None
|
||||
and key == run_key
|
||||
and previous is not None
|
||||
and previous.end is not None
|
||||
and incident.start is not None
|
||||
and incident.start - previous.end <= window
|
||||
)
|
||||
if not contiguous:
|
||||
flush()
|
||||
run = []
|
||||
run_key = key
|
||||
if key is not None and incident.start is not None:
|
||||
run.append(incident)
|
||||
flush()
|
||||
return campaigns
|
||||
|
||||
|
||||
def _incident_sort_key(incident: IncidentCluster) -> tuple[bool, datetime, str, str]:
|
||||
return (
|
||||
incident.start is None,
|
||||
incident.start or datetime.max.replace(tzinfo=UTC),
|
||||
incident.rule_id,
|
||||
incident.id,
|
||||
)
|
||||
|
||||
|
||||
def correlate_with_campaigns(events: list[NormalizedEvent]) -> tuple[list[IncidentCluster], list[Campaign]]:
|
||||
ordered = sorted(events, key=_event_sort_key)
|
||||
incidents: list[IncidentCluster] = []
|
||||
campaigns: list[Campaign] = []
|
||||
consumed: set[str] = set()
|
||||
for rule in RULES:
|
||||
keyed = [(key, _build(rule, group)) for key, group in _groups_for_rule(ordered, rule)]
|
||||
for _, incident in keyed:
|
||||
consumed.update(incident.evidence_event_ids)
|
||||
incidents.extend(incident for _, incident in keyed)
|
||||
campaigns.extend(_campaigns_for(rule.id, rule.window_seconds, keyed))
|
||||
|
||||
remaining = [event for event in ordered if event.id not in consumed and _attention(event)]
|
||||
keyed_fallback = [
|
||||
(key, _build_fallback(group))
|
||||
for key, group in _windowed_groups(remaining, _always, _fallback_key, FALLBACK_WINDOW_SECONDS)
|
||||
]
|
||||
incidents.extend(incident for _, incident in keyed_fallback)
|
||||
campaigns.extend(_campaigns_for(FALLBACK_RULE_ID, FALLBACK_WINDOW_SECONDS, keyed_fallback))
|
||||
# Untimed attention events stay visible as evidence but never join a temporal group.
|
||||
incidents.extend(
|
||||
_build_fallback([event]) for event in remaining if event.timestamp is None or not event.timestamp_valid
|
||||
)
|
||||
campaigns.sort(key=lambda campaign: (campaign.start, campaign.rule_id, campaign.id))
|
||||
return sorted(incidents, key=_incident_sort_key), campaigns
|
||||
|
||||
|
||||
def correlate(events: list[NormalizedEvent]) -> list[IncidentCluster]:
|
||||
ordered = sorted(events, key=_event_sort_key)
|
||||
incidents: list[IncidentCluster] = []
|
||||
consumed: set[str] = set()
|
||||
for rule in RULES:
|
||||
for group in _groups_for_rule(ordered, rule):
|
||||
incidents.append(_build(rule, group))
|
||||
consumed.update(event.id for event in group)
|
||||
fallback_rule_id = "windows.generic.attention"
|
||||
for event in ordered:
|
||||
if event.id not in consumed and event.level.casefold() in {"critical", "error", "warning"}:
|
||||
incidents.append(
|
||||
IncidentCluster(
|
||||
id=_incident_id(fallback_rule_id, [event]),
|
||||
rule_id=fallback_rule_id,
|
||||
start=event.timestamp if event.timestamp_valid else None,
|
||||
end=event.timestamp if event.timestamp_valid else None,
|
||||
title=event.summary or "Unclassified attention event",
|
||||
category=event.category,
|
||||
severity="warning",
|
||||
reasons=["An error or warning was not matched by a more specific correlation rule."],
|
||||
evidence_event_ids=[event.id],
|
||||
original_count=1,
|
||||
compressed_count=1,
|
||||
)
|
||||
)
|
||||
return sorted(
|
||||
incidents,
|
||||
key=lambda incident: (
|
||||
incident.start is None,
|
||||
incident.start or datetime.max.replace(tzinfo=UTC),
|
||||
incident.rule_id,
|
||||
incident.id,
|
||||
),
|
||||
)
|
||||
return correlate_with_campaigns(events)[0]
|
||||
+164
-102
@@ -1,22 +1,30 @@
|
||||
import asyncio
|
||||
import gzip
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from collections import Counter
|
||||
from collections.abc import AsyncIterator
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from fastapi import FastAPI, File, HTTPException, UploadFile
|
||||
from fastapi import FastAPI, HTTPException, Request, Response
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from starlette.datastructures import UploadFile as StarletteUploadFile
|
||||
from starlette.formparsers import MultiPartException
|
||||
from starlette.requests import ClientDisconnect
|
||||
|
||||
from .correlate import correlate
|
||||
from .correlate import correlate_with_campaigns
|
||||
from .demo import load_demo
|
||||
from .models import AnalysisResponse, ParseIssue, ParseStats
|
||||
from .models import AnalysisResponse, ParseIssue, ParseStats, ParseStatus, SourceInfo
|
||||
from .parser import (
|
||||
MAX_UPLOAD_BYTES,
|
||||
EvtxParseError,
|
||||
ParseResult,
|
||||
UnexpectedParserError,
|
||||
parse_evtx_bytes,
|
||||
)
|
||||
@@ -43,10 +51,11 @@ app.add_middleware(
|
||||
|
||||
MAX_MULTIPART_OVERHEAD_BYTES = 1024 * 1024
|
||||
MAX_REQUEST_BYTES = MAX_UPLOAD_BYTES + MAX_MULTIPART_OVERHEAD_BYTES
|
||||
MAX_REPORTED_ISSUES = 200
|
||||
GZIP_LEVEL = 1
|
||||
CONTENT_SECURITY_POLICY = (
|
||||
"default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; "
|
||||
"img-src 'self' data:; connect-src 'self' http://localhost:8000 http://127.0.0.1:8000 "
|
||||
"http://localhost:5173 http://127.0.0.1:5173 http://127.0.0.1:4173; "
|
||||
"img-src 'self' data:; connect-src 'self'; "
|
||||
"font-src 'self'; object-src 'none'; base-uri 'none'; frame-ancestors 'none'; form-action 'self'"
|
||||
)
|
||||
SECURITY_HEADERS = {
|
||||
@@ -176,49 +185,67 @@ def health():
|
||||
@app.get("/api/demo", response_model=AnalysisResponse)
|
||||
def demo():
|
||||
events = load_demo()
|
||||
return AnalysisResponse(source="bundled-demo", events=events, incidents=correlate(events))
|
||||
incidents, campaigns = correlate_with_campaigns(events)
|
||||
return AnalysisResponse(source="bundled-demo", events=events, incidents=incidents, campaigns=campaigns)
|
||||
|
||||
|
||||
async def _parse_with_admission(data: bytes):
|
||||
def _capacity_error(message: str) -> HTTPException:
|
||||
return HTTPException(
|
||||
503,
|
||||
detail={"code": "parse_capacity_exceeded", "message": message},
|
||||
headers={"Retry-After": "1"},
|
||||
)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def _admission() -> AsyncIterator[None]:
|
||||
"""Reserve an analysis slot before the multipart body is spooled.
|
||||
|
||||
Starlette spools uploads to TMPDIR while parsing the form. Admitting first
|
||||
bounds the number of concurrently spooled uploads to concurrency + queue,
|
||||
so a burst of large uploads gets a fast 503 instead of exhausting tmpfs.
|
||||
"""
|
||||
global _parse_inflight
|
||||
async with _parse_admission_lock:
|
||||
if _parse_inflight >= PARSE_CONCURRENCY + PARSE_QUEUE_LIMIT:
|
||||
raise HTTPException(
|
||||
503,
|
||||
detail={
|
||||
"code": "parse_capacity_exceeded",
|
||||
"message": "EventLens is already analyzing its maximum concurrent uploads; retry shortly.",
|
||||
},
|
||||
headers={"Retry-After": "1"},
|
||||
)
|
||||
raise _capacity_error("EventLens is already analyzing its maximum concurrent uploads; retry shortly.")
|
||||
_parse_inflight += 1
|
||||
acquired = False
|
||||
try:
|
||||
try:
|
||||
await asyncio.wait_for(_parse_slots.acquire(), timeout=PARSE_QUEUE_TIMEOUT_SECONDS)
|
||||
acquired = True
|
||||
except TimeoutError as exc:
|
||||
raise HTTPException(
|
||||
503,
|
||||
detail={
|
||||
"code": "parse_capacity_exceeded",
|
||||
"message": "EventLens is busy analyzing another upload; retry shortly.",
|
||||
},
|
||||
headers={"Retry-After": "1"},
|
||||
) from exc
|
||||
yield
|
||||
finally:
|
||||
if not acquired:
|
||||
async with _parse_admission_lock:
|
||||
_parse_inflight -= 1
|
||||
try:
|
||||
return await asyncio.to_thread(parse_evtx_bytes, data)
|
||||
finally:
|
||||
if acquired:
|
||||
_parse_slots.release()
|
||||
async with _parse_admission_lock:
|
||||
_parse_inflight -= 1
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def _parse_slot() -> AsyncIterator[None]:
|
||||
try:
|
||||
await asyncio.wait_for(_parse_slots.acquire(), timeout=PARSE_QUEUE_TIMEOUT_SECONDS)
|
||||
except TimeoutError as exc:
|
||||
raise _capacity_error("EventLens is busy analyzing another upload; retry shortly.") from exc
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_parse_slots.release()
|
||||
|
||||
|
||||
def _stats(parsed: ParseResult, status: ParseStatus | None = None) -> ParseStats:
|
||||
counts = Counter(issue.code for issue in parsed.issues)
|
||||
return ParseStats(
|
||||
status=status or parsed.status,
|
||||
parsed_events=len(parsed.events) if status != "failed" else 0,
|
||||
records_seen=parsed.records_seen,
|
||||
skipped_records=parsed.skipped_records,
|
||||
issue_count=len(parsed.issues),
|
||||
truncated=parsed.truncated,
|
||||
truncation_reason=parsed.truncation_reason,
|
||||
duration_ms=parsed.duration_ms,
|
||||
issues=parsed.issues[:MAX_REPORTED_ISSUES],
|
||||
issues_truncated=len(parsed.issues) > MAX_REPORTED_ISSUES,
|
||||
issue_counts=dict(sorted(counts.items())),
|
||||
)
|
||||
|
||||
|
||||
def _failed_parse_stats() -> ParseStats:
|
||||
return ParseStats(
|
||||
status="failed",
|
||||
@@ -230,77 +257,112 @@ def _failed_parse_stats() -> ParseStats:
|
||||
truncation_reason=None,
|
||||
duration_ms=0,
|
||||
issues=[ParseIssue(code="parser_rejected_input", message="The EVTX parser rejected the input.")],
|
||||
issue_counts={"parser_rejected_input": 1},
|
||||
)
|
||||
|
||||
|
||||
@app.post("/api/analyze", response_model=AnalysisResponse)
|
||||
async def analyze(file: UploadFile = File(...)): # noqa: B008
|
||||
def _analyze_bytes(source: str, data: bytes | bytearray, compress: bool = False) -> bytes:
|
||||
"""Parse, correlate, serialize and optionally gzip off the event loop."""
|
||||
source_info = SourceInfo(sha256=hashlib.sha256(data).hexdigest(), size_bytes=len(data))
|
||||
try:
|
||||
source = _safe_source_name(file.filename)
|
||||
if not source.lower().endswith(".evtx"):
|
||||
raise HTTPException(400, "Upload an .evtx file")
|
||||
data = bytearray()
|
||||
while True:
|
||||
chunk = await file.read(min(1024 * 1024, MAX_UPLOAD_BYTES + 1 - len(data)))
|
||||
if not chunk:
|
||||
break
|
||||
data.extend(chunk)
|
||||
if len(data) > MAX_UPLOAD_BYTES:
|
||||
raise HTTPException(413, "EVTX file exceeds the 50 MiB MVP limit")
|
||||
try:
|
||||
parsed = await _parse_with_admission(bytes(data))
|
||||
except EvtxParseError as exc:
|
||||
raise HTTPException(
|
||||
422,
|
||||
detail={
|
||||
"code": "parser_rejected_input",
|
||||
"message": str(exc),
|
||||
"parse": _failed_parse_stats().model_dump(mode="json"),
|
||||
},
|
||||
) from exc
|
||||
except UnexpectedParserError as exc:
|
||||
logger.error("Unexpected EVTX parser failure reached API exception_type=%s", type(exc).__name__)
|
||||
raise HTTPException(
|
||||
500,
|
||||
detail={"code": "parser_internal_error", "message": "EVTX analysis failed unexpectedly; retry the request."},
|
||||
) from exc
|
||||
if not parsed.events:
|
||||
raise HTTPException(
|
||||
422,
|
||||
detail={
|
||||
"code": "no_readable_events",
|
||||
"message": "No readable Windows events were found in this file.",
|
||||
"parse": ParseStats(
|
||||
status="failed",
|
||||
parsed_events=0,
|
||||
records_seen=parsed.records_seen,
|
||||
skipped_records=parsed.skipped_records,
|
||||
issue_count=len(parsed.issues),
|
||||
truncated=parsed.truncated,
|
||||
truncation_reason=parsed.truncation_reason,
|
||||
duration_ms=parsed.duration_ms,
|
||||
issues=parsed.issues,
|
||||
).model_dump(mode="json"),
|
||||
},
|
||||
)
|
||||
return AnalysisResponse(
|
||||
source=source,
|
||||
events=parsed.events,
|
||||
incidents=correlate(parsed.events),
|
||||
parse=ParseStats(
|
||||
status=parsed.status,
|
||||
parsed_events=len(parsed.events),
|
||||
records_seen=parsed.records_seen,
|
||||
skipped_records=parsed.skipped_records,
|
||||
issue_count=len(parsed.issues),
|
||||
truncated=parsed.truncated,
|
||||
truncation_reason=parsed.truncation_reason,
|
||||
duration_ms=parsed.duration_ms,
|
||||
issues=parsed.issues,
|
||||
),
|
||||
parsed = parse_evtx_bytes(data)
|
||||
except EvtxParseError as exc:
|
||||
raise HTTPException(
|
||||
422,
|
||||
detail={
|
||||
"code": "parser_rejected_input",
|
||||
"message": str(exc),
|
||||
"parse": _failed_parse_stats().model_dump(mode="json"),
|
||||
},
|
||||
) from exc
|
||||
if not parsed.events:
|
||||
raise HTTPException(
|
||||
422,
|
||||
detail={
|
||||
"code": "no_readable_events",
|
||||
"message": "No readable Windows events were found in this file.",
|
||||
"parse": _stats(parsed, "failed").model_dump(mode="json"),
|
||||
},
|
||||
)
|
||||
finally:
|
||||
await file.close()
|
||||
incidents, campaigns = correlate_with_campaigns(parsed.events)
|
||||
response = AnalysisResponse(
|
||||
source=source,
|
||||
events=parsed.events,
|
||||
incidents=incidents,
|
||||
campaigns=campaigns,
|
||||
parse=_stats(parsed),
|
||||
source_info=source_info,
|
||||
)
|
||||
payload = response.model_dump_json().encode("utf-8")
|
||||
# Event JSON is highly repetitive: level 1 shrinks a 100k-event response
|
||||
# ~20x for about a second of worker-thread CPU, never on the event loop.
|
||||
return gzip.compress(payload, compresslevel=GZIP_LEVEL, mtime=0) if compress else payload
|
||||
|
||||
|
||||
async def _read_bounded(upload: StarletteUploadFile) -> bytearray:
|
||||
data = bytearray()
|
||||
while True:
|
||||
chunk = await upload.read(min(1024 * 1024, MAX_UPLOAD_BYTES + 1 - len(data)))
|
||||
if not chunk:
|
||||
return data
|
||||
data.extend(chunk)
|
||||
if len(data) > MAX_UPLOAD_BYTES:
|
||||
raise HTTPException(413, "EVTX file exceeds the 50 MiB MVP limit")
|
||||
|
||||
|
||||
ANALYZE_OPENAPI = {
|
||||
"requestBody": {
|
||||
"required": True,
|
||||
"content": {
|
||||
"multipart/form-data": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"required": ["file"],
|
||||
"properties": {"file": {"type": "string", "format": "binary"}},
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@app.post("/api/analyze", response_model=AnalysisResponse, openapi_extra=ANALYZE_OPENAPI)
|
||||
async def analyze(request: Request):
|
||||
compress = "gzip" in request.headers.get("accept-encoding", "").lower()
|
||||
async with _admission():
|
||||
try:
|
||||
form = await request.form(max_files=1, max_fields=4)
|
||||
except MultiPartException as exc:
|
||||
raise HTTPException(400, "Upload one .evtx file as multipart form data") from exc
|
||||
try:
|
||||
upload = form.get("file")
|
||||
if not isinstance(upload, StarletteUploadFile):
|
||||
raise HTTPException(422, "Upload one .evtx file in the 'file' form field")
|
||||
source = _safe_source_name(upload.filename)
|
||||
if not source.lower().endswith(".evtx"):
|
||||
raise HTTPException(400, "Upload an .evtx file")
|
||||
data = await _read_bounded(upload)
|
||||
async with _parse_slot():
|
||||
try:
|
||||
payload = await asyncio.to_thread(_analyze_bytes, source, data, compress)
|
||||
except UnexpectedParserError as exc:
|
||||
logger.error(
|
||||
"Unexpected EVTX parser failure reached API exception_type=%s", type(exc).__name__
|
||||
)
|
||||
raise HTTPException(
|
||||
500,
|
||||
detail={
|
||||
"code": "parser_internal_error",
|
||||
"message": "EVTX analysis failed unexpectedly; retry the request.",
|
||||
},
|
||||
) from exc
|
||||
del data
|
||||
finally:
|
||||
await form.close()
|
||||
headers = {"Vary": "Accept-Encoding"}
|
||||
if compress:
|
||||
headers["Content-Encoding"] = "gzip"
|
||||
return Response(content=payload, media_type="application/json", headers=headers)
|
||||
|
||||
|
||||
def _safe_source_name(filename: str | None) -> str:
|
||||
|
||||
+36
-2
@@ -3,7 +3,11 @@ from typing import Literal
|
||||
|
||||
from pydantic import BaseModel, Field, field_validator, model_validator
|
||||
|
||||
Category = Literal["system", "service", "network", "authentication", "application", "crash", "other"]
|
||||
Category = Literal[
|
||||
"system", "service", "network", "authentication", "security", "application", "crash", "other"
|
||||
]
|
||||
Severity = Literal["critical", "error", "warning", "info"]
|
||||
AuditOutcome = Literal["success", "failure"]
|
||||
ParseStatus = Literal["complete", "partial", "failed"]
|
||||
TimestampSource = Literal["event", "record", "missing", "invalid"]
|
||||
|
||||
@@ -20,6 +24,7 @@ class NormalizedEvent(BaseModel):
|
||||
computer: str = Field(default="", max_length=256)
|
||||
category: Category = "other"
|
||||
summary: str = Field(default="", max_length=512)
|
||||
outcome: AuditOutcome | None = None
|
||||
fields: dict[str, str] = Field(default_factory=dict, max_length=200)
|
||||
|
||||
@field_validator("timestamp")
|
||||
@@ -49,11 +54,31 @@ class IncidentCluster(BaseModel):
|
||||
end: datetime | None = None
|
||||
title: str = Field(max_length=256)
|
||||
category: Category
|
||||
severity: str = Field(max_length=32)
|
||||
severity: Severity
|
||||
reasons: list[str] = Field(max_length=8)
|
||||
evidence_event_ids: list[str]
|
||||
original_count: int = Field(ge=1)
|
||||
compressed_count: int = Field(ge=1)
|
||||
campaign_id: str | None = Field(default=None, max_length=128)
|
||||
|
||||
|
||||
class Campaign(BaseModel):
|
||||
"""Consecutive incidents of one rule and grouping key, linked for summary only.
|
||||
|
||||
Incident windows stay bounded by their rule span; a campaign only states
|
||||
that the same rule fired for the same known dimensions in adjacent windows.
|
||||
"""
|
||||
|
||||
id: str = Field(max_length=128)
|
||||
rule_id: str = Field(max_length=128)
|
||||
title: str = Field(max_length=256)
|
||||
category: Category
|
||||
severity: Severity
|
||||
start: datetime
|
||||
end: datetime
|
||||
incident_ids: list[str] = Field(min_length=2)
|
||||
event_count: int = Field(ge=2)
|
||||
reasons: list[str] = Field(max_length=8)
|
||||
|
||||
|
||||
class ParseIssue(BaseModel):
|
||||
@@ -74,10 +99,19 @@ class ParseStats(BaseModel):
|
||||
truncation_reason: str | None = Field(default=None, max_length=64)
|
||||
duration_ms: int = Field(ge=0)
|
||||
issues: list[ParseIssue] = Field(default_factory=list)
|
||||
issues_truncated: bool = False
|
||||
issue_counts: dict[str, int] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class SourceInfo(BaseModel):
|
||||
sha256: str = Field(pattern=r"^[0-9a-f]{64}$")
|
||||
size_bytes: int = Field(ge=0)
|
||||
|
||||
|
||||
class AnalysisResponse(BaseModel):
|
||||
source: str
|
||||
events: list[NormalizedEvent]
|
||||
incidents: list[IncidentCluster]
|
||||
campaigns: list[Campaign] = Field(default_factory=list)
|
||||
parse: ParseStats | None = None
|
||||
source_info: SourceInfo | None = None
|
||||
+90
-16
@@ -3,9 +3,12 @@ import json
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from .models import Category, NormalizedEvent, TimestampSource
|
||||
from .models import AuditOutcome, Category, NormalizedEvent, TimestampSource
|
||||
|
||||
AUTH_EVENT_IDS = {4624, 4625, 4634, 4647, 4648, 4672, 4768, 4769, 4771, 4776}
|
||||
AUTH_EVENT_IDS = {4624, 4625, 4634, 4647, 4648, 4672, 4740, 4768, 4769, 4771, 4776}
|
||||
ACCOUNT_CHANGE_EVENT_IDS = {4720, 4728, 4732, 4756}
|
||||
AUDIT_FAILURE_KEYWORD = 0x0010_0000_0000_0000
|
||||
AUDIT_SUCCESS_KEYWORD = 0x0020_0000_0000_0000
|
||||
LEVELS = {0: "information", 1: "critical", 2: "error", 3: "warning", 4: "information", 5: "verbose"}
|
||||
MAX_EVENT_ID = 4_294_967_295
|
||||
MAX_RECORD_ID = 9_223_372_036_854_775_807
|
||||
@@ -86,6 +89,21 @@ def _provider(system: dict[str, Any]) -> str:
|
||||
return _bounded_text(value, MAX_PROVIDER_CHARS, "Unknown provider")
|
||||
|
||||
|
||||
def _outcome(system: dict[str, Any]) -> AuditOutcome | None:
|
||||
raw = _scalar(system.get("Keywords"))
|
||||
if raw is None or isinstance(raw, bool):
|
||||
return None
|
||||
try:
|
||||
value = int(raw, 0) if isinstance(raw, str) else int(raw)
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
return None
|
||||
if value & AUDIT_FAILURE_KEYWORD:
|
||||
return "failure"
|
||||
if value & AUDIT_SUCCESS_KEYWORD:
|
||||
return "success"
|
||||
return None
|
||||
|
||||
|
||||
def _store_field(output: dict[str, str], key: str, value: str, limit: int) -> None:
|
||||
bounded_key = _bounded_text(key or "value", MAX_FIELD_KEY_CHARS)
|
||||
if bounded_key not in output:
|
||||
@@ -127,47 +145,78 @@ def _flatten(
|
||||
if len(output) >= limit:
|
||||
return
|
||||
return
|
||||
_store_field(output, prefix or "value", _render_scalar(value)[:MAX_FIELD_VALUE_CHARS], limit)
|
||||
|
||||
|
||||
def _render_scalar(value: Any) -> str:
|
||||
"""Render a scalar exactly like ``json.dumps`` would, with fast paths for common types."""
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
if value is None:
|
||||
return "null"
|
||||
if value is True:
|
||||
return "true"
|
||||
if value is False:
|
||||
return "false"
|
||||
if type(value) is int:
|
||||
return str(value)
|
||||
try:
|
||||
rendered = json.dumps(value, ensure_ascii=False) if not isinstance(value, str) else value
|
||||
return json.dumps(value, ensure_ascii=False)
|
||||
except (TypeError, ValueError):
|
||||
rendered = "[unrenderable value]"
|
||||
_store_field(output, prefix or "value", rendered[:MAX_FIELD_VALUE_CHARS], limit)
|
||||
return "[unrenderable value]"
|
||||
|
||||
|
||||
def _fold(value: str) -> str:
|
||||
return value.strip().casefold()
|
||||
|
||||
|
||||
EVENTLOG_PROVIDERS = frozenset({"eventlog", "microsoft-windows-eventlog"})
|
||||
|
||||
|
||||
def _channel_matches(channel: str, *accepted: str) -> bool:
|
||||
return channel.strip().casefold() in {value.casefold() for value in accepted}
|
||||
folded = _fold(channel)
|
||||
return any(folded == value.casefold() for value in accepted)
|
||||
|
||||
|
||||
def is_security_audit(provider: str, channel: str) -> bool:
|
||||
return provider.strip().casefold() == "microsoft-windows-security-auditing" and _channel_matches(channel, "Security")
|
||||
return _fold(provider) == "microsoft-windows-security-auditing" and _fold(channel) == "security"
|
||||
|
||||
|
||||
def is_service_control_manager(provider: str, channel: str) -> bool:
|
||||
return provider.strip().casefold() == "service control manager" and _channel_matches(channel, "System")
|
||||
return _fold(provider) == "service control manager" and _fold(channel) == "system"
|
||||
|
||||
|
||||
def is_dns_client(provider: str, channel: str) -> bool:
|
||||
return provider.strip().casefold() == "microsoft-windows-dns-client" and _channel_matches(channel, "System")
|
||||
return _fold(provider) == "microsoft-windows-dns-client" and _fold(channel) == "system"
|
||||
|
||||
|
||||
def is_application_error(provider: str, channel: str, event_id: int | None) -> bool:
|
||||
return provider.casefold().strip() == "application error" and event_id == 1000 and _channel_matches(channel, "Application")
|
||||
return event_id == 1000 and _fold(provider) == "application error" and _fold(channel) == "application"
|
||||
|
||||
|
||||
def is_windows_error_reporting(provider: str, channel: str, event_id: int | None) -> bool:
|
||||
return provider.casefold().strip() == "windows error reporting" and event_id == 1001 and _channel_matches(channel, "Application")
|
||||
return event_id == 1001 and _fold(provider) == "windows error reporting" and _fold(channel) == "application"
|
||||
|
||||
|
||||
def is_event_log(provider: str, channel: str) -> bool:
|
||||
return provider.casefold().strip() in {"eventlog", "microsoft-windows-eventlog"} and _channel_matches(channel, "System")
|
||||
return _fold(provider) in EVENTLOG_PROVIDERS and _fold(channel) == "system"
|
||||
|
||||
|
||||
def is_log_cleared(provider: str, channel: str, event_id: int | None) -> bool:
|
||||
"""Security 1102 (audit log cleared) or System 104 (an event log was cleared)."""
|
||||
if event_id not in {1102, 104} or _fold(provider) not in EVENTLOG_PROVIDERS:
|
||||
return False
|
||||
return (event_id == 1102 and _channel_matches(channel, "Security")) or (
|
||||
event_id == 104 and _channel_matches(channel, "System")
|
||||
)
|
||||
|
||||
|
||||
def is_kernel_general(provider: str, channel: str) -> bool:
|
||||
return provider.strip().casefold() == "microsoft-windows-kernel-general" and _channel_matches(channel, "System")
|
||||
return _fold(provider) == "microsoft-windows-kernel-general" and _fold(channel) == "system"
|
||||
|
||||
|
||||
def is_kernel_power(provider: str, channel: str) -> bool:
|
||||
return provider.strip().casefold() == "microsoft-windows-kernel-power" and _channel_matches(channel, "System")
|
||||
return _fold(provider) == "microsoft-windows-kernel-power" and _fold(channel) == "system"
|
||||
|
||||
|
||||
def classify(provider: str, event_id: int | None, channel: str) -> Category:
|
||||
@@ -181,6 +230,10 @@ def classify(provider: str, event_id: int | None, channel: str) -> Category:
|
||||
return "network"
|
||||
if is_security_audit(provider, channel) and event_id in AUTH_EVENT_IDS:
|
||||
return "authentication"
|
||||
if is_log_cleared(provider, channel, event_id) or (
|
||||
is_security_audit(provider, channel) and event_id in ACCOUNT_CHANGE_EVENT_IDS
|
||||
):
|
||||
return "security"
|
||||
if is_application_error(provider, channel, event_id) or is_windows_error_reporting(provider, channel, event_id):
|
||||
return "crash"
|
||||
if (
|
||||
@@ -195,6 +248,23 @@ def classify(provider: str, event_id: int | None, channel: str) -> Category:
|
||||
|
||||
|
||||
def summarize(provider: str, event_id: int | None, category: Category, fields: dict[str, str]) -> str:
|
||||
if category == "service" and event_id == 7045:
|
||||
service = fields.get("EventData.ServiceName") or fields.get("service")
|
||||
return f"Service installed: {service}" if service else "Service installed"
|
||||
if category == "security" and event_id == 1102:
|
||||
return "Security audit log cleared"
|
||||
if category == "security" and event_id == 104:
|
||||
cleared = fields.get("UserData.LogFileCleared.Channel")
|
||||
return f"Event log cleared: {cleared}" if cleared else "Event log cleared"
|
||||
if category == "security" and event_id == 4720:
|
||||
account = fields.get("EventData.TargetUserName")
|
||||
return f"User account created: {account}" if account else "User account created"
|
||||
if category == "security" and event_id in {4728, 4732, 4756}:
|
||||
group = fields.get("EventData.TargetUserName")
|
||||
return f"Member added to security group {group}" if group else "Member added to a security group"
|
||||
if category == "authentication" and event_id == 4740:
|
||||
account = fields.get("EventData.TargetUserName")
|
||||
return f"Account locked out: {account}" if account else "Account locked out"
|
||||
if category == "service" and event_id in {7023, 7024, 7031, 7034}:
|
||||
service = fields.get("EventData.param1") or fields.get("UserData.param1") or fields.get("service")
|
||||
return f"{service} terminated unexpectedly" if service else "Service failure recorded"
|
||||
@@ -245,11 +315,14 @@ def normalize_record(
|
||||
stable_id = f"event-{hashlib.sha256(seed).hexdigest()[:16]}"
|
||||
timestamp, timestamp_source = _timestamp(system, record_timestamp)
|
||||
summary = summarize(provider, event_id, category, fields)[:MAX_SUMMARY_CHARS]
|
||||
return NormalizedEvent(
|
||||
# Every value above is already bounded and typed exactly as NormalizedEvent
|
||||
# requires (see tests/test_performance.py, which re-validates the output),
|
||||
# so skip per-event pydantic validation on this hot path.
|
||||
return NormalizedEvent.model_construct(
|
||||
id=stable_id,
|
||||
timestamp=timestamp,
|
||||
timestamp_source=timestamp_source,
|
||||
timestamp_valid=timestamp is not None,
|
||||
timestamp_valid=timestamp is not None and timestamp_source in {"event", "record"},
|
||||
provider=provider,
|
||||
event_id=event_id,
|
||||
level=level[:32],
|
||||
@@ -257,5 +330,6 @@ def normalize_record(
|
||||
computer=computer,
|
||||
category=category,
|
||||
summary=summary,
|
||||
outcome=_outcome(system),
|
||||
fields=fields,
|
||||
)
|
||||
@@ -52,7 +52,7 @@ def _fingerprint(stage: str, exc: BaseException) -> str:
|
||||
|
||||
|
||||
def parse_evtx_bytes(
|
||||
data: bytes,
|
||||
data: bytes | bytearray,
|
||||
max_events: int = MAX_EVENTS,
|
||||
max_seconds: float = MAX_PARSE_SECONDS,
|
||||
) -> ParseResult:
|
||||
@@ -113,9 +113,11 @@ def parse_evtx_bytes(
|
||||
raw_data = record["data"]
|
||||
if not isinstance(raw_data, str):
|
||||
raise TypeError("record data was not textual")
|
||||
if (
|
||||
len(raw_data) > MAX_RECORD_JSON_BYTES
|
||||
or len(raw_data.encode("utf-8")) > MAX_RECORD_JSON_BYTES
|
||||
# A str of n characters encodes to at most 4n UTF-8 bytes, so the
|
||||
# costly encode is only needed for records near the limit.
|
||||
if len(raw_data) > MAX_RECORD_JSON_BYTES or (
|
||||
len(raw_data) * 4 > MAX_RECORD_JSON_BYTES
|
||||
and len(raw_data.encode("utf-8")) > MAX_RECORD_JSON_BYTES
|
||||
):
|
||||
add_issue(
|
||||
"record_json_limit_exceeded",
|
||||
|
||||
Reference in new issue
Block a user