EventLens: publish v1.1.0 application and standalone Compose

Includes campaign summaries, expanded Windows event correlation, worker-based redaction, updated demo assets and a local-only Compose example.

Publish curated EventLens source from 6a2ca12f81a2dcf816c61a86bd8811fcc95e8d8d
This commit is contained in:
Public Source Publisher committed 2026-11-20 13:15:35 +01:00
1 parent c0ade21029
commit 4e282639a4
63 files changed
+7381 -1420

No files matched your search

+387 -56
View File
@@ -3,13 +3,14 @@ from collections.abc import Callable
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from .models import Category, IncidentCluster, NormalizedEvent
from .models import Campaign, Category, IncidentCluster, NormalizedEvent, Severity
from .normalize import (
is_application_error,
is_dns_client,
is_event_log,
is_kernel_general,
is_kernel_power,
is_log_cleared,
is_security_audit,
is_service_control_manager,
is_windows_error_reporting,
@@ -17,18 +18,28 @@ from .normalize import (
TitleBuilder = Callable[[list[NormalizedEvent]], str]
ReasonBuilder = Callable[[list[NormalizedEvent]], str]
GroupFilter = Callable[[list[NormalizedEvent]], bool]
Group = tuple[str | None, list[NormalizedEvent]]
MAX_TITLE_CHARS = 256
FALLBACK_RULE_ID = "windows.generic.attention"
FALLBACK_WINDOW_SECONDS = 300
SPRAY_MIN_DISTINCT_ACCOUNTS = 3
SEVERITY_RANK: dict[str, int] = {"info": 0, "warning": 1, "error": 2, "critical": 3}
LEVEL_SEVERITY: dict[str, Severity] = {"critical": "critical", "error": "error", "warning": "warning"}
@dataclass(frozen=True)
class Rule:
id: str
category: Category
severity: str
severity: Severity
window_seconds: int
predicate: Callable[[NormalizedEvent], bool]
key: Callable[[NormalizedEvent], str | None]
title_for: TitleBuilder
reason_for: ReasonBuilder
accept: GroupFilter | None = None
def _field(event: NormalizedEvent, *names: str) -> str:
@@ -66,16 +77,36 @@ def _service(event: NormalizedEvent) -> str | None:
return _dimension(_field(event, "service", "EventData.param1", "UserData.param1"))
def _new_service(event: NormalizedEvent) -> str | None:
return _dimension(_field(event, "EventData.ServiceName", "service"))
def _account(event: NormalizedEvent) -> str | None:
return _dimension(_field(event, "EventData.TargetUserName", "TargetUserName"))
def _source_ip(event: NormalizedEvent) -> str | None:
return _dimension(_field(event, "EventData.IpAddress", "IpAddress"))
def _group_key(*dimensions: str | None) -> str | None:
if any(value is None for value in dimensions):
return None
return "|".join(value.casefold() for value in dimensions if value is not None)
def _bounded_title(value: str, fallback: str) -> str:
text = " ".join(value.split()) or fallback
return text if len(text) <= MAX_TITLE_CHARS else f"{text[: MAX_TITLE_CHARS - 1]}…"
def _distinct_accounts(evidence: list[NormalizedEvent]) -> int:
return len({account.casefold() for event in evidence if (account := _account(event))})
# --- predicates -------------------------------------------------------------
def _crash(event: NormalizedEvent) -> bool:
return is_application_error(event.provider, event.channel, event.event_id) or is_windows_error_reporting(
event.provider, event.channel, event.event_id
@@ -86,6 +117,10 @@ def _service_failure(event: NormalizedEvent) -> bool:
return is_service_control_manager(event.provider, event.channel) and event.event_id in {7023, 7024, 7031, 7034}
def _service_installed(event: NormalizedEvent) -> bool:
return is_service_control_manager(event.provider, event.channel) and event.event_id == 7045
def _dns_timeout(event: NormalizedEvent) -> bool:
return is_dns_client(event.provider, event.channel) and event.event_id == 1014
@@ -94,6 +129,18 @@ def _failed_logon(event: NormalizedEvent) -> bool:
return is_security_audit(event.provider, event.channel) and event.event_id == 4625
def _account_lockout(event: NormalizedEvent) -> bool:
return is_security_audit(event.provider, event.channel) and event.event_id == 4740
def _account_change(event: NormalizedEvent) -> bool:
return is_security_audit(event.provider, event.channel) and event.event_id in {4720, 4728, 4732, 4756}
def _log_cleared(event: NormalizedEvent) -> bool:
return is_log_cleared(event.provider, event.channel, event.event_id)
def _normal_startup(event: NormalizedEvent) -> bool:
return (is_kernel_general(event.provider, event.channel) and event.event_id == 12) or (
is_event_log(event.provider, event.channel) and event.event_id == 6005
@@ -112,6 +159,9 @@ def _unexpected_shutdown(event: NormalizedEvent) -> bool:
)
# --- titles and reasons -----------------------------------------------------
def _crash_title(evidence: list[NormalizedEvent]) -> str:
ids = {event.event_id for event in evidence}
if {1000, 1001}.issubset(ids):
@@ -139,6 +189,20 @@ def _service_reason(evidence: list[NormalizedEvent]) -> str:
return "Service Control Manager reported an abnormal service termination for the same known service and host."
def _service_installed_key(event: NormalizedEvent) -> str | None:
return _group_key(_host(event), _new_service(event))
def _service_installed_title(evidence: list[NormalizedEvent]) -> str:
return "New service installed" if len(evidence) == 1 else "New service installation activity"
def _service_installed_reason(evidence: list[NormalizedEvent]) -> str:
if _service_installed_key(evidence[0]) is None:
return "Service Control Manager recorded a service installation (7045); no service/host grouping was applied because a grouping dimension was unavailable."
return "Service Control Manager recorded installation of the same known service on the same host (7045); installation alone is not evidence of malice."
def _dns_title(evidence: list[NormalizedEvent]) -> str:
return "DNS timeout activity" if len(evidence) > 1 else "DNS timeout recorded"
@@ -163,6 +227,53 @@ def _auth_reason(evidence: list[NormalizedEvent]) -> str:
return "Security Auditing reported failed sign-ins for the same known account and host; this is evidence of activity, not proof of cause."
def _spray_title(evidence: list[NormalizedEvent]) -> str:
return "Failed sign-ins across multiple accounts from one source"
def _spray_reason(evidence: list[NormalizedEvent]) -> str:
return (
f"Security Auditing reported failed sign-ins (4625) for {_distinct_accounts(evidence)} distinct accounts "
"from the same known source address on the same host; this pattern is consistent with, but does not prove, password spraying."
)
def _lockout_title(evidence: list[NormalizedEvent]) -> str:
return "Account lockout" if len(evidence) == 1 else "Repeated account lockouts"
def _lockout_reason(evidence: list[NormalizedEvent]) -> str:
if _group_key(_host(evidence[0]), _account(evidence[0])) is None:
return "Security Auditing recorded an account lockout (4740); no account/host grouping was applied because a grouping dimension was unavailable."
return "Security Auditing recorded lockouts (4740) for the same known account on the same host."
def _account_change_title(evidence: list[NormalizedEvent]) -> str:
ids = {event.event_id for event in evidence}
if 4720 in ids and ids & {4728, 4732, 4756}:
return "Account created and added to a security group"
if 4720 in ids:
return "User account created"
return "Security group membership changed"
def _account_change_reason(evidence: list[NormalizedEvent]) -> str:
if _host(evidence[0]) is None:
return "Security Auditing recorded an account or security-group change; no host grouping was applied because the computer identity was unavailable."
return "Security Auditing recorded account creation or security-group membership changes (4720/4728/4732/4756) on the same known host in one bounded window."
def _log_cleared_title(evidence: list[NormalizedEvent]) -> str:
return "Event log cleared" if len(evidence) == 1 else "Event logs cleared"
def _log_cleared_reason(evidence: list[NormalizedEvent]) -> str:
return (
"EventLog recorded that an event log was cleared (Security 1102 / System 104); records from before this point "
"may be missing. The record identifies the clearing account only as far as its own fields state."
)
def _startup_title(evidence: list[NormalizedEvent]) -> str:
return "Normal startup sequence" if len(evidence) > 1 else "Normal startup evidence"
@@ -188,6 +299,16 @@ def _unexpected_reason(evidence: list[NormalizedEvent]) -> str:
RULES = (
Rule(
"windows.audit.log-cleared",
"security",
"critical",
300,
_log_cleared,
lambda event: _group_key(_host(event)),
_log_cleared_title,
_log_cleared_reason,
),
Rule(
"windows.crash.application-report",
"crash",
@@ -208,6 +329,16 @@ RULES = (
_service_title,
_service_reason,
),
Rule(
"windows.service.installed",
"service",
"warning",
300,
_service_installed,
_service_installed_key,
_service_installed_title,
_service_installed_reason,
),
Rule(
"windows.network.dns-timeout",
"network",
@@ -228,6 +359,37 @@ RULES = (
_auth_title,
_auth_reason,
),
Rule(
"windows.auth.password-spray",
"authentication",
"warning",
300,
_failed_logon,
lambda event: _group_key(_host(event), _source_ip(event)),
_spray_title,
_spray_reason,
accept=lambda evidence: _distinct_accounts(evidence) >= SPRAY_MIN_DISTINCT_ACCOUNTS,
),
Rule(
"windows.auth.account-lockout",
"authentication",
"warning",
300,
_account_lockout,
lambda event: _group_key(_host(event), _account(event)),
_lockout_title,
_lockout_reason,
),
Rule(
"windows.account.change",
"security",
"warning",
300,
_account_change,
lambda event: _group_key(_host(event)),
_account_change_title,
_account_change_reason,
),
Rule(
"windows.system.startup",
"system",
@@ -261,6 +423,9 @@ RULES = (
)
# --- grouping ---------------------------------------------------------------
def _event_sort_key(event: NormalizedEvent) -> tuple[bool, datetime, str]:
reliable_timestamp = event.timestamp is not None and event.timestamp_valid
return (
@@ -270,19 +435,56 @@ def _event_sort_key(event: NormalizedEvent) -> tuple[bool, datetime, str]:
)
def _groups_for_rule(events: list[NormalizedEvent], rule: Rule) -> list[list[NormalizedEvent]]:
buckets: dict[str, list[NormalizedEvent]] = {}
isolated_number = 0
Signature = tuple[str, str, int | None]
def _signature(event: NormalizedEvent) -> Signature:
return (event.provider, event.channel, event.event_id)
def _matching(events: list[NormalizedEvent], predicate: Callable[[NormalizedEvent], bool]) -> list[NormalizedEvent]:
"""Filter events by a rule predicate, evaluating it once per distinct signature.
Every rule predicate depends only on provider, channel and event ID, so a
100,000-event log with a few hundred signatures needs a few hundred
predicate calls per rule instead of 100,000.
"""
verdicts: dict[Signature, bool] = {}
matched: list[NormalizedEvent] = []
for event in events:
if not rule.predicate(event) or event.timestamp is None or not event.timestamp_valid:
signature = _signature(event)
verdict = verdicts.get(signature)
if verdict is None:
verdict = verdicts[signature] = predicate(event)
if verdict:
matched.append(event)
return matched
def _windowed_groups(
events: list[NormalizedEvent],
predicate: Callable[[NormalizedEvent], bool],
key_for: Callable[[NormalizedEvent], str | None],
window_seconds: int,
) -> list[Group]:
"""Bucket timed events by key and split each bucket into bounded windows.
A window is anchored on its first event, so a group can never chain beyond
``window_seconds``. Events whose key is unknown become isolated groups
(key ``None``) so that missing dimensions never merge unrelated evidence.
"""
buckets: dict[str, list[NormalizedEvent]] = {}
isolated: list[NormalizedEvent] = []
for event in events:
if not predicate(event) or event.timestamp is None or not event.timestamp_valid:
continue
key = rule.key(event)
key = key_for(event)
if key is None:
isolated_number += 1
key = f"__isolated__{isolated_number}:{event.id}"
buckets.setdefault(key, []).append(event)
groups: list[list[NormalizedEvent]] = []
window = timedelta(seconds=rule.window_seconds)
isolated.append(event)
else:
buckets.setdefault(key, []).append(event)
groups: list[Group] = []
window = timedelta(seconds=window_seconds)
for key in sorted(buckets):
ordered = sorted(buckets[key], key=_event_sort_key)
current: list[NormalizedEvent] = []
@@ -290,73 +492,202 @@ def _groups_for_rule(events: list[NormalizedEvent], rule: Rule) -> list[list[Nor
if not current or event.timestamp - current[0].timestamp <= window:
current.append(event)
else:
groups.append(current)
groups.append((key, current))
current = [event]
if current:
groups.append(current)
groups.append((key, current))
groups.extend((None, [event]) for event in sorted(isolated, key=_event_sort_key))
return groups
def _always(_event: NormalizedEvent) -> bool:
return True
def _groups_for_rule(events: list[NormalizedEvent], rule: Rule) -> list[Group]:
groups = _windowed_groups(_matching(events, rule.predicate), _always, rule.key, rule.window_seconds)
if rule.accept is None:
return groups
return [group for group in groups if rule.accept(group[1])]
def _incident_id(rule_id: str, evidence: list[NormalizedEvent]) -> str:
seed = f"{rule_id}|{'|'.join(event.id for event in evidence)}".encode()
return f"incident-{hashlib.sha256(seed).hexdigest()[:16]}"
def _build(rule: Rule, evidence: list[NormalizedEvent]) -> IncidentCluster:
signatures = {(event.provider, event.event_id, event.summary) for event in evidence}
reasons = [rule.reason_for(evidence)]
if len(evidence) > 1:
def _signatures(evidence: list[NormalizedEvent]) -> set[tuple[str, int | None, str]]:
return {(event.provider, event.event_id, event.summary) for event in evidence}
def _compression_reasons(evidence: list[NormalizedEvent]) -> list[str]:
reasons: list[str] = []
timed = all(event.timestamp is not None and event.timestamp_valid for event in evidence)
if len(evidence) > 1 and timed:
seconds = int((evidence[-1].timestamp - evidence[0].timestamp).total_seconds())
reasons.append(f"{len(evidence)} evidence events occurred within {seconds} second(s).")
signatures = _signatures(evidence)
if len(signatures) < len(evidence):
reasons.append(f"Repeated signals compress from {len(evidence)} records to {len(signatures)} distinct signature(s).")
return IncidentCluster(
return reasons
def _build(rule: Rule, evidence: list[NormalizedEvent]) -> IncidentCluster:
return IncidentCluster.model_construct(
id=_incident_id(rule.id, evidence),
rule_id=rule.id,
start=evidence[0].timestamp,
end=evidence[-1].timestamp,
title=rule.title_for(evidence),
title=_bounded_title(rule.title_for(evidence), "Correlated evidence"),
category=rule.category,
severity=rule.severity,
reasons=reasons,
reasons=[rule.reason_for(evidence), *_compression_reasons(evidence)],
evidence_event_ids=[event.id for event in evidence],
original_count=len(evidence),
compressed_count=max(1, len(signatures)),
compressed_count=max(1, len(_signatures(evidence))),
)
def _attention(event: NormalizedEvent) -> bool:
return event.level.casefold() in LEVEL_SEVERITY
def _fallback_key(event: NormalizedEvent) -> str | None:
provider = _dimension(event.provider)
if provider is None or provider.casefold() == "unknown provider" or event.event_id is None:
return None
return _group_key(_host(event), provider, str(event.event_id))
def _max_severity(levels: list[Severity]) -> Severity:
return max(levels, key=lambda level: SEVERITY_RANK[level])
def _build_fallback(evidence: list[NormalizedEvent]) -> IncidentCluster:
first = evidence[0]
timed = first.timestamp is not None and first.timestamp_valid
if len(evidence) == 1:
reason = "An error or warning was not matched by a more specific correlation rule."
else:
reason = (
"Errors or warnings with the same provider, event ID and known host were not matched by a more specific "
"rule and were compressed into one bounded window; repetition is not a causal claim."
)
return IncidentCluster.model_construct(
id=_incident_id(FALLBACK_RULE_ID, evidence),
rule_id=FALLBACK_RULE_ID,
start=first.timestamp if timed else None,
end=evidence[-1].timestamp if timed else None,
title=_bounded_title(first.summary, "Unclassified attention event"),
category=first.category,
severity=_max_severity([LEVEL_SEVERITY.get(event.level.casefold(), "warning") for event in evidence]),
reasons=[reason, *_compression_reasons(evidence)],
evidence_event_ids=[event.id for event in evidence],
original_count=len(evidence),
compressed_count=max(1, len(_signatures(evidence))),
)
# --- campaigns --------------------------------------------------------------
def _campaigns_for(
rule_id: str, window_seconds: int, keyed: list[tuple[str | None, IncidentCluster]]
) -> list[Campaign]:
"""Link adjacent windows of one rule and one known key into a summary campaign.
``keyed`` is ordered by key and then by time, as produced by
:func:`_windowed_groups`. Incidents keep their bounded windows; a campaign
only records that the same rule recurred for the same known dimensions.
"""
campaigns: list[Campaign] = []
window = timedelta(seconds=window_seconds)
run: list[IncidentCluster] = []
run_key: str | None = None
def flush() -> None:
if len(run) < 2:
return
first, last = run[0], run[-1]
incident_ids = [incident.id for incident in run]
seed = f"{rule_id}|{'|'.join(incident_ids)}".encode()
campaign = Campaign.model_construct(
id=f"campaign-{hashlib.sha256(seed).hexdigest()[:16]}",
rule_id=rule_id,
title=_bounded_title(f"Recurring: {first.title}", "Recurring activity"),
category=first.category,
severity=_max_severity([incident.severity for incident in run]),
start=first.start,
end=last.end,
incident_ids=incident_ids,
event_count=sum(incident.original_count for incident in run),
reasons=[
(
f"The same rule fired for the same known grouping dimensions in {len(run)} adjacent windows "
f"(gaps of at most {window_seconds} second(s))."
),
"Each incident stays bounded by its rule span; the campaign only summarizes recurrence and makes no causal claim.",
],
)
for incident in run:
incident.campaign_id = campaign.id
campaigns.append(campaign)
for key, incident in keyed:
previous = run[-1] if run else None
contiguous = (
key is not None
and key == run_key
and previous is not None
and previous.end is not None
and incident.start is not None
and incident.start - previous.end <= window
)
if not contiguous:
flush()
run = []
run_key = key
if key is not None and incident.start is not None:
run.append(incident)
flush()
return campaigns
def _incident_sort_key(incident: IncidentCluster) -> tuple[bool, datetime, str, str]:
return (
incident.start is None,
incident.start or datetime.max.replace(tzinfo=UTC),
incident.rule_id,
incident.id,
)
def correlate_with_campaigns(events: list[NormalizedEvent]) -> tuple[list[IncidentCluster], list[Campaign]]:
ordered = sorted(events, key=_event_sort_key)
incidents: list[IncidentCluster] = []
campaigns: list[Campaign] = []
consumed: set[str] = set()
for rule in RULES:
keyed = [(key, _build(rule, group)) for key, group in _groups_for_rule(ordered, rule)]
for _, incident in keyed:
consumed.update(incident.evidence_event_ids)
incidents.extend(incident for _, incident in keyed)
campaigns.extend(_campaigns_for(rule.id, rule.window_seconds, keyed))
remaining = [event for event in ordered if event.id not in consumed and _attention(event)]
keyed_fallback = [
(key, _build_fallback(group))
for key, group in _windowed_groups(remaining, _always, _fallback_key, FALLBACK_WINDOW_SECONDS)
]
incidents.extend(incident for _, incident in keyed_fallback)
campaigns.extend(_campaigns_for(FALLBACK_RULE_ID, FALLBACK_WINDOW_SECONDS, keyed_fallback))
# Untimed attention events stay visible as evidence but never join a temporal group.
incidents.extend(
_build_fallback([event]) for event in remaining if event.timestamp is None or not event.timestamp_valid
)
campaigns.sort(key=lambda campaign: (campaign.start, campaign.rule_id, campaign.id))
return sorted(incidents, key=_incident_sort_key), campaigns
def correlate(events: list[NormalizedEvent]) -> list[IncidentCluster]:
ordered = sorted(events, key=_event_sort_key)
incidents: list[IncidentCluster] = []
consumed: set[str] = set()
for rule in RULES:
for group in _groups_for_rule(ordered, rule):
incidents.append(_build(rule, group))
consumed.update(event.id for event in group)
fallback_rule_id = "windows.generic.attention"
for event in ordered:
if event.id not in consumed and event.level.casefold() in {"critical", "error", "warning"}:
incidents.append(
IncidentCluster(
id=_incident_id(fallback_rule_id, [event]),
rule_id=fallback_rule_id,
start=event.timestamp if event.timestamp_valid else None,
end=event.timestamp if event.timestamp_valid else None,
title=event.summary or "Unclassified attention event",
category=event.category,
severity="warning",
reasons=["An error or warning was not matched by a more specific correlation rule."],
evidence_event_ids=[event.id],
original_count=1,
compressed_count=1,
)
)
return sorted(
incidents,
key=lambda incident: (
incident.start is None,
incident.start or datetime.max.replace(tzinfo=UTC),
incident.rule_id,
incident.id,
),
)
return correlate_with_campaigns(events)[0]
+164 -102
View File
@@ -1,22 +1,30 @@
import asyncio
import gzip
import hashlib
import json
import logging
import os
import re
from collections import Counter
from collections.abc import AsyncIterator
from contextlib import asynccontextmanager
from pathlib import Path
from typing import Any
from fastapi import FastAPI, File, HTTPException, UploadFile
from fastapi import FastAPI, HTTPException, Request, Response
from fastapi.middleware.cors import CORSMiddleware
from fastapi.staticfiles import StaticFiles
from starlette.datastructures import UploadFile as StarletteUploadFile
from starlette.formparsers import MultiPartException
from starlette.requests import ClientDisconnect
from .correlate import correlate
from .correlate import correlate_with_campaigns
from .demo import load_demo
from .models import AnalysisResponse, ParseIssue, ParseStats
from .models import AnalysisResponse, ParseIssue, ParseStats, ParseStatus, SourceInfo
from .parser import (
MAX_UPLOAD_BYTES,
EvtxParseError,
ParseResult,
UnexpectedParserError,
parse_evtx_bytes,
)
@@ -43,10 +51,11 @@ app.add_middleware(
MAX_MULTIPART_OVERHEAD_BYTES = 1024 * 1024
MAX_REQUEST_BYTES = MAX_UPLOAD_BYTES + MAX_MULTIPART_OVERHEAD_BYTES
MAX_REPORTED_ISSUES = 200
GZIP_LEVEL = 1
CONTENT_SECURITY_POLICY = (
"default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; "
"img-src 'self' data:; connect-src 'self' http://localhost:8000 http://127.0.0.1:8000 "
"http://localhost:5173 http://127.0.0.1:5173 http://127.0.0.1:4173; "
"img-src 'self' data:; connect-src 'self'; "
"font-src 'self'; object-src 'none'; base-uri 'none'; frame-ancestors 'none'; form-action 'self'"
)
SECURITY_HEADERS = {
@@ -176,49 +185,67 @@ def health():
@app.get("/api/demo", response_model=AnalysisResponse)
def demo():
events = load_demo()
return AnalysisResponse(source="bundled-demo", events=events, incidents=correlate(events))
incidents, campaigns = correlate_with_campaigns(events)
return AnalysisResponse(source="bundled-demo", events=events, incidents=incidents, campaigns=campaigns)
async def _parse_with_admission(data: bytes):
def _capacity_error(message: str) -> HTTPException:
return HTTPException(
503,
detail={"code": "parse_capacity_exceeded", "message": message},
headers={"Retry-After": "1"},
)
@asynccontextmanager
async def _admission() -> AsyncIterator[None]:
"""Reserve an analysis slot before the multipart body is spooled.
Starlette spools uploads to TMPDIR while parsing the form. Admitting first
bounds the number of concurrently spooled uploads to concurrency + queue,
so a burst of large uploads gets a fast 503 instead of exhausting tmpfs.
"""
global _parse_inflight
async with _parse_admission_lock:
if _parse_inflight >= PARSE_CONCURRENCY + PARSE_QUEUE_LIMIT:
raise HTTPException(
503,
detail={
"code": "parse_capacity_exceeded",
"message": "EventLens is already analyzing its maximum concurrent uploads; retry shortly.",
},
headers={"Retry-After": "1"},
)
raise _capacity_error("EventLens is already analyzing its maximum concurrent uploads; retry shortly.")
_parse_inflight += 1
acquired = False
try:
try:
await asyncio.wait_for(_parse_slots.acquire(), timeout=PARSE_QUEUE_TIMEOUT_SECONDS)
acquired = True
except TimeoutError as exc:
raise HTTPException(
503,
detail={
"code": "parse_capacity_exceeded",
"message": "EventLens is busy analyzing another upload; retry shortly.",
},
headers={"Retry-After": "1"},
) from exc
yield
finally:
if not acquired:
async with _parse_admission_lock:
_parse_inflight -= 1
try:
return await asyncio.to_thread(parse_evtx_bytes, data)
finally:
if acquired:
_parse_slots.release()
async with _parse_admission_lock:
_parse_inflight -= 1
@asynccontextmanager
async def _parse_slot() -> AsyncIterator[None]:
try:
await asyncio.wait_for(_parse_slots.acquire(), timeout=PARSE_QUEUE_TIMEOUT_SECONDS)
except TimeoutError as exc:
raise _capacity_error("EventLens is busy analyzing another upload; retry shortly.") from exc
try:
yield
finally:
_parse_slots.release()
def _stats(parsed: ParseResult, status: ParseStatus | None = None) -> ParseStats:
counts = Counter(issue.code for issue in parsed.issues)
return ParseStats(
status=status or parsed.status,
parsed_events=len(parsed.events) if status != "failed" else 0,
records_seen=parsed.records_seen,
skipped_records=parsed.skipped_records,
issue_count=len(parsed.issues),
truncated=parsed.truncated,
truncation_reason=parsed.truncation_reason,
duration_ms=parsed.duration_ms,
issues=parsed.issues[:MAX_REPORTED_ISSUES],
issues_truncated=len(parsed.issues) > MAX_REPORTED_ISSUES,
issue_counts=dict(sorted(counts.items())),
)
def _failed_parse_stats() -> ParseStats:
return ParseStats(
status="failed",
@@ -230,77 +257,112 @@ def _failed_parse_stats() -> ParseStats:
truncation_reason=None,
duration_ms=0,
issues=[ParseIssue(code="parser_rejected_input", message="The EVTX parser rejected the input.")],
issue_counts={"parser_rejected_input": 1},
)
@app.post("/api/analyze", response_model=AnalysisResponse)
async def analyze(file: UploadFile = File(...)): # noqa: B008
def _analyze_bytes(source: str, data: bytes | bytearray, compress: bool = False) -> bytes:
"""Parse, correlate, serialize and optionally gzip off the event loop."""
source_info = SourceInfo(sha256=hashlib.sha256(data).hexdigest(), size_bytes=len(data))
try:
source = _safe_source_name(file.filename)
if not source.lower().endswith(".evtx"):
raise HTTPException(400, "Upload an .evtx file")
data = bytearray()
while True:
chunk = await file.read(min(1024 * 1024, MAX_UPLOAD_BYTES + 1 - len(data)))
if not chunk:
break
data.extend(chunk)
if len(data) > MAX_UPLOAD_BYTES:
raise HTTPException(413, "EVTX file exceeds the 50 MiB MVP limit")
try:
parsed = await _parse_with_admission(bytes(data))
except EvtxParseError as exc:
raise HTTPException(
422,
detail={
"code": "parser_rejected_input",
"message": str(exc),
"parse": _failed_parse_stats().model_dump(mode="json"),
},
) from exc
except UnexpectedParserError as exc:
logger.error("Unexpected EVTX parser failure reached API exception_type=%s", type(exc).__name__)
raise HTTPException(
500,
detail={"code": "parser_internal_error", "message": "EVTX analysis failed unexpectedly; retry the request."},
) from exc
if not parsed.events:
raise HTTPException(
422,
detail={
"code": "no_readable_events",
"message": "No readable Windows events were found in this file.",
"parse": ParseStats(
status="failed",
parsed_events=0,
records_seen=parsed.records_seen,
skipped_records=parsed.skipped_records,
issue_count=len(parsed.issues),
truncated=parsed.truncated,
truncation_reason=parsed.truncation_reason,
duration_ms=parsed.duration_ms,
issues=parsed.issues,
).model_dump(mode="json"),
},
)
return AnalysisResponse(
source=source,
events=parsed.events,
incidents=correlate(parsed.events),
parse=ParseStats(
status=parsed.status,
parsed_events=len(parsed.events),
records_seen=parsed.records_seen,
skipped_records=parsed.skipped_records,
issue_count=len(parsed.issues),
truncated=parsed.truncated,
truncation_reason=parsed.truncation_reason,
duration_ms=parsed.duration_ms,
issues=parsed.issues,
),
parsed = parse_evtx_bytes(data)
except EvtxParseError as exc:
raise HTTPException(
422,
detail={
"code": "parser_rejected_input",
"message": str(exc),
"parse": _failed_parse_stats().model_dump(mode="json"),
},
) from exc
if not parsed.events:
raise HTTPException(
422,
detail={
"code": "no_readable_events",
"message": "No readable Windows events were found in this file.",
"parse": _stats(parsed, "failed").model_dump(mode="json"),
},
)
finally:
await file.close()
incidents, campaigns = correlate_with_campaigns(parsed.events)
response = AnalysisResponse(
source=source,
events=parsed.events,
incidents=incidents,
campaigns=campaigns,
parse=_stats(parsed),
source_info=source_info,
)
payload = response.model_dump_json().encode("utf-8")
# Event JSON is highly repetitive: level 1 shrinks a 100k-event response
# ~20x for about a second of worker-thread CPU, never on the event loop.
return gzip.compress(payload, compresslevel=GZIP_LEVEL, mtime=0) if compress else payload
async def _read_bounded(upload: StarletteUploadFile) -> bytearray:
data = bytearray()
while True:
chunk = await upload.read(min(1024 * 1024, MAX_UPLOAD_BYTES + 1 - len(data)))
if not chunk:
return data
data.extend(chunk)
if len(data) > MAX_UPLOAD_BYTES:
raise HTTPException(413, "EVTX file exceeds the 50 MiB MVP limit")
ANALYZE_OPENAPI = {
"requestBody": {
"required": True,
"content": {
"multipart/form-data": {
"schema": {
"type": "object",
"required": ["file"],
"properties": {"file": {"type": "string", "format": "binary"}},
}
}
},
}
}
@app.post("/api/analyze", response_model=AnalysisResponse, openapi_extra=ANALYZE_OPENAPI)
async def analyze(request: Request):
compress = "gzip" in request.headers.get("accept-encoding", "").lower()
async with _admission():
try:
form = await request.form(max_files=1, max_fields=4)
except MultiPartException as exc:
raise HTTPException(400, "Upload one .evtx file as multipart form data") from exc
try:
upload = form.get("file")
if not isinstance(upload, StarletteUploadFile):
raise HTTPException(422, "Upload one .evtx file in the 'file' form field")
source = _safe_source_name(upload.filename)
if not source.lower().endswith(".evtx"):
raise HTTPException(400, "Upload an .evtx file")
data = await _read_bounded(upload)
async with _parse_slot():
try:
payload = await asyncio.to_thread(_analyze_bytes, source, data, compress)
except UnexpectedParserError as exc:
logger.error(
"Unexpected EVTX parser failure reached API exception_type=%s", type(exc).__name__
)
raise HTTPException(
500,
detail={
"code": "parser_internal_error",
"message": "EVTX analysis failed unexpectedly; retry the request.",
},
) from exc
del data
finally:
await form.close()
headers = {"Vary": "Accept-Encoding"}
if compress:
headers["Content-Encoding"] = "gzip"
return Response(content=payload, media_type="application/json", headers=headers)
def _safe_source_name(filename: str | None) -> str:
+36 -2
View File
@@ -3,7 +3,11 @@ from typing import Literal
from pydantic import BaseModel, Field, field_validator, model_validator
Category = Literal["system", "service", "network", "authentication", "application", "crash", "other"]
Category = Literal[
"system", "service", "network", "authentication", "security", "application", "crash", "other"
]
Severity = Literal["critical", "error", "warning", "info"]
AuditOutcome = Literal["success", "failure"]
ParseStatus = Literal["complete", "partial", "failed"]
TimestampSource = Literal["event", "record", "missing", "invalid"]
@@ -20,6 +24,7 @@ class NormalizedEvent(BaseModel):
computer: str = Field(default="", max_length=256)
category: Category = "other"
summary: str = Field(default="", max_length=512)
outcome: AuditOutcome | None = None
fields: dict[str, str] = Field(default_factory=dict, max_length=200)
@field_validator("timestamp")
@@ -49,11 +54,31 @@ class IncidentCluster(BaseModel):
end: datetime | None = None
title: str = Field(max_length=256)
category: Category
severity: str = Field(max_length=32)
severity: Severity
reasons: list[str] = Field(max_length=8)
evidence_event_ids: list[str]
original_count: int = Field(ge=1)
compressed_count: int = Field(ge=1)
campaign_id: str | None = Field(default=None, max_length=128)
class Campaign(BaseModel):
"""Consecutive incidents of one rule and grouping key, linked for summary only.
Incident windows stay bounded by their rule span; a campaign only states
that the same rule fired for the same known dimensions in adjacent windows.
"""
id: str = Field(max_length=128)
rule_id: str = Field(max_length=128)
title: str = Field(max_length=256)
category: Category
severity: Severity
start: datetime
end: datetime
incident_ids: list[str] = Field(min_length=2)
event_count: int = Field(ge=2)
reasons: list[str] = Field(max_length=8)
class ParseIssue(BaseModel):
@@ -74,10 +99,19 @@ class ParseStats(BaseModel):
truncation_reason: str | None = Field(default=None, max_length=64)
duration_ms: int = Field(ge=0)
issues: list[ParseIssue] = Field(default_factory=list)
issues_truncated: bool = False
issue_counts: dict[str, int] = Field(default_factory=dict)
class SourceInfo(BaseModel):
sha256: str = Field(pattern=r"^[0-9a-f]{64}$")
size_bytes: int = Field(ge=0)
class AnalysisResponse(BaseModel):
source: str
events: list[NormalizedEvent]
incidents: list[IncidentCluster]
campaigns: list[Campaign] = Field(default_factory=list)
parse: ParseStats | None = None
source_info: SourceInfo | None = None
+90 -16
View File
@@ -3,9 +3,12 @@ import json
from datetime import UTC, datetime
from typing import Any
from .models import Category, NormalizedEvent, TimestampSource
from .models import AuditOutcome, Category, NormalizedEvent, TimestampSource
AUTH_EVENT_IDS = {4624, 4625, 4634, 4647, 4648, 4672, 4768, 4769, 4771, 4776}
AUTH_EVENT_IDS = {4624, 4625, 4634, 4647, 4648, 4672, 4740, 4768, 4769, 4771, 4776}
ACCOUNT_CHANGE_EVENT_IDS = {4720, 4728, 4732, 4756}
AUDIT_FAILURE_KEYWORD = 0x0010_0000_0000_0000
AUDIT_SUCCESS_KEYWORD = 0x0020_0000_0000_0000
LEVELS = {0: "information", 1: "critical", 2: "error", 3: "warning", 4: "information", 5: "verbose"}
MAX_EVENT_ID = 4_294_967_295
MAX_RECORD_ID = 9_223_372_036_854_775_807
@@ -86,6 +89,21 @@ def _provider(system: dict[str, Any]) -> str:
return _bounded_text(value, MAX_PROVIDER_CHARS, "Unknown provider")
def _outcome(system: dict[str, Any]) -> AuditOutcome | None:
raw = _scalar(system.get("Keywords"))
if raw is None or isinstance(raw, bool):
return None
try:
value = int(raw, 0) if isinstance(raw, str) else int(raw)
except (TypeError, ValueError, OverflowError):
return None
if value & AUDIT_FAILURE_KEYWORD:
return "failure"
if value & AUDIT_SUCCESS_KEYWORD:
return "success"
return None
def _store_field(output: dict[str, str], key: str, value: str, limit: int) -> None:
bounded_key = _bounded_text(key or "value", MAX_FIELD_KEY_CHARS)
if bounded_key not in output:
@@ -127,47 +145,78 @@ def _flatten(
if len(output) >= limit:
return
return
_store_field(output, prefix or "value", _render_scalar(value)[:MAX_FIELD_VALUE_CHARS], limit)
def _render_scalar(value: Any) -> str:
"""Render a scalar exactly like ``json.dumps`` would, with fast paths for common types."""
if isinstance(value, str):
return value
if value is None:
return "null"
if value is True:
return "true"
if value is False:
return "false"
if type(value) is int:
return str(value)
try:
rendered = json.dumps(value, ensure_ascii=False) if not isinstance(value, str) else value
return json.dumps(value, ensure_ascii=False)
except (TypeError, ValueError):
rendered = "[unrenderable value]"
_store_field(output, prefix or "value", rendered[:MAX_FIELD_VALUE_CHARS], limit)
return "[unrenderable value]"
def _fold(value: str) -> str:
return value.strip().casefold()
EVENTLOG_PROVIDERS = frozenset({"eventlog", "microsoft-windows-eventlog"})
def _channel_matches(channel: str, *accepted: str) -> bool:
return channel.strip().casefold() in {value.casefold() for value in accepted}
folded = _fold(channel)
return any(folded == value.casefold() for value in accepted)
def is_security_audit(provider: str, channel: str) -> bool:
return provider.strip().casefold() == "microsoft-windows-security-auditing" and _channel_matches(channel, "Security")
return _fold(provider) == "microsoft-windows-security-auditing" and _fold(channel) == "security"
def is_service_control_manager(provider: str, channel: str) -> bool:
return provider.strip().casefold() == "service control manager" and _channel_matches(channel, "System")
return _fold(provider) == "service control manager" and _fold(channel) == "system"
def is_dns_client(provider: str, channel: str) -> bool:
return provider.strip().casefold() == "microsoft-windows-dns-client" and _channel_matches(channel, "System")
return _fold(provider) == "microsoft-windows-dns-client" and _fold(channel) == "system"
def is_application_error(provider: str, channel: str, event_id: int | None) -> bool:
return provider.casefold().strip() == "application error" and event_id == 1000 and _channel_matches(channel, "Application")
return event_id == 1000 and _fold(provider) == "application error" and _fold(channel) == "application"
def is_windows_error_reporting(provider: str, channel: str, event_id: int | None) -> bool:
return provider.casefold().strip() == "windows error reporting" and event_id == 1001 and _channel_matches(channel, "Application")
return event_id == 1001 and _fold(provider) == "windows error reporting" and _fold(channel) == "application"
def is_event_log(provider: str, channel: str) -> bool:
return provider.casefold().strip() in {"eventlog", "microsoft-windows-eventlog"} and _channel_matches(channel, "System")
return _fold(provider) in EVENTLOG_PROVIDERS and _fold(channel) == "system"
def is_log_cleared(provider: str, channel: str, event_id: int | None) -> bool:
"""Security 1102 (audit log cleared) or System 104 (an event log was cleared)."""
if event_id not in {1102, 104} or _fold(provider) not in EVENTLOG_PROVIDERS:
return False
return (event_id == 1102 and _channel_matches(channel, "Security")) or (
event_id == 104 and _channel_matches(channel, "System")
)
def is_kernel_general(provider: str, channel: str) -> bool:
return provider.strip().casefold() == "microsoft-windows-kernel-general" and _channel_matches(channel, "System")
return _fold(provider) == "microsoft-windows-kernel-general" and _fold(channel) == "system"
def is_kernel_power(provider: str, channel: str) -> bool:
return provider.strip().casefold() == "microsoft-windows-kernel-power" and _channel_matches(channel, "System")
return _fold(provider) == "microsoft-windows-kernel-power" and _fold(channel) == "system"
def classify(provider: str, event_id: int | None, channel: str) -> Category:
@@ -181,6 +230,10 @@ def classify(provider: str, event_id: int | None, channel: str) -> Category:
return "network"
if is_security_audit(provider, channel) and event_id in AUTH_EVENT_IDS:
return "authentication"
if is_log_cleared(provider, channel, event_id) or (
is_security_audit(provider, channel) and event_id in ACCOUNT_CHANGE_EVENT_IDS
):
return "security"
if is_application_error(provider, channel, event_id) or is_windows_error_reporting(provider, channel, event_id):
return "crash"
if (
@@ -195,6 +248,23 @@ def classify(provider: str, event_id: int | None, channel: str) -> Category:
def summarize(provider: str, event_id: int | None, category: Category, fields: dict[str, str]) -> str:
if category == "service" and event_id == 7045:
service = fields.get("EventData.ServiceName") or fields.get("service")
return f"Service installed: {service}" if service else "Service installed"
if category == "security" and event_id == 1102:
return "Security audit log cleared"
if category == "security" and event_id == 104:
cleared = fields.get("UserData.LogFileCleared.Channel")
return f"Event log cleared: {cleared}" if cleared else "Event log cleared"
if category == "security" and event_id == 4720:
account = fields.get("EventData.TargetUserName")
return f"User account created: {account}" if account else "User account created"
if category == "security" and event_id in {4728, 4732, 4756}:
group = fields.get("EventData.TargetUserName")
return f"Member added to security group {group}" if group else "Member added to a security group"
if category == "authentication" and event_id == 4740:
account = fields.get("EventData.TargetUserName")
return f"Account locked out: {account}" if account else "Account locked out"
if category == "service" and event_id in {7023, 7024, 7031, 7034}:
service = fields.get("EventData.param1") or fields.get("UserData.param1") or fields.get("service")
return f"{service} terminated unexpectedly" if service else "Service failure recorded"
@@ -245,11 +315,14 @@ def normalize_record(
stable_id = f"event-{hashlib.sha256(seed).hexdigest()[:16]}"
timestamp, timestamp_source = _timestamp(system, record_timestamp)
summary = summarize(provider, event_id, category, fields)[:MAX_SUMMARY_CHARS]
return NormalizedEvent(
# Every value above is already bounded and typed exactly as NormalizedEvent
# requires (see tests/test_performance.py, which re-validates the output),
# so skip per-event pydantic validation on this hot path.
return NormalizedEvent.model_construct(
id=stable_id,
timestamp=timestamp,
timestamp_source=timestamp_source,
timestamp_valid=timestamp is not None,
timestamp_valid=timestamp is not None and timestamp_source in {"event", "record"},
provider=provider,
event_id=event_id,
level=level[:32],
@@ -257,5 +330,6 @@ def normalize_record(
computer=computer,
category=category,
summary=summary,
outcome=_outcome(system),
fields=fields,
)
+6 -4
View File
@@ -52,7 +52,7 @@ def _fingerprint(stage: str, exc: BaseException) -> str:
def parse_evtx_bytes(
data: bytes,
data: bytes | bytearray,
max_events: int = MAX_EVENTS,
max_seconds: float = MAX_PARSE_SECONDS,
) -> ParseResult:
@@ -113,9 +113,11 @@ def parse_evtx_bytes(
raw_data = record["data"]
if not isinstance(raw_data, str):
raise TypeError("record data was not textual")
if (
len(raw_data) > MAX_RECORD_JSON_BYTES
or len(raw_data.encode("utf-8")) > MAX_RECORD_JSON_BYTES
# A str of n characters encodes to at most 4n UTF-8 bytes, so the
# costly encode is only needed for records near the limit.
if len(raw_data) > MAX_RECORD_JSON_BYTES or (
len(raw_data) * 4 > MAX_RECORD_JSON_BYTES
and len(raw_data.encode("utf-8")) > MAX_RECORD_JSON_BYTES
):
add_issue(
"record_json_limit_exceeded",