Files
EventLens-Public/backend/app/normalize.py
T
2026-11-20 12:20:18 +00:00

336 lines
13 KiB
Python

import hashlib
import json
from datetime import UTC, datetime
from typing import Any
from .models import AuditOutcome, Category, NormalizedEvent, TimestampSource
AUTH_EVENT_IDS = {4624, 4625, 4634, 4647, 4648, 4672, 4740, 4768, 4769, 4771, 4776}
ACCOUNT_CHANGE_EVENT_IDS = {4720, 4728, 4732, 4756}
AUDIT_FAILURE_KEYWORD = 0x0010_0000_0000_0000
AUDIT_SUCCESS_KEYWORD = 0x0020_0000_0000_0000
LEVELS = {0: "information", 1: "critical", 2: "error", 3: "warning", 4: "information", 5: "verbose"}
MAX_EVENT_ID = 4_294_967_295
MAX_RECORD_ID = 9_223_372_036_854_775_807
MAX_FIELDS = 200
MAX_FIELD_KEY_CHARS = 256
MAX_FIELD_VALUE_CHARS = 1000
MAX_NESTING_DEPTH = 16
MAX_PROVIDER_CHARS = 256
MAX_CHANNEL_CHARS = 256
MAX_COMPUTER_CHARS = 256
MAX_SUMMARY_CHARS = 512
def _bounded_text(value: Any, limit: int, fallback: str = "") -> str:
if value is None:
return fallback
return str(value)[:limit]
def _attributes(value: Any) -> dict[str, Any]:
return value.get("#attributes", {}) if isinstance(value, dict) else {}
def _scalar(value: Any) -> Any:
if isinstance(value, dict):
for key in ("#text", "Value", "value"):
if key in value:
return value[key]
return value
def _integer(value: Any, minimum: int = 0, maximum: int = MAX_RECORD_ID) -> int | None:
value = _scalar(value)
if isinstance(value, bool):
return None
try:
parsed = int(value)
except (TypeError, ValueError, OverflowError):
return None
return parsed if minimum <= parsed <= maximum else None
def _coerce_timestamp(value: Any) -> datetime | None:
if isinstance(value, datetime):
return value if value.tzinfo else value.replace(tzinfo=UTC)
if value is None:
return None
text = str(value).strip()
if not text:
return None
if text.endswith("Z"):
text = f"{text[:-1]}+00:00"
try:
parsed = datetime.fromisoformat(text)
except (TypeError, ValueError):
return None
return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC)
def _timestamp(system: dict[str, Any], fallback: Any) -> tuple[datetime | None, TimestampSource]:
event_raw = _attributes(system.get("TimeCreated")).get("SystemTime")
if event_raw is not None:
parsed = _coerce_timestamp(event_raw)
if parsed is not None:
return parsed, "event"
record_parsed = _coerce_timestamp(fallback)
return (record_parsed, "record" if record_parsed is not None else "invalid")
if fallback is not None:
record_parsed = _coerce_timestamp(fallback)
return (record_parsed, "record" if record_parsed is not None else "invalid")
return None, "missing"
def _provider(system: dict[str, Any]) -> str:
value = system.get("Provider")
if isinstance(value, dict):
return _bounded_text(_attributes(value).get("Name") or value.get("Name"), MAX_PROVIDER_CHARS, "Unknown provider")
return _bounded_text(value, MAX_PROVIDER_CHARS, "Unknown provider")
def _outcome(system: dict[str, Any]) -> AuditOutcome | None:
raw = _scalar(system.get("Keywords"))
if raw is None or isinstance(raw, bool):
return None
try:
value = int(raw, 0) if isinstance(raw, str) else int(raw)
except (TypeError, ValueError, OverflowError):
return None
if value & AUDIT_FAILURE_KEYWORD:
return "failure"
if value & AUDIT_SUCCESS_KEYWORD:
return "success"
return None
def _store_field(output: dict[str, str], key: str, value: str, limit: int) -> None:
bounded_key = _bounded_text(key or "value", MAX_FIELD_KEY_CHARS)
if bounded_key not in output:
output[bounded_key] = value
return
suffix = 2
while len(output) < limit:
suffix_key = _bounded_text(f"{bounded_key}#{suffix}", MAX_FIELD_KEY_CHARS)
if suffix_key not in output:
output[suffix_key] = value
return
suffix += 1
def _flatten(
prefix: str,
value: Any,
output: dict[str, str],
limit: int = MAX_FIELDS,
depth: int = 0,
) -> None:
if len(output) >= limit:
return
if depth > MAX_NESTING_DEPTH:
_store_field(output, prefix or "value", "[nested value truncated]", limit)
return
if isinstance(value, dict):
for key, child in value.items():
if key == "#attributes":
continue
next_prefix = f"{prefix}.{key}" if prefix else str(key)
_flatten(next_prefix, child, output, limit, depth + 1)
if len(output) >= limit:
return
return
if isinstance(value, list):
for index, child in enumerate(value):
_flatten(f"{prefix}[{index}]", child, output, limit, depth + 1)
if len(output) >= limit:
return
return
_store_field(output, prefix or "value", _render_scalar(value)[:MAX_FIELD_VALUE_CHARS], limit)
def _render_scalar(value: Any) -> str:
"""Render a scalar exactly like ``json.dumps`` would, with fast paths for common types."""
if isinstance(value, str):
return value
if value is None:
return "null"
if value is True:
return "true"
if value is False:
return "false"
if type(value) is int:
return str(value)
try:
return json.dumps(value, ensure_ascii=False)
except (TypeError, ValueError):
return "[unrenderable value]"
def _fold(value: str) -> str:
return value.strip().casefold()
EVENTLOG_PROVIDERS = frozenset({"eventlog", "microsoft-windows-eventlog"})
def _channel_matches(channel: str, *accepted: str) -> bool:
folded = _fold(channel)
return any(folded == value.casefold() for value in accepted)
def is_security_audit(provider: str, channel: str) -> bool:
return _fold(provider) == "microsoft-windows-security-auditing" and _fold(channel) == "security"
def is_service_control_manager(provider: str, channel: str) -> bool:
return _fold(provider) == "service control manager" and _fold(channel) == "system"
def is_dns_client(provider: str, channel: str) -> bool:
return _fold(provider) == "microsoft-windows-dns-client" and _fold(channel) == "system"
def is_application_error(provider: str, channel: str, event_id: int | None) -> bool:
return event_id == 1000 and _fold(provider) == "application error" and _fold(channel) == "application"
def is_windows_error_reporting(provider: str, channel: str, event_id: int | None) -> bool:
return event_id == 1001 and _fold(provider) == "windows error reporting" and _fold(channel) == "application"
def is_event_log(provider: str, channel: str) -> bool:
return _fold(provider) in EVENTLOG_PROVIDERS and _fold(channel) == "system"
def is_log_cleared(provider: str, channel: str, event_id: int | None) -> bool:
"""Security 1102 (audit log cleared) or System 104 (an event log was cleared)."""
if event_id not in {1102, 104} or _fold(provider) not in EVENTLOG_PROVIDERS:
return False
return (event_id == 1102 and _channel_matches(channel, "Security")) or (
event_id == 104 and _channel_matches(channel, "System")
)
def is_kernel_general(provider: str, channel: str) -> bool:
return _fold(provider) == "microsoft-windows-kernel-general" and _fold(channel) == "system"
def is_kernel_power(provider: str, channel: str) -> bool:
return _fold(provider) == "microsoft-windows-kernel-power" and _fold(channel) == "system"
def classify(provider: str, event_id: int | None, channel: str) -> Category:
provider_lower = provider.casefold()
channel_lower = channel.casefold()
if is_service_control_manager(provider, channel):
return "service"
if is_dns_client(provider, channel) or any(
token in provider_lower for token in ("networkprofile", "tcpip", "dhcp", "nlasvc")
):
return "network"
if is_security_audit(provider, channel) and event_id in AUTH_EVENT_IDS:
return "authentication"
if is_log_cleared(provider, channel, event_id) or (
is_security_audit(provider, channel) and event_id in ACCOUNT_CHANGE_EVENT_IDS
):
return "security"
if is_application_error(provider, channel, event_id) or is_windows_error_reporting(provider, channel, event_id):
return "crash"
if (
(is_kernel_general(provider, channel) and event_id in {12, 13})
or (is_kernel_power(provider, channel) and event_id == 41)
or (is_event_log(provider, channel) and event_id in {6005, 6006, 6008})
):
return "system"
if "application" in channel_lower:
return "application"
return "other"
def summarize(provider: str, event_id: int | None, category: Category, fields: dict[str, str]) -> str:
if category == "service" and event_id == 7045:
service = fields.get("EventData.ServiceName") or fields.get("service")
return f"Service installed: {service}" if service else "Service installed"
if category == "security" and event_id == 1102:
return "Security audit log cleared"
if category == "security" and event_id == 104:
cleared = fields.get("UserData.LogFileCleared.Channel")
return f"Event log cleared: {cleared}" if cleared else "Event log cleared"
if category == "security" and event_id == 4720:
account = fields.get("EventData.TargetUserName")
return f"User account created: {account}" if account else "User account created"
if category == "security" and event_id in {4728, 4732, 4756}:
group = fields.get("EventData.TargetUserName")
return f"Member added to security group {group}" if group else "Member added to a security group"
if category == "authentication" and event_id == 4740:
account = fields.get("EventData.TargetUserName")
return f"Account locked out: {account}" if account else "Account locked out"
if category == "service" and event_id in {7023, 7024, 7031, 7034}:
service = fields.get("EventData.param1") or fields.get("UserData.param1") or fields.get("service")
return f"{service} terminated unexpectedly" if service else "Service failure recorded"
if category == "crash" and event_id == 1000:
process = fields.get("EventData.AppName") or fields.get("EventData.Data[0]") or fields.get("process")
return f"{process} crashed" if process else "Application crash recorded"
if category == "authentication" and event_id == 4625:
account = fields.get("EventData.TargetUserName") or fields.get("TargetUserName")
return f"Failed sign-in for {account}" if account else "Failed sign-in recorded"
return f"{provider} event {event_id}" if event_id is not None else f"{provider} event"
def normalize_record(
payload: dict[str, Any],
record_id: Any = None,
record_timestamp: Any = None,
record_ordinal: int | None = None,
) -> NormalizedEvent:
if not isinstance(payload, dict):
raise TypeError("Record does not contain an Event object")
event = payload.get("Event", payload)
if not isinstance(event, dict):
raise TypeError("Record does not contain an Event object")
system = event.get("System") if isinstance(event.get("System"), dict) else {}
event_id = _integer(system.get("EventID"), maximum=MAX_EVENT_ID)
provider = _provider(system)
channel = _bounded_text(_scalar(system.get("Channel")), MAX_CHANNEL_CHARS)
computer = _bounded_text(_scalar(system.get("Computer")), MAX_COMPUTER_CHARS)
level_value = _integer(system.get("Level"), maximum=5)
level = LEVELS.get(level_value, str(_scalar(system.get("Level")) or "information").lower())
fields: dict[str, str] = {}
if "EventData" in event:
_flatten("EventData", event.get("EventData"), fields)
if "UserData" in event:
_flatten("UserData", event.get("UserData"), fields)
category = classify(provider, event_id, channel)
normalized_record_id = _integer(system.get("EventRecordID"))
if normalized_record_id is None:
normalized_record_id = _integer(record_id)
if normalized_record_id is not None:
stable_id = str(normalized_record_id)
else:
canonical_event = json.dumps(event, sort_keys=True, ensure_ascii=False, separators=(",", ":"), default=str)
ordinal = "" if record_ordinal is None else str(record_ordinal)
seed = f"{provider}|{event_id}|{record_timestamp}|{ordinal}|{canonical_event}".encode(
"utf-8", errors="replace"
)
stable_id = f"event-{hashlib.sha256(seed).hexdigest()[:16]}"
timestamp, timestamp_source = _timestamp(system, record_timestamp)
summary = summarize(provider, event_id, category, fields)[:MAX_SUMMARY_CHARS]
# Every value above is already bounded and typed exactly as NormalizedEvent
# requires (see tests/test_performance.py, which re-validates the output),
# so skip per-event pydantic validation on this hot path.
return NormalizedEvent.model_construct(
id=stable_id,
timestamp=timestamp,
timestamp_source=timestamp_source,
timestamp_valid=timestamp is not None and timestamp_source in {"event", "record"},
provider=provider,
event_id=event_id,
level=level[:32],
channel=channel,
computer=computer,
category=category,
summary=summary,
outcome=_outcome(system),
fields=fields,
)