Publish EventLens v1.1.0 (#1)

This commit was merged in pull request #1.
This commit is contained in:
Jens committed 2026-11-20 12:20:18 +00:00
1 parent c0ade21029
commit 87432f3283
63 files changed
+7381 -1420

No files matched your search

+164 -102
View File
@@ -1,22 +1,30 @@
import asyncio
import gzip
import hashlib
import json
import logging
import os
import re
from collections import Counter
from collections.abc import AsyncIterator
from contextlib import asynccontextmanager
from pathlib import Path
from typing import Any
from fastapi import FastAPI, File, HTTPException, UploadFile
from fastapi import FastAPI, HTTPException, Request, Response
from fastapi.middleware.cors import CORSMiddleware
from fastapi.staticfiles import StaticFiles
from starlette.datastructures import UploadFile as StarletteUploadFile
from starlette.formparsers import MultiPartException
from starlette.requests import ClientDisconnect
from .correlate import correlate
from .correlate import correlate_with_campaigns
from .demo import load_demo
from .models import AnalysisResponse, ParseIssue, ParseStats
from .models import AnalysisResponse, ParseIssue, ParseStats, ParseStatus, SourceInfo
from .parser import (
MAX_UPLOAD_BYTES,
EvtxParseError,
ParseResult,
UnexpectedParserError,
parse_evtx_bytes,
)
@@ -43,10 +51,11 @@ app.add_middleware(
MAX_MULTIPART_OVERHEAD_BYTES = 1024 * 1024
MAX_REQUEST_BYTES = MAX_UPLOAD_BYTES + MAX_MULTIPART_OVERHEAD_BYTES
MAX_REPORTED_ISSUES = 200
GZIP_LEVEL = 1
CONTENT_SECURITY_POLICY = (
"default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; "
"img-src 'self' data:; connect-src 'self' http://localhost:8000 http://127.0.0.1:8000 "
"http://localhost:5173 http://127.0.0.1:5173 http://127.0.0.1:4173; "
"img-src 'self' data:; connect-src 'self'; "
"font-src 'self'; object-src 'none'; base-uri 'none'; frame-ancestors 'none'; form-action 'self'"
)
SECURITY_HEADERS = {
@@ -176,49 +185,67 @@ def health():
@app.get("/api/demo", response_model=AnalysisResponse)
def demo():
events = load_demo()
return AnalysisResponse(source="bundled-demo", events=events, incidents=correlate(events))
incidents, campaigns = correlate_with_campaigns(events)
return AnalysisResponse(source="bundled-demo", events=events, incidents=incidents, campaigns=campaigns)
async def _parse_with_admission(data: bytes):
def _capacity_error(message: str) -> HTTPException:
return HTTPException(
503,
detail={"code": "parse_capacity_exceeded", "message": message},
headers={"Retry-After": "1"},
)
@asynccontextmanager
async def _admission() -> AsyncIterator[None]:
"""Reserve an analysis slot before the multipart body is spooled.
Starlette spools uploads to TMPDIR while parsing the form. Admitting first
bounds the number of concurrently spooled uploads to concurrency + queue,
so a burst of large uploads gets a fast 503 instead of exhausting tmpfs.
"""
global _parse_inflight
async with _parse_admission_lock:
if _parse_inflight >= PARSE_CONCURRENCY + PARSE_QUEUE_LIMIT:
raise HTTPException(
503,
detail={
"code": "parse_capacity_exceeded",
"message": "EventLens is already analyzing its maximum concurrent uploads; retry shortly.",
},
headers={"Retry-After": "1"},
)
raise _capacity_error("EventLens is already analyzing its maximum concurrent uploads; retry shortly.")
_parse_inflight += 1
acquired = False
try:
try:
await asyncio.wait_for(_parse_slots.acquire(), timeout=PARSE_QUEUE_TIMEOUT_SECONDS)
acquired = True
except TimeoutError as exc:
raise HTTPException(
503,
detail={
"code": "parse_capacity_exceeded",
"message": "EventLens is busy analyzing another upload; retry shortly.",
},
headers={"Retry-After": "1"},
) from exc
yield
finally:
if not acquired:
async with _parse_admission_lock:
_parse_inflight -= 1
try:
return await asyncio.to_thread(parse_evtx_bytes, data)
finally:
if acquired:
_parse_slots.release()
async with _parse_admission_lock:
_parse_inflight -= 1
@asynccontextmanager
async def _parse_slot() -> AsyncIterator[None]:
try:
await asyncio.wait_for(_parse_slots.acquire(), timeout=PARSE_QUEUE_TIMEOUT_SECONDS)
except TimeoutError as exc:
raise _capacity_error("EventLens is busy analyzing another upload; retry shortly.") from exc
try:
yield
finally:
_parse_slots.release()
def _stats(parsed: ParseResult, status: ParseStatus | None = None) -> ParseStats:
counts = Counter(issue.code for issue in parsed.issues)
return ParseStats(
status=status or parsed.status,
parsed_events=len(parsed.events) if status != "failed" else 0,
records_seen=parsed.records_seen,
skipped_records=parsed.skipped_records,
issue_count=len(parsed.issues),
truncated=parsed.truncated,
truncation_reason=parsed.truncation_reason,
duration_ms=parsed.duration_ms,
issues=parsed.issues[:MAX_REPORTED_ISSUES],
issues_truncated=len(parsed.issues) > MAX_REPORTED_ISSUES,
issue_counts=dict(sorted(counts.items())),
)
def _failed_parse_stats() -> ParseStats:
return ParseStats(
status="failed",
@@ -230,77 +257,112 @@ def _failed_parse_stats() -> ParseStats:
truncation_reason=None,
duration_ms=0,
issues=[ParseIssue(code="parser_rejected_input", message="The EVTX parser rejected the input.")],
issue_counts={"parser_rejected_input": 1},
)
@app.post("/api/analyze", response_model=AnalysisResponse)
async def analyze(file: UploadFile = File(...)): # noqa: B008
def _analyze_bytes(source: str, data: bytes | bytearray, compress: bool = False) -> bytes:
"""Parse, correlate, serialize and optionally gzip off the event loop."""
source_info = SourceInfo(sha256=hashlib.sha256(data).hexdigest(), size_bytes=len(data))
try:
source = _safe_source_name(file.filename)
if not source.lower().endswith(".evtx"):
raise HTTPException(400, "Upload an .evtx file")
data = bytearray()
while True:
chunk = await file.read(min(1024 * 1024, MAX_UPLOAD_BYTES + 1 - len(data)))
if not chunk:
break
data.extend(chunk)
if len(data) > MAX_UPLOAD_BYTES:
raise HTTPException(413, "EVTX file exceeds the 50 MiB MVP limit")
try:
parsed = await _parse_with_admission(bytes(data))
except EvtxParseError as exc:
raise HTTPException(
422,
detail={
"code": "parser_rejected_input",
"message": str(exc),
"parse": _failed_parse_stats().model_dump(mode="json"),
},
) from exc
except UnexpectedParserError as exc:
logger.error("Unexpected EVTX parser failure reached API exception_type=%s", type(exc).__name__)
raise HTTPException(
500,
detail={"code": "parser_internal_error", "message": "EVTX analysis failed unexpectedly; retry the request."},
) from exc
if not parsed.events:
raise HTTPException(
422,
detail={
"code": "no_readable_events",
"message": "No readable Windows events were found in this file.",
"parse": ParseStats(
status="failed",
parsed_events=0,
records_seen=parsed.records_seen,
skipped_records=parsed.skipped_records,
issue_count=len(parsed.issues),
truncated=parsed.truncated,
truncation_reason=parsed.truncation_reason,
duration_ms=parsed.duration_ms,
issues=parsed.issues,
).model_dump(mode="json"),
},
)
return AnalysisResponse(
source=source,
events=parsed.events,
incidents=correlate(parsed.events),
parse=ParseStats(
status=parsed.status,
parsed_events=len(parsed.events),
records_seen=parsed.records_seen,
skipped_records=parsed.skipped_records,
issue_count=len(parsed.issues),
truncated=parsed.truncated,
truncation_reason=parsed.truncation_reason,
duration_ms=parsed.duration_ms,
issues=parsed.issues,
),
parsed = parse_evtx_bytes(data)
except EvtxParseError as exc:
raise HTTPException(
422,
detail={
"code": "parser_rejected_input",
"message": str(exc),
"parse": _failed_parse_stats().model_dump(mode="json"),
},
) from exc
if not parsed.events:
raise HTTPException(
422,
detail={
"code": "no_readable_events",
"message": "No readable Windows events were found in this file.",
"parse": _stats(parsed, "failed").model_dump(mode="json"),
},
)
finally:
await file.close()
incidents, campaigns = correlate_with_campaigns(parsed.events)
response = AnalysisResponse(
source=source,
events=parsed.events,
incidents=incidents,
campaigns=campaigns,
parse=_stats(parsed),
source_info=source_info,
)
payload = response.model_dump_json().encode("utf-8")
# Event JSON is highly repetitive: level 1 shrinks a 100k-event response
# ~20x for about a second of worker-thread CPU, never on the event loop.
return gzip.compress(payload, compresslevel=GZIP_LEVEL, mtime=0) if compress else payload
async def _read_bounded(upload: StarletteUploadFile) -> bytearray:
data = bytearray()
while True:
chunk = await upload.read(min(1024 * 1024, MAX_UPLOAD_BYTES + 1 - len(data)))
if not chunk:
return data
data.extend(chunk)
if len(data) > MAX_UPLOAD_BYTES:
raise HTTPException(413, "EVTX file exceeds the 50 MiB MVP limit")
ANALYZE_OPENAPI = {
"requestBody": {
"required": True,
"content": {
"multipart/form-data": {
"schema": {
"type": "object",
"required": ["file"],
"properties": {"file": {"type": "string", "format": "binary"}},
}
}
},
}
}
@app.post("/api/analyze", response_model=AnalysisResponse, openapi_extra=ANALYZE_OPENAPI)
async def analyze(request: Request):
compress = "gzip" in request.headers.get("accept-encoding", "").lower()
async with _admission():
try:
form = await request.form(max_files=1, max_fields=4)
except MultiPartException as exc:
raise HTTPException(400, "Upload one .evtx file as multipart form data") from exc
try:
upload = form.get("file")
if not isinstance(upload, StarletteUploadFile):
raise HTTPException(422, "Upload one .evtx file in the 'file' form field")
source = _safe_source_name(upload.filename)
if not source.lower().endswith(".evtx"):
raise HTTPException(400, "Upload an .evtx file")
data = await _read_bounded(upload)
async with _parse_slot():
try:
payload = await asyncio.to_thread(_analyze_bytes, source, data, compress)
except UnexpectedParserError as exc:
logger.error(
"Unexpected EVTX parser failure reached API exception_type=%s", type(exc).__name__
)
raise HTTPException(
500,
detail={
"code": "parser_internal_error",
"message": "EVTX analysis failed unexpectedly; retry the request.",
},
) from exc
del data
finally:
await form.close()
headers = {"Vary": "Accept-Encoding"}
if compress:
headers["Content-Encoding"] = "gzip"
return Response(content=payload, media_type="application/json", headers=headers)
def _safe_source_name(filename: str | None) -> str: