Files
EventLens-Public/backend/app/main.py
T
Public Source Publisher 4e282639a4 EventLens: publish v1.1.0 application and standalone Compose
Includes campaign summaries, expanded Windows event correlation, worker-based redaction, updated demo assets and a local-only Compose example.

Publish curated EventLens source from 6a2ca12f81a2dcf816c61a86bd8811fcc95e8d8d
2026-11-20 13:15:35 +01:00

382 lines
13 KiB
Python

import asyncio
import gzip
import hashlib
import json
import logging
import os
import re
from collections import Counter
from collections.abc import AsyncIterator
from contextlib import asynccontextmanager
from pathlib import Path
from typing import Any
from fastapi import FastAPI, HTTPException, Request, Response
from fastapi.middleware.cors import CORSMiddleware
from fastapi.staticfiles import StaticFiles
from starlette.datastructures import UploadFile as StarletteUploadFile
from starlette.formparsers import MultiPartException
from starlette.requests import ClientDisconnect
from .correlate import correlate_with_campaigns
from .demo import load_demo
from .models import AnalysisResponse, ParseIssue, ParseStats, ParseStatus, SourceInfo
from .parser import (
MAX_UPLOAD_BYTES,
EvtxParseError,
ParseResult,
UnexpectedParserError,
parse_evtx_bytes,
)
from .version import VERSION, read_revision
logger = logging.getLogger(__name__)
app = FastAPI(title="EventLens API", version=VERSION)
cors_origins = [
origin.strip()
for origin in os.environ.get(
"EVENTLENS_CORS_ORIGINS",
"http://localhost:5173,http://127.0.0.1:5173,http://127.0.0.1:4173",
).split(",")
if origin.strip()
]
app.add_middleware(
CORSMiddleware,
allow_origins=cors_origins,
allow_credentials=False,
allow_methods=["GET", "POST"],
allow_headers=["Content-Type"],
)
MAX_MULTIPART_OVERHEAD_BYTES = 1024 * 1024
MAX_REQUEST_BYTES = MAX_UPLOAD_BYTES + MAX_MULTIPART_OVERHEAD_BYTES
MAX_REPORTED_ISSUES = 200
GZIP_LEVEL = 1
CONTENT_SECURITY_POLICY = (
"default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; "
"img-src 'self' data:; connect-src 'self'; "
"font-src 'self'; object-src 'none'; base-uri 'none'; frame-ancestors 'none'; form-action 'self'"
)
SECURITY_HEADERS = {
"X-Content-Type-Options": "nosniff",
"Referrer-Policy": "no-referrer",
"Permissions-Policy": "camera=(), microphone=(), geolocation=()",
"Content-Security-Policy": CONTENT_SECURITY_POLICY,
"X-Frame-Options": "DENY",
}
def _bounded_env_int(name: str, default: int, minimum: int, maximum: int) -> int:
try:
value = int(os.environ.get(name, str(default)))
except (TypeError, ValueError):
value = default
return min(max(value, minimum), maximum)
PARSE_CONCURRENCY = _bounded_env_int("EVENTLENS_PARSE_CONCURRENCY", 1, 1, 2)
PARSE_QUEUE_LIMIT = _bounded_env_int("EVENTLENS_PARSE_QUEUE", 1, 0, 4)
PARSE_QUEUE_TIMEOUT_SECONDS = 0.25
_parse_slots = asyncio.Semaphore(PARSE_CONCURRENCY)
_parse_admission_lock = asyncio.Lock()
_parse_inflight = 0
class RequestBodyLimitMiddleware:
"""Bound request bytes before Starlette's multipart parser expands the upload."""
def __init__(self, app: Any, max_bytes: int) -> None:
self.app = app
self.max_bytes = max_bytes
async def __call__(self, scope: dict[str, Any], receive: Any, send: Any) -> None:
if scope.get("type") != "http" or scope.get("path") != "/api/analyze" or scope.get("method") != "POST":
await self.app(scope, receive, send)
return
headers = {key.lower(): value for key, value in scope.get("headers", [])}
content_length = headers.get(b"content-length")
if content_length is not None:
try:
declared = int(content_length)
except (TypeError, ValueError):
declared = 0
if declared > self.max_bytes:
await self._send_too_large(send)
return
consumed = 0
too_large = False
response_started = False
async def limited_receive() -> dict[str, Any]:
nonlocal consumed, too_large
message = await receive()
if message.get("type") == "http.request":
consumed += len(message.get("body", b""))
if consumed > self.max_bytes:
too_large = True
return {"type": "http.disconnect"}
return message
async def guarded_send(message: dict[str, Any]) -> None:
nonlocal response_started
if too_large:
if message.get("type") == "http.response.start" and not response_started:
response_started = True
await self._send_too_large(send)
return
await send(message)
try:
await self.app(scope, limited_receive, guarded_send)
except ClientDisconnect:
if not too_large:
raise
if too_large and not response_started:
await self._send_too_large(send)
async def _send_too_large(self, send: Any) -> None:
body = json.dumps(
{"detail": {"code": "request_body_too_large", "message": "Upload request exceeds the bounded request limit."}}
).encode("utf-8")
headers = [
(b"content-type", b"application/json"),
(b"content-length", str(len(body)).encode()),
(b"cache-control", b"no-store"),
(b"pragma", b"no-cache"),
]
headers.extend((name.lower().encode("ascii"), value.encode("ascii")) for name, value in SECURITY_HEADERS.items())
await send(
{
"type": "http.response.start",
"status": 413,
"headers": headers,
}
)
await send({"type": "http.response.body", "body": body})
app.add_middleware(RequestBodyLimitMiddleware, max_bytes=MAX_REQUEST_BYTES)
@app.middleware("http")
async def response_security_headers(request, call_next):
response = await call_next(request)
for name, value in SECURITY_HEADERS.items():
response.headers[name] = value
if request.url.path.startswith("/api/") or request.url.path == "/health":
response.headers["Cache-Control"] = "no-store"
response.headers["Pragma"] = "no-cache"
return response
@app.get("/health")
def health():
return {
"status": "ok",
"evtx_ingestion": True,
"version": VERSION,
"revision": read_revision(),
}
@app.get("/api/demo", response_model=AnalysisResponse)
def demo():
events = load_demo()
incidents, campaigns = correlate_with_campaigns(events)
return AnalysisResponse(source="bundled-demo", events=events, incidents=incidents, campaigns=campaigns)
def _capacity_error(message: str) -> HTTPException:
return HTTPException(
503,
detail={"code": "parse_capacity_exceeded", "message": message},
headers={"Retry-After": "1"},
)
@asynccontextmanager
async def _admission() -> AsyncIterator[None]:
"""Reserve an analysis slot before the multipart body is spooled.
Starlette spools uploads to TMPDIR while parsing the form. Admitting first
bounds the number of concurrently spooled uploads to concurrency + queue,
so a burst of large uploads gets a fast 503 instead of exhausting tmpfs.
"""
global _parse_inflight
async with _parse_admission_lock:
if _parse_inflight >= PARSE_CONCURRENCY + PARSE_QUEUE_LIMIT:
raise _capacity_error("EventLens is already analyzing its maximum concurrent uploads; retry shortly.")
_parse_inflight += 1
try:
yield
finally:
async with _parse_admission_lock:
_parse_inflight -= 1
@asynccontextmanager
async def _parse_slot() -> AsyncIterator[None]:
try:
await asyncio.wait_for(_parse_slots.acquire(), timeout=PARSE_QUEUE_TIMEOUT_SECONDS)
except TimeoutError as exc:
raise _capacity_error("EventLens is busy analyzing another upload; retry shortly.") from exc
try:
yield
finally:
_parse_slots.release()
def _stats(parsed: ParseResult, status: ParseStatus | None = None) -> ParseStats:
counts = Counter(issue.code for issue in parsed.issues)
return ParseStats(
status=status or parsed.status,
parsed_events=len(parsed.events) if status != "failed" else 0,
records_seen=parsed.records_seen,
skipped_records=parsed.skipped_records,
issue_count=len(parsed.issues),
truncated=parsed.truncated,
truncation_reason=parsed.truncation_reason,
duration_ms=parsed.duration_ms,
issues=parsed.issues[:MAX_REPORTED_ISSUES],
issues_truncated=len(parsed.issues) > MAX_REPORTED_ISSUES,
issue_counts=dict(sorted(counts.items())),
)
def _failed_parse_stats() -> ParseStats:
return ParseStats(
status="failed",
parsed_events=0,
records_seen=0,
skipped_records=0,
issue_count=1,
truncated=False,
truncation_reason=None,
duration_ms=0,
issues=[ParseIssue(code="parser_rejected_input", message="The EVTX parser rejected the input.")],
issue_counts={"parser_rejected_input": 1},
)
def _analyze_bytes(source: str, data: bytes | bytearray, compress: bool = False) -> bytes:
"""Parse, correlate, serialize and optionally gzip off the event loop."""
source_info = SourceInfo(sha256=hashlib.sha256(data).hexdigest(), size_bytes=len(data))
try:
parsed = parse_evtx_bytes(data)
except EvtxParseError as exc:
raise HTTPException(
422,
detail={
"code": "parser_rejected_input",
"message": str(exc),
"parse": _failed_parse_stats().model_dump(mode="json"),
},
) from exc
if not parsed.events:
raise HTTPException(
422,
detail={
"code": "no_readable_events",
"message": "No readable Windows events were found in this file.",
"parse": _stats(parsed, "failed").model_dump(mode="json"),
},
)
incidents, campaigns = correlate_with_campaigns(parsed.events)
response = AnalysisResponse(
source=source,
events=parsed.events,
incidents=incidents,
campaigns=campaigns,
parse=_stats(parsed),
source_info=source_info,
)
payload = response.model_dump_json().encode("utf-8")
# Event JSON is highly repetitive: level 1 shrinks a 100k-event response
# ~20x for about a second of worker-thread CPU, never on the event loop.
return gzip.compress(payload, compresslevel=GZIP_LEVEL, mtime=0) if compress else payload
async def _read_bounded(upload: StarletteUploadFile) -> bytearray:
data = bytearray()
while True:
chunk = await upload.read(min(1024 * 1024, MAX_UPLOAD_BYTES + 1 - len(data)))
if not chunk:
return data
data.extend(chunk)
if len(data) > MAX_UPLOAD_BYTES:
raise HTTPException(413, "EVTX file exceeds the 50 MiB MVP limit")
ANALYZE_OPENAPI = {
"requestBody": {
"required": True,
"content": {
"multipart/form-data": {
"schema": {
"type": "object",
"required": ["file"],
"properties": {"file": {"type": "string", "format": "binary"}},
}
}
},
}
}
@app.post("/api/analyze", response_model=AnalysisResponse, openapi_extra=ANALYZE_OPENAPI)
async def analyze(request: Request):
compress = "gzip" in request.headers.get("accept-encoding", "").lower()
async with _admission():
try:
form = await request.form(max_files=1, max_fields=4)
except MultiPartException as exc:
raise HTTPException(400, "Upload one .evtx file as multipart form data") from exc
try:
upload = form.get("file")
if not isinstance(upload, StarletteUploadFile):
raise HTTPException(422, "Upload one .evtx file in the 'file' form field")
source = _safe_source_name(upload.filename)
if not source.lower().endswith(".evtx"):
raise HTTPException(400, "Upload an .evtx file")
data = await _read_bounded(upload)
async with _parse_slot():
try:
payload = await asyncio.to_thread(_analyze_bytes, source, data, compress)
except UnexpectedParserError as exc:
logger.error(
"Unexpected EVTX parser failure reached API exception_type=%s", type(exc).__name__
)
raise HTTPException(
500,
detail={
"code": "parser_internal_error",
"message": "EVTX analysis failed unexpectedly; retry the request.",
},
) from exc
del data
finally:
await form.close()
headers = {"Vary": "Accept-Encoding"}
if compress:
headers["Content-Encoding"] = "gzip"
return Response(content=payload, media_type="application/json", headers=headers)
def _safe_source_name(filename: str | None) -> str:
"""Return a short display/export name without path or control characters."""
candidate = re.split(r"[\\/]", filename or "")[-1]
candidate = "".join(char for char in candidate if char.isprintable()).strip()
candidate = re.sub(r"[^A-Za-z0-9._ -]", "_", candidate)
candidate = candidate[:128]
return candidate or "upload.evtx"
frontend_dir = os.environ.get("EVENTLENS_FRONTEND_DIR")
if frontend_dir:
path = Path(frontend_dir)
if path.is_dir():
app.mount("/", StaticFiles(directory=str(path), html=True), name="frontend")