Includes campaign summaries, expanded Windows event correlation, worker-based redaction, updated demo assets and a local-only Compose example. Publish curated EventLens source from 6a2ca12f81a2dcf816c61a86bd8811fcc95e8d8d
382 lines
13 KiB
Python
382 lines
13 KiB
Python
import asyncio
|
|
import gzip
|
|
import hashlib
|
|
import json
|
|
import logging
|
|
import os
|
|
import re
|
|
from collections import Counter
|
|
from collections.abc import AsyncIterator
|
|
from contextlib import asynccontextmanager
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from fastapi import FastAPI, HTTPException, Request, Response
|
|
from fastapi.middleware.cors import CORSMiddleware
|
|
from fastapi.staticfiles import StaticFiles
|
|
from starlette.datastructures import UploadFile as StarletteUploadFile
|
|
from starlette.formparsers import MultiPartException
|
|
from starlette.requests import ClientDisconnect
|
|
|
|
from .correlate import correlate_with_campaigns
|
|
from .demo import load_demo
|
|
from .models import AnalysisResponse, ParseIssue, ParseStats, ParseStatus, SourceInfo
|
|
from .parser import (
|
|
MAX_UPLOAD_BYTES,
|
|
EvtxParseError,
|
|
ParseResult,
|
|
UnexpectedParserError,
|
|
parse_evtx_bytes,
|
|
)
|
|
from .version import VERSION, read_revision
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
app = FastAPI(title="EventLens API", version=VERSION)
|
|
cors_origins = [
|
|
origin.strip()
|
|
for origin in os.environ.get(
|
|
"EVENTLENS_CORS_ORIGINS",
|
|
"http://localhost:5173,http://127.0.0.1:5173,http://127.0.0.1:4173",
|
|
).split(",")
|
|
if origin.strip()
|
|
]
|
|
app.add_middleware(
|
|
CORSMiddleware,
|
|
allow_origins=cors_origins,
|
|
allow_credentials=False,
|
|
allow_methods=["GET", "POST"],
|
|
allow_headers=["Content-Type"],
|
|
)
|
|
|
|
MAX_MULTIPART_OVERHEAD_BYTES = 1024 * 1024
|
|
MAX_REQUEST_BYTES = MAX_UPLOAD_BYTES + MAX_MULTIPART_OVERHEAD_BYTES
|
|
MAX_REPORTED_ISSUES = 200
|
|
GZIP_LEVEL = 1
|
|
CONTENT_SECURITY_POLICY = (
|
|
"default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; "
|
|
"img-src 'self' data:; connect-src 'self'; "
|
|
"font-src 'self'; object-src 'none'; base-uri 'none'; frame-ancestors 'none'; form-action 'self'"
|
|
)
|
|
SECURITY_HEADERS = {
|
|
"X-Content-Type-Options": "nosniff",
|
|
"Referrer-Policy": "no-referrer",
|
|
"Permissions-Policy": "camera=(), microphone=(), geolocation=()",
|
|
"Content-Security-Policy": CONTENT_SECURITY_POLICY,
|
|
"X-Frame-Options": "DENY",
|
|
}
|
|
|
|
|
|
def _bounded_env_int(name: str, default: int, minimum: int, maximum: int) -> int:
|
|
try:
|
|
value = int(os.environ.get(name, str(default)))
|
|
except (TypeError, ValueError):
|
|
value = default
|
|
return min(max(value, minimum), maximum)
|
|
|
|
|
|
PARSE_CONCURRENCY = _bounded_env_int("EVENTLENS_PARSE_CONCURRENCY", 1, 1, 2)
|
|
PARSE_QUEUE_LIMIT = _bounded_env_int("EVENTLENS_PARSE_QUEUE", 1, 0, 4)
|
|
PARSE_QUEUE_TIMEOUT_SECONDS = 0.25
|
|
_parse_slots = asyncio.Semaphore(PARSE_CONCURRENCY)
|
|
_parse_admission_lock = asyncio.Lock()
|
|
_parse_inflight = 0
|
|
|
|
|
|
class RequestBodyLimitMiddleware:
|
|
"""Bound request bytes before Starlette's multipart parser expands the upload."""
|
|
|
|
def __init__(self, app: Any, max_bytes: int) -> None:
|
|
self.app = app
|
|
self.max_bytes = max_bytes
|
|
|
|
async def __call__(self, scope: dict[str, Any], receive: Any, send: Any) -> None:
|
|
if scope.get("type") != "http" or scope.get("path") != "/api/analyze" or scope.get("method") != "POST":
|
|
await self.app(scope, receive, send)
|
|
return
|
|
|
|
headers = {key.lower(): value for key, value in scope.get("headers", [])}
|
|
content_length = headers.get(b"content-length")
|
|
if content_length is not None:
|
|
try:
|
|
declared = int(content_length)
|
|
except (TypeError, ValueError):
|
|
declared = 0
|
|
if declared > self.max_bytes:
|
|
await self._send_too_large(send)
|
|
return
|
|
|
|
consumed = 0
|
|
too_large = False
|
|
response_started = False
|
|
|
|
async def limited_receive() -> dict[str, Any]:
|
|
nonlocal consumed, too_large
|
|
message = await receive()
|
|
if message.get("type") == "http.request":
|
|
consumed += len(message.get("body", b""))
|
|
if consumed > self.max_bytes:
|
|
too_large = True
|
|
return {"type": "http.disconnect"}
|
|
return message
|
|
|
|
async def guarded_send(message: dict[str, Any]) -> None:
|
|
nonlocal response_started
|
|
if too_large:
|
|
if message.get("type") == "http.response.start" and not response_started:
|
|
response_started = True
|
|
await self._send_too_large(send)
|
|
return
|
|
await send(message)
|
|
|
|
try:
|
|
await self.app(scope, limited_receive, guarded_send)
|
|
except ClientDisconnect:
|
|
if not too_large:
|
|
raise
|
|
if too_large and not response_started:
|
|
await self._send_too_large(send)
|
|
|
|
async def _send_too_large(self, send: Any) -> None:
|
|
body = json.dumps(
|
|
{"detail": {"code": "request_body_too_large", "message": "Upload request exceeds the bounded request limit."}}
|
|
).encode("utf-8")
|
|
headers = [
|
|
(b"content-type", b"application/json"),
|
|
(b"content-length", str(len(body)).encode()),
|
|
(b"cache-control", b"no-store"),
|
|
(b"pragma", b"no-cache"),
|
|
]
|
|
headers.extend((name.lower().encode("ascii"), value.encode("ascii")) for name, value in SECURITY_HEADERS.items())
|
|
await send(
|
|
{
|
|
"type": "http.response.start",
|
|
"status": 413,
|
|
"headers": headers,
|
|
}
|
|
)
|
|
await send({"type": "http.response.body", "body": body})
|
|
|
|
|
|
app.add_middleware(RequestBodyLimitMiddleware, max_bytes=MAX_REQUEST_BYTES)
|
|
|
|
|
|
@app.middleware("http")
|
|
async def response_security_headers(request, call_next):
|
|
response = await call_next(request)
|
|
for name, value in SECURITY_HEADERS.items():
|
|
response.headers[name] = value
|
|
if request.url.path.startswith("/api/") or request.url.path == "/health":
|
|
response.headers["Cache-Control"] = "no-store"
|
|
response.headers["Pragma"] = "no-cache"
|
|
return response
|
|
|
|
|
|
@app.get("/health")
|
|
def health():
|
|
return {
|
|
"status": "ok",
|
|
"evtx_ingestion": True,
|
|
"version": VERSION,
|
|
"revision": read_revision(),
|
|
}
|
|
|
|
|
|
@app.get("/api/demo", response_model=AnalysisResponse)
|
|
def demo():
|
|
events = load_demo()
|
|
incidents, campaigns = correlate_with_campaigns(events)
|
|
return AnalysisResponse(source="bundled-demo", events=events, incidents=incidents, campaigns=campaigns)
|
|
|
|
|
|
def _capacity_error(message: str) -> HTTPException:
|
|
return HTTPException(
|
|
503,
|
|
detail={"code": "parse_capacity_exceeded", "message": message},
|
|
headers={"Retry-After": "1"},
|
|
)
|
|
|
|
|
|
@asynccontextmanager
|
|
async def _admission() -> AsyncIterator[None]:
|
|
"""Reserve an analysis slot before the multipart body is spooled.
|
|
|
|
Starlette spools uploads to TMPDIR while parsing the form. Admitting first
|
|
bounds the number of concurrently spooled uploads to concurrency + queue,
|
|
so a burst of large uploads gets a fast 503 instead of exhausting tmpfs.
|
|
"""
|
|
global _parse_inflight
|
|
async with _parse_admission_lock:
|
|
if _parse_inflight >= PARSE_CONCURRENCY + PARSE_QUEUE_LIMIT:
|
|
raise _capacity_error("EventLens is already analyzing its maximum concurrent uploads; retry shortly.")
|
|
_parse_inflight += 1
|
|
try:
|
|
yield
|
|
finally:
|
|
async with _parse_admission_lock:
|
|
_parse_inflight -= 1
|
|
|
|
|
|
@asynccontextmanager
|
|
async def _parse_slot() -> AsyncIterator[None]:
|
|
try:
|
|
await asyncio.wait_for(_parse_slots.acquire(), timeout=PARSE_QUEUE_TIMEOUT_SECONDS)
|
|
except TimeoutError as exc:
|
|
raise _capacity_error("EventLens is busy analyzing another upload; retry shortly.") from exc
|
|
try:
|
|
yield
|
|
finally:
|
|
_parse_slots.release()
|
|
|
|
|
|
def _stats(parsed: ParseResult, status: ParseStatus | None = None) -> ParseStats:
|
|
counts = Counter(issue.code for issue in parsed.issues)
|
|
return ParseStats(
|
|
status=status or parsed.status,
|
|
parsed_events=len(parsed.events) if status != "failed" else 0,
|
|
records_seen=parsed.records_seen,
|
|
skipped_records=parsed.skipped_records,
|
|
issue_count=len(parsed.issues),
|
|
truncated=parsed.truncated,
|
|
truncation_reason=parsed.truncation_reason,
|
|
duration_ms=parsed.duration_ms,
|
|
issues=parsed.issues[:MAX_REPORTED_ISSUES],
|
|
issues_truncated=len(parsed.issues) > MAX_REPORTED_ISSUES,
|
|
issue_counts=dict(sorted(counts.items())),
|
|
)
|
|
|
|
|
|
def _failed_parse_stats() -> ParseStats:
|
|
return ParseStats(
|
|
status="failed",
|
|
parsed_events=0,
|
|
records_seen=0,
|
|
skipped_records=0,
|
|
issue_count=1,
|
|
truncated=False,
|
|
truncation_reason=None,
|
|
duration_ms=0,
|
|
issues=[ParseIssue(code="parser_rejected_input", message="The EVTX parser rejected the input.")],
|
|
issue_counts={"parser_rejected_input": 1},
|
|
)
|
|
|
|
|
|
def _analyze_bytes(source: str, data: bytes | bytearray, compress: bool = False) -> bytes:
|
|
"""Parse, correlate, serialize and optionally gzip off the event loop."""
|
|
source_info = SourceInfo(sha256=hashlib.sha256(data).hexdigest(), size_bytes=len(data))
|
|
try:
|
|
parsed = parse_evtx_bytes(data)
|
|
except EvtxParseError as exc:
|
|
raise HTTPException(
|
|
422,
|
|
detail={
|
|
"code": "parser_rejected_input",
|
|
"message": str(exc),
|
|
"parse": _failed_parse_stats().model_dump(mode="json"),
|
|
},
|
|
) from exc
|
|
if not parsed.events:
|
|
raise HTTPException(
|
|
422,
|
|
detail={
|
|
"code": "no_readable_events",
|
|
"message": "No readable Windows events were found in this file.",
|
|
"parse": _stats(parsed, "failed").model_dump(mode="json"),
|
|
},
|
|
)
|
|
incidents, campaigns = correlate_with_campaigns(parsed.events)
|
|
response = AnalysisResponse(
|
|
source=source,
|
|
events=parsed.events,
|
|
incidents=incidents,
|
|
campaigns=campaigns,
|
|
parse=_stats(parsed),
|
|
source_info=source_info,
|
|
)
|
|
payload = response.model_dump_json().encode("utf-8")
|
|
# Event JSON is highly repetitive: level 1 shrinks a 100k-event response
|
|
# ~20x for about a second of worker-thread CPU, never on the event loop.
|
|
return gzip.compress(payload, compresslevel=GZIP_LEVEL, mtime=0) if compress else payload
|
|
|
|
|
|
async def _read_bounded(upload: StarletteUploadFile) -> bytearray:
|
|
data = bytearray()
|
|
while True:
|
|
chunk = await upload.read(min(1024 * 1024, MAX_UPLOAD_BYTES + 1 - len(data)))
|
|
if not chunk:
|
|
return data
|
|
data.extend(chunk)
|
|
if len(data) > MAX_UPLOAD_BYTES:
|
|
raise HTTPException(413, "EVTX file exceeds the 50 MiB MVP limit")
|
|
|
|
|
|
ANALYZE_OPENAPI = {
|
|
"requestBody": {
|
|
"required": True,
|
|
"content": {
|
|
"multipart/form-data": {
|
|
"schema": {
|
|
"type": "object",
|
|
"required": ["file"],
|
|
"properties": {"file": {"type": "string", "format": "binary"}},
|
|
}
|
|
}
|
|
},
|
|
}
|
|
}
|
|
|
|
|
|
@app.post("/api/analyze", response_model=AnalysisResponse, openapi_extra=ANALYZE_OPENAPI)
|
|
async def analyze(request: Request):
|
|
compress = "gzip" in request.headers.get("accept-encoding", "").lower()
|
|
async with _admission():
|
|
try:
|
|
form = await request.form(max_files=1, max_fields=4)
|
|
except MultiPartException as exc:
|
|
raise HTTPException(400, "Upload one .evtx file as multipart form data") from exc
|
|
try:
|
|
upload = form.get("file")
|
|
if not isinstance(upload, StarletteUploadFile):
|
|
raise HTTPException(422, "Upload one .evtx file in the 'file' form field")
|
|
source = _safe_source_name(upload.filename)
|
|
if not source.lower().endswith(".evtx"):
|
|
raise HTTPException(400, "Upload an .evtx file")
|
|
data = await _read_bounded(upload)
|
|
async with _parse_slot():
|
|
try:
|
|
payload = await asyncio.to_thread(_analyze_bytes, source, data, compress)
|
|
except UnexpectedParserError as exc:
|
|
logger.error(
|
|
"Unexpected EVTX parser failure reached API exception_type=%s", type(exc).__name__
|
|
)
|
|
raise HTTPException(
|
|
500,
|
|
detail={
|
|
"code": "parser_internal_error",
|
|
"message": "EVTX analysis failed unexpectedly; retry the request.",
|
|
},
|
|
) from exc
|
|
del data
|
|
finally:
|
|
await form.close()
|
|
headers = {"Vary": "Accept-Encoding"}
|
|
if compress:
|
|
headers["Content-Encoding"] = "gzip"
|
|
return Response(content=payload, media_type="application/json", headers=headers)
|
|
|
|
|
|
def _safe_source_name(filename: str | None) -> str:
|
|
"""Return a short display/export name without path or control characters."""
|
|
candidate = re.split(r"[\\/]", filename or "")[-1]
|
|
candidate = "".join(char for char in candidate if char.isprintable()).strip()
|
|
candidate = re.sub(r"[^A-Za-z0-9._ -]", "_", candidate)
|
|
candidate = candidate[:128]
|
|
return candidate or "upload.evtx"
|
|
|
|
|
|
frontend_dir = os.environ.get("EVENTLENS_FRONTEND_DIR")
|
|
if frontend_dir:
|
|
path = Path(frontend_dir)
|
|
if path.is_dir():
|
|
app.mount("/", StaticFiles(directory=str(path), html=True), name="frontend")
|