EventLens: publish v1.1.0 application and standalone Compose
Includes campaign summaries, expanded Windows event correlation, worker-based redaction, updated demo assets and a local-only Compose example. Publish curated EventLens source from 6a2ca12f81a2dcf816c61a86bd8811fcc95e8d8d
This commit is contained in:
1 parent
c0ade21029
commit
4e282639a4
63 files changed
+7381
-1420
No files matched your search
+164
-102
@@ -1,22 +1,30 @@
|
||||
import asyncio
|
||||
import gzip
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from collections import Counter
|
||||
from collections.abc import AsyncIterator
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from fastapi import FastAPI, File, HTTPException, UploadFile
|
||||
from fastapi import FastAPI, HTTPException, Request, Response
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from starlette.datastructures import UploadFile as StarletteUploadFile
|
||||
from starlette.formparsers import MultiPartException
|
||||
from starlette.requests import ClientDisconnect
|
||||
|
||||
from .correlate import correlate
|
||||
from .correlate import correlate_with_campaigns
|
||||
from .demo import load_demo
|
||||
from .models import AnalysisResponse, ParseIssue, ParseStats
|
||||
from .models import AnalysisResponse, ParseIssue, ParseStats, ParseStatus, SourceInfo
|
||||
from .parser import (
|
||||
MAX_UPLOAD_BYTES,
|
||||
EvtxParseError,
|
||||
ParseResult,
|
||||
UnexpectedParserError,
|
||||
parse_evtx_bytes,
|
||||
)
|
||||
@@ -43,10 +51,11 @@ app.add_middleware(
|
||||
|
||||
MAX_MULTIPART_OVERHEAD_BYTES = 1024 * 1024
|
||||
MAX_REQUEST_BYTES = MAX_UPLOAD_BYTES + MAX_MULTIPART_OVERHEAD_BYTES
|
||||
MAX_REPORTED_ISSUES = 200
|
||||
GZIP_LEVEL = 1
|
||||
CONTENT_SECURITY_POLICY = (
|
||||
"default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; "
|
||||
"img-src 'self' data:; connect-src 'self' http://localhost:8000 http://127.0.0.1:8000 "
|
||||
"http://localhost:5173 http://127.0.0.1:5173 http://127.0.0.1:4173; "
|
||||
"img-src 'self' data:; connect-src 'self'; "
|
||||
"font-src 'self'; object-src 'none'; base-uri 'none'; frame-ancestors 'none'; form-action 'self'"
|
||||
)
|
||||
SECURITY_HEADERS = {
|
||||
@@ -176,49 +185,67 @@ def health():
|
||||
@app.get("/api/demo", response_model=AnalysisResponse)
|
||||
def demo():
|
||||
events = load_demo()
|
||||
return AnalysisResponse(source="bundled-demo", events=events, incidents=correlate(events))
|
||||
incidents, campaigns = correlate_with_campaigns(events)
|
||||
return AnalysisResponse(source="bundled-demo", events=events, incidents=incidents, campaigns=campaigns)
|
||||
|
||||
|
||||
async def _parse_with_admission(data: bytes):
|
||||
def _capacity_error(message: str) -> HTTPException:
|
||||
return HTTPException(
|
||||
503,
|
||||
detail={"code": "parse_capacity_exceeded", "message": message},
|
||||
headers={"Retry-After": "1"},
|
||||
)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def _admission() -> AsyncIterator[None]:
|
||||
"""Reserve an analysis slot before the multipart body is spooled.
|
||||
|
||||
Starlette spools uploads to TMPDIR while parsing the form. Admitting first
|
||||
bounds the number of concurrently spooled uploads to concurrency + queue,
|
||||
so a burst of large uploads gets a fast 503 instead of exhausting tmpfs.
|
||||
"""
|
||||
global _parse_inflight
|
||||
async with _parse_admission_lock:
|
||||
if _parse_inflight >= PARSE_CONCURRENCY + PARSE_QUEUE_LIMIT:
|
||||
raise HTTPException(
|
||||
503,
|
||||
detail={
|
||||
"code": "parse_capacity_exceeded",
|
||||
"message": "EventLens is already analyzing its maximum concurrent uploads; retry shortly.",
|
||||
},
|
||||
headers={"Retry-After": "1"},
|
||||
)
|
||||
raise _capacity_error("EventLens is already analyzing its maximum concurrent uploads; retry shortly.")
|
||||
_parse_inflight += 1
|
||||
acquired = False
|
||||
try:
|
||||
try:
|
||||
await asyncio.wait_for(_parse_slots.acquire(), timeout=PARSE_QUEUE_TIMEOUT_SECONDS)
|
||||
acquired = True
|
||||
except TimeoutError as exc:
|
||||
raise HTTPException(
|
||||
503,
|
||||
detail={
|
||||
"code": "parse_capacity_exceeded",
|
||||
"message": "EventLens is busy analyzing another upload; retry shortly.",
|
||||
},
|
||||
headers={"Retry-After": "1"},
|
||||
) from exc
|
||||
yield
|
||||
finally:
|
||||
if not acquired:
|
||||
async with _parse_admission_lock:
|
||||
_parse_inflight -= 1
|
||||
try:
|
||||
return await asyncio.to_thread(parse_evtx_bytes, data)
|
||||
finally:
|
||||
if acquired:
|
||||
_parse_slots.release()
|
||||
async with _parse_admission_lock:
|
||||
_parse_inflight -= 1
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def _parse_slot() -> AsyncIterator[None]:
|
||||
try:
|
||||
await asyncio.wait_for(_parse_slots.acquire(), timeout=PARSE_QUEUE_TIMEOUT_SECONDS)
|
||||
except TimeoutError as exc:
|
||||
raise _capacity_error("EventLens is busy analyzing another upload; retry shortly.") from exc
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_parse_slots.release()
|
||||
|
||||
|
||||
def _stats(parsed: ParseResult, status: ParseStatus | None = None) -> ParseStats:
|
||||
counts = Counter(issue.code for issue in parsed.issues)
|
||||
return ParseStats(
|
||||
status=status or parsed.status,
|
||||
parsed_events=len(parsed.events) if status != "failed" else 0,
|
||||
records_seen=parsed.records_seen,
|
||||
skipped_records=parsed.skipped_records,
|
||||
issue_count=len(parsed.issues),
|
||||
truncated=parsed.truncated,
|
||||
truncation_reason=parsed.truncation_reason,
|
||||
duration_ms=parsed.duration_ms,
|
||||
issues=parsed.issues[:MAX_REPORTED_ISSUES],
|
||||
issues_truncated=len(parsed.issues) > MAX_REPORTED_ISSUES,
|
||||
issue_counts=dict(sorted(counts.items())),
|
||||
)
|
||||
|
||||
|
||||
def _failed_parse_stats() -> ParseStats:
|
||||
return ParseStats(
|
||||
status="failed",
|
||||
@@ -230,77 +257,112 @@ def _failed_parse_stats() -> ParseStats:
|
||||
truncation_reason=None,
|
||||
duration_ms=0,
|
||||
issues=[ParseIssue(code="parser_rejected_input", message="The EVTX parser rejected the input.")],
|
||||
issue_counts={"parser_rejected_input": 1},
|
||||
)
|
||||
|
||||
|
||||
@app.post("/api/analyze", response_model=AnalysisResponse)
|
||||
async def analyze(file: UploadFile = File(...)): # noqa: B008
|
||||
def _analyze_bytes(source: str, data: bytes | bytearray, compress: bool = False) -> bytes:
|
||||
"""Parse, correlate, serialize and optionally gzip off the event loop."""
|
||||
source_info = SourceInfo(sha256=hashlib.sha256(data).hexdigest(), size_bytes=len(data))
|
||||
try:
|
||||
source = _safe_source_name(file.filename)
|
||||
if not source.lower().endswith(".evtx"):
|
||||
raise HTTPException(400, "Upload an .evtx file")
|
||||
data = bytearray()
|
||||
while True:
|
||||
chunk = await file.read(min(1024 * 1024, MAX_UPLOAD_BYTES + 1 - len(data)))
|
||||
if not chunk:
|
||||
break
|
||||
data.extend(chunk)
|
||||
if len(data) > MAX_UPLOAD_BYTES:
|
||||
raise HTTPException(413, "EVTX file exceeds the 50 MiB MVP limit")
|
||||
try:
|
||||
parsed = await _parse_with_admission(bytes(data))
|
||||
except EvtxParseError as exc:
|
||||
raise HTTPException(
|
||||
422,
|
||||
detail={
|
||||
"code": "parser_rejected_input",
|
||||
"message": str(exc),
|
||||
"parse": _failed_parse_stats().model_dump(mode="json"),
|
||||
},
|
||||
) from exc
|
||||
except UnexpectedParserError as exc:
|
||||
logger.error("Unexpected EVTX parser failure reached API exception_type=%s", type(exc).__name__)
|
||||
raise HTTPException(
|
||||
500,
|
||||
detail={"code": "parser_internal_error", "message": "EVTX analysis failed unexpectedly; retry the request."},
|
||||
) from exc
|
||||
if not parsed.events:
|
||||
raise HTTPException(
|
||||
422,
|
||||
detail={
|
||||
"code": "no_readable_events",
|
||||
"message": "No readable Windows events were found in this file.",
|
||||
"parse": ParseStats(
|
||||
status="failed",
|
||||
parsed_events=0,
|
||||
records_seen=parsed.records_seen,
|
||||
skipped_records=parsed.skipped_records,
|
||||
issue_count=len(parsed.issues),
|
||||
truncated=parsed.truncated,
|
||||
truncation_reason=parsed.truncation_reason,
|
||||
duration_ms=parsed.duration_ms,
|
||||
issues=parsed.issues,
|
||||
).model_dump(mode="json"),
|
||||
},
|
||||
)
|
||||
return AnalysisResponse(
|
||||
source=source,
|
||||
events=parsed.events,
|
||||
incidents=correlate(parsed.events),
|
||||
parse=ParseStats(
|
||||
status=parsed.status,
|
||||
parsed_events=len(parsed.events),
|
||||
records_seen=parsed.records_seen,
|
||||
skipped_records=parsed.skipped_records,
|
||||
issue_count=len(parsed.issues),
|
||||
truncated=parsed.truncated,
|
||||
truncation_reason=parsed.truncation_reason,
|
||||
duration_ms=parsed.duration_ms,
|
||||
issues=parsed.issues,
|
||||
),
|
||||
parsed = parse_evtx_bytes(data)
|
||||
except EvtxParseError as exc:
|
||||
raise HTTPException(
|
||||
422,
|
||||
detail={
|
||||
"code": "parser_rejected_input",
|
||||
"message": str(exc),
|
||||
"parse": _failed_parse_stats().model_dump(mode="json"),
|
||||
},
|
||||
) from exc
|
||||
if not parsed.events:
|
||||
raise HTTPException(
|
||||
422,
|
||||
detail={
|
||||
"code": "no_readable_events",
|
||||
"message": "No readable Windows events were found in this file.",
|
||||
"parse": _stats(parsed, "failed").model_dump(mode="json"),
|
||||
},
|
||||
)
|
||||
finally:
|
||||
await file.close()
|
||||
incidents, campaigns = correlate_with_campaigns(parsed.events)
|
||||
response = AnalysisResponse(
|
||||
source=source,
|
||||
events=parsed.events,
|
||||
incidents=incidents,
|
||||
campaigns=campaigns,
|
||||
parse=_stats(parsed),
|
||||
source_info=source_info,
|
||||
)
|
||||
payload = response.model_dump_json().encode("utf-8")
|
||||
# Event JSON is highly repetitive: level 1 shrinks a 100k-event response
|
||||
# ~20x for about a second of worker-thread CPU, never on the event loop.
|
||||
return gzip.compress(payload, compresslevel=GZIP_LEVEL, mtime=0) if compress else payload
|
||||
|
||||
|
||||
async def _read_bounded(upload: StarletteUploadFile) -> bytearray:
|
||||
data = bytearray()
|
||||
while True:
|
||||
chunk = await upload.read(min(1024 * 1024, MAX_UPLOAD_BYTES + 1 - len(data)))
|
||||
if not chunk:
|
||||
return data
|
||||
data.extend(chunk)
|
||||
if len(data) > MAX_UPLOAD_BYTES:
|
||||
raise HTTPException(413, "EVTX file exceeds the 50 MiB MVP limit")
|
||||
|
||||
|
||||
ANALYZE_OPENAPI = {
|
||||
"requestBody": {
|
||||
"required": True,
|
||||
"content": {
|
||||
"multipart/form-data": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"required": ["file"],
|
||||
"properties": {"file": {"type": "string", "format": "binary"}},
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@app.post("/api/analyze", response_model=AnalysisResponse, openapi_extra=ANALYZE_OPENAPI)
|
||||
async def analyze(request: Request):
|
||||
compress = "gzip" in request.headers.get("accept-encoding", "").lower()
|
||||
async with _admission():
|
||||
try:
|
||||
form = await request.form(max_files=1, max_fields=4)
|
||||
except MultiPartException as exc:
|
||||
raise HTTPException(400, "Upload one .evtx file as multipart form data") from exc
|
||||
try:
|
||||
upload = form.get("file")
|
||||
if not isinstance(upload, StarletteUploadFile):
|
||||
raise HTTPException(422, "Upload one .evtx file in the 'file' form field")
|
||||
source = _safe_source_name(upload.filename)
|
||||
if not source.lower().endswith(".evtx"):
|
||||
raise HTTPException(400, "Upload an .evtx file")
|
||||
data = await _read_bounded(upload)
|
||||
async with _parse_slot():
|
||||
try:
|
||||
payload = await asyncio.to_thread(_analyze_bytes, source, data, compress)
|
||||
except UnexpectedParserError as exc:
|
||||
logger.error(
|
||||
"Unexpected EVTX parser failure reached API exception_type=%s", type(exc).__name__
|
||||
)
|
||||
raise HTTPException(
|
||||
500,
|
||||
detail={
|
||||
"code": "parser_internal_error",
|
||||
"message": "EVTX analysis failed unexpectedly; retry the request.",
|
||||
},
|
||||
) from exc
|
||||
del data
|
||||
finally:
|
||||
await form.close()
|
||||
headers = {"Vary": "Accept-Encoding"}
|
||||
if compress:
|
||||
headers["Content-Encoding"] = "gzip"
|
||||
return Response(content=payload, media_type="application/json", headers=headers)
|
||||
|
||||
|
||||
def _safe_source_name(filename: str | None) -> str:
|
||||
|
||||
Reference in new issue
Block a user