EventLens: publish v1.1.0 application and standalone Compose
Includes campaign summaries, expanded Windows event correlation, worker-based redaction, updated demo assets and a local-only Compose example. Publish curated EventLens source from 6a2ca12f81a2dcf816c61a86bd8811fcc95e8d8d
This commit is contained in:
1 parent
c0ade21029
commit
4e282639a4
63 files changed
+7381
-1420
No files matched your search
@@ -0,0 +1,298 @@
|
||||
"""Regression tests for the November 2026 audit: bounded titles, severities,
|
||||
fallback burst compression, campaigns, new scoped rules, audit outcome,
|
||||
source integrity metadata, bounded issue lists and off-loop analysis."""
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import threading
|
||||
import time
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
import httpx
|
||||
|
||||
from app import main
|
||||
from app.correlate import MAX_TITLE_CHARS, correlate, correlate_with_campaigns
|
||||
from app.models import NormalizedEvent, ParseIssue
|
||||
from app.normalize import normalize_record
|
||||
from app.parser import ParseResult
|
||||
|
||||
START = datetime(2026, 11, 20, 8, 0, tzinfo=UTC)
|
||||
|
||||
|
||||
def event(
|
||||
identifier: str,
|
||||
seconds: int,
|
||||
*,
|
||||
provider: str = "Microsoft-Windows-Security-Auditing",
|
||||
event_id: int = 4625,
|
||||
channel: str = "Security",
|
||||
computer: str = "LAB-PC",
|
||||
level: str = "information",
|
||||
summary: str = "Synthetic event",
|
||||
fields: dict[str, str] | None = None,
|
||||
) -> NormalizedEvent:
|
||||
return NormalizedEvent(
|
||||
id=identifier,
|
||||
timestamp=START + timedelta(seconds=seconds),
|
||||
provider=provider,
|
||||
event_id=event_id,
|
||||
level=level,
|
||||
channel=channel,
|
||||
computer=computer,
|
||||
summary=summary,
|
||||
fields=fields or {},
|
||||
)
|
||||
|
||||
|
||||
def request(method: str, url: str, **kwargs) -> httpx.Response:
|
||||
async def send() -> httpx.Response:
|
||||
transport = httpx.ASGITransport(app=main.app)
|
||||
async with httpx.AsyncClient(transport=transport, base_url="http://testserver") as client:
|
||||
return await client.request(method, url, **kwargs)
|
||||
|
||||
return asyncio.run(send())
|
||||
|
||||
|
||||
# --- correlation ------------------------------------------------------------
|
||||
|
||||
|
||||
def test_unmatched_event_with_long_summary_gets_a_bounded_title():
|
||||
long = event("1", 0, provider="P" * 250, event_id=4_294_967_295, level="warning", summary="P" * 300 + " event")
|
||||
incident = correlate([long])[0]
|
||||
assert len(incident.title) <= MAX_TITLE_CHARS
|
||||
assert incident.title.endswith("…")
|
||||
|
||||
|
||||
def test_fallback_severity_preserves_critical_and_error_levels():
|
||||
critical = event("1", 0, provider="Disk", event_id=11, channel="System", level="critical")
|
||||
error = event("2", 0, provider="Ntfs", event_id=55, channel="System", level="error")
|
||||
severities = {incident.evidence_event_ids[0]: incident.severity for incident in correlate([critical, error])}
|
||||
assert severities == {"1": "critical", "2": "error"}
|
||||
|
||||
|
||||
def test_identical_unmatched_warnings_are_burst_compressed():
|
||||
events = [
|
||||
event(str(index), index % 60, provider="Disk", event_id=153, channel="System", level="warning", summary="Disk event 153")
|
||||
for index in range(5_000)
|
||||
]
|
||||
incidents = correlate(events)
|
||||
assert len(incidents) == 1
|
||||
assert incidents[0].rule_id == "windows.generic.attention"
|
||||
assert incidents[0].original_count == 5_000
|
||||
assert incidents[0].compressed_count == 1
|
||||
assert any("compress" in reason for reason in incidents[0].reasons)
|
||||
|
||||
|
||||
def test_fallback_never_merges_unknown_hosts_or_different_event_ids():
|
||||
events = [
|
||||
event("1", 0, provider="Disk", event_id=153, channel="System", computer="", level="warning"),
|
||||
event("2", 1, provider="Disk", event_id=153, channel="System", computer="", level="warning"),
|
||||
event("3", 2, provider="Disk", event_id=154, channel="System", level="warning"),
|
||||
event("4", 3, provider="Disk", event_id=153, channel="System", level="warning"),
|
||||
]
|
||||
assert sorted(len(incident.evidence_event_ids) for incident in correlate(events)) == [1, 1, 1, 1]
|
||||
|
||||
|
||||
def test_sustained_activity_is_summarized_as_a_campaign_without_widening_incidents():
|
||||
events = [
|
||||
event(str(index), index * 60, fields={"EventData.TargetUserName": "alice"}) for index in range(30)
|
||||
]
|
||||
incidents, campaigns = correlate_with_campaigns(events)
|
||||
failed = [incident for incident in incidents if incident.rule_id == "windows.auth.failed-logon"]
|
||||
assert len(failed) > 1
|
||||
assert all(
|
||||
(incident.end - incident.start).total_seconds() <= 120 for incident in failed if incident.start and incident.end
|
||||
)
|
||||
assert len(campaigns) == 1
|
||||
campaign = campaigns[0]
|
||||
assert campaign.incident_ids == [incident.id for incident in failed]
|
||||
assert campaign.event_count == 30
|
||||
assert {incident.campaign_id for incident in failed} == {campaign.id}
|
||||
|
||||
|
||||
def test_separated_activity_does_not_form_a_campaign():
|
||||
events = [
|
||||
event("1", 0, fields={"EventData.TargetUserName": "alice"}),
|
||||
event("2", 3_600, fields={"EventData.TargetUserName": "alice"}),
|
||||
]
|
||||
incidents, campaigns = correlate_with_campaigns(events)
|
||||
assert campaigns == []
|
||||
assert all(incident.campaign_id is None for incident in incidents)
|
||||
|
||||
|
||||
def test_log_clear_is_a_critical_security_incident():
|
||||
cleared = normalize_record(
|
||||
{
|
||||
"Event": {
|
||||
"System": {
|
||||
"Provider": {"#attributes": {"Name": "Microsoft-Windows-Eventlog"}},
|
||||
"EventID": 1102,
|
||||
"Channel": "Security",
|
||||
"Computer": "LAB-PC",
|
||||
"TimeCreated": {"#attributes": {"SystemTime": "2026-11-20T08:00:00Z"}},
|
||||
},
|
||||
"UserData": {"LogFileCleared": {"SubjectUserName": "admin"}},
|
||||
}
|
||||
}
|
||||
)
|
||||
assert cleared.category == "security"
|
||||
assert cleared.summary == "Security audit log cleared"
|
||||
incident = correlate([cleared])[0]
|
||||
assert incident.rule_id == "windows.audit.log-cleared"
|
||||
assert incident.severity == "critical"
|
||||
|
||||
|
||||
def test_system_log_clear_is_detected_and_scoped_to_its_channel():
|
||||
system_clear = event("1", 0, provider="Microsoft-Windows-Eventlog", event_id=104, channel="System")
|
||||
wrong_channel = event("2", 0, provider="Microsoft-Windows-Eventlog", event_id=104, channel="Application")
|
||||
rules = {incident.evidence_event_ids[0]: incident.rule_id for incident in correlate([system_clear, wrong_channel])}
|
||||
assert rules.get("1") == "windows.audit.log-cleared"
|
||||
assert rules.get("2") != "windows.audit.log-cleared"
|
||||
|
||||
|
||||
def test_service_installation_lockout_and_account_change_rules():
|
||||
events = [
|
||||
event("1", 0, provider="Service Control Manager", event_id=7045, channel="System", fields={"EventData.ServiceName": "Updater"}),
|
||||
event("2", 10, event_id=4740, fields={"EventData.TargetUserName": "alice"}),
|
||||
event("3", 20, event_id=4720, fields={"EventData.TargetUserName": "backup"}),
|
||||
event("4", 25, event_id=4732, fields={"EventData.TargetUserName": "Administrators"}),
|
||||
]
|
||||
by_rule = {incident.rule_id: incident for incident in correlate(events)}
|
||||
assert by_rule["windows.service.installed"].evidence_event_ids == ["1"]
|
||||
assert by_rule["windows.auth.account-lockout"].evidence_event_ids == ["2"]
|
||||
change = by_rule["windows.account.change"]
|
||||
assert change.evidence_event_ids == ["3", "4"]
|
||||
assert change.title == "Account created and added to a security group"
|
||||
|
||||
|
||||
def test_password_spray_requires_distinct_accounts_from_one_known_source():
|
||||
spray = [
|
||||
event(str(index), index, fields={"EventData.TargetUserName": f"user{index}", "EventData.IpAddress": "192.0.2.9"})
|
||||
for index in range(3)
|
||||
]
|
||||
two_accounts = [
|
||||
event(f"b{index}", index, fields={"EventData.TargetUserName": f"user{index % 2}", "EventData.IpAddress": "192.0.2.8"})
|
||||
for index in range(4)
|
||||
]
|
||||
unknown_source = [
|
||||
event(f"c{index}", index, fields={"EventData.TargetUserName": f"user{index}", "EventData.IpAddress": "-"})
|
||||
for index in range(5)
|
||||
]
|
||||
sprays = [
|
||||
incident for incident in correlate(spray + two_accounts + unknown_source) if incident.rule_id == "windows.auth.password-spray"
|
||||
]
|
||||
assert len(sprays) == 1
|
||||
assert sprays[0].evidence_event_ids == ["0", "1", "2"]
|
||||
assert "3 distinct accounts" in sprays[0].reasons[0]
|
||||
|
||||
|
||||
def test_audit_failure_keyword_becomes_outcome():
|
||||
record = {
|
||||
"Event": {
|
||||
"System": {
|
||||
"Provider": {"#attributes": {"Name": "Microsoft-Windows-Security-Auditing"}},
|
||||
"EventID": 4625,
|
||||
"Level": 0,
|
||||
"Keywords": "0x8010000000000000",
|
||||
"Channel": "Security",
|
||||
}
|
||||
}
|
||||
}
|
||||
assert normalize_record(record).outcome == "failure"
|
||||
record["Event"]["System"]["Keywords"] = "0x8020000000000000"
|
||||
assert normalize_record(record).outcome == "success"
|
||||
record["Event"]["System"]["Keywords"] = "not-hex"
|
||||
assert normalize_record(record).outcome is None
|
||||
|
||||
|
||||
def test_correlation_of_large_inputs_stays_within_budget():
|
||||
events = [
|
||||
event(str(index), index, fields={"EventData.TargetUserName": f"u{index % 50}", "EventData.IpAddress": "192.0.2.1"})
|
||||
for index in range(50_000)
|
||||
]
|
||||
started = time.perf_counter()
|
||||
incidents, campaigns = correlate_with_campaigns(events)
|
||||
assert time.perf_counter() - started < 20
|
||||
assert len(campaigns) >= 50
|
||||
assert incidents
|
||||
|
||||
|
||||
# --- API --------------------------------------------------------------------
|
||||
|
||||
|
||||
def _parse_with_issues(issue_count: int):
|
||||
def fake(_data) -> ParseResult:
|
||||
events = [event("1", 0, provider="Test", event_id=1000, channel="Application", level="error")]
|
||||
issues = [ParseIssue(code="invalid_timestamp", message="synthetic", record_id=index) for index in range(issue_count)]
|
||||
return ParseResult(events=events, issues=issues, truncated=False, duration_ms=1)
|
||||
|
||||
return fake
|
||||
|
||||
|
||||
def test_analysis_reports_source_sha256_and_size(monkeypatch):
|
||||
monkeypatch.setattr(main, "parse_evtx_bytes", _parse_with_issues(0))
|
||||
payload = b"synthetic-evtx-bytes"
|
||||
response = request("POST", "/api/analyze", files={"file": ("sample.evtx", payload, "application/octet-stream")})
|
||||
assert response.status_code == 200
|
||||
assert response.json()["source_info"] == {"sha256": hashlib.sha256(payload).hexdigest(), "size_bytes": len(payload)}
|
||||
|
||||
|
||||
def test_demo_has_no_source_integrity_claim_and_returns_campaign_list():
|
||||
body = request("GET", "/api/demo").json()
|
||||
assert body["source_info"] is None
|
||||
assert isinstance(body["campaigns"], list)
|
||||
|
||||
|
||||
def test_parse_issue_list_is_bounded_with_counts(monkeypatch):
|
||||
monkeypatch.setattr(main, "parse_evtx_bytes", _parse_with_issues(main.MAX_REPORTED_ISSUES + 50))
|
||||
response = request("POST", "/api/analyze", files={"file": ("sample.evtx", b"x", "application/octet-stream")})
|
||||
parse = response.json()["parse"]
|
||||
assert parse["issue_count"] == main.MAX_REPORTED_ISSUES + 50
|
||||
assert len(parse["issues"]) == main.MAX_REPORTED_ISSUES
|
||||
assert parse["issues_truncated"] is True
|
||||
assert parse["issue_counts"] == {"invalid_timestamp": main.MAX_REPORTED_ISSUES + 50}
|
||||
|
||||
|
||||
def test_parsing_and_correlation_run_off_the_event_loop(monkeypatch):
|
||||
threads: list[int] = []
|
||||
original = main.correlate_with_campaigns
|
||||
|
||||
def recording(events):
|
||||
threads.append(threading.get_ident())
|
||||
return original(events)
|
||||
|
||||
monkeypatch.setattr(main, "parse_evtx_bytes", _parse_with_issues(0))
|
||||
monkeypatch.setattr(main, "correlate_with_campaigns", recording)
|
||||
loop_thread: list[int] = []
|
||||
|
||||
async def send() -> httpx.Response:
|
||||
loop_thread.append(threading.get_ident())
|
||||
transport = httpx.ASGITransport(app=main.app)
|
||||
async with httpx.AsyncClient(transport=transport, base_url="http://testserver") as client:
|
||||
return await client.post("/api/analyze", files={"file": ("a.evtx", b"x", "application/octet-stream")})
|
||||
|
||||
assert asyncio.run(send()).status_code == 200
|
||||
assert threads and threads[0] != loop_thread[0]
|
||||
|
||||
|
||||
def test_full_admission_rejects_before_reading_the_upload(monkeypatch):
|
||||
def must_not_parse(_data):
|
||||
raise AssertionError("parser must not run when admission is full")
|
||||
|
||||
monkeypatch.setattr(main, "parse_evtx_bytes", must_not_parse)
|
||||
monkeypatch.setattr(main, "_parse_inflight", main.PARSE_CONCURRENCY + main.PARSE_QUEUE_LIMIT)
|
||||
response = request("POST", "/api/analyze", files={"file": ("a.evtx", b"x", "application/octet-stream")})
|
||||
assert response.status_code == 503
|
||||
assert response.json()["detail"]["code"] == "parse_capacity_exceeded"
|
||||
assert response.headers["retry-after"] == "1"
|
||||
|
||||
|
||||
def test_missing_file_field_is_a_client_error():
|
||||
response = request("POST", "/api/analyze", data={"other": "value"})
|
||||
assert response.status_code == 422
|
||||
|
||||
|
||||
def test_production_csp_only_allows_same_origin_connections():
|
||||
csp = request("GET", "/health").headers["content-security-policy"]
|
||||
assert "connect-src 'self';" in csp
|
||||
assert "localhost" not in csp
|
||||
@@ -0,0 +1,99 @@
|
||||
"""Guards for the performance shortcuts: they must never change results or
|
||||
bypass the normalized-event contract."""
|
||||
|
||||
import asyncio
|
||||
import gzip
|
||||
from datetime import UTC, datetime
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from app import correlate as correlate_module
|
||||
from app import main
|
||||
from app.models import NormalizedEvent
|
||||
from app.normalize import _render_scalar, normalize_record
|
||||
from app.parser import ParseResult
|
||||
|
||||
RECORDS = [
|
||||
{"Event": {"System": {"Provider": {"#attributes": {"Name": "Service Control Manager"}}, "EventID": {"#text": 7031, "#attributes": {"Qualifiers": 49152}}, "Level": 2, "Channel": "System", "Computer": "LAB-PC", "EventRecordID": 9, "TimeCreated": {"#attributes": {"SystemTime": "2026-11-20T08:00:00.123456Z"}}}, "EventData": {"param1": "Sync", "param2": 1, "flag": True, "none": None, "ratio": 1.5, "huge": 10**30}}},
|
||||
{"Event": {"System": {"Provider": "P" * 400, "EventID": "not-a-number", "Level": "weird", "Channel": "C" * 400, "Computer": "H" * 400}, "UserData": {"Nested": {"List": [{"a": "x" * 5000}, [1, [2, [3]]]]}}}},
|
||||
{"Event": {"System": {"Provider": {"Name": "Legacy"}, "EventID": 1, "TimeCreated": {"#attributes": {"SystemTime": "garbage"}}}}},
|
||||
{"Event": {"System": {"EventID": 4294967296, "Keywords": "0x8010000000000000"}, "EventData": {f"k{index}": index for index in range(400)}}},
|
||||
{"System": {"Provider": {"#attributes": {"Name": "Microsoft-Windows-Eventlog"}}, "EventID": 1102, "Channel": "Security"}},
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("payload", RECORDS)
|
||||
def test_constructed_events_satisfy_the_validated_contract(payload):
|
||||
event = normalize_record(payload, record_timestamp="2026-11-20T08:00:00Z", record_ordinal=3)
|
||||
revalidated = NormalizedEvent.model_validate(event.model_dump())
|
||||
assert revalidated == event
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", [None, True, False, 0, -7, 10**30, 1.5, float("inf"), [1, "a"], {"a": 1}])
|
||||
def test_fast_scalar_rendering_matches_json_dumps(value):
|
||||
import json
|
||||
|
||||
assert _render_scalar(value) == json.dumps(value, ensure_ascii=False)
|
||||
|
||||
|
||||
def test_strings_are_kept_verbatim():
|
||||
assert _render_scalar('a "quoted" value') == 'a "quoted" value'
|
||||
|
||||
|
||||
def test_rule_predicates_run_once_per_signature(monkeypatch):
|
||||
calls = 0
|
||||
original = correlate_module._failed_logon
|
||||
|
||||
def counting(event):
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
return original(event)
|
||||
|
||||
rules = tuple(
|
||||
correlate_module.Rule(**{**rule.__dict__, "predicate": counting}) if rule.predicate is original else rule
|
||||
for rule in correlate_module.RULES
|
||||
)
|
||||
monkeypatch.setattr(correlate_module, "RULES", rules)
|
||||
events = [
|
||||
NormalizedEvent(id=str(index), timestamp=datetime(2026, 11, 20, tzinfo=UTC), provider="Microsoft-Windows-Security-Auditing", channel="Security", computer="LAB-PC", event_id=4625, fields={"EventData.TargetUserName": "alice"})
|
||||
for index in range(1_000)
|
||||
]
|
||||
correlate_module.correlate(events)
|
||||
# two rules use the predicate (failed-logon and password-spray), one signature each
|
||||
assert calls == 2
|
||||
|
||||
|
||||
def _request(headers: dict[str, str]) -> httpx.Response:
|
||||
async def send() -> httpx.Response:
|
||||
transport = httpx.ASGITransport(app=main.app)
|
||||
async with httpx.AsyncClient(transport=transport, base_url="http://testserver") as client:
|
||||
return await client.post("/api/analyze", headers=headers, files={"file": ("a.evtx", b"x", "application/octet-stream")})
|
||||
|
||||
return asyncio.run(send())
|
||||
|
||||
|
||||
def _fake_parse(_data) -> ParseResult:
|
||||
event = NormalizedEvent(id="1", timestamp=datetime(2026, 11, 20, tzinfo=UTC), provider="Test", event_id=1, level="error", channel="Application", computer="LAB-PC", summary="x")
|
||||
return ParseResult(events=[event], issues=[], truncated=False, duration_ms=1)
|
||||
|
||||
|
||||
def test_analysis_response_is_gzipped_when_accepted(monkeypatch):
|
||||
monkeypatch.setattr(main, "parse_evtx_bytes", _fake_parse)
|
||||
response = _request({"Accept-Encoding": "gzip"})
|
||||
assert response.status_code == 200
|
||||
assert response.headers["content-encoding"] == "gzip"
|
||||
assert response.headers["vary"] == "Accept-Encoding"
|
||||
assert response.json()["events"][0]["id"] == "1"
|
||||
|
||||
|
||||
def test_analysis_response_is_plain_without_gzip_support(monkeypatch):
|
||||
monkeypatch.setattr(main, "parse_evtx_bytes", _fake_parse)
|
||||
response = _request({"Accept-Encoding": "identity"})
|
||||
assert "content-encoding" not in response.headers
|
||||
assert response.json()["events"][0]["id"] == "1"
|
||||
|
||||
|
||||
def test_gzip_output_is_deterministic():
|
||||
payload = b'{"events": []}' * 100
|
||||
assert gzip.compress(payload, compresslevel=main.GZIP_LEVEL, mtime=0) == gzip.compress(payload, compresslevel=main.GZIP_LEVEL, mtime=0)
|
||||
Reference in new issue
Block a user