299 lines
12 KiB
Python
299 lines
12 KiB
Python
"""Regression tests for the November 2026 audit: bounded titles, severities,
|
|
fallback burst compression, campaigns, new scoped rules, audit outcome,
|
|
source integrity metadata, bounded issue lists and off-loop analysis."""
|
|
|
|
import asyncio
|
|
import hashlib
|
|
import threading
|
|
import time
|
|
from datetime import UTC, datetime, timedelta
|
|
|
|
import httpx
|
|
|
|
from app import main
|
|
from app.correlate import MAX_TITLE_CHARS, correlate, correlate_with_campaigns
|
|
from app.models import NormalizedEvent, ParseIssue
|
|
from app.normalize import normalize_record
|
|
from app.parser import ParseResult
|
|
|
|
START = datetime(2026, 11, 20, 8, 0, tzinfo=UTC)
|
|
|
|
|
|
def event(
|
|
identifier: str,
|
|
seconds: int,
|
|
*,
|
|
provider: str = "Microsoft-Windows-Security-Auditing",
|
|
event_id: int = 4625,
|
|
channel: str = "Security",
|
|
computer: str = "LAB-PC",
|
|
level: str = "information",
|
|
summary: str = "Synthetic event",
|
|
fields: dict[str, str] | None = None,
|
|
) -> NormalizedEvent:
|
|
return NormalizedEvent(
|
|
id=identifier,
|
|
timestamp=START + timedelta(seconds=seconds),
|
|
provider=provider,
|
|
event_id=event_id,
|
|
level=level,
|
|
channel=channel,
|
|
computer=computer,
|
|
summary=summary,
|
|
fields=fields or {},
|
|
)
|
|
|
|
|
|
def request(method: str, url: str, **kwargs) -> httpx.Response:
|
|
async def send() -> httpx.Response:
|
|
transport = httpx.ASGITransport(app=main.app)
|
|
async with httpx.AsyncClient(transport=transport, base_url="http://testserver") as client:
|
|
return await client.request(method, url, **kwargs)
|
|
|
|
return asyncio.run(send())
|
|
|
|
|
|
# --- correlation ------------------------------------------------------------
|
|
|
|
|
|
def test_unmatched_event_with_long_summary_gets_a_bounded_title():
|
|
long = event("1", 0, provider="P" * 250, event_id=4_294_967_295, level="warning", summary="P" * 300 + " event")
|
|
incident = correlate([long])[0]
|
|
assert len(incident.title) <= MAX_TITLE_CHARS
|
|
assert incident.title.endswith("…")
|
|
|
|
|
|
def test_fallback_severity_preserves_critical_and_error_levels():
|
|
critical = event("1", 0, provider="Disk", event_id=11, channel="System", level="critical")
|
|
error = event("2", 0, provider="Ntfs", event_id=55, channel="System", level="error")
|
|
severities = {incident.evidence_event_ids[0]: incident.severity for incident in correlate([critical, error])}
|
|
assert severities == {"1": "critical", "2": "error"}
|
|
|
|
|
|
def test_identical_unmatched_warnings_are_burst_compressed():
|
|
events = [
|
|
event(str(index), index % 60, provider="Disk", event_id=153, channel="System", level="warning", summary="Disk event 153")
|
|
for index in range(5_000)
|
|
]
|
|
incidents = correlate(events)
|
|
assert len(incidents) == 1
|
|
assert incidents[0].rule_id == "windows.generic.attention"
|
|
assert incidents[0].original_count == 5_000
|
|
assert incidents[0].compressed_count == 1
|
|
assert any("compress" in reason for reason in incidents[0].reasons)
|
|
|
|
|
|
def test_fallback_never_merges_unknown_hosts_or_different_event_ids():
|
|
events = [
|
|
event("1", 0, provider="Disk", event_id=153, channel="System", computer="", level="warning"),
|
|
event("2", 1, provider="Disk", event_id=153, channel="System", computer="", level="warning"),
|
|
event("3", 2, provider="Disk", event_id=154, channel="System", level="warning"),
|
|
event("4", 3, provider="Disk", event_id=153, channel="System", level="warning"),
|
|
]
|
|
assert sorted(len(incident.evidence_event_ids) for incident in correlate(events)) == [1, 1, 1, 1]
|
|
|
|
|
|
def test_sustained_activity_is_summarized_as_a_campaign_without_widening_incidents():
|
|
events = [
|
|
event(str(index), index * 60, fields={"EventData.TargetUserName": "alice"}) for index in range(30)
|
|
]
|
|
incidents, campaigns = correlate_with_campaigns(events)
|
|
failed = [incident for incident in incidents if incident.rule_id == "windows.auth.failed-logon"]
|
|
assert len(failed) > 1
|
|
assert all(
|
|
(incident.end - incident.start).total_seconds() <= 120 for incident in failed if incident.start and incident.end
|
|
)
|
|
assert len(campaigns) == 1
|
|
campaign = campaigns[0]
|
|
assert campaign.incident_ids == [incident.id for incident in failed]
|
|
assert campaign.event_count == 30
|
|
assert {incident.campaign_id for incident in failed} == {campaign.id}
|
|
|
|
|
|
def test_separated_activity_does_not_form_a_campaign():
|
|
events = [
|
|
event("1", 0, fields={"EventData.TargetUserName": "alice"}),
|
|
event("2", 3_600, fields={"EventData.TargetUserName": "alice"}),
|
|
]
|
|
incidents, campaigns = correlate_with_campaigns(events)
|
|
assert campaigns == []
|
|
assert all(incident.campaign_id is None for incident in incidents)
|
|
|
|
|
|
def test_log_clear_is_a_critical_security_incident():
|
|
cleared = normalize_record(
|
|
{
|
|
"Event": {
|
|
"System": {
|
|
"Provider": {"#attributes": {"Name": "Microsoft-Windows-Eventlog"}},
|
|
"EventID": 1102,
|
|
"Channel": "Security",
|
|
"Computer": "LAB-PC",
|
|
"TimeCreated": {"#attributes": {"SystemTime": "2026-11-20T08:00:00Z"}},
|
|
},
|
|
"UserData": {"LogFileCleared": {"SubjectUserName": "admin"}},
|
|
}
|
|
}
|
|
)
|
|
assert cleared.category == "security"
|
|
assert cleared.summary == "Security audit log cleared"
|
|
incident = correlate([cleared])[0]
|
|
assert incident.rule_id == "windows.audit.log-cleared"
|
|
assert incident.severity == "critical"
|
|
|
|
|
|
def test_system_log_clear_is_detected_and_scoped_to_its_channel():
|
|
system_clear = event("1", 0, provider="Microsoft-Windows-Eventlog", event_id=104, channel="System")
|
|
wrong_channel = event("2", 0, provider="Microsoft-Windows-Eventlog", event_id=104, channel="Application")
|
|
rules = {incident.evidence_event_ids[0]: incident.rule_id for incident in correlate([system_clear, wrong_channel])}
|
|
assert rules.get("1") == "windows.audit.log-cleared"
|
|
assert rules.get("2") != "windows.audit.log-cleared"
|
|
|
|
|
|
def test_service_installation_lockout_and_account_change_rules():
|
|
events = [
|
|
event("1", 0, provider="Service Control Manager", event_id=7045, channel="System", fields={"EventData.ServiceName": "Updater"}),
|
|
event("2", 10, event_id=4740, fields={"EventData.TargetUserName": "alice"}),
|
|
event("3", 20, event_id=4720, fields={"EventData.TargetUserName": "backup"}),
|
|
event("4", 25, event_id=4732, fields={"EventData.TargetUserName": "Administrators"}),
|
|
]
|
|
by_rule = {incident.rule_id: incident for incident in correlate(events)}
|
|
assert by_rule["windows.service.installed"].evidence_event_ids == ["1"]
|
|
assert by_rule["windows.auth.account-lockout"].evidence_event_ids == ["2"]
|
|
change = by_rule["windows.account.change"]
|
|
assert change.evidence_event_ids == ["3", "4"]
|
|
assert change.title == "Account created and added to a security group"
|
|
|
|
|
|
def test_password_spray_requires_distinct_accounts_from_one_known_source():
|
|
spray = [
|
|
event(str(index), index, fields={"EventData.TargetUserName": f"user{index}", "EventData.IpAddress": "192.0.2.9"})
|
|
for index in range(3)
|
|
]
|
|
two_accounts = [
|
|
event(f"b{index}", index, fields={"EventData.TargetUserName": f"user{index % 2}", "EventData.IpAddress": "192.0.2.8"})
|
|
for index in range(4)
|
|
]
|
|
unknown_source = [
|
|
event(f"c{index}", index, fields={"EventData.TargetUserName": f"user{index}", "EventData.IpAddress": "-"})
|
|
for index in range(5)
|
|
]
|
|
sprays = [
|
|
incident for incident in correlate(spray + two_accounts + unknown_source) if incident.rule_id == "windows.auth.password-spray"
|
|
]
|
|
assert len(sprays) == 1
|
|
assert sprays[0].evidence_event_ids == ["0", "1", "2"]
|
|
assert "3 distinct accounts" in sprays[0].reasons[0]
|
|
|
|
|
|
def test_audit_failure_keyword_becomes_outcome():
|
|
record = {
|
|
"Event": {
|
|
"System": {
|
|
"Provider": {"#attributes": {"Name": "Microsoft-Windows-Security-Auditing"}},
|
|
"EventID": 4625,
|
|
"Level": 0,
|
|
"Keywords": "0x8010000000000000",
|
|
"Channel": "Security",
|
|
}
|
|
}
|
|
}
|
|
assert normalize_record(record).outcome == "failure"
|
|
record["Event"]["System"]["Keywords"] = "0x8020000000000000"
|
|
assert normalize_record(record).outcome == "success"
|
|
record["Event"]["System"]["Keywords"] = "not-hex"
|
|
assert normalize_record(record).outcome is None
|
|
|
|
|
|
def test_correlation_of_large_inputs_stays_within_budget():
|
|
events = [
|
|
event(str(index), index, fields={"EventData.TargetUserName": f"u{index % 50}", "EventData.IpAddress": "192.0.2.1"})
|
|
for index in range(50_000)
|
|
]
|
|
started = time.perf_counter()
|
|
incidents, campaigns = correlate_with_campaigns(events)
|
|
assert time.perf_counter() - started < 20
|
|
assert len(campaigns) >= 50
|
|
assert incidents
|
|
|
|
|
|
# --- API --------------------------------------------------------------------
|
|
|
|
|
|
def _parse_with_issues(issue_count: int):
|
|
def fake(_data) -> ParseResult:
|
|
events = [event("1", 0, provider="Test", event_id=1000, channel="Application", level="error")]
|
|
issues = [ParseIssue(code="invalid_timestamp", message="synthetic", record_id=index) for index in range(issue_count)]
|
|
return ParseResult(events=events, issues=issues, truncated=False, duration_ms=1)
|
|
|
|
return fake
|
|
|
|
|
|
def test_analysis_reports_source_sha256_and_size(monkeypatch):
|
|
monkeypatch.setattr(main, "parse_evtx_bytes", _parse_with_issues(0))
|
|
payload = b"synthetic-evtx-bytes"
|
|
response = request("POST", "/api/analyze", files={"file": ("sample.evtx", payload, "application/octet-stream")})
|
|
assert response.status_code == 200
|
|
assert response.json()["source_info"] == {"sha256": hashlib.sha256(payload).hexdigest(), "size_bytes": len(payload)}
|
|
|
|
|
|
def test_demo_has_no_source_integrity_claim_and_returns_campaign_list():
|
|
body = request("GET", "/api/demo").json()
|
|
assert body["source_info"] is None
|
|
assert isinstance(body["campaigns"], list)
|
|
|
|
|
|
def test_parse_issue_list_is_bounded_with_counts(monkeypatch):
|
|
monkeypatch.setattr(main, "parse_evtx_bytes", _parse_with_issues(main.MAX_REPORTED_ISSUES + 50))
|
|
response = request("POST", "/api/analyze", files={"file": ("sample.evtx", b"x", "application/octet-stream")})
|
|
parse = response.json()["parse"]
|
|
assert parse["issue_count"] == main.MAX_REPORTED_ISSUES + 50
|
|
assert len(parse["issues"]) == main.MAX_REPORTED_ISSUES
|
|
assert parse["issues_truncated"] is True
|
|
assert parse["issue_counts"] == {"invalid_timestamp": main.MAX_REPORTED_ISSUES + 50}
|
|
|
|
|
|
def test_parsing_and_correlation_run_off_the_event_loop(monkeypatch):
|
|
threads: list[int] = []
|
|
original = main.correlate_with_campaigns
|
|
|
|
def recording(events):
|
|
threads.append(threading.get_ident())
|
|
return original(events)
|
|
|
|
monkeypatch.setattr(main, "parse_evtx_bytes", _parse_with_issues(0))
|
|
monkeypatch.setattr(main, "correlate_with_campaigns", recording)
|
|
loop_thread: list[int] = []
|
|
|
|
async def send() -> httpx.Response:
|
|
loop_thread.append(threading.get_ident())
|
|
transport = httpx.ASGITransport(app=main.app)
|
|
async with httpx.AsyncClient(transport=transport, base_url="http://testserver") as client:
|
|
return await client.post("/api/analyze", files={"file": ("a.evtx", b"x", "application/octet-stream")})
|
|
|
|
assert asyncio.run(send()).status_code == 200
|
|
assert threads and threads[0] != loop_thread[0]
|
|
|
|
|
|
def test_full_admission_rejects_before_reading_the_upload(monkeypatch):
|
|
def must_not_parse(_data):
|
|
raise AssertionError("parser must not run when admission is full")
|
|
|
|
monkeypatch.setattr(main, "parse_evtx_bytes", must_not_parse)
|
|
monkeypatch.setattr(main, "_parse_inflight", main.PARSE_CONCURRENCY + main.PARSE_QUEUE_LIMIT)
|
|
response = request("POST", "/api/analyze", files={"file": ("a.evtx", b"x", "application/octet-stream")})
|
|
assert response.status_code == 503
|
|
assert response.json()["detail"]["code"] == "parse_capacity_exceeded"
|
|
assert response.headers["retry-after"] == "1"
|
|
|
|
|
|
def test_missing_file_field_is_a_client_error():
|
|
response = request("POST", "/api/analyze", data={"other": "value"})
|
|
assert response.status_code == 422
|
|
|
|
|
|
def test_production_csp_only_allows_same_origin_connections():
|
|
csp = request("GET", "/health").headers["content-security-policy"]
|
|
assert "connect-src 'self';" in csp
|
|
assert "localhost" not in csp
|