RepoPulse: update backend/app and dependencies (+3 areas)
Public changes compared with the previous published export: 26 files changed. Affected areas: - backend/app: 11 files - dependencies: 5 files - deployment and CI: 3 files - documentation: 2 files - tests: 5 files Changed files: - Updated: "Dockerfile.unraid" - Updated: "README.md" - Updated: "backend/Dockerfile" - Updated: "backend/app/acquisition.py" - Updated: "backend/app/architecture.py" - Updated: "backend/app/git_analysis.py" - Updated: "backend/app/gitea.py" - Updated: "backend/app/jobs.py" - Updated: "backend/app/main.py" - Added: "backend/app/processes.py" - Updated: "backend/app/rules.py" - Updated: "backend/app/scanner.py" - Updated: "backend/app/signals.py" - Updated: "backend/app/store.py" - Updated: "backend/pyproject.toml" - Added: "backend/tests/test_api_responsiveness.py" - Added: "backend/tests/test_processes.py" - Updated: "backend/tests/test_scanner.py" - Updated: "backend/tests/test_signals_v11.py" - Updated: "backend/tests/test_store.py" - Updated: "docker-compose.yml" - Updated: "docs/ARCHITECTURE.md" - Updated: "frontend/package-lock.json" - Updated: "frontend/package.json" - Updated: "package-lock.json" - Updated: "package.json" Export provenance/manifests refreshed; excluded from the change summary. Publish curated RepoPulse source from a251f826238908b156485299061da76482780ed2
This commit is contained in:
1 parent
23a4a721f1
commit
61ae49f9d0
28 files changed
+1083
-408
No files matched your search
@@ -0,0 +1,107 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
import app.store as store_module
|
||||
from app import main as main_module
|
||||
from app.models import Finding, Repository, Severity
|
||||
from app.rules import enrich_finding, enrich_findings
|
||||
from app.store import ScanStore
|
||||
|
||||
|
||||
def test_database_bound_endpoints_never_run_blocking_sqlite_on_the_event_loop() -> None:
|
||||
# A sync endpoint runs in the threadpool; an async one must offload its store work.
|
||||
assert not inspect.iscoroutinefunction(main_module.create_repository_scan)
|
||||
assert not inspect.iscoroutinefunction(main_module._handle_gitea_webhook)
|
||||
webhook_source = inspect.getsource(main_module.gitea_webhook)
|
||||
assert "store." not in webhook_source
|
||||
assert "run_in_threadpool(_handle_gitea_webhook" in webhook_source
|
||||
assert "asyncio.to_thread(store.save" in inspect.getsource(main_module.create_scan)
|
||||
|
||||
|
||||
def test_large_api_responses_are_gzip_compressed(tmp_path: Path) -> None:
|
||||
main_module.store = ScanStore(str(tmp_path / "history.db"))
|
||||
main_module.job_runner = None
|
||||
for index in range(12):
|
||||
main_module.store.upsert_repository(
|
||||
Repository(
|
||||
id=f"gitea:{index}",
|
||||
provider="gitea",
|
||||
provider_repository_id=str(index),
|
||||
owner="Jens",
|
||||
name=f"Repository{index}",
|
||||
full_name=f"Jens/Repository{index}",
|
||||
default_branch="main",
|
||||
)
|
||||
)
|
||||
client = TestClient(main_module.app)
|
||||
|
||||
compressed = client.get("/api/v1/repositories", headers={"Accept-Encoding": "gzip"})
|
||||
small = client.get("/health", headers={"Accept-Encoding": "gzip"})
|
||||
|
||||
assert compressed.status_code == 200
|
||||
assert compressed.headers.get("content-encoding") == "gzip"
|
||||
assert len(compressed.json()) == 12
|
||||
# Responses below the 1 KiB threshold are not worth compressing.
|
||||
assert small.headers.get("content-encoding") is None
|
||||
|
||||
|
||||
def test_hashed_frontend_assets_are_immutable_and_index_revalidates(tmp_path: Path) -> None:
|
||||
(tmp_path / "assets").mkdir()
|
||||
(tmp_path / "index.html").write_text("<html></html>", encoding="utf-8")
|
||||
(tmp_path / "assets" / "index-abc123.js").write_text("console.log(1)", encoding="utf-8")
|
||||
app = FastAPI()
|
||||
app.mount("/", main_module._FrontendStaticFiles(directory=tmp_path, html=True), name="frontend")
|
||||
client = TestClient(app)
|
||||
|
||||
asset = client.get("/assets/index-abc123.js")
|
||||
index = client.get("/")
|
||||
|
||||
assert asset.headers["cache-control"] == "public, max-age=31536000, immutable"
|
||||
assert index.headers["cache-control"] == "no-cache"
|
||||
|
||||
|
||||
def test_finding_guidance_is_shared_and_unchanged() -> None:
|
||||
finding = Finding(
|
||||
code="security.secret-assignment",
|
||||
category="Security basics",
|
||||
severity=Severity.BLOCKER,
|
||||
message="m",
|
||||
deduction=20,
|
||||
)
|
||||
first, second = enrich_findings([finding, finding.model_copy(update={"path": "b.py"})])
|
||||
|
||||
assert first.guidance is second.guidance
|
||||
assert first.guidance.priority == 800
|
||||
assert first.guidance.priority_rationale == "Blocker signal; 20 health-point impact."
|
||||
assert enrich_finding(first) is first
|
||||
|
||||
|
||||
def test_expired_mirror_stats_are_served_while_refreshing_in_background(tmp_path: Path, monkeypatch) -> None:
|
||||
store = ScanStore(str(tmp_path / "history.db"))
|
||||
mirrors = tmp_path / "mirrors"
|
||||
mirrors.mkdir()
|
||||
calls: list[float] = []
|
||||
|
||||
def slow_walk(_root: Path) -> tuple[int, int]:
|
||||
calls.append(time.monotonic())
|
||||
time.sleep(0.3)
|
||||
return len(calls), len(calls) * 100
|
||||
|
||||
monkeypatch.setattr(store, "_tree_stats", slow_walk)
|
||||
assert store._cached_mirror_stats(mirrors) == (1, 100)
|
||||
|
||||
monkeypatch.setattr(store_module, "MIRROR_STATS_TTL_SECONDS", 0.0)
|
||||
started = time.monotonic()
|
||||
assert store._cached_mirror_stats(mirrors) == (1, 100)
|
||||
assert time.monotonic() - started < 0.2
|
||||
|
||||
deadline = time.monotonic() + 3
|
||||
while store._mirror_stats_cache[2] == (1, 100) and time.monotonic() < deadline:
|
||||
time.sleep(0.05)
|
||||
assert store._mirror_stats_cache[2] == (2, 200)
|
||||
@@ -0,0 +1,101 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app.gitea import GiteaClient
|
||||
from app.jobs import ScanJobRunner
|
||||
from app.processes import git_environment, run_bounded
|
||||
from app.store import ScanStore
|
||||
|
||||
|
||||
def test_git_environment_forces_foreground_housekeeping_and_keeps_extra_config() -> None:
|
||||
env = git_environment({}, extra_config=[("http.extraHeader", "Authorization: token t")])
|
||||
|
||||
pairs = {
|
||||
env[f"GIT_CONFIG_KEY_{index}"]: env[f"GIT_CONFIG_VALUE_{index}"]
|
||||
for index in range(int(env["GIT_CONFIG_COUNT"]))
|
||||
}
|
||||
assert pairs == {
|
||||
"gc.autoDetach": "false",
|
||||
"maintenance.autoDetach": "false",
|
||||
"http.extraHeader": "Authorization: token t",
|
||||
}
|
||||
|
||||
|
||||
def test_run_bounded_returns_output_and_raises_on_failure() -> None:
|
||||
ok = run_bounded([sys.executable, "-c", "print('hi')"], timeout=10)
|
||||
assert ok.stdout.strip() == b"hi"
|
||||
with pytest.raises(subprocess.CalledProcessError):
|
||||
run_bounded([sys.executable, "-c", "raise SystemExit(3)"], timeout=10)
|
||||
|
||||
|
||||
@pytest.mark.skipif(os.name != "posix", reason="process groups are POSIX-only")
|
||||
def test_run_bounded_timeout_kills_grandchildren(tmp_path: Path) -> None:
|
||||
marker = tmp_path / "grandchild.pid"
|
||||
script = (
|
||||
"import subprocess, sys, time\n"
|
||||
"child = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(60)'])\n"
|
||||
f"open({str(marker)!r}, 'w').write(str(child.pid))\n"
|
||||
"time.sleep(60)\n"
|
||||
)
|
||||
with pytest.raises(subprocess.TimeoutExpired):
|
||||
run_bounded([sys.executable, "-c", script], timeout=2)
|
||||
|
||||
grandchild = int(marker.read_text())
|
||||
deadline = time.monotonic() + 5
|
||||
while time.monotonic() < deadline:
|
||||
if _process_gone_or_zombie(grandchild):
|
||||
return
|
||||
time.sleep(0.05)
|
||||
pytest.fail("grandchild survived the timeout kill")
|
||||
|
||||
|
||||
def _process_gone_or_zombie(pid: int) -> bool:
|
||||
"""A killed orphan stays a zombie until PID 1 reaps it (tini does in production).
|
||||
|
||||
Inside a CI job container PID 1 may not reap, so a zombie counts as killed.
|
||||
"""
|
||||
try:
|
||||
stat = Path(f"/proc/{pid}/stat").read_text()
|
||||
except (FileNotFoundError, ProcessLookupError):
|
||||
return True
|
||||
# Field 3, after the parenthesised command name, is the process state.
|
||||
return stat.rsplit(")", 1)[1].split()[0] in {"Z", "X"}
|
||||
|
||||
|
||||
def test_idle_claim_never_takes_the_write_lock(tmp_path: Path) -> None:
|
||||
store = ScanStore(str(tmp_path / "repopulse.db"))
|
||||
writer = sqlite3.connect(tmp_path / "repopulse.db", timeout=0)
|
||||
try:
|
||||
writer.execute("BEGIN IMMEDIATE")
|
||||
started = time.monotonic()
|
||||
assert store.claim_next_job() is None
|
||||
# Without the read-only probe this would block on busy_timeout (5 s).
|
||||
assert time.monotonic() - started < 2
|
||||
finally:
|
||||
writer.rollback()
|
||||
writer.close()
|
||||
|
||||
|
||||
def test_idle_worker_backs_off_and_enqueue_wakes_it(tmp_path: Path) -> None:
|
||||
runner = ScanJobRunner(ScanStore(str(tmp_path / "repopulse.db")), gitea=GiteaClient())
|
||||
runner.poll_seconds = 0.5
|
||||
runner.idle_poll_seconds = 5.0
|
||||
|
||||
delays = []
|
||||
for iterations in range(1, 7):
|
||||
runner._idle_iterations = iterations
|
||||
delays.append(runner._idle_delay())
|
||||
assert delays == [0.5, 1.0, 2.0, 4.0, 5.0, 5.0]
|
||||
|
||||
runner.wake()
|
||||
started = time.monotonic()
|
||||
assert runner._wake.wait(1)
|
||||
assert time.monotonic() - started < 0.5
|
||||
@@ -221,3 +221,47 @@ def test_empty_git_repository_does_not_fallback_to_untracked_files(tmp_path: Pat
|
||||
assert report.git.tracked_files == 0
|
||||
assert report.metrics.files == 0
|
||||
assert report.metrics.source_files == 0
|
||||
|
||||
|
||||
def test_single_pass_python_analysis_matches_per_function_branch_counts() -> None:
|
||||
import ast
|
||||
|
||||
from app.scanner import PYTHON_BRANCH_TYPES, _analyze_python
|
||||
|
||||
inner = "\n".join(f" if x == {i}:\n x += 1" for i in range(12))
|
||||
outer = "\n".join(f" for i{i} in range(2):\n pass" for i in range(10))
|
||||
source = (
|
||||
"import os\nfrom . import sibling\n\n"
|
||||
f"def outer(x):\n{outer}\n def inner(x):\n{inner}\n return x\n return inner(x)\n"
|
||||
)
|
||||
findings, imports = _analyze_python("pkg/mod.py", source)
|
||||
|
||||
expected = {
|
||||
node.name: sum(1 for child in ast.walk(node) if isinstance(child, PYTHON_BRANCH_TYPES))
|
||||
for node in ast.walk(ast.parse(source))
|
||||
if isinstance(node, ast.FunctionDef)
|
||||
}
|
||||
assert expected == {"outer": 22, "inner": 12}
|
||||
branch_messages = [item.message for item in findings if item.code == "maintainability.branch-heavy-function"]
|
||||
assert branch_messages == ["Function outer contains 22 branch constructs"]
|
||||
assert imports == ("os", "pkg")
|
||||
|
||||
|
||||
def test_js_import_resolution_is_lexical_and_never_escapes_the_root(tmp_path: Path) -> None:
|
||||
from app.architecture import build_architecture_from_imports
|
||||
|
||||
imports = {
|
||||
"web/app.ts": ("./util", "../../outside/secret", "../web/util.js"),
|
||||
"web/util.ts": (),
|
||||
}
|
||||
graph, _ = build_architecture_from_imports(
|
||||
tmp_path,
|
||||
imports,
|
||||
{path: "TypeScript" for path in imports},
|
||||
{path: 1 for path in imports},
|
||||
{path: 0 for path in imports},
|
||||
{},
|
||||
)
|
||||
|
||||
assert {(edge.source, edge.target) for edge in graph.edges} == {("web/app.ts", "web/util.ts")}
|
||||
assert graph.total_edges == 1
|
||||
@@ -16,7 +16,7 @@ def test_non_git_fallback_applies_root_gitignore_rules(tmp_path: Path) -> None:
|
||||
(tmp_path / "ignored-dir").mkdir()
|
||||
(tmp_path / "ignored-dir" / "nested.py").write_text("print('nested')\n", encoding="utf-8")
|
||||
|
||||
files = {path.relative_to(tmp_path).as_posix() for path in _walk_files(tmp_path)}
|
||||
files = {relative for relative, _path in _walk_files(tmp_path)}
|
||||
|
||||
assert "kept.py" in files
|
||||
assert "keep.log" in files
|
||||
|
||||
@@ -159,3 +159,38 @@ def test_catalog_summary_uses_denormalized_fields_without_report_json_parsing(tm
|
||||
assert storage["scan_count"] == 1
|
||||
assert storage["mirror_count"] >= 0
|
||||
assert storage["worktree_count"] >= 0
|
||||
|
||||
|
||||
def test_history_listings_keep_counts_for_compacted_reports_without_report_parsing(
|
||||
tmp_path: Path, monkeypatch
|
||||
) -> None:
|
||||
repository = tmp_path / "repo"
|
||||
repository.mkdir()
|
||||
(repository / "app.py").write_text("print('hello')\n", encoding="utf-8")
|
||||
monkeypatch.setenv("REPOPULSE_FULL_REPORTS_PER_REPOSITORY", "1")
|
||||
store = ScanStore(str(tmp_path / "repopulse.db"))
|
||||
first = store.save(scan_repository(str(repository)))
|
||||
second = store.save(scan_repository(str(repository)))
|
||||
assert store.get(first.id).compacted is True
|
||||
assert first.findings
|
||||
|
||||
def fail_if_called(*_args, **_kwargs):
|
||||
raise AssertionError("history listings must not deserialize full reports")
|
||||
|
||||
monkeypatch.setattr(store, "_report_from_row", fail_if_called)
|
||||
repository_id = second.source.repository_id
|
||||
active = [item for item in first.findings if not item.suppressed]
|
||||
|
||||
scans = store.repository_scans(repository_id)
|
||||
assert [item.id for item in scans] == [second.id, first.id]
|
||||
assert scans[0].findings == len(second.findings)
|
||||
assert scans[0].duration_ms == second.duration_ms
|
||||
assert scans[0].release_readiness == second.release_readiness
|
||||
# A compacted report keeps its retained summary count instead of reporting zero.
|
||||
assert scans[1].findings == len(active)
|
||||
|
||||
trend = store.trend(repository_id)
|
||||
assert [item.scan_id for item in trend] == [first.id, second.id]
|
||||
assert trend[0].findings == len(active)
|
||||
assert trend[0].warnings == sum(1 for item in active if item.severity == "warning")
|
||||
assert trend[1].findings == len(second.findings)
|
||||
Reference in new issue
Block a user