RepoPulse: update backend/app and dependencies (+3 areas)

Public changes compared with the previous published export:
26 files changed.

Affected areas:
- backend/app: 11 files
- dependencies: 5 files
- deployment and CI: 3 files
- documentation: 2 files
- tests: 5 files

Changed files:
- Updated: "Dockerfile.unraid"
- Updated: "README.md"
- Updated: "backend/Dockerfile"
- Updated: "backend/app/acquisition.py"
- Updated: "backend/app/architecture.py"
- Updated: "backend/app/git_analysis.py"
- Updated: "backend/app/gitea.py"
- Updated: "backend/app/jobs.py"
- Updated: "backend/app/main.py"
- Added: "backend/app/processes.py"
- Updated: "backend/app/rules.py"
- Updated: "backend/app/scanner.py"
- Updated: "backend/app/signals.py"
- Updated: "backend/app/store.py"
- Updated: "backend/pyproject.toml"
- Added: "backend/tests/test_api_responsiveness.py"
- Added: "backend/tests/test_processes.py"
- Updated: "backend/tests/test_scanner.py"
- Updated: "backend/tests/test_signals_v11.py"
- Updated: "backend/tests/test_store.py"
- Updated: "docker-compose.yml"
- Updated: "docs/ARCHITECTURE.md"
- Updated: "frontend/package-lock.json"
- Updated: "frontend/package.json"
- Updated: "package-lock.json"
- Updated: "package.json"

Export provenance/manifests refreshed; excluded from the change summary.

Publish curated RepoPulse source from a251f826238908b156485299061da76482780ed2
This commit is contained in:
Public Source Publisher committed 2026-11-17 21:10:30 +00:00
1 parent 23a4a721f1
commit 61ae49f9d0
28 files changed
+1083 -408

No files matched your search

+5 -20
View File
@@ -10,6 +10,7 @@ from threading import Lock, RLock
from urllib.parse import urlsplit
from .models import Repository
from .processes import git_environment, run_bounded
SAFE_REVISION = re.compile(r"^[0-9a-fA-F]{7,64}$")
SAFE_BRANCH = re.compile(r"^[A-Za-z0-9._/-]+$")
@@ -72,35 +73,19 @@ class GitAcquirer:
return self._repository_locks.setdefault(repository_id, RLock())
def _git_env(self) -> dict[str, str]:
env = os.environ.copy()
if self.token:
env.update(
{
"GIT_CONFIG_COUNT": "1",
"GIT_CONFIG_KEY_0": "http.extraHeader",
"GIT_CONFIG_VALUE_0": f"Authorization: token {self.token}",
}
)
return env
extra = [("http.extraHeader", f"Authorization: token {self.token}")] if self.token else []
return git_environment(extra_config=extra)
def _run(self, args: list[str]) -> str:
try:
result = subprocess.run(
args,
check=True,
capture_output=True,
text=True,
errors="replace",
timeout=self.timeout,
env=self._git_env(),
)
result = run_bounded(args, timeout=self.timeout, env=self._git_env())
except subprocess.TimeoutExpired as exc:
raise AcquisitionError("timeout", "Git source acquisition timed out") from exc
except (OSError, subprocess.CalledProcessError) as exc:
# Git stderr may contain remote URLs or server-specific details; do
# not persist or expose it because the job API is user-visible.
raise AcquisitionError("git_failed", "Git source acquisition failed") from exc
return result.stdout.strip()
return result.stdout.decode("utf-8", errors="replace").strip()
def _mirror_path(self, repository: Repository) -> Path:
identity = f"{repository.provider}-{repository.provider_repository_id}"
+138 -68
View File
@@ -4,6 +4,7 @@ import ast
import json
import math
import os
import posixpath
import re
from collections import Counter
from pathlib import Path
@@ -66,8 +67,10 @@ def _ts_path_aliases(root: Path) -> list[tuple[str, str]]:
return aliases[:100]
def _workspace_packages(root: Path) -> dict[str, Path]:
packages: dict[str, Path] = {}
def _workspace_packages(root: Path) -> dict[str, str]:
"""Map local package names to their root-relative POSIX directory."""
packages: dict[str, str] = {}
for manifest in _bounded_config_files(root, {"package.json"}, maximum=100):
try:
if manifest.stat().st_size > 256_000:
@@ -75,7 +78,7 @@ def _workspace_packages(root: Path) -> dict[str, Path]:
data = json.loads(manifest.read_text(encoding="utf-8"))
name = data.get("name") if isinstance(data, dict) else None
if isinstance(name, str) and name and not name.startswith((".", "/")):
packages.setdefault(name, manifest.parent)
packages.setdefault(name, manifest.parent.relative_to(root).as_posix())
except (OSError, ValueError, json.JSONDecodeError, TypeError):
continue
return packages
@@ -116,79 +119,118 @@ def _resolve_python_target(module: str, module_map: dict[str, str]) -> str | Non
return None
def _python_imports(path: str, text: str, module_map: dict[str, str]) -> set[str]:
try:
tree = ast.parse(text)
except SyntaxError:
return set()
def python_package_parts(path: str) -> list[str]:
"""Return the package that relative imports in ``path`` resolve against."""
current_names = _module_names(path)
current = current_names[0] if current_names else ""
current_parts = current.split(".") if current else []
if not path.endswith("/__init__.py") and current_parts:
package_parts = current_parts[:-1]
else:
package_parts = current_parts
return current_parts[:-1]
return current_parts
imports: set[str] = set()
def python_import_modules(node: ast.AST, package_parts: list[str]) -> list[str]:
"""Return the absolute (still unresolved) module names referenced by one import node."""
if isinstance(node, ast.Import):
return [alias.name for alias in node.names]
if isinstance(node, ast.ImportFrom):
module = node.module or ""
if node.level:
trim = max(0, node.level - 1)
prefix = package_parts[:-trim] if trim else package_parts
return [".".join([*prefix, *([module] if module else [])])]
return [module]
return []
def extract_python_imports(path: str, tree: ast.AST) -> tuple[str, ...]:
package_parts = python_package_parts(path)
modules: dict[str, None] = {}
for node in ast.walk(tree):
if isinstance(node, ast.Import):
for alias in node.names:
target = _resolve_python_target(alias.name, module_map)
if target and target != path:
imports.add(target)
elif isinstance(node, ast.ImportFrom):
module = node.module or ""
if node.level:
trim = max(0, node.level - 1)
prefix = package_parts[:-trim] if trim else package_parts
absolute = ".".join([*prefix, *([module] if module else [])])
else:
absolute = module
target = _resolve_python_target(absolute, module_map)
if target and target != path:
imports.add(target)
return imports
if isinstance(node, (ast.Import, ast.ImportFrom)):
for module in python_import_modules(node, package_parts):
modules.setdefault(module)
return tuple(modules)
def extract_js_specifiers(text: str) -> tuple[str, ...]:
return tuple(dict.fromkeys(match.group("target") for match in JS_IMPORT_RE.finditer(text)))
def extract_imports(path: str, text: str, language: str) -> tuple[str, ...]:
"""Extract compact, unresolved import references so source text need not be retained."""
if language == "Python":
try:
tree = ast.parse(text)
except (SyntaxError, ValueError):
return ()
return extract_python_imports(path, tree)
if language in {"TypeScript", "JavaScript"}:
return extract_js_specifiers(text)
return ()
def _lexical_relative(path: str) -> str | None:
"""Normalize a root-relative POSIX path without filesystem access.
Candidates are only matched against already safety-checked scanner paths, so
symlink resolution adds nothing; a path escaping the root yields no target.
"""
if not path or path.startswith("/"):
return None
normalized = posixpath.normpath(path)
if normalized == ".." or normalized.startswith("../"):
return None
return normalized
def _resolve_js_target(
root: Path,
source_path: str,
specifier: str,
known: set[str],
aliases: list[tuple[str, str]],
workspace_packages: dict[str, Path],
workspace_packages: dict[str, str],
) -> str | None:
relative_base: str | None = None
if specifier.startswith("."):
source_dir = (root / source_path).parent
base = (source_dir / specifier).resolve()
relative_base = _lexical_relative(posixpath.join(posixpath.dirname(source_path), specifier))
else:
base = None
matched_alias = False
for alias, target in aliases:
if alias.endswith("/*") and specifier.startswith(alias[:-1]):
base = (root / target.replace("*", specifier[len(alias) - 1 :])).resolve()
relative_base = _lexical_relative(target.replace("*", specifier[len(alias) - 1 :]))
matched_alias = True
break
if alias == specifier:
base = (root / target).resolve()
relative_base = _lexical_relative(target)
matched_alias = True
break
if base is None:
if not matched_alias:
package_name = next(
(name for name in workspace_packages if specifier == name or specifier.startswith(f"{name}/")),
(
name
for name in workspace_packages
if specifier == name or specifier.startswith(f"{name}/")
),
None,
)
if package_name:
package_root = workspace_packages[package_name]
suffix = specifier.removeprefix(package_name).lstrip("/")
base = package_root / suffix if suffix else package_root / "src" / "index"
if base is None:
return None
try:
relative_base = base.relative_to(root.resolve()).as_posix()
except ValueError:
relative_base = _lexical_relative(
posixpath.join(package_root, suffix)
if suffix
else posixpath.join(package_root, "src", "index")
)
if relative_base is None:
return None
candidates = [relative_base]
suffix = Path(relative_base).suffix.lower()
suffix = posixpath.splitext(relative_base)[1].lower()
if suffix in JS_EXTENSIONS:
extensionless = relative_base[: -len(suffix)]
candidates.extend(extensionless + extension for extension in JS_EXTENSIONS)
@@ -199,22 +241,6 @@ def _resolve_js_target(
return next((candidate for candidate in candidates if candidate in known), None)
def _js_imports(
root: Path,
path: str,
text: str,
known: set[str],
aliases: list[tuple[str, str]],
workspace_packages: dict[str, Path],
) -> set[str]:
imports: set[str] = set()
for match in JS_IMPORT_RE.finditer(text):
target = _resolve_js_target(root, path, match.group("target"), known, aliases, workspace_packages)
if target and target != path:
imports.add(target)
return imports
def build_architecture(
root: Path,
source_texts: dict[str, str],
@@ -226,27 +252,71 @@ def build_architecture(
max_nodes: int = 160,
max_edges: int = 600,
) -> tuple[ArchitectureGraph, list[Hotspot]]:
"""Compatibility entry point that accepts full source texts."""
imports_by_path = {
path: extract_imports(path, text, language_by_path.get(path, ""))
for path, text in source_texts.items()
}
return build_architecture_from_imports(
root,
imports_by_path,
language_by_path,
loc_by_path,
debt_by_path,
churn_by_path,
max_nodes=max_nodes,
max_edges=max_edges,
)
def build_architecture_from_imports(
root: Path,
imports_by_path: dict[str, tuple[str, ...]],
language_by_path: dict[str, str],
loc_by_path: dict[str, int],
debt_by_path: dict[str, int],
churn_by_path: dict[str, int],
*,
max_nodes: int = 160,
max_edges: int = 600,
) -> tuple[ArchitectureGraph, list[Hotspot]]:
"""Build the bounded graph from compact import references; no source text is retained."""
max_nodes = max(1, max_nodes)
max_edges = max(0, max_edges)
known_paths = set(source_texts)
known_paths = set(imports_by_path)
module_map = _python_module_map(list(known_paths))
ts_aliases = _ts_path_aliases(root)
workspace_packages = _workspace_packages(root)
js_cache: dict[tuple[str, str], str | None] = {}
edge_pairs: set[tuple[str, str]] = set()
total_inbound: Counter[str] = Counter()
total_outbound: Counter[str] = Counter()
total_edges = 0
truncated = len(known_paths) > max_nodes
for path in sorted(source_texts):
text = source_texts[path]
for path in sorted(imports_by_path):
references = imports_by_path[path]
language = language_by_path.get(path, "")
targets: set[str] = set()
if language == "Python":
targets = _python_imports(path, text, module_map)
for module in references:
target = _resolve_python_target(module, module_map)
if target and target != path:
targets.add(target)
elif language in {"TypeScript", "JavaScript"}:
targets = _js_imports(root, path, text, known_paths, ts_aliases, workspace_packages)
else:
targets = set()
directory = posixpath.dirname(path)
for specifier in references:
# Only relative specifiers depend on the importing directory.
key = (directory if specifier.startswith(".") else "", specifier)
if key not in js_cache:
js_cache[key] = _resolve_js_target(
path, specifier, known_paths, ts_aliases, workspace_packages
)
target = js_cache[key]
if target and target != path:
targets.add(target)
for target in sorted(targets):
if target not in known_paths:
continue
+4 -10
View File
@@ -6,21 +6,15 @@ from datetime import datetime
from pathlib import Path
from .models import GitSignals
from .processes import git_environment, run_bounded
def _run(root: Path, *args: str, timeout: int = 8) -> str | None:
try:
result = subprocess.run(
["git", "-C", str(root), *args],
check=True,
capture_output=True,
text=True,
errors="replace",
timeout=timeout,
)
except (OSError, UnicodeError, subprocess.SubprocessError):
result = run_bounded(["git", "-C", str(root), *args], timeout=timeout, env=git_environment())
except (OSError, subprocess.SubprocessError):
return None
return result.stdout
return result.stdout.decode("utf-8", errors="replace")
def analyze_git(root: Path) -> tuple[GitSignals, dict[str, int]]:
+2 -2
View File
@@ -238,7 +238,7 @@ class GiteaClient:
listed, discovered, branch_errors = self.discover_with_errors()
except GiteaError as exc:
error = f"{exc.code}: {exc.detail}"
for existing in store.list_repositories(limit=1000):
for existing in store.list_repositories(limit=-1):
if existing.provider == "gitea" and existing.available:
store.update_repository_sync(existing, synced_at=synced_at, error=error)
result = RepositorySyncResult(
@@ -290,7 +290,7 @@ class GiteaClient:
updated += 1
unavailable = 0
if self._listing_complete:
for existing in store.list_repositories(limit=1000):
for existing in store.list_repositories(limit=-1):
if existing.provider == "gitea" and existing.id not in listed_ids and existing.available:
store.update_repository_sync(
existing.model_copy(update={"available": False}),
+82 -7
View File
@@ -1,7 +1,12 @@
from __future__ import annotations
import ctypes
import ctypes.util
import gc
import logging
import os
import sys
from collections.abc import Callable
from datetime import UTC, datetime
from threading import Event, Lock, Thread, current_thread
from time import monotonic
@@ -21,6 +26,40 @@ from .scanner import scan_repository
from .store import MaintenanceModeError, QueueFullError, ScanStore, job_lease_seconds
LOGGER = logging.getLogger(__name__)
WORKER_HEARTBEAT_SECONDS = 5.0
WORKER_IDLE_POLL_SECONDS = 5.0
def _load_malloc_trim() -> Callable[[int], int] | None:
if not sys.platform.startswith("linux"):
return None
try:
libc = ctypes.CDLL(ctypes.util.find_library("c") or "libc.so.6")
trim = libc.malloc_trim
except (AttributeError, OSError):
return None # e.g. musl libc has no malloc_trim
trim.argtypes = [ctypes.c_size_t]
trim.restype = ctypes.c_int
return trim
_MALLOC_TRIM = _load_malloc_trim()
def release_memory() -> None:
"""Return memory freed by a finished scan to the OS.
A scan allocates many short-lived objects (ASTs, file texts, findings). CPython frees
them, but glibc keeps the arenas mapped, so the long-running worker's RSS would stay
at its largest scan. A collection plus malloc_trim hands those pages back.
"""
gc.collect()
if _MALLOC_TRIM is not None:
try:
_MALLOC_TRIM(0)
except OSError:
pass
def _positive_int(name: str, default: int, *, maximum: int | None = None) -> int:
@@ -60,8 +99,17 @@ class ScanJobRunner:
)
self.max_queue = max_queue or _positive_int("REPOPULSE_MAX_QUEUED_SCANS", 100, maximum=1000)
self._stop = Event()
self._wake = Event()
self._idle_iterations = 0
self.idle_poll_seconds = max(
self.poll_seconds, _positive_float("REPOPULSE_WORKER_IDLE_POLL_SECONDS", WORKER_IDLE_POLL_SECONDS)
)
self._thread: Thread | None = None
self._last_sync = float("-inf")
self._last_heartbeat = float("-inf")
self.heartbeat_interval_seconds = _positive_float(
"REPOPULSE_WORKER_HEARTBEAT_SECONDS", WORKER_HEARTBEAT_SECONDS
)
self._sync_lock = Lock()
@property
@@ -72,11 +120,13 @@ class ScanJobRunner:
if self.started:
return
self._stop.clear()
self._wake.clear()
self._thread = Thread(target=self._run, name="repopulse-scan-worker", daemon=True)
self._thread.start()
def stop(self) -> None:
self._stop.set()
self._wake.set()
thread = self._thread
if thread and thread is not current_thread():
thread.join(timeout=5)
@@ -94,19 +144,39 @@ class ScanJobRunner:
LOGGER.exception("RepoPulse worker loop recovered from an unexpected failure")
job = None
if job is None:
self._stop.wait(self.poll_seconds)
self._idle_iterations += 1
self._wake.wait(self._idle_delay())
self._wake.clear()
else:
self._idle_iterations = 0
def _idle_delay(self) -> float:
"""Back off exponentially while idle; in-process enqueues wake the loop at once."""
exponent = min(self._idle_iterations - 1, 16)
return min(self.idle_poll_seconds, self.poll_seconds * (2**exponent))
def wake(self) -> None:
self._wake.set()
def _run_iteration(self) -> ScanJob | None:
"""Perform one scheduler pass unless durable maintenance has paused work."""
if self.store.maintenance_status()["enabled"]:
return None
if not self.store.touch_worker_heartbeat_if_active(datetime.now(UTC).isoformat()):
return None
# The heartbeat is a liveness signal, not a lock: claims and enqueues re-check
# maintenance inside their own transactions, so writing it on every 0.5 s poll
# only churns the WAL. Throttle the write to the heartbeat interval.
now = monotonic()
if now - self._last_heartbeat >= self.heartbeat_interval_seconds:
if not self.store.touch_worker_heartbeat_if_active(datetime.now(UTC).isoformat()):
return None
self._last_heartbeat = now
if self.gitea.configured and monotonic() - self._last_sync >= self.sync_interval_seconds:
self.sync_now()
if self.store.maintenance_status()["enabled"]:
return None
# Synchronization can take a while; re-check before claiming new work.
if self.store.maintenance_status()["enabled"]:
return None
return self.process_one()
def sync_now(self) -> RepositorySyncResult:
@@ -135,7 +205,8 @@ class ScanJobRunner:
def _schedule_changed_repositories(self) -> None:
if self.store.maintenance_status()["enabled"]:
return
for summary in self.store.list_repositories(limit=1000):
# No silent catalog cap: every repository is reconciled, however large the catalog.
for summary in self.store.list_repositories(limit=-1):
if summary.provider != "gitea" or summary.archived or not summary.latest_revision:
continue
if not summary.policy.auto_scan:
@@ -166,13 +237,15 @@ class ScanJobRunner:
trigger: ScanTrigger = ScanTrigger.MANUAL,
) -> ScanJob:
requested_revision = repository.latest_revision
return self.store.enqueue_job(
job = self.store.enqueue_job(
repository.id,
requested_revision,
trigger,
force=force,
max_queue=self.max_queue,
)
self.wake()
return job
def process_one(self) -> ScanJob | None:
job = self.store.claim_next_job()
@@ -194,6 +267,8 @@ class ScanJobRunner:
)
except Exception:
LOGGER.exception("RepoPulse could not persist an unexpected job failure")
finally:
release_memory()
return self.store.get_job(job.id)
class _LeaseHeartbeat:
+29 -6
View File
@@ -11,8 +11,10 @@ from threading import BoundedSemaphore, Lock
from fastapi import FastAPI, HTTPException, Query, Request, Response, status
from fastapi.concurrency import run_in_threadpool
from fastapi.middleware.gzip import GZipMiddleware
from fastapi.responses import JSONResponse
from fastapi.staticfiles import StaticFiles
from starlette.types import Scope
from .acquisition import GitAcquirer
from .gitea import GiteaClient
@@ -35,7 +37,7 @@ from .reporting import content_disposition, render_html_report, safe_report_file
from .scanner import scan_repository
from .store import MaintenanceModeError, QueueFullError, ScanStore
VERSION = "1.5.0"
VERSION = "1.5.1"
store = ScanStore()
SCAN_ROOT = os.getenv("REPOPULSE_SCAN_ROOT")
@@ -82,6 +84,8 @@ async def lifespan(_app: FastAPI):
app = FastAPI(title="RepoPulse API", version=VERSION, lifespan=lifespan)
# Catalog, report and bundle payloads are large, repetitive JSON/JS; compress them.
app.add_middleware(GZipMiddleware, minimum_size=1024, compresslevel=5)
@app.middleware("http")
@@ -167,7 +171,7 @@ def meta() -> dict[str, object]:
return {
"name": "RepoPulse",
"version": VERSION,
"stage": "v1.5.0",
"stage": "v1.5.1",
"scan_root_restricted": bool(SCAN_ROOT),
"scan_root": SCAN_ROOT,
"max_concurrent_scans": MAX_CONCURRENT_SCANS,
@@ -242,7 +246,8 @@ async def create_scan(payload: ScanRequest, request: Request) -> ScanReport:
raise HTTPException(status_code=422, detail="Repository path cannot be scanned") from exc
finally:
SCAN_SLOTS.release()
return store.save(report)
# SQLite writes may wait on busy_timeout; never block the event loop with them.
return await asyncio.to_thread(store.save, report)
@app.get("/api/v1/repositories", response_model=list[RepositorySummary])
@@ -296,7 +301,7 @@ def repository_trend(
@app.post("/api/v1/repositories/{repository_id}/scans", response_model=ScanJob, status_code=202)
async def create_repository_scan(
def create_repository_scan(
request: Request,
repository_id: str,
payload: RepositoryScanRequest | None = None,
@@ -357,6 +362,12 @@ async def gitea_webhook(request: Request) -> WebhookResponse:
raise HTTPException(status_code=400, detail="Webhook payload is invalid JSON") from exc
if not isinstance(payload, dict):
raise HTTPException(status_code=400, detail="Webhook payload must be an object")
event = request.headers.get("x-gitea-event", "push").casefold()
# Catalog reads and the enqueue transaction are blocking SQLite work.
return await run_in_threadpool(_handle_gitea_webhook, payload, event)
def _handle_gitea_webhook(payload: dict, event: str) -> WebhookResponse:
repository_payload = payload.get("repository")
if not isinstance(repository_payload, dict):
raise HTTPException(status_code=400, detail="Webhook repository is missing")
@@ -368,7 +379,6 @@ async def gitea_webhook(request: Request) -> WebhookResponse:
accepted=False,
message="Repository is not present in the synchronized catalog",
)
event = request.headers.get("x-gitea-event", "push").casefold()
if event != "push":
return WebhookResponse(
accepted=True,
@@ -486,6 +496,19 @@ def export_scan(scan_id: str, format: str = Query(default="html", pattern="^(htm
)
class _FrontendStaticFiles(StaticFiles):
"""Serve the SPA; content-hashed build assets are immutable and cached long-term."""
async def get_response(self, path: str, scope: Scope) -> Response:
response = await super().get_response(path, scope)
if response.status_code == 200:
if path.replace("\\", "/").startswith("assets/"):
response.headers["Cache-Control"] = "public, max-age=31536000, immutable"
else:
response.headers["Cache-Control"] = "no-cache"
return response
static_dir = Path(os.getenv("REPOPULSE_STATIC_DIR", "/app/static"))
if static_dir.is_dir():
app.mount("/", StaticFiles(directory=static_dir, html=True), name="frontend")
app.mount("/", _FrontendStaticFiles(directory=static_dir, html=True), name="frontend")
+77
View File
@@ -0,0 +1,77 @@
"""Bounded child-process execution that never leaves orphaned descendants behind."""
from __future__ import annotations
import os
import signal
import subprocess
from collections.abc import Mapping, Sequence
# Git may daemonize automatic housekeeping (``gc --auto``/``maintenance --auto``) after a
# fetch. A detached child escapes the timeout and is reparented to PID 1, where it becomes
# a zombie if PID 1 does not reap. Housekeeping therefore runs in the foreground.
GIT_FOREGROUND_CONFIG: tuple[tuple[str, str], ...] = (
("gc.autoDetach", "false"),
("maintenance.autoDetach", "false"),
)
def git_environment(
base: Mapping[str, str] | None = None,
extra_config: Sequence[tuple[str, str]] = (),
) -> dict[str, str]:
"""Return an environment that applies foreground housekeeping plus ``extra_config``."""
env = dict(os.environ if base is None else base)
pairs = [*GIT_FOREGROUND_CONFIG, *extra_config]
env["GIT_CONFIG_COUNT"] = str(len(pairs))
for index, (key, value) in enumerate(pairs):
env[f"GIT_CONFIG_KEY_{index}"] = key
env[f"GIT_CONFIG_VALUE_{index}"] = value
return env
def _kill_process_tree(process: subprocess.Popen[bytes]) -> None:
if os.name == "posix":
try:
os.killpg(process.pid, signal.SIGKILL)
except (ProcessLookupError, PermissionError):
pass
else:
process.kill()
def run_bounded(
args: Sequence[str],
*,
timeout: float,
env: Mapping[str, str] | None = None,
check: bool = True,
) -> subprocess.CompletedProcess[bytes]:
"""Run ``args`` in its own process group and reap it and all descendants.
On timeout the whole group is killed (not only the direct child), then the child is
waited for so no zombie remains. Raises the same exceptions as ``subprocess.run``.
"""
process = subprocess.Popen(
list(args),
stdin=subprocess.DEVNULL,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=None if env is None else dict(env),
start_new_session=os.name == "posix",
)
try:
stdout, stderr = process.communicate(timeout=timeout)
except subprocess.TimeoutExpired:
_kill_process_tree(process)
process.communicate()
raise
except BaseException:
_kill_process_tree(process)
process.wait()
raise
if check and process.returncode:
raise subprocess.CalledProcessError(process.returncode, list(args), stdout, stderr)
return subprocess.CompletedProcess(list(args), process.returncode, stdout, stderr)
+18 -6
View File
@@ -1,3 +1,5 @@
from functools import lru_cache
from .models import Finding, FindingGuidance, Severity
_RULES: dict[str, tuple[str, str, str, str]] = {
@@ -63,12 +65,16 @@ def _default_rule(finding: Finding) -> tuple[str, str, str, str]:
)
def enrich_finding(finding: Finding) -> Finding:
remediation_class, why, action, confidence = _RULES.get(finding.code, _default_rule(finding))
severity_weight = {Severity.BLOCKER: 700, Severity.WARNING: 400, Severity.INFO: 100}[finding.severity]
priority = min(1000, severity_weight + finding.deduction * 5)
rationale = f"{finding.severity.value.title()} signal; {finding.deduction} health-point impact."
guidance = FindingGuidance(
@lru_cache(maxsize=4096)
def _guidance(code: str, severity: Severity, deduction: int) -> FindingGuidance:
"""Guidance depends only on code, severity and deduction; share one immutable instance."""
probe = Finding(code=code, category="", severity=severity, message="", deduction=deduction)
remediation_class, why, action, confidence = _RULES.get(code, _default_rule(probe))
severity_weight = {Severity.BLOCKER: 700, Severity.WARNING: 400, Severity.INFO: 100}[severity]
priority = min(1000, severity_weight + deduction * 5)
rationale = f"{severity.value.title()} signal; {deduction} health-point impact."
return FindingGuidance(
remediation_class=remediation_class,
confidence=confidence,
why=why,
@@ -76,6 +82,12 @@ def enrich_finding(finding: Finding) -> Finding:
priority=priority,
priority_rationale=rationale,
)
def enrich_finding(finding: Finding) -> Finding:
guidance = _guidance(finding.code, finding.severity, finding.deduction)
if finding.guidance is guidance:
return finding
return finding.model_copy(update={"guidance": guidance})
+140 -69
View File
@@ -5,13 +5,18 @@ import fnmatch
import os
import re
import subprocess
from collections import Counter
from collections import Counter, deque
from datetime import UTC, datetime
from pathlib import Path
from time import perf_counter
from uuid import uuid4
from .architecture import build_architecture
from .architecture import (
build_architecture_from_imports,
extract_js_specifiers,
python_import_modules,
python_package_parts,
)
from .git_analysis import analyze_git
from .models import (
Finding,
@@ -25,9 +30,10 @@ from .models import (
finding_fingerprint,
)
from .policy import load_policy
from .processes import git_environment, run_bounded
from .release_gate import build_release_gate
from .rules import enrich_findings
from .signals import analyze_dependencies, detect_test_signals
from .signals import analyze_dependencies, detect_test_signals, is_signal_text_path
LANGUAGES = {
".py": "Python",
@@ -224,11 +230,8 @@ class _GitIgnoreMatcher:
def _git_worktree_root(root: Path) -> Path | None:
try:
result = subprocess.run(
["git", "-C", str(root), "rev-parse", "--show-toplevel"],
check=True,
capture_output=True,
timeout=5,
result = run_bounded(
["git", "-C", str(root), "rev-parse", "--show-toplevel"], timeout=5, env=git_environment()
)
except (OSError, subprocess.SubprocessError):
return None
@@ -238,42 +241,61 @@ def _git_worktree_root(root: Path) -> Path | None:
return None
class _SafePathChecker:
"""Validate candidate files below one root, caching directories already proven safe.
Every path component is still checked for symlinks, but a directory shared by many
files is inspected once instead of once per file.
"""
def __init__(self, root: Path) -> None:
self.root = root
self.root_resolved = root.resolve()
self._safe_dirs: set[Path] = set()
def check(self, candidate: Path) -> Path | None:
"""Return a resolved regular file only when every path component is safe."""
try:
relative = candidate.relative_to(self.root)
if _ignored(str(relative)):
return None
current = self.root
parts = relative.parts
for index, part in enumerate(parts):
current /= part
is_last = index == len(parts) - 1
if not is_last and current in self._safe_dirs:
continue
if current.is_symlink():
return None
if not is_last:
self._safe_dirs.add(current)
resolved = candidate.resolve(strict=True)
root_resolved = self.root_resolved
if resolved == root_resolved or root_resolved not in resolved.parents:
return None
if not resolved.is_file():
return None
except (OSError, RuntimeError, ValueError):
return None
return resolved
def _safe_file(root: Path, candidate: Path) -> Path | None:
"""Return a resolved regular file only when every path component is safe."""
try:
relative = candidate.relative_to(root)
if _ignored(str(relative)):
return None
current = root
for part in Path(relative).parts:
current /= part
if current.is_symlink():
return None
resolved = candidate.resolve(strict=True)
root_resolved = root.resolve()
if resolved == root_resolved or root_resolved not in resolved.parents:
return None
if not resolved.is_file():
return None
except (OSError, RuntimeError, ValueError):
return None
return resolved
return _SafePathChecker(root).check(candidate)
def _tracked_files(root: Path) -> list[Path] | None:
def _tracked_files(root: Path) -> list[tuple[str, Path]] | None:
if _git_worktree_root(root) != root:
return None
try:
result = subprocess.run(
["git", "-C", str(root), "ls-files", "-z"],
check=True,
capture_output=True,
timeout=5,
)
result = run_bounded(["git", "-C", str(root), "ls-files", "-z"], timeout=5, env=git_environment())
except (OSError, subprocess.SubprocessError):
return None
paths: list[Path] = []
checker = _SafePathChecker(root)
paths: list[tuple[str, Path]] = []
for raw in result.stdout.split(b"\0"):
if not raw:
continue
@@ -281,14 +303,15 @@ def _tracked_files(root: Path) -> list[Path] | None:
relative_path = Path(relative)
if relative_path.is_absolute() or _ignored(relative):
continue
safe = _safe_file(root, root / relative_path)
safe = checker.check(root / relative_path)
if safe is not None:
paths.append(safe)
paths.append((relative_path.as_posix(), safe))
return paths
def _walk_files(root: Path) -> list[Path]:
files: list[Path] = []
def _walk_files(root: Path) -> list[tuple[str, Path]]:
files: list[tuple[str, Path]] = []
checker = _SafePathChecker(root)
gitignore = _GitIgnoreMatcher(root)
for current, dirs, names in os.walk(root, followlinks=False):
current_path = Path(current)
@@ -312,13 +335,14 @@ def _walk_files(root: Path) -> list[Path]:
continue
if gitignore.is_ignored(relative, is_dir=False):
continue
safe = _safe_file(root, candidate)
safe = checker.check(candidate)
if safe is not None:
files.append(safe)
files.append((relative.replace("\\", "/"), safe))
return files
def _read_text(path: Path, max_bytes: int = MAX_TEXT_BYTES) -> str | None:
def _read_text_bytes(path: Path, max_bytes: int = MAX_TEXT_BYTES) -> tuple[str, int] | None:
"""Return decoded text plus the number of bytes read, or None for binary/oversized files."""
try:
if path.stat().st_size > max_bytes:
return None
@@ -327,7 +351,12 @@ def _read_text(path: Path, max_bytes: int = MAX_TEXT_BYTES) -> str | None:
return None
if b"\0" in raw[:8192]:
return None
return raw.decode("utf-8", errors="replace")
return raw.decode("utf-8", errors="replace"), len(raw)
def _read_text(path: Path, max_bytes: int = MAX_TEXT_BYTES) -> str | None:
result = _read_text_bytes(path, max_bytes)
return result[0] if result else None
def _is_test_file(relative: str) -> bool:
@@ -343,19 +372,45 @@ def _is_test_file(relative: str) -> bool:
)
def _python_function_findings(relative: str, text: str) -> list[Finding]:
PYTHON_BRANCH_TYPES = (ast.If, ast.For, ast.AsyncFor, ast.While, ast.Try, ast.Match, ast.BoolOp)
def _analyze_python(relative: str, text: str) -> tuple[list[Finding], tuple[str, ...]]:
"""Parse one Python file once and return function findings plus unresolved imports.
A single breadth-first walk (the same order as ``ast.walk``) attributes each branch
construct to every enclosing function, matching a per-function ``ast.walk`` count
without re-walking nested subtrees.
"""
try:
tree = ast.parse(text)
except SyntaxError:
return []
except (SyntaxError, ValueError):
return [], ()
package_parts = python_package_parts(relative)
imports: dict[str, None] = {}
functions: list[ast.FunctionDef | ast.AsyncFunctionDef] = []
branch_counts: list[int] = []
pending: deque[tuple[ast.AST, tuple[int, ...]]] = deque([(tree, ())])
while pending:
node, enclosing = pending.popleft()
if isinstance(node, PYTHON_BRANCH_TYPES):
for index in enclosing:
branch_counts[index] += 1
elif isinstance(node, (ast.Import, ast.ImportFrom)):
for module in python_import_modules(node, package_parts):
imports.setdefault(module)
child_enclosing = enclosing
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
functions.append(node)
branch_counts.append(0)
child_enclosing = (*enclosing, len(functions) - 1)
pending.extend((child, child_enclosing) for child in ast.iter_child_nodes(node))
del tree, pending
findings: list[Finding] = []
branch_types = (ast.If, ast.For, ast.AsyncFor, ast.While, ast.Try, ast.Match, ast.BoolOp)
for node in ast.walk(tree):
if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
continue
for node, branches in zip(functions, branch_counts, strict=True):
end_line = getattr(node, "end_lineno", node.lineno)
length = max(1, end_line - node.lineno + 1)
branches = sum(1 for child in ast.walk(node) if isinstance(child, branch_types))
if length >= 140:
findings.append(
Finding(
@@ -380,7 +435,11 @@ def _python_function_findings(relative: str, text: str) -> list[Finding]:
deduction=4,
)
)
return findings[:12]
return findings[:12], tuple(imports)
def _python_function_findings(relative: str, text: str) -> list[Finding]:
return _analyze_python(relative, text)[0]
def _is_placeholder_secret(match: re.Match[str]) -> bool:
@@ -408,7 +467,7 @@ def scan_repository(
limits = limits or scan_limits_from_environment()
tracked_files = _tracked_files(root)
all_files = _walk_files(root) if tracked_files is None else tracked_files
all_files.sort(key=lambda item: str(item.relative_to(root)).replace("\\", "/").casefold())
all_files.sort(key=lambda item: item[0].casefold())
files = all_files[: limits.max_files]
coverage = ScanCoverage(
limits=limits,
@@ -426,8 +485,11 @@ def scan_repository(
metrics = RepositoryMetrics(files=len(all_files))
files_by_name: dict[str, list[Path]] = {}
root_entries = {entry.name.lower() for entry in root.iterdir()}
text_by_path: dict[str, str] = {}
source_texts: dict[str, str] = {}
# Only small derived values are retained per file. Full text is held for a single
# file at a time, except the few CI/manifest texts that test-signal detection needs.
signal_texts: dict[str, str] = {}
readable_paths: list[str] = []
imports_by_path: dict[str, tuple[str, ...]] = {}
language_by_path: dict[str, str] = {}
loc_by_path: dict[str, int] = {}
debt_by_path: dict[str, int] = {}
@@ -435,11 +497,10 @@ def scan_repository(
secret_findings_truncated = False
text_budget_exhausted = False
for path_obj in files:
for relative, path_obj in files:
try:
relative = str(path_obj.relative_to(root)).replace("\\", "/")
size = path_obj.stat().st_size
except (OSError, ValueError):
except OSError:
continue
files_by_name.setdefault(path_obj.name, []).append(path_obj)
@@ -479,21 +540,28 @@ def scan_repository(
coverage.skipped_files_by_text_budget += 1
text_budget_exhausted = True
continue
text = _read_text(path_obj, limits.max_text_bytes_per_file)
if text is None:
read = _read_text_bytes(path_obj, limits.max_text_bytes_per_file)
if read is None:
continue
text, byte_count = read
coverage.scanned_files += 1
coverage.text_bytes_scanned += len(text.encode("utf-8", errors="replace"))
text_by_path[relative] = text
coverage.text_bytes_scanned += byte_count
readable_paths.append(relative)
if is_signal_text_path(relative):
signal_texts[relative] = text
lines = text.splitlines()
if language:
loc = sum(1 for line in text.splitlines() if line.strip())
loc = sum(1 for line in lines if line.strip())
metrics.source_files += 1
metrics.lines_of_code += loc
languages[language] += loc
source_texts[relative] = text
language_by_path[relative] = language
loc_by_path[relative] = loc
if language in {"TypeScript", "JavaScript"}:
imports_by_path[relative] = extract_js_specifiers(text)
elif language != "Python":
imports_by_path[relative] = ()
if loc > 1000:
metrics.large_source_files += 1
findings.append(
@@ -507,7 +575,8 @@ def scan_repository(
)
)
if language == "Python":
findings.extend(_python_function_findings(relative, text))
python_findings, imports_by_path[relative] = _analyze_python(relative, text)
findings.extend(python_findings)
if _is_test_file(relative):
metrics.test_files += 1
@@ -520,7 +589,7 @@ def scan_repository(
debt_by_path[relative] = sum(debt.values())
if _is_secret_scan_candidate(path_obj, relative, language) and secret_findings < limits.max_secret_findings:
for line_number, line in enumerate(text.splitlines(), start=1):
for line_number, line in enumerate(lines, start=1):
match = SECRET_PATTERN.search(line)
if match is None or _is_placeholder_secret(match):
continue
@@ -567,11 +636,13 @@ def scan_repository(
findings.append(Finding(code="hygiene.missing-env-example", category="Repository hygiene", severity=Severity.INFO, message="Environment configuration exists without a root .env.example", deduction=3))
test_signals = detect_test_signals(
[str(path.relative_to(root)).replace("\\", "/") for path in files],
[relative for relative, _path in files],
metrics.source_files,
metrics.test_files,
text_by_path,
signal_texts,
readable_paths,
)
del signal_texts, readable_paths
if metrics.source_files and metrics.test_files == 0:
findings.append(Finding(code="testing.none-detected", category="Testing", severity=Severity.WARNING, message="No test files detected", deduction=30))
if not test_signals.ci_workflows:
@@ -593,9 +664,9 @@ def scan_repository(
findings.extend(dependency_findings)
git_signals, churn_by_path = analyze_git(root)
architecture, hotspots = build_architecture(
architecture, hotspots = build_architecture_from_imports(
root,
source_texts,
imports_by_path,
language_by_path,
loc_by_path,
debt_by_path,
+34 -9
View File
@@ -5,6 +5,7 @@ import re
import tomllib
import xml.etree.ElementTree as ET
from collections import defaultdict
from collections.abc import Iterable
from pathlib import Path
from .models import DependencySignals, Finding, Severity, TestSignals
@@ -32,6 +33,24 @@ COVERAGE_NAMES = {
COVERAGE_NAMES_CASEFOLD = {item.casefold() for item in COVERAGE_NAMES}
CI_WORKFLOW_PREFIXES = (".github/workflows/", ".gitea/workflows/")
CI_WORKFLOW_NAMES = {".gitlab-ci.yml", "jenkinsfile", "azure-pipelines.yml"}
SIGNAL_TEXT_NAMES = {"package.json", "pyproject.toml"}
def is_ci_workflow_path(relative: str) -> bool:
normalized = relative.replace("\\", "/")
return normalized.lower().startswith(CI_WORKFLOW_PREFIXES) or (
Path(normalized).name.casefold() in CI_WORKFLOW_NAMES
)
def is_signal_text_path(relative: str) -> bool:
"""Return True when detect_test_signals needs this file's text content."""
return is_ci_workflow_path(relative) or Path(relative).name.casefold() in SIGNAL_TEXT_NAMES
def _python_dependency_name(declaration: str) -> str:
return re.split(r"[<>=!~ @;\[]", declaration, maxsplit=1)[0].strip()
@@ -69,7 +88,14 @@ def detect_test_signals(
source_files: int,
test_files: int,
text_by_path: dict[str, str],
readable_paths: Iterable[str] | None = None,
) -> TestSignals:
"""Derive test/CI signals.
``text_by_path`` only needs the texts accepted by :func:`is_signal_text_path`;
``readable_paths`` lists every successfully read text file (defaults to the
``text_by_path`` keys) so coverage evidence does not require retaining text.
"""
frameworks: set[str] = set()
coverage: list[str] = []
ci: list[str] = []
@@ -85,11 +111,7 @@ def detect_test_signals(
lower = normalized.lower()
if name_lower in COVERAGE_NAMES_CASEFOLD or "/coverage/" in f"/{lower}":
coverage.append(normalized)
if lower.startswith((".github/workflows/", ".gitea/workflows/")) or name_lower in {
".gitlab-ci.yml",
"jenkinsfile",
"azure-pipelines.yml",
}:
if is_ci_workflow_path(normalized):
ci.append(normalized)
if (name_lower.startswith("test_") or "/tests/" in f"/{lower}") and name_lower.endswith(".py"):
@@ -103,11 +125,16 @@ def detect_test_signals(
if name_lower.endswith(("tests.cs", "test.cs")):
frameworks.add(".NET test")
ci_paths = set(ci)
for path in text_by_path if readable_paths is None else readable_paths:
if Path(path).name.casefold() in COVERAGE_NAMES_CASEFOLD:
coverage_evidence.append(path.replace("\\", "/"))
for path, text in text_by_path.items():
name = Path(path).name
normalized = path.replace("\\", "/")
lowered_text = text.casefold()
if normalized in ci:
if normalized in ci_paths:
lowered_text = text.casefold()
validates = any(
token in lowered_text
for token in ("pytest", "vitest", "jest", "npm test", "pnpm test", "go test", "dotnet test", "ruff", "eslint", "typecheck", "py_compile")
@@ -125,8 +152,6 @@ def detect_test_signals(
ci_deploy_only.append(normalized)
if any(token in lowered_text for token in ("--cov", "coverage", "lcov", "codecov")):
coverage_evidence.append(normalized)
if name.casefold() in COVERAGE_NAMES_CASEFOLD:
coverage_evidence.append(normalized)
if name.casefold() == "package.json":
try:
data = json.loads(text)
+183 -154
View File
@@ -9,6 +9,8 @@ from contextlib import contextmanager
from datetime import UTC, datetime, timedelta
from hashlib import sha256
from pathlib import Path
from threading import Lock, Thread
from time import monotonic
from uuid import uuid4
from .models import (
@@ -35,6 +37,7 @@ from .rules import enrich_findings
MAX_HISTORY_PER_REPOSITORY = 200
SCHEMA_VERSION = 4
MIRROR_STATS_TTL_SECONDS = 300.0
JOB_LEASE_SECONDS = 900
ACTIVE_JOB_STATUSES = (
ScanJobStatus.QUEUED.value,
@@ -108,6 +111,8 @@ class ScanStore:
self.path = Path(raw_path)
self.path.parent.mkdir(parents=True, exist_ok=True)
self.recovery_metadata: dict[str, str] | None = None
self._mirror_stats_cache: tuple[float, str, tuple[int, int]] | None = None
self._mirror_stats_refresh = Lock()
try:
self._validate_existing_database()
self._init_schema()
@@ -349,7 +354,7 @@ class ScanStore:
def _backfill_legacy_catalog(self, db: sqlite3.Connection) -> None:
rows = db.execute(
"""
SELECT id, repository_path, repository_name, report_json, repository_id, history_key
SELECT id, repository_path, repository_name, repository_id, history_key
FROM scans
WHERE repository_id IS NULL OR history_key IS NULL
"""
@@ -412,11 +417,19 @@ class ScanStore:
}
def _backfill_scan_summaries(self, db: sqlite3.Connection) -> None:
rows = db.execute(
"SELECT rowid AS scan_rowid, * FROM scans "
"WHERE finding_count IS NULL OR blocker_count IS NULL OR release_readiness IS NULL"
).fetchall()
for row in rows:
# Collect ids first and load one report at a time so a large legacy history is
# never materialized in memory at once during migration.
scan_ids = [
str(item[0])
for item in db.execute(
"SELECT id FROM scans "
"WHERE finding_count IS NULL OR blocker_count IS NULL OR release_readiness IS NULL"
).fetchall()
]
for scan_id in scan_ids:
row = db.execute("SELECT rowid AS scan_rowid, * FROM scans WHERE id = ?", (scan_id,)).fetchone()
if row is None:
continue
report = self._report_from_row(row)
previous_row = db.execute(
"""
@@ -560,112 +573,6 @@ class ScanStore:
row = db.execute("SELECT source_path FROM repositories WHERE id = ?", (repository_id,)).fetchone()
return row["source_path"] if row else None
def _repository_summary(self, db: sqlite3.Connection, row: sqlite3.Row) -> RepositorySummary:
repository = self._repository_from_row(row)
latest = db.execute(
"""
SELECT * FROM scans WHERE repository_id = ?
ORDER BY scanned_at DESC, rowid DESC LIMIT 1
""",
(repository.id,),
).fetchone()
active = db.execute(
"""
SELECT * FROM scan_jobs WHERE repository_id = ? AND status IN (?, ?, ?, ?, ?)
ORDER BY created_at DESC LIMIT 1
""",
(repository.id, *ACTIVE_JOB_STATUSES),
).fetchone()
last_job = db.execute(
"SELECT * FROM scan_jobs WHERE repository_id = ? ORDER BY created_at DESC LIMIT 1",
(repository.id,),
).fetchone()
last_scan = self._report_from_row(latest) if latest else None
active_findings = [item for item in (last_scan.findings if last_scan else []) if not item.suppressed]
severity_rank = {Severity.BLOCKER: 0, Severity.WARNING: 1, Severity.INFO: 2}
primary_finding = min(
active_findings,
key=lambda item: (severity_rank[item.severity], -item.deduction, item.category, item.code),
default=None,
)
primary_signal = (
RemediationSignal(
code=primary_finding.code,
category=primary_finding.category,
severity=primary_finding.severity,
message=primary_finding.message,
count=sum(1 for item in active_findings if item.code == primary_finding.code),
)
if primary_finding
else None
)
previous_row = (
db.execute(
"""
SELECT * FROM scans
WHERE repository_id = ? AND id != ?
ORDER BY scanned_at DESC, rowid DESC LIMIT 1
""",
(repository.id, last_scan.id),
).fetchone()
if last_scan
else None
)
previous_scan = self._report_from_row(previous_row) if previous_row else None
previous_findings = [item for item in (previous_scan.findings if previous_scan else []) if not item.suppressed]
active_fingerprints = {self._finding_key(item) for item in active_findings}
previous_fingerprints = {self._finding_key(item) for item in previous_findings}
last_revision = (
last_scan.source.actual_revision
if last_scan and last_scan.source.actual_revision
else (latest["actual_revision"] if latest else None)
)
stale = bool(repository.latest_revision and last_revision and repository.latest_revision != last_revision)
if repository.archived:
state = RepositoryState.ARCHIVED
elif not repository.available:
state = RepositoryState.UNAVAILABLE
elif active:
state = (
RepositoryState.QUEUED
if active["status"] == ScanJobStatus.QUEUED.value
else RepositoryState.SCANNING
)
elif last_job and last_job["status"] == ScanJobStatus.FAILED.value:
last_job_at = _datetime(last_job["created_at"])
if last_scan is None or (last_job_at and last_job_at > last_scan.scanned_at):
state = RepositoryState.FAILED
elif stale:
state = RepositoryState.STALE
else:
state = RepositoryState.CURRENT
elif last_scan is None:
state = RepositoryState.NEVER_SCANNED
elif stale:
state = RepositoryState.STALE
else:
state = RepositoryState.CURRENT
data = repository.model_dump()
data.update(
{
"state": state,
"stale": stale,
"active_job_id": active["id"] if active else None,
"last_scan_id": last_scan.id if last_scan else None,
"last_scan_at": last_scan.scanned_at if last_scan else None,
"last_scan_health": last_scan.health_score if last_scan else None,
"last_scan_readiness": last_scan.release_gate.status if last_scan and last_scan.release_gate else (last_scan.release_readiness if last_scan else None),
"last_scan_revision": last_revision,
"last_scan_blockers": sum(1 for item in active_findings if item.severity == Severity.BLOCKER),
"last_scan_warnings": sum(1 for item in active_findings if item.severity == Severity.WARNING),
"last_scan_primary_signal": primary_signal,
"last_scan_health_delta": (last_scan.health_score - previous_scan.health_score) if previous_scan and last_scan else None,
"last_scan_added_findings": len(active_fingerprints - previous_fingerprints) if previous_scan else 0,
"last_scan_resolved_findings": len(previous_fingerprints - active_fingerprints) if previous_scan else 0,
}
)
return RepositorySummary(**data)
def _catalog_rows(
self, db: sqlite3.Connection, *, limit: int, offset: int = 0, repository_id: str | None = None
) -> list[sqlite3.Row]:
@@ -674,7 +581,11 @@ class ScanStore:
return db.execute(
f"""
WITH ranked_scans AS (
SELECT s.*, ROW_NUMBER() OVER (
-- Summary columns only: report_json must never be materialized here.
SELECT s.id, s.repository_id, s.scanned_at, s.health_score, s.release_readiness,
s.actual_revision, s.finding_count, s.blocker_count, s.warning_count,
s.primary_signal_json, s.added_findings, s.resolved_findings,
s.triage_priority, s.triage_rationale, ROW_NUMBER() OVER (
PARTITION BY s.repository_id ORDER BY s.scanned_at DESC, s.rowid DESC
) AS scan_rank
FROM scans AS s WHERE s.repository_id IS NOT NULL
@@ -777,7 +688,7 @@ class ScanStore:
def repository_state_counts(self) -> dict[str, int]:
counts: Counter[str] = Counter()
for repository in self.list_repositories(limit=1000):
for repository in self.list_repositories(limit=-1):
counts[repository.state.value] += 1
return dict(counts)
@@ -793,31 +704,70 @@ class ScanStore:
).fetchone()
return RepositoryDetail(**summary.model_dump(), latest_scan=self._report_from_row(latest) if latest else None)
@staticmethod
def _merged_source_value(report_value: object, column_value: object, field: str) -> object:
"""Mirror _report_from_row: a non-null column fills an absent/manual report value."""
if column_value is None or report_value not in (None, ScanTrigger.MANUAL.value):
return report_value
if field == "trigger":
try:
return ScanTrigger(str(column_value))
except ValueError:
return report_value
return column_value
def repository_scans(self, repository_id: str, limit: int = 50) -> list[ScanSummary]:
"""List scan summaries without deserializing full report payloads.
Finding counts come from the retained report; compacted reports no longer hold
findings, so their retained active-finding summary count is reported instead of 0.
"""
with self._connection() as db:
rows = db.execute(
"SELECT * FROM scans WHERE repository_id = ? ORDER BY scanned_at DESC, rowid DESC LIMIT ?",
"""
SELECT id, scanned_at, health_score, report_compacted, finding_count,
release_readiness AS column_readiness,
branch, requested_revision, actual_revision, trigger,
json_extract(report_json, '$.source.branch') AS report_branch,
json_extract(report_json, '$.source.requested_revision') AS report_requested_revision,
json_extract(report_json, '$.source.actual_revision') AS report_actual_revision,
json_extract(report_json, '$.source.trigger') AS report_trigger,
json_extract(report_json, '$.release_readiness') AS report_readiness,
json_extract(report_json, '$.duration_ms') AS duration_ms,
json_array_length(report_json, '$.findings') AS report_findings
FROM scans WHERE repository_id = ?
ORDER BY scanned_at DESC, rowid DESC LIMIT ?
""",
(repository_id, limit),
).fetchall()
result: list[ScanSummary] = []
for row in rows:
report = self._report_from_row(row)
result.append(
ScanSummary(
id=report.id,
repository_id=repository_id,
scanned_at=report.scanned_at,
branch=report.source.branch,
requested_revision=report.source.requested_revision,
actual_revision=report.source.actual_revision,
trigger=report.source.trigger,
health_score=report.health_score,
release_readiness=report.release_readiness,
findings=len(report.findings),
duration_ms=report.duration_ms,
)
result: list[ScanSummary] = []
for row in rows:
source = {
field: self._merged_source_value(row[f"report_{field}"], row[field], field)
for field in ("branch", "requested_revision", "actual_revision", "trigger")
}
if row["report_compacted"]:
findings = int(row["finding_count"] or 0)
else:
findings = int(row["report_findings"] or 0)
result.append(
ScanSummary(
id=row["id"],
repository_id=repository_id,
scanned_at=row["scanned_at"],
branch=source["branch"],
requested_revision=source["requested_revision"],
actual_revision=source["actual_revision"],
trigger=source["trigger"] or ScanTrigger.MANUAL,
health_score=row["health_score"],
release_readiness=row["report_readiness"] or row["column_readiness"] or "",
findings=findings,
duration_ms=float(row["duration_ms"] or 0.0),
)
return result
)
return result
def _report_from_row(self, row: sqlite3.Row | None) -> ScanReport:
if row is None:
@@ -948,14 +898,20 @@ class ScanStore:
)
cutoff = (_now() - timedelta(days=report_retention_days())).isoformat()
db.execute("DELETE FROM scans WHERE history_key = ? AND scanned_at < ?", (history_key, cutoff))
rows = db.execute(
"""
SELECT * FROM scans WHERE history_key = ? AND report_compacted = 0
ORDER BY scanned_at DESC, rowid DESC LIMIT -1 OFFSET ?
""",
(history_key, full_reports_per_repository()),
).fetchall()
for row in rows:
scan_ids = [
str(item[0])
for item in db.execute(
"""
SELECT id FROM scans WHERE history_key = ? AND report_compacted = 0
ORDER BY scanned_at DESC, rowid DESC LIMIT -1 OFFSET ?
""",
(history_key, full_reports_per_repository()),
).fetchall()
]
for scan_id in scan_ids:
row = db.execute("SELECT * FROM scans WHERE id = ?", (scan_id,)).fetchone()
if row is None:
continue
report = self._report_from_row(row)
compact = report.model_copy(
update={"findings": [], "architecture": ArchitectureGraph(), "hotspots": [], "compacted": True}
@@ -1055,21 +1011,52 @@ class ScanStore:
)
def trend(self, repository_key: str, limit: int = 50) -> list[TrendPoint]:
reports = list(reversed(self.recent_for_repository(repository_key, limit)))
"""Return chronological trend points computed in SQLite, not from full reports.
Counts include every retained finding, as before. Compacted reports no longer
hold findings, so their retained active-finding summary counts are used rather
than reporting a misleading zero.
"""
with self._connection() as db:
rows: list[sqlite3.Row] = []
for candidate in self._history_candidates(repository_key):
rows = db.execute(
"""
SELECT id, scanned_at, health_score, report_compacted,
finding_count, blocker_count, warning_count,
json_array_length(report_json, '$.findings') AS report_findings,
(
SELECT COUNT(*) FROM json_each(scans.report_json, '$.findings')
WHERE json_extract(value, '$.severity') = 'blocker'
) AS report_blockers,
(
SELECT COUNT(*) FROM json_each(scans.report_json, '$.findings')
WHERE json_extract(value, '$.severity') = 'warning'
) AS report_warnings,
COALESCE(json_extract(report_json, '$.metrics.todo'), 0)
+ COALESCE(json_extract(report_json, '$.metrics.fixme'), 0)
+ COALESCE(json_extract(report_json, '$.metrics.hack'), 0)
+ COALESCE(json_extract(report_json, '$.metrics.xxx'), 0) AS debt_markers
FROM scans WHERE history_key = ?
ORDER BY scanned_at DESC, rowid DESC LIMIT ?
""",
(candidate, limit),
).fetchall()
if rows:
break
points: list[TrendPoint] = []
for report in reports:
blockers = sum(1 for item in report.findings if item.severity == Severity.BLOCKER)
warnings = sum(1 for item in report.findings if item.severity == Severity.WARNING)
debt = report.metrics.todo + report.metrics.fixme + report.metrics.hack + report.metrics.xxx
for row in reversed(rows):
compacted = bool(row["report_compacted"])
points.append(
TrendPoint(
scan_id=report.id,
scanned_at=report.scanned_at,
health_score=report.health_score,
findings=len(report.findings),
blockers=blockers,
warnings=warnings,
debt_markers=debt,
scan_id=row["id"],
scanned_at=row["scanned_at"],
health_score=row["health_score"],
findings=int((row["finding_count"] if compacted else row["report_findings"]) or 0),
blockers=int((row["blocker_count"] if compacted else row["report_blockers"]) or 0),
warnings=int((row["warning_count"] if compacted else row["report_warnings"]) or 0),
debt_markers=int(row["debt_markers"] or 0),
)
)
return points
@@ -1220,11 +1207,29 @@ class ScanStore:
row = db.execute("SELECT * FROM scan_jobs WHERE id = ?", (job_id,)).fetchone()
return self._job_from_row(row) if row else None
@staticmethod
def _claim_work_may_exist(db: sqlite3.Connection, now: str) -> bool:
placeholders = ", ".join("?" for _ in ACTIVE_JOB_STATUSES[1:])
row = db.execute(
f"""
SELECT EXISTS(SELECT 1 FROM scan_jobs WHERE status = 'queued')
OR EXISTS(
SELECT 1 FROM scan_jobs
WHERE status IN ({placeholders}) AND lease_until IS NOT NULL AND lease_until < ?
)
""",
(*ACTIVE_JOB_STATUSES[1:], now),
).fetchone()
return bool(row[0])
def claim_next_job(self) -> ScanJob | None:
now_dt = _now()
now = now_dt.isoformat()
lease_until = (now_dt + timedelta(seconds=job_lease_seconds())).isoformat()
with self._connection() as db:
# Cheap read-only probe: an idle queue never takes the write lock.
if not self._claim_work_may_exist(db, now):
return None
db.execute("BEGIN IMMEDIATE")
if self._maintenance_enabled(db):
return None
@@ -1443,6 +1448,30 @@ class ScanStore:
except OSError:
return 0
def _cached_mirror_stats(self, mirrors_root: Path) -> tuple[int, int]:
"""Return mirror-cache size without making /meta wait on a large tree walk.
The walk (up to 50k files) runs synchronously only when no value exists yet.
Afterwards an expired value is served while one background thread refreshes it.
"""
key = str(mirrors_root)
now = monotonic()
cached = self._mirror_stats_cache
if cached is None or cached[1] != key:
stats = self._tree_stats(mirrors_root)
self._mirror_stats_cache = (monotonic(), key, stats)
return stats
if now - cached[0] >= MIRROR_STATS_TTL_SECONDS and self._mirror_stats_refresh.acquire(blocking=False):
def refresh() -> None:
try:
self._mirror_stats_cache = (monotonic(), key, self._tree_stats(mirrors_root))
finally:
self._mirror_stats_refresh.release()
Thread(target=refresh, name="repopulse-mirror-stats", daemon=True).start()
return cached[2]
def storage_stats(self) -> dict[str, int | str | None]:
with self._connection() as db:
row = db.execute(
@@ -1462,7 +1491,7 @@ class ScanStore:
cache_root = Path(os.getenv("REPOPULSE_REPOSITORY_CACHE", "data/repos")).expanduser()
mirrors_root = cache_root / "mirrors"
worktrees_root = cache_root / "worktrees"
_mirror_files, mirror_cache_bytes = self._tree_stats(mirrors_root)
_mirror_files, mirror_cache_bytes = self._cached_mirror_stats(mirrors_root)
mirror_count = self._managed_entry_count(mirrors_root)
worktree_count = self._managed_entry_count(worktrees_root)
return {