RepoPulse: update backend/app and dependencies (+3 areas)
Public changes compared with the previous published export: 26 files changed. Affected areas: - backend/app: 11 files - dependencies: 5 files - deployment and CI: 3 files - documentation: 2 files - tests: 5 files Changed files: - Updated: "Dockerfile.unraid" - Updated: "README.md" - Updated: "backend/Dockerfile" - Updated: "backend/app/acquisition.py" - Updated: "backend/app/architecture.py" - Updated: "backend/app/git_analysis.py" - Updated: "backend/app/gitea.py" - Updated: "backend/app/jobs.py" - Updated: "backend/app/main.py" - Added: "backend/app/processes.py" - Updated: "backend/app/rules.py" - Updated: "backend/app/scanner.py" - Updated: "backend/app/signals.py" - Updated: "backend/app/store.py" - Updated: "backend/pyproject.toml" - Added: "backend/tests/test_api_responsiveness.py" - Added: "backend/tests/test_processes.py" - Updated: "backend/tests/test_scanner.py" - Updated: "backend/tests/test_signals_v11.py" - Updated: "backend/tests/test_store.py" - Updated: "docker-compose.yml" - Updated: "docs/ARCHITECTURE.md" - Updated: "frontend/package-lock.json" - Updated: "frontend/package.json" - Updated: "package-lock.json" - Updated: "package.json" Export provenance/manifests refreshed; excluded from the change summary. Publish curated RepoPulse source from a251f826238908b156485299061da76482780ed2
This commit is contained in:
1 parent
23a4a721f1
commit
61ae49f9d0
28 files changed
+1083
-408
No files matched your search
+3
-1
@@ -7,7 +7,7 @@ ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
REPOPULSE_SCAN_ROOT=/scan
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends git \
|
||||
&& apt-get install -y --no-install-recommends git tini \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY backend/pyproject.toml ./pyproject.toml
|
||||
@@ -16,4 +16,6 @@ RUN pip install --no-cache-dir .
|
||||
|
||||
VOLUME ["/data", "/scan"]
|
||||
EXPOSE 8000
|
||||
# tini is PID 1 so orphaned git descendants are always reaped (no zombies).
|
||||
ENTRYPOINT ["/usr/bin/tini", "--"]
|
||||
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]
|
||||
@@ -10,6 +10,7 @@ from threading import Lock, RLock
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from .models import Repository
|
||||
from .processes import git_environment, run_bounded
|
||||
|
||||
SAFE_REVISION = re.compile(r"^[0-9a-fA-F]{7,64}$")
|
||||
SAFE_BRANCH = re.compile(r"^[A-Za-z0-9._/-]+$")
|
||||
@@ -72,35 +73,19 @@ class GitAcquirer:
|
||||
return self._repository_locks.setdefault(repository_id, RLock())
|
||||
|
||||
def _git_env(self) -> dict[str, str]:
|
||||
env = os.environ.copy()
|
||||
if self.token:
|
||||
env.update(
|
||||
{
|
||||
"GIT_CONFIG_COUNT": "1",
|
||||
"GIT_CONFIG_KEY_0": "http.extraHeader",
|
||||
"GIT_CONFIG_VALUE_0": f"Authorization: token {self.token}",
|
||||
}
|
||||
)
|
||||
return env
|
||||
extra = [("http.extraHeader", f"Authorization: token {self.token}")] if self.token else []
|
||||
return git_environment(extra_config=extra)
|
||||
|
||||
def _run(self, args: list[str]) -> str:
|
||||
try:
|
||||
result = subprocess.run(
|
||||
args,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
errors="replace",
|
||||
timeout=self.timeout,
|
||||
env=self._git_env(),
|
||||
)
|
||||
result = run_bounded(args, timeout=self.timeout, env=self._git_env())
|
||||
except subprocess.TimeoutExpired as exc:
|
||||
raise AcquisitionError("timeout", "Git source acquisition timed out") from exc
|
||||
except (OSError, subprocess.CalledProcessError) as exc:
|
||||
# Git stderr may contain remote URLs or server-specific details; do
|
||||
# not persist or expose it because the job API is user-visible.
|
||||
raise AcquisitionError("git_failed", "Git source acquisition failed") from exc
|
||||
return result.stdout.strip()
|
||||
return result.stdout.decode("utf-8", errors="replace").strip()
|
||||
|
||||
def _mirror_path(self, repository: Repository) -> Path:
|
||||
identity = f"{repository.provider}-{repository.provider_repository_id}"
|
||||
|
||||
+138
-68
@@ -4,6 +4,7 @@ import ast
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import posixpath
|
||||
import re
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
@@ -66,8 +67,10 @@ def _ts_path_aliases(root: Path) -> list[tuple[str, str]]:
|
||||
return aliases[:100]
|
||||
|
||||
|
||||
def _workspace_packages(root: Path) -> dict[str, Path]:
|
||||
packages: dict[str, Path] = {}
|
||||
def _workspace_packages(root: Path) -> dict[str, str]:
|
||||
"""Map local package names to their root-relative POSIX directory."""
|
||||
|
||||
packages: dict[str, str] = {}
|
||||
for manifest in _bounded_config_files(root, {"package.json"}, maximum=100):
|
||||
try:
|
||||
if manifest.stat().st_size > 256_000:
|
||||
@@ -75,7 +78,7 @@ def _workspace_packages(root: Path) -> dict[str, Path]:
|
||||
data = json.loads(manifest.read_text(encoding="utf-8"))
|
||||
name = data.get("name") if isinstance(data, dict) else None
|
||||
if isinstance(name, str) and name and not name.startswith((".", "/")):
|
||||
packages.setdefault(name, manifest.parent)
|
||||
packages.setdefault(name, manifest.parent.relative_to(root).as_posix())
|
||||
except (OSError, ValueError, json.JSONDecodeError, TypeError):
|
||||
continue
|
||||
return packages
|
||||
@@ -116,79 +119,118 @@ def _resolve_python_target(module: str, module_map: dict[str, str]) -> str | Non
|
||||
return None
|
||||
|
||||
|
||||
def _python_imports(path: str, text: str, module_map: dict[str, str]) -> set[str]:
|
||||
try:
|
||||
tree = ast.parse(text)
|
||||
except SyntaxError:
|
||||
return set()
|
||||
def python_package_parts(path: str) -> list[str]:
|
||||
"""Return the package that relative imports in ``path`` resolve against."""
|
||||
|
||||
current_names = _module_names(path)
|
||||
current = current_names[0] if current_names else ""
|
||||
current_parts = current.split(".") if current else []
|
||||
if not path.endswith("/__init__.py") and current_parts:
|
||||
package_parts = current_parts[:-1]
|
||||
else:
|
||||
package_parts = current_parts
|
||||
return current_parts[:-1]
|
||||
return current_parts
|
||||
|
||||
imports: set[str] = set()
|
||||
|
||||
def python_import_modules(node: ast.AST, package_parts: list[str]) -> list[str]:
|
||||
"""Return the absolute (still unresolved) module names referenced by one import node."""
|
||||
|
||||
if isinstance(node, ast.Import):
|
||||
return [alias.name for alias in node.names]
|
||||
if isinstance(node, ast.ImportFrom):
|
||||
module = node.module or ""
|
||||
if node.level:
|
||||
trim = max(0, node.level - 1)
|
||||
prefix = package_parts[:-trim] if trim else package_parts
|
||||
return [".".join([*prefix, *([module] if module else [])])]
|
||||
return [module]
|
||||
return []
|
||||
|
||||
|
||||
def extract_python_imports(path: str, tree: ast.AST) -> tuple[str, ...]:
|
||||
package_parts = python_package_parts(path)
|
||||
modules: dict[str, None] = {}
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Import):
|
||||
for alias in node.names:
|
||||
target = _resolve_python_target(alias.name, module_map)
|
||||
if target and target != path:
|
||||
imports.add(target)
|
||||
elif isinstance(node, ast.ImportFrom):
|
||||
module = node.module or ""
|
||||
if node.level:
|
||||
trim = max(0, node.level - 1)
|
||||
prefix = package_parts[:-trim] if trim else package_parts
|
||||
absolute = ".".join([*prefix, *([module] if module else [])])
|
||||
else:
|
||||
absolute = module
|
||||
target = _resolve_python_target(absolute, module_map)
|
||||
if target and target != path:
|
||||
imports.add(target)
|
||||
return imports
|
||||
if isinstance(node, (ast.Import, ast.ImportFrom)):
|
||||
for module in python_import_modules(node, package_parts):
|
||||
modules.setdefault(module)
|
||||
return tuple(modules)
|
||||
|
||||
|
||||
def extract_js_specifiers(text: str) -> tuple[str, ...]:
|
||||
return tuple(dict.fromkeys(match.group("target") for match in JS_IMPORT_RE.finditer(text)))
|
||||
|
||||
|
||||
def extract_imports(path: str, text: str, language: str) -> tuple[str, ...]:
|
||||
"""Extract compact, unresolved import references so source text need not be retained."""
|
||||
|
||||
if language == "Python":
|
||||
try:
|
||||
tree = ast.parse(text)
|
||||
except (SyntaxError, ValueError):
|
||||
return ()
|
||||
return extract_python_imports(path, tree)
|
||||
if language in {"TypeScript", "JavaScript"}:
|
||||
return extract_js_specifiers(text)
|
||||
return ()
|
||||
|
||||
|
||||
def _lexical_relative(path: str) -> str | None:
|
||||
"""Normalize a root-relative POSIX path without filesystem access.
|
||||
|
||||
Candidates are only matched against already safety-checked scanner paths, so
|
||||
symlink resolution adds nothing; a path escaping the root yields no target.
|
||||
"""
|
||||
|
||||
if not path or path.startswith("/"):
|
||||
return None
|
||||
normalized = posixpath.normpath(path)
|
||||
if normalized == ".." or normalized.startswith("../"):
|
||||
return None
|
||||
return normalized
|
||||
|
||||
|
||||
def _resolve_js_target(
|
||||
root: Path,
|
||||
source_path: str,
|
||||
specifier: str,
|
||||
known: set[str],
|
||||
aliases: list[tuple[str, str]],
|
||||
workspace_packages: dict[str, Path],
|
||||
workspace_packages: dict[str, str],
|
||||
) -> str | None:
|
||||
relative_base: str | None = None
|
||||
if specifier.startswith("."):
|
||||
source_dir = (root / source_path).parent
|
||||
base = (source_dir / specifier).resolve()
|
||||
relative_base = _lexical_relative(posixpath.join(posixpath.dirname(source_path), specifier))
|
||||
else:
|
||||
base = None
|
||||
matched_alias = False
|
||||
for alias, target in aliases:
|
||||
if alias.endswith("/*") and specifier.startswith(alias[:-1]):
|
||||
base = (root / target.replace("*", specifier[len(alias) - 1 :])).resolve()
|
||||
relative_base = _lexical_relative(target.replace("*", specifier[len(alias) - 1 :]))
|
||||
matched_alias = True
|
||||
break
|
||||
if alias == specifier:
|
||||
base = (root / target).resolve()
|
||||
relative_base = _lexical_relative(target)
|
||||
matched_alias = True
|
||||
break
|
||||
if base is None:
|
||||
if not matched_alias:
|
||||
package_name = next(
|
||||
(name for name in workspace_packages if specifier == name or specifier.startswith(f"{name}/")),
|
||||
(
|
||||
name
|
||||
for name in workspace_packages
|
||||
if specifier == name or specifier.startswith(f"{name}/")
|
||||
),
|
||||
None,
|
||||
)
|
||||
if package_name:
|
||||
package_root = workspace_packages[package_name]
|
||||
suffix = specifier.removeprefix(package_name).lstrip("/")
|
||||
base = package_root / suffix if suffix else package_root / "src" / "index"
|
||||
if base is None:
|
||||
return None
|
||||
try:
|
||||
relative_base = base.relative_to(root.resolve()).as_posix()
|
||||
except ValueError:
|
||||
relative_base = _lexical_relative(
|
||||
posixpath.join(package_root, suffix)
|
||||
if suffix
|
||||
else posixpath.join(package_root, "src", "index")
|
||||
)
|
||||
if relative_base is None:
|
||||
return None
|
||||
|
||||
candidates = [relative_base]
|
||||
suffix = Path(relative_base).suffix.lower()
|
||||
suffix = posixpath.splitext(relative_base)[1].lower()
|
||||
if suffix in JS_EXTENSIONS:
|
||||
extensionless = relative_base[: -len(suffix)]
|
||||
candidates.extend(extensionless + extension for extension in JS_EXTENSIONS)
|
||||
@@ -199,22 +241,6 @@ def _resolve_js_target(
|
||||
return next((candidate for candidate in candidates if candidate in known), None)
|
||||
|
||||
|
||||
def _js_imports(
|
||||
root: Path,
|
||||
path: str,
|
||||
text: str,
|
||||
known: set[str],
|
||||
aliases: list[tuple[str, str]],
|
||||
workspace_packages: dict[str, Path],
|
||||
) -> set[str]:
|
||||
imports: set[str] = set()
|
||||
for match in JS_IMPORT_RE.finditer(text):
|
||||
target = _resolve_js_target(root, path, match.group("target"), known, aliases, workspace_packages)
|
||||
if target and target != path:
|
||||
imports.add(target)
|
||||
return imports
|
||||
|
||||
|
||||
def build_architecture(
|
||||
root: Path,
|
||||
source_texts: dict[str, str],
|
||||
@@ -226,27 +252,71 @@ def build_architecture(
|
||||
max_nodes: int = 160,
|
||||
max_edges: int = 600,
|
||||
) -> tuple[ArchitectureGraph, list[Hotspot]]:
|
||||
"""Compatibility entry point that accepts full source texts."""
|
||||
|
||||
imports_by_path = {
|
||||
path: extract_imports(path, text, language_by_path.get(path, ""))
|
||||
for path, text in source_texts.items()
|
||||
}
|
||||
return build_architecture_from_imports(
|
||||
root,
|
||||
imports_by_path,
|
||||
language_by_path,
|
||||
loc_by_path,
|
||||
debt_by_path,
|
||||
churn_by_path,
|
||||
max_nodes=max_nodes,
|
||||
max_edges=max_edges,
|
||||
)
|
||||
|
||||
|
||||
def build_architecture_from_imports(
|
||||
root: Path,
|
||||
imports_by_path: dict[str, tuple[str, ...]],
|
||||
language_by_path: dict[str, str],
|
||||
loc_by_path: dict[str, int],
|
||||
debt_by_path: dict[str, int],
|
||||
churn_by_path: dict[str, int],
|
||||
*,
|
||||
max_nodes: int = 160,
|
||||
max_edges: int = 600,
|
||||
) -> tuple[ArchitectureGraph, list[Hotspot]]:
|
||||
"""Build the bounded graph from compact import references; no source text is retained."""
|
||||
|
||||
max_nodes = max(1, max_nodes)
|
||||
max_edges = max(0, max_edges)
|
||||
known_paths = set(source_texts)
|
||||
known_paths = set(imports_by_path)
|
||||
module_map = _python_module_map(list(known_paths))
|
||||
ts_aliases = _ts_path_aliases(root)
|
||||
workspace_packages = _workspace_packages(root)
|
||||
js_cache: dict[tuple[str, str], str | None] = {}
|
||||
edge_pairs: set[tuple[str, str]] = set()
|
||||
total_inbound: Counter[str] = Counter()
|
||||
total_outbound: Counter[str] = Counter()
|
||||
total_edges = 0
|
||||
truncated = len(known_paths) > max_nodes
|
||||
|
||||
for path in sorted(source_texts):
|
||||
text = source_texts[path]
|
||||
for path in sorted(imports_by_path):
|
||||
references = imports_by_path[path]
|
||||
language = language_by_path.get(path, "")
|
||||
targets: set[str] = set()
|
||||
if language == "Python":
|
||||
targets = _python_imports(path, text, module_map)
|
||||
for module in references:
|
||||
target = _resolve_python_target(module, module_map)
|
||||
if target and target != path:
|
||||
targets.add(target)
|
||||
elif language in {"TypeScript", "JavaScript"}:
|
||||
targets = _js_imports(root, path, text, known_paths, ts_aliases, workspace_packages)
|
||||
else:
|
||||
targets = set()
|
||||
directory = posixpath.dirname(path)
|
||||
for specifier in references:
|
||||
# Only relative specifiers depend on the importing directory.
|
||||
key = (directory if specifier.startswith(".") else "", specifier)
|
||||
if key not in js_cache:
|
||||
js_cache[key] = _resolve_js_target(
|
||||
path, specifier, known_paths, ts_aliases, workspace_packages
|
||||
)
|
||||
target = js_cache[key]
|
||||
if target and target != path:
|
||||
targets.add(target)
|
||||
for target in sorted(targets):
|
||||
if target not in known_paths:
|
||||
continue
|
||||
|
||||
@@ -6,21 +6,15 @@ from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
from .models import GitSignals
|
||||
from .processes import git_environment, run_bounded
|
||||
|
||||
|
||||
def _run(root: Path, *args: str, timeout: int = 8) -> str | None:
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(root), *args],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
errors="replace",
|
||||
timeout=timeout,
|
||||
)
|
||||
except (OSError, UnicodeError, subprocess.SubprocessError):
|
||||
result = run_bounded(["git", "-C", str(root), *args], timeout=timeout, env=git_environment())
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
return None
|
||||
return result.stdout
|
||||
return result.stdout.decode("utf-8", errors="replace")
|
||||
|
||||
|
||||
def analyze_git(root: Path) -> tuple[GitSignals, dict[str, int]]:
|
||||
|
||||
@@ -238,7 +238,7 @@ class GiteaClient:
|
||||
listed, discovered, branch_errors = self.discover_with_errors()
|
||||
except GiteaError as exc:
|
||||
error = f"{exc.code}: {exc.detail}"
|
||||
for existing in store.list_repositories(limit=1000):
|
||||
for existing in store.list_repositories(limit=-1):
|
||||
if existing.provider == "gitea" and existing.available:
|
||||
store.update_repository_sync(existing, synced_at=synced_at, error=error)
|
||||
result = RepositorySyncResult(
|
||||
@@ -290,7 +290,7 @@ class GiteaClient:
|
||||
updated += 1
|
||||
unavailable = 0
|
||||
if self._listing_complete:
|
||||
for existing in store.list_repositories(limit=1000):
|
||||
for existing in store.list_repositories(limit=-1):
|
||||
if existing.provider == "gitea" and existing.id not in listed_ids and existing.available:
|
||||
store.update_repository_sync(
|
||||
existing.model_copy(update={"available": False}),
|
||||
|
||||
+82
-7
@@ -1,7 +1,12 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import ctypes
|
||||
import ctypes.util
|
||||
import gc
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime
|
||||
from threading import Event, Lock, Thread, current_thread
|
||||
from time import monotonic
|
||||
@@ -21,6 +26,40 @@ from .scanner import scan_repository
|
||||
from .store import MaintenanceModeError, QueueFullError, ScanStore, job_lease_seconds
|
||||
|
||||
LOGGER = logging.getLogger(__name__)
|
||||
WORKER_HEARTBEAT_SECONDS = 5.0
|
||||
WORKER_IDLE_POLL_SECONDS = 5.0
|
||||
|
||||
|
||||
def _load_malloc_trim() -> Callable[[int], int] | None:
|
||||
if not sys.platform.startswith("linux"):
|
||||
return None
|
||||
try:
|
||||
libc = ctypes.CDLL(ctypes.util.find_library("c") or "libc.so.6")
|
||||
trim = libc.malloc_trim
|
||||
except (AttributeError, OSError):
|
||||
return None # e.g. musl libc has no malloc_trim
|
||||
trim.argtypes = [ctypes.c_size_t]
|
||||
trim.restype = ctypes.c_int
|
||||
return trim
|
||||
|
||||
|
||||
_MALLOC_TRIM = _load_malloc_trim()
|
||||
|
||||
|
||||
def release_memory() -> None:
|
||||
"""Return memory freed by a finished scan to the OS.
|
||||
|
||||
A scan allocates many short-lived objects (ASTs, file texts, findings). CPython frees
|
||||
them, but glibc keeps the arenas mapped, so the long-running worker's RSS would stay
|
||||
at its largest scan. A collection plus malloc_trim hands those pages back.
|
||||
"""
|
||||
|
||||
gc.collect()
|
||||
if _MALLOC_TRIM is not None:
|
||||
try:
|
||||
_MALLOC_TRIM(0)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def _positive_int(name: str, default: int, *, maximum: int | None = None) -> int:
|
||||
@@ -60,8 +99,17 @@ class ScanJobRunner:
|
||||
)
|
||||
self.max_queue = max_queue or _positive_int("REPOPULSE_MAX_QUEUED_SCANS", 100, maximum=1000)
|
||||
self._stop = Event()
|
||||
self._wake = Event()
|
||||
self._idle_iterations = 0
|
||||
self.idle_poll_seconds = max(
|
||||
self.poll_seconds, _positive_float("REPOPULSE_WORKER_IDLE_POLL_SECONDS", WORKER_IDLE_POLL_SECONDS)
|
||||
)
|
||||
self._thread: Thread | None = None
|
||||
self._last_sync = float("-inf")
|
||||
self._last_heartbeat = float("-inf")
|
||||
self.heartbeat_interval_seconds = _positive_float(
|
||||
"REPOPULSE_WORKER_HEARTBEAT_SECONDS", WORKER_HEARTBEAT_SECONDS
|
||||
)
|
||||
self._sync_lock = Lock()
|
||||
|
||||
@property
|
||||
@@ -72,11 +120,13 @@ class ScanJobRunner:
|
||||
if self.started:
|
||||
return
|
||||
self._stop.clear()
|
||||
self._wake.clear()
|
||||
self._thread = Thread(target=self._run, name="repopulse-scan-worker", daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
def stop(self) -> None:
|
||||
self._stop.set()
|
||||
self._wake.set()
|
||||
thread = self._thread
|
||||
if thread and thread is not current_thread():
|
||||
thread.join(timeout=5)
|
||||
@@ -94,19 +144,39 @@ class ScanJobRunner:
|
||||
LOGGER.exception("RepoPulse worker loop recovered from an unexpected failure")
|
||||
job = None
|
||||
if job is None:
|
||||
self._stop.wait(self.poll_seconds)
|
||||
self._idle_iterations += 1
|
||||
self._wake.wait(self._idle_delay())
|
||||
self._wake.clear()
|
||||
else:
|
||||
self._idle_iterations = 0
|
||||
|
||||
def _idle_delay(self) -> float:
|
||||
"""Back off exponentially while idle; in-process enqueues wake the loop at once."""
|
||||
|
||||
exponent = min(self._idle_iterations - 1, 16)
|
||||
return min(self.idle_poll_seconds, self.poll_seconds * (2**exponent))
|
||||
|
||||
def wake(self) -> None:
|
||||
self._wake.set()
|
||||
|
||||
def _run_iteration(self) -> ScanJob | None:
|
||||
"""Perform one scheduler pass unless durable maintenance has paused work."""
|
||||
|
||||
if self.store.maintenance_status()["enabled"]:
|
||||
return None
|
||||
if not self.store.touch_worker_heartbeat_if_active(datetime.now(UTC).isoformat()):
|
||||
return None
|
||||
# The heartbeat is a liveness signal, not a lock: claims and enqueues re-check
|
||||
# maintenance inside their own transactions, so writing it on every 0.5 s poll
|
||||
# only churns the WAL. Throttle the write to the heartbeat interval.
|
||||
now = monotonic()
|
||||
if now - self._last_heartbeat >= self.heartbeat_interval_seconds:
|
||||
if not self.store.touch_worker_heartbeat_if_active(datetime.now(UTC).isoformat()):
|
||||
return None
|
||||
self._last_heartbeat = now
|
||||
if self.gitea.configured and monotonic() - self._last_sync >= self.sync_interval_seconds:
|
||||
self.sync_now()
|
||||
if self.store.maintenance_status()["enabled"]:
|
||||
return None
|
||||
# Synchronization can take a while; re-check before claiming new work.
|
||||
if self.store.maintenance_status()["enabled"]:
|
||||
return None
|
||||
return self.process_one()
|
||||
|
||||
def sync_now(self) -> RepositorySyncResult:
|
||||
@@ -135,7 +205,8 @@ class ScanJobRunner:
|
||||
def _schedule_changed_repositories(self) -> None:
|
||||
if self.store.maintenance_status()["enabled"]:
|
||||
return
|
||||
for summary in self.store.list_repositories(limit=1000):
|
||||
# No silent catalog cap: every repository is reconciled, however large the catalog.
|
||||
for summary in self.store.list_repositories(limit=-1):
|
||||
if summary.provider != "gitea" or summary.archived or not summary.latest_revision:
|
||||
continue
|
||||
if not summary.policy.auto_scan:
|
||||
@@ -166,13 +237,15 @@ class ScanJobRunner:
|
||||
trigger: ScanTrigger = ScanTrigger.MANUAL,
|
||||
) -> ScanJob:
|
||||
requested_revision = repository.latest_revision
|
||||
return self.store.enqueue_job(
|
||||
job = self.store.enqueue_job(
|
||||
repository.id,
|
||||
requested_revision,
|
||||
trigger,
|
||||
force=force,
|
||||
max_queue=self.max_queue,
|
||||
)
|
||||
self.wake()
|
||||
return job
|
||||
|
||||
def process_one(self) -> ScanJob | None:
|
||||
job = self.store.claim_next_job()
|
||||
@@ -194,6 +267,8 @@ class ScanJobRunner:
|
||||
)
|
||||
except Exception:
|
||||
LOGGER.exception("RepoPulse could not persist an unexpected job failure")
|
||||
finally:
|
||||
release_memory()
|
||||
return self.store.get_job(job.id)
|
||||
|
||||
class _LeaseHeartbeat:
|
||||
|
||||
+29
-6
@@ -11,8 +11,10 @@ from threading import BoundedSemaphore, Lock
|
||||
|
||||
from fastapi import FastAPI, HTTPException, Query, Request, Response, status
|
||||
from fastapi.concurrency import run_in_threadpool
|
||||
from fastapi.middleware.gzip import GZipMiddleware
|
||||
from fastapi.responses import JSONResponse
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from starlette.types import Scope
|
||||
|
||||
from .acquisition import GitAcquirer
|
||||
from .gitea import GiteaClient
|
||||
@@ -35,7 +37,7 @@ from .reporting import content_disposition, render_html_report, safe_report_file
|
||||
from .scanner import scan_repository
|
||||
from .store import MaintenanceModeError, QueueFullError, ScanStore
|
||||
|
||||
VERSION = "1.5.0"
|
||||
VERSION = "1.5.1"
|
||||
store = ScanStore()
|
||||
SCAN_ROOT = os.getenv("REPOPULSE_SCAN_ROOT")
|
||||
|
||||
@@ -82,6 +84,8 @@ async def lifespan(_app: FastAPI):
|
||||
|
||||
|
||||
app = FastAPI(title="RepoPulse API", version=VERSION, lifespan=lifespan)
|
||||
# Catalog, report and bundle payloads are large, repetitive JSON/JS; compress them.
|
||||
app.add_middleware(GZipMiddleware, minimum_size=1024, compresslevel=5)
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
@@ -167,7 +171,7 @@ def meta() -> dict[str, object]:
|
||||
return {
|
||||
"name": "RepoPulse",
|
||||
"version": VERSION,
|
||||
"stage": "v1.5.0",
|
||||
"stage": "v1.5.1",
|
||||
"scan_root_restricted": bool(SCAN_ROOT),
|
||||
"scan_root": SCAN_ROOT,
|
||||
"max_concurrent_scans": MAX_CONCURRENT_SCANS,
|
||||
@@ -242,7 +246,8 @@ async def create_scan(payload: ScanRequest, request: Request) -> ScanReport:
|
||||
raise HTTPException(status_code=422, detail="Repository path cannot be scanned") from exc
|
||||
finally:
|
||||
SCAN_SLOTS.release()
|
||||
return store.save(report)
|
||||
# SQLite writes may wait on busy_timeout; never block the event loop with them.
|
||||
return await asyncio.to_thread(store.save, report)
|
||||
|
||||
|
||||
@app.get("/api/v1/repositories", response_model=list[RepositorySummary])
|
||||
@@ -296,7 +301,7 @@ def repository_trend(
|
||||
|
||||
|
||||
@app.post("/api/v1/repositories/{repository_id}/scans", response_model=ScanJob, status_code=202)
|
||||
async def create_repository_scan(
|
||||
def create_repository_scan(
|
||||
request: Request,
|
||||
repository_id: str,
|
||||
payload: RepositoryScanRequest | None = None,
|
||||
@@ -357,6 +362,12 @@ async def gitea_webhook(request: Request) -> WebhookResponse:
|
||||
raise HTTPException(status_code=400, detail="Webhook payload is invalid JSON") from exc
|
||||
if not isinstance(payload, dict):
|
||||
raise HTTPException(status_code=400, detail="Webhook payload must be an object")
|
||||
event = request.headers.get("x-gitea-event", "push").casefold()
|
||||
# Catalog reads and the enqueue transaction are blocking SQLite work.
|
||||
return await run_in_threadpool(_handle_gitea_webhook, payload, event)
|
||||
|
||||
|
||||
def _handle_gitea_webhook(payload: dict, event: str) -> WebhookResponse:
|
||||
repository_payload = payload.get("repository")
|
||||
if not isinstance(repository_payload, dict):
|
||||
raise HTTPException(status_code=400, detail="Webhook repository is missing")
|
||||
@@ -368,7 +379,6 @@ async def gitea_webhook(request: Request) -> WebhookResponse:
|
||||
accepted=False,
|
||||
message="Repository is not present in the synchronized catalog",
|
||||
)
|
||||
event = request.headers.get("x-gitea-event", "push").casefold()
|
||||
if event != "push":
|
||||
return WebhookResponse(
|
||||
accepted=True,
|
||||
@@ -486,6 +496,19 @@ def export_scan(scan_id: str, format: str = Query(default="html", pattern="^(htm
|
||||
)
|
||||
|
||||
|
||||
class _FrontendStaticFiles(StaticFiles):
|
||||
"""Serve the SPA; content-hashed build assets are immutable and cached long-term."""
|
||||
|
||||
async def get_response(self, path: str, scope: Scope) -> Response:
|
||||
response = await super().get_response(path, scope)
|
||||
if response.status_code == 200:
|
||||
if path.replace("\\", "/").startswith("assets/"):
|
||||
response.headers["Cache-Control"] = "public, max-age=31536000, immutable"
|
||||
else:
|
||||
response.headers["Cache-Control"] = "no-cache"
|
||||
return response
|
||||
|
||||
|
||||
static_dir = Path(os.getenv("REPOPULSE_STATIC_DIR", "/app/static"))
|
||||
if static_dir.is_dir():
|
||||
app.mount("/", StaticFiles(directory=static_dir, html=True), name="frontend")
|
||||
app.mount("/", _FrontendStaticFiles(directory=static_dir, html=True), name="frontend")
|
||||
@@ -0,0 +1,77 @@
|
||||
"""Bounded child-process execution that never leaves orphaned descendants behind."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
from collections.abc import Mapping, Sequence
|
||||
|
||||
# Git may daemonize automatic housekeeping (``gc --auto``/``maintenance --auto``) after a
|
||||
# fetch. A detached child escapes the timeout and is reparented to PID 1, where it becomes
|
||||
# a zombie if PID 1 does not reap. Housekeeping therefore runs in the foreground.
|
||||
GIT_FOREGROUND_CONFIG: tuple[tuple[str, str], ...] = (
|
||||
("gc.autoDetach", "false"),
|
||||
("maintenance.autoDetach", "false"),
|
||||
)
|
||||
|
||||
|
||||
def git_environment(
|
||||
base: Mapping[str, str] | None = None,
|
||||
extra_config: Sequence[tuple[str, str]] = (),
|
||||
) -> dict[str, str]:
|
||||
"""Return an environment that applies foreground housekeeping plus ``extra_config``."""
|
||||
|
||||
env = dict(os.environ if base is None else base)
|
||||
pairs = [*GIT_FOREGROUND_CONFIG, *extra_config]
|
||||
env["GIT_CONFIG_COUNT"] = str(len(pairs))
|
||||
for index, (key, value) in enumerate(pairs):
|
||||
env[f"GIT_CONFIG_KEY_{index}"] = key
|
||||
env[f"GIT_CONFIG_VALUE_{index}"] = value
|
||||
return env
|
||||
|
||||
|
||||
def _kill_process_tree(process: subprocess.Popen[bytes]) -> None:
|
||||
if os.name == "posix":
|
||||
try:
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
except (ProcessLookupError, PermissionError):
|
||||
pass
|
||||
else:
|
||||
process.kill()
|
||||
|
||||
|
||||
def run_bounded(
|
||||
args: Sequence[str],
|
||||
*,
|
||||
timeout: float,
|
||||
env: Mapping[str, str] | None = None,
|
||||
check: bool = True,
|
||||
) -> subprocess.CompletedProcess[bytes]:
|
||||
"""Run ``args`` in its own process group and reap it and all descendants.
|
||||
|
||||
On timeout the whole group is killed (not only the direct child), then the child is
|
||||
waited for so no zombie remains. Raises the same exceptions as ``subprocess.run``.
|
||||
"""
|
||||
|
||||
process = subprocess.Popen(
|
||||
list(args),
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
env=None if env is None else dict(env),
|
||||
start_new_session=os.name == "posix",
|
||||
)
|
||||
try:
|
||||
stdout, stderr = process.communicate(timeout=timeout)
|
||||
except subprocess.TimeoutExpired:
|
||||
_kill_process_tree(process)
|
||||
process.communicate()
|
||||
raise
|
||||
except BaseException:
|
||||
_kill_process_tree(process)
|
||||
process.wait()
|
||||
raise
|
||||
if check and process.returncode:
|
||||
raise subprocess.CalledProcessError(process.returncode, list(args), stdout, stderr)
|
||||
return subprocess.CompletedProcess(list(args), process.returncode, stdout, stderr)
|
||||
+18
-6
@@ -1,3 +1,5 @@
|
||||
from functools import lru_cache
|
||||
|
||||
from .models import Finding, FindingGuidance, Severity
|
||||
|
||||
_RULES: dict[str, tuple[str, str, str, str]] = {
|
||||
@@ -63,12 +65,16 @@ def _default_rule(finding: Finding) -> tuple[str, str, str, str]:
|
||||
)
|
||||
|
||||
|
||||
def enrich_finding(finding: Finding) -> Finding:
|
||||
remediation_class, why, action, confidence = _RULES.get(finding.code, _default_rule(finding))
|
||||
severity_weight = {Severity.BLOCKER: 700, Severity.WARNING: 400, Severity.INFO: 100}[finding.severity]
|
||||
priority = min(1000, severity_weight + finding.deduction * 5)
|
||||
rationale = f"{finding.severity.value.title()} signal; {finding.deduction} health-point impact."
|
||||
guidance = FindingGuidance(
|
||||
@lru_cache(maxsize=4096)
|
||||
def _guidance(code: str, severity: Severity, deduction: int) -> FindingGuidance:
|
||||
"""Guidance depends only on code, severity and deduction; share one immutable instance."""
|
||||
|
||||
probe = Finding(code=code, category="", severity=severity, message="", deduction=deduction)
|
||||
remediation_class, why, action, confidence = _RULES.get(code, _default_rule(probe))
|
||||
severity_weight = {Severity.BLOCKER: 700, Severity.WARNING: 400, Severity.INFO: 100}[severity]
|
||||
priority = min(1000, severity_weight + deduction * 5)
|
||||
rationale = f"{severity.value.title()} signal; {deduction} health-point impact."
|
||||
return FindingGuidance(
|
||||
remediation_class=remediation_class,
|
||||
confidence=confidence,
|
||||
why=why,
|
||||
@@ -76,6 +82,12 @@ def enrich_finding(finding: Finding) -> Finding:
|
||||
priority=priority,
|
||||
priority_rationale=rationale,
|
||||
)
|
||||
|
||||
|
||||
def enrich_finding(finding: Finding) -> Finding:
|
||||
guidance = _guidance(finding.code, finding.severity, finding.deduction)
|
||||
if finding.guidance is guidance:
|
||||
return finding
|
||||
return finding.model_copy(update={"guidance": guidance})
|
||||
|
||||
|
||||
|
||||
+140
-69
@@ -5,13 +5,18 @@ import fnmatch
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
from collections import Counter
|
||||
from collections import Counter, deque
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from time import perf_counter
|
||||
from uuid import uuid4
|
||||
|
||||
from .architecture import build_architecture
|
||||
from .architecture import (
|
||||
build_architecture_from_imports,
|
||||
extract_js_specifiers,
|
||||
python_import_modules,
|
||||
python_package_parts,
|
||||
)
|
||||
from .git_analysis import analyze_git
|
||||
from .models import (
|
||||
Finding,
|
||||
@@ -25,9 +30,10 @@ from .models import (
|
||||
finding_fingerprint,
|
||||
)
|
||||
from .policy import load_policy
|
||||
from .processes import git_environment, run_bounded
|
||||
from .release_gate import build_release_gate
|
||||
from .rules import enrich_findings
|
||||
from .signals import analyze_dependencies, detect_test_signals
|
||||
from .signals import analyze_dependencies, detect_test_signals, is_signal_text_path
|
||||
|
||||
LANGUAGES = {
|
||||
".py": "Python",
|
||||
@@ -224,11 +230,8 @@ class _GitIgnoreMatcher:
|
||||
|
||||
def _git_worktree_root(root: Path) -> Path | None:
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(root), "rev-parse", "--show-toplevel"],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
timeout=5,
|
||||
result = run_bounded(
|
||||
["git", "-C", str(root), "rev-parse", "--show-toplevel"], timeout=5, env=git_environment()
|
||||
)
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
return None
|
||||
@@ -238,42 +241,61 @@ def _git_worktree_root(root: Path) -> Path | None:
|
||||
return None
|
||||
|
||||
|
||||
class _SafePathChecker:
|
||||
"""Validate candidate files below one root, caching directories already proven safe.
|
||||
|
||||
Every path component is still checked for symlinks, but a directory shared by many
|
||||
files is inspected once instead of once per file.
|
||||
"""
|
||||
|
||||
def __init__(self, root: Path) -> None:
|
||||
self.root = root
|
||||
self.root_resolved = root.resolve()
|
||||
self._safe_dirs: set[Path] = set()
|
||||
|
||||
def check(self, candidate: Path) -> Path | None:
|
||||
"""Return a resolved regular file only when every path component is safe."""
|
||||
try:
|
||||
relative = candidate.relative_to(self.root)
|
||||
if _ignored(str(relative)):
|
||||
return None
|
||||
current = self.root
|
||||
parts = relative.parts
|
||||
for index, part in enumerate(parts):
|
||||
current /= part
|
||||
is_last = index == len(parts) - 1
|
||||
if not is_last and current in self._safe_dirs:
|
||||
continue
|
||||
if current.is_symlink():
|
||||
return None
|
||||
if not is_last:
|
||||
self._safe_dirs.add(current)
|
||||
resolved = candidate.resolve(strict=True)
|
||||
root_resolved = self.root_resolved
|
||||
if resolved == root_resolved or root_resolved not in resolved.parents:
|
||||
return None
|
||||
if not resolved.is_file():
|
||||
return None
|
||||
except (OSError, RuntimeError, ValueError):
|
||||
return None
|
||||
return resolved
|
||||
|
||||
|
||||
def _safe_file(root: Path, candidate: Path) -> Path | None:
|
||||
"""Return a resolved regular file only when every path component is safe."""
|
||||
try:
|
||||
relative = candidate.relative_to(root)
|
||||
if _ignored(str(relative)):
|
||||
return None
|
||||
current = root
|
||||
for part in Path(relative).parts:
|
||||
current /= part
|
||||
if current.is_symlink():
|
||||
return None
|
||||
resolved = candidate.resolve(strict=True)
|
||||
root_resolved = root.resolve()
|
||||
if resolved == root_resolved or root_resolved not in resolved.parents:
|
||||
return None
|
||||
if not resolved.is_file():
|
||||
return None
|
||||
except (OSError, RuntimeError, ValueError):
|
||||
return None
|
||||
return resolved
|
||||
return _SafePathChecker(root).check(candidate)
|
||||
|
||||
|
||||
def _tracked_files(root: Path) -> list[Path] | None:
|
||||
def _tracked_files(root: Path) -> list[tuple[str, Path]] | None:
|
||||
if _git_worktree_root(root) != root:
|
||||
return None
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(root), "ls-files", "-z"],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
timeout=5,
|
||||
)
|
||||
result = run_bounded(["git", "-C", str(root), "ls-files", "-z"], timeout=5, env=git_environment())
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
return None
|
||||
|
||||
paths: list[Path] = []
|
||||
checker = _SafePathChecker(root)
|
||||
paths: list[tuple[str, Path]] = []
|
||||
for raw in result.stdout.split(b"\0"):
|
||||
if not raw:
|
||||
continue
|
||||
@@ -281,14 +303,15 @@ def _tracked_files(root: Path) -> list[Path] | None:
|
||||
relative_path = Path(relative)
|
||||
if relative_path.is_absolute() or _ignored(relative):
|
||||
continue
|
||||
safe = _safe_file(root, root / relative_path)
|
||||
safe = checker.check(root / relative_path)
|
||||
if safe is not None:
|
||||
paths.append(safe)
|
||||
paths.append((relative_path.as_posix(), safe))
|
||||
return paths
|
||||
|
||||
|
||||
def _walk_files(root: Path) -> list[Path]:
|
||||
files: list[Path] = []
|
||||
def _walk_files(root: Path) -> list[tuple[str, Path]]:
|
||||
files: list[tuple[str, Path]] = []
|
||||
checker = _SafePathChecker(root)
|
||||
gitignore = _GitIgnoreMatcher(root)
|
||||
for current, dirs, names in os.walk(root, followlinks=False):
|
||||
current_path = Path(current)
|
||||
@@ -312,13 +335,14 @@ def _walk_files(root: Path) -> list[Path]:
|
||||
continue
|
||||
if gitignore.is_ignored(relative, is_dir=False):
|
||||
continue
|
||||
safe = _safe_file(root, candidate)
|
||||
safe = checker.check(candidate)
|
||||
if safe is not None:
|
||||
files.append(safe)
|
||||
files.append((relative.replace("\\", "/"), safe))
|
||||
return files
|
||||
|
||||
|
||||
def _read_text(path: Path, max_bytes: int = MAX_TEXT_BYTES) -> str | None:
|
||||
def _read_text_bytes(path: Path, max_bytes: int = MAX_TEXT_BYTES) -> tuple[str, int] | None:
|
||||
"""Return decoded text plus the number of bytes read, or None for binary/oversized files."""
|
||||
try:
|
||||
if path.stat().st_size > max_bytes:
|
||||
return None
|
||||
@@ -327,7 +351,12 @@ def _read_text(path: Path, max_bytes: int = MAX_TEXT_BYTES) -> str | None:
|
||||
return None
|
||||
if b"\0" in raw[:8192]:
|
||||
return None
|
||||
return raw.decode("utf-8", errors="replace")
|
||||
return raw.decode("utf-8", errors="replace"), len(raw)
|
||||
|
||||
|
||||
def _read_text(path: Path, max_bytes: int = MAX_TEXT_BYTES) -> str | None:
|
||||
result = _read_text_bytes(path, max_bytes)
|
||||
return result[0] if result else None
|
||||
|
||||
|
||||
def _is_test_file(relative: str) -> bool:
|
||||
@@ -343,19 +372,45 @@ def _is_test_file(relative: str) -> bool:
|
||||
)
|
||||
|
||||
|
||||
def _python_function_findings(relative: str, text: str) -> list[Finding]:
|
||||
PYTHON_BRANCH_TYPES = (ast.If, ast.For, ast.AsyncFor, ast.While, ast.Try, ast.Match, ast.BoolOp)
|
||||
|
||||
|
||||
def _analyze_python(relative: str, text: str) -> tuple[list[Finding], tuple[str, ...]]:
|
||||
"""Parse one Python file once and return function findings plus unresolved imports.
|
||||
|
||||
A single breadth-first walk (the same order as ``ast.walk``) attributes each branch
|
||||
construct to every enclosing function, matching a per-function ``ast.walk`` count
|
||||
without re-walking nested subtrees.
|
||||
"""
|
||||
try:
|
||||
tree = ast.parse(text)
|
||||
except SyntaxError:
|
||||
return []
|
||||
except (SyntaxError, ValueError):
|
||||
return [], ()
|
||||
package_parts = python_package_parts(relative)
|
||||
imports: dict[str, None] = {}
|
||||
functions: list[ast.FunctionDef | ast.AsyncFunctionDef] = []
|
||||
branch_counts: list[int] = []
|
||||
pending: deque[tuple[ast.AST, tuple[int, ...]]] = deque([(tree, ())])
|
||||
while pending:
|
||||
node, enclosing = pending.popleft()
|
||||
if isinstance(node, PYTHON_BRANCH_TYPES):
|
||||
for index in enclosing:
|
||||
branch_counts[index] += 1
|
||||
elif isinstance(node, (ast.Import, ast.ImportFrom)):
|
||||
for module in python_import_modules(node, package_parts):
|
||||
imports.setdefault(module)
|
||||
child_enclosing = enclosing
|
||||
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
||||
functions.append(node)
|
||||
branch_counts.append(0)
|
||||
child_enclosing = (*enclosing, len(functions) - 1)
|
||||
pending.extend((child, child_enclosing) for child in ast.iter_child_nodes(node))
|
||||
del tree, pending
|
||||
|
||||
findings: list[Finding] = []
|
||||
branch_types = (ast.If, ast.For, ast.AsyncFor, ast.While, ast.Try, ast.Match, ast.BoolOp)
|
||||
for node in ast.walk(tree):
|
||||
if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
||||
continue
|
||||
for node, branches in zip(functions, branch_counts, strict=True):
|
||||
end_line = getattr(node, "end_lineno", node.lineno)
|
||||
length = max(1, end_line - node.lineno + 1)
|
||||
branches = sum(1 for child in ast.walk(node) if isinstance(child, branch_types))
|
||||
if length >= 140:
|
||||
findings.append(
|
||||
Finding(
|
||||
@@ -380,7 +435,11 @@ def _python_function_findings(relative: str, text: str) -> list[Finding]:
|
||||
deduction=4,
|
||||
)
|
||||
)
|
||||
return findings[:12]
|
||||
return findings[:12], tuple(imports)
|
||||
|
||||
|
||||
def _python_function_findings(relative: str, text: str) -> list[Finding]:
|
||||
return _analyze_python(relative, text)[0]
|
||||
|
||||
|
||||
def _is_placeholder_secret(match: re.Match[str]) -> bool:
|
||||
@@ -408,7 +467,7 @@ def scan_repository(
|
||||
limits = limits or scan_limits_from_environment()
|
||||
tracked_files = _tracked_files(root)
|
||||
all_files = _walk_files(root) if tracked_files is None else tracked_files
|
||||
all_files.sort(key=lambda item: str(item.relative_to(root)).replace("\\", "/").casefold())
|
||||
all_files.sort(key=lambda item: item[0].casefold())
|
||||
files = all_files[: limits.max_files]
|
||||
coverage = ScanCoverage(
|
||||
limits=limits,
|
||||
@@ -426,8 +485,11 @@ def scan_repository(
|
||||
metrics = RepositoryMetrics(files=len(all_files))
|
||||
files_by_name: dict[str, list[Path]] = {}
|
||||
root_entries = {entry.name.lower() for entry in root.iterdir()}
|
||||
text_by_path: dict[str, str] = {}
|
||||
source_texts: dict[str, str] = {}
|
||||
# Only small derived values are retained per file. Full text is held for a single
|
||||
# file at a time, except the few CI/manifest texts that test-signal detection needs.
|
||||
signal_texts: dict[str, str] = {}
|
||||
readable_paths: list[str] = []
|
||||
imports_by_path: dict[str, tuple[str, ...]] = {}
|
||||
language_by_path: dict[str, str] = {}
|
||||
loc_by_path: dict[str, int] = {}
|
||||
debt_by_path: dict[str, int] = {}
|
||||
@@ -435,11 +497,10 @@ def scan_repository(
|
||||
secret_findings_truncated = False
|
||||
text_budget_exhausted = False
|
||||
|
||||
for path_obj in files:
|
||||
for relative, path_obj in files:
|
||||
try:
|
||||
relative = str(path_obj.relative_to(root)).replace("\\", "/")
|
||||
size = path_obj.stat().st_size
|
||||
except (OSError, ValueError):
|
||||
except OSError:
|
||||
continue
|
||||
|
||||
files_by_name.setdefault(path_obj.name, []).append(path_obj)
|
||||
@@ -479,21 +540,28 @@ def scan_repository(
|
||||
coverage.skipped_files_by_text_budget += 1
|
||||
text_budget_exhausted = True
|
||||
continue
|
||||
text = _read_text(path_obj, limits.max_text_bytes_per_file)
|
||||
if text is None:
|
||||
read = _read_text_bytes(path_obj, limits.max_text_bytes_per_file)
|
||||
if read is None:
|
||||
continue
|
||||
text, byte_count = read
|
||||
coverage.scanned_files += 1
|
||||
coverage.text_bytes_scanned += len(text.encode("utf-8", errors="replace"))
|
||||
text_by_path[relative] = text
|
||||
coverage.text_bytes_scanned += byte_count
|
||||
readable_paths.append(relative)
|
||||
if is_signal_text_path(relative):
|
||||
signal_texts[relative] = text
|
||||
lines = text.splitlines()
|
||||
|
||||
if language:
|
||||
loc = sum(1 for line in text.splitlines() if line.strip())
|
||||
loc = sum(1 for line in lines if line.strip())
|
||||
metrics.source_files += 1
|
||||
metrics.lines_of_code += loc
|
||||
languages[language] += loc
|
||||
source_texts[relative] = text
|
||||
language_by_path[relative] = language
|
||||
loc_by_path[relative] = loc
|
||||
if language in {"TypeScript", "JavaScript"}:
|
||||
imports_by_path[relative] = extract_js_specifiers(text)
|
||||
elif language != "Python":
|
||||
imports_by_path[relative] = ()
|
||||
if loc > 1000:
|
||||
metrics.large_source_files += 1
|
||||
findings.append(
|
||||
@@ -507,7 +575,8 @@ def scan_repository(
|
||||
)
|
||||
)
|
||||
if language == "Python":
|
||||
findings.extend(_python_function_findings(relative, text))
|
||||
python_findings, imports_by_path[relative] = _analyze_python(relative, text)
|
||||
findings.extend(python_findings)
|
||||
|
||||
if _is_test_file(relative):
|
||||
metrics.test_files += 1
|
||||
@@ -520,7 +589,7 @@ def scan_repository(
|
||||
debt_by_path[relative] = sum(debt.values())
|
||||
|
||||
if _is_secret_scan_candidate(path_obj, relative, language) and secret_findings < limits.max_secret_findings:
|
||||
for line_number, line in enumerate(text.splitlines(), start=1):
|
||||
for line_number, line in enumerate(lines, start=1):
|
||||
match = SECRET_PATTERN.search(line)
|
||||
if match is None or _is_placeholder_secret(match):
|
||||
continue
|
||||
@@ -567,11 +636,13 @@ def scan_repository(
|
||||
findings.append(Finding(code="hygiene.missing-env-example", category="Repository hygiene", severity=Severity.INFO, message="Environment configuration exists without a root .env.example", deduction=3))
|
||||
|
||||
test_signals = detect_test_signals(
|
||||
[str(path.relative_to(root)).replace("\\", "/") for path in files],
|
||||
[relative for relative, _path in files],
|
||||
metrics.source_files,
|
||||
metrics.test_files,
|
||||
text_by_path,
|
||||
signal_texts,
|
||||
readable_paths,
|
||||
)
|
||||
del signal_texts, readable_paths
|
||||
if metrics.source_files and metrics.test_files == 0:
|
||||
findings.append(Finding(code="testing.none-detected", category="Testing", severity=Severity.WARNING, message="No test files detected", deduction=30))
|
||||
if not test_signals.ci_workflows:
|
||||
@@ -593,9 +664,9 @@ def scan_repository(
|
||||
findings.extend(dependency_findings)
|
||||
|
||||
git_signals, churn_by_path = analyze_git(root)
|
||||
architecture, hotspots = build_architecture(
|
||||
architecture, hotspots = build_architecture_from_imports(
|
||||
root,
|
||||
source_texts,
|
||||
imports_by_path,
|
||||
language_by_path,
|
||||
loc_by_path,
|
||||
debt_by_path,
|
||||
|
||||
+34
-9
@@ -5,6 +5,7 @@ import re
|
||||
import tomllib
|
||||
import xml.etree.ElementTree as ET
|
||||
from collections import defaultdict
|
||||
from collections.abc import Iterable
|
||||
from pathlib import Path
|
||||
|
||||
from .models import DependencySignals, Finding, Severity, TestSignals
|
||||
@@ -32,6 +33,24 @@ COVERAGE_NAMES = {
|
||||
COVERAGE_NAMES_CASEFOLD = {item.casefold() for item in COVERAGE_NAMES}
|
||||
|
||||
|
||||
CI_WORKFLOW_PREFIXES = (".github/workflows/", ".gitea/workflows/")
|
||||
CI_WORKFLOW_NAMES = {".gitlab-ci.yml", "jenkinsfile", "azure-pipelines.yml"}
|
||||
SIGNAL_TEXT_NAMES = {"package.json", "pyproject.toml"}
|
||||
|
||||
|
||||
def is_ci_workflow_path(relative: str) -> bool:
|
||||
normalized = relative.replace("\\", "/")
|
||||
return normalized.lower().startswith(CI_WORKFLOW_PREFIXES) or (
|
||||
Path(normalized).name.casefold() in CI_WORKFLOW_NAMES
|
||||
)
|
||||
|
||||
|
||||
def is_signal_text_path(relative: str) -> bool:
|
||||
"""Return True when detect_test_signals needs this file's text content."""
|
||||
|
||||
return is_ci_workflow_path(relative) or Path(relative).name.casefold() in SIGNAL_TEXT_NAMES
|
||||
|
||||
|
||||
def _python_dependency_name(declaration: str) -> str:
|
||||
return re.split(r"[<>=!~ @;\[]", declaration, maxsplit=1)[0].strip()
|
||||
|
||||
@@ -69,7 +88,14 @@ def detect_test_signals(
|
||||
source_files: int,
|
||||
test_files: int,
|
||||
text_by_path: dict[str, str],
|
||||
readable_paths: Iterable[str] | None = None,
|
||||
) -> TestSignals:
|
||||
"""Derive test/CI signals.
|
||||
|
||||
``text_by_path`` only needs the texts accepted by :func:`is_signal_text_path`;
|
||||
``readable_paths`` lists every successfully read text file (defaults to the
|
||||
``text_by_path`` keys) so coverage evidence does not require retaining text.
|
||||
"""
|
||||
frameworks: set[str] = set()
|
||||
coverage: list[str] = []
|
||||
ci: list[str] = []
|
||||
@@ -85,11 +111,7 @@ def detect_test_signals(
|
||||
lower = normalized.lower()
|
||||
if name_lower in COVERAGE_NAMES_CASEFOLD or "/coverage/" in f"/{lower}":
|
||||
coverage.append(normalized)
|
||||
if lower.startswith((".github/workflows/", ".gitea/workflows/")) or name_lower in {
|
||||
".gitlab-ci.yml",
|
||||
"jenkinsfile",
|
||||
"azure-pipelines.yml",
|
||||
}:
|
||||
if is_ci_workflow_path(normalized):
|
||||
ci.append(normalized)
|
||||
|
||||
if (name_lower.startswith("test_") or "/tests/" in f"/{lower}") and name_lower.endswith(".py"):
|
||||
@@ -103,11 +125,16 @@ def detect_test_signals(
|
||||
if name_lower.endswith(("tests.cs", "test.cs")):
|
||||
frameworks.add(".NET test")
|
||||
|
||||
ci_paths = set(ci)
|
||||
for path in text_by_path if readable_paths is None else readable_paths:
|
||||
if Path(path).name.casefold() in COVERAGE_NAMES_CASEFOLD:
|
||||
coverage_evidence.append(path.replace("\\", "/"))
|
||||
|
||||
for path, text in text_by_path.items():
|
||||
name = Path(path).name
|
||||
normalized = path.replace("\\", "/")
|
||||
lowered_text = text.casefold()
|
||||
if normalized in ci:
|
||||
if normalized in ci_paths:
|
||||
lowered_text = text.casefold()
|
||||
validates = any(
|
||||
token in lowered_text
|
||||
for token in ("pytest", "vitest", "jest", "npm test", "pnpm test", "go test", "dotnet test", "ruff", "eslint", "typecheck", "py_compile")
|
||||
@@ -125,8 +152,6 @@ def detect_test_signals(
|
||||
ci_deploy_only.append(normalized)
|
||||
if any(token in lowered_text for token in ("--cov", "coverage", "lcov", "codecov")):
|
||||
coverage_evidence.append(normalized)
|
||||
if name.casefold() in COVERAGE_NAMES_CASEFOLD:
|
||||
coverage_evidence.append(normalized)
|
||||
if name.casefold() == "package.json":
|
||||
try:
|
||||
data = json.loads(text)
|
||||
|
||||
+183
-154
@@ -9,6 +9,8 @@ from contextlib import contextmanager
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from hashlib import sha256
|
||||
from pathlib import Path
|
||||
from threading import Lock, Thread
|
||||
from time import monotonic
|
||||
from uuid import uuid4
|
||||
|
||||
from .models import (
|
||||
@@ -35,6 +37,7 @@ from .rules import enrich_findings
|
||||
|
||||
MAX_HISTORY_PER_REPOSITORY = 200
|
||||
SCHEMA_VERSION = 4
|
||||
MIRROR_STATS_TTL_SECONDS = 300.0
|
||||
JOB_LEASE_SECONDS = 900
|
||||
ACTIVE_JOB_STATUSES = (
|
||||
ScanJobStatus.QUEUED.value,
|
||||
@@ -108,6 +111,8 @@ class ScanStore:
|
||||
self.path = Path(raw_path)
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self.recovery_metadata: dict[str, str] | None = None
|
||||
self._mirror_stats_cache: tuple[float, str, tuple[int, int]] | None = None
|
||||
self._mirror_stats_refresh = Lock()
|
||||
try:
|
||||
self._validate_existing_database()
|
||||
self._init_schema()
|
||||
@@ -349,7 +354,7 @@ class ScanStore:
|
||||
def _backfill_legacy_catalog(self, db: sqlite3.Connection) -> None:
|
||||
rows = db.execute(
|
||||
"""
|
||||
SELECT id, repository_path, repository_name, report_json, repository_id, history_key
|
||||
SELECT id, repository_path, repository_name, repository_id, history_key
|
||||
FROM scans
|
||||
WHERE repository_id IS NULL OR history_key IS NULL
|
||||
"""
|
||||
@@ -412,11 +417,19 @@ class ScanStore:
|
||||
}
|
||||
|
||||
def _backfill_scan_summaries(self, db: sqlite3.Connection) -> None:
|
||||
rows = db.execute(
|
||||
"SELECT rowid AS scan_rowid, * FROM scans "
|
||||
"WHERE finding_count IS NULL OR blocker_count IS NULL OR release_readiness IS NULL"
|
||||
).fetchall()
|
||||
for row in rows:
|
||||
# Collect ids first and load one report at a time so a large legacy history is
|
||||
# never materialized in memory at once during migration.
|
||||
scan_ids = [
|
||||
str(item[0])
|
||||
for item in db.execute(
|
||||
"SELECT id FROM scans "
|
||||
"WHERE finding_count IS NULL OR blocker_count IS NULL OR release_readiness IS NULL"
|
||||
).fetchall()
|
||||
]
|
||||
for scan_id in scan_ids:
|
||||
row = db.execute("SELECT rowid AS scan_rowid, * FROM scans WHERE id = ?", (scan_id,)).fetchone()
|
||||
if row is None:
|
||||
continue
|
||||
report = self._report_from_row(row)
|
||||
previous_row = db.execute(
|
||||
"""
|
||||
@@ -560,112 +573,6 @@ class ScanStore:
|
||||
row = db.execute("SELECT source_path FROM repositories WHERE id = ?", (repository_id,)).fetchone()
|
||||
return row["source_path"] if row else None
|
||||
|
||||
def _repository_summary(self, db: sqlite3.Connection, row: sqlite3.Row) -> RepositorySummary:
|
||||
repository = self._repository_from_row(row)
|
||||
latest = db.execute(
|
||||
"""
|
||||
SELECT * FROM scans WHERE repository_id = ?
|
||||
ORDER BY scanned_at DESC, rowid DESC LIMIT 1
|
||||
""",
|
||||
(repository.id,),
|
||||
).fetchone()
|
||||
active = db.execute(
|
||||
"""
|
||||
SELECT * FROM scan_jobs WHERE repository_id = ? AND status IN (?, ?, ?, ?, ?)
|
||||
ORDER BY created_at DESC LIMIT 1
|
||||
""",
|
||||
(repository.id, *ACTIVE_JOB_STATUSES),
|
||||
).fetchone()
|
||||
last_job = db.execute(
|
||||
"SELECT * FROM scan_jobs WHERE repository_id = ? ORDER BY created_at DESC LIMIT 1",
|
||||
(repository.id,),
|
||||
).fetchone()
|
||||
last_scan = self._report_from_row(latest) if latest else None
|
||||
active_findings = [item for item in (last_scan.findings if last_scan else []) if not item.suppressed]
|
||||
severity_rank = {Severity.BLOCKER: 0, Severity.WARNING: 1, Severity.INFO: 2}
|
||||
primary_finding = min(
|
||||
active_findings,
|
||||
key=lambda item: (severity_rank[item.severity], -item.deduction, item.category, item.code),
|
||||
default=None,
|
||||
)
|
||||
primary_signal = (
|
||||
RemediationSignal(
|
||||
code=primary_finding.code,
|
||||
category=primary_finding.category,
|
||||
severity=primary_finding.severity,
|
||||
message=primary_finding.message,
|
||||
count=sum(1 for item in active_findings if item.code == primary_finding.code),
|
||||
)
|
||||
if primary_finding
|
||||
else None
|
||||
)
|
||||
previous_row = (
|
||||
db.execute(
|
||||
"""
|
||||
SELECT * FROM scans
|
||||
WHERE repository_id = ? AND id != ?
|
||||
ORDER BY scanned_at DESC, rowid DESC LIMIT 1
|
||||
""",
|
||||
(repository.id, last_scan.id),
|
||||
).fetchone()
|
||||
if last_scan
|
||||
else None
|
||||
)
|
||||
previous_scan = self._report_from_row(previous_row) if previous_row else None
|
||||
previous_findings = [item for item in (previous_scan.findings if previous_scan else []) if not item.suppressed]
|
||||
active_fingerprints = {self._finding_key(item) for item in active_findings}
|
||||
previous_fingerprints = {self._finding_key(item) for item in previous_findings}
|
||||
last_revision = (
|
||||
last_scan.source.actual_revision
|
||||
if last_scan and last_scan.source.actual_revision
|
||||
else (latest["actual_revision"] if latest else None)
|
||||
)
|
||||
stale = bool(repository.latest_revision and last_revision and repository.latest_revision != last_revision)
|
||||
if repository.archived:
|
||||
state = RepositoryState.ARCHIVED
|
||||
elif not repository.available:
|
||||
state = RepositoryState.UNAVAILABLE
|
||||
elif active:
|
||||
state = (
|
||||
RepositoryState.QUEUED
|
||||
if active["status"] == ScanJobStatus.QUEUED.value
|
||||
else RepositoryState.SCANNING
|
||||
)
|
||||
elif last_job and last_job["status"] == ScanJobStatus.FAILED.value:
|
||||
last_job_at = _datetime(last_job["created_at"])
|
||||
if last_scan is None or (last_job_at and last_job_at > last_scan.scanned_at):
|
||||
state = RepositoryState.FAILED
|
||||
elif stale:
|
||||
state = RepositoryState.STALE
|
||||
else:
|
||||
state = RepositoryState.CURRENT
|
||||
elif last_scan is None:
|
||||
state = RepositoryState.NEVER_SCANNED
|
||||
elif stale:
|
||||
state = RepositoryState.STALE
|
||||
else:
|
||||
state = RepositoryState.CURRENT
|
||||
data = repository.model_dump()
|
||||
data.update(
|
||||
{
|
||||
"state": state,
|
||||
"stale": stale,
|
||||
"active_job_id": active["id"] if active else None,
|
||||
"last_scan_id": last_scan.id if last_scan else None,
|
||||
"last_scan_at": last_scan.scanned_at if last_scan else None,
|
||||
"last_scan_health": last_scan.health_score if last_scan else None,
|
||||
"last_scan_readiness": last_scan.release_gate.status if last_scan and last_scan.release_gate else (last_scan.release_readiness if last_scan else None),
|
||||
"last_scan_revision": last_revision,
|
||||
"last_scan_blockers": sum(1 for item in active_findings if item.severity == Severity.BLOCKER),
|
||||
"last_scan_warnings": sum(1 for item in active_findings if item.severity == Severity.WARNING),
|
||||
"last_scan_primary_signal": primary_signal,
|
||||
"last_scan_health_delta": (last_scan.health_score - previous_scan.health_score) if previous_scan and last_scan else None,
|
||||
"last_scan_added_findings": len(active_fingerprints - previous_fingerprints) if previous_scan else 0,
|
||||
"last_scan_resolved_findings": len(previous_fingerprints - active_fingerprints) if previous_scan else 0,
|
||||
}
|
||||
)
|
||||
return RepositorySummary(**data)
|
||||
|
||||
def _catalog_rows(
|
||||
self, db: sqlite3.Connection, *, limit: int, offset: int = 0, repository_id: str | None = None
|
||||
) -> list[sqlite3.Row]:
|
||||
@@ -674,7 +581,11 @@ class ScanStore:
|
||||
return db.execute(
|
||||
f"""
|
||||
WITH ranked_scans AS (
|
||||
SELECT s.*, ROW_NUMBER() OVER (
|
||||
-- Summary columns only: report_json must never be materialized here.
|
||||
SELECT s.id, s.repository_id, s.scanned_at, s.health_score, s.release_readiness,
|
||||
s.actual_revision, s.finding_count, s.blocker_count, s.warning_count,
|
||||
s.primary_signal_json, s.added_findings, s.resolved_findings,
|
||||
s.triage_priority, s.triage_rationale, ROW_NUMBER() OVER (
|
||||
PARTITION BY s.repository_id ORDER BY s.scanned_at DESC, s.rowid DESC
|
||||
) AS scan_rank
|
||||
FROM scans AS s WHERE s.repository_id IS NOT NULL
|
||||
@@ -777,7 +688,7 @@ class ScanStore:
|
||||
|
||||
def repository_state_counts(self) -> dict[str, int]:
|
||||
counts: Counter[str] = Counter()
|
||||
for repository in self.list_repositories(limit=1000):
|
||||
for repository in self.list_repositories(limit=-1):
|
||||
counts[repository.state.value] += 1
|
||||
return dict(counts)
|
||||
|
||||
@@ -793,31 +704,70 @@ class ScanStore:
|
||||
).fetchone()
|
||||
return RepositoryDetail(**summary.model_dump(), latest_scan=self._report_from_row(latest) if latest else None)
|
||||
|
||||
@staticmethod
|
||||
def _merged_source_value(report_value: object, column_value: object, field: str) -> object:
|
||||
"""Mirror _report_from_row: a non-null column fills an absent/manual report value."""
|
||||
|
||||
if column_value is None or report_value not in (None, ScanTrigger.MANUAL.value):
|
||||
return report_value
|
||||
if field == "trigger":
|
||||
try:
|
||||
return ScanTrigger(str(column_value))
|
||||
except ValueError:
|
||||
return report_value
|
||||
return column_value
|
||||
|
||||
def repository_scans(self, repository_id: str, limit: int = 50) -> list[ScanSummary]:
|
||||
"""List scan summaries without deserializing full report payloads.
|
||||
|
||||
Finding counts come from the retained report; compacted reports no longer hold
|
||||
findings, so their retained active-finding summary count is reported instead of 0.
|
||||
"""
|
||||
|
||||
with self._connection() as db:
|
||||
rows = db.execute(
|
||||
"SELECT * FROM scans WHERE repository_id = ? ORDER BY scanned_at DESC, rowid DESC LIMIT ?",
|
||||
"""
|
||||
SELECT id, scanned_at, health_score, report_compacted, finding_count,
|
||||
release_readiness AS column_readiness,
|
||||
branch, requested_revision, actual_revision, trigger,
|
||||
json_extract(report_json, '$.source.branch') AS report_branch,
|
||||
json_extract(report_json, '$.source.requested_revision') AS report_requested_revision,
|
||||
json_extract(report_json, '$.source.actual_revision') AS report_actual_revision,
|
||||
json_extract(report_json, '$.source.trigger') AS report_trigger,
|
||||
json_extract(report_json, '$.release_readiness') AS report_readiness,
|
||||
json_extract(report_json, '$.duration_ms') AS duration_ms,
|
||||
json_array_length(report_json, '$.findings') AS report_findings
|
||||
FROM scans WHERE repository_id = ?
|
||||
ORDER BY scanned_at DESC, rowid DESC LIMIT ?
|
||||
""",
|
||||
(repository_id, limit),
|
||||
).fetchall()
|
||||
result: list[ScanSummary] = []
|
||||
for row in rows:
|
||||
report = self._report_from_row(row)
|
||||
result.append(
|
||||
ScanSummary(
|
||||
id=report.id,
|
||||
repository_id=repository_id,
|
||||
scanned_at=report.scanned_at,
|
||||
branch=report.source.branch,
|
||||
requested_revision=report.source.requested_revision,
|
||||
actual_revision=report.source.actual_revision,
|
||||
trigger=report.source.trigger,
|
||||
health_score=report.health_score,
|
||||
release_readiness=report.release_readiness,
|
||||
findings=len(report.findings),
|
||||
duration_ms=report.duration_ms,
|
||||
)
|
||||
result: list[ScanSummary] = []
|
||||
for row in rows:
|
||||
source = {
|
||||
field: self._merged_source_value(row[f"report_{field}"], row[field], field)
|
||||
for field in ("branch", "requested_revision", "actual_revision", "trigger")
|
||||
}
|
||||
if row["report_compacted"]:
|
||||
findings = int(row["finding_count"] or 0)
|
||||
else:
|
||||
findings = int(row["report_findings"] or 0)
|
||||
result.append(
|
||||
ScanSummary(
|
||||
id=row["id"],
|
||||
repository_id=repository_id,
|
||||
scanned_at=row["scanned_at"],
|
||||
branch=source["branch"],
|
||||
requested_revision=source["requested_revision"],
|
||||
actual_revision=source["actual_revision"],
|
||||
trigger=source["trigger"] or ScanTrigger.MANUAL,
|
||||
health_score=row["health_score"],
|
||||
release_readiness=row["report_readiness"] or row["column_readiness"] or "",
|
||||
findings=findings,
|
||||
duration_ms=float(row["duration_ms"] or 0.0),
|
||||
)
|
||||
return result
|
||||
)
|
||||
return result
|
||||
|
||||
def _report_from_row(self, row: sqlite3.Row | None) -> ScanReport:
|
||||
if row is None:
|
||||
@@ -948,14 +898,20 @@ class ScanStore:
|
||||
)
|
||||
cutoff = (_now() - timedelta(days=report_retention_days())).isoformat()
|
||||
db.execute("DELETE FROM scans WHERE history_key = ? AND scanned_at < ?", (history_key, cutoff))
|
||||
rows = db.execute(
|
||||
"""
|
||||
SELECT * FROM scans WHERE history_key = ? AND report_compacted = 0
|
||||
ORDER BY scanned_at DESC, rowid DESC LIMIT -1 OFFSET ?
|
||||
""",
|
||||
(history_key, full_reports_per_repository()),
|
||||
).fetchall()
|
||||
for row in rows:
|
||||
scan_ids = [
|
||||
str(item[0])
|
||||
for item in db.execute(
|
||||
"""
|
||||
SELECT id FROM scans WHERE history_key = ? AND report_compacted = 0
|
||||
ORDER BY scanned_at DESC, rowid DESC LIMIT -1 OFFSET ?
|
||||
""",
|
||||
(history_key, full_reports_per_repository()),
|
||||
).fetchall()
|
||||
]
|
||||
for scan_id in scan_ids:
|
||||
row = db.execute("SELECT * FROM scans WHERE id = ?", (scan_id,)).fetchone()
|
||||
if row is None:
|
||||
continue
|
||||
report = self._report_from_row(row)
|
||||
compact = report.model_copy(
|
||||
update={"findings": [], "architecture": ArchitectureGraph(), "hotspots": [], "compacted": True}
|
||||
@@ -1055,21 +1011,52 @@ class ScanStore:
|
||||
)
|
||||
|
||||
def trend(self, repository_key: str, limit: int = 50) -> list[TrendPoint]:
|
||||
reports = list(reversed(self.recent_for_repository(repository_key, limit)))
|
||||
"""Return chronological trend points computed in SQLite, not from full reports.
|
||||
|
||||
Counts include every retained finding, as before. Compacted reports no longer
|
||||
hold findings, so their retained active-finding summary counts are used rather
|
||||
than reporting a misleading zero.
|
||||
"""
|
||||
|
||||
with self._connection() as db:
|
||||
rows: list[sqlite3.Row] = []
|
||||
for candidate in self._history_candidates(repository_key):
|
||||
rows = db.execute(
|
||||
"""
|
||||
SELECT id, scanned_at, health_score, report_compacted,
|
||||
finding_count, blocker_count, warning_count,
|
||||
json_array_length(report_json, '$.findings') AS report_findings,
|
||||
(
|
||||
SELECT COUNT(*) FROM json_each(scans.report_json, '$.findings')
|
||||
WHERE json_extract(value, '$.severity') = 'blocker'
|
||||
) AS report_blockers,
|
||||
(
|
||||
SELECT COUNT(*) FROM json_each(scans.report_json, '$.findings')
|
||||
WHERE json_extract(value, '$.severity') = 'warning'
|
||||
) AS report_warnings,
|
||||
COALESCE(json_extract(report_json, '$.metrics.todo'), 0)
|
||||
+ COALESCE(json_extract(report_json, '$.metrics.fixme'), 0)
|
||||
+ COALESCE(json_extract(report_json, '$.metrics.hack'), 0)
|
||||
+ COALESCE(json_extract(report_json, '$.metrics.xxx'), 0) AS debt_markers
|
||||
FROM scans WHERE history_key = ?
|
||||
ORDER BY scanned_at DESC, rowid DESC LIMIT ?
|
||||
""",
|
||||
(candidate, limit),
|
||||
).fetchall()
|
||||
if rows:
|
||||
break
|
||||
points: list[TrendPoint] = []
|
||||
for report in reports:
|
||||
blockers = sum(1 for item in report.findings if item.severity == Severity.BLOCKER)
|
||||
warnings = sum(1 for item in report.findings if item.severity == Severity.WARNING)
|
||||
debt = report.metrics.todo + report.metrics.fixme + report.metrics.hack + report.metrics.xxx
|
||||
for row in reversed(rows):
|
||||
compacted = bool(row["report_compacted"])
|
||||
points.append(
|
||||
TrendPoint(
|
||||
scan_id=report.id,
|
||||
scanned_at=report.scanned_at,
|
||||
health_score=report.health_score,
|
||||
findings=len(report.findings),
|
||||
blockers=blockers,
|
||||
warnings=warnings,
|
||||
debt_markers=debt,
|
||||
scan_id=row["id"],
|
||||
scanned_at=row["scanned_at"],
|
||||
health_score=row["health_score"],
|
||||
findings=int((row["finding_count"] if compacted else row["report_findings"]) or 0),
|
||||
blockers=int((row["blocker_count"] if compacted else row["report_blockers"]) or 0),
|
||||
warnings=int((row["warning_count"] if compacted else row["report_warnings"]) or 0),
|
||||
debt_markers=int(row["debt_markers"] or 0),
|
||||
)
|
||||
)
|
||||
return points
|
||||
@@ -1220,11 +1207,29 @@ class ScanStore:
|
||||
row = db.execute("SELECT * FROM scan_jobs WHERE id = ?", (job_id,)).fetchone()
|
||||
return self._job_from_row(row) if row else None
|
||||
|
||||
@staticmethod
|
||||
def _claim_work_may_exist(db: sqlite3.Connection, now: str) -> bool:
|
||||
placeholders = ", ".join("?" for _ in ACTIVE_JOB_STATUSES[1:])
|
||||
row = db.execute(
|
||||
f"""
|
||||
SELECT EXISTS(SELECT 1 FROM scan_jobs WHERE status = 'queued')
|
||||
OR EXISTS(
|
||||
SELECT 1 FROM scan_jobs
|
||||
WHERE status IN ({placeholders}) AND lease_until IS NOT NULL AND lease_until < ?
|
||||
)
|
||||
""",
|
||||
(*ACTIVE_JOB_STATUSES[1:], now),
|
||||
).fetchone()
|
||||
return bool(row[0])
|
||||
|
||||
def claim_next_job(self) -> ScanJob | None:
|
||||
now_dt = _now()
|
||||
now = now_dt.isoformat()
|
||||
lease_until = (now_dt + timedelta(seconds=job_lease_seconds())).isoformat()
|
||||
with self._connection() as db:
|
||||
# Cheap read-only probe: an idle queue never takes the write lock.
|
||||
if not self._claim_work_may_exist(db, now):
|
||||
return None
|
||||
db.execute("BEGIN IMMEDIATE")
|
||||
if self._maintenance_enabled(db):
|
||||
return None
|
||||
@@ -1443,6 +1448,30 @@ class ScanStore:
|
||||
except OSError:
|
||||
return 0
|
||||
|
||||
def _cached_mirror_stats(self, mirrors_root: Path) -> tuple[int, int]:
|
||||
"""Return mirror-cache size without making /meta wait on a large tree walk.
|
||||
|
||||
The walk (up to 50k files) runs synchronously only when no value exists yet.
|
||||
Afterwards an expired value is served while one background thread refreshes it.
|
||||
"""
|
||||
|
||||
key = str(mirrors_root)
|
||||
now = monotonic()
|
||||
cached = self._mirror_stats_cache
|
||||
if cached is None or cached[1] != key:
|
||||
stats = self._tree_stats(mirrors_root)
|
||||
self._mirror_stats_cache = (monotonic(), key, stats)
|
||||
return stats
|
||||
if now - cached[0] >= MIRROR_STATS_TTL_SECONDS and self._mirror_stats_refresh.acquire(blocking=False):
|
||||
def refresh() -> None:
|
||||
try:
|
||||
self._mirror_stats_cache = (monotonic(), key, self._tree_stats(mirrors_root))
|
||||
finally:
|
||||
self._mirror_stats_refresh.release()
|
||||
|
||||
Thread(target=refresh, name="repopulse-mirror-stats", daemon=True).start()
|
||||
return cached[2]
|
||||
|
||||
def storage_stats(self) -> dict[str, int | str | None]:
|
||||
with self._connection() as db:
|
||||
row = db.execute(
|
||||
@@ -1462,7 +1491,7 @@ class ScanStore:
|
||||
cache_root = Path(os.getenv("REPOPULSE_REPOSITORY_CACHE", "data/repos")).expanduser()
|
||||
mirrors_root = cache_root / "mirrors"
|
||||
worktrees_root = cache_root / "worktrees"
|
||||
_mirror_files, mirror_cache_bytes = self._tree_stats(mirrors_root)
|
||||
_mirror_files, mirror_cache_bytes = self._cached_mirror_stats(mirrors_root)
|
||||
mirror_count = self._managed_entry_count(mirrors_root)
|
||||
worktree_count = self._managed_entry_count(worktrees_root)
|
||||
return {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[project]
|
||||
name = "repopulse-backend"
|
||||
version = "1.5.0"
|
||||
version = "1.5.1"
|
||||
description = "RepoPulse repository intelligence API and deterministic analysis engine"
|
||||
requires-python = ">=3.12"
|
||||
dependencies = [
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
import app.store as store_module
|
||||
from app import main as main_module
|
||||
from app.models import Finding, Repository, Severity
|
||||
from app.rules import enrich_finding, enrich_findings
|
||||
from app.store import ScanStore
|
||||
|
||||
|
||||
def test_database_bound_endpoints_never_run_blocking_sqlite_on_the_event_loop() -> None:
|
||||
# A sync endpoint runs in the threadpool; an async one must offload its store work.
|
||||
assert not inspect.iscoroutinefunction(main_module.create_repository_scan)
|
||||
assert not inspect.iscoroutinefunction(main_module._handle_gitea_webhook)
|
||||
webhook_source = inspect.getsource(main_module.gitea_webhook)
|
||||
assert "store." not in webhook_source
|
||||
assert "run_in_threadpool(_handle_gitea_webhook" in webhook_source
|
||||
assert "asyncio.to_thread(store.save" in inspect.getsource(main_module.create_scan)
|
||||
|
||||
|
||||
def test_large_api_responses_are_gzip_compressed(tmp_path: Path) -> None:
|
||||
main_module.store = ScanStore(str(tmp_path / "history.db"))
|
||||
main_module.job_runner = None
|
||||
for index in range(12):
|
||||
main_module.store.upsert_repository(
|
||||
Repository(
|
||||
id=f"gitea:{index}",
|
||||
provider="gitea",
|
||||
provider_repository_id=str(index),
|
||||
owner="Jens",
|
||||
name=f"Repository{index}",
|
||||
full_name=f"Jens/Repository{index}",
|
||||
default_branch="main",
|
||||
)
|
||||
)
|
||||
client = TestClient(main_module.app)
|
||||
|
||||
compressed = client.get("/api/v1/repositories", headers={"Accept-Encoding": "gzip"})
|
||||
small = client.get("/health", headers={"Accept-Encoding": "gzip"})
|
||||
|
||||
assert compressed.status_code == 200
|
||||
assert compressed.headers.get("content-encoding") == "gzip"
|
||||
assert len(compressed.json()) == 12
|
||||
# Responses below the 1 KiB threshold are not worth compressing.
|
||||
assert small.headers.get("content-encoding") is None
|
||||
|
||||
|
||||
def test_hashed_frontend_assets_are_immutable_and_index_revalidates(tmp_path: Path) -> None:
|
||||
(tmp_path / "assets").mkdir()
|
||||
(tmp_path / "index.html").write_text("<html></html>", encoding="utf-8")
|
||||
(tmp_path / "assets" / "index-abc123.js").write_text("console.log(1)", encoding="utf-8")
|
||||
app = FastAPI()
|
||||
app.mount("/", main_module._FrontendStaticFiles(directory=tmp_path, html=True), name="frontend")
|
||||
client = TestClient(app)
|
||||
|
||||
asset = client.get("/assets/index-abc123.js")
|
||||
index = client.get("/")
|
||||
|
||||
assert asset.headers["cache-control"] == "public, max-age=31536000, immutable"
|
||||
assert index.headers["cache-control"] == "no-cache"
|
||||
|
||||
|
||||
def test_finding_guidance_is_shared_and_unchanged() -> None:
|
||||
finding = Finding(
|
||||
code="security.secret-assignment",
|
||||
category="Security basics",
|
||||
severity=Severity.BLOCKER,
|
||||
message="m",
|
||||
deduction=20,
|
||||
)
|
||||
first, second = enrich_findings([finding, finding.model_copy(update={"path": "b.py"})])
|
||||
|
||||
assert first.guidance is second.guidance
|
||||
assert first.guidance.priority == 800
|
||||
assert first.guidance.priority_rationale == "Blocker signal; 20 health-point impact."
|
||||
assert enrich_finding(first) is first
|
||||
|
||||
|
||||
def test_expired_mirror_stats_are_served_while_refreshing_in_background(tmp_path: Path, monkeypatch) -> None:
|
||||
store = ScanStore(str(tmp_path / "history.db"))
|
||||
mirrors = tmp_path / "mirrors"
|
||||
mirrors.mkdir()
|
||||
calls: list[float] = []
|
||||
|
||||
def slow_walk(_root: Path) -> tuple[int, int]:
|
||||
calls.append(time.monotonic())
|
||||
time.sleep(0.3)
|
||||
return len(calls), len(calls) * 100
|
||||
|
||||
monkeypatch.setattr(store, "_tree_stats", slow_walk)
|
||||
assert store._cached_mirror_stats(mirrors) == (1, 100)
|
||||
|
||||
monkeypatch.setattr(store_module, "MIRROR_STATS_TTL_SECONDS", 0.0)
|
||||
started = time.monotonic()
|
||||
assert store._cached_mirror_stats(mirrors) == (1, 100)
|
||||
assert time.monotonic() - started < 0.2
|
||||
|
||||
deadline = time.monotonic() + 3
|
||||
while store._mirror_stats_cache[2] == (1, 100) and time.monotonic() < deadline:
|
||||
time.sleep(0.05)
|
||||
assert store._mirror_stats_cache[2] == (2, 200)
|
||||
@@ -0,0 +1,101 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app.gitea import GiteaClient
|
||||
from app.jobs import ScanJobRunner
|
||||
from app.processes import git_environment, run_bounded
|
||||
from app.store import ScanStore
|
||||
|
||||
|
||||
def test_git_environment_forces_foreground_housekeeping_and_keeps_extra_config() -> None:
|
||||
env = git_environment({}, extra_config=[("http.extraHeader", "Authorization: token t")])
|
||||
|
||||
pairs = {
|
||||
env[f"GIT_CONFIG_KEY_{index}"]: env[f"GIT_CONFIG_VALUE_{index}"]
|
||||
for index in range(int(env["GIT_CONFIG_COUNT"]))
|
||||
}
|
||||
assert pairs == {
|
||||
"gc.autoDetach": "false",
|
||||
"maintenance.autoDetach": "false",
|
||||
"http.extraHeader": "Authorization: token t",
|
||||
}
|
||||
|
||||
|
||||
def test_run_bounded_returns_output_and_raises_on_failure() -> None:
|
||||
ok = run_bounded([sys.executable, "-c", "print('hi')"], timeout=10)
|
||||
assert ok.stdout.strip() == b"hi"
|
||||
with pytest.raises(subprocess.CalledProcessError):
|
||||
run_bounded([sys.executable, "-c", "raise SystemExit(3)"], timeout=10)
|
||||
|
||||
|
||||
@pytest.mark.skipif(os.name != "posix", reason="process groups are POSIX-only")
|
||||
def test_run_bounded_timeout_kills_grandchildren(tmp_path: Path) -> None:
|
||||
marker = tmp_path / "grandchild.pid"
|
||||
script = (
|
||||
"import subprocess, sys, time\n"
|
||||
"child = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(60)'])\n"
|
||||
f"open({str(marker)!r}, 'w').write(str(child.pid))\n"
|
||||
"time.sleep(60)\n"
|
||||
)
|
||||
with pytest.raises(subprocess.TimeoutExpired):
|
||||
run_bounded([sys.executable, "-c", script], timeout=2)
|
||||
|
||||
grandchild = int(marker.read_text())
|
||||
deadline = time.monotonic() + 5
|
||||
while time.monotonic() < deadline:
|
||||
if _process_gone_or_zombie(grandchild):
|
||||
return
|
||||
time.sleep(0.05)
|
||||
pytest.fail("grandchild survived the timeout kill")
|
||||
|
||||
|
||||
def _process_gone_or_zombie(pid: int) -> bool:
|
||||
"""A killed orphan stays a zombie until PID 1 reaps it (tini does in production).
|
||||
|
||||
Inside a CI job container PID 1 may not reap, so a zombie counts as killed.
|
||||
"""
|
||||
try:
|
||||
stat = Path(f"/proc/{pid}/stat").read_text()
|
||||
except (FileNotFoundError, ProcessLookupError):
|
||||
return True
|
||||
# Field 3, after the parenthesised command name, is the process state.
|
||||
return stat.rsplit(")", 1)[1].split()[0] in {"Z", "X"}
|
||||
|
||||
|
||||
def test_idle_claim_never_takes_the_write_lock(tmp_path: Path) -> None:
|
||||
store = ScanStore(str(tmp_path / "repopulse.db"))
|
||||
writer = sqlite3.connect(tmp_path / "repopulse.db", timeout=0)
|
||||
try:
|
||||
writer.execute("BEGIN IMMEDIATE")
|
||||
started = time.monotonic()
|
||||
assert store.claim_next_job() is None
|
||||
# Without the read-only probe this would block on busy_timeout (5 s).
|
||||
assert time.monotonic() - started < 2
|
||||
finally:
|
||||
writer.rollback()
|
||||
writer.close()
|
||||
|
||||
|
||||
def test_idle_worker_backs_off_and_enqueue_wakes_it(tmp_path: Path) -> None:
|
||||
runner = ScanJobRunner(ScanStore(str(tmp_path / "repopulse.db")), gitea=GiteaClient())
|
||||
runner.poll_seconds = 0.5
|
||||
runner.idle_poll_seconds = 5.0
|
||||
|
||||
delays = []
|
||||
for iterations in range(1, 7):
|
||||
runner._idle_iterations = iterations
|
||||
delays.append(runner._idle_delay())
|
||||
assert delays == [0.5, 1.0, 2.0, 4.0, 5.0, 5.0]
|
||||
|
||||
runner.wake()
|
||||
started = time.monotonic()
|
||||
assert runner._wake.wait(1)
|
||||
assert time.monotonic() - started < 0.5
|
||||
@@ -221,3 +221,47 @@ def test_empty_git_repository_does_not_fallback_to_untracked_files(tmp_path: Pat
|
||||
assert report.git.tracked_files == 0
|
||||
assert report.metrics.files == 0
|
||||
assert report.metrics.source_files == 0
|
||||
|
||||
|
||||
def test_single_pass_python_analysis_matches_per_function_branch_counts() -> None:
|
||||
import ast
|
||||
|
||||
from app.scanner import PYTHON_BRANCH_TYPES, _analyze_python
|
||||
|
||||
inner = "\n".join(f" if x == {i}:\n x += 1" for i in range(12))
|
||||
outer = "\n".join(f" for i{i} in range(2):\n pass" for i in range(10))
|
||||
source = (
|
||||
"import os\nfrom . import sibling\n\n"
|
||||
f"def outer(x):\n{outer}\n def inner(x):\n{inner}\n return x\n return inner(x)\n"
|
||||
)
|
||||
findings, imports = _analyze_python("pkg/mod.py", source)
|
||||
|
||||
expected = {
|
||||
node.name: sum(1 for child in ast.walk(node) if isinstance(child, PYTHON_BRANCH_TYPES))
|
||||
for node in ast.walk(ast.parse(source))
|
||||
if isinstance(node, ast.FunctionDef)
|
||||
}
|
||||
assert expected == {"outer": 22, "inner": 12}
|
||||
branch_messages = [item.message for item in findings if item.code == "maintainability.branch-heavy-function"]
|
||||
assert branch_messages == ["Function outer contains 22 branch constructs"]
|
||||
assert imports == ("os", "pkg")
|
||||
|
||||
|
||||
def test_js_import_resolution_is_lexical_and_never_escapes_the_root(tmp_path: Path) -> None:
|
||||
from app.architecture import build_architecture_from_imports
|
||||
|
||||
imports = {
|
||||
"web/app.ts": ("./util", "../../outside/secret", "../web/util.js"),
|
||||
"web/util.ts": (),
|
||||
}
|
||||
graph, _ = build_architecture_from_imports(
|
||||
tmp_path,
|
||||
imports,
|
||||
{path: "TypeScript" for path in imports},
|
||||
{path: 1 for path in imports},
|
||||
{path: 0 for path in imports},
|
||||
{},
|
||||
)
|
||||
|
||||
assert {(edge.source, edge.target) for edge in graph.edges} == {("web/app.ts", "web/util.ts")}
|
||||
assert graph.total_edges == 1
|
||||
@@ -16,7 +16,7 @@ def test_non_git_fallback_applies_root_gitignore_rules(tmp_path: Path) -> None:
|
||||
(tmp_path / "ignored-dir").mkdir()
|
||||
(tmp_path / "ignored-dir" / "nested.py").write_text("print('nested')\n", encoding="utf-8")
|
||||
|
||||
files = {path.relative_to(tmp_path).as_posix() for path in _walk_files(tmp_path)}
|
||||
files = {relative for relative, _path in _walk_files(tmp_path)}
|
||||
|
||||
assert "kept.py" in files
|
||||
assert "keep.log" in files
|
||||
|
||||
@@ -159,3 +159,38 @@ def test_catalog_summary_uses_denormalized_fields_without_report_json_parsing(tm
|
||||
assert storage["scan_count"] == 1
|
||||
assert storage["mirror_count"] >= 0
|
||||
assert storage["worktree_count"] >= 0
|
||||
|
||||
|
||||
def test_history_listings_keep_counts_for_compacted_reports_without_report_parsing(
|
||||
tmp_path: Path, monkeypatch
|
||||
) -> None:
|
||||
repository = tmp_path / "repo"
|
||||
repository.mkdir()
|
||||
(repository / "app.py").write_text("print('hello')\n", encoding="utf-8")
|
||||
monkeypatch.setenv("REPOPULSE_FULL_REPORTS_PER_REPOSITORY", "1")
|
||||
store = ScanStore(str(tmp_path / "repopulse.db"))
|
||||
first = store.save(scan_repository(str(repository)))
|
||||
second = store.save(scan_repository(str(repository)))
|
||||
assert store.get(first.id).compacted is True
|
||||
assert first.findings
|
||||
|
||||
def fail_if_called(*_args, **_kwargs):
|
||||
raise AssertionError("history listings must not deserialize full reports")
|
||||
|
||||
monkeypatch.setattr(store, "_report_from_row", fail_if_called)
|
||||
repository_id = second.source.repository_id
|
||||
active = [item for item in first.findings if not item.suppressed]
|
||||
|
||||
scans = store.repository_scans(repository_id)
|
||||
assert [item.id for item in scans] == [second.id, first.id]
|
||||
assert scans[0].findings == len(second.findings)
|
||||
assert scans[0].duration_ms == second.duration_ms
|
||||
assert scans[0].release_readiness == second.release_readiness
|
||||
# A compacted report keeps its retained summary count instead of reporting zero.
|
||||
assert scans[1].findings == len(active)
|
||||
|
||||
trend = store.trend(repository_id)
|
||||
assert [item.scan_id for item in trend] == [first.id, second.id]
|
||||
assert trend[0].findings == len(active)
|
||||
assert trend[0].warnings == sum(1 for item in active if item.severity == "warning")
|
||||
assert trend[1].findings == len(second.findings)
|
||||
Reference in new issue
Block a user