RepoPulse: update backend/app and dependencies (+3 areas)

Public changes compared with the previous published export:
26 files changed.

Affected areas:
- backend/app: 11 files
- dependencies: 5 files
- deployment and CI: 3 files
- documentation: 2 files
- tests: 5 files

Changed files:
- Updated: "Dockerfile.unraid"
- Updated: "README.md"
- Updated: "backend/Dockerfile"
- Updated: "backend/app/acquisition.py"
- Updated: "backend/app/architecture.py"
- Updated: "backend/app/git_analysis.py"
- Updated: "backend/app/gitea.py"
- Updated: "backend/app/jobs.py"
- Updated: "backend/app/main.py"
- Added: "backend/app/processes.py"
- Updated: "backend/app/rules.py"
- Updated: "backend/app/scanner.py"
- Updated: "backend/app/signals.py"
- Updated: "backend/app/store.py"
- Updated: "backend/pyproject.toml"
- Added: "backend/tests/test_api_responsiveness.py"
- Added: "backend/tests/test_processes.py"
- Updated: "backend/tests/test_scanner.py"
- Updated: "backend/tests/test_signals_v11.py"
- Updated: "backend/tests/test_store.py"
- Updated: "docker-compose.yml"
- Updated: "docs/ARCHITECTURE.md"
- Updated: "frontend/package-lock.json"
- Updated: "frontend/package.json"
- Updated: "package-lock.json"
- Updated: "package.json"

Export provenance/manifests refreshed; excluded from the change summary.

Publish curated RepoPulse source from a251f826238908b156485299061da76482780ed2
This commit is contained in:
Public Source Publisher committed 2026-11-17 21:10:30 +00:00
1 parent 23a4a721f1
commit 61ae49f9d0
28 files changed
+1083 -408

No files matched your search

+3 -1
View File
@@ -16,7 +16,7 @@ ENV PYTHONDONTWRITEBYTECODE=1 \
REPOPULSE_STATIC_DIR=/app/static REPOPULSE_STATIC_DIR=/app/static
RUN apt-get update \ RUN apt-get update \
&& apt-get install -y --no-install-recommends git \ && apt-get install -y --no-install-recommends git tini \
&& rm -rf /var/lib/apt/lists/* && rm -rf /var/lib/apt/lists/*
COPY backend/pyproject.toml ./pyproject.toml COPY backend/pyproject.toml ./pyproject.toml
@@ -26,4 +26,6 @@ COPY --from=frontend-build /frontend/dist ./static
VOLUME ["/data", "/scan"] VOLUME ["/data", "/scan"]
EXPOSE 8000 EXPOSE 8000
# tini is PID 1 so orphaned git descendants are always reaped (no zombies).
ENTRYPOINT ["/usr/bin/tini", "--"]
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"] CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]
+1 -1
View File
@@ -1,3 +1,3 @@
# Public source export # Public source export
This source snapshot was generated from the private canonical repository at revision `d5e4a9e3bf5b0a127b751c9fba917b5e40be41d5`. It contains no private Git history or operational evidence. Changes are published from the canonical repository. This source snapshot was generated from the private canonical repository at revision `a251f826238908b156485299061da76482780ed2`. It contains no private Git history or operational evidence. Changes are published from the canonical repository.
+56 -41
View File
@@ -1,6 +1,6 @@
{ {
"schemaVersion": 1, "schemaVersion": 1,
"sourceRevision": "d5e4a9e3bf5b0a127b751c9fba917b5e40be41d5", "sourceRevision": "a251f826238908b156485299061da76482780ed2",
"files": [ "files": [
{ {
"path": ".dockerignore", "path": ".dockerignore",
@@ -24,18 +24,18 @@
}, },
{ {
"path": "Dockerfile.unraid", "path": "Dockerfile.unraid",
"sha256": "69f829fca37035eb45a73bd4121ffe8e383d61f92813f22a95b555b3f8bf8f60", "sha256": "3abf243a64d45e040aa9c0f5ed3f4727ebfe3b09aa540fe6c03f353780a7e868",
"bytes": 759 "bytes": 875
}, },
{ {
"path": "README.md", "path": "README.md",
"sha256": "a70daf2bbd2ac9fcf74b407901f07d9324ca05a486e117e7eb89d722d35e1ae9", "sha256": "3e4db609694a889bd8f33e3703791e1bfea4058c191fec1a3570b08649ff4418",
"bytes": 5879 "bytes": 5879
}, },
{ {
"path": "backend/Dockerfile", "path": "backend/Dockerfile",
"sha256": "41e92e2246c2811719b0b15911a8d277e9ea2383df12410fc132b0a7fae5a66a", "sha256": "58470a33f7741b5952eff7a7fc76a43cf2859b1418d712af3fc3deefbb00d3b1",
"bytes": 486 "bytes": 602
}, },
{ {
"path": "backend/app/__init__.py", "path": "backend/app/__init__.py",
@@ -44,33 +44,33 @@
}, },
{ {
"path": "backend/app/acquisition.py", "path": "backend/app/acquisition.py",
"sha256": "92beed04a0d6eda5bb7fa9a852343edd510ca510a751f8935fd91a61a8a7f15c", "sha256": "1c1877d1d720c66d62a6a6f518a1752f649f6b0a2fa2fda766c093d39a4f6367",
"bytes": 10788 "bytes": 10499
}, },
{ {
"path": "backend/app/architecture.py", "path": "backend/app/architecture.py",
"sha256": "57eb1be36636fedf599dcd8b88d996165610de07fb2ec047e120c933caa20b07", "sha256": "596fb586c80316dfff4ecec40c57e7e79e93b497fc9bbde7701d07be0dc7228d",
"bytes": 12912 "bytes": 15586
}, },
{ {
"path": "backend/app/git_analysis.py", "path": "backend/app/git_analysis.py",
"sha256": "7530d55350c2acf3b62f8d3842679a05b612d3833fb854165f306a3fa17eefc4", "sha256": "12c4a85dab82def2067a564e6c25e463a7f33c97049e414bef762506f3e5cab9",
"bytes": 3760 "bytes": 3707
}, },
{ {
"path": "backend/app/gitea.py", "path": "backend/app/gitea.py",
"sha256": "b6572bb2a32ecefdd575f40197835229bd8163c5b028ced1c01a02c277542c9c", "sha256": "e4795d7d60fe15fc133b2216a33b11b02086d86c4baa87de0d17d3e55ab0dbbb",
"bytes": 13962 "bytes": 13958
}, },
{ {
"path": "backend/app/jobs.py", "path": "backend/app/jobs.py",
"sha256": "4f11fb25917b5400eadc796227ba0c55ed78489396ec720a0ba63ab8672b1b1d", "sha256": "f4deb3732d995f602980f41623e16bba5255b7125cf1dff03534cea5f571a0b4",
"bytes": 12450 "bytes": 15128
}, },
{ {
"path": "backend/app/main.py", "path": "backend/app/main.py",
"sha256": "281fb5cf04cac0122dc64c55f49dbb99064f26e2608ace679a27c662f04829b3", "sha256": "b8898a7ddb4d328b52d2e7141d50610f6ee4ac27eaa57276d095590cd4d708ba",
"bytes": 19153 "bytes": 20288
}, },
{ {
"path": "backend/app/models.py", "path": "backend/app/models.py",
@@ -82,6 +82,11 @@
"sha256": "a0f05788bbb6f64de261287b19d241ebe116cb79c131b6a255a51037c06c6e6b", "sha256": "a0f05788bbb6f64de261287b19d241ebe116cb79c131b6a255a51037c06c6e6b",
"bytes": 3184 "bytes": 3184
}, },
{
"path": "backend/app/processes.py",
"sha256": "7864e7bbe7128c6484e29b2722a5f3d471d8e4eadb524e35a08a42b5c61f5fa6",
"bytes": 2601
},
{ {
"path": "backend/app/release_gate.py", "path": "backend/app/release_gate.py",
"sha256": "a21927144cc15cf115cd3f42ce84f8d67411e98e9b7ef9f17228c40e0ec9e3fb", "sha256": "a21927144cc15cf115cd3f42ce84f8d67411e98e9b7ef9f17228c40e0ec9e3fb",
@@ -94,27 +99,27 @@
}, },
{ {
"path": "backend/app/rules.py", "path": "backend/app/rules.py",
"sha256": "25accd0a6fe0d25d81e26bbf45ff7da73290ec4a75b85e2eb3e558314ac4cb8d", "sha256": "cd12c89f706fb2b0df9f37ed8a5eb7bfe75f1a76d22bec55fea819570e8ffb24",
"bytes": 3401 "bytes": 3824
}, },
{ {
"path": "backend/app/scanner.py", "path": "backend/app/scanner.py",
"sha256": "0bbe6cdb2f3ce99c46277d53b001549c99a3eb835e7d75e17f485dddbc77700a", "sha256": "ea806338e0c7c6eb4e9daa868e8185c2d918888618f3f1dc8136bc9a1c90af3e",
"bytes": 25747 "bytes": 29045
}, },
{ {
"path": "backend/app/signals.py", "path": "backend/app/signals.py",
"sha256": "65494200f87e6538873fc12a3cfa2eae8838285b3fc3f239ba956955e532df1e", "sha256": "4b4ed3325f88ffb0a382b574ca27280d7f0cbebfcec8d2ffb053be530a3e1817",
"bytes": 19857 "bytes": 20868
}, },
{ {
"path": "backend/app/store.py", "path": "backend/app/store.py",
"sha256": "d7cdaf94afea9b33e720f3a6b4dbf4257c57af402d4977ac0b86c87051a1ff6c", "sha256": "0af079a9c54abc1571caeb78648a6ac08ff4aaf9c20cc11290d216588135b5a8",
"bytes": 68791 "bytes": 70464
}, },
{ {
"path": "backend/pyproject.toml", "path": "backend/pyproject.toml",
"sha256": "48280f85cbc8a9f44acab05f7bdc76825597ad7438bee7012bcb2be66cd3b6b2", "sha256": "3efd954e6b6493bc707483a019d0aea5cb0e3ba573e6dd42d5fdeda6e91b03f6",
"bytes": 446 "bytes": 446
}, },
{ {
@@ -122,6 +127,11 @@
"sha256": "452151143df1130f26efb1ee6554d2ffe22a0c301ef603c1cf5ac902a26ad932", "sha256": "452151143df1130f26efb1ee6554d2ffe22a0c301ef603c1cf5ac902a26ad932",
"bytes": 4773 "bytes": 4773
}, },
{
"path": "backend/tests/test_api_responsiveness.py",
"sha256": "66399c27f86c33f3798fa2a670eab25439a774a1cab981be3e95f486c4264355",
"bytes": 4182
},
{ {
"path": "backend/tests/test_api_v11.py", "path": "backend/tests/test_api_v11.py",
"sha256": "d7b34896a0fb7a3c87f42810d4d598f973995b8a44d5296153f9f6be4d58ff83", "sha256": "d7b34896a0fb7a3c87f42810d4d598f973995b8a44d5296153f9f6be4d58ff83",
@@ -152,6 +162,11 @@
"sha256": "e41669f8acf5f6fe80097d24281caf9732515263e32870adf56cbba2611528ca", "sha256": "e41669f8acf5f6fe80097d24281caf9732515263e32870adf56cbba2611528ca",
"bytes": 2786 "bytes": 2786
}, },
{
"path": "backend/tests/test_processes.py",
"sha256": "f7495ae414b0ce04dcc7bde4f911165291330e9562eb3b8836c776c51aec7fa1",
"bytes": 3487
},
{ {
"path": "backend/tests/test_reporting.py", "path": "backend/tests/test_reporting.py",
"sha256": "1d46130c2b08b936cdf8a5a32219b9e7722e08348ff01d37335de0aaed6daeb6", "sha256": "1d46130c2b08b936cdf8a5a32219b9e7722e08348ff01d37335de0aaed6daeb6",
@@ -159,8 +174,8 @@
}, },
{ {
"path": "backend/tests/test_scanner.py", "path": "backend/tests/test_scanner.py",
"sha256": "e750da9c9a2ff98064bcd20ee666fcc2bb50cd90b0bfe2cc870e9fc979f1a6ed", "sha256": "18f740260217d62ff71319dbd746c51f34160b8c30f227188841768bd201cbf8",
"bytes": 9629 "bytes": 11341
}, },
{ {
"path": "backend/tests/test_signals.py", "path": "backend/tests/test_signals.py",
@@ -169,13 +184,13 @@
}, },
{ {
"path": "backend/tests/test_signals_v11.py", "path": "backend/tests/test_signals_v11.py",
"sha256": "f25ee36ddb02b99a36283535f9ab4aa58734452ca3a306fa1a685e7c06191bfd", "sha256": "f7afe0d14dac93feaadfcfa9041085e7439fbe3bacfd52581ec4cf66d1a6c52e",
"bytes": 2992 "bytes": 2974
}, },
{ {
"path": "backend/tests/test_store.py", "path": "backend/tests/test_store.py",
"sha256": "69dbf585d6a801036f9c87ea99c37ad16d15b4c8df75b6c78ba81ee790a0a3f8", "sha256": "f8fc769bbc064f5a0886d7350da89e6cba1b602018114278a9969c530cb44cad",
"bytes": 6737 "bytes": 8338
}, },
{ {
"path": "compose.yaml", "path": "compose.yaml",
@@ -184,13 +199,13 @@
}, },
{ {
"path": "docker-compose.yml", "path": "docker-compose.yml",
"sha256": "002a1126ea5a6131de9bbf4f11eb6a3eaae7097bd20bcaccffbf694d3f6003c5", "sha256": "efa0684eb4d79d9eab4afee7d73bb68b60ec397f96b99b428310bccc62501cfa",
"bytes": 2438 "bytes": 2484
}, },
{ {
"path": "docs/ARCHITECTURE.md", "path": "docs/ARCHITECTURE.md",
"sha256": "3431db93b6ebce5d24f224d7a62de1fef47fa38f055ac0ea5f61b3ee8992ed82", "sha256": "41253e1aa1f0a816316bdf2e0d559575beb3936c538509d4d7becd4e2cd7f012",
"bytes": 13328 "bytes": 15543
}, },
{ {
"path": "docs/screenshots/demo-explorer.png", "path": "docs/screenshots/demo-explorer.png",
@@ -234,12 +249,12 @@
}, },
{ {
"path": "frontend/package-lock.json", "path": "frontend/package-lock.json",
"sha256": "f12d141b6ea3655dc9ac2843f7534e5f76838e490a9a08ac839570ed88598218", "sha256": "afeb669147a6a98f76c28f7f2a706595b05db881a0c76a0d7c15348c64717d4b",
"bytes": 113084 "bytes": 113084
}, },
{ {
"path": "frontend/package.json", "path": "frontend/package.json",
"sha256": "0bc4b2f7c0406d8f8faada4991737cb625c74c1f2e8304992daf68d486adbfa9", "sha256": "ac706aaddfd7774ce5f0b302b3640c7570eb06c2505709c0277d97a31cbfcb2c",
"bytes": 724 "bytes": 724
}, },
{ {
@@ -429,12 +444,12 @@
}, },
{ {
"path": "package-lock.json", "path": "package-lock.json",
"sha256": "f293d47c364d320785df5fba11c3ea6db8f38cf487345a0a32f7932309076427", "sha256": "9479d451836a22b5299b96a4108b47962a42cd1bb76528486447bcffc6aabdf2",
"bytes": 201 "bytes": 201
}, },
{ {
"path": "package.json", "path": "package.json",
"sha256": "f56ba0a04cedf9ba3600fb75fb736e3712f340381f2a593ba503cbeeafd92b89", "sha256": "18c0e87a4d7c2dba357c1cdac7728a6ca7983d098f2e71a2a055152eccf2c928",
"bytes": 246 "bytes": 246
}, },
{ {
+1 -1
View File
@@ -91,6 +91,6 @@ Run `python -m pytest`, `npm --prefix frontend test`, `npm --prefix frontend run
## Source distribution and licensing ## Source distribution and licensing
This is the curated public source edition of RepoPulse 1.5.0. Reviewed updates flow from the private canonical repository after merges to `main`, without private history, repository inventories, runtime stores or internal CI/deployment workflows. Configure the generic deployment templates for your own environment. This is the curated public source edition of RepoPulse 1.5.1. Reviewed updates flow from the private canonical repository after merges to `main`, without private history, repository inventories, runtime stores or internal CI/deployment workflows. Configure the generic deployment templates for your own environment.
No project-wide license is included. Public visibility does not grant reuse or redistribution rights; obtain the owner's terms and review third-party dependencies before redistribution. No project-wide license is included. Public visibility does not grant reuse or redistribution rights; obtain the owner's terms and review third-party dependencies before redistribution.
+3 -1
View File
@@ -7,7 +7,7 @@ ENV PYTHONDONTWRITEBYTECODE=1 \
REPOPULSE_SCAN_ROOT=/scan REPOPULSE_SCAN_ROOT=/scan
RUN apt-get update \ RUN apt-get update \
&& apt-get install -y --no-install-recommends git \ && apt-get install -y --no-install-recommends git tini \
&& rm -rf /var/lib/apt/lists/* && rm -rf /var/lib/apt/lists/*
COPY backend/pyproject.toml ./pyproject.toml COPY backend/pyproject.toml ./pyproject.toml
@@ -16,4 +16,6 @@ RUN pip install --no-cache-dir .
VOLUME ["/data", "/scan"] VOLUME ["/data", "/scan"]
EXPOSE 8000 EXPOSE 8000
# tini is PID 1 so orphaned git descendants are always reaped (no zombies).
ENTRYPOINT ["/usr/bin/tini", "--"]
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"] CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]
+5 -20
View File
@@ -10,6 +10,7 @@ from threading import Lock, RLock
from urllib.parse import urlsplit from urllib.parse import urlsplit
from .models import Repository from .models import Repository
from .processes import git_environment, run_bounded
SAFE_REVISION = re.compile(r"^[0-9a-fA-F]{7,64}$") SAFE_REVISION = re.compile(r"^[0-9a-fA-F]{7,64}$")
SAFE_BRANCH = re.compile(r"^[A-Za-z0-9._/-]+$") SAFE_BRANCH = re.compile(r"^[A-Za-z0-9._/-]+$")
@@ -72,35 +73,19 @@ class GitAcquirer:
return self._repository_locks.setdefault(repository_id, RLock()) return self._repository_locks.setdefault(repository_id, RLock())
def _git_env(self) -> dict[str, str]: def _git_env(self) -> dict[str, str]:
env = os.environ.copy() extra = [("http.extraHeader", f"Authorization: token {self.token}")] if self.token else []
if self.token: return git_environment(extra_config=extra)
env.update(
{
"GIT_CONFIG_COUNT": "1",
"GIT_CONFIG_KEY_0": "http.extraHeader",
"GIT_CONFIG_VALUE_0": f"Authorization: token {self.token}",
}
)
return env
def _run(self, args: list[str]) -> str: def _run(self, args: list[str]) -> str:
try: try:
result = subprocess.run( result = run_bounded(args, timeout=self.timeout, env=self._git_env())
args,
check=True,
capture_output=True,
text=True,
errors="replace",
timeout=self.timeout,
env=self._git_env(),
)
except subprocess.TimeoutExpired as exc: except subprocess.TimeoutExpired as exc:
raise AcquisitionError("timeout", "Git source acquisition timed out") from exc raise AcquisitionError("timeout", "Git source acquisition timed out") from exc
except (OSError, subprocess.CalledProcessError) as exc: except (OSError, subprocess.CalledProcessError) as exc:
# Git stderr may contain remote URLs or server-specific details; do # Git stderr may contain remote URLs or server-specific details; do
# not persist or expose it because the job API is user-visible. # not persist or expose it because the job API is user-visible.
raise AcquisitionError("git_failed", "Git source acquisition failed") from exc raise AcquisitionError("git_failed", "Git source acquisition failed") from exc
return result.stdout.strip() return result.stdout.decode("utf-8", errors="replace").strip()
def _mirror_path(self, repository: Repository) -> Path: def _mirror_path(self, repository: Repository) -> Path:
identity = f"{repository.provider}-{repository.provider_repository_id}" identity = f"{repository.provider}-{repository.provider_repository_id}"
+138 -68
View File
@@ -4,6 +4,7 @@ import ast
import json import json
import math import math
import os import os
import posixpath
import re import re
from collections import Counter from collections import Counter
from pathlib import Path from pathlib import Path
@@ -66,8 +67,10 @@ def _ts_path_aliases(root: Path) -> list[tuple[str, str]]:
return aliases[:100] return aliases[:100]
def _workspace_packages(root: Path) -> dict[str, Path]: def _workspace_packages(root: Path) -> dict[str, str]:
packages: dict[str, Path] = {} """Map local package names to their root-relative POSIX directory."""
packages: dict[str, str] = {}
for manifest in _bounded_config_files(root, {"package.json"}, maximum=100): for manifest in _bounded_config_files(root, {"package.json"}, maximum=100):
try: try:
if manifest.stat().st_size > 256_000: if manifest.stat().st_size > 256_000:
@@ -75,7 +78,7 @@ def _workspace_packages(root: Path) -> dict[str, Path]:
data = json.loads(manifest.read_text(encoding="utf-8")) data = json.loads(manifest.read_text(encoding="utf-8"))
name = data.get("name") if isinstance(data, dict) else None name = data.get("name") if isinstance(data, dict) else None
if isinstance(name, str) and name and not name.startswith((".", "/")): if isinstance(name, str) and name and not name.startswith((".", "/")):
packages.setdefault(name, manifest.parent) packages.setdefault(name, manifest.parent.relative_to(root).as_posix())
except (OSError, ValueError, json.JSONDecodeError, TypeError): except (OSError, ValueError, json.JSONDecodeError, TypeError):
continue continue
return packages return packages
@@ -116,79 +119,118 @@ def _resolve_python_target(module: str, module_map: dict[str, str]) -> str | Non
return None return None
def _python_imports(path: str, text: str, module_map: dict[str, str]) -> set[str]: def python_package_parts(path: str) -> list[str]:
try: """Return the package that relative imports in ``path`` resolve against."""
tree = ast.parse(text)
except SyntaxError:
return set()
current_names = _module_names(path) current_names = _module_names(path)
current = current_names[0] if current_names else "" current = current_names[0] if current_names else ""
current_parts = current.split(".") if current else [] current_parts = current.split(".") if current else []
if not path.endswith("/__init__.py") and current_parts: if not path.endswith("/__init__.py") and current_parts:
package_parts = current_parts[:-1] return current_parts[:-1]
else: return current_parts
package_parts = current_parts
imports: set[str] = set()
def python_import_modules(node: ast.AST, package_parts: list[str]) -> list[str]:
"""Return the absolute (still unresolved) module names referenced by one import node."""
if isinstance(node, ast.Import):
return [alias.name for alias in node.names]
if isinstance(node, ast.ImportFrom):
module = node.module or ""
if node.level:
trim = max(0, node.level - 1)
prefix = package_parts[:-trim] if trim else package_parts
return [".".join([*prefix, *([module] if module else [])])]
return [module]
return []
def extract_python_imports(path: str, tree: ast.AST) -> tuple[str, ...]:
package_parts = python_package_parts(path)
modules: dict[str, None] = {}
for node in ast.walk(tree): for node in ast.walk(tree):
if isinstance(node, ast.Import): if isinstance(node, (ast.Import, ast.ImportFrom)):
for alias in node.names: for module in python_import_modules(node, package_parts):
target = _resolve_python_target(alias.name, module_map) modules.setdefault(module)
if target and target != path: return tuple(modules)
imports.add(target)
elif isinstance(node, ast.ImportFrom):
module = node.module or "" def extract_js_specifiers(text: str) -> tuple[str, ...]:
if node.level: return tuple(dict.fromkeys(match.group("target") for match in JS_IMPORT_RE.finditer(text)))
trim = max(0, node.level - 1)
prefix = package_parts[:-trim] if trim else package_parts
absolute = ".".join([*prefix, *([module] if module else [])]) def extract_imports(path: str, text: str, language: str) -> tuple[str, ...]:
else: """Extract compact, unresolved import references so source text need not be retained."""
absolute = module
target = _resolve_python_target(absolute, module_map) if language == "Python":
if target and target != path: try:
imports.add(target) tree = ast.parse(text)
return imports except (SyntaxError, ValueError):
return ()
return extract_python_imports(path, tree)
if language in {"TypeScript", "JavaScript"}:
return extract_js_specifiers(text)
return ()
def _lexical_relative(path: str) -> str | None:
"""Normalize a root-relative POSIX path without filesystem access.
Candidates are only matched against already safety-checked scanner paths, so
symlink resolution adds nothing; a path escaping the root yields no target.
"""
if not path or path.startswith("/"):
return None
normalized = posixpath.normpath(path)
if normalized == ".." or normalized.startswith("../"):
return None
return normalized
def _resolve_js_target( def _resolve_js_target(
root: Path,
source_path: str, source_path: str,
specifier: str, specifier: str,
known: set[str], known: set[str],
aliases: list[tuple[str, str]], aliases: list[tuple[str, str]],
workspace_packages: dict[str, Path], workspace_packages: dict[str, str],
) -> str | None: ) -> str | None:
relative_base: str | None = None
if specifier.startswith("."): if specifier.startswith("."):
source_dir = (root / source_path).parent relative_base = _lexical_relative(posixpath.join(posixpath.dirname(source_path), specifier))
base = (source_dir / specifier).resolve()
else: else:
base = None matched_alias = False
for alias, target in aliases: for alias, target in aliases:
if alias.endswith("/*") and specifier.startswith(alias[:-1]): if alias.endswith("/*") and specifier.startswith(alias[:-1]):
base = (root / target.replace("*", specifier[len(alias) - 1 :])).resolve() relative_base = _lexical_relative(target.replace("*", specifier[len(alias) - 1 :]))
matched_alias = True
break break
if alias == specifier: if alias == specifier:
base = (root / target).resolve() relative_base = _lexical_relative(target)
matched_alias = True
break break
if base is None: if not matched_alias:
package_name = next( package_name = next(
(name for name in workspace_packages if specifier == name or specifier.startswith(f"{name}/")), (
name
for name in workspace_packages
if specifier == name or specifier.startswith(f"{name}/")
),
None, None,
) )
if package_name: if package_name:
package_root = workspace_packages[package_name] package_root = workspace_packages[package_name]
suffix = specifier.removeprefix(package_name).lstrip("/") suffix = specifier.removeprefix(package_name).lstrip("/")
base = package_root / suffix if suffix else package_root / "src" / "index" relative_base = _lexical_relative(
if base is None: posixpath.join(package_root, suffix)
return None if suffix
try: else posixpath.join(package_root, "src", "index")
relative_base = base.relative_to(root.resolve()).as_posix() )
except ValueError: if relative_base is None:
return None return None
candidates = [relative_base] candidates = [relative_base]
suffix = Path(relative_base).suffix.lower() suffix = posixpath.splitext(relative_base)[1].lower()
if suffix in JS_EXTENSIONS: if suffix in JS_EXTENSIONS:
extensionless = relative_base[: -len(suffix)] extensionless = relative_base[: -len(suffix)]
candidates.extend(extensionless + extension for extension in JS_EXTENSIONS) candidates.extend(extensionless + extension for extension in JS_EXTENSIONS)
@@ -199,22 +241,6 @@ def _resolve_js_target(
return next((candidate for candidate in candidates if candidate in known), None) return next((candidate for candidate in candidates if candidate in known), None)
def _js_imports(
root: Path,
path: str,
text: str,
known: set[str],
aliases: list[tuple[str, str]],
workspace_packages: dict[str, Path],
) -> set[str]:
imports: set[str] = set()
for match in JS_IMPORT_RE.finditer(text):
target = _resolve_js_target(root, path, match.group("target"), known, aliases, workspace_packages)
if target and target != path:
imports.add(target)
return imports
def build_architecture( def build_architecture(
root: Path, root: Path,
source_texts: dict[str, str], source_texts: dict[str, str],
@@ -226,27 +252,71 @@ def build_architecture(
max_nodes: int = 160, max_nodes: int = 160,
max_edges: int = 600, max_edges: int = 600,
) -> tuple[ArchitectureGraph, list[Hotspot]]: ) -> tuple[ArchitectureGraph, list[Hotspot]]:
"""Compatibility entry point that accepts full source texts."""
imports_by_path = {
path: extract_imports(path, text, language_by_path.get(path, ""))
for path, text in source_texts.items()
}
return build_architecture_from_imports(
root,
imports_by_path,
language_by_path,
loc_by_path,
debt_by_path,
churn_by_path,
max_nodes=max_nodes,
max_edges=max_edges,
)
def build_architecture_from_imports(
root: Path,
imports_by_path: dict[str, tuple[str, ...]],
language_by_path: dict[str, str],
loc_by_path: dict[str, int],
debt_by_path: dict[str, int],
churn_by_path: dict[str, int],
*,
max_nodes: int = 160,
max_edges: int = 600,
) -> tuple[ArchitectureGraph, list[Hotspot]]:
"""Build the bounded graph from compact import references; no source text is retained."""
max_nodes = max(1, max_nodes) max_nodes = max(1, max_nodes)
max_edges = max(0, max_edges) max_edges = max(0, max_edges)
known_paths = set(source_texts) known_paths = set(imports_by_path)
module_map = _python_module_map(list(known_paths)) module_map = _python_module_map(list(known_paths))
ts_aliases = _ts_path_aliases(root) ts_aliases = _ts_path_aliases(root)
workspace_packages = _workspace_packages(root) workspace_packages = _workspace_packages(root)
js_cache: dict[tuple[str, str], str | None] = {}
edge_pairs: set[tuple[str, str]] = set() edge_pairs: set[tuple[str, str]] = set()
total_inbound: Counter[str] = Counter() total_inbound: Counter[str] = Counter()
total_outbound: Counter[str] = Counter() total_outbound: Counter[str] = Counter()
total_edges = 0 total_edges = 0
truncated = len(known_paths) > max_nodes truncated = len(known_paths) > max_nodes
for path in sorted(source_texts): for path in sorted(imports_by_path):
text = source_texts[path] references = imports_by_path[path]
language = language_by_path.get(path, "") language = language_by_path.get(path, "")
targets: set[str] = set()
if language == "Python": if language == "Python":
targets = _python_imports(path, text, module_map) for module in references:
target = _resolve_python_target(module, module_map)
if target and target != path:
targets.add(target)
elif language in {"TypeScript", "JavaScript"}: elif language in {"TypeScript", "JavaScript"}:
targets = _js_imports(root, path, text, known_paths, ts_aliases, workspace_packages) directory = posixpath.dirname(path)
else: for specifier in references:
targets = set() # Only relative specifiers depend on the importing directory.
key = (directory if specifier.startswith(".") else "", specifier)
if key not in js_cache:
js_cache[key] = _resolve_js_target(
path, specifier, known_paths, ts_aliases, workspace_packages
)
target = js_cache[key]
if target and target != path:
targets.add(target)
for target in sorted(targets): for target in sorted(targets):
if target not in known_paths: if target not in known_paths:
continue continue
+4 -10
View File
@@ -6,21 +6,15 @@ from datetime import datetime
from pathlib import Path from pathlib import Path
from .models import GitSignals from .models import GitSignals
from .processes import git_environment, run_bounded
def _run(root: Path, *args: str, timeout: int = 8) -> str | None: def _run(root: Path, *args: str, timeout: int = 8) -> str | None:
try: try:
result = subprocess.run( result = run_bounded(["git", "-C", str(root), *args], timeout=timeout, env=git_environment())
["git", "-C", str(root), *args], except (OSError, subprocess.SubprocessError):
check=True,
capture_output=True,
text=True,
errors="replace",
timeout=timeout,
)
except (OSError, UnicodeError, subprocess.SubprocessError):
return None return None
return result.stdout return result.stdout.decode("utf-8", errors="replace")
def analyze_git(root: Path) -> tuple[GitSignals, dict[str, int]]: def analyze_git(root: Path) -> tuple[GitSignals, dict[str, int]]:
+2 -2
View File
@@ -238,7 +238,7 @@ class GiteaClient:
listed, discovered, branch_errors = self.discover_with_errors() listed, discovered, branch_errors = self.discover_with_errors()
except GiteaError as exc: except GiteaError as exc:
error = f"{exc.code}: {exc.detail}" error = f"{exc.code}: {exc.detail}"
for existing in store.list_repositories(limit=1000): for existing in store.list_repositories(limit=-1):
if existing.provider == "gitea" and existing.available: if existing.provider == "gitea" and existing.available:
store.update_repository_sync(existing, synced_at=synced_at, error=error) store.update_repository_sync(existing, synced_at=synced_at, error=error)
result = RepositorySyncResult( result = RepositorySyncResult(
@@ -290,7 +290,7 @@ class GiteaClient:
updated += 1 updated += 1
unavailable = 0 unavailable = 0
if self._listing_complete: if self._listing_complete:
for existing in store.list_repositories(limit=1000): for existing in store.list_repositories(limit=-1):
if existing.provider == "gitea" and existing.id not in listed_ids and existing.available: if existing.provider == "gitea" and existing.id not in listed_ids and existing.available:
store.update_repository_sync( store.update_repository_sync(
existing.model_copy(update={"available": False}), existing.model_copy(update={"available": False}),
+82 -7
View File
@@ -1,7 +1,12 @@
from __future__ import annotations from __future__ import annotations
import ctypes
import ctypes.util
import gc
import logging import logging
import os import os
import sys
from collections.abc import Callable
from datetime import UTC, datetime from datetime import UTC, datetime
from threading import Event, Lock, Thread, current_thread from threading import Event, Lock, Thread, current_thread
from time import monotonic from time import monotonic
@@ -21,6 +26,40 @@ from .scanner import scan_repository
from .store import MaintenanceModeError, QueueFullError, ScanStore, job_lease_seconds from .store import MaintenanceModeError, QueueFullError, ScanStore, job_lease_seconds
LOGGER = logging.getLogger(__name__) LOGGER = logging.getLogger(__name__)
WORKER_HEARTBEAT_SECONDS = 5.0
WORKER_IDLE_POLL_SECONDS = 5.0
def _load_malloc_trim() -> Callable[[int], int] | None:
if not sys.platform.startswith("linux"):
return None
try:
libc = ctypes.CDLL(ctypes.util.find_library("c") or "libc.so.6")
trim = libc.malloc_trim
except (AttributeError, OSError):
return None # e.g. musl libc has no malloc_trim
trim.argtypes = [ctypes.c_size_t]
trim.restype = ctypes.c_int
return trim
_MALLOC_TRIM = _load_malloc_trim()
def release_memory() -> None:
"""Return memory freed by a finished scan to the OS.
A scan allocates many short-lived objects (ASTs, file texts, findings). CPython frees
them, but glibc keeps the arenas mapped, so the long-running worker's RSS would stay
at its largest scan. A collection plus malloc_trim hands those pages back.
"""
gc.collect()
if _MALLOC_TRIM is not None:
try:
_MALLOC_TRIM(0)
except OSError:
pass
def _positive_int(name: str, default: int, *, maximum: int | None = None) -> int: def _positive_int(name: str, default: int, *, maximum: int | None = None) -> int:
@@ -60,8 +99,17 @@ class ScanJobRunner:
) )
self.max_queue = max_queue or _positive_int("REPOPULSE_MAX_QUEUED_SCANS", 100, maximum=1000) self.max_queue = max_queue or _positive_int("REPOPULSE_MAX_QUEUED_SCANS", 100, maximum=1000)
self._stop = Event() self._stop = Event()
self._wake = Event()
self._idle_iterations = 0
self.idle_poll_seconds = max(
self.poll_seconds, _positive_float("REPOPULSE_WORKER_IDLE_POLL_SECONDS", WORKER_IDLE_POLL_SECONDS)
)
self._thread: Thread | None = None self._thread: Thread | None = None
self._last_sync = float("-inf") self._last_sync = float("-inf")
self._last_heartbeat = float("-inf")
self.heartbeat_interval_seconds = _positive_float(
"REPOPULSE_WORKER_HEARTBEAT_SECONDS", WORKER_HEARTBEAT_SECONDS
)
self._sync_lock = Lock() self._sync_lock = Lock()
@property @property
@@ -72,11 +120,13 @@ class ScanJobRunner:
if self.started: if self.started:
return return
self._stop.clear() self._stop.clear()
self._wake.clear()
self._thread = Thread(target=self._run, name="repopulse-scan-worker", daemon=True) self._thread = Thread(target=self._run, name="repopulse-scan-worker", daemon=True)
self._thread.start() self._thread.start()
def stop(self) -> None: def stop(self) -> None:
self._stop.set() self._stop.set()
self._wake.set()
thread = self._thread thread = self._thread
if thread and thread is not current_thread(): if thread and thread is not current_thread():
thread.join(timeout=5) thread.join(timeout=5)
@@ -94,19 +144,39 @@ class ScanJobRunner:
LOGGER.exception("RepoPulse worker loop recovered from an unexpected failure") LOGGER.exception("RepoPulse worker loop recovered from an unexpected failure")
job = None job = None
if job is None: if job is None:
self._stop.wait(self.poll_seconds) self._idle_iterations += 1
self._wake.wait(self._idle_delay())
self._wake.clear()
else:
self._idle_iterations = 0
def _idle_delay(self) -> float:
"""Back off exponentially while idle; in-process enqueues wake the loop at once."""
exponent = min(self._idle_iterations - 1, 16)
return min(self.idle_poll_seconds, self.poll_seconds * (2**exponent))
def wake(self) -> None:
self._wake.set()
def _run_iteration(self) -> ScanJob | None: def _run_iteration(self) -> ScanJob | None:
"""Perform one scheduler pass unless durable maintenance has paused work.""" """Perform one scheduler pass unless durable maintenance has paused work."""
if self.store.maintenance_status()["enabled"]: if self.store.maintenance_status()["enabled"]:
return None return None
if not self.store.touch_worker_heartbeat_if_active(datetime.now(UTC).isoformat()): # The heartbeat is a liveness signal, not a lock: claims and enqueues re-check
return None # maintenance inside their own transactions, so writing it on every 0.5 s poll
# only churns the WAL. Throttle the write to the heartbeat interval.
now = monotonic()
if now - self._last_heartbeat >= self.heartbeat_interval_seconds:
if not self.store.touch_worker_heartbeat_if_active(datetime.now(UTC).isoformat()):
return None
self._last_heartbeat = now
if self.gitea.configured and monotonic() - self._last_sync >= self.sync_interval_seconds: if self.gitea.configured and monotonic() - self._last_sync >= self.sync_interval_seconds:
self.sync_now() self.sync_now()
if self.store.maintenance_status()["enabled"]: # Synchronization can take a while; re-check before claiming new work.
return None if self.store.maintenance_status()["enabled"]:
return None
return self.process_one() return self.process_one()
def sync_now(self) -> RepositorySyncResult: def sync_now(self) -> RepositorySyncResult:
@@ -135,7 +205,8 @@ class ScanJobRunner:
def _schedule_changed_repositories(self) -> None: def _schedule_changed_repositories(self) -> None:
if self.store.maintenance_status()["enabled"]: if self.store.maintenance_status()["enabled"]:
return return
for summary in self.store.list_repositories(limit=1000): # No silent catalog cap: every repository is reconciled, however large the catalog.
for summary in self.store.list_repositories(limit=-1):
if summary.provider != "gitea" or summary.archived or not summary.latest_revision: if summary.provider != "gitea" or summary.archived or not summary.latest_revision:
continue continue
if not summary.policy.auto_scan: if not summary.policy.auto_scan:
@@ -166,13 +237,15 @@ class ScanJobRunner:
trigger: ScanTrigger = ScanTrigger.MANUAL, trigger: ScanTrigger = ScanTrigger.MANUAL,
) -> ScanJob: ) -> ScanJob:
requested_revision = repository.latest_revision requested_revision = repository.latest_revision
return self.store.enqueue_job( job = self.store.enqueue_job(
repository.id, repository.id,
requested_revision, requested_revision,
trigger, trigger,
force=force, force=force,
max_queue=self.max_queue, max_queue=self.max_queue,
) )
self.wake()
return job
def process_one(self) -> ScanJob | None: def process_one(self) -> ScanJob | None:
job = self.store.claim_next_job() job = self.store.claim_next_job()
@@ -194,6 +267,8 @@ class ScanJobRunner:
) )
except Exception: except Exception:
LOGGER.exception("RepoPulse could not persist an unexpected job failure") LOGGER.exception("RepoPulse could not persist an unexpected job failure")
finally:
release_memory()
return self.store.get_job(job.id) return self.store.get_job(job.id)
class _LeaseHeartbeat: class _LeaseHeartbeat:
+29 -6
View File
@@ -11,8 +11,10 @@ from threading import BoundedSemaphore, Lock
from fastapi import FastAPI, HTTPException, Query, Request, Response, status from fastapi import FastAPI, HTTPException, Query, Request, Response, status
from fastapi.concurrency import run_in_threadpool from fastapi.concurrency import run_in_threadpool
from fastapi.middleware.gzip import GZipMiddleware
from fastapi.responses import JSONResponse from fastapi.responses import JSONResponse
from fastapi.staticfiles import StaticFiles from fastapi.staticfiles import StaticFiles
from starlette.types import Scope
from .acquisition import GitAcquirer from .acquisition import GitAcquirer
from .gitea import GiteaClient from .gitea import GiteaClient
@@ -35,7 +37,7 @@ from .reporting import content_disposition, render_html_report, safe_report_file
from .scanner import scan_repository from .scanner import scan_repository
from .store import MaintenanceModeError, QueueFullError, ScanStore from .store import MaintenanceModeError, QueueFullError, ScanStore
VERSION = "1.5.0" VERSION = "1.5.1"
store = ScanStore() store = ScanStore()
SCAN_ROOT = os.getenv("REPOPULSE_SCAN_ROOT") SCAN_ROOT = os.getenv("REPOPULSE_SCAN_ROOT")
@@ -82,6 +84,8 @@ async def lifespan(_app: FastAPI):
app = FastAPI(title="RepoPulse API", version=VERSION, lifespan=lifespan) app = FastAPI(title="RepoPulse API", version=VERSION, lifespan=lifespan)
# Catalog, report and bundle payloads are large, repetitive JSON/JS; compress them.
app.add_middleware(GZipMiddleware, minimum_size=1024, compresslevel=5)
@app.middleware("http") @app.middleware("http")
@@ -167,7 +171,7 @@ def meta() -> dict[str, object]:
return { return {
"name": "RepoPulse", "name": "RepoPulse",
"version": VERSION, "version": VERSION,
"stage": "v1.5.0", "stage": "v1.5.1",
"scan_root_restricted": bool(SCAN_ROOT), "scan_root_restricted": bool(SCAN_ROOT),
"scan_root": SCAN_ROOT, "scan_root": SCAN_ROOT,
"max_concurrent_scans": MAX_CONCURRENT_SCANS, "max_concurrent_scans": MAX_CONCURRENT_SCANS,
@@ -242,7 +246,8 @@ async def create_scan(payload: ScanRequest, request: Request) -> ScanReport:
raise HTTPException(status_code=422, detail="Repository path cannot be scanned") from exc raise HTTPException(status_code=422, detail="Repository path cannot be scanned") from exc
finally: finally:
SCAN_SLOTS.release() SCAN_SLOTS.release()
return store.save(report) # SQLite writes may wait on busy_timeout; never block the event loop with them.
return await asyncio.to_thread(store.save, report)
@app.get("/api/v1/repositories", response_model=list[RepositorySummary]) @app.get("/api/v1/repositories", response_model=list[RepositorySummary])
@@ -296,7 +301,7 @@ def repository_trend(
@app.post("/api/v1/repositories/{repository_id}/scans", response_model=ScanJob, status_code=202) @app.post("/api/v1/repositories/{repository_id}/scans", response_model=ScanJob, status_code=202)
async def create_repository_scan( def create_repository_scan(
request: Request, request: Request,
repository_id: str, repository_id: str,
payload: RepositoryScanRequest | None = None, payload: RepositoryScanRequest | None = None,
@@ -357,6 +362,12 @@ async def gitea_webhook(request: Request) -> WebhookResponse:
raise HTTPException(status_code=400, detail="Webhook payload is invalid JSON") from exc raise HTTPException(status_code=400, detail="Webhook payload is invalid JSON") from exc
if not isinstance(payload, dict): if not isinstance(payload, dict):
raise HTTPException(status_code=400, detail="Webhook payload must be an object") raise HTTPException(status_code=400, detail="Webhook payload must be an object")
event = request.headers.get("x-gitea-event", "push").casefold()
# Catalog reads and the enqueue transaction are blocking SQLite work.
return await run_in_threadpool(_handle_gitea_webhook, payload, event)
def _handle_gitea_webhook(payload: dict, event: str) -> WebhookResponse:
repository_payload = payload.get("repository") repository_payload = payload.get("repository")
if not isinstance(repository_payload, dict): if not isinstance(repository_payload, dict):
raise HTTPException(status_code=400, detail="Webhook repository is missing") raise HTTPException(status_code=400, detail="Webhook repository is missing")
@@ -368,7 +379,6 @@ async def gitea_webhook(request: Request) -> WebhookResponse:
accepted=False, accepted=False,
message="Repository is not present in the synchronized catalog", message="Repository is not present in the synchronized catalog",
) )
event = request.headers.get("x-gitea-event", "push").casefold()
if event != "push": if event != "push":
return WebhookResponse( return WebhookResponse(
accepted=True, accepted=True,
@@ -486,6 +496,19 @@ def export_scan(scan_id: str, format: str = Query(default="html", pattern="^(htm
) )
class _FrontendStaticFiles(StaticFiles):
"""Serve the SPA; content-hashed build assets are immutable and cached long-term."""
async def get_response(self, path: str, scope: Scope) -> Response:
response = await super().get_response(path, scope)
if response.status_code == 200:
if path.replace("\\", "/").startswith("assets/"):
response.headers["Cache-Control"] = "public, max-age=31536000, immutable"
else:
response.headers["Cache-Control"] = "no-cache"
return response
static_dir = Path(os.getenv("REPOPULSE_STATIC_DIR", "/app/static")) static_dir = Path(os.getenv("REPOPULSE_STATIC_DIR", "/app/static"))
if static_dir.is_dir(): if static_dir.is_dir():
app.mount("/", StaticFiles(directory=static_dir, html=True), name="frontend") app.mount("/", _FrontendStaticFiles(directory=static_dir, html=True), name="frontend")
+77
View File
@@ -0,0 +1,77 @@
"""Bounded child-process execution that never leaves orphaned descendants behind."""
from __future__ import annotations
import os
import signal
import subprocess
from collections.abc import Mapping, Sequence
# Git may daemonize automatic housekeeping (``gc --auto``/``maintenance --auto``) after a
# fetch. A detached child escapes the timeout and is reparented to PID 1, where it becomes
# a zombie if PID 1 does not reap. Housekeeping therefore runs in the foreground.
GIT_FOREGROUND_CONFIG: tuple[tuple[str, str], ...] = (
("gc.autoDetach", "false"),
("maintenance.autoDetach", "false"),
)
def git_environment(
base: Mapping[str, str] | None = None,
extra_config: Sequence[tuple[str, str]] = (),
) -> dict[str, str]:
"""Return an environment that applies foreground housekeeping plus ``extra_config``."""
env = dict(os.environ if base is None else base)
pairs = [*GIT_FOREGROUND_CONFIG, *extra_config]
env["GIT_CONFIG_COUNT"] = str(len(pairs))
for index, (key, value) in enumerate(pairs):
env[f"GIT_CONFIG_KEY_{index}"] = key
env[f"GIT_CONFIG_VALUE_{index}"] = value
return env
def _kill_process_tree(process: subprocess.Popen[bytes]) -> None:
if os.name == "posix":
try:
os.killpg(process.pid, signal.SIGKILL)
except (ProcessLookupError, PermissionError):
pass
else:
process.kill()
def run_bounded(
args: Sequence[str],
*,
timeout: float,
env: Mapping[str, str] | None = None,
check: bool = True,
) -> subprocess.CompletedProcess[bytes]:
"""Run ``args`` in its own process group and reap it and all descendants.
On timeout the whole group is killed (not only the direct child), then the child is
waited for so no zombie remains. Raises the same exceptions as ``subprocess.run``.
"""
process = subprocess.Popen(
list(args),
stdin=subprocess.DEVNULL,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=None if env is None else dict(env),
start_new_session=os.name == "posix",
)
try:
stdout, stderr = process.communicate(timeout=timeout)
except subprocess.TimeoutExpired:
_kill_process_tree(process)
process.communicate()
raise
except BaseException:
_kill_process_tree(process)
process.wait()
raise
if check and process.returncode:
raise subprocess.CalledProcessError(process.returncode, list(args), stdout, stderr)
return subprocess.CompletedProcess(list(args), process.returncode, stdout, stderr)
+18 -6
View File
@@ -1,3 +1,5 @@
from functools import lru_cache
from .models import Finding, FindingGuidance, Severity from .models import Finding, FindingGuidance, Severity
_RULES: dict[str, tuple[str, str, str, str]] = { _RULES: dict[str, tuple[str, str, str, str]] = {
@@ -63,12 +65,16 @@ def _default_rule(finding: Finding) -> tuple[str, str, str, str]:
) )
def enrich_finding(finding: Finding) -> Finding: @lru_cache(maxsize=4096)
remediation_class, why, action, confidence = _RULES.get(finding.code, _default_rule(finding)) def _guidance(code: str, severity: Severity, deduction: int) -> FindingGuidance:
severity_weight = {Severity.BLOCKER: 700, Severity.WARNING: 400, Severity.INFO: 100}[finding.severity] """Guidance depends only on code, severity and deduction; share one immutable instance."""
priority = min(1000, severity_weight + finding.deduction * 5)
rationale = f"{finding.severity.value.title()} signal; {finding.deduction} health-point impact." probe = Finding(code=code, category="", severity=severity, message="", deduction=deduction)
guidance = FindingGuidance( remediation_class, why, action, confidence = _RULES.get(code, _default_rule(probe))
severity_weight = {Severity.BLOCKER: 700, Severity.WARNING: 400, Severity.INFO: 100}[severity]
priority = min(1000, severity_weight + deduction * 5)
rationale = f"{severity.value.title()} signal; {deduction} health-point impact."
return FindingGuidance(
remediation_class=remediation_class, remediation_class=remediation_class,
confidence=confidence, confidence=confidence,
why=why, why=why,
@@ -76,6 +82,12 @@ def enrich_finding(finding: Finding) -> Finding:
priority=priority, priority=priority,
priority_rationale=rationale, priority_rationale=rationale,
) )
def enrich_finding(finding: Finding) -> Finding:
guidance = _guidance(finding.code, finding.severity, finding.deduction)
if finding.guidance is guidance:
return finding
return finding.model_copy(update={"guidance": guidance}) return finding.model_copy(update={"guidance": guidance})
+140 -69
View File
@@ -5,13 +5,18 @@ import fnmatch
import os import os
import re import re
import subprocess import subprocess
from collections import Counter from collections import Counter, deque
from datetime import UTC, datetime from datetime import UTC, datetime
from pathlib import Path from pathlib import Path
from time import perf_counter from time import perf_counter
from uuid import uuid4 from uuid import uuid4
from .architecture import build_architecture from .architecture import (
build_architecture_from_imports,
extract_js_specifiers,
python_import_modules,
python_package_parts,
)
from .git_analysis import analyze_git from .git_analysis import analyze_git
from .models import ( from .models import (
Finding, Finding,
@@ -25,9 +30,10 @@ from .models import (
finding_fingerprint, finding_fingerprint,
) )
from .policy import load_policy from .policy import load_policy
from .processes import git_environment, run_bounded
from .release_gate import build_release_gate from .release_gate import build_release_gate
from .rules import enrich_findings from .rules import enrich_findings
from .signals import analyze_dependencies, detect_test_signals from .signals import analyze_dependencies, detect_test_signals, is_signal_text_path
LANGUAGES = { LANGUAGES = {
".py": "Python", ".py": "Python",
@@ -224,11 +230,8 @@ class _GitIgnoreMatcher:
def _git_worktree_root(root: Path) -> Path | None: def _git_worktree_root(root: Path) -> Path | None:
try: try:
result = subprocess.run( result = run_bounded(
["git", "-C", str(root), "rev-parse", "--show-toplevel"], ["git", "-C", str(root), "rev-parse", "--show-toplevel"], timeout=5, env=git_environment()
check=True,
capture_output=True,
timeout=5,
) )
except (OSError, subprocess.SubprocessError): except (OSError, subprocess.SubprocessError):
return None return None
@@ -238,42 +241,61 @@ def _git_worktree_root(root: Path) -> Path | None:
return None return None
class _SafePathChecker:
"""Validate candidate files below one root, caching directories already proven safe.
Every path component is still checked for symlinks, but a directory shared by many
files is inspected once instead of once per file.
"""
def __init__(self, root: Path) -> None:
self.root = root
self.root_resolved = root.resolve()
self._safe_dirs: set[Path] = set()
def check(self, candidate: Path) -> Path | None:
"""Return a resolved regular file only when every path component is safe."""
try:
relative = candidate.relative_to(self.root)
if _ignored(str(relative)):
return None
current = self.root
parts = relative.parts
for index, part in enumerate(parts):
current /= part
is_last = index == len(parts) - 1
if not is_last and current in self._safe_dirs:
continue
if current.is_symlink():
return None
if not is_last:
self._safe_dirs.add(current)
resolved = candidate.resolve(strict=True)
root_resolved = self.root_resolved
if resolved == root_resolved or root_resolved not in resolved.parents:
return None
if not resolved.is_file():
return None
except (OSError, RuntimeError, ValueError):
return None
return resolved
def _safe_file(root: Path, candidate: Path) -> Path | None: def _safe_file(root: Path, candidate: Path) -> Path | None:
"""Return a resolved regular file only when every path component is safe.""" """Return a resolved regular file only when every path component is safe."""
try: return _SafePathChecker(root).check(candidate)
relative = candidate.relative_to(root)
if _ignored(str(relative)):
return None
current = root
for part in Path(relative).parts:
current /= part
if current.is_symlink():
return None
resolved = candidate.resolve(strict=True)
root_resolved = root.resolve()
if resolved == root_resolved or root_resolved not in resolved.parents:
return None
if not resolved.is_file():
return None
except (OSError, RuntimeError, ValueError):
return None
return resolved
def _tracked_files(root: Path) -> list[Path] | None: def _tracked_files(root: Path) -> list[tuple[str, Path]] | None:
if _git_worktree_root(root) != root: if _git_worktree_root(root) != root:
return None return None
try: try:
result = subprocess.run( result = run_bounded(["git", "-C", str(root), "ls-files", "-z"], timeout=5, env=git_environment())
["git", "-C", str(root), "ls-files", "-z"],
check=True,
capture_output=True,
timeout=5,
)
except (OSError, subprocess.SubprocessError): except (OSError, subprocess.SubprocessError):
return None return None
paths: list[Path] = [] checker = _SafePathChecker(root)
paths: list[tuple[str, Path]] = []
for raw in result.stdout.split(b"\0"): for raw in result.stdout.split(b"\0"):
if not raw: if not raw:
continue continue
@@ -281,14 +303,15 @@ def _tracked_files(root: Path) -> list[Path] | None:
relative_path = Path(relative) relative_path = Path(relative)
if relative_path.is_absolute() or _ignored(relative): if relative_path.is_absolute() or _ignored(relative):
continue continue
safe = _safe_file(root, root / relative_path) safe = checker.check(root / relative_path)
if safe is not None: if safe is not None:
paths.append(safe) paths.append((relative_path.as_posix(), safe))
return paths return paths
def _walk_files(root: Path) -> list[Path]: def _walk_files(root: Path) -> list[tuple[str, Path]]:
files: list[Path] = [] files: list[tuple[str, Path]] = []
checker = _SafePathChecker(root)
gitignore = _GitIgnoreMatcher(root) gitignore = _GitIgnoreMatcher(root)
for current, dirs, names in os.walk(root, followlinks=False): for current, dirs, names in os.walk(root, followlinks=False):
current_path = Path(current) current_path = Path(current)
@@ -312,13 +335,14 @@ def _walk_files(root: Path) -> list[Path]:
continue continue
if gitignore.is_ignored(relative, is_dir=False): if gitignore.is_ignored(relative, is_dir=False):
continue continue
safe = _safe_file(root, candidate) safe = checker.check(candidate)
if safe is not None: if safe is not None:
files.append(safe) files.append((relative.replace("\\", "/"), safe))
return files return files
def _read_text(path: Path, max_bytes: int = MAX_TEXT_BYTES) -> str | None: def _read_text_bytes(path: Path, max_bytes: int = MAX_TEXT_BYTES) -> tuple[str, int] | None:
"""Return decoded text plus the number of bytes read, or None for binary/oversized files."""
try: try:
if path.stat().st_size > max_bytes: if path.stat().st_size > max_bytes:
return None return None
@@ -327,7 +351,12 @@ def _read_text(path: Path, max_bytes: int = MAX_TEXT_BYTES) -> str | None:
return None return None
if b"\0" in raw[:8192]: if b"\0" in raw[:8192]:
return None return None
return raw.decode("utf-8", errors="replace") return raw.decode("utf-8", errors="replace"), len(raw)
def _read_text(path: Path, max_bytes: int = MAX_TEXT_BYTES) -> str | None:
result = _read_text_bytes(path, max_bytes)
return result[0] if result else None
def _is_test_file(relative: str) -> bool: def _is_test_file(relative: str) -> bool:
@@ -343,19 +372,45 @@ def _is_test_file(relative: str) -> bool:
) )
def _python_function_findings(relative: str, text: str) -> list[Finding]: PYTHON_BRANCH_TYPES = (ast.If, ast.For, ast.AsyncFor, ast.While, ast.Try, ast.Match, ast.BoolOp)
def _analyze_python(relative: str, text: str) -> tuple[list[Finding], tuple[str, ...]]:
"""Parse one Python file once and return function findings plus unresolved imports.
A single breadth-first walk (the same order as ``ast.walk``) attributes each branch
construct to every enclosing function, matching a per-function ``ast.walk`` count
without re-walking nested subtrees.
"""
try: try:
tree = ast.parse(text) tree = ast.parse(text)
except SyntaxError: except (SyntaxError, ValueError):
return [] return [], ()
package_parts = python_package_parts(relative)
imports: dict[str, None] = {}
functions: list[ast.FunctionDef | ast.AsyncFunctionDef] = []
branch_counts: list[int] = []
pending: deque[tuple[ast.AST, tuple[int, ...]]] = deque([(tree, ())])
while pending:
node, enclosing = pending.popleft()
if isinstance(node, PYTHON_BRANCH_TYPES):
for index in enclosing:
branch_counts[index] += 1
elif isinstance(node, (ast.Import, ast.ImportFrom)):
for module in python_import_modules(node, package_parts):
imports.setdefault(module)
child_enclosing = enclosing
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
functions.append(node)
branch_counts.append(0)
child_enclosing = (*enclosing, len(functions) - 1)
pending.extend((child, child_enclosing) for child in ast.iter_child_nodes(node))
del tree, pending
findings: list[Finding] = [] findings: list[Finding] = []
branch_types = (ast.If, ast.For, ast.AsyncFor, ast.While, ast.Try, ast.Match, ast.BoolOp) for node, branches in zip(functions, branch_counts, strict=True):
for node in ast.walk(tree):
if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
continue
end_line = getattr(node, "end_lineno", node.lineno) end_line = getattr(node, "end_lineno", node.lineno)
length = max(1, end_line - node.lineno + 1) length = max(1, end_line - node.lineno + 1)
branches = sum(1 for child in ast.walk(node) if isinstance(child, branch_types))
if length >= 140: if length >= 140:
findings.append( findings.append(
Finding( Finding(
@@ -380,7 +435,11 @@ def _python_function_findings(relative: str, text: str) -> list[Finding]:
deduction=4, deduction=4,
) )
) )
return findings[:12] return findings[:12], tuple(imports)
def _python_function_findings(relative: str, text: str) -> list[Finding]:
return _analyze_python(relative, text)[0]
def _is_placeholder_secret(match: re.Match[str]) -> bool: def _is_placeholder_secret(match: re.Match[str]) -> bool:
@@ -408,7 +467,7 @@ def scan_repository(
limits = limits or scan_limits_from_environment() limits = limits or scan_limits_from_environment()
tracked_files = _tracked_files(root) tracked_files = _tracked_files(root)
all_files = _walk_files(root) if tracked_files is None else tracked_files all_files = _walk_files(root) if tracked_files is None else tracked_files
all_files.sort(key=lambda item: str(item.relative_to(root)).replace("\\", "/").casefold()) all_files.sort(key=lambda item: item[0].casefold())
files = all_files[: limits.max_files] files = all_files[: limits.max_files]
coverage = ScanCoverage( coverage = ScanCoverage(
limits=limits, limits=limits,
@@ -426,8 +485,11 @@ def scan_repository(
metrics = RepositoryMetrics(files=len(all_files)) metrics = RepositoryMetrics(files=len(all_files))
files_by_name: dict[str, list[Path]] = {} files_by_name: dict[str, list[Path]] = {}
root_entries = {entry.name.lower() for entry in root.iterdir()} root_entries = {entry.name.lower() for entry in root.iterdir()}
text_by_path: dict[str, str] = {} # Only small derived values are retained per file. Full text is held for a single
source_texts: dict[str, str] = {} # file at a time, except the few CI/manifest texts that test-signal detection needs.
signal_texts: dict[str, str] = {}
readable_paths: list[str] = []
imports_by_path: dict[str, tuple[str, ...]] = {}
language_by_path: dict[str, str] = {} language_by_path: dict[str, str] = {}
loc_by_path: dict[str, int] = {} loc_by_path: dict[str, int] = {}
debt_by_path: dict[str, int] = {} debt_by_path: dict[str, int] = {}
@@ -435,11 +497,10 @@ def scan_repository(
secret_findings_truncated = False secret_findings_truncated = False
text_budget_exhausted = False text_budget_exhausted = False
for path_obj in files: for relative, path_obj in files:
try: try:
relative = str(path_obj.relative_to(root)).replace("\\", "/")
size = path_obj.stat().st_size size = path_obj.stat().st_size
except (OSError, ValueError): except OSError:
continue continue
files_by_name.setdefault(path_obj.name, []).append(path_obj) files_by_name.setdefault(path_obj.name, []).append(path_obj)
@@ -479,21 +540,28 @@ def scan_repository(
coverage.skipped_files_by_text_budget += 1 coverage.skipped_files_by_text_budget += 1
text_budget_exhausted = True text_budget_exhausted = True
continue continue
text = _read_text(path_obj, limits.max_text_bytes_per_file) read = _read_text_bytes(path_obj, limits.max_text_bytes_per_file)
if text is None: if read is None:
continue continue
text, byte_count = read
coverage.scanned_files += 1 coverage.scanned_files += 1
coverage.text_bytes_scanned += len(text.encode("utf-8", errors="replace")) coverage.text_bytes_scanned += byte_count
text_by_path[relative] = text readable_paths.append(relative)
if is_signal_text_path(relative):
signal_texts[relative] = text
lines = text.splitlines()
if language: if language:
loc = sum(1 for line in text.splitlines() if line.strip()) loc = sum(1 for line in lines if line.strip())
metrics.source_files += 1 metrics.source_files += 1
metrics.lines_of_code += loc metrics.lines_of_code += loc
languages[language] += loc languages[language] += loc
source_texts[relative] = text
language_by_path[relative] = language language_by_path[relative] = language
loc_by_path[relative] = loc loc_by_path[relative] = loc
if language in {"TypeScript", "JavaScript"}:
imports_by_path[relative] = extract_js_specifiers(text)
elif language != "Python":
imports_by_path[relative] = ()
if loc > 1000: if loc > 1000:
metrics.large_source_files += 1 metrics.large_source_files += 1
findings.append( findings.append(
@@ -507,7 +575,8 @@ def scan_repository(
) )
) )
if language == "Python": if language == "Python":
findings.extend(_python_function_findings(relative, text)) python_findings, imports_by_path[relative] = _analyze_python(relative, text)
findings.extend(python_findings)
if _is_test_file(relative): if _is_test_file(relative):
metrics.test_files += 1 metrics.test_files += 1
@@ -520,7 +589,7 @@ def scan_repository(
debt_by_path[relative] = sum(debt.values()) debt_by_path[relative] = sum(debt.values())
if _is_secret_scan_candidate(path_obj, relative, language) and secret_findings < limits.max_secret_findings: if _is_secret_scan_candidate(path_obj, relative, language) and secret_findings < limits.max_secret_findings:
for line_number, line in enumerate(text.splitlines(), start=1): for line_number, line in enumerate(lines, start=1):
match = SECRET_PATTERN.search(line) match = SECRET_PATTERN.search(line)
if match is None or _is_placeholder_secret(match): if match is None or _is_placeholder_secret(match):
continue continue
@@ -567,11 +636,13 @@ def scan_repository(
findings.append(Finding(code="hygiene.missing-env-example", category="Repository hygiene", severity=Severity.INFO, message="Environment configuration exists without a root .env.example", deduction=3)) findings.append(Finding(code="hygiene.missing-env-example", category="Repository hygiene", severity=Severity.INFO, message="Environment configuration exists without a root .env.example", deduction=3))
test_signals = detect_test_signals( test_signals = detect_test_signals(
[str(path.relative_to(root)).replace("\\", "/") for path in files], [relative for relative, _path in files],
metrics.source_files, metrics.source_files,
metrics.test_files, metrics.test_files,
text_by_path, signal_texts,
readable_paths,
) )
del signal_texts, readable_paths
if metrics.source_files and metrics.test_files == 0: if metrics.source_files and metrics.test_files == 0:
findings.append(Finding(code="testing.none-detected", category="Testing", severity=Severity.WARNING, message="No test files detected", deduction=30)) findings.append(Finding(code="testing.none-detected", category="Testing", severity=Severity.WARNING, message="No test files detected", deduction=30))
if not test_signals.ci_workflows: if not test_signals.ci_workflows:
@@ -593,9 +664,9 @@ def scan_repository(
findings.extend(dependency_findings) findings.extend(dependency_findings)
git_signals, churn_by_path = analyze_git(root) git_signals, churn_by_path = analyze_git(root)
architecture, hotspots = build_architecture( architecture, hotspots = build_architecture_from_imports(
root, root,
source_texts, imports_by_path,
language_by_path, language_by_path,
loc_by_path, loc_by_path,
debt_by_path, debt_by_path,
+34 -9
View File
@@ -5,6 +5,7 @@ import re
import tomllib import tomllib
import xml.etree.ElementTree as ET import xml.etree.ElementTree as ET
from collections import defaultdict from collections import defaultdict
from collections.abc import Iterable
from pathlib import Path from pathlib import Path
from .models import DependencySignals, Finding, Severity, TestSignals from .models import DependencySignals, Finding, Severity, TestSignals
@@ -32,6 +33,24 @@ COVERAGE_NAMES = {
COVERAGE_NAMES_CASEFOLD = {item.casefold() for item in COVERAGE_NAMES} COVERAGE_NAMES_CASEFOLD = {item.casefold() for item in COVERAGE_NAMES}
CI_WORKFLOW_PREFIXES = (".github/workflows/", ".gitea/workflows/")
CI_WORKFLOW_NAMES = {".gitlab-ci.yml", "jenkinsfile", "azure-pipelines.yml"}
SIGNAL_TEXT_NAMES = {"package.json", "pyproject.toml"}
def is_ci_workflow_path(relative: str) -> bool:
normalized = relative.replace("\\", "/")
return normalized.lower().startswith(CI_WORKFLOW_PREFIXES) or (
Path(normalized).name.casefold() in CI_WORKFLOW_NAMES
)
def is_signal_text_path(relative: str) -> bool:
"""Return True when detect_test_signals needs this file's text content."""
return is_ci_workflow_path(relative) or Path(relative).name.casefold() in SIGNAL_TEXT_NAMES
def _python_dependency_name(declaration: str) -> str: def _python_dependency_name(declaration: str) -> str:
return re.split(r"[<>=!~ @;\[]", declaration, maxsplit=1)[0].strip() return re.split(r"[<>=!~ @;\[]", declaration, maxsplit=1)[0].strip()
@@ -69,7 +88,14 @@ def detect_test_signals(
source_files: int, source_files: int,
test_files: int, test_files: int,
text_by_path: dict[str, str], text_by_path: dict[str, str],
readable_paths: Iterable[str] | None = None,
) -> TestSignals: ) -> TestSignals:
"""Derive test/CI signals.
``text_by_path`` only needs the texts accepted by :func:`is_signal_text_path`;
``readable_paths`` lists every successfully read text file (defaults to the
``text_by_path`` keys) so coverage evidence does not require retaining text.
"""
frameworks: set[str] = set() frameworks: set[str] = set()
coverage: list[str] = [] coverage: list[str] = []
ci: list[str] = [] ci: list[str] = []
@@ -85,11 +111,7 @@ def detect_test_signals(
lower = normalized.lower() lower = normalized.lower()
if name_lower in COVERAGE_NAMES_CASEFOLD or "/coverage/" in f"/{lower}": if name_lower in COVERAGE_NAMES_CASEFOLD or "/coverage/" in f"/{lower}":
coverage.append(normalized) coverage.append(normalized)
if lower.startswith((".github/workflows/", ".gitea/workflows/")) or name_lower in { if is_ci_workflow_path(normalized):
".gitlab-ci.yml",
"jenkinsfile",
"azure-pipelines.yml",
}:
ci.append(normalized) ci.append(normalized)
if (name_lower.startswith("test_") or "/tests/" in f"/{lower}") and name_lower.endswith(".py"): if (name_lower.startswith("test_") or "/tests/" in f"/{lower}") and name_lower.endswith(".py"):
@@ -103,11 +125,16 @@ def detect_test_signals(
if name_lower.endswith(("tests.cs", "test.cs")): if name_lower.endswith(("tests.cs", "test.cs")):
frameworks.add(".NET test") frameworks.add(".NET test")
ci_paths = set(ci)
for path in text_by_path if readable_paths is None else readable_paths:
if Path(path).name.casefold() in COVERAGE_NAMES_CASEFOLD:
coverage_evidence.append(path.replace("\\", "/"))
for path, text in text_by_path.items(): for path, text in text_by_path.items():
name = Path(path).name name = Path(path).name
normalized = path.replace("\\", "/") normalized = path.replace("\\", "/")
lowered_text = text.casefold() if normalized in ci_paths:
if normalized in ci: lowered_text = text.casefold()
validates = any( validates = any(
token in lowered_text token in lowered_text
for token in ("pytest", "vitest", "jest", "npm test", "pnpm test", "go test", "dotnet test", "ruff", "eslint", "typecheck", "py_compile") for token in ("pytest", "vitest", "jest", "npm test", "pnpm test", "go test", "dotnet test", "ruff", "eslint", "typecheck", "py_compile")
@@ -125,8 +152,6 @@ def detect_test_signals(
ci_deploy_only.append(normalized) ci_deploy_only.append(normalized)
if any(token in lowered_text for token in ("--cov", "coverage", "lcov", "codecov")): if any(token in lowered_text for token in ("--cov", "coverage", "lcov", "codecov")):
coverage_evidence.append(normalized) coverage_evidence.append(normalized)
if name.casefold() in COVERAGE_NAMES_CASEFOLD:
coverage_evidence.append(normalized)
if name.casefold() == "package.json": if name.casefold() == "package.json":
try: try:
data = json.loads(text) data = json.loads(text)
+183 -154
View File
@@ -9,6 +9,8 @@ from contextlib import contextmanager
from datetime import UTC, datetime, timedelta from datetime import UTC, datetime, timedelta
from hashlib import sha256 from hashlib import sha256
from pathlib import Path from pathlib import Path
from threading import Lock, Thread
from time import monotonic
from uuid import uuid4 from uuid import uuid4
from .models import ( from .models import (
@@ -35,6 +37,7 @@ from .rules import enrich_findings
MAX_HISTORY_PER_REPOSITORY = 200 MAX_HISTORY_PER_REPOSITORY = 200
SCHEMA_VERSION = 4 SCHEMA_VERSION = 4
MIRROR_STATS_TTL_SECONDS = 300.0
JOB_LEASE_SECONDS = 900 JOB_LEASE_SECONDS = 900
ACTIVE_JOB_STATUSES = ( ACTIVE_JOB_STATUSES = (
ScanJobStatus.QUEUED.value, ScanJobStatus.QUEUED.value,
@@ -108,6 +111,8 @@ class ScanStore:
self.path = Path(raw_path) self.path = Path(raw_path)
self.path.parent.mkdir(parents=True, exist_ok=True) self.path.parent.mkdir(parents=True, exist_ok=True)
self.recovery_metadata: dict[str, str] | None = None self.recovery_metadata: dict[str, str] | None = None
self._mirror_stats_cache: tuple[float, str, tuple[int, int]] | None = None
self._mirror_stats_refresh = Lock()
try: try:
self._validate_existing_database() self._validate_existing_database()
self._init_schema() self._init_schema()
@@ -349,7 +354,7 @@ class ScanStore:
def _backfill_legacy_catalog(self, db: sqlite3.Connection) -> None: def _backfill_legacy_catalog(self, db: sqlite3.Connection) -> None:
rows = db.execute( rows = db.execute(
""" """
SELECT id, repository_path, repository_name, report_json, repository_id, history_key SELECT id, repository_path, repository_name, repository_id, history_key
FROM scans FROM scans
WHERE repository_id IS NULL OR history_key IS NULL WHERE repository_id IS NULL OR history_key IS NULL
""" """
@@ -412,11 +417,19 @@ class ScanStore:
} }
def _backfill_scan_summaries(self, db: sqlite3.Connection) -> None: def _backfill_scan_summaries(self, db: sqlite3.Connection) -> None:
rows = db.execute( # Collect ids first and load one report at a time so a large legacy history is
"SELECT rowid AS scan_rowid, * FROM scans " # never materialized in memory at once during migration.
"WHERE finding_count IS NULL OR blocker_count IS NULL OR release_readiness IS NULL" scan_ids = [
).fetchall() str(item[0])
for row in rows: for item in db.execute(
"SELECT id FROM scans "
"WHERE finding_count IS NULL OR blocker_count IS NULL OR release_readiness IS NULL"
).fetchall()
]
for scan_id in scan_ids:
row = db.execute("SELECT rowid AS scan_rowid, * FROM scans WHERE id = ?", (scan_id,)).fetchone()
if row is None:
continue
report = self._report_from_row(row) report = self._report_from_row(row)
previous_row = db.execute( previous_row = db.execute(
""" """
@@ -560,112 +573,6 @@ class ScanStore:
row = db.execute("SELECT source_path FROM repositories WHERE id = ?", (repository_id,)).fetchone() row = db.execute("SELECT source_path FROM repositories WHERE id = ?", (repository_id,)).fetchone()
return row["source_path"] if row else None return row["source_path"] if row else None
def _repository_summary(self, db: sqlite3.Connection, row: sqlite3.Row) -> RepositorySummary:
repository = self._repository_from_row(row)
latest = db.execute(
"""
SELECT * FROM scans WHERE repository_id = ?
ORDER BY scanned_at DESC, rowid DESC LIMIT 1
""",
(repository.id,),
).fetchone()
active = db.execute(
"""
SELECT * FROM scan_jobs WHERE repository_id = ? AND status IN (?, ?, ?, ?, ?)
ORDER BY created_at DESC LIMIT 1
""",
(repository.id, *ACTIVE_JOB_STATUSES),
).fetchone()
last_job = db.execute(
"SELECT * FROM scan_jobs WHERE repository_id = ? ORDER BY created_at DESC LIMIT 1",
(repository.id,),
).fetchone()
last_scan = self._report_from_row(latest) if latest else None
active_findings = [item for item in (last_scan.findings if last_scan else []) if not item.suppressed]
severity_rank = {Severity.BLOCKER: 0, Severity.WARNING: 1, Severity.INFO: 2}
primary_finding = min(
active_findings,
key=lambda item: (severity_rank[item.severity], -item.deduction, item.category, item.code),
default=None,
)
primary_signal = (
RemediationSignal(
code=primary_finding.code,
category=primary_finding.category,
severity=primary_finding.severity,
message=primary_finding.message,
count=sum(1 for item in active_findings if item.code == primary_finding.code),
)
if primary_finding
else None
)
previous_row = (
db.execute(
"""
SELECT * FROM scans
WHERE repository_id = ? AND id != ?
ORDER BY scanned_at DESC, rowid DESC LIMIT 1
""",
(repository.id, last_scan.id),
).fetchone()
if last_scan
else None
)
previous_scan = self._report_from_row(previous_row) if previous_row else None
previous_findings = [item for item in (previous_scan.findings if previous_scan else []) if not item.suppressed]
active_fingerprints = {self._finding_key(item) for item in active_findings}
previous_fingerprints = {self._finding_key(item) for item in previous_findings}
last_revision = (
last_scan.source.actual_revision
if last_scan and last_scan.source.actual_revision
else (latest["actual_revision"] if latest else None)
)
stale = bool(repository.latest_revision and last_revision and repository.latest_revision != last_revision)
if repository.archived:
state = RepositoryState.ARCHIVED
elif not repository.available:
state = RepositoryState.UNAVAILABLE
elif active:
state = (
RepositoryState.QUEUED
if active["status"] == ScanJobStatus.QUEUED.value
else RepositoryState.SCANNING
)
elif last_job and last_job["status"] == ScanJobStatus.FAILED.value:
last_job_at = _datetime(last_job["created_at"])
if last_scan is None or (last_job_at and last_job_at > last_scan.scanned_at):
state = RepositoryState.FAILED
elif stale:
state = RepositoryState.STALE
else:
state = RepositoryState.CURRENT
elif last_scan is None:
state = RepositoryState.NEVER_SCANNED
elif stale:
state = RepositoryState.STALE
else:
state = RepositoryState.CURRENT
data = repository.model_dump()
data.update(
{
"state": state,
"stale": stale,
"active_job_id": active["id"] if active else None,
"last_scan_id": last_scan.id if last_scan else None,
"last_scan_at": last_scan.scanned_at if last_scan else None,
"last_scan_health": last_scan.health_score if last_scan else None,
"last_scan_readiness": last_scan.release_gate.status if last_scan and last_scan.release_gate else (last_scan.release_readiness if last_scan else None),
"last_scan_revision": last_revision,
"last_scan_blockers": sum(1 for item in active_findings if item.severity == Severity.BLOCKER),
"last_scan_warnings": sum(1 for item in active_findings if item.severity == Severity.WARNING),
"last_scan_primary_signal": primary_signal,
"last_scan_health_delta": (last_scan.health_score - previous_scan.health_score) if previous_scan and last_scan else None,
"last_scan_added_findings": len(active_fingerprints - previous_fingerprints) if previous_scan else 0,
"last_scan_resolved_findings": len(previous_fingerprints - active_fingerprints) if previous_scan else 0,
}
)
return RepositorySummary(**data)
def _catalog_rows( def _catalog_rows(
self, db: sqlite3.Connection, *, limit: int, offset: int = 0, repository_id: str | None = None self, db: sqlite3.Connection, *, limit: int, offset: int = 0, repository_id: str | None = None
) -> list[sqlite3.Row]: ) -> list[sqlite3.Row]:
@@ -674,7 +581,11 @@ class ScanStore:
return db.execute( return db.execute(
f""" f"""
WITH ranked_scans AS ( WITH ranked_scans AS (
SELECT s.*, ROW_NUMBER() OVER ( -- Summary columns only: report_json must never be materialized here.
SELECT s.id, s.repository_id, s.scanned_at, s.health_score, s.release_readiness,
s.actual_revision, s.finding_count, s.blocker_count, s.warning_count,
s.primary_signal_json, s.added_findings, s.resolved_findings,
s.triage_priority, s.triage_rationale, ROW_NUMBER() OVER (
PARTITION BY s.repository_id ORDER BY s.scanned_at DESC, s.rowid DESC PARTITION BY s.repository_id ORDER BY s.scanned_at DESC, s.rowid DESC
) AS scan_rank ) AS scan_rank
FROM scans AS s WHERE s.repository_id IS NOT NULL FROM scans AS s WHERE s.repository_id IS NOT NULL
@@ -777,7 +688,7 @@ class ScanStore:
def repository_state_counts(self) -> dict[str, int]: def repository_state_counts(self) -> dict[str, int]:
counts: Counter[str] = Counter() counts: Counter[str] = Counter()
for repository in self.list_repositories(limit=1000): for repository in self.list_repositories(limit=-1):
counts[repository.state.value] += 1 counts[repository.state.value] += 1
return dict(counts) return dict(counts)
@@ -793,31 +704,70 @@ class ScanStore:
).fetchone() ).fetchone()
return RepositoryDetail(**summary.model_dump(), latest_scan=self._report_from_row(latest) if latest else None) return RepositoryDetail(**summary.model_dump(), latest_scan=self._report_from_row(latest) if latest else None)
@staticmethod
def _merged_source_value(report_value: object, column_value: object, field: str) -> object:
"""Mirror _report_from_row: a non-null column fills an absent/manual report value."""
if column_value is None or report_value not in (None, ScanTrigger.MANUAL.value):
return report_value
if field == "trigger":
try:
return ScanTrigger(str(column_value))
except ValueError:
return report_value
return column_value
def repository_scans(self, repository_id: str, limit: int = 50) -> list[ScanSummary]: def repository_scans(self, repository_id: str, limit: int = 50) -> list[ScanSummary]:
"""List scan summaries without deserializing full report payloads.
Finding counts come from the retained report; compacted reports no longer hold
findings, so their retained active-finding summary count is reported instead of 0.
"""
with self._connection() as db: with self._connection() as db:
rows = db.execute( rows = db.execute(
"SELECT * FROM scans WHERE repository_id = ? ORDER BY scanned_at DESC, rowid DESC LIMIT ?", """
SELECT id, scanned_at, health_score, report_compacted, finding_count,
release_readiness AS column_readiness,
branch, requested_revision, actual_revision, trigger,
json_extract(report_json, '$.source.branch') AS report_branch,
json_extract(report_json, '$.source.requested_revision') AS report_requested_revision,
json_extract(report_json, '$.source.actual_revision') AS report_actual_revision,
json_extract(report_json, '$.source.trigger') AS report_trigger,
json_extract(report_json, '$.release_readiness') AS report_readiness,
json_extract(report_json, '$.duration_ms') AS duration_ms,
json_array_length(report_json, '$.findings') AS report_findings
FROM scans WHERE repository_id = ?
ORDER BY scanned_at DESC, rowid DESC LIMIT ?
""",
(repository_id, limit), (repository_id, limit),
).fetchall() ).fetchall()
result: list[ScanSummary] = [] result: list[ScanSummary] = []
for row in rows: for row in rows:
report = self._report_from_row(row) source = {
result.append( field: self._merged_source_value(row[f"report_{field}"], row[field], field)
ScanSummary( for field in ("branch", "requested_revision", "actual_revision", "trigger")
id=report.id, }
repository_id=repository_id, if row["report_compacted"]:
scanned_at=report.scanned_at, findings = int(row["finding_count"] or 0)
branch=report.source.branch, else:
requested_revision=report.source.requested_revision, findings = int(row["report_findings"] or 0)
actual_revision=report.source.actual_revision, result.append(
trigger=report.source.trigger, ScanSummary(
health_score=report.health_score, id=row["id"],
release_readiness=report.release_readiness, repository_id=repository_id,
findings=len(report.findings), scanned_at=row["scanned_at"],
duration_ms=report.duration_ms, branch=source["branch"],
) requested_revision=source["requested_revision"],
actual_revision=source["actual_revision"],
trigger=source["trigger"] or ScanTrigger.MANUAL,
health_score=row["health_score"],
release_readiness=row["report_readiness"] or row["column_readiness"] or "",
findings=findings,
duration_ms=float(row["duration_ms"] or 0.0),
) )
return result )
return result
def _report_from_row(self, row: sqlite3.Row | None) -> ScanReport: def _report_from_row(self, row: sqlite3.Row | None) -> ScanReport:
if row is None: if row is None:
@@ -948,14 +898,20 @@ class ScanStore:
) )
cutoff = (_now() - timedelta(days=report_retention_days())).isoformat() cutoff = (_now() - timedelta(days=report_retention_days())).isoformat()
db.execute("DELETE FROM scans WHERE history_key = ? AND scanned_at < ?", (history_key, cutoff)) db.execute("DELETE FROM scans WHERE history_key = ? AND scanned_at < ?", (history_key, cutoff))
rows = db.execute( scan_ids = [
""" str(item[0])
SELECT * FROM scans WHERE history_key = ? AND report_compacted = 0 for item in db.execute(
ORDER BY scanned_at DESC, rowid DESC LIMIT -1 OFFSET ? """
""", SELECT id FROM scans WHERE history_key = ? AND report_compacted = 0
(history_key, full_reports_per_repository()), ORDER BY scanned_at DESC, rowid DESC LIMIT -1 OFFSET ?
).fetchall() """,
for row in rows: (history_key, full_reports_per_repository()),
).fetchall()
]
for scan_id in scan_ids:
row = db.execute("SELECT * FROM scans WHERE id = ?", (scan_id,)).fetchone()
if row is None:
continue
report = self._report_from_row(row) report = self._report_from_row(row)
compact = report.model_copy( compact = report.model_copy(
update={"findings": [], "architecture": ArchitectureGraph(), "hotspots": [], "compacted": True} update={"findings": [], "architecture": ArchitectureGraph(), "hotspots": [], "compacted": True}
@@ -1055,21 +1011,52 @@ class ScanStore:
) )
def trend(self, repository_key: str, limit: int = 50) -> list[TrendPoint]: def trend(self, repository_key: str, limit: int = 50) -> list[TrendPoint]:
reports = list(reversed(self.recent_for_repository(repository_key, limit))) """Return chronological trend points computed in SQLite, not from full reports.
Counts include every retained finding, as before. Compacted reports no longer
hold findings, so their retained active-finding summary counts are used rather
than reporting a misleading zero.
"""
with self._connection() as db:
rows: list[sqlite3.Row] = []
for candidate in self._history_candidates(repository_key):
rows = db.execute(
"""
SELECT id, scanned_at, health_score, report_compacted,
finding_count, blocker_count, warning_count,
json_array_length(report_json, '$.findings') AS report_findings,
(
SELECT COUNT(*) FROM json_each(scans.report_json, '$.findings')
WHERE json_extract(value, '$.severity') = 'blocker'
) AS report_blockers,
(
SELECT COUNT(*) FROM json_each(scans.report_json, '$.findings')
WHERE json_extract(value, '$.severity') = 'warning'
) AS report_warnings,
COALESCE(json_extract(report_json, '$.metrics.todo'), 0)
+ COALESCE(json_extract(report_json, '$.metrics.fixme'), 0)
+ COALESCE(json_extract(report_json, '$.metrics.hack'), 0)
+ COALESCE(json_extract(report_json, '$.metrics.xxx'), 0) AS debt_markers
FROM scans WHERE history_key = ?
ORDER BY scanned_at DESC, rowid DESC LIMIT ?
""",
(candidate, limit),
).fetchall()
if rows:
break
points: list[TrendPoint] = [] points: list[TrendPoint] = []
for report in reports: for row in reversed(rows):
blockers = sum(1 for item in report.findings if item.severity == Severity.BLOCKER) compacted = bool(row["report_compacted"])
warnings = sum(1 for item in report.findings if item.severity == Severity.WARNING)
debt = report.metrics.todo + report.metrics.fixme + report.metrics.hack + report.metrics.xxx
points.append( points.append(
TrendPoint( TrendPoint(
scan_id=report.id, scan_id=row["id"],
scanned_at=report.scanned_at, scanned_at=row["scanned_at"],
health_score=report.health_score, health_score=row["health_score"],
findings=len(report.findings), findings=int((row["finding_count"] if compacted else row["report_findings"]) or 0),
blockers=blockers, blockers=int((row["blocker_count"] if compacted else row["report_blockers"]) or 0),
warnings=warnings, warnings=int((row["warning_count"] if compacted else row["report_warnings"]) or 0),
debt_markers=debt, debt_markers=int(row["debt_markers"] or 0),
) )
) )
return points return points
@@ -1220,11 +1207,29 @@ class ScanStore:
row = db.execute("SELECT * FROM scan_jobs WHERE id = ?", (job_id,)).fetchone() row = db.execute("SELECT * FROM scan_jobs WHERE id = ?", (job_id,)).fetchone()
return self._job_from_row(row) if row else None return self._job_from_row(row) if row else None
@staticmethod
def _claim_work_may_exist(db: sqlite3.Connection, now: str) -> bool:
placeholders = ", ".join("?" for _ in ACTIVE_JOB_STATUSES[1:])
row = db.execute(
f"""
SELECT EXISTS(SELECT 1 FROM scan_jobs WHERE status = 'queued')
OR EXISTS(
SELECT 1 FROM scan_jobs
WHERE status IN ({placeholders}) AND lease_until IS NOT NULL AND lease_until < ?
)
""",
(*ACTIVE_JOB_STATUSES[1:], now),
).fetchone()
return bool(row[0])
def claim_next_job(self) -> ScanJob | None: def claim_next_job(self) -> ScanJob | None:
now_dt = _now() now_dt = _now()
now = now_dt.isoformat() now = now_dt.isoformat()
lease_until = (now_dt + timedelta(seconds=job_lease_seconds())).isoformat() lease_until = (now_dt + timedelta(seconds=job_lease_seconds())).isoformat()
with self._connection() as db: with self._connection() as db:
# Cheap read-only probe: an idle queue never takes the write lock.
if not self._claim_work_may_exist(db, now):
return None
db.execute("BEGIN IMMEDIATE") db.execute("BEGIN IMMEDIATE")
if self._maintenance_enabled(db): if self._maintenance_enabled(db):
return None return None
@@ -1443,6 +1448,30 @@ class ScanStore:
except OSError: except OSError:
return 0 return 0
def _cached_mirror_stats(self, mirrors_root: Path) -> tuple[int, int]:
"""Return mirror-cache size without making /meta wait on a large tree walk.
The walk (up to 50k files) runs synchronously only when no value exists yet.
Afterwards an expired value is served while one background thread refreshes it.
"""
key = str(mirrors_root)
now = monotonic()
cached = self._mirror_stats_cache
if cached is None or cached[1] != key:
stats = self._tree_stats(mirrors_root)
self._mirror_stats_cache = (monotonic(), key, stats)
return stats
if now - cached[0] >= MIRROR_STATS_TTL_SECONDS and self._mirror_stats_refresh.acquire(blocking=False):
def refresh() -> None:
try:
self._mirror_stats_cache = (monotonic(), key, self._tree_stats(mirrors_root))
finally:
self._mirror_stats_refresh.release()
Thread(target=refresh, name="repopulse-mirror-stats", daemon=True).start()
return cached[2]
def storage_stats(self) -> dict[str, int | str | None]: def storage_stats(self) -> dict[str, int | str | None]:
with self._connection() as db: with self._connection() as db:
row = db.execute( row = db.execute(
@@ -1462,7 +1491,7 @@ class ScanStore:
cache_root = Path(os.getenv("REPOPULSE_REPOSITORY_CACHE", "data/repos")).expanduser() cache_root = Path(os.getenv("REPOPULSE_REPOSITORY_CACHE", "data/repos")).expanduser()
mirrors_root = cache_root / "mirrors" mirrors_root = cache_root / "mirrors"
worktrees_root = cache_root / "worktrees" worktrees_root = cache_root / "worktrees"
_mirror_files, mirror_cache_bytes = self._tree_stats(mirrors_root) _mirror_files, mirror_cache_bytes = self._cached_mirror_stats(mirrors_root)
mirror_count = self._managed_entry_count(mirrors_root) mirror_count = self._managed_entry_count(mirrors_root)
worktree_count = self._managed_entry_count(worktrees_root) worktree_count = self._managed_entry_count(worktrees_root)
return { return {
+1 -1
View File
@@ -1,6 +1,6 @@
[project] [project]
name = "repopulse-backend" name = "repopulse-backend"
version = "1.5.0" version = "1.5.1"
description = "RepoPulse repository intelligence API and deterministic analysis engine" description = "RepoPulse repository intelligence API and deterministic analysis engine"
requires-python = ">=3.12" requires-python = ">=3.12"
dependencies = [ dependencies = [
+107
View File
@@ -0,0 +1,107 @@
from __future__ import annotations
import inspect
import time
from pathlib import Path
from fastapi import FastAPI
from fastapi.testclient import TestClient
import app.store as store_module
from app import main as main_module
from app.models import Finding, Repository, Severity
from app.rules import enrich_finding, enrich_findings
from app.store import ScanStore
def test_database_bound_endpoints_never_run_blocking_sqlite_on_the_event_loop() -> None:
# A sync endpoint runs in the threadpool; an async one must offload its store work.
assert not inspect.iscoroutinefunction(main_module.create_repository_scan)
assert not inspect.iscoroutinefunction(main_module._handle_gitea_webhook)
webhook_source = inspect.getsource(main_module.gitea_webhook)
assert "store." not in webhook_source
assert "run_in_threadpool(_handle_gitea_webhook" in webhook_source
assert "asyncio.to_thread(store.save" in inspect.getsource(main_module.create_scan)
def test_large_api_responses_are_gzip_compressed(tmp_path: Path) -> None:
main_module.store = ScanStore(str(tmp_path / "history.db"))
main_module.job_runner = None
for index in range(12):
main_module.store.upsert_repository(
Repository(
id=f"gitea:{index}",
provider="gitea",
provider_repository_id=str(index),
owner="Jens",
name=f"Repository{index}",
full_name=f"Jens/Repository{index}",
default_branch="main",
)
)
client = TestClient(main_module.app)
compressed = client.get("/api/v1/repositories", headers={"Accept-Encoding": "gzip"})
small = client.get("/health", headers={"Accept-Encoding": "gzip"})
assert compressed.status_code == 200
assert compressed.headers.get("content-encoding") == "gzip"
assert len(compressed.json()) == 12
# Responses below the 1 KiB threshold are not worth compressing.
assert small.headers.get("content-encoding") is None
def test_hashed_frontend_assets_are_immutable_and_index_revalidates(tmp_path: Path) -> None:
(tmp_path / "assets").mkdir()
(tmp_path / "index.html").write_text("<html></html>", encoding="utf-8")
(tmp_path / "assets" / "index-abc123.js").write_text("console.log(1)", encoding="utf-8")
app = FastAPI()
app.mount("/", main_module._FrontendStaticFiles(directory=tmp_path, html=True), name="frontend")
client = TestClient(app)
asset = client.get("/assets/index-abc123.js")
index = client.get("/")
assert asset.headers["cache-control"] == "public, max-age=31536000, immutable"
assert index.headers["cache-control"] == "no-cache"
def test_finding_guidance_is_shared_and_unchanged() -> None:
finding = Finding(
code="security.secret-assignment",
category="Security basics",
severity=Severity.BLOCKER,
message="m",
deduction=20,
)
first, second = enrich_findings([finding, finding.model_copy(update={"path": "b.py"})])
assert first.guidance is second.guidance
assert first.guidance.priority == 800
assert first.guidance.priority_rationale == "Blocker signal; 20 health-point impact."
assert enrich_finding(first) is first
def test_expired_mirror_stats_are_served_while_refreshing_in_background(tmp_path: Path, monkeypatch) -> None:
store = ScanStore(str(tmp_path / "history.db"))
mirrors = tmp_path / "mirrors"
mirrors.mkdir()
calls: list[float] = []
def slow_walk(_root: Path) -> tuple[int, int]:
calls.append(time.monotonic())
time.sleep(0.3)
return len(calls), len(calls) * 100
monkeypatch.setattr(store, "_tree_stats", slow_walk)
assert store._cached_mirror_stats(mirrors) == (1, 100)
monkeypatch.setattr(store_module, "MIRROR_STATS_TTL_SECONDS", 0.0)
started = time.monotonic()
assert store._cached_mirror_stats(mirrors) == (1, 100)
assert time.monotonic() - started < 0.2
deadline = time.monotonic() + 3
while store._mirror_stats_cache[2] == (1, 100) and time.monotonic() < deadline:
time.sleep(0.05)
assert store._mirror_stats_cache[2] == (2, 200)
+101
View File
@@ -0,0 +1,101 @@
from __future__ import annotations
import os
import sqlite3
import subprocess
import sys
import time
from pathlib import Path
import pytest
from app.gitea import GiteaClient
from app.jobs import ScanJobRunner
from app.processes import git_environment, run_bounded
from app.store import ScanStore
def test_git_environment_forces_foreground_housekeeping_and_keeps_extra_config() -> None:
env = git_environment({}, extra_config=[("http.extraHeader", "Authorization: token t")])
pairs = {
env[f"GIT_CONFIG_KEY_{index}"]: env[f"GIT_CONFIG_VALUE_{index}"]
for index in range(int(env["GIT_CONFIG_COUNT"]))
}
assert pairs == {
"gc.autoDetach": "false",
"maintenance.autoDetach": "false",
"http.extraHeader": "Authorization: token t",
}
def test_run_bounded_returns_output_and_raises_on_failure() -> None:
ok = run_bounded([sys.executable, "-c", "print('hi')"], timeout=10)
assert ok.stdout.strip() == b"hi"
with pytest.raises(subprocess.CalledProcessError):
run_bounded([sys.executable, "-c", "raise SystemExit(3)"], timeout=10)
@pytest.mark.skipif(os.name != "posix", reason="process groups are POSIX-only")
def test_run_bounded_timeout_kills_grandchildren(tmp_path: Path) -> None:
marker = tmp_path / "grandchild.pid"
script = (
"import subprocess, sys, time\n"
"child = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(60)'])\n"
f"open({str(marker)!r}, 'w').write(str(child.pid))\n"
"time.sleep(60)\n"
)
with pytest.raises(subprocess.TimeoutExpired):
run_bounded([sys.executable, "-c", script], timeout=2)
grandchild = int(marker.read_text())
deadline = time.monotonic() + 5
while time.monotonic() < deadline:
if _process_gone_or_zombie(grandchild):
return
time.sleep(0.05)
pytest.fail("grandchild survived the timeout kill")
def _process_gone_or_zombie(pid: int) -> bool:
"""A killed orphan stays a zombie until PID 1 reaps it (tini does in production).
Inside a CI job container PID 1 may not reap, so a zombie counts as killed.
"""
try:
stat = Path(f"/proc/{pid}/stat").read_text()
except (FileNotFoundError, ProcessLookupError):
return True
# Field 3, after the parenthesised command name, is the process state.
return stat.rsplit(")", 1)[1].split()[0] in {"Z", "X"}
def test_idle_claim_never_takes_the_write_lock(tmp_path: Path) -> None:
store = ScanStore(str(tmp_path / "repopulse.db"))
writer = sqlite3.connect(tmp_path / "repopulse.db", timeout=0)
try:
writer.execute("BEGIN IMMEDIATE")
started = time.monotonic()
assert store.claim_next_job() is None
# Without the read-only probe this would block on busy_timeout (5 s).
assert time.monotonic() - started < 2
finally:
writer.rollback()
writer.close()
def test_idle_worker_backs_off_and_enqueue_wakes_it(tmp_path: Path) -> None:
runner = ScanJobRunner(ScanStore(str(tmp_path / "repopulse.db")), gitea=GiteaClient())
runner.poll_seconds = 0.5
runner.idle_poll_seconds = 5.0
delays = []
for iterations in range(1, 7):
runner._idle_iterations = iterations
delays.append(runner._idle_delay())
assert delays == [0.5, 1.0, 2.0, 4.0, 5.0, 5.0]
runner.wake()
started = time.monotonic()
assert runner._wake.wait(1)
assert time.monotonic() - started < 0.5
+44
View File
@@ -221,3 +221,47 @@ def test_empty_git_repository_does_not_fallback_to_untracked_files(tmp_path: Pat
assert report.git.tracked_files == 0 assert report.git.tracked_files == 0
assert report.metrics.files == 0 assert report.metrics.files == 0
assert report.metrics.source_files == 0 assert report.metrics.source_files == 0
def test_single_pass_python_analysis_matches_per_function_branch_counts() -> None:
import ast
from app.scanner import PYTHON_BRANCH_TYPES, _analyze_python
inner = "\n".join(f" if x == {i}:\n x += 1" for i in range(12))
outer = "\n".join(f" for i{i} in range(2):\n pass" for i in range(10))
source = (
"import os\nfrom . import sibling\n\n"
f"def outer(x):\n{outer}\n def inner(x):\n{inner}\n return x\n return inner(x)\n"
)
findings, imports = _analyze_python("pkg/mod.py", source)
expected = {
node.name: sum(1 for child in ast.walk(node) if isinstance(child, PYTHON_BRANCH_TYPES))
for node in ast.walk(ast.parse(source))
if isinstance(node, ast.FunctionDef)
}
assert expected == {"outer": 22, "inner": 12}
branch_messages = [item.message for item in findings if item.code == "maintainability.branch-heavy-function"]
assert branch_messages == ["Function outer contains 22 branch constructs"]
assert imports == ("os", "pkg")
def test_js_import_resolution_is_lexical_and_never_escapes_the_root(tmp_path: Path) -> None:
from app.architecture import build_architecture_from_imports
imports = {
"web/app.ts": ("./util", "../../outside/secret", "../web/util.js"),
"web/util.ts": (),
}
graph, _ = build_architecture_from_imports(
tmp_path,
imports,
{path: "TypeScript" for path in imports},
{path: 1 for path in imports},
{path: 0 for path in imports},
{},
)
assert {(edge.source, edge.target) for edge in graph.edges} == {("web/app.ts", "web/util.ts")}
assert graph.total_edges == 1
+1 -1
View File
@@ -16,7 +16,7 @@ def test_non_git_fallback_applies_root_gitignore_rules(tmp_path: Path) -> None:
(tmp_path / "ignored-dir").mkdir() (tmp_path / "ignored-dir").mkdir()
(tmp_path / "ignored-dir" / "nested.py").write_text("print('nested')\n", encoding="utf-8") (tmp_path / "ignored-dir" / "nested.py").write_text("print('nested')\n", encoding="utf-8")
files = {path.relative_to(tmp_path).as_posix() for path in _walk_files(tmp_path)} files = {relative for relative, _path in _walk_files(tmp_path)}
assert "kept.py" in files assert "kept.py" in files
assert "keep.log" in files assert "keep.log" in files
+35
View File
@@ -159,3 +159,38 @@ def test_catalog_summary_uses_denormalized_fields_without_report_json_parsing(tm
assert storage["scan_count"] == 1 assert storage["scan_count"] == 1
assert storage["mirror_count"] >= 0 assert storage["mirror_count"] >= 0
assert storage["worktree_count"] >= 0 assert storage["worktree_count"] >= 0
def test_history_listings_keep_counts_for_compacted_reports_without_report_parsing(
tmp_path: Path, monkeypatch
) -> None:
repository = tmp_path / "repo"
repository.mkdir()
(repository / "app.py").write_text("print('hello')\n", encoding="utf-8")
monkeypatch.setenv("REPOPULSE_FULL_REPORTS_PER_REPOSITORY", "1")
store = ScanStore(str(tmp_path / "repopulse.db"))
first = store.save(scan_repository(str(repository)))
second = store.save(scan_repository(str(repository)))
assert store.get(first.id).compacted is True
assert first.findings
def fail_if_called(*_args, **_kwargs):
raise AssertionError("history listings must not deserialize full reports")
monkeypatch.setattr(store, "_report_from_row", fail_if_called)
repository_id = second.source.repository_id
active = [item for item in first.findings if not item.suppressed]
scans = store.repository_scans(repository_id)
assert [item.id for item in scans] == [second.id, first.id]
assert scans[0].findings == len(second.findings)
assert scans[0].duration_ms == second.duration_ms
assert scans[0].release_readiness == second.release_readiness
# A compacted report keeps its retained summary count instead of reporting zero.
assert scans[1].findings == len(active)
trend = store.trend(repository_id)
assert [item.scan_id for item in trend] == [first.id, second.id]
assert trend[0].findings == len(active)
assert trend[0].warnings == sum(1 for item in active if item.severity == "warning")
assert trend[1].findings == len(second.findings)
+2
View File
@@ -7,6 +7,8 @@ services:
dockerfile: Dockerfile.unraid dockerfile: Dockerfile.unraid
container_name: repopulse container_name: repopulse
restart: unless-stopped restart: unless-stopped
mem_limit: 1024m
memswap_limit: 1024m
environment: environment:
REPOPULSE_SCAN_ROOT: /scan REPOPULSE_SCAN_ROOT: /scan
REPOPULSE_DB: /data/repopulse.db REPOPULSE_DB: /data/repopulse.db
+10 -4
View File
@@ -49,6 +49,8 @@ The AppOps-managed runtime candidate is defined separately in `.itworx/deploymen
The scanner prefers `git ls-files` for a top-level worktree so ignored/untracked build artifacts do not distort results. An empty Git worktree remains empty rather than silently falling back. Non-Git directories use a filesystem traversal that excludes common generated/vendor directories and applies bounded root/nested `.gitignore` rules. Text files larger than the configured internal read threshold are not parsed as source, and binary files are skipped. The scanner prefers `git ls-files` for a top-level worktree so ignored/untracked build artifacts do not distort results. An empty Git worktree remains empty rather than silently falling back. Non-Git directories use a filesystem traversal that excludes common generated/vendor directories and applies bounded root/nested `.gitignore` rules. Text files larger than the configured internal read threshold are not parsed as source, and binary files are skipped.
The scanner processes inventory in one streaming pass and holds the full text of only one file at a time. It retains small derived values per file (LOC, debt counts, unresolved import references) plus the texts of the few CI workflow, `package.json` and `pyproject.toml` files needed for test signals, so peak scan memory no longer grows with the total text budget. Each Python file is parsed once for both function heuristics and imports. `text_bytes_scanned` counts the bytes actually read from disk.
Every report includes `ScanCoverage`: bounded file, per-file-text, total-text, finding and credential-finding limits plus the observed skipped counts. A limit produces a deterministic partial result and an analysis-coverage warning rather than an unqualified completeness claim or an artificial blocker. Every report includes `ScanCoverage`: bounded file, per-file-text, total-text, finding and credential-finding limits plus the observed skipped counts. A limit produces a deterministic partial result and an analysis-coverage warning rather than an unqualified completeness claim or an artificial blocker.
## Signals ## Signals
@@ -69,7 +71,7 @@ Each finding is enriched by a backend rule definition with confidence, why-it-ma
## Architecture graph ## Architecture graph
Python imports are parsed using the standard-library AST. Local JavaScript and TypeScript relative imports, bounded `baseUrl`/`paths` aliases from `tsconfig.json`/`jsconfig.json`, and simple local workspace package imports are resolved against known source files. Only local edges are retained; this remains heuristic source analysis, not a TypeScript compiler. The displayed graph is bounded to 160 nodes and 600 edges to keep large repositories responsive; the response explicitly indicates truncation, reports total discovered nodes/edges and distinguishes visible coupling from total coupling used in hotspot ranking. Extensionless, index and `.js`-specifier-to-TypeScript resolution are supported. Python imports are parsed using the standard-library AST. Local JavaScript and TypeScript relative imports, bounded `baseUrl`/`paths` aliases from `tsconfig.json`/`jsconfig.json`, and simple local workspace package imports are resolved against known source files. Import targets are resolved lexically against the known, already safety-checked file set rather than through filesystem resolution; a specifier escaping the scan root never yields an edge. Only local edges are retained; this remains heuristic source analysis, not a TypeScript compiler. The displayed graph is bounded to 160 nodes and 600 edges to keep large repositories responsive; the response explicitly indicates truncation, reports total discovered nodes/edges and distinguishes visible coupling from total coupling used in hotspot ranking. Extensionless, index and `.js`-specifier-to-TypeScript resolution are supported.
## Hotspot model ## Hotspot model
@@ -81,7 +83,7 @@ Category scores start at 100 and are reduced only by findings with an explicit d
## History and comparison ## History and comparison
Every report is serialized as a snapshot in SQLite using WAL. Lightweight scan summary columns (readiness, counts, primary signal, deltas and triage priority) drive catalog CTE queries without deserializing full reports. Comparison calculates metric/category deltas and added/resolved findings against the chronological previous scan of the same repository identity. Findings carry stable semantic fingerprints so line movement does not appear as a new risk. Trend endpoints return bounded chronological snapshots for UI visualization. By default, up to 200 snapshots are retained per repository, the latest 30 remain detailed, older retained payloads compact to summary evidence, and snapshots older than 365 days are removed. v1 path history is backfilled into a local catalog identity and is never deleted solely for migration. Every report is serialized as a snapshot in SQLite using WAL. Lightweight scan summary columns (readiness, counts, primary signal, deltas and triage priority) drive catalog CTE queries without deserializing full reports. Comparison calculates metric/category deltas and added/resolved findings against the chronological previous scan of the same repository identity. Findings carry stable semantic fingerprints so line movement does not appear as a new risk. Trend and scan-history endpoints return bounded chronological snapshots computed in SQLite from summary columns and JSON extraction, without deserializing full reports; a compacted snapshot reports its retained active-finding summary counts instead of a misleading zero. By default, up to 200 snapshots are retained per repository, the latest 30 remain detailed, older retained payloads compact to summary evidence, and snapshots older than 365 days are removed. v1 path history is backfilled into a local catalog identity and is never deleted solely for migration.
## Reporting ## Reporting
@@ -89,7 +91,11 @@ RepoPulse can export the complete JSON report or a standalone HTML report. HTML
## Job state and failure isolation ## Job state and failure isolation
The persisted queue records repository, trigger, requested/actual revision, phase, progress, attempt, timestamps, resulting scan and sanitized error information. Claims use `BEGIN IMMEDIATE` plus a status guard, so two processes cannot own the same queued job. They move through `queued → acquiring_source → scanning → analyzing → persisting → completed` or an isolated `failed` outcome. A heartbeat renews each active lease; only a crashed worker’s expired lease returns work to `queued` for recovery. Older unstarted revisions become `superseded`; explicit cancellation states remain representable. One repository failure does not terminate the worker loop or affect other queued repositories. The persisted queue records repository, trigger, requested/actual revision, phase, progress, attempt, timestamps, resulting scan and sanitized error information. Claims use `BEGIN IMMEDIATE` plus a status guard, so two processes cannot own the same queued job. They move through `queued → acquiring_source → scanning → analyzing → persisting → completed` or an isolated `failed` outcome. The idle scheduler writes its liveness heartbeat at most every `REPOPULSE_WORKER_HEARTBEAT_SECONDS` (default 5) instead of on every poll; claims still re-check maintenance inside their own transaction. An idle worker backs off exponentially from the 0.5 s poll to `REPOPULSE_WORKER_IDLE_POLL_SECONDS` (default 5), and in-process enqueues (API, webhook) wake it immediately. A claim first runs a read-only probe for queued work or expired leases, so an idle queue never takes the SQLite write lock. After each job the worker runs a garbage collection and, on glibc, `malloc_trim` so the long-running process returns a large scan's memory to the OS. A heartbeat renews each active lease; only a crashed worker’s expired lease returns work to `queued` for recovery. Older unstarted revisions become `superseded`; explicit cancellation states remain representable. One repository failure does not terminate the worker loop or affect other queued repositories.
## Child processes
Every Git subprocess runs in its own process group with a hard timeout; on timeout the whole group is killed and reaped, so no descendant (for example `git-remote-https` or `index-pack`) is orphaned. Git automatic housekeeping is forced to the foreground (`gc.autoDetach=false`, `maintenance.autoDetach=false`) so it stays within that bound. The container images run `tini` as PID 1, which reaps any remaining orphan; previously uvicorn ran as PID 1 and accumulated zombie `git` processes.
## Operator maintenance mode ## Operator maintenance mode
@@ -97,4 +103,4 @@ The `app_state` key `maintenance_mode` durably records whether operator maintena
## Operational configuration ## Operational configuration
`REPOPULSE_GITEA_URL` and `REPOPULSE_GITEA_TOKEN` enable the provider. `REPOPULSE_GITEA_SYNC_INTERVAL_SECONDS`, `REPOPULSE_GITEA_BRANCH_WORKERS`, `REPOPULSE_MAX_QUEUED_SCANS`, `REPOPULSE_JOB_LEASE_SECONDS`, scan-budget and retention variables, and `REPOPULSE_REPOSITORY_CACHE` bound reconciliation, queue depth, active leases, analysis and cache placement. The durable Gitea runner deliberately has one worker; `/api/v1/meta` exposes requested versus actual capacity. `REPOPULSE_GITEA_WEBHOOK_SECRET` enables signed default-branch push intake; non-push, non-default-branch, invalid and branch-deletion events are acknowledged without scheduling work. `REPOPULSE_OPERATOR_TOKEN` protects mutating endpoints when an authenticated proxy perimeter is not the selected boundary. `/api/v1/meta` exposes repository state counts, queue depth, storage/retention metrics, recovery status, worker heartbeat, connector URL (without token) and last sync result. `REPOPULSE_GITEA_URL` and `REPOPULSE_GITEA_TOKEN` enable the provider. `REPOPULSE_GITEA_SYNC_INTERVAL_SECONDS`, `REPOPULSE_GITEA_BRANCH_WORKERS`, `REPOPULSE_MAX_QUEUED_SCANS`, `REPOPULSE_JOB_LEASE_SECONDS`, scan-budget and retention variables, and `REPOPULSE_REPOSITORY_CACHE` bound reconciliation, queue depth, active leases, analysis and cache placement. The durable Gitea runner deliberately has one worker; `/api/v1/meta` exposes requested versus actual capacity. `REPOPULSE_GITEA_WEBHOOK_SECRET` enables signed default-branch push intake; non-push, non-default-branch, invalid and branch-deletion events are acknowledged without scheduling work. `REPOPULSE_OPERATOR_TOKEN` protects mutating endpoints when an authenticated proxy perimeter is not the selected boundary. `/api/v1/meta` exposes repository state counts over the complete catalog (no silent cap), queue depth, storage/retention metrics (the mirror-cache walk runs in the background at most every 5 minutes, so the endpoint never waits on it after the first call), recovery status, worker heartbeat, connector URL (without token) and last sync result.
+2 -2
View File
@@ -1,12 +1,12 @@
{ {
"name": "repopulse-frontend", "name": "repopulse-frontend",
"version": "1.5.0", "version": "1.5.1",
"lockfileVersion": 3, "lockfileVersion": 3,
"requires": true, "requires": true,
"packages": { "packages": {
"": { "": {
"name": "repopulse-frontend", "name": "repopulse-frontend",
"version": "1.5.0", "version": "1.5.1",
"dependencies": { "dependencies": {
"@xyflow/react": "^12.8.5", "@xyflow/react": "^12.8.5",
"react": "^19.1.1", "react": "^19.1.1",
+1 -1
View File
@@ -1,7 +1,7 @@
{ {
"name": "repopulse-frontend", "name": "repopulse-frontend",
"private": true, "private": true,
"version": "1.5.0", "version": "1.5.1",
"type": "module", "type": "module",
"scripts": { "scripts": {
"dev": "vite", "dev": "vite",
+2 -2
View File
@@ -1,12 +1,12 @@
{ {
"name": "repopulse-workspace", "name": "repopulse-workspace",
"version": "1.5.0", "version": "1.5.1",
"lockfileVersion": 3, "lockfileVersion": 3,
"requires": true, "requires": true,
"packages": { "packages": {
"": { "": {
"name": "repopulse-workspace", "name": "repopulse-workspace",
"version": "1.5.0" "version": "1.5.1"
} }
} }
} }
+1 -1
View File
@@ -1,7 +1,7 @@
{ {
"name": "repopulse-workspace", "name": "repopulse-workspace",
"private": true, "private": true,
"version": "1.5.0", "version": "1.5.1",
"scripts": { "scripts": {
"test": "npm --prefix frontend run test", "test": "npm --prefix frontend run test",
"typecheck": "npm --prefix frontend run typecheck", "typecheck": "npm --prefix frontend run typecheck",