RepoPulse: update backend/app and dependencies (+3 areas)
Public changes compared with the previous published export: 26 files changed. Affected areas: - backend/app: 11 files - dependencies: 5 files - deployment and CI: 3 files - documentation: 2 files - tests: 5 files Changed files: - Updated: "Dockerfile.unraid" - Updated: "README.md" - Updated: "backend/Dockerfile" - Updated: "backend/app/acquisition.py" - Updated: "backend/app/architecture.py" - Updated: "backend/app/git_analysis.py" - Updated: "backend/app/gitea.py" - Updated: "backend/app/jobs.py" - Updated: "backend/app/main.py" - Added: "backend/app/processes.py" - Updated: "backend/app/rules.py" - Updated: "backend/app/scanner.py" - Updated: "backend/app/signals.py" - Updated: "backend/app/store.py" - Updated: "backend/pyproject.toml" - Added: "backend/tests/test_api_responsiveness.py" - Added: "backend/tests/test_processes.py" - Updated: "backend/tests/test_scanner.py" - Updated: "backend/tests/test_signals_v11.py" - Updated: "backend/tests/test_store.py" - Updated: "docker-compose.yml" - Updated: "docs/ARCHITECTURE.md" - Updated: "frontend/package-lock.json" - Updated: "frontend/package.json" - Updated: "package-lock.json" - Updated: "package.json" Export provenance/manifests refreshed; excluded from the change summary. Publish curated RepoPulse source from a251f826238908b156485299061da76482780ed2
This commit is contained in:
1 parent
23a4a721f1
commit
61ae49f9d0
28 files changed
+1047
-372
No files matched your search
+3
-1
@@ -16,7 +16,7 @@ ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
REPOPULSE_STATIC_DIR=/app/static
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends git \
|
||||
&& apt-get install -y --no-install-recommends git tini \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY backend/pyproject.toml ./pyproject.toml
|
||||
@@ -26,4 +26,6 @@ COPY --from=frontend-build /frontend/dist ./static
|
||||
|
||||
VOLUME ["/data", "/scan"]
|
||||
EXPOSE 8000
|
||||
# tini is PID 1 so orphaned git descendants are always reaped (no zombies).
|
||||
ENTRYPOINT ["/usr/bin/tini", "--"]
|
||||
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]
|
||||
@@ -1,3 +1,3 @@
|
||||
# Public source export
|
||||
|
||||
This source snapshot was generated from the private canonical repository at revision `d5e4a9e3bf5b0a127b751c9fba917b5e40be41d5`. It contains no private Git history or operational evidence. Changes are published from the canonical repository.
|
||||
This source snapshot was generated from the private canonical repository at revision `a251f826238908b156485299061da76482780ed2`. It contains no private Git history or operational evidence. Changes are published from the canonical repository.
|
||||
+56
-41
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"sourceRevision": "d5e4a9e3bf5b0a127b751c9fba917b5e40be41d5",
|
||||
"sourceRevision": "a251f826238908b156485299061da76482780ed2",
|
||||
"files": [
|
||||
{
|
||||
"path": ".dockerignore",
|
||||
@@ -24,18 +24,18 @@
|
||||
},
|
||||
{
|
||||
"path": "Dockerfile.unraid",
|
||||
"sha256": "69f829fca37035eb45a73bd4121ffe8e383d61f92813f22a95b555b3f8bf8f60",
|
||||
"bytes": 759
|
||||
"sha256": "3abf243a64d45e040aa9c0f5ed3f4727ebfe3b09aa540fe6c03f353780a7e868",
|
||||
"bytes": 875
|
||||
},
|
||||
{
|
||||
"path": "README.md",
|
||||
"sha256": "a70daf2bbd2ac9fcf74b407901f07d9324ca05a486e117e7eb89d722d35e1ae9",
|
||||
"sha256": "3e4db609694a889bd8f33e3703791e1bfea4058c191fec1a3570b08649ff4418",
|
||||
"bytes": 5879
|
||||
},
|
||||
{
|
||||
"path": "backend/Dockerfile",
|
||||
"sha256": "41e92e2246c2811719b0b15911a8d277e9ea2383df12410fc132b0a7fae5a66a",
|
||||
"bytes": 486
|
||||
"sha256": "58470a33f7741b5952eff7a7fc76a43cf2859b1418d712af3fc3deefbb00d3b1",
|
||||
"bytes": 602
|
||||
},
|
||||
{
|
||||
"path": "backend/app/__init__.py",
|
||||
@@ -44,33 +44,33 @@
|
||||
},
|
||||
{
|
||||
"path": "backend/app/acquisition.py",
|
||||
"sha256": "92beed04a0d6eda5bb7fa9a852343edd510ca510a751f8935fd91a61a8a7f15c",
|
||||
"bytes": 10788
|
||||
"sha256": "1c1877d1d720c66d62a6a6f518a1752f649f6b0a2fa2fda766c093d39a4f6367",
|
||||
"bytes": 10499
|
||||
},
|
||||
{
|
||||
"path": "backend/app/architecture.py",
|
||||
"sha256": "57eb1be36636fedf599dcd8b88d996165610de07fb2ec047e120c933caa20b07",
|
||||
"bytes": 12912
|
||||
"sha256": "596fb586c80316dfff4ecec40c57e7e79e93b497fc9bbde7701d07be0dc7228d",
|
||||
"bytes": 15586
|
||||
},
|
||||
{
|
||||
"path": "backend/app/git_analysis.py",
|
||||
"sha256": "7530d55350c2acf3b62f8d3842679a05b612d3833fb854165f306a3fa17eefc4",
|
||||
"bytes": 3760
|
||||
"sha256": "12c4a85dab82def2067a564e6c25e463a7f33c97049e414bef762506f3e5cab9",
|
||||
"bytes": 3707
|
||||
},
|
||||
{
|
||||
"path": "backend/app/gitea.py",
|
||||
"sha256": "b6572bb2a32ecefdd575f40197835229bd8163c5b028ced1c01a02c277542c9c",
|
||||
"bytes": 13962
|
||||
"sha256": "e4795d7d60fe15fc133b2216a33b11b02086d86c4baa87de0d17d3e55ab0dbbb",
|
||||
"bytes": 13958
|
||||
},
|
||||
{
|
||||
"path": "backend/app/jobs.py",
|
||||
"sha256": "4f11fb25917b5400eadc796227ba0c55ed78489396ec720a0ba63ab8672b1b1d",
|
||||
"bytes": 12450
|
||||
"sha256": "f4deb3732d995f602980f41623e16bba5255b7125cf1dff03534cea5f571a0b4",
|
||||
"bytes": 15128
|
||||
},
|
||||
{
|
||||
"path": "backend/app/main.py",
|
||||
"sha256": "281fb5cf04cac0122dc64c55f49dbb99064f26e2608ace679a27c662f04829b3",
|
||||
"bytes": 19153
|
||||
"sha256": "b8898a7ddb4d328b52d2e7141d50610f6ee4ac27eaa57276d095590cd4d708ba",
|
||||
"bytes": 20288
|
||||
},
|
||||
{
|
||||
"path": "backend/app/models.py",
|
||||
@@ -82,6 +82,11 @@
|
||||
"sha256": "a0f05788bbb6f64de261287b19d241ebe116cb79c131b6a255a51037c06c6e6b",
|
||||
"bytes": 3184
|
||||
},
|
||||
{
|
||||
"path": "backend/app/processes.py",
|
||||
"sha256": "7864e7bbe7128c6484e29b2722a5f3d471d8e4eadb524e35a08a42b5c61f5fa6",
|
||||
"bytes": 2601
|
||||
},
|
||||
{
|
||||
"path": "backend/app/release_gate.py",
|
||||
"sha256": "a21927144cc15cf115cd3f42ce84f8d67411e98e9b7ef9f17228c40e0ec9e3fb",
|
||||
@@ -94,27 +99,27 @@
|
||||
},
|
||||
{
|
||||
"path": "backend/app/rules.py",
|
||||
"sha256": "25accd0a6fe0d25d81e26bbf45ff7da73290ec4a75b85e2eb3e558314ac4cb8d",
|
||||
"bytes": 3401
|
||||
"sha256": "cd12c89f706fb2b0df9f37ed8a5eb7bfe75f1a76d22bec55fea819570e8ffb24",
|
||||
"bytes": 3824
|
||||
},
|
||||
{
|
||||
"path": "backend/app/scanner.py",
|
||||
"sha256": "0bbe6cdb2f3ce99c46277d53b001549c99a3eb835e7d75e17f485dddbc77700a",
|
||||
"bytes": 25747
|
||||
"sha256": "ea806338e0c7c6eb4e9daa868e8185c2d918888618f3f1dc8136bc9a1c90af3e",
|
||||
"bytes": 29045
|
||||
},
|
||||
{
|
||||
"path": "backend/app/signals.py",
|
||||
"sha256": "65494200f87e6538873fc12a3cfa2eae8838285b3fc3f239ba956955e532df1e",
|
||||
"bytes": 19857
|
||||
"sha256": "4b4ed3325f88ffb0a382b574ca27280d7f0cbebfcec8d2ffb053be530a3e1817",
|
||||
"bytes": 20868
|
||||
},
|
||||
{
|
||||
"path": "backend/app/store.py",
|
||||
"sha256": "d7cdaf94afea9b33e720f3a6b4dbf4257c57af402d4977ac0b86c87051a1ff6c",
|
||||
"bytes": 68791
|
||||
"sha256": "0af079a9c54abc1571caeb78648a6ac08ff4aaf9c20cc11290d216588135b5a8",
|
||||
"bytes": 70464
|
||||
},
|
||||
{
|
||||
"path": "backend/pyproject.toml",
|
||||
"sha256": "48280f85cbc8a9f44acab05f7bdc76825597ad7438bee7012bcb2be66cd3b6b2",
|
||||
"sha256": "3efd954e6b6493bc707483a019d0aea5cb0e3ba573e6dd42d5fdeda6e91b03f6",
|
||||
"bytes": 446
|
||||
},
|
||||
{
|
||||
@@ -122,6 +127,11 @@
|
||||
"sha256": "452151143df1130f26efb1ee6554d2ffe22a0c301ef603c1cf5ac902a26ad932",
|
||||
"bytes": 4773
|
||||
},
|
||||
{
|
||||
"path": "backend/tests/test_api_responsiveness.py",
|
||||
"sha256": "66399c27f86c33f3798fa2a670eab25439a774a1cab981be3e95f486c4264355",
|
||||
"bytes": 4182
|
||||
},
|
||||
{
|
||||
"path": "backend/tests/test_api_v11.py",
|
||||
"sha256": "d7b34896a0fb7a3c87f42810d4d598f973995b8a44d5296153f9f6be4d58ff83",
|
||||
@@ -152,6 +162,11 @@
|
||||
"sha256": "e41669f8acf5f6fe80097d24281caf9732515263e32870adf56cbba2611528ca",
|
||||
"bytes": 2786
|
||||
},
|
||||
{
|
||||
"path": "backend/tests/test_processes.py",
|
||||
"sha256": "f7495ae414b0ce04dcc7bde4f911165291330e9562eb3b8836c776c51aec7fa1",
|
||||
"bytes": 3487
|
||||
},
|
||||
{
|
||||
"path": "backend/tests/test_reporting.py",
|
||||
"sha256": "1d46130c2b08b936cdf8a5a32219b9e7722e08348ff01d37335de0aaed6daeb6",
|
||||
@@ -159,8 +174,8 @@
|
||||
},
|
||||
{
|
||||
"path": "backend/tests/test_scanner.py",
|
||||
"sha256": "e750da9c9a2ff98064bcd20ee666fcc2bb50cd90b0bfe2cc870e9fc979f1a6ed",
|
||||
"bytes": 9629
|
||||
"sha256": "18f740260217d62ff71319dbd746c51f34160b8c30f227188841768bd201cbf8",
|
||||
"bytes": 11341
|
||||
},
|
||||
{
|
||||
"path": "backend/tests/test_signals.py",
|
||||
@@ -169,13 +184,13 @@
|
||||
},
|
||||
{
|
||||
"path": "backend/tests/test_signals_v11.py",
|
||||
"sha256": "f25ee36ddb02b99a36283535f9ab4aa58734452ca3a306fa1a685e7c06191bfd",
|
||||
"bytes": 2992
|
||||
"sha256": "f7afe0d14dac93feaadfcfa9041085e7439fbe3bacfd52581ec4cf66d1a6c52e",
|
||||
"bytes": 2974
|
||||
},
|
||||
{
|
||||
"path": "backend/tests/test_store.py",
|
||||
"sha256": "69dbf585d6a801036f9c87ea99c37ad16d15b4c8df75b6c78ba81ee790a0a3f8",
|
||||
"bytes": 6737
|
||||
"sha256": "f8fc769bbc064f5a0886d7350da89e6cba1b602018114278a9969c530cb44cad",
|
||||
"bytes": 8338
|
||||
},
|
||||
{
|
||||
"path": "compose.yaml",
|
||||
@@ -184,13 +199,13 @@
|
||||
},
|
||||
{
|
||||
"path": "docker-compose.yml",
|
||||
"sha256": "002a1126ea5a6131de9bbf4f11eb6a3eaae7097bd20bcaccffbf694d3f6003c5",
|
||||
"bytes": 2438
|
||||
"sha256": "efa0684eb4d79d9eab4afee7d73bb68b60ec397f96b99b428310bccc62501cfa",
|
||||
"bytes": 2484
|
||||
},
|
||||
{
|
||||
"path": "docs/ARCHITECTURE.md",
|
||||
"sha256": "3431db93b6ebce5d24f224d7a62de1fef47fa38f055ac0ea5f61b3ee8992ed82",
|
||||
"bytes": 13328
|
||||
"sha256": "41253e1aa1f0a816316bdf2e0d559575beb3936c538509d4d7becd4e2cd7f012",
|
||||
"bytes": 15543
|
||||
},
|
||||
{
|
||||
"path": "docs/screenshots/demo-explorer.png",
|
||||
@@ -234,12 +249,12 @@
|
||||
},
|
||||
{
|
||||
"path": "frontend/package-lock.json",
|
||||
"sha256": "f12d141b6ea3655dc9ac2843f7534e5f76838e490a9a08ac839570ed88598218",
|
||||
"sha256": "afeb669147a6a98f76c28f7f2a706595b05db881a0c76a0d7c15348c64717d4b",
|
||||
"bytes": 113084
|
||||
},
|
||||
{
|
||||
"path": "frontend/package.json",
|
||||
"sha256": "0bc4b2f7c0406d8f8faada4991737cb625c74c1f2e8304992daf68d486adbfa9",
|
||||
"sha256": "ac706aaddfd7774ce5f0b302b3640c7570eb06c2505709c0277d97a31cbfcb2c",
|
||||
"bytes": 724
|
||||
},
|
||||
{
|
||||
@@ -429,12 +444,12 @@
|
||||
},
|
||||
{
|
||||
"path": "package-lock.json",
|
||||
"sha256": "f293d47c364d320785df5fba11c3ea6db8f38cf487345a0a32f7932309076427",
|
||||
"sha256": "9479d451836a22b5299b96a4108b47962a42cd1bb76528486447bcffc6aabdf2",
|
||||
"bytes": 201
|
||||
},
|
||||
{
|
||||
"path": "package.json",
|
||||
"sha256": "f56ba0a04cedf9ba3600fb75fb736e3712f340381f2a593ba503cbeeafd92b89",
|
||||
"sha256": "18c0e87a4d7c2dba357c1cdac7728a6ca7983d098f2e71a2a055152eccf2c928",
|
||||
"bytes": 246
|
||||
},
|
||||
{
|
||||
|
||||
@@ -91,6 +91,6 @@ Run `python -m pytest`, `npm --prefix frontend test`, `npm --prefix frontend run
|
||||
|
||||
## Source distribution and licensing
|
||||
|
||||
This is the curated public source edition of RepoPulse 1.5.0. Reviewed updates flow from the private canonical repository after merges to `main`, without private history, repository inventories, runtime stores or internal CI/deployment workflows. Configure the generic deployment templates for your own environment.
|
||||
This is the curated public source edition of RepoPulse 1.5.1. Reviewed updates flow from the private canonical repository after merges to `main`, without private history, repository inventories, runtime stores or internal CI/deployment workflows. Configure the generic deployment templates for your own environment.
|
||||
|
||||
No project-wide license is included. Public visibility does not grant reuse or redistribution rights; obtain the owner's terms and review third-party dependencies before redistribution.
|
||||
+3
-1
@@ -7,7 +7,7 @@ ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
REPOPULSE_SCAN_ROOT=/scan
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends git \
|
||||
&& apt-get install -y --no-install-recommends git tini \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY backend/pyproject.toml ./pyproject.toml
|
||||
@@ -16,4 +16,6 @@ RUN pip install --no-cache-dir .
|
||||
|
||||
VOLUME ["/data", "/scan"]
|
||||
EXPOSE 8000
|
||||
# tini is PID 1 so orphaned git descendants are always reaped (no zombies).
|
||||
ENTRYPOINT ["/usr/bin/tini", "--"]
|
||||
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]
|
||||
@@ -10,6 +10,7 @@ from threading import Lock, RLock
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from .models import Repository
|
||||
from .processes import git_environment, run_bounded
|
||||
|
||||
SAFE_REVISION = re.compile(r"^[0-9a-fA-F]{7,64}$")
|
||||
SAFE_BRANCH = re.compile(r"^[A-Za-z0-9._/-]+$")
|
||||
@@ -72,35 +73,19 @@ class GitAcquirer:
|
||||
return self._repository_locks.setdefault(repository_id, RLock())
|
||||
|
||||
def _git_env(self) -> dict[str, str]:
|
||||
env = os.environ.copy()
|
||||
if self.token:
|
||||
env.update(
|
||||
{
|
||||
"GIT_CONFIG_COUNT": "1",
|
||||
"GIT_CONFIG_KEY_0": "http.extraHeader",
|
||||
"GIT_CONFIG_VALUE_0": f"Authorization: token {self.token}",
|
||||
}
|
||||
)
|
||||
return env
|
||||
extra = [("http.extraHeader", f"Authorization: token {self.token}")] if self.token else []
|
||||
return git_environment(extra_config=extra)
|
||||
|
||||
def _run(self, args: list[str]) -> str:
|
||||
try:
|
||||
result = subprocess.run(
|
||||
args,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
errors="replace",
|
||||
timeout=self.timeout,
|
||||
env=self._git_env(),
|
||||
)
|
||||
result = run_bounded(args, timeout=self.timeout, env=self._git_env())
|
||||
except subprocess.TimeoutExpired as exc:
|
||||
raise AcquisitionError("timeout", "Git source acquisition timed out") from exc
|
||||
except (OSError, subprocess.CalledProcessError) as exc:
|
||||
# Git stderr may contain remote URLs or server-specific details; do
|
||||
# not persist or expose it because the job API is user-visible.
|
||||
raise AcquisitionError("git_failed", "Git source acquisition failed") from exc
|
||||
return result.stdout.strip()
|
||||
return result.stdout.decode("utf-8", errors="replace").strip()
|
||||
|
||||
def _mirror_path(self, repository: Repository) -> Path:
|
||||
identity = f"{repository.provider}-{repository.provider_repository_id}"
|
||||
|
||||
+134
-64
@@ -4,6 +4,7 @@ import ast
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import posixpath
|
||||
import re
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
@@ -66,8 +67,10 @@ def _ts_path_aliases(root: Path) -> list[tuple[str, str]]:
|
||||
return aliases[:100]
|
||||
|
||||
|
||||
def _workspace_packages(root: Path) -> dict[str, Path]:
|
||||
packages: dict[str, Path] = {}
|
||||
def _workspace_packages(root: Path) -> dict[str, str]:
|
||||
"""Map local package names to their root-relative POSIX directory."""
|
||||
|
||||
packages: dict[str, str] = {}
|
||||
for manifest in _bounded_config_files(root, {"package.json"}, maximum=100):
|
||||
try:
|
||||
if manifest.stat().st_size > 256_000:
|
||||
@@ -75,7 +78,7 @@ def _workspace_packages(root: Path) -> dict[str, Path]:
|
||||
data = json.loads(manifest.read_text(encoding="utf-8"))
|
||||
name = data.get("name") if isinstance(data, dict) else None
|
||||
if isinstance(name, str) and name and not name.startswith((".", "/")):
|
||||
packages.setdefault(name, manifest.parent)
|
||||
packages.setdefault(name, manifest.parent.relative_to(root).as_posix())
|
||||
except (OSError, ValueError, json.JSONDecodeError, TypeError):
|
||||
continue
|
||||
return packages
|
||||
@@ -116,79 +119,118 @@ def _resolve_python_target(module: str, module_map: dict[str, str]) -> str | Non
|
||||
return None
|
||||
|
||||
|
||||
def _python_imports(path: str, text: str, module_map: dict[str, str]) -> set[str]:
|
||||
try:
|
||||
tree = ast.parse(text)
|
||||
except SyntaxError:
|
||||
return set()
|
||||
def python_package_parts(path: str) -> list[str]:
|
||||
"""Return the package that relative imports in ``path`` resolve against."""
|
||||
|
||||
current_names = _module_names(path)
|
||||
current = current_names[0] if current_names else ""
|
||||
current_parts = current.split(".") if current else []
|
||||
if not path.endswith("/__init__.py") and current_parts:
|
||||
package_parts = current_parts[:-1]
|
||||
else:
|
||||
package_parts = current_parts
|
||||
return current_parts[:-1]
|
||||
return current_parts
|
||||
|
||||
|
||||
def python_import_modules(node: ast.AST, package_parts: list[str]) -> list[str]:
|
||||
"""Return the absolute (still unresolved) module names referenced by one import node."""
|
||||
|
||||
imports: set[str] = set()
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Import):
|
||||
for alias in node.names:
|
||||
target = _resolve_python_target(alias.name, module_map)
|
||||
if target and target != path:
|
||||
imports.add(target)
|
||||
elif isinstance(node, ast.ImportFrom):
|
||||
return [alias.name for alias in node.names]
|
||||
if isinstance(node, ast.ImportFrom):
|
||||
module = node.module or ""
|
||||
if node.level:
|
||||
trim = max(0, node.level - 1)
|
||||
prefix = package_parts[:-trim] if trim else package_parts
|
||||
absolute = ".".join([*prefix, *([module] if module else [])])
|
||||
else:
|
||||
absolute = module
|
||||
target = _resolve_python_target(absolute, module_map)
|
||||
if target and target != path:
|
||||
imports.add(target)
|
||||
return imports
|
||||
return [".".join([*prefix, *([module] if module else [])])]
|
||||
return [module]
|
||||
return []
|
||||
|
||||
|
||||
def extract_python_imports(path: str, tree: ast.AST) -> tuple[str, ...]:
|
||||
package_parts = python_package_parts(path)
|
||||
modules: dict[str, None] = {}
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, (ast.Import, ast.ImportFrom)):
|
||||
for module in python_import_modules(node, package_parts):
|
||||
modules.setdefault(module)
|
||||
return tuple(modules)
|
||||
|
||||
|
||||
def extract_js_specifiers(text: str) -> tuple[str, ...]:
|
||||
return tuple(dict.fromkeys(match.group("target") for match in JS_IMPORT_RE.finditer(text)))
|
||||
|
||||
|
||||
def extract_imports(path: str, text: str, language: str) -> tuple[str, ...]:
|
||||
"""Extract compact, unresolved import references so source text need not be retained."""
|
||||
|
||||
if language == "Python":
|
||||
try:
|
||||
tree = ast.parse(text)
|
||||
except (SyntaxError, ValueError):
|
||||
return ()
|
||||
return extract_python_imports(path, tree)
|
||||
if language in {"TypeScript", "JavaScript"}:
|
||||
return extract_js_specifiers(text)
|
||||
return ()
|
||||
|
||||
|
||||
def _lexical_relative(path: str) -> str | None:
|
||||
"""Normalize a root-relative POSIX path without filesystem access.
|
||||
|
||||
Candidates are only matched against already safety-checked scanner paths, so
|
||||
symlink resolution adds nothing; a path escaping the root yields no target.
|
||||
"""
|
||||
|
||||
if not path or path.startswith("/"):
|
||||
return None
|
||||
normalized = posixpath.normpath(path)
|
||||
if normalized == ".." or normalized.startswith("../"):
|
||||
return None
|
||||
return normalized
|
||||
|
||||
|
||||
def _resolve_js_target(
|
||||
root: Path,
|
||||
source_path: str,
|
||||
specifier: str,
|
||||
known: set[str],
|
||||
aliases: list[tuple[str, str]],
|
||||
workspace_packages: dict[str, Path],
|
||||
workspace_packages: dict[str, str],
|
||||
) -> str | None:
|
||||
relative_base: str | None = None
|
||||
if specifier.startswith("."):
|
||||
source_dir = (root / source_path).parent
|
||||
base = (source_dir / specifier).resolve()
|
||||
relative_base = _lexical_relative(posixpath.join(posixpath.dirname(source_path), specifier))
|
||||
else:
|
||||
base = None
|
||||
matched_alias = False
|
||||
for alias, target in aliases:
|
||||
if alias.endswith("/*") and specifier.startswith(alias[:-1]):
|
||||
base = (root / target.replace("*", specifier[len(alias) - 1 :])).resolve()
|
||||
relative_base = _lexical_relative(target.replace("*", specifier[len(alias) - 1 :]))
|
||||
matched_alias = True
|
||||
break
|
||||
if alias == specifier:
|
||||
base = (root / target).resolve()
|
||||
relative_base = _lexical_relative(target)
|
||||
matched_alias = True
|
||||
break
|
||||
if base is None:
|
||||
if not matched_alias:
|
||||
package_name = next(
|
||||
(name for name in workspace_packages if specifier == name or specifier.startswith(f"{name}/")),
|
||||
(
|
||||
name
|
||||
for name in workspace_packages
|
||||
if specifier == name or specifier.startswith(f"{name}/")
|
||||
),
|
||||
None,
|
||||
)
|
||||
if package_name:
|
||||
package_root = workspace_packages[package_name]
|
||||
suffix = specifier.removeprefix(package_name).lstrip("/")
|
||||
base = package_root / suffix if suffix else package_root / "src" / "index"
|
||||
if base is None:
|
||||
return None
|
||||
try:
|
||||
relative_base = base.relative_to(root.resolve()).as_posix()
|
||||
except ValueError:
|
||||
relative_base = _lexical_relative(
|
||||
posixpath.join(package_root, suffix)
|
||||
if suffix
|
||||
else posixpath.join(package_root, "src", "index")
|
||||
)
|
||||
if relative_base is None:
|
||||
return None
|
||||
|
||||
candidates = [relative_base]
|
||||
suffix = Path(relative_base).suffix.lower()
|
||||
suffix = posixpath.splitext(relative_base)[1].lower()
|
||||
if suffix in JS_EXTENSIONS:
|
||||
extensionless = relative_base[: -len(suffix)]
|
||||
candidates.extend(extensionless + extension for extension in JS_EXTENSIONS)
|
||||
@@ -199,22 +241,6 @@ def _resolve_js_target(
|
||||
return next((candidate for candidate in candidates if candidate in known), None)
|
||||
|
||||
|
||||
def _js_imports(
|
||||
root: Path,
|
||||
path: str,
|
||||
text: str,
|
||||
known: set[str],
|
||||
aliases: list[tuple[str, str]],
|
||||
workspace_packages: dict[str, Path],
|
||||
) -> set[str]:
|
||||
imports: set[str] = set()
|
||||
for match in JS_IMPORT_RE.finditer(text):
|
||||
target = _resolve_js_target(root, path, match.group("target"), known, aliases, workspace_packages)
|
||||
if target and target != path:
|
||||
imports.add(target)
|
||||
return imports
|
||||
|
||||
|
||||
def build_architecture(
|
||||
root: Path,
|
||||
source_texts: dict[str, str],
|
||||
@@ -226,27 +252,71 @@ def build_architecture(
|
||||
max_nodes: int = 160,
|
||||
max_edges: int = 600,
|
||||
) -> tuple[ArchitectureGraph, list[Hotspot]]:
|
||||
"""Compatibility entry point that accepts full source texts."""
|
||||
|
||||
imports_by_path = {
|
||||
path: extract_imports(path, text, language_by_path.get(path, ""))
|
||||
for path, text in source_texts.items()
|
||||
}
|
||||
return build_architecture_from_imports(
|
||||
root,
|
||||
imports_by_path,
|
||||
language_by_path,
|
||||
loc_by_path,
|
||||
debt_by_path,
|
||||
churn_by_path,
|
||||
max_nodes=max_nodes,
|
||||
max_edges=max_edges,
|
||||
)
|
||||
|
||||
|
||||
def build_architecture_from_imports(
|
||||
root: Path,
|
||||
imports_by_path: dict[str, tuple[str, ...]],
|
||||
language_by_path: dict[str, str],
|
||||
loc_by_path: dict[str, int],
|
||||
debt_by_path: dict[str, int],
|
||||
churn_by_path: dict[str, int],
|
||||
*,
|
||||
max_nodes: int = 160,
|
||||
max_edges: int = 600,
|
||||
) -> tuple[ArchitectureGraph, list[Hotspot]]:
|
||||
"""Build the bounded graph from compact import references; no source text is retained."""
|
||||
|
||||
max_nodes = max(1, max_nodes)
|
||||
max_edges = max(0, max_edges)
|
||||
known_paths = set(source_texts)
|
||||
known_paths = set(imports_by_path)
|
||||
module_map = _python_module_map(list(known_paths))
|
||||
ts_aliases = _ts_path_aliases(root)
|
||||
workspace_packages = _workspace_packages(root)
|
||||
js_cache: dict[tuple[str, str], str | None] = {}
|
||||
edge_pairs: set[tuple[str, str]] = set()
|
||||
total_inbound: Counter[str] = Counter()
|
||||
total_outbound: Counter[str] = Counter()
|
||||
total_edges = 0
|
||||
truncated = len(known_paths) > max_nodes
|
||||
|
||||
for path in sorted(source_texts):
|
||||
text = source_texts[path]
|
||||
for path in sorted(imports_by_path):
|
||||
references = imports_by_path[path]
|
||||
language = language_by_path.get(path, "")
|
||||
targets: set[str] = set()
|
||||
if language == "Python":
|
||||
targets = _python_imports(path, text, module_map)
|
||||
for module in references:
|
||||
target = _resolve_python_target(module, module_map)
|
||||
if target and target != path:
|
||||
targets.add(target)
|
||||
elif language in {"TypeScript", "JavaScript"}:
|
||||
targets = _js_imports(root, path, text, known_paths, ts_aliases, workspace_packages)
|
||||
else:
|
||||
targets = set()
|
||||
directory = posixpath.dirname(path)
|
||||
for specifier in references:
|
||||
# Only relative specifiers depend on the importing directory.
|
||||
key = (directory if specifier.startswith(".") else "", specifier)
|
||||
if key not in js_cache:
|
||||
js_cache[key] = _resolve_js_target(
|
||||
path, specifier, known_paths, ts_aliases, workspace_packages
|
||||
)
|
||||
target = js_cache[key]
|
||||
if target and target != path:
|
||||
targets.add(target)
|
||||
for target in sorted(targets):
|
||||
if target not in known_paths:
|
||||
continue
|
||||
|
||||
@@ -6,21 +6,15 @@ from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
from .models import GitSignals
|
||||
from .processes import git_environment, run_bounded
|
||||
|
||||
|
||||
def _run(root: Path, *args: str, timeout: int = 8) -> str | None:
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(root), *args],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
errors="replace",
|
||||
timeout=timeout,
|
||||
)
|
||||
except (OSError, UnicodeError, subprocess.SubprocessError):
|
||||
result = run_bounded(["git", "-C", str(root), *args], timeout=timeout, env=git_environment())
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
return None
|
||||
return result.stdout
|
||||
return result.stdout.decode("utf-8", errors="replace")
|
||||
|
||||
|
||||
def analyze_git(root: Path) -> tuple[GitSignals, dict[str, int]]:
|
||||
|
||||
@@ -238,7 +238,7 @@ class GiteaClient:
|
||||
listed, discovered, branch_errors = self.discover_with_errors()
|
||||
except GiteaError as exc:
|
||||
error = f"{exc.code}: {exc.detail}"
|
||||
for existing in store.list_repositories(limit=1000):
|
||||
for existing in store.list_repositories(limit=-1):
|
||||
if existing.provider == "gitea" and existing.available:
|
||||
store.update_repository_sync(existing, synced_at=synced_at, error=error)
|
||||
result = RepositorySyncResult(
|
||||
@@ -290,7 +290,7 @@ class GiteaClient:
|
||||
updated += 1
|
||||
unavailable = 0
|
||||
if self._listing_complete:
|
||||
for existing in store.list_repositories(limit=1000):
|
||||
for existing in store.list_repositories(limit=-1):
|
||||
if existing.provider == "gitea" and existing.id not in listed_ids and existing.available:
|
||||
store.update_repository_sync(
|
||||
existing.model_copy(update={"available": False}),
|
||||
|
||||
+78
-3
@@ -1,7 +1,12 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import ctypes
|
||||
import ctypes.util
|
||||
import gc
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime
|
||||
from threading import Event, Lock, Thread, current_thread
|
||||
from time import monotonic
|
||||
@@ -21,6 +26,40 @@ from .scanner import scan_repository
|
||||
from .store import MaintenanceModeError, QueueFullError, ScanStore, job_lease_seconds
|
||||
|
||||
LOGGER = logging.getLogger(__name__)
|
||||
WORKER_HEARTBEAT_SECONDS = 5.0
|
||||
WORKER_IDLE_POLL_SECONDS = 5.0
|
||||
|
||||
|
||||
def _load_malloc_trim() -> Callable[[int], int] | None:
|
||||
if not sys.platform.startswith("linux"):
|
||||
return None
|
||||
try:
|
||||
libc = ctypes.CDLL(ctypes.util.find_library("c") or "libc.so.6")
|
||||
trim = libc.malloc_trim
|
||||
except (AttributeError, OSError):
|
||||
return None # e.g. musl libc has no malloc_trim
|
||||
trim.argtypes = [ctypes.c_size_t]
|
||||
trim.restype = ctypes.c_int
|
||||
return trim
|
||||
|
||||
|
||||
_MALLOC_TRIM = _load_malloc_trim()
|
||||
|
||||
|
||||
def release_memory() -> None:
|
||||
"""Return memory freed by a finished scan to the OS.
|
||||
|
||||
A scan allocates many short-lived objects (ASTs, file texts, findings). CPython frees
|
||||
them, but glibc keeps the arenas mapped, so the long-running worker's RSS would stay
|
||||
at its largest scan. A collection plus malloc_trim hands those pages back.
|
||||
"""
|
||||
|
||||
gc.collect()
|
||||
if _MALLOC_TRIM is not None:
|
||||
try:
|
||||
_MALLOC_TRIM(0)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def _positive_int(name: str, default: int, *, maximum: int | None = None) -> int:
|
||||
@@ -60,8 +99,17 @@ class ScanJobRunner:
|
||||
)
|
||||
self.max_queue = max_queue or _positive_int("REPOPULSE_MAX_QUEUED_SCANS", 100, maximum=1000)
|
||||
self._stop = Event()
|
||||
self._wake = Event()
|
||||
self._idle_iterations = 0
|
||||
self.idle_poll_seconds = max(
|
||||
self.poll_seconds, _positive_float("REPOPULSE_WORKER_IDLE_POLL_SECONDS", WORKER_IDLE_POLL_SECONDS)
|
||||
)
|
||||
self._thread: Thread | None = None
|
||||
self._last_sync = float("-inf")
|
||||
self._last_heartbeat = float("-inf")
|
||||
self.heartbeat_interval_seconds = _positive_float(
|
||||
"REPOPULSE_WORKER_HEARTBEAT_SECONDS", WORKER_HEARTBEAT_SECONDS
|
||||
)
|
||||
self._sync_lock = Lock()
|
||||
|
||||
@property
|
||||
@@ -72,11 +120,13 @@ class ScanJobRunner:
|
||||
if self.started:
|
||||
return
|
||||
self._stop.clear()
|
||||
self._wake.clear()
|
||||
self._thread = Thread(target=self._run, name="repopulse-scan-worker", daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
def stop(self) -> None:
|
||||
self._stop.set()
|
||||
self._wake.set()
|
||||
thread = self._thread
|
||||
if thread and thread is not current_thread():
|
||||
thread.join(timeout=5)
|
||||
@@ -94,17 +144,37 @@ class ScanJobRunner:
|
||||
LOGGER.exception("RepoPulse worker loop recovered from an unexpected failure")
|
||||
job = None
|
||||
if job is None:
|
||||
self._stop.wait(self.poll_seconds)
|
||||
self._idle_iterations += 1
|
||||
self._wake.wait(self._idle_delay())
|
||||
self._wake.clear()
|
||||
else:
|
||||
self._idle_iterations = 0
|
||||
|
||||
def _idle_delay(self) -> float:
|
||||
"""Back off exponentially while idle; in-process enqueues wake the loop at once."""
|
||||
|
||||
exponent = min(self._idle_iterations - 1, 16)
|
||||
return min(self.idle_poll_seconds, self.poll_seconds * (2**exponent))
|
||||
|
||||
def wake(self) -> None:
|
||||
self._wake.set()
|
||||
|
||||
def _run_iteration(self) -> ScanJob | None:
|
||||
"""Perform one scheduler pass unless durable maintenance has paused work."""
|
||||
|
||||
if self.store.maintenance_status()["enabled"]:
|
||||
return None
|
||||
# The heartbeat is a liveness signal, not a lock: claims and enqueues re-check
|
||||
# maintenance inside their own transactions, so writing it on every 0.5 s poll
|
||||
# only churns the WAL. Throttle the write to the heartbeat interval.
|
||||
now = monotonic()
|
||||
if now - self._last_heartbeat >= self.heartbeat_interval_seconds:
|
||||
if not self.store.touch_worker_heartbeat_if_active(datetime.now(UTC).isoformat()):
|
||||
return None
|
||||
self._last_heartbeat = now
|
||||
if self.gitea.configured and monotonic() - self._last_sync >= self.sync_interval_seconds:
|
||||
self.sync_now()
|
||||
# Synchronization can take a while; re-check before claiming new work.
|
||||
if self.store.maintenance_status()["enabled"]:
|
||||
return None
|
||||
return self.process_one()
|
||||
@@ -135,7 +205,8 @@ class ScanJobRunner:
|
||||
def _schedule_changed_repositories(self) -> None:
|
||||
if self.store.maintenance_status()["enabled"]:
|
||||
return
|
||||
for summary in self.store.list_repositories(limit=1000):
|
||||
# No silent catalog cap: every repository is reconciled, however large the catalog.
|
||||
for summary in self.store.list_repositories(limit=-1):
|
||||
if summary.provider != "gitea" or summary.archived or not summary.latest_revision:
|
||||
continue
|
||||
if not summary.policy.auto_scan:
|
||||
@@ -166,13 +237,15 @@ class ScanJobRunner:
|
||||
trigger: ScanTrigger = ScanTrigger.MANUAL,
|
||||
) -> ScanJob:
|
||||
requested_revision = repository.latest_revision
|
||||
return self.store.enqueue_job(
|
||||
job = self.store.enqueue_job(
|
||||
repository.id,
|
||||
requested_revision,
|
||||
trigger,
|
||||
force=force,
|
||||
max_queue=self.max_queue,
|
||||
)
|
||||
self.wake()
|
||||
return job
|
||||
|
||||
def process_one(self) -> ScanJob | None:
|
||||
job = self.store.claim_next_job()
|
||||
@@ -194,6 +267,8 @@ class ScanJobRunner:
|
||||
)
|
||||
except Exception:
|
||||
LOGGER.exception("RepoPulse could not persist an unexpected job failure")
|
||||
finally:
|
||||
release_memory()
|
||||
return self.store.get_job(job.id)
|
||||
|
||||
class _LeaseHeartbeat:
|
||||
|
||||
+29
-6
@@ -11,8 +11,10 @@ from threading import BoundedSemaphore, Lock
|
||||
|
||||
from fastapi import FastAPI, HTTPException, Query, Request, Response, status
|
||||
from fastapi.concurrency import run_in_threadpool
|
||||
from fastapi.middleware.gzip import GZipMiddleware
|
||||
from fastapi.responses import JSONResponse
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from starlette.types import Scope
|
||||
|
||||
from .acquisition import GitAcquirer
|
||||
from .gitea import GiteaClient
|
||||
@@ -35,7 +37,7 @@ from .reporting import content_disposition, render_html_report, safe_report_file
|
||||
from .scanner import scan_repository
|
||||
from .store import MaintenanceModeError, QueueFullError, ScanStore
|
||||
|
||||
VERSION = "1.5.0"
|
||||
VERSION = "1.5.1"
|
||||
store = ScanStore()
|
||||
SCAN_ROOT = os.getenv("REPOPULSE_SCAN_ROOT")
|
||||
|
||||
@@ -82,6 +84,8 @@ async def lifespan(_app: FastAPI):
|
||||
|
||||
|
||||
app = FastAPI(title="RepoPulse API", version=VERSION, lifespan=lifespan)
|
||||
# Catalog, report and bundle payloads are large, repetitive JSON/JS; compress them.
|
||||
app.add_middleware(GZipMiddleware, minimum_size=1024, compresslevel=5)
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
@@ -167,7 +171,7 @@ def meta() -> dict[str, object]:
|
||||
return {
|
||||
"name": "RepoPulse",
|
||||
"version": VERSION,
|
||||
"stage": "v1.5.0",
|
||||
"stage": "v1.5.1",
|
||||
"scan_root_restricted": bool(SCAN_ROOT),
|
||||
"scan_root": SCAN_ROOT,
|
||||
"max_concurrent_scans": MAX_CONCURRENT_SCANS,
|
||||
@@ -242,7 +246,8 @@ async def create_scan(payload: ScanRequest, request: Request) -> ScanReport:
|
||||
raise HTTPException(status_code=422, detail="Repository path cannot be scanned") from exc
|
||||
finally:
|
||||
SCAN_SLOTS.release()
|
||||
return store.save(report)
|
||||
# SQLite writes may wait on busy_timeout; never block the event loop with them.
|
||||
return await asyncio.to_thread(store.save, report)
|
||||
|
||||
|
||||
@app.get("/api/v1/repositories", response_model=list[RepositorySummary])
|
||||
@@ -296,7 +301,7 @@ def repository_trend(
|
||||
|
||||
|
||||
@app.post("/api/v1/repositories/{repository_id}/scans", response_model=ScanJob, status_code=202)
|
||||
async def create_repository_scan(
|
||||
def create_repository_scan(
|
||||
request: Request,
|
||||
repository_id: str,
|
||||
payload: RepositoryScanRequest | None = None,
|
||||
@@ -357,6 +362,12 @@ async def gitea_webhook(request: Request) -> WebhookResponse:
|
||||
raise HTTPException(status_code=400, detail="Webhook payload is invalid JSON") from exc
|
||||
if not isinstance(payload, dict):
|
||||
raise HTTPException(status_code=400, detail="Webhook payload must be an object")
|
||||
event = request.headers.get("x-gitea-event", "push").casefold()
|
||||
# Catalog reads and the enqueue transaction are blocking SQLite work.
|
||||
return await run_in_threadpool(_handle_gitea_webhook, payload, event)
|
||||
|
||||
|
||||
def _handle_gitea_webhook(payload: dict, event: str) -> WebhookResponse:
|
||||
repository_payload = payload.get("repository")
|
||||
if not isinstance(repository_payload, dict):
|
||||
raise HTTPException(status_code=400, detail="Webhook repository is missing")
|
||||
@@ -368,7 +379,6 @@ async def gitea_webhook(request: Request) -> WebhookResponse:
|
||||
accepted=False,
|
||||
message="Repository is not present in the synchronized catalog",
|
||||
)
|
||||
event = request.headers.get("x-gitea-event", "push").casefold()
|
||||
if event != "push":
|
||||
return WebhookResponse(
|
||||
accepted=True,
|
||||
@@ -486,6 +496,19 @@ def export_scan(scan_id: str, format: str = Query(default="html", pattern="^(htm
|
||||
)
|
||||
|
||||
|
||||
class _FrontendStaticFiles(StaticFiles):
|
||||
"""Serve the SPA; content-hashed build assets are immutable and cached long-term."""
|
||||
|
||||
async def get_response(self, path: str, scope: Scope) -> Response:
|
||||
response = await super().get_response(path, scope)
|
||||
if response.status_code == 200:
|
||||
if path.replace("\\", "/").startswith("assets/"):
|
||||
response.headers["Cache-Control"] = "public, max-age=31536000, immutable"
|
||||
else:
|
||||
response.headers["Cache-Control"] = "no-cache"
|
||||
return response
|
||||
|
||||
|
||||
static_dir = Path(os.getenv("REPOPULSE_STATIC_DIR", "/app/static"))
|
||||
if static_dir.is_dir():
|
||||
app.mount("/", StaticFiles(directory=static_dir, html=True), name="frontend")
|
||||
app.mount("/", _FrontendStaticFiles(directory=static_dir, html=True), name="frontend")
|
||||
@@ -0,0 +1,77 @@
|
||||
"""Bounded child-process execution that never leaves orphaned descendants behind."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
from collections.abc import Mapping, Sequence
|
||||
|
||||
# Git may daemonize automatic housekeeping (``gc --auto``/``maintenance --auto``) after a
|
||||
# fetch. A detached child escapes the timeout and is reparented to PID 1, where it becomes
|
||||
# a zombie if PID 1 does not reap. Housekeeping therefore runs in the foreground.
|
||||
GIT_FOREGROUND_CONFIG: tuple[tuple[str, str], ...] = (
|
||||
("gc.autoDetach", "false"),
|
||||
("maintenance.autoDetach", "false"),
|
||||
)
|
||||
|
||||
|
||||
def git_environment(
|
||||
base: Mapping[str, str] | None = None,
|
||||
extra_config: Sequence[tuple[str, str]] = (),
|
||||
) -> dict[str, str]:
|
||||
"""Return an environment that applies foreground housekeeping plus ``extra_config``."""
|
||||
|
||||
env = dict(os.environ if base is None else base)
|
||||
pairs = [*GIT_FOREGROUND_CONFIG, *extra_config]
|
||||
env["GIT_CONFIG_COUNT"] = str(len(pairs))
|
||||
for index, (key, value) in enumerate(pairs):
|
||||
env[f"GIT_CONFIG_KEY_{index}"] = key
|
||||
env[f"GIT_CONFIG_VALUE_{index}"] = value
|
||||
return env
|
||||
|
||||
|
||||
def _kill_process_tree(process: subprocess.Popen[bytes]) -> None:
|
||||
if os.name == "posix":
|
||||
try:
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
except (ProcessLookupError, PermissionError):
|
||||
pass
|
||||
else:
|
||||
process.kill()
|
||||
|
||||
|
||||
def run_bounded(
|
||||
args: Sequence[str],
|
||||
*,
|
||||
timeout: float,
|
||||
env: Mapping[str, str] | None = None,
|
||||
check: bool = True,
|
||||
) -> subprocess.CompletedProcess[bytes]:
|
||||
"""Run ``args`` in its own process group and reap it and all descendants.
|
||||
|
||||
On timeout the whole group is killed (not only the direct child), then the child is
|
||||
waited for so no zombie remains. Raises the same exceptions as ``subprocess.run``.
|
||||
"""
|
||||
|
||||
process = subprocess.Popen(
|
||||
list(args),
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
env=None if env is None else dict(env),
|
||||
start_new_session=os.name == "posix",
|
||||
)
|
||||
try:
|
||||
stdout, stderr = process.communicate(timeout=timeout)
|
||||
except subprocess.TimeoutExpired:
|
||||
_kill_process_tree(process)
|
||||
process.communicate()
|
||||
raise
|
||||
except BaseException:
|
||||
_kill_process_tree(process)
|
||||
process.wait()
|
||||
raise
|
||||
if check and process.returncode:
|
||||
raise subprocess.CalledProcessError(process.returncode, list(args), stdout, stderr)
|
||||
return subprocess.CompletedProcess(list(args), process.returncode, stdout, stderr)
|
||||
+18
-6
@@ -1,3 +1,5 @@
|
||||
from functools import lru_cache
|
||||
|
||||
from .models import Finding, FindingGuidance, Severity
|
||||
|
||||
_RULES: dict[str, tuple[str, str, str, str]] = {
|
||||
@@ -63,12 +65,16 @@ def _default_rule(finding: Finding) -> tuple[str, str, str, str]:
|
||||
)
|
||||
|
||||
|
||||
def enrich_finding(finding: Finding) -> Finding:
|
||||
remediation_class, why, action, confidence = _RULES.get(finding.code, _default_rule(finding))
|
||||
severity_weight = {Severity.BLOCKER: 700, Severity.WARNING: 400, Severity.INFO: 100}[finding.severity]
|
||||
priority = min(1000, severity_weight + finding.deduction * 5)
|
||||
rationale = f"{finding.severity.value.title()} signal; {finding.deduction} health-point impact."
|
||||
guidance = FindingGuidance(
|
||||
@lru_cache(maxsize=4096)
|
||||
def _guidance(code: str, severity: Severity, deduction: int) -> FindingGuidance:
|
||||
"""Guidance depends only on code, severity and deduction; share one immutable instance."""
|
||||
|
||||
probe = Finding(code=code, category="", severity=severity, message="", deduction=deduction)
|
||||
remediation_class, why, action, confidence = _RULES.get(code, _default_rule(probe))
|
||||
severity_weight = {Severity.BLOCKER: 700, Severity.WARNING: 400, Severity.INFO: 100}[severity]
|
||||
priority = min(1000, severity_weight + deduction * 5)
|
||||
rationale = f"{severity.value.title()} signal; {deduction} health-point impact."
|
||||
return FindingGuidance(
|
||||
remediation_class=remediation_class,
|
||||
confidence=confidence,
|
||||
why=why,
|
||||
@@ -76,6 +82,12 @@ def enrich_finding(finding: Finding) -> Finding:
|
||||
priority=priority,
|
||||
priority_rationale=rationale,
|
||||
)
|
||||
|
||||
|
||||
def enrich_finding(finding: Finding) -> Finding:
|
||||
guidance = _guidance(finding.code, finding.severity, finding.deduction)
|
||||
if finding.guidance is guidance:
|
||||
return finding
|
||||
return finding.model_copy(update={"guidance": guidance})
|
||||
|
||||
|
||||
|
||||
+127
-56
@@ -5,13 +5,18 @@ import fnmatch
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
from collections import Counter
|
||||
from collections import Counter, deque
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from time import perf_counter
|
||||
from uuid import uuid4
|
||||
|
||||
from .architecture import build_architecture
|
||||
from .architecture import (
|
||||
build_architecture_from_imports,
|
||||
extract_js_specifiers,
|
||||
python_import_modules,
|
||||
python_package_parts,
|
||||
)
|
||||
from .git_analysis import analyze_git
|
||||
from .models import (
|
||||
Finding,
|
||||
@@ -25,9 +30,10 @@ from .models import (
|
||||
finding_fingerprint,
|
||||
)
|
||||
from .policy import load_policy
|
||||
from .processes import git_environment, run_bounded
|
||||
from .release_gate import build_release_gate
|
||||
from .rules import enrich_findings
|
||||
from .signals import analyze_dependencies, detect_test_signals
|
||||
from .signals import analyze_dependencies, detect_test_signals, is_signal_text_path
|
||||
|
||||
LANGUAGES = {
|
||||
".py": "Python",
|
||||
@@ -224,11 +230,8 @@ class _GitIgnoreMatcher:
|
||||
|
||||
def _git_worktree_root(root: Path) -> Path | None:
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(root), "rev-parse", "--show-toplevel"],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
timeout=5,
|
||||
result = run_bounded(
|
||||
["git", "-C", str(root), "rev-parse", "--show-toplevel"], timeout=5, env=git_environment()
|
||||
)
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
return None
|
||||
@@ -238,19 +241,37 @@ def _git_worktree_root(root: Path) -> Path | None:
|
||||
return None
|
||||
|
||||
|
||||
def _safe_file(root: Path, candidate: Path) -> Path | None:
|
||||
class _SafePathChecker:
|
||||
"""Validate candidate files below one root, caching directories already proven safe.
|
||||
|
||||
Every path component is still checked for symlinks, but a directory shared by many
|
||||
files is inspected once instead of once per file.
|
||||
"""
|
||||
|
||||
def __init__(self, root: Path) -> None:
|
||||
self.root = root
|
||||
self.root_resolved = root.resolve()
|
||||
self._safe_dirs: set[Path] = set()
|
||||
|
||||
def check(self, candidate: Path) -> Path | None:
|
||||
"""Return a resolved regular file only when every path component is safe."""
|
||||
try:
|
||||
relative = candidate.relative_to(root)
|
||||
relative = candidate.relative_to(self.root)
|
||||
if _ignored(str(relative)):
|
||||
return None
|
||||
current = root
|
||||
for part in Path(relative).parts:
|
||||
current = self.root
|
||||
parts = relative.parts
|
||||
for index, part in enumerate(parts):
|
||||
current /= part
|
||||
is_last = index == len(parts) - 1
|
||||
if not is_last and current in self._safe_dirs:
|
||||
continue
|
||||
if current.is_symlink():
|
||||
return None
|
||||
if not is_last:
|
||||
self._safe_dirs.add(current)
|
||||
resolved = candidate.resolve(strict=True)
|
||||
root_resolved = root.resolve()
|
||||
root_resolved = self.root_resolved
|
||||
if resolved == root_resolved or root_resolved not in resolved.parents:
|
||||
return None
|
||||
if not resolved.is_file():
|
||||
@@ -260,20 +281,21 @@ def _safe_file(root: Path, candidate: Path) -> Path | None:
|
||||
return resolved
|
||||
|
||||
|
||||
def _tracked_files(root: Path) -> list[Path] | None:
|
||||
def _safe_file(root: Path, candidate: Path) -> Path | None:
|
||||
"""Return a resolved regular file only when every path component is safe."""
|
||||
return _SafePathChecker(root).check(candidate)
|
||||
|
||||
|
||||
def _tracked_files(root: Path) -> list[tuple[str, Path]] | None:
|
||||
if _git_worktree_root(root) != root:
|
||||
return None
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(root), "ls-files", "-z"],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
timeout=5,
|
||||
)
|
||||
result = run_bounded(["git", "-C", str(root), "ls-files", "-z"], timeout=5, env=git_environment())
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
return None
|
||||
|
||||
paths: list[Path] = []
|
||||
checker = _SafePathChecker(root)
|
||||
paths: list[tuple[str, Path]] = []
|
||||
for raw in result.stdout.split(b"\0"):
|
||||
if not raw:
|
||||
continue
|
||||
@@ -281,14 +303,15 @@ def _tracked_files(root: Path) -> list[Path] | None:
|
||||
relative_path = Path(relative)
|
||||
if relative_path.is_absolute() or _ignored(relative):
|
||||
continue
|
||||
safe = _safe_file(root, root / relative_path)
|
||||
safe = checker.check(root / relative_path)
|
||||
if safe is not None:
|
||||
paths.append(safe)
|
||||
paths.append((relative_path.as_posix(), safe))
|
||||
return paths
|
||||
|
||||
|
||||
def _walk_files(root: Path) -> list[Path]:
|
||||
files: list[Path] = []
|
||||
def _walk_files(root: Path) -> list[tuple[str, Path]]:
|
||||
files: list[tuple[str, Path]] = []
|
||||
checker = _SafePathChecker(root)
|
||||
gitignore = _GitIgnoreMatcher(root)
|
||||
for current, dirs, names in os.walk(root, followlinks=False):
|
||||
current_path = Path(current)
|
||||
@@ -312,13 +335,14 @@ def _walk_files(root: Path) -> list[Path]:
|
||||
continue
|
||||
if gitignore.is_ignored(relative, is_dir=False):
|
||||
continue
|
||||
safe = _safe_file(root, candidate)
|
||||
safe = checker.check(candidate)
|
||||
if safe is not None:
|
||||
files.append(safe)
|
||||
files.append((relative.replace("\\", "/"), safe))
|
||||
return files
|
||||
|
||||
|
||||
def _read_text(path: Path, max_bytes: int = MAX_TEXT_BYTES) -> str | None:
|
||||
def _read_text_bytes(path: Path, max_bytes: int = MAX_TEXT_BYTES) -> tuple[str, int] | None:
|
||||
"""Return decoded text plus the number of bytes read, or None for binary/oversized files."""
|
||||
try:
|
||||
if path.stat().st_size > max_bytes:
|
||||
return None
|
||||
@@ -327,7 +351,12 @@ def _read_text(path: Path, max_bytes: int = MAX_TEXT_BYTES) -> str | None:
|
||||
return None
|
||||
if b"\0" in raw[:8192]:
|
||||
return None
|
||||
return raw.decode("utf-8", errors="replace")
|
||||
return raw.decode("utf-8", errors="replace"), len(raw)
|
||||
|
||||
|
||||
def _read_text(path: Path, max_bytes: int = MAX_TEXT_BYTES) -> str | None:
|
||||
result = _read_text_bytes(path, max_bytes)
|
||||
return result[0] if result else None
|
||||
|
||||
|
||||
def _is_test_file(relative: str) -> bool:
|
||||
@@ -343,19 +372,45 @@ def _is_test_file(relative: str) -> bool:
|
||||
)
|
||||
|
||||
|
||||
def _python_function_findings(relative: str, text: str) -> list[Finding]:
|
||||
PYTHON_BRANCH_TYPES = (ast.If, ast.For, ast.AsyncFor, ast.While, ast.Try, ast.Match, ast.BoolOp)
|
||||
|
||||
|
||||
def _analyze_python(relative: str, text: str) -> tuple[list[Finding], tuple[str, ...]]:
|
||||
"""Parse one Python file once and return function findings plus unresolved imports.
|
||||
|
||||
A single breadth-first walk (the same order as ``ast.walk``) attributes each branch
|
||||
construct to every enclosing function, matching a per-function ``ast.walk`` count
|
||||
without re-walking nested subtrees.
|
||||
"""
|
||||
try:
|
||||
tree = ast.parse(text)
|
||||
except SyntaxError:
|
||||
return []
|
||||
except (SyntaxError, ValueError):
|
||||
return [], ()
|
||||
package_parts = python_package_parts(relative)
|
||||
imports: dict[str, None] = {}
|
||||
functions: list[ast.FunctionDef | ast.AsyncFunctionDef] = []
|
||||
branch_counts: list[int] = []
|
||||
pending: deque[tuple[ast.AST, tuple[int, ...]]] = deque([(tree, ())])
|
||||
while pending:
|
||||
node, enclosing = pending.popleft()
|
||||
if isinstance(node, PYTHON_BRANCH_TYPES):
|
||||
for index in enclosing:
|
||||
branch_counts[index] += 1
|
||||
elif isinstance(node, (ast.Import, ast.ImportFrom)):
|
||||
for module in python_import_modules(node, package_parts):
|
||||
imports.setdefault(module)
|
||||
child_enclosing = enclosing
|
||||
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
||||
functions.append(node)
|
||||
branch_counts.append(0)
|
||||
child_enclosing = (*enclosing, len(functions) - 1)
|
||||
pending.extend((child, child_enclosing) for child in ast.iter_child_nodes(node))
|
||||
del tree, pending
|
||||
|
||||
findings: list[Finding] = []
|
||||
branch_types = (ast.If, ast.For, ast.AsyncFor, ast.While, ast.Try, ast.Match, ast.BoolOp)
|
||||
for node in ast.walk(tree):
|
||||
if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
||||
continue
|
||||
for node, branches in zip(functions, branch_counts, strict=True):
|
||||
end_line = getattr(node, "end_lineno", node.lineno)
|
||||
length = max(1, end_line - node.lineno + 1)
|
||||
branches = sum(1 for child in ast.walk(node) if isinstance(child, branch_types))
|
||||
if length >= 140:
|
||||
findings.append(
|
||||
Finding(
|
||||
@@ -380,7 +435,11 @@ def _python_function_findings(relative: str, text: str) -> list[Finding]:
|
||||
deduction=4,
|
||||
)
|
||||
)
|
||||
return findings[:12]
|
||||
return findings[:12], tuple(imports)
|
||||
|
||||
|
||||
def _python_function_findings(relative: str, text: str) -> list[Finding]:
|
||||
return _analyze_python(relative, text)[0]
|
||||
|
||||
|
||||
def _is_placeholder_secret(match: re.Match[str]) -> bool:
|
||||
@@ -408,7 +467,7 @@ def scan_repository(
|
||||
limits = limits or scan_limits_from_environment()
|
||||
tracked_files = _tracked_files(root)
|
||||
all_files = _walk_files(root) if tracked_files is None else tracked_files
|
||||
all_files.sort(key=lambda item: str(item.relative_to(root)).replace("\\", "/").casefold())
|
||||
all_files.sort(key=lambda item: item[0].casefold())
|
||||
files = all_files[: limits.max_files]
|
||||
coverage = ScanCoverage(
|
||||
limits=limits,
|
||||
@@ -426,8 +485,11 @@ def scan_repository(
|
||||
metrics = RepositoryMetrics(files=len(all_files))
|
||||
files_by_name: dict[str, list[Path]] = {}
|
||||
root_entries = {entry.name.lower() for entry in root.iterdir()}
|
||||
text_by_path: dict[str, str] = {}
|
||||
source_texts: dict[str, str] = {}
|
||||
# Only small derived values are retained per file. Full text is held for a single
|
||||
# file at a time, except the few CI/manifest texts that test-signal detection needs.
|
||||
signal_texts: dict[str, str] = {}
|
||||
readable_paths: list[str] = []
|
||||
imports_by_path: dict[str, tuple[str, ...]] = {}
|
||||
language_by_path: dict[str, str] = {}
|
||||
loc_by_path: dict[str, int] = {}
|
||||
debt_by_path: dict[str, int] = {}
|
||||
@@ -435,11 +497,10 @@ def scan_repository(
|
||||
secret_findings_truncated = False
|
||||
text_budget_exhausted = False
|
||||
|
||||
for path_obj in files:
|
||||
for relative, path_obj in files:
|
||||
try:
|
||||
relative = str(path_obj.relative_to(root)).replace("\\", "/")
|
||||
size = path_obj.stat().st_size
|
||||
except (OSError, ValueError):
|
||||
except OSError:
|
||||
continue
|
||||
|
||||
files_by_name.setdefault(path_obj.name, []).append(path_obj)
|
||||
@@ -479,21 +540,28 @@ def scan_repository(
|
||||
coverage.skipped_files_by_text_budget += 1
|
||||
text_budget_exhausted = True
|
||||
continue
|
||||
text = _read_text(path_obj, limits.max_text_bytes_per_file)
|
||||
if text is None:
|
||||
read = _read_text_bytes(path_obj, limits.max_text_bytes_per_file)
|
||||
if read is None:
|
||||
continue
|
||||
text, byte_count = read
|
||||
coverage.scanned_files += 1
|
||||
coverage.text_bytes_scanned += len(text.encode("utf-8", errors="replace"))
|
||||
text_by_path[relative] = text
|
||||
coverage.text_bytes_scanned += byte_count
|
||||
readable_paths.append(relative)
|
||||
if is_signal_text_path(relative):
|
||||
signal_texts[relative] = text
|
||||
lines = text.splitlines()
|
||||
|
||||
if language:
|
||||
loc = sum(1 for line in text.splitlines() if line.strip())
|
||||
loc = sum(1 for line in lines if line.strip())
|
||||
metrics.source_files += 1
|
||||
metrics.lines_of_code += loc
|
||||
languages[language] += loc
|
||||
source_texts[relative] = text
|
||||
language_by_path[relative] = language
|
||||
loc_by_path[relative] = loc
|
||||
if language in {"TypeScript", "JavaScript"}:
|
||||
imports_by_path[relative] = extract_js_specifiers(text)
|
||||
elif language != "Python":
|
||||
imports_by_path[relative] = ()
|
||||
if loc > 1000:
|
||||
metrics.large_source_files += 1
|
||||
findings.append(
|
||||
@@ -507,7 +575,8 @@ def scan_repository(
|
||||
)
|
||||
)
|
||||
if language == "Python":
|
||||
findings.extend(_python_function_findings(relative, text))
|
||||
python_findings, imports_by_path[relative] = _analyze_python(relative, text)
|
||||
findings.extend(python_findings)
|
||||
|
||||
if _is_test_file(relative):
|
||||
metrics.test_files += 1
|
||||
@@ -520,7 +589,7 @@ def scan_repository(
|
||||
debt_by_path[relative] = sum(debt.values())
|
||||
|
||||
if _is_secret_scan_candidate(path_obj, relative, language) and secret_findings < limits.max_secret_findings:
|
||||
for line_number, line in enumerate(text.splitlines(), start=1):
|
||||
for line_number, line in enumerate(lines, start=1):
|
||||
match = SECRET_PATTERN.search(line)
|
||||
if match is None or _is_placeholder_secret(match):
|
||||
continue
|
||||
@@ -567,11 +636,13 @@ def scan_repository(
|
||||
findings.append(Finding(code="hygiene.missing-env-example", category="Repository hygiene", severity=Severity.INFO, message="Environment configuration exists without a root .env.example", deduction=3))
|
||||
|
||||
test_signals = detect_test_signals(
|
||||
[str(path.relative_to(root)).replace("\\", "/") for path in files],
|
||||
[relative for relative, _path in files],
|
||||
metrics.source_files,
|
||||
metrics.test_files,
|
||||
text_by_path,
|
||||
signal_texts,
|
||||
readable_paths,
|
||||
)
|
||||
del signal_texts, readable_paths
|
||||
if metrics.source_files and metrics.test_files == 0:
|
||||
findings.append(Finding(code="testing.none-detected", category="Testing", severity=Severity.WARNING, message="No test files detected", deduction=30))
|
||||
if not test_signals.ci_workflows:
|
||||
@@ -593,9 +664,9 @@ def scan_repository(
|
||||
findings.extend(dependency_findings)
|
||||
|
||||
git_signals, churn_by_path = analyze_git(root)
|
||||
architecture, hotspots = build_architecture(
|
||||
architecture, hotspots = build_architecture_from_imports(
|
||||
root,
|
||||
source_texts,
|
||||
imports_by_path,
|
||||
language_by_path,
|
||||
loc_by_path,
|
||||
debt_by_path,
|
||||
|
||||
+33
-8
@@ -5,6 +5,7 @@ import re
|
||||
import tomllib
|
||||
import xml.etree.ElementTree as ET
|
||||
from collections import defaultdict
|
||||
from collections.abc import Iterable
|
||||
from pathlib import Path
|
||||
|
||||
from .models import DependencySignals, Finding, Severity, TestSignals
|
||||
@@ -32,6 +33,24 @@ COVERAGE_NAMES = {
|
||||
COVERAGE_NAMES_CASEFOLD = {item.casefold() for item in COVERAGE_NAMES}
|
||||
|
||||
|
||||
CI_WORKFLOW_PREFIXES = (".github/workflows/", ".gitea/workflows/")
|
||||
CI_WORKFLOW_NAMES = {".gitlab-ci.yml", "jenkinsfile", "azure-pipelines.yml"}
|
||||
SIGNAL_TEXT_NAMES = {"package.json", "pyproject.toml"}
|
||||
|
||||
|
||||
def is_ci_workflow_path(relative: str) -> bool:
|
||||
normalized = relative.replace("\\", "/")
|
||||
return normalized.lower().startswith(CI_WORKFLOW_PREFIXES) or (
|
||||
Path(normalized).name.casefold() in CI_WORKFLOW_NAMES
|
||||
)
|
||||
|
||||
|
||||
def is_signal_text_path(relative: str) -> bool:
|
||||
"""Return True when detect_test_signals needs this file's text content."""
|
||||
|
||||
return is_ci_workflow_path(relative) or Path(relative).name.casefold() in SIGNAL_TEXT_NAMES
|
||||
|
||||
|
||||
def _python_dependency_name(declaration: str) -> str:
|
||||
return re.split(r"[<>=!~ @;\[]", declaration, maxsplit=1)[0].strip()
|
||||
|
||||
@@ -69,7 +88,14 @@ def detect_test_signals(
|
||||
source_files: int,
|
||||
test_files: int,
|
||||
text_by_path: dict[str, str],
|
||||
readable_paths: Iterable[str] | None = None,
|
||||
) -> TestSignals:
|
||||
"""Derive test/CI signals.
|
||||
|
||||
``text_by_path`` only needs the texts accepted by :func:`is_signal_text_path`;
|
||||
``readable_paths`` lists every successfully read text file (defaults to the
|
||||
``text_by_path`` keys) so coverage evidence does not require retaining text.
|
||||
"""
|
||||
frameworks: set[str] = set()
|
||||
coverage: list[str] = []
|
||||
ci: list[str] = []
|
||||
@@ -85,11 +111,7 @@ def detect_test_signals(
|
||||
lower = normalized.lower()
|
||||
if name_lower in COVERAGE_NAMES_CASEFOLD or "/coverage/" in f"/{lower}":
|
||||
coverage.append(normalized)
|
||||
if lower.startswith((".github/workflows/", ".gitea/workflows/")) or name_lower in {
|
||||
".gitlab-ci.yml",
|
||||
"jenkinsfile",
|
||||
"azure-pipelines.yml",
|
||||
}:
|
||||
if is_ci_workflow_path(normalized):
|
||||
ci.append(normalized)
|
||||
|
||||
if (name_lower.startswith("test_") or "/tests/" in f"/{lower}") and name_lower.endswith(".py"):
|
||||
@@ -103,11 +125,16 @@ def detect_test_signals(
|
||||
if name_lower.endswith(("tests.cs", "test.cs")):
|
||||
frameworks.add(".NET test")
|
||||
|
||||
ci_paths = set(ci)
|
||||
for path in text_by_path if readable_paths is None else readable_paths:
|
||||
if Path(path).name.casefold() in COVERAGE_NAMES_CASEFOLD:
|
||||
coverage_evidence.append(path.replace("\\", "/"))
|
||||
|
||||
for path, text in text_by_path.items():
|
||||
name = Path(path).name
|
||||
normalized = path.replace("\\", "/")
|
||||
if normalized in ci_paths:
|
||||
lowered_text = text.casefold()
|
||||
if normalized in ci:
|
||||
validates = any(
|
||||
token in lowered_text
|
||||
for token in ("pytest", "vitest", "jest", "npm test", "pnpm test", "go test", "dotnet test", "ruff", "eslint", "typecheck", "py_compile")
|
||||
@@ -125,8 +152,6 @@ def detect_test_signals(
|
||||
ci_deploy_only.append(normalized)
|
||||
if any(token in lowered_text for token in ("--cov", "coverage", "lcov", "codecov")):
|
||||
coverage_evidence.append(normalized)
|
||||
if name.casefold() in COVERAGE_NAMES_CASEFOLD:
|
||||
coverage_evidence.append(normalized)
|
||||
if name.casefold() == "package.json":
|
||||
try:
|
||||
data = json.loads(text)
|
||||
|
||||
+169
-140
@@ -9,6 +9,8 @@ from contextlib import contextmanager
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from hashlib import sha256
|
||||
from pathlib import Path
|
||||
from threading import Lock, Thread
|
||||
from time import monotonic
|
||||
from uuid import uuid4
|
||||
|
||||
from .models import (
|
||||
@@ -35,6 +37,7 @@ from .rules import enrich_findings
|
||||
|
||||
MAX_HISTORY_PER_REPOSITORY = 200
|
||||
SCHEMA_VERSION = 4
|
||||
MIRROR_STATS_TTL_SECONDS = 300.0
|
||||
JOB_LEASE_SECONDS = 900
|
||||
ACTIVE_JOB_STATUSES = (
|
||||
ScanJobStatus.QUEUED.value,
|
||||
@@ -108,6 +111,8 @@ class ScanStore:
|
||||
self.path = Path(raw_path)
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self.recovery_metadata: dict[str, str] | None = None
|
||||
self._mirror_stats_cache: tuple[float, str, tuple[int, int]] | None = None
|
||||
self._mirror_stats_refresh = Lock()
|
||||
try:
|
||||
self._validate_existing_database()
|
||||
self._init_schema()
|
||||
@@ -349,7 +354,7 @@ class ScanStore:
|
||||
def _backfill_legacy_catalog(self, db: sqlite3.Connection) -> None:
|
||||
rows = db.execute(
|
||||
"""
|
||||
SELECT id, repository_path, repository_name, report_json, repository_id, history_key
|
||||
SELECT id, repository_path, repository_name, repository_id, history_key
|
||||
FROM scans
|
||||
WHERE repository_id IS NULL OR history_key IS NULL
|
||||
"""
|
||||
@@ -412,11 +417,19 @@ class ScanStore:
|
||||
}
|
||||
|
||||
def _backfill_scan_summaries(self, db: sqlite3.Connection) -> None:
|
||||
rows = db.execute(
|
||||
"SELECT rowid AS scan_rowid, * FROM scans "
|
||||
# Collect ids first and load one report at a time so a large legacy history is
|
||||
# never materialized in memory at once during migration.
|
||||
scan_ids = [
|
||||
str(item[0])
|
||||
for item in db.execute(
|
||||
"SELECT id FROM scans "
|
||||
"WHERE finding_count IS NULL OR blocker_count IS NULL OR release_readiness IS NULL"
|
||||
).fetchall()
|
||||
for row in rows:
|
||||
]
|
||||
for scan_id in scan_ids:
|
||||
row = db.execute("SELECT rowid AS scan_rowid, * FROM scans WHERE id = ?", (scan_id,)).fetchone()
|
||||
if row is None:
|
||||
continue
|
||||
report = self._report_from_row(row)
|
||||
previous_row = db.execute(
|
||||
"""
|
||||
@@ -560,112 +573,6 @@ class ScanStore:
|
||||
row = db.execute("SELECT source_path FROM repositories WHERE id = ?", (repository_id,)).fetchone()
|
||||
return row["source_path"] if row else None
|
||||
|
||||
def _repository_summary(self, db: sqlite3.Connection, row: sqlite3.Row) -> RepositorySummary:
|
||||
repository = self._repository_from_row(row)
|
||||
latest = db.execute(
|
||||
"""
|
||||
SELECT * FROM scans WHERE repository_id = ?
|
||||
ORDER BY scanned_at DESC, rowid DESC LIMIT 1
|
||||
""",
|
||||
(repository.id,),
|
||||
).fetchone()
|
||||
active = db.execute(
|
||||
"""
|
||||
SELECT * FROM scan_jobs WHERE repository_id = ? AND status IN (?, ?, ?, ?, ?)
|
||||
ORDER BY created_at DESC LIMIT 1
|
||||
""",
|
||||
(repository.id, *ACTIVE_JOB_STATUSES),
|
||||
).fetchone()
|
||||
last_job = db.execute(
|
||||
"SELECT * FROM scan_jobs WHERE repository_id = ? ORDER BY created_at DESC LIMIT 1",
|
||||
(repository.id,),
|
||||
).fetchone()
|
||||
last_scan = self._report_from_row(latest) if latest else None
|
||||
active_findings = [item for item in (last_scan.findings if last_scan else []) if not item.suppressed]
|
||||
severity_rank = {Severity.BLOCKER: 0, Severity.WARNING: 1, Severity.INFO: 2}
|
||||
primary_finding = min(
|
||||
active_findings,
|
||||
key=lambda item: (severity_rank[item.severity], -item.deduction, item.category, item.code),
|
||||
default=None,
|
||||
)
|
||||
primary_signal = (
|
||||
RemediationSignal(
|
||||
code=primary_finding.code,
|
||||
category=primary_finding.category,
|
||||
severity=primary_finding.severity,
|
||||
message=primary_finding.message,
|
||||
count=sum(1 for item in active_findings if item.code == primary_finding.code),
|
||||
)
|
||||
if primary_finding
|
||||
else None
|
||||
)
|
||||
previous_row = (
|
||||
db.execute(
|
||||
"""
|
||||
SELECT * FROM scans
|
||||
WHERE repository_id = ? AND id != ?
|
||||
ORDER BY scanned_at DESC, rowid DESC LIMIT 1
|
||||
""",
|
||||
(repository.id, last_scan.id),
|
||||
).fetchone()
|
||||
if last_scan
|
||||
else None
|
||||
)
|
||||
previous_scan = self._report_from_row(previous_row) if previous_row else None
|
||||
previous_findings = [item for item in (previous_scan.findings if previous_scan else []) if not item.suppressed]
|
||||
active_fingerprints = {self._finding_key(item) for item in active_findings}
|
||||
previous_fingerprints = {self._finding_key(item) for item in previous_findings}
|
||||
last_revision = (
|
||||
last_scan.source.actual_revision
|
||||
if last_scan and last_scan.source.actual_revision
|
||||
else (latest["actual_revision"] if latest else None)
|
||||
)
|
||||
stale = bool(repository.latest_revision and last_revision and repository.latest_revision != last_revision)
|
||||
if repository.archived:
|
||||
state = RepositoryState.ARCHIVED
|
||||
elif not repository.available:
|
||||
state = RepositoryState.UNAVAILABLE
|
||||
elif active:
|
||||
state = (
|
||||
RepositoryState.QUEUED
|
||||
if active["status"] == ScanJobStatus.QUEUED.value
|
||||
else RepositoryState.SCANNING
|
||||
)
|
||||
elif last_job and last_job["status"] == ScanJobStatus.FAILED.value:
|
||||
last_job_at = _datetime(last_job["created_at"])
|
||||
if last_scan is None or (last_job_at and last_job_at > last_scan.scanned_at):
|
||||
state = RepositoryState.FAILED
|
||||
elif stale:
|
||||
state = RepositoryState.STALE
|
||||
else:
|
||||
state = RepositoryState.CURRENT
|
||||
elif last_scan is None:
|
||||
state = RepositoryState.NEVER_SCANNED
|
||||
elif stale:
|
||||
state = RepositoryState.STALE
|
||||
else:
|
||||
state = RepositoryState.CURRENT
|
||||
data = repository.model_dump()
|
||||
data.update(
|
||||
{
|
||||
"state": state,
|
||||
"stale": stale,
|
||||
"active_job_id": active["id"] if active else None,
|
||||
"last_scan_id": last_scan.id if last_scan else None,
|
||||
"last_scan_at": last_scan.scanned_at if last_scan else None,
|
||||
"last_scan_health": last_scan.health_score if last_scan else None,
|
||||
"last_scan_readiness": last_scan.release_gate.status if last_scan and last_scan.release_gate else (last_scan.release_readiness if last_scan else None),
|
||||
"last_scan_revision": last_revision,
|
||||
"last_scan_blockers": sum(1 for item in active_findings if item.severity == Severity.BLOCKER),
|
||||
"last_scan_warnings": sum(1 for item in active_findings if item.severity == Severity.WARNING),
|
||||
"last_scan_primary_signal": primary_signal,
|
||||
"last_scan_health_delta": (last_scan.health_score - previous_scan.health_score) if previous_scan and last_scan else None,
|
||||
"last_scan_added_findings": len(active_fingerprints - previous_fingerprints) if previous_scan else 0,
|
||||
"last_scan_resolved_findings": len(previous_fingerprints - active_fingerprints) if previous_scan else 0,
|
||||
}
|
||||
)
|
||||
return RepositorySummary(**data)
|
||||
|
||||
def _catalog_rows(
|
||||
self, db: sqlite3.Connection, *, limit: int, offset: int = 0, repository_id: str | None = None
|
||||
) -> list[sqlite3.Row]:
|
||||
@@ -674,7 +581,11 @@ class ScanStore:
|
||||
return db.execute(
|
||||
f"""
|
||||
WITH ranked_scans AS (
|
||||
SELECT s.*, ROW_NUMBER() OVER (
|
||||
-- Summary columns only: report_json must never be materialized here.
|
||||
SELECT s.id, s.repository_id, s.scanned_at, s.health_score, s.release_readiness,
|
||||
s.actual_revision, s.finding_count, s.blocker_count, s.warning_count,
|
||||
s.primary_signal_json, s.added_findings, s.resolved_findings,
|
||||
s.triage_priority, s.triage_rationale, ROW_NUMBER() OVER (
|
||||
PARTITION BY s.repository_id ORDER BY s.scanned_at DESC, s.rowid DESC
|
||||
) AS scan_rank
|
||||
FROM scans AS s WHERE s.repository_id IS NOT NULL
|
||||
@@ -777,7 +688,7 @@ class ScanStore:
|
||||
|
||||
def repository_state_counts(self) -> dict[str, int]:
|
||||
counts: Counter[str] = Counter()
|
||||
for repository in self.list_repositories(limit=1000):
|
||||
for repository in self.list_repositories(limit=-1):
|
||||
counts[repository.state.value] += 1
|
||||
return dict(counts)
|
||||
|
||||
@@ -793,28 +704,67 @@ class ScanStore:
|
||||
).fetchone()
|
||||
return RepositoryDetail(**summary.model_dump(), latest_scan=self._report_from_row(latest) if latest else None)
|
||||
|
||||
@staticmethod
|
||||
def _merged_source_value(report_value: object, column_value: object, field: str) -> object:
|
||||
"""Mirror _report_from_row: a non-null column fills an absent/manual report value."""
|
||||
|
||||
if column_value is None or report_value not in (None, ScanTrigger.MANUAL.value):
|
||||
return report_value
|
||||
if field == "trigger":
|
||||
try:
|
||||
return ScanTrigger(str(column_value))
|
||||
except ValueError:
|
||||
return report_value
|
||||
return column_value
|
||||
|
||||
def repository_scans(self, repository_id: str, limit: int = 50) -> list[ScanSummary]:
|
||||
"""List scan summaries without deserializing full report payloads.
|
||||
|
||||
Finding counts come from the retained report; compacted reports no longer hold
|
||||
findings, so their retained active-finding summary count is reported instead of 0.
|
||||
"""
|
||||
|
||||
with self._connection() as db:
|
||||
rows = db.execute(
|
||||
"SELECT * FROM scans WHERE repository_id = ? ORDER BY scanned_at DESC, rowid DESC LIMIT ?",
|
||||
"""
|
||||
SELECT id, scanned_at, health_score, report_compacted, finding_count,
|
||||
release_readiness AS column_readiness,
|
||||
branch, requested_revision, actual_revision, trigger,
|
||||
json_extract(report_json, '$.source.branch') AS report_branch,
|
||||
json_extract(report_json, '$.source.requested_revision') AS report_requested_revision,
|
||||
json_extract(report_json, '$.source.actual_revision') AS report_actual_revision,
|
||||
json_extract(report_json, '$.source.trigger') AS report_trigger,
|
||||
json_extract(report_json, '$.release_readiness') AS report_readiness,
|
||||
json_extract(report_json, '$.duration_ms') AS duration_ms,
|
||||
json_array_length(report_json, '$.findings') AS report_findings
|
||||
FROM scans WHERE repository_id = ?
|
||||
ORDER BY scanned_at DESC, rowid DESC LIMIT ?
|
||||
""",
|
||||
(repository_id, limit),
|
||||
).fetchall()
|
||||
result: list[ScanSummary] = []
|
||||
for row in rows:
|
||||
report = self._report_from_row(row)
|
||||
source = {
|
||||
field: self._merged_source_value(row[f"report_{field}"], row[field], field)
|
||||
for field in ("branch", "requested_revision", "actual_revision", "trigger")
|
||||
}
|
||||
if row["report_compacted"]:
|
||||
findings = int(row["finding_count"] or 0)
|
||||
else:
|
||||
findings = int(row["report_findings"] or 0)
|
||||
result.append(
|
||||
ScanSummary(
|
||||
id=report.id,
|
||||
id=row["id"],
|
||||
repository_id=repository_id,
|
||||
scanned_at=report.scanned_at,
|
||||
branch=report.source.branch,
|
||||
requested_revision=report.source.requested_revision,
|
||||
actual_revision=report.source.actual_revision,
|
||||
trigger=report.source.trigger,
|
||||
health_score=report.health_score,
|
||||
release_readiness=report.release_readiness,
|
||||
findings=len(report.findings),
|
||||
duration_ms=report.duration_ms,
|
||||
scanned_at=row["scanned_at"],
|
||||
branch=source["branch"],
|
||||
requested_revision=source["requested_revision"],
|
||||
actual_revision=source["actual_revision"],
|
||||
trigger=source["trigger"] or ScanTrigger.MANUAL,
|
||||
health_score=row["health_score"],
|
||||
release_readiness=row["report_readiness"] or row["column_readiness"] or "",
|
||||
findings=findings,
|
||||
duration_ms=float(row["duration_ms"] or 0.0),
|
||||
)
|
||||
)
|
||||
return result
|
||||
@@ -948,14 +898,20 @@ class ScanStore:
|
||||
)
|
||||
cutoff = (_now() - timedelta(days=report_retention_days())).isoformat()
|
||||
db.execute("DELETE FROM scans WHERE history_key = ? AND scanned_at < ?", (history_key, cutoff))
|
||||
rows = db.execute(
|
||||
scan_ids = [
|
||||
str(item[0])
|
||||
for item in db.execute(
|
||||
"""
|
||||
SELECT * FROM scans WHERE history_key = ? AND report_compacted = 0
|
||||
SELECT id FROM scans WHERE history_key = ? AND report_compacted = 0
|
||||
ORDER BY scanned_at DESC, rowid DESC LIMIT -1 OFFSET ?
|
||||
""",
|
||||
(history_key, full_reports_per_repository()),
|
||||
).fetchall()
|
||||
for row in rows:
|
||||
]
|
||||
for scan_id in scan_ids:
|
||||
row = db.execute("SELECT * FROM scans WHERE id = ?", (scan_id,)).fetchone()
|
||||
if row is None:
|
||||
continue
|
||||
report = self._report_from_row(row)
|
||||
compact = report.model_copy(
|
||||
update={"findings": [], "architecture": ArchitectureGraph(), "hotspots": [], "compacted": True}
|
||||
@@ -1055,21 +1011,52 @@ class ScanStore:
|
||||
)
|
||||
|
||||
def trend(self, repository_key: str, limit: int = 50) -> list[TrendPoint]:
|
||||
reports = list(reversed(self.recent_for_repository(repository_key, limit)))
|
||||
"""Return chronological trend points computed in SQLite, not from full reports.
|
||||
|
||||
Counts include every retained finding, as before. Compacted reports no longer
|
||||
hold findings, so their retained active-finding summary counts are used rather
|
||||
than reporting a misleading zero.
|
||||
"""
|
||||
|
||||
with self._connection() as db:
|
||||
rows: list[sqlite3.Row] = []
|
||||
for candidate in self._history_candidates(repository_key):
|
||||
rows = db.execute(
|
||||
"""
|
||||
SELECT id, scanned_at, health_score, report_compacted,
|
||||
finding_count, blocker_count, warning_count,
|
||||
json_array_length(report_json, '$.findings') AS report_findings,
|
||||
(
|
||||
SELECT COUNT(*) FROM json_each(scans.report_json, '$.findings')
|
||||
WHERE json_extract(value, '$.severity') = 'blocker'
|
||||
) AS report_blockers,
|
||||
(
|
||||
SELECT COUNT(*) FROM json_each(scans.report_json, '$.findings')
|
||||
WHERE json_extract(value, '$.severity') = 'warning'
|
||||
) AS report_warnings,
|
||||
COALESCE(json_extract(report_json, '$.metrics.todo'), 0)
|
||||
+ COALESCE(json_extract(report_json, '$.metrics.fixme'), 0)
|
||||
+ COALESCE(json_extract(report_json, '$.metrics.hack'), 0)
|
||||
+ COALESCE(json_extract(report_json, '$.metrics.xxx'), 0) AS debt_markers
|
||||
FROM scans WHERE history_key = ?
|
||||
ORDER BY scanned_at DESC, rowid DESC LIMIT ?
|
||||
""",
|
||||
(candidate, limit),
|
||||
).fetchall()
|
||||
if rows:
|
||||
break
|
||||
points: list[TrendPoint] = []
|
||||
for report in reports:
|
||||
blockers = sum(1 for item in report.findings if item.severity == Severity.BLOCKER)
|
||||
warnings = sum(1 for item in report.findings if item.severity == Severity.WARNING)
|
||||
debt = report.metrics.todo + report.metrics.fixme + report.metrics.hack + report.metrics.xxx
|
||||
for row in reversed(rows):
|
||||
compacted = bool(row["report_compacted"])
|
||||
points.append(
|
||||
TrendPoint(
|
||||
scan_id=report.id,
|
||||
scanned_at=report.scanned_at,
|
||||
health_score=report.health_score,
|
||||
findings=len(report.findings),
|
||||
blockers=blockers,
|
||||
warnings=warnings,
|
||||
debt_markers=debt,
|
||||
scan_id=row["id"],
|
||||
scanned_at=row["scanned_at"],
|
||||
health_score=row["health_score"],
|
||||
findings=int((row["finding_count"] if compacted else row["report_findings"]) or 0),
|
||||
blockers=int((row["blocker_count"] if compacted else row["report_blockers"]) or 0),
|
||||
warnings=int((row["warning_count"] if compacted else row["report_warnings"]) or 0),
|
||||
debt_markers=int(row["debt_markers"] or 0),
|
||||
)
|
||||
)
|
||||
return points
|
||||
@@ -1220,11 +1207,29 @@ class ScanStore:
|
||||
row = db.execute("SELECT * FROM scan_jobs WHERE id = ?", (job_id,)).fetchone()
|
||||
return self._job_from_row(row) if row else None
|
||||
|
||||
@staticmethod
|
||||
def _claim_work_may_exist(db: sqlite3.Connection, now: str) -> bool:
|
||||
placeholders = ", ".join("?" for _ in ACTIVE_JOB_STATUSES[1:])
|
||||
row = db.execute(
|
||||
f"""
|
||||
SELECT EXISTS(SELECT 1 FROM scan_jobs WHERE status = 'queued')
|
||||
OR EXISTS(
|
||||
SELECT 1 FROM scan_jobs
|
||||
WHERE status IN ({placeholders}) AND lease_until IS NOT NULL AND lease_until < ?
|
||||
)
|
||||
""",
|
||||
(*ACTIVE_JOB_STATUSES[1:], now),
|
||||
).fetchone()
|
||||
return bool(row[0])
|
||||
|
||||
def claim_next_job(self) -> ScanJob | None:
|
||||
now_dt = _now()
|
||||
now = now_dt.isoformat()
|
||||
lease_until = (now_dt + timedelta(seconds=job_lease_seconds())).isoformat()
|
||||
with self._connection() as db:
|
||||
# Cheap read-only probe: an idle queue never takes the write lock.
|
||||
if not self._claim_work_may_exist(db, now):
|
||||
return None
|
||||
db.execute("BEGIN IMMEDIATE")
|
||||
if self._maintenance_enabled(db):
|
||||
return None
|
||||
@@ -1443,6 +1448,30 @@ class ScanStore:
|
||||
except OSError:
|
||||
return 0
|
||||
|
||||
def _cached_mirror_stats(self, mirrors_root: Path) -> tuple[int, int]:
|
||||
"""Return mirror-cache size without making /meta wait on a large tree walk.
|
||||
|
||||
The walk (up to 50k files) runs synchronously only when no value exists yet.
|
||||
Afterwards an expired value is served while one background thread refreshes it.
|
||||
"""
|
||||
|
||||
key = str(mirrors_root)
|
||||
now = monotonic()
|
||||
cached = self._mirror_stats_cache
|
||||
if cached is None or cached[1] != key:
|
||||
stats = self._tree_stats(mirrors_root)
|
||||
self._mirror_stats_cache = (monotonic(), key, stats)
|
||||
return stats
|
||||
if now - cached[0] >= MIRROR_STATS_TTL_SECONDS and self._mirror_stats_refresh.acquire(blocking=False):
|
||||
def refresh() -> None:
|
||||
try:
|
||||
self._mirror_stats_cache = (monotonic(), key, self._tree_stats(mirrors_root))
|
||||
finally:
|
||||
self._mirror_stats_refresh.release()
|
||||
|
||||
Thread(target=refresh, name="repopulse-mirror-stats", daemon=True).start()
|
||||
return cached[2]
|
||||
|
||||
def storage_stats(self) -> dict[str, int | str | None]:
|
||||
with self._connection() as db:
|
||||
row = db.execute(
|
||||
@@ -1462,7 +1491,7 @@ class ScanStore:
|
||||
cache_root = Path(os.getenv("REPOPULSE_REPOSITORY_CACHE", "data/repos")).expanduser()
|
||||
mirrors_root = cache_root / "mirrors"
|
||||
worktrees_root = cache_root / "worktrees"
|
||||
_mirror_files, mirror_cache_bytes = self._tree_stats(mirrors_root)
|
||||
_mirror_files, mirror_cache_bytes = self._cached_mirror_stats(mirrors_root)
|
||||
mirror_count = self._managed_entry_count(mirrors_root)
|
||||
worktree_count = self._managed_entry_count(worktrees_root)
|
||||
return {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[project]
|
||||
name = "repopulse-backend"
|
||||
version = "1.5.0"
|
||||
version = "1.5.1"
|
||||
description = "RepoPulse repository intelligence API and deterministic analysis engine"
|
||||
requires-python = ">=3.12"
|
||||
dependencies = [
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
import app.store as store_module
|
||||
from app import main as main_module
|
||||
from app.models import Finding, Repository, Severity
|
||||
from app.rules import enrich_finding, enrich_findings
|
||||
from app.store import ScanStore
|
||||
|
||||
|
||||
def test_database_bound_endpoints_never_run_blocking_sqlite_on_the_event_loop() -> None:
|
||||
# A sync endpoint runs in the threadpool; an async one must offload its store work.
|
||||
assert not inspect.iscoroutinefunction(main_module.create_repository_scan)
|
||||
assert not inspect.iscoroutinefunction(main_module._handle_gitea_webhook)
|
||||
webhook_source = inspect.getsource(main_module.gitea_webhook)
|
||||
assert "store." not in webhook_source
|
||||
assert "run_in_threadpool(_handle_gitea_webhook" in webhook_source
|
||||
assert "asyncio.to_thread(store.save" in inspect.getsource(main_module.create_scan)
|
||||
|
||||
|
||||
def test_large_api_responses_are_gzip_compressed(tmp_path: Path) -> None:
|
||||
main_module.store = ScanStore(str(tmp_path / "history.db"))
|
||||
main_module.job_runner = None
|
||||
for index in range(12):
|
||||
main_module.store.upsert_repository(
|
||||
Repository(
|
||||
id=f"gitea:{index}",
|
||||
provider="gitea",
|
||||
provider_repository_id=str(index),
|
||||
owner="Jens",
|
||||
name=f"Repository{index}",
|
||||
full_name=f"Jens/Repository{index}",
|
||||
default_branch="main",
|
||||
)
|
||||
)
|
||||
client = TestClient(main_module.app)
|
||||
|
||||
compressed = client.get("/api/v1/repositories", headers={"Accept-Encoding": "gzip"})
|
||||
small = client.get("/health", headers={"Accept-Encoding": "gzip"})
|
||||
|
||||
assert compressed.status_code == 200
|
||||
assert compressed.headers.get("content-encoding") == "gzip"
|
||||
assert len(compressed.json()) == 12
|
||||
# Responses below the 1 KiB threshold are not worth compressing.
|
||||
assert small.headers.get("content-encoding") is None
|
||||
|
||||
|
||||
def test_hashed_frontend_assets_are_immutable_and_index_revalidates(tmp_path: Path) -> None:
|
||||
(tmp_path / "assets").mkdir()
|
||||
(tmp_path / "index.html").write_text("<html></html>", encoding="utf-8")
|
||||
(tmp_path / "assets" / "index-abc123.js").write_text("console.log(1)", encoding="utf-8")
|
||||
app = FastAPI()
|
||||
app.mount("/", main_module._FrontendStaticFiles(directory=tmp_path, html=True), name="frontend")
|
||||
client = TestClient(app)
|
||||
|
||||
asset = client.get("/assets/index-abc123.js")
|
||||
index = client.get("/")
|
||||
|
||||
assert asset.headers["cache-control"] == "public, max-age=31536000, immutable"
|
||||
assert index.headers["cache-control"] == "no-cache"
|
||||
|
||||
|
||||
def test_finding_guidance_is_shared_and_unchanged() -> None:
|
||||
finding = Finding(
|
||||
code="security.secret-assignment",
|
||||
category="Security basics",
|
||||
severity=Severity.BLOCKER,
|
||||
message="m",
|
||||
deduction=20,
|
||||
)
|
||||
first, second = enrich_findings([finding, finding.model_copy(update={"path": "b.py"})])
|
||||
|
||||
assert first.guidance is second.guidance
|
||||
assert first.guidance.priority == 800
|
||||
assert first.guidance.priority_rationale == "Blocker signal; 20 health-point impact."
|
||||
assert enrich_finding(first) is first
|
||||
|
||||
|
||||
def test_expired_mirror_stats_are_served_while_refreshing_in_background(tmp_path: Path, monkeypatch) -> None:
|
||||
store = ScanStore(str(tmp_path / "history.db"))
|
||||
mirrors = tmp_path / "mirrors"
|
||||
mirrors.mkdir()
|
||||
calls: list[float] = []
|
||||
|
||||
def slow_walk(_root: Path) -> tuple[int, int]:
|
||||
calls.append(time.monotonic())
|
||||
time.sleep(0.3)
|
||||
return len(calls), len(calls) * 100
|
||||
|
||||
monkeypatch.setattr(store, "_tree_stats", slow_walk)
|
||||
assert store._cached_mirror_stats(mirrors) == (1, 100)
|
||||
|
||||
monkeypatch.setattr(store_module, "MIRROR_STATS_TTL_SECONDS", 0.0)
|
||||
started = time.monotonic()
|
||||
assert store._cached_mirror_stats(mirrors) == (1, 100)
|
||||
assert time.monotonic() - started < 0.2
|
||||
|
||||
deadline = time.monotonic() + 3
|
||||
while store._mirror_stats_cache[2] == (1, 100) and time.monotonic() < deadline:
|
||||
time.sleep(0.05)
|
||||
assert store._mirror_stats_cache[2] == (2, 200)
|
||||
@@ -0,0 +1,101 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app.gitea import GiteaClient
|
||||
from app.jobs import ScanJobRunner
|
||||
from app.processes import git_environment, run_bounded
|
||||
from app.store import ScanStore
|
||||
|
||||
|
||||
def test_git_environment_forces_foreground_housekeeping_and_keeps_extra_config() -> None:
|
||||
env = git_environment({}, extra_config=[("http.extraHeader", "Authorization: token t")])
|
||||
|
||||
pairs = {
|
||||
env[f"GIT_CONFIG_KEY_{index}"]: env[f"GIT_CONFIG_VALUE_{index}"]
|
||||
for index in range(int(env["GIT_CONFIG_COUNT"]))
|
||||
}
|
||||
assert pairs == {
|
||||
"gc.autoDetach": "false",
|
||||
"maintenance.autoDetach": "false",
|
||||
"http.extraHeader": "Authorization: token t",
|
||||
}
|
||||
|
||||
|
||||
def test_run_bounded_returns_output_and_raises_on_failure() -> None:
|
||||
ok = run_bounded([sys.executable, "-c", "print('hi')"], timeout=10)
|
||||
assert ok.stdout.strip() == b"hi"
|
||||
with pytest.raises(subprocess.CalledProcessError):
|
||||
run_bounded([sys.executable, "-c", "raise SystemExit(3)"], timeout=10)
|
||||
|
||||
|
||||
@pytest.mark.skipif(os.name != "posix", reason="process groups are POSIX-only")
|
||||
def test_run_bounded_timeout_kills_grandchildren(tmp_path: Path) -> None:
|
||||
marker = tmp_path / "grandchild.pid"
|
||||
script = (
|
||||
"import subprocess, sys, time\n"
|
||||
"child = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(60)'])\n"
|
||||
f"open({str(marker)!r}, 'w').write(str(child.pid))\n"
|
||||
"time.sleep(60)\n"
|
||||
)
|
||||
with pytest.raises(subprocess.TimeoutExpired):
|
||||
run_bounded([sys.executable, "-c", script], timeout=2)
|
||||
|
||||
grandchild = int(marker.read_text())
|
||||
deadline = time.monotonic() + 5
|
||||
while time.monotonic() < deadline:
|
||||
if _process_gone_or_zombie(grandchild):
|
||||
return
|
||||
time.sleep(0.05)
|
||||
pytest.fail("grandchild survived the timeout kill")
|
||||
|
||||
|
||||
def _process_gone_or_zombie(pid: int) -> bool:
|
||||
"""A killed orphan stays a zombie until PID 1 reaps it (tini does in production).
|
||||
|
||||
Inside a CI job container PID 1 may not reap, so a zombie counts as killed.
|
||||
"""
|
||||
try:
|
||||
stat = Path(f"/proc/{pid}/stat").read_text()
|
||||
except (FileNotFoundError, ProcessLookupError):
|
||||
return True
|
||||
# Field 3, after the parenthesised command name, is the process state.
|
||||
return stat.rsplit(")", 1)[1].split()[0] in {"Z", "X"}
|
||||
|
||||
|
||||
def test_idle_claim_never_takes_the_write_lock(tmp_path: Path) -> None:
|
||||
store = ScanStore(str(tmp_path / "repopulse.db"))
|
||||
writer = sqlite3.connect(tmp_path / "repopulse.db", timeout=0)
|
||||
try:
|
||||
writer.execute("BEGIN IMMEDIATE")
|
||||
started = time.monotonic()
|
||||
assert store.claim_next_job() is None
|
||||
# Without the read-only probe this would block on busy_timeout (5 s).
|
||||
assert time.monotonic() - started < 2
|
||||
finally:
|
||||
writer.rollback()
|
||||
writer.close()
|
||||
|
||||
|
||||
def test_idle_worker_backs_off_and_enqueue_wakes_it(tmp_path: Path) -> None:
|
||||
runner = ScanJobRunner(ScanStore(str(tmp_path / "repopulse.db")), gitea=GiteaClient())
|
||||
runner.poll_seconds = 0.5
|
||||
runner.idle_poll_seconds = 5.0
|
||||
|
||||
delays = []
|
||||
for iterations in range(1, 7):
|
||||
runner._idle_iterations = iterations
|
||||
delays.append(runner._idle_delay())
|
||||
assert delays == [0.5, 1.0, 2.0, 4.0, 5.0, 5.0]
|
||||
|
||||
runner.wake()
|
||||
started = time.monotonic()
|
||||
assert runner._wake.wait(1)
|
||||
assert time.monotonic() - started < 0.5
|
||||
@@ -221,3 +221,47 @@ def test_empty_git_repository_does_not_fallback_to_untracked_files(tmp_path: Pat
|
||||
assert report.git.tracked_files == 0
|
||||
assert report.metrics.files == 0
|
||||
assert report.metrics.source_files == 0
|
||||
|
||||
|
||||
def test_single_pass_python_analysis_matches_per_function_branch_counts() -> None:
|
||||
import ast
|
||||
|
||||
from app.scanner import PYTHON_BRANCH_TYPES, _analyze_python
|
||||
|
||||
inner = "\n".join(f" if x == {i}:\n x += 1" for i in range(12))
|
||||
outer = "\n".join(f" for i{i} in range(2):\n pass" for i in range(10))
|
||||
source = (
|
||||
"import os\nfrom . import sibling\n\n"
|
||||
f"def outer(x):\n{outer}\n def inner(x):\n{inner}\n return x\n return inner(x)\n"
|
||||
)
|
||||
findings, imports = _analyze_python("pkg/mod.py", source)
|
||||
|
||||
expected = {
|
||||
node.name: sum(1 for child in ast.walk(node) if isinstance(child, PYTHON_BRANCH_TYPES))
|
||||
for node in ast.walk(ast.parse(source))
|
||||
if isinstance(node, ast.FunctionDef)
|
||||
}
|
||||
assert expected == {"outer": 22, "inner": 12}
|
||||
branch_messages = [item.message for item in findings if item.code == "maintainability.branch-heavy-function"]
|
||||
assert branch_messages == ["Function outer contains 22 branch constructs"]
|
||||
assert imports == ("os", "pkg")
|
||||
|
||||
|
||||
def test_js_import_resolution_is_lexical_and_never_escapes_the_root(tmp_path: Path) -> None:
|
||||
from app.architecture import build_architecture_from_imports
|
||||
|
||||
imports = {
|
||||
"web/app.ts": ("./util", "../../outside/secret", "../web/util.js"),
|
||||
"web/util.ts": (),
|
||||
}
|
||||
graph, _ = build_architecture_from_imports(
|
||||
tmp_path,
|
||||
imports,
|
||||
{path: "TypeScript" for path in imports},
|
||||
{path: 1 for path in imports},
|
||||
{path: 0 for path in imports},
|
||||
{},
|
||||
)
|
||||
|
||||
assert {(edge.source, edge.target) for edge in graph.edges} == {("web/app.ts", "web/util.ts")}
|
||||
assert graph.total_edges == 1
|
||||
@@ -16,7 +16,7 @@ def test_non_git_fallback_applies_root_gitignore_rules(tmp_path: Path) -> None:
|
||||
(tmp_path / "ignored-dir").mkdir()
|
||||
(tmp_path / "ignored-dir" / "nested.py").write_text("print('nested')\n", encoding="utf-8")
|
||||
|
||||
files = {path.relative_to(tmp_path).as_posix() for path in _walk_files(tmp_path)}
|
||||
files = {relative for relative, _path in _walk_files(tmp_path)}
|
||||
|
||||
assert "kept.py" in files
|
||||
assert "keep.log" in files
|
||||
|
||||
@@ -159,3 +159,38 @@ def test_catalog_summary_uses_denormalized_fields_without_report_json_parsing(tm
|
||||
assert storage["scan_count"] == 1
|
||||
assert storage["mirror_count"] >= 0
|
||||
assert storage["worktree_count"] >= 0
|
||||
|
||||
|
||||
def test_history_listings_keep_counts_for_compacted_reports_without_report_parsing(
|
||||
tmp_path: Path, monkeypatch
|
||||
) -> None:
|
||||
repository = tmp_path / "repo"
|
||||
repository.mkdir()
|
||||
(repository / "app.py").write_text("print('hello')\n", encoding="utf-8")
|
||||
monkeypatch.setenv("REPOPULSE_FULL_REPORTS_PER_REPOSITORY", "1")
|
||||
store = ScanStore(str(tmp_path / "repopulse.db"))
|
||||
first = store.save(scan_repository(str(repository)))
|
||||
second = store.save(scan_repository(str(repository)))
|
||||
assert store.get(first.id).compacted is True
|
||||
assert first.findings
|
||||
|
||||
def fail_if_called(*_args, **_kwargs):
|
||||
raise AssertionError("history listings must not deserialize full reports")
|
||||
|
||||
monkeypatch.setattr(store, "_report_from_row", fail_if_called)
|
||||
repository_id = second.source.repository_id
|
||||
active = [item for item in first.findings if not item.suppressed]
|
||||
|
||||
scans = store.repository_scans(repository_id)
|
||||
assert [item.id for item in scans] == [second.id, first.id]
|
||||
assert scans[0].findings == len(second.findings)
|
||||
assert scans[0].duration_ms == second.duration_ms
|
||||
assert scans[0].release_readiness == second.release_readiness
|
||||
# A compacted report keeps its retained summary count instead of reporting zero.
|
||||
assert scans[1].findings == len(active)
|
||||
|
||||
trend = store.trend(repository_id)
|
||||
assert [item.scan_id for item in trend] == [first.id, second.id]
|
||||
assert trend[0].findings == len(active)
|
||||
assert trend[0].warnings == sum(1 for item in active if item.severity == "warning")
|
||||
assert trend[1].findings == len(second.findings)
|
||||
@@ -7,6 +7,8 @@ services:
|
||||
dockerfile: Dockerfile.unraid
|
||||
container_name: repopulse
|
||||
restart: unless-stopped
|
||||
mem_limit: 1024m
|
||||
memswap_limit: 1024m
|
||||
environment:
|
||||
REPOPULSE_SCAN_ROOT: /scan
|
||||
REPOPULSE_DB: /data/repopulse.db
|
||||
|
||||
+10
-4
@@ -49,6 +49,8 @@ The AppOps-managed runtime candidate is defined separately in `.itworx/deploymen
|
||||
|
||||
The scanner prefers `git ls-files` for a top-level worktree so ignored/untracked build artifacts do not distort results. An empty Git worktree remains empty rather than silently falling back. Non-Git directories use a filesystem traversal that excludes common generated/vendor directories and applies bounded root/nested `.gitignore` rules. Text files larger than the configured internal read threshold are not parsed as source, and binary files are skipped.
|
||||
|
||||
The scanner processes inventory in one streaming pass and holds the full text of only one file at a time. It retains small derived values per file (LOC, debt counts, unresolved import references) plus the texts of the few CI workflow, `package.json` and `pyproject.toml` files needed for test signals, so peak scan memory no longer grows with the total text budget. Each Python file is parsed once for both function heuristics and imports. `text_bytes_scanned` counts the bytes actually read from disk.
|
||||
|
||||
Every report includes `ScanCoverage`: bounded file, per-file-text, total-text, finding and credential-finding limits plus the observed skipped counts. A limit produces a deterministic partial result and an analysis-coverage warning rather than an unqualified completeness claim or an artificial blocker.
|
||||
|
||||
## Signals
|
||||
@@ -69,7 +71,7 @@ Each finding is enriched by a backend rule definition with confidence, why-it-ma
|
||||
|
||||
## Architecture graph
|
||||
|
||||
Python imports are parsed using the standard-library AST. Local JavaScript and TypeScript relative imports, bounded `baseUrl`/`paths` aliases from `tsconfig.json`/`jsconfig.json`, and simple local workspace package imports are resolved against known source files. Only local edges are retained; this remains heuristic source analysis, not a TypeScript compiler. The displayed graph is bounded to 160 nodes and 600 edges to keep large repositories responsive; the response explicitly indicates truncation, reports total discovered nodes/edges and distinguishes visible coupling from total coupling used in hotspot ranking. Extensionless, index and `.js`-specifier-to-TypeScript resolution are supported.
|
||||
Python imports are parsed using the standard-library AST. Local JavaScript and TypeScript relative imports, bounded `baseUrl`/`paths` aliases from `tsconfig.json`/`jsconfig.json`, and simple local workspace package imports are resolved against known source files. Import targets are resolved lexically against the known, already safety-checked file set rather than through filesystem resolution; a specifier escaping the scan root never yields an edge. Only local edges are retained; this remains heuristic source analysis, not a TypeScript compiler. The displayed graph is bounded to 160 nodes and 600 edges to keep large repositories responsive; the response explicitly indicates truncation, reports total discovered nodes/edges and distinguishes visible coupling from total coupling used in hotspot ranking. Extensionless, index and `.js`-specifier-to-TypeScript resolution are supported.
|
||||
|
||||
## Hotspot model
|
||||
|
||||
@@ -81,7 +83,7 @@ Category scores start at 100 and are reduced only by findings with an explicit d
|
||||
|
||||
## History and comparison
|
||||
|
||||
Every report is serialized as a snapshot in SQLite using WAL. Lightweight scan summary columns (readiness, counts, primary signal, deltas and triage priority) drive catalog CTE queries without deserializing full reports. Comparison calculates metric/category deltas and added/resolved findings against the chronological previous scan of the same repository identity. Findings carry stable semantic fingerprints so line movement does not appear as a new risk. Trend endpoints return bounded chronological snapshots for UI visualization. By default, up to 200 snapshots are retained per repository, the latest 30 remain detailed, older retained payloads compact to summary evidence, and snapshots older than 365 days are removed. v1 path history is backfilled into a local catalog identity and is never deleted solely for migration.
|
||||
Every report is serialized as a snapshot in SQLite using WAL. Lightweight scan summary columns (readiness, counts, primary signal, deltas and triage priority) drive catalog CTE queries without deserializing full reports. Comparison calculates metric/category deltas and added/resolved findings against the chronological previous scan of the same repository identity. Findings carry stable semantic fingerprints so line movement does not appear as a new risk. Trend and scan-history endpoints return bounded chronological snapshots computed in SQLite from summary columns and JSON extraction, without deserializing full reports; a compacted snapshot reports its retained active-finding summary counts instead of a misleading zero. By default, up to 200 snapshots are retained per repository, the latest 30 remain detailed, older retained payloads compact to summary evidence, and snapshots older than 365 days are removed. v1 path history is backfilled into a local catalog identity and is never deleted solely for migration.
|
||||
|
||||
## Reporting
|
||||
|
||||
@@ -89,7 +91,11 @@ RepoPulse can export the complete JSON report or a standalone HTML report. HTML
|
||||
|
||||
## Job state and failure isolation
|
||||
|
||||
The persisted queue records repository, trigger, requested/actual revision, phase, progress, attempt, timestamps, resulting scan and sanitized error information. Claims use `BEGIN IMMEDIATE` plus a status guard, so two processes cannot own the same queued job. They move through `queued → acquiring_source → scanning → analyzing → persisting → completed` or an isolated `failed` outcome. A heartbeat renews each active lease; only a crashed worker’s expired lease returns work to `queued` for recovery. Older unstarted revisions become `superseded`; explicit cancellation states remain representable. One repository failure does not terminate the worker loop or affect other queued repositories.
|
||||
The persisted queue records repository, trigger, requested/actual revision, phase, progress, attempt, timestamps, resulting scan and sanitized error information. Claims use `BEGIN IMMEDIATE` plus a status guard, so two processes cannot own the same queued job. They move through `queued → acquiring_source → scanning → analyzing → persisting → completed` or an isolated `failed` outcome. The idle scheduler writes its liveness heartbeat at most every `REPOPULSE_WORKER_HEARTBEAT_SECONDS` (default 5) instead of on every poll; claims still re-check maintenance inside their own transaction. An idle worker backs off exponentially from the 0.5 s poll to `REPOPULSE_WORKER_IDLE_POLL_SECONDS` (default 5), and in-process enqueues (API, webhook) wake it immediately. A claim first runs a read-only probe for queued work or expired leases, so an idle queue never takes the SQLite write lock. After each job the worker runs a garbage collection and, on glibc, `malloc_trim` so the long-running process returns a large scan's memory to the OS. A heartbeat renews each active lease; only a crashed worker’s expired lease returns work to `queued` for recovery. Older unstarted revisions become `superseded`; explicit cancellation states remain representable. One repository failure does not terminate the worker loop or affect other queued repositories.
|
||||
|
||||
## Child processes
|
||||
|
||||
Every Git subprocess runs in its own process group with a hard timeout; on timeout the whole group is killed and reaped, so no descendant (for example `git-remote-https` or `index-pack`) is orphaned. Git automatic housekeeping is forced to the foreground (`gc.autoDetach=false`, `maintenance.autoDetach=false`) so it stays within that bound. The container images run `tini` as PID 1, which reaps any remaining orphan; previously uvicorn ran as PID 1 and accumulated zombie `git` processes.
|
||||
|
||||
## Operator maintenance mode
|
||||
|
||||
@@ -97,4 +103,4 @@ The `app_state` key `maintenance_mode` durably records whether operator maintena
|
||||
|
||||
## Operational configuration
|
||||
|
||||
`REPOPULSE_GITEA_URL` and `REPOPULSE_GITEA_TOKEN` enable the provider. `REPOPULSE_GITEA_SYNC_INTERVAL_SECONDS`, `REPOPULSE_GITEA_BRANCH_WORKERS`, `REPOPULSE_MAX_QUEUED_SCANS`, `REPOPULSE_JOB_LEASE_SECONDS`, scan-budget and retention variables, and `REPOPULSE_REPOSITORY_CACHE` bound reconciliation, queue depth, active leases, analysis and cache placement. The durable Gitea runner deliberately has one worker; `/api/v1/meta` exposes requested versus actual capacity. `REPOPULSE_GITEA_WEBHOOK_SECRET` enables signed default-branch push intake; non-push, non-default-branch, invalid and branch-deletion events are acknowledged without scheduling work. `REPOPULSE_OPERATOR_TOKEN` protects mutating endpoints when an authenticated proxy perimeter is not the selected boundary. `/api/v1/meta` exposes repository state counts, queue depth, storage/retention metrics, recovery status, worker heartbeat, connector URL (without token) and last sync result.
|
||||
`REPOPULSE_GITEA_URL` and `REPOPULSE_GITEA_TOKEN` enable the provider. `REPOPULSE_GITEA_SYNC_INTERVAL_SECONDS`, `REPOPULSE_GITEA_BRANCH_WORKERS`, `REPOPULSE_MAX_QUEUED_SCANS`, `REPOPULSE_JOB_LEASE_SECONDS`, scan-budget and retention variables, and `REPOPULSE_REPOSITORY_CACHE` bound reconciliation, queue depth, active leases, analysis and cache placement. The durable Gitea runner deliberately has one worker; `/api/v1/meta` exposes requested versus actual capacity. `REPOPULSE_GITEA_WEBHOOK_SECRET` enables signed default-branch push intake; non-push, non-default-branch, invalid and branch-deletion events are acknowledged without scheduling work. `REPOPULSE_OPERATOR_TOKEN` protects mutating endpoints when an authenticated proxy perimeter is not the selected boundary. `/api/v1/meta` exposes repository state counts over the complete catalog (no silent cap), queue depth, storage/retention metrics (the mirror-cache walk runs in the background at most every 5 minutes, so the endpoint never waits on it after the first call), recovery status, worker heartbeat, connector URL (without token) and last sync result.
|
||||
Generated
+2
-2
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "repopulse-frontend",
|
||||
"version": "1.5.0",
|
||||
"version": "1.5.1",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "repopulse-frontend",
|
||||
"version": "1.5.0",
|
||||
"version": "1.5.1",
|
||||
"dependencies": {
|
||||
"@xyflow/react": "^12.8.5",
|
||||
"react": "^19.1.1",
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "repopulse-frontend",
|
||||
"private": true,
|
||||
"version": "1.5.0",
|
||||
"version": "1.5.1",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "vite",
|
||||
|
||||
Generated
+2
-2
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "repopulse-workspace",
|
||||
"version": "1.5.0",
|
||||
"version": "1.5.1",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "repopulse-workspace",
|
||||
"version": "1.5.0"
|
||||
"version": "1.5.1"
|
||||
}
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "repopulse-workspace",
|
||||
"private": true,
|
||||
"version": "1.5.0",
|
||||
"version": "1.5.1",
|
||||
"scripts": {
|
||||
"test": "npm --prefix frontend run test",
|
||||
"typecheck": "npm --prefix frontend run typecheck",
|
||||
|
||||
Reference in new issue
Block a user