RepoPulse: update backend/app and dependencies (+3 areas)

Public changes compared with the previous published export:
26 files changed.

Affected areas:
- backend/app: 11 files
- dependencies: 5 files
- deployment and CI: 3 files
- documentation: 2 files
- tests: 5 files

Changed files:
- Updated: "Dockerfile.unraid"
- Updated: "README.md"
- Updated: "backend/Dockerfile"
- Updated: "backend/app/acquisition.py"
- Updated: "backend/app/architecture.py"
- Updated: "backend/app/git_analysis.py"
- Updated: "backend/app/gitea.py"
- Updated: "backend/app/jobs.py"
- Updated: "backend/app/main.py"
- Added: "backend/app/processes.py"
- Updated: "backend/app/rules.py"
- Updated: "backend/app/scanner.py"
- Updated: "backend/app/signals.py"
- Updated: "backend/app/store.py"
- Updated: "backend/pyproject.toml"
- Added: "backend/tests/test_api_responsiveness.py"
- Added: "backend/tests/test_processes.py"
- Updated: "backend/tests/test_scanner.py"
- Updated: "backend/tests/test_signals_v11.py"
- Updated: "backend/tests/test_store.py"
- Updated: "docker-compose.yml"
- Updated: "docs/ARCHITECTURE.md"
- Updated: "frontend/package-lock.json"
- Updated: "frontend/package.json"
- Updated: "package-lock.json"
- Updated: "package.json"

Export provenance/manifests refreshed; excluded from the change summary.

Publish curated RepoPulse source from a251f826238908b156485299061da76482780ed2
This commit is contained in:
Public Source Publisher committed 2026-11-17 21:10:30 +00:00
1 parent 23a4a721f1
commit 61ae49f9d0
28 files changed
+1083 -408

No files matched your search

+140 -69
View File
@@ -5,13 +5,18 @@ import fnmatch
import os
import re
import subprocess
from collections import Counter
from collections import Counter, deque
from datetime import UTC, datetime
from pathlib import Path
from time import perf_counter
from uuid import uuid4
from .architecture import build_architecture
from .architecture import (
build_architecture_from_imports,
extract_js_specifiers,
python_import_modules,
python_package_parts,
)
from .git_analysis import analyze_git
from .models import (
Finding,
@@ -25,9 +30,10 @@ from .models import (
finding_fingerprint,
)
from .policy import load_policy
from .processes import git_environment, run_bounded
from .release_gate import build_release_gate
from .rules import enrich_findings
from .signals import analyze_dependencies, detect_test_signals
from .signals import analyze_dependencies, detect_test_signals, is_signal_text_path
LANGUAGES = {
".py": "Python",
@@ -224,11 +230,8 @@ class _GitIgnoreMatcher:
def _git_worktree_root(root: Path) -> Path | None:
try:
result = subprocess.run(
["git", "-C", str(root), "rev-parse", "--show-toplevel"],
check=True,
capture_output=True,
timeout=5,
result = run_bounded(
["git", "-C", str(root), "rev-parse", "--show-toplevel"], timeout=5, env=git_environment()
)
except (OSError, subprocess.SubprocessError):
return None
@@ -238,42 +241,61 @@ def _git_worktree_root(root: Path) -> Path | None:
return None
class _SafePathChecker:
"""Validate candidate files below one root, caching directories already proven safe.
Every path component is still checked for symlinks, but a directory shared by many
files is inspected once instead of once per file.
"""
def __init__(self, root: Path) -> None:
self.root = root
self.root_resolved = root.resolve()
self._safe_dirs: set[Path] = set()
def check(self, candidate: Path) -> Path | None:
"""Return a resolved regular file only when every path component is safe."""
try:
relative = candidate.relative_to(self.root)
if _ignored(str(relative)):
return None
current = self.root
parts = relative.parts
for index, part in enumerate(parts):
current /= part
is_last = index == len(parts) - 1
if not is_last and current in self._safe_dirs:
continue
if current.is_symlink():
return None
if not is_last:
self._safe_dirs.add(current)
resolved = candidate.resolve(strict=True)
root_resolved = self.root_resolved
if resolved == root_resolved or root_resolved not in resolved.parents:
return None
if not resolved.is_file():
return None
except (OSError, RuntimeError, ValueError):
return None
return resolved
def _safe_file(root: Path, candidate: Path) -> Path | None:
"""Return a resolved regular file only when every path component is safe."""
try:
relative = candidate.relative_to(root)
if _ignored(str(relative)):
return None
current = root
for part in Path(relative).parts:
current /= part
if current.is_symlink():
return None
resolved = candidate.resolve(strict=True)
root_resolved = root.resolve()
if resolved == root_resolved or root_resolved not in resolved.parents:
return None
if not resolved.is_file():
return None
except (OSError, RuntimeError, ValueError):
return None
return resolved
return _SafePathChecker(root).check(candidate)
def _tracked_files(root: Path) -> list[Path] | None:
def _tracked_files(root: Path) -> list[tuple[str, Path]] | None:
if _git_worktree_root(root) != root:
return None
try:
result = subprocess.run(
["git", "-C", str(root), "ls-files", "-z"],
check=True,
capture_output=True,
timeout=5,
)
result = run_bounded(["git", "-C", str(root), "ls-files", "-z"], timeout=5, env=git_environment())
except (OSError, subprocess.SubprocessError):
return None
paths: list[Path] = []
checker = _SafePathChecker(root)
paths: list[tuple[str, Path]] = []
for raw in result.stdout.split(b"\0"):
if not raw:
continue
@@ -281,14 +303,15 @@ def _tracked_files(root: Path) -> list[Path] | None:
relative_path = Path(relative)
if relative_path.is_absolute() or _ignored(relative):
continue
safe = _safe_file(root, root / relative_path)
safe = checker.check(root / relative_path)
if safe is not None:
paths.append(safe)
paths.append((relative_path.as_posix(), safe))
return paths
def _walk_files(root: Path) -> list[Path]:
files: list[Path] = []
def _walk_files(root: Path) -> list[tuple[str, Path]]:
files: list[tuple[str, Path]] = []
checker = _SafePathChecker(root)
gitignore = _GitIgnoreMatcher(root)
for current, dirs, names in os.walk(root, followlinks=False):
current_path = Path(current)
@@ -312,13 +335,14 @@ def _walk_files(root: Path) -> list[Path]:
continue
if gitignore.is_ignored(relative, is_dir=False):
continue
safe = _safe_file(root, candidate)
safe = checker.check(candidate)
if safe is not None:
files.append(safe)
files.append((relative.replace("\\", "/"), safe))
return files
def _read_text(path: Path, max_bytes: int = MAX_TEXT_BYTES) -> str | None:
def _read_text_bytes(path: Path, max_bytes: int = MAX_TEXT_BYTES) -> tuple[str, int] | None:
"""Return decoded text plus the number of bytes read, or None for binary/oversized files."""
try:
if path.stat().st_size > max_bytes:
return None
@@ -327,7 +351,12 @@ def _read_text(path: Path, max_bytes: int = MAX_TEXT_BYTES) -> str | None:
return None
if b"\0" in raw[:8192]:
return None
return raw.decode("utf-8", errors="replace")
return raw.decode("utf-8", errors="replace"), len(raw)
def _read_text(path: Path, max_bytes: int = MAX_TEXT_BYTES) -> str | None:
result = _read_text_bytes(path, max_bytes)
return result[0] if result else None
def _is_test_file(relative: str) -> bool:
@@ -343,19 +372,45 @@ def _is_test_file(relative: str) -> bool:
)
def _python_function_findings(relative: str, text: str) -> list[Finding]:
PYTHON_BRANCH_TYPES = (ast.If, ast.For, ast.AsyncFor, ast.While, ast.Try, ast.Match, ast.BoolOp)
def _analyze_python(relative: str, text: str) -> tuple[list[Finding], tuple[str, ...]]:
"""Parse one Python file once and return function findings plus unresolved imports.
A single breadth-first walk (the same order as ``ast.walk``) attributes each branch
construct to every enclosing function, matching a per-function ``ast.walk`` count
without re-walking nested subtrees.
"""
try:
tree = ast.parse(text)
except SyntaxError:
return []
except (SyntaxError, ValueError):
return [], ()
package_parts = python_package_parts(relative)
imports: dict[str, None] = {}
functions: list[ast.FunctionDef | ast.AsyncFunctionDef] = []
branch_counts: list[int] = []
pending: deque[tuple[ast.AST, tuple[int, ...]]] = deque([(tree, ())])
while pending:
node, enclosing = pending.popleft()
if isinstance(node, PYTHON_BRANCH_TYPES):
for index in enclosing:
branch_counts[index] += 1
elif isinstance(node, (ast.Import, ast.ImportFrom)):
for module in python_import_modules(node, package_parts):
imports.setdefault(module)
child_enclosing = enclosing
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
functions.append(node)
branch_counts.append(0)
child_enclosing = (*enclosing, len(functions) - 1)
pending.extend((child, child_enclosing) for child in ast.iter_child_nodes(node))
del tree, pending
findings: list[Finding] = []
branch_types = (ast.If, ast.For, ast.AsyncFor, ast.While, ast.Try, ast.Match, ast.BoolOp)
for node in ast.walk(tree):
if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
continue
for node, branches in zip(functions, branch_counts, strict=True):
end_line = getattr(node, "end_lineno", node.lineno)
length = max(1, end_line - node.lineno + 1)
branches = sum(1 for child in ast.walk(node) if isinstance(child, branch_types))
if length >= 140:
findings.append(
Finding(
@@ -380,7 +435,11 @@ def _python_function_findings(relative: str, text: str) -> list[Finding]:
deduction=4,
)
)
return findings[:12]
return findings[:12], tuple(imports)
def _python_function_findings(relative: str, text: str) -> list[Finding]:
return _analyze_python(relative, text)[0]
def _is_placeholder_secret(match: re.Match[str]) -> bool:
@@ -408,7 +467,7 @@ def scan_repository(
limits = limits or scan_limits_from_environment()
tracked_files = _tracked_files(root)
all_files = _walk_files(root) if tracked_files is None else tracked_files
all_files.sort(key=lambda item: str(item.relative_to(root)).replace("\\", "/").casefold())
all_files.sort(key=lambda item: item[0].casefold())
files = all_files[: limits.max_files]
coverage = ScanCoverage(
limits=limits,
@@ -426,8 +485,11 @@ def scan_repository(
metrics = RepositoryMetrics(files=len(all_files))
files_by_name: dict[str, list[Path]] = {}
root_entries = {entry.name.lower() for entry in root.iterdir()}
text_by_path: dict[str, str] = {}
source_texts: dict[str, str] = {}
# Only small derived values are retained per file. Full text is held for a single
# file at a time, except the few CI/manifest texts that test-signal detection needs.
signal_texts: dict[str, str] = {}
readable_paths: list[str] = []
imports_by_path: dict[str, tuple[str, ...]] = {}
language_by_path: dict[str, str] = {}
loc_by_path: dict[str, int] = {}
debt_by_path: dict[str, int] = {}
@@ -435,11 +497,10 @@ def scan_repository(
secret_findings_truncated = False
text_budget_exhausted = False
for path_obj in files:
for relative, path_obj in files:
try:
relative = str(path_obj.relative_to(root)).replace("\\", "/")
size = path_obj.stat().st_size
except (OSError, ValueError):
except OSError:
continue
files_by_name.setdefault(path_obj.name, []).append(path_obj)
@@ -479,21 +540,28 @@ def scan_repository(
coverage.skipped_files_by_text_budget += 1
text_budget_exhausted = True
continue
text = _read_text(path_obj, limits.max_text_bytes_per_file)
if text is None:
read = _read_text_bytes(path_obj, limits.max_text_bytes_per_file)
if read is None:
continue
text, byte_count = read
coverage.scanned_files += 1
coverage.text_bytes_scanned += len(text.encode("utf-8", errors="replace"))
text_by_path[relative] = text
coverage.text_bytes_scanned += byte_count
readable_paths.append(relative)
if is_signal_text_path(relative):
signal_texts[relative] = text
lines = text.splitlines()
if language:
loc = sum(1 for line in text.splitlines() if line.strip())
loc = sum(1 for line in lines if line.strip())
metrics.source_files += 1
metrics.lines_of_code += loc
languages[language] += loc
source_texts[relative] = text
language_by_path[relative] = language
loc_by_path[relative] = loc
if language in {"TypeScript", "JavaScript"}:
imports_by_path[relative] = extract_js_specifiers(text)
elif language != "Python":
imports_by_path[relative] = ()
if loc > 1000:
metrics.large_source_files += 1
findings.append(
@@ -507,7 +575,8 @@ def scan_repository(
)
)
if language == "Python":
findings.extend(_python_function_findings(relative, text))
python_findings, imports_by_path[relative] = _analyze_python(relative, text)
findings.extend(python_findings)
if _is_test_file(relative):
metrics.test_files += 1
@@ -520,7 +589,7 @@ def scan_repository(
debt_by_path[relative] = sum(debt.values())
if _is_secret_scan_candidate(path_obj, relative, language) and secret_findings < limits.max_secret_findings:
for line_number, line in enumerate(text.splitlines(), start=1):
for line_number, line in enumerate(lines, start=1):
match = SECRET_PATTERN.search(line)
if match is None or _is_placeholder_secret(match):
continue
@@ -567,11 +636,13 @@ def scan_repository(
findings.append(Finding(code="hygiene.missing-env-example", category="Repository hygiene", severity=Severity.INFO, message="Environment configuration exists without a root .env.example", deduction=3))
test_signals = detect_test_signals(
[str(path.relative_to(root)).replace("\\", "/") for path in files],
[relative for relative, _path in files],
metrics.source_files,
metrics.test_files,
text_by_path,
signal_texts,
readable_paths,
)
del signal_texts, readable_paths
if metrics.source_files and metrics.test_files == 0:
findings.append(Finding(code="testing.none-detected", category="Testing", severity=Severity.WARNING, message="No test files detected", deduction=30))
if not test_signals.ci_workflows:
@@ -593,9 +664,9 @@ def scan_repository(
findings.extend(dependency_findings)
git_signals, churn_by_path = analyze_git(root)
architecture, hotspots = build_architecture(
architecture, hotspots = build_architecture_from_imports(
root,
source_texts,
imports_by_path,
language_by_path,
loc_by_path,
debt_by_path,