RepoPulse: update backend/app and dependencies (+3 areas)
Public changes compared with the previous published export: 26 files changed. Affected areas: - backend/app: 11 files - dependencies: 5 files - deployment and CI: 3 files - documentation: 2 files - tests: 5 files Changed files: - Updated: "Dockerfile.unraid" - Updated: "README.md" - Updated: "backend/Dockerfile" - Updated: "backend/app/acquisition.py" - Updated: "backend/app/architecture.py" - Updated: "backend/app/git_analysis.py" - Updated: "backend/app/gitea.py" - Updated: "backend/app/jobs.py" - Updated: "backend/app/main.py" - Added: "backend/app/processes.py" - Updated: "backend/app/rules.py" - Updated: "backend/app/scanner.py" - Updated: "backend/app/signals.py" - Updated: "backend/app/store.py" - Updated: "backend/pyproject.toml" - Added: "backend/tests/test_api_responsiveness.py" - Added: "backend/tests/test_processes.py" - Updated: "backend/tests/test_scanner.py" - Updated: "backend/tests/test_signals_v11.py" - Updated: "backend/tests/test_store.py" - Updated: "docker-compose.yml" - Updated: "docs/ARCHITECTURE.md" - Updated: "frontend/package-lock.json" - Updated: "frontend/package.json" - Updated: "package-lock.json" - Updated: "package.json" Export provenance/manifests refreshed; excluded from the change summary. Publish curated RepoPulse source from a251f826238908b156485299061da76482780ed2
This commit is contained in:
1 parent
23a4a721f1
commit
61ae49f9d0
28 files changed
+1083
-408
No files matched your search
+140
-69
@@ -5,13 +5,18 @@ import fnmatch
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
from collections import Counter
|
||||
from collections import Counter, deque
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from time import perf_counter
|
||||
from uuid import uuid4
|
||||
|
||||
from .architecture import build_architecture
|
||||
from .architecture import (
|
||||
build_architecture_from_imports,
|
||||
extract_js_specifiers,
|
||||
python_import_modules,
|
||||
python_package_parts,
|
||||
)
|
||||
from .git_analysis import analyze_git
|
||||
from .models import (
|
||||
Finding,
|
||||
@@ -25,9 +30,10 @@ from .models import (
|
||||
finding_fingerprint,
|
||||
)
|
||||
from .policy import load_policy
|
||||
from .processes import git_environment, run_bounded
|
||||
from .release_gate import build_release_gate
|
||||
from .rules import enrich_findings
|
||||
from .signals import analyze_dependencies, detect_test_signals
|
||||
from .signals import analyze_dependencies, detect_test_signals, is_signal_text_path
|
||||
|
||||
LANGUAGES = {
|
||||
".py": "Python",
|
||||
@@ -224,11 +230,8 @@ class _GitIgnoreMatcher:
|
||||
|
||||
def _git_worktree_root(root: Path) -> Path | None:
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(root), "rev-parse", "--show-toplevel"],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
timeout=5,
|
||||
result = run_bounded(
|
||||
["git", "-C", str(root), "rev-parse", "--show-toplevel"], timeout=5, env=git_environment()
|
||||
)
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
return None
|
||||
@@ -238,42 +241,61 @@ def _git_worktree_root(root: Path) -> Path | None:
|
||||
return None
|
||||
|
||||
|
||||
class _SafePathChecker:
|
||||
"""Validate candidate files below one root, caching directories already proven safe.
|
||||
|
||||
Every path component is still checked for symlinks, but a directory shared by many
|
||||
files is inspected once instead of once per file.
|
||||
"""
|
||||
|
||||
def __init__(self, root: Path) -> None:
|
||||
self.root = root
|
||||
self.root_resolved = root.resolve()
|
||||
self._safe_dirs: set[Path] = set()
|
||||
|
||||
def check(self, candidate: Path) -> Path | None:
|
||||
"""Return a resolved regular file only when every path component is safe."""
|
||||
try:
|
||||
relative = candidate.relative_to(self.root)
|
||||
if _ignored(str(relative)):
|
||||
return None
|
||||
current = self.root
|
||||
parts = relative.parts
|
||||
for index, part in enumerate(parts):
|
||||
current /= part
|
||||
is_last = index == len(parts) - 1
|
||||
if not is_last and current in self._safe_dirs:
|
||||
continue
|
||||
if current.is_symlink():
|
||||
return None
|
||||
if not is_last:
|
||||
self._safe_dirs.add(current)
|
||||
resolved = candidate.resolve(strict=True)
|
||||
root_resolved = self.root_resolved
|
||||
if resolved == root_resolved or root_resolved not in resolved.parents:
|
||||
return None
|
||||
if not resolved.is_file():
|
||||
return None
|
||||
except (OSError, RuntimeError, ValueError):
|
||||
return None
|
||||
return resolved
|
||||
|
||||
|
||||
def _safe_file(root: Path, candidate: Path) -> Path | None:
|
||||
"""Return a resolved regular file only when every path component is safe."""
|
||||
try:
|
||||
relative = candidate.relative_to(root)
|
||||
if _ignored(str(relative)):
|
||||
return None
|
||||
current = root
|
||||
for part in Path(relative).parts:
|
||||
current /= part
|
||||
if current.is_symlink():
|
||||
return None
|
||||
resolved = candidate.resolve(strict=True)
|
||||
root_resolved = root.resolve()
|
||||
if resolved == root_resolved or root_resolved not in resolved.parents:
|
||||
return None
|
||||
if not resolved.is_file():
|
||||
return None
|
||||
except (OSError, RuntimeError, ValueError):
|
||||
return None
|
||||
return resolved
|
||||
return _SafePathChecker(root).check(candidate)
|
||||
|
||||
|
||||
def _tracked_files(root: Path) -> list[Path] | None:
|
||||
def _tracked_files(root: Path) -> list[tuple[str, Path]] | None:
|
||||
if _git_worktree_root(root) != root:
|
||||
return None
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(root), "ls-files", "-z"],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
timeout=5,
|
||||
)
|
||||
result = run_bounded(["git", "-C", str(root), "ls-files", "-z"], timeout=5, env=git_environment())
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
return None
|
||||
|
||||
paths: list[Path] = []
|
||||
checker = _SafePathChecker(root)
|
||||
paths: list[tuple[str, Path]] = []
|
||||
for raw in result.stdout.split(b"\0"):
|
||||
if not raw:
|
||||
continue
|
||||
@@ -281,14 +303,15 @@ def _tracked_files(root: Path) -> list[Path] | None:
|
||||
relative_path = Path(relative)
|
||||
if relative_path.is_absolute() or _ignored(relative):
|
||||
continue
|
||||
safe = _safe_file(root, root / relative_path)
|
||||
safe = checker.check(root / relative_path)
|
||||
if safe is not None:
|
||||
paths.append(safe)
|
||||
paths.append((relative_path.as_posix(), safe))
|
||||
return paths
|
||||
|
||||
|
||||
def _walk_files(root: Path) -> list[Path]:
|
||||
files: list[Path] = []
|
||||
def _walk_files(root: Path) -> list[tuple[str, Path]]:
|
||||
files: list[tuple[str, Path]] = []
|
||||
checker = _SafePathChecker(root)
|
||||
gitignore = _GitIgnoreMatcher(root)
|
||||
for current, dirs, names in os.walk(root, followlinks=False):
|
||||
current_path = Path(current)
|
||||
@@ -312,13 +335,14 @@ def _walk_files(root: Path) -> list[Path]:
|
||||
continue
|
||||
if gitignore.is_ignored(relative, is_dir=False):
|
||||
continue
|
||||
safe = _safe_file(root, candidate)
|
||||
safe = checker.check(candidate)
|
||||
if safe is not None:
|
||||
files.append(safe)
|
||||
files.append((relative.replace("\\", "/"), safe))
|
||||
return files
|
||||
|
||||
|
||||
def _read_text(path: Path, max_bytes: int = MAX_TEXT_BYTES) -> str | None:
|
||||
def _read_text_bytes(path: Path, max_bytes: int = MAX_TEXT_BYTES) -> tuple[str, int] | None:
|
||||
"""Return decoded text plus the number of bytes read, or None for binary/oversized files."""
|
||||
try:
|
||||
if path.stat().st_size > max_bytes:
|
||||
return None
|
||||
@@ -327,7 +351,12 @@ def _read_text(path: Path, max_bytes: int = MAX_TEXT_BYTES) -> str | None:
|
||||
return None
|
||||
if b"\0" in raw[:8192]:
|
||||
return None
|
||||
return raw.decode("utf-8", errors="replace")
|
||||
return raw.decode("utf-8", errors="replace"), len(raw)
|
||||
|
||||
|
||||
def _read_text(path: Path, max_bytes: int = MAX_TEXT_BYTES) -> str | None:
|
||||
result = _read_text_bytes(path, max_bytes)
|
||||
return result[0] if result else None
|
||||
|
||||
|
||||
def _is_test_file(relative: str) -> bool:
|
||||
@@ -343,19 +372,45 @@ def _is_test_file(relative: str) -> bool:
|
||||
)
|
||||
|
||||
|
||||
def _python_function_findings(relative: str, text: str) -> list[Finding]:
|
||||
PYTHON_BRANCH_TYPES = (ast.If, ast.For, ast.AsyncFor, ast.While, ast.Try, ast.Match, ast.BoolOp)
|
||||
|
||||
|
||||
def _analyze_python(relative: str, text: str) -> tuple[list[Finding], tuple[str, ...]]:
|
||||
"""Parse one Python file once and return function findings plus unresolved imports.
|
||||
|
||||
A single breadth-first walk (the same order as ``ast.walk``) attributes each branch
|
||||
construct to every enclosing function, matching a per-function ``ast.walk`` count
|
||||
without re-walking nested subtrees.
|
||||
"""
|
||||
try:
|
||||
tree = ast.parse(text)
|
||||
except SyntaxError:
|
||||
return []
|
||||
except (SyntaxError, ValueError):
|
||||
return [], ()
|
||||
package_parts = python_package_parts(relative)
|
||||
imports: dict[str, None] = {}
|
||||
functions: list[ast.FunctionDef | ast.AsyncFunctionDef] = []
|
||||
branch_counts: list[int] = []
|
||||
pending: deque[tuple[ast.AST, tuple[int, ...]]] = deque([(tree, ())])
|
||||
while pending:
|
||||
node, enclosing = pending.popleft()
|
||||
if isinstance(node, PYTHON_BRANCH_TYPES):
|
||||
for index in enclosing:
|
||||
branch_counts[index] += 1
|
||||
elif isinstance(node, (ast.Import, ast.ImportFrom)):
|
||||
for module in python_import_modules(node, package_parts):
|
||||
imports.setdefault(module)
|
||||
child_enclosing = enclosing
|
||||
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
||||
functions.append(node)
|
||||
branch_counts.append(0)
|
||||
child_enclosing = (*enclosing, len(functions) - 1)
|
||||
pending.extend((child, child_enclosing) for child in ast.iter_child_nodes(node))
|
||||
del tree, pending
|
||||
|
||||
findings: list[Finding] = []
|
||||
branch_types = (ast.If, ast.For, ast.AsyncFor, ast.While, ast.Try, ast.Match, ast.BoolOp)
|
||||
for node in ast.walk(tree):
|
||||
if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
||||
continue
|
||||
for node, branches in zip(functions, branch_counts, strict=True):
|
||||
end_line = getattr(node, "end_lineno", node.lineno)
|
||||
length = max(1, end_line - node.lineno + 1)
|
||||
branches = sum(1 for child in ast.walk(node) if isinstance(child, branch_types))
|
||||
if length >= 140:
|
||||
findings.append(
|
||||
Finding(
|
||||
@@ -380,7 +435,11 @@ def _python_function_findings(relative: str, text: str) -> list[Finding]:
|
||||
deduction=4,
|
||||
)
|
||||
)
|
||||
return findings[:12]
|
||||
return findings[:12], tuple(imports)
|
||||
|
||||
|
||||
def _python_function_findings(relative: str, text: str) -> list[Finding]:
|
||||
return _analyze_python(relative, text)[0]
|
||||
|
||||
|
||||
def _is_placeholder_secret(match: re.Match[str]) -> bool:
|
||||
@@ -408,7 +467,7 @@ def scan_repository(
|
||||
limits = limits or scan_limits_from_environment()
|
||||
tracked_files = _tracked_files(root)
|
||||
all_files = _walk_files(root) if tracked_files is None else tracked_files
|
||||
all_files.sort(key=lambda item: str(item.relative_to(root)).replace("\\", "/").casefold())
|
||||
all_files.sort(key=lambda item: item[0].casefold())
|
||||
files = all_files[: limits.max_files]
|
||||
coverage = ScanCoverage(
|
||||
limits=limits,
|
||||
@@ -426,8 +485,11 @@ def scan_repository(
|
||||
metrics = RepositoryMetrics(files=len(all_files))
|
||||
files_by_name: dict[str, list[Path]] = {}
|
||||
root_entries = {entry.name.lower() for entry in root.iterdir()}
|
||||
text_by_path: dict[str, str] = {}
|
||||
source_texts: dict[str, str] = {}
|
||||
# Only small derived values are retained per file. Full text is held for a single
|
||||
# file at a time, except the few CI/manifest texts that test-signal detection needs.
|
||||
signal_texts: dict[str, str] = {}
|
||||
readable_paths: list[str] = []
|
||||
imports_by_path: dict[str, tuple[str, ...]] = {}
|
||||
language_by_path: dict[str, str] = {}
|
||||
loc_by_path: dict[str, int] = {}
|
||||
debt_by_path: dict[str, int] = {}
|
||||
@@ -435,11 +497,10 @@ def scan_repository(
|
||||
secret_findings_truncated = False
|
||||
text_budget_exhausted = False
|
||||
|
||||
for path_obj in files:
|
||||
for relative, path_obj in files:
|
||||
try:
|
||||
relative = str(path_obj.relative_to(root)).replace("\\", "/")
|
||||
size = path_obj.stat().st_size
|
||||
except (OSError, ValueError):
|
||||
except OSError:
|
||||
continue
|
||||
|
||||
files_by_name.setdefault(path_obj.name, []).append(path_obj)
|
||||
@@ -479,21 +540,28 @@ def scan_repository(
|
||||
coverage.skipped_files_by_text_budget += 1
|
||||
text_budget_exhausted = True
|
||||
continue
|
||||
text = _read_text(path_obj, limits.max_text_bytes_per_file)
|
||||
if text is None:
|
||||
read = _read_text_bytes(path_obj, limits.max_text_bytes_per_file)
|
||||
if read is None:
|
||||
continue
|
||||
text, byte_count = read
|
||||
coverage.scanned_files += 1
|
||||
coverage.text_bytes_scanned += len(text.encode("utf-8", errors="replace"))
|
||||
text_by_path[relative] = text
|
||||
coverage.text_bytes_scanned += byte_count
|
||||
readable_paths.append(relative)
|
||||
if is_signal_text_path(relative):
|
||||
signal_texts[relative] = text
|
||||
lines = text.splitlines()
|
||||
|
||||
if language:
|
||||
loc = sum(1 for line in text.splitlines() if line.strip())
|
||||
loc = sum(1 for line in lines if line.strip())
|
||||
metrics.source_files += 1
|
||||
metrics.lines_of_code += loc
|
||||
languages[language] += loc
|
||||
source_texts[relative] = text
|
||||
language_by_path[relative] = language
|
||||
loc_by_path[relative] = loc
|
||||
if language in {"TypeScript", "JavaScript"}:
|
||||
imports_by_path[relative] = extract_js_specifiers(text)
|
||||
elif language != "Python":
|
||||
imports_by_path[relative] = ()
|
||||
if loc > 1000:
|
||||
metrics.large_source_files += 1
|
||||
findings.append(
|
||||
@@ -507,7 +575,8 @@ def scan_repository(
|
||||
)
|
||||
)
|
||||
if language == "Python":
|
||||
findings.extend(_python_function_findings(relative, text))
|
||||
python_findings, imports_by_path[relative] = _analyze_python(relative, text)
|
||||
findings.extend(python_findings)
|
||||
|
||||
if _is_test_file(relative):
|
||||
metrics.test_files += 1
|
||||
@@ -520,7 +589,7 @@ def scan_repository(
|
||||
debt_by_path[relative] = sum(debt.values())
|
||||
|
||||
if _is_secret_scan_candidate(path_obj, relative, language) and secret_findings < limits.max_secret_findings:
|
||||
for line_number, line in enumerate(text.splitlines(), start=1):
|
||||
for line_number, line in enumerate(lines, start=1):
|
||||
match = SECRET_PATTERN.search(line)
|
||||
if match is None or _is_placeholder_secret(match):
|
||||
continue
|
||||
@@ -567,11 +636,13 @@ def scan_repository(
|
||||
findings.append(Finding(code="hygiene.missing-env-example", category="Repository hygiene", severity=Severity.INFO, message="Environment configuration exists without a root .env.example", deduction=3))
|
||||
|
||||
test_signals = detect_test_signals(
|
||||
[str(path.relative_to(root)).replace("\\", "/") for path in files],
|
||||
[relative for relative, _path in files],
|
||||
metrics.source_files,
|
||||
metrics.test_files,
|
||||
text_by_path,
|
||||
signal_texts,
|
||||
readable_paths,
|
||||
)
|
||||
del signal_texts, readable_paths
|
||||
if metrics.source_files and metrics.test_files == 0:
|
||||
findings.append(Finding(code="testing.none-detected", category="Testing", severity=Severity.WARNING, message="No test files detected", deduction=30))
|
||||
if not test_signals.ci_workflows:
|
||||
@@ -593,9 +664,9 @@ def scan_repository(
|
||||
findings.extend(dependency_findings)
|
||||
|
||||
git_signals, churn_by_path = analyze_git(root)
|
||||
architecture, hotspots = build_architecture(
|
||||
architecture, hotspots = build_architecture_from_imports(
|
||||
root,
|
||||
source_texts,
|
||||
imports_by_path,
|
||||
language_by_path,
|
||||
loc_by_path,
|
||||
debt_by_path,
|
||||
|
||||
Reference in new issue
Block a user