RepoPulse: update backend/app and dependencies (+3 areas)

Public changes compared with the previous published export:
26 files changed.

Affected areas:
- backend/app: 11 files
- dependencies: 5 files
- deployment and CI: 3 files
- documentation: 2 files
- tests: 5 files

Changed files:
- Updated: "Dockerfile.unraid"
- Updated: "README.md"
- Updated: "backend/Dockerfile"
- Updated: "backend/app/acquisition.py"
- Updated: "backend/app/architecture.py"
- Updated: "backend/app/git_analysis.py"
- Updated: "backend/app/gitea.py"
- Updated: "backend/app/jobs.py"
- Updated: "backend/app/main.py"
- Added: "backend/app/processes.py"
- Updated: "backend/app/rules.py"
- Updated: "backend/app/scanner.py"
- Updated: "backend/app/signals.py"
- Updated: "backend/app/store.py"
- Updated: "backend/pyproject.toml"
- Added: "backend/tests/test_api_responsiveness.py"
- Added: "backend/tests/test_processes.py"
- Updated: "backend/tests/test_scanner.py"
- Updated: "backend/tests/test_signals_v11.py"
- Updated: "backend/tests/test_store.py"
- Updated: "docker-compose.yml"
- Updated: "docs/ARCHITECTURE.md"
- Updated: "frontend/package-lock.json"
- Updated: "frontend/package.json"
- Updated: "package-lock.json"
- Updated: "package.json"

Export provenance/manifests refreshed; excluded from the change summary.

Publish curated RepoPulse source from a251f826238908b156485299061da76482780ed2
This commit is contained in:
Public Source Publisher committed 2026-11-17 21:10:30 +00:00
1 parent 23a4a721f1
commit 61ae49f9d0
28 files changed
+1083 -408

No files matched your search

+183 -154
View File
@@ -9,6 +9,8 @@ from contextlib import contextmanager
from datetime import UTC, datetime, timedelta
from hashlib import sha256
from pathlib import Path
from threading import Lock, Thread
from time import monotonic
from uuid import uuid4
from .models import (
@@ -35,6 +37,7 @@ from .rules import enrich_findings
MAX_HISTORY_PER_REPOSITORY = 200
SCHEMA_VERSION = 4
MIRROR_STATS_TTL_SECONDS = 300.0
JOB_LEASE_SECONDS = 900
ACTIVE_JOB_STATUSES = (
ScanJobStatus.QUEUED.value,
@@ -108,6 +111,8 @@ class ScanStore:
self.path = Path(raw_path)
self.path.parent.mkdir(parents=True, exist_ok=True)
self.recovery_metadata: dict[str, str] | None = None
self._mirror_stats_cache: tuple[float, str, tuple[int, int]] | None = None
self._mirror_stats_refresh = Lock()
try:
self._validate_existing_database()
self._init_schema()
@@ -349,7 +354,7 @@ class ScanStore:
def _backfill_legacy_catalog(self, db: sqlite3.Connection) -> None:
rows = db.execute(
"""
SELECT id, repository_path, repository_name, report_json, repository_id, history_key
SELECT id, repository_path, repository_name, repository_id, history_key
FROM scans
WHERE repository_id IS NULL OR history_key IS NULL
"""
@@ -412,11 +417,19 @@ class ScanStore:
}
def _backfill_scan_summaries(self, db: sqlite3.Connection) -> None:
rows = db.execute(
"SELECT rowid AS scan_rowid, * FROM scans "
"WHERE finding_count IS NULL OR blocker_count IS NULL OR release_readiness IS NULL"
).fetchall()
for row in rows:
# Collect ids first and load one report at a time so a large legacy history is
# never materialized in memory at once during migration.
scan_ids = [
str(item[0])
for item in db.execute(
"SELECT id FROM scans "
"WHERE finding_count IS NULL OR blocker_count IS NULL OR release_readiness IS NULL"
).fetchall()
]
for scan_id in scan_ids:
row = db.execute("SELECT rowid AS scan_rowid, * FROM scans WHERE id = ?", (scan_id,)).fetchone()
if row is None:
continue
report = self._report_from_row(row)
previous_row = db.execute(
"""
@@ -560,112 +573,6 @@ class ScanStore:
row = db.execute("SELECT source_path FROM repositories WHERE id = ?", (repository_id,)).fetchone()
return row["source_path"] if row else None
def _repository_summary(self, db: sqlite3.Connection, row: sqlite3.Row) -> RepositorySummary:
repository = self._repository_from_row(row)
latest = db.execute(
"""
SELECT * FROM scans WHERE repository_id = ?
ORDER BY scanned_at DESC, rowid DESC LIMIT 1
""",
(repository.id,),
).fetchone()
active = db.execute(
"""
SELECT * FROM scan_jobs WHERE repository_id = ? AND status IN (?, ?, ?, ?, ?)
ORDER BY created_at DESC LIMIT 1
""",
(repository.id, *ACTIVE_JOB_STATUSES),
).fetchone()
last_job = db.execute(
"SELECT * FROM scan_jobs WHERE repository_id = ? ORDER BY created_at DESC LIMIT 1",
(repository.id,),
).fetchone()
last_scan = self._report_from_row(latest) if latest else None
active_findings = [item for item in (last_scan.findings if last_scan else []) if not item.suppressed]
severity_rank = {Severity.BLOCKER: 0, Severity.WARNING: 1, Severity.INFO: 2}
primary_finding = min(
active_findings,
key=lambda item: (severity_rank[item.severity], -item.deduction, item.category, item.code),
default=None,
)
primary_signal = (
RemediationSignal(
code=primary_finding.code,
category=primary_finding.category,
severity=primary_finding.severity,
message=primary_finding.message,
count=sum(1 for item in active_findings if item.code == primary_finding.code),
)
if primary_finding
else None
)
previous_row = (
db.execute(
"""
SELECT * FROM scans
WHERE repository_id = ? AND id != ?
ORDER BY scanned_at DESC, rowid DESC LIMIT 1
""",
(repository.id, last_scan.id),
).fetchone()
if last_scan
else None
)
previous_scan = self._report_from_row(previous_row) if previous_row else None
previous_findings = [item for item in (previous_scan.findings if previous_scan else []) if not item.suppressed]
active_fingerprints = {self._finding_key(item) for item in active_findings}
previous_fingerprints = {self._finding_key(item) for item in previous_findings}
last_revision = (
last_scan.source.actual_revision
if last_scan and last_scan.source.actual_revision
else (latest["actual_revision"] if latest else None)
)
stale = bool(repository.latest_revision and last_revision and repository.latest_revision != last_revision)
if repository.archived:
state = RepositoryState.ARCHIVED
elif not repository.available:
state = RepositoryState.UNAVAILABLE
elif active:
state = (
RepositoryState.QUEUED
if active["status"] == ScanJobStatus.QUEUED.value
else RepositoryState.SCANNING
)
elif last_job and last_job["status"] == ScanJobStatus.FAILED.value:
last_job_at = _datetime(last_job["created_at"])
if last_scan is None or (last_job_at and last_job_at > last_scan.scanned_at):
state = RepositoryState.FAILED
elif stale:
state = RepositoryState.STALE
else:
state = RepositoryState.CURRENT
elif last_scan is None:
state = RepositoryState.NEVER_SCANNED
elif stale:
state = RepositoryState.STALE
else:
state = RepositoryState.CURRENT
data = repository.model_dump()
data.update(
{
"state": state,
"stale": stale,
"active_job_id": active["id"] if active else None,
"last_scan_id": last_scan.id if last_scan else None,
"last_scan_at": last_scan.scanned_at if last_scan else None,
"last_scan_health": last_scan.health_score if last_scan else None,
"last_scan_readiness": last_scan.release_gate.status if last_scan and last_scan.release_gate else (last_scan.release_readiness if last_scan else None),
"last_scan_revision": last_revision,
"last_scan_blockers": sum(1 for item in active_findings if item.severity == Severity.BLOCKER),
"last_scan_warnings": sum(1 for item in active_findings if item.severity == Severity.WARNING),
"last_scan_primary_signal": primary_signal,
"last_scan_health_delta": (last_scan.health_score - previous_scan.health_score) if previous_scan and last_scan else None,
"last_scan_added_findings": len(active_fingerprints - previous_fingerprints) if previous_scan else 0,
"last_scan_resolved_findings": len(previous_fingerprints - active_fingerprints) if previous_scan else 0,
}
)
return RepositorySummary(**data)
def _catalog_rows(
self, db: sqlite3.Connection, *, limit: int, offset: int = 0, repository_id: str | None = None
) -> list[sqlite3.Row]:
@@ -674,7 +581,11 @@ class ScanStore:
return db.execute(
f"""
WITH ranked_scans AS (
SELECT s.*, ROW_NUMBER() OVER (
-- Summary columns only: report_json must never be materialized here.
SELECT s.id, s.repository_id, s.scanned_at, s.health_score, s.release_readiness,
s.actual_revision, s.finding_count, s.blocker_count, s.warning_count,
s.primary_signal_json, s.added_findings, s.resolved_findings,
s.triage_priority, s.triage_rationale, ROW_NUMBER() OVER (
PARTITION BY s.repository_id ORDER BY s.scanned_at DESC, s.rowid DESC
) AS scan_rank
FROM scans AS s WHERE s.repository_id IS NOT NULL
@@ -777,7 +688,7 @@ class ScanStore:
def repository_state_counts(self) -> dict[str, int]:
counts: Counter[str] = Counter()
for repository in self.list_repositories(limit=1000):
for repository in self.list_repositories(limit=-1):
counts[repository.state.value] += 1
return dict(counts)
@@ -793,31 +704,70 @@ class ScanStore:
).fetchone()
return RepositoryDetail(**summary.model_dump(), latest_scan=self._report_from_row(latest) if latest else None)
@staticmethod
def _merged_source_value(report_value: object, column_value: object, field: str) -> object:
"""Mirror _report_from_row: a non-null column fills an absent/manual report value."""
if column_value is None or report_value not in (None, ScanTrigger.MANUAL.value):
return report_value
if field == "trigger":
try:
return ScanTrigger(str(column_value))
except ValueError:
return report_value
return column_value
def repository_scans(self, repository_id: str, limit: int = 50) -> list[ScanSummary]:
"""List scan summaries without deserializing full report payloads.
Finding counts come from the retained report; compacted reports no longer hold
findings, so their retained active-finding summary count is reported instead of 0.
"""
with self._connection() as db:
rows = db.execute(
"SELECT * FROM scans WHERE repository_id = ? ORDER BY scanned_at DESC, rowid DESC LIMIT ?",
"""
SELECT id, scanned_at, health_score, report_compacted, finding_count,
release_readiness AS column_readiness,
branch, requested_revision, actual_revision, trigger,
json_extract(report_json, '$.source.branch') AS report_branch,
json_extract(report_json, '$.source.requested_revision') AS report_requested_revision,
json_extract(report_json, '$.source.actual_revision') AS report_actual_revision,
json_extract(report_json, '$.source.trigger') AS report_trigger,
json_extract(report_json, '$.release_readiness') AS report_readiness,
json_extract(report_json, '$.duration_ms') AS duration_ms,
json_array_length(report_json, '$.findings') AS report_findings
FROM scans WHERE repository_id = ?
ORDER BY scanned_at DESC, rowid DESC LIMIT ?
""",
(repository_id, limit),
).fetchall()
result: list[ScanSummary] = []
for row in rows:
report = self._report_from_row(row)
result.append(
ScanSummary(
id=report.id,
repository_id=repository_id,
scanned_at=report.scanned_at,
branch=report.source.branch,
requested_revision=report.source.requested_revision,
actual_revision=report.source.actual_revision,
trigger=report.source.trigger,
health_score=report.health_score,
release_readiness=report.release_readiness,
findings=len(report.findings),
duration_ms=report.duration_ms,
)
result: list[ScanSummary] = []
for row in rows:
source = {
field: self._merged_source_value(row[f"report_{field}"], row[field], field)
for field in ("branch", "requested_revision", "actual_revision", "trigger")
}
if row["report_compacted"]:
findings = int(row["finding_count"] or 0)
else:
findings = int(row["report_findings"] or 0)
result.append(
ScanSummary(
id=row["id"],
repository_id=repository_id,
scanned_at=row["scanned_at"],
branch=source["branch"],
requested_revision=source["requested_revision"],
actual_revision=source["actual_revision"],
trigger=source["trigger"] or ScanTrigger.MANUAL,
health_score=row["health_score"],
release_readiness=row["report_readiness"] or row["column_readiness"] or "",
findings=findings,
duration_ms=float(row["duration_ms"] or 0.0),
)
return result
)
return result
def _report_from_row(self, row: sqlite3.Row | None) -> ScanReport:
if row is None:
@@ -948,14 +898,20 @@ class ScanStore:
)
cutoff = (_now() - timedelta(days=report_retention_days())).isoformat()
db.execute("DELETE FROM scans WHERE history_key = ? AND scanned_at < ?", (history_key, cutoff))
rows = db.execute(
"""
SELECT * FROM scans WHERE history_key = ? AND report_compacted = 0
ORDER BY scanned_at DESC, rowid DESC LIMIT -1 OFFSET ?
""",
(history_key, full_reports_per_repository()),
).fetchall()
for row in rows:
scan_ids = [
str(item[0])
for item in db.execute(
"""
SELECT id FROM scans WHERE history_key = ? AND report_compacted = 0
ORDER BY scanned_at DESC, rowid DESC LIMIT -1 OFFSET ?
""",
(history_key, full_reports_per_repository()),
).fetchall()
]
for scan_id in scan_ids:
row = db.execute("SELECT * FROM scans WHERE id = ?", (scan_id,)).fetchone()
if row is None:
continue
report = self._report_from_row(row)
compact = report.model_copy(
update={"findings": [], "architecture": ArchitectureGraph(), "hotspots": [], "compacted": True}
@@ -1055,21 +1011,52 @@ class ScanStore:
)
def trend(self, repository_key: str, limit: int = 50) -> list[TrendPoint]:
reports = list(reversed(self.recent_for_repository(repository_key, limit)))
"""Return chronological trend points computed in SQLite, not from full reports.
Counts include every retained finding, as before. Compacted reports no longer
hold findings, so their retained active-finding summary counts are used rather
than reporting a misleading zero.
"""
with self._connection() as db:
rows: list[sqlite3.Row] = []
for candidate in self._history_candidates(repository_key):
rows = db.execute(
"""
SELECT id, scanned_at, health_score, report_compacted,
finding_count, blocker_count, warning_count,
json_array_length(report_json, '$.findings') AS report_findings,
(
SELECT COUNT(*) FROM json_each(scans.report_json, '$.findings')
WHERE json_extract(value, '$.severity') = 'blocker'
) AS report_blockers,
(
SELECT COUNT(*) FROM json_each(scans.report_json, '$.findings')
WHERE json_extract(value, '$.severity') = 'warning'
) AS report_warnings,
COALESCE(json_extract(report_json, '$.metrics.todo'), 0)
+ COALESCE(json_extract(report_json, '$.metrics.fixme'), 0)
+ COALESCE(json_extract(report_json, '$.metrics.hack'), 0)
+ COALESCE(json_extract(report_json, '$.metrics.xxx'), 0) AS debt_markers
FROM scans WHERE history_key = ?
ORDER BY scanned_at DESC, rowid DESC LIMIT ?
""",
(candidate, limit),
).fetchall()
if rows:
break
points: list[TrendPoint] = []
for report in reports:
blockers = sum(1 for item in report.findings if item.severity == Severity.BLOCKER)
warnings = sum(1 for item in report.findings if item.severity == Severity.WARNING)
debt = report.metrics.todo + report.metrics.fixme + report.metrics.hack + report.metrics.xxx
for row in reversed(rows):
compacted = bool(row["report_compacted"])
points.append(
TrendPoint(
scan_id=report.id,
scanned_at=report.scanned_at,
health_score=report.health_score,
findings=len(report.findings),
blockers=blockers,
warnings=warnings,
debt_markers=debt,
scan_id=row["id"],
scanned_at=row["scanned_at"],
health_score=row["health_score"],
findings=int((row["finding_count"] if compacted else row["report_findings"]) or 0),
blockers=int((row["blocker_count"] if compacted else row["report_blockers"]) or 0),
warnings=int((row["warning_count"] if compacted else row["report_warnings"]) or 0),
debt_markers=int(row["debt_markers"] or 0),
)
)
return points
@@ -1220,11 +1207,29 @@ class ScanStore:
row = db.execute("SELECT * FROM scan_jobs WHERE id = ?", (job_id,)).fetchone()
return self._job_from_row(row) if row else None
@staticmethod
def _claim_work_may_exist(db: sqlite3.Connection, now: str) -> bool:
placeholders = ", ".join("?" for _ in ACTIVE_JOB_STATUSES[1:])
row = db.execute(
f"""
SELECT EXISTS(SELECT 1 FROM scan_jobs WHERE status = 'queued')
OR EXISTS(
SELECT 1 FROM scan_jobs
WHERE status IN ({placeholders}) AND lease_until IS NOT NULL AND lease_until < ?
)
""",
(*ACTIVE_JOB_STATUSES[1:], now),
).fetchone()
return bool(row[0])
def claim_next_job(self) -> ScanJob | None:
now_dt = _now()
now = now_dt.isoformat()
lease_until = (now_dt + timedelta(seconds=job_lease_seconds())).isoformat()
with self._connection() as db:
# Cheap read-only probe: an idle queue never takes the write lock.
if not self._claim_work_may_exist(db, now):
return None
db.execute("BEGIN IMMEDIATE")
if self._maintenance_enabled(db):
return None
@@ -1443,6 +1448,30 @@ class ScanStore:
except OSError:
return 0
def _cached_mirror_stats(self, mirrors_root: Path) -> tuple[int, int]:
"""Return mirror-cache size without making /meta wait on a large tree walk.
The walk (up to 50k files) runs synchronously only when no value exists yet.
Afterwards an expired value is served while one background thread refreshes it.
"""
key = str(mirrors_root)
now = monotonic()
cached = self._mirror_stats_cache
if cached is None or cached[1] != key:
stats = self._tree_stats(mirrors_root)
self._mirror_stats_cache = (monotonic(), key, stats)
return stats
if now - cached[0] >= MIRROR_STATS_TTL_SECONDS and self._mirror_stats_refresh.acquire(blocking=False):
def refresh() -> None:
try:
self._mirror_stats_cache = (monotonic(), key, self._tree_stats(mirrors_root))
finally:
self._mirror_stats_refresh.release()
Thread(target=refresh, name="repopulse-mirror-stats", daemon=True).start()
return cached[2]
def storage_stats(self) -> dict[str, int | str | None]:
with self._connection() as db:
row = db.execute(
@@ -1462,7 +1491,7 @@ class ScanStore:
cache_root = Path(os.getenv("REPOPULSE_REPOSITORY_CACHE", "data/repos")).expanduser()
mirrors_root = cache_root / "mirrors"
worktrees_root = cache_root / "worktrees"
_mirror_files, mirror_cache_bytes = self._tree_stats(mirrors_root)
_mirror_files, mirror_cache_bytes = self._cached_mirror_stats(mirrors_root)
mirror_count = self._managed_entry_count(mirrors_root)
worktree_count = self._managed_entry_count(worktrees_root)
return {