RepoPulse: update backend/app and dependencies (+3 areas)
Public changes compared with the previous published export: 26 files changed. Affected areas: - backend/app: 11 files - dependencies: 5 files - deployment and CI: 3 files - documentation: 2 files - tests: 5 files Changed files: - Updated: "Dockerfile.unraid" - Updated: "README.md" - Updated: "backend/Dockerfile" - Updated: "backend/app/acquisition.py" - Updated: "backend/app/architecture.py" - Updated: "backend/app/git_analysis.py" - Updated: "backend/app/gitea.py" - Updated: "backend/app/jobs.py" - Updated: "backend/app/main.py" - Added: "backend/app/processes.py" - Updated: "backend/app/rules.py" - Updated: "backend/app/scanner.py" - Updated: "backend/app/signals.py" - Updated: "backend/app/store.py" - Updated: "backend/pyproject.toml" - Added: "backend/tests/test_api_responsiveness.py" - Added: "backend/tests/test_processes.py" - Updated: "backend/tests/test_scanner.py" - Updated: "backend/tests/test_signals_v11.py" - Updated: "backend/tests/test_store.py" - Updated: "docker-compose.yml" - Updated: "docs/ARCHITECTURE.md" - Updated: "frontend/package-lock.json" - Updated: "frontend/package.json" - Updated: "package-lock.json" - Updated: "package.json" Export provenance/manifests refreshed; excluded from the change summary. Publish curated RepoPulse source from a251f826238908b156485299061da76482780ed2
This commit is contained in:
1 parent
23a4a721f1
commit
61ae49f9d0
28 files changed
+1083
-408
No files matched your search
+183
-154
@@ -9,6 +9,8 @@ from contextlib import contextmanager
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from hashlib import sha256
|
||||
from pathlib import Path
|
||||
from threading import Lock, Thread
|
||||
from time import monotonic
|
||||
from uuid import uuid4
|
||||
|
||||
from .models import (
|
||||
@@ -35,6 +37,7 @@ from .rules import enrich_findings
|
||||
|
||||
MAX_HISTORY_PER_REPOSITORY = 200
|
||||
SCHEMA_VERSION = 4
|
||||
MIRROR_STATS_TTL_SECONDS = 300.0
|
||||
JOB_LEASE_SECONDS = 900
|
||||
ACTIVE_JOB_STATUSES = (
|
||||
ScanJobStatus.QUEUED.value,
|
||||
@@ -108,6 +111,8 @@ class ScanStore:
|
||||
self.path = Path(raw_path)
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self.recovery_metadata: dict[str, str] | None = None
|
||||
self._mirror_stats_cache: tuple[float, str, tuple[int, int]] | None = None
|
||||
self._mirror_stats_refresh = Lock()
|
||||
try:
|
||||
self._validate_existing_database()
|
||||
self._init_schema()
|
||||
@@ -349,7 +354,7 @@ class ScanStore:
|
||||
def _backfill_legacy_catalog(self, db: sqlite3.Connection) -> None:
|
||||
rows = db.execute(
|
||||
"""
|
||||
SELECT id, repository_path, repository_name, report_json, repository_id, history_key
|
||||
SELECT id, repository_path, repository_name, repository_id, history_key
|
||||
FROM scans
|
||||
WHERE repository_id IS NULL OR history_key IS NULL
|
||||
"""
|
||||
@@ -412,11 +417,19 @@ class ScanStore:
|
||||
}
|
||||
|
||||
def _backfill_scan_summaries(self, db: sqlite3.Connection) -> None:
|
||||
rows = db.execute(
|
||||
"SELECT rowid AS scan_rowid, * FROM scans "
|
||||
"WHERE finding_count IS NULL OR blocker_count IS NULL OR release_readiness IS NULL"
|
||||
).fetchall()
|
||||
for row in rows:
|
||||
# Collect ids first and load one report at a time so a large legacy history is
|
||||
# never materialized in memory at once during migration.
|
||||
scan_ids = [
|
||||
str(item[0])
|
||||
for item in db.execute(
|
||||
"SELECT id FROM scans "
|
||||
"WHERE finding_count IS NULL OR blocker_count IS NULL OR release_readiness IS NULL"
|
||||
).fetchall()
|
||||
]
|
||||
for scan_id in scan_ids:
|
||||
row = db.execute("SELECT rowid AS scan_rowid, * FROM scans WHERE id = ?", (scan_id,)).fetchone()
|
||||
if row is None:
|
||||
continue
|
||||
report = self._report_from_row(row)
|
||||
previous_row = db.execute(
|
||||
"""
|
||||
@@ -560,112 +573,6 @@ class ScanStore:
|
||||
row = db.execute("SELECT source_path FROM repositories WHERE id = ?", (repository_id,)).fetchone()
|
||||
return row["source_path"] if row else None
|
||||
|
||||
def _repository_summary(self, db: sqlite3.Connection, row: sqlite3.Row) -> RepositorySummary:
|
||||
repository = self._repository_from_row(row)
|
||||
latest = db.execute(
|
||||
"""
|
||||
SELECT * FROM scans WHERE repository_id = ?
|
||||
ORDER BY scanned_at DESC, rowid DESC LIMIT 1
|
||||
""",
|
||||
(repository.id,),
|
||||
).fetchone()
|
||||
active = db.execute(
|
||||
"""
|
||||
SELECT * FROM scan_jobs WHERE repository_id = ? AND status IN (?, ?, ?, ?, ?)
|
||||
ORDER BY created_at DESC LIMIT 1
|
||||
""",
|
||||
(repository.id, *ACTIVE_JOB_STATUSES),
|
||||
).fetchone()
|
||||
last_job = db.execute(
|
||||
"SELECT * FROM scan_jobs WHERE repository_id = ? ORDER BY created_at DESC LIMIT 1",
|
||||
(repository.id,),
|
||||
).fetchone()
|
||||
last_scan = self._report_from_row(latest) if latest else None
|
||||
active_findings = [item for item in (last_scan.findings if last_scan else []) if not item.suppressed]
|
||||
severity_rank = {Severity.BLOCKER: 0, Severity.WARNING: 1, Severity.INFO: 2}
|
||||
primary_finding = min(
|
||||
active_findings,
|
||||
key=lambda item: (severity_rank[item.severity], -item.deduction, item.category, item.code),
|
||||
default=None,
|
||||
)
|
||||
primary_signal = (
|
||||
RemediationSignal(
|
||||
code=primary_finding.code,
|
||||
category=primary_finding.category,
|
||||
severity=primary_finding.severity,
|
||||
message=primary_finding.message,
|
||||
count=sum(1 for item in active_findings if item.code == primary_finding.code),
|
||||
)
|
||||
if primary_finding
|
||||
else None
|
||||
)
|
||||
previous_row = (
|
||||
db.execute(
|
||||
"""
|
||||
SELECT * FROM scans
|
||||
WHERE repository_id = ? AND id != ?
|
||||
ORDER BY scanned_at DESC, rowid DESC LIMIT 1
|
||||
""",
|
||||
(repository.id, last_scan.id),
|
||||
).fetchone()
|
||||
if last_scan
|
||||
else None
|
||||
)
|
||||
previous_scan = self._report_from_row(previous_row) if previous_row else None
|
||||
previous_findings = [item for item in (previous_scan.findings if previous_scan else []) if not item.suppressed]
|
||||
active_fingerprints = {self._finding_key(item) for item in active_findings}
|
||||
previous_fingerprints = {self._finding_key(item) for item in previous_findings}
|
||||
last_revision = (
|
||||
last_scan.source.actual_revision
|
||||
if last_scan and last_scan.source.actual_revision
|
||||
else (latest["actual_revision"] if latest else None)
|
||||
)
|
||||
stale = bool(repository.latest_revision and last_revision and repository.latest_revision != last_revision)
|
||||
if repository.archived:
|
||||
state = RepositoryState.ARCHIVED
|
||||
elif not repository.available:
|
||||
state = RepositoryState.UNAVAILABLE
|
||||
elif active:
|
||||
state = (
|
||||
RepositoryState.QUEUED
|
||||
if active["status"] == ScanJobStatus.QUEUED.value
|
||||
else RepositoryState.SCANNING
|
||||
)
|
||||
elif last_job and last_job["status"] == ScanJobStatus.FAILED.value:
|
||||
last_job_at = _datetime(last_job["created_at"])
|
||||
if last_scan is None or (last_job_at and last_job_at > last_scan.scanned_at):
|
||||
state = RepositoryState.FAILED
|
||||
elif stale:
|
||||
state = RepositoryState.STALE
|
||||
else:
|
||||
state = RepositoryState.CURRENT
|
||||
elif last_scan is None:
|
||||
state = RepositoryState.NEVER_SCANNED
|
||||
elif stale:
|
||||
state = RepositoryState.STALE
|
||||
else:
|
||||
state = RepositoryState.CURRENT
|
||||
data = repository.model_dump()
|
||||
data.update(
|
||||
{
|
||||
"state": state,
|
||||
"stale": stale,
|
||||
"active_job_id": active["id"] if active else None,
|
||||
"last_scan_id": last_scan.id if last_scan else None,
|
||||
"last_scan_at": last_scan.scanned_at if last_scan else None,
|
||||
"last_scan_health": last_scan.health_score if last_scan else None,
|
||||
"last_scan_readiness": last_scan.release_gate.status if last_scan and last_scan.release_gate else (last_scan.release_readiness if last_scan else None),
|
||||
"last_scan_revision": last_revision,
|
||||
"last_scan_blockers": sum(1 for item in active_findings if item.severity == Severity.BLOCKER),
|
||||
"last_scan_warnings": sum(1 for item in active_findings if item.severity == Severity.WARNING),
|
||||
"last_scan_primary_signal": primary_signal,
|
||||
"last_scan_health_delta": (last_scan.health_score - previous_scan.health_score) if previous_scan and last_scan else None,
|
||||
"last_scan_added_findings": len(active_fingerprints - previous_fingerprints) if previous_scan else 0,
|
||||
"last_scan_resolved_findings": len(previous_fingerprints - active_fingerprints) if previous_scan else 0,
|
||||
}
|
||||
)
|
||||
return RepositorySummary(**data)
|
||||
|
||||
def _catalog_rows(
|
||||
self, db: sqlite3.Connection, *, limit: int, offset: int = 0, repository_id: str | None = None
|
||||
) -> list[sqlite3.Row]:
|
||||
@@ -674,7 +581,11 @@ class ScanStore:
|
||||
return db.execute(
|
||||
f"""
|
||||
WITH ranked_scans AS (
|
||||
SELECT s.*, ROW_NUMBER() OVER (
|
||||
-- Summary columns only: report_json must never be materialized here.
|
||||
SELECT s.id, s.repository_id, s.scanned_at, s.health_score, s.release_readiness,
|
||||
s.actual_revision, s.finding_count, s.blocker_count, s.warning_count,
|
||||
s.primary_signal_json, s.added_findings, s.resolved_findings,
|
||||
s.triage_priority, s.triage_rationale, ROW_NUMBER() OVER (
|
||||
PARTITION BY s.repository_id ORDER BY s.scanned_at DESC, s.rowid DESC
|
||||
) AS scan_rank
|
||||
FROM scans AS s WHERE s.repository_id IS NOT NULL
|
||||
@@ -777,7 +688,7 @@ class ScanStore:
|
||||
|
||||
def repository_state_counts(self) -> dict[str, int]:
|
||||
counts: Counter[str] = Counter()
|
||||
for repository in self.list_repositories(limit=1000):
|
||||
for repository in self.list_repositories(limit=-1):
|
||||
counts[repository.state.value] += 1
|
||||
return dict(counts)
|
||||
|
||||
@@ -793,31 +704,70 @@ class ScanStore:
|
||||
).fetchone()
|
||||
return RepositoryDetail(**summary.model_dump(), latest_scan=self._report_from_row(latest) if latest else None)
|
||||
|
||||
@staticmethod
|
||||
def _merged_source_value(report_value: object, column_value: object, field: str) -> object:
|
||||
"""Mirror _report_from_row: a non-null column fills an absent/manual report value."""
|
||||
|
||||
if column_value is None or report_value not in (None, ScanTrigger.MANUAL.value):
|
||||
return report_value
|
||||
if field == "trigger":
|
||||
try:
|
||||
return ScanTrigger(str(column_value))
|
||||
except ValueError:
|
||||
return report_value
|
||||
return column_value
|
||||
|
||||
def repository_scans(self, repository_id: str, limit: int = 50) -> list[ScanSummary]:
|
||||
"""List scan summaries without deserializing full report payloads.
|
||||
|
||||
Finding counts come from the retained report; compacted reports no longer hold
|
||||
findings, so their retained active-finding summary count is reported instead of 0.
|
||||
"""
|
||||
|
||||
with self._connection() as db:
|
||||
rows = db.execute(
|
||||
"SELECT * FROM scans WHERE repository_id = ? ORDER BY scanned_at DESC, rowid DESC LIMIT ?",
|
||||
"""
|
||||
SELECT id, scanned_at, health_score, report_compacted, finding_count,
|
||||
release_readiness AS column_readiness,
|
||||
branch, requested_revision, actual_revision, trigger,
|
||||
json_extract(report_json, '$.source.branch') AS report_branch,
|
||||
json_extract(report_json, '$.source.requested_revision') AS report_requested_revision,
|
||||
json_extract(report_json, '$.source.actual_revision') AS report_actual_revision,
|
||||
json_extract(report_json, '$.source.trigger') AS report_trigger,
|
||||
json_extract(report_json, '$.release_readiness') AS report_readiness,
|
||||
json_extract(report_json, '$.duration_ms') AS duration_ms,
|
||||
json_array_length(report_json, '$.findings') AS report_findings
|
||||
FROM scans WHERE repository_id = ?
|
||||
ORDER BY scanned_at DESC, rowid DESC LIMIT ?
|
||||
""",
|
||||
(repository_id, limit),
|
||||
).fetchall()
|
||||
result: list[ScanSummary] = []
|
||||
for row in rows:
|
||||
report = self._report_from_row(row)
|
||||
result.append(
|
||||
ScanSummary(
|
||||
id=report.id,
|
||||
repository_id=repository_id,
|
||||
scanned_at=report.scanned_at,
|
||||
branch=report.source.branch,
|
||||
requested_revision=report.source.requested_revision,
|
||||
actual_revision=report.source.actual_revision,
|
||||
trigger=report.source.trigger,
|
||||
health_score=report.health_score,
|
||||
release_readiness=report.release_readiness,
|
||||
findings=len(report.findings),
|
||||
duration_ms=report.duration_ms,
|
||||
)
|
||||
result: list[ScanSummary] = []
|
||||
for row in rows:
|
||||
source = {
|
||||
field: self._merged_source_value(row[f"report_{field}"], row[field], field)
|
||||
for field in ("branch", "requested_revision", "actual_revision", "trigger")
|
||||
}
|
||||
if row["report_compacted"]:
|
||||
findings = int(row["finding_count"] or 0)
|
||||
else:
|
||||
findings = int(row["report_findings"] or 0)
|
||||
result.append(
|
||||
ScanSummary(
|
||||
id=row["id"],
|
||||
repository_id=repository_id,
|
||||
scanned_at=row["scanned_at"],
|
||||
branch=source["branch"],
|
||||
requested_revision=source["requested_revision"],
|
||||
actual_revision=source["actual_revision"],
|
||||
trigger=source["trigger"] or ScanTrigger.MANUAL,
|
||||
health_score=row["health_score"],
|
||||
release_readiness=row["report_readiness"] or row["column_readiness"] or "",
|
||||
findings=findings,
|
||||
duration_ms=float(row["duration_ms"] or 0.0),
|
||||
)
|
||||
return result
|
||||
)
|
||||
return result
|
||||
|
||||
def _report_from_row(self, row: sqlite3.Row | None) -> ScanReport:
|
||||
if row is None:
|
||||
@@ -948,14 +898,20 @@ class ScanStore:
|
||||
)
|
||||
cutoff = (_now() - timedelta(days=report_retention_days())).isoformat()
|
||||
db.execute("DELETE FROM scans WHERE history_key = ? AND scanned_at < ?", (history_key, cutoff))
|
||||
rows = db.execute(
|
||||
"""
|
||||
SELECT * FROM scans WHERE history_key = ? AND report_compacted = 0
|
||||
ORDER BY scanned_at DESC, rowid DESC LIMIT -1 OFFSET ?
|
||||
""",
|
||||
(history_key, full_reports_per_repository()),
|
||||
).fetchall()
|
||||
for row in rows:
|
||||
scan_ids = [
|
||||
str(item[0])
|
||||
for item in db.execute(
|
||||
"""
|
||||
SELECT id FROM scans WHERE history_key = ? AND report_compacted = 0
|
||||
ORDER BY scanned_at DESC, rowid DESC LIMIT -1 OFFSET ?
|
||||
""",
|
||||
(history_key, full_reports_per_repository()),
|
||||
).fetchall()
|
||||
]
|
||||
for scan_id in scan_ids:
|
||||
row = db.execute("SELECT * FROM scans WHERE id = ?", (scan_id,)).fetchone()
|
||||
if row is None:
|
||||
continue
|
||||
report = self._report_from_row(row)
|
||||
compact = report.model_copy(
|
||||
update={"findings": [], "architecture": ArchitectureGraph(), "hotspots": [], "compacted": True}
|
||||
@@ -1055,21 +1011,52 @@ class ScanStore:
|
||||
)
|
||||
|
||||
def trend(self, repository_key: str, limit: int = 50) -> list[TrendPoint]:
|
||||
reports = list(reversed(self.recent_for_repository(repository_key, limit)))
|
||||
"""Return chronological trend points computed in SQLite, not from full reports.
|
||||
|
||||
Counts include every retained finding, as before. Compacted reports no longer
|
||||
hold findings, so their retained active-finding summary counts are used rather
|
||||
than reporting a misleading zero.
|
||||
"""
|
||||
|
||||
with self._connection() as db:
|
||||
rows: list[sqlite3.Row] = []
|
||||
for candidate in self._history_candidates(repository_key):
|
||||
rows = db.execute(
|
||||
"""
|
||||
SELECT id, scanned_at, health_score, report_compacted,
|
||||
finding_count, blocker_count, warning_count,
|
||||
json_array_length(report_json, '$.findings') AS report_findings,
|
||||
(
|
||||
SELECT COUNT(*) FROM json_each(scans.report_json, '$.findings')
|
||||
WHERE json_extract(value, '$.severity') = 'blocker'
|
||||
) AS report_blockers,
|
||||
(
|
||||
SELECT COUNT(*) FROM json_each(scans.report_json, '$.findings')
|
||||
WHERE json_extract(value, '$.severity') = 'warning'
|
||||
) AS report_warnings,
|
||||
COALESCE(json_extract(report_json, '$.metrics.todo'), 0)
|
||||
+ COALESCE(json_extract(report_json, '$.metrics.fixme'), 0)
|
||||
+ COALESCE(json_extract(report_json, '$.metrics.hack'), 0)
|
||||
+ COALESCE(json_extract(report_json, '$.metrics.xxx'), 0) AS debt_markers
|
||||
FROM scans WHERE history_key = ?
|
||||
ORDER BY scanned_at DESC, rowid DESC LIMIT ?
|
||||
""",
|
||||
(candidate, limit),
|
||||
).fetchall()
|
||||
if rows:
|
||||
break
|
||||
points: list[TrendPoint] = []
|
||||
for report in reports:
|
||||
blockers = sum(1 for item in report.findings if item.severity == Severity.BLOCKER)
|
||||
warnings = sum(1 for item in report.findings if item.severity == Severity.WARNING)
|
||||
debt = report.metrics.todo + report.metrics.fixme + report.metrics.hack + report.metrics.xxx
|
||||
for row in reversed(rows):
|
||||
compacted = bool(row["report_compacted"])
|
||||
points.append(
|
||||
TrendPoint(
|
||||
scan_id=report.id,
|
||||
scanned_at=report.scanned_at,
|
||||
health_score=report.health_score,
|
||||
findings=len(report.findings),
|
||||
blockers=blockers,
|
||||
warnings=warnings,
|
||||
debt_markers=debt,
|
||||
scan_id=row["id"],
|
||||
scanned_at=row["scanned_at"],
|
||||
health_score=row["health_score"],
|
||||
findings=int((row["finding_count"] if compacted else row["report_findings"]) or 0),
|
||||
blockers=int((row["blocker_count"] if compacted else row["report_blockers"]) or 0),
|
||||
warnings=int((row["warning_count"] if compacted else row["report_warnings"]) or 0),
|
||||
debt_markers=int(row["debt_markers"] or 0),
|
||||
)
|
||||
)
|
||||
return points
|
||||
@@ -1220,11 +1207,29 @@ class ScanStore:
|
||||
row = db.execute("SELECT * FROM scan_jobs WHERE id = ?", (job_id,)).fetchone()
|
||||
return self._job_from_row(row) if row else None
|
||||
|
||||
@staticmethod
|
||||
def _claim_work_may_exist(db: sqlite3.Connection, now: str) -> bool:
|
||||
placeholders = ", ".join("?" for _ in ACTIVE_JOB_STATUSES[1:])
|
||||
row = db.execute(
|
||||
f"""
|
||||
SELECT EXISTS(SELECT 1 FROM scan_jobs WHERE status = 'queued')
|
||||
OR EXISTS(
|
||||
SELECT 1 FROM scan_jobs
|
||||
WHERE status IN ({placeholders}) AND lease_until IS NOT NULL AND lease_until < ?
|
||||
)
|
||||
""",
|
||||
(*ACTIVE_JOB_STATUSES[1:], now),
|
||||
).fetchone()
|
||||
return bool(row[0])
|
||||
|
||||
def claim_next_job(self) -> ScanJob | None:
|
||||
now_dt = _now()
|
||||
now = now_dt.isoformat()
|
||||
lease_until = (now_dt + timedelta(seconds=job_lease_seconds())).isoformat()
|
||||
with self._connection() as db:
|
||||
# Cheap read-only probe: an idle queue never takes the write lock.
|
||||
if not self._claim_work_may_exist(db, now):
|
||||
return None
|
||||
db.execute("BEGIN IMMEDIATE")
|
||||
if self._maintenance_enabled(db):
|
||||
return None
|
||||
@@ -1443,6 +1448,30 @@ class ScanStore:
|
||||
except OSError:
|
||||
return 0
|
||||
|
||||
def _cached_mirror_stats(self, mirrors_root: Path) -> tuple[int, int]:
|
||||
"""Return mirror-cache size without making /meta wait on a large tree walk.
|
||||
|
||||
The walk (up to 50k files) runs synchronously only when no value exists yet.
|
||||
Afterwards an expired value is served while one background thread refreshes it.
|
||||
"""
|
||||
|
||||
key = str(mirrors_root)
|
||||
now = monotonic()
|
||||
cached = self._mirror_stats_cache
|
||||
if cached is None or cached[1] != key:
|
||||
stats = self._tree_stats(mirrors_root)
|
||||
self._mirror_stats_cache = (monotonic(), key, stats)
|
||||
return stats
|
||||
if now - cached[0] >= MIRROR_STATS_TTL_SECONDS and self._mirror_stats_refresh.acquire(blocking=False):
|
||||
def refresh() -> None:
|
||||
try:
|
||||
self._mirror_stats_cache = (monotonic(), key, self._tree_stats(mirrors_root))
|
||||
finally:
|
||||
self._mirror_stats_refresh.release()
|
||||
|
||||
Thread(target=refresh, name="repopulse-mirror-stats", daemon=True).start()
|
||||
return cached[2]
|
||||
|
||||
def storage_stats(self) -> dict[str, int | str | None]:
|
||||
with self._connection() as db:
|
||||
row = db.execute(
|
||||
@@ -1462,7 +1491,7 @@ class ScanStore:
|
||||
cache_root = Path(os.getenv("REPOPULSE_REPOSITORY_CACHE", "data/repos")).expanduser()
|
||||
mirrors_root = cache_root / "mirrors"
|
||||
worktrees_root = cache_root / "worktrees"
|
||||
_mirror_files, mirror_cache_bytes = self._tree_stats(mirrors_root)
|
||||
_mirror_files, mirror_cache_bytes = self._cached_mirror_stats(mirrors_root)
|
||||
mirror_count = self._managed_entry_count(mirrors_root)
|
||||
worktree_count = self._managed_entry_count(worktrees_root)
|
||||
return {
|
||||
|
||||
Reference in new issue
Block a user