Files
Public Source Publisher 61ae49f9d0 RepoPulse: update backend/app and dependencies (+3 areas)
Public changes compared with the previous published export:
26 files changed.

Affected areas:
- backend/app: 11 files
- dependencies: 5 files
- deployment and CI: 3 files
- documentation: 2 files
- tests: 5 files

Changed files:
- Updated: "Dockerfile.unraid"
- Updated: "README.md"
- Updated: "backend/Dockerfile"
- Updated: "backend/app/acquisition.py"
- Updated: "backend/app/architecture.py"
- Updated: "backend/app/git_analysis.py"
- Updated: "backend/app/gitea.py"
- Updated: "backend/app/jobs.py"
- Updated: "backend/app/main.py"
- Added: "backend/app/processes.py"
- Updated: "backend/app/rules.py"
- Updated: "backend/app/scanner.py"
- Updated: "backend/app/signals.py"
- Updated: "backend/app/store.py"
- Updated: "backend/pyproject.toml"
- Added: "backend/tests/test_api_responsiveness.py"
- Added: "backend/tests/test_processes.py"
- Updated: "backend/tests/test_scanner.py"
- Updated: "backend/tests/test_signals_v11.py"
- Updated: "backend/tests/test_store.py"
- Updated: "docker-compose.yml"
- Updated: "docs/ARCHITECTURE.md"
- Updated: "frontend/package-lock.json"
- Updated: "frontend/package.json"
- Updated: "package-lock.json"
- Updated: "package.json"

Export provenance/manifests refreshed; excluded from the change summary.

Publish curated RepoPulse source from a251f826238908b156485299061da76482780ed2
2026-11-17 21:10:30 +00:00

406 lines
15 KiB
Python

from __future__ import annotations
import ctypes
import ctypes.util
import gc
import logging
import os
import sys
from collections.abc import Callable
from datetime import UTC, datetime
from threading import Event, Lock, Thread, current_thread
from time import monotonic
from .acquisition import AcquisitionError, GitAcquirer
from .gitea import GiteaClient
from .models import (
Repository,
RepositorySyncResult,
ScanJob,
ScanJobStatus,
ScanSource,
ScanTrigger,
SourceOrigin,
)
from .scanner import scan_repository
from .store import MaintenanceModeError, QueueFullError, ScanStore, job_lease_seconds
LOGGER = logging.getLogger(__name__)
WORKER_HEARTBEAT_SECONDS = 5.0
WORKER_IDLE_POLL_SECONDS = 5.0
def _load_malloc_trim() -> Callable[[int], int] | None:
if not sys.platform.startswith("linux"):
return None
try:
libc = ctypes.CDLL(ctypes.util.find_library("c") or "libc.so.6")
trim = libc.malloc_trim
except (AttributeError, OSError):
return None # e.g. musl libc has no malloc_trim
trim.argtypes = [ctypes.c_size_t]
trim.restype = ctypes.c_int
return trim
_MALLOC_TRIM = _load_malloc_trim()
def release_memory() -> None:
"""Return memory freed by a finished scan to the OS.
A scan allocates many short-lived objects (ASTs, file texts, findings). CPython frees
them, but glibc keeps the arenas mapped, so the long-running worker's RSS would stay
at its largest scan. A collection plus malloc_trim hands those pages back.
"""
gc.collect()
if _MALLOC_TRIM is not None:
try:
_MALLOC_TRIM(0)
except OSError:
pass
def _positive_int(name: str, default: int, *, maximum: int | None = None) -> int:
try:
value = max(1, int(os.getenv(name, str(default))))
except ValueError:
value = default
return min(value, maximum) if maximum else value
def _positive_float(name: str, default: float) -> float:
try:
return max(0.2, float(os.getenv(name, str(default))))
except ValueError:
return default
class ScanJobRunner:
"""Single-process durable worker and bounded Gitea scheduler."""
def __init__(
self,
store: ScanStore,
*,
gitea: GiteaClient | None = None,
acquirer: GitAcquirer | None = None,
poll_seconds: float = 0.5,
sync_interval_seconds: float | None = None,
max_queue: int | None = None,
) -> None:
self.store = store
self.gitea = gitea or GiteaClient()
self.acquirer = acquirer or GitAcquirer()
self.poll_seconds = max(0.2, poll_seconds)
self.sync_interval_seconds = sync_interval_seconds or _positive_float(
"REPOPULSE_GITEA_SYNC_INTERVAL_SECONDS", 300.0
)
self.max_queue = max_queue or _positive_int("REPOPULSE_MAX_QUEUED_SCANS", 100, maximum=1000)
self._stop = Event()
self._wake = Event()
self._idle_iterations = 0
self.idle_poll_seconds = max(
self.poll_seconds, _positive_float("REPOPULSE_WORKER_IDLE_POLL_SECONDS", WORKER_IDLE_POLL_SECONDS)
)
self._thread: Thread | None = None
self._last_sync = float("-inf")
self._last_heartbeat = float("-inf")
self.heartbeat_interval_seconds = _positive_float(
"REPOPULSE_WORKER_HEARTBEAT_SECONDS", WORKER_HEARTBEAT_SECONDS
)
self._sync_lock = Lock()
@property
def started(self) -> bool:
return self._thread is not None and self._thread.is_alive()
def start(self) -> None:
if self.started:
return
self._stop.clear()
self._wake.clear()
self._thread = Thread(target=self._run, name="repopulse-scan-worker", daemon=True)
self._thread.start()
def stop(self) -> None:
self._stop.set()
self._wake.set()
thread = self._thread
if thread and thread is not current_thread():
thread.join(timeout=5)
if thread is None or not thread.is_alive():
self._thread = None
def _run(self) -> None:
while not self._stop.is_set():
try:
job = self._run_iteration()
except MaintenanceModeError:
job = None
except Exception:
self._last_sync = monotonic()
LOGGER.exception("RepoPulse worker loop recovered from an unexpected failure")
job = None
if job is None:
self._idle_iterations += 1
self._wake.wait(self._idle_delay())
self._wake.clear()
else:
self._idle_iterations = 0
def _idle_delay(self) -> float:
"""Back off exponentially while idle; in-process enqueues wake the loop at once."""
exponent = min(self._idle_iterations - 1, 16)
return min(self.idle_poll_seconds, self.poll_seconds * (2**exponent))
def wake(self) -> None:
self._wake.set()
def _run_iteration(self) -> ScanJob | None:
"""Perform one scheduler pass unless durable maintenance has paused work."""
if self.store.maintenance_status()["enabled"]:
return None
# The heartbeat is a liveness signal, not a lock: claims and enqueues re-check
# maintenance inside their own transactions, so writing it on every 0.5 s poll
# only churns the WAL. Throttle the write to the heartbeat interval.
now = monotonic()
if now - self._last_heartbeat >= self.heartbeat_interval_seconds:
if not self.store.touch_worker_heartbeat_if_active(datetime.now(UTC).isoformat()):
return None
self._last_heartbeat = now
if self.gitea.configured and monotonic() - self._last_sync >= self.sync_interval_seconds:
self.sync_now()
# Synchronization can take a while; re-check before claiming new work.
if self.store.maintenance_status()["enabled"]:
return None
return self.process_one()
def sync_now(self) -> RepositorySyncResult:
with self._sync_lock:
if self.store.maintenance_status()["enabled"]:
raise MaintenanceModeError("RepoPulse is in maintenance mode")
if not self.gitea.configured:
result = RepositorySyncResult(
provider="gitea",
errors=["not_configured: Gitea connector is not configured"],
synced_at=datetime.now(UTC),
)
self.store.record_sync_result(result)
self._last_sync = monotonic()
return result
try:
result = self.gitea.sync(self.store)
except Exception:
self._last_sync = monotonic()
raise
self._last_sync = monotonic()
if not result.errors:
self._schedule_changed_repositories()
return result
def _schedule_changed_repositories(self) -> None:
if self.store.maintenance_status()["enabled"]:
return
# No silent catalog cap: every repository is reconciled, however large the catalog.
for summary in self.store.list_repositories(limit=-1):
if summary.provider != "gitea" or summary.archived or not summary.latest_revision:
continue
if not summary.policy.auto_scan:
continue
if summary.last_scan_revision and summary.last_scan_revision == summary.latest_revision:
continue
try:
self.store.enqueue_job(
summary.id,
summary.latest_revision,
ScanTrigger.SCHEDULED,
max_queue=self.max_queue,
)
except (MaintenanceModeError, QueueFullError, KeyError):
continue
def set_maintenance_mode(self, enabled: bool) -> dict[str, object]:
"""Serialize maintenance changes with catalog reconciliation and scan scheduling."""
with self._sync_lock:
return self.store.set_maintenance_mode(enabled)
def enqueue(
self,
repository: Repository,
*,
force: bool = False,
trigger: ScanTrigger = ScanTrigger.MANUAL,
) -> ScanJob:
requested_revision = repository.latest_revision
job = self.store.enqueue_job(
repository.id,
requested_revision,
trigger,
force=force,
max_queue=self.max_queue,
)
self.wake()
return job
def process_one(self) -> ScanJob | None:
job = self.store.claim_next_job()
if job is None:
return None
try:
self._process(job)
except Exception:
LOGGER.exception("RepoPulse scan job failed outside the isolated job handler")
try:
self.store.update_job(
job.id,
status=ScanJobStatus.FAILED,
phase="failed",
progress=100,
error_code="internal_error",
error_detail="Unexpected repository analysis failure",
message="Repository analysis failed",
)
except Exception:
LOGGER.exception("RepoPulse could not persist an unexpected job failure")
finally:
release_memory()
return self.store.get_job(job.id)
class _LeaseHeartbeat:
def __init__(self, store: ScanStore, job_id: str) -> None:
self.store = store
self.job_id = job_id
self.stop_event = Event()
self.thread = Thread(target=self._run, name=f"repopulse-lease-{job_id[:8]}", daemon=True)
def start(self) -> None:
self.thread.start()
def stop(self) -> None:
self.stop_event.set()
self.thread.join(timeout=2)
def _run(self) -> None:
interval = max(1.0, job_lease_seconds() / 3)
while not self.stop_event.wait(interval):
try:
self.store.renew_job_lease(self.job_id)
except Exception:
LOGGER.warning("RepoPulse could not renew a scan-job lease", exc_info=True)
def _process(self, job: ScanJob) -> None:
repository = self.store.get_repository(job.repository_id)
if repository is None:
self.store.update_job(
job.id,
status=ScanJobStatus.FAILED,
phase="failed",
progress=100,
error_code="repository_not_found",
error_detail="Repository catalog entry no longer exists",
)
return
source_path = None
acquired = None
lease_heartbeat = self._LeaseHeartbeat(self.store, job.id)
lease_heartbeat.start()
try:
if repository.provider != "gitea":
source_path = self.store.get_repository_source_path(repository.id)
if not source_path:
raise AcquisitionError("source_unavailable", "Local repository source is unavailable")
actual_revision = job.requested_revision
branch = repository.default_branch
else:
acquired = self.acquirer.acquire(repository, job.id, job.requested_revision)
source_path = str(acquired.path)
actual_revision = acquired.actual_revision
branch = acquired.branch
self.store.update_job(
job.id,
status=ScanJobStatus.SCANNING,
phase="scanning",
progress=30,
actual_revision=actual_revision,
message="Running deterministic repository analysis",
)
source = ScanSource(
origin=SourceOrigin.GITEA if repository.provider == "gitea" else SourceOrigin.LOCAL,
provider=repository.provider,
repository_id=repository.id,
repository_full_name=repository.full_name,
branch=branch,
requested_revision=job.requested_revision,
actual_revision=actual_revision,
trigger=job.trigger,
)
report = scan_repository(source_path, source=source, display_name=repository.name)
self.store.update_job(
job.id,
status=ScanJobStatus.ANALYZING,
phase="analyzing",
progress=80,
message="Persisting report and comparison baseline",
)
self.store.update_job(
job.id,
status=ScanJobStatus.PERSISTING,
phase="persisting",
progress=90,
message="Saving report and revision history",
)
saved = self.store.save(report)
self.store.upsert_repository(repository.model_copy(update={"policy": saved.policy}))
self.store.update_job(
job.id,
status=ScanJobStatus.COMPLETED,
phase="completed",
progress=100,
actual_revision=saved.source.actual_revision,
scan_id=saved.id,
message="Scan completed",
)
except AcquisitionError as exc:
self.store.update_job(
job.id,
status=ScanJobStatus.FAILED,
phase="failed",
progress=100,
error_code=exc.code,
error_detail=exc.detail,
message="Repository source acquisition failed",
)
except (ValueError, PermissionError, OSError) as exc:
self.store.update_job(
job.id,
status=ScanJobStatus.FAILED,
phase="failed",
progress=100,
error_code="scan_failed",
error_detail=str(exc) if isinstance(exc, ValueError) else "Repository analysis failed",
message="Repository analysis failed",
)
except Exception:
LOGGER.exception("RepoPulse scan job failed without exposing job details")
self.store.update_job(
job.id,
status=ScanJobStatus.FAILED,
phase="failed",
progress=100,
error_code="internal_error",
error_detail="Unexpected repository analysis failure",
message="Repository analysis failed",
)
finally:
lease_heartbeat.stop()
if acquired is not None and repository is not None:
try:
self.acquirer.cleanup(acquired, repository)
except AcquisitionError:
LOGGER.warning("Unable to clean one RepoPulse worktree")