Public changes compared with the previous published export: 26 files changed. Affected areas: - backend/app: 11 files - dependencies: 5 files - deployment and CI: 3 files - documentation: 2 files - tests: 5 files Changed files: - Updated: "Dockerfile.unraid" - Updated: "README.md" - Updated: "backend/Dockerfile" - Updated: "backend/app/acquisition.py" - Updated: "backend/app/architecture.py" - Updated: "backend/app/git_analysis.py" - Updated: "backend/app/gitea.py" - Updated: "backend/app/jobs.py" - Updated: "backend/app/main.py" - Added: "backend/app/processes.py" - Updated: "backend/app/rules.py" - Updated: "backend/app/scanner.py" - Updated: "backend/app/signals.py" - Updated: "backend/app/store.py" - Updated: "backend/pyproject.toml" - Added: "backend/tests/test_api_responsiveness.py" - Added: "backend/tests/test_processes.py" - Updated: "backend/tests/test_scanner.py" - Updated: "backend/tests/test_signals_v11.py" - Updated: "backend/tests/test_store.py" - Updated: "docker-compose.yml" - Updated: "docs/ARCHITECTURE.md" - Updated: "frontend/package-lock.json" - Updated: "frontend/package.json" - Updated: "package-lock.json" - Updated: "package.json" Export provenance/manifests refreshed; excluded from the change summary. Publish curated RepoPulse source from a251f826238908b156485299061da76482780ed2
406 lines
15 KiB
Python
406 lines
15 KiB
Python
from __future__ import annotations
|
|
|
|
import ctypes
|
|
import ctypes.util
|
|
import gc
|
|
import logging
|
|
import os
|
|
import sys
|
|
from collections.abc import Callable
|
|
from datetime import UTC, datetime
|
|
from threading import Event, Lock, Thread, current_thread
|
|
from time import monotonic
|
|
|
|
from .acquisition import AcquisitionError, GitAcquirer
|
|
from .gitea import GiteaClient
|
|
from .models import (
|
|
Repository,
|
|
RepositorySyncResult,
|
|
ScanJob,
|
|
ScanJobStatus,
|
|
ScanSource,
|
|
ScanTrigger,
|
|
SourceOrigin,
|
|
)
|
|
from .scanner import scan_repository
|
|
from .store import MaintenanceModeError, QueueFullError, ScanStore, job_lease_seconds
|
|
|
|
LOGGER = logging.getLogger(__name__)
|
|
WORKER_HEARTBEAT_SECONDS = 5.0
|
|
WORKER_IDLE_POLL_SECONDS = 5.0
|
|
|
|
|
|
def _load_malloc_trim() -> Callable[[int], int] | None:
|
|
if not sys.platform.startswith("linux"):
|
|
return None
|
|
try:
|
|
libc = ctypes.CDLL(ctypes.util.find_library("c") or "libc.so.6")
|
|
trim = libc.malloc_trim
|
|
except (AttributeError, OSError):
|
|
return None # e.g. musl libc has no malloc_trim
|
|
trim.argtypes = [ctypes.c_size_t]
|
|
trim.restype = ctypes.c_int
|
|
return trim
|
|
|
|
|
|
_MALLOC_TRIM = _load_malloc_trim()
|
|
|
|
|
|
def release_memory() -> None:
|
|
"""Return memory freed by a finished scan to the OS.
|
|
|
|
A scan allocates many short-lived objects (ASTs, file texts, findings). CPython frees
|
|
them, but glibc keeps the arenas mapped, so the long-running worker's RSS would stay
|
|
at its largest scan. A collection plus malloc_trim hands those pages back.
|
|
"""
|
|
|
|
gc.collect()
|
|
if _MALLOC_TRIM is not None:
|
|
try:
|
|
_MALLOC_TRIM(0)
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
def _positive_int(name: str, default: int, *, maximum: int | None = None) -> int:
|
|
try:
|
|
value = max(1, int(os.getenv(name, str(default))))
|
|
except ValueError:
|
|
value = default
|
|
return min(value, maximum) if maximum else value
|
|
|
|
|
|
def _positive_float(name: str, default: float) -> float:
|
|
try:
|
|
return max(0.2, float(os.getenv(name, str(default))))
|
|
except ValueError:
|
|
return default
|
|
|
|
|
|
class ScanJobRunner:
|
|
"""Single-process durable worker and bounded Gitea scheduler."""
|
|
|
|
def __init__(
|
|
self,
|
|
store: ScanStore,
|
|
*,
|
|
gitea: GiteaClient | None = None,
|
|
acquirer: GitAcquirer | None = None,
|
|
poll_seconds: float = 0.5,
|
|
sync_interval_seconds: float | None = None,
|
|
max_queue: int | None = None,
|
|
) -> None:
|
|
self.store = store
|
|
self.gitea = gitea or GiteaClient()
|
|
self.acquirer = acquirer or GitAcquirer()
|
|
self.poll_seconds = max(0.2, poll_seconds)
|
|
self.sync_interval_seconds = sync_interval_seconds or _positive_float(
|
|
"REPOPULSE_GITEA_SYNC_INTERVAL_SECONDS", 300.0
|
|
)
|
|
self.max_queue = max_queue or _positive_int("REPOPULSE_MAX_QUEUED_SCANS", 100, maximum=1000)
|
|
self._stop = Event()
|
|
self._wake = Event()
|
|
self._idle_iterations = 0
|
|
self.idle_poll_seconds = max(
|
|
self.poll_seconds, _positive_float("REPOPULSE_WORKER_IDLE_POLL_SECONDS", WORKER_IDLE_POLL_SECONDS)
|
|
)
|
|
self._thread: Thread | None = None
|
|
self._last_sync = float("-inf")
|
|
self._last_heartbeat = float("-inf")
|
|
self.heartbeat_interval_seconds = _positive_float(
|
|
"REPOPULSE_WORKER_HEARTBEAT_SECONDS", WORKER_HEARTBEAT_SECONDS
|
|
)
|
|
self._sync_lock = Lock()
|
|
|
|
@property
|
|
def started(self) -> bool:
|
|
return self._thread is not None and self._thread.is_alive()
|
|
|
|
def start(self) -> None:
|
|
if self.started:
|
|
return
|
|
self._stop.clear()
|
|
self._wake.clear()
|
|
self._thread = Thread(target=self._run, name="repopulse-scan-worker", daemon=True)
|
|
self._thread.start()
|
|
|
|
def stop(self) -> None:
|
|
self._stop.set()
|
|
self._wake.set()
|
|
thread = self._thread
|
|
if thread and thread is not current_thread():
|
|
thread.join(timeout=5)
|
|
if thread is None or not thread.is_alive():
|
|
self._thread = None
|
|
|
|
def _run(self) -> None:
|
|
while not self._stop.is_set():
|
|
try:
|
|
job = self._run_iteration()
|
|
except MaintenanceModeError:
|
|
job = None
|
|
except Exception:
|
|
self._last_sync = monotonic()
|
|
LOGGER.exception("RepoPulse worker loop recovered from an unexpected failure")
|
|
job = None
|
|
if job is None:
|
|
self._idle_iterations += 1
|
|
self._wake.wait(self._idle_delay())
|
|
self._wake.clear()
|
|
else:
|
|
self._idle_iterations = 0
|
|
|
|
def _idle_delay(self) -> float:
|
|
"""Back off exponentially while idle; in-process enqueues wake the loop at once."""
|
|
|
|
exponent = min(self._idle_iterations - 1, 16)
|
|
return min(self.idle_poll_seconds, self.poll_seconds * (2**exponent))
|
|
|
|
def wake(self) -> None:
|
|
self._wake.set()
|
|
|
|
def _run_iteration(self) -> ScanJob | None:
|
|
"""Perform one scheduler pass unless durable maintenance has paused work."""
|
|
|
|
if self.store.maintenance_status()["enabled"]:
|
|
return None
|
|
# The heartbeat is a liveness signal, not a lock: claims and enqueues re-check
|
|
# maintenance inside their own transactions, so writing it on every 0.5 s poll
|
|
# only churns the WAL. Throttle the write to the heartbeat interval.
|
|
now = monotonic()
|
|
if now - self._last_heartbeat >= self.heartbeat_interval_seconds:
|
|
if not self.store.touch_worker_heartbeat_if_active(datetime.now(UTC).isoformat()):
|
|
return None
|
|
self._last_heartbeat = now
|
|
if self.gitea.configured and monotonic() - self._last_sync >= self.sync_interval_seconds:
|
|
self.sync_now()
|
|
# Synchronization can take a while; re-check before claiming new work.
|
|
if self.store.maintenance_status()["enabled"]:
|
|
return None
|
|
return self.process_one()
|
|
|
|
def sync_now(self) -> RepositorySyncResult:
|
|
with self._sync_lock:
|
|
if self.store.maintenance_status()["enabled"]:
|
|
raise MaintenanceModeError("RepoPulse is in maintenance mode")
|
|
if not self.gitea.configured:
|
|
result = RepositorySyncResult(
|
|
provider="gitea",
|
|
errors=["not_configured: Gitea connector is not configured"],
|
|
synced_at=datetime.now(UTC),
|
|
)
|
|
self.store.record_sync_result(result)
|
|
self._last_sync = monotonic()
|
|
return result
|
|
try:
|
|
result = self.gitea.sync(self.store)
|
|
except Exception:
|
|
self._last_sync = monotonic()
|
|
raise
|
|
self._last_sync = monotonic()
|
|
if not result.errors:
|
|
self._schedule_changed_repositories()
|
|
return result
|
|
|
|
def _schedule_changed_repositories(self) -> None:
|
|
if self.store.maintenance_status()["enabled"]:
|
|
return
|
|
# No silent catalog cap: every repository is reconciled, however large the catalog.
|
|
for summary in self.store.list_repositories(limit=-1):
|
|
if summary.provider != "gitea" or summary.archived or not summary.latest_revision:
|
|
continue
|
|
if not summary.policy.auto_scan:
|
|
continue
|
|
if summary.last_scan_revision and summary.last_scan_revision == summary.latest_revision:
|
|
continue
|
|
try:
|
|
self.store.enqueue_job(
|
|
summary.id,
|
|
summary.latest_revision,
|
|
ScanTrigger.SCHEDULED,
|
|
max_queue=self.max_queue,
|
|
)
|
|
except (MaintenanceModeError, QueueFullError, KeyError):
|
|
continue
|
|
|
|
def set_maintenance_mode(self, enabled: bool) -> dict[str, object]:
|
|
"""Serialize maintenance changes with catalog reconciliation and scan scheduling."""
|
|
|
|
with self._sync_lock:
|
|
return self.store.set_maintenance_mode(enabled)
|
|
|
|
def enqueue(
|
|
self,
|
|
repository: Repository,
|
|
*,
|
|
force: bool = False,
|
|
trigger: ScanTrigger = ScanTrigger.MANUAL,
|
|
) -> ScanJob:
|
|
requested_revision = repository.latest_revision
|
|
job = self.store.enqueue_job(
|
|
repository.id,
|
|
requested_revision,
|
|
trigger,
|
|
force=force,
|
|
max_queue=self.max_queue,
|
|
)
|
|
self.wake()
|
|
return job
|
|
|
|
def process_one(self) -> ScanJob | None:
|
|
job = self.store.claim_next_job()
|
|
if job is None:
|
|
return None
|
|
try:
|
|
self._process(job)
|
|
except Exception:
|
|
LOGGER.exception("RepoPulse scan job failed outside the isolated job handler")
|
|
try:
|
|
self.store.update_job(
|
|
job.id,
|
|
status=ScanJobStatus.FAILED,
|
|
phase="failed",
|
|
progress=100,
|
|
error_code="internal_error",
|
|
error_detail="Unexpected repository analysis failure",
|
|
message="Repository analysis failed",
|
|
)
|
|
except Exception:
|
|
LOGGER.exception("RepoPulse could not persist an unexpected job failure")
|
|
finally:
|
|
release_memory()
|
|
return self.store.get_job(job.id)
|
|
|
|
class _LeaseHeartbeat:
|
|
def __init__(self, store: ScanStore, job_id: str) -> None:
|
|
self.store = store
|
|
self.job_id = job_id
|
|
self.stop_event = Event()
|
|
self.thread = Thread(target=self._run, name=f"repopulse-lease-{job_id[:8]}", daemon=True)
|
|
|
|
def start(self) -> None:
|
|
self.thread.start()
|
|
|
|
def stop(self) -> None:
|
|
self.stop_event.set()
|
|
self.thread.join(timeout=2)
|
|
|
|
def _run(self) -> None:
|
|
interval = max(1.0, job_lease_seconds() / 3)
|
|
while not self.stop_event.wait(interval):
|
|
try:
|
|
self.store.renew_job_lease(self.job_id)
|
|
except Exception:
|
|
LOGGER.warning("RepoPulse could not renew a scan-job lease", exc_info=True)
|
|
|
|
def _process(self, job: ScanJob) -> None:
|
|
repository = self.store.get_repository(job.repository_id)
|
|
if repository is None:
|
|
self.store.update_job(
|
|
job.id,
|
|
status=ScanJobStatus.FAILED,
|
|
phase="failed",
|
|
progress=100,
|
|
error_code="repository_not_found",
|
|
error_detail="Repository catalog entry no longer exists",
|
|
)
|
|
return
|
|
source_path = None
|
|
acquired = None
|
|
lease_heartbeat = self._LeaseHeartbeat(self.store, job.id)
|
|
lease_heartbeat.start()
|
|
try:
|
|
if repository.provider != "gitea":
|
|
source_path = self.store.get_repository_source_path(repository.id)
|
|
if not source_path:
|
|
raise AcquisitionError("source_unavailable", "Local repository source is unavailable")
|
|
actual_revision = job.requested_revision
|
|
branch = repository.default_branch
|
|
else:
|
|
acquired = self.acquirer.acquire(repository, job.id, job.requested_revision)
|
|
source_path = str(acquired.path)
|
|
actual_revision = acquired.actual_revision
|
|
branch = acquired.branch
|
|
self.store.update_job(
|
|
job.id,
|
|
status=ScanJobStatus.SCANNING,
|
|
phase="scanning",
|
|
progress=30,
|
|
actual_revision=actual_revision,
|
|
message="Running deterministic repository analysis",
|
|
)
|
|
source = ScanSource(
|
|
origin=SourceOrigin.GITEA if repository.provider == "gitea" else SourceOrigin.LOCAL,
|
|
provider=repository.provider,
|
|
repository_id=repository.id,
|
|
repository_full_name=repository.full_name,
|
|
branch=branch,
|
|
requested_revision=job.requested_revision,
|
|
actual_revision=actual_revision,
|
|
trigger=job.trigger,
|
|
)
|
|
report = scan_repository(source_path, source=source, display_name=repository.name)
|
|
self.store.update_job(
|
|
job.id,
|
|
status=ScanJobStatus.ANALYZING,
|
|
phase="analyzing",
|
|
progress=80,
|
|
message="Persisting report and comparison baseline",
|
|
)
|
|
self.store.update_job(
|
|
job.id,
|
|
status=ScanJobStatus.PERSISTING,
|
|
phase="persisting",
|
|
progress=90,
|
|
message="Saving report and revision history",
|
|
)
|
|
saved = self.store.save(report)
|
|
self.store.upsert_repository(repository.model_copy(update={"policy": saved.policy}))
|
|
self.store.update_job(
|
|
job.id,
|
|
status=ScanJobStatus.COMPLETED,
|
|
phase="completed",
|
|
progress=100,
|
|
actual_revision=saved.source.actual_revision,
|
|
scan_id=saved.id,
|
|
message="Scan completed",
|
|
)
|
|
except AcquisitionError as exc:
|
|
self.store.update_job(
|
|
job.id,
|
|
status=ScanJobStatus.FAILED,
|
|
phase="failed",
|
|
progress=100,
|
|
error_code=exc.code,
|
|
error_detail=exc.detail,
|
|
message="Repository source acquisition failed",
|
|
)
|
|
except (ValueError, PermissionError, OSError) as exc:
|
|
self.store.update_job(
|
|
job.id,
|
|
status=ScanJobStatus.FAILED,
|
|
phase="failed",
|
|
progress=100,
|
|
error_code="scan_failed",
|
|
error_detail=str(exc) if isinstance(exc, ValueError) else "Repository analysis failed",
|
|
message="Repository analysis failed",
|
|
)
|
|
except Exception:
|
|
LOGGER.exception("RepoPulse scan job failed without exposing job details")
|
|
self.store.update_job(
|
|
job.id,
|
|
status=ScanJobStatus.FAILED,
|
|
phase="failed",
|
|
progress=100,
|
|
error_code="internal_error",
|
|
error_detail="Unexpected repository analysis failure",
|
|
message="Repository analysis failed",
|
|
)
|
|
finally:
|
|
lease_heartbeat.stop()
|
|
if acquired is not None and repository is not None:
|
|
try:
|
|
self.acquirer.cleanup(acquired, repository)
|
|
except AcquisitionError:
|
|
LOGGER.warning("Unable to clean one RepoPulse worktree")
|