RepoPulse: update backend/app and dependencies (+3 areas)

Public changes compared with the previous published export:
26 files changed.

Affected areas:
- backend/app: 11 files
- dependencies: 5 files
- deployment and CI: 3 files
- documentation: 2 files
- tests: 5 files

Changed files:
- Updated: "Dockerfile.unraid"
- Updated: "README.md"
- Updated: "backend/Dockerfile"
- Updated: "backend/app/acquisition.py"
- Updated: "backend/app/architecture.py"
- Updated: "backend/app/git_analysis.py"
- Updated: "backend/app/gitea.py"
- Updated: "backend/app/jobs.py"
- Updated: "backend/app/main.py"
- Added: "backend/app/processes.py"
- Updated: "backend/app/rules.py"
- Updated: "backend/app/scanner.py"
- Updated: "backend/app/signals.py"
- Updated: "backend/app/store.py"
- Updated: "backend/pyproject.toml"
- Added: "backend/tests/test_api_responsiveness.py"
- Added: "backend/tests/test_processes.py"
- Updated: "backend/tests/test_scanner.py"
- Updated: "backend/tests/test_signals_v11.py"
- Updated: "backend/tests/test_store.py"
- Updated: "docker-compose.yml"
- Updated: "docs/ARCHITECTURE.md"
- Updated: "frontend/package-lock.json"
- Updated: "frontend/package.json"
- Updated: "package-lock.json"
- Updated: "package.json"

Export provenance/manifests refreshed; excluded from the change summary.

Publish curated RepoPulse source from a251f826238908b156485299061da76482780ed2
This commit is contained in:
Public Source Publisher committed 2026-11-17 21:10:30 +00:00
1 parent 23a4a721f1
commit 61ae49f9d0
28 files changed
+1083 -408

No files matched your search

+82 -7
View File
@@ -1,7 +1,12 @@
from __future__ import annotations
import ctypes
import ctypes.util
import gc
import logging
import os
import sys
from collections.abc import Callable
from datetime import UTC, datetime
from threading import Event, Lock, Thread, current_thread
from time import monotonic
@@ -21,6 +26,40 @@ from .scanner import scan_repository
from .store import MaintenanceModeError, QueueFullError, ScanStore, job_lease_seconds
LOGGER = logging.getLogger(__name__)
WORKER_HEARTBEAT_SECONDS = 5.0
WORKER_IDLE_POLL_SECONDS = 5.0
def _load_malloc_trim() -> Callable[[int], int] | None:
if not sys.platform.startswith("linux"):
return None
try:
libc = ctypes.CDLL(ctypes.util.find_library("c") or "libc.so.6")
trim = libc.malloc_trim
except (AttributeError, OSError):
return None # e.g. musl libc has no malloc_trim
trim.argtypes = [ctypes.c_size_t]
trim.restype = ctypes.c_int
return trim
_MALLOC_TRIM = _load_malloc_trim()
def release_memory() -> None:
"""Return memory freed by a finished scan to the OS.
A scan allocates many short-lived objects (ASTs, file texts, findings). CPython frees
them, but glibc keeps the arenas mapped, so the long-running worker's RSS would stay
at its largest scan. A collection plus malloc_trim hands those pages back.
"""
gc.collect()
if _MALLOC_TRIM is not None:
try:
_MALLOC_TRIM(0)
except OSError:
pass
def _positive_int(name: str, default: int, *, maximum: int | None = None) -> int:
@@ -60,8 +99,17 @@ class ScanJobRunner:
)
self.max_queue = max_queue or _positive_int("REPOPULSE_MAX_QUEUED_SCANS", 100, maximum=1000)
self._stop = Event()
self._wake = Event()
self._idle_iterations = 0
self.idle_poll_seconds = max(
self.poll_seconds, _positive_float("REPOPULSE_WORKER_IDLE_POLL_SECONDS", WORKER_IDLE_POLL_SECONDS)
)
self._thread: Thread | None = None
self._last_sync = float("-inf")
self._last_heartbeat = float("-inf")
self.heartbeat_interval_seconds = _positive_float(
"REPOPULSE_WORKER_HEARTBEAT_SECONDS", WORKER_HEARTBEAT_SECONDS
)
self._sync_lock = Lock()
@property
@@ -72,11 +120,13 @@ class ScanJobRunner:
if self.started:
return
self._stop.clear()
self._wake.clear()
self._thread = Thread(target=self._run, name="repopulse-scan-worker", daemon=True)
self._thread.start()
def stop(self) -> None:
self._stop.set()
self._wake.set()
thread = self._thread
if thread and thread is not current_thread():
thread.join(timeout=5)
@@ -94,19 +144,39 @@ class ScanJobRunner:
LOGGER.exception("RepoPulse worker loop recovered from an unexpected failure")
job = None
if job is None:
self._stop.wait(self.poll_seconds)
self._idle_iterations += 1
self._wake.wait(self._idle_delay())
self._wake.clear()
else:
self._idle_iterations = 0
def _idle_delay(self) -> float:
"""Back off exponentially while idle; in-process enqueues wake the loop at once."""
exponent = min(self._idle_iterations - 1, 16)
return min(self.idle_poll_seconds, self.poll_seconds * (2**exponent))
def wake(self) -> None:
self._wake.set()
def _run_iteration(self) -> ScanJob | None:
"""Perform one scheduler pass unless durable maintenance has paused work."""
if self.store.maintenance_status()["enabled"]:
return None
if not self.store.touch_worker_heartbeat_if_active(datetime.now(UTC).isoformat()):
return None
# The heartbeat is a liveness signal, not a lock: claims and enqueues re-check
# maintenance inside their own transactions, so writing it on every 0.5 s poll
# only churns the WAL. Throttle the write to the heartbeat interval.
now = monotonic()
if now - self._last_heartbeat >= self.heartbeat_interval_seconds:
if not self.store.touch_worker_heartbeat_if_active(datetime.now(UTC).isoformat()):
return None
self._last_heartbeat = now
if self.gitea.configured and monotonic() - self._last_sync >= self.sync_interval_seconds:
self.sync_now()
if self.store.maintenance_status()["enabled"]:
return None
# Synchronization can take a while; re-check before claiming new work.
if self.store.maintenance_status()["enabled"]:
return None
return self.process_one()
def sync_now(self) -> RepositorySyncResult:
@@ -135,7 +205,8 @@ class ScanJobRunner:
def _schedule_changed_repositories(self) -> None:
if self.store.maintenance_status()["enabled"]:
return
for summary in self.store.list_repositories(limit=1000):
# No silent catalog cap: every repository is reconciled, however large the catalog.
for summary in self.store.list_repositories(limit=-1):
if summary.provider != "gitea" or summary.archived or not summary.latest_revision:
continue
if not summary.policy.auto_scan:
@@ -166,13 +237,15 @@ class ScanJobRunner:
trigger: ScanTrigger = ScanTrigger.MANUAL,
) -> ScanJob:
requested_revision = repository.latest_revision
return self.store.enqueue_job(
job = self.store.enqueue_job(
repository.id,
requested_revision,
trigger,
force=force,
max_queue=self.max_queue,
)
self.wake()
return job
def process_one(self) -> ScanJob | None:
job = self.store.claim_next_job()
@@ -194,6 +267,8 @@ class ScanJobRunner:
)
except Exception:
LOGGER.exception("RepoPulse could not persist an unexpected job failure")
finally:
release_memory()
return self.store.get_job(job.id)
class _LeaseHeartbeat: