RepoPulse: update backend/app and dependencies (+3 areas)
Public changes compared with the previous published export: 26 files changed. Affected areas: - backend/app: 11 files - dependencies: 5 files - deployment and CI: 3 files - documentation: 2 files - tests: 5 files Changed files: - Updated: "Dockerfile.unraid" - Updated: "README.md" - Updated: "backend/Dockerfile" - Updated: "backend/app/acquisition.py" - Updated: "backend/app/architecture.py" - Updated: "backend/app/git_analysis.py" - Updated: "backend/app/gitea.py" - Updated: "backend/app/jobs.py" - Updated: "backend/app/main.py" - Added: "backend/app/processes.py" - Updated: "backend/app/rules.py" - Updated: "backend/app/scanner.py" - Updated: "backend/app/signals.py" - Updated: "backend/app/store.py" - Updated: "backend/pyproject.toml" - Added: "backend/tests/test_api_responsiveness.py" - Added: "backend/tests/test_processes.py" - Updated: "backend/tests/test_scanner.py" - Updated: "backend/tests/test_signals_v11.py" - Updated: "backend/tests/test_store.py" - Updated: "docker-compose.yml" - Updated: "docs/ARCHITECTURE.md" - Updated: "frontend/package-lock.json" - Updated: "frontend/package.json" - Updated: "package-lock.json" - Updated: "package.json" Export provenance/manifests refreshed; excluded from the change summary. Publish curated RepoPulse source from a251f826238908b156485299061da76482780ed2
This commit is contained in:
1 parent
23a4a721f1
commit
61ae49f9d0
28 files changed
+1083
-408
No files matched your search
+82
-7
@@ -1,7 +1,12 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import ctypes
|
||||
import ctypes.util
|
||||
import gc
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime
|
||||
from threading import Event, Lock, Thread, current_thread
|
||||
from time import monotonic
|
||||
@@ -21,6 +26,40 @@ from .scanner import scan_repository
|
||||
from .store import MaintenanceModeError, QueueFullError, ScanStore, job_lease_seconds
|
||||
|
||||
LOGGER = logging.getLogger(__name__)
|
||||
WORKER_HEARTBEAT_SECONDS = 5.0
|
||||
WORKER_IDLE_POLL_SECONDS = 5.0
|
||||
|
||||
|
||||
def _load_malloc_trim() -> Callable[[int], int] | None:
|
||||
if not sys.platform.startswith("linux"):
|
||||
return None
|
||||
try:
|
||||
libc = ctypes.CDLL(ctypes.util.find_library("c") or "libc.so.6")
|
||||
trim = libc.malloc_trim
|
||||
except (AttributeError, OSError):
|
||||
return None # e.g. musl libc has no malloc_trim
|
||||
trim.argtypes = [ctypes.c_size_t]
|
||||
trim.restype = ctypes.c_int
|
||||
return trim
|
||||
|
||||
|
||||
_MALLOC_TRIM = _load_malloc_trim()
|
||||
|
||||
|
||||
def release_memory() -> None:
|
||||
"""Return memory freed by a finished scan to the OS.
|
||||
|
||||
A scan allocates many short-lived objects (ASTs, file texts, findings). CPython frees
|
||||
them, but glibc keeps the arenas mapped, so the long-running worker's RSS would stay
|
||||
at its largest scan. A collection plus malloc_trim hands those pages back.
|
||||
"""
|
||||
|
||||
gc.collect()
|
||||
if _MALLOC_TRIM is not None:
|
||||
try:
|
||||
_MALLOC_TRIM(0)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def _positive_int(name: str, default: int, *, maximum: int | None = None) -> int:
|
||||
@@ -60,8 +99,17 @@ class ScanJobRunner:
|
||||
)
|
||||
self.max_queue = max_queue or _positive_int("REPOPULSE_MAX_QUEUED_SCANS", 100, maximum=1000)
|
||||
self._stop = Event()
|
||||
self._wake = Event()
|
||||
self._idle_iterations = 0
|
||||
self.idle_poll_seconds = max(
|
||||
self.poll_seconds, _positive_float("REPOPULSE_WORKER_IDLE_POLL_SECONDS", WORKER_IDLE_POLL_SECONDS)
|
||||
)
|
||||
self._thread: Thread | None = None
|
||||
self._last_sync = float("-inf")
|
||||
self._last_heartbeat = float("-inf")
|
||||
self.heartbeat_interval_seconds = _positive_float(
|
||||
"REPOPULSE_WORKER_HEARTBEAT_SECONDS", WORKER_HEARTBEAT_SECONDS
|
||||
)
|
||||
self._sync_lock = Lock()
|
||||
|
||||
@property
|
||||
@@ -72,11 +120,13 @@ class ScanJobRunner:
|
||||
if self.started:
|
||||
return
|
||||
self._stop.clear()
|
||||
self._wake.clear()
|
||||
self._thread = Thread(target=self._run, name="repopulse-scan-worker", daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
def stop(self) -> None:
|
||||
self._stop.set()
|
||||
self._wake.set()
|
||||
thread = self._thread
|
||||
if thread and thread is not current_thread():
|
||||
thread.join(timeout=5)
|
||||
@@ -94,19 +144,39 @@ class ScanJobRunner:
|
||||
LOGGER.exception("RepoPulse worker loop recovered from an unexpected failure")
|
||||
job = None
|
||||
if job is None:
|
||||
self._stop.wait(self.poll_seconds)
|
||||
self._idle_iterations += 1
|
||||
self._wake.wait(self._idle_delay())
|
||||
self._wake.clear()
|
||||
else:
|
||||
self._idle_iterations = 0
|
||||
|
||||
def _idle_delay(self) -> float:
|
||||
"""Back off exponentially while idle; in-process enqueues wake the loop at once."""
|
||||
|
||||
exponent = min(self._idle_iterations - 1, 16)
|
||||
return min(self.idle_poll_seconds, self.poll_seconds * (2**exponent))
|
||||
|
||||
def wake(self) -> None:
|
||||
self._wake.set()
|
||||
|
||||
def _run_iteration(self) -> ScanJob | None:
|
||||
"""Perform one scheduler pass unless durable maintenance has paused work."""
|
||||
|
||||
if self.store.maintenance_status()["enabled"]:
|
||||
return None
|
||||
if not self.store.touch_worker_heartbeat_if_active(datetime.now(UTC).isoformat()):
|
||||
return None
|
||||
# The heartbeat is a liveness signal, not a lock: claims and enqueues re-check
|
||||
# maintenance inside their own transactions, so writing it on every 0.5 s poll
|
||||
# only churns the WAL. Throttle the write to the heartbeat interval.
|
||||
now = monotonic()
|
||||
if now - self._last_heartbeat >= self.heartbeat_interval_seconds:
|
||||
if not self.store.touch_worker_heartbeat_if_active(datetime.now(UTC).isoformat()):
|
||||
return None
|
||||
self._last_heartbeat = now
|
||||
if self.gitea.configured and monotonic() - self._last_sync >= self.sync_interval_seconds:
|
||||
self.sync_now()
|
||||
if self.store.maintenance_status()["enabled"]:
|
||||
return None
|
||||
# Synchronization can take a while; re-check before claiming new work.
|
||||
if self.store.maintenance_status()["enabled"]:
|
||||
return None
|
||||
return self.process_one()
|
||||
|
||||
def sync_now(self) -> RepositorySyncResult:
|
||||
@@ -135,7 +205,8 @@ class ScanJobRunner:
|
||||
def _schedule_changed_repositories(self) -> None:
|
||||
if self.store.maintenance_status()["enabled"]:
|
||||
return
|
||||
for summary in self.store.list_repositories(limit=1000):
|
||||
# No silent catalog cap: every repository is reconciled, however large the catalog.
|
||||
for summary in self.store.list_repositories(limit=-1):
|
||||
if summary.provider != "gitea" or summary.archived or not summary.latest_revision:
|
||||
continue
|
||||
if not summary.policy.auto_scan:
|
||||
@@ -166,13 +237,15 @@ class ScanJobRunner:
|
||||
trigger: ScanTrigger = ScanTrigger.MANUAL,
|
||||
) -> ScanJob:
|
||||
requested_revision = repository.latest_revision
|
||||
return self.store.enqueue_job(
|
||||
job = self.store.enqueue_job(
|
||||
repository.id,
|
||||
requested_revision,
|
||||
trigger,
|
||||
force=force,
|
||||
max_queue=self.max_queue,
|
||||
)
|
||||
self.wake()
|
||||
return job
|
||||
|
||||
def process_one(self) -> ScanJob | None:
|
||||
job = self.store.claim_next_job()
|
||||
@@ -194,6 +267,8 @@ class ScanJobRunner:
|
||||
)
|
||||
except Exception:
|
||||
LOGGER.exception("RepoPulse could not persist an unexpected job failure")
|
||||
finally:
|
||||
release_memory()
|
||||
return self.store.get_job(job.id)
|
||||
|
||||
class _LeaseHeartbeat:
|
||||
|
||||
Reference in new issue
Block a user