Files
RepoPulse-Public/backend/app/main.py
T
Public Source Publisher 131096340e RepoPulse: update "backend/app/main.py" and "backend/tests/test_frontend_routes.py"
Public changes compared with the previous published export:
2 files changed.

Affected areas:
- backend/app: 1 file
- tests: 1 file

Changed files:
- Updated: "backend/app/main.py"
- Added: "backend/tests/test_frontend_routes.py"

Export provenance/manifests refreshed; excluded from the change summary.

Publish curated RepoPulse source from 1201120636c6141d94a4aee81b48b5b001cea045
2026-11-20 05:43:52 +00:00

564 lines
22 KiB
Python

from __future__ import annotations
import asyncio
import hashlib
import hmac
import json
import os
from contextlib import asynccontextmanager
from pathlib import Path
from threading import BoundedSemaphore, Lock
from fastapi import FastAPI, HTTPException, Query, Request, Response, status
from fastapi.concurrency import run_in_threadpool
from fastapi.middleware.gzip import GZipMiddleware
from fastapi.responses import JSONResponse
from fastapi.staticfiles import StaticFiles
from starlette.exceptions import HTTPException as StarletteHTTPException
from starlette.types import Scope
from .acquisition import GitAcquirer
from .gitea import GiteaClient
from .icons import SAFE_REVISION as SAFE_ICON_REVISION
from .icons import ProjectIconResolver
from .jobs import ScanJobRunner
from .models import (
RepositoryDetail,
RepositoryScanRequest,
RepositorySummary,
RepositorySyncResult,
ScanComparison,
ScanJob,
ScanReport,
ScanRequest,
ScanSummary,
ScanTrigger,
TrendPoint,
WebhookResponse,
)
from .reporting import content_disposition, render_html_report, safe_report_filename
from .scanner import scan_repository
from .store import MaintenanceModeError, QueueFullError, ScanStore
VERSION = "1.5.1"
store = ScanStore()
SCAN_ROOT = os.getenv("REPOPULSE_SCAN_ROOT")
def _positive_int_env(name: str, default: int) -> int:
try:
return max(1, int(os.getenv(name, str(default))))
except ValueError:
return default
# The durable queue has one real worker. Keep direct legacy path scans at the
# same capacity rather than advertising concurrency that cannot be delivered.
REQUESTED_MAX_CONCURRENT_SCANS = _positive_int_env("REPOPULSE_MAX_CONCURRENT_SCANS", 1)
MAX_CONCURRENT_SCANS = 1
SCAN_SLOTS = BoundedSemaphore(MAX_CONCURRENT_SCANS)
gitea_client = GiteaClient()
repository_acquirer = GitAcquirer()
project_icons = ProjectIconResolver(repository_acquirer.mirrors_root)
job_runner: ScanJobRunner | None = None
_api_mutation_lock = Lock()
_MAINTENANCE_ENTER_PATH = "/api/v1/operator/maintenance/enter"
_MAINTENANCE_EXIT_PATH = "/api/v1/operator/maintenance/exit"
def _get_runner() -> ScanJobRunner:
global job_runner
if job_runner is None or job_runner.store is not store:
job_runner = ScanJobRunner(
store,
gitea=gitea_client,
acquirer=repository_acquirer,
)
return job_runner
@asynccontextmanager
async def lifespan(_app: FastAPI):
runner = _get_runner()
runner.start()
try:
yield
finally:
runner.stop()
app = FastAPI(title="RepoPulse API", version=VERSION, lifespan=lifespan)
# Catalog, report and bundle payloads are large, repetitive JSON/JS; compress them.
app.add_middleware(GZipMiddleware, minimum_size=1024, compresslevel=5)
@app.middleware("http")
async def maintenance_mutation_gate(request: Request, call_next):
"""Serialize external API writes with maintenance transitions and reject writes while paused."""
if not request.url.path.startswith("/api/v1/") or request.method in {"GET", "HEAD", "OPTIONS"}:
return await call_next(request)
if request.url.path in {_MAINTENANCE_ENTER_PATH, _MAINTENANCE_EXIT_PATH}:
return await call_next(request)
await run_in_threadpool(_api_mutation_lock.acquire)
try:
if store.maintenance_status()["enabled"]:
return JSONResponse(
status_code=status.HTTP_423_LOCKED,
content={"detail": "RepoPulse is in maintenance mode"},
headers={"Retry-After": "30"},
)
return await call_next(request)
finally:
_api_mutation_lock.release()
def _maintenance_snapshot() -> dict[str, object]:
mode = store.maintenance_status()
stats = store.queue_stats()
queued = int(stats["queued"])
active = int(stats["active"])
return {
**mode,
"queue_depth": queued,
"active_jobs": active,
"active_drained": active == 0,
"drained": queued == 0 and active == 0,
}
def _require_mutation_authorization(request: Request, *, strict: bool = False) -> None:
"""Use an operator token when configured; deployment perimeter remains explicit otherwise."""
token = os.getenv("REPOPULSE_OPERATOR_TOKEN", "")
if strict:
token = os.getenv("REPOPULSE_MAINTENANCE_TOKEN", "") or token
if not token:
if strict:
raise HTTPException(status_code=503, detail="Operator token is not configured")
return
supplied = request.headers.get("x-repopulse-token", "")
if not hmac.compare_digest(supplied, token):
raise HTTPException(status_code=401, detail="Operator authorization is required")
def _resolve_scan_path(raw_path: str) -> Path:
supplied = Path(raw_path).expanduser()
if SCAN_ROOT:
root = Path(SCAN_ROOT).expanduser().resolve()
candidate = (root / supplied).resolve() if not supplied.is_absolute() else supplied.resolve()
if candidate != root and root not in candidate.parents:
raise ValueError(f"Path must be inside configured scan root {root}")
return candidate
return supplied.resolve()
@app.get("/health")
def health() -> dict[str, str]:
recovery = store.database_recovery_status()
status = "degraded" if recovery.get("status") == "degraded_recovered" else "ok"
return {"status": status}
@app.get("/api/v1/meta")
def meta() -> dict[str, object]:
stats = store.queue_stats()
repository_states = store.repository_state_counts()
last_sync = store.get_state("last_gitea_sync")
parsed_sync: object = None
if last_sync:
try:
parsed_sync = json.loads(last_sync)
except json.JSONDecodeError:
parsed_sync = last_sync
return {
"name": "RepoPulse",
"version": VERSION,
"stage": "v1.5.1",
"scan_root_restricted": bool(SCAN_ROOT),
"scan_root": SCAN_ROOT,
"max_concurrent_scans": MAX_CONCURRENT_SCANS,
"requested_max_concurrent_scans": REQUESTED_MAX_CONCURRENT_SCANS,
"repository_count": store.repository_count(),
"repository_states": repository_states,
"current_repositories": repository_states.get("current", 0),
"stale_repositories": repository_states.get("stale", 0),
"queue_depth": stats["queued"],
"active_jobs": stats["active"],
"completed_jobs": stats["completed"],
"failed_jobs": stats["failed"],
"worker_heartbeat": stats["worker_heartbeat"],
"gitea": {
"configured": gitea_client.configured,
"url": gitea_client.base_url or None,
"last_sync": parsed_sync,
},
"storage": store.storage_stats(),
"database_recovery": store.database_recovery_status(),
"maintenance": _maintenance_snapshot(),
"mutation_authentication": "operator_token" if os.getenv("REPOPULSE_OPERATOR_TOKEN") else "perimeter_required",
}
@app.get("/api/v1/operator/maintenance")
def get_maintenance_status(request: Request) -> dict[str, object]:
_require_mutation_authorization(request, strict=True)
return _maintenance_snapshot()
@app.post("/api/v1/operator/maintenance/enter")
def enter_maintenance(request: Request) -> dict[str, object]:
_require_mutation_authorization(request, strict=True)
with _api_mutation_lock:
if store.maintenance_status()["enabled"]:
raise HTTPException(status_code=423, detail="RepoPulse is already in maintenance mode")
_get_runner().set_maintenance_mode(True)
return _maintenance_snapshot()
@app.post("/api/v1/operator/maintenance/exit")
def exit_maintenance(request: Request) -> dict[str, object]:
_require_mutation_authorization(request, strict=True)
with _api_mutation_lock:
_get_runner().set_maintenance_mode(False)
return _maintenance_snapshot()
@app.post("/api/v1/scans", response_model=ScanReport, status_code=status.HTTP_201_CREATED)
async def create_scan(payload: ScanRequest, request: Request) -> ScanReport:
_require_mutation_authorization(request)
try:
repository_path = _resolve_scan_path(payload.path)
except ValueError as exc:
raise HTTPException(status_code=422, detail=str(exc)) from exc
if not SCAN_SLOTS.acquire(blocking=False):
raise HTTPException(
status_code=429,
detail="Repository scan capacity is currently in use; retry after the active scan completes",
headers={"Retry-After": "2"},
)
try:
report = await asyncio.to_thread(scan_repository, str(repository_path))
except ValueError as exc:
raise HTTPException(status_code=422, detail=str(exc)) from exc
except PermissionError as exc:
raise HTTPException(status_code=403, detail="Repository path cannot be read") from exc
except OSError as exc:
raise HTTPException(status_code=422, detail="Repository path cannot be scanned") from exc
finally:
SCAN_SLOTS.release()
# SQLite writes may wait on busy_timeout; never block the event loop with them.
return await asyncio.to_thread(store.save, report)
@app.get("/api/v1/repositories", response_model=list[RepositorySummary])
def list_repositories(
limit: int = Query(default=100, ge=1, le=1000),
offset: int = Query(default=0, ge=0, le=100_000),
) -> list[RepositorySummary]:
return store.list_repositories(limit, offset)
@app.post("/api/v1/repositories/sync", response_model=RepositorySyncResult)
async def sync_repositories(request: Request) -> RepositorySyncResult:
_require_mutation_authorization(request)
if not gitea_client.configured:
raise HTTPException(status_code=503, detail="Gitea connector is not configured")
try:
result = await asyncio.to_thread(_get_runner().sync_now)
except MaintenanceModeError as exc:
raise HTTPException(status_code=423, detail=str(exc), headers={"Retry-After": "30"}) from exc
if result.errors and result.branch_failures == 0:
raise HTTPException(status_code=502, detail=result.errors[0])
return result
@app.get("/api/v1/repositories/{repository_id}", response_model=RepositoryDetail)
def get_repository(repository_id: str) -> RepositoryDetail:
repository = store.get_repository_detail(repository_id)
if repository is None:
raise HTTPException(status_code=404, detail="Repository not found")
return repository
ICON_HEADERS = {
"Cache-Control": "public, max-age=86400",
"X-Content-Type-Options": "nosniff",
# Repository-supplied SVG is untrusted: never allow script, network or embedding.
"Content-Security-Policy": "default-src 'none'; style-src 'unsafe-inline'; img-src data:; sandbox",
"Cross-Origin-Resource-Policy": "same-origin",
}
@app.get("/api/v1/repositories/{repository_id}/icon")
def repository_icon(repository_id: str, rev: str | None = Query(default=None, max_length=64)) -> Response:
"""Serve the project's own favicon or logo from its mirror at the retained revision."""
repository = store.get_repository(repository_id)
if repository is None:
raise HTTPException(status_code=404, detail="Repository not found")
known = {value.casefold() for value in (repository.last_scan_revision, repository.latest_revision) if value}
revision = rev if rev and SAFE_ICON_REVISION.fullmatch(rev) and rev.casefold() in known else repository.last_scan_revision or repository.latest_revision
icon = project_icons.resolve(repository_acquirer.mirror_path(repository), revision) if revision else None
if icon is None:
return Response(status_code=status.HTTP_404_NOT_FOUND, headers={"Cache-Control": "public, max-age=3600"})
return Response(icon.content, media_type=icon.media_type, headers=ICON_HEADERS)
@app.get("/api/v1/repositories/{repository_id}/scans", response_model=list[ScanSummary])
def list_repository_scans(
repository_id: str,
limit: int = Query(default=50, ge=1, le=200),
) -> list[ScanSummary]:
if store.get_repository(repository_id) is None:
raise HTTPException(status_code=404, detail="Repository not found")
return store.repository_scans(repository_id, limit)
@app.get("/api/v1/repositories/{repository_id}/trend", response_model=list[TrendPoint])
def repository_trend(
repository_id: str,
limit: int = Query(default=50, ge=1, le=200),
) -> list[TrendPoint]:
if store.get_repository(repository_id) is None:
raise HTTPException(status_code=404, detail="Repository not found")
return store.trend(repository_id, limit)
@app.post("/api/v1/repositories/{repository_id}/scans", response_model=ScanJob, status_code=202)
def create_repository_scan(
request: Request,
repository_id: str,
payload: RepositoryScanRequest | None = None,
) -> ScanJob:
_require_mutation_authorization(request)
repository = store.get_repository(repository_id)
if repository is None:
raise HTTPException(status_code=404, detail="Repository not found")
if repository.archived:
raise HTTPException(status_code=409, detail="Archived repositories are not scanned")
if not repository.available:
raise HTTPException(status_code=409, detail="Repository is no longer available to the Gitea connector")
if repository.provider == "gitea" and not gitea_client.configured:
raise HTTPException(status_code=503, detail="Gitea connector is not configured")
try:
return _get_runner().enqueue(repository, force=bool(payload and payload.force))
except MaintenanceModeError as exc:
raise HTTPException(status_code=423, detail=str(exc), headers={"Retry-After": "30"}) from exc
except QueueFullError as exc:
raise HTTPException(
status_code=429,
detail=str(exc),
headers={"Retry-After": "5"},
) from exc
@app.get("/api/v1/scan-jobs/{job_id}", response_model=ScanJob)
def get_scan_job(job_id: str) -> ScanJob:
job = store.get_job(job_id)
if job is None:
raise HTTPException(status_code=404, detail="Scan job not found")
return job
@app.get("/api/v1/repositories/{repository_id}/scans/{scan_id}", response_model=ScanReport)
def get_repository_scan(repository_id: str, scan_id: str) -> ScanReport:
report = store.get(scan_id)
if report is None or report.source.repository_id != repository_id:
raise HTTPException(status_code=404, detail="Repository scan not found")
return report
@app.post("/api/v1/webhooks/gitea", response_model=WebhookResponse)
async def gitea_webhook(request: Request) -> WebhookResponse:
secret = os.getenv("REPOPULSE_GITEA_WEBHOOK_SECRET", "")
if not secret:
raise HTTPException(status_code=404, detail="Gitea webhook integration is not configured")
body = await request.body()
if len(body) > 2_000_000:
raise HTTPException(status_code=413, detail="Webhook payload is too large")
signature = request.headers.get("x-gitea-signature", "").removeprefix("sha256=")
expected = hmac.new(secret.encode("utf-8"), body, hashlib.sha256).hexdigest()
if not signature or not hmac.compare_digest(signature.casefold(), expected.casefold()):
raise HTTPException(status_code=401, detail="Invalid webhook signature")
try:
payload = json.loads(body.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise HTTPException(status_code=400, detail="Webhook payload is invalid JSON") from exc
if not isinstance(payload, dict):
raise HTTPException(status_code=400, detail="Webhook payload must be an object")
event = request.headers.get("x-gitea-event", "push").casefold()
# Catalog reads and the enqueue transaction are blocking SQLite work.
return await run_in_threadpool(_handle_gitea_webhook, payload, event)
def _handle_gitea_webhook(payload: dict, event: str) -> WebhookResponse:
repository_payload = payload.get("repository")
if not isinstance(repository_payload, dict):
raise HTTPException(status_code=400, detail="Webhook repository is missing")
raw_id = repository_payload.get("id")
repository_id = f"gitea:{raw_id}" if str(raw_id).isdigit() else None
repository = store.get_repository(repository_id) if repository_id else None
if repository is None:
return WebhookResponse(
accepted=False,
message="Repository is not present in the synchronized catalog",
)
if event != "push":
return WebhookResponse(
accepted=True,
repository_id=repository.id,
message="Non-push webhook acknowledged without scheduling a scan",
)
ref = payload.get("ref")
if ref != f"refs/heads/{repository.default_branch}":
return WebhookResponse(
accepted=True,
repository_id=repository.id,
message="Non-default branch push acknowledged without scheduling a scan",
)
after = payload.get("after")
if (
not isinstance(after, str)
or not 7 <= len(after) <= 64
or not all(char in "0123456789abcdefABCDEF" for char in after)
or not set(after) - {"0"}
):
return WebhookResponse(
accepted=True,
repository_id=repository.id,
message="Push acknowledged without a usable commit revision",
)
if not gitea_client.configured:
raise HTTPException(status_code=503, detail="Gitea connector is not configured")
updated = repository.model_copy(update={"latest_revision": after, "sync_error": None})
store.upsert_repository(updated)
try:
job = _get_runner().enqueue(updated, trigger=ScanTrigger.WEBHOOK)
except MaintenanceModeError as exc:
raise HTTPException(status_code=423, detail=str(exc), headers={"Retry-After": "30"}) from exc
except QueueFullError as exc:
raise HTTPException(status_code=429, detail=str(exc), headers={"Retry-After": "5"}) from exc
return WebhookResponse(
accepted=True,
repository_id=repository.id,
job_id=job.id,
message="Push accepted and scan scheduled",
)
@app.post("/api/v1/operator/database-recovery/acknowledge", status_code=status.HTTP_204_NO_CONTENT)
def acknowledge_database_recovery(request: Request) -> Response:
_require_mutation_authorization(request, strict=True)
store.acknowledge_database_recovery()
return Response(status_code=status.HTTP_204_NO_CONTENT)
@app.get("/api/v1/scans", response_model=list[ScanReport])
def list_scans(limit: int = Query(default=20, ge=1, le=100)) -> list[ScanReport]:
return store.recent(limit)
@app.get("/api/v1/scans/{scan_id}", response_model=ScanReport)
def get_scan(scan_id: str) -> ScanReport:
report = store.get(scan_id)
if report is None:
raise HTTPException(status_code=404, detail="Scan not found")
return report
@app.get("/api/v1/scans/{scan_id}/comparison", response_model=ScanComparison)
def get_scan_comparison(scan_id: str) -> ScanComparison:
report = store.get(scan_id)
if report is None:
raise HTTPException(status_code=404, detail="Scan not found")
return store.compare(report)
@app.get("/api/v1/scans/{scan_id}/delta")
def get_scan_delta(scan_id: str) -> dict[str, int | None]:
report = store.get(scan_id)
if report is None:
raise HTTPException(status_code=404, detail="Scan not found")
previous = store.previous_for_repository(
report.source.repository_id or report.repository_path,
report.id,
)
return {
"previous_health_score": previous.health_score if previous else None,
"health_score_delta": report.health_score - previous.health_score if previous else None,
"previous_finding_count": len(previous.findings) if previous else None,
"finding_count_delta": len(report.findings) - len(previous.findings) if previous else None,
}
@app.get("/api/v1/trend", response_model=list[TrendPoint])
def trend(path: str, limit: int = Query(default=30, ge=1, le=100)) -> list[TrendPoint]:
try:
resolved = _resolve_scan_path(path)
except ValueError as exc:
raise HTTPException(status_code=422, detail=str(exc)) from exc
return store.trend(str(resolved), limit)
@app.get("/api/v1/scans/{scan_id}/export")
def export_scan(scan_id: str, format: str = Query(default="html", pattern="^(html|json)$")) -> Response:
report = store.get(scan_id)
if report is None:
raise HTTPException(status_code=404, detail="Scan not found")
if format == "json":
filename = safe_report_filename(report.repository_name, "json")
return Response(
report.model_dump_json(indent=2),
media_type="application/json",
headers={"Content-Disposition": content_disposition(filename)},
)
filename = safe_report_filename(report.repository_name, "html")
return Response(
render_html_report(report),
media_type="text/html",
headers={"Content-Disposition": content_disposition(filename)},
)
class _FrontendStaticFiles(StaticFiles):
"""Serve the SPA; content-hashed build assets are immutable and cached long-term.
Client-side routes such as ``/repositories/{id}/findings`` have no file on disk. They
fall back to ``index.html`` so a refresh or a shared link opens the right view, while
API paths, build assets and anything that looks like a file keep an honest 404.
"""
@staticmethod
def _is_client_route(path: str) -> bool:
normalized = path.replace("\\", "/").strip("/")
if not normalized or normalized.split("/", 1)[0] in {"api", "assets", "health"}:
return False
return "." not in normalized.rsplit("/", 1)[-1]
async def get_response(self, path: str, scope: Scope) -> Response:
try:
response = await super().get_response(path, scope)
except (StarletteHTTPException, OSError) as exc:
# Route segments such as "gitea:64" are not valid file names on every platform.
missing = isinstance(exc, OSError) or exc.status_code == 404
if not missing or scope.get("method") not in {"GET", "HEAD"} or not self._is_client_route(path):
if isinstance(exc, OSError):
raise StarletteHTTPException(status_code=404) from exc
raise
response = await super().get_response("index.html", scope)
if response.status_code == 200:
if path.replace("\\", "/").startswith("assets/"):
response.headers["Cache-Control"] = "public, max-age=31536000, immutable"
else:
response.headers["Cache-Control"] = "no-cache"
return response
static_dir = Path(os.getenv("REPOPULSE_STATIC_DIR", "/app/static"))
if static_dir.is_dir():
app.mount("/", _FrontendStaticFiles(directory=static_dir, html=True), name="frontend")