RepoPulse: update frontend/src and backend/app (+4 areas)
Public changes compared with the previous published export: 19 files changed. Affected areas: - backend/app: 3 files - dependencies: 2 files - documentation: 1 file - frontend: 2 files - frontend/src: 10 files - tests: 1 file Changed files: - Updated: "backend/app/acquisition.py" - Added: "backend/app/icons.py" - Updated: "backend/app/main.py" - Added: "backend/tests/test_icons.py" - Updated: "docs/ARCHITECTURE.md" - Updated: "frontend/package-lock.json" - Updated: "frontend/package.json" - Updated: "frontend/playwright.config.ts" - Updated: "frontend/src/App.tsx" - Updated: "frontend/src/components/DashboardView.tsx" - Added: "frontend/src/components/HealthRing.tsx" - Updated: "frontend/src/components/HistoryView.tsx" - Added: "frontend/src/components/Icon.tsx" - Updated: "frontend/src/components/PortfolioAtlas.tsx" - Updated: "frontend/src/components/RemediationCenter.tsx" - Added: "frontend/src/components/RepoAvatar.tsx" - Updated: "frontend/src/main.tsx" - Updated: "frontend/src/styles.css" - Updated: "frontend/vite.config.ts" Export provenance/manifests refreshed; excluded from the change summary. Publish curated RepoPulse source from 7a60c53ee2c1daebecd66a01656e323f1db72e56
This commit is contained in:
1 parent
d5ab22ddf1
commit
787a2b1dc3
21 files changed
+1543
-759
No files matched your search
@@ -92,6 +92,10 @@ class GitAcquirer:
|
||||
safe = re.sub(r"[^A-Za-z0-9._-]", "_", identity)
|
||||
return self.mirrors_root / f"{safe}.git"
|
||||
|
||||
def mirror_path(self, repository: Repository) -> Path:
|
||||
"""Return the bare mirror location for read-only consumers such as icon discovery."""
|
||||
return self._mirror_path(repository)
|
||||
|
||||
def _worktree_path(self, job_id: str) -> Path:
|
||||
if not SAFE_JOB_ID.fullmatch(job_id):
|
||||
raise AcquisitionError("invalid_job", "Scan job identity is invalid")
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
"""Read-only project icon discovery from retained bare mirrors.
|
||||
|
||||
A repository's own favicon or logo makes it recognisable in the catalog. The icon is
|
||||
resolved from the existing mirror at an exact revision, never from a worktree and never
|
||||
by fetching: the scanned repository and the mirror are not modified. Candidates are
|
||||
ranked deterministically, size-bounded and content-checked before they are served.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import subprocess
|
||||
from collections import OrderedDict
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from threading import Lock
|
||||
|
||||
from .processes import git_environment, run_bounded
|
||||
|
||||
SAFE_REVISION = re.compile(r"^[0-9a-fA-F]{7,64}$")
|
||||
SAFE_OBJECT = re.compile(r"^[0-9a-f]{40,64}$")
|
||||
MAX_ICON_BYTES = 128 * 1024
|
||||
MAX_TREE_ENTRIES = 200_000
|
||||
MIN_RASTER_BYTES = 3 * 1024
|
||||
|
||||
_ICON_NAME = re.compile(r"^(favicon|icon|app-icon|logo|logo-mark|apple-touch-icon)([._-][a-z0-9-]*)?\.(svg|png|webp|ico)$", re.IGNORECASE)
|
||||
_EXCLUDED_SEGMENT = re.compile(
|
||||
r"^(node_modules|vendor|third_party|\.git|dist|build-cache|coverage|tests?|__tests__|fixtures?|examples?|screens|screenshots|mockups?|templates?)$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_PREFERRED_DIRECTORY = {"public": 0, "static": 1, "app": 1, "assets": 2, "icons": 2, "images": 3, "wwwroot": 1, "media": 3, "build": 4, "resources": 4, "build-resources": 4}
|
||||
_NAME_RANK = {"favicon": 0, "icon": 1, "app-icon": 1, "logo-mark": 2, "logo": 3, "apple-touch-icon": 4}
|
||||
_FORMAT_RANK = {"svg": 0, "png": 1, "webp": 2, "ico": 3}
|
||||
MEDIA_TYPES = {"svg": "image/svg+xml", "png": "image/png", "webp": "image/webp", "ico": "image/x-icon"}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ProjectIcon:
|
||||
path: str
|
||||
media_type: str
|
||||
content: bytes
|
||||
|
||||
|
||||
def _rank(path: str, size: int) -> tuple[int, int, int, int, int, int, str] | None:
|
||||
parts = path.split("/")
|
||||
name = parts[-1]
|
||||
match = _ICON_NAME.fullmatch(name)
|
||||
if not match or size <= 0 or size > MAX_ICON_BYTES:
|
||||
return None
|
||||
directories = parts[:-1]
|
||||
if any(_EXCLUDED_SEGMENT.fullmatch(segment) for segment in directories):
|
||||
return None
|
||||
if len(directories) > 6:
|
||||
return None
|
||||
directory_rank = min((_PREFERRED_DIRECTORY.get(segment.casefold(), 6) for segment in directories), default=5)
|
||||
extension = match.group(3).casefold()
|
||||
sized_variant = 1 if match.group(2) and re.search(r"\d", match.group(2)) else 0
|
||||
# Tiny raster favicons (16/32 px) look blurry at avatar size; prefer a larger raster.
|
||||
low_resolution = 1 if extension != "svg" and size < MIN_RASTER_BYTES else 0
|
||||
return (
|
||||
_FORMAT_RANK[extension],
|
||||
low_resolution,
|
||||
_NAME_RANK[match.group(1).casefold()],
|
||||
directory_rank + sized_variant,
|
||||
len(directories),
|
||||
size,
|
||||
path,
|
||||
)
|
||||
|
||||
|
||||
def _looks_like(content: bytes, extension: str) -> bool:
|
||||
if extension == "png":
|
||||
return content.startswith(b"\x89PNG\r\n\x1a\n")
|
||||
if extension == "ico":
|
||||
return content[:4] == b"\x00\x00\x01\x00"
|
||||
if extension == "webp":
|
||||
return content[:4] == b"RIFF" and content[8:12] == b"WEBP"
|
||||
head = content[:2048].lstrip().lower()
|
||||
return head.startswith((b"<?xml", b"<svg", b"<!--")) and b"<svg" in content[:8192].lower()
|
||||
|
||||
|
||||
class ProjectIconResolver:
|
||||
"""Resolve and cache the most representative icon file at an exact revision."""
|
||||
|
||||
def __init__(self, mirrors_root: Path, *, timeout: float = 15, cache_size: int = 512) -> None:
|
||||
self.mirrors_root = mirrors_root
|
||||
self.timeout = timeout
|
||||
self.cache_size = cache_size
|
||||
self._cache: OrderedDict[tuple[str, str], ProjectIcon | None] = OrderedDict()
|
||||
self._lock = Lock()
|
||||
|
||||
def _git(self, mirror: Path, args: list[str]) -> bytes:
|
||||
result = run_bounded(["git", "--git-dir", str(mirror), *args], timeout=self.timeout, env=git_environment())
|
||||
return result.stdout
|
||||
|
||||
def resolve(self, mirror: Path, revision: str) -> ProjectIcon | None:
|
||||
if not SAFE_REVISION.fullmatch(revision):
|
||||
return None
|
||||
try:
|
||||
mirror.resolve().relative_to(self.mirrors_root.resolve())
|
||||
except (OSError, RuntimeError, ValueError):
|
||||
return None
|
||||
key = (str(mirror), revision.casefold())
|
||||
with self._lock:
|
||||
if key in self._cache:
|
||||
self._cache.move_to_end(key)
|
||||
return self._cache[key]
|
||||
icon = self._discover(mirror, revision) if mirror.is_dir() else None
|
||||
with self._lock:
|
||||
self._cache[key] = icon
|
||||
self._cache.move_to_end(key)
|
||||
while len(self._cache) > self.cache_size:
|
||||
self._cache.popitem(last=False)
|
||||
return icon
|
||||
|
||||
def _discover(self, mirror: Path, revision: str) -> ProjectIcon | None:
|
||||
try:
|
||||
listing = self._git(mirror, ["ls-tree", "-r", "-l", "--full-tree", "-z", f"{revision}^{{tree}}"])
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
return None
|
||||
candidates: list[tuple[tuple[int, int, int, int, int, int, str], str, str]] = []
|
||||
for index, entry in enumerate(listing.split(b"\x00")):
|
||||
if index >= MAX_TREE_ENTRIES or not entry:
|
||||
continue
|
||||
try:
|
||||
meta, raw_path = entry.split(b"\t", 1)
|
||||
_mode, kind, obj, size = meta.split()
|
||||
path = raw_path.decode("utf-8")
|
||||
except ValueError:
|
||||
continue
|
||||
if kind != b"blob" or not size.isdigit():
|
||||
continue
|
||||
obj_id = obj.decode("ascii", errors="ignore")
|
||||
rank = _rank(path, int(size))
|
||||
if rank is not None and SAFE_OBJECT.fullmatch(obj_id):
|
||||
candidates.append((rank, obj_id, path))
|
||||
for _rank_key, obj_id, path in sorted(candidates)[:8]:
|
||||
try:
|
||||
content = self._git(mirror, ["cat-file", "blob", obj_id])
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
continue
|
||||
extension = path.rsplit(".", 1)[-1].casefold()
|
||||
if len(content) <= MAX_ICON_BYTES and _looks_like(content, extension):
|
||||
return ProjectIcon(path=path, media_type=MEDIA_TYPES[extension], content=content)
|
||||
return None
|
||||
@@ -18,6 +18,8 @@ from starlette.types import Scope
|
||||
|
||||
from .acquisition import GitAcquirer
|
||||
from .gitea import GiteaClient
|
||||
from .icons import SAFE_REVISION as SAFE_ICON_REVISION
|
||||
from .icons import ProjectIconResolver
|
||||
from .jobs import ScanJobRunner
|
||||
from .models import (
|
||||
RepositoryDetail,
|
||||
@@ -56,6 +58,7 @@ MAX_CONCURRENT_SCANS = 1
|
||||
SCAN_SLOTS = BoundedSemaphore(MAX_CONCURRENT_SCANS)
|
||||
gitea_client = GiteaClient()
|
||||
repository_acquirer = GitAcquirer()
|
||||
project_icons = ProjectIconResolver(repository_acquirer.mirrors_root)
|
||||
job_runner: ScanJobRunner | None = None
|
||||
_api_mutation_lock = Lock()
|
||||
_MAINTENANCE_ENTER_PATH = "/api/v1/operator/maintenance/enter"
|
||||
@@ -280,6 +283,30 @@ def get_repository(repository_id: str) -> RepositoryDetail:
|
||||
return repository
|
||||
|
||||
|
||||
ICON_HEADERS = {
|
||||
"Cache-Control": "public, max-age=86400",
|
||||
"X-Content-Type-Options": "nosniff",
|
||||
# Repository-supplied SVG is untrusted: never allow script, network or embedding.
|
||||
"Content-Security-Policy": "default-src 'none'; style-src 'unsafe-inline'; img-src data:; sandbox",
|
||||
"Cross-Origin-Resource-Policy": "same-origin",
|
||||
}
|
||||
|
||||
|
||||
@app.get("/api/v1/repositories/{repository_id}/icon")
|
||||
def repository_icon(repository_id: str, rev: str | None = Query(default=None, max_length=64)) -> Response:
|
||||
"""Serve the project's own favicon or logo from its mirror at the retained revision."""
|
||||
|
||||
repository = store.get_repository(repository_id)
|
||||
if repository is None:
|
||||
raise HTTPException(status_code=404, detail="Repository not found")
|
||||
known = {value.casefold() for value in (repository.last_scan_revision, repository.latest_revision) if value}
|
||||
revision = rev if rev and SAFE_ICON_REVISION.fullmatch(rev) and rev.casefold() in known else repository.last_scan_revision or repository.latest_revision
|
||||
icon = project_icons.resolve(repository_acquirer.mirror_path(repository), revision) if revision else None
|
||||
if icon is None:
|
||||
return Response(status_code=status.HTTP_404_NOT_FOUND, headers={"Cache-Control": "public, max-age=3600"})
|
||||
return Response(icon.content, media_type=icon.media_type, headers=ICON_HEADERS)
|
||||
|
||||
|
||||
@app.get("/api/v1/repositories/{repository_id}/scans", response_model=list[ScanSummary])
|
||||
def list_repository_scans(
|
||||
repository_id: str,
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
from app.icons import MAX_ICON_BYTES, ProjectIconResolver
|
||||
|
||||
PNG = b"\x89PNG\r\n\x1a\n" + b"\x00" * 32
|
||||
SVG = b'<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 8 8"><rect width="8" height="8"/></svg>'
|
||||
|
||||
|
||||
def _mirror(tmp_path: Path, files: dict[str, bytes]) -> tuple[Path, Path, str]:
|
||||
source = tmp_path / "source"
|
||||
source.mkdir()
|
||||
for relative, content in files.items():
|
||||
target = source / relative
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_bytes(content)
|
||||
subprocess.run(["git", "init", "--quiet", str(source)], check=True)
|
||||
subprocess.run(["git", "-C", str(source), "config", "user.email", "test@example.com"], check=True)
|
||||
subprocess.run(["git", "-C", str(source), "config", "user.name", "RepoPulse Test"], check=True)
|
||||
subprocess.run(["git", "-C", str(source), "add", "."], check=True)
|
||||
subprocess.run(["git", "-C", str(source), "commit", "--quiet", "-m", "initial"], check=True)
|
||||
revision = subprocess.run(["git", "-C", str(source), "rev-parse", "HEAD"], check=True, capture_output=True, text=True).stdout.strip()
|
||||
mirrors = tmp_path / "mirrors"
|
||||
mirrors.mkdir()
|
||||
mirror = mirrors / "gitea-1.git"
|
||||
subprocess.run(["git", "clone", "--quiet", "--mirror", str(source), str(mirror)], check=True)
|
||||
return mirrors, mirror, revision
|
||||
|
||||
|
||||
def test_prefers_a_public_svg_favicon_over_logos_and_excluded_fixtures(tmp_path: Path) -> None:
|
||||
mirrors, mirror, revision = _mirror(tmp_path, {
|
||||
"README.md": b"# Demo\n",
|
||||
"docs/logo.png": PNG,
|
||||
"tests/fixtures/favicon.svg": SVG,
|
||||
"node_modules/pkg/favicon.svg": SVG,
|
||||
"frontend/public/favicon.svg": SVG,
|
||||
"frontend/public/icons.svg": SVG,
|
||||
})
|
||||
icon = ProjectIconResolver(mirrors).resolve(mirror, revision)
|
||||
assert icon is not None
|
||||
assert icon.path == "frontend/public/favicon.svg"
|
||||
assert icon.media_type == "image/svg+xml"
|
||||
assert icon.content == SVG
|
||||
|
||||
|
||||
def test_rejects_content_that_does_not_match_its_extension_and_oversized_files(tmp_path: Path) -> None:
|
||||
mirrors, mirror, revision = _mirror(tmp_path, {
|
||||
"public/favicon.svg": b"#!/bin/sh\necho not an image\n",
|
||||
"public/logo.png": b"not a png",
|
||||
"public/icon.png": PNG + b"\x00" * MAX_ICON_BYTES,
|
||||
"app/icon.png": PNG,
|
||||
})
|
||||
icon = ProjectIconResolver(mirrors).resolve(mirror, revision)
|
||||
assert icon is not None
|
||||
assert icon.path == "app/icon.png"
|
||||
assert icon.media_type == "image/png"
|
||||
|
||||
|
||||
def test_prefers_a_larger_raster_over_a_tiny_favicon(tmp_path: Path) -> None:
|
||||
mirrors, mirror, revision = _mirror(tmp_path, {
|
||||
"frontend/public/icons/favicon-16.png": PNG,
|
||||
"frontend/public/icons/apple-touch-icon.png": PNG + bytes(8000),
|
||||
})
|
||||
icon = ProjectIconResolver(mirrors).resolve(mirror, revision)
|
||||
assert icon is not None
|
||||
assert icon.path == "frontend/public/icons/apple-touch-icon.png"
|
||||
|
||||
|
||||
def test_missing_icons_unknown_revisions_and_foreign_mirrors_resolve_to_none(tmp_path: Path) -> None:
|
||||
mirrors, mirror, revision = _mirror(tmp_path, {"README.md": b"# Demo\n"})
|
||||
resolver = ProjectIconResolver(mirrors)
|
||||
assert resolver.resolve(mirror, revision) is None
|
||||
assert resolver.resolve(mirror, "not-a-revision") is None
|
||||
assert resolver.resolve(mirror, "0" * 40) is None
|
||||
assert resolver.resolve(tmp_path / "source" / ".git", revision) is None
|
||||
|
||||
|
||||
def test_icon_endpoint_serves_the_mirror_icon_with_strict_headers(tmp_path: Path, monkeypatch) -> None:
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from app import main as main_module
|
||||
from app.models import Repository
|
||||
from app.store import ScanStore
|
||||
|
||||
mirrors, _mirror_path, revision = _mirror(tmp_path, {"public/favicon.svg": SVG})
|
||||
store = ScanStore(str(tmp_path / "history.db"))
|
||||
repository = Repository(id="gitea:1", provider="gitea", provider_repository_id="1", owner="Jens", name="Demo", full_name="Jens/Demo", latest_revision=revision)
|
||||
store.upsert_repository(repository)
|
||||
empty = Repository(id="gitea:2", provider="gitea", provider_repository_id="2", owner="Jens", name="Empty", full_name="Jens/Empty")
|
||||
store.upsert_repository(empty)
|
||||
monkeypatch.setattr(main_module, "store", store)
|
||||
monkeypatch.setattr(main_module, "project_icons", ProjectIconResolver(mirrors))
|
||||
monkeypatch.setattr(main_module.repository_acquirer, "mirrors_root", mirrors)
|
||||
client = TestClient(main_module.app)
|
||||
|
||||
response = client.get(f"/api/v1/repositories/gitea:1/icon?rev={revision}")
|
||||
assert response.status_code == 200
|
||||
assert response.content == SVG
|
||||
assert response.headers["content-type"].startswith("image/svg+xml")
|
||||
assert "sandbox" in response.headers["content-security-policy"]
|
||||
assert response.headers["x-content-type-options"] == "nosniff"
|
||||
|
||||
assert client.get("/api/v1/repositories/gitea:2/icon").status_code == 404
|
||||
assert client.get("/api/v1/repositories/gitea:404/icon").status_code == 404
|
||||
Reference in new issue
Block a user