Publish ITWorx Pulse source

internal/metricquery/handler.go

@@ -0,0 +1,125 @@
+package metricquery
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"io"
+	"net/http"
+	"strings"
+
+	"github.com/itworx/pulse/internal/auth"
+	"github.com/itworx/pulse/internal/problem"
+	"github.com/itworx/pulse/internal/promqlbinding"
+	"github.com/itworx/pulse/internal/queryplan"
+)
+
+const maxRequestBody = 64 << 10
+
+type Handler struct{ Service *Service }
+
+func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodPost {
+		http.NotFound(w, r)
+		return
+	}
+	if _, ok := auth.PrincipalFromContext(r.Context()); !ok {
+		problem.Write(w, r, http.StatusUnauthorized, "UNAUTHORIZED", "Authentication required", "Authentication is required to query metrics.", nil)
+		return
+	}
+	if h.Service == nil {
+		problem.Write(w, r, http.StatusServiceUnavailable, "METRIC_SOURCE_UNAVAILABLE", "Metric source unavailable", "The configured metric source is not available.", nil)
+		return
+	}
+	var response Response
+	var err error
+	switch strings.TrimPrefix(r.URL.Path, "/api/v1/metrics/") {
+	case "inspect":
+		var request queryplan.Request
+		if !decodeJSON(w, r, &request) {
+			return
+		}
+		inspector, inspectErr := h.Service.Inspect(r.Context(), request)
+		if inspectErr != nil {
+			writeQueryError(w, r, inspectErr)
+			return
+		}
+		w.Header().Set("Content-Type", "application/json")
+		_ = json.NewEncoder(w).Encode(map[string]any{"inspector": inspector})
+		return
+	case "query-range":
+		var request queryplan.Request
+		if decodeJSON(w, r, &request) {
+			response, err = h.Service.ExecuteRange(r.Context(), request)
+		}
+	case "query":
+		var request InstantRequest
+		if decodeJSON(w, r, &request) {
+			response, err = h.Service.ExecuteInstant(r.Context(), request)
+		}
+	default:
+		http.NotFound(w, r)
+		return
+	}
+	if err != nil {
+		writeQueryError(w, r, err)
+		return
+	}
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(response)
+}
+
+func decodeJSON(w http.ResponseWriter, r *http.Request, destination any) bool {
+	body, readErr := io.ReadAll(io.LimitReader(r.Body, maxRequestBody+1))
+	if readErr != nil || len(body) > maxRequestBody {
+		problem.Write(w, r, http.StatusRequestEntityTooLarge, "QUERY_BODY_LIMIT", "Query body too large", "The metric query body exceeds the allowed size.", nil)
+		return false
+	}
+	decoder := json.NewDecoder(bytes.NewReader(body))
+	decoder.DisallowUnknownFields()
+	if err := decoder.Decode(destination); err != nil {
+		problem.Write(w, r, http.StatusBadRequest, "QUERY_BODY_INVALID", "Invalid query body", "The metric query body is invalid.", nil)
+		return false
+	}
+	var extra any
+	if err := decoder.Decode(&extra); err != io.EOF {
+		problem.Write(w, r, http.StatusBadRequest, "QUERY_BODY_INVALID", "Invalid query body", "The metric query body contains trailing data.", nil)
+		return false
+	}
+	return true
+}
+
+func writeQueryError(w http.ResponseWriter, r *http.Request, err error) {
+	if errors.Is(err, ErrInspectorUnauthorized) {
+		problem.Write(w, r, http.StatusForbidden, "QUERY_INSPECTOR_FORBIDDEN", "Query inspector forbidden", "The query inspector requires operate permission.", nil)
+		return
+	}
+	if errors.Is(err, context.Canceled) {
+		return
+	}
+	var sourceErr sourceError
+	if errors.Is(err, ErrSourceUnavailable) || errors.As(err, &sourceErr) {
+		problem.Write(w, r, http.StatusServiceUnavailable, "QUERY_SOURCE_UNAVAILABLE", "Metric source unavailable", "The metric source could not be queried.", nil)
+		return
+	}
+	var planError queryplan.Error
+	if errors.As(err, &planError) {
+		status := http.StatusUnprocessableEntity
+		if planError.Code == queryplan.ErrUnauthorized.Code {
+			status = http.StatusUnauthorized
+		}
+		problem.Write(w, r, status, planError.Code, "Metric query rejected", planError.Detail, map[string]string{"field": planError.Field})
+		return
+	}
+	var bindingError promqlbinding.Error
+	if errors.As(err, &bindingError) {
+		status := http.StatusUnprocessableEntity
+		if bindingError.Code == "PROMQL_RAW_UNAUTHORIZED" {
+			status = http.StatusForbidden
+		}
+		problem.Write(w, r, status, bindingError.Code, "Metric query rejected", bindingError.Detail, map[string]string{"field": bindingError.Field})
+		return
+	}
+	problem.Write(w, r, http.StatusServiceUnavailable, "QUERY_UNAVAILABLE", "Metric query unavailable", "The metric query could not be completed.", nil)
+}

internal/metricquery/handler_test.go

@@ -0,0 +1,119 @@
+package metricquery
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/itworx/pulse/internal/auth"
+	"github.com/itworx/pulse/internal/metriccatalog"
+	"github.com/itworx/pulse/internal/prometheus"
+	"github.com/itworx/pulse/internal/queryplan"
+)
+
+func handlerService(t *testing.T) *Service {
+	return testService(t, &fakeSource{result: prometheus.QueryResult{Status: "success", Data: []byte(`{"resultType":"matrix","result":[]}`)}}, NewCache(8, 1<<20))
+}
+func authenticatedRequest(method, path, body string) *http.Request {
+	request := httptest.NewRequest(method, path, strings.NewReader(body))
+	return request.WithContext(auth.WithPrincipal(context.Background(), auth.Principal{Subject: "viewer", Role: auth.RoleViewer}))
+}
+func operatorRequest(method, path, body string) *http.Request {
+	request := httptest.NewRequest(method, path, strings.NewReader(body))
+	return request.WithContext(auth.WithPrincipal(context.Background(), auth.Principal{Subject: "operator", Role: auth.RoleOperator}))
+}
+
+func TestHandlerRequiresAuthAndMapsUnavailableService(t *testing.T) {
+	response := httptest.NewRecorder()
+	Handler{Service: handlerService(t)}.ServeHTTP(response, httptest.NewRequest(http.MethodPost, "/api/v1/metrics/query-range", strings.NewReader(`{}`)))
+	if response.Code != http.StatusUnauthorized {
+		t.Fatalf("status=%d", response.Code)
+	}
+	response = httptest.NewRecorder()
+	Handler{}.ServeHTTP(response, authenticatedRequest(http.MethodPost, "/api/v1/metrics/query-range", `{}`))
+	if response.Code != http.StatusServiceUnavailable {
+		t.Fatalf("status=%d", response.Code)
+	}
+}
+func TestHandlerRejectsMalformedAndOversizedBodies(t *testing.T) {
+	service := handlerService(t)
+	response := httptest.NewRecorder()
+	Handler{Service: service}.ServeHTTP(response, authenticatedRequest(http.MethodPost, "/api/v1/metrics/query-range", `{"metric":`))
+	if response.Code != http.StatusBadRequest {
+		t.Fatalf("malformed=%d", response.Code)
+	}
+	response = httptest.NewRecorder()
+	oversized := `{"padding":"` + strings.Repeat("x", maxRequestBody) + `"}`
+	Handler{Service: service}.ServeHTTP(response, authenticatedRequest(http.MethodPost, "/api/v1/metrics/query-range", oversized))
+	if response.Code != http.StatusRequestEntityTooLarge {
+		t.Fatalf("oversized=%d", response.Code)
+	}
+}
+func TestHandlerPermissionedInspector(t *testing.T) {
+	service := handlerService(t)
+	payload, _ := json.Marshal(testRequest())
+	response := httptest.NewRecorder()
+	Handler{Service: service}.ServeHTTP(response, authenticatedRequest(http.MethodPost, "/api/v1/metrics/inspect", string(payload)))
+	if response.Code != http.StatusForbidden {
+		t.Fatalf("viewer status=%d body=%s", response.Code, response.Body.String())
+	}
+	response = httptest.NewRecorder()
+	Handler{Service: service}.ServeHTTP(response, operatorRequest(http.MethodPost, "/api/v1/metrics/inspect", string(payload)))
+	if response.Code != http.StatusOK {
+		t.Fatalf("operator status=%d body=%s", response.Code, response.Body.String())
+	}
+	var decoded struct {
+		Inspector Inspector `json:"inspector"`
+	}
+	if err := json.Unmarshal(response.Body.Bytes(), &decoded); err != nil {
+		t.Fatal(err)
+	}
+	if decoded.Inspector.GeneratedQuery == "" || decoded.Inspector.Cost.Series < 1 || decoded.Inspector.Limits.MaxPoints < 1 {
+		t.Fatalf("inspector=%+v", decoded.Inspector)
+	}
+}
+
+func TestHandlerReturnsBoundedQueryResponse(t *testing.T) {
+	service := handlerService(t)
+	request := testRequest()
+	payload, _ := json.Marshal(request)
+	response := httptest.NewRecorder()
+	Handler{Service: service}.ServeHTTP(response, authenticatedRequest(http.MethodPost, "/api/v1/metrics/query-range", string(payload)))
+	if response.Code != http.StatusOK {
+		t.Fatalf("status=%d body=%s", response.Code, response.Body.String())
+	}
+	var decoded Response
+	if err := json.Unmarshal(response.Body.Bytes(), &decoded); err != nil {
+		t.Fatal(err)
+	}
+	if decoded.Provenance.Source != "prometheus" || decoded.Freshness != "fresh" {
+		t.Fatalf("response=%+v", decoded)
+	}
+}
+func TestHandlerMapsSourceErrorsWithoutLeakingDetails(t *testing.T) {
+	service := testService(t, &fakeSource{err: context.DeadlineExceeded}, nil)
+	payload, _ := json.Marshal(testRequest())
+	response := httptest.NewRecorder()
+	Handler{Service: service}.ServeHTTP(response, authenticatedRequest(http.MethodPost, "/api/v1/metrics/query-range", string(payload)))
+	if response.Code != http.StatusServiceUnavailable || strings.Contains(response.Body.String(), "DeadlineExceeded") {
+		t.Fatalf("status=%d body=%s", response.Code, response.Body.String())
+	}
+}
+func TestHandlerInstantRoute(t *testing.T) {
+	registry, err := metriccatalog.DefaultRegistry()
+	if err != nil {
+		t.Fatal(err)
+	}
+	source := &fakeSource{result: prometheus.QueryResult{Status: "success", Data: []byte(`{}`)}}
+	service := NewService(queryplan.NewPlanner(registry, queryplan.Limits{}), source, nil)
+	payload := `{"metric":"host.cpu.utilization","scope":{"serverId":"server-1"},"at":"` + time.Date(2026, 8, 1, 12, 0, 0, 0, time.UTC).Format(time.RFC3339) + `"}`
+	response := httptest.NewRecorder()
+	Handler{Service: service}.ServeHTTP(response, authenticatedRequest(http.MethodPost, "/api/v1/metrics/query", payload))
+	if response.Code != http.StatusOK || source.instantCalls != 1 {
+		t.Fatalf("status=%d calls=%d body=%s", response.Code, source.instantCalls, response.Body.String())
+	}
+}

internal/metricquery/service.go

@@ -0,0 +1,325 @@
+package metricquery
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"sync"
+	"time"
+
+	"github.com/itworx/pulse/internal/auth"
+	"github.com/itworx/pulse/internal/metriccatalog"
+	"github.com/itworx/pulse/internal/prometheus"
+	"github.com/itworx/pulse/internal/promqlbinding"
+	"github.com/itworx/pulse/internal/queryplan"
+	"github.com/itworx/pulse/internal/redaction"
+)
+
+var ErrSourceUnavailable = errors.New("metric source unavailable")
+var ErrInspectorUnauthorized = errors.New("metric inspector requires operate permission")
+
+const (
+	defaultCacheEntries = 128
+	defaultCacheBytes   = 16 << 20
+	defaultCacheTTL     = 15 * time.Second
+)
+
+// Freshness states carried by every metric response and live sample.
+const (
+	FreshnessFresh   = "fresh"
+	FreshnessStale   = "stale"
+	FreshnessUnknown = "unknown"
+)
+
+// FreshnessWindow is the freshness policy of a semantic metric definition.
+func FreshnessWindow(definition metriccatalog.Definition) time.Duration {
+	return time.Duration(definition.FreshnessSeconds) * time.Second
+}
+
+// ClassifyFreshness maps an observation timestamp onto the shared freshness
+// vocabulary. Telemetry without a timestamp or without a policy is unknown and
+// telemetry older than its policy window is stale; neither is ever fresh
+// (ADR-0008).
+func ClassifyFreshness(observedAt, now time.Time, maxAge time.Duration) string {
+	if observedAt.IsZero() || maxAge <= 0 {
+		return FreshnessUnknown
+	}
+	if now.Sub(observedAt) > maxAge {
+		return FreshnessStale
+	}
+	return FreshnessFresh
+}
+
+type Source interface {
+	Query(context.Context, string, *time.Time) (prometheus.QueryResult, error)
+	QueryRange(context.Context, string, time.Time, time.Time, time.Duration) (prometheus.QueryResult, error)
+}
+
+type Provenance struct {
+	Source         string `json:"source"`
+	Metric         string `json:"metric"`
+	CatalogVersion string `json:"catalogVersion"`
+	CacheKey       string `json:"cacheKey"`
+}
+
+type InspectorLimits struct {
+	MaxSeries int `json:"maxSeries"`
+	MaxPoints int `json:"maxPoints"`
+}
+
+type Inspector struct {
+	SemanticMetric string          `json:"semanticMetric"`
+	GeneratedQuery string          `json:"generatedQuery"`
+	Cost           queryplan.Cost  `json:"cost"`
+	Limits         InspectorLimits `json:"limits"`
+}
+
+type Response struct {
+	Status           string          `json:"status"`
+	Data             json.RawMessage `json:"data"`
+	Warnings         []string        `json:"warnings,omitempty"`
+	Provenance       Provenance      `json:"provenance"`
+	SourceObservedAt time.Time       `json:"sourceObservedAt"`
+	ReceivedAt       time.Time       `json:"receivedAt"`
+	Freshness        string          `json:"freshness"`
+	CacheHit         bool            `json:"cacheHit"`
+	Inspector        *Inspector      `json:"inspector,omitempty"`
+}
+
+type Service struct {
+	planner  queryplan.Planner
+	source   Source
+	cache    *Cache
+	mu       sync.Mutex
+	inflight map[string]*call
+	cacheTTL time.Duration
+}
+
+type call struct {
+	done     chan struct{}
+	response Response
+	err      error
+}
+
+func NewService(planner queryplan.Planner, source Source, cache *Cache) *Service {
+	if cache == nil {
+		cache = NewCache(defaultCacheEntries, defaultCacheBytes)
+	}
+	return &Service{planner: planner, source: source, cache: cache, inflight: make(map[string]*call), cacheTTL: defaultCacheTTL}
+}
+func (s *Service) Inspect(ctx context.Context, request queryplan.Request) (Inspector, error) {
+	if s == nil {
+		return Inspector{}, ErrSourceUnavailable
+	}
+	principal, ok := principalFrom(ctx)
+	if !ok || !auth.Allows(principal.Role, auth.PermissionOperate) {
+		return Inspector{}, ErrInspectorUnauthorized
+	}
+	plan, err := s.planner.Plan(ctx, request)
+	if err != nil {
+		return Inspector{}, err
+	}
+	query, err := promqlbinding.CompilePlan(plan)
+	if err != nil {
+		return Inspector{}, err
+	}
+	inspector := inspectorFor(principal, plan, query)
+	if inspector == nil {
+		return Inspector{}, ErrInspectorUnauthorized
+	}
+	return *inspector, nil
+}
+
+func (s *Service) ExecuteRange(ctx context.Context, request queryplan.Request) (Response, error) {
+	return s.execute(ctx, request, false, time.Time{})
+}
+func (s *Service) ExecuteInstant(ctx context.Context, request InstantRequest) (Response, error) {
+	at := request.At
+	if at.IsZero() {
+		at = time.Now().UTC()
+	}
+	queryRequest := queryplan.Request{Metric: request.Metric, Scope: request.Scope, Aggregation: request.Aggregation, GroupBy: request.GroupBy, MaxSeries: request.MaxSeries, MaxPoints: 10, Range: queryplan.Range{From: at.Add(-time.Minute), To: at, StepSeconds: 60}}
+	return s.execute(ctx, queryRequest, true, at.UTC())
+}
+
+type InstantRequest struct {
+	Metric      string            `json:"metric"`
+	Scope       map[string]string `json:"scope,omitempty"`
+	At          time.Time         `json:"at,omitempty"`
+	Aggregation string            `json:"aggregation,omitempty"`
+	GroupBy     []string          `json:"groupBy,omitempty"`
+	MaxSeries   int               `json:"maxSeries,omitempty"`
+}
+
+type sourceError struct{ err error }
+
+func (e sourceError) Error() string { return ErrSourceUnavailable.Error() }
+func (e sourceError) Unwrap() error { return e.err }
+func (e sourceError) Is(target error) bool {
+	return target == ErrSourceUnavailable || errors.Is(e.err, target)
+}
+
+func (s *Service) execute(ctx context.Context, request queryplan.Request, instant bool, at time.Time) (Response, error) {
+	if s == nil || s.source == nil {
+		return Response{}, ErrSourceUnavailable
+	}
+	plan, err := s.planner.Plan(ctx, request)
+	if err != nil {
+		return Response{}, err
+	}
+	query, err := promqlbinding.CompilePlan(plan)
+	if err != nil {
+		return Response{}, err
+	}
+	principal, _ := principalFrom(ctx)
+	key := scopedKey(principal.Subject, plan.CacheKey)
+	if cached, ok := s.cache.Get(key); ok {
+		cached.CacheHit = true
+		cached.Inspector = inspectorFor(principal, plan, query)
+		return cached, nil
+	}
+	s.mu.Lock()
+	if existing, ok := s.inflight[key]; ok {
+		s.mu.Unlock()
+		select {
+		case <-existing.done:
+			cachedResponse := existing.response
+			cachedResponse.CacheHit = true
+			cachedResponse.Inspector = inspectorFor(principal, plan, query)
+			return cachedResponse, existing.err
+		case <-ctx.Done():
+			return Response{}, ctx.Err()
+		}
+	}
+	current := &call{done: make(chan struct{})}
+	s.inflight[key] = current
+	s.mu.Unlock()
+	response, err := s.fetch(ctx, plan, query, instant, at, key)
+	if err == nil {
+		response.Inspector = inspectorFor(principal, plan, query)
+		cacheResponse := response
+		cacheResponse.Inspector = nil
+		s.cache.Put(key, cacheResponse, s.cacheTTL)
+	}
+	s.mu.Lock()
+	current.response, current.err = response, err
+	delete(s.inflight, key)
+	close(current.done)
+	s.mu.Unlock()
+	return response, err
+}
+
+func (s *Service) fetch(ctx context.Context, plan queryplan.Plan, query string, instant bool, at time.Time, key string) (Response, error) {
+	now := time.Now().UTC()
+	var result prometheus.QueryResult
+	var err error
+	if instant {
+		result, err = s.source.Query(ctx, query, &at)
+	} else {
+		result, err = s.source.QueryRange(ctx, query, plan.Request.Range.From, plan.Request.Range.To, time.Duration(plan.Request.Range.StepSeconds)*time.Second)
+	}
+	if err != nil {
+		return Response{}, sourceError{err: err}
+	}
+	warnings := make([]string, 0, len(result.Warnings))
+	for _, warning := range result.Warnings {
+		warnings = append(warnings, redaction.String(warning))
+	}
+	return Response{Status: result.Status, Data: append(json.RawMessage(nil), result.Data...), Warnings: warnings, Provenance: Provenance{Source: "prometheus", Metric: plan.Metric.SemanticName, CatalogVersion: s.planner.CatalogVersion(), CacheKey: key}, SourceObservedAt: now, ReceivedAt: now, Freshness: ClassifyFreshness(now, now, FreshnessWindow(plan.Metric))}, nil
+}
+
+func inspectorFor(principal auth.Principal, plan queryplan.Plan, query string) *Inspector {
+	if !auth.Allows(principal.Role, auth.PermissionOperate) {
+		return nil
+	}
+	return &Inspector{SemanticMetric: plan.Metric.SemanticName, GeneratedQuery: redaction.String(query), Cost: plan.Cost, Limits: InspectorLimits{MaxSeries: plan.Request.MaxSeries, MaxPoints: plan.Request.MaxPoints}}
+}
+
+func principalFrom(ctx context.Context) (auth.Principal, bool) { return auth.PrincipalFromContext(ctx) }
+func scopedKey(subject, planKey string) string {
+	digest := sha256.Sum256([]byte(subject + "\x00" + planKey))
+	return hex.EncodeToString(digest[:])
+}
+
+func cloneResponse(response Response) Response {
+	copy := response
+	copy.Data = append(json.RawMessage(nil), response.Data...)
+	copy.Warnings = append([]string(nil), response.Warnings...)
+	return copy
+}
+
+type cacheEntry struct {
+	response  Response
+	expiresAt time.Time
+	lastUsed  time.Time
+	size      int64
+}
+type Cache struct {
+	mu         sync.Mutex
+	entries    map[string]cacheEntry
+	maxEntries int
+	maxBytes   int64
+	bytes      int64
+}
+
+func NewCache(maxEntries int, maxBytes int64) *Cache {
+	if maxEntries <= 0 {
+		maxEntries = defaultCacheEntries
+	}
+	if maxBytes <= 0 {
+		maxBytes = defaultCacheBytes
+	}
+	return &Cache{entries: make(map[string]cacheEntry), maxEntries: maxEntries, maxBytes: maxBytes}
+}
+func (c *Cache) Get(key string) (Response, bool) {
+	now := time.Now().UTC()
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	entry, ok := c.entries[key]
+	if !ok {
+		return Response{}, false
+	}
+	if !now.Before(entry.expiresAt) {
+		c.bytes -= entry.size
+		delete(c.entries, key)
+		return Response{}, false
+	}
+	entry.lastUsed = now
+	c.entries[key] = entry
+	return cloneResponse(entry.response), true
+}
+func (c *Cache) Put(key string, response Response, ttl time.Duration) {
+	if ttl <= 0 {
+		ttl = defaultCacheTTL
+	}
+	size := int64(len(response.Data)) + int64(len(response.Warnings))*64 + 512
+	if size > c.maxBytes {
+		return
+	}
+	now := time.Now().UTC()
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if old, ok := c.entries[key]; ok {
+		c.bytes -= old.size
+	}
+	for len(c.entries) >= c.maxEntries || c.bytes+size > c.maxBytes {
+		oldestKey := ""
+		var oldest time.Time
+		for candidateKey, candidate := range c.entries {
+			if oldestKey == "" || candidate.lastUsed.Before(oldest) {
+				oldestKey, oldest = candidateKey, candidate.lastUsed
+			}
+		}
+		if oldestKey == "" {
+			break
+		}
+		c.bytes -= c.entries[oldestKey].size
+		delete(c.entries, oldestKey)
+	}
+	c.entries[key] = cacheEntry{response: cloneResponse(response), expiresAt: now.Add(ttl), lastUsed: now, size: size}
+	c.bytes += size
+}
+func (c *Cache) Len() int     { c.mu.Lock(); defer c.mu.Unlock(); return len(c.entries) }
+func (c *Cache) Bytes() int64 { c.mu.Lock(); defer c.mu.Unlock(); return c.bytes }

internal/metricquery/service_test.go

@@ -0,0 +1,257 @@
+package metricquery
+
+import (
+	"context"
+	"errors"
+	"sort"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/itworx/pulse/internal/auth"
+	"github.com/itworx/pulse/internal/metriccatalog"
+	"github.com/itworx/pulse/internal/prometheus"
+	"github.com/itworx/pulse/internal/queryplan"
+)
+
+type fakeSource struct {
+	mu           sync.Mutex
+	rangeCalls   int
+	instantCalls int
+	started      chan struct{}
+	release      chan struct{}
+	result       prometheus.QueryResult
+	err          error
+}
+
+func (f *fakeSource) Query(ctx context.Context, _ string, _ *time.Time) (prometheus.QueryResult, error) {
+	f.mu.Lock()
+	f.instantCalls++
+	f.mu.Unlock()
+	return f.result, f.err
+}
+func (f *fakeSource) QueryRange(ctx context.Context, _ string, _, _ time.Time, _ time.Duration) (prometheus.QueryResult, error) {
+	f.mu.Lock()
+	f.rangeCalls++
+	started := f.started
+	release := f.release
+	f.mu.Unlock()
+	if started != nil {
+		select {
+		case started <- struct{}{}:
+		default:
+		}
+	}
+	if release != nil {
+		select {
+		case <-release:
+		case <-ctx.Done():
+			return prometheus.QueryResult{}, ctx.Err()
+		}
+	}
+	return f.result, f.err
+}
+func (f *fakeSource) calls() int { f.mu.Lock(); defer f.mu.Unlock(); return f.rangeCalls }
+func testService(t *testing.T, source Source, cache *Cache) *Service {
+	t.Helper()
+	registry, err := metriccatalog.DefaultRegistry()
+	if err != nil {
+		t.Fatal(err)
+	}
+	return NewService(queryplan.NewPlanner(registry, queryplan.Limits{}), source, cache)
+}
+func testRequest() queryplan.Request {
+	now := time.Date(2026, 8, 1, 12, 0, 0, 0, time.UTC)
+	return queryplan.Request{Metric: "container.cpu.utilization", Scope: map[string]string{"containerId": "media_server"}, Range: queryplan.Range{From: now.Add(-time.Hour), To: now, StepSeconds: 60}, GroupBy: []string{"container"}}
+}
+func userContext(subject string) context.Context {
+	return auth.WithPrincipal(context.Background(), auth.Principal{Subject: subject, Role: auth.RoleViewer})
+}
+
+func TestServiceDeduplicatesConcurrentEquivalentQueriesAndCachesBySubject(t *testing.T) {
+	source := &fakeSource{started: make(chan struct{}, 1), release: make(chan struct{}), result: prometheus.QueryResult{Status: "success", Data: []byte(`{"resultType":"matrix","result":[]}`)}}
+	service := testService(t, source, NewCache(8, 1<<20))
+	firstDone := make(chan struct{})
+	var firstErr error
+	go func() { _, firstErr = service.ExecuteRange(userContext("alice"), testRequest()); close(firstDone) }()
+	<-source.started
+	equivalent := testRequest()
+	equivalent.GroupBy = []string{"container"}
+	secondDone := make(chan struct{})
+	var second Response
+	go func() { second, _ = service.ExecuteRange(userContext("alice"), equivalent); close(secondDone) }()
+	close(source.release)
+	<-firstDone
+	<-secondDone
+	if firstErr != nil {
+		t.Fatal(firstErr)
+	}
+	if source.calls() != 1 {
+		t.Fatalf("source calls=%d", source.calls())
+	}
+	if second.CacheHit != true {
+		t.Fatal("deduplicated waiter was not marked cache hit")
+	}
+	cached, err := service.ExecuteRange(userContext("alice"), testRequest())
+	if err != nil || !cached.CacheHit || source.calls() != 1 {
+		t.Fatalf("cache=%+v err=%v calls=%d", cached, err, source.calls())
+	}
+	_, err = service.ExecuteRange(userContext("bob"), testRequest())
+	if err != nil || source.calls() != 2 {
+		t.Fatalf("subject cache isolation err=%v calls=%d", err, source.calls())
+	}
+}
+
+func TestServiceMapsSourceErrorsAndDoesNotCacheFailures(t *testing.T) {
+	source := &fakeSource{err: errors.New("upstream private detail")}
+	service := testService(t, source, NewCache(8, 1<<20))
+	_, err := service.ExecuteRange(userContext("alice"), testRequest())
+	if err == nil || !errors.Is(err, ErrSourceUnavailable) {
+		t.Fatalf("err=%v", err)
+	}
+	if source.calls() != 1 {
+		t.Fatal("unexpected calls")
+	}
+	_, err = service.ExecuteRange(userContext("alice"), testRequest())
+	if err == nil || source.calls() != 2 {
+		t.Fatalf("failure was cached err=%v calls=%d", err, source.calls())
+	}
+}
+
+func TestServiceFollowerHonorsCancellation(t *testing.T) {
+	source := &fakeSource{started: make(chan struct{}, 1), release: make(chan struct{}), result: prometheus.QueryResult{Status: "success", Data: []byte(`{}`)}}
+	service := testService(t, source, nil)
+	go func() { _, _ = service.ExecuteRange(userContext("alice"), testRequest()) }()
+	<-source.started
+	ctx, cancel := context.WithCancel(userContext("alice"))
+	cancel()
+	if _, err := service.ExecuteRange(ctx, testRequest()); !errors.Is(err, context.Canceled) {
+		t.Fatalf("err=%v", err)
+	}
+	close(source.release)
+}
+
+func TestServiceInstantReturnsProvenanceAndFreshness(t *testing.T) {
+	source := &fakeSource{result: prometheus.QueryResult{Status: "success", Data: []byte(`{"resultType":"vector","result":[]`)}}
+	service := testService(t, source, nil)
+	response, err := service.ExecuteInstant(userContext("alice"), InstantRequest{Metric: "host.cpu.utilization", Scope: map[string]string{"serverId": "server-1"}, At: time.Date(2026, 8, 1, 12, 0, 0, 0, time.UTC)})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if response.Provenance.Source != "prometheus" || response.Provenance.CatalogVersion == "" || response.Freshness != "fresh" || source.instantCalls != 1 {
+		t.Fatalf("response=%+v calls=%d", response, source.instantCalls)
+	}
+}
+
+func TestClassifyFreshnessNeverReportsMissingOrStaleAsFresh(t *testing.T) {
+	registry, err := metriccatalog.DefaultRegistry()
+	if err != nil {
+		t.Fatal(err)
+	}
+	definition, ok := registry.Find("container.cpu.utilization")
+	if !ok {
+		t.Fatal("seed metric is missing")
+	}
+	window := FreshnessWindow(definition)
+	if window != 30*time.Second {
+		t.Fatalf("window=%s", window)
+	}
+	now := time.Date(2026, 8, 1, 12, 0, 0, 0, time.UTC)
+	if state := ClassifyFreshness(now.Add(-window), now, window); state != FreshnessFresh {
+		t.Fatalf("state=%s", state)
+	}
+	if state := ClassifyFreshness(now.Add(-window-time.Second), now, window); state != FreshnessStale {
+		t.Fatalf("state=%s", state)
+	}
+	if state := ClassifyFreshness(time.Time{}, now, window); state != FreshnessUnknown {
+		t.Fatalf("state=%s", state)
+	}
+	if state := ClassifyFreshness(now, now, 0); state != FreshnessUnknown {
+		t.Fatalf("state=%s", state)
+	}
+}
+
+func TestCacheIsBoundedByEntriesAndBytes(t *testing.T) {
+	cache := NewCache(1, 1024)
+	response := Response{Data: []byte(`{"large":"payload"}`)}
+	cache.Put("a", response, time.Minute)
+	cache.Put("b", response, time.Minute)
+	if cache.Len() != 1 || cache.Bytes() <= 0 {
+		t.Fatalf("len=%d bytes=%d", cache.Len(), cache.Bytes())
+	}
+}
+
+func operatorContext(subject string) context.Context {
+	return auth.WithPrincipal(context.Background(), auth.Principal{Subject: subject, Role: auth.RoleOperator})
+}
+
+func TestInspectorIsPermissionedAndWarningsAreRedacted(t *testing.T) {
+	source := &fakeSource{result: prometheus.QueryResult{Status: "success", Data: []byte("{}"), Warnings: []string{"authorization=Bearer abc123"}}}
+	service := testService(t, source, NewCache(8, 1<<20))
+	viewer, err := service.ExecuteRange(userContext("viewer"), testRequest())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if viewer.Inspector != nil {
+		t.Fatal("viewer received query inspector")
+	}
+	if len(viewer.Warnings) != 1 || strings.Contains(viewer.Warnings[0], "abc123") {
+		t.Fatalf("warnings leaked: %#v", viewer.Warnings)
+	}
+	operator, err := service.ExecuteRange(operatorContext("operator"), testRequest())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if operator.Inspector == nil || operator.Inspector.SemanticMetric != "container.cpu.utilization" || operator.Inspector.GeneratedQuery == "" {
+		t.Fatalf("inspector missing: %+v", operator.Inspector)
+	}
+	if operator.Inspector.Cost.Series < 1 || operator.Inspector.Cost.Points < 1 {
+		t.Fatalf("unexpected cost: %+v", operator.Inspector.Cost)
+	}
+	if strings.Contains(operator.Inspector.GeneratedQuery, "abc123") || strings.Contains(strings.Join(operator.Warnings, " "), "abc123") {
+		t.Fatal("secret fixture leaked")
+	}
+}
+
+func BenchmarkServiceCacheHit(b *testing.B) {
+	source := &fakeSource{result: prometheus.QueryResult{Status: "success", Data: []byte(`{}`)}}
+	registry, err := metriccatalog.DefaultRegistry()
+	if err != nil {
+		b.Fatal(err)
+	}
+	service := NewService(queryplan.NewPlanner(registry, queryplan.Limits{}), source, NewCache(8, 1<<20))
+	ctx := userContext("benchmark")
+	request := testRequest()
+	if _, err := service.ExecuteRange(ctx, request); err != nil {
+		b.Fatal(err)
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		if _, err := service.ExecuteRange(ctx, request); err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestServiceCacheHitP95Budget(t *testing.T) {
+	source := &fakeSource{result: prometheus.QueryResult{Status: "success", Data: []byte(`{}`)}}
+	service := testService(t, source, NewCache(8, 1<<20))
+	ctx := userContext("p95")
+	request := testRequest()
+	if _, err := service.ExecuteRange(ctx, request); err != nil {
+		t.Fatal(err)
+	}
+	durations := make([]time.Duration, 200)
+	for i := range durations {
+		started := time.Now()
+		if _, err := service.ExecuteRange(ctx, request); err != nil {
+			t.Fatal(err)
+		}
+		durations[i] = time.Since(started)
+	}
+	sort.Slice(durations, func(i, j int) bool { return durations[i] < durations[j] })
+	if p95 := durations[189]; p95 > 750*time.Millisecond {
+		t.Fatalf("local cache p95=%s exceeds 750ms budget", p95)
+	}
+}