Initial public ModelForge release
This commit is contained in:
Jens
@@ -0,0 +1,5 @@
|
||||
__pycache__
|
||||
*.pyc
|
||||
.pytest_cache
|
||||
.ruff_cache
|
||||
tests
|
||||
@@ -0,0 +1,52 @@
|
||||
FROM python:3.12-alpine3.23@sha256:31a768b01976652c222e318fe5bd6e7c252f056cbf489c88fa256f1bf0af58e3
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1
|
||||
|
||||
# PostgreSQL 17 client tools. ModelForge owns its own consistent logical backup and restore and
|
||||
# must never fall back to copying a live data directory, so pg_dump/pg_restore/psql ship with the
|
||||
# control plane and are pinned to the same major version as the server. Upgrade the signed Alpine
|
||||
# repository packages during the build so a digest-pinned base does not retain already-fixed CVEs.
|
||||
ARG POSTGRES_MAJOR=17
|
||||
RUN set -eux; \
|
||||
apk upgrade --no-cache; \
|
||||
apk add --no-cache ca-certificates "postgresql${POSTGRES_MAJOR}-client"
|
||||
|
||||
# Build identity. These are stamped in at build time so a running container can say exactly
|
||||
# where it came from; an argument that is never passed stays empty and is reported as null
|
||||
# rather than becoming a claimed commit.
|
||||
ARG MODELFORGE_VERSION=0.0.0
|
||||
ARG MODELFORGE_COMMIT=""
|
||||
ARG MODELFORGE_BUILT_AT=""
|
||||
ENV MODELFORGE_BUILD_COMMIT=${MODELFORGE_COMMIT}
|
||||
ENV MODELFORGE_BUILD_TIMESTAMP=${MODELFORGE_BUILT_AT}
|
||||
LABEL org.opencontainers.image.title="ITWorx ModelForge control plane"
|
||||
LABEL org.opencontainers.image.description="Capability-first local AI ModelOps and GPU control plane"
|
||||
LABEL org.opencontainers.image.version="${MODELFORGE_VERSION}"
|
||||
LABEL org.opencontainers.image.revision="${MODELFORGE_COMMIT}"
|
||||
LABEL org.opencontainers.image.created="${MODELFORGE_BUILT_AT}"
|
||||
LABEL org.opencontainers.image.source="https://git.example.com/example/modelforge.git"
|
||||
LABEL org.opencontainers.image.vendor="ITWorx"
|
||||
LABEL org.opencontainers.image.licenses="AGPL-3.0-or-later"
|
||||
|
||||
WORKDIR /app
|
||||
COPY pyproject.toml ./
|
||||
COPY src ./src
|
||||
COPY alembic.ini ./
|
||||
COPY alembic ./alembic
|
||||
# Upgrade the installer before it resolves anything: the pinned base image ships a pip
|
||||
# carrying archive-extraction advisories. pip never runs at runtime, but a release image
|
||||
# should not carry a known-vulnerable installer.
|
||||
RUN pip install --no-cache-dir --upgrade pip "setuptools>=78.1.1" "msgpack>=1.2.1" \
|
||||
&& pip install --no-cache-dir . \
|
||||
&& pip check \
|
||||
&& python -m pip uninstall --yes pip setuptools
|
||||
|
||||
RUN addgroup -S modelforge && adduser -S -G modelforge -h /app modelforge \
|
||||
&& mkdir -p /data/state /data/hf-cache /data/artifacts /data/quarantine \
|
||||
/data/backups /data/restore \
|
||||
&& chown -R modelforge:modelforge /app /data
|
||||
USER modelforge
|
||||
|
||||
EXPOSE 8000
|
||||
CMD ["uvicorn", "modelforge_api.main:app", "--host", "0.0.0.0", "--port", "8000"]
|
||||
@@ -0,0 +1,34 @@
|
||||
[alembic]
|
||||
script_location = alembic
|
||||
prepend_sys_path = src
|
||||
# Left empty deliberately. The URL comes from -x db_url=..., from a programmatic
|
||||
# set_main_option, or from MODELFORGE_DATABASE_URL — in that order. A hardcoded value
|
||||
# here is one an operator can migrate the wrong database with.
|
||||
sqlalchemy.url =
|
||||
|
||||
[loggers]
|
||||
keys = root,sqlalchemy,alembic
|
||||
[handlers]
|
||||
keys = console
|
||||
[formatters]
|
||||
keys = generic
|
||||
[logger_root]
|
||||
level = WARN
|
||||
handlers = console
|
||||
qualname =
|
||||
[logger_sqlalchemy]
|
||||
level = WARN
|
||||
handlers =
|
||||
qualname = sqlalchemy.engine
|
||||
[logger_alembic]
|
||||
level = INFO
|
||||
handlers =
|
||||
qualname = alembic
|
||||
[handler_console]
|
||||
class = StreamHandler
|
||||
args = (sys.stderr,)
|
||||
level = NOTSET
|
||||
formatter = generic
|
||||
[formatter_generic]
|
||||
format = %(levelname)-5.5s [%(name)s] %(message)s
|
||||
datefmt = %H:%M:%S
|
||||
@@ -0,0 +1,76 @@
|
||||
from logging.config import fileConfig
|
||||
|
||||
from sqlalchemy import engine_from_config, pool
|
||||
|
||||
from alembic import context
|
||||
from modelforge_api.persistence.models import Base
|
||||
from modelforge_api.settings import get_settings
|
||||
|
||||
config = context.config
|
||||
if config.config_file_name is not None:
|
||||
fileConfig(config.config_file_name)
|
||||
|
||||
|
||||
def _database_url() -> str:
|
||||
"""Where the migration runs, in order of explicitness.
|
||||
|
||||
1. `-x db_url=...` on the command line
|
||||
2. a URL the caller set programmatically, or in alembic.ini
|
||||
3. MODELFORGE_MIGRATION_DATABASE_URL
|
||||
4. MODELFORGE_DATABASE_URL only outside production
|
||||
|
||||
env.py used to overwrite whatever the caller had set with the settings default
|
||||
unconditionally, so both explicit forms were silently discarded. A bootstrap or upgrade
|
||||
rehearsal aimed at an isolated copy would have migrated the deployment's own database while
|
||||
reporting success against the copy.
|
||||
"""
|
||||
|
||||
supplied = context.get_x_argument(as_dictionary=True).get("db_url")
|
||||
if supplied:
|
||||
return str(supplied)
|
||||
configured = config.get_main_option("sqlalchemy.url", None)
|
||||
if configured:
|
||||
return configured
|
||||
settings = get_settings()
|
||||
if settings.migration_database_url is not None:
|
||||
return settings.migration_database_url.get_secret_value()
|
||||
if settings.env == "production":
|
||||
raise RuntimeError(
|
||||
"production migrations require MODELFORGE_MIGRATION_DATABASE_URL; the API runtime "
|
||||
"credential is intentionally not a migration credential"
|
||||
)
|
||||
return settings.database_url
|
||||
|
||||
|
||||
config.set_main_option("sqlalchemy.url", _database_url())
|
||||
target_metadata = Base.metadata
|
||||
|
||||
|
||||
def run_migrations_offline() -> None:
|
||||
context.configure(
|
||||
url=config.get_main_option("sqlalchemy.url"),
|
||||
target_metadata=target_metadata,
|
||||
literal_binds=True,
|
||||
dialect_opts={"paramstyle": "named"},
|
||||
compare_type=True,
|
||||
)
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
def run_migrations_online() -> None:
|
||||
connectable = engine_from_config(
|
||||
config.get_section(config.config_ini_section, {}),
|
||||
prefix="sqlalchemy.",
|
||||
poolclass=pool.NullPool,
|
||||
)
|
||||
with connectable.connect() as connection:
|
||||
context.configure(connection=connection, target_metadata=target_metadata, compare_type=True)
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
if context.is_offline_mode():
|
||||
run_migrations_offline()
|
||||
else:
|
||||
run_migrations_online()
|
||||
@@ -0,0 +1,20 @@
|
||||
"""${message}
|
||||
|
||||
Revision ID: ${up_revision}
|
||||
Revises: ${down_revision | comma,n}
|
||||
Create Date: ${create_date}
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
${imports if imports else ""}
|
||||
revision: str = ${repr(up_revision)}
|
||||
down_revision: Union[str, None] = ${repr(down_revision)}
|
||||
branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
|
||||
depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
|
||||
|
||||
def upgrade() -> None:
|
||||
${upgrades if upgrades else "pass"}
|
||||
|
||||
def downgrade() -> None:
|
||||
${downgrades if downgrades else "pass"}
|
||||
@@ -0,0 +1,222 @@
|
||||
"""M0 domain foundation.
|
||||
|
||||
Revision ID: 20260824_0001
|
||||
Revises: None
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from alembic import op
|
||||
from modelforge_api.persistence.models import Base
|
||||
|
||||
revision = "20260824_0001"
|
||||
down_revision = None
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# The declarative metadata is the reviewed M0 schema contract. create_all is
|
||||
# used only from this baseline migration; later changes require explicit ops.
|
||||
Base.metadata.create_all(bind=op.get_bind(), checkfirst=False)
|
||||
# Freeze the original M0 shape even though the declarative metadata evolves.
|
||||
# Remove newer foreign-key columns before dropping the tables they target.
|
||||
for column in (
|
||||
"profile_config",
|
||||
"health_contract",
|
||||
"device_policy",
|
||||
"modality",
|
||||
"dtype",
|
||||
"version",
|
||||
"artifact_set_id",
|
||||
"runtime_environment_id",
|
||||
):
|
||||
op.drop_column("runtime_profiles", column)
|
||||
for table in (
|
||||
# Later milestone tables are present in the evolving declarative metadata
|
||||
# used to build this baseline. Drop dependants before their M5/M0 targets
|
||||
# so a fresh upgrade still reconstructs each historical revision exactly.
|
||||
"audit_chain_heads",
|
||||
"node_decommission_operations",
|
||||
# M15 recovery tables come first: artifact_recovery_operations references
|
||||
# artifact_jobs, download_plans, artifact_sets, model_revisions and storage_roots,
|
||||
# all of which are dropped further down this list.
|
||||
"artifact_recovery_operations",
|
||||
"restore_operation_events",
|
||||
"restore_operations",
|
||||
"restore_plans",
|
||||
"backup_manifest_entries",
|
||||
"backup_sets",
|
||||
"recovery_asset_records",
|
||||
"recovery_policy_revisions",
|
||||
"incident_timeline_events",
|
||||
"alert_history_events",
|
||||
"operational_alerts",
|
||||
"operational_incidents",
|
||||
"maintenance_windows",
|
||||
"alert_rule_revisions",
|
||||
"slo_evaluations",
|
||||
"slo_policy_revisions",
|
||||
"service_level_indicators",
|
||||
"capacity_aggregates",
|
||||
"capacity_snapshots",
|
||||
"migration_events",
|
||||
"migration_cutover_operations",
|
||||
"migration_rollback_snapshots",
|
||||
"migration_shadow_sessions",
|
||||
"migration_validation_snapshots",
|
||||
"migration_batch_checkpoints",
|
||||
"migration_plans",
|
||||
"migration_validation_policy_revisions",
|
||||
"artifact_location_removal_records",
|
||||
"lifecycle_canary_runs",
|
||||
"lifecycle_operations",
|
||||
"lifecycle_rollback_snapshots",
|
||||
"lifecycle_promotion_plans",
|
||||
"lifecycle_approval_evidence",
|
||||
"lifecycle_approval_requests",
|
||||
"lifecycle_cleanup_plans",
|
||||
"lifecycle_retention_records",
|
||||
"retention_policy_revisions",
|
||||
"lifecycle_events",
|
||||
"lifecycle_policy_revisions",
|
||||
"lifecycle_subjects",
|
||||
"project_fit_evidence",
|
||||
"scheduler_evictions",
|
||||
"co_residency_evidence",
|
||||
"placement_plans",
|
||||
"scheduler_policy_revisions",
|
||||
"scheduler_accelerator_states",
|
||||
"capability_evaluation_runs",
|
||||
"capability_evaluation_suites",
|
||||
"reranking_case_results",
|
||||
"reranking_evaluation_runs",
|
||||
"retrieval_candidate_pools",
|
||||
"advisor_recommendations",
|
||||
"retrieval_pipeline_identities",
|
||||
"discovery_candidate_assessments",
|
||||
"advisor_policies",
|
||||
"model_comparisons",
|
||||
"evaluation_comparisons",
|
||||
"evaluation_case_results",
|
||||
"evaluation_runs",
|
||||
"evaluation_cases",
|
||||
"evaluation_suite_revisions",
|
||||
"evaluation_suites",
|
||||
"embedding_migrations",
|
||||
"project_evaluation_bindings",
|
||||
"serving_gpu_leases",
|
||||
"serving_jobs",
|
||||
"gateway_requests",
|
||||
"residency_allocations",
|
||||
"service_credentials",
|
||||
"service_clients",
|
||||
"capability_resource_envelopes",
|
||||
"capability_experiment_routes",
|
||||
"capability_deployments",
|
||||
"embedding_spaces",
|
||||
"production_execution_approvals",
|
||||
"deployment_candidates",
|
||||
"runtime_probe_metrics",
|
||||
"runtime_probes",
|
||||
"execution_approvals",
|
||||
"runtime_compatibility_assessments",
|
||||
"artifact_set_members",
|
||||
"artifact_inspections",
|
||||
"artifact_job_attempts",
|
||||
"artifact_jobs",
|
||||
"download_plan_files",
|
||||
"download_plans",
|
||||
"artifact_sets",
|
||||
"upstream_files",
|
||||
"upstream_snapshots",
|
||||
"artifact_locations",
|
||||
"derived_artifact_sources",
|
||||
"storage_roots",
|
||||
):
|
||||
op.drop_table(table)
|
||||
op.drop_table("runtime_environments")
|
||||
for column in ("source_type", "upstream_metadata", "local_metadata", "interpretation_metadata", "description"):
|
||||
op.drop_column("models", column)
|
||||
op.drop_column("model_revisions", "archived_at")
|
||||
op.drop_column("audit_events", "canonical_payload")
|
||||
op.drop_column("audit_events", "hash_format")
|
||||
for column in ("status", "verification_details", "archived_at"):
|
||||
op.drop_column("model_artifacts", column)
|
||||
for column in ("transformation_type", "environment_snapshot", "status", "verification_details"):
|
||||
op.drop_column("derived_artifacts", column)
|
||||
for table in (
|
||||
"node_credentials",
|
||||
"node_enrollments",
|
||||
"hardware_inventory_runs",
|
||||
"storage_volume_states",
|
||||
"accelerator_telemetry_latest",
|
||||
"host_telemetry_latest",
|
||||
):
|
||||
op.drop_table(table)
|
||||
for column in (
|
||||
"inventory_at",
|
||||
"last_seen_at",
|
||||
"first_seen_at",
|
||||
"inventory_source",
|
||||
"status_reason",
|
||||
"status",
|
||||
"mig_mode_current",
|
||||
"total_vram_bytes",
|
||||
"compute_capability_minor",
|
||||
"compute_capability_major",
|
||||
"architecture",
|
||||
"vendor",
|
||||
"pci_bus_id",
|
||||
):
|
||||
op.drop_column("accelerators", column)
|
||||
for column in (
|
||||
"decommissioned_by",
|
||||
"decommission_reason",
|
||||
"decommissioned_at",
|
||||
"generation",
|
||||
"telemetry_sequence",
|
||||
"inventory_sequence",
|
||||
"last_connection_error",
|
||||
"last_telemetry_observed_at",
|
||||
"last_telemetry_received_at",
|
||||
"last_inventory_observed_at",
|
||||
"last_inventory_received_at",
|
||||
"last_heartbeat_at",
|
||||
"agent_started_at",
|
||||
"agent_capabilities",
|
||||
"agent_protocol_version",
|
||||
"liveness_state",
|
||||
"observation_source",
|
||||
"benchmark_eligible",
|
||||
"lab_eligible",
|
||||
"production_eligible",
|
||||
"labels",
|
||||
"role",
|
||||
"enabled",
|
||||
):
|
||||
op.drop_column("compute_nodes", column)
|
||||
for column in (
|
||||
"inventory_at",
|
||||
"first_seen_at",
|
||||
"status_reason",
|
||||
"agent_version",
|
||||
"total_ram_bytes",
|
||||
"physical_core_count",
|
||||
"logical_cpu_count",
|
||||
"cpu_model",
|
||||
"kernel_version",
|
||||
"architecture",
|
||||
"os_version",
|
||||
"os_name",
|
||||
"identity_source",
|
||||
"display_name",
|
||||
):
|
||||
op.drop_column("compute_nodes", column)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
existing = set(sa.inspect(op.get_bind()).get_table_names())
|
||||
for table in reversed(Base.metadata.sorted_tables):
|
||||
if table.name in existing:
|
||||
op.drop_table(table.name)
|
||||
@@ -0,0 +1,194 @@
|
||||
"""M1 hardware plane persistence.
|
||||
|
||||
Revision ID: 20260824_0002
|
||||
Revises: 20260824_0001
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision = "20260824_0002"
|
||||
down_revision = "20260824_0001"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
for column in (
|
||||
sa.Column("display_name", sa.String(255), nullable=False, server_default=""),
|
||||
sa.Column("identity_source", sa.String(32), nullable=False, server_default="unknown"),
|
||||
sa.Column("os_name", sa.String(128)),
|
||||
sa.Column("os_version", sa.Text()),
|
||||
sa.Column("architecture", sa.String(128)),
|
||||
sa.Column("kernel_version", sa.String(255)),
|
||||
sa.Column("cpu_model", sa.String(255)),
|
||||
sa.Column("logical_cpu_count", sa.Integer()),
|
||||
sa.Column("physical_core_count", sa.Integer()),
|
||||
sa.Column("total_ram_bytes", sa.BigInteger()),
|
||||
sa.Column("agent_version", sa.String(64)),
|
||||
sa.Column("status_reason", sa.Text()),
|
||||
sa.Column(
|
||||
"first_seen_at",
|
||||
sa.DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=sa.func.now(),
|
||||
),
|
||||
sa.Column("inventory_at", sa.DateTime(timezone=True)),
|
||||
):
|
||||
op.add_column("compute_nodes", column)
|
||||
for column in (
|
||||
sa.Column("pci_bus_id", sa.String(64)),
|
||||
sa.Column("vendor", sa.String(64), nullable=False, server_default="NVIDIA"),
|
||||
sa.Column("architecture", sa.String(64)),
|
||||
sa.Column("compute_capability_major", sa.Integer()),
|
||||
sa.Column("compute_capability_minor", sa.Integer()),
|
||||
sa.Column("total_vram_bytes", sa.BigInteger()),
|
||||
sa.Column("mig_mode_current", sa.Boolean()),
|
||||
sa.Column("status", sa.String(32), nullable=False, server_default="active"),
|
||||
sa.Column("status_reason", sa.Text()),
|
||||
sa.Column("inventory_source", sa.String(32), nullable=False, server_default="nvidia_nvml"),
|
||||
sa.Column(
|
||||
"first_seen_at",
|
||||
sa.DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=sa.func.now(),
|
||||
),
|
||||
sa.Column("last_seen_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("inventory_at", sa.DateTime(timezone=True)),
|
||||
):
|
||||
op.add_column("accelerators", column)
|
||||
op.create_table(
|
||||
"host_telemetry_latest",
|
||||
sa.Column(
|
||||
"compute_node_id",
|
||||
sa.Uuid(),
|
||||
sa.ForeignKey("compute_nodes.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
unique=True,
|
||||
),
|
||||
sa.Column("available_ram_bytes", sa.BigInteger()),
|
||||
sa.Column("availability", sa.JSON(), nullable=False),
|
||||
sa.Column("observed_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("id", sa.Uuid(), primary_key=True),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_host_telemetry_latest_compute_node_id",
|
||||
"host_telemetry_latest",
|
||||
["compute_node_id"],
|
||||
unique=True,
|
||||
)
|
||||
op.create_table(
|
||||
"accelerator_telemetry_latest",
|
||||
sa.Column(
|
||||
"accelerator_id",
|
||||
sa.Uuid(),
|
||||
sa.ForeignKey("accelerators.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
unique=True,
|
||||
),
|
||||
sa.Column("used_vram_bytes", sa.BigInteger()),
|
||||
sa.Column("free_vram_bytes", sa.BigInteger()),
|
||||
sa.Column("gpu_utilization_percent", sa.Integer()),
|
||||
sa.Column("memory_utilization_percent", sa.Integer()),
|
||||
sa.Column("temperature_c", sa.Integer()),
|
||||
sa.Column("power_draw_w", sa.Float()),
|
||||
sa.Column("power_limit_w", sa.Float()),
|
||||
sa.Column("graphics_clock_mhz", sa.Integer()),
|
||||
sa.Column("memory_clock_mhz", sa.Integer()),
|
||||
sa.Column("fan_speed_percent", sa.Integer()),
|
||||
sa.Column("performance_state", sa.String(32)),
|
||||
sa.Column("availability", sa.JSON(), nullable=False),
|
||||
sa.Column("observed_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("id", sa.Uuid(), primary_key=True),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_accelerator_telemetry_latest_accelerator_id",
|
||||
"accelerator_telemetry_latest",
|
||||
["accelerator_id"],
|
||||
unique=True,
|
||||
)
|
||||
op.create_table(
|
||||
"storage_volume_states",
|
||||
sa.Column(
|
||||
"compute_node_id",
|
||||
sa.Uuid(),
|
||||
sa.ForeignKey("compute_nodes.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("purpose", sa.String(64), nullable=False),
|
||||
sa.Column("path", sa.Text(), nullable=False),
|
||||
sa.Column("total_bytes", sa.BigInteger()),
|
||||
sa.Column("used_bytes", sa.BigInteger()),
|
||||
sa.Column("free_bytes", sa.BigInteger()),
|
||||
sa.Column("availability", sa.JSON(), nullable=False),
|
||||
sa.Column("observed_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("id", sa.Uuid(), primary_key=True),
|
||||
sa.UniqueConstraint("compute_node_id", "purpose", "path", name="uq_node_storage_path"),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_storage_volume_states_compute_node_id", "storage_volume_states", ["compute_node_id"]
|
||||
)
|
||||
op.create_table(
|
||||
"hardware_inventory_runs",
|
||||
sa.Column(
|
||||
"compute_node_id", sa.Uuid(), sa.ForeignKey("compute_nodes.id", ondelete="SET NULL")
|
||||
),
|
||||
sa.Column("status", sa.String(32), nullable=False),
|
||||
sa.Column("source", sa.String(64), nullable=False),
|
||||
sa.Column("fingerprint", sa.String(64)),
|
||||
sa.Column("summary", sa.JSON(), nullable=False),
|
||||
sa.Column("error", sa.Text()),
|
||||
sa.Column("started_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("completed_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("id", sa.Uuid(), primary_key=True),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_hardware_inventory_runs_compute_node_id", "hardware_inventory_runs", ["compute_node_id"]
|
||||
)
|
||||
op.create_index(
|
||||
"ix_hardware_inventory_runs_fingerprint", "hardware_inventory_runs", ["fingerprint"]
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
for table in (
|
||||
"hardware_inventory_runs",
|
||||
"storage_volume_states",
|
||||
"accelerator_telemetry_latest",
|
||||
"host_telemetry_latest",
|
||||
):
|
||||
op.drop_table(table)
|
||||
for column in (
|
||||
"inventory_at",
|
||||
"last_seen_at",
|
||||
"first_seen_at",
|
||||
"inventory_source",
|
||||
"status_reason",
|
||||
"status",
|
||||
"mig_mode_current",
|
||||
"total_vram_bytes",
|
||||
"compute_capability_minor",
|
||||
"compute_capability_major",
|
||||
"architecture",
|
||||
"vendor",
|
||||
"pci_bus_id",
|
||||
):
|
||||
op.drop_column("accelerators", column)
|
||||
for column in (
|
||||
"inventory_at",
|
||||
"first_seen_at",
|
||||
"status_reason",
|
||||
"agent_version",
|
||||
"total_ram_bytes",
|
||||
"physical_core_count",
|
||||
"logical_cpu_count",
|
||||
"cpu_model",
|
||||
"kernel_version",
|
||||
"architecture",
|
||||
"os_version",
|
||||
"os_name",
|
||||
"identity_source",
|
||||
"display_name",
|
||||
):
|
||||
op.drop_column("compute_nodes", column)
|
||||
@@ -0,0 +1,141 @@
|
||||
"""M1.5 remote compute node agent.
|
||||
|
||||
Revision ID: 20260825_0003
|
||||
Revises: 20260824_0002
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision = "20260825_0003"
|
||||
down_revision = "20260824_0002"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
for column in (
|
||||
sa.Column("enabled", sa.Boolean(), nullable=False, server_default=sa.true()),
|
||||
sa.Column("role", sa.String(64)),
|
||||
sa.Column("labels", sa.JSON(), nullable=False, server_default=sa.text("'{}'")),
|
||||
sa.Column("production_eligible", sa.Boolean(), nullable=False, server_default=sa.false()),
|
||||
sa.Column("lab_eligible", sa.Boolean(), nullable=False, server_default=sa.true()),
|
||||
sa.Column("benchmark_eligible", sa.Boolean(), nullable=False, server_default=sa.false()),
|
||||
sa.Column(
|
||||
"observation_source",
|
||||
sa.String(32),
|
||||
nullable=False,
|
||||
server_default="local_control_plane",
|
||||
),
|
||||
sa.Column("liveness_state", sa.String(32), nullable=False, server_default="offline"),
|
||||
sa.Column("agent_protocol_version", sa.Integer()),
|
||||
sa.Column("agent_capabilities", sa.JSON(), nullable=False, server_default=sa.text("'[]'")),
|
||||
sa.Column("agent_started_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("last_heartbeat_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("last_inventory_received_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("last_inventory_observed_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("last_telemetry_received_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("last_telemetry_observed_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("last_connection_error", sa.Text()),
|
||||
sa.Column("inventory_sequence", sa.BigInteger(), nullable=False, server_default="0"),
|
||||
sa.Column("telemetry_sequence", sa.BigInteger(), nullable=False, server_default="0"),
|
||||
):
|
||||
op.add_column("compute_nodes", column)
|
||||
|
||||
for table in (
|
||||
"host_telemetry_latest",
|
||||
"accelerator_telemetry_latest",
|
||||
"storage_volume_states",
|
||||
):
|
||||
op.add_column(table, sa.Column("received_at", sa.DateTime(timezone=True)))
|
||||
for column in (
|
||||
sa.Column("observation_sequence", sa.BigInteger()),
|
||||
sa.Column("observed_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("received_at", sa.DateTime(timezone=True)),
|
||||
):
|
||||
op.add_column("hardware_inventory_runs", column)
|
||||
|
||||
op.create_table(
|
||||
"node_enrollments",
|
||||
sa.Column("token_hash", sa.String(64), nullable=False, unique=True),
|
||||
sa.Column("scope", sa.String(64), nullable=False),
|
||||
sa.Column("requested_metadata", sa.JSON(), nullable=False),
|
||||
sa.Column(
|
||||
"created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()
|
||||
),
|
||||
sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("used_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("revoked_at", sa.DateTime(timezone=True)),
|
||||
sa.Column(
|
||||
"enrolled_node_id",
|
||||
sa.Uuid(),
|
||||
sa.ForeignKey("compute_nodes.id", ondelete="SET NULL"),
|
||||
),
|
||||
sa.Column("id", sa.Uuid(), primary_key=True),
|
||||
)
|
||||
op.create_index("ix_node_enrollments_expires_at", "node_enrollments", ["expires_at"])
|
||||
op.create_index(
|
||||
"ix_node_enrollments_enrolled_node_id", "node_enrollments", ["enrolled_node_id"]
|
||||
)
|
||||
op.create_table(
|
||||
"node_credentials",
|
||||
sa.Column(
|
||||
"compute_node_id",
|
||||
sa.Uuid(),
|
||||
sa.ForeignKey("compute_nodes.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("secret_hash", sa.String(64), nullable=False, unique=True),
|
||||
sa.Column("scope", sa.String(64), nullable=False),
|
||||
sa.Column(
|
||||
"created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()
|
||||
),
|
||||
sa.Column("last_used_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("revoked_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("id", sa.Uuid(), primary_key=True),
|
||||
)
|
||||
op.create_index("ix_node_credentials_compute_node_id", "node_credentials", ["compute_node_id"])
|
||||
op.create_index(
|
||||
"uq_active_node_credential",
|
||||
"node_credentials",
|
||||
["compute_node_id"],
|
||||
unique=True,
|
||||
postgresql_where=sa.text("revoked_at IS NULL"),
|
||||
sqlite_where=sa.text("revoked_at IS NULL"),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("node_credentials")
|
||||
op.drop_table("node_enrollments")
|
||||
for column in ("received_at", "observed_at", "observation_sequence"):
|
||||
op.drop_column("hardware_inventory_runs", column)
|
||||
for table in (
|
||||
"storage_volume_states",
|
||||
"accelerator_telemetry_latest",
|
||||
"host_telemetry_latest",
|
||||
):
|
||||
op.drop_column(table, "received_at")
|
||||
for column in (
|
||||
"telemetry_sequence",
|
||||
"inventory_sequence",
|
||||
"last_connection_error",
|
||||
"last_telemetry_observed_at",
|
||||
"last_telemetry_received_at",
|
||||
"last_inventory_observed_at",
|
||||
"last_inventory_received_at",
|
||||
"last_heartbeat_at",
|
||||
"agent_started_at",
|
||||
"agent_capabilities",
|
||||
"agent_protocol_version",
|
||||
"liveness_state",
|
||||
"observation_source",
|
||||
"benchmark_eligible",
|
||||
"lab_eligible",
|
||||
"production_eligible",
|
||||
"labels",
|
||||
"role",
|
||||
"enabled",
|
||||
):
|
||||
op.drop_column("compute_nodes", column)
|
||||
@@ -0,0 +1,170 @@
|
||||
"""M2 operational model registry and artifact provenance.
|
||||
|
||||
Revision ID: 20260825_0004
|
||||
Revises: 20260825_0003
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision = "20260825_0004"
|
||||
down_revision = "20260825_0003"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
for column in (
|
||||
sa.Column("source_type", sa.String(32), nullable=False, server_default="huggingface"),
|
||||
sa.Column("upstream_metadata", sa.JSON(), nullable=False, server_default=sa.text("'{}'")),
|
||||
sa.Column("local_metadata", sa.JSON(), nullable=False, server_default=sa.text("'{}'")),
|
||||
sa.Column(
|
||||
"interpretation_metadata", sa.JSON(), nullable=False, server_default=sa.text("'{}'")
|
||||
),
|
||||
sa.Column("description", sa.Text()),
|
||||
):
|
||||
op.add_column("models", column)
|
||||
op.add_column("model_revisions", sa.Column("archived_at", sa.DateTime(timezone=True)))
|
||||
for column in (
|
||||
sa.Column("status", sa.String(32), nullable=False, server_default="remote"),
|
||||
sa.Column(
|
||||
"verification_details", sa.JSON(), nullable=False, server_default=sa.text("'{}'")
|
||||
),
|
||||
sa.Column("archived_at", sa.DateTime(timezone=True)),
|
||||
):
|
||||
op.add_column("model_artifacts", column)
|
||||
op.alter_column("model_artifacts", "storage_uri", existing_type=sa.Text(), nullable=True)
|
||||
for column in (
|
||||
sa.Column(
|
||||
"transformation_type", sa.String(64), nullable=False, server_default="conversion"
|
||||
),
|
||||
sa.Column(
|
||||
"environment_snapshot", sa.JSON(), nullable=False, server_default=sa.text("'{}'")
|
||||
),
|
||||
sa.Column("status", sa.String(32), nullable=False, server_default="remote"),
|
||||
sa.Column(
|
||||
"verification_details", sa.JSON(), nullable=False, server_default=sa.text("'{}'")
|
||||
),
|
||||
):
|
||||
op.add_column("derived_artifacts", column)
|
||||
op.alter_column("derived_artifacts", "storage_uri", existing_type=sa.Text(), nullable=True)
|
||||
op.alter_column(
|
||||
"derived_artifacts", "source_artifact_id", existing_type=sa.Uuid(), nullable=True
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"storage_roots",
|
||||
sa.Column("compute_node_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("name", sa.String(128), nullable=False),
|
||||
sa.Column("purpose", sa.String(64), nullable=False, server_default="model_artifacts"),
|
||||
sa.Column("path", sa.Text(), nullable=False),
|
||||
sa.Column("status", sa.String(32), nullable=False, server_default="unknown"),
|
||||
sa.Column("writable", sa.Boolean(), nullable=False, server_default=sa.false()),
|
||||
sa.Column("capacity_bytes", sa.BigInteger()),
|
||||
sa.Column("free_bytes", sa.BigInteger()),
|
||||
sa.Column("reserve_bytes", sa.BigInteger(), nullable=False, server_default="0"),
|
||||
sa.Column("reserve_percent", sa.Integer(), nullable=False, server_default="10"),
|
||||
sa.Column("capacity_observed_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("validation_details", sa.JSON(), nullable=False, server_default=sa.text("'{}'")),
|
||||
sa.Column("deprecated_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("id", sa.Uuid(), primary_key=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.ForeignKeyConstraint(["compute_node_id"], ["compute_nodes.id"], ondelete="RESTRICT"),
|
||||
sa.UniqueConstraint("compute_node_id", "path", name="uq_storage_root_node_path"),
|
||||
sa.CheckConstraint("reserve_bytes >= 0", name="ck_storage_root_reserve_bytes"),
|
||||
sa.CheckConstraint(
|
||||
"reserve_percent >= 0 AND reserve_percent <= 100",
|
||||
name="ck_storage_root_reserve_percent",
|
||||
),
|
||||
)
|
||||
op.create_index("ix_storage_roots_compute_node_id", "storage_roots", ["compute_node_id"])
|
||||
op.create_table(
|
||||
"derived_artifact_sources",
|
||||
sa.Column("derived_artifact_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("source_artifact_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("ordinal", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("source_sha256", sa.String(64), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["derived_artifact_id"], ["derived_artifacts.id"], ondelete="RESTRICT"
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["source_artifact_id"], ["model_artifacts.id"], ondelete="RESTRICT"
|
||||
),
|
||||
sa.PrimaryKeyConstraint("derived_artifact_id", "source_artifact_id"),
|
||||
sa.UniqueConstraint(
|
||||
"derived_artifact_id", "source_artifact_id", name="uq_derived_artifact_source"
|
||||
),
|
||||
)
|
||||
op.create_table(
|
||||
"artifact_locations",
|
||||
sa.Column("artifact_id", sa.Uuid()),
|
||||
sa.Column("derived_artifact_id", sa.Uuid()),
|
||||
sa.Column("storage_root_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("relative_path", sa.Text(), nullable=False),
|
||||
sa.Column("status", sa.String(32), nullable=False, server_default="unknown"),
|
||||
sa.Column("size_bytes", sa.BigInteger()),
|
||||
sa.Column("observed_sha256", sa.String(64)),
|
||||
sa.Column("last_checked_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("id", sa.Uuid(), primary_key=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.ForeignKeyConstraint(["artifact_id"], ["model_artifacts.id"], ondelete="RESTRICT"),
|
||||
sa.ForeignKeyConstraint(
|
||||
["derived_artifact_id"], ["derived_artifacts.id"], ondelete="RESTRICT"
|
||||
),
|
||||
sa.ForeignKeyConstraint(["storage_root_id"], ["storage_roots.id"], ondelete="RESTRICT"),
|
||||
sa.CheckConstraint(
|
||||
"(artifact_id IS NOT NULL AND derived_artifact_id IS NULL) OR "
|
||||
"(artifact_id IS NULL AND derived_artifact_id IS NOT NULL)",
|
||||
name="ck_artifact_location_one_owner",
|
||||
),
|
||||
sa.UniqueConstraint(
|
||||
"storage_root_id", "relative_path", name="uq_storage_root_relative_path"
|
||||
),
|
||||
)
|
||||
op.create_index("ix_artifact_locations_artifact_id", "artifact_locations", ["artifact_id"])
|
||||
op.create_index(
|
||||
"ix_artifact_locations_derived_artifact_id",
|
||||
"artifact_locations",
|
||||
["derived_artifact_id"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_artifact_locations_storage_root_id", "artifact_locations", ["storage_root_id"]
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("artifact_locations")
|
||||
op.drop_table("derived_artifact_sources")
|
||||
op.drop_table("storage_roots")
|
||||
op.execute(
|
||||
"UPDATE model_artifacts SET storage_uri = 'registry://artifact/' || id::text "
|
||||
"WHERE storage_uri IS NULL"
|
||||
)
|
||||
op.execute(
|
||||
"UPDATE derived_artifacts SET storage_uri = 'registry://derived/' || id::text "
|
||||
"WHERE storage_uri IS NULL"
|
||||
)
|
||||
op.alter_column("derived_artifacts", "source_artifact_id", existing_type=sa.Uuid(), nullable=False)
|
||||
op.alter_column("derived_artifacts", "storage_uri", existing_type=sa.Text(), nullable=False)
|
||||
for column in (
|
||||
"verification_details",
|
||||
"status",
|
||||
"environment_snapshot",
|
||||
"transformation_type",
|
||||
):
|
||||
op.drop_column("derived_artifacts", column)
|
||||
op.alter_column("model_artifacts", "storage_uri", existing_type=sa.Text(), nullable=False)
|
||||
for column in ("archived_at", "verification_details", "status"):
|
||||
op.drop_column("model_artifacts", column)
|
||||
op.drop_column("model_revisions", "archived_at")
|
||||
for column in (
|
||||
"description",
|
||||
"interpretation_metadata",
|
||||
"local_metadata",
|
||||
"upstream_metadata",
|
||||
"source_type",
|
||||
):
|
||||
op.drop_column("models", column)
|
||||
@@ -0,0 +1,210 @@
|
||||
"""M3 Hugging Face discovery and node-local artifact acquisition.
|
||||
|
||||
Revision ID: 20260825_0005
|
||||
Revises: 20260825_0004
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision = "20260825_0005"
|
||||
down_revision = "20260825_0004"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _identity() -> list[sa.Column]:
|
||||
return [
|
||||
sa.Column("id", sa.Uuid(), primary_key=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
]
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("storage_roots", sa.Column("agent_path", sa.Text()))
|
||||
op.create_table(
|
||||
"upstream_snapshots",
|
||||
sa.Column("model_id", sa.Uuid()),
|
||||
sa.Column("provider", sa.String(64), nullable=False, server_default="huggingface"),
|
||||
sa.Column("repository_id", sa.String(255), nullable=False),
|
||||
sa.Column("requested_revision", sa.String(255), nullable=False, server_default="main"),
|
||||
sa.Column("resolved_commit_sha", sa.String(64), nullable=False),
|
||||
sa.Column("access_state", sa.String(32), nullable=False, server_default="public"),
|
||||
sa.Column("metadata_snapshot", sa.JSON(), nullable=False, server_default=sa.text("'{}'")),
|
||||
sa.Column("card_metadata", sa.JSON(), nullable=False, server_default=sa.text("'{}'")),
|
||||
sa.Column("security_metadata", sa.JSON(), nullable=False, server_default=sa.text("'{}'")),
|
||||
sa.Column("source_updated_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("observed_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
|
||||
sa.Column("stale_after", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("id", sa.Uuid(), primary_key=True),
|
||||
sa.ForeignKeyConstraint(["model_id"], ["models.id"], ondelete="RESTRICT"),
|
||||
sa.CheckConstraint("length(resolved_commit_sha) >= 40", name="ck_snapshot_commit_length"),
|
||||
)
|
||||
op.create_index("ix_upstream_snapshots_model_id", "upstream_snapshots", ["model_id"])
|
||||
op.create_index("ix_upstream_snapshots_repository_id", "upstream_snapshots", ["repository_id"])
|
||||
op.create_index("ix_upstream_snapshots_resolved_commit_sha", "upstream_snapshots", ["resolved_commit_sha"])
|
||||
op.create_table(
|
||||
"upstream_files",
|
||||
sa.Column("snapshot_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("path", sa.Text(), nullable=False),
|
||||
sa.Column("size_bytes", sa.BigInteger()),
|
||||
sa.Column("blob_id", sa.String(128)),
|
||||
sa.Column("upstream_sha256", sa.String(64)),
|
||||
sa.Column("file_format", sa.String(64), nullable=False, server_default="unknown"),
|
||||
sa.Column("role", sa.String(64), nullable=False, server_default="other"),
|
||||
sa.Column("risk_flags", sa.JSON(), nullable=False, server_default=sa.text("'[]'")),
|
||||
sa.Column("metadata_snapshot", sa.JSON(), nullable=False, server_default=sa.text("'{}'")),
|
||||
sa.Column("id", sa.Uuid(), primary_key=True),
|
||||
sa.ForeignKeyConstraint(["snapshot_id"], ["upstream_snapshots.id"], ondelete="RESTRICT"),
|
||||
sa.UniqueConstraint("snapshot_id", "path", name="uq_upstream_file_snapshot_path"),
|
||||
sa.CheckConstraint("size_bytes IS NULL OR size_bytes >= 0", name="ck_upstream_file_size"),
|
||||
)
|
||||
op.create_index("ix_upstream_files_snapshot_id", "upstream_files", ["snapshot_id"])
|
||||
op.create_table(
|
||||
"artifact_sets",
|
||||
sa.Column("revision_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("snapshot_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("variant_key", sa.String(128), nullable=False),
|
||||
sa.Column("label", sa.String(255), nullable=False),
|
||||
sa.Column("selection_reason", sa.Text(), nullable=False),
|
||||
sa.Column("selected_paths", sa.JSON(), nullable=False, server_default=sa.text("'[]'")),
|
||||
sa.Column("total_size_bytes", sa.BigInteger(), nullable=False, server_default="0"),
|
||||
sa.Column("file_count", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("availability", sa.String(32), nullable=False, server_default="remote"),
|
||||
sa.Column("status", sa.String(32), nullable=False, server_default="remote"),
|
||||
sa.Column("completeness", sa.String(32), nullable=False, server_default="planned"),
|
||||
sa.Column("security_status", sa.String(32), nullable=False, server_default="unverified"),
|
||||
sa.Column("license_status", sa.String(32), nullable=False, server_default="unknown"),
|
||||
sa.Column("immutable_at", sa.DateTime(timezone=True), nullable=False),
|
||||
*_identity(),
|
||||
sa.ForeignKeyConstraint(["revision_id"], ["model_revisions.id"], ondelete="RESTRICT"),
|
||||
sa.ForeignKeyConstraint(["snapshot_id"], ["upstream_snapshots.id"], ondelete="RESTRICT"),
|
||||
sa.UniqueConstraint("revision_id", "variant_key", name="uq_artifact_set_revision_variant"),
|
||||
sa.CheckConstraint("total_size_bytes >= 0", name="ck_artifact_set_size"),
|
||||
)
|
||||
op.create_index("ix_artifact_sets_revision_id", "artifact_sets", ["revision_id"])
|
||||
op.create_index("ix_artifact_sets_snapshot_id", "artifact_sets", ["snapshot_id"])
|
||||
op.create_table(
|
||||
"download_plans",
|
||||
sa.Column("artifact_set_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("compute_node_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("storage_root_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("repository_id", sa.String(255), nullable=False),
|
||||
sa.Column("resolved_commit_sha", sa.String(64), nullable=False),
|
||||
sa.Column("total_size_bytes", sa.BigInteger(), nullable=False),
|
||||
sa.Column("file_count", sa.Integer(), nullable=False),
|
||||
sa.Column("status", sa.String(32), nullable=False, server_default="ready"),
|
||||
sa.Column("idempotency_key", sa.String(64), nullable=False),
|
||||
sa.Column("preflight", sa.JSON(), nullable=False, server_default=sa.text("'{}'")),
|
||||
sa.Column("immutable_payload", sa.JSON(), nullable=False),
|
||||
sa.Column("planned_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
|
||||
sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("immutable_at", sa.DateTime(timezone=True), nullable=False),
|
||||
*_identity(),
|
||||
sa.ForeignKeyConstraint(["artifact_set_id"], ["artifact_sets.id"], ondelete="RESTRICT"),
|
||||
sa.ForeignKeyConstraint(["compute_node_id"], ["compute_nodes.id"], ondelete="RESTRICT"),
|
||||
sa.ForeignKeyConstraint(["storage_root_id"], ["storage_roots.id"], ondelete="RESTRICT"),
|
||||
sa.UniqueConstraint("idempotency_key", name="uq_download_plan_idempotency"),
|
||||
sa.CheckConstraint("total_size_bytes >= 0", name="ck_download_plan_size"),
|
||||
)
|
||||
for col in ("artifact_set_id", "compute_node_id", "storage_root_id"):
|
||||
op.create_index(f"ix_download_plans_{col}", "download_plans", [col])
|
||||
op.create_table(
|
||||
"download_plan_files",
|
||||
sa.Column("plan_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("ordinal", sa.Integer(), nullable=False),
|
||||
sa.Column("path", sa.Text(), nullable=False),
|
||||
sa.Column("size_bytes", sa.BigInteger(), nullable=False),
|
||||
sa.Column("upstream_sha256", sa.String(64)),
|
||||
sa.Column("file_format", sa.String(64), nullable=False),
|
||||
sa.Column("role", sa.String(64), nullable=False),
|
||||
sa.Column("risk_flags", sa.JSON(), nullable=False, server_default=sa.text("'[]'")),
|
||||
sa.Column("id", sa.Uuid(), primary_key=True),
|
||||
sa.ForeignKeyConstraint(["plan_id"], ["download_plans.id"], ondelete="RESTRICT"),
|
||||
sa.UniqueConstraint("plan_id", "path", name="uq_download_plan_file_path"),
|
||||
sa.UniqueConstraint("plan_id", "ordinal", name="uq_download_plan_file_ordinal"),
|
||||
)
|
||||
op.create_index("ix_download_plan_files_plan_id", "download_plan_files", ["plan_id"])
|
||||
op.create_table(
|
||||
"artifact_jobs",
|
||||
sa.Column("plan_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("compute_node_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("storage_root_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("status", sa.String(32), nullable=False, server_default="queued"),
|
||||
sa.Column("idempotency_key", sa.String(64), nullable=False),
|
||||
sa.Column("attempt_count", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("lease_token_hash", sa.String(64)),
|
||||
sa.Column("lease_expires_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("progress_bytes", sa.BigInteger(), nullable=False, server_default="0"),
|
||||
sa.Column("total_bytes", sa.BigInteger(), nullable=False),
|
||||
sa.Column("current_file", sa.Text()),
|
||||
sa.Column("cancel_requested", sa.Boolean(), nullable=False, server_default=sa.false()),
|
||||
sa.Column("quarantine_relative_path", sa.Text()),
|
||||
sa.Column("promoted_relative_path", sa.Text()),
|
||||
sa.Column("error_code", sa.String(64)),
|
||||
sa.Column("error_message", sa.Text()),
|
||||
sa.Column("result", sa.JSON(), nullable=False, server_default=sa.text("'{}'")),
|
||||
sa.Column("started_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("completed_at", sa.DateTime(timezone=True)),
|
||||
*_identity(),
|
||||
sa.ForeignKeyConstraint(["plan_id"], ["download_plans.id"], ondelete="RESTRICT"),
|
||||
sa.ForeignKeyConstraint(["compute_node_id"], ["compute_nodes.id"], ondelete="RESTRICT"),
|
||||
sa.ForeignKeyConstraint(["storage_root_id"], ["storage_roots.id"], ondelete="RESTRICT"),
|
||||
sa.UniqueConstraint("plan_id", name="uq_artifact_job_plan"),
|
||||
sa.UniqueConstraint("idempotency_key", name="uq_artifact_job_idempotency"),
|
||||
)
|
||||
for col in ("plan_id", "compute_node_id", "storage_root_id", "status"):
|
||||
op.create_index(f"ix_artifact_jobs_{col}", "artifact_jobs", [col])
|
||||
op.create_table(
|
||||
"artifact_job_attempts",
|
||||
sa.Column("job_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("attempt", sa.Integer(), nullable=False),
|
||||
sa.Column("status", sa.String(32), nullable=False),
|
||||
sa.Column("started_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
|
||||
sa.Column("completed_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("details", sa.JSON(), nullable=False, server_default=sa.text("'{}'")),
|
||||
sa.Column("id", sa.Uuid(), primary_key=True),
|
||||
sa.ForeignKeyConstraint(["job_id"], ["artifact_jobs.id"], ondelete="RESTRICT"),
|
||||
)
|
||||
op.create_index("ix_artifact_job_attempts_job_id", "artifact_job_attempts", ["job_id"])
|
||||
op.create_table(
|
||||
"artifact_inspections",
|
||||
sa.Column("job_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("file_path", sa.Text(), nullable=False),
|
||||
sa.Column("inspection_type", sa.String(64), nullable=False),
|
||||
sa.Column("status", sa.String(32), nullable=False),
|
||||
sa.Column("severity", sa.String(32), nullable=False),
|
||||
sa.Column("evidence", sa.JSON(), nullable=False, server_default=sa.text("'{}'")),
|
||||
sa.Column("observed_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
|
||||
sa.Column("id", sa.Uuid(), primary_key=True),
|
||||
sa.ForeignKeyConstraint(["job_id"], ["artifact_jobs.id"], ondelete="RESTRICT"),
|
||||
)
|
||||
op.create_index("ix_artifact_inspections_job_id", "artifact_inspections", ["job_id"])
|
||||
op.create_table(
|
||||
"artifact_set_members",
|
||||
sa.Column("artifact_set_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("artifact_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("ordinal", sa.Integer(), nullable=False),
|
||||
sa.Column("required", sa.Boolean(), nullable=False, server_default=sa.true()),
|
||||
sa.ForeignKeyConstraint(["artifact_set_id"], ["artifact_sets.id"], ondelete="RESTRICT"),
|
||||
sa.ForeignKeyConstraint(["artifact_id"], ["model_artifacts.id"], ondelete="RESTRICT"),
|
||||
sa.PrimaryKeyConstraint("artifact_set_id", "artifact_id"),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
for table in (
|
||||
"artifact_set_members",
|
||||
"artifact_inspections",
|
||||
"artifact_job_attempts",
|
||||
"artifact_jobs",
|
||||
"download_plan_files",
|
||||
"download_plans",
|
||||
"artifact_sets",
|
||||
"upstream_files",
|
||||
"upstream_snapshots",
|
||||
):
|
||||
op.drop_table(table)
|
||||
op.drop_column("storage_roots", "agent_path")
|
||||
@@ -0,0 +1,221 @@
|
||||
"""M4 runtime compatibility and runtime plane.
|
||||
|
||||
Revision ID: 20260825_0006
|
||||
Revises: 20260825_0005
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision = "20260825_0006"
|
||||
down_revision = "20260825_0005"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _timestamps() -> list[sa.Column]:
|
||||
return [
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
]
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"runtime_environments",
|
||||
sa.Column("name", sa.String(255), nullable=False),
|
||||
sa.Column("adapter", sa.String(64), nullable=False),
|
||||
sa.Column("runtime_version", sa.String(128), nullable=False),
|
||||
sa.Column("image_repository", sa.String(255), nullable=False),
|
||||
sa.Column("image_digest", sa.String(71), nullable=False),
|
||||
sa.Column("python_version", sa.String(64), nullable=False),
|
||||
sa.Column("cuda_runtime_version", sa.String(64)),
|
||||
sa.Column("package_versions", sa.JSON(), nullable=False, server_default=sa.text("'{}'")),
|
||||
sa.Column("supported_model_types", sa.JSON(), nullable=False, server_default=sa.text("'[]'")),
|
||||
sa.Column("supported_formats", sa.JSON(), nullable=False, server_default=sa.text("'[]'")),
|
||||
sa.Column("supported_modalities", sa.JSON(), nullable=False, server_default=sa.text("'[]'")),
|
||||
sa.Column("network_policy", sa.String(64), nullable=False),
|
||||
sa.Column("fingerprint", sa.String(64), nullable=False),
|
||||
sa.Column("immutable_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.Column("id", sa.Uuid(), primary_key=True),
|
||||
sa.UniqueConstraint("fingerprint", name="uq_runtime_environment_fingerprint"),
|
||||
)
|
||||
op.create_index("ix_runtime_environments_adapter", "runtime_environments", ["adapter"])
|
||||
for column in (
|
||||
sa.Column("runtime_environment_id", sa.Uuid()),
|
||||
sa.Column("artifact_set_id", sa.Uuid()),
|
||||
sa.Column("version", sa.Integer(), nullable=False, server_default="1"),
|
||||
sa.Column("dtype", sa.String(32)),
|
||||
sa.Column("modality", sa.String(32)),
|
||||
sa.Column("device_policy", sa.String(32), nullable=False, server_default="cuda_required"),
|
||||
sa.Column("health_contract", sa.JSON(), nullable=False, server_default=sa.text("'{}'")),
|
||||
sa.Column("profile_config", sa.JSON(), nullable=False, server_default=sa.text("'{}'")),
|
||||
):
|
||||
op.add_column("runtime_profiles", column)
|
||||
op.create_foreign_key(
|
||||
"fk_runtime_profiles_environment",
|
||||
"runtime_profiles",
|
||||
"runtime_environments",
|
||||
["runtime_environment_id"],
|
||||
["id"],
|
||||
ondelete="RESTRICT",
|
||||
)
|
||||
op.create_foreign_key(
|
||||
"fk_runtime_profiles_artifact_set",
|
||||
"runtime_profiles",
|
||||
"artifact_sets",
|
||||
["artifact_set_id"],
|
||||
["id"],
|
||||
ondelete="RESTRICT",
|
||||
)
|
||||
op.create_index("ix_runtime_profiles_runtime_environment_id", "runtime_profiles", ["runtime_environment_id"])
|
||||
op.create_index("ix_runtime_profiles_artifact_set_id", "runtime_profiles", ["artifact_set_id"])
|
||||
|
||||
op.create_table(
|
||||
"runtime_compatibility_assessments",
|
||||
sa.Column("artifact_set_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("runtime_profile_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("compute_node_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("adapter", sa.String(64), nullable=False),
|
||||
sa.Column("runtime_version", sa.String(128), nullable=False),
|
||||
sa.Column("status", sa.String(32), nullable=False),
|
||||
sa.Column("static_result", sa.JSON(), nullable=False, server_default=sa.text("'{}'")),
|
||||
sa.Column("evidence", sa.JSON(), nullable=False, server_default=sa.text("'{}'")),
|
||||
sa.Column("blockers", sa.JSON(), nullable=False, server_default=sa.text("'[]'")),
|
||||
sa.Column("warnings", sa.JSON(), nullable=False, server_default=sa.text("'[]'")),
|
||||
sa.Column("required_approvals", sa.JSON(), nullable=False, server_default=sa.text("'[]'")),
|
||||
sa.Column("hardware_facts", sa.JSON(), nullable=False, server_default=sa.text("'{}'")),
|
||||
sa.Column("artifact_facts", sa.JSON(), nullable=False, server_default=sa.text("'{}'")),
|
||||
sa.Column("environment_fingerprint", sa.String(64), nullable=False),
|
||||
sa.Column("stale", sa.Boolean(), nullable=False, server_default=sa.false()),
|
||||
sa.Column("stale_reason", sa.Text()),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.Column("id", sa.Uuid(), primary_key=True),
|
||||
sa.ForeignKeyConstraint(["artifact_set_id"], ["artifact_sets.id"], ondelete="RESTRICT"),
|
||||
sa.ForeignKeyConstraint(["runtime_profile_id"], ["runtime_profiles.id"], ondelete="RESTRICT"),
|
||||
sa.ForeignKeyConstraint(["compute_node_id"], ["compute_nodes.id"], ondelete="RESTRICT"),
|
||||
)
|
||||
for column in ("artifact_set_id", "runtime_profile_id", "compute_node_id", "status", "environment_fingerprint"):
|
||||
op.create_index(f"ix_runtime_compatibility_assessments_{column}", "runtime_compatibility_assessments", [column])
|
||||
|
||||
op.create_table(
|
||||
"execution_approvals",
|
||||
sa.Column("artifact_set_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("scope", sa.String(32), nullable=False),
|
||||
sa.Column("status", sa.String(32), nullable=False),
|
||||
sa.Column("evidence_fingerprint", sa.String(64), nullable=False),
|
||||
sa.Column("reason", sa.Text(), nullable=False),
|
||||
sa.Column("approved_by", sa.String(255), nullable=False),
|
||||
sa.Column("approved_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.Column("expires_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("revoked_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("id", sa.Uuid(), primary_key=True),
|
||||
sa.ForeignKeyConstraint(["artifact_set_id"], ["artifact_sets.id"], ondelete="RESTRICT"),
|
||||
)
|
||||
op.create_index("ix_execution_approvals_artifact_set_id", "execution_approvals", ["artifact_set_id"])
|
||||
op.create_index("ix_execution_approvals_status", "execution_approvals", ["status"])
|
||||
|
||||
op.create_table(
|
||||
"runtime_probes",
|
||||
sa.Column("artifact_set_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("runtime_profile_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("compute_node_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("compatibility_assessment_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("execution_approval_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("status", sa.String(32), nullable=False, server_default="queued"),
|
||||
sa.Column("phase", sa.String(64)),
|
||||
sa.Column("probe_input", sa.Text(), nullable=False),
|
||||
sa.Column("idempotency_key", sa.String(64), nullable=False),
|
||||
sa.Column("attempt_count", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("lease_token_hash", sa.String(64)),
|
||||
sa.Column("lease_expires_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("cancel_requested", sa.Boolean(), nullable=False, server_default=sa.false()),
|
||||
sa.Column("load_result", sa.JSON(), nullable=False, server_default=sa.text("'{}'")),
|
||||
sa.Column("health_result", sa.JSON(), nullable=False, server_default=sa.text("'{}'")),
|
||||
sa.Column("inference_result", sa.JSON(), nullable=False, server_default=sa.text("'{}'")),
|
||||
sa.Column("unload_result", sa.JSON(), nullable=False, server_default=sa.text("'{}'")),
|
||||
sa.Column("measured_resources", sa.JSON(), nullable=False, server_default=sa.text("'{}'")),
|
||||
sa.Column("runtime_facts", sa.JSON(), nullable=False, server_default=sa.text("'{}'")),
|
||||
sa.Column("environment_fingerprint", sa.String(64), nullable=False),
|
||||
sa.Column("failure_code", sa.String(64)),
|
||||
sa.Column("failure_message", sa.Text()),
|
||||
sa.Column("started_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("finished_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("id", sa.Uuid(), primary_key=True),
|
||||
*_timestamps(),
|
||||
sa.ForeignKeyConstraint(["artifact_set_id"], ["artifact_sets.id"], ondelete="RESTRICT"),
|
||||
sa.ForeignKeyConstraint(["runtime_profile_id"], ["runtime_profiles.id"], ondelete="RESTRICT"),
|
||||
sa.ForeignKeyConstraint(["compute_node_id"], ["compute_nodes.id"], ondelete="RESTRICT"),
|
||||
sa.ForeignKeyConstraint(["compatibility_assessment_id"], ["runtime_compatibility_assessments.id"], ondelete="RESTRICT"),
|
||||
sa.ForeignKeyConstraint(["execution_approval_id"], ["execution_approvals.id"], ondelete="RESTRICT"),
|
||||
sa.UniqueConstraint("idempotency_key", name="uq_runtime_probe_idempotency"),
|
||||
)
|
||||
for column in ("artifact_set_id", "runtime_profile_id", "compute_node_id", "compatibility_assessment_id", "execution_approval_id", "status"):
|
||||
op.create_index(f"ix_runtime_probes_{column}", "runtime_probes", [column])
|
||||
|
||||
op.create_table(
|
||||
"runtime_probe_metrics",
|
||||
sa.Column("runtime_probe_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("phase", sa.String(64), nullable=False),
|
||||
sa.Column("measurement_type", sa.String(64), nullable=False),
|
||||
sa.Column("source", sa.String(64), nullable=False),
|
||||
sa.Column("values", sa.JSON(), nullable=False, server_default=sa.text("'{}'")),
|
||||
sa.Column("observed_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("id", sa.Uuid(), primary_key=True),
|
||||
sa.ForeignKeyConstraint(["runtime_probe_id"], ["runtime_probes.id"], ondelete="RESTRICT"),
|
||||
)
|
||||
op.create_index("ix_runtime_probe_metrics_runtime_probe_id", "runtime_probe_metrics", ["runtime_probe_id"])
|
||||
|
||||
op.create_table(
|
||||
"deployment_candidates",
|
||||
sa.Column("artifact_set_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("runtime_profile_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("compute_node_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("compatibility_assessment_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("runtime_probe_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("channel", sa.String(32), nullable=False, server_default="lab"),
|
||||
sa.Column("status", sa.String(32), nullable=False, server_default="lab_ready"),
|
||||
sa.Column("production", sa.Boolean(), nullable=False, server_default=sa.false()),
|
||||
sa.Column("health_contract", sa.JSON(), nullable=False, server_default=sa.text("'{}'")),
|
||||
sa.Column("measured_resources", sa.JSON(), nullable=False, server_default=sa.text("'{}'")),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.Column("id", sa.Uuid(), primary_key=True),
|
||||
sa.ForeignKeyConstraint(["artifact_set_id"], ["artifact_sets.id"], ondelete="RESTRICT"),
|
||||
sa.ForeignKeyConstraint(["runtime_profile_id"], ["runtime_profiles.id"], ondelete="RESTRICT"),
|
||||
sa.ForeignKeyConstraint(["compute_node_id"], ["compute_nodes.id"], ondelete="RESTRICT"),
|
||||
sa.ForeignKeyConstraint(["compatibility_assessment_id"], ["runtime_compatibility_assessments.id"], ondelete="RESTRICT"),
|
||||
sa.ForeignKeyConstraint(["runtime_probe_id"], ["runtime_probes.id"], ondelete="RESTRICT"),
|
||||
sa.UniqueConstraint("runtime_probe_id", name="uq_deployment_candidate_probe"),
|
||||
)
|
||||
for column in ("artifact_set_id", "runtime_profile_id", "compute_node_id", "runtime_probe_id", "status"):
|
||||
op.create_index(f"ix_deployment_candidates_{column}", "deployment_candidates", [column])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
for table in (
|
||||
"deployment_candidates",
|
||||
"runtime_probe_metrics",
|
||||
"runtime_probes",
|
||||
"execution_approvals",
|
||||
"runtime_compatibility_assessments",
|
||||
):
|
||||
op.drop_table(table)
|
||||
op.drop_index("ix_runtime_profiles_artifact_set_id", table_name="runtime_profiles")
|
||||
op.drop_index("ix_runtime_profiles_runtime_environment_id", table_name="runtime_profiles")
|
||||
op.drop_constraint("fk_runtime_profiles_artifact_set", "runtime_profiles", type_="foreignkey")
|
||||
op.drop_constraint("fk_runtime_profiles_environment", "runtime_profiles", type_="foreignkey")
|
||||
for column in (
|
||||
"profile_config",
|
||||
"health_contract",
|
||||
"device_policy",
|
||||
"modality",
|
||||
"dtype",
|
||||
"version",
|
||||
"artifact_set_id",
|
||||
"runtime_environment_id",
|
||||
):
|
||||
op.drop_column("runtime_profiles", column)
|
||||
op.drop_table("runtime_environments")
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
"""M4 bounded runtime log reference.
|
||||
|
||||
Revision ID: 20260825_0007
|
||||
Revises: 20260825_0006
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "20260825_0007"
|
||||
down_revision: str | None = "20260825_0006"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("runtime_probes", sa.Column("logs_reference", sa.String(512)))
|
||||
op.execute(
|
||||
"UPDATE runtime_probes SET logs_reference = "
|
||||
"'runtime-worker://historical-probe/' || CAST(id AS VARCHAR)"
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("runtime_probes", "logs_reference")
|
||||
@@ -0,0 +1,382 @@
|
||||
"""M5 capability serving, gateway and GPU scheduling.
|
||||
|
||||
Revision ID: 20260825_0008
|
||||
Revises: 20260825_0007
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision = "20260825_0008"
|
||||
down_revision = "20260825_0007"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _identity_columns() -> list[sa.Column]:
|
||||
return [sa.Column("id", sa.Uuid(), primary_key=True)]
|
||||
|
||||
|
||||
def _timestamps() -> list[sa.Column]:
|
||||
return [
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
]
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"production_execution_approvals",
|
||||
*_identity_columns(),
|
||||
sa.Column("deployment_candidate_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("capability_contract_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("artifact_set_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("runtime_profile_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("compute_node_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("deployment_config", sa.JSON(), nullable=False),
|
||||
sa.Column("supply_chain_evidence", sa.JSON(), nullable=False),
|
||||
sa.Column("evidence_fingerprint", sa.String(64), nullable=False),
|
||||
sa.Column("status", sa.String(32), nullable=False, server_default="approved"),
|
||||
sa.Column("approved_by", sa.String(255), nullable=False),
|
||||
sa.Column("reason", sa.Text(), nullable=False),
|
||||
sa.Column("approved_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.Column("immutable_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.Column("revoked_at", sa.DateTime(timezone=True)),
|
||||
sa.ForeignKeyConstraint(["deployment_candidate_id"], ["deployment_candidates.id"]),
|
||||
sa.ForeignKeyConstraint(["capability_contract_id"], ["capability_contracts.id"]),
|
||||
sa.ForeignKeyConstraint(["artifact_set_id"], ["artifact_sets.id"]),
|
||||
sa.ForeignKeyConstraint(["runtime_profile_id"], ["runtime_profiles.id"]),
|
||||
sa.ForeignKeyConstraint(["compute_node_id"], ["compute_nodes.id"]),
|
||||
sa.UniqueConstraint("evidence_fingerprint", name="uq_production_approval_fingerprint"),
|
||||
)
|
||||
for column in (
|
||||
"deployment_candidate_id",
|
||||
"capability_contract_id",
|
||||
"artifact_set_id",
|
||||
"runtime_profile_id",
|
||||
"compute_node_id",
|
||||
"status",
|
||||
):
|
||||
op.create_index(
|
||||
f"ix_production_execution_approvals_{column}",
|
||||
"production_execution_approvals",
|
||||
[column],
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"embedding_spaces",
|
||||
*_identity_columns(),
|
||||
sa.Column("capability_contract_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("artifact_set_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("runtime_profile_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("identity_digest", sa.String(64), nullable=False),
|
||||
sa.Column("dimension", sa.Integer(), nullable=False),
|
||||
sa.Column("normalized", sa.Boolean(), nullable=False),
|
||||
sa.Column("migration_class", sa.String(32), nullable=False),
|
||||
sa.Column("identity_facts", sa.JSON(), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.Column("immutable_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.ForeignKeyConstraint(["capability_contract_id"], ["capability_contracts.id"]),
|
||||
sa.ForeignKeyConstraint(["artifact_set_id"], ["artifact_sets.id"]),
|
||||
sa.ForeignKeyConstraint(["runtime_profile_id"], ["runtime_profiles.id"]),
|
||||
sa.UniqueConstraint("identity_digest", name="uq_embedding_space_digest"),
|
||||
)
|
||||
for column in ("capability_contract_id", "artifact_set_id", "runtime_profile_id"):
|
||||
op.create_index(f"ix_embedding_spaces_{column}", "embedding_spaces", [column])
|
||||
|
||||
op.create_table(
|
||||
"capability_deployments",
|
||||
*_identity_columns(),
|
||||
*_timestamps(),
|
||||
sa.Column("capability_contract_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("deployment_candidate_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("production_approval_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("embedding_space_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("artifact_set_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("runtime_profile_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("compute_node_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("accelerator_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("channel", sa.String(32), nullable=False, server_default="stable"),
|
||||
sa.Column("status", sa.String(32), nullable=False, server_default="approved"),
|
||||
sa.Column("production", sa.Boolean(), nullable=False, server_default=sa.false()),
|
||||
sa.Column("health_status", sa.String(32), nullable=False, server_default="ready_on_demand"),
|
||||
sa.Column("routing_weight", sa.Integer(), nullable=False, server_default="100"),
|
||||
sa.Column("fallback_policy", sa.JSON(), nullable=False),
|
||||
sa.Column("residency_policy", sa.String(32), nullable=False),
|
||||
sa.Column("keep_warm_seconds", sa.Integer(), nullable=False),
|
||||
sa.Column("max_concurrency", sa.Integer(), nullable=False),
|
||||
sa.Column("max_queue_depth", sa.Integer(), nullable=False),
|
||||
sa.Column("config_fingerprint", sa.String(64), nullable=False),
|
||||
sa.Column("provenance", sa.JSON(), nullable=False),
|
||||
sa.Column("rollback_policy", sa.JSON(), nullable=False),
|
||||
sa.Column("promoted_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("draining_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("deprecated_at", sa.DateTime(timezone=True)),
|
||||
sa.ForeignKeyConstraint(["capability_contract_id"], ["capability_contracts.id"]),
|
||||
sa.ForeignKeyConstraint(["deployment_candidate_id"], ["deployment_candidates.id"]),
|
||||
sa.ForeignKeyConstraint(["production_approval_id"], ["production_execution_approvals.id"]),
|
||||
sa.ForeignKeyConstraint(["embedding_space_id"], ["embedding_spaces.id"]),
|
||||
sa.ForeignKeyConstraint(["artifact_set_id"], ["artifact_sets.id"]),
|
||||
sa.ForeignKeyConstraint(["runtime_profile_id"], ["runtime_profiles.id"]),
|
||||
sa.ForeignKeyConstraint(["compute_node_id"], ["compute_nodes.id"]),
|
||||
sa.ForeignKeyConstraint(["accelerator_id"], ["accelerators.id"]),
|
||||
sa.UniqueConstraint("config_fingerprint", name="uq_capability_deployment_fingerprint"),
|
||||
)
|
||||
for column in (
|
||||
"capability_contract_id",
|
||||
"deployment_candidate_id",
|
||||
"production_approval_id",
|
||||
"embedding_space_id",
|
||||
"artifact_set_id",
|
||||
"runtime_profile_id",
|
||||
"compute_node_id",
|
||||
"accelerator_id",
|
||||
"channel",
|
||||
"status",
|
||||
):
|
||||
op.create_index(f"ix_capability_deployments_{column}", "capability_deployments", [column])
|
||||
op.create_index(
|
||||
"ix_capability_deployments_contract_channel_status",
|
||||
"capability_deployments",
|
||||
["capability_contract_id", "channel", "status"],
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"capability_resource_envelopes",
|
||||
*_identity_columns(),
|
||||
*_timestamps(),
|
||||
sa.Column("capability_deployment_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("runtime_probe_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("accelerator_kind", sa.String(255), nullable=False),
|
||||
sa.Column("accelerator_uuid", sa.String(255), nullable=False),
|
||||
sa.Column("environment_fingerprint", sa.String(64), nullable=False),
|
||||
sa.Column("concurrency", sa.Integer(), nullable=False),
|
||||
sa.Column("batch_size", sa.Integer(), nullable=False),
|
||||
sa.Column("max_sequence_length", sa.Integer(), nullable=False),
|
||||
sa.Column("baseline_vram_bytes", sa.BigInteger(), nullable=False),
|
||||
sa.Column("resident_vram_bytes", sa.BigInteger(), nullable=False),
|
||||
sa.Column("peak_vram_bytes", sa.BigInteger(), nullable=False),
|
||||
sa.Column("required_vram_bytes", sa.BigInteger(), nullable=False),
|
||||
sa.Column("cold_load_time_ms", sa.Float(), nullable=False),
|
||||
sa.Column("inference_latency_ms", sa.Float(), nullable=False),
|
||||
sa.Column("evidence", sa.JSON(), nullable=False),
|
||||
sa.Column("stale", sa.Boolean(), nullable=False, server_default=sa.false()),
|
||||
sa.Column("stale_reason", sa.Text()),
|
||||
sa.Column("immutable_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.ForeignKeyConstraint(["capability_deployment_id"], ["capability_deployments.id"]),
|
||||
sa.ForeignKeyConstraint(["runtime_probe_id"], ["runtime_probes.id"]),
|
||||
sa.UniqueConstraint("capability_deployment_id", name="uq_capability_resource_envelope"),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_capability_resource_envelopes_capability_deployment_id",
|
||||
"capability_resource_envelopes",
|
||||
["capability_deployment_id"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_capability_resource_envelopes_runtime_probe_id",
|
||||
"capability_resource_envelopes",
|
||||
["runtime_probe_id"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_capability_resource_envelopes_environment_fingerprint",
|
||||
"capability_resource_envelopes",
|
||||
["environment_fingerprint"],
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"service_clients",
|
||||
*_identity_columns(),
|
||||
*_timestamps(),
|
||||
sa.Column("name", sa.String(255), nullable=False, unique=True),
|
||||
sa.Column("status", sa.String(32), nullable=False, server_default="active"),
|
||||
sa.Column("allowed_capabilities", sa.JSON(), nullable=False),
|
||||
sa.Column("requests_per_minute", sa.Integer(), nullable=False),
|
||||
sa.Column("max_concurrent_requests", sa.Integer(), nullable=False),
|
||||
sa.Column("last_used_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("disabled_at", sa.DateTime(timezone=True)),
|
||||
)
|
||||
op.create_index("ix_service_clients_status", "service_clients", ["status"])
|
||||
op.create_table(
|
||||
"service_credentials",
|
||||
*_identity_columns(),
|
||||
sa.Column("service_client_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("secret_hash", sa.String(64), nullable=False, unique=True),
|
||||
sa.Column("secret_prefix", sa.String(16), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.Column("expires_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("last_used_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("revoked_at", sa.DateTime(timezone=True)),
|
||||
sa.ForeignKeyConstraint(["service_client_id"], ["service_clients.id"], ondelete="CASCADE"),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_service_credentials_service_client_id", "service_credentials", ["service_client_id"]
|
||||
)
|
||||
op.create_index(
|
||||
"uq_active_service_client_credential",
|
||||
"service_credentials",
|
||||
["service_client_id"],
|
||||
unique=True,
|
||||
postgresql_where=sa.text("revoked_at IS NULL"),
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"residency_allocations",
|
||||
*_identity_columns(),
|
||||
*_timestamps(),
|
||||
sa.Column("capability_deployment_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("compute_node_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("accelerator_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("state", sa.String(32), nullable=False, server_default="cold"),
|
||||
sa.Column("worker_instance_id", sa.String(255)),
|
||||
sa.Column("load_count", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("active_requests", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column(
|
||||
"measured_resident_vram_bytes", sa.BigInteger(), nullable=False, server_default="0"
|
||||
),
|
||||
sa.Column(
|
||||
"external_baseline_vram_bytes", sa.BigInteger(), nullable=False, server_default="0"
|
||||
),
|
||||
sa.Column("health", sa.JSON(), nullable=False),
|
||||
sa.Column("resident_since", sa.DateTime(timezone=True)),
|
||||
sa.Column("last_used_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("transition_started_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("failure_code", sa.String(64)),
|
||||
sa.Column("failure_message", sa.Text()),
|
||||
sa.ForeignKeyConstraint(["capability_deployment_id"], ["capability_deployments.id"]),
|
||||
sa.ForeignKeyConstraint(["compute_node_id"], ["compute_nodes.id"]),
|
||||
sa.ForeignKeyConstraint(["accelerator_id"], ["accelerators.id"]),
|
||||
sa.UniqueConstraint("capability_deployment_id", name="uq_residency_deployment"),
|
||||
)
|
||||
for column in ("capability_deployment_id", "compute_node_id", "accelerator_id", "state"):
|
||||
op.create_index(f"ix_residency_allocations_{column}", "residency_allocations", [column])
|
||||
|
||||
op.create_table(
|
||||
"serving_gpu_leases",
|
||||
*_identity_columns(),
|
||||
sa.Column("accelerator_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("capability_deployment_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("request_id", sa.Uuid()),
|
||||
sa.Column("reserved_vram_bytes", sa.BigInteger(), nullable=False),
|
||||
sa.Column("priority", sa.String(32), nullable=False),
|
||||
sa.Column("state", sa.String(32), nullable=False, server_default="pending"),
|
||||
sa.Column("owner", sa.String(255), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("acquired_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("heartbeat_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("released_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("failure_code", sa.String(64)),
|
||||
sa.ForeignKeyConstraint(["accelerator_id"], ["accelerators.id"]),
|
||||
sa.ForeignKeyConstraint(["capability_deployment_id"], ["capability_deployments.id"]),
|
||||
)
|
||||
for column in (
|
||||
"accelerator_id",
|
||||
"capability_deployment_id",
|
||||
"request_id",
|
||||
"state",
|
||||
"expires_at",
|
||||
):
|
||||
op.create_index(f"ix_serving_gpu_leases_{column}", "serving_gpu_leases", [column])
|
||||
|
||||
op.create_table(
|
||||
"gateway_requests",
|
||||
*_identity_columns(),
|
||||
sa.Column("request_id", sa.Uuid(), nullable=False, unique=True),
|
||||
sa.Column("service_client_id", sa.Uuid()),
|
||||
sa.Column("capability_key", sa.String(128), nullable=False),
|
||||
sa.Column("capability_version", sa.Integer(), nullable=False),
|
||||
sa.Column("capability_deployment_id", sa.Uuid()),
|
||||
sa.Column("compute_node_id", sa.Uuid()),
|
||||
sa.Column("status", sa.String(32), nullable=False),
|
||||
sa.Column("priority", sa.String(32), nullable=False),
|
||||
sa.Column("input_sha256", sa.String(64), nullable=False),
|
||||
sa.Column("input_count", sa.Integer(), nullable=False),
|
||||
sa.Column("cold", sa.Boolean()),
|
||||
sa.Column("queue_time_ms", sa.Float()),
|
||||
sa.Column("load_time_ms", sa.Float()),
|
||||
sa.Column("inference_time_ms", sa.Float()),
|
||||
sa.Column("total_latency_ms", sa.Float()),
|
||||
sa.Column("failure_code", sa.String(64)),
|
||||
sa.Column("decision_evidence", sa.JSON(), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.Column("finished_at", sa.DateTime(timezone=True)),
|
||||
sa.ForeignKeyConstraint(["service_client_id"], ["service_clients.id"], ondelete="SET NULL"),
|
||||
sa.ForeignKeyConstraint(
|
||||
["capability_deployment_id"], ["capability_deployments.id"], ondelete="SET NULL"
|
||||
),
|
||||
sa.ForeignKeyConstraint(["compute_node_id"], ["compute_nodes.id"], ondelete="SET NULL"),
|
||||
)
|
||||
for column in (
|
||||
"request_id",
|
||||
"service_client_id",
|
||||
"capability_key",
|
||||
"capability_deployment_id",
|
||||
"compute_node_id",
|
||||
"status",
|
||||
"created_at",
|
||||
):
|
||||
op.create_index(f"ix_gateway_requests_{column}", "gateway_requests", [column])
|
||||
op.create_index(
|
||||
"ix_gateway_requests_client_created",
|
||||
"gateway_requests",
|
||||
["service_client_id", "created_at"],
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"serving_jobs",
|
||||
*_identity_columns(),
|
||||
*_timestamps(),
|
||||
sa.Column("capability_deployment_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("compute_node_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("gateway_request_id", sa.Uuid()),
|
||||
sa.Column("operation", sa.String(32), nullable=False),
|
||||
sa.Column("status", sa.String(32), nullable=False, server_default="queued"),
|
||||
sa.Column("priority", sa.String(32), nullable=False),
|
||||
sa.Column("idempotency_key", sa.String(64), nullable=False),
|
||||
sa.Column("payload_reference", sa.String(128)),
|
||||
sa.Column("attempt_count", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("lease_token_hash", sa.String(64)),
|
||||
sa.Column("lease_expires_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("result_reference", sa.String(128)),
|
||||
sa.Column("result_summary", sa.JSON(), nullable=False),
|
||||
sa.Column("failure_code", sa.String(64)),
|
||||
sa.Column("failure_message", sa.Text()),
|
||||
sa.Column("started_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("finished_at", sa.DateTime(timezone=True)),
|
||||
sa.ForeignKeyConstraint(["capability_deployment_id"], ["capability_deployments.id"]),
|
||||
sa.ForeignKeyConstraint(["compute_node_id"], ["compute_nodes.id"]),
|
||||
sa.ForeignKeyConstraint(
|
||||
["gateway_request_id"], ["gateway_requests.id"], ondelete="SET NULL"
|
||||
),
|
||||
sa.UniqueConstraint("idempotency_key", name="uq_serving_job_idempotency"),
|
||||
)
|
||||
for column in (
|
||||
"capability_deployment_id",
|
||||
"compute_node_id",
|
||||
"gateway_request_id",
|
||||
"operation",
|
||||
"status",
|
||||
):
|
||||
op.create_index(f"ix_serving_jobs_{column}", "serving_jobs", [column])
|
||||
op.create_index(
|
||||
"ix_serving_jobs_node_status_priority",
|
||||
"serving_jobs",
|
||||
["compute_node_id", "status", "priority"],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
for table in (
|
||||
"serving_jobs",
|
||||
"gateway_requests",
|
||||
"serving_gpu_leases",
|
||||
"residency_allocations",
|
||||
"service_credentials",
|
||||
"service_clients",
|
||||
"capability_resource_envelopes",
|
||||
"capability_deployments",
|
||||
"embedding_spaces",
|
||||
"production_execution_approvals",
|
||||
):
|
||||
op.drop_table(table)
|
||||
@@ -0,0 +1,165 @@
|
||||
"""M6 project evaluation and isolated embedding migrations.
|
||||
|
||||
Revision ID: 20260825_0009
|
||||
Revises: 20260825_0008
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision = "20260825_0009"
|
||||
down_revision = "20260825_0008"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _id() -> sa.Column:
|
||||
return sa.Column("id", sa.Uuid(), primary_key=True)
|
||||
|
||||
|
||||
def _created() -> sa.Column:
|
||||
return sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now())
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"service_clients",
|
||||
sa.Column("workload_priority", sa.String(32), nullable=False, server_default="production"),
|
||||
)
|
||||
op.create_table(
|
||||
"project_evaluation_bindings", _id(), _created(),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.Column("project_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("adapter_kind", sa.String(64), nullable=False),
|
||||
sa.Column("endpoint", sa.Text(), nullable=False),
|
||||
sa.Column("current_target", sa.String(128), nullable=False),
|
||||
sa.Column("shadow_target", sa.String(128)),
|
||||
sa.Column("enabled", sa.Boolean(), nullable=False, server_default=sa.true()),
|
||||
sa.Column("configuration", sa.JSON(), nullable=False),
|
||||
sa.ForeignKeyConstraint(["project_id"], ["projects.id"], ondelete="RESTRICT"),
|
||||
sa.UniqueConstraint("project_id", name="uq_project_evaluation_binding"),
|
||||
)
|
||||
op.create_index("ix_project_evaluation_bindings_project_id", "project_evaluation_bindings", ["project_id"])
|
||||
|
||||
op.create_table(
|
||||
"embedding_migrations", _id(), _created(),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.Column("project_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("source_embedding_space", sa.String(255), nullable=False),
|
||||
sa.Column("target_embedding_space_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("source_index_ref", sa.Text(), nullable=False),
|
||||
sa.Column("target_index_ref", sa.Text(), nullable=False, unique=True),
|
||||
sa.Column("corpus_revision", sa.String(128), nullable=False),
|
||||
sa.Column("status", sa.String(32), nullable=False, server_default="planned"),
|
||||
sa.Column("total_chunks", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("completed_chunks", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("failed_chunks", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("retried_chunks", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("batch_size", sa.Integer(), nullable=False),
|
||||
sa.Column("concurrency", sa.Integer(), nullable=False),
|
||||
sa.Column("priority", sa.String(32), nullable=False, server_default="background"),
|
||||
sa.Column("preflight_evidence", sa.JSON(), nullable=False),
|
||||
sa.Column("progress_evidence", sa.JSON(), nullable=False),
|
||||
sa.Column("validation_evidence", sa.JSON(), nullable=False),
|
||||
sa.Column("operational_metrics", sa.JSON(), nullable=False),
|
||||
sa.Column("evaluation_eligibility", sa.Boolean(), nullable=False, server_default=sa.false()),
|
||||
sa.Column("cancel_requested", sa.Boolean(), nullable=False, server_default=sa.false()),
|
||||
sa.Column("failure_code", sa.String(64)), sa.Column("failure_message", sa.Text()),
|
||||
sa.Column("started_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("finished_at", sa.DateTime(timezone=True)),
|
||||
sa.ForeignKeyConstraint(["project_id"], ["projects.id"], ondelete="RESTRICT"),
|
||||
sa.ForeignKeyConstraint(["target_embedding_space_id"], ["embedding_spaces.id"], ondelete="RESTRICT"),
|
||||
)
|
||||
for column in ("project_id", "target_embedding_space_id", "status"):
|
||||
op.create_index(f"ix_embedding_migrations_{column}", "embedding_migrations", [column])
|
||||
|
||||
op.create_table(
|
||||
"evaluation_suites", _id(), _created(),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.Column("project_id", sa.Uuid(), nullable=False), sa.Column("key", sa.String(128), nullable=False),
|
||||
sa.Column("name", sa.String(255), nullable=False), sa.Column("description", sa.Text(), nullable=False),
|
||||
sa.ForeignKeyConstraint(["project_id"], ["projects.id"], ondelete="RESTRICT"),
|
||||
sa.UniqueConstraint("project_id", "key", name="uq_evaluation_suite_key"),
|
||||
)
|
||||
op.create_index("ix_evaluation_suites_project_id", "evaluation_suites", ["project_id"])
|
||||
op.create_table(
|
||||
"evaluation_suite_revisions", _id(), _created(),
|
||||
sa.Column("suite_id", sa.Uuid(), nullable=False), sa.Column("revision", sa.String(128), nullable=False),
|
||||
sa.Column("dataset_revision", sa.String(128), nullable=False),
|
||||
sa.Column("definition_digest", sa.String(64), nullable=False), sa.Column("metrics", sa.JSON(), nullable=False),
|
||||
sa.Column("top_k", sa.Integer(), nullable=False), sa.Column("retrieval_settings", sa.JSON(), nullable=False),
|
||||
sa.Column("thresholds", sa.JSON(), nullable=False),
|
||||
sa.Column("immutable_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.ForeignKeyConstraint(["suite_id"], ["evaluation_suites.id"], ondelete="RESTRICT"),
|
||||
sa.UniqueConstraint("suite_id", "revision", name="uq_evaluation_suite_revision"),
|
||||
sa.UniqueConstraint("definition_digest", name="uq_evaluation_revision_digest"),
|
||||
)
|
||||
op.create_index("ix_evaluation_suite_revisions_suite_id", "evaluation_suite_revisions", ["suite_id"])
|
||||
op.create_table(
|
||||
"evaluation_cases", _id(), _created(),
|
||||
sa.Column("suite_revision_id", sa.Uuid(), nullable=False), sa.Column("case_key", sa.String(128), nullable=False),
|
||||
sa.Column("query", sa.Text(), nullable=False), sa.Column("relevant_chunk_ids", sa.JSON(), nullable=False),
|
||||
sa.Column("relevant_document_ids", sa.JSON(), nullable=False), sa.Column("relevance_grades", sa.JSON(), nullable=False),
|
||||
sa.Column("label_provenance", sa.JSON(), nullable=False), sa.Column("critical", sa.Boolean(), nullable=False),
|
||||
sa.Column("review_status", sa.String(32), nullable=False),
|
||||
sa.Column("immutable_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.ForeignKeyConstraint(["suite_revision_id"], ["evaluation_suite_revisions.id"], ondelete="RESTRICT"),
|
||||
sa.UniqueConstraint("suite_revision_id", "case_key", name="uq_evaluation_case_key"),
|
||||
)
|
||||
op.create_index("ix_evaluation_cases_suite_revision_id", "evaluation_cases", ["suite_revision_id"])
|
||||
op.create_index("ix_evaluation_cases_critical", "evaluation_cases", ["critical"])
|
||||
op.create_table(
|
||||
"evaluation_runs", _id(), _created(),
|
||||
sa.Column("project_id", sa.Uuid(), nullable=False), sa.Column("suite_revision_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("target_kind", sa.String(32), nullable=False), sa.Column("target_index_ref", sa.Text(), nullable=False),
|
||||
sa.Column("embedding_space_ref", sa.String(255), nullable=False), sa.Column("capability_deployment_id", sa.Uuid()),
|
||||
sa.Column("status", sa.String(32), nullable=False), sa.Column("corpus_revision", sa.String(128), nullable=False),
|
||||
sa.Column("retrieval_config_digest", sa.String(64), nullable=False),
|
||||
sa.Column("environment_fingerprint", sa.JSON(), nullable=False), sa.Column("environment_digest", sa.String(64), nullable=False),
|
||||
sa.Column("expected_cases", sa.Integer(), nullable=False), sa.Column("completed_cases", sa.Integer(), nullable=False),
|
||||
sa.Column("error_count", sa.Integer(), nullable=False), sa.Column("aggregate_metrics", sa.JSON(), nullable=False),
|
||||
sa.Column("started_at", sa.DateTime(timezone=True)), sa.Column("completed_at", sa.DateTime(timezone=True)),
|
||||
sa.ForeignKeyConstraint(["project_id"], ["projects.id"], ondelete="RESTRICT"),
|
||||
sa.ForeignKeyConstraint(["suite_revision_id"], ["evaluation_suite_revisions.id"], ondelete="RESTRICT"),
|
||||
sa.ForeignKeyConstraint(["capability_deployment_id"], ["capability_deployments.id"], ondelete="RESTRICT"),
|
||||
)
|
||||
for column in ("project_id", "suite_revision_id", "capability_deployment_id", "status", "environment_digest"):
|
||||
op.create_index(f"ix_evaluation_runs_{column}", "evaluation_runs", [column])
|
||||
op.create_table(
|
||||
"evaluation_case_results", _id(), _created(),
|
||||
sa.Column("run_id", sa.Uuid(), nullable=False), sa.Column("case_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("ranked_results", sa.JSON(), nullable=False), sa.Column("relevant_results", sa.JSON(), nullable=False),
|
||||
sa.Column("first_relevant_rank", sa.Integer()), sa.Column("metrics", sa.JSON(), nullable=False),
|
||||
sa.Column("latency_ms", sa.Float(), nullable=False), sa.Column("error_code", sa.String(64)),
|
||||
sa.ForeignKeyConstraint(["run_id"], ["evaluation_runs.id"], ondelete="RESTRICT"),
|
||||
sa.ForeignKeyConstraint(["case_id"], ["evaluation_cases.id"], ondelete="RESTRICT"),
|
||||
sa.UniqueConstraint("run_id", "case_id", name="uq_evaluation_case_result"),
|
||||
)
|
||||
op.create_index("ix_evaluation_case_results_run_id", "evaluation_case_results", ["run_id"])
|
||||
op.create_index("ix_evaluation_case_results_case_id", "evaluation_case_results", ["case_id"])
|
||||
op.create_table(
|
||||
"evaluation_comparisons", _id(), _created(),
|
||||
sa.Column("project_id", sa.Uuid(), nullable=False), sa.Column("baseline_run_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("candidate_run_id", sa.Uuid(), nullable=False), sa.Column("comparability", sa.String(32), nullable=False),
|
||||
sa.Column("comparability_evidence", sa.JSON(), nullable=False), sa.Column("metric_deltas", sa.JSON(), nullable=False),
|
||||
sa.Column("improved_cases", sa.Integer(), nullable=False), sa.Column("unchanged_cases", sa.Integer(), nullable=False),
|
||||
sa.Column("regressed_cases", sa.Integer(), nullable=False), sa.Column("critical_regressions", sa.Integer(), nullable=False),
|
||||
sa.Column("case_comparisons", sa.JSON(), nullable=False), sa.Column("promotion_eligibility", sa.String(32), nullable=False),
|
||||
sa.Column("eligibility_evidence", sa.JSON(), nullable=False),
|
||||
sa.ForeignKeyConstraint(["project_id"], ["projects.id"], ondelete="RESTRICT"),
|
||||
sa.ForeignKeyConstraint(["baseline_run_id"], ["evaluation_runs.id"], ondelete="RESTRICT"),
|
||||
sa.ForeignKeyConstraint(["candidate_run_id"], ["evaluation_runs.id"], ondelete="RESTRICT"),
|
||||
sa.UniqueConstraint("baseline_run_id", "candidate_run_id", name="uq_evaluation_comparison"),
|
||||
)
|
||||
for column in ("project_id", "baseline_run_id", "candidate_run_id"):
|
||||
op.create_index(f"ix_evaluation_comparisons_{column}", "evaluation_comparisons", [column])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
for table in (
|
||||
"evaluation_comparisons", "evaluation_case_results", "evaluation_runs", "evaluation_cases",
|
||||
"evaluation_suite_revisions", "evaluation_suites", "embedding_migrations", "project_evaluation_bindings",
|
||||
):
|
||||
op.drop_table(table)
|
||||
op.drop_column("service_clients", "workload_priority")
|
||||
@@ -0,0 +1,103 @@
|
||||
"""M7 candidate capability routes and lab approval provenance.
|
||||
|
||||
Revision ID: 20260825_0010
|
||||
Revises: 20260825_0009
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "20260825_0010"
|
||||
down_revision: str | None = "20260825_0009"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.alter_column(
|
||||
"capability_deployments",
|
||||
"production_approval_id",
|
||||
existing_type=sa.Uuid(),
|
||||
nullable=True,
|
||||
)
|
||||
op.add_column(
|
||||
"capability_deployments",
|
||||
sa.Column("execution_approval_id", sa.Uuid(), nullable=True),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_capability_deployments_execution_approval_id",
|
||||
"capability_deployments",
|
||||
["execution_approval_id"],
|
||||
)
|
||||
op.create_foreign_key(
|
||||
"fk_capability_deployments_execution_approval_id",
|
||||
"capability_deployments",
|
||||
"execution_approvals",
|
||||
["execution_approval_id"],
|
||||
["id"],
|
||||
ondelete="RESTRICT",
|
||||
)
|
||||
op.create_table(
|
||||
"capability_experiment_routes",
|
||||
sa.Column("route_key", sa.String(length=128), nullable=False),
|
||||
sa.Column("capability_contract_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("capability_deployment_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("status", sa.String(length=32), nullable=False),
|
||||
sa.Column("purpose", sa.Text(), nullable=False),
|
||||
sa.Column("evidence", sa.JSON(), nullable=False),
|
||||
sa.Column("id", sa.Uuid(), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["capability_contract_id"], ["capability_contracts.id"], ondelete="RESTRICT"
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["capability_deployment_id"], ["capability_deployments.id"], ondelete="RESTRICT"
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
sa.UniqueConstraint("route_key"),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_capability_experiment_routes_route_key",
|
||||
"capability_experiment_routes",
|
||||
["route_key"],
|
||||
unique=True,
|
||||
)
|
||||
op.create_index(
|
||||
"ix_capability_experiment_routes_capability_contract_id",
|
||||
"capability_experiment_routes",
|
||||
["capability_contract_id"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_capability_experiment_routes_capability_deployment_id",
|
||||
"capability_experiment_routes",
|
||||
["capability_deployment_id"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_capability_experiment_routes_status",
|
||||
"capability_experiment_routes",
|
||||
["status"],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("capability_experiment_routes")
|
||||
op.drop_constraint(
|
||||
"fk_capability_deployments_execution_approval_id",
|
||||
"capability_deployments",
|
||||
type_="foreignkey",
|
||||
)
|
||||
op.drop_index(
|
||||
"ix_capability_deployments_execution_approval_id",
|
||||
table_name="capability_deployments",
|
||||
)
|
||||
op.drop_column("capability_deployments", "execution_approval_id")
|
||||
op.alter_column(
|
||||
"capability_deployments",
|
||||
"production_approval_id",
|
||||
existing_type=sa.Uuid(),
|
||||
nullable=False,
|
||||
)
|
||||
@@ -0,0 +1,109 @@
|
||||
"""M7 first-class model comparisons and deterministic advisor evidence.
|
||||
|
||||
Revision ID: 20260825_0011
|
||||
Revises: 20260825_0010
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "20260825_0011"
|
||||
down_revision: str | None = "20260825_0010"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"model_comparisons",
|
||||
sa.Column("project_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("capability_contract_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("suite_revision_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("current_run_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("title", sa.String(length=255), nullable=False),
|
||||
sa.Column("candidates", sa.JSON(), nullable=False),
|
||||
sa.Column("comparability", sa.String(length=32), nullable=False),
|
||||
sa.Column("evidence_fingerprint", sa.String(length=64), nullable=False),
|
||||
sa.Column("id", sa.Uuid(), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
|
||||
sa.ForeignKeyConstraint(["project_id"], ["projects.id"], ondelete="RESTRICT"),
|
||||
sa.ForeignKeyConstraint(["capability_contract_id"], ["capability_contracts.id"], ondelete="RESTRICT"),
|
||||
sa.ForeignKeyConstraint(["suite_revision_id"], ["evaluation_suite_revisions.id"], ondelete="RESTRICT"),
|
||||
sa.ForeignKeyConstraint(["current_run_id"], ["evaluation_runs.id"], ondelete="RESTRICT"),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
sa.UniqueConstraint("evidence_fingerprint"),
|
||||
)
|
||||
op.create_index("ix_model_comparisons_project_id", "model_comparisons", ["project_id"])
|
||||
op.create_index("ix_model_comparisons_capability_contract_id", "model_comparisons", ["capability_contract_id"])
|
||||
op.create_index("ix_model_comparisons_suite_revision_id", "model_comparisons", ["suite_revision_id"])
|
||||
|
||||
op.create_table(
|
||||
"advisor_policies",
|
||||
sa.Column("key", sa.String(length=128), nullable=False),
|
||||
sa.Column("required_evidence_level", sa.String(length=8), nullable=False),
|
||||
sa.Column("critical_regression_hard_block", sa.Boolean(), nullable=False),
|
||||
sa.Column("maximum_latency_p95_regression_ratio", sa.Float(), nullable=False),
|
||||
sa.Column("minimum_metric_deltas", sa.JSON(), nullable=False),
|
||||
sa.Column("require_verified_supply_chain", sa.Boolean(), nullable=False),
|
||||
sa.Column("require_runtime_fit", sa.Boolean(), nullable=False),
|
||||
sa.Column("rationale", sa.Text(), nullable=False),
|
||||
sa.Column("version", sa.Integer(), nullable=False),
|
||||
sa.Column("id", sa.Uuid(), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
sa.UniqueConstraint("key"),
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"advisor_recommendations",
|
||||
sa.Column("comparison_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("project_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("capability_contract_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("policy_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("candidate_key", sa.String(length=128), nullable=False),
|
||||
sa.Column("current_deployment_id", sa.Uuid(), nullable=True),
|
||||
sa.Column("candidate_deployment_id", sa.Uuid(), nullable=True),
|
||||
sa.Column("current_embedding_space", sa.String(length=255), nullable=False),
|
||||
sa.Column("candidate_embedding_space", sa.String(length=255), nullable=True),
|
||||
sa.Column("verdict", sa.String(length=64), nullable=False),
|
||||
sa.Column("confidence", sa.String(length=16), nullable=False),
|
||||
sa.Column("evidence_level", sa.String(length=8), nullable=False),
|
||||
sa.Column("quality_deltas", sa.JSON(), nullable=False),
|
||||
sa.Column("latency_deltas", sa.JSON(), nullable=False),
|
||||
sa.Column("resource_deltas", sa.JSON(), nullable=False),
|
||||
sa.Column("migration_impact", sa.JSON(), nullable=False),
|
||||
sa.Column("security_state", sa.JSON(), nullable=False),
|
||||
sa.Column("key_improvements", sa.JSON(), nullable=False),
|
||||
sa.Column("blockers", sa.JSON(), nullable=False),
|
||||
sa.Column("policy_snapshot", sa.JSON(), nullable=False),
|
||||
sa.Column("evidence_fingerprint", sa.String(length=64), nullable=False),
|
||||
sa.Column("status", sa.String(length=32), nullable=False),
|
||||
sa.Column("generated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("dismissed_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("dismissed_by", sa.String(length=255), nullable=True),
|
||||
sa.Column("dismissal_reason", sa.Text(), nullable=True),
|
||||
sa.Column("id", sa.Uuid(), nullable=False),
|
||||
sa.ForeignKeyConstraint(["comparison_id"], ["model_comparisons.id"], ondelete="RESTRICT"),
|
||||
sa.ForeignKeyConstraint(["project_id"], ["projects.id"], ondelete="RESTRICT"),
|
||||
sa.ForeignKeyConstraint(["capability_contract_id"], ["capability_contracts.id"], ondelete="RESTRICT"),
|
||||
sa.ForeignKeyConstraint(["policy_id"], ["advisor_policies.id"], ondelete="RESTRICT"),
|
||||
sa.ForeignKeyConstraint(["current_deployment_id"], ["capability_deployments.id"], ondelete="RESTRICT"),
|
||||
sa.ForeignKeyConstraint(["candidate_deployment_id"], ["capability_deployments.id"], ondelete="RESTRICT"),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
sa.UniqueConstraint("evidence_fingerprint"),
|
||||
)
|
||||
op.create_index("ix_advisor_recommendations_comparison_id", "advisor_recommendations", ["comparison_id"])
|
||||
op.create_index("ix_advisor_recommendations_project_id", "advisor_recommendations", ["project_id"])
|
||||
op.create_index("ix_advisor_recommendations_capability_contract_id", "advisor_recommendations", ["capability_contract_id"])
|
||||
op.create_index("ix_advisor_recommendations_verdict", "advisor_recommendations", ["verdict"])
|
||||
op.create_index("ix_advisor_recommendations_status", "advisor_recommendations", ["status"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("advisor_recommendations")
|
||||
op.drop_table("advisor_policies")
|
||||
op.drop_table("model_comparisons")
|
||||
@@ -0,0 +1,276 @@
|
||||
"""M8 frozen reranking pools and retrieval pipeline evidence.
|
||||
|
||||
Revision ID: 20260825_0012
|
||||
Revises: 20260825_0011
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "20260825_0012"
|
||||
down_revision: str | None = "20260825_0011"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.alter_column("capability_deployments", "embedding_space_id", nullable=True)
|
||||
|
||||
op.create_table(
|
||||
"retrieval_candidate_pools",
|
||||
sa.Column("project_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("suite_revision_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("evaluation_case_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("source_embedding_space", sa.String(length=255), nullable=False),
|
||||
sa.Column("source_index_ref", sa.Text(), nullable=False),
|
||||
sa.Column("corpus_revision", sa.String(length=128), nullable=False),
|
||||
sa.Column("retrieval_config_digest", sa.String(length=64), nullable=False),
|
||||
sa.Column("candidate_count", sa.Integer(), nullable=False),
|
||||
sa.Column("ordered_candidates", sa.JSON(), nullable=False),
|
||||
sa.Column("fingerprint", sa.String(length=64), nullable=False),
|
||||
sa.Column("id", sa.Uuid(), nullable=False),
|
||||
sa.Column(
|
||||
"created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
|
||||
),
|
||||
sa.Column(
|
||||
"immutable_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"candidate_count > 0 AND candidate_count <= 40", name="ck_candidate_pool_count"
|
||||
),
|
||||
sa.ForeignKeyConstraint(["project_id"], ["projects.id"], ondelete="RESTRICT"),
|
||||
sa.ForeignKeyConstraint(
|
||||
["suite_revision_id"], ["evaluation_suite_revisions.id"], ondelete="RESTRICT"
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["evaluation_case_id"], ["evaluation_cases.id"], ondelete="RESTRICT"
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
sa.UniqueConstraint(
|
||||
"evaluation_case_id",
|
||||
"source_index_ref",
|
||||
"fingerprint",
|
||||
name="uq_candidate_pool_case_source_fingerprint",
|
||||
),
|
||||
sa.UniqueConstraint("fingerprint"),
|
||||
)
|
||||
op.create_index("ix_candidate_pools_project_id", "retrieval_candidate_pools", ["project_id"])
|
||||
op.create_index(
|
||||
"ix_candidate_pools_suite_revision_id", "retrieval_candidate_pools", ["suite_revision_id"]
|
||||
)
|
||||
op.create_index(
|
||||
"ix_candidate_pools_evaluation_case_id", "retrieval_candidate_pools", ["evaluation_case_id"]
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"retrieval_pipeline_identities",
|
||||
sa.Column("project_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("embedding_space_ref", sa.String(length=255), nullable=False),
|
||||
sa.Column("sparse_config_digest", sa.String(length=64), nullable=False),
|
||||
sa.Column("fusion_config_digest", sa.String(length=64), nullable=False),
|
||||
sa.Column("reranker_deployment_id", sa.Uuid(), nullable=True),
|
||||
sa.Column("reranker_config_digest", sa.String(length=64), nullable=True),
|
||||
sa.Column("candidate_k", sa.Integer(), nullable=False),
|
||||
sa.Column("output_k", sa.Integer(), nullable=False),
|
||||
sa.Column("identity_digest", sa.String(length=64), nullable=False),
|
||||
sa.Column("configuration", sa.JSON(), nullable=False),
|
||||
sa.Column("migration_class", sa.String(length=32), nullable=False),
|
||||
sa.Column("id", sa.Uuid(), nullable=False),
|
||||
sa.Column(
|
||||
"created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
|
||||
),
|
||||
sa.Column(
|
||||
"immutable_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
|
||||
),
|
||||
sa.CheckConstraint("candidate_k > 0 AND candidate_k <= 40", name="ck_pipeline_candidate_k"),
|
||||
sa.CheckConstraint("output_k > 0 AND output_k <= candidate_k", name="ck_pipeline_output_k"),
|
||||
sa.ForeignKeyConstraint(["project_id"], ["projects.id"], ondelete="RESTRICT"),
|
||||
sa.ForeignKeyConstraint(
|
||||
["reranker_deployment_id"], ["capability_deployments.id"], ondelete="RESTRICT"
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
sa.UniqueConstraint("identity_digest"),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_pipeline_identities_project_id", "retrieval_pipeline_identities", ["project_id"]
|
||||
)
|
||||
op.create_index(
|
||||
"ix_pipeline_identities_reranker_deployment_id",
|
||||
"retrieval_pipeline_identities",
|
||||
["reranker_deployment_id"],
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"reranking_evaluation_runs",
|
||||
sa.Column("project_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("suite_revision_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("pipeline_identity_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("control_pipeline_identity_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("reranker_deployment_id", sa.Uuid(), nullable=True),
|
||||
sa.Column("status", sa.String(length=32), nullable=False),
|
||||
sa.Column("corpus_revision", sa.String(length=128), nullable=False),
|
||||
sa.Column("candidate_pool_set_fingerprint", sa.String(length=64), nullable=False),
|
||||
sa.Column("environment_fingerprint", sa.JSON(), nullable=False),
|
||||
sa.Column("environment_digest", sa.String(length=64), nullable=False),
|
||||
sa.Column("expected_cases", sa.Integer(), nullable=False),
|
||||
sa.Column("completed_cases", sa.Integer(), nullable=False),
|
||||
sa.Column("error_count", sa.Integer(), nullable=False),
|
||||
sa.Column("aggregate_metrics", sa.JSON(), nullable=False),
|
||||
sa.Column("latency_metrics", sa.JSON(), nullable=False),
|
||||
sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("id", sa.Uuid(), nullable=False),
|
||||
sa.Column(
|
||||
"created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
|
||||
),
|
||||
sa.ForeignKeyConstraint(["project_id"], ["projects.id"], ondelete="RESTRICT"),
|
||||
sa.ForeignKeyConstraint(
|
||||
["suite_revision_id"], ["evaluation_suite_revisions.id"], ondelete="RESTRICT"
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["pipeline_identity_id"], ["retrieval_pipeline_identities.id"], ondelete="RESTRICT"
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["control_pipeline_identity_id"],
|
||||
["retrieval_pipeline_identities.id"],
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["reranker_deployment_id"], ["capability_deployments.id"], ondelete="RESTRICT"
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
)
|
||||
op.create_index("ix_reranking_runs_project_id", "reranking_evaluation_runs", ["project_id"])
|
||||
op.create_index(
|
||||
"ix_reranking_runs_suite_revision_id", "reranking_evaluation_runs", ["suite_revision_id"]
|
||||
)
|
||||
op.create_index(
|
||||
"ix_reranking_runs_pipeline_identity_id",
|
||||
"reranking_evaluation_runs",
|
||||
["pipeline_identity_id"],
|
||||
)
|
||||
op.create_index("ix_reranking_runs_status", "reranking_evaluation_runs", ["status"])
|
||||
|
||||
op.create_table(
|
||||
"reranking_case_results",
|
||||
sa.Column("run_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("case_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("candidate_pool_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("ranked_results", sa.JSON(), nullable=False),
|
||||
sa.Column("relevant_results", sa.JSON(), nullable=False),
|
||||
sa.Column("first_relevant_rank", sa.Integer(), nullable=True),
|
||||
sa.Column("metrics", sa.JSON(), nullable=False),
|
||||
sa.Column("retrieval_latency_ms", sa.Float(), nullable=False),
|
||||
sa.Column("rerank_latency_ms", sa.Float(), nullable=False),
|
||||
sa.Column("total_latency_ms", sa.Float(), nullable=False),
|
||||
sa.Column("error_code", sa.String(length=64), nullable=True),
|
||||
sa.Column("id", sa.Uuid(), nullable=False),
|
||||
sa.Column(
|
||||
"created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
|
||||
),
|
||||
sa.ForeignKeyConstraint(["run_id"], ["reranking_evaluation_runs.id"], ondelete="RESTRICT"),
|
||||
sa.ForeignKeyConstraint(["case_id"], ["evaluation_cases.id"], ondelete="RESTRICT"),
|
||||
sa.ForeignKeyConstraint(
|
||||
["candidate_pool_id"], ["retrieval_candidate_pools.id"], ondelete="RESTRICT"
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
sa.UniqueConstraint("run_id", "case_id", name="uq_reranking_case_result"),
|
||||
)
|
||||
op.create_index("ix_reranking_case_results_run_id", "reranking_case_results", ["run_id"])
|
||||
op.create_index("ix_reranking_case_results_case_id", "reranking_case_results", ["case_id"])
|
||||
op.create_index(
|
||||
"ix_reranking_case_results_pool_id", "reranking_case_results", ["candidate_pool_id"]
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"discovery_candidate_assessments",
|
||||
sa.Column("model_id", sa.Uuid(), nullable=True),
|
||||
sa.Column("upstream_snapshot_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("candidate_key", sa.String(length=128), nullable=False),
|
||||
sa.Column("repository_id", sa.String(length=255), nullable=False),
|
||||
sa.Column("resolved_commit_sha", sa.String(length=64), nullable=False),
|
||||
sa.Column("artifact_evidence", sa.JSON(), nullable=False),
|
||||
sa.Column("security_state", sa.JSON(), nullable=False),
|
||||
sa.Column("license_state", sa.JSON(), nullable=False),
|
||||
sa.Column("gpu_fit", sa.JSON(), nullable=False),
|
||||
sa.Column("status", sa.String(length=32), nullable=False),
|
||||
sa.Column("rationale", sa.Text(), nullable=False),
|
||||
sa.Column("evidence_fingerprint", sa.String(length=64), nullable=False),
|
||||
sa.Column("id", sa.Uuid(), nullable=False),
|
||||
sa.Column(
|
||||
"created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
|
||||
),
|
||||
sa.Column(
|
||||
"immutable_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
|
||||
),
|
||||
sa.ForeignKeyConstraint(["model_id"], ["models.id"], ondelete="RESTRICT"),
|
||||
sa.ForeignKeyConstraint(
|
||||
["upstream_snapshot_id"], ["upstream_snapshots.id"], ondelete="RESTRICT"
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
sa.UniqueConstraint("evidence_fingerprint"),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_discovery_assessments_model_id", "discovery_candidate_assessments", ["model_id"]
|
||||
)
|
||||
op.create_index(
|
||||
"ix_discovery_assessments_snapshot_id",
|
||||
"discovery_candidate_assessments",
|
||||
["upstream_snapshot_id"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_discovery_assessments_status", "discovery_candidate_assessments", ["status"]
|
||||
)
|
||||
|
||||
op.add_column(
|
||||
"advisor_recommendations",
|
||||
sa.Column(
|
||||
"target_kind",
|
||||
sa.String(length=32),
|
||||
nullable=False,
|
||||
server_default="embedding_deployment",
|
||||
),
|
||||
)
|
||||
op.add_column(
|
||||
"advisor_recommendations",
|
||||
sa.Column("current_pipeline_identity_id", sa.Uuid(), nullable=True),
|
||||
)
|
||||
op.add_column(
|
||||
"advisor_recommendations",
|
||||
sa.Column("candidate_pipeline_identity_id", sa.Uuid(), nullable=True),
|
||||
)
|
||||
op.create_foreign_key(
|
||||
"fk_advisor_current_pipeline",
|
||||
"advisor_recommendations",
|
||||
"retrieval_pipeline_identities",
|
||||
["current_pipeline_identity_id"],
|
||||
["id"],
|
||||
ondelete="RESTRICT",
|
||||
)
|
||||
op.create_foreign_key(
|
||||
"fk_advisor_candidate_pipeline",
|
||||
"advisor_recommendations",
|
||||
"retrieval_pipeline_identities",
|
||||
["candidate_pipeline_identity_id"],
|
||||
["id"],
|
||||
ondelete="RESTRICT",
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_constraint(
|
||||
"fk_advisor_candidate_pipeline", "advisor_recommendations", type_="foreignkey"
|
||||
)
|
||||
op.drop_constraint("fk_advisor_current_pipeline", "advisor_recommendations", type_="foreignkey")
|
||||
op.drop_column("advisor_recommendations", "candidate_pipeline_identity_id")
|
||||
op.drop_column("advisor_recommendations", "current_pipeline_identity_id")
|
||||
op.drop_column("advisor_recommendations", "target_kind")
|
||||
op.drop_table("discovery_candidate_assessments")
|
||||
op.drop_table("reranking_case_results")
|
||||
op.drop_table("reranking_evaluation_runs")
|
||||
op.drop_table("retrieval_pipeline_identities")
|
||||
op.drop_table("retrieval_candidate_pools")
|
||||
op.alter_column("capability_deployments", "embedding_space_id", nullable=False)
|
||||
@@ -0,0 +1,74 @@
|
||||
"""M9 multimodal capability evaluation foundation.
|
||||
|
||||
Revision ID: 20260826_0013
|
||||
Revises: 20260825_0012
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "20260826_0013"
|
||||
down_revision: str | None = "20260825_0012"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"capability_evaluation_suites",
|
||||
sa.Column("id", sa.Uuid(), nullable=False),
|
||||
sa.Column("capability_contract_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("key", sa.String(length=128), nullable=False),
|
||||
sa.Column("evaluation_type", sa.String(length=32), nullable=False),
|
||||
sa.Column("revision", sa.String(length=128), nullable=False),
|
||||
sa.Column("dataset_revision", sa.String(length=128), nullable=False),
|
||||
sa.Column("definition_digest", sa.String(length=64), nullable=False),
|
||||
sa.Column("metric_definitions", sa.JSON(), nullable=False),
|
||||
sa.Column("case_definitions", sa.JSON(), nullable=False),
|
||||
sa.Column("thresholds", sa.JSON(), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
|
||||
sa.Column("immutable_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
|
||||
sa.ForeignKeyConstraint(["capability_contract_id"], ["capability_contracts.id"], ondelete="RESTRICT"),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
sa.UniqueConstraint("capability_contract_id", "key", "revision", name="uq_capability_eval_suite"),
|
||||
sa.UniqueConstraint("definition_digest", name="uq_capability_eval_definition_digest"),
|
||||
)
|
||||
op.create_index("ix_capability_evaluation_suites_capability_contract_id", "capability_evaluation_suites", ["capability_contract_id"])
|
||||
op.create_index("ix_capability_evaluation_suites_evaluation_type", "capability_evaluation_suites", ["evaluation_type"])
|
||||
op.create_table(
|
||||
"capability_evaluation_runs",
|
||||
sa.Column("id", sa.Uuid(), nullable=False),
|
||||
sa.Column("suite_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("capability_deployment_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("status", sa.String(length=32), nullable=False),
|
||||
sa.Column("metric_values", sa.JSON(), nullable=False),
|
||||
sa.Column("case_results", sa.JSON(), nullable=False),
|
||||
sa.Column("resource_metrics", sa.JSON(), nullable=False),
|
||||
sa.Column("environment_fingerprint", sa.String(length=64), nullable=False),
|
||||
sa.Column("evidence_digest", sa.String(length=64), nullable=False),
|
||||
sa.Column("evidence", sa.JSON(), nullable=False),
|
||||
sa.Column("started_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("completed_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
|
||||
sa.ForeignKeyConstraint(["suite_id"], ["capability_evaluation_suites.id"], ondelete="RESTRICT"),
|
||||
sa.ForeignKeyConstraint(["capability_deployment_id"], ["capability_deployments.id"], ondelete="RESTRICT"),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
sa.UniqueConstraint("evidence_digest", name="uq_capability_eval_evidence"),
|
||||
)
|
||||
op.create_index("ix_capability_evaluation_runs_suite_id", "capability_evaluation_runs", ["suite_id"])
|
||||
op.create_index("ix_capability_evaluation_runs_capability_deployment_id", "capability_evaluation_runs", ["capability_deployment_id"])
|
||||
op.create_index("ix_capability_evaluation_runs_status", "capability_evaluation_runs", ["status"])
|
||||
op.create_index("ix_capability_evaluation_runs_environment_fingerprint", "capability_evaluation_runs", ["environment_fingerprint"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_capability_evaluation_runs_environment_fingerprint", table_name="capability_evaluation_runs")
|
||||
op.drop_index("ix_capability_evaluation_runs_status", table_name="capability_evaluation_runs")
|
||||
op.drop_index("ix_capability_evaluation_runs_capability_deployment_id", table_name="capability_evaluation_runs")
|
||||
op.drop_index("ix_capability_evaluation_runs_suite_id", table_name="capability_evaluation_runs")
|
||||
op.drop_table("capability_evaluation_runs")
|
||||
op.drop_index("ix_capability_evaluation_suites_evaluation_type", table_name="capability_evaluation_suites")
|
||||
op.drop_index("ix_capability_evaluation_suites_capability_contract_id", table_name="capability_evaluation_suites")
|
||||
op.drop_table("capability_evaluation_suites")
|
||||
@@ -0,0 +1,232 @@
|
||||
"""M10 advanced residency and GPU scheduling.
|
||||
|
||||
Revision ID: 20260826_0014
|
||||
Revises: 20260826_0013
|
||||
"""
|
||||
|
||||
import uuid
|
||||
from collections.abc import Sequence
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "20260826_0014"
|
||||
down_revision: str | None = "20260826_0013"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"residency_allocations",
|
||||
sa.Column("generation", sa.Integer(), server_default="1", nullable=False),
|
||||
)
|
||||
op.add_column(
|
||||
"residency_allocations", sa.Column("transition_reason", sa.String(length=64), nullable=True)
|
||||
)
|
||||
op.add_column(
|
||||
"serving_gpu_leases",
|
||||
sa.Column(
|
||||
"lease_type", sa.String(length=32), server_default="request_execution", nullable=False
|
||||
),
|
||||
)
|
||||
op.add_column(
|
||||
"serving_gpu_leases",
|
||||
sa.Column("materialized_vram_bytes", sa.BigInteger(), server_default="0", nullable=False),
|
||||
)
|
||||
op.add_column("serving_gpu_leases", sa.Column("serving_job_id", sa.Uuid(), nullable=True))
|
||||
op.add_column(
|
||||
"serving_gpu_leases",
|
||||
sa.Column("generation", sa.Integer(), server_default="1", nullable=False),
|
||||
)
|
||||
op.create_foreign_key(
|
||||
"fk_serving_gpu_leases_job",
|
||||
"serving_gpu_leases",
|
||||
"serving_jobs",
|
||||
["serving_job_id"],
|
||||
["id"],
|
||||
ondelete="SET NULL",
|
||||
)
|
||||
op.create_index(
|
||||
"ix_serving_gpu_leases_serving_job_id", "serving_gpu_leases", ["serving_job_id"]
|
||||
)
|
||||
op.create_table(
|
||||
"scheduler_policy_revisions",
|
||||
sa.Column("id", sa.Uuid(), nullable=False),
|
||||
sa.Column("revision", sa.String(length=64), nullable=False),
|
||||
sa.Column("configuration", sa.JSON(), nullable=False),
|
||||
sa.Column("active", sa.Boolean(), nullable=False),
|
||||
sa.Column(
|
||||
"created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
sa.UniqueConstraint("revision"),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_scheduler_policy_revisions_active", "scheduler_policy_revisions", ["active"]
|
||||
)
|
||||
policy_table = sa.table(
|
||||
"scheduler_policy_revisions",
|
||||
sa.column("id", sa.Uuid()),
|
||||
sa.column("revision", sa.String()),
|
||||
sa.column("configuration", sa.JSON()),
|
||||
sa.column("active", sa.Boolean()),
|
||||
)
|
||||
op.bulk_insert(
|
||||
policy_table,
|
||||
[
|
||||
{
|
||||
"id": uuid.UUID("8e4d6a85-8e50-4f78-a182-000000000010"),
|
||||
"revision": "m10-v1",
|
||||
"configuration": {
|
||||
"reserve_minimum_bytes": 1073741824,
|
||||
"reserve_percentage": 0.05,
|
||||
"runtime_margin_bytes": 268435456,
|
||||
"deployment_margin_minimum_bytes": 134217728,
|
||||
"deployment_margin_percentage": 0.1,
|
||||
"request_execution_floor_bytes": 67108864,
|
||||
"pressure_stable_seconds": 30,
|
||||
"eviction_cooldown_seconds": 60,
|
||||
"global_queue_limit": 128,
|
||||
"placement_history_limit": 500,
|
||||
"lab_paused": False,
|
||||
},
|
||||
"active": True,
|
||||
}
|
||||
],
|
||||
)
|
||||
op.create_table(
|
||||
"scheduler_accelerator_states",
|
||||
sa.Column("id", sa.Uuid(), nullable=False),
|
||||
sa.Column("accelerator_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("pressure_state", sa.String(length=16), nullable=False),
|
||||
sa.Column("recovery_candidate", sa.String(length=16), nullable=True),
|
||||
sa.Column("recovery_candidate_since", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column(
|
||||
"pressure_changed_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.func.now(),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"last_observed_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.func.now(),
|
||||
nullable=False,
|
||||
),
|
||||
sa.ForeignKeyConstraint(["accelerator_id"], ["accelerators.id"], ondelete="CASCADE"),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
sa.UniqueConstraint("accelerator_id"),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_scheduler_accelerator_states_accelerator_id",
|
||||
"scheduler_accelerator_states",
|
||||
["accelerator_id"],
|
||||
unique=True,
|
||||
)
|
||||
op.create_table(
|
||||
"placement_plans",
|
||||
sa.Column("id", sa.Uuid(), nullable=False),
|
||||
sa.Column("capability_deployment_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("request_id", sa.Uuid(), nullable=True),
|
||||
sa.Column("policy_revision", sa.String(length=64), nullable=False),
|
||||
sa.Column("verdict", sa.String(length=32), nullable=False),
|
||||
sa.Column("reason_codes", sa.JSON(), nullable=False),
|
||||
sa.Column("evidence", sa.JSON(), nullable=False),
|
||||
sa.Column("evidence_fingerprint", sa.String(length=64), nullable=False),
|
||||
sa.Column("dry_run", sa.Boolean(), nullable=False),
|
||||
sa.Column(
|
||||
"created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["capability_deployment_id"], ["capability_deployments.id"], ondelete="RESTRICT"
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
)
|
||||
for column in (
|
||||
"capability_deployment_id",
|
||||
"request_id",
|
||||
"verdict",
|
||||
"evidence_fingerprint",
|
||||
"created_at",
|
||||
):
|
||||
op.create_index(f"ix_placement_plans_{column}", "placement_plans", [column])
|
||||
op.create_table(
|
||||
"co_residency_evidence",
|
||||
sa.Column("id", sa.Uuid(), nullable=False),
|
||||
sa.Column("left_deployment_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("right_deployment_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("status", sa.String(length=32), nullable=False),
|
||||
sa.Column("expected_combined_bytes", sa.BigInteger(), nullable=False),
|
||||
sa.Column("measured_combined_bytes", sa.BigInteger(), nullable=True),
|
||||
sa.Column("evidence", sa.JSON(), nullable=False),
|
||||
sa.Column("measured_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column(
|
||||
"created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["left_deployment_id"], ["capability_deployments.id"], ondelete="RESTRICT"
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["right_deployment_id"], ["capability_deployments.id"], ondelete="RESTRICT"
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
sa.UniqueConstraint(
|
||||
"left_deployment_id", "right_deployment_id", name="uq_co_residency_pair"
|
||||
),
|
||||
)
|
||||
for column in ("left_deployment_id", "right_deployment_id", "status"):
|
||||
op.create_index(f"ix_co_residency_evidence_{column}", "co_residency_evidence", [column])
|
||||
op.create_table(
|
||||
"scheduler_evictions",
|
||||
sa.Column("id", sa.Uuid(), nullable=False),
|
||||
sa.Column("capability_deployment_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("requested_deployment_id", sa.Uuid(), nullable=True),
|
||||
sa.Column("reason_code", sa.String(length=64), nullable=False),
|
||||
sa.Column("expected_reclaimed_bytes", sa.BigInteger(), nullable=False),
|
||||
sa.Column("actual_reclaimed_bytes", sa.BigInteger(), nullable=True),
|
||||
sa.Column("state", sa.String(length=32), nullable=False),
|
||||
sa.Column(
|
||||
"created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["capability_deployment_id"], ["capability_deployments.id"], ondelete="RESTRICT"
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["requested_deployment_id"], ["capability_deployments.id"], ondelete="SET NULL"
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
)
|
||||
for column in ("capability_deployment_id", "requested_deployment_id", "reason_code"):
|
||||
op.create_index(f"ix_scheduler_evictions_{column}", "scheduler_evictions", [column])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index(
|
||||
"ix_scheduler_accelerator_states_accelerator_id", table_name="scheduler_accelerator_states"
|
||||
)
|
||||
op.drop_table("scheduler_accelerator_states")
|
||||
for column in ("capability_deployment_id", "requested_deployment_id", "reason_code"):
|
||||
op.drop_index(f"ix_scheduler_evictions_{column}", table_name="scheduler_evictions")
|
||||
op.drop_table("scheduler_evictions")
|
||||
for column in ("left_deployment_id", "right_deployment_id", "status"):
|
||||
op.drop_index(f"ix_co_residency_evidence_{column}", table_name="co_residency_evidence")
|
||||
op.drop_table("co_residency_evidence")
|
||||
for column in (
|
||||
"capability_deployment_id",
|
||||
"request_id",
|
||||
"verdict",
|
||||
"evidence_fingerprint",
|
||||
"created_at",
|
||||
):
|
||||
op.drop_index(f"ix_placement_plans_{column}", table_name="placement_plans")
|
||||
op.drop_table("placement_plans")
|
||||
op.drop_index("ix_scheduler_policy_revisions_active", table_name="scheduler_policy_revisions")
|
||||
op.drop_table("scheduler_policy_revisions")
|
||||
op.drop_index("ix_serving_gpu_leases_serving_job_id", table_name="serving_gpu_leases")
|
||||
op.drop_constraint("fk_serving_gpu_leases_job", "serving_gpu_leases", type_="foreignkey")
|
||||
for column in ("generation", "serving_job_id", "materialized_vram_bytes", "lease_type"):
|
||||
op.drop_column("serving_gpu_leases", column)
|
||||
op.drop_column("residency_allocations", "transition_reason")
|
||||
op.drop_column("residency_allocations", "generation")
|
||||
@@ -0,0 +1,54 @@
|
||||
"""Use 64-bit scheduler and worker generations.
|
||||
|
||||
Revision ID: 20260826_0015
|
||||
Revises: 20260826_0014
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "20260826_0015"
|
||||
down_revision: str | None = "20260826_0014"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.alter_column(
|
||||
"residency_allocations",
|
||||
"generation",
|
||||
existing_type=sa.Integer(),
|
||||
type_=sa.BigInteger(),
|
||||
existing_nullable=False,
|
||||
existing_server_default="1",
|
||||
)
|
||||
op.alter_column(
|
||||
"serving_gpu_leases",
|
||||
"generation",
|
||||
existing_type=sa.Integer(),
|
||||
type_=sa.BigInteger(),
|
||||
existing_nullable=False,
|
||||
existing_server_default="1",
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.alter_column(
|
||||
"serving_gpu_leases",
|
||||
"generation",
|
||||
existing_type=sa.BigInteger(),
|
||||
type_=sa.Integer(),
|
||||
existing_nullable=False,
|
||||
existing_server_default="1",
|
||||
)
|
||||
op.alter_column(
|
||||
"residency_allocations",
|
||||
"generation",
|
||||
existing_type=sa.BigInteger(),
|
||||
type_=sa.Integer(),
|
||||
existing_nullable=False,
|
||||
existing_server_default="1",
|
||||
)
|
||||
@@ -0,0 +1,65 @@
|
||||
"""Add operational project integrations and project-fit evidence.
|
||||
|
||||
Revision ID: 20260826_0016
|
||||
Revises: 20260826_0015
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "20260826_0016"
|
||||
down_revision: str | None = "20260826_0015"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("service_clients", sa.Column("project_binding_id", sa.Uuid(), nullable=True))
|
||||
op.add_column("service_clients", sa.Column("integration_environment", sa.String(32), nullable=True))
|
||||
op.add_column("service_clients", sa.Column("purpose", sa.Text(), nullable=True))
|
||||
op.create_foreign_key(
|
||||
"fk_service_clients_project_binding",
|
||||
"service_clients",
|
||||
"project_bindings",
|
||||
["project_binding_id"],
|
||||
["id"],
|
||||
ondelete="RESTRICT",
|
||||
)
|
||||
op.create_index("ix_service_clients_project_binding_id", "service_clients", ["project_binding_id"])
|
||||
op.create_table(
|
||||
"project_fit_evidence",
|
||||
sa.Column("id", sa.Uuid(), nullable=False),
|
||||
sa.Column("project_binding_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("capability_deployment_id", sa.Uuid(), nullable=True),
|
||||
sa.Column("environment", sa.String(32), nullable=False),
|
||||
sa.Column("recommendation", sa.String(32), nullable=False),
|
||||
sa.Column("case_count", sa.Integer(), nullable=False),
|
||||
sa.Column("metric_values", sa.JSON(), nullable=False),
|
||||
sa.Column("critical_errors", sa.Integer(), nullable=False),
|
||||
sa.Column("blockers", sa.JSON(), nullable=False),
|
||||
sa.Column("evidence_digest", sa.String(64), nullable=False),
|
||||
sa.Column("evidence", sa.JSON(), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
|
||||
sa.ForeignKeyConstraint(["project_binding_id"], ["project_bindings.id"], ondelete="RESTRICT"),
|
||||
sa.ForeignKeyConstraint(["capability_deployment_id"], ["capability_deployments.id"], ondelete="SET NULL"),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
sa.UniqueConstraint("evidence_digest", name="uq_project_fit_evidence_digest"),
|
||||
)
|
||||
op.create_index("ix_project_fit_evidence_project_binding_id", "project_fit_evidence", ["project_binding_id"])
|
||||
op.create_index("ix_project_fit_evidence_capability_deployment_id", "project_fit_evidence", ["capability_deployment_id"])
|
||||
op.create_index("ix_project_fit_evidence_recommendation", "project_fit_evidence", ["recommendation"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_project_fit_evidence_recommendation", table_name="project_fit_evidence")
|
||||
op.drop_index("ix_project_fit_evidence_capability_deployment_id", table_name="project_fit_evidence")
|
||||
op.drop_index("ix_project_fit_evidence_project_binding_id", table_name="project_fit_evidence")
|
||||
op.drop_table("project_fit_evidence")
|
||||
op.drop_index("ix_service_clients_project_binding_id", table_name="service_clients")
|
||||
op.drop_constraint("fk_service_clients_project_binding", "service_clients", type_="foreignkey")
|
||||
op.drop_column("service_clients", "purpose")
|
||||
op.drop_column("service_clients", "integration_environment")
|
||||
op.drop_column("service_clients", "project_binding_id")
|
||||
@@ -0,0 +1,82 @@
|
||||
"""Classify project-fit evidence and defer production validation explicitly.
|
||||
|
||||
Revision ID: 20260826_0017
|
||||
Revises: 20260826_0016
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "20260826_0017"
|
||||
down_revision: str | None = "20260826_0016"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"project_fit_evidence",
|
||||
sa.Column(
|
||||
"evidence_class",
|
||||
sa.String(32),
|
||||
server_default="CATALOG_REFERENCE",
|
||||
nullable=False,
|
||||
),
|
||||
)
|
||||
op.add_column(
|
||||
"project_fit_evidence",
|
||||
sa.Column(
|
||||
"engineering_integration",
|
||||
sa.String(32),
|
||||
server_default="PASS",
|
||||
nullable=False,
|
||||
),
|
||||
)
|
||||
op.add_column(
|
||||
"project_fit_evidence",
|
||||
sa.Column(
|
||||
"production_validation",
|
||||
sa.String(48),
|
||||
server_default="DEFERRED_EXTERNAL_VALIDATION",
|
||||
nullable=False,
|
||||
),
|
||||
)
|
||||
op.add_column(
|
||||
"project_fit_evidence",
|
||||
sa.Column("production_action", sa.String(32), server_default="NONE", nullable=False),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_project_fit_evidence_evidence_class",
|
||||
"project_fit_evidence",
|
||||
["evidence_class"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_project_fit_evidence_production_validation",
|
||||
"project_fit_evidence",
|
||||
["production_validation"],
|
||||
)
|
||||
for column in (
|
||||
"evidence_class",
|
||||
"engineering_integration",
|
||||
"production_validation",
|
||||
"production_action",
|
||||
):
|
||||
op.alter_column("project_fit_evidence", column, server_default=None)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index(
|
||||
"ix_project_fit_evidence_production_validation",
|
||||
table_name="project_fit_evidence",
|
||||
)
|
||||
op.drop_index(
|
||||
"ix_project_fit_evidence_evidence_class",
|
||||
table_name="project_fit_evidence",
|
||||
)
|
||||
op.drop_column("project_fit_evidence", "production_action")
|
||||
op.drop_column("project_fit_evidence", "production_validation")
|
||||
op.drop_column("project_fit_evidence", "engineering_integration")
|
||||
op.drop_column("project_fit_evidence", "evidence_class")
|
||||
@@ -0,0 +1,431 @@
|
||||
"""Add evidence-bound lifecycle, rollback, retention and cleanup records.
|
||||
|
||||
Revision ID: 20260826_0018
|
||||
Revises: 20260826_0017
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "20260826_0018"
|
||||
down_revision: str | None = "20260826_0017"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def id_column() -> sa.Column[object]:
|
||||
return sa.Column("id", sa.Uuid(), primary_key=True, nullable=False)
|
||||
|
||||
|
||||
def created_at() -> sa.Column[object]:
|
||||
return sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now())
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"lifecycle_subjects",
|
||||
id_column(),
|
||||
sa.Column("target_type", sa.String(48), nullable=False),
|
||||
sa.Column("target_ref", sa.String(255), nullable=False),
|
||||
sa.Column("environment", sa.String(32), nullable=False),
|
||||
sa.Column("state", sa.String(48), nullable=False),
|
||||
sa.Column("version", sa.Integer(), nullable=False, server_default="1"),
|
||||
sa.Column("superseded_by_ref", sa.String(255)),
|
||||
created_at(),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.CheckConstraint("version >= 1", name="ck_lifecycle_subject_version"),
|
||||
sa.UniqueConstraint(
|
||||
"target_type", "target_ref", "environment", name="uq_lifecycle_subject_target"
|
||||
),
|
||||
)
|
||||
op.create_index("ix_lifecycle_subjects_target_type", "lifecycle_subjects", ["target_type"])
|
||||
op.create_index("ix_lifecycle_subjects_target_ref", "lifecycle_subjects", ["target_ref"])
|
||||
op.create_index("ix_lifecycle_subjects_environment", "lifecycle_subjects", ["environment"])
|
||||
op.create_index("ix_lifecycle_subjects_state", "lifecycle_subjects", ["state"])
|
||||
|
||||
op.create_table(
|
||||
"lifecycle_policy_revisions",
|
||||
id_column(),
|
||||
sa.Column("key", sa.String(128), nullable=False),
|
||||
sa.Column("revision", sa.Integer(), nullable=False),
|
||||
sa.Column("scope", sa.String(48), nullable=False),
|
||||
sa.Column("requirements", sa.JSON(), nullable=False),
|
||||
sa.Column("fingerprint", sa.String(64), nullable=False),
|
||||
sa.Column("active", sa.Boolean(), nullable=False, server_default=sa.true()),
|
||||
sa.Column("created_by", sa.String(255), nullable=False),
|
||||
created_at(),
|
||||
sa.Column("immutable_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.UniqueConstraint("key", "revision", name="uq_lifecycle_policy_revision"),
|
||||
sa.UniqueConstraint("fingerprint", name="uq_lifecycle_policy_fingerprint"),
|
||||
)
|
||||
op.create_index("ix_lifecycle_policy_revisions_key", "lifecycle_policy_revisions", ["key"])
|
||||
op.create_index("ix_lifecycle_policy_revisions_scope", "lifecycle_policy_revisions", ["scope"])
|
||||
op.create_index(
|
||||
"ix_lifecycle_policy_revisions_active", "lifecycle_policy_revisions", ["active"]
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"lifecycle_approval_requests",
|
||||
id_column(),
|
||||
sa.Column(
|
||||
"policy_revision_id",
|
||||
sa.Uuid(),
|
||||
sa.ForeignKey("lifecycle_policy_revisions.id", ondelete="RESTRICT"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"subject_id",
|
||||
sa.Uuid(),
|
||||
sa.ForeignKey("lifecycle_subjects.id", ondelete="RESTRICT"),
|
||||
),
|
||||
sa.Column("target_type", sa.String(48), nullable=False),
|
||||
sa.Column("target_ref", sa.String(255), nullable=False),
|
||||
sa.Column("environment", sa.String(32), nullable=False),
|
||||
sa.Column("requested_transition", sa.String(48), nullable=False),
|
||||
sa.Column("evidence_snapshot", sa.JSON(), nullable=False),
|
||||
sa.Column("evidence_fingerprint", sa.String(64), nullable=False),
|
||||
sa.Column("status", sa.String(32), nullable=False),
|
||||
sa.Column("blockers", sa.JSON(), nullable=False),
|
||||
sa.Column("warnings", sa.JSON(), nullable=False),
|
||||
sa.Column("requested_by", sa.String(255), nullable=False),
|
||||
sa.Column("approved_by", sa.String(255)),
|
||||
sa.Column("reason", sa.Text(), nullable=False),
|
||||
sa.Column("expires_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("stale_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("decided_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("version", sa.Integer(), nullable=False, server_default="1"),
|
||||
created_at(),
|
||||
sa.CheckConstraint("version >= 1", name="ck_lifecycle_approval_version"),
|
||||
)
|
||||
for column in (
|
||||
"policy_revision_id",
|
||||
"subject_id",
|
||||
"target_type",
|
||||
"target_ref",
|
||||
"environment",
|
||||
"evidence_fingerprint",
|
||||
"status",
|
||||
"expires_at",
|
||||
"created_at",
|
||||
):
|
||||
op.create_index(
|
||||
f"ix_lifecycle_approval_requests_{column}", "lifecycle_approval_requests", [column]
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"lifecycle_approval_evidence",
|
||||
sa.Column(
|
||||
"approval_request_id",
|
||||
sa.Uuid(),
|
||||
sa.ForeignKey("lifecycle_approval_requests.id", ondelete="RESTRICT"),
|
||||
primary_key=True,
|
||||
),
|
||||
sa.Column("evidence_type", sa.String(64), primary_key=True),
|
||||
sa.Column("evidence_ref", sa.String(255), primary_key=True),
|
||||
sa.Column("evidence_digest", sa.String(64), nullable=False),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_lifecycle_approval_evidence_evidence_ref",
|
||||
"lifecycle_approval_evidence",
|
||||
["evidence_ref"],
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"lifecycle_promotion_plans",
|
||||
id_column(),
|
||||
sa.Column(
|
||||
"approval_request_id",
|
||||
sa.Uuid(),
|
||||
sa.ForeignKey("lifecycle_approval_requests.id", ondelete="RESTRICT"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"subject_id",
|
||||
sa.Uuid(),
|
||||
sa.ForeignKey("lifecycle_subjects.id", ondelete="RESTRICT"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("current_state", sa.String(48), nullable=False),
|
||||
sa.Column("desired_state", sa.String(48), nullable=False),
|
||||
sa.Column("migration_class", sa.String(32), nullable=False),
|
||||
sa.Column(
|
||||
"candidate_deployment_id",
|
||||
sa.Uuid(),
|
||||
sa.ForeignKey("capability_deployments.id", ondelete="RESTRICT"),
|
||||
),
|
||||
sa.Column("rollback_target_ref", sa.String(255), nullable=False),
|
||||
sa.Column("project_consumers", sa.JSON(), nullable=False),
|
||||
sa.Column("affected_identities", sa.JSON(), nullable=False),
|
||||
sa.Column("impact_analysis", sa.JSON(), nullable=False),
|
||||
sa.Column(
|
||||
"migration_id",
|
||||
sa.Uuid(),
|
||||
sa.ForeignKey("embedding_migrations.id", ondelete="RESTRICT"),
|
||||
),
|
||||
sa.Column("canary_strategy", sa.JSON(), nullable=False),
|
||||
sa.Column("drain_strategy", sa.JSON(), nullable=False),
|
||||
sa.Column("health_gates", sa.JSON(), nullable=False),
|
||||
sa.Column("automatic_abort_conditions", sa.JSON(), nullable=False),
|
||||
sa.Column("plan_fingerprint", sa.String(64), nullable=False),
|
||||
sa.Column("status", sa.String(32), nullable=False, server_default="DRAFT"),
|
||||
sa.Column("version", sa.Integer(), nullable=False, server_default="1"),
|
||||
sa.Column("created_by", sa.String(255), nullable=False),
|
||||
sa.Column("approved_by", sa.String(255)),
|
||||
sa.Column("immutable_at", sa.DateTime(timezone=True)),
|
||||
created_at(),
|
||||
sa.CheckConstraint("version >= 1", name="ck_lifecycle_plan_version"),
|
||||
sa.UniqueConstraint("plan_fingerprint", name="uq_lifecycle_plan_fingerprint"),
|
||||
)
|
||||
for column in (
|
||||
"approval_request_id",
|
||||
"subject_id",
|
||||
"migration_class",
|
||||
"candidate_deployment_id",
|
||||
"migration_id",
|
||||
"status",
|
||||
"created_at",
|
||||
):
|
||||
op.create_index(
|
||||
f"ix_lifecycle_promotion_plans_{column}", "lifecycle_promotion_plans", [column]
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"lifecycle_rollback_snapshots",
|
||||
id_column(),
|
||||
sa.Column(
|
||||
"promotion_plan_id",
|
||||
sa.Uuid(),
|
||||
sa.ForeignKey("lifecycle_promotion_plans.id", ondelete="RESTRICT"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("snapshot", sa.JSON(), nullable=False),
|
||||
sa.Column("snapshot_digest", sa.String(64), nullable=False),
|
||||
created_at(),
|
||||
sa.Column("immutable_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.UniqueConstraint("promotion_plan_id", name="uq_lifecycle_rollback_plan"),
|
||||
sa.UniqueConstraint("snapshot_digest", name="uq_lifecycle_rollback_digest"),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_lifecycle_rollback_snapshots_promotion_plan_id",
|
||||
"lifecycle_rollback_snapshots",
|
||||
["promotion_plan_id"],
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"lifecycle_operations",
|
||||
id_column(),
|
||||
sa.Column(
|
||||
"promotion_plan_id",
|
||||
sa.Uuid(),
|
||||
sa.ForeignKey("lifecycle_promotion_plans.id", ondelete="RESTRICT"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("stage", sa.String(32), nullable=False),
|
||||
sa.Column("idempotency_key", sa.String(128), nullable=False),
|
||||
sa.Column("expected_subject_version", sa.Integer(), nullable=False),
|
||||
sa.Column("requester", sa.String(255), nullable=False),
|
||||
sa.Column("approver", sa.String(255), nullable=False),
|
||||
sa.Column("executor", sa.String(255), nullable=False),
|
||||
sa.Column("failure_code", sa.String(64)),
|
||||
sa.Column("failure_details", sa.JSON(), nullable=False),
|
||||
sa.Column("rollback_duration_ms", sa.Float()),
|
||||
sa.Column("version", sa.Integer(), nullable=False, server_default="1"),
|
||||
sa.Column("started_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.Column("finished_at", sa.DateTime(timezone=True)),
|
||||
sa.CheckConstraint("version >= 1", name="ck_lifecycle_operation_version"),
|
||||
sa.UniqueConstraint("idempotency_key", name="uq_lifecycle_operation_idempotency"),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_lifecycle_operations_promotion_plan_id", "lifecycle_operations", ["promotion_plan_id"]
|
||||
)
|
||||
op.create_index("ix_lifecycle_operations_stage", "lifecycle_operations", ["stage"])
|
||||
|
||||
op.create_table(
|
||||
"lifecycle_canary_runs",
|
||||
id_column(),
|
||||
sa.Column(
|
||||
"operation_id",
|
||||
sa.Uuid(),
|
||||
sa.ForeignKey("lifecycle_operations.id", ondelete="RESTRICT"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("mode", sa.String(48), nullable=False),
|
||||
sa.Column("traffic_percent", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("eligible_clients", sa.JSON(), nullable=False),
|
||||
sa.Column("eligible_projects", sa.JSON(), nullable=False),
|
||||
sa.Column("target_request_count", sa.Integer(), nullable=False),
|
||||
sa.Column("thresholds", sa.JSON(), nullable=False),
|
||||
sa.Column("status", sa.String(48), nullable=False),
|
||||
sa.Column("request_count", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("error_count", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("latency_p95_ms", sa.Float()),
|
||||
sa.Column("abort_trigger", sa.String(64)),
|
||||
sa.Column("result", sa.JSON(), nullable=False),
|
||||
sa.Column("started_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.Column("finished_at", sa.DateTime(timezone=True)),
|
||||
sa.UniqueConstraint("operation_id", name="uq_lifecycle_canary_operation"),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_lifecycle_canary_runs_operation_id", "lifecycle_canary_runs", ["operation_id"]
|
||||
)
|
||||
op.create_index("ix_lifecycle_canary_runs_status", "lifecycle_canary_runs", ["status"])
|
||||
|
||||
op.create_table(
|
||||
"retention_policy_revisions",
|
||||
id_column(),
|
||||
sa.Column("key", sa.String(128), nullable=False),
|
||||
sa.Column("revision", sa.Integer(), nullable=False),
|
||||
sa.Column("minimum_rollback_days", sa.Integer(), nullable=False),
|
||||
sa.Column("requirements", sa.JSON(), nullable=False),
|
||||
sa.Column("fingerprint", sa.String(64), nullable=False),
|
||||
sa.Column("active", sa.Boolean(), nullable=False, server_default=sa.true()),
|
||||
sa.Column("created_by", sa.String(255), nullable=False),
|
||||
created_at(),
|
||||
sa.Column("immutable_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.UniqueConstraint("key", "revision", name="uq_retention_policy_revision"),
|
||||
sa.UniqueConstraint("fingerprint", name="uq_retention_policy_fingerprint"),
|
||||
)
|
||||
op.create_index("ix_retention_policy_revisions_key", "retention_policy_revisions", ["key"])
|
||||
op.create_index(
|
||||
"ix_retention_policy_revisions_active", "retention_policy_revisions", ["active"]
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"lifecycle_retention_records",
|
||||
id_column(),
|
||||
sa.Column(
|
||||
"policy_revision_id",
|
||||
sa.Uuid(),
|
||||
sa.ForeignKey("retention_policy_revisions.id", ondelete="RESTRICT"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("target_type", sa.String(48), nullable=False),
|
||||
sa.Column("target_ref", sa.String(255), nullable=False),
|
||||
sa.Column("state", sa.String(32), nullable=False),
|
||||
sa.Column("retained_until", sa.DateTime(timezone=True)),
|
||||
sa.Column("legal_hold", sa.Boolean(), nullable=False, server_default=sa.false()),
|
||||
sa.Column("reason", sa.Text(), nullable=False),
|
||||
created_at(),
|
||||
sa.UniqueConstraint("target_type", "target_ref", name="uq_lifecycle_retention_target"),
|
||||
)
|
||||
for column in ("policy_revision_id", "target_type", "target_ref", "state", "retained_until"):
|
||||
op.create_index(
|
||||
f"ix_lifecycle_retention_records_{column}", "lifecycle_retention_records", [column]
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"lifecycle_cleanup_plans",
|
||||
id_column(),
|
||||
sa.Column("target_type", sa.String(48), nullable=False),
|
||||
sa.Column("target_ref", sa.String(255), nullable=False),
|
||||
sa.Column("action", sa.String(48), nullable=False),
|
||||
sa.Column("dependencies", sa.JSON(), nullable=False),
|
||||
sa.Column("dependency_digest", sa.String(64), nullable=False),
|
||||
sa.Column("reclaimable_bytes", sa.BigInteger(), nullable=False),
|
||||
sa.Column("retention_state", sa.String(32), nullable=False),
|
||||
sa.Column("blockers", sa.JSON(), nullable=False),
|
||||
sa.Column("status", sa.String(32), nullable=False),
|
||||
sa.Column("created_by", sa.String(255), nullable=False),
|
||||
created_at(),
|
||||
sa.Column("executed_at", sa.DateTime(timezone=True)),
|
||||
)
|
||||
for column in ("target_type", "target_ref", "status", "created_at"):
|
||||
op.create_index(f"ix_lifecycle_cleanup_plans_{column}", "lifecycle_cleanup_plans", [column])
|
||||
|
||||
op.create_table(
|
||||
"artifact_location_removal_records",
|
||||
id_column(),
|
||||
sa.Column(
|
||||
"artifact_location_id",
|
||||
sa.Uuid(),
|
||||
sa.ForeignKey("artifact_locations.id", ondelete="RESTRICT"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"cleanup_plan_id",
|
||||
sa.Uuid(),
|
||||
sa.ForeignKey("lifecycle_cleanup_plans.id", ondelete="RESTRICT"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("prior_status", sa.String(32), nullable=False),
|
||||
sa.Column("prior_observed_sha256", sa.String(64)),
|
||||
sa.Column("prior_size_bytes", sa.BigInteger()),
|
||||
sa.Column("removed_by", sa.String(255), nullable=False),
|
||||
sa.Column("removed_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_artifact_location_removal_records_artifact_location_id",
|
||||
"artifact_location_removal_records",
|
||||
["artifact_location_id"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_artifact_location_removal_records_cleanup_plan_id",
|
||||
"artifact_location_removal_records",
|
||||
["cleanup_plan_id"],
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"lifecycle_events",
|
||||
id_column(),
|
||||
sa.Column("event_type", sa.String(64), nullable=False),
|
||||
sa.Column("object_type", sa.String(48), nullable=False),
|
||||
sa.Column("object_ref", sa.String(255), nullable=False),
|
||||
sa.Column("from_state", sa.String(48)),
|
||||
sa.Column("to_state", sa.String(48)),
|
||||
sa.Column("actor", sa.String(255), nullable=False),
|
||||
sa.Column("actor_role", sa.String(32), nullable=False),
|
||||
sa.Column(
|
||||
"policy_revision_id",
|
||||
sa.Uuid(),
|
||||
sa.ForeignKey("lifecycle_policy_revisions.id", ondelete="RESTRICT"),
|
||||
),
|
||||
sa.Column("evidence_ids", sa.JSON(), nullable=False),
|
||||
sa.Column("reason", sa.Text(), nullable=False),
|
||||
sa.Column("change_id", sa.String(64), nullable=False),
|
||||
sa.Column("details", sa.JSON(), nullable=False),
|
||||
sa.Column("occurred_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
)
|
||||
for column in (
|
||||
"event_type",
|
||||
"object_type",
|
||||
"object_ref",
|
||||
"policy_revision_id",
|
||||
"change_id",
|
||||
"occurred_at",
|
||||
):
|
||||
op.create_index(f"ix_lifecycle_events_{column}", "lifecycle_events", [column])
|
||||
|
||||
op.create_index(
|
||||
"uq_capability_deployment_one_production_stable",
|
||||
"capability_deployments",
|
||||
["capability_contract_id"],
|
||||
unique=True,
|
||||
postgresql_where=sa.text("production = true AND status = 'stable'"),
|
||||
sqlite_where=sa.text("production = 1 AND status = 'stable'"),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index(
|
||||
"uq_capability_deployment_one_production_stable",
|
||||
table_name="capability_deployments",
|
||||
)
|
||||
for table in (
|
||||
"lifecycle_events",
|
||||
"artifact_location_removal_records",
|
||||
"lifecycle_cleanup_plans",
|
||||
"lifecycle_retention_records",
|
||||
"retention_policy_revisions",
|
||||
"lifecycle_canary_runs",
|
||||
"lifecycle_operations",
|
||||
"lifecycle_rollback_snapshots",
|
||||
"lifecycle_promotion_plans",
|
||||
"lifecycle_approval_evidence",
|
||||
"lifecycle_approval_requests",
|
||||
"lifecycle_policy_revisions",
|
||||
"lifecycle_subjects",
|
||||
):
|
||||
op.drop_table(table)
|
||||
@@ -0,0 +1,415 @@
|
||||
"""Add typed resumable migration-engine records.
|
||||
|
||||
Revision ID: 20260826_0019
|
||||
Revises: 20260826_0018
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "20260826_0019"
|
||||
down_revision: str | None = "20260826_0018"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def _id() -> sa.Column[object]:
|
||||
return sa.Column("id", sa.Uuid(), primary_key=True, nullable=False)
|
||||
|
||||
|
||||
def _created() -> sa.Column[object]:
|
||||
return sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now())
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"migration_validation_policy_revisions",
|
||||
_id(),
|
||||
sa.Column("key", sa.String(128), nullable=False),
|
||||
sa.Column("revision", sa.Integer(), nullable=False),
|
||||
sa.Column("required_completeness", sa.Float(), nullable=False),
|
||||
sa.Column("allowed_failures", sa.Integer(), nullable=False),
|
||||
sa.Column("required_evaluation", sa.Boolean(), nullable=False),
|
||||
sa.Column("critical_regressions_allowed", sa.Integer(), nullable=False),
|
||||
sa.Column("maximum_latency_regression_ratio", sa.Float()),
|
||||
sa.Column("require_project_fit", sa.Boolean(), nullable=False),
|
||||
sa.Column("require_external_validation", sa.Boolean(), nullable=False),
|
||||
sa.Column("require_security_approved", sa.Boolean(), nullable=False),
|
||||
sa.Column("allow_isolated_lab_cutover", sa.Boolean(), nullable=False),
|
||||
sa.Column("fingerprint", sa.String(64), nullable=False),
|
||||
sa.Column("active", sa.Boolean(), nullable=False, server_default=sa.true()),
|
||||
sa.Column("created_by", sa.String(255), nullable=False),
|
||||
_created(),
|
||||
sa.Column("immutable_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.UniqueConstraint("key", "revision", name="uq_migration_validation_policy_revision"),
|
||||
sa.UniqueConstraint("fingerprint", name="uq_migration_validation_policy_fingerprint"),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_migration_validation_policy_revisions_key",
|
||||
"migration_validation_policy_revisions",
|
||||
["key"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_migration_validation_policy_revisions_active",
|
||||
"migration_validation_policy_revisions",
|
||||
["active"],
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"migration_plans",
|
||||
_id(),
|
||||
sa.Column(
|
||||
"project_id",
|
||||
sa.Uuid(),
|
||||
sa.ForeignKey("projects.id", ondelete="RESTRICT"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"project_binding_id",
|
||||
sa.Uuid(),
|
||||
sa.ForeignKey("project_bindings.id", ondelete="RESTRICT"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"capability_contract_id",
|
||||
sa.Uuid(),
|
||||
sa.ForeignKey("capability_contracts.id", ondelete="RESTRICT"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("migration_class", sa.String(32), nullable=False),
|
||||
sa.Column("environment", sa.String(32), nullable=False),
|
||||
sa.Column("adapter", sa.JSON(), nullable=False),
|
||||
sa.Column("source_identity", sa.JSON(), nullable=False),
|
||||
sa.Column("target_identity", sa.JSON(), nullable=False),
|
||||
sa.Column("source_data_target", sa.Text(), nullable=False),
|
||||
sa.Column("target_shadow_target", sa.Text(), nullable=False),
|
||||
sa.Column("source_space_ref", sa.String(255), nullable=False),
|
||||
sa.Column(
|
||||
"target_space_id",
|
||||
sa.Uuid(),
|
||||
sa.ForeignKey("embedding_spaces.id", ondelete="RESTRICT"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("corpus_revision", sa.String(128), nullable=False),
|
||||
sa.Column("migration_policy_revision", sa.String(128), nullable=False),
|
||||
sa.Column(
|
||||
"validation_policy_revision_id",
|
||||
sa.Uuid(),
|
||||
sa.ForeignKey("migration_validation_policy_revisions.id", ondelete="RESTRICT"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"lifecycle_approval_id",
|
||||
sa.Uuid(),
|
||||
sa.ForeignKey("lifecycle_approval_requests.id", ondelete="RESTRICT"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"promotion_plan_id",
|
||||
sa.Uuid(),
|
||||
sa.ForeignKey("lifecycle_promotion_plans.id", ondelete="RESTRICT"),
|
||||
),
|
||||
sa.Column("rollback_target_ref", sa.Text(), nullable=False),
|
||||
sa.Column("total_expected_items", sa.Integer(), nullable=False),
|
||||
sa.Column("batch_size", sa.Integer(), nullable=False),
|
||||
sa.Column("max_in_flight_batches", sa.Integer(), nullable=False),
|
||||
sa.Column("concurrency", sa.Integer(), nullable=False),
|
||||
sa.Column("priority", sa.String(32), nullable=False),
|
||||
sa.Column("target_storage", sa.JSON(), nullable=False),
|
||||
sa.Column("shadow_policy", sa.JSON(), nullable=False),
|
||||
sa.Column("cutover_policy", sa.JSON(), nullable=False),
|
||||
sa.Column("rollback_retention_days", sa.Integer(), nullable=False),
|
||||
sa.Column("environment_fingerprint", sa.String(64), nullable=False),
|
||||
sa.Column("idempotency_key", sa.String(128), nullable=False),
|
||||
sa.Column("created_by", sa.String(255), nullable=False),
|
||||
sa.Column("irreversible", sa.Boolean(), nullable=False, server_default=sa.false()),
|
||||
sa.Column("schema_steps", sa.JSON(), nullable=False),
|
||||
sa.Column("state", sa.String(48), nullable=False, server_default="PLANNED"),
|
||||
sa.Column("version", sa.Integer(), nullable=False, server_default="1"),
|
||||
sa.Column("generation", sa.BigInteger(), nullable=False, server_default="1"),
|
||||
sa.Column("plan_fingerprint", sa.String(64), nullable=False),
|
||||
sa.Column("approval_fingerprint", sa.String(64), nullable=False),
|
||||
sa.Column("completed_items", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("failed_items", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("retryable_items", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("permanent_failed_items", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("last_cursor", sa.String(255)),
|
||||
sa.Column("cancel_requested", sa.Boolean(), nullable=False, server_default=sa.false()),
|
||||
sa.Column("failure_code", sa.String(64)),
|
||||
sa.Column("failure_details", sa.JSON(), nullable=False),
|
||||
sa.Column("started_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("completed_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("immutable_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
_created(),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.UniqueConstraint("idempotency_key", name="uq_migration_plan_idempotency"),
|
||||
sa.UniqueConstraint("plan_fingerprint", name="uq_migration_plan_fingerprint"),
|
||||
sa.UniqueConstraint("target_shadow_target", name="uq_migration_plans_target_shadow_target"),
|
||||
sa.CheckConstraint("version >= 1", name="ck_migration_plan_version"),
|
||||
sa.CheckConstraint("generation >= 1", name="ck_migration_plan_generation"),
|
||||
sa.CheckConstraint(
|
||||
"completed_items >= 0 AND failed_items >= 0", name="ck_migration_plan_progress"
|
||||
),
|
||||
)
|
||||
for column in (
|
||||
"project_id",
|
||||
"project_binding_id",
|
||||
"capability_contract_id",
|
||||
"migration_class",
|
||||
"environment",
|
||||
"target_space_id",
|
||||
"corpus_revision",
|
||||
"validation_policy_revision_id",
|
||||
"lifecycle_approval_id",
|
||||
"promotion_plan_id",
|
||||
"state",
|
||||
):
|
||||
op.create_index(f"ix_migration_plans_{column}", "migration_plans", [column])
|
||||
op.create_index(
|
||||
"uq_migration_one_production_active",
|
||||
"migration_plans",
|
||||
["project_id", "capability_contract_id"],
|
||||
unique=True,
|
||||
postgresql_where=sa.text(
|
||||
"environment = 'PRODUCTION' AND state NOT IN "
|
||||
"('CUTOVER_COMMITTED','ROLLED_BACK','FAILED','CANCELLED')"
|
||||
),
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"migration_batch_checkpoints",
|
||||
_id(),
|
||||
sa.Column(
|
||||
"migration_plan_id",
|
||||
sa.Uuid(),
|
||||
sa.ForeignKey("migration_plans.id", ondelete="RESTRICT"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("batch_number", sa.Integer(), nullable=False),
|
||||
sa.Column("generation", sa.BigInteger(), nullable=False),
|
||||
sa.Column("cursor_start", sa.String(255), nullable=False),
|
||||
sa.Column("cursor_end", sa.String(255), nullable=False),
|
||||
sa.Column("item_count", sa.Integer(), nullable=False),
|
||||
sa.Column("completed_items", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("failed_items", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("retryable_items", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("permanent_failed_items", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("item_fingerprint", sa.String(64), nullable=False),
|
||||
sa.Column("result_fingerprint", sa.String(64)),
|
||||
sa.Column("status", sa.String(32), nullable=False),
|
||||
sa.Column("attempts", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("duration_ms", sa.Float()),
|
||||
sa.Column("error_code", sa.String(64)),
|
||||
sa.Column("bounded_errors", sa.JSON(), nullable=False),
|
||||
sa.Column("started_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("completed_at", sa.DateTime(timezone=True)),
|
||||
sa.UniqueConstraint(
|
||||
"migration_plan_id", "batch_number", "generation", name="uq_migration_batch"
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_migration_batch_checkpoints_migration_plan_id",
|
||||
"migration_batch_checkpoints",
|
||||
["migration_plan_id"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_migration_batch_checkpoints_status", "migration_batch_checkpoints", ["status"]
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"migration_validation_snapshots",
|
||||
_id(),
|
||||
sa.Column(
|
||||
"migration_plan_id",
|
||||
sa.Uuid(),
|
||||
sa.ForeignKey("migration_plans.id", ondelete="RESTRICT"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"validation_policy_revision_id",
|
||||
sa.Uuid(),
|
||||
sa.ForeignKey("migration_validation_policy_revisions.id", ondelete="RESTRICT"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("generation", sa.BigInteger(), nullable=False),
|
||||
sa.Column("expected_count", sa.Integer(), nullable=False),
|
||||
sa.Column("actual_count", sa.Integer(), nullable=False),
|
||||
sa.Column("missing_count", sa.Integer(), nullable=False),
|
||||
sa.Column("duplicate_count", sa.Integer(), nullable=False),
|
||||
sa.Column("malformed_count", sa.Integer(), nullable=False),
|
||||
sa.Column("non_finite_count", sa.Integer(), nullable=False),
|
||||
sa.Column("wrong_dimension_count", sa.Integer(), nullable=False),
|
||||
sa.Column("content_hash_mismatch_count", sa.Integer(), nullable=False),
|
||||
sa.Column("wrong_space_count", sa.Integer(), nullable=False),
|
||||
sa.Column("index_schema_matches", sa.Boolean(), nullable=False),
|
||||
sa.Column("distance_metric_matches", sa.Boolean(), nullable=False),
|
||||
sa.Column("payload_integrity", sa.Boolean(), nullable=False),
|
||||
sa.Column("target_fingerprint", sa.String(64), nullable=False),
|
||||
sa.Column("evaluation_run_ids", sa.JSON(), nullable=False),
|
||||
sa.Column("comparable", sa.Boolean(), nullable=False),
|
||||
sa.Column("critical_regressions", sa.Integer(), nullable=False),
|
||||
sa.Column("latency_regression_ratio", sa.Float()),
|
||||
sa.Column("project_fit_eligible", sa.Boolean(), nullable=False),
|
||||
sa.Column("external_validation_satisfied", sa.Boolean(), nullable=False),
|
||||
sa.Column("security_approved", sa.Boolean(), nullable=False),
|
||||
sa.Column("evidence", sa.JSON(), nullable=False),
|
||||
sa.Column("snapshot_fingerprint", sa.String(64), nullable=False),
|
||||
sa.Column("passed", sa.Boolean(), nullable=False),
|
||||
sa.Column("technical_cutover_eligible", sa.Boolean(), nullable=False),
|
||||
sa.Column("project_promotion_eligible", sa.Boolean(), nullable=False),
|
||||
sa.Column("blockers", sa.JSON(), nullable=False),
|
||||
_created(),
|
||||
sa.Column("immutable_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.UniqueConstraint("snapshot_fingerprint", name="uq_migration_validation_snapshot"),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_migration_validation_snapshots_migration_plan_id",
|
||||
"migration_validation_snapshots",
|
||||
["migration_plan_id"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_migration_validation_snapshots_validation_policy_revision_id",
|
||||
"migration_validation_snapshots",
|
||||
["validation_policy_revision_id"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_migration_validation_snapshots_passed", "migration_validation_snapshots", ["passed"]
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"migration_shadow_sessions",
|
||||
_id(),
|
||||
sa.Column(
|
||||
"migration_plan_id",
|
||||
sa.Uuid(),
|
||||
sa.ForeignKey("migration_plans.id", ondelete="RESTRICT"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("generation", sa.BigInteger(), nullable=False),
|
||||
sa.Column("status", sa.String(32), nullable=False),
|
||||
sa.Column("request_count", sa.Integer(), nullable=False),
|
||||
sa.Column("source_error_count", sa.Integer(), nullable=False),
|
||||
sa.Column("target_error_count", sa.Integer(), nullable=False),
|
||||
sa.Column("source_latency_p95_ms", sa.Float(), nullable=False),
|
||||
sa.Column("target_latency_p95_ms", sa.Float(), nullable=False),
|
||||
sa.Column("critical_regressions", sa.Integer(), nullable=False),
|
||||
sa.Column("metrics", sa.JSON(), nullable=False),
|
||||
sa.Column("evidence_refs", sa.JSON(), nullable=False),
|
||||
sa.Column("result_fingerprint", sa.String(64), nullable=False),
|
||||
sa.Column("started_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.Column("completed_at", sa.DateTime(timezone=True)),
|
||||
sa.UniqueConstraint("result_fingerprint", name="uq_migration_shadow_result_fingerprint"),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_migration_shadow_sessions_migration_plan_id",
|
||||
"migration_shadow_sessions",
|
||||
["migration_plan_id"],
|
||||
)
|
||||
op.create_index("ix_migration_shadow_sessions_status", "migration_shadow_sessions", ["status"])
|
||||
|
||||
op.create_table(
|
||||
"migration_rollback_snapshots",
|
||||
_id(),
|
||||
sa.Column(
|
||||
"migration_plan_id",
|
||||
sa.Uuid(),
|
||||
sa.ForeignKey("migration_plans.id", ondelete="RESTRICT"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("snapshot", sa.JSON(), nullable=False),
|
||||
sa.Column("snapshot_fingerprint", sa.String(64), nullable=False),
|
||||
sa.Column("retain_until", sa.DateTime(timezone=True), nullable=False),
|
||||
_created(),
|
||||
sa.Column("immutable_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.UniqueConstraint("migration_plan_id", name="uq_migration_rollback_plan"),
|
||||
sa.UniqueConstraint("snapshot_fingerprint", name="uq_migration_rollback_fingerprint"),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_migration_rollback_snapshots_migration_plan_id",
|
||||
"migration_rollback_snapshots",
|
||||
["migration_plan_id"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_migration_rollback_snapshots_retain_until",
|
||||
"migration_rollback_snapshots",
|
||||
["retain_until"],
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"migration_cutover_operations",
|
||||
_id(),
|
||||
sa.Column(
|
||||
"migration_plan_id",
|
||||
sa.Uuid(),
|
||||
sa.ForeignKey("migration_plans.id", ondelete="RESTRICT"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("stage", sa.String(32), nullable=False),
|
||||
sa.Column("idempotency_key", sa.String(128), nullable=False),
|
||||
sa.Column("generation", sa.BigInteger(), nullable=False),
|
||||
sa.Column("expected_plan_version", sa.Integer(), nullable=False),
|
||||
sa.Column("source_before", sa.Text(), nullable=False),
|
||||
sa.Column("target_after", sa.Text(), nullable=False),
|
||||
sa.Column("external_state_fingerprint", sa.String(64), nullable=False),
|
||||
sa.Column("health_evidence", sa.JSON(), nullable=False),
|
||||
sa.Column("failure_code", sa.String(64)),
|
||||
sa.Column("failure_details", sa.JSON(), nullable=False),
|
||||
sa.Column("switch_duration_ms", sa.Float()),
|
||||
sa.Column("rollback_duration_ms", sa.Float()),
|
||||
sa.Column("started_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.Column("finished_at", sa.DateTime(timezone=True)),
|
||||
sa.UniqueConstraint("idempotency_key", name="uq_migration_cutover_idempotency"),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_migration_cutover_operations_migration_plan_id",
|
||||
"migration_cutover_operations",
|
||||
["migration_plan_id"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_migration_cutover_operations_stage", "migration_cutover_operations", ["stage"]
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"migration_events",
|
||||
_id(),
|
||||
sa.Column(
|
||||
"migration_plan_id",
|
||||
sa.Uuid(),
|
||||
sa.ForeignKey("migration_plans.id", ondelete="RESTRICT"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"operation_id",
|
||||
sa.Uuid(),
|
||||
sa.ForeignKey("migration_cutover_operations.id", ondelete="RESTRICT"),
|
||||
),
|
||||
sa.Column("event_type", sa.String(64), nullable=False),
|
||||
sa.Column("before_state", sa.String(48)),
|
||||
sa.Column("after_state", sa.String(48)),
|
||||
sa.Column("actor", sa.String(255), nullable=False),
|
||||
sa.Column("policy_revision", sa.String(128), nullable=False),
|
||||
sa.Column("evidence_refs", sa.JSON(), nullable=False),
|
||||
sa.Column("reason", sa.Text(), nullable=False),
|
||||
sa.Column("source_identity", sa.JSON(), nullable=False),
|
||||
sa.Column("target_identity", sa.JSON(), nullable=False),
|
||||
sa.Column("generation", sa.BigInteger(), nullable=False),
|
||||
sa.Column("change_id", sa.String(128), nullable=False),
|
||||
sa.Column("details", sa.JSON(), nullable=False),
|
||||
sa.Column("occurred_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
)
|
||||
for column in ("migration_plan_id", "operation_id", "event_type", "change_id", "occurred_at"):
|
||||
op.create_index(f"ix_migration_events_{column}", "migration_events", [column])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("migration_events")
|
||||
op.drop_table("migration_cutover_operations")
|
||||
op.drop_table("migration_rollback_snapshots")
|
||||
op.drop_table("migration_shadow_sessions")
|
||||
op.drop_table("migration_validation_snapshots")
|
||||
op.drop_table("migration_batch_checkpoints")
|
||||
op.drop_table("migration_plans")
|
||||
op.drop_table("migration_validation_policy_revisions")
|
||||
@@ -0,0 +1,257 @@
|
||||
"""Add operational SLO, alert and bounded capacity history records.
|
||||
|
||||
Revision ID: 20260827_0020
|
||||
Revises: 20260826_0019
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "20260827_0020"
|
||||
down_revision: str | None = "20260826_0019"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def _id() -> sa.Column[object]:
|
||||
return sa.Column("id", sa.Uuid(), primary_key=True, nullable=False)
|
||||
|
||||
|
||||
def _created() -> sa.Column[object]:
|
||||
return sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now())
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"service_level_indicators",
|
||||
_id(),
|
||||
sa.Column("key", sa.String(128), nullable=False, unique=True),
|
||||
sa.Column("name", sa.String(255), nullable=False),
|
||||
sa.Column("service", sa.String(128), nullable=False),
|
||||
sa.Column("capability", sa.String(128)),
|
||||
sa.Column("measurement", sa.String(32), nullable=False),
|
||||
sa.Column("valid_population", sa.JSON(), nullable=False),
|
||||
sa.Column("success_condition", sa.JSON(), nullable=False),
|
||||
sa.Column("default_window_seconds", sa.Integer(), nullable=False),
|
||||
sa.Column("enabled", sa.Boolean(), nullable=False, server_default=sa.true()),
|
||||
_created(),
|
||||
)
|
||||
for column in ("key", "service", "capability", "enabled"):
|
||||
op.create_index(f"ix_service_level_indicators_{column}", "service_level_indicators", [column])
|
||||
|
||||
op.create_table(
|
||||
"slo_policy_revisions",
|
||||
_id(),
|
||||
sa.Column("key", sa.String(128), nullable=False),
|
||||
sa.Column("revision", sa.Integer(), nullable=False),
|
||||
sa.Column("sli_definition_id", sa.Uuid(), sa.ForeignKey("service_level_indicators.id", ondelete="RESTRICT"), nullable=False),
|
||||
sa.Column("objective", sa.Float(), nullable=False),
|
||||
sa.Column("threshold_ms", sa.Float()),
|
||||
sa.Column("rolling_window_seconds", sa.Integer(), nullable=False),
|
||||
sa.Column("minimum_sample_count", sa.Integer(), nullable=False),
|
||||
sa.Column("severity", sa.String(16), nullable=False),
|
||||
sa.Column("environment", sa.String(16), nullable=False),
|
||||
sa.Column("rationale", sa.Text(), nullable=False),
|
||||
sa.Column("effective_from", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("active", sa.Boolean(), nullable=False, server_default=sa.true()),
|
||||
sa.Column("fingerprint", sa.String(64), nullable=False, unique=True),
|
||||
sa.Column("created_by", sa.String(255), nullable=False),
|
||||
_created(),
|
||||
sa.Column("immutable_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.UniqueConstraint("key", "revision", name="uq_slo_policy_key_revision"),
|
||||
)
|
||||
for column in ("key", "sli_definition_id", "environment", "active"):
|
||||
op.create_index(f"ix_slo_policy_revisions_{column}", "slo_policy_revisions", [column])
|
||||
|
||||
op.create_table(
|
||||
"slo_evaluations",
|
||||
_id(),
|
||||
sa.Column("policy_id", sa.Uuid(), sa.ForeignKey("slo_policy_revisions.id", ondelete="RESTRICT"), nullable=False),
|
||||
sa.Column("window_start", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("window_end", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("observed_value", sa.Float()),
|
||||
sa.Column("sample_count", sa.Integer(), nullable=False),
|
||||
sa.Column("good_count", sa.Integer(), nullable=False),
|
||||
sa.Column("bad_count", sa.Integer(), nullable=False),
|
||||
sa.Column("state", sa.String(32), nullable=False),
|
||||
sa.Column("freshness_seconds", sa.Float()),
|
||||
sa.Column("allowed_bad", sa.Float()),
|
||||
sa.Column("consumed_bad", sa.Integer()),
|
||||
sa.Column("remaining_bad", sa.Float()),
|
||||
sa.Column("short_burn_rate", sa.Float()),
|
||||
sa.Column("long_burn_rate", sa.Float()),
|
||||
sa.Column("evidence", sa.JSON(), nullable=False),
|
||||
sa.Column("observed_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.UniqueConstraint("policy_id", "window_end", name="uq_slo_evaluation_window"),
|
||||
)
|
||||
for column in ("policy_id", "window_start", "window_end", "state"):
|
||||
op.create_index(f"ix_slo_evaluations_{column}", "slo_evaluations", [column])
|
||||
|
||||
op.create_table(
|
||||
"alert_rule_revisions",
|
||||
_id(),
|
||||
sa.Column("key", sa.String(128), nullable=False),
|
||||
sa.Column("revision", sa.Integer(), nullable=False),
|
||||
sa.Column("alert_type", sa.String(64), nullable=False),
|
||||
sa.Column("signal", sa.String(128), nullable=False),
|
||||
sa.Column("slo_policy_id", sa.Uuid(), sa.ForeignKey("slo_policy_revisions.id", ondelete="RESTRICT")),
|
||||
sa.Column("condition", sa.JSON(), nullable=False),
|
||||
sa.Column("pending_seconds", sa.Integer(), nullable=False),
|
||||
sa.Column("severity", sa.String(16), nullable=False),
|
||||
sa.Column("labels", sa.JSON(), nullable=False),
|
||||
sa.Column("cooldown_seconds", sa.Integer(), nullable=False),
|
||||
sa.Column("recovery_condition", sa.JSON(), nullable=False),
|
||||
sa.Column("active", sa.Boolean(), nullable=False, server_default=sa.true()),
|
||||
sa.Column("fingerprint", sa.String(64), nullable=False, unique=True),
|
||||
sa.Column("created_by", sa.String(255), nullable=False),
|
||||
_created(),
|
||||
sa.Column("immutable_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.UniqueConstraint("key", "revision", name="uq_alert_rule_key_revision"),
|
||||
)
|
||||
for column in ("key", "alert_type", "slo_policy_id", "severity", "active"):
|
||||
op.create_index(f"ix_alert_rule_revisions_{column}", "alert_rule_revisions", [column])
|
||||
|
||||
op.create_table(
|
||||
"maintenance_windows",
|
||||
_id(),
|
||||
sa.Column("name", sa.String(255), nullable=False),
|
||||
sa.Column("starts_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("ends_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("matcher", sa.JSON(), nullable=False),
|
||||
sa.Column("reason", sa.Text(), nullable=False),
|
||||
sa.Column("active", sa.Boolean(), nullable=False, server_default=sa.true()),
|
||||
sa.Column("created_by", sa.String(255), nullable=False),
|
||||
_created(),
|
||||
)
|
||||
for column in ("starts_at", "ends_at", "active"):
|
||||
op.create_index(f"ix_maintenance_windows_{column}", "maintenance_windows", [column])
|
||||
|
||||
op.create_table(
|
||||
"operational_incidents",
|
||||
_id(),
|
||||
sa.Column("fingerprint", sa.String(64), nullable=False, unique=True),
|
||||
sa.Column("title", sa.String(255), nullable=False),
|
||||
sa.Column("state", sa.String(32), nullable=False),
|
||||
sa.Column("severity", sa.String(16), nullable=False),
|
||||
sa.Column("root_subject_type", sa.String(64), nullable=False),
|
||||
sa.Column("root_subject_ref", sa.String(255), nullable=False),
|
||||
sa.Column("correlation", sa.String(32), nullable=False),
|
||||
sa.Column("first_seen_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("resolved_at", sa.DateTime(timezone=True)),
|
||||
)
|
||||
op.create_index("ix_operational_incidents_state", "operational_incidents", ["state"])
|
||||
|
||||
op.create_table(
|
||||
"operational_alerts",
|
||||
_id(),
|
||||
sa.Column("rule_id", sa.Uuid(), sa.ForeignKey("alert_rule_revisions.id", ondelete="RESTRICT"), nullable=False),
|
||||
sa.Column("fingerprint", sa.String(64), nullable=False, unique=True),
|
||||
sa.Column("alert_type", sa.String(64), nullable=False),
|
||||
sa.Column("severity", sa.String(16), nullable=False),
|
||||
sa.Column("state", sa.String(32), nullable=False),
|
||||
sa.Column("source", sa.String(128), nullable=False),
|
||||
sa.Column("subject_type", sa.String(64), nullable=False),
|
||||
sa.Column("subject_ref", sa.String(255), nullable=False),
|
||||
sa.Column("summary", sa.Text(), nullable=False),
|
||||
sa.Column("details", sa.JSON(), nullable=False),
|
||||
sa.Column("first_seen_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("firing_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("acknowledged_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("acknowledged_by", sa.String(255)),
|
||||
sa.Column("acknowledgement_reason", sa.Text()),
|
||||
sa.Column("resolved_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("suppressed_until", sa.DateTime(timezone=True)),
|
||||
sa.Column("occurrence_count", sa.Integer(), nullable=False, server_default="1"),
|
||||
sa.Column("cooldown_until", sa.DateTime(timezone=True)),
|
||||
sa.Column("incident_id", sa.Uuid(), sa.ForeignKey("operational_incidents.id", ondelete="SET NULL")),
|
||||
)
|
||||
for column in ("rule_id", "alert_type", "severity", "state", "subject_ref", "first_seen_at", "last_seen_at", "incident_id"):
|
||||
op.create_index(f"ix_operational_alerts_{column}", "operational_alerts", [column])
|
||||
|
||||
op.create_table(
|
||||
"alert_history_events",
|
||||
_id(),
|
||||
sa.Column("alert_id", sa.Uuid(), sa.ForeignKey("operational_alerts.id", ondelete="RESTRICT"), nullable=False),
|
||||
sa.Column("from_state", sa.String(32)),
|
||||
sa.Column("to_state", sa.String(32), nullable=False),
|
||||
sa.Column("actor", sa.String(255), nullable=False),
|
||||
sa.Column("reason", sa.Text(), nullable=False),
|
||||
sa.Column("evidence", sa.JSON(), nullable=False),
|
||||
sa.Column("occurred_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
)
|
||||
for column in ("alert_id", "to_state", "occurred_at"):
|
||||
op.create_index(f"ix_alert_history_events_{column}", "alert_history_events", [column])
|
||||
|
||||
op.create_table(
|
||||
"incident_timeline_events",
|
||||
_id(),
|
||||
sa.Column("incident_id", sa.Uuid(), sa.ForeignKey("operational_incidents.id", ondelete="RESTRICT"), nullable=False),
|
||||
sa.Column("alert_id", sa.Uuid(), sa.ForeignKey("operational_alerts.id", ondelete="SET NULL")),
|
||||
sa.Column("event_type", sa.String(64), nullable=False),
|
||||
sa.Column("relation", sa.String(32), nullable=False),
|
||||
sa.Column("summary", sa.Text(), nullable=False),
|
||||
sa.Column("occurred_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
)
|
||||
for column in ("incident_id", "alert_id", "occurred_at"):
|
||||
op.create_index(f"ix_incident_timeline_events_{column}", "incident_timeline_events", [column])
|
||||
|
||||
op.create_table(
|
||||
"capacity_snapshots",
|
||||
_id(),
|
||||
sa.Column("observed_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("received_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("node_id", sa.Uuid(), sa.ForeignKey("compute_nodes.id", ondelete="RESTRICT"), nullable=False),
|
||||
sa.Column("node_name", sa.String(255), nullable=False),
|
||||
sa.Column("accelerator_id", sa.Uuid(), sa.ForeignKey("accelerators.id", ondelete="RESTRICT")),
|
||||
sa.Column("gpu_total_bytes", sa.BigInteger()),
|
||||
sa.Column("gpu_observed_bytes", sa.BigInteger()),
|
||||
sa.Column("gpu_external_bytes", sa.BigInteger()),
|
||||
sa.Column("gpu_managed_resident_bytes", sa.BigInteger()),
|
||||
sa.Column("gpu_leased_bytes", sa.BigInteger()),
|
||||
sa.Column("gpu_reserve_bytes", sa.BigInteger()),
|
||||
sa.Column("gpu_schedulable_bytes", sa.BigInteger()),
|
||||
sa.Column("pressure_state", sa.String(16), nullable=False),
|
||||
sa.Column("system_ram_total_bytes", sa.BigInteger()),
|
||||
sa.Column("system_ram_available_bytes", sa.BigInteger()),
|
||||
sa.Column("storage_total_bytes", sa.BigInteger()),
|
||||
sa.Column("storage_free_bytes", sa.BigInteger()),
|
||||
sa.Column("availability", sa.String(32), nullable=False),
|
||||
sa.Column("freshness_seconds", sa.Float(), nullable=False),
|
||||
sa.UniqueConstraint("node_id", "accelerator_id", "observed_at", name="uq_capacity_observation"),
|
||||
)
|
||||
for column in ("observed_at", "received_at", "node_id", "accelerator_id"):
|
||||
op.create_index(f"ix_capacity_snapshots_{column}", "capacity_snapshots", [column])
|
||||
|
||||
op.create_table(
|
||||
"capacity_aggregates",
|
||||
_id(),
|
||||
sa.Column("node_id", sa.Uuid(), sa.ForeignKey("compute_nodes.id", ondelete="RESTRICT"), nullable=False),
|
||||
sa.Column("accelerator_id", sa.Uuid(), sa.ForeignKey("accelerators.id", ondelete="RESTRICT")),
|
||||
sa.Column("bucket_start", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("bucket_seconds", sa.Integer(), nullable=False),
|
||||
sa.Column("sample_count", sa.Integer(), nullable=False),
|
||||
sa.Column("metrics", sa.JSON(), nullable=False),
|
||||
_created(),
|
||||
sa.UniqueConstraint("node_id", "accelerator_id", "bucket_start", name="uq_capacity_bucket"),
|
||||
)
|
||||
for column in ("node_id", "accelerator_id", "bucket_start"):
|
||||
op.create_index(f"ix_capacity_aggregates_{column}", "capacity_aggregates", [column])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("capacity_aggregates")
|
||||
op.drop_table("capacity_snapshots")
|
||||
op.drop_table("incident_timeline_events")
|
||||
op.drop_table("alert_history_events")
|
||||
op.drop_table("operational_alerts")
|
||||
op.drop_table("operational_incidents")
|
||||
op.drop_table("maintenance_windows")
|
||||
op.drop_table("alert_rule_revisions")
|
||||
op.drop_table("slo_evaluations")
|
||||
op.drop_table("slo_policy_revisions")
|
||||
op.drop_table("service_level_indicators")
|
||||
@@ -0,0 +1,333 @@
|
||||
"""Add versioned recovery policies, immutable backup sets, restore journals and artifact recovery.
|
||||
|
||||
Revision ID: 20260827_0021
|
||||
Revises: 20260827_0020
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "20260827_0021"
|
||||
down_revision: str | None = "20260827_0020"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def _id() -> sa.Column[object]:
|
||||
return sa.Column("id", sa.Uuid(), primary_key=True, nullable=False)
|
||||
|
||||
|
||||
def _created() -> sa.Column[object]:
|
||||
return sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now())
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"recovery_policy_revisions",
|
||||
_id(),
|
||||
sa.Column("key", sa.String(128), nullable=False),
|
||||
sa.Column("revision", sa.Integer(), nullable=False),
|
||||
sa.Column("name", sa.String(255), nullable=False),
|
||||
sa.Column("asset_class", sa.String(32), nullable=False),
|
||||
sa.Column("backup_method", sa.String(48), nullable=False),
|
||||
sa.Column("retention_days", sa.Integer(), nullable=False),
|
||||
sa.Column("minimum_verified_backups", sa.Integer(), nullable=False),
|
||||
sa.Column("rpo_seconds", sa.Integer()),
|
||||
sa.Column("rto_target_seconds", sa.Integer()),
|
||||
sa.Column("restore_verification", sa.String(32), nullable=False),
|
||||
sa.Column("encryption_required", sa.Boolean(), nullable=False, server_default=sa.false()),
|
||||
sa.Column("external_dependency", sa.Boolean(), nullable=False, server_default=sa.false()),
|
||||
sa.Column("rehydration_allowed", sa.Boolean(), nullable=False, server_default=sa.false()),
|
||||
sa.Column("secret_class", sa.String(32)),
|
||||
sa.Column("rationale", sa.Text(), nullable=False),
|
||||
sa.Column("active", sa.Boolean(), nullable=False, server_default=sa.true()),
|
||||
sa.Column("fingerprint", sa.String(64), nullable=False, unique=True),
|
||||
sa.Column("created_by", sa.String(255), nullable=False),
|
||||
_created(),
|
||||
sa.Column("immutable_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.UniqueConstraint("key", "revision", name="uq_recovery_policy_key_revision"),
|
||||
sa.CheckConstraint("retention_days >= 1", name="ck_recovery_policy_retention"),
|
||||
sa.CheckConstraint(
|
||||
"minimum_verified_backups >= 1", name="ck_recovery_policy_minimum_verified"
|
||||
),
|
||||
)
|
||||
for column in ("key", "asset_class", "active"):
|
||||
op.create_index(
|
||||
f"ix_recovery_policy_revisions_{column}", "recovery_policy_revisions", [column]
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"recovery_asset_records",
|
||||
_id(),
|
||||
sa.Column("key", sa.String(128), nullable=False, unique=True),
|
||||
sa.Column("name", sa.String(255), nullable=False),
|
||||
sa.Column("asset_class", sa.String(32), nullable=False),
|
||||
sa.Column("owner", sa.String(128), nullable=False),
|
||||
sa.Column("location", sa.Text(), nullable=False),
|
||||
sa.Column("backup_method", sa.String(48), nullable=False),
|
||||
sa.Column("restore_method", sa.Text(), nullable=False),
|
||||
sa.Column("rebuild_method", sa.Text()),
|
||||
sa.Column("rpo_seconds", sa.Integer()),
|
||||
sa.Column("readiness", sa.String(32), nullable=False),
|
||||
sa.Column("dependencies", sa.JSON(), nullable=False),
|
||||
sa.Column("notes", sa.Text(), nullable=False, server_default=""),
|
||||
sa.Column(
|
||||
"policy_revision_id",
|
||||
sa.Uuid(),
|
||||
sa.ForeignKey("recovery_policy_revisions.id", ondelete="RESTRICT"),
|
||||
nullable=False,
|
||||
),
|
||||
_created(),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
)
|
||||
for column in ("key", "asset_class", "readiness", "policy_revision_id"):
|
||||
op.create_index(f"ix_recovery_asset_records_{column}", "recovery_asset_records", [column])
|
||||
|
||||
op.create_table(
|
||||
"backup_sets",
|
||||
_id(),
|
||||
sa.Column("backup_id", sa.String(64), nullable=False),
|
||||
sa.Column("state", sa.String(32), nullable=False, server_default="PLANNED"),
|
||||
sa.Column(
|
||||
"policy_revision_id",
|
||||
sa.Uuid(),
|
||||
sa.ForeignKey("recovery_policy_revisions.id", ondelete="RESTRICT"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("modelforge_version", sa.String(64), nullable=False),
|
||||
sa.Column("modelforge_commit", sa.String(64)),
|
||||
sa.Column("source_repository", sa.Text()),
|
||||
sa.Column("source_reference", sa.String(255)),
|
||||
sa.Column("schema_revision", sa.String(64)),
|
||||
sa.Column("environment_fingerprint", sa.JSON(), nullable=False),
|
||||
sa.Column("database_identity", sa.JSON(), nullable=False),
|
||||
sa.Column("destination_root", sa.Text(), nullable=False),
|
||||
sa.Column("payload_relative_path", sa.Text()),
|
||||
sa.Column("payload_sha256", sa.String(64)),
|
||||
sa.Column("manifest_relative_path", sa.Text()),
|
||||
sa.Column("manifest_sha256", sa.String(64)),
|
||||
sa.Column("included_asset_classes", sa.JSON(), nullable=False),
|
||||
sa.Column("excluded_asset_classes", sa.JSON(), nullable=False),
|
||||
sa.Column("payload_bytes", sa.BigInteger(), nullable=False, server_default="0"),
|
||||
sa.Column("encrypted", sa.Boolean(), nullable=False, server_default=sa.false()),
|
||||
sa.Column("encryption_algorithm", sa.String(64)),
|
||||
sa.Column("encryption_key_id", sa.String(128)),
|
||||
sa.Column("verification_details", sa.JSON(), nullable=False),
|
||||
sa.Column("verified_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("failure_code", sa.String(64)),
|
||||
sa.Column("failure_reason", sa.Text()),
|
||||
sa.Column("milestone", sa.String(64)),
|
||||
sa.Column("legal_hold", sa.Boolean(), nullable=False, server_default=sa.false()),
|
||||
sa.Column("reason", sa.Text(), nullable=False),
|
||||
sa.Column("created_by", sa.String(255), nullable=False),
|
||||
sa.Column("started_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("completed_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("expires_at", sa.DateTime(timezone=True)),
|
||||
_created(),
|
||||
sa.Column("immutable_at", sa.DateTime(timezone=True)),
|
||||
sa.UniqueConstraint("backup_id", name="uq_backup_set_backup_id"),
|
||||
sa.CheckConstraint("payload_bytes >= 0", name="ck_backup_set_payload_bytes"),
|
||||
)
|
||||
for column in (
|
||||
"backup_id",
|
||||
"state",
|
||||
"policy_revision_id",
|
||||
"schema_revision",
|
||||
"manifest_sha256",
|
||||
"verified_at",
|
||||
"milestone",
|
||||
"legal_hold",
|
||||
"expires_at",
|
||||
):
|
||||
op.create_index(f"ix_backup_sets_{column}", "backup_sets", [column])
|
||||
|
||||
op.create_table(
|
||||
"backup_manifest_entries",
|
||||
_id(),
|
||||
sa.Column(
|
||||
"backup_set_id",
|
||||
sa.Uuid(),
|
||||
sa.ForeignKey("backup_sets.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("logical_asset_type", sa.String(64), nullable=False),
|
||||
sa.Column("object_name", sa.String(255), nullable=False),
|
||||
sa.Column("relative_path", sa.Text(), nullable=False),
|
||||
sa.Column("size_bytes", sa.BigInteger(), nullable=False),
|
||||
sa.Column("sha256", sa.String(64), nullable=False),
|
||||
sa.Column("source_generation", sa.String(128), nullable=False),
|
||||
sa.Column("schema_version", sa.String(64)),
|
||||
sa.Column("dependency_refs", sa.JSON(), nullable=False),
|
||||
_created(),
|
||||
sa.UniqueConstraint("backup_set_id", "object_name", name="uq_backup_manifest_object"),
|
||||
sa.CheckConstraint("length(sha256) = 64", name="ck_backup_manifest_sha256_length"),
|
||||
sa.CheckConstraint("size_bytes >= 0", name="ck_backup_manifest_size"),
|
||||
)
|
||||
for column in ("backup_set_id", "logical_asset_type"):
|
||||
op.create_index(
|
||||
f"ix_backup_manifest_entries_{column}", "backup_manifest_entries", [column]
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"restore_plans",
|
||||
_id(),
|
||||
sa.Column(
|
||||
"backup_set_id",
|
||||
sa.Uuid(),
|
||||
sa.ForeignKey("backup_sets.id", ondelete="RESTRICT"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("mode", sa.String(32), nullable=False),
|
||||
sa.Column("state", sa.String(32), nullable=False, server_default="DRAFT"),
|
||||
sa.Column("target_environment", sa.String(32), nullable=False),
|
||||
sa.Column("target_label", sa.String(128), nullable=False),
|
||||
sa.Column("database_destination", sa.Text(), nullable=False),
|
||||
sa.Column("artifact_strategy", sa.String(32), nullable=False),
|
||||
sa.Column("secret_strategy", sa.String(32), nullable=False),
|
||||
sa.Column("node_strategy", sa.String(32), nullable=False),
|
||||
sa.Column("expected_modelforge_version", sa.String(64)),
|
||||
sa.Column("preflight", sa.JSON(), nullable=False),
|
||||
sa.Column("validation_requirements", sa.JSON(), nullable=False),
|
||||
sa.Column("fingerprint", sa.String(64), nullable=False),
|
||||
sa.Column("reason", sa.Text(), nullable=False),
|
||||
sa.Column("created_by", sa.String(255), nullable=False),
|
||||
_created(),
|
||||
sa.UniqueConstraint("fingerprint", name="uq_restore_plan_fingerprint"),
|
||||
)
|
||||
for column in ("backup_set_id", "mode", "state", "target_environment"):
|
||||
op.create_index(f"ix_restore_plans_{column}", "restore_plans", [column])
|
||||
|
||||
op.create_table(
|
||||
"restore_operations",
|
||||
_id(),
|
||||
sa.Column(
|
||||
"plan_id",
|
||||
sa.Uuid(),
|
||||
sa.ForeignKey("restore_plans.id", ondelete="RESTRICT"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"backup_set_id",
|
||||
sa.Uuid(),
|
||||
sa.ForeignKey("backup_sets.id", ondelete="RESTRICT"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("state", sa.String(48), nullable=False, server_default="PLANNED"),
|
||||
sa.Column("attempt", sa.Integer(), nullable=False, server_default="1"),
|
||||
sa.Column("idempotency_key", sa.String(64), nullable=False),
|
||||
sa.Column("preflight_result", sa.JSON(), nullable=False),
|
||||
sa.Column("phase_durations", sa.JSON(), nullable=False),
|
||||
sa.Column("source_fingerprint", sa.JSON(), nullable=False),
|
||||
sa.Column("restored_fingerprint", sa.JSON(), nullable=False),
|
||||
sa.Column("fingerprint_diff", sa.JSON(), nullable=False),
|
||||
sa.Column("validation_result", sa.JSON(), nullable=False),
|
||||
sa.Column("rpo_seconds", sa.Float()),
|
||||
sa.Column("rto_seconds", sa.Float()),
|
||||
sa.Column("failure_code", sa.String(64)),
|
||||
sa.Column("failure_reason", sa.Text()),
|
||||
sa.Column("started_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.Column("ready_at", sa.DateTime(timezone=True)),
|
||||
sa.UniqueConstraint("idempotency_key", name="uq_restore_operation_idempotency"),
|
||||
)
|
||||
for column in ("plan_id", "backup_set_id", "state"):
|
||||
op.create_index(f"ix_restore_operations_{column}", "restore_operations", [column])
|
||||
|
||||
op.create_table(
|
||||
"restore_operation_events",
|
||||
_id(),
|
||||
sa.Column(
|
||||
"restore_operation_id",
|
||||
sa.Uuid(),
|
||||
sa.ForeignKey("restore_operations.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("from_state", sa.String(48)),
|
||||
sa.Column("to_state", sa.String(48), nullable=False),
|
||||
sa.Column("phase", sa.String(48), nullable=False),
|
||||
sa.Column("actor", sa.String(255), nullable=False),
|
||||
sa.Column("reason", sa.Text(), nullable=False),
|
||||
sa.Column("evidence", sa.JSON(), nullable=False),
|
||||
sa.Column("occurred_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
)
|
||||
for column in ("restore_operation_id", "to_state", "occurred_at"):
|
||||
op.create_index(
|
||||
f"ix_restore_operation_events_{column}", "restore_operation_events", [column]
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"artifact_recovery_operations",
|
||||
_id(),
|
||||
sa.Column(
|
||||
"restore_operation_id",
|
||||
sa.Uuid(),
|
||||
sa.ForeignKey("restore_operations.id", ondelete="SET NULL"),
|
||||
),
|
||||
sa.Column(
|
||||
"artifact_set_id",
|
||||
sa.Uuid(),
|
||||
sa.ForeignKey("artifact_sets.id", ondelete="RESTRICT"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"model_revision_id",
|
||||
sa.Uuid(),
|
||||
sa.ForeignKey("model_revisions.id", ondelete="RESTRICT"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("recovery_class", sa.String(32), nullable=False),
|
||||
sa.Column("state", sa.String(32), nullable=False, server_default="PLANNED"),
|
||||
sa.Column("upstream_repository", sa.Text()),
|
||||
sa.Column("upstream_commit_sha", sa.String(64)),
|
||||
sa.Column(
|
||||
"target_storage_root_id",
|
||||
sa.Uuid(),
|
||||
sa.ForeignKey("storage_roots.id", ondelete="RESTRICT"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("expected_files", sa.JSON(), nullable=False),
|
||||
sa.Column("verified_files", sa.JSON(), nullable=False),
|
||||
sa.Column("bytes_total", sa.BigInteger(), nullable=False, server_default="0"),
|
||||
sa.Column("bytes_recovered", sa.BigInteger(), nullable=False, server_default="0"),
|
||||
sa.Column(
|
||||
"download_plan_id", sa.Uuid(), sa.ForeignKey("download_plans.id", ondelete="SET NULL")
|
||||
),
|
||||
sa.Column(
|
||||
"artifact_job_id", sa.Uuid(), sa.ForeignKey("artifact_jobs.id", ondelete="SET NULL")
|
||||
),
|
||||
sa.Column("lineage", sa.JSON(), nullable=False),
|
||||
sa.Column("duration_seconds", sa.Float()),
|
||||
sa.Column("failure_code", sa.String(64)),
|
||||
sa.Column("failure_reason", sa.Text()),
|
||||
sa.Column("reason", sa.Text(), nullable=False),
|
||||
sa.Column("created_by", sa.String(255), nullable=False),
|
||||
sa.Column("started_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.Column("completed_at", sa.DateTime(timezone=True)),
|
||||
)
|
||||
for column in (
|
||||
"restore_operation_id",
|
||||
"artifact_set_id",
|
||||
"model_revision_id",
|
||||
"recovery_class",
|
||||
"state",
|
||||
"target_storage_root_id",
|
||||
"download_plan_id",
|
||||
"artifact_job_id",
|
||||
):
|
||||
op.create_index(
|
||||
f"ix_artifact_recovery_operations_{column}", "artifact_recovery_operations", [column]
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("artifact_recovery_operations")
|
||||
op.drop_table("restore_operation_events")
|
||||
op.drop_table("restore_operations")
|
||||
op.drop_table("restore_plans")
|
||||
op.drop_table("backup_manifest_entries")
|
||||
op.drop_table("backup_sets")
|
||||
op.drop_table("recovery_asset_records")
|
||||
op.drop_table("recovery_policy_revisions")
|
||||
@@ -0,0 +1,71 @@
|
||||
"""Add audited compute-node decommission tombstones and operation records.
|
||||
|
||||
Revision ID: 20260828_0022
|
||||
Revises: 20260827_0021
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "20260828_0022"
|
||||
down_revision: str | None = "20260827_0021"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"compute_nodes",
|
||||
sa.Column("generation", sa.BigInteger(), nullable=False, server_default="1"),
|
||||
)
|
||||
op.add_column("compute_nodes", sa.Column("decommissioned_at", sa.DateTime(timezone=True)))
|
||||
op.add_column("compute_nodes", sa.Column("decommission_reason", sa.Text()))
|
||||
op.add_column("compute_nodes", sa.Column("decommissioned_by", sa.String(255)))
|
||||
op.create_index("ix_compute_nodes_decommissioned_at", "compute_nodes", ["decommissioned_at"])
|
||||
op.create_table(
|
||||
"node_decommission_operations",
|
||||
sa.Column("id", sa.Uuid(), primary_key=True, nullable=False),
|
||||
sa.Column(
|
||||
"compute_node_id",
|
||||
sa.Uuid(),
|
||||
sa.ForeignKey("compute_nodes.id", ondelete="RESTRICT"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("persisted_identity", sa.String(128), nullable=False),
|
||||
sa.Column("idempotency_key", sa.String(128), nullable=False),
|
||||
sa.Column("expected_generation", sa.BigInteger(), nullable=False),
|
||||
sa.Column("preview_digest", sa.String(64), nullable=False),
|
||||
sa.Column("reason", sa.Text(), nullable=False),
|
||||
sa.Column("operator", sa.String(255), nullable=False),
|
||||
sa.Column("previous_state", sa.JSON(), nullable=False),
|
||||
sa.Column("cleanup_summary", sa.JSON(), nullable=False),
|
||||
sa.Column("status", sa.String(32), nullable=False, server_default="completed"),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=sa.func.now(),
|
||||
),
|
||||
sa.Column("decommissioned_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.UniqueConstraint("compute_node_id", name="uq_node_decommission_node"),
|
||||
sa.UniqueConstraint("idempotency_key", name="uq_node_decommission_idempotency"),
|
||||
sa.CheckConstraint("expected_generation >= 1", name="ck_node_decommission_generation"),
|
||||
)
|
||||
for column in ("compute_node_id", "persisted_identity", "status", "decommissioned_at"):
|
||||
op.create_index(
|
||||
f"ix_node_decommission_operations_{column}",
|
||||
"node_decommission_operations",
|
||||
[column],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("node_decommission_operations")
|
||||
op.drop_index("ix_compute_nodes_decommissioned_at", table_name="compute_nodes")
|
||||
op.drop_column("compute_nodes", "decommissioned_by")
|
||||
op.drop_column("compute_nodes", "decommission_reason")
|
||||
op.drop_column("compute_nodes", "decommissioned_at")
|
||||
op.drop_column("compute_nodes", "generation")
|
||||
@@ -0,0 +1,56 @@
|
||||
"""Enforce the two fixed Node Agent credential scopes.
|
||||
|
||||
Revision ID: 20260830_0023
|
||||
Revises: 20260828_0022
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "20260830_0023"
|
||||
down_revision: str | None = "20260828_0022"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
_SCOPE_CONSTRAINTS = (
|
||||
("node_enrollments", "node.enroll", "ck_node_enrollment_scope"),
|
||||
("node_credentials", "node.publish", "ck_node_credential_scope"),
|
||||
)
|
||||
|
||||
|
||||
def _validate_existing_scopes() -> None:
|
||||
connection = op.get_bind()
|
||||
for table_name, expected_scope, _constraint_name in _SCOPE_CONSTRAINTS:
|
||||
table = sa.table(table_name, sa.column("scope", sa.String(64)))
|
||||
invalid_rows = connection.scalar(
|
||||
sa.select(sa.func.count())
|
||||
.select_from(table)
|
||||
.where(sa.or_(table.c.scope.is_(None), table.c.scope != expected_scope))
|
||||
)
|
||||
if invalid_rows:
|
||||
raise RuntimeError(
|
||||
f"refusing node-scope migration: {table_name} contains "
|
||||
f"{invalid_rows} row(s) outside {expected_scope!r}"
|
||||
)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# Validate every table before changing either one. A malformed production row therefore aborts
|
||||
# the migration without leaving a partially hardened schema.
|
||||
_validate_existing_scopes()
|
||||
for table_name, expected_scope, constraint_name in _SCOPE_CONSTRAINTS:
|
||||
with op.batch_alter_table(table_name) as batch_op:
|
||||
batch_op.create_check_constraint(
|
||||
constraint_name,
|
||||
f"scope = '{expected_scope}'",
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
for table_name, _expected_scope, constraint_name in reversed(_SCOPE_CONSTRAINTS):
|
||||
with op.batch_alter_table(table_name) as batch_op:
|
||||
batch_op.drop_constraint(constraint_name, type_="check")
|
||||
@@ -0,0 +1,717 @@
|
||||
"""Version and checkpoint the tamper-evident audit chain.
|
||||
|
||||
Revision ID: 20260830_0024
|
||||
Revises: 20260830_0023
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.engine import Connection
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision = "20260830_0024"
|
||||
down_revision = "20260830_0023"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_LEGACY_PREFIX_DOMAIN = b"modelforge:audit:legacy-prefix:v1\n"
|
||||
_SHA256 = re.compile(r"^[0-9a-f]{64}$")
|
||||
_AUDIT_CHAIN_LOCK_KEY = int.from_bytes(b"MF_AUDIT", byteorder="big", signed=False)
|
||||
_AUDIT_OWNER_ROLE = "modelforge"
|
||||
_AUDIT_RUNTIME_ROLE = "modelforge_runtime"
|
||||
|
||||
_POSTGRES_AUDIT_BOUNDARY_SQL = r"""
|
||||
create schema if not exists modelforge_audit authorization modelforge;
|
||||
alter schema modelforge_audit owner to modelforge;
|
||||
revoke all on schema modelforge_audit from public;
|
||||
revoke create on schema public from modelforge_runtime;
|
||||
grant usage on schema public, modelforge_audit to modelforge_runtime;
|
||||
|
||||
create or replace function modelforge_audit.enforce_owner_mutation()
|
||||
returns trigger
|
||||
language plpgsql
|
||||
security invoker
|
||||
set search_path = pg_catalog
|
||||
as $guard$
|
||||
begin
|
||||
if current_user <> 'modelforge' then
|
||||
raise exception 'audit tables are writable only through the canonical append function'
|
||||
using errcode = '42501';
|
||||
end if;
|
||||
if tg_op = 'DELETE' then
|
||||
return old;
|
||||
elsif tg_op = 'TRUNCATE' then
|
||||
return null;
|
||||
end if;
|
||||
return new;
|
||||
end
|
||||
$guard$;
|
||||
alter function modelforge_audit.enforce_owner_mutation() owner to modelforge;
|
||||
revoke all on function modelforge_audit.enforce_owner_mutation() from public;
|
||||
revoke all on function modelforge_audit.enforce_owner_mutation() from modelforge_runtime;
|
||||
|
||||
drop trigger if exists trg_modelforge_audit_events_owner on public.audit_events;
|
||||
create trigger trg_modelforge_audit_events_owner
|
||||
before insert or update or delete on public.audit_events
|
||||
for each row execute function modelforge_audit.enforce_owner_mutation();
|
||||
drop trigger if exists trg_modelforge_audit_events_truncate_owner on public.audit_events;
|
||||
create trigger trg_modelforge_audit_events_truncate_owner
|
||||
before truncate on public.audit_events
|
||||
for each statement execute function modelforge_audit.enforce_owner_mutation();
|
||||
drop trigger if exists trg_modelforge_audit_head_owner on public.audit_chain_heads;
|
||||
create trigger trg_modelforge_audit_head_owner
|
||||
before insert or update or delete on public.audit_chain_heads
|
||||
for each row execute function modelforge_audit.enforce_owner_mutation();
|
||||
drop trigger if exists trg_modelforge_audit_head_truncate_owner on public.audit_chain_heads;
|
||||
create trigger trg_modelforge_audit_head_truncate_owner
|
||||
before truncate on public.audit_chain_heads
|
||||
for each statement execute function modelforge_audit.enforce_owner_mutation();
|
||||
|
||||
create or replace function modelforge_audit.append_event_v2(
|
||||
p_event_id uuid,
|
||||
p_occurred_at timestamptz,
|
||||
p_correlation_id text,
|
||||
p_actor_type text,
|
||||
p_actor_id text,
|
||||
p_action text,
|
||||
p_resource_type text,
|
||||
p_resource_id text,
|
||||
p_outcome text,
|
||||
p_details jsonb,
|
||||
p_expected_event_count bigint,
|
||||
p_expected_last_sequence bigint,
|
||||
p_expected_last_event_hash text,
|
||||
p_expected_hash_format text,
|
||||
p_expected_v2_start_sequence bigint,
|
||||
p_expected_legacy_prefix_count bigint,
|
||||
p_expected_legacy_prefix_seal text
|
||||
)
|
||||
returns table(event_id uuid, sequence bigint, event_hash text, occurred_at timestamptz)
|
||||
language plpgsql
|
||||
security definer
|
||||
set search_path = pg_catalog
|
||||
as $append$
|
||||
declare
|
||||
v_head public.audit_chain_heads%rowtype;
|
||||
v_tail public.audit_events%rowtype;
|
||||
v_max_sequence bigint;
|
||||
v_predecessor_hash text;
|
||||
v_sequence bigint;
|
||||
v_payload text;
|
||||
v_hash text;
|
||||
v_updated bigint;
|
||||
begin
|
||||
perform pg_catalog.pg_advisory_xact_lock(5568242723498248532);
|
||||
|
||||
if p_event_id is null or p_occurred_at is null or p_correlation_id is null
|
||||
or p_actor_type is null or p_actor_id is null or p_action is null
|
||||
or p_resource_type is null or p_outcome is null or p_details is null then
|
||||
raise exception 'canonical audit append arguments must not be null'
|
||||
using errcode = '23502';
|
||||
end if;
|
||||
if not pg_catalog.isfinite(p_occurred_at) then
|
||||
raise exception 'canonical audit occurred_at must be finite' using errcode = '22008';
|
||||
end if;
|
||||
if pg_catalog.jsonb_typeof(p_details) <> 'object' then
|
||||
raise exception 'canonical audit details must be a JSON object'
|
||||
using errcode = '22023';
|
||||
end if;
|
||||
|
||||
select head.* into v_head
|
||||
from public.audit_chain_heads as head
|
||||
where head.singleton_id = 1
|
||||
for update;
|
||||
if not found then
|
||||
raise exception 'audit checkpoint is missing' using errcode = '23514';
|
||||
end if;
|
||||
if v_head.event_count is distinct from p_expected_event_count
|
||||
or v_head.last_sequence is distinct from p_expected_last_sequence
|
||||
or v_head.last_event_hash is distinct from p_expected_last_event_hash
|
||||
or v_head.hash_format is distinct from p_expected_hash_format
|
||||
or v_head.v2_start_sequence is distinct from p_expected_v2_start_sequence
|
||||
or v_head.legacy_prefix_count is distinct from p_expected_legacy_prefix_count
|
||||
or v_head.legacy_prefix_seal is distinct from p_expected_legacy_prefix_seal then
|
||||
raise exception 'audit checkpoint changed before canonical append'
|
||||
using errcode = '40001';
|
||||
end if;
|
||||
if v_head.hash_format <> 'v2'
|
||||
or v_head.event_count <> v_head.last_sequence
|
||||
or v_head.event_count < 0
|
||||
or v_head.v2_start_sequence < 1
|
||||
or v_head.legacy_prefix_count <> v_head.v2_start_sequence - 1
|
||||
or v_head.legacy_prefix_count > v_head.event_count
|
||||
or v_head.legacy_prefix_seal !~ '^[0-9a-f]{64}$' then
|
||||
raise exception 'audit checkpoint invariants are invalid' using errcode = '23514';
|
||||
end if;
|
||||
|
||||
select events.sequence into v_max_sequence
|
||||
from public.audit_events as events
|
||||
order by events.sequence desc, events.id desc
|
||||
limit 1;
|
||||
if v_head.event_count = 0 then
|
||||
if v_max_sequence is not null or v_head.last_event_hash is not null then
|
||||
raise exception 'empty audit checkpoint has retained events' using errcode = '23514';
|
||||
end if;
|
||||
else
|
||||
if v_max_sequence is distinct from v_head.last_sequence
|
||||
or v_head.last_event_hash is null then
|
||||
raise exception 'audit checkpoint does not identify the retained tail'
|
||||
using errcode = '23514';
|
||||
end if;
|
||||
select events.* into v_tail
|
||||
from public.audit_events as events
|
||||
where events.sequence = v_head.last_sequence;
|
||||
if not found or v_tail.event_hash is distinct from v_head.last_event_hash
|
||||
or v_tail.event_hash !~ '^[0-9a-f]{64}$' then
|
||||
raise exception 'audit retained tail is missing or does not match the checkpoint'
|
||||
using errcode = '23514';
|
||||
end if;
|
||||
if v_tail.hash_format = 'v2' then
|
||||
if v_tail.canonical_payload is null
|
||||
or pg_catalog.encode(
|
||||
pg_catalog.sha256(pg_catalog.convert_to(v_tail.canonical_payload, 'UTF8')),
|
||||
'hex'
|
||||
) <> v_tail.event_hash
|
||||
or v_tail.canonical_payload::jsonb is distinct from pg_catalog.jsonb_build_object(
|
||||
'hash_format', 'v2',
|
||||
'id', v_tail.id::text,
|
||||
'occurred_at', pg_catalog.to_char(
|
||||
v_tail.occurred_at at time zone 'UTC',
|
||||
'YYYY-MM-DD"T"HH24:MI:SS.US"Z"'
|
||||
),
|
||||
'correlation_id', v_tail.correlation_id,
|
||||
'actor_type', v_tail.actor_type,
|
||||
'actor_id', v_tail.actor_id,
|
||||
'action', v_tail.action,
|
||||
'resource_type', v_tail.resource_type,
|
||||
'resource_id', v_tail.resource_id,
|
||||
'outcome', v_tail.outcome,
|
||||
'details', v_tail.details::jsonb,
|
||||
'previous_event_hash', v_tail.previous_event_hash
|
||||
) then
|
||||
raise exception 'v2 audit retained tail payload is invalid'
|
||||
using errcode = '23514';
|
||||
end if;
|
||||
elsif v_tail.hash_format <> 'v1'
|
||||
or v_tail.sequence <> v_head.v2_start_sequence - 1 then
|
||||
raise exception 'audit retained tail hash format is invalid' using errcode = '23514';
|
||||
end if;
|
||||
if v_tail.sequence = 1 then
|
||||
if v_tail.previous_event_hash is not null then
|
||||
raise exception 'first audit event has a previous hash' using errcode = '23514';
|
||||
end if;
|
||||
else
|
||||
select events.event_hash into v_predecessor_hash
|
||||
from public.audit_events as events
|
||||
where events.sequence = v_tail.sequence - 1;
|
||||
if not found or v_tail.previous_event_hash is distinct from v_predecessor_hash then
|
||||
raise exception 'audit retained tail link is invalid' using errcode = '23514';
|
||||
end if;
|
||||
end if;
|
||||
end if;
|
||||
|
||||
v_sequence := v_head.last_sequence + 1;
|
||||
v_payload := pg_catalog.jsonb_build_object(
|
||||
'hash_format', 'v2',
|
||||
'id', p_event_id::text,
|
||||
'occurred_at', pg_catalog.to_char(
|
||||
p_occurred_at at time zone 'UTC',
|
||||
'YYYY-MM-DD"T"HH24:MI:SS.US"Z"'
|
||||
),
|
||||
'correlation_id', p_correlation_id,
|
||||
'actor_type', p_actor_type,
|
||||
'actor_id', p_actor_id,
|
||||
'action', p_action,
|
||||
'resource_type', p_resource_type,
|
||||
'resource_id', p_resource_id,
|
||||
'outcome', p_outcome,
|
||||
'details', p_details,
|
||||
'previous_event_hash', v_head.last_event_hash
|
||||
)::text;
|
||||
v_hash := pg_catalog.encode(
|
||||
pg_catalog.sha256(pg_catalog.convert_to(v_payload, 'UTF8')), 'hex'
|
||||
);
|
||||
|
||||
insert into public.audit_events (
|
||||
id, sequence, occurred_at, correlation_id, actor_type, actor_id, action,
|
||||
resource_type, resource_id, outcome, details, previous_event_hash, event_hash,
|
||||
hash_format, canonical_payload
|
||||
) values (
|
||||
p_event_id, v_sequence, p_occurred_at, p_correlation_id, p_actor_type, p_actor_id,
|
||||
p_action, p_resource_type, p_resource_id, p_outcome, p_details,
|
||||
v_head.last_event_hash, v_hash, 'v2', v_payload
|
||||
);
|
||||
update public.audit_chain_heads as head
|
||||
set event_count = v_head.event_count + 1,
|
||||
last_sequence = v_sequence,
|
||||
last_event_hash = v_hash,
|
||||
updated_at = p_occurred_at
|
||||
where head.singleton_id = 1
|
||||
and head.event_count = v_head.event_count
|
||||
and head.last_sequence = v_head.last_sequence
|
||||
and head.last_event_hash is not distinct from v_head.last_event_hash
|
||||
and head.hash_format = v_head.hash_format
|
||||
and head.v2_start_sequence = v_head.v2_start_sequence
|
||||
and head.legacy_prefix_count = v_head.legacy_prefix_count
|
||||
and head.legacy_prefix_seal = v_head.legacy_prefix_seal;
|
||||
get diagnostics v_updated = row_count;
|
||||
if v_updated <> 1 then
|
||||
raise exception 'audit checkpoint compare-and-set failed' using errcode = '40001';
|
||||
end if;
|
||||
return query select p_event_id, v_sequence, v_hash, p_occurred_at;
|
||||
end
|
||||
$append$;
|
||||
alter function modelforge_audit.append_event_v2(
|
||||
uuid, timestamptz, text, text, text, text, text, text, text, jsonb,
|
||||
bigint, bigint, text, text, bigint, bigint, text
|
||||
) owner to modelforge;
|
||||
revoke all on function modelforge_audit.append_event_v2(
|
||||
uuid, timestamptz, text, text, text, text, text, text, text, jsonb,
|
||||
bigint, bigint, text, text, bigint, bigint, text
|
||||
) from public;
|
||||
grant execute on function modelforge_audit.append_event_v2(
|
||||
uuid, timestamptz, text, text, text, text, text, text, text, jsonb,
|
||||
bigint, bigint, text, text, bigint, bigint, text
|
||||
) to modelforge_runtime;
|
||||
|
||||
grant select, insert, update, delete on all tables in schema public to modelforge_runtime;
|
||||
grant usage, select on all sequences in schema public to modelforge_runtime;
|
||||
revoke execute on all functions in schema public from public, modelforge_runtime;
|
||||
revoke insert, update, delete, truncate, references, trigger
|
||||
on public.audit_events, public.audit_chain_heads from modelforge_runtime;
|
||||
grant select on public.audit_events, public.audit_chain_heads to modelforge_runtime;
|
||||
alter default privileges for role modelforge in schema public
|
||||
grant select, insert, update, delete on tables to modelforge_runtime;
|
||||
alter default privileges for role modelforge in schema public
|
||||
grant usage, select on sequences to modelforge_runtime;
|
||||
alter default privileges for role modelforge in schema public
|
||||
revoke execute on functions from public;
|
||||
|
||||
do $database_privileges$
|
||||
begin
|
||||
execute pg_catalog.format(
|
||||
'revoke create, temporary on database %I from modelforge_runtime',
|
||||
pg_catalog.current_database()
|
||||
);
|
||||
end
|
||||
$database_privileges$;
|
||||
"""
|
||||
|
||||
|
||||
def _normalise_timestamp(value: datetime | str) -> str:
|
||||
if isinstance(value, datetime):
|
||||
moment = value
|
||||
elif isinstance(value, str):
|
||||
candidate = value.strip()
|
||||
if candidate.endswith("Z"):
|
||||
candidate = candidate[:-1] + "+00:00"
|
||||
try:
|
||||
moment = datetime.fromisoformat(candidate)
|
||||
except ValueError as error:
|
||||
raise RuntimeError("legacy audit event has an invalid occurred_at") from error
|
||||
else:
|
||||
raise RuntimeError("legacy audit event has an invalid occurred_at")
|
||||
if moment.tzinfo is None:
|
||||
moment = moment.replace(tzinfo=UTC)
|
||||
return moment.astimezone(UTC).isoformat(timespec="microseconds").replace("+00:00", "Z")
|
||||
|
||||
|
||||
def _details(value: Any) -> dict[str, Any]:
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
value = json.loads(value)
|
||||
except json.JSONDecodeError as error:
|
||||
raise RuntimeError("legacy audit event details are not valid JSON") from error
|
||||
if not isinstance(value, dict):
|
||||
raise RuntimeError("legacy audit event details must be a JSON object")
|
||||
return value
|
||||
|
||||
|
||||
def _legacy_hash(row: sa.RowMapping) -> str:
|
||||
payload = {
|
||||
"correlation_id": row["correlation_id"],
|
||||
"actor_type": row["actor_type"],
|
||||
"actor_id": row["actor_id"],
|
||||
"action": row["action"],
|
||||
"resource_type": row["resource_type"],
|
||||
"resource_id": row["resource_id"],
|
||||
"outcome": row["outcome"],
|
||||
"details": _details(row["details"]),
|
||||
"previous_event_hash": row["previous_event_hash"],
|
||||
}
|
||||
encoded = json.dumps(payload, sort_keys=True, separators=(",", ":"))
|
||||
return hashlib.sha256(encoded.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _prefix_entry(row: sa.RowMapping) -> bytes:
|
||||
try:
|
||||
event_id = str(uuid.UUID(str(row["id"])))
|
||||
except (AttributeError, TypeError, ValueError) as error:
|
||||
raise RuntimeError("legacy audit event id is not a UUID") from error
|
||||
payload = {
|
||||
"sequence": int(row["sequence"]),
|
||||
"id": event_id,
|
||||
"occurred_at": _normalise_timestamp(row["occurred_at"]),
|
||||
"event_hash": str(row["event_hash"]),
|
||||
}
|
||||
return json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8") + b"\n"
|
||||
|
||||
|
||||
def _validate_legacy_rows(rows: list[sa.RowMapping]) -> dict[str, Any]:
|
||||
"""Validate the production-shaped v1 chain and return its immutable cutover state."""
|
||||
|
||||
previous_hash: str | None = None
|
||||
prefix = hashlib.sha256()
|
||||
prefix.update(_LEGACY_PREFIX_DOMAIN)
|
||||
for expected_sequence, row in enumerate(rows, start=1):
|
||||
try:
|
||||
sequence = int(row["sequence"])
|
||||
except (TypeError, ValueError) as error:
|
||||
raise RuntimeError("legacy audit event sequence is not an integer") from error
|
||||
if sequence != expected_sequence:
|
||||
raise RuntimeError(
|
||||
f"legacy audit chain has sequence {sequence}; expected {expected_sequence}"
|
||||
)
|
||||
event_hash = str(row["event_hash"])
|
||||
if _SHA256.fullmatch(event_hash) is None:
|
||||
raise RuntimeError(f"legacy audit event {sequence} has a malformed event hash")
|
||||
if row["previous_event_hash"] != previous_hash:
|
||||
raise RuntimeError(f"legacy audit event {sequence} has an invalid previous hash")
|
||||
if _legacy_hash(row) != event_hash:
|
||||
raise RuntimeError(f"legacy audit event {sequence} content hash is invalid")
|
||||
prefix.update(_prefix_entry(row))
|
||||
previous_hash = event_hash
|
||||
count = len(rows)
|
||||
return {
|
||||
"event_count": count,
|
||||
"last_sequence": count,
|
||||
"last_event_hash": previous_hash,
|
||||
"v2_start_sequence": count + 1,
|
||||
"legacy_prefix_count": count,
|
||||
"legacy_prefix_seal": prefix.hexdigest(),
|
||||
}
|
||||
|
||||
|
||||
def _read_and_validate_legacy_chain(connection: Connection) -> dict[str, Any]:
|
||||
rows = list(
|
||||
connection.execute(
|
||||
sa.text(
|
||||
"select id, sequence, occurred_at, correlation_id, actor_type, actor_id, "
|
||||
"action, resource_type, resource_id, outcome, details, previous_event_hash, "
|
||||
"event_hash from audit_events order by sequence, id"
|
||||
)
|
||||
).mappings()
|
||||
)
|
||||
return _validate_legacy_rows(rows)
|
||||
|
||||
|
||||
def _lock_legacy_audit_chain(connection: Connection) -> None:
|
||||
"""Serialize validation and checkpoint seed against every legacy writer.
|
||||
|
||||
The shared advisory key coordinates with 0024-aware writers. ``ACCESS EXCLUSIVE`` also blocks
|
||||
pre-0024 applications, which do not know that key, until this migration transaction commits.
|
||||
SQLite is test-only; a no-op write upgrades its deferred transaction to a writer before the
|
||||
validation read so another test connection cannot append into the validation/seed window.
|
||||
"""
|
||||
|
||||
dialect = connection.dialect.name
|
||||
if dialect == "postgresql":
|
||||
connection.execute(
|
||||
sa.text("select pg_advisory_xact_lock(:lock_key)"),
|
||||
{"lock_key": _AUDIT_CHAIN_LOCK_KEY},
|
||||
)
|
||||
connection.execute(sa.text("lock table audit_events in access exclusive mode"))
|
||||
return
|
||||
if dialect == "sqlite":
|
||||
connection.execute(sa.text("update audit_events set event_hash = event_hash where 1 = 0"))
|
||||
return
|
||||
raise RuntimeError(f"audit-chain migration does not support the {dialect!r} dialect")
|
||||
|
||||
|
||||
def _validate_postgres_role_preflight(connection: Connection) -> None:
|
||||
"""Require the separately provisioned non-superuser owner/runtime roles before DDL.
|
||||
|
||||
Existing 1.2.1 installations commonly made ``modelforge`` the cluster bootstrap superuser.
|
||||
That credential cannot be converted into the API boundary implicitly by an application
|
||||
migration. Operators must first run the documented admin-owned provisioning step; failure is
|
||||
deliberately before this migration changes a column or seeds a checkpoint.
|
||||
"""
|
||||
|
||||
if connection.dialect.name != "postgresql":
|
||||
return
|
||||
roles = list(
|
||||
connection.execute(
|
||||
sa.text(
|
||||
"select rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, "
|
||||
"rolcanlogin, rolreplication, rolbypassrls from pg_catalog.pg_roles "
|
||||
"where rolname in (:owner_role, :runtime_role) order by rolname"
|
||||
),
|
||||
{"owner_role": _AUDIT_OWNER_ROLE, "runtime_role": _AUDIT_RUNTIME_ROLE},
|
||||
).mappings()
|
||||
)
|
||||
by_name = {str(row["rolname"]): row for row in roles}
|
||||
if set(by_name) != {_AUDIT_OWNER_ROLE, _AUDIT_RUNTIME_ROLE}:
|
||||
raise RuntimeError(
|
||||
"audit migration preflight requires separately provisioned modelforge owner and "
|
||||
"modelforge_runtime roles; run the v1.2.1-to-schema-0024 role provisioning step"
|
||||
)
|
||||
current_role = str(connection.scalar(sa.text("select current_user")))
|
||||
if current_role != _AUDIT_OWNER_ROLE:
|
||||
raise RuntimeError(
|
||||
"audit migration must run with the non-superuser modelforge owner credential"
|
||||
)
|
||||
session_role = str(connection.scalar(sa.text("select session_user")))
|
||||
if session_role != _AUDIT_OWNER_ROLE:
|
||||
raise RuntimeError(
|
||||
"audit migration must authenticate directly as modelforge, not SET ROLE from admin"
|
||||
)
|
||||
for role_name, require_noinherit in (
|
||||
(_AUDIT_OWNER_ROLE, False),
|
||||
(_AUDIT_RUNTIME_ROLE, True),
|
||||
):
|
||||
role = by_name[role_name]
|
||||
forbidden = any(
|
||||
bool(role[field])
|
||||
for field in (
|
||||
"rolsuper",
|
||||
"rolcreaterole",
|
||||
"rolcreatedb",
|
||||
"rolreplication",
|
||||
"rolbypassrls",
|
||||
)
|
||||
)
|
||||
if forbidden or not bool(role["rolcanlogin"]):
|
||||
raise RuntimeError(f"database role {role_name} has forbidden administrative powers")
|
||||
if require_noinherit and bool(role["rolinherit"]):
|
||||
raise RuntimeError("modelforge_runtime must be provisioned NOINHERIT")
|
||||
app_role_membership_count = int(
|
||||
connection.scalar(
|
||||
sa.text(
|
||||
"select count(*) from pg_catalog.pg_auth_members as membership "
|
||||
"join pg_catalog.pg_roles as member on member.oid = membership.member "
|
||||
"where member.rolname in (:runtime_role, :owner_role)"
|
||||
),
|
||||
{"runtime_role": _AUDIT_RUNTIME_ROLE, "owner_role": _AUDIT_OWNER_ROLE},
|
||||
)
|
||||
or 0
|
||||
)
|
||||
if app_role_membership_count:
|
||||
raise RuntimeError(
|
||||
"modelforge and modelforge_runtime must have no SET ROLE-capable memberships"
|
||||
)
|
||||
|
||||
|
||||
def _install_postgres_audit_boundary(connection: Connection) -> None:
|
||||
if connection.dialect.name == "postgresql":
|
||||
for statement in _postgres_sql_statements(_POSTGRES_AUDIT_BOUNDARY_SQL):
|
||||
_exec_postgres_sql(connection, statement)
|
||||
|
||||
|
||||
def _exec_postgres_sql(connection: Connection, statement: str) -> None:
|
||||
"""Execute trusted static SQL without exposing PostgreSQL percent syntax to DBAPI parsing."""
|
||||
|
||||
paramstyle = getattr(connection.dialect, "paramstyle", None)
|
||||
driver_statement = (
|
||||
statement.replace("%", "%%")
|
||||
if paramstyle in {"format", "pyformat"}
|
||||
else statement
|
||||
)
|
||||
connection.exec_driver_sql(driver_statement)
|
||||
|
||||
|
||||
def _postgres_sql_statements(script: str) -> list[str]:
|
||||
"""Split this migration's trusted static SQL without splitting function bodies."""
|
||||
|
||||
statements: list[str] = []
|
||||
start = 0
|
||||
index = 0
|
||||
quote: str | None = None
|
||||
while index < len(script):
|
||||
if quote is not None:
|
||||
if quote == "'" and script.startswith("''", index):
|
||||
index += 2
|
||||
continue
|
||||
if script.startswith(quote, index):
|
||||
index += len(quote)
|
||||
quote = None
|
||||
continue
|
||||
index += 1
|
||||
continue
|
||||
character = script[index]
|
||||
if character == "'":
|
||||
quote = "'"
|
||||
index += 1
|
||||
continue
|
||||
if character == "$":
|
||||
delimiter = re.match(r"\$[A-Za-z_][A-Za-z0-9_]*\$|\$\$", script[index:])
|
||||
if delimiter is not None:
|
||||
quote = delimiter.group(0)
|
||||
index += len(quote)
|
||||
continue
|
||||
if character == ";":
|
||||
statement = script[start:index].strip()
|
||||
if statement:
|
||||
statements.append(statement)
|
||||
start = index + 1
|
||||
index += 1
|
||||
trailing = script[start:].strip()
|
||||
if quote is not None:
|
||||
raise RuntimeError("generated PostgreSQL audit boundary SQL has an unterminated literal")
|
||||
if trailing:
|
||||
statements.append(trailing)
|
||||
return statements
|
||||
|
||||
|
||||
def _remove_postgres_audit_boundary(connection: Connection) -> None:
|
||||
if connection.dialect.name != "postgresql":
|
||||
return
|
||||
script = (
|
||||
"drop trigger if exists trg_modelforge_audit_events_owner on public.audit_events; "
|
||||
"drop trigger if exists trg_modelforge_audit_events_truncate_owner "
|
||||
"on public.audit_events; "
|
||||
"drop trigger if exists trg_modelforge_audit_head_owner on public.audit_chain_heads; "
|
||||
"drop trigger if exists trg_modelforge_audit_head_truncate_owner "
|
||||
"on public.audit_chain_heads; "
|
||||
"drop function if exists modelforge_audit.append_event_v2("
|
||||
"uuid, timestamptz, text, text, text, text, text, text, text, jsonb, "
|
||||
"bigint, bigint, text, text, bigint, bigint, text); "
|
||||
"drop function if exists modelforge_audit.enforce_owner_mutation(); "
|
||||
"drop schema if exists modelforge_audit; "
|
||||
"grant select, insert on public.audit_events to modelforge_runtime"
|
||||
)
|
||||
for statement in _postgres_sql_statements(script):
|
||||
_exec_postgres_sql(connection, statement)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
connection = op.get_bind()
|
||||
_validate_postgres_role_preflight(connection)
|
||||
_lock_legacy_audit_chain(connection)
|
||||
# Validation deliberately precedes every schema mutation. In particular, legacy recovery
|
||||
# markers written with a random hash stop the migration instead of being blessed by a seal.
|
||||
legacy = _read_and_validate_legacy_chain(connection)
|
||||
|
||||
op.add_column(
|
||||
"audit_events",
|
||||
sa.Column("hash_format", sa.String(length=16), nullable=True),
|
||||
)
|
||||
op.add_column(
|
||||
"audit_events",
|
||||
sa.Column("canonical_payload", sa.Text(), nullable=True),
|
||||
)
|
||||
connection.execute(sa.text("update audit_events set hash_format = 'v1'"))
|
||||
with op.batch_alter_table("audit_events") as batch:
|
||||
batch.alter_column(
|
||||
"hash_format",
|
||||
existing_type=sa.String(length=16),
|
||||
nullable=False,
|
||||
)
|
||||
batch.create_check_constraint(
|
||||
"ck_audit_event_hash_format", "hash_format IN ('v1', 'v2')"
|
||||
)
|
||||
batch.create_check_constraint(
|
||||
"ck_audit_event_canonical_payload",
|
||||
"((hash_format = 'v1' AND canonical_payload IS NULL) OR "
|
||||
"(hash_format = 'v2' AND canonical_payload IS NOT NULL))",
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"audit_chain_heads",
|
||||
sa.Column("singleton_id", sa.Integer(), nullable=False),
|
||||
sa.Column("event_count", sa.BigInteger(), nullable=False),
|
||||
sa.Column("last_sequence", sa.BigInteger(), nullable=False),
|
||||
sa.Column("last_event_hash", sa.String(length=64), nullable=True),
|
||||
sa.Column("hash_format", sa.String(length=16), nullable=False),
|
||||
sa.Column("v2_start_sequence", sa.BigInteger(), nullable=False),
|
||||
sa.Column("legacy_prefix_count", sa.BigInteger(), nullable=False),
|
||||
sa.Column("legacy_prefix_seal", sa.String(length=64), nullable=False),
|
||||
sa.Column(
|
||||
"updated_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("CURRENT_TIMESTAMP"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.CheckConstraint("singleton_id = 1", name="ck_audit_chain_head_singleton"),
|
||||
sa.CheckConstraint("event_count >= 0", name="ck_audit_chain_head_count"),
|
||||
sa.CheckConstraint("last_sequence >= 0", name="ck_audit_chain_head_sequence"),
|
||||
sa.CheckConstraint(
|
||||
"event_count = last_sequence",
|
||||
name="ck_audit_chain_head_count_sequence",
|
||||
),
|
||||
sa.CheckConstraint("v2_start_sequence >= 1", name="ck_audit_chain_head_cutover"),
|
||||
sa.CheckConstraint(
|
||||
"legacy_prefix_count = v2_start_sequence - 1",
|
||||
name="ck_audit_chain_head_prefix_count",
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"legacy_prefix_count <= event_count",
|
||||
name="ck_audit_chain_head_prefix_within_chain",
|
||||
),
|
||||
sa.CheckConstraint("hash_format = 'v2'", name="ck_audit_chain_head_hash_format"),
|
||||
sa.CheckConstraint(
|
||||
"length(legacy_prefix_seal) = 64",
|
||||
name="ck_audit_chain_head_prefix_seal",
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"((event_count = 0 AND last_sequence = 0 AND last_event_hash IS NULL) OR "
|
||||
"(event_count > 0 AND last_sequence > 0 AND last_event_hash IS NOT NULL))",
|
||||
name="ck_audit_chain_head_shape",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("singleton_id"),
|
||||
)
|
||||
connection.execute(
|
||||
sa.text(
|
||||
"insert into audit_chain_heads (singleton_id, event_count, last_sequence, "
|
||||
"last_event_hash, hash_format, v2_start_sequence, legacy_prefix_count, "
|
||||
"legacy_prefix_seal) values (1, :event_count, :last_sequence, :last_event_hash, "
|
||||
"'v2', :v2_start_sequence, :legacy_prefix_count, :legacy_prefix_seal)"
|
||||
),
|
||||
legacy,
|
||||
)
|
||||
_install_postgres_audit_boundary(connection)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
connection = op.get_bind()
|
||||
_validate_postgres_role_preflight(connection)
|
||||
_lock_legacy_audit_chain(connection)
|
||||
non_legacy = int(
|
||||
connection.scalar(
|
||||
sa.text("select count(*) from audit_events where hash_format <> 'v1'")
|
||||
)
|
||||
or 0
|
||||
)
|
||||
if non_legacy:
|
||||
raise RuntimeError(
|
||||
"cannot downgrade audit hash format after v2 events exist without rewriting history"
|
||||
)
|
||||
legacy = _read_and_validate_legacy_chain(connection)
|
||||
head = connection.execute(
|
||||
sa.text(
|
||||
"select event_count, last_sequence, last_event_hash, hash_format, "
|
||||
"v2_start_sequence, legacy_prefix_count, legacy_prefix_seal "
|
||||
"from audit_chain_heads where singleton_id = 1"
|
||||
)
|
||||
).mappings().one_or_none()
|
||||
if head is None or head["hash_format"] != "v2":
|
||||
raise RuntimeError("cannot downgrade a missing or malformed audit checkpoint")
|
||||
for key, expected in legacy.items():
|
||||
if head[key] != expected:
|
||||
raise RuntimeError(f"cannot downgrade: audit checkpoint {key} is inconsistent")
|
||||
|
||||
_remove_postgres_audit_boundary(connection)
|
||||
op.drop_table("audit_chain_heads")
|
||||
with op.batch_alter_table("audit_events") as batch:
|
||||
batch.drop_constraint("ck_audit_event_canonical_payload", type_="check")
|
||||
batch.drop_constraint("ck_audit_event_hash_format", type_="check")
|
||||
batch.drop_column("canonical_payload")
|
||||
batch.drop_column("hash_format")
|
||||
@@ -0,0 +1,65 @@
|
||||
[build-system]
|
||||
requires = ["hatchling"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[project]
|
||||
name = "modelforge-api"
|
||||
version = "1.2.2"
|
||||
description = "ITWorx ModelForge control-plane API"
|
||||
license = "AGPL-3.0-or-later"
|
||||
requires-python = ">=3.12"
|
||||
dependencies = [
|
||||
"fastapi==0.141.1",
|
||||
"uvicorn[standard]==0.52.4",
|
||||
"pydantic==2.13.4",
|
||||
"pydantic-settings==2.15.0",
|
||||
"sqlalchemy==2.0.52",
|
||||
"alembic==1.19.1",
|
||||
"psycopg[binary]==3.3.4",
|
||||
"cryptography==50.0.1",
|
||||
"redis==6.4.0",
|
||||
"httpx==0.28.1",
|
||||
"huggingface-hub==1.29.0",
|
||||
"nvidia-ml-py==13.610.43",
|
||||
"psutil==7.2.2",
|
||||
"pyyaml==6.0.3",
|
||||
"structlog==25.5.0",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
dev = [
|
||||
"pytest==8.4.2",
|
||||
"pytest-asyncio==1.4.0",
|
||||
"ruff==0.16.5",
|
||||
"mypy==1.20.2",
|
||||
]
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
packages = ["src/modelforge_api"]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
pythonpath = ["src"]
|
||||
testpaths = ["tests"]
|
||||
|
||||
[tool.mypy]
|
||||
# Strict, and analysed for the platform the images actually run on.
|
||||
#
|
||||
# Both of these were assumed rather than configured. There was no [tool.mypy] section at all, so
|
||||
# `mypy src` ran with defaults while every milestone report described it as strict; and it analysed
|
||||
# the developer's platform, so a Windows-only winreg branch type-checked cleanly here and failed in
|
||||
# CI on identical code. Turning strict on cost three stale `type: ignore` comments.
|
||||
strict = true
|
||||
platform = "linux"
|
||||
|
||||
[tool.ruff]
|
||||
line-length = 100
|
||||
target-version = "py312"
|
||||
|
||||
[tool.ruff.lint]
|
||||
extend-select = ["B", "BLE", "I", "S", "SIM", "UP"]
|
||||
ignore = ["B008"]
|
||||
|
||||
[tool.ruff.lint.per-file-ignores]
|
||||
"tests/*" = ["S101"]
|
||||
"src/modelforge_api/settings.py" = ["S104"]
|
||||
"src/modelforge_api/hardware/collectors.py" = ["B023"]
|
||||
@@ -0,0 +1,11 @@
|
||||
"""ITWorx ModelForge control plane.
|
||||
|
||||
The version is derived from the repository's VERSION file rather than declared again here. Four
|
||||
independent copies of the product version is three chances to publish a release that misdescribes
|
||||
itself, which is exactly the class of mistake a release gate is supposed to make impossible.
|
||||
"""
|
||||
|
||||
from modelforge_api.domain.release import PRODUCT_VERSION
|
||||
|
||||
__all__ = ["__version__"]
|
||||
__version__ = PRODUCT_VERSION
|
||||
@@ -0,0 +1,208 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hmac
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from enum import StrEnum
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import Depends, Header, HTTPException, Request
|
||||
|
||||
from modelforge_api.settings import Settings, get_settings
|
||||
|
||||
|
||||
class PrincipalRole(StrEnum):
|
||||
"""Closed operator-console roles, ordered from read-only to security administration."""
|
||||
|
||||
VIEWER = "viewer"
|
||||
OPERATOR = "operator"
|
||||
ADMIN = "admin"
|
||||
|
||||
|
||||
class AccessBoundary(StrEnum):
|
||||
"""Mutually exclusive request authentication boundaries.
|
||||
|
||||
The control-plane boundary is deliberately the default. New endpoints therefore fail closed
|
||||
behind the human operator credential until their narrower machine/public policy is explicitly
|
||||
recorded here and in the route inventory tests.
|
||||
"""
|
||||
|
||||
PUBLIC = "public"
|
||||
SINGLE_USE_ENROLLMENT = "single_use_enrollment"
|
||||
NODE = "node"
|
||||
CAPABILITY_CLIENT = "capability_client"
|
||||
CONTROL_PLANE = "control_plane"
|
||||
|
||||
|
||||
_PUBLIC_REQUESTS = {
|
||||
("GET", "/"),
|
||||
("GET", "/api/v1/health/live"),
|
||||
("GET", "/api/v1/health/ready"),
|
||||
("GET", "/api/v1/version"),
|
||||
}
|
||||
_INTERACTIVE_API_PATHS = {"/docs", "/docs/oauth2-redirect", "/redoc", "/openapi.json"}
|
||||
_NODE_REQUESTS = {
|
||||
("POST", "/api/v1/agent/heartbeat"),
|
||||
("PUT", "/api/v1/agent/inventory"),
|
||||
("PUT", "/api/v1/agent/telemetry"),
|
||||
("GET", "/api/v1/agent/artifact-jobs/next"),
|
||||
("POST", "/api/v1/agent/artifact-jobs/{job_id}/progress"),
|
||||
("POST", "/api/v1/agent/artifact-jobs/{job_id}/complete"),
|
||||
("POST", "/api/v1/agent/artifact-jobs/{job_id}/fail"),
|
||||
("GET", "/api/v1/agent/runtime-probes/next"),
|
||||
("POST", "/api/v1/agent/runtime-probes/{probe_id}/progress"),
|
||||
("POST", "/api/v1/agent/runtime-probes/{probe_id}/complete"),
|
||||
("POST", "/api/v1/agent/runtime-probes/{probe_id}/fail"),
|
||||
("GET", "/api/v1/agent/serving-jobs/next"),
|
||||
("POST", "/api/v1/agent/serving-jobs/{job_id}/complete"),
|
||||
("POST", "/api/v1/agent/serving-jobs/{job_id}/fail"),
|
||||
("POST", "/api/v1/agent/serving-state"),
|
||||
}
|
||||
_NODE_PARAMETERIZED_REQUESTS = (
|
||||
("POST", re.compile(r"^/api/v1/agent/artifact-jobs/[^/]+/(?:progress|complete|fail)$")),
|
||||
("POST", re.compile(r"^/api/v1/agent/runtime-probes/[^/]+/(?:progress|complete|fail)$")),
|
||||
("POST", re.compile(r"^/api/v1/agent/serving-jobs/[^/]+/(?:complete|fail)$")),
|
||||
)
|
||||
_CAPABILITY_REQUIREMENTS = {
|
||||
("POST", "/api/v1/capabilities/rag.embedding@1/invoke"): "rag.embedding@1",
|
||||
("POST", "/api/v1/capabilities/rag.reranking@1/invoke"): "rag.reranking@1",
|
||||
("POST", "/api/v1/capabilities/document.ocr@1/invoke"): "document.ocr@1",
|
||||
("POST", "/api/v1/capabilities/vision.embedding@1/invoke"): "vision.embedding@1",
|
||||
("POST", "/api/v1/capabilities/speech.transcription@1/invoke"): "speech.transcription@1",
|
||||
("POST", "/api/v1/capability-experiments/{route_key}/invoke"): "rag.embedding@1",
|
||||
("POST", "/v1/embeddings"): "rag.embedding@1",
|
||||
}
|
||||
_CAPABILITY_CLIENT_REQUESTS = set(_CAPABILITY_REQUIREMENTS)
|
||||
_CAPABILITY_EXPERIMENT_REQUEST = re.compile(r"^/api/v1/capability-experiments/[^/]+/invoke$")
|
||||
|
||||
|
||||
def access_boundary_for_request(
|
||||
method: str,
|
||||
path: str,
|
||||
) -> AccessBoundary:
|
||||
"""Classify an HTTP request without reading its body or accepting credential aliases."""
|
||||
|
||||
normalized_method = method.upper()
|
||||
if (normalized_method, path) in _PUBLIC_REQUESTS:
|
||||
return AccessBoundary.PUBLIC
|
||||
if path in _INTERACTIVE_API_PATHS:
|
||||
# These routes exist only when the FastAPI app is explicitly constructed for development.
|
||||
# In test/production they must reach routing unauthenticated so the disabled surface is a
|
||||
# genuine 404 rather than an operator-auth challenge that reveals a hidden endpoint.
|
||||
return AccessBoundary.PUBLIC
|
||||
if normalized_method == "POST" and path == "/api/v1/agent/enroll":
|
||||
return AccessBoundary.SINGLE_USE_ENROLLMENT
|
||||
if (normalized_method, path) in _NODE_REQUESTS or any(
|
||||
normalized_method == rule_method and pattern.fullmatch(path)
|
||||
for rule_method, pattern in _NODE_PARAMETERIZED_REQUESTS
|
||||
):
|
||||
return AccessBoundary.NODE
|
||||
if (normalized_method, path) in _CAPABILITY_CLIENT_REQUESTS or (
|
||||
normalized_method == "POST" and _CAPABILITY_EXPERIMENT_REQUEST.fullmatch(path)
|
||||
):
|
||||
return AccessBoundary.CAPABILITY_CLIENT
|
||||
return AccessBoundary.CONTROL_PLANE
|
||||
|
||||
|
||||
def required_capability_for_request(method: str, path: str) -> str | None:
|
||||
"""Return the exact capability scope for a registered project-facing route."""
|
||||
|
||||
normalized_method = method.upper()
|
||||
requirement = _CAPABILITY_REQUIREMENTS.get((normalized_method, path))
|
||||
if requirement is not None:
|
||||
return requirement
|
||||
if normalized_method == "POST" and _CAPABILITY_EXPERIMENT_REQUEST.fullmatch(path):
|
||||
return "rag.embedding@1"
|
||||
return None
|
||||
|
||||
|
||||
def request_body_limit_bytes(settings: Settings, boundary: AccessBoundary) -> int:
|
||||
"""Select an explicit pre-parser body limit for every request boundary."""
|
||||
|
||||
if boundary is AccessBoundary.CAPABILITY_CLIENT:
|
||||
return settings.gateway_max_payload_bytes
|
||||
if boundary in {AccessBoundary.NODE, AccessBoundary.SINGLE_USE_ENROLLMENT}:
|
||||
return settings.node_agent_max_payload_bytes
|
||||
return settings.control_plane_max_payload_bytes
|
||||
|
||||
|
||||
_ROLE_RANK = {
|
||||
PrincipalRole.VIEWER: 0,
|
||||
PrincipalRole.OPERATOR: 1,
|
||||
PrincipalRole.ADMIN: 2,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Principal:
|
||||
"""An authenticated human/control-plane principal.
|
||||
|
||||
Node and capability credentials deliberately never become a ``Principal``. Their separate
|
||||
dependencies remain the only way into node-agent and inference surfaces, preventing credential
|
||||
confusion at the type and dependency boundaries.
|
||||
"""
|
||||
|
||||
subject: str
|
||||
role: PrincipalRole
|
||||
authentication_method: str
|
||||
|
||||
def permits(self, required: PrincipalRole) -> bool:
|
||||
return _ROLE_RANK[self.role] >= _ROLE_RANK[required]
|
||||
|
||||
|
||||
def authenticate_operator_token(settings: Settings, token: str | None) -> Principal:
|
||||
"""Authenticate the backward-compatible operator key as an admin principal.
|
||||
|
||||
The legacy key is the sole human credential in this release. OIDC/session authentication can
|
||||
add another principal producer later without changing route authorization policy.
|
||||
"""
|
||||
|
||||
configured = settings.operator_api_key
|
||||
if configured is None or not configured.get_secret_value():
|
||||
raise HTTPException(status_code=503, detail="operator API authentication is not configured")
|
||||
if token is None or not hmac.compare_digest(token, configured.get_secret_value()):
|
||||
raise HTTPException(status_code=401, detail="invalid operator credential")
|
||||
return Principal(
|
||||
subject="legacy-operator",
|
||||
role=PrincipalRole.ADMIN,
|
||||
authentication_method="legacy_admin_token",
|
||||
)
|
||||
|
||||
|
||||
def authenticate_operator_principal(
|
||||
request: Request,
|
||||
settings: Annotated[Settings, Depends(get_settings)],
|
||||
token: Annotated[str | None, Header(alias="X-ModelForge-Admin-Token")] = None,
|
||||
) -> Principal:
|
||||
"""Reuse the pre-body principal and retain a safe direct-dependency fallback."""
|
||||
|
||||
principal = getattr(request.state, "principal", None)
|
||||
if isinstance(principal, Principal):
|
||||
return principal
|
||||
return authenticate_operator_token(settings, token)
|
||||
|
||||
|
||||
AuthenticatedPrincipal = Annotated[Principal, Depends(authenticate_operator_principal)]
|
||||
|
||||
|
||||
def _require_role(principal: Principal, required: PrincipalRole) -> Principal:
|
||||
if not principal.permits(required):
|
||||
raise HTTPException(status_code=403, detail="operator role is not authorized")
|
||||
return principal
|
||||
|
||||
|
||||
def require_viewer(principal: AuthenticatedPrincipal) -> Principal:
|
||||
return _require_role(principal, PrincipalRole.VIEWER)
|
||||
|
||||
|
||||
def require_operator(principal: AuthenticatedPrincipal) -> Principal:
|
||||
return _require_role(principal, PrincipalRole.OPERATOR)
|
||||
|
||||
|
||||
def require_admin(principal: AuthenticatedPrincipal) -> Principal:
|
||||
return _require_role(principal, PrincipalRole.ADMIN)
|
||||
|
||||
|
||||
Viewer = Annotated[Principal, Depends(require_viewer)]
|
||||
Operator = Annotated[Principal, Depends(require_operator)]
|
||||
Admin = Annotated[Principal, Depends(require_admin)]
|
||||
@@ -0,0 +1,205 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from starlette.responses import JSONResponse
|
||||
from starlette.types import ASGIApp, Message, Receive, Scope, Send
|
||||
|
||||
MAX_CONSECUTIVE_EMPTY_REQUEST_EVENTS = 32
|
||||
MAX_TOTAL_EMPTY_REQUEST_EVENTS = 128
|
||||
MAX_REQUEST_BODY_EVENTS = 4096
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RequestBodyTooLarge(Exception):
|
||||
limit_bytes: int
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RequestBodyProgressExhausted(Exception):
|
||||
received_events: int
|
||||
empty_events: int
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class InvalidContentLength(Exception):
|
||||
message: str
|
||||
|
||||
|
||||
class RequestBodyLimitMiddleware:
|
||||
"""Reject declared or streamed oversized bodies before application parsing.
|
||||
|
||||
The receive wrapper forwards chunks only while the cumulative size is within the configured
|
||||
boundary. It raises as soon as the next chunk crosses the limit and never drains or buffers the
|
||||
remaining request body. It also rejects a body stream that exceeds the fixed request-event or
|
||||
empty-progress budgets, preventing an immediately-ready sequence of empty ASGI frames from
|
||||
spinning indefinitely without yielding useful input.
|
||||
"""
|
||||
|
||||
def __init__(self, app: ASGIApp) -> None:
|
||||
self.app = app
|
||||
|
||||
@staticmethod
|
||||
def _declared_length(scope: Scope) -> int | None:
|
||||
values = [
|
||||
value for name, value in scope.get("headers", []) if name.lower() == b"content-length"
|
||||
]
|
||||
if not values:
|
||||
return None
|
||||
if len(values) != 1:
|
||||
raise ValueError("multiple content-length headers are not accepted")
|
||||
try:
|
||||
rendered = values[0].decode("ascii")
|
||||
if not rendered.isdecimal():
|
||||
raise ValueError("content-length must be an unsigned decimal integer")
|
||||
return int(rendered)
|
||||
except UnicodeDecodeError as exc:
|
||||
raise ValueError("content-length must be ASCII") from exc
|
||||
|
||||
@staticmethod
|
||||
async def _error_response(
|
||||
scope: Scope,
|
||||
receive: Receive,
|
||||
send: Send,
|
||||
*,
|
||||
status_code: int,
|
||||
code: str,
|
||||
message: str,
|
||||
details: dict[str, int] | None = None,
|
||||
) -> None:
|
||||
state = scope.get("state", {})
|
||||
correlation_id = state.get("correlation_id", "unknown")
|
||||
response = JSONResponse(
|
||||
status_code=status_code,
|
||||
content={
|
||||
"error": {
|
||||
"code": code,
|
||||
"message": message,
|
||||
"correlation_id": correlation_id,
|
||||
"details": details or {},
|
||||
}
|
||||
},
|
||||
headers={"X-Correlation-ID": str(correlation_id)},
|
||||
)
|
||||
await response(scope, receive, send)
|
||||
|
||||
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
|
||||
if scope["type"] != "http":
|
||||
await self.app(scope, receive, send)
|
||||
return
|
||||
|
||||
state = scope.setdefault("state", {})
|
||||
configured_limit: int | None = None
|
||||
headers_checked = False
|
||||
received_bytes = 0
|
||||
received_events = 0
|
||||
empty_events = 0
|
||||
consecutive_empty_events = 0
|
||||
request_error: (
|
||||
InvalidContentLength | RequestBodyTooLarge | RequestBodyProgressExhausted | None
|
||||
) = None
|
||||
|
||||
async def limited_receive() -> Message:
|
||||
nonlocal configured_limit, headers_checked
|
||||
nonlocal consecutive_empty_events, empty_events, received_bytes
|
||||
nonlocal received_events, request_error
|
||||
if not headers_checked:
|
||||
candidate_limit = state.get("request_body_limit_bytes")
|
||||
if not isinstance(candidate_limit, int) or candidate_limit < 0:
|
||||
raise RuntimeError("request body limit was not established before routing")
|
||||
configured_limit = candidate_limit
|
||||
headers_checked = True
|
||||
try:
|
||||
declared_length = self._declared_length(scope)
|
||||
except ValueError as exc:
|
||||
request_error = InvalidContentLength(str(exc))
|
||||
state["request_body_error_status_code"] = 400
|
||||
raise request_error from exc
|
||||
if declared_length is not None and declared_length > configured_limit:
|
||||
request_error = RequestBodyTooLarge(configured_limit)
|
||||
state["request_body_error_status_code"] = 413
|
||||
raise request_error
|
||||
message = await receive()
|
||||
if message["type"] == "http.request":
|
||||
if configured_limit is None: # pragma: no cover - guarded before receive
|
||||
raise RuntimeError("request body limit was not established before routing")
|
||||
received_events += 1
|
||||
body = message.get("body", b"")
|
||||
received_bytes += len(body)
|
||||
if received_bytes > configured_limit:
|
||||
request_error = RequestBodyTooLarge(configured_limit)
|
||||
state["request_body_error_status_code"] = 413
|
||||
raise request_error
|
||||
if body:
|
||||
consecutive_empty_events = 0
|
||||
elif message.get("more_body", False):
|
||||
empty_events += 1
|
||||
consecutive_empty_events += 1
|
||||
else:
|
||||
consecutive_empty_events = 0
|
||||
if (
|
||||
received_events > MAX_REQUEST_BODY_EVENTS
|
||||
or empty_events > MAX_TOTAL_EMPTY_REQUEST_EVENTS
|
||||
or consecutive_empty_events > MAX_CONSECUTIVE_EMPTY_REQUEST_EVENTS
|
||||
):
|
||||
request_error = RequestBodyProgressExhausted(
|
||||
received_events=received_events,
|
||||
empty_events=empty_events,
|
||||
)
|
||||
state["request_body_error_status_code"] = 400
|
||||
raise request_error
|
||||
return message
|
||||
|
||||
response_started = False
|
||||
|
||||
async def tracked_send(message: Message) -> None:
|
||||
nonlocal response_started
|
||||
# FastAPI deliberately converts arbitrary request-body receive failures to a generic
|
||||
# 400. Once this middleware has observed a boundary violation, suppress that parser
|
||||
# response and emit the boundary's typed response after the inner app unwinds.
|
||||
if request_error is not None:
|
||||
return
|
||||
if message["type"] == "http.response.start":
|
||||
response_started = True
|
||||
await send(message)
|
||||
|
||||
try:
|
||||
await self.app(scope, limited_receive, tracked_send)
|
||||
except (InvalidContentLength, RequestBodyProgressExhausted, RequestBodyTooLarge) as exc:
|
||||
request_error = exc
|
||||
if request_error is not None:
|
||||
if response_started:
|
||||
raise request_error
|
||||
if isinstance(request_error, InvalidContentLength):
|
||||
await self._error_response(
|
||||
scope,
|
||||
receive,
|
||||
send,
|
||||
status_code=400,
|
||||
code="invalid_content_length",
|
||||
message=request_error.message,
|
||||
)
|
||||
elif isinstance(request_error, RequestBodyTooLarge):
|
||||
await self._error_response(
|
||||
scope,
|
||||
receive,
|
||||
send,
|
||||
status_code=413,
|
||||
code="request_body_too_large",
|
||||
message="Request body exceeds the permitted boundary",
|
||||
details={"limit_bytes": request_error.limit_bytes},
|
||||
)
|
||||
else:
|
||||
await self._error_response(
|
||||
scope,
|
||||
receive,
|
||||
send,
|
||||
status_code=400,
|
||||
code="request_body_progress_exhausted",
|
||||
message="Request body made insufficient bounded progress",
|
||||
details={
|
||||
"max_consecutive_empty_events": MAX_CONSECUTIVE_EMPTY_REQUEST_EVENTS,
|
||||
"max_empty_events": MAX_TOTAL_EMPTY_REQUEST_EVENTS,
|
||||
"max_request_events": MAX_REQUEST_BODY_EVENTS,
|
||||
},
|
||||
)
|
||||
@@ -0,0 +1,134 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from modelforge_api.api.authorization import require_admin, require_operator, require_viewer
|
||||
from modelforge_api.db import get_session
|
||||
from modelforge_api.domain.acquisition import (
|
||||
ArtifactJobResponse,
|
||||
ArtifactSetResponse,
|
||||
DiscoveryCandidate,
|
||||
DiscoverySearchRequest,
|
||||
DownloadPlanCreate,
|
||||
DownloadPlanResponse,
|
||||
UpstreamRefreshRequest,
|
||||
UpstreamSnapshotResponse,
|
||||
)
|
||||
from modelforge_api.providers.huggingface import OfficialHuggingFaceProvider
|
||||
from modelforge_api.services.acquisition import AcquisitionService
|
||||
from modelforge_api.settings import Settings, get_settings
|
||||
|
||||
router = APIRouter(
|
||||
prefix="/api/v1",
|
||||
tags=["artifact-acquisition"],
|
||||
dependencies=[Depends(require_viewer)],
|
||||
)
|
||||
|
||||
|
||||
def get_acquisition_service(
|
||||
session: Annotated[Session, Depends(get_session)],
|
||||
settings: Annotated[Settings, Depends(get_settings)],
|
||||
) -> AcquisitionService:
|
||||
token = settings.hf_token.get_secret_value() if settings.hf_token else None
|
||||
provider = OfficialHuggingFaceProvider(token=token, timeout=settings.hf_timeout_seconds)
|
||||
return AcquisitionService(session, settings, provider)
|
||||
|
||||
|
||||
Service = Annotated[AcquisitionService, Depends(get_acquisition_service)]
|
||||
|
||||
|
||||
@router.post(
|
||||
"/discovery/search",
|
||||
response_model=list[DiscoveryCandidate],
|
||||
dependencies=[Depends(require_operator)],
|
||||
)
|
||||
def search(request: DiscoverySearchRequest, service: Service) -> list[DiscoveryCandidate]:
|
||||
return service.search(request)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/models/{model_id}/refresh-upstream",
|
||||
response_model=UpstreamSnapshotResponse,
|
||||
dependencies=[Depends(require_operator)],
|
||||
)
|
||||
def refresh_upstream(
|
||||
model_id: uuid.UUID, request: UpstreamRefreshRequest, service: Service
|
||||
) -> UpstreamSnapshotResponse:
|
||||
return service.refresh_model(model_id, request.revision)
|
||||
|
||||
|
||||
@router.get("/models/{model_id}/upstream", response_model=UpstreamSnapshotResponse)
|
||||
def model_upstream(model_id: uuid.UUID, service: Service) -> UpstreamSnapshotResponse:
|
||||
return service.latest_snapshot(model_id)
|
||||
|
||||
|
||||
@router.get("/revisions/{revision_id}/artifact-sets", response_model=list[ArtifactSetResponse])
|
||||
def artifact_sets(revision_id: uuid.UUID, service: Service) -> list[ArtifactSetResponse]:
|
||||
return service.artifact_sets(revision_id)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/download-plans",
|
||||
response_model=DownloadPlanResponse,
|
||||
status_code=201,
|
||||
dependencies=[Depends(require_operator)],
|
||||
)
|
||||
def create_download_plan(request: DownloadPlanCreate, service: Service) -> DownloadPlanResponse:
|
||||
return service.create_plan(request)
|
||||
|
||||
|
||||
@router.get("/download-plans/{plan_id}", response_model=DownloadPlanResponse)
|
||||
def download_plan(plan_id: uuid.UUID, service: Service) -> DownloadPlanResponse:
|
||||
return service.plan_response(plan_id)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/download-plans/{plan_id}/approve",
|
||||
response_model=DownloadPlanResponse,
|
||||
dependencies=[Depends(require_admin)],
|
||||
)
|
||||
def approve_download_plan(plan_id: uuid.UUID, service: Service) -> DownloadPlanResponse:
|
||||
return service.approve_plan(plan_id)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/download-plans/{plan_id}/execute",
|
||||
response_model=ArtifactJobResponse,
|
||||
dependencies=[Depends(require_operator)],
|
||||
)
|
||||
def execute_download_plan(plan_id: uuid.UUID, service: Service) -> ArtifactJobResponse:
|
||||
return service.execute_plan(plan_id)
|
||||
|
||||
|
||||
@router.get("/artifact-jobs", response_model=list[ArtifactJobResponse])
|
||||
def artifact_jobs(
|
||||
service: Service, limit: Annotated[int, Query(ge=1, le=100)] = 100
|
||||
) -> list[ArtifactJobResponse]:
|
||||
return service.jobs()[:limit]
|
||||
|
||||
|
||||
@router.get("/artifact-jobs/{job_id}", response_model=ArtifactJobResponse)
|
||||
def artifact_job(job_id: uuid.UUID, service: Service) -> ArtifactJobResponse:
|
||||
return service.job(job_id)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/artifact-jobs/{job_id}/cancel",
|
||||
response_model=ArtifactJobResponse,
|
||||
dependencies=[Depends(require_operator)],
|
||||
)
|
||||
def cancel_artifact_job(job_id: uuid.UUID, service: Service) -> ArtifactJobResponse:
|
||||
return service.cancel(job_id)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/artifact-jobs/{job_id}/retry",
|
||||
response_model=ArtifactJobResponse,
|
||||
dependencies=[Depends(require_operator)],
|
||||
)
|
||||
def retry_artifact_job(job_id: uuid.UUID, service: Service) -> ArtifactJobResponse:
|
||||
return service.retry(job_id)
|
||||
@@ -0,0 +1,371 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
import uuid
|
||||
from collections import defaultdict, deque
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Depends, Header, HTTPException, Request, Response, status
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from modelforge_api.api.authorization import Admin
|
||||
from modelforge_api.db import get_session
|
||||
from modelforge_api.domain.acquisition import (
|
||||
AgentArtifactJobLease,
|
||||
AgentJobComplete,
|
||||
AgentJobControl,
|
||||
AgentJobFailure,
|
||||
AgentJobProgress,
|
||||
ArtifactJobResponse,
|
||||
)
|
||||
from modelforge_api.domain.agent_protocol import (
|
||||
EnrollmentRequest,
|
||||
EnrollmentResponse,
|
||||
EnrollmentTokenCreate,
|
||||
EnrollmentTokenCreated,
|
||||
EnrollmentTokenSummary,
|
||||
HeartbeatRequest,
|
||||
InventoryReport,
|
||||
NodeCredentialCreated,
|
||||
NodeManagementUpdate,
|
||||
ObservationAck,
|
||||
TelemetryReport,
|
||||
)
|
||||
from modelforge_api.domain.node_decommission import (
|
||||
NodeDecommissionExecute,
|
||||
NodeDecommissionPreview,
|
||||
NodeDecommissionResult,
|
||||
)
|
||||
from modelforge_api.domain.runtime import (
|
||||
AgentRuntimeProbeComplete,
|
||||
AgentRuntimeProbeControl,
|
||||
AgentRuntimeProbeFailure,
|
||||
AgentRuntimeProbeLease,
|
||||
AgentRuntimeProbeProgress,
|
||||
RuntimeProbeResponse,
|
||||
)
|
||||
from modelforge_api.persistence.models import ComputeNode, NodeCredential
|
||||
from modelforge_api.providers.huggingface import OfficialHuggingFaceProvider
|
||||
from modelforge_api.services.acquisition import AcquisitionService
|
||||
from modelforge_api.services.node_agent import NodeAgentService, NodeAuthenticationEvidence
|
||||
from modelforge_api.services.node_decommission import NodeDecommissionService
|
||||
from modelforge_api.services.runtime import RuntimeService
|
||||
from modelforge_api.settings import Settings, get_settings
|
||||
|
||||
router = APIRouter(tags=["node-agent"])
|
||||
|
||||
|
||||
class ActionResponse(BaseModel):
|
||||
status: str = "ok"
|
||||
|
||||
|
||||
class AttemptLimiter:
|
||||
def __init__(self, limit: int = 10, window_seconds: int = 60) -> None:
|
||||
self.limit = limit
|
||||
self.window_seconds = window_seconds
|
||||
self.attempts: dict[str, deque[float]] = defaultdict(deque)
|
||||
|
||||
def check(self, key: str) -> None:
|
||||
now = time.monotonic()
|
||||
bucket = self.attempts[key]
|
||||
while bucket and bucket[0] <= now - self.window_seconds:
|
||||
bucket.popleft()
|
||||
if len(bucket) >= self.limit:
|
||||
raise HTTPException(status_code=429, detail="enrollment rate limit exceeded")
|
||||
bucket.append(now)
|
||||
|
||||
|
||||
enrollment_limiter = AttemptLimiter()
|
||||
|
||||
|
||||
def get_agent_service(
|
||||
session: Annotated[Session, Depends(get_session)],
|
||||
settings: Annotated[Settings, Depends(get_settings)],
|
||||
) -> NodeAgentService:
|
||||
return NodeAgentService(session, settings)
|
||||
|
||||
|
||||
Service = Annotated[NodeAgentService, Depends(get_agent_service)]
|
||||
|
||||
|
||||
def get_decommission_service(
|
||||
session: Annotated[Session, Depends(get_session)],
|
||||
) -> NodeDecommissionService:
|
||||
return NodeDecommissionService(session)
|
||||
|
||||
|
||||
DecommissionService = Annotated[NodeDecommissionService, Depends(get_decommission_service)]
|
||||
|
||||
|
||||
@router.post(
|
||||
"/api/v1/admin/node-enrollments",
|
||||
response_model=EnrollmentTokenCreated,
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
)
|
||||
def create_enrollment(
|
||||
request: EnrollmentTokenCreate, service: Service, _admin: Admin, response: Response
|
||||
) -> EnrollmentTokenCreated:
|
||||
response.headers["Cache-Control"] = "no-store"
|
||||
return service.create_enrollment(request)
|
||||
|
||||
|
||||
@router.get("/api/v1/admin/node-enrollments", response_model=list[EnrollmentTokenSummary])
|
||||
def list_enrollments(service: Service, _admin: Admin) -> list[EnrollmentTokenSummary]:
|
||||
return [
|
||||
EnrollmentTokenSummary(
|
||||
id=str(item.id),
|
||||
created_at=item.created_at,
|
||||
expires_at=item.expires_at,
|
||||
used_at=item.used_at,
|
||||
revoked_at=item.revoked_at,
|
||||
)
|
||||
for item in service.repository.enrollments()
|
||||
]
|
||||
|
||||
|
||||
@router.delete("/api/v1/admin/node-enrollments/{enrollment_id}", response_model=ActionResponse)
|
||||
def revoke_enrollment(enrollment_id: uuid.UUID, service: Service, _admin: Admin) -> ActionResponse:
|
||||
service.revoke_enrollment(enrollment_id)
|
||||
return ActionResponse()
|
||||
|
||||
|
||||
@router.patch("/api/v1/admin/hardware/nodes/{node_id}", response_model=ActionResponse)
|
||||
def update_node(
|
||||
node_id: uuid.UUID,
|
||||
request: NodeManagementUpdate,
|
||||
service: Service,
|
||||
_admin: Admin,
|
||||
) -> ActionResponse:
|
||||
service.update_node(node_id, request)
|
||||
return ActionResponse()
|
||||
|
||||
|
||||
@router.post(
|
||||
"/api/v1/admin/hardware/nodes/{node_id}/decommission/preview",
|
||||
response_model=NodeDecommissionPreview,
|
||||
)
|
||||
def preview_node_decommission(
|
||||
node_id: uuid.UUID, service: DecommissionService, _admin: Admin
|
||||
) -> NodeDecommissionPreview:
|
||||
return service.preview(node_id)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/api/v1/admin/hardware/nodes/{node_id}/decommission",
|
||||
response_model=NodeDecommissionResult,
|
||||
)
|
||||
def execute_node_decommission(
|
||||
node_id: uuid.UUID,
|
||||
request: NodeDecommissionExecute,
|
||||
service: DecommissionService,
|
||||
_admin: Admin,
|
||||
) -> NodeDecommissionResult:
|
||||
return service.execute(node_id, request)
|
||||
|
||||
|
||||
@router.delete("/api/v1/admin/hardware/nodes/{node_id}/credential", response_model=ActionResponse)
|
||||
def revoke_credential(node_id: uuid.UUID, service: Service, _admin: Admin) -> ActionResponse:
|
||||
service.revoke_credential(node_id)
|
||||
return ActionResponse()
|
||||
|
||||
|
||||
@router.post(
|
||||
"/api/v1/admin/hardware/nodes/{node_id}/credential/rotate",
|
||||
response_model=NodeCredentialCreated,
|
||||
)
|
||||
def rotate_credential(
|
||||
node_id: uuid.UUID, service: Service, _admin: Admin, response: Response
|
||||
) -> NodeCredentialCreated:
|
||||
response.headers["Cache-Control"] = "no-store"
|
||||
return service.rotate_credential(node_id)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/api/v1/agent/enroll",
|
||||
response_model=EnrollmentResponse,
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
)
|
||||
def enroll(
|
||||
request: EnrollmentRequest, http_request: Request, service: Service, response: Response
|
||||
) -> EnrollmentResponse:
|
||||
enrollment_limiter.check(http_request.client.host if http_request.client else "unknown")
|
||||
response.headers["Cache-Control"] = "no-store"
|
||||
return service.enroll(request)
|
||||
|
||||
|
||||
def authenticated_node(
|
||||
request: Request,
|
||||
service: Service,
|
||||
authorization: Annotated[str | None, Header(alias="Authorization")] = None,
|
||||
) -> tuple[NodeCredential, ComputeNode]:
|
||||
evidence = getattr(request.state, "node_authentication", None)
|
||||
if isinstance(evidence, NodeAuthenticationEvidence):
|
||||
return service.reuse_authentication(evidence, authorization)
|
||||
return service.authenticate(authorization)
|
||||
|
||||
|
||||
NodeIdentity = tuple[NodeCredential, ComputeNode]
|
||||
AgentIdentity = Annotated[NodeIdentity, Depends(authenticated_node)]
|
||||
|
||||
|
||||
def get_agent_acquisition_service(
|
||||
session: Annotated[Session, Depends(get_session)],
|
||||
settings: Annotated[Settings, Depends(get_settings)],
|
||||
) -> AcquisitionService:
|
||||
token = settings.hf_token.get_secret_value() if settings.hf_token else None
|
||||
return AcquisitionService(
|
||||
session,
|
||||
settings,
|
||||
OfficialHuggingFaceProvider(token=token, timeout=settings.hf_timeout_seconds),
|
||||
actor_type="node_agent",
|
||||
actor_id="authenticated-node",
|
||||
)
|
||||
|
||||
|
||||
AgentAcquisition = Annotated[AcquisitionService, Depends(get_agent_acquisition_service)]
|
||||
|
||||
|
||||
def get_agent_runtime_service(
|
||||
session: Annotated[Session, Depends(get_session)],
|
||||
settings: Annotated[Settings, Depends(get_settings)],
|
||||
) -> RuntimeService:
|
||||
return RuntimeService(
|
||||
session,
|
||||
settings,
|
||||
actor_type="runtime_worker",
|
||||
actor_id="authenticated-node",
|
||||
)
|
||||
|
||||
|
||||
AgentRuntime = Annotated[RuntimeService, Depends(get_agent_runtime_service)]
|
||||
|
||||
|
||||
@router.post("/api/v1/agent/heartbeat", response_model=ObservationAck)
|
||||
def heartbeat(
|
||||
request: HeartbeatRequest, service: Service, identity: AgentIdentity
|
||||
) -> ObservationAck:
|
||||
_credential, node = identity
|
||||
return service.heartbeat(node, request)
|
||||
|
||||
|
||||
@router.put("/api/v1/agent/inventory", response_model=ObservationAck)
|
||||
def publish_inventory(
|
||||
request: InventoryReport, service: Service, identity: AgentIdentity
|
||||
) -> ObservationAck:
|
||||
_credential, node = identity
|
||||
return service.publish_inventory(node, request)
|
||||
|
||||
|
||||
@router.put("/api/v1/agent/telemetry", response_model=ObservationAck)
|
||||
def publish_telemetry(
|
||||
request: TelemetryReport, service: Service, identity: AgentIdentity
|
||||
) -> ObservationAck:
|
||||
_credential, node = identity
|
||||
return service.publish_telemetry(node, request)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/api/v1/agent/artifact-jobs/next",
|
||||
response_model=AgentArtifactJobLease | None,
|
||||
)
|
||||
def claim_artifact_job(
|
||||
acquisition: AgentAcquisition, identity: AgentIdentity
|
||||
) -> AgentArtifactJobLease | None:
|
||||
_credential, node = identity
|
||||
return acquisition.claim_next(node)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/api/v1/agent/artifact-jobs/{job_id}/progress",
|
||||
response_model=AgentJobControl,
|
||||
)
|
||||
def artifact_job_progress(
|
||||
job_id: uuid.UUID,
|
||||
request: AgentJobProgress,
|
||||
acquisition: AgentAcquisition,
|
||||
identity: AgentIdentity,
|
||||
) -> AgentJobControl:
|
||||
_credential, node = identity
|
||||
return acquisition.progress(job_id, node, request)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/api/v1/agent/artifact-jobs/{job_id}/complete",
|
||||
response_model=ArtifactJobResponse,
|
||||
)
|
||||
def artifact_job_complete(
|
||||
job_id: uuid.UUID,
|
||||
request: AgentJobComplete,
|
||||
acquisition: AgentAcquisition,
|
||||
identity: AgentIdentity,
|
||||
) -> ArtifactJobResponse:
|
||||
_credential, node = identity
|
||||
return acquisition.complete(job_id, node, request)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/api/v1/agent/artifact-jobs/{job_id}/fail",
|
||||
response_model=ArtifactJobResponse,
|
||||
)
|
||||
def artifact_job_fail(
|
||||
job_id: uuid.UUID,
|
||||
request: AgentJobFailure,
|
||||
acquisition: AgentAcquisition,
|
||||
identity: AgentIdentity,
|
||||
) -> ArtifactJobResponse:
|
||||
_credential, node = identity
|
||||
return acquisition.fail(job_id, node, request)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/api/v1/agent/runtime-probes/next",
|
||||
response_model=AgentRuntimeProbeLease | None,
|
||||
)
|
||||
def claim_runtime_probe(
|
||||
runtime: AgentRuntime, identity: AgentIdentity
|
||||
) -> AgentRuntimeProbeLease | None:
|
||||
_credential, node = identity
|
||||
return runtime.claim_next(node)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/api/v1/agent/runtime-probes/{probe_id}/progress",
|
||||
response_model=AgentRuntimeProbeControl,
|
||||
)
|
||||
def runtime_probe_progress(
|
||||
probe_id: uuid.UUID,
|
||||
request: AgentRuntimeProbeProgress,
|
||||
runtime: AgentRuntime,
|
||||
identity: AgentIdentity,
|
||||
) -> AgentRuntimeProbeControl:
|
||||
_credential, node = identity
|
||||
return runtime.progress(probe_id, node, request)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/api/v1/agent/runtime-probes/{probe_id}/complete",
|
||||
response_model=RuntimeProbeResponse,
|
||||
)
|
||||
def runtime_probe_complete(
|
||||
probe_id: uuid.UUID,
|
||||
request: AgentRuntimeProbeComplete,
|
||||
runtime: AgentRuntime,
|
||||
identity: AgentIdentity,
|
||||
) -> RuntimeProbeResponse:
|
||||
_credential, node = identity
|
||||
return runtime.complete(probe_id, node, request)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/api/v1/agent/runtime-probes/{probe_id}/fail",
|
||||
response_model=RuntimeProbeResponse,
|
||||
)
|
||||
def runtime_probe_fail(
|
||||
probe_id: uuid.UUID,
|
||||
request: AgentRuntimeProbeFailure,
|
||||
runtime: AgentRuntime,
|
||||
identity: AgentIdentity,
|
||||
) -> RuntimeProbeResponse:
|
||||
_credential, node = identity
|
||||
return runtime.fail(probe_id, node, request)
|
||||
@@ -0,0 +1,273 @@
|
||||
from fastapi import APIRouter, Depends
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from modelforge_api.api.authorization import require_viewer
|
||||
from modelforge_api.db import get_session
|
||||
from modelforge_api.domain.schemas import (
|
||||
CapabilityContractResponse,
|
||||
CapabilityEstateResponse,
|
||||
InstallationDependencyResponse,
|
||||
ModelInstallationRationaleResponse,
|
||||
ProjectBindingResponse,
|
||||
ProjectResponse,
|
||||
)
|
||||
from modelforge_api.persistence.models import (
|
||||
ArtifactSet,
|
||||
Capability,
|
||||
CapabilityContract,
|
||||
CapabilityDeployment,
|
||||
CapabilityEvaluationRun,
|
||||
CapabilityResourceEnvelope,
|
||||
Model,
|
||||
ModelRevision,
|
||||
Project,
|
||||
ProjectBinding,
|
||||
ProjectFitEvidence,
|
||||
ResidencyAllocation,
|
||||
RuntimeProfile,
|
||||
ServiceClient,
|
||||
)
|
||||
from modelforge_api.services.manifest_registry import ManifestRegistry, get_manifest_registry
|
||||
|
||||
router = APIRouter(
|
||||
prefix="/api/v1", tags=["registry"], dependencies=[Depends(require_viewer)]
|
||||
)
|
||||
|
||||
|
||||
@router.get("/capabilities", response_model=list[CapabilityContractResponse])
|
||||
def list_capabilities(
|
||||
registry: ManifestRegistry = Depends(get_manifest_registry),
|
||||
session: Session = Depends(get_session),
|
||||
) -> list[CapabilityContractResponse]:
|
||||
responses = []
|
||||
for contract in registry.capabilities():
|
||||
stable = session.execute(
|
||||
select(CapabilityDeployment)
|
||||
.join(
|
||||
CapabilityContract,
|
||||
CapabilityContract.id == CapabilityDeployment.capability_contract_id,
|
||||
)
|
||||
.join(Capability, Capability.id == CapabilityContract.capability_id)
|
||||
.where(
|
||||
Capability.key == contract.capability,
|
||||
CapabilityContract.version == contract.version,
|
||||
CapabilityDeployment.status == "stable",
|
||||
CapabilityDeployment.production.is_(True),
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
responses.append(
|
||||
CapabilityContractResponse(
|
||||
key=contract.capability,
|
||||
version=contract.version,
|
||||
description=contract.description,
|
||||
contract=contract,
|
||||
stable_deployment=(
|
||||
{
|
||||
"id": str(stable.id),
|
||||
"status": stable.status,
|
||||
"health": stable.health_status,
|
||||
"production": stable.production,
|
||||
}
|
||||
if stable
|
||||
else None
|
||||
),
|
||||
)
|
||||
)
|
||||
return responses
|
||||
|
||||
|
||||
@router.get("/capability-estate", response_model=list[CapabilityEstateResponse])
|
||||
def capability_estate(
|
||||
registry: ManifestRegistry = Depends(get_manifest_registry),
|
||||
session: Session = Depends(get_session),
|
||||
) -> list[CapabilityEstateResponse]:
|
||||
responses: list[CapabilityEstateResponse] = []
|
||||
for manifest in registry.capabilities():
|
||||
contract = session.scalar(
|
||||
select(CapabilityContract)
|
||||
.join(Capability, Capability.id == CapabilityContract.capability_id)
|
||||
.where(Capability.key == manifest.capability, CapabilityContract.version == manifest.version)
|
||||
)
|
||||
deployment = None
|
||||
model = None
|
||||
revision = None
|
||||
profile = None
|
||||
envelope = None
|
||||
latest_evaluation = None
|
||||
if contract:
|
||||
deployment = session.scalar(
|
||||
select(CapabilityDeployment)
|
||||
.where(CapabilityDeployment.capability_contract_id == contract.id)
|
||||
.order_by(
|
||||
CapabilityDeployment.production.desc(),
|
||||
CapabilityDeployment.created_at.desc(),
|
||||
)
|
||||
.limit(1)
|
||||
)
|
||||
if deployment:
|
||||
artifact_set = session.get(ArtifactSet, deployment.artifact_set_id)
|
||||
revision = session.get(ModelRevision, artifact_set.revision_id) if artifact_set else None
|
||||
model = session.get(Model, revision.model_id) if revision else None
|
||||
profile = session.get(RuntimeProfile, deployment.runtime_profile_id)
|
||||
envelope = session.scalar(
|
||||
select(CapabilityResourceEnvelope).where(
|
||||
CapabilityResourceEnvelope.capability_deployment_id == deployment.id
|
||||
)
|
||||
)
|
||||
latest_evaluation = session.scalar(
|
||||
select(CapabilityEvaluationRun)
|
||||
.where(CapabilityEvaluationRun.capability_deployment_id == deployment.id)
|
||||
.order_by(CapabilityEvaluationRun.created_at.desc())
|
||||
.limit(1)
|
||||
)
|
||||
operational_state = (
|
||||
deployment.status
|
||||
if deployment
|
||||
else ("blocked" if manifest.estate.stability == "blocked" else "not_deployed")
|
||||
)
|
||||
responses.append(
|
||||
CapabilityEstateResponse(
|
||||
capability=manifest.capability,
|
||||
version=manifest.version,
|
||||
category=manifest.estate.category,
|
||||
purpose=manifest.estate.purpose,
|
||||
declared_stability=manifest.estate.stability,
|
||||
operational_state=operational_state,
|
||||
current_deployment_id=deployment.id if deployment else None,
|
||||
model=model.display_name if model else None,
|
||||
revision=revision.resolved_commit_sha if revision else None,
|
||||
runtime=profile.runtime_type if profile else None,
|
||||
node=str(deployment.compute_node_id) if deployment else None,
|
||||
resource_class=manifest.estate.resource_class,
|
||||
measured_required_vram_bytes=envelope.required_vram_bytes if envelope else None,
|
||||
consumers=manifest.estate.consumers,
|
||||
privacy_class=manifest.privacy.classification,
|
||||
evaluation_type=manifest.estate.evaluation_type,
|
||||
evaluation_state=latest_evaluation.status if latest_evaluation else "not_evaluated",
|
||||
)
|
||||
)
|
||||
return responses
|
||||
|
||||
|
||||
@router.get(
|
||||
"/models/installation-rationale",
|
||||
response_model=list[ModelInstallationRationaleResponse],
|
||||
)
|
||||
def model_installation_rationale(
|
||||
session: Session = Depends(get_session),
|
||||
) -> list[ModelInstallationRationaleResponse]:
|
||||
responses: list[ModelInstallationRationaleResponse] = []
|
||||
for model in session.scalars(select(Model).order_by(Model.display_name)).all():
|
||||
revisions = session.scalars(
|
||||
select(ModelRevision).where(ModelRevision.model_id == model.id)
|
||||
).all()
|
||||
revision_ids = [item.id for item in revisions]
|
||||
sets = (
|
||||
session.scalars(select(ArtifactSet).where(ArtifactSet.revision_id.in_(revision_ids))).all()
|
||||
if revision_ids
|
||||
else []
|
||||
)
|
||||
installed_sets = [item for item in sets if item.availability == "local"]
|
||||
dependencies: list[InstallationDependencyResponse] = []
|
||||
for artifact_set in installed_sets:
|
||||
deployments = session.scalars(
|
||||
select(CapabilityDeployment).where(
|
||||
CapabilityDeployment.artifact_set_id == artifact_set.id
|
||||
)
|
||||
).all()
|
||||
for deployment in deployments:
|
||||
contract = session.get(CapabilityContract, deployment.capability_contract_id)
|
||||
capability = session.get(Capability, contract.capability_id) if contract else None
|
||||
if not contract or not capability:
|
||||
continue
|
||||
projects = session.scalars(
|
||||
select(Project)
|
||||
.join(ProjectBinding, ProjectBinding.project_id == Project.id)
|
||||
.where(
|
||||
ProjectBinding.capability_contract_id == contract.id,
|
||||
ProjectBinding.deprecated_at.is_(None),
|
||||
)
|
||||
.distinct()
|
||||
).all()
|
||||
active_projects = session.scalars(
|
||||
select(Project)
|
||||
.join(ProjectBinding, ProjectBinding.project_id == Project.id)
|
||||
.join(ServiceClient, ServiceClient.project_binding_id == ProjectBinding.id)
|
||||
.where(
|
||||
ProjectBinding.capability_contract_id == contract.id,
|
||||
ServiceClient.status == "active",
|
||||
)
|
||||
.distinct()
|
||||
).all()
|
||||
project_fit_ids = session.scalars(
|
||||
select(ProjectFitEvidence.id)
|
||||
.join(
|
||||
ProjectBinding,
|
||||
ProjectBinding.id == ProjectFitEvidence.project_binding_id,
|
||||
)
|
||||
.where(ProjectBinding.capability_contract_id == contract.id)
|
||||
).all()
|
||||
evaluations = session.scalars(
|
||||
select(CapabilityEvaluationRun).where(
|
||||
CapabilityEvaluationRun.capability_deployment_id == deployment.id
|
||||
)
|
||||
).all()
|
||||
residency = session.scalar(
|
||||
select(ResidencyAllocation).where(
|
||||
ResidencyAllocation.capability_deployment_id == deployment.id
|
||||
)
|
||||
)
|
||||
dependencies.append(
|
||||
InstallationDependencyResponse(
|
||||
capability=capability.key,
|
||||
version=contract.version,
|
||||
deployment_id=deployment.id,
|
||||
channel=deployment.channel,
|
||||
production=deployment.production,
|
||||
project_consumers=[item.key for item in projects],
|
||||
active_project_consumers=[item.key for item in active_projects],
|
||||
project_fit_evidence_ids=list(project_fit_ids),
|
||||
evaluation_run_ids=[item.id for item in evaluations],
|
||||
last_used_at=residency.last_used_at if residency else None,
|
||||
)
|
||||
)
|
||||
blockers = []
|
||||
if dependencies:
|
||||
blockers.append("capability_deployment_dependency")
|
||||
if any(item.production for item in dependencies):
|
||||
blockers.append("production_dependency")
|
||||
responses.append(
|
||||
ModelInstallationRationaleResponse(
|
||||
model_id=model.id,
|
||||
display_name=model.display_name,
|
||||
upstream_source=model.upstream_source,
|
||||
installed=bool(installed_sets),
|
||||
installed_bytes=sum(item.total_size_bytes for item in installed_sets),
|
||||
dependencies=dependencies,
|
||||
can_delete=bool(installed_sets) and not blockers,
|
||||
deletion_blockers=blockers,
|
||||
)
|
||||
)
|
||||
return responses
|
||||
|
||||
|
||||
@router.get("/projects", response_model=list[ProjectResponse])
|
||||
def list_projects(
|
||||
registry: ManifestRegistry = Depends(get_manifest_registry),
|
||||
) -> list[ProjectResponse]:
|
||||
return [
|
||||
ProjectResponse(
|
||||
**project.project.model_dump(),
|
||||
notes=project.notes,
|
||||
bindings=[
|
||||
ProjectBindingResponse(
|
||||
capability=capability,
|
||||
contract_version=binding.contract_version,
|
||||
binding=binding,
|
||||
)
|
||||
for capability, binding in project.bindings.items()
|
||||
],
|
||||
)
|
||||
for project in registry.projects()
|
||||
]
|
||||
@@ -0,0 +1,387 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Depends, status
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from modelforge_api.api.authorization import Admin, require_viewer
|
||||
from modelforge_api.db import get_session
|
||||
from modelforge_api.domain.capability_evaluation import (
|
||||
CapabilityAdvisorResponse,
|
||||
CapabilityEvaluationRunCreate,
|
||||
CapabilityEvaluationRunResponse,
|
||||
CapabilityEvaluationSuiteCreate,
|
||||
CapabilityEvaluationSuiteResponse,
|
||||
)
|
||||
from modelforge_api.domain.evaluation import (
|
||||
AdvisorPolicyResponse,
|
||||
AdvisorPolicyUpdate,
|
||||
AdvisorRecommendationCreate,
|
||||
AdvisorRecommendationDismiss,
|
||||
AdvisorRecommendationResponse,
|
||||
DiscoveryCandidateAssessmentCreate,
|
||||
DiscoveryCandidateAssessmentResponse,
|
||||
EmbeddingMigrationCreate,
|
||||
EmbeddingMigrationResponse,
|
||||
EvaluationCaseDefinitionResponse,
|
||||
EvaluationCaseResultResponse,
|
||||
EvaluationComparisonCreate,
|
||||
EvaluationComparisonResponse,
|
||||
EvaluationRunComplete,
|
||||
EvaluationRunCreate,
|
||||
EvaluationRunResponse,
|
||||
EvaluationSuiteCreate,
|
||||
EvaluationSuiteResponse,
|
||||
MigrationUpdate,
|
||||
ModelComparisonCreate,
|
||||
ModelComparisonResponse,
|
||||
RerankingCaseResultResponse,
|
||||
RerankingRunComplete,
|
||||
RerankingRunCreate,
|
||||
RerankingRunResponse,
|
||||
RetrievalCandidatePoolCreate,
|
||||
RetrievalCandidatePoolResponse,
|
||||
RetrievalPipelineIdentityCreate,
|
||||
RetrievalPipelineIdentityResponse,
|
||||
)
|
||||
from modelforge_api.services.capability_evaluation import CapabilityEvaluationService
|
||||
from modelforge_api.services.evaluation import EvaluationService
|
||||
|
||||
router = APIRouter(
|
||||
prefix="/api/v1",
|
||||
tags=["project-evaluation"],
|
||||
dependencies=[Depends(require_viewer)],
|
||||
)
|
||||
|
||||
|
||||
def service(session: Annotated[Session, Depends(get_session)]) -> EvaluationService:
|
||||
return EvaluationService(session)
|
||||
|
||||
|
||||
Service = Annotated[EvaluationService, Depends(service)]
|
||||
|
||||
|
||||
def capability_service(
|
||||
session: Annotated[Session, Depends(get_session)],
|
||||
) -> CapabilityEvaluationService:
|
||||
return CapabilityEvaluationService(session)
|
||||
|
||||
|
||||
CapabilityService = Annotated[CapabilityEvaluationService, Depends(capability_service)]
|
||||
|
||||
|
||||
@router.get("/capability-advisor", response_model=list[CapabilityAdvisorResponse])
|
||||
def capability_advisor(svc: CapabilityService) -> list[CapabilityAdvisorResponse]:
|
||||
return svc.advisor()
|
||||
|
||||
|
||||
@router.get(
|
||||
"/capability-evaluation-suites",
|
||||
response_model=list[CapabilityEvaluationSuiteResponse],
|
||||
)
|
||||
def capability_suites(svc: CapabilityService) -> list[CapabilityEvaluationSuiteResponse]:
|
||||
return svc.suites()
|
||||
|
||||
|
||||
@router.post(
|
||||
"/capability-evaluation-suites",
|
||||
response_model=CapabilityEvaluationSuiteResponse,
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
)
|
||||
def create_capability_suite(
|
||||
request: CapabilityEvaluationSuiteCreate, svc: CapabilityService, _admin: Admin
|
||||
) -> CapabilityEvaluationSuiteResponse:
|
||||
return svc.create_suite(request)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/capability-evaluation-runs",
|
||||
response_model=list[CapabilityEvaluationRunResponse],
|
||||
)
|
||||
def capability_runs(svc: CapabilityService) -> list[CapabilityEvaluationRunResponse]:
|
||||
return svc.runs()
|
||||
|
||||
|
||||
@router.post(
|
||||
"/capability-evaluation-runs",
|
||||
response_model=CapabilityEvaluationRunResponse,
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
)
|
||||
def create_capability_run(
|
||||
request: CapabilityEvaluationRunCreate, svc: CapabilityService, _admin: Admin
|
||||
) -> CapabilityEvaluationRunResponse:
|
||||
return svc.create_run(request)
|
||||
|
||||
|
||||
@router.get("/evaluation-suites", response_model=list[EvaluationSuiteResponse])
|
||||
def suites(svc: Service) -> list[EvaluationSuiteResponse]:
|
||||
return svc.suites()
|
||||
|
||||
|
||||
@router.post(
|
||||
"/evaluation-suites",
|
||||
response_model=EvaluationSuiteResponse,
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
)
|
||||
def create_suite(
|
||||
request: EvaluationSuiteCreate, svc: Service, _admin: Admin
|
||||
) -> EvaluationSuiteResponse:
|
||||
return svc.create_suite(request)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/evaluation-suites/{suite_id}/revisions/{revision_id}/cases",
|
||||
response_model=list[EvaluationCaseDefinitionResponse],
|
||||
)
|
||||
def suite_cases(
|
||||
suite_id: uuid.UUID, revision_id: uuid.UUID, svc: Service
|
||||
) -> list[EvaluationCaseDefinitionResponse]:
|
||||
return svc.suite_cases(suite_id, revision_id)
|
||||
|
||||
|
||||
@router.get("/evaluation-runs", response_model=list[EvaluationRunResponse])
|
||||
def runs(svc: Service) -> list[EvaluationRunResponse]:
|
||||
return svc.runs()
|
||||
|
||||
|
||||
@router.post(
|
||||
"/evaluation-runs", response_model=EvaluationRunResponse, status_code=status.HTTP_201_CREATED
|
||||
)
|
||||
def create_run(request: EvaluationRunCreate, svc: Service, _admin: Admin) -> EvaluationRunResponse:
|
||||
return svc.create_run(request)
|
||||
|
||||
|
||||
@router.get("/evaluation-runs/{run_id}", response_model=EvaluationRunResponse)
|
||||
def run(run_id: uuid.UUID, svc: Service) -> EvaluationRunResponse:
|
||||
return svc.run(run_id)
|
||||
|
||||
|
||||
@router.post("/evaluation-runs/{run_id}/complete", response_model=EvaluationRunResponse)
|
||||
def complete_run(
|
||||
run_id: uuid.UUID, request: EvaluationRunComplete, svc: Service, _admin: Admin
|
||||
) -> EvaluationRunResponse:
|
||||
return svc.complete_run(run_id, request)
|
||||
|
||||
|
||||
@router.get("/evaluation-runs/{run_id}/cases", response_model=list[EvaluationCaseResultResponse])
|
||||
def case_results(run_id: uuid.UUID, svc: Service) -> list[EvaluationCaseResultResponse]:
|
||||
return svc.case_results(run_id)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/retrieval-candidate-pools",
|
||||
response_model=RetrievalCandidatePoolResponse,
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
)
|
||||
def create_candidate_pool(
|
||||
request: RetrievalCandidatePoolCreate, svc: Service, _admin: Admin
|
||||
) -> RetrievalCandidatePoolResponse:
|
||||
return svc.create_candidate_pool(request)
|
||||
|
||||
|
||||
@router.get("/retrieval-candidate-pools", response_model=list[RetrievalCandidatePoolResponse])
|
||||
def candidate_pools(svc: Service) -> list[RetrievalCandidatePoolResponse]:
|
||||
return svc.candidate_pools()
|
||||
|
||||
|
||||
@router.post(
|
||||
"/retrieval-pipeline-identities",
|
||||
response_model=RetrievalPipelineIdentityResponse,
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
)
|
||||
def create_pipeline_identity(
|
||||
request: RetrievalPipelineIdentityCreate, svc: Service, _admin: Admin
|
||||
) -> RetrievalPipelineIdentityResponse:
|
||||
return svc.create_pipeline_identity(request)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/retrieval-pipeline-identities", response_model=list[RetrievalPipelineIdentityResponse]
|
||||
)
|
||||
def pipeline_identities(svc: Service) -> list[RetrievalPipelineIdentityResponse]:
|
||||
return svc.pipeline_identities()
|
||||
|
||||
|
||||
@router.post(
|
||||
"/reranking-runs",
|
||||
response_model=RerankingRunResponse,
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
)
|
||||
def create_reranking_run(
|
||||
request: RerankingRunCreate, svc: Service, _admin: Admin
|
||||
) -> RerankingRunResponse:
|
||||
return svc.create_reranking_run(request)
|
||||
|
||||
|
||||
@router.get("/reranking-runs", response_model=list[RerankingRunResponse])
|
||||
def reranking_runs(svc: Service) -> list[RerankingRunResponse]:
|
||||
return svc.reranking_runs()
|
||||
|
||||
|
||||
@router.post("/reranking-runs/{run_id}/complete", response_model=RerankingRunResponse)
|
||||
def complete_reranking_run(
|
||||
run_id: uuid.UUID, request: RerankingRunComplete, svc: Service, _admin: Admin
|
||||
) -> RerankingRunResponse:
|
||||
return svc.complete_reranking_run(run_id, request)
|
||||
|
||||
|
||||
@router.get("/reranking-runs/{run_id}/cases", response_model=list[RerankingCaseResultResponse])
|
||||
def reranking_case_results(run_id: uuid.UUID, svc: Service) -> list[RerankingCaseResultResponse]:
|
||||
return svc.reranking_case_results(run_id)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/discovery-assessments",
|
||||
response_model=DiscoveryCandidateAssessmentResponse,
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
)
|
||||
def create_discovery_assessment(
|
||||
request: DiscoveryCandidateAssessmentCreate, svc: Service, _admin: Admin
|
||||
) -> DiscoveryCandidateAssessmentResponse:
|
||||
return svc.create_discovery_assessment(request)
|
||||
|
||||
|
||||
@router.get("/discovery-assessments", response_model=list[DiscoveryCandidateAssessmentResponse])
|
||||
def discovery_assessments(svc: Service) -> list[DiscoveryCandidateAssessmentResponse]:
|
||||
return svc.discovery_assessments()
|
||||
|
||||
|
||||
@router.post(
|
||||
"/evaluation-comparisons",
|
||||
response_model=EvaluationComparisonResponse,
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
)
|
||||
def compare(
|
||||
request: EvaluationComparisonCreate, svc: Service, _admin: Admin
|
||||
) -> EvaluationComparisonResponse:
|
||||
return svc.compare(request)
|
||||
|
||||
|
||||
@router.get("/evaluation-comparisons", response_model=list[EvaluationComparisonResponse])
|
||||
def comparisons(svc: Service) -> list[EvaluationComparisonResponse]:
|
||||
return svc.comparisons()
|
||||
|
||||
|
||||
@router.get("/evaluation-comparisons/{comparison_id}", response_model=EvaluationComparisonResponse)
|
||||
def comparison(comparison_id: uuid.UUID, svc: Service) -> EvaluationComparisonResponse:
|
||||
return svc.comparison(comparison_id)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/model-comparisons",
|
||||
response_model=ModelComparisonResponse,
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
)
|
||||
def create_model_comparison(
|
||||
request: ModelComparisonCreate, svc: Service, _admin: Admin
|
||||
) -> ModelComparisonResponse:
|
||||
return svc.create_model_comparison(request)
|
||||
|
||||
|
||||
@router.get("/model-comparisons", response_model=list[ModelComparisonResponse])
|
||||
def model_comparisons(svc: Service) -> list[ModelComparisonResponse]:
|
||||
return svc.model_comparisons()
|
||||
|
||||
|
||||
@router.get("/model-comparisons/{comparison_id}", response_model=ModelComparisonResponse)
|
||||
def model_comparison(comparison_id: uuid.UUID, svc: Service) -> ModelComparisonResponse:
|
||||
return svc.model_comparison(comparison_id)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/model-comparisons/{comparison_id}/recommendations",
|
||||
response_model=AdvisorRecommendationResponse,
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
)
|
||||
def create_recommendation(
|
||||
comparison_id: uuid.UUID,
|
||||
request: AdvisorRecommendationCreate,
|
||||
svc: Service,
|
||||
_admin: Admin,
|
||||
) -> AdvisorRecommendationResponse:
|
||||
return svc.recommend(comparison_id, request)
|
||||
|
||||
|
||||
@router.get("/recommendations", response_model=list[AdvisorRecommendationResponse])
|
||||
def recommendations(svc: Service) -> list[AdvisorRecommendationResponse]:
|
||||
return svc.recommendations()
|
||||
|
||||
|
||||
@router.get("/recommendations/{recommendation_id}", response_model=AdvisorRecommendationResponse)
|
||||
def recommendation(recommendation_id: uuid.UUID, svc: Service) -> AdvisorRecommendationResponse:
|
||||
return svc.recommendation(recommendation_id)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/recommendations/{recommendation_id}/dismiss",
|
||||
response_model=AdvisorRecommendationResponse,
|
||||
)
|
||||
def dismiss_recommendation(
|
||||
recommendation_id: uuid.UUID,
|
||||
request: AdvisorRecommendationDismiss,
|
||||
svc: Service,
|
||||
_admin: Admin,
|
||||
) -> AdvisorRecommendationResponse:
|
||||
return svc.dismiss_recommendation(recommendation_id, request)
|
||||
|
||||
|
||||
@router.get("/admin/advisor-policies/current", response_model=AdvisorPolicyResponse)
|
||||
def advisor_policy(svc: Service, _admin: Admin) -> AdvisorPolicyResponse:
|
||||
return svc.advisor_policy()
|
||||
|
||||
|
||||
@router.patch("/admin/advisor-policies/current", response_model=AdvisorPolicyResponse)
|
||||
def update_advisor_policy(
|
||||
request: AdvisorPolicyUpdate, svc: Service, _admin: Admin
|
||||
) -> AdvisorPolicyResponse:
|
||||
return svc.update_advisor_policy(request)
|
||||
|
||||
|
||||
@router.get("/embedding-migrations", response_model=list[EmbeddingMigrationResponse])
|
||||
def migrations(svc: Service) -> list[EmbeddingMigrationResponse]:
|
||||
return svc.migrations()
|
||||
|
||||
|
||||
@router.post(
|
||||
"/projects/{project_id}/embedding-migrations",
|
||||
response_model=EmbeddingMigrationResponse,
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
)
|
||||
def create_migration(
|
||||
project_id: uuid.UUID, request: EmbeddingMigrationCreate, svc: Service, _admin: Admin
|
||||
) -> EmbeddingMigrationResponse:
|
||||
return svc.create_migration(project_id, request)
|
||||
|
||||
|
||||
@router.get("/embedding-migrations/{migration_id}", response_model=EmbeddingMigrationResponse)
|
||||
def migration(migration_id: uuid.UUID, svc: Service) -> EmbeddingMigrationResponse:
|
||||
return svc.migration(migration_id)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/embedding-migrations/{migration_id}/state", response_model=EmbeddingMigrationResponse
|
||||
)
|
||||
def update_migration(
|
||||
migration_id: uuid.UUID, request: MigrationUpdate, svc: Service, _admin: Admin
|
||||
) -> EmbeddingMigrationResponse:
|
||||
return svc.update_migration(migration_id, request)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/embedding-migrations/{migration_id}/start", response_model=EmbeddingMigrationResponse
|
||||
)
|
||||
def start_migration(
|
||||
migration_id: uuid.UUID, svc: Service, _admin: Admin
|
||||
) -> EmbeddingMigrationResponse:
|
||||
return svc.start_migration(migration_id)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/embedding-migrations/{migration_id}/cancel", response_model=EmbeddingMigrationResponse
|
||||
)
|
||||
def cancel_migration(
|
||||
migration_id: uuid.UUID, svc: Service, _admin: Admin
|
||||
) -> EmbeddingMigrationResponse:
|
||||
return svc.cancel_migration(migration_id)
|
||||
@@ -0,0 +1,82 @@
|
||||
import uuid
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from modelforge_api.api.authorization import require_operator, require_viewer
|
||||
from modelforge_api.db import get_session
|
||||
from modelforge_api.domain.hardware import AcceleratorState, HardwareState, NodeState
|
||||
from modelforge_api.services.hardware_factory import build_hardware_service
|
||||
from modelforge_api.services.hardware_inventory import HardwareInventoryService, HardwareRefreshBusy
|
||||
from modelforge_api.settings import Settings, get_settings
|
||||
|
||||
router = APIRouter(
|
||||
prefix="/api/v1/hardware", tags=["hardware"], dependencies=[Depends(require_viewer)]
|
||||
)
|
||||
|
||||
|
||||
def get_hardware_service(
|
||||
session: Annotated[Session, Depends(get_session)],
|
||||
settings: Annotated[Settings, Depends(get_settings)],
|
||||
) -> HardwareInventoryService:
|
||||
return build_hardware_service(session, settings)
|
||||
|
||||
|
||||
Service = Annotated[HardwareInventoryService, Depends(get_hardware_service)]
|
||||
|
||||
|
||||
@router.get("", response_model=HardwareState)
|
||||
def hardware_overview(service: Service) -> HardwareState:
|
||||
return service.state()
|
||||
|
||||
|
||||
@router.get("/nodes", response_model=list[NodeState])
|
||||
def list_nodes(service: Service) -> list[NodeState]:
|
||||
return service.state().nodes
|
||||
|
||||
|
||||
@router.get("/nodes/{node_id}", response_model=NodeState)
|
||||
def get_node(node_id: uuid.UUID, service: Service) -> NodeState:
|
||||
node = next((item for item in service.state().nodes if item.id == str(node_id)), None)
|
||||
if node is None:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="compute node not found")
|
||||
return node
|
||||
|
||||
|
||||
@router.get("/accelerators", response_model=list[AcceleratorState])
|
||||
def list_accelerators(service: Service) -> list[AcceleratorState]:
|
||||
return [accelerator for node in service.state().nodes for accelerator in node.accelerators]
|
||||
|
||||
|
||||
@router.get("/accelerators/{accelerator_id}", response_model=AcceleratorState)
|
||||
def get_accelerator(accelerator_id: uuid.UUID, service: Service) -> AcceleratorState:
|
||||
accelerator = next(
|
||||
(
|
||||
item
|
||||
for node in service.state().nodes
|
||||
for item in node.accelerators
|
||||
if item.id == str(accelerator_id)
|
||||
),
|
||||
None,
|
||||
)
|
||||
if accelerator is None:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="accelerator not found")
|
||||
return accelerator
|
||||
|
||||
|
||||
@router.post(
|
||||
"/refresh",
|
||||
response_model=HardwareState,
|
||||
dependencies=[Depends(require_operator)],
|
||||
)
|
||||
def refresh_hardware(service: Service) -> HardwareState:
|
||||
try:
|
||||
return service.refresh()
|
||||
except HardwareRefreshBusy as exc:
|
||||
raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(exc)) from exc
|
||||
except Exception as exc:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail=f"hardware inventory failed: {type(exc).__name__}",
|
||||
) from exc
|
||||
@@ -0,0 +1,109 @@
|
||||
from fastapi import APIRouter, Depends
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from modelforge_api import __version__
|
||||
from modelforge_api.api.authorization import require_viewer
|
||||
from modelforge_api.db import get_session
|
||||
from modelforge_api.domain.enums import HealthStatus
|
||||
from modelforge_api.domain.release import (
|
||||
CURRENT_AGENT_PROTOCOL_VERSION,
|
||||
MINIMUM_POSTGRES_MAJOR,
|
||||
MINIMUM_UPGRADE_SOURCE,
|
||||
PRODUCT_NAME,
|
||||
RELEASE_CHANNEL,
|
||||
SUPPORTED_AGENT_PROTOCOL_VERSIONS,
|
||||
SUPPORTED_SCHEMA_REVISIONS,
|
||||
TARGET_SCHEMA_REVISION,
|
||||
build_identity,
|
||||
)
|
||||
from modelforge_api.domain.schemas import (
|
||||
HealthResponse,
|
||||
ReadinessResponse,
|
||||
ReleaseCompatibility,
|
||||
ReleaseInfo,
|
||||
SystemMetadata,
|
||||
)
|
||||
from modelforge_api.persistence.models import CapabilityDeployment
|
||||
from modelforge_api.services.manifest_registry import ManifestRegistry, get_manifest_registry
|
||||
from modelforge_api.settings import Settings, get_settings
|
||||
|
||||
router = APIRouter(prefix="/api/v1", tags=["system"])
|
||||
|
||||
|
||||
@router.get("/health/live", response_model=HealthResponse)
|
||||
def liveness() -> HealthResponse:
|
||||
return HealthResponse(version=__version__)
|
||||
|
||||
|
||||
@router.get("/health/ready", response_model=ReadinessResponse)
|
||||
def readiness(registry: ManifestRegistry = Depends(get_manifest_registry)) -> ReadinessResponse:
|
||||
registry.capabilities()
|
||||
registry.projects()
|
||||
registry.candidates()
|
||||
registry.benchmarks()
|
||||
registry.policies()
|
||||
return ReadinessResponse(
|
||||
status=HealthStatus.HEALTHY,
|
||||
checks={"manifests": HealthStatus.HEALTHY},
|
||||
version=__version__,
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/system",
|
||||
response_model=SystemMetadata,
|
||||
dependencies=[Depends(require_viewer)],
|
||||
)
|
||||
def system_metadata(
|
||||
settings: Settings = Depends(get_settings),
|
||||
session: Session = Depends(get_session),
|
||||
) -> SystemMetadata:
|
||||
production_count = int(
|
||||
session.scalar(
|
||||
select(func.count(CapabilityDeployment.id)).where(
|
||||
CapabilityDeployment.status == "stable",
|
||||
CapabilityDeployment.production.is_(True),
|
||||
)
|
||||
)
|
||||
or 0
|
||||
)
|
||||
return SystemMetadata(
|
||||
name=PRODUCT_NAME,
|
||||
version=__version__,
|
||||
environment=settings.env,
|
||||
release_channel=RELEASE_CHANNEL,
|
||||
production_inference_available=production_count > 0,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/version", response_model=ReleaseInfo)
|
||||
def release_info(settings: Settings = Depends(get_settings)) -> ReleaseInfo:
|
||||
"""Build identity and compatibility for the running process.
|
||||
|
||||
Unauthenticated on purpose: an operator diagnosing a deployment needs to know which build is
|
||||
answering before they have credentials for it, and everything here is already implied by the
|
||||
image they are running. Nothing configuration-derived or sensitive is exposed.
|
||||
"""
|
||||
|
||||
identity = build_identity(
|
||||
source_commit=settings.build_commit,
|
||||
built_at=settings.build_timestamp,
|
||||
image_digest=settings.build_image_digest,
|
||||
)
|
||||
return ReleaseInfo(
|
||||
name=PRODUCT_NAME,
|
||||
version=identity.version,
|
||||
release_channel=identity.channel,
|
||||
source_commit=identity.source_commit,
|
||||
built_at=identity.built_at,
|
||||
image_digest=identity.image_digest,
|
||||
compatibility=ReleaseCompatibility(
|
||||
schema_revision=TARGET_SCHEMA_REVISION,
|
||||
supported_schema_revisions=list(SUPPORTED_SCHEMA_REVISIONS),
|
||||
agent_protocol_version=CURRENT_AGENT_PROTOCOL_VERSION,
|
||||
supported_agent_protocol_versions=list(SUPPORTED_AGENT_PROTOCOL_VERSIONS),
|
||||
minimum_upgrade_source=MINIMUM_UPGRADE_SOURCE,
|
||||
minimum_postgres_major=MINIMUM_POSTGRES_MAJOR,
|
||||
),
|
||||
)
|
||||
@@ -0,0 +1,248 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Depends, Query, status
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from modelforge_api.api.authorization import Admin, require_viewer
|
||||
from modelforge_api.db import get_session
|
||||
from modelforge_api.domain.lifecycle_contracts import (
|
||||
ApprovalDecision,
|
||||
ApprovalPolicyCreate,
|
||||
ApprovalPolicyResponse,
|
||||
ApprovalRequestCreate,
|
||||
ApprovalRequestResponse,
|
||||
CanaryObservation,
|
||||
CleanupExecutionCreate,
|
||||
CleanupPlanCreate,
|
||||
CleanupPlanResponse,
|
||||
LifecycleEventResponse,
|
||||
LifecycleOperationResponse,
|
||||
LifecycleSubjectCreate,
|
||||
LifecycleSubjectResponse,
|
||||
PlanExecutionCreate,
|
||||
PromotionPlanCreate,
|
||||
PromotionPlanResponse,
|
||||
RetentionPolicyCreate,
|
||||
RetentionPolicyResponse,
|
||||
)
|
||||
from modelforge_api.services.lifecycle import LifecycleService
|
||||
|
||||
router = APIRouter(
|
||||
prefix="/api/v1/admin/lifecycle",
|
||||
tags=["lifecycle"],
|
||||
dependencies=[Depends(require_viewer)],
|
||||
)
|
||||
|
||||
|
||||
def get_lifecycle_service(
|
||||
session: Annotated[Session, Depends(get_session)],
|
||||
) -> LifecycleService:
|
||||
return LifecycleService(session)
|
||||
|
||||
|
||||
Service = Annotated[LifecycleService, Depends(get_lifecycle_service)]
|
||||
|
||||
|
||||
@router.get("/approval-policies", response_model=list[ApprovalPolicyResponse])
|
||||
def approval_policies(service: Service, _admin: Admin) -> list[ApprovalPolicyResponse]:
|
||||
return service.policies()
|
||||
|
||||
|
||||
@router.post(
|
||||
"/approval-policies",
|
||||
response_model=ApprovalPolicyResponse,
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
)
|
||||
def create_approval_policy(
|
||||
request: ApprovalPolicyCreate, service: Service, _admin: Admin
|
||||
) -> ApprovalPolicyResponse:
|
||||
return service.create_policy(request)
|
||||
|
||||
|
||||
@router.get("/subjects", response_model=list[LifecycleSubjectResponse])
|
||||
def subjects(service: Service, _admin: Admin) -> list[LifecycleSubjectResponse]:
|
||||
return service.subjects()
|
||||
|
||||
|
||||
@router.post(
|
||||
"/subjects",
|
||||
response_model=LifecycleSubjectResponse,
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
)
|
||||
def create_subject(
|
||||
request: LifecycleSubjectCreate, service: Service, _admin: Admin
|
||||
) -> LifecycleSubjectResponse:
|
||||
return service.create_subject(request)
|
||||
|
||||
|
||||
@router.get("/approval-requests", response_model=list[ApprovalRequestResponse])
|
||||
def approval_requests(service: Service, _admin: Admin) -> list[ApprovalRequestResponse]:
|
||||
return service.approval_requests()
|
||||
|
||||
|
||||
@router.post(
|
||||
"/approval-requests",
|
||||
response_model=ApprovalRequestResponse,
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
)
|
||||
def request_approval(
|
||||
request: ApprovalRequestCreate, service: Service, _admin: Admin
|
||||
) -> ApprovalRequestResponse:
|
||||
return service.request_approval(request)
|
||||
|
||||
|
||||
@router.get("/approval-requests/{approval_id}", response_model=ApprovalRequestResponse)
|
||||
def approval_request(
|
||||
approval_id: uuid.UUID, service: Service, _admin: Admin
|
||||
) -> ApprovalRequestResponse:
|
||||
return service.approval(approval_id)
|
||||
|
||||
|
||||
@router.post("/approval-requests/{approval_id}/approve", response_model=ApprovalRequestResponse)
|
||||
def approve_request(
|
||||
approval_id: uuid.UUID, request: ApprovalDecision, service: Service, _admin: Admin
|
||||
) -> ApprovalRequestResponse:
|
||||
return service.decide_approval(approval_id, request, "approve")
|
||||
|
||||
|
||||
@router.post("/approval-requests/{approval_id}/reject", response_model=ApprovalRequestResponse)
|
||||
def reject_request(
|
||||
approval_id: uuid.UUID, request: ApprovalDecision, service: Service, _admin: Admin
|
||||
) -> ApprovalRequestResponse:
|
||||
return service.decide_approval(approval_id, request, "reject")
|
||||
|
||||
|
||||
@router.post("/approval-requests/{approval_id}/revoke", response_model=ApprovalRequestResponse)
|
||||
def revoke_request(
|
||||
approval_id: uuid.UUID, request: ApprovalDecision, service: Service, _admin: Admin
|
||||
) -> ApprovalRequestResponse:
|
||||
return service.decide_approval(approval_id, request, "revoke")
|
||||
|
||||
|
||||
@router.get("/promotion-plans", response_model=list[PromotionPlanResponse])
|
||||
def promotion_plans(service: Service, _admin: Admin) -> list[PromotionPlanResponse]:
|
||||
return service.plans()
|
||||
|
||||
|
||||
@router.post(
|
||||
"/promotion-plans",
|
||||
response_model=PromotionPlanResponse,
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
)
|
||||
def create_promotion_plan(
|
||||
request: PromotionPlanCreate, service: Service, _admin: Admin
|
||||
) -> PromotionPlanResponse:
|
||||
return service.create_plan(request)
|
||||
|
||||
|
||||
@router.get("/promotion-plans/{plan_id}", response_model=PromotionPlanResponse)
|
||||
def promotion_plan(plan_id: uuid.UUID, service: Service, _admin: Admin) -> PromotionPlanResponse:
|
||||
return service.plan(plan_id)
|
||||
|
||||
|
||||
@router.post("/promotion-plans/{plan_id}/approve", response_model=PromotionPlanResponse)
|
||||
def approve_promotion_plan(
|
||||
plan_id: uuid.UUID, request: ApprovalDecision, service: Service, _admin: Admin
|
||||
) -> PromotionPlanResponse:
|
||||
return service.approve_plan(plan_id, request)
|
||||
|
||||
|
||||
@router.post("/promotion-plans/{plan_id}/execute", response_model=LifecycleOperationResponse)
|
||||
def execute_promotion_plan(
|
||||
plan_id: uuid.UUID, request: PlanExecutionCreate, service: Service, _admin: Admin
|
||||
) -> LifecycleOperationResponse:
|
||||
return service.execute_plan(plan_id, request)
|
||||
|
||||
|
||||
@router.get("/operations", response_model=list[LifecycleOperationResponse])
|
||||
def operations(service: Service, _admin: Admin) -> list[LifecycleOperationResponse]:
|
||||
return service.operations()
|
||||
|
||||
|
||||
@router.get("/operations/{operation_id}", response_model=LifecycleOperationResponse)
|
||||
def operation(
|
||||
operation_id: uuid.UUID, service: Service, _admin: Admin
|
||||
) -> LifecycleOperationResponse:
|
||||
return service.operation(operation_id)
|
||||
|
||||
|
||||
@router.post("/operations/{operation_id}/canary", response_model=LifecycleOperationResponse)
|
||||
def observe_canary(
|
||||
operation_id: uuid.UUID, request: CanaryObservation, service: Service, _admin: Admin
|
||||
) -> LifecycleOperationResponse:
|
||||
return service.observe_canary(operation_id, request)
|
||||
|
||||
|
||||
@router.post("/operations/{operation_id}/commit", response_model=LifecycleOperationResponse)
|
||||
def commit_operation(
|
||||
operation_id: uuid.UUID, request: ApprovalDecision, service: Service, _admin: Admin
|
||||
) -> LifecycleOperationResponse:
|
||||
return service.commit_operation(operation_id, request)
|
||||
|
||||
|
||||
@router.post("/operations/{operation_id}/rollback", response_model=LifecycleOperationResponse)
|
||||
def rollback_operation(
|
||||
operation_id: uuid.UUID, request: ApprovalDecision, service: Service, _admin: Admin
|
||||
) -> LifecycleOperationResponse:
|
||||
return service.rollback_operation(operation_id, request)
|
||||
|
||||
|
||||
@router.post("/reconcile", response_model=dict[str, int])
|
||||
def reconcile(service: Service, _admin: Admin) -> dict[str, int]:
|
||||
return {"reconciled_operations": service.reconcile_incomplete()}
|
||||
|
||||
|
||||
@router.get("/retention-policies", response_model=list[RetentionPolicyResponse])
|
||||
def retention_policies(service: Service, _admin: Admin) -> list[RetentionPolicyResponse]:
|
||||
return service.retention_policies()
|
||||
|
||||
|
||||
@router.post(
|
||||
"/retention-policies",
|
||||
response_model=RetentionPolicyResponse,
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
)
|
||||
def create_retention_policy(
|
||||
request: RetentionPolicyCreate, service: Service, _admin: Admin
|
||||
) -> RetentionPolicyResponse:
|
||||
return service.create_retention_policy(request)
|
||||
|
||||
|
||||
@router.get("/cleanup-plans", response_model=list[CleanupPlanResponse])
|
||||
def cleanup_plans(service: Service, _admin: Admin) -> list[CleanupPlanResponse]:
|
||||
return service.cleanup_plans()
|
||||
|
||||
|
||||
@router.post(
|
||||
"/cleanup-plans",
|
||||
response_model=CleanupPlanResponse,
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
)
|
||||
def create_cleanup_plan(
|
||||
request: CleanupPlanCreate, service: Service, _admin: Admin
|
||||
) -> CleanupPlanResponse:
|
||||
return service.create_cleanup_plan(request)
|
||||
|
||||
|
||||
@router.get("/cleanup-plans/{plan_id}", response_model=CleanupPlanResponse)
|
||||
def cleanup_plan(plan_id: uuid.UUID, service: Service, _admin: Admin) -> CleanupPlanResponse:
|
||||
return service.cleanup_plan(plan_id)
|
||||
|
||||
|
||||
@router.post("/cleanup-plans/{plan_id}/execute", response_model=CleanupPlanResponse)
|
||||
def execute_cleanup_plan(
|
||||
plan_id: uuid.UUID, request: CleanupExecutionCreate, service: Service, _admin: Admin
|
||||
) -> CleanupPlanResponse:
|
||||
return service.execute_cleanup(plan_id, request)
|
||||
|
||||
|
||||
@router.get("/events", response_model=list[LifecycleEventResponse])
|
||||
def lifecycle_events(
|
||||
service: Service,
|
||||
_admin: Admin,
|
||||
limit: Annotated[int, Query(ge=1, le=1000)] = 200,
|
||||
) -> list[LifecycleEventResponse]:
|
||||
return service.events(limit)
|
||||
@@ -0,0 +1,191 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Depends, Query, status
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from modelforge_api.api.authorization import Admin, require_viewer
|
||||
from modelforge_api.db import get_session
|
||||
from modelforge_api.domain.migration_contracts import (
|
||||
BatchReport,
|
||||
CutoverOperationResponse,
|
||||
CutoverPrepare,
|
||||
CutoverReport,
|
||||
MigrationBatchResponse,
|
||||
MigrationEventResponse,
|
||||
MigrationPlanCreate,
|
||||
MigrationPlanResponse,
|
||||
MigrationValidationPolicyCreate,
|
||||
MigrationValidationPolicyResponse,
|
||||
PreflightReport,
|
||||
ReconciliationReport,
|
||||
RollbackReport,
|
||||
ShadowReport,
|
||||
StateAction,
|
||||
ValidationReport,
|
||||
ValidationSnapshotResponse,
|
||||
)
|
||||
from modelforge_api.services.migration_engine import MigrationEngineService
|
||||
|
||||
router = APIRouter(
|
||||
prefix="/api/v1/admin/migrations",
|
||||
tags=["migrations"],
|
||||
dependencies=[Depends(require_viewer)],
|
||||
)
|
||||
|
||||
|
||||
def get_service(session: Annotated[Session, Depends(get_session)]) -> MigrationEngineService:
|
||||
return MigrationEngineService(session)
|
||||
|
||||
|
||||
Service = Annotated[MigrationEngineService, Depends(get_service)]
|
||||
|
||||
|
||||
@router.get("/validation-policies", response_model=list[MigrationValidationPolicyResponse])
|
||||
def validation_policies(service: Service, _admin: Admin) -> list[MigrationValidationPolicyResponse]:
|
||||
return service.validation_policies()
|
||||
|
||||
|
||||
@router.post(
|
||||
"/validation-policies",
|
||||
response_model=MigrationValidationPolicyResponse,
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
)
|
||||
def create_validation_policy(
|
||||
request: MigrationValidationPolicyCreate, service: Service, _admin: Admin
|
||||
) -> MigrationValidationPolicyResponse:
|
||||
return service.create_validation_policy(request)
|
||||
|
||||
|
||||
@router.get("/plans", response_model=list[MigrationPlanResponse])
|
||||
def plans(service: Service, _admin: Admin) -> list[MigrationPlanResponse]:
|
||||
return service.plans()
|
||||
|
||||
|
||||
@router.post("/plans", response_model=MigrationPlanResponse, status_code=status.HTTP_201_CREATED)
|
||||
def create_plan(
|
||||
request: MigrationPlanCreate, service: Service, _admin: Admin
|
||||
) -> MigrationPlanResponse:
|
||||
return service.create_plan(request)
|
||||
|
||||
|
||||
@router.get("/plans/{plan_id}", response_model=MigrationPlanResponse)
|
||||
def plan(plan_id: uuid.UUID, service: Service, _admin: Admin) -> MigrationPlanResponse:
|
||||
return service.plan(plan_id)
|
||||
|
||||
|
||||
@router.post("/plans/{plan_id}/preflight", response_model=MigrationPlanResponse)
|
||||
def preflight(
|
||||
plan_id: uuid.UUID, request: PreflightReport, service: Service, _admin: Admin
|
||||
) -> MigrationPlanResponse:
|
||||
return service.preflight(plan_id, request)
|
||||
|
||||
|
||||
@router.post("/plans/{plan_id}/backfill/start", response_model=MigrationPlanResponse)
|
||||
def start_backfill(
|
||||
plan_id: uuid.UUID, request: StateAction, service: Service, _admin: Admin
|
||||
) -> MigrationPlanResponse:
|
||||
return service.start_backfill(plan_id, request)
|
||||
|
||||
|
||||
@router.post("/plans/{plan_id}/backfill/pause", response_model=MigrationPlanResponse)
|
||||
def pause_backfill(
|
||||
plan_id: uuid.UUID, request: StateAction, service: Service, _admin: Admin
|
||||
) -> MigrationPlanResponse:
|
||||
return service.pause_backfill(plan_id, request)
|
||||
|
||||
|
||||
@router.post("/plans/{plan_id}/batches", response_model=MigrationBatchResponse)
|
||||
def record_batch(
|
||||
plan_id: uuid.UUID, request: BatchReport, service: Service, _admin: Admin
|
||||
) -> MigrationBatchResponse:
|
||||
return service.record_batch(plan_id, request)
|
||||
|
||||
|
||||
@router.get("/plans/{plan_id}/batches", response_model=list[MigrationBatchResponse])
|
||||
def batches(plan_id: uuid.UUID, service: Service, _admin: Admin) -> list[MigrationBatchResponse]:
|
||||
return service.batches(plan_id)
|
||||
|
||||
|
||||
@router.post("/plans/{plan_id}/validation", response_model=ValidationSnapshotResponse)
|
||||
def validate_target(
|
||||
plan_id: uuid.UUID, request: ValidationReport, service: Service, _admin: Admin
|
||||
) -> ValidationSnapshotResponse:
|
||||
return service.validate(plan_id, request)
|
||||
|
||||
|
||||
@router.get("/plans/{plan_id}/validation", response_model=list[ValidationSnapshotResponse])
|
||||
def validation_snapshots(
|
||||
plan_id: uuid.UUID, service: Service, _admin: Admin
|
||||
) -> list[ValidationSnapshotResponse]:
|
||||
return service.validation_snapshots(plan_id)
|
||||
|
||||
|
||||
@router.post("/plans/{plan_id}/shadow/start", response_model=MigrationPlanResponse)
|
||||
def start_shadow(
|
||||
plan_id: uuid.UUID, request: StateAction, service: Service, _admin: Admin
|
||||
) -> MigrationPlanResponse:
|
||||
return service.start_shadow(plan_id, request)
|
||||
|
||||
|
||||
@router.post("/plans/{plan_id}/shadow/complete", response_model=MigrationPlanResponse)
|
||||
def complete_shadow(
|
||||
plan_id: uuid.UUID, request: ShadowReport, service: Service, _admin: Admin
|
||||
) -> MigrationPlanResponse:
|
||||
return service.complete_shadow(plan_id, request)
|
||||
|
||||
|
||||
@router.post("/plans/{plan_id}/cutover/prepare", response_model=CutoverOperationResponse)
|
||||
def prepare_cutover(
|
||||
plan_id: uuid.UUID, request: CutoverPrepare, service: Service, _admin: Admin
|
||||
) -> CutoverOperationResponse:
|
||||
return service.prepare_cutover(plan_id, request)
|
||||
|
||||
|
||||
@router.post("/plans/{plan_id}/cutover/report", response_model=CutoverOperationResponse)
|
||||
def report_cutover(
|
||||
plan_id: uuid.UUID, request: CutoverReport, service: Service, _admin: Admin
|
||||
) -> CutoverOperationResponse:
|
||||
return service.report_cutover(plan_id, request)
|
||||
|
||||
|
||||
@router.post("/plans/{plan_id}/rollback", response_model=CutoverOperationResponse)
|
||||
def rollback(
|
||||
plan_id: uuid.UUID, request: RollbackReport, service: Service, _admin: Admin
|
||||
) -> CutoverOperationResponse:
|
||||
return service.rollback(plan_id, request)
|
||||
|
||||
|
||||
@router.post("/reconcile", response_model=CutoverOperationResponse)
|
||||
def reconcile(
|
||||
request: ReconciliationReport, service: Service, _admin: Admin
|
||||
) -> CutoverOperationResponse:
|
||||
return service.reconcile(request)
|
||||
|
||||
|
||||
@router.post("/plans/{plan_id}/cancel", response_model=MigrationPlanResponse)
|
||||
def cancel(
|
||||
plan_id: uuid.UUID, request: StateAction, service: Service, _admin: Admin
|
||||
) -> MigrationPlanResponse:
|
||||
return service.cancel(plan_id, request)
|
||||
|
||||
|
||||
@router.get("/cutovers", response_model=list[CutoverOperationResponse])
|
||||
def cutovers(
|
||||
service: Service,
|
||||
_admin: Admin,
|
||||
plan_id: Annotated[uuid.UUID | None, Query()] = None,
|
||||
) -> list[CutoverOperationResponse]:
|
||||
return service.operations(plan_id)
|
||||
|
||||
|
||||
@router.get("/events", response_model=list[MigrationEventResponse])
|
||||
def events(
|
||||
service: Service,
|
||||
_admin: Admin,
|
||||
plan_id: Annotated[uuid.UUID | None, Query()] = None,
|
||||
limit: Annotated[int, Query(ge=1, le=1000)] = 200,
|
||||
) -> list[MigrationEventResponse]:
|
||||
return service.events(plan_id, limit)
|
||||
@@ -0,0 +1,237 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from typing import Annotated, Any
|
||||
|
||||
from fastapi import APIRouter, Depends, Query, Response, status
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.exc import SQLAlchemyError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from modelforge_api.api.authorization import Admin, require_viewer
|
||||
from modelforge_api.db import get_session
|
||||
from modelforge_api.domain.observability import (
|
||||
AlertAction,
|
||||
AlertHistoryResponse,
|
||||
AlertResponse,
|
||||
AlertRuleCreate,
|
||||
AlertRuleResponse,
|
||||
CapacitySnapshotResponse,
|
||||
IncidentResponse,
|
||||
MaintenanceWindowCreate,
|
||||
MaintenanceWindowResponse,
|
||||
OperationsOverview,
|
||||
SLIDefinitionResponse,
|
||||
SLOEvaluationResponse,
|
||||
SLOPolicyCreate,
|
||||
SLOPolicyResponse,
|
||||
TrendResponse,
|
||||
metrics,
|
||||
)
|
||||
from modelforge_api.persistence.models import (
|
||||
AlertHistoryEvent,
|
||||
IncidentTimelineEvent,
|
||||
OperationalIncident,
|
||||
)
|
||||
from modelforge_api.services.observability import ObservabilityService
|
||||
|
||||
router = APIRouter(tags=["operations"], dependencies=[Depends(require_viewer)])
|
||||
|
||||
|
||||
def get_service(session: Annotated[Session, Depends(get_session)]) -> ObservabilityService:
|
||||
return ObservabilityService(session)
|
||||
|
||||
|
||||
Service = Annotated[ObservabilityService, Depends(get_service)]
|
||||
|
||||
|
||||
@router.get("/metrics", response_class=Response)
|
||||
def prometheus_metrics(service: Service, _admin: Admin) -> Response:
|
||||
"""Admin-isolated Prometheus exposition; process metrics survive DB failure."""
|
||||
try:
|
||||
content = service.prometheus()
|
||||
metrics.gauge("modelforge_observability_degraded", {}, 0)
|
||||
except SQLAlchemyError:
|
||||
service.session.rollback()
|
||||
metrics.gauge("modelforge_observability_degraded", {}, 1)
|
||||
content = metrics.render()
|
||||
return Response(content=content, media_type="text/plain; version=0.0.4; charset=utf-8")
|
||||
|
||||
|
||||
@router.get("/api/v1/admin/operations/overview", response_model=OperationsOverview)
|
||||
def overview(service: Service, _admin: Admin) -> OperationsOverview:
|
||||
return service.overview()
|
||||
|
||||
|
||||
@router.get(
|
||||
"/api/v1/admin/operations/slis", response_model=list[SLIDefinitionResponse]
|
||||
)
|
||||
def sli_definitions(service: Service, _admin: Admin) -> list[SLIDefinitionResponse]:
|
||||
return service.definitions()
|
||||
|
||||
|
||||
@router.get(
|
||||
"/api/v1/admin/operations/slo-policies", response_model=list[SLOPolicyResponse]
|
||||
)
|
||||
def slo_policies(service: Service, _admin: Admin) -> list[SLOPolicyResponse]:
|
||||
return service.policies()
|
||||
|
||||
|
||||
@router.post(
|
||||
"/api/v1/admin/operations/slo-policies",
|
||||
response_model=SLOPolicyResponse,
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
)
|
||||
def create_slo_policy(
|
||||
request: SLOPolicyCreate, service: Service, _admin: Admin
|
||||
) -> SLOPolicyResponse:
|
||||
return service.create_policy(request)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/api/v1/admin/operations/slo-evaluations", response_model=list[SLOEvaluationResponse]
|
||||
)
|
||||
def slo_evaluations(
|
||||
service: Service, _admin: Admin, limit: int = Query(default=100, ge=1, le=1000)
|
||||
) -> list[SLOEvaluationResponse]:
|
||||
return service.evaluations(limit)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/api/v1/admin/operations/slo-evaluations/run",
|
||||
response_model=list[SLOEvaluationResponse],
|
||||
)
|
||||
def evaluate_slos(service: Service, _admin: Admin) -> list[SLOEvaluationResponse]:
|
||||
return service.evaluate_slos()
|
||||
|
||||
|
||||
@router.get(
|
||||
"/api/v1/admin/operations/alert-rules", response_model=list[AlertRuleResponse]
|
||||
)
|
||||
def alert_rules(service: Service, _admin: Admin) -> list[AlertRuleResponse]:
|
||||
return service.rules()
|
||||
|
||||
|
||||
@router.post(
|
||||
"/api/v1/admin/operations/alert-rules",
|
||||
response_model=AlertRuleResponse,
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
)
|
||||
def create_alert_rule(
|
||||
request: AlertRuleCreate, service: Service, _admin: Admin
|
||||
) -> AlertRuleResponse:
|
||||
return service.create_rule(request)
|
||||
|
||||
|
||||
@router.get("/api/v1/admin/operations/alerts", response_model=list[AlertResponse])
|
||||
def alerts(
|
||||
service: Service, _admin: Admin, limit: int = Query(default=200, ge=1, le=1000)
|
||||
) -> list[AlertResponse]:
|
||||
return service.alerts(limit)
|
||||
|
||||
|
||||
@router.post("/api/v1/admin/operations/alerts/evaluate", response_model=list[AlertResponse])
|
||||
def evaluate_alerts(service: Service, _admin: Admin) -> list[AlertResponse]:
|
||||
return service.evaluate_alerts()
|
||||
|
||||
|
||||
@router.post(
|
||||
"/api/v1/admin/operations/alerts/{alert_id}/acknowledge", response_model=AlertResponse
|
||||
)
|
||||
def acknowledge_alert(
|
||||
alert_id: uuid.UUID, request: AlertAction, service: Service, _admin: Admin
|
||||
) -> AlertResponse:
|
||||
return service.acknowledge(alert_id, request)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/api/v1/admin/operations/alerts/{alert_id}/history",
|
||||
response_model=list[AlertHistoryResponse],
|
||||
)
|
||||
def alert_history(
|
||||
alert_id: uuid.UUID, service: Service, _admin: Admin
|
||||
) -> list[AlertHistoryResponse]:
|
||||
rows = service.session.scalars(
|
||||
select(AlertHistoryEvent)
|
||||
.where(AlertHistoryEvent.alert_id == alert_id)
|
||||
.order_by(AlertHistoryEvent.occurred_at)
|
||||
)
|
||||
return [AlertHistoryResponse.model_validate(item) for item in rows]
|
||||
|
||||
|
||||
@router.get(
|
||||
"/api/v1/admin/operations/maintenance-windows",
|
||||
response_model=list[MaintenanceWindowResponse],
|
||||
)
|
||||
def maintenance_windows(service: Service, _admin: Admin) -> list[MaintenanceWindowResponse]:
|
||||
return service.maintenance_windows()
|
||||
|
||||
|
||||
@router.post(
|
||||
"/api/v1/admin/operations/maintenance-windows",
|
||||
response_model=MaintenanceWindowResponse,
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
)
|
||||
def create_maintenance_window(
|
||||
request: MaintenanceWindowCreate, service: Service, _admin: Admin
|
||||
) -> MaintenanceWindowResponse:
|
||||
return service.create_maintenance_window(request)
|
||||
|
||||
|
||||
@router.get("/api/v1/admin/operations/capacity", response_model=list[CapacitySnapshotResponse])
|
||||
def capacity(
|
||||
service: Service, _admin: Admin, limit: int = Query(default=500, ge=1, le=5000)
|
||||
) -> list[CapacitySnapshotResponse]:
|
||||
return service.capacity(limit)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/api/v1/admin/operations/capacity/collect", response_model=list[CapacitySnapshotResponse]
|
||||
)
|
||||
def collect_capacity(service: Service, _admin: Admin) -> list[CapacitySnapshotResponse]:
|
||||
return service.collect_capacity()
|
||||
|
||||
|
||||
@router.get("/api/v1/admin/operations/capacity/{node_id}/trend", response_model=TrendResponse)
|
||||
def capacity_trend(
|
||||
node_id: uuid.UUID,
|
||||
service: Service,
|
||||
_admin: Admin,
|
||||
hours: int = Query(default=24, ge=1, le=2160),
|
||||
) -> TrendResponse:
|
||||
return service.capacity_trend(node_id, hours)
|
||||
|
||||
|
||||
@router.post("/api/v1/admin/operations/retention/run", response_model=dict[str, int])
|
||||
def apply_retention(service: Service, _admin: Admin) -> dict[str, int]:
|
||||
return service.apply_retention()
|
||||
|
||||
|
||||
@router.get("/api/v1/admin/operations/incidents", response_model=list[IncidentResponse])
|
||||
def incidents(service: Service, _admin: Admin) -> list[IncidentResponse]:
|
||||
rows = service.session.scalars(
|
||||
select(OperationalIncident).order_by(OperationalIncident.last_seen_at.desc())
|
||||
)
|
||||
return [IncidentResponse.model_validate(item) for item in rows]
|
||||
|
||||
|
||||
@router.get("/api/v1/admin/operations/incidents/{incident_id}/timeline")
|
||||
def incident_timeline(
|
||||
incident_id: uuid.UUID, service: Service, _admin: Admin
|
||||
) -> list[dict[str, Any]]:
|
||||
rows = service.session.scalars(
|
||||
select(IncidentTimelineEvent)
|
||||
.where(IncidentTimelineEvent.incident_id == incident_id)
|
||||
.order_by(IncidentTimelineEvent.occurred_at)
|
||||
)
|
||||
return [
|
||||
{
|
||||
"id": str(item.id),
|
||||
"alert_id": str(item.alert_id) if item.alert_id else None,
|
||||
"event_type": item.event_type,
|
||||
"relation": item.relation,
|
||||
"summary": item.summary,
|
||||
"occurred_at": item.occurred_at,
|
||||
}
|
||||
for item in rows
|
||||
]
|
||||
@@ -0,0 +1,215 @@
|
||||
"""Operator-only recovery API.
|
||||
|
||||
Backup creation, verification and restore *planning* are safe control-plane operations. Restore
|
||||
*execution* is deliberately restricted: it can only run against an isolated destination that is
|
||||
not this control plane's own database, and replacing a production database stays an operator
|
||||
runbook/CLI action rather than a remote call.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from typing import Annotated, Any
|
||||
|
||||
from fastapi import APIRouter, Depends, Query, status
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from modelforge_api.api.authorization import Admin, require_viewer
|
||||
from modelforge_api.db import get_session
|
||||
from modelforge_api.domain.recovery import (
|
||||
ArtifactRecoveryCreate,
|
||||
ArtifactRecoveryResponse,
|
||||
BackupCapacityEstimate,
|
||||
BackupSetCreate,
|
||||
BackupSetResponse,
|
||||
RecoveryAssetResponse,
|
||||
RecoveryDashboard,
|
||||
RecoveryPolicyCreate,
|
||||
RecoveryPolicyResponse,
|
||||
RestoreAdvanceRequest,
|
||||
RestoreOperationEventResponse,
|
||||
RestoreOperationResponse,
|
||||
RestorePlanCreate,
|
||||
RestorePlanResponse,
|
||||
)
|
||||
from modelforge_api.services.recovery import RecoveryService
|
||||
from modelforge_api.settings import Settings, get_settings
|
||||
|
||||
router = APIRouter(
|
||||
prefix="/api/v1/admin/recovery",
|
||||
tags=["recovery"],
|
||||
dependencies=[Depends(require_viewer)],
|
||||
)
|
||||
|
||||
|
||||
def get_service(
|
||||
session: Annotated[Session, Depends(get_session)],
|
||||
settings: Annotated[Settings, Depends(get_settings)],
|
||||
) -> RecoveryService:
|
||||
return RecoveryService(session, settings)
|
||||
|
||||
|
||||
Service = Annotated[RecoveryService, Depends(get_service)]
|
||||
|
||||
|
||||
@router.get("/dashboard", response_model=RecoveryDashboard)
|
||||
def dashboard(service: Service, _admin: Admin) -> RecoveryDashboard:
|
||||
return service.dashboard()
|
||||
|
||||
|
||||
@router.get("/policies", response_model=list[RecoveryPolicyResponse])
|
||||
def policies(service: Service, _admin: Admin) -> list[RecoveryPolicyResponse]:
|
||||
return service.policies()
|
||||
|
||||
|
||||
@router.post(
|
||||
"/policies",
|
||||
response_model=RecoveryPolicyResponse,
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
)
|
||||
def create_policy(
|
||||
request: RecoveryPolicyCreate, service: Service, _admin: Admin
|
||||
) -> RecoveryPolicyResponse:
|
||||
return service.create_policy(request)
|
||||
|
||||
|
||||
@router.get("/assets", response_model=list[RecoveryAssetResponse])
|
||||
def assets(service: Service, _admin: Admin) -> list[RecoveryAssetResponse]:
|
||||
return service.assets()
|
||||
|
||||
|
||||
@router.get("/capacity", response_model=BackupCapacityEstimate)
|
||||
def capacity(service: Service, _admin: Admin) -> BackupCapacityEstimate:
|
||||
return service.estimate_capacity()
|
||||
|
||||
|
||||
@router.get("/backups", response_model=list[BackupSetResponse])
|
||||
def backups(
|
||||
service: Service, _admin: Admin, limit: int = Query(default=100, ge=1, le=1000)
|
||||
) -> list[BackupSetResponse]:
|
||||
return service.backups(limit)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/backups", response_model=BackupSetResponse, status_code=status.HTTP_201_CREATED
|
||||
)
|
||||
def create_backup(
|
||||
request: BackupSetCreate, service: Service, _admin: Admin
|
||||
) -> BackupSetResponse:
|
||||
return service.create_backup(request)
|
||||
|
||||
|
||||
@router.get("/backups/{backup_set_id}", response_model=BackupSetResponse)
|
||||
def backup(backup_set_id: uuid.UUID, service: Service, _admin: Admin) -> BackupSetResponse:
|
||||
return service.backup(backup_set_id)
|
||||
|
||||
|
||||
@router.post("/backups/{backup_set_id}/verify", response_model=BackupSetResponse)
|
||||
def verify_backup(
|
||||
backup_set_id: uuid.UUID, service: Service, _admin: Admin
|
||||
) -> BackupSetResponse:
|
||||
return service.verify_backup(backup_set_id)
|
||||
|
||||
|
||||
@router.post("/retention/run", response_model=dict[str, int])
|
||||
def apply_retention(service: Service, _admin: Admin) -> dict[str, int]:
|
||||
return service.apply_retention()
|
||||
|
||||
|
||||
@router.get("/restore-plans", response_model=list[RestorePlanResponse])
|
||||
def restore_plans(
|
||||
service: Service, _admin: Admin, limit: int = Query(default=100, ge=1, le=1000)
|
||||
) -> list[RestorePlanResponse]:
|
||||
return service.restore_plans(limit)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/restore-plans", response_model=RestorePlanResponse, status_code=status.HTTP_201_CREATED
|
||||
)
|
||||
def create_restore_plan(
|
||||
request: RestorePlanCreate, service: Service, _admin: Admin
|
||||
) -> RestorePlanResponse:
|
||||
return service.create_restore_plan(request)
|
||||
|
||||
|
||||
@router.get("/restore-plans/{plan_id}", response_model=RestorePlanResponse)
|
||||
def restore_plan(plan_id: uuid.UUID, service: Service, _admin: Admin) -> RestorePlanResponse:
|
||||
return service.restore_plan(plan_id)
|
||||
|
||||
|
||||
@router.post("/restore-plans/{plan_id}/preflight", response_model=RestorePlanResponse)
|
||||
def preflight(plan_id: uuid.UUID, service: Service, _admin: Admin) -> RestorePlanResponse:
|
||||
return service.preflight(plan_id)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/restore-plans/{plan_id}/start",
|
||||
response_model=RestoreOperationResponse,
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
)
|
||||
def start_restore(
|
||||
plan_id: uuid.UUID,
|
||||
request: RestoreAdvanceRequest,
|
||||
service: Service,
|
||||
_admin: Admin,
|
||||
) -> RestoreOperationResponse:
|
||||
return service.start_restore(plan_id, request)
|
||||
|
||||
|
||||
@router.get("/restore-operations", response_model=list[RestoreOperationResponse])
|
||||
def restore_operations(
|
||||
service: Service, _admin: Admin, limit: int = Query(default=100, ge=1, le=1000)
|
||||
) -> list[RestoreOperationResponse]:
|
||||
return service.restore_operations(limit)
|
||||
|
||||
|
||||
@router.get("/restore-operations/{operation_id}", response_model=RestoreOperationResponse)
|
||||
def restore_operation(
|
||||
operation_id: uuid.UUID, service: Service, _admin: Admin
|
||||
) -> RestoreOperationResponse:
|
||||
return service.restore_operation(operation_id)
|
||||
|
||||
|
||||
@router.post("/restore-operations/{operation_id}/advance", response_model=RestoreOperationResponse)
|
||||
def advance_restore(
|
||||
operation_id: uuid.UUID,
|
||||
request: RestoreAdvanceRequest,
|
||||
service: Service,
|
||||
_admin: Admin,
|
||||
) -> RestoreOperationResponse:
|
||||
return service.advance_restore(operation_id, request)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/restore-operations/{operation_id}/events",
|
||||
response_model=list[RestoreOperationEventResponse],
|
||||
)
|
||||
def restore_events(
|
||||
operation_id: uuid.UUID, service: Service, _admin: Admin
|
||||
) -> list[RestoreOperationEventResponse]:
|
||||
return service.restore_events(operation_id)
|
||||
|
||||
|
||||
@router.get("/artifact-recoveries", response_model=list[ArtifactRecoveryResponse])
|
||||
def artifact_recoveries(
|
||||
service: Service, _admin: Admin, limit: int = Query(default=100, ge=1, le=1000)
|
||||
) -> list[ArtifactRecoveryResponse]:
|
||||
return service.artifact_recoveries(limit)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/artifact-recoveries",
|
||||
response_model=ArtifactRecoveryResponse,
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
)
|
||||
def plan_artifact_recovery(
|
||||
request: ArtifactRecoveryCreate, service: Service, _admin: Admin
|
||||
) -> ArtifactRecoveryResponse:
|
||||
return service.plan_artifact_recovery(request)
|
||||
|
||||
|
||||
@router.get("/fingerprint", response_model=dict[str, Any])
|
||||
def fingerprint(service: Service, _admin: Admin) -> dict[str, Any]:
|
||||
"""Bounded semantic fingerprint of the live control plane; contains no secret material."""
|
||||
|
||||
return service.fingerprint()
|
||||
@@ -0,0 +1,291 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import uuid
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Depends, Query, Response, status
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from modelforge_api.api.authorization import require_admin, require_operator, require_viewer
|
||||
from modelforge_api.db import get_session
|
||||
from modelforge_api.domain.registry import (
|
||||
ArtifactCreate,
|
||||
ArtifactResponse,
|
||||
CapacityDecision,
|
||||
DerivedArtifactCreate,
|
||||
DerivedArtifactResponse,
|
||||
ModelCreate,
|
||||
ModelResponse,
|
||||
ModelUpdate,
|
||||
Page,
|
||||
RevisionCreate,
|
||||
RevisionResponse,
|
||||
StorageRootCreate,
|
||||
StorageRootObservation,
|
||||
StorageRootResponse,
|
||||
StorageRootUpdate,
|
||||
VerifyResponse,
|
||||
)
|
||||
from modelforge_api.services.registry import RegistryService
|
||||
|
||||
router = APIRouter(
|
||||
prefix="/api/v1", tags=["model-registry"], dependencies=[Depends(require_viewer)]
|
||||
)
|
||||
|
||||
|
||||
def get_registry_service(session: Annotated[Session, Depends(get_session)]) -> RegistryService:
|
||||
return RegistryService(session)
|
||||
|
||||
|
||||
Service = Annotated[RegistryService, Depends(get_registry_service)]
|
||||
|
||||
|
||||
def page_values[T: BaseModel](items: list[T], page: int, page_size: int) -> Page[T]:
|
||||
total = len(items)
|
||||
start = (page - 1) * page_size
|
||||
return Page(
|
||||
items=items[start : start + page_size],
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
total=total,
|
||||
pages=math.ceil(total / page_size) if total else 0,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/models", response_model=Page[ModelResponse])
|
||||
def list_models(
|
||||
service: Service,
|
||||
page: Annotated[int, Query(ge=1)] = 1,
|
||||
page_size: Annotated[int, Query(ge=1, le=100)] = 20,
|
||||
search: str | None = None,
|
||||
lifecycle: str | None = None,
|
||||
source_type: str | None = None,
|
||||
) -> Page[ModelResponse]:
|
||||
return service.list_models(
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
search=search,
|
||||
lifecycle=lifecycle,
|
||||
source_type=source_type,
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/models",
|
||||
response_model=ModelResponse,
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
dependencies=[Depends(require_operator)],
|
||||
)
|
||||
def create_model(request: ModelCreate, service: Service) -> ModelResponse:
|
||||
return service.create_model(request)
|
||||
|
||||
|
||||
@router.get("/models/{model_id}", response_model=ModelResponse)
|
||||
def get_model(model_id: uuid.UUID, service: Service) -> ModelResponse:
|
||||
return service.get_model(model_id)
|
||||
|
||||
|
||||
@router.patch(
|
||||
"/models/{model_id}",
|
||||
response_model=ModelResponse,
|
||||
dependencies=[Depends(require_operator)],
|
||||
)
|
||||
def update_model(model_id: uuid.UUID, request: ModelUpdate, service: Service) -> ModelResponse:
|
||||
return service.update_model(model_id, request)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/models/{model_id}/deprecate",
|
||||
response_model=ModelResponse,
|
||||
dependencies=[Depends(require_admin)],
|
||||
)
|
||||
def deprecate_model(model_id: uuid.UUID, service: Service) -> ModelResponse:
|
||||
return service.deprecate_model(model_id)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/models/{model_id}/archive",
|
||||
response_model=ModelResponse,
|
||||
dependencies=[Depends(require_admin)],
|
||||
)
|
||||
def archive_model(model_id: uuid.UUID, service: Service) -> ModelResponse:
|
||||
return service.archive_model(model_id)
|
||||
|
||||
|
||||
@router.delete(
|
||||
"/models/{model_id}",
|
||||
status_code=status.HTTP_204_NO_CONTENT,
|
||||
dependencies=[Depends(require_admin)],
|
||||
)
|
||||
def delete_model(model_id: uuid.UUID, service: Service) -> Response:
|
||||
service.delete("model", model_id)
|
||||
return Response(status_code=status.HTTP_204_NO_CONTENT)
|
||||
|
||||
|
||||
@router.get("/models/{model_id}/revisions", response_model=Page[RevisionResponse])
|
||||
def list_revisions(
|
||||
model_id: uuid.UUID,
|
||||
service: Service,
|
||||
page: Annotated[int, Query(ge=1)] = 1,
|
||||
page_size: Annotated[int, Query(ge=1, le=100)] = 20,
|
||||
) -> Page[RevisionResponse]:
|
||||
return page_values(service.revisions(model_id), page, page_size)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/models/{model_id}/revisions",
|
||||
response_model=RevisionResponse,
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
dependencies=[Depends(require_operator)],
|
||||
)
|
||||
def create_revision(
|
||||
model_id: uuid.UUID, request: RevisionCreate, service: Service
|
||||
) -> RevisionResponse:
|
||||
return service.create_revision(model_id, request)
|
||||
|
||||
|
||||
@router.delete(
|
||||
"/revisions/{revision_id}",
|
||||
status_code=status.HTTP_204_NO_CONTENT,
|
||||
dependencies=[Depends(require_admin)],
|
||||
)
|
||||
def delete_revision(revision_id: uuid.UUID, service: Service) -> Response:
|
||||
service.delete("model_revision", revision_id)
|
||||
return Response(status_code=status.HTTP_204_NO_CONTENT)
|
||||
|
||||
|
||||
@router.get("/revisions/{revision_id}/artifacts", response_model=Page[ArtifactResponse])
|
||||
def list_artifacts(
|
||||
revision_id: uuid.UUID,
|
||||
service: Service,
|
||||
page: Annotated[int, Query(ge=1)] = 1,
|
||||
page_size: Annotated[int, Query(ge=1, le=100)] = 20,
|
||||
) -> Page[ArtifactResponse]:
|
||||
return page_values(service.artifacts(revision_id), page, page_size)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/revisions/{revision_id}/artifacts",
|
||||
response_model=ArtifactResponse,
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
dependencies=[Depends(require_operator)],
|
||||
)
|
||||
def create_artifact(
|
||||
revision_id: uuid.UUID, request: ArtifactCreate, service: Service
|
||||
) -> ArtifactResponse:
|
||||
return service.create_artifact(revision_id, request)
|
||||
|
||||
|
||||
@router.delete(
|
||||
"/artifacts/{artifact_id}",
|
||||
status_code=status.HTTP_204_NO_CONTENT,
|
||||
dependencies=[Depends(require_admin)],
|
||||
)
|
||||
def delete_artifact(artifact_id: uuid.UUID, service: Service) -> Response:
|
||||
service.delete("model_artifact", artifact_id)
|
||||
return Response(status_code=status.HTTP_204_NO_CONTENT)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/artifacts/{artifact_id}/verify",
|
||||
response_model=VerifyResponse,
|
||||
dependencies=[Depends(require_operator)],
|
||||
)
|
||||
def verify_artifact(
|
||||
artifact_id: uuid.UUID, location_id: uuid.UUID, service: Service
|
||||
) -> VerifyResponse:
|
||||
return service.verify_artifact(artifact_id, location_id)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/revisions/{revision_id}/derived-artifacts", response_model=Page[DerivedArtifactResponse]
|
||||
)
|
||||
def list_derived_artifacts(
|
||||
revision_id: uuid.UUID,
|
||||
service: Service,
|
||||
page: Annotated[int, Query(ge=1)] = 1,
|
||||
page_size: Annotated[int, Query(ge=1, le=100)] = 20,
|
||||
) -> Page[DerivedArtifactResponse]:
|
||||
return page_values(service.derived(revision_id), page, page_size)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/derived-artifacts",
|
||||
response_model=DerivedArtifactResponse,
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
dependencies=[Depends(require_operator)],
|
||||
)
|
||||
def create_derived_artifact(
|
||||
request: DerivedArtifactCreate, service: Service
|
||||
) -> DerivedArtifactResponse:
|
||||
return service.create_derived(request)
|
||||
|
||||
|
||||
@router.delete(
|
||||
"/derived-artifacts/{artifact_id}",
|
||||
status_code=status.HTTP_204_NO_CONTENT,
|
||||
dependencies=[Depends(require_admin)],
|
||||
)
|
||||
def delete_derived_artifact(artifact_id: uuid.UUID, service: Service) -> Response:
|
||||
service.delete("derived_artifact", artifact_id)
|
||||
return Response(status_code=status.HTTP_204_NO_CONTENT)
|
||||
|
||||
|
||||
@router.get("/storage-roots", response_model=list[StorageRootResponse])
|
||||
def list_storage_roots(
|
||||
service: Service, node_id: uuid.UUID | None = None
|
||||
) -> list[StorageRootResponse]:
|
||||
return service.storage_roots(node_id)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/storage-roots",
|
||||
response_model=StorageRootResponse,
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
dependencies=[Depends(require_admin)],
|
||||
)
|
||||
def create_storage_root(request: StorageRootCreate, service: Service) -> StorageRootResponse:
|
||||
return service.create_storage_root(request)
|
||||
|
||||
|
||||
@router.patch(
|
||||
"/storage-roots/{root_id}",
|
||||
response_model=StorageRootResponse,
|
||||
dependencies=[Depends(require_admin)],
|
||||
)
|
||||
def update_storage_root(
|
||||
root_id: uuid.UUID, request: StorageRootUpdate, service: Service
|
||||
) -> StorageRootResponse:
|
||||
return service.update_storage_root(root_id, request)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/storage-roots/{root_id}/observations",
|
||||
response_model=StorageRootResponse,
|
||||
dependencies=[Depends(require_operator)],
|
||||
)
|
||||
def observe_storage_root(
|
||||
root_id: uuid.UUID, request: StorageRootObservation, service: Service
|
||||
) -> StorageRootResponse:
|
||||
return service.observe_storage_root(root_id, request)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/storage-roots/{root_id}/capacity",
|
||||
response_model=CapacityDecision,
|
||||
dependencies=[Depends(require_operator)],
|
||||
)
|
||||
def check_capacity(root_id: uuid.UUID, requested_bytes: int, service: Service) -> CapacityDecision:
|
||||
return service.check_capacity(root_id, requested_bytes)
|
||||
|
||||
|
||||
@router.delete(
|
||||
"/storage-roots/{root_id}",
|
||||
status_code=status.HTTP_204_NO_CONTENT,
|
||||
dependencies=[Depends(require_admin)],
|
||||
)
|
||||
def delete_storage_root(root_id: uuid.UUID, service: Service) -> Response:
|
||||
service.delete("storage_root", root_id)
|
||||
return Response(status_code=status.HTTP_204_NO_CONTENT)
|
||||
@@ -0,0 +1,187 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from modelforge_api.api.authorization import require_admin, require_operator, require_viewer
|
||||
from modelforge_api.db import get_session
|
||||
from modelforge_api.domain.runtime import (
|
||||
CompatibilityAssessmentCreate,
|
||||
CompatibilityAssessmentResponse,
|
||||
DeploymentCandidateResponse,
|
||||
ExecutionApprovalCreate,
|
||||
ExecutionApprovalResponse,
|
||||
RuntimeEnvironmentCreate,
|
||||
RuntimeEnvironmentResponse,
|
||||
RuntimeProbeCreate,
|
||||
RuntimeProbeResponse,
|
||||
RuntimeProfileCreate,
|
||||
RuntimeProfileResponse,
|
||||
)
|
||||
from modelforge_api.services.runtime import RuntimeService
|
||||
from modelforge_api.settings import Settings, get_settings
|
||||
|
||||
router = APIRouter(
|
||||
prefix="/api/v1", tags=["runtime-plane"], dependencies=[Depends(require_viewer)]
|
||||
)
|
||||
|
||||
|
||||
def get_runtime_service(
|
||||
session: Annotated[Session, Depends(get_session)],
|
||||
settings: Annotated[Settings, Depends(get_settings)],
|
||||
) -> RuntimeService:
|
||||
return RuntimeService(session, settings)
|
||||
|
||||
|
||||
Service = Annotated[RuntimeService, Depends(get_runtime_service)]
|
||||
|
||||
|
||||
@router.post(
|
||||
"/runtime-environments",
|
||||
response_model=RuntimeEnvironmentResponse,
|
||||
status_code=201,
|
||||
dependencies=[Depends(require_operator)],
|
||||
)
|
||||
def create_environment(
|
||||
request: RuntimeEnvironmentCreate, service: Service
|
||||
) -> RuntimeEnvironmentResponse:
|
||||
return service.create_environment(request)
|
||||
|
||||
|
||||
@router.get("/runtime-environments", response_model=list[RuntimeEnvironmentResponse])
|
||||
def environments(service: Service) -> list[RuntimeEnvironmentResponse]:
|
||||
return service.environments()
|
||||
|
||||
|
||||
@router.get("/runtime-environments/{environment_id}", response_model=RuntimeEnvironmentResponse)
|
||||
def environment(environment_id: uuid.UUID, service: Service) -> RuntimeEnvironmentResponse:
|
||||
return service.environment(environment_id)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/runtime-profiles",
|
||||
response_model=RuntimeProfileResponse,
|
||||
status_code=201,
|
||||
dependencies=[Depends(require_operator)],
|
||||
)
|
||||
def create_profile(request: RuntimeProfileCreate, service: Service) -> RuntimeProfileResponse:
|
||||
return service.create_profile(request)
|
||||
|
||||
|
||||
@router.get("/runtime-profiles", response_model=list[RuntimeProfileResponse])
|
||||
def profiles(service: Service) -> list[RuntimeProfileResponse]:
|
||||
return service.profiles()
|
||||
|
||||
|
||||
@router.get("/runtime-profiles/{profile_id}", response_model=RuntimeProfileResponse)
|
||||
def profile(profile_id: uuid.UUID, service: Service) -> RuntimeProfileResponse:
|
||||
return service.profile(profile_id)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/artifact-sets/{artifact_set_id}/compatibility-assessments",
|
||||
response_model=CompatibilityAssessmentResponse,
|
||||
status_code=201,
|
||||
dependencies=[Depends(require_operator)],
|
||||
)
|
||||
def assess(
|
||||
artifact_set_id: uuid.UUID,
|
||||
request: CompatibilityAssessmentCreate,
|
||||
service: Service,
|
||||
) -> CompatibilityAssessmentResponse:
|
||||
profile = service.repo.profile(request.runtime_profile_id)
|
||||
if not profile or profile.artifact_set_id != artifact_set_id:
|
||||
from modelforge_api.services.registry import RegistryConflict
|
||||
|
||||
raise RegistryConflict("runtime profile belongs to another artifact set")
|
||||
return service.assess(request)
|
||||
|
||||
|
||||
@router.get("/compatibility-assessments", response_model=list[CompatibilityAssessmentResponse])
|
||||
def assessments(
|
||||
service: Service,
|
||||
artifact_set_id: Annotated[uuid.UUID | None, Query()] = None,
|
||||
model_id: Annotated[uuid.UUID | None, Query()] = None,
|
||||
compute_node_id: Annotated[uuid.UUID | None, Query()] = None,
|
||||
runtime_profile_id: Annotated[uuid.UUID | None, Query()] = None,
|
||||
status: Annotated[str | None, Query()] = None,
|
||||
) -> list[CompatibilityAssessmentResponse]:
|
||||
return service.assessments(
|
||||
artifact_set_id=artifact_set_id,
|
||||
model_id=model_id,
|
||||
compute_node_id=compute_node_id,
|
||||
runtime_profile_id=runtime_profile_id,
|
||||
status=status,
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/compatibility-assessments/{assessment_id}",
|
||||
response_model=CompatibilityAssessmentResponse,
|
||||
)
|
||||
def assessment(assessment_id: uuid.UUID, service: Service) -> CompatibilityAssessmentResponse:
|
||||
return service.assessment(assessment_id)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/artifact-sets/{artifact_set_id}/execution-approvals",
|
||||
response_model=ExecutionApprovalResponse,
|
||||
status_code=201,
|
||||
dependencies=[Depends(require_admin)],
|
||||
)
|
||||
def approve_execution(
|
||||
artifact_set_id: uuid.UUID,
|
||||
request: ExecutionApprovalCreate,
|
||||
service: Service,
|
||||
) -> ExecutionApprovalResponse:
|
||||
return service.approve(artifact_set_id, request)
|
||||
|
||||
|
||||
@router.get("/execution-approvals", response_model=list[ExecutionApprovalResponse])
|
||||
def approvals(
|
||||
service: Service,
|
||||
artifact_set_id: Annotated[uuid.UUID | None, Query()] = None,
|
||||
) -> list[ExecutionApprovalResponse]:
|
||||
return service.approvals(artifact_set_id)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/runtime-probes",
|
||||
response_model=RuntimeProbeResponse,
|
||||
status_code=201,
|
||||
dependencies=[Depends(require_operator)],
|
||||
)
|
||||
def create_probe(request: RuntimeProbeCreate, service: Service) -> RuntimeProbeResponse:
|
||||
return service.create_probe(request)
|
||||
|
||||
|
||||
@router.get("/runtime-probes", response_model=list[RuntimeProbeResponse])
|
||||
def probes(service: Service) -> list[RuntimeProbeResponse]:
|
||||
return service.probes()
|
||||
|
||||
|
||||
@router.get("/runtime-probes/{probe_id}", response_model=RuntimeProbeResponse)
|
||||
def probe(probe_id: uuid.UUID, service: Service) -> RuntimeProbeResponse:
|
||||
return service.probe(probe_id)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/runtime-probes/{probe_id}/cancel",
|
||||
response_model=RuntimeProbeResponse,
|
||||
dependencies=[Depends(require_operator)],
|
||||
)
|
||||
def cancel_probe(probe_id: uuid.UUID, service: Service) -> RuntimeProbeResponse:
|
||||
return service.cancel(probe_id)
|
||||
|
||||
|
||||
@router.get("/deployment-candidates", response_model=list[DeploymentCandidateResponse])
|
||||
def deployment_candidates(service: Service) -> list[DeploymentCandidateResponse]:
|
||||
return service.candidates()
|
||||
|
||||
|
||||
@router.get("/deployment-candidates/{candidate_id}", response_model=DeploymentCandidateResponse)
|
||||
def deployment_candidate(candidate_id: uuid.UUID, service: Service) -> DeploymentCandidateResponse:
|
||||
return service.candidate(candidate_id)
|
||||
@@ -0,0 +1,582 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from functools import lru_cache
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Depends, Header, Query, Request, Response, status
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from modelforge_api.api.authorization import Admin, require_viewer
|
||||
from modelforge_api.api.routes.agent import AgentIdentity
|
||||
from modelforge_api.db import get_session
|
||||
from modelforge_api.domain.serving import (
|
||||
AgentServingJobComplete,
|
||||
AgentServingJobFailure,
|
||||
AgentServingJobLease,
|
||||
AgentServingStateAck,
|
||||
AgentServingStateReport,
|
||||
CapabilityDeploymentResponse,
|
||||
CapabilityExperimentCreate,
|
||||
CapabilityExperimentResponse,
|
||||
CapabilityPromotionCreate,
|
||||
CoResidencyEvidenceCreate,
|
||||
CoResidencyEvidenceResponse,
|
||||
EmbeddingInvokeRequest,
|
||||
EmbeddingInvokeResponse,
|
||||
GatewayRequestResponse,
|
||||
OCRInvokeRequest,
|
||||
OCRInvokeResponse,
|
||||
OpenAIEmbeddingItem,
|
||||
OpenAIEmbeddingRequest,
|
||||
OpenAIEmbeddingResponse,
|
||||
OpenAIUsage,
|
||||
PlacementPlanRequest,
|
||||
PlacementPlanResponse,
|
||||
ProductionApprovalCreate,
|
||||
ProductionApprovalResponse,
|
||||
ProjectFitEvidenceCreate,
|
||||
ProjectFitEvidenceResponse,
|
||||
ProjectIntegrationResponse,
|
||||
RerankingInvokeRequest,
|
||||
RerankingInvokeResponse,
|
||||
ResidencyPolicyUpdate,
|
||||
SchedulerBudgetResponse,
|
||||
SchedulerMetricsResponse,
|
||||
SchedulerPolicyResponse,
|
||||
SchedulerPolicyUpdate,
|
||||
ServiceClientCreate,
|
||||
ServiceClientCreated,
|
||||
ServiceClientResponse,
|
||||
SpeechTranscriptionInvokeRequest,
|
||||
SpeechTranscriptionInvokeResponse,
|
||||
StableEmbeddingInvokeResponse,
|
||||
VisionEmbeddingInvokeRequest,
|
||||
VisionEmbeddingInvokeResponse,
|
||||
)
|
||||
from modelforge_api.persistence.models import ServiceClient
|
||||
from modelforge_api.services.manifest_registry import ManifestRegistry, get_manifest_registry
|
||||
from modelforge_api.services.serving import (
|
||||
CapabilityAuthenticationEvidence,
|
||||
ServingError,
|
||||
ServingService,
|
||||
)
|
||||
from modelforge_api.services.transient_payloads import RedisPayloadStore
|
||||
from modelforge_api.settings import Settings, get_settings
|
||||
|
||||
router = APIRouter(tags=["capability-serving"])
|
||||
|
||||
|
||||
@lru_cache
|
||||
def payload_store() -> RedisPayloadStore:
|
||||
return RedisPayloadStore(get_settings().redis_url)
|
||||
|
||||
|
||||
def get_serving_service(
|
||||
session: Annotated[Session, Depends(get_session)],
|
||||
settings: Annotated[Settings, Depends(get_settings)],
|
||||
manifests: Annotated[ManifestRegistry, Depends(get_manifest_registry)],
|
||||
) -> ServingService:
|
||||
return ServingService(session, settings, manifests, payload_store())
|
||||
|
||||
|
||||
Service = Annotated[ServingService, Depends(get_serving_service)]
|
||||
|
||||
|
||||
def _authenticate_capability_client(
|
||||
request: Request,
|
||||
service: ServingService,
|
||||
authorization: str | None,
|
||||
capability: str,
|
||||
) -> ServiceClient:
|
||||
evidence = getattr(request.state, "capability_authentication", None)
|
||||
if isinstance(evidence, CapabilityAuthenticationEvidence):
|
||||
return service.reuse_authentication(evidence, authorization, capability)
|
||||
return service.authenticate(authorization, capability)
|
||||
|
||||
|
||||
def authenticate_rag_embedding_client(
|
||||
request: Request,
|
||||
service: Service,
|
||||
authorization: Annotated[str | None, Header(alias="Authorization")] = None,
|
||||
) -> ServiceClient:
|
||||
return _authenticate_capability_client(request, service, authorization, "rag.embedding@1")
|
||||
|
||||
|
||||
def authenticate_rag_reranking_client(
|
||||
request: Request,
|
||||
service: Service,
|
||||
authorization: Annotated[str | None, Header(alias="Authorization")] = None,
|
||||
) -> ServiceClient:
|
||||
return _authenticate_capability_client(request, service, authorization, "rag.reranking@1")
|
||||
|
||||
|
||||
def authenticate_document_ocr_client(
|
||||
request: Request,
|
||||
service: Service,
|
||||
authorization: Annotated[str | None, Header(alias="Authorization")] = None,
|
||||
) -> ServiceClient:
|
||||
return _authenticate_capability_client(request, service, authorization, "document.ocr@1")
|
||||
|
||||
|
||||
def authenticate_vision_embedding_client(
|
||||
request: Request,
|
||||
service: Service,
|
||||
authorization: Annotated[str | None, Header(alias="Authorization")] = None,
|
||||
) -> ServiceClient:
|
||||
return _authenticate_capability_client(request, service, authorization, "vision.embedding@1")
|
||||
|
||||
|
||||
def authenticate_speech_transcription_client(
|
||||
request: Request,
|
||||
service: Service,
|
||||
authorization: Annotated[str | None, Header(alias="Authorization")] = None,
|
||||
) -> ServiceClient:
|
||||
return _authenticate_capability_client(
|
||||
request, service, authorization, "speech.transcription@1"
|
||||
)
|
||||
|
||||
|
||||
RagEmbeddingClient = Annotated[ServiceClient, Depends(authenticate_rag_embedding_client)]
|
||||
RagRerankingClient = Annotated[ServiceClient, Depends(authenticate_rag_reranking_client)]
|
||||
DocumentOcrClient = Annotated[ServiceClient, Depends(authenticate_document_ocr_client)]
|
||||
VisionEmbeddingClient = Annotated[ServiceClient, Depends(authenticate_vision_embedding_client)]
|
||||
SpeechTranscriptionClient = Annotated[
|
||||
ServiceClient, Depends(authenticate_speech_transcription_client)
|
||||
]
|
||||
|
||||
|
||||
@router.post(
|
||||
"/api/v1/admin/deployment-candidates/{candidate_id}/production-approvals",
|
||||
response_model=ProductionApprovalResponse,
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
)
|
||||
def approve_production(
|
||||
candidate_id: uuid.UUID,
|
||||
request: ProductionApprovalCreate,
|
||||
service: Service,
|
||||
_admin: Admin,
|
||||
) -> ProductionApprovalResponse:
|
||||
return service.approve_production(candidate_id, request)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/api/v1/admin/production-approvals",
|
||||
response_model=list[ProductionApprovalResponse],
|
||||
)
|
||||
def production_approvals(service: Service, _admin: Admin) -> list[ProductionApprovalResponse]:
|
||||
return service.approvals()
|
||||
|
||||
|
||||
@router.post(
|
||||
"/api/v1/admin/deployment-candidates/{candidate_id}/promote",
|
||||
response_model=CapabilityDeploymentResponse,
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
)
|
||||
def promote_candidate(
|
||||
candidate_id: uuid.UUID,
|
||||
request: CapabilityPromotionCreate,
|
||||
service: Service,
|
||||
_admin: Admin,
|
||||
) -> CapabilityDeploymentResponse:
|
||||
return service.promote(candidate_id, request)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/api/v1/capability-deployments",
|
||||
response_model=list[CapabilityDeploymentResponse],
|
||||
dependencies=[Depends(require_viewer)],
|
||||
)
|
||||
def capability_deployments(service: Service) -> list[CapabilityDeploymentResponse]:
|
||||
return service.deployments()
|
||||
|
||||
|
||||
@router.post(
|
||||
"/api/v1/admin/deployment-candidates/{candidate_id}/experiments",
|
||||
response_model=CapabilityExperimentResponse,
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
)
|
||||
def create_capability_experiment(
|
||||
candidate_id: uuid.UUID,
|
||||
request: CapabilityExperimentCreate,
|
||||
service: Service,
|
||||
_admin: Admin,
|
||||
) -> CapabilityExperimentResponse:
|
||||
return service.create_experiment(candidate_id, request)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/api/v1/capability-experiments",
|
||||
response_model=list[CapabilityExperimentResponse],
|
||||
dependencies=[Depends(require_viewer)],
|
||||
)
|
||||
def capability_experiments(service: Service) -> list[CapabilityExperimentResponse]:
|
||||
return service.experiments()
|
||||
|
||||
|
||||
@router.post(
|
||||
"/api/v1/admin/capability-experiments/{experiment_id}/deactivate",
|
||||
response_model=CapabilityExperimentResponse,
|
||||
)
|
||||
def deactivate_capability_experiment(
|
||||
experiment_id: uuid.UUID, service: Service, _admin: Admin
|
||||
) -> CapabilityExperimentResponse:
|
||||
return service.deactivate_experiment(experiment_id)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/api/v1/admin/capability-deployments/{deployment_id}/unload",
|
||||
response_model=CapabilityDeploymentResponse,
|
||||
)
|
||||
def unload_deployment(
|
||||
deployment_id: uuid.UUID, service: Service, _admin: Admin
|
||||
) -> CapabilityDeploymentResponse:
|
||||
return service.request_unload(deployment_id)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/api/v1/admin/capability-deployments/{deployment_id}/drain",
|
||||
response_model=CapabilityDeploymentResponse,
|
||||
)
|
||||
def drain_deployment(
|
||||
deployment_id: uuid.UUID, service: Service, _admin: Admin
|
||||
) -> CapabilityDeploymentResponse:
|
||||
return service.request_unload(deployment_id, drain=True)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/api/v1/admin/service-clients",
|
||||
response_model=ServiceClientCreated,
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
)
|
||||
def create_service_client(
|
||||
request: ServiceClientCreate,
|
||||
service: Service,
|
||||
_admin: Admin,
|
||||
response: Response,
|
||||
) -> ServiceClientCreated:
|
||||
response.headers["Cache-Control"] = "no-store"
|
||||
return service.create_client(request)
|
||||
|
||||
|
||||
@router.get("/api/v1/admin/service-clients", response_model=list[ServiceClientResponse])
|
||||
def service_clients(service: Service, _admin: Admin) -> list[ServiceClientResponse]:
|
||||
return service.clients()
|
||||
|
||||
|
||||
@router.delete(
|
||||
"/api/v1/admin/service-clients/{client_id}/credential",
|
||||
response_model=ServiceClientResponse,
|
||||
)
|
||||
def revoke_service_credential(
|
||||
client_id: uuid.UUID, service: Service, _admin: Admin
|
||||
) -> ServiceClientResponse:
|
||||
return service.revoke_client_credential(client_id)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/api/v1/admin/service-clients/{client_id}/credential/rotate",
|
||||
response_model=ServiceClientCreated,
|
||||
)
|
||||
def rotate_service_credential(
|
||||
client_id: uuid.UUID, service: Service, _admin: Admin, response: Response
|
||||
) -> ServiceClientCreated:
|
||||
response.headers["Cache-Control"] = "no-store"
|
||||
return service.rotate_client_credential(client_id)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/api/v1/project-integrations",
|
||||
response_model=list[ProjectIntegrationResponse],
|
||||
dependencies=[Depends(require_viewer)],
|
||||
)
|
||||
def project_integrations(service: Service) -> list[ProjectIntegrationResponse]:
|
||||
return service.project_integrations()
|
||||
|
||||
|
||||
@router.post(
|
||||
"/api/v1/admin/project-fit-evidence",
|
||||
response_model=ProjectFitEvidenceResponse,
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
)
|
||||
def record_project_fit_evidence(
|
||||
request: ProjectFitEvidenceCreate, service: Service, _admin: Admin
|
||||
) -> ProjectFitEvidenceResponse:
|
||||
return service.record_project_fit(request)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/api/v1/scheduler",
|
||||
response_model=list[SchedulerBudgetResponse],
|
||||
dependencies=[Depends(require_viewer)],
|
||||
)
|
||||
def scheduler_overview(service: Service) -> list[SchedulerBudgetResponse]:
|
||||
return service.scheduler_overview()
|
||||
|
||||
|
||||
@router.get(
|
||||
"/api/v1/scheduler/co-residency",
|
||||
response_model=list[CoResidencyEvidenceResponse],
|
||||
dependencies=[Depends(require_viewer)],
|
||||
)
|
||||
def co_residency_matrix(service: Service) -> list[CoResidencyEvidenceResponse]:
|
||||
return service.co_residency_matrix()
|
||||
|
||||
|
||||
@router.post(
|
||||
"/api/v1/admin/scheduler/co-residency-evidence",
|
||||
response_model=CoResidencyEvidenceResponse,
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
)
|
||||
def record_co_residency_evidence(
|
||||
request: CoResidencyEvidenceCreate, service: Service, _admin: Admin
|
||||
) -> CoResidencyEvidenceResponse:
|
||||
return service.record_co_residency_evidence(request)
|
||||
|
||||
|
||||
@router.get("/api/v1/admin/scheduler/policy", response_model=SchedulerPolicyResponse)
|
||||
def scheduler_policy(service: Service, _admin: Admin) -> SchedulerPolicyResponse:
|
||||
return service.scheduler_policy()
|
||||
|
||||
|
||||
@router.get("/api/v1/admin/scheduler/metrics", response_model=SchedulerMetricsResponse)
|
||||
def scheduler_metrics(service: Service, _admin: Admin) -> SchedulerMetricsResponse:
|
||||
return service.scheduler_metrics()
|
||||
|
||||
|
||||
@router.put("/api/v1/admin/scheduler/policy", response_model=SchedulerPolicyResponse)
|
||||
def update_scheduler_policy(
|
||||
request: SchedulerPolicyUpdate, service: Service, _admin: Admin
|
||||
) -> SchedulerPolicyResponse:
|
||||
return service.update_scheduler_policy(request)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/api/v1/admin/scheduler/placements/{deployment_id}/dry-run",
|
||||
response_model=PlacementPlanResponse,
|
||||
)
|
||||
def dry_run_placement(
|
||||
deployment_id: uuid.UUID,
|
||||
request: PlacementPlanRequest,
|
||||
service: Service,
|
||||
_admin: Admin,
|
||||
) -> PlacementPlanResponse:
|
||||
return service.dry_run_placement(deployment_id, request)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/api/v1/admin/scheduler/placements",
|
||||
response_model=list[PlacementPlanResponse],
|
||||
)
|
||||
def placement_history(
|
||||
service: Service,
|
||||
_admin: Admin,
|
||||
limit: Annotated[int, Query(ge=1, le=500)] = 100,
|
||||
) -> list[PlacementPlanResponse]:
|
||||
return service.placement_history(limit)
|
||||
|
||||
|
||||
@router.put(
|
||||
"/api/v1/admin/capability-deployments/{deployment_id}/residency-policy",
|
||||
response_model=CapabilityDeploymentResponse,
|
||||
)
|
||||
def update_residency_policy(
|
||||
deployment_id: uuid.UUID,
|
||||
request: ResidencyPolicyUpdate,
|
||||
service: Service,
|
||||
_admin: Admin,
|
||||
) -> CapabilityDeploymentResponse:
|
||||
return service.update_residency_policy(deployment_id, request)
|
||||
|
||||
|
||||
@router.get("/api/v1/gateway/requests", response_model=list[GatewayRequestResponse])
|
||||
def gateway_requests(
|
||||
service: Service,
|
||||
_admin: Admin,
|
||||
limit: Annotated[int, Query(ge=1, le=500)] = 100,
|
||||
) -> list[GatewayRequestResponse]:
|
||||
return service.request_history(limit)
|
||||
|
||||
|
||||
@router.get("/api/v1/admin/latency-traces", response_model=list[GatewayRequestResponse])
|
||||
def latency_traces(
|
||||
service: Service,
|
||||
_admin: Admin,
|
||||
limit: Annotated[int, Query(ge=1, le=500)] = 100,
|
||||
) -> list[GatewayRequestResponse]:
|
||||
"""Return bounded span summaries; request content and input digests are excluded."""
|
||||
return service.request_history(limit)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/api/v1/capabilities/rag.embedding@1/invoke",
|
||||
response_model=StableEmbeddingInvokeResponse,
|
||||
)
|
||||
def invoke_embedding(
|
||||
request: EmbeddingInvokeRequest,
|
||||
service: Service,
|
||||
client: RagEmbeddingClient,
|
||||
) -> StableEmbeddingInvokeResponse:
|
||||
return StableEmbeddingInvokeResponse.model_validate(service.invoke(request, client))
|
||||
|
||||
|
||||
@router.post(
|
||||
"/api/v1/capabilities/rag.reranking@1/invoke",
|
||||
response_model=RerankingInvokeResponse,
|
||||
)
|
||||
def invoke_reranking(
|
||||
request: RerankingInvokeRequest,
|
||||
service: Service,
|
||||
client: RagRerankingClient,
|
||||
) -> RerankingInvokeResponse:
|
||||
return service.invoke_reranking(request, client)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/api/v1/capabilities/document.ocr@1/invoke",
|
||||
response_model=OCRInvokeResponse,
|
||||
)
|
||||
def invoke_ocr(
|
||||
request: OCRInvokeRequest,
|
||||
service: Service,
|
||||
client: DocumentOcrClient,
|
||||
) -> OCRInvokeResponse:
|
||||
result, request_id, execution = service.invoke_modality(
|
||||
"document.ocr", request.model_dump(mode="json"), client
|
||||
)
|
||||
return OCRInvokeResponse(request_id=request_id, execution=execution, **result)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/api/v1/capabilities/vision.embedding@1/invoke",
|
||||
response_model=VisionEmbeddingInvokeResponse,
|
||||
)
|
||||
def invoke_vision_embedding(
|
||||
request: VisionEmbeddingInvokeRequest,
|
||||
service: Service,
|
||||
client: VisionEmbeddingClient,
|
||||
) -> VisionEmbeddingInvokeResponse:
|
||||
result, request_id, execution = service.invoke_modality(
|
||||
"vision.embedding", request.model_dump(mode="json"), client
|
||||
)
|
||||
return VisionEmbeddingInvokeResponse(
|
||||
request_id=request_id,
|
||||
execution=execution,
|
||||
dimension=int(result["dimension"]),
|
||||
embedding_space_id=result["embedding_space_id"],
|
||||
data=result["vectors"],
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/api/v1/capabilities/speech.transcription@1/invoke",
|
||||
response_model=SpeechTranscriptionInvokeResponse,
|
||||
)
|
||||
def invoke_speech_transcription(
|
||||
request: SpeechTranscriptionInvokeRequest,
|
||||
service: Service,
|
||||
client: SpeechTranscriptionClient,
|
||||
) -> SpeechTranscriptionInvokeResponse:
|
||||
result, request_id, execution = service.invoke_modality(
|
||||
"speech.transcription", request.model_dump(mode="json"), client
|
||||
)
|
||||
return SpeechTranscriptionInvokeResponse(
|
||||
request_id=request_id,
|
||||
execution=execution,
|
||||
**result,
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/api/v1/capability-experiments/{route_key}/invoke",
|
||||
response_model=EmbeddingInvokeResponse,
|
||||
)
|
||||
def invoke_embedding_experiment(
|
||||
route_key: str,
|
||||
request: EmbeddingInvokeRequest,
|
||||
service: Service,
|
||||
client: RagEmbeddingClient,
|
||||
) -> EmbeddingInvokeResponse:
|
||||
route = service.repo.experiment_route(route_key)
|
||||
deployment = (
|
||||
service.repo.deployment(route.capability_deployment_id)
|
||||
if route and route.status == "active"
|
||||
else None
|
||||
)
|
||||
if not deployment:
|
||||
raise ServingError(404, "EXPERIMENT_NOT_FOUND", "capability experiment is unavailable")
|
||||
return service.invoke(
|
||||
request,
|
||||
client,
|
||||
deployment=deployment,
|
||||
experiment_route=route_key,
|
||||
)
|
||||
|
||||
|
||||
@router.post("/v1/embeddings", response_model=OpenAIEmbeddingResponse)
|
||||
def openai_embeddings(
|
||||
request: OpenAIEmbeddingRequest,
|
||||
service: Service,
|
||||
client: RagEmbeddingClient,
|
||||
) -> OpenAIEmbeddingResponse:
|
||||
native = service.invoke(EmbeddingInvokeRequest(input=request.input), client)
|
||||
return OpenAIEmbeddingResponse(
|
||||
data=[
|
||||
OpenAIEmbeddingItem(index=index, embedding=embedding)
|
||||
for index, embedding in enumerate(native.data)
|
||||
],
|
||||
usage=OpenAIUsage(
|
||||
prompt_tokens=native.usage.input_tokens,
|
||||
total_tokens=native.usage.input_tokens,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/api/v1/agent/serving-jobs/next",
|
||||
response_model=AgentServingJobLease | None,
|
||||
)
|
||||
def claim_serving_job(
|
||||
service: Service,
|
||||
identity: AgentIdentity,
|
||||
wait_seconds: Annotated[float, Query(ge=0.0, le=5.0)] = 0.0,
|
||||
) -> AgentServingJobLease | None:
|
||||
_credential, node = identity
|
||||
return service.claim_next(node, wait_seconds=wait_seconds)
|
||||
|
||||
|
||||
@router.post("/api/v1/agent/serving-jobs/{job_id}/complete")
|
||||
def complete_serving_job(
|
||||
job_id: uuid.UUID,
|
||||
request: AgentServingJobComplete,
|
||||
service: Service,
|
||||
identity: AgentIdentity,
|
||||
) -> dict[str, str]:
|
||||
_credential, node = identity
|
||||
service.complete_job(job_id, node, request)
|
||||
return {"status": "accepted"}
|
||||
|
||||
|
||||
@router.post("/api/v1/agent/serving-jobs/{job_id}/fail")
|
||||
def fail_serving_job(
|
||||
job_id: uuid.UUID,
|
||||
request: AgentServingJobFailure,
|
||||
service: Service,
|
||||
identity: AgentIdentity,
|
||||
) -> dict[str, str]:
|
||||
_credential, node = identity
|
||||
service.fail_job(job_id, node, request)
|
||||
return {"status": "accepted"}
|
||||
|
||||
|
||||
@router.post(
|
||||
"/api/v1/agent/serving-state",
|
||||
response_model=AgentServingStateAck,
|
||||
)
|
||||
def report_serving_state(
|
||||
request: AgentServingStateReport,
|
||||
service: Service,
|
||||
identity: AgentIdentity,
|
||||
) -> AgentServingStateAck:
|
||||
_credential, node = identity
|
||||
return service.report_state(node, request)
|
||||
@@ -0,0 +1 @@
|
||||
"""ModelForge operator command-line surfaces."""
|
||||
@@ -0,0 +1,334 @@
|
||||
"""Operator disaster-recovery CLI.
|
||||
|
||||
Restoring over a live database is an operator action with a runbook behind it, not a remote API
|
||||
call, so the destructive half of recovery lives here. Every subcommand takes typed arguments and
|
||||
runs fixed operations: there is no pass-through shell, and no argument reaches a shell.
|
||||
|
||||
python -m modelforge_api.cli.dr create-backup --backup-id m15-rehearsal-a --reason "..."
|
||||
python -m modelforge_api.cli.dr verify-backup --backup-id m15-rehearsal-a
|
||||
python -m modelforge_api.cli.dr plan-restore --backup-id m15-rehearsal-a --target-url ...
|
||||
python -m modelforge_api.cli.dr validate-restore --plan-id <uuid>
|
||||
python -m modelforge_api.cli.dr run-restore --plan-id <uuid> --reason "..."
|
||||
python -m modelforge_api.cli.dr readiness
|
||||
python -m modelforge_api.cli.dr bundle --backup-id m15-rehearsal-a
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import uuid
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from modelforge_api.db import build_engine
|
||||
from modelforge_api.domain.recovery import (
|
||||
POINT_IN_TIME_SUPPORT,
|
||||
BackupSetCreate,
|
||||
RestoreAdvanceRequest,
|
||||
RestoreMode,
|
||||
RestorePlanCreate,
|
||||
redact_database_url,
|
||||
)
|
||||
from modelforge_api.persistence.models import BackupSet
|
||||
from modelforge_api.services.recovery import RecoveryError, RecoveryService
|
||||
from modelforge_api.settings import get_settings
|
||||
|
||||
|
||||
def _service(session: Session) -> RecoveryService:
|
||||
return RecoveryService(session, get_settings(), "operator", "dr-cli")
|
||||
|
||||
|
||||
def _emit(payload: Any) -> None:
|
||||
print(json.dumps(payload, indent=2, default=str))
|
||||
|
||||
|
||||
def _backup_by_id(session: Session, backup_id: str) -> BackupSet:
|
||||
record = session.scalar(select(BackupSet).where(BackupSet.backup_id == backup_id))
|
||||
if record is None:
|
||||
raise RecoveryError(404, "backup_not_found", f"no backup set named {backup_id}")
|
||||
return record
|
||||
|
||||
|
||||
def _cmd_create_backup(session: Session, args: argparse.Namespace) -> int:
|
||||
service = _service(session)
|
||||
service.ensure_defaults()
|
||||
response = service.create_backup(
|
||||
BackupSetCreate(
|
||||
backup_id=args.backup_id,
|
||||
reason=args.reason,
|
||||
milestone=args.milestone,
|
||||
legal_hold=args.legal_hold,
|
||||
created_by=args.actor,
|
||||
)
|
||||
)
|
||||
_emit(json.loads(response.model_dump_json()))
|
||||
return 0 if response.state.value in {"CREATED", "VERIFIED"} else 1
|
||||
|
||||
|
||||
def _cmd_verify_backup(session: Session, args: argparse.Namespace) -> int:
|
||||
service = _service(session)
|
||||
record = _backup_by_id(session, args.backup_id)
|
||||
response = service.verify_backup(record.id)
|
||||
_emit(
|
||||
{
|
||||
"backup_id": response.backup_id,
|
||||
"state": response.state.value,
|
||||
"restore_eligible": response.restore_eligible,
|
||||
"manifest_sha256": response.manifest_sha256,
|
||||
"failure_code": response.failure_code,
|
||||
"failure_reason": response.failure_reason,
|
||||
"verification": response.verification_details.get("verification", {}),
|
||||
}
|
||||
)
|
||||
return 0 if response.restore_eligible else 1
|
||||
|
||||
|
||||
def _cmd_list_backups(session: Session, args: argparse.Namespace) -> int:
|
||||
service = _service(session)
|
||||
_emit(
|
||||
[
|
||||
{
|
||||
"backup_id": item.backup_id,
|
||||
"state": item.state.value,
|
||||
"restore_eligible": item.restore_eligible,
|
||||
"schema_revision": item.schema_revision,
|
||||
"payload_bytes": item.payload_bytes,
|
||||
"encrypted": item.encrypted,
|
||||
"verified_at": item.verified_at,
|
||||
"expires_at": item.expires_at,
|
||||
}
|
||||
for item in service.backups(args.limit)
|
||||
]
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
def _cmd_plan_restore(session: Session, args: argparse.Namespace) -> int:
|
||||
service = _service(session)
|
||||
record = _backup_by_id(session, args.backup_id)
|
||||
response = service.create_restore_plan(
|
||||
RestorePlanCreate(
|
||||
backup_set_id=record.id,
|
||||
mode=RestoreMode(args.mode),
|
||||
target_environment=args.target_environment,
|
||||
target_label=args.target_label,
|
||||
database_destination=args.target_url,
|
||||
artifact_strategy=args.artifact_strategy,
|
||||
secret_strategy=args.secret_strategy,
|
||||
node_strategy=args.node_strategy,
|
||||
reason=args.reason,
|
||||
created_by=args.actor,
|
||||
)
|
||||
)
|
||||
_emit(json.loads(response.model_dump_json()))
|
||||
return 0
|
||||
|
||||
|
||||
def _cmd_validate_restore(session: Session, args: argparse.Namespace) -> int:
|
||||
service = _service(session)
|
||||
response = service.preflight(uuid.UUID(args.plan_id))
|
||||
_emit(
|
||||
{
|
||||
"plan_id": str(response.id),
|
||||
"backup_id": response.backup_id,
|
||||
"state": response.state.value,
|
||||
"destination": response.database_destination_redacted,
|
||||
"preflight": response.preflight,
|
||||
}
|
||||
)
|
||||
return 0 if response.preflight.get("status") == "PASS" else 1
|
||||
|
||||
|
||||
def _cmd_run_restore(session: Session, args: argparse.Namespace) -> int:
|
||||
service = _service(session)
|
||||
plan_id = uuid.UUID(args.plan_id)
|
||||
request = RestoreAdvanceRequest(actor=args.actor, reason=args.reason)
|
||||
operation = service.start_restore(plan_id, request)
|
||||
operation = service.advance_restore(operation.id, request)
|
||||
_emit(
|
||||
{
|
||||
"operation_id": str(operation.id),
|
||||
"backup_id": operation.backup_id,
|
||||
"state": operation.state.value,
|
||||
"attempt": operation.attempt,
|
||||
"phase_durations": operation.phase_durations,
|
||||
"rto_seconds": operation.rto_seconds,
|
||||
"rpo_seconds": operation.rpo_seconds,
|
||||
"validation": operation.validation_result,
|
||||
"fingerprint_diff": operation.fingerprint_diff,
|
||||
"failure_code": operation.failure_code,
|
||||
"failure_reason": operation.failure_reason,
|
||||
}
|
||||
)
|
||||
return 0 if operation.state.value == "READY" else 1
|
||||
|
||||
|
||||
def _cmd_readiness(session: Session, _args: argparse.Namespace) -> int:
|
||||
service = _service(session)
|
||||
dashboard = service.dashboard()
|
||||
_emit(json.loads(dashboard.model_dump_json()))
|
||||
return 0 if not dashboard.unprotected_assets and not dashboard.stale_backup else 1
|
||||
|
||||
|
||||
def _cmd_bundle(session: Session, args: argparse.Namespace) -> int:
|
||||
"""Print the bounded DR bundle manifest: what an operator needs to rebuild from nothing."""
|
||||
|
||||
service = _service(session)
|
||||
record = _backup_by_id(session, args.backup_id)
|
||||
backup = service.backup(record.id)
|
||||
settings = get_settings()
|
||||
_emit(
|
||||
{
|
||||
"modelforge": {
|
||||
"version": backup.modelforge_version,
|
||||
"commit": backup.modelforge_commit,
|
||||
"reference": backup.environment_fingerprint.get("source_reference"),
|
||||
"repository": record.source_repository,
|
||||
},
|
||||
"database_backup": {
|
||||
"backup_id": backup.backup_id,
|
||||
"state": backup.state.value,
|
||||
"restore_eligible": backup.restore_eligible,
|
||||
"destination_root": backup.destination_root,
|
||||
"manifest": backup.manifest_relative_path,
|
||||
"manifest_sha256": backup.manifest_sha256,
|
||||
"payload_bytes": backup.payload_bytes,
|
||||
"schema_revision": backup.schema_revision,
|
||||
"database_identity": backup.database_identity,
|
||||
"point_in_time_support": POINT_IN_TIME_SUPPORT,
|
||||
},
|
||||
"encryption": {
|
||||
"encrypted": backup.encrypted,
|
||||
"algorithm": backup.encryption_algorithm,
|
||||
"key_id": backup.encryption_key_id,
|
||||
"key_requirement": (
|
||||
"MODELFORGE_BACKUP_ENCRYPTION_KEY must be supplied by the operator; it is "
|
||||
"never written into a backup"
|
||||
),
|
||||
},
|
||||
"host_configuration": {
|
||||
"backup_root": str(settings.backup_root),
|
||||
"artifact_root": settings.artifact_root,
|
||||
"quarantine_root": settings.quarantine_root,
|
||||
"config_root": str(settings.config_root),
|
||||
"database_url": redact_database_url(settings.database_url),
|
||||
},
|
||||
"artifact_recovery_plan": [
|
||||
{
|
||||
"object": entry.object_name,
|
||||
"type": entry.logical_asset_type,
|
||||
"sha256": entry.sha256,
|
||||
"size_bytes": entry.size_bytes,
|
||||
}
|
||||
for entry in backup.entries
|
||||
],
|
||||
"node_enrollment": (
|
||||
"Issue a fresh enrollment token through /api/v1/admin/node-enrollments and "
|
||||
"revoke any credential believed lost; hardware identity is persisted so a "
|
||||
"recovered node keeps its node id."
|
||||
),
|
||||
"external_dependencies": [
|
||||
item.asset_key
|
||||
for item in service.readiness()
|
||||
if item.readiness.value == "EXTERNAL_DEPENDENCY"
|
||||
],
|
||||
"runbooks": [
|
||||
"docs/operations/RUNBOOK_FULL_DR.md",
|
||||
"docs/operations/RUNBOOK_DATABASE_RESTORE.md",
|
||||
"docs/operations/RUNBOOK_CONTROL_PLANE_LOSS.md",
|
||||
"docs/operations/RUNBOOK_NODE_LOSS.md",
|
||||
"docs/operations/RUNBOOK_ARTIFACT_LOSS.md",
|
||||
"docs/operations/RUNBOOK_BACKUP_FAILURE.md",
|
||||
],
|
||||
}
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="modelforge-dr", description="ModelForge disaster-recovery operator commands"
|
||||
)
|
||||
parser.add_argument("--database-url", default=None, help="override the control-plane database")
|
||||
subparsers = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
create = subparsers.add_parser("create-backup", help="create a new backup set")
|
||||
create.add_argument("--backup-id", required=True)
|
||||
create.add_argument("--reason", required=True)
|
||||
create.add_argument("--milestone", default=None)
|
||||
create.add_argument("--legal-hold", action="store_true")
|
||||
create.add_argument("--actor", default="operator")
|
||||
create.set_defaults(handler=_cmd_create_backup)
|
||||
|
||||
verify = subparsers.add_parser("verify-backup", help="verify a backup set end to end")
|
||||
verify.add_argument("--backup-id", required=True)
|
||||
verify.set_defaults(handler=_cmd_verify_backup)
|
||||
|
||||
listing = subparsers.add_parser("list-backups", help="list journaled backup sets")
|
||||
listing.add_argument("--limit", type=int, default=25)
|
||||
listing.set_defaults(handler=_cmd_list_backups)
|
||||
|
||||
plan = subparsers.add_parser("plan-restore", help="create a restore plan")
|
||||
plan.add_argument("--backup-id", required=True)
|
||||
plan.add_argument("--target-url", required=True)
|
||||
plan.add_argument("--target-label", required=True)
|
||||
plan.add_argument(
|
||||
"--mode", choices=[item.value for item in RestoreMode], default=RestoreMode.VALIDATION.value
|
||||
)
|
||||
plan.add_argument(
|
||||
"--target-environment", choices=["ISOLATED", "STAGING", "PRODUCTION"], default="ISOLATED"
|
||||
)
|
||||
plan.add_argument(
|
||||
"--artifact-strategy",
|
||||
choices=["NONE", "MANIFEST_ONLY", "REHYDRATE_MISSING", "RESTORE_LOCAL"],
|
||||
default="MANIFEST_ONLY",
|
||||
)
|
||||
plan.add_argument(
|
||||
"--secret-strategy", choices=["ROTATE", "RESTORE_HASHES", "MANUAL"], default="RESTORE_HASHES"
|
||||
)
|
||||
plan.add_argument(
|
||||
"--node-strategy", choices=["REUSE_CREDENTIAL", "RE_ENROLL", "NONE"], default="NONE"
|
||||
)
|
||||
plan.add_argument("--reason", required=True)
|
||||
plan.add_argument("--actor", default="operator")
|
||||
plan.set_defaults(handler=_cmd_plan_restore)
|
||||
|
||||
validate = subparsers.add_parser("validate-restore", help="run the restore preflight")
|
||||
validate.add_argument("--plan-id", required=True)
|
||||
validate.set_defaults(handler=_cmd_validate_restore)
|
||||
|
||||
run = subparsers.add_parser("run-restore", help="execute a restore plan to completion")
|
||||
run.add_argument("--plan-id", required=True)
|
||||
run.add_argument("--reason", required=True)
|
||||
run.add_argument("--actor", default="operator")
|
||||
run.set_defaults(handler=_cmd_run_restore)
|
||||
|
||||
readiness = subparsers.add_parser("readiness", help="print recovery readiness")
|
||||
readiness.set_defaults(handler=_cmd_readiness)
|
||||
|
||||
bundle = subparsers.add_parser("bundle", help="print the operator DR bundle manifest")
|
||||
bundle.add_argument("--backup-id", required=True)
|
||||
bundle.set_defaults(handler=_cmd_bundle)
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = build_parser().parse_args(argv)
|
||||
engine = build_engine(args.database_url)
|
||||
try:
|
||||
with Session(engine) as session:
|
||||
handler = args.handler
|
||||
return int(handler(session, args))
|
||||
except RecoveryError as error:
|
||||
_emit({"error": {"code": error.code, "message": error.message, "details": error.details}})
|
||||
return 2
|
||||
finally:
|
||||
engine.dispose()
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover - operator entry point
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,38 @@
|
||||
from collections.abc import Generator
|
||||
from typing import Any, cast
|
||||
|
||||
from sqlalchemy import CursorResult, Engine, create_engine
|
||||
from sqlalchemy.engine import Result
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from modelforge_api.persistence.runtime_engine import register_application_engine
|
||||
from modelforge_api.settings import get_settings
|
||||
|
||||
|
||||
def rows_affected(result: Result[Any]) -> int:
|
||||
"""How many rows a DML statement actually changed.
|
||||
|
||||
`Session.execute` is typed as returning `Result`, which carries no `rowcount`; a DML statement
|
||||
always returns a `CursorResult`, which does. SQLAlchemy 2.0.52 narrowed that return type, so
|
||||
reading `.rowcount` directly stopped type-checking — while continuing to work at runtime.
|
||||
|
||||
The cast is where that knowledge lives, once, rather than at ten call sites. It matters more
|
||||
than it looks: every single-use claim in the platform is a conditional UPDATE whose row count
|
||||
decides the winner, and that is what makes enrolment tokens and lifecycle claims atomic instead
|
||||
of merely usually-correct.
|
||||
"""
|
||||
|
||||
return cast("CursorResult[Any]", result).rowcount
|
||||
|
||||
|
||||
def build_engine(database_url: str | None = None) -> Engine:
|
||||
url = database_url or get_settings().database_url
|
||||
return register_application_engine(create_engine(url, pool_pre_ping=True))
|
||||
|
||||
|
||||
engine = build_engine()
|
||||
|
||||
|
||||
def get_session() -> Generator[Session, None, None]:
|
||||
with Session(engine) as session:
|
||||
yield session
|
||||
@@ -0,0 +1,217 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator
|
||||
|
||||
|
||||
class AcquisitionModel(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", from_attributes=True)
|
||||
|
||||
|
||||
class DiscoverySearchRequest(AcquisitionModel):
|
||||
query: str = Field(min_length=1, max_length=255)
|
||||
limit: int = Field(default=20, ge=1, le=100)
|
||||
sort: Literal["downloads", "likes", "last_modified"] = "downloads"
|
||||
pipeline_tag: str | None = Field(default=None, max_length=128)
|
||||
|
||||
|
||||
class DiscoveryCandidate(AcquisitionModel):
|
||||
repository_id: str
|
||||
resolved_commit_sha: str | None = None
|
||||
access_state: str
|
||||
pipeline_tag: str | None = None
|
||||
library_name: str | None = None
|
||||
tags: list[str] = Field(default_factory=list)
|
||||
downloads: int | None = None
|
||||
likes: int | None = None
|
||||
last_modified: datetime | None = None
|
||||
matched_model_id: uuid.UUID | None = None
|
||||
upstream_facts: dict[str, Any] = Field(default_factory=dict)
|
||||
local_interpretation: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class UpstreamRefreshRequest(AcquisitionModel):
|
||||
revision: str = Field(default="main", min_length=1, max_length=255)
|
||||
|
||||
|
||||
class UpstreamFileResponse(AcquisitionModel):
|
||||
id: uuid.UUID
|
||||
path: str
|
||||
size_bytes: int | None
|
||||
blob_id: str | None
|
||||
upstream_sha256: str | None
|
||||
file_format: str
|
||||
role: str
|
||||
risk_flags: list[str]
|
||||
metadata_snapshot: dict[str, Any]
|
||||
|
||||
|
||||
class UpstreamSnapshotResponse(AcquisitionModel):
|
||||
id: uuid.UUID
|
||||
model_id: uuid.UUID | None
|
||||
provider: str
|
||||
repository_id: str
|
||||
requested_revision: str
|
||||
resolved_commit_sha: str
|
||||
access_state: str
|
||||
metadata_snapshot: dict[str, Any]
|
||||
card_metadata: dict[str, Any]
|
||||
security_metadata: dict[str, Any]
|
||||
source_updated_at: datetime | None
|
||||
observed_at: datetime
|
||||
stale_after: datetime
|
||||
stale: bool
|
||||
files: list[UpstreamFileResponse]
|
||||
|
||||
|
||||
class ArtifactSetResponse(AcquisitionModel):
|
||||
id: uuid.UUID
|
||||
revision_id: uuid.UUID
|
||||
snapshot_id: uuid.UUID
|
||||
variant_key: str
|
||||
label: str
|
||||
selection_reason: str
|
||||
selected_paths: list[str]
|
||||
total_size_bytes: int
|
||||
file_count: int
|
||||
availability: str
|
||||
status: str
|
||||
completeness: str
|
||||
security_status: str
|
||||
license_status: str
|
||||
immutable_at: datetime
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
|
||||
|
||||
class DownloadPlanCreate(AcquisitionModel):
|
||||
artifact_set_id: uuid.UUID
|
||||
compute_node_id: uuid.UUID
|
||||
storage_root_id: uuid.UUID
|
||||
expires_in_seconds: int = Field(default=3600, ge=300, le=86400)
|
||||
|
||||
|
||||
class DownloadPlanFileResponse(AcquisitionModel):
|
||||
ordinal: int
|
||||
path: str
|
||||
size_bytes: int
|
||||
upstream_sha256: str | None
|
||||
file_format: str
|
||||
role: str
|
||||
risk_flags: list[str]
|
||||
|
||||
|
||||
class DownloadPlanResponse(AcquisitionModel):
|
||||
id: uuid.UUID
|
||||
artifact_set_id: uuid.UUID
|
||||
compute_node_id: uuid.UUID
|
||||
storage_root_id: uuid.UUID
|
||||
repository_id: str
|
||||
resolved_commit_sha: str
|
||||
total_size_bytes: int
|
||||
file_count: int
|
||||
status: str
|
||||
idempotency_key: str
|
||||
preflight: dict[str, Any]
|
||||
immutable_payload: dict[str, Any]
|
||||
planned_at: datetime
|
||||
expires_at: datetime
|
||||
immutable_at: datetime
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
stale: bool
|
||||
files: list[DownloadPlanFileResponse]
|
||||
|
||||
|
||||
class ArtifactJobResponse(AcquisitionModel):
|
||||
id: uuid.UUID
|
||||
plan_id: uuid.UUID
|
||||
compute_node_id: uuid.UUID
|
||||
storage_root_id: uuid.UUID
|
||||
status: str
|
||||
attempt_count: int
|
||||
progress_bytes: int
|
||||
total_bytes: int
|
||||
current_file: str | None
|
||||
cancel_requested: bool
|
||||
quarantine_relative_path: str | None
|
||||
promoted_relative_path: str | None
|
||||
error_code: str | None
|
||||
error_message: str | None
|
||||
result: dict[str, Any]
|
||||
started_at: datetime | None
|
||||
completed_at: datetime | None
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
|
||||
|
||||
class AgentArtifactJobFile(AcquisitionModel):
|
||||
ordinal: int
|
||||
path: str
|
||||
size_bytes: int
|
||||
upstream_sha256: str | None
|
||||
file_format: str
|
||||
role: str
|
||||
risk_flags: list[str]
|
||||
|
||||
|
||||
class AgentArtifactJobLease(AcquisitionModel):
|
||||
job_id: uuid.UUID
|
||||
lease_token: str
|
||||
lease_expires_at: datetime
|
||||
repository_id: str
|
||||
resolved_commit_sha: str
|
||||
storage_root_id: uuid.UUID
|
||||
target_root: str
|
||||
total_size_bytes: int
|
||||
reserve_bytes: int
|
||||
reserve_percent: int
|
||||
files: list[AgentArtifactJobFile]
|
||||
|
||||
|
||||
class AgentJobProgress(AcquisitionModel):
|
||||
lease_token: str = Field(min_length=32, max_length=512)
|
||||
status: Literal["claimed", "downloading", "verifying", "promoting"]
|
||||
progress_bytes: int = Field(ge=0)
|
||||
current_file: str | None = Field(default=None, max_length=2048)
|
||||
quarantine_relative_path: str | None = Field(default=None, max_length=2048)
|
||||
|
||||
|
||||
class AgentJobControl(AcquisitionModel):
|
||||
accepted: bool
|
||||
cancel_requested: bool
|
||||
lease_expires_at: datetime
|
||||
|
||||
|
||||
class CompletedFile(AcquisitionModel):
|
||||
path: str
|
||||
relative_path: str
|
||||
size_bytes: int = Field(ge=0)
|
||||
sha256: str = Field(pattern=r"^[0-9a-f]{64}$")
|
||||
inspections: list[dict[str, Any]] = Field(default_factory=list)
|
||||
|
||||
@field_validator("path", "relative_path")
|
||||
@classmethod
|
||||
def safe_path(cls, value: str) -> str:
|
||||
normalized = value.replace("\\", "/")
|
||||
if normalized.startswith("/") or ".." in normalized.split("/"):
|
||||
raise ValueError("path must be relative and confined")
|
||||
return normalized
|
||||
|
||||
|
||||
class AgentJobComplete(AcquisitionModel):
|
||||
lease_token: str = Field(min_length=32, max_length=512)
|
||||
promoted_relative_path: str
|
||||
capacity_observation: dict[str, Any]
|
||||
files: list[CompletedFile] = Field(min_length=1)
|
||||
|
||||
|
||||
class AgentJobFailure(AcquisitionModel):
|
||||
lease_token: str = Field(min_length=32, max_length=512)
|
||||
error_code: str = Field(min_length=1, max_length=64)
|
||||
error_message: str = Field(min_length=1, max_length=2000)
|
||||
retryable: bool = False
|
||||
details: dict[str, Any] = Field(default_factory=dict)
|
||||
@@ -0,0 +1,135 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from modelforge_api.domain.enums import Availability
|
||||
from modelforge_api.domain.hardware import (
|
||||
AcceleratorInventory,
|
||||
AcceleratorTelemetry,
|
||||
HostInventory,
|
||||
ObservedValue,
|
||||
StorageObservation,
|
||||
)
|
||||
|
||||
AGENT_PROTOCOL_VERSION = 1
|
||||
AGENT_PROTOCOL_CAPABILITIES = [
|
||||
"hardware.inventory",
|
||||
"hardware.telemetry",
|
||||
"artifact.acquire.v1",
|
||||
"runtime.probe.v1",
|
||||
"runtime.health.v1",
|
||||
"runtime.unload.v1",
|
||||
"deployment.load.v1",
|
||||
"deployment.invoke.v1",
|
||||
"deployment.health.v1",
|
||||
"deployment.drain.v1",
|
||||
"deployment.unload.v1",
|
||||
]
|
||||
|
||||
|
||||
class AgentProtocolModel(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AgentMetadata(AgentProtocolModel):
|
||||
agent_version: str
|
||||
protocol_version: int
|
||||
supported_capabilities: list[str] = Field(default_factory=list)
|
||||
started_at: datetime
|
||||
|
||||
|
||||
class EnrollmentRequest(AgentProtocolModel):
|
||||
enrollment_token: str = Field(min_length=32, max_length=512)
|
||||
identity_key: str = Field(min_length=1, max_length=128)
|
||||
identity_source: str = Field(min_length=1, max_length=32)
|
||||
hostname: str = Field(min_length=1, max_length=255)
|
||||
display_name: str = Field(min_length=1, max_length=255)
|
||||
metadata: AgentMetadata
|
||||
|
||||
|
||||
class EnrollmentResponse(AgentProtocolModel):
|
||||
node_id: str
|
||||
credential_id: str
|
||||
node_credential: str
|
||||
protocol_version: int = AGENT_PROTOCOL_VERSION
|
||||
|
||||
|
||||
class NodeCredentialCreated(AgentProtocolModel):
|
||||
node_id: str
|
||||
credential_id: str
|
||||
node_credential: str
|
||||
|
||||
|
||||
class HeartbeatRequest(AgentProtocolModel):
|
||||
identity_key: str
|
||||
metadata: AgentMetadata
|
||||
observed_at: datetime
|
||||
last_error: str | None = Field(default=None, max_length=1000)
|
||||
|
||||
|
||||
class InventoryNvidiaPayload(AgentProtocolModel):
|
||||
availability: Availability
|
||||
reason: str | None = None
|
||||
inventory: list[AcceleratorInventory] = Field(default_factory=list)
|
||||
|
||||
|
||||
class InventoryReport(AgentProtocolModel):
|
||||
identity_key: str
|
||||
protocol_version: int
|
||||
sequence: int = Field(ge=1, le=9_223_372_036_854_775_807)
|
||||
observed_at: datetime
|
||||
host: HostInventory
|
||||
nvidia: InventoryNvidiaPayload
|
||||
|
||||
|
||||
class TelemetryReport(AgentProtocolModel):
|
||||
identity_key: str
|
||||
protocol_version: int
|
||||
sequence: int = Field(ge=1, le=9_223_372_036_854_775_807)
|
||||
observed_at: datetime
|
||||
available_ram_bytes: ObservedValue[int]
|
||||
storage: list[StorageObservation] = Field(default_factory=list)
|
||||
accelerators: list[AcceleratorTelemetry] = Field(default_factory=list)
|
||||
|
||||
|
||||
class ObservationAck(AgentProtocolModel):
|
||||
accepted: bool
|
||||
reason: str | None = None
|
||||
received_at: datetime
|
||||
|
||||
|
||||
class EnrollmentTokenCreate(AgentProtocolModel):
|
||||
expires_in_seconds: int = Field(default=900, ge=60, le=86400)
|
||||
display_name: str | None = Field(default=None, max_length=255)
|
||||
role: str | None = Field(default=None, max_length=64)
|
||||
labels: dict[str, str | bool] = Field(default_factory=dict)
|
||||
production_eligible: bool = False
|
||||
lab_eligible: bool = True
|
||||
benchmark_eligible: bool = False
|
||||
|
||||
|
||||
class EnrollmentTokenCreated(AgentProtocolModel):
|
||||
id: str
|
||||
enrollment_token: str
|
||||
expires_at: datetime
|
||||
setup_environment: dict[str, str]
|
||||
|
||||
|
||||
class EnrollmentTokenSummary(AgentProtocolModel):
|
||||
id: str
|
||||
created_at: datetime
|
||||
expires_at: datetime
|
||||
used_at: datetime | None
|
||||
revoked_at: datetime | None
|
||||
|
||||
|
||||
class NodeManagementUpdate(AgentProtocolModel):
|
||||
display_name: str | None = Field(default=None, min_length=1, max_length=255)
|
||||
role: str | None = Field(default=None, max_length=64)
|
||||
labels: dict[str, str | bool] | None = None
|
||||
enabled: bool | None = None
|
||||
production_eligible: bool | None = None
|
||||
lab_eligible: bool | None = None
|
||||
benchmark_eligible: bool | None = None
|
||||
@@ -0,0 +1,140 @@
|
||||
"""Stable audit-chain format identifiers shared by persistence and services."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
AUDIT_CHAIN_SINGLETON_ID = 1
|
||||
AUDIT_HASH_FORMAT_V1 = "v1"
|
||||
AUDIT_HASH_FORMAT_V2 = "v2"
|
||||
AUDIT_CURRENT_HASH_FORMAT = AUDIT_HASH_FORMAT_V2
|
||||
AUDIT_LEGACY_PREFIX_DOMAIN = b"modelforge:audit:legacy-prefix:v1\n"
|
||||
AUDIT_EMPTY_LEGACY_PREFIX_SEAL = hashlib.sha256(AUDIT_LEGACY_PREFIX_DOMAIN).hexdigest()
|
||||
|
||||
|
||||
def normalise_audit_timestamp(value: datetime | str) -> str:
|
||||
"""Return the stable UTC/microsecond representation used by v2 audit hashes."""
|
||||
|
||||
moment: datetime
|
||||
if isinstance(value, datetime):
|
||||
moment = value
|
||||
elif isinstance(value, str):
|
||||
candidate = value.strip()
|
||||
if candidate.endswith("Z"):
|
||||
candidate = candidate[:-1] + "+00:00"
|
||||
try:
|
||||
moment = datetime.fromisoformat(candidate)
|
||||
except ValueError as error:
|
||||
raise ValueError("audit occurred_at is not an ISO-8601 timestamp") from error
|
||||
else:
|
||||
raise TypeError("audit occurred_at must be a datetime or ISO-8601 string")
|
||||
if moment.tzinfo is None:
|
||||
moment = moment.replace(tzinfo=UTC)
|
||||
return moment.astimezone(UTC).isoformat(timespec="microseconds").replace("+00:00", "Z")
|
||||
|
||||
|
||||
def normalise_audit_event_id(value: Any) -> str:
|
||||
try:
|
||||
return str(uuid.UUID(str(value)))
|
||||
except (AttributeError, TypeError, ValueError) as error:
|
||||
raise ValueError("audit event id is not a UUID") from error
|
||||
|
||||
|
||||
def canonical_audit_payload_and_hash(
|
||||
*,
|
||||
correlation_id: str,
|
||||
actor_type: str,
|
||||
actor_id: str,
|
||||
action: str,
|
||||
resource_type: str,
|
||||
resource_id: str | None,
|
||||
outcome: str,
|
||||
details: dict[str, Any],
|
||||
previous_event_hash: str | None,
|
||||
hash_format: str = AUDIT_HASH_FORMAT_V1,
|
||||
event_id: Any | None = None,
|
||||
occurred_at: datetime | str | None = None,
|
||||
) -> tuple[dict[str, Any], str]:
|
||||
"""Return detached stored fields and their versioned SHA-256 identity."""
|
||||
|
||||
stored_payload, _encoded_hash, event_hash = canonical_audit_payload_text_and_hash(
|
||||
correlation_id=correlation_id,
|
||||
actor_type=actor_type,
|
||||
actor_id=actor_id,
|
||||
action=action,
|
||||
resource_type=resource_type,
|
||||
resource_id=resource_id,
|
||||
outcome=outcome,
|
||||
details=details,
|
||||
previous_event_hash=previous_event_hash,
|
||||
hash_format=hash_format,
|
||||
event_id=event_id,
|
||||
occurred_at=occurred_at,
|
||||
)
|
||||
return stored_payload, event_hash
|
||||
|
||||
|
||||
def canonical_audit_payload_text_and_hash(
|
||||
*,
|
||||
correlation_id: str,
|
||||
actor_type: str,
|
||||
actor_id: str,
|
||||
action: str,
|
||||
resource_type: str,
|
||||
resource_id: str | None,
|
||||
outcome: str,
|
||||
details: dict[str, Any],
|
||||
previous_event_hash: str | None,
|
||||
hash_format: str = AUDIT_HASH_FORMAT_V1,
|
||||
event_id: Any | None = None,
|
||||
occurred_at: datetime | str | None = None,
|
||||
) -> tuple[dict[str, Any], str, str]:
|
||||
"""Return stored fields, the exact hashed text, and its SHA-256 digest.
|
||||
|
||||
The exact text is persisted for v2 events. That makes PostgreSQL's SECURITY DEFINER writer
|
||||
authoritative for its own canonical encoding without asking Python and PostgreSQL to reproduce
|
||||
each other's JSON lexical representation. Strict verification hashes the stored bytes and then
|
||||
independently checks that the decoded object is exactly the event's semantic payload.
|
||||
"""
|
||||
|
||||
if not isinstance(details, dict):
|
||||
raise TypeError("audit details must be an object")
|
||||
stored_payload: dict[str, Any] = {
|
||||
"correlation_id": correlation_id,
|
||||
"actor_type": actor_type,
|
||||
"actor_id": actor_id,
|
||||
"action": action,
|
||||
"resource_type": resource_type,
|
||||
"resource_id": resource_id,
|
||||
"outcome": outcome,
|
||||
"details": details,
|
||||
"previous_event_hash": previous_event_hash,
|
||||
}
|
||||
encoded_stored = json.dumps(stored_payload, sort_keys=True, separators=(",", ":"))
|
||||
canonical_stored = json.loads(encoded_stored)
|
||||
if not isinstance(canonical_stored, dict): # pragma: no cover - constructed above
|
||||
raise TypeError("canonical audit payload must be an object")
|
||||
|
||||
if hash_format == AUDIT_HASH_FORMAT_V1:
|
||||
hash_payload = canonical_stored
|
||||
elif hash_format == AUDIT_HASH_FORMAT_V2:
|
||||
if event_id is None or occurred_at is None:
|
||||
raise ValueError("v2 audit hashes require an event id and occurred_at")
|
||||
hash_payload = {
|
||||
"hash_format": AUDIT_HASH_FORMAT_V2,
|
||||
"id": normalise_audit_event_id(event_id),
|
||||
"occurred_at": normalise_audit_timestamp(occurred_at),
|
||||
**canonical_stored,
|
||||
}
|
||||
else:
|
||||
raise ValueError(f"unsupported audit hash format {hash_format!r}")
|
||||
encoded_hash = json.dumps(hash_payload, sort_keys=True, separators=(",", ":"))
|
||||
return (
|
||||
canonical_stored,
|
||||
encoded_hash,
|
||||
hashlib.sha256(encoded_hash.encode("utf-8")).hexdigest(),
|
||||
)
|
||||
@@ -0,0 +1,171 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import re
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
|
||||
EvaluationType = Literal[
|
||||
"retrieval", "ocr", "visual-retrieval", "asr", "tts", "generation", "llm"
|
||||
]
|
||||
CapabilityRecommendationState = Literal[
|
||||
"KEEP_CURRENT",
|
||||
"LAB_READY",
|
||||
"PROMOTION_ELIGIBLE",
|
||||
"REQUIRES_MORE_EVIDENCE",
|
||||
"BLOCKED",
|
||||
]
|
||||
MetricDirection = Literal["higher_is_better", "lower_is_better", "informational"]
|
||||
_KEY = re.compile(r"^[a-z0-9][a-z0-9_-]{0,127}$")
|
||||
|
||||
ALLOWED_METRICS: dict[str, frozenset[str]] = {
|
||||
"retrieval": frozenset({"recall_at_5", "recall_at_10", "mrr", "ndcg_at_10"}),
|
||||
"ocr": frozenset({"cer", "wer", "text_accuracy", "field_accuracy", "layout_accuracy", "latency_ms", "peak_vram_bytes"}),
|
||||
"visual-retrieval": frozenset({"recall_at_1", "recall_at_5", "recall_at_10", "mrr", "latency_ms", "peak_vram_bytes"}),
|
||||
"asr": frozenset({"wer", "real_time_factor", "latency_ms", "peak_vram_bytes"}),
|
||||
"tts": frozenset({"latency_ms", "real_time_factor", "peak_vram_bytes"}),
|
||||
"generation": frozenset({"latency_ms", "peak_vram_bytes"}),
|
||||
"llm": frozenset({"task_success", "structured_output_validity", "latency_ms", "peak_vram_bytes"}),
|
||||
}
|
||||
|
||||
|
||||
class EvaluationModel(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", from_attributes=True)
|
||||
|
||||
|
||||
class MetricDefinition(EvaluationModel):
|
||||
name: str
|
||||
direction: MetricDirection
|
||||
unit: str = Field(min_length=1, max_length=64)
|
||||
minimum: float | None = None
|
||||
maximum: float | None = None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def valid_range(self) -> MetricDefinition:
|
||||
if self.minimum is not None and self.maximum is not None and self.minimum > self.maximum:
|
||||
raise ValueError("metric minimum cannot exceed maximum")
|
||||
return self
|
||||
|
||||
|
||||
class CapabilityEvaluationCase(EvaluationModel):
|
||||
key: str
|
||||
fixture_ref: str = Field(min_length=1, max_length=512)
|
||||
fixture_sha256: str = Field(pattern=r"^[a-f0-9]{64}$")
|
||||
ground_truth: dict[str, Any]
|
||||
labels: list[str] = Field(default_factory=list)
|
||||
critical: bool = False
|
||||
|
||||
@field_validator("key")
|
||||
@classmethod
|
||||
def valid_key(cls, value: str) -> str:
|
||||
if not _KEY.fullmatch(value):
|
||||
raise ValueError("case key must be a safe identifier")
|
||||
return value
|
||||
|
||||
|
||||
class CapabilityEvaluationSuiteCreate(EvaluationModel):
|
||||
capability: str = Field(pattern=r"^[a-z][a-z0-9_]*(?:\.[a-z][a-z0-9_]*)+$")
|
||||
contract_version: int = Field(default=1, ge=1)
|
||||
key: str
|
||||
evaluation_type: EvaluationType
|
||||
revision: str
|
||||
dataset_revision: str
|
||||
metrics: list[MetricDefinition] = Field(min_length=1)
|
||||
cases: list[CapabilityEvaluationCase] = Field(min_length=1, max_length=500)
|
||||
thresholds: dict[str, float] = Field(default_factory=dict)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_definition(self) -> CapabilityEvaluationSuiteCreate:
|
||||
for key in (self.key, self.revision, self.dataset_revision):
|
||||
if not _KEY.fullmatch(key):
|
||||
raise ValueError("suite identifiers must be safe")
|
||||
metric_names = [item.name for item in self.metrics]
|
||||
if len(metric_names) != len(set(metric_names)):
|
||||
raise ValueError("metric names must be unique")
|
||||
unsupported = set(metric_names) - ALLOWED_METRICS[self.evaluation_type]
|
||||
if unsupported:
|
||||
raise ValueError(f"metrics are invalid for {self.evaluation_type}: {sorted(unsupported)}")
|
||||
if set(self.thresholds) - set(metric_names):
|
||||
raise ValueError("thresholds must refer to declared metrics")
|
||||
if len({item.key for item in self.cases}) != len(self.cases):
|
||||
raise ValueError("case keys must be unique")
|
||||
return self
|
||||
|
||||
|
||||
class CapabilityEvaluationSuiteResponse(EvaluationModel):
|
||||
id: uuid.UUID
|
||||
capability: str
|
||||
contract_version: int
|
||||
key: str
|
||||
evaluation_type: EvaluationType
|
||||
revision: str
|
||||
dataset_revision: str
|
||||
definition_digest: str
|
||||
metrics: list[MetricDefinition]
|
||||
cases: list[CapabilityEvaluationCase]
|
||||
thresholds: dict[str, float]
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class CapabilityCaseResult(EvaluationModel):
|
||||
case_key: str
|
||||
metrics: dict[str, float]
|
||||
output_sha256: str = Field(pattern=r"^[a-f0-9]{64}$")
|
||||
status: Literal["passed", "failed", "error"]
|
||||
error_code: str | None = Field(default=None, max_length=64)
|
||||
|
||||
|
||||
class CapabilityEvaluationRunCreate(EvaluationModel):
|
||||
suite_id: uuid.UUID
|
||||
capability_deployment_id: uuid.UUID
|
||||
status: Literal["completed", "failed"]
|
||||
metrics: dict[str, float]
|
||||
cases: list[CapabilityCaseResult] = Field(min_length=1, max_length=500)
|
||||
resource_metrics: dict[str, float]
|
||||
environment_fingerprint: str = Field(pattern=r"^[a-f0-9]{64}$")
|
||||
evidence: dict[str, Any] = Field(default_factory=dict)
|
||||
started_at: datetime
|
||||
completed_at: datetime
|
||||
|
||||
@field_validator("metrics", "resource_metrics")
|
||||
@classmethod
|
||||
def finite_values(cls, value: dict[str, float]) -> dict[str, float]:
|
||||
if any(not math.isfinite(item) for item in value.values()):
|
||||
raise ValueError("metric values must be finite")
|
||||
return value
|
||||
|
||||
@model_validator(mode="after")
|
||||
def chronological(self) -> CapabilityEvaluationRunCreate:
|
||||
if self.completed_at < self.started_at:
|
||||
raise ValueError("evaluation completion precedes its start")
|
||||
return self
|
||||
|
||||
|
||||
class CapabilityEvaluationRunResponse(EvaluationModel):
|
||||
id: uuid.UUID
|
||||
suite_id: uuid.UUID
|
||||
capability_deployment_id: uuid.UUID
|
||||
evaluation_type: EvaluationType
|
||||
status: str
|
||||
metrics: dict[str, float]
|
||||
cases: list[CapabilityCaseResult]
|
||||
resource_metrics: dict[str, float]
|
||||
environment_fingerprint: str
|
||||
evidence_digest: str
|
||||
evidence: dict[str, Any]
|
||||
started_at: datetime
|
||||
completed_at: datetime
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class CapabilityAdvisorResponse(EvaluationModel):
|
||||
capability: str
|
||||
contract_version: int
|
||||
state: CapabilityRecommendationState
|
||||
deployment_id: uuid.UUID | None = None
|
||||
evaluation_run_id: uuid.UUID | None = None
|
||||
reasons: list[str]
|
||||
automatic_promotion: Literal[False] = False
|
||||
@@ -0,0 +1,282 @@
|
||||
"""Documentation for every setting, held next to the settings themselves.
|
||||
|
||||
`.env.example` and `docs/CONFIGURATION.md` are generated from this table joined with the typed
|
||||
defaults, and a test fails when a setting exists without an entry here. Hand-maintained
|
||||
configuration documentation drifts silently — the operator finds out when a production deployment
|
||||
does something the manual said it would not.
|
||||
|
||||
`required_in_production` means startup validation refuses to run production without it, not merely
|
||||
that it is recommended.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from enum import StrEnum
|
||||
|
||||
|
||||
class Sensitivity(StrEnum):
|
||||
PUBLIC = "public"
|
||||
#: Reveals deployment topology. Not a credential, but not for a public issue tracker either.
|
||||
INTERNAL = "internal"
|
||||
#: A credential or key. Never logged, never packaged, never echoed in an error.
|
||||
SECRET = "secret" # noqa: S105 - a classification label, not a credential
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SettingDoc:
|
||||
description: str
|
||||
sensitivity: Sensitivity = Sensitivity.PUBLIC
|
||||
required_in_production: bool = False
|
||||
#: False when the value is re-read per request or per poll rather than only at startup.
|
||||
restart_required: bool = True
|
||||
|
||||
|
||||
def _p(description: str, **kwargs: object) -> SettingDoc:
|
||||
return SettingDoc(description, **kwargs) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def _secret(description: str, *, required: bool = False) -> SettingDoc:
|
||||
return SettingDoc(description, Sensitivity.SECRET, required_in_production=required)
|
||||
|
||||
|
||||
def _internal(description: str, *, required: bool = False) -> SettingDoc:
|
||||
return SettingDoc(description, Sensitivity.INTERNAL, required_in_production=required)
|
||||
|
||||
|
||||
SETTING_DOCS: dict[str, SettingDoc] = {
|
||||
# ---------------------------------------------------------------- process
|
||||
"env": _p(
|
||||
"Deployment profile. 'production' turns on every fail-closed startup rule; "
|
||||
"'development' and 'test' report the same problems without refusing to start."
|
||||
),
|
||||
"api_host": _p("Interface the API binds inside its container. Leave at 0.0.0.0."),
|
||||
"api_port": _p("Port the API listens on inside its container."),
|
||||
"control_plane_max_payload_bytes": _p(
|
||||
"Pre-parser request-body limit for public and operator control-plane routes."
|
||||
),
|
||||
"service_name": _p("Name this process reports in logs and audit events."),
|
||||
"log_level": _p("Structured log level: DEBUG, INFO, WARNING or ERROR.", restart_required=True),
|
||||
# ---------------------------------------------------------------- dependencies
|
||||
"database_url": _secret(
|
||||
"SQLAlchemy URL for the API's non-owner modelforge_runtime role. It may read the audit "
|
||||
"trail and execute the canonical append function, but cannot mutate audit tables directly.",
|
||||
required=True,
|
||||
),
|
||||
"migration_database_url": _secret(
|
||||
"SQLAlchemy URL for the non-superuser modelforge schema-owner role. Set only in the "
|
||||
"one-shot migration process; production API startup refuses when this secret is present."
|
||||
),
|
||||
"redis_url": _internal("Redis URL for transient request payloads and queues.", required=True),
|
||||
"cors_origins": _p(
|
||||
"Comma-separated exact origins allowed to call the API from a browser. "
|
||||
"A wildcard is refused in production because requests are credentialed."
|
||||
),
|
||||
# ---------------------------------------------------------------- credentials
|
||||
"operator_api_key": _secret(
|
||||
"Operator API key guarding every admin route. Generate at least 32 random characters; "
|
||||
"ModelForge never mints one for you.",
|
||||
required=True,
|
||||
),
|
||||
"hf_token": _secret(
|
||||
"Optional Hugging Face token, used only for acquiring gated repositories. "
|
||||
"It is never passed to a runtime and never leaves the control plane."
|
||||
),
|
||||
"backup_encryption_key": _secret(
|
||||
"Base64 AES-256 key for backup encryption. Without it no backup can be produced, and "
|
||||
"without the same key no backup can be restored — store it outside this deployment.",
|
||||
required=True,
|
||||
),
|
||||
"backup_encryption_key_id": _p(
|
||||
"Identifier recorded in each backup manifest so a restore can name the key it needs."
|
||||
),
|
||||
# ---------------------------------------------------------------- storage
|
||||
"hf_home": _p("Hugging Face cache root inside the container."),
|
||||
"artifact_root": _p("Verified model artifact root. Must exist and be writable."),
|
||||
"quarantine_root": _p("Where acquired artifacts are held until their checks pass."),
|
||||
"runtime_artifact_root": _p("Artifact root as a runtime worker sees it on a compute node."),
|
||||
"config_root": _p("Directory holding the capability, project and policy manifests."),
|
||||
"backup_root": _p("Backup destination. Must exist and be writable, or backups fail closed."),
|
||||
"backup_restore_root": _p("Working directory a restore stages into before it commits."),
|
||||
"alembic_directory": _p("Override for the migration directory. Leave empty in a container."),
|
||||
# ---------------------------------------------------------------- acquisition
|
||||
"hf_timeout_seconds": _p("Per-request timeout for Hugging Face metadata calls."),
|
||||
"hf_snapshot_ttl_seconds": _p("How long a resolved upstream snapshot stays cached."),
|
||||
"allow_remote_code": _p(
|
||||
"Whether model repositories may execute their own Python. Always false in production; "
|
||||
"startup refuses any other value there."
|
||||
),
|
||||
# ---------------------------------------------------------------- hardware and nodes
|
||||
"enable_gpu_telemetry": _p("Collect GPU telemetry on this host."),
|
||||
"hardware_refresh_on_startup": _p("Run a hardware inventory pass when the process starts."),
|
||||
"hardware_poll_interval_seconds": _p("Interval between hardware inventory passes."),
|
||||
"node_identity": _internal("Explicit node identity. Leave empty to use the persisted file."),
|
||||
"node_identity_mode": _p("'persisted' keeps a node's identity across restarts; 'auto' derives it."),
|
||||
"node_identity_file": _p("Where a persisted node identity is stored."),
|
||||
"node_stale_after_seconds": _p("Silence after which a node is considered stale."),
|
||||
"node_offline_after_seconds": _p(
|
||||
"Silence after which a node is considered offline. Must exceed the stale threshold."
|
||||
),
|
||||
"liveness_poll_interval_seconds": _p("How often node liveness is re-evaluated."),
|
||||
"agent_max_clock_skew_seconds": _p("Clock skew tolerated on an agent report before refusal."),
|
||||
"node_agent_max_payload_bytes": _p(
|
||||
"Pre-parser request-body limit for enrollment and authenticated Node Agent reports."
|
||||
),
|
||||
"node_liveness_monitor_enabled": _p("Run the node liveness monitor in this process."),
|
||||
"agent_protocol_version": _p("Agent protocol version this control plane speaks."),
|
||||
# ---------------------------------------------------------------- gateway
|
||||
"gateway_max_batch_size": _p("Maximum inputs accepted in a single capability invocation."),
|
||||
"gateway_max_input_characters": _p("Maximum characters per input item."),
|
||||
"gateway_max_payload_bytes": _p("Maximum accepted request body size."),
|
||||
"gateway_request_timeout_seconds": _p(
|
||||
"Total time a capability invocation may take. Must exceed the queue timeout."
|
||||
),
|
||||
"gateway_queue_timeout_seconds": _p("How long a request may wait for capacity before rejection."),
|
||||
"serving_job_lease_seconds": _p("Lease held by a serving job before it is reclaimed."),
|
||||
"serving_payload_ttl_seconds": _p("How long a request payload survives in Redis."),
|
||||
# ---------------------------------------------------------------- scheduler
|
||||
"scheduler_safety_reserve_bytes": _p("VRAM never offered to a placement, as an absolute floor."),
|
||||
"scheduler_safety_reserve_percentage": _p(
|
||||
"VRAM never offered to a placement, as a fraction. Half a device leaves nothing schedulable."
|
||||
),
|
||||
"scheduler_runtime_margin_bytes": _p("Headroom reserved for runtime overhead per node."),
|
||||
"scheduler_deployment_margin_bytes": _p("Absolute headroom added to each deployment estimate."),
|
||||
"scheduler_deployment_margin_percentage": _p("Proportional headroom added to each estimate."),
|
||||
"scheduler_global_queue_limit": _p("Queued requests accepted before capacity rejection begins."),
|
||||
"scheduler_telemetry_stale_seconds": _p(
|
||||
"Telemetry age past which admission is blocked rather than extrapolated."
|
||||
),
|
||||
"scheduler_pressure_stable_seconds": _p("How long pressure must hold before the state changes."),
|
||||
"scheduler_eviction_cooldown_seconds": _p("Minimum interval between evictions on a node."),
|
||||
"scheduler_placement_history_limit": _p("Placement decisions retained for inspection."),
|
||||
# ---------------------------------------------------------------- background work
|
||||
"registry_seed_on_startup": _p("Seed the candidate and project registries from manifests."),
|
||||
"serving_reconciliation_enabled": _p("Reconcile abandoned serving work in this process."),
|
||||
"serving_reconciliation_interval_seconds": _p("Interval between serving reconciliation passes."),
|
||||
"lifecycle_reconciliation_enabled": _p("Roll back incomplete lifecycle operations at startup."),
|
||||
"migration_reconciliation_enabled": _p(
|
||||
"Report interrupted migration cutovers at startup. They are never auto-resolved: external "
|
||||
"alias truth cannot be inferred after a crash."
|
||||
),
|
||||
"observability_monitor_enabled": _p("Run SLO and alert evaluation in this process."),
|
||||
"observability_poll_interval_seconds": _p("Interval between observability evaluation passes."),
|
||||
"recovery_reconciliation_enabled": _p("Reconcile interrupted backups and restores at startup."),
|
||||
# ---------------------------------------------------------------- recovery
|
||||
"backup_pg_dump_path": _p("pg_dump executable. Must match the server major version."),
|
||||
"backup_pg_restore_path": _p("pg_restore executable."),
|
||||
"backup_psql_path": _p("psql executable."),
|
||||
"backup_command_timeout_seconds": _p("Timeout for a dump or restore command."),
|
||||
"backup_stale_after_seconds": _p("Age past which the newest verified backup raises BACKUP_STALE."),
|
||||
"backup_minimum_free_bytes": _p("Free space below which a backup refuses to start."),
|
||||
"backup_capacity_headroom_ratio": _p("Required free space as a multiple of the estimated size."),
|
||||
"restore_allow_production_target": _p(
|
||||
"Whether a restore may overwrite the live database. Keep false outside a rehearsal."
|
||||
),
|
||||
# ---------------------------------------------------------------- build identity
|
||||
"build_commit": _p("Source commit stamped into the image at build time. Never set by hand."),
|
||||
"build_timestamp": _p("Build time stamped into the image. Never set by hand."),
|
||||
"build_image_digest": _p("Image digest recorded at deployment. Never set by hand."),
|
||||
}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- deployment variables
|
||||
#
|
||||
# Variables the deployment reads rather than the control-plane process: Compose interpolation, the
|
||||
# Node Agent and the Runtime Worker. They are not `Settings` fields, but an operator still has to
|
||||
# set them, so leaving them out of the reference would recreate exactly the undocumented-variable
|
||||
# problem this table exists to prevent.
|
||||
|
||||
DEPLOYMENT_DOCS: dict[str, SettingDoc] = {
|
||||
# ---------------------------------------------------------------- network bindings
|
||||
"MODELFORGE_POSTGRES_BIND": _p(
|
||||
"Host address the control-plane database is published on. Defaults to 127.0.0.1; "
|
||||
"publishing it more widely exposes provenance, credential hashes and the audit trail."
|
||||
),
|
||||
"MODELFORGE_REDIS_BIND": _p("Host address Redis is published on. Defaults to 127.0.0.1."),
|
||||
"MODELFORGE_API_BIND": _p(
|
||||
"Host address the API is published on. Defaults to 0.0.0.0 deliberately: the console and "
|
||||
"compute nodes need it, and every admin route is operator-authenticated."
|
||||
),
|
||||
"MODELFORGE_WEB_BIND": _p("Host address the console is published on. Defaults to 127.0.0.1."),
|
||||
"MODELFORGE_POSTGRES_PORT": _p("Host port the database is published on. Defaults to 5432."),
|
||||
"MODELFORGE_REDIS_PORT": _p("Host port Redis is published on. Defaults to 6379."),
|
||||
"MODELFORGE_API_PUBLISHED_PORT": _p("Host port the API is published on. Defaults to 8000."),
|
||||
"MODELFORGE_WEB_PORT": _p("Host port the console is published on. Defaults to 3000."),
|
||||
"MODELFORGE_DR_POSTGRES_BIND": _p("Host address for the DR rehearsal database. Loopback only."),
|
||||
"MODELFORGE_DR_API_BIND": _p("Host address for the DR rehearsal API. Loopback only."),
|
||||
"VITE_API_BASE_URL": _p(
|
||||
"API base URL compiled into the console. Vite inlines it at build time, so changing it "
|
||||
"requires rebuilding the console image, not restarting it."
|
||||
),
|
||||
# ---------------------------------------------------------------- production identity
|
||||
"MODELFORGE_POSTGRES_DB": _p("Production database name. Required by the production overlay."),
|
||||
"MODELFORGE_POSTGRES_ADMIN_USER": _internal(
|
||||
"Bootstrap/admin role used only by PostgreSQL provisioning; defaults to postgres."
|
||||
),
|
||||
"MODELFORGE_POSTGRES_ADMIN_PASSWORD": _secret(
|
||||
"Bootstrap/admin password; never passed to the migration or API container.", required=True
|
||||
),
|
||||
"MODELFORGE_MIGRATION_DB_PASSWORD": _secret(
|
||||
"Raw password supplied to provisioning for the non-superuser modelforge owner role.",
|
||||
required=True,
|
||||
),
|
||||
"MODELFORGE_RUNTIME_DB_PASSWORD": _secret(
|
||||
"Raw password supplied to provisioning for the non-owner modelforge_runtime role.",
|
||||
required=True,
|
||||
),
|
||||
"MODELFORGE_RUNTIME_DATABASE_URL": _secret(
|
||||
"Non-owner runtime-role URL passed only to the API container.", required=True
|
||||
),
|
||||
"MODELFORGE_VERSION": _p(
|
||||
"Exact version tag applied to built images and required when the production overlay is "
|
||||
"not given explicit API and web image references."
|
||||
),
|
||||
"MODELFORGE_COMMIT": _p("Source commit stamped into images at build time."),
|
||||
"MODELFORGE_BUILT_AT": _p("Build timestamp stamped into images."),
|
||||
"MODELFORGE_API_IMAGE": _p(
|
||||
"Exact tag or digest the production overlay runs for the API; never use latest."
|
||||
),
|
||||
"MODELFORGE_WEB_IMAGE": _p(
|
||||
"Exact tag or digest the production overlay runs for the console; never use latest."
|
||||
),
|
||||
"MODELFORGE_NODE_AGENT_IMAGE": _p(
|
||||
"Exact release tag or digest for the standalone Node Agent. The local-build fallback is "
|
||||
"named local and never resolves to latest."
|
||||
),
|
||||
"MODELFORGE_API_IMAGE_DIGEST": _p("Digest recorded as the running API build identity."),
|
||||
# ---------------------------------------------------------------- volumes
|
||||
"MODELFORGE_BACKUP_VOLUME": _p("Volume or bind path backing the backup root."),
|
||||
"MODELFORGE_RESTORE_VOLUME": _p("Volume or bind path backing the restore staging root."),
|
||||
"MODELFORGE_AGENT_STATE_VOLUME": _p("Volume or bind path holding the agent's persisted identity."),
|
||||
"MODELFORGE_AGENT_HF_CACHE_VOLUME": _p("Volume or bind path for the agent's Hugging Face cache."),
|
||||
"MODELFORGE_AGENT_ARTIFACT_VOLUME": _p("Volume or bind path for verified artifacts on a node."),
|
||||
"MODELFORGE_AGENT_QUARANTINE_VOLUME": _p("Volume or bind path for the node's quarantine area."),
|
||||
# ---------------------------------------------------------------- node agent
|
||||
"MODELFORGE_AGENT_CONTROL_PLANE_URL": _p(
|
||||
"URL the agent reports to. Outbound only; the control plane never dials a node."
|
||||
),
|
||||
"MODELFORGE_AGENT_ENROLLMENT_TOKEN": _secret(
|
||||
"Single-use enrolment token. Consumed atomically: a storm against one token produces "
|
||||
"exactly one identity."
|
||||
),
|
||||
"MODELFORGE_AGENT_HOSTNAME": _p("Hostname the agent enrols under."),
|
||||
"MODELFORGE_AGENT_ACCELERATOR_MODE": _p(
|
||||
"Accelerator contract: nvidia fails closed unless NVML inventory and telemetry are valid; "
|
||||
"cpu permits a legitimate CPU-only node; auto requires NVIDIA when injected devices are "
|
||||
"observed. Canonical GPU Compose deployments set nvidia explicitly."
|
||||
),
|
||||
"MODELFORGE_AGENT_TLS_VERIFY": _p(
|
||||
"Whether the agent verifies the control plane's certificate. True wherever TLS is real."
|
||||
),
|
||||
"MODELFORGE_AGENT_CONTROL_PLANE_HOST_ADDRESS": _internal(
|
||||
"Host address mapped for a private-CA deployment."
|
||||
),
|
||||
"MODELFORGE_AGENT_CA_CERT_PATH": _p("Path to the private CA certificate the agent trusts."),
|
||||
# ---------------------------------------------------------------- runtime worker
|
||||
"MODELFORGE_RUNTIME_WORKER_ARTIFACT_ROOT": _p("Artifact root as the runtime worker sees it."),
|
||||
"MODELFORGE_RUNTIME_WORKER_POLL_INTERVAL_SECONDS": _p("Worker poll interval, in seconds."),
|
||||
# ---------------------------------------------------------------- provenance passthrough
|
||||
"MODELFORGE_SOURCE_COMMIT": _p("Source commit reported by the deployment."),
|
||||
"MODELFORGE_SOURCE_REFERENCE": _p("Git reference reported by the deployment."),
|
||||
"MODELFORGE_SOURCE_REPOSITORY": _p("Repository URL reported by the deployment."),
|
||||
}
|
||||
@@ -0,0 +1,432 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any, Literal, Protocol
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
|
||||
from .enums import (
|
||||
DeploymentChannel,
|
||||
FailureCode,
|
||||
FailureOwner,
|
||||
HealthStatus,
|
||||
MigrationStatus,
|
||||
ResidencyPolicy,
|
||||
UpgradeClass,
|
||||
VerificationStatus,
|
||||
WorkloadPriority,
|
||||
)
|
||||
|
||||
CAPABILITY_KEY = re.compile(r"^[a-z][a-z0-9_]*(?:\.[a-z][a-z0-9_]*)+$")
|
||||
SHA256 = re.compile(r"^[a-f0-9]{64}$")
|
||||
COMMIT_SHA = re.compile(r"^[a-f0-9]{40,64}$")
|
||||
|
||||
|
||||
class StrictModel(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class SchemaDocument(StrictModel):
|
||||
schema_: dict[str, Any] = Field(alias="schema")
|
||||
|
||||
|
||||
class ModalityContract(StrictModel):
|
||||
input: list[str] = Field(min_length=1)
|
||||
output: list[str] = Field(min_length=1)
|
||||
|
||||
|
||||
class VectorContract(StrictModel):
|
||||
dimensionality: int | None = Field(default=None, gt=0)
|
||||
normalized: bool | None = None
|
||||
cross_deployment_compatible: bool = False
|
||||
|
||||
|
||||
class SLOContract(StrictModel):
|
||||
latency_p95_ms: int | None = Field(default=None, gt=0)
|
||||
availability_percent: float | None = Field(default=None, ge=0, le=100)
|
||||
max_concurrency: int | None = Field(default=None, gt=0)
|
||||
|
||||
|
||||
class PrivacyContract(StrictModel):
|
||||
classification: Literal["public", "internal", "confidential", "restricted"] = "internal"
|
||||
allow_persistence: bool = False
|
||||
allow_logging_payloads: bool = False
|
||||
allow_network_egress: bool = False
|
||||
|
||||
|
||||
class ResourceRequirement(StrictModel):
|
||||
accelerator_required: bool = False
|
||||
minimum_vram_mb: int | None = Field(default=None, ge=0)
|
||||
cpu_fallback_allowed: bool = False
|
||||
|
||||
|
||||
CapabilityCategory = Literal[
|
||||
"TEXT", "RAG", "DOCUMENT", "VISION", "AUDIO", "GENERATION", "ASSISTANTS"
|
||||
]
|
||||
CapabilityStability = Literal["stable", "experimental", "blocked", "planned"]
|
||||
EvaluationType = Literal[
|
||||
"retrieval", "ocr", "visual-retrieval", "asr", "tts", "generation", "llm"
|
||||
]
|
||||
ResourceClass = Literal["LIGHT", "MEDIUM", "HEAVY", "EXCLUSIVE_GPU"]
|
||||
|
||||
|
||||
class PayloadLimits(StrictModel):
|
||||
max_bytes: int = Field(ge=1, le=67_108_864)
|
||||
max_batch_count: int = Field(default=1, ge=1, le=64)
|
||||
max_width: int | None = Field(default=None, ge=1, le=16_384)
|
||||
max_height: int | None = Field(default=None, ge=1, le=16_384)
|
||||
max_pages: int | None = Field(default=None, ge=1, le=128)
|
||||
max_duration_seconds: float | None = Field(default=None, gt=0, le=3600)
|
||||
|
||||
|
||||
class CapabilityEstateMetadata(StrictModel):
|
||||
category: CapabilityCategory
|
||||
purpose: str = Field(min_length=1, max_length=1000)
|
||||
stability: CapabilityStability
|
||||
resource_class: ResourceClass
|
||||
evaluation_type: EvaluationType
|
||||
consumers: list[str] = Field(default_factory=list)
|
||||
payload_limits: PayloadLimits
|
||||
|
||||
|
||||
class FallbackContract(StrictModel):
|
||||
allowed: bool = False
|
||||
mode: Literal["none", "compatible_deployment", "degrade", "hard_fail"] = "hard_fail"
|
||||
capability: str | None = None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_fallback(self) -> FallbackContract:
|
||||
if not self.allowed and self.mode not in {"none", "hard_fail"}:
|
||||
raise ValueError("disabled fallback must use none or hard_fail")
|
||||
return self
|
||||
|
||||
|
||||
class CapabilityContractManifest(StrictModel):
|
||||
capability: str
|
||||
version: int = Field(ge=1)
|
||||
description: str = Field(min_length=1)
|
||||
input_schema: dict[str, Any]
|
||||
output_schema: dict[str, Any]
|
||||
modalities: ModalityContract
|
||||
languages: list[str] = Field(default_factory=list)
|
||||
streaming: bool = False
|
||||
structured_output: bool = False
|
||||
vector: VectorContract | None = None
|
||||
slo: SLOContract = Field(default_factory=SLOContract)
|
||||
quality_metrics: list[str] = Field(min_length=1)
|
||||
upgrade_class: UpgradeClass
|
||||
fallback: FallbackContract = Field(default_factory=FallbackContract)
|
||||
privacy: PrivacyContract = Field(default_factory=PrivacyContract)
|
||||
resources: ResourceRequirement = Field(default_factory=ResourceRequirement)
|
||||
production_priority: WorkloadPriority
|
||||
default_residency: ResidencyPolicy
|
||||
estate: CapabilityEstateMetadata
|
||||
|
||||
@field_validator("capability")
|
||||
@classmethod
|
||||
def validate_key(cls, value: str) -> str:
|
||||
if not CAPABILITY_KEY.fullmatch(value):
|
||||
raise ValueError("capability must be a dotted, lowercase logical key")
|
||||
return value
|
||||
|
||||
@model_validator(mode="after")
|
||||
def vector_safety(self) -> CapabilityContractManifest:
|
||||
vector_output = self.vector is not None or "embedding" in self.capability
|
||||
if vector_output and not self.vector:
|
||||
raise ValueError("vector-producing capabilities require a vector contract")
|
||||
if (
|
||||
vector_output
|
||||
and self.vector is not None
|
||||
and not self.vector.cross_deployment_compatible
|
||||
and self.upgrade_class is not UpgradeClass.REQUIRES_REINDEX
|
||||
):
|
||||
raise ValueError("incompatible vector spaces require requires_reindex")
|
||||
return self
|
||||
|
||||
|
||||
class ProjectDefinition(StrictModel):
|
||||
id: str = Field(pattern=r"^[a-z][a-z0-9-]*$")
|
||||
name: str = Field(min_length=1)
|
||||
description: str = Field(min_length=1)
|
||||
|
||||
|
||||
class ProjectBindingManifest(StrictModel):
|
||||
contract_version: int = Field(ge=1)
|
||||
channel: DeploymentChannel
|
||||
priority: WorkloadPriority
|
||||
optional: bool = False
|
||||
fallback: FallbackContract = Field(default_factory=FallbackContract)
|
||||
migration_support: Literal["none", "reindex", "schema"] = "none"
|
||||
slo: SLOContract = Field(default_factory=SLOContract)
|
||||
benchmark_suites: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
class ProjectManifest(StrictModel):
|
||||
project: ProjectDefinition
|
||||
bindings: dict[str, ProjectBindingManifest]
|
||||
notes: list[str] = Field(default_factory=list)
|
||||
|
||||
@field_validator("bindings")
|
||||
@classmethod
|
||||
def validate_bindings(
|
||||
cls, value: dict[str, ProjectBindingManifest]
|
||||
) -> dict[str, ProjectBindingManifest]:
|
||||
for key in value:
|
||||
if not CAPABILITY_KEY.fullmatch(key):
|
||||
raise ValueError(f"invalid capability binding key: {key}")
|
||||
return value
|
||||
|
||||
|
||||
class CandidateManifest(StrictModel):
|
||||
id: str
|
||||
display_name: str
|
||||
source: str
|
||||
intended_capabilities: list[str] = Field(min_length=1)
|
||||
proposed_role: str
|
||||
preferred_runtime: str | None = None
|
||||
verification_status: VerificationStatus = VerificationStatus.UNVERIFIED
|
||||
deployment_status: Literal["not_deployed"] = "not_deployed"
|
||||
|
||||
|
||||
class CandidateRegistryManifest(StrictModel):
|
||||
candidates: list[CandidateManifest]
|
||||
|
||||
|
||||
class BenchmarkSuiteDefinition(StrictModel):
|
||||
id: str
|
||||
capability: str
|
||||
version: int = Field(ge=1)
|
||||
purpose: str
|
||||
dataset_revision: str
|
||||
runnable: bool = False
|
||||
|
||||
|
||||
class BenchmarkSuiteManifest(StrictModel):
|
||||
suite: BenchmarkSuiteDefinition
|
||||
metrics: dict[str, list[str]]
|
||||
performance: dict[str, list[str]]
|
||||
release_policy: dict[str, Any]
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_pinned_dataset_for_runnable_suite(self) -> BenchmarkSuiteManifest:
|
||||
if self.suite.runnable and self.suite.dataset_revision.startswith("pending"):
|
||||
raise ValueError("runnable benchmark suites require a pinned dataset revision")
|
||||
return self
|
||||
|
||||
|
||||
class SecurityDefaults(StrictModel):
|
||||
trust_remote_code: Literal[False] = False
|
||||
prefer_safetensors: Literal[True] = True
|
||||
require_exact_revision_for_approval: Literal[True] = True
|
||||
require_artifact_digest: Literal[True] = True
|
||||
inference_network_egress: Literal[False] = False
|
||||
expose_runtime_workers: Literal[False] = False
|
||||
|
||||
|
||||
class LifecycleDefaults(StrictModel):
|
||||
automatic_production_promotion: Literal[False] = False
|
||||
require_local_benchmark_for_stable: Literal[True] = True
|
||||
retain_previous_stable_as_rollback_target: Literal[True] = True
|
||||
destructive_cleanup_requires_explicit_approval: Literal[True] = True
|
||||
|
||||
|
||||
class SchedulerDefaults(StrictModel):
|
||||
priority_order: list[WorkloadPriority]
|
||||
reserve_vram_mb: int = Field(ge=0)
|
||||
default_warm_ttl_seconds: int = Field(ge=0)
|
||||
benchmark_may_disrupt_production: Literal[False] = False
|
||||
|
||||
|
||||
class UpgradeDefaults(StrictModel):
|
||||
default_behavioral_change_class: Literal[UpgradeClass.BEHAVIORAL]
|
||||
vector_producer_default_change_class: Literal[UpgradeClass.REQUIRES_REINDEX]
|
||||
schema_change_requires_contract_version_bump: Literal[True] = True
|
||||
|
||||
|
||||
class PolicyDefaults(StrictModel):
|
||||
security: SecurityDefaults
|
||||
lifecycle: LifecycleDefaults
|
||||
scheduler: SchedulerDefaults
|
||||
upgrades: UpgradeDefaults
|
||||
|
||||
|
||||
class ArtifactProvenance(StrictModel):
|
||||
upstream_repository: str
|
||||
resolved_commit_sha: str
|
||||
filename: str
|
||||
artifact_type: str
|
||||
sha256: str
|
||||
source_artifact_sha256: str | None = None
|
||||
derivation_tool: str | None = None
|
||||
derivation_tool_version: str | None = None
|
||||
derivation_arguments: dict[str, Any] = Field(default_factory=dict)
|
||||
imported_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
||||
|
||||
@field_validator("resolved_commit_sha")
|
||||
@classmethod
|
||||
def validate_commit(cls, value: str) -> str:
|
||||
value = value.lower()
|
||||
if not COMMIT_SHA.fullmatch(value):
|
||||
raise ValueError("resolved commit must be a 40-64 character hexadecimal digest")
|
||||
return value
|
||||
|
||||
@field_validator("sha256", "source_artifact_sha256")
|
||||
@classmethod
|
||||
def validate_digest(cls, value: str | None) -> str | None:
|
||||
if value is not None and not SHA256.fullmatch(value.lower()):
|
||||
raise ValueError("artifact digest must be lowercase SHA-256")
|
||||
return value.lower() if value else None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_lineage(self) -> ArtifactProvenance:
|
||||
derived = self.source_artifact_sha256 is not None
|
||||
if derived and (not self.derivation_tool or not self.derivation_tool_version):
|
||||
raise ValueError("derived artifacts require derivation tool and version")
|
||||
return self
|
||||
|
||||
|
||||
class RuntimeProfileContract(StrictModel):
|
||||
runtime_type: Literal["vllm", "transformers", "diffusers", "llama_cpp", "custom"]
|
||||
runtime_version: str
|
||||
runtime_image_digest: str | None = Field(default=None, pattern=r"^sha256:[a-f0-9]{64}$")
|
||||
artifact_sha256: str = Field(pattern=r"^[a-f0-9]{64}$")
|
||||
quantization: str | None = None
|
||||
context_length: int | None = Field(default=None, gt=0)
|
||||
max_concurrency: int = Field(default=1, gt=0)
|
||||
launch_arguments: dict[str, Any] = Field(default_factory=dict)
|
||||
environment_constraints: dict[str, Any] = Field(default_factory=dict)
|
||||
trust_remote_code: bool = False
|
||||
network_egress: bool = False
|
||||
|
||||
@property
|
||||
def fingerprint(self) -> str:
|
||||
payload = self.model_dump(mode="json", exclude_none=True)
|
||||
return hashlib.sha256(json.dumps(payload, sort_keys=True).encode()).hexdigest()
|
||||
|
||||
|
||||
class ResourceEnvelopeContract(StrictModel):
|
||||
deployment_id: str
|
||||
accelerator_kind: str
|
||||
context_length: int | None = Field(default=None, gt=0)
|
||||
concurrency: int = Field(ge=1)
|
||||
batch_size: int = Field(ge=1)
|
||||
idle_vram_mb: int = Field(ge=0)
|
||||
peak_vram_mb: int = Field(ge=0)
|
||||
safety_margin_mb: int = Field(default=1024, ge=0)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def peak_not_below_idle(self) -> ResourceEnvelopeContract:
|
||||
if self.peak_vram_mb < self.idle_vram_mb:
|
||||
raise ValueError("peak VRAM cannot be below idle VRAM")
|
||||
return self
|
||||
|
||||
|
||||
class LeaseRequest(StrictModel):
|
||||
deployment_id: str
|
||||
priority: WorkloadPriority
|
||||
residency: ResidencyPolicy
|
||||
envelope: ResourceEnvelopeContract
|
||||
warm_ttl_seconds: int = Field(default=900, ge=0)
|
||||
exclusive: bool = False
|
||||
allow_cpu_fallback: bool = False
|
||||
|
||||
|
||||
class BenchmarkEnvironmentFingerprint(StrictModel):
|
||||
runtime_type: str
|
||||
runtime_version: str
|
||||
runtime_image_digest: str | None = None
|
||||
launch_arguments: dict[str, Any]
|
||||
cuda_version: str | None
|
||||
driver_version: str | None
|
||||
accelerator_name: str
|
||||
accelerator_uuid: str | None
|
||||
context_length: int | None
|
||||
concurrency: int = Field(ge=1)
|
||||
seed: int | None
|
||||
operating_system: str
|
||||
python_version: str | None = None
|
||||
|
||||
@property
|
||||
def digest(self) -> str:
|
||||
raw = json.dumps(self.model_dump(mode="json"), sort_keys=True, separators=(",", ":"))
|
||||
return hashlib.sha256(raw.encode()).hexdigest()
|
||||
|
||||
def comparable_with(self, other: BenchmarkEnvironmentFingerprint) -> bool:
|
||||
relevant = (
|
||||
"runtime_type",
|
||||
"runtime_version",
|
||||
"runtime_image_digest",
|
||||
"launch_arguments",
|
||||
"cuda_version",
|
||||
"driver_version",
|
||||
"accelerator_name",
|
||||
"context_length",
|
||||
"concurrency",
|
||||
)
|
||||
return all(getattr(self, field) == getattr(other, field) for field in relevant)
|
||||
|
||||
|
||||
class BenchmarkRunContract(StrictModel):
|
||||
deployment_id: str
|
||||
model_revision_sha: str = Field(pattern=r"^[a-f0-9]{40,64}$")
|
||||
artifact_sha256: str = Field(pattern=r"^[a-f0-9]{64}$")
|
||||
suite_key: str
|
||||
suite_revision: str
|
||||
dataset_revision: str
|
||||
environment: BenchmarkEnvironmentFingerprint
|
||||
started_at: datetime
|
||||
completed_at: datetime | None = None
|
||||
|
||||
|
||||
class MigrationContract(StrictModel):
|
||||
project_id: str
|
||||
capability: str
|
||||
source_deployment_id: str
|
||||
target_deployment_id: str
|
||||
upgrade_class: UpgradeClass
|
||||
current_index_ref: str
|
||||
shadow_index_ref: str
|
||||
status: MigrationStatus = MigrationStatus.PLANNED
|
||||
rollback_retain_until: datetime
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_reindex(self) -> MigrationContract:
|
||||
if (
|
||||
"embedding" in self.capability
|
||||
and self.upgrade_class is not UpgradeClass.REQUIRES_REINDEX
|
||||
):
|
||||
raise ValueError("embedding migrations must be classified requires_reindex")
|
||||
if self.current_index_ref == self.shadow_index_ref:
|
||||
raise ValueError("shadow index must be distinct from current production index")
|
||||
return self
|
||||
|
||||
|
||||
class LayerHealth(StrictModel):
|
||||
process: HealthStatus
|
||||
runtime: HealthStatus
|
||||
model: HealthStatus
|
||||
capability: HealthStatus
|
||||
project: HealthStatus
|
||||
reasons: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
class FailurePolicy(StrictModel):
|
||||
code: FailureCode
|
||||
owner: FailureOwner
|
||||
retryable: bool
|
||||
fallback_allowed: bool
|
||||
hard_failure: bool
|
||||
|
||||
|
||||
class RuntimeAdapter(Protocol):
|
||||
"""Stable control-plane boundary; concrete adapters arrive in M5."""
|
||||
|
||||
runtime_type: str
|
||||
|
||||
def validate_profile(self, profile: RuntimeProfileContract) -> None: ...
|
||||
def probe(self, profile: RuntimeProfileContract) -> LayerHealth: ...
|
||||
def load(self, profile: RuntimeProfileContract, lease_id: str) -> str: ...
|
||||
def unload(self, deployment_id: str) -> None: ...
|
||||
@@ -0,0 +1,200 @@
|
||||
from enum import StrEnum
|
||||
|
||||
|
||||
class ModelLifecycle(StrEnum):
|
||||
DISCOVERED = "discovered"
|
||||
CANDIDATE = "candidate"
|
||||
DOWNLOADING = "downloading"
|
||||
QUARANTINED = "quarantined"
|
||||
VERIFIED = "verified"
|
||||
TESTING = "testing"
|
||||
APPROVED = "approved"
|
||||
STANDBY = "standby"
|
||||
ACTIVE = "active"
|
||||
DEPRECATED = "deprecated"
|
||||
ARCHIVED = "archived"
|
||||
REJECTED = "rejected"
|
||||
INCOMPATIBLE = "incompatible"
|
||||
SECURITY_BLOCKED = "security_blocked"
|
||||
LICENSE_BLOCKED = "license_blocked"
|
||||
|
||||
|
||||
class UpgradeClass(StrEnum):
|
||||
TRANSPARENT = "transparent"
|
||||
BEHAVIORAL = "behavioral"
|
||||
REQUIRES_REINDEX = "requires_reindex"
|
||||
SCHEMA_BREAKING = "schema_breaking"
|
||||
|
||||
|
||||
class DeploymentChannel(StrEnum):
|
||||
STABLE = "stable"
|
||||
CANDIDATE = "candidate"
|
||||
EXPERIMENTAL = "experimental"
|
||||
ARCHIVE = "archive"
|
||||
|
||||
|
||||
class ResidencyPolicy(StrEnum):
|
||||
ALWAYS_WARM = "always_warm"
|
||||
KEEP_WARM = "keep_warm"
|
||||
LOAD_ON_DEMAND = "load_on_demand"
|
||||
EXCLUSIVE = "exclusive"
|
||||
LAB_ONLY = "lab_only"
|
||||
|
||||
|
||||
class WorkloadPriority(StrEnum):
|
||||
PRODUCTION = "production"
|
||||
INTERACTIVE = "interactive"
|
||||
BACKGROUND = "background"
|
||||
BENCHMARK = "benchmark"
|
||||
MAINTENANCE = "maintenance"
|
||||
|
||||
|
||||
class VerificationStatus(StrEnum):
|
||||
UNKNOWN = "unknown"
|
||||
UNVERIFIED = "unverified"
|
||||
QUARANTINED = "quarantined"
|
||||
VERIFIED = "verified"
|
||||
BLOCKED = "blocked"
|
||||
|
||||
|
||||
class LicenseStatus(StrEnum):
|
||||
UNKNOWN = "unknown"
|
||||
REVIEW_REQUIRED = "review_required"
|
||||
APPROVED = "approved"
|
||||
BLOCKED = "blocked"
|
||||
|
||||
|
||||
class ArtifactStatus(StrEnum):
|
||||
REMOTE = "remote"
|
||||
LOCAL = "local"
|
||||
VERIFYING = "verifying"
|
||||
VERIFIED = "verified"
|
||||
MISSING = "missing"
|
||||
CORRUPT = "corrupt"
|
||||
QUARANTINED = "quarantined"
|
||||
ARCHIVED = "archived"
|
||||
UNREACHABLE = "unreachable"
|
||||
|
||||
|
||||
class StorageRootStatus(StrEnum):
|
||||
UNKNOWN = "unknown"
|
||||
READY = "ready"
|
||||
READ_ONLY = "read_only"
|
||||
CAPACITY_BLOCKED = "capacity_blocked"
|
||||
UNAVAILABLE = "unavailable"
|
||||
DEPRECATED = "deprecated"
|
||||
|
||||
|
||||
class DeploymentStatus(StrEnum):
|
||||
DRAFT = "draft"
|
||||
VALIDATING = "validating"
|
||||
READY = "ready"
|
||||
ACTIVE = "active"
|
||||
UNHEALTHY = "unhealthy"
|
||||
RETIRED = "retired"
|
||||
|
||||
|
||||
class MigrationStatus(StrEnum):
|
||||
PLANNED = "planned"
|
||||
BACKFILLING = "backfilling"
|
||||
VALIDATING = "validating"
|
||||
SHADOWING = "shadowing"
|
||||
READY = "ready"
|
||||
PROMOTED = "promoted"
|
||||
ROLLED_BACK = "rolled_back"
|
||||
FAILED = "failed"
|
||||
|
||||
|
||||
class HealthStatus(StrEnum):
|
||||
HEALTHY = "healthy"
|
||||
DEGRADED = "degraded"
|
||||
UNHEALTHY = "unhealthy"
|
||||
UNAVAILABLE = "unavailable"
|
||||
UNKNOWN = "unknown"
|
||||
|
||||
|
||||
class FailureCode(StrEnum):
|
||||
MODEL_LOAD_FAILED = "MODEL_LOAD_FAILED"
|
||||
GPU_OOM = "GPU_OOM"
|
||||
RUNTIME_CRASH = "RUNTIME_CRASH"
|
||||
TIMEOUT = "TIMEOUT"
|
||||
INVALID_OUTPUT = "INVALID_OUTPUT"
|
||||
HEALTHCHECK_FAILED = "HEALTHCHECK_FAILED"
|
||||
ARTIFACT_CORRUPT = "ARTIFACT_CORRUPT"
|
||||
DRIVER_ERROR = "DRIVER_ERROR"
|
||||
CAPABILITY_UNAVAILABLE = "CAPABILITY_UNAVAILABLE"
|
||||
RUNTIME_INCOMPATIBLE = "RUNTIME_INCOMPATIBLE"
|
||||
ARTIFACT_INCOMPLETE = "ARTIFACT_INCOMPLETE"
|
||||
EXECUTION_NOT_APPROVED = "EXECUTION_NOT_APPROVED"
|
||||
RUNTIME_DEPENDENCY_MISSING = "RUNTIME_DEPENDENCY_MISSING"
|
||||
OFFLINE_LOAD_VIOLATION = "OFFLINE_LOAD_VIOLATION"
|
||||
UNLOAD_FAILED = "UNLOAD_FAILED"
|
||||
GPU_MEMORY_NOT_RECLAIMED = "GPU_MEMORY_NOT_RECLAIMED"
|
||||
NO_ELIGIBLE_NODE = "NO_ELIGIBLE_NODE"
|
||||
INSUFFICIENT_SCHEDULABLE_VRAM = "INSUFFICIENT_SCHEDULABLE_VRAM"
|
||||
QUEUE_FULL = "QUEUE_FULL"
|
||||
LEASE_TIMEOUT = "LEASE_TIMEOUT"
|
||||
RESIDENCY_LOAD_FAILED = "RESIDENCY_LOAD_FAILED"
|
||||
RESIDENCY_UNLOAD_FAILED = "RESIDENCY_UNLOAD_FAILED"
|
||||
AUTHORIZATION_DENIED = "AUTHORIZATION_DENIED"
|
||||
RATE_LIMITED = "RATE_LIMITED"
|
||||
EXTERNAL_GPU_PRESSURE = "EXTERNAL_GPU_PRESSURE"
|
||||
STALE_RESOURCE_ENVELOPE = "STALE_RESOURCE_ENVELOPE"
|
||||
|
||||
|
||||
class FailureOwner(StrEnum):
|
||||
RUNTIME_ADAPTER = "runtime_adapter"
|
||||
SCHEDULER = "scheduler"
|
||||
GATEWAY = "gateway"
|
||||
CONTROL_PLANE = "control_plane"
|
||||
OPERATOR = "operator"
|
||||
|
||||
|
||||
class Availability(StrEnum):
|
||||
KNOWN = "known"
|
||||
UNKNOWN = "unknown"
|
||||
UNSUPPORTED = "unsupported"
|
||||
UNAVAILABLE = "unavailable"
|
||||
TEMPORARILY_FAILED = "temporarily_failed"
|
||||
|
||||
|
||||
class HardwareStatus(StrEnum):
|
||||
UNKNOWN = "unknown"
|
||||
ACTIVE = "active"
|
||||
MISSING = "missing"
|
||||
UNAVAILABLE = "unavailable"
|
||||
DEGRADED = "degraded"
|
||||
PENDING = "pending"
|
||||
DECOMMISSIONED = "decommissioned"
|
||||
|
||||
|
||||
class InventorySource(StrEnum):
|
||||
HOST = "host"
|
||||
NVIDIA_NVML = "nvidia_nvml"
|
||||
|
||||
|
||||
class InventoryRunStatus(StrEnum):
|
||||
RUNNING = "running"
|
||||
SUCCEEDED = "succeeded"
|
||||
DEGRADED = "degraded"
|
||||
FAILED = "failed"
|
||||
|
||||
|
||||
class NodeLiveness(StrEnum):
|
||||
ONLINE = "online"
|
||||
STALE = "stale"
|
||||
OFFLINE = "offline"
|
||||
DISABLED = "disabled"
|
||||
DECOMMISSIONED = "decommissioned"
|
||||
|
||||
|
||||
class AgentHealth(StrEnum):
|
||||
HEALTHY = "healthy"
|
||||
INCOMPATIBLE = "incompatible"
|
||||
REVOKED = "revoked"
|
||||
UNKNOWN = "unknown"
|
||||
|
||||
|
||||
class ObservationSource(StrEnum):
|
||||
LOCAL_CONTROL_PLANE = "local_control_plane"
|
||||
REMOTE_AGENT = "remote_agent"
|
||||
@@ -0,0 +1,593 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import re
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
|
||||
MetricName = Literal["recall_at_5", "recall_at_10", "mrr", "ndcg_at_10"]
|
||||
TargetKind = Literal["current", "shadow"]
|
||||
_KEY = re.compile(r"^[a-z0-9][a-z0-9_-]{0,127}$")
|
||||
|
||||
|
||||
class EvaluationModel(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", from_attributes=True)
|
||||
|
||||
|
||||
class EvaluationCaseCreate(EvaluationModel):
|
||||
case_key: str
|
||||
query: str = Field(min_length=1, max_length=4000)
|
||||
relevant_chunk_ids: list[uuid.UUID] = Field(min_length=1, max_length=100)
|
||||
relevant_document_ids: list[uuid.UUID] = Field(default_factory=list, max_length=100)
|
||||
relevance_grades: dict[str, int] = Field(default_factory=dict)
|
||||
label_provenance: dict[str, Any]
|
||||
critical: bool = False
|
||||
review_status: Literal["reviewed", "approved"]
|
||||
|
||||
@field_validator("case_key")
|
||||
@classmethod
|
||||
def valid_key(cls, value: str) -> str:
|
||||
if not _KEY.fullmatch(value):
|
||||
raise ValueError("case_key must be a safe identifier")
|
||||
return value
|
||||
|
||||
@field_validator("relevant_chunk_ids")
|
||||
@classmethod
|
||||
def unique_relevance(cls, value: list[uuid.UUID]) -> list[uuid.UUID]:
|
||||
if len(set(value)) != len(value):
|
||||
raise ValueError("relevant chunk ids must be unique")
|
||||
return value
|
||||
|
||||
|
||||
class EvaluationRevisionCreate(EvaluationModel):
|
||||
revision: str
|
||||
dataset_revision: str
|
||||
metrics: list[MetricName] = ["recall_at_5", "recall_at_10", "mrr", "ndcg_at_10"]
|
||||
top_k: int = Field(default=10, ge=10, le=100)
|
||||
retrieval_settings: dict[str, Any]
|
||||
thresholds: dict[str, float] = Field(default_factory=dict)
|
||||
cases: list[EvaluationCaseCreate] = Field(min_length=1, max_length=500)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_revision(self) -> EvaluationRevisionCreate:
|
||||
if not _KEY.fullmatch(self.revision) or not _KEY.fullmatch(self.dataset_revision):
|
||||
raise ValueError("revision identifiers must be safe")
|
||||
if len(set(self.metrics)) != len(self.metrics):
|
||||
raise ValueError("metrics must be unique")
|
||||
if len({case.case_key for case in self.cases}) != len(self.cases):
|
||||
raise ValueError("case keys must be unique")
|
||||
return self
|
||||
|
||||
|
||||
class EvaluationSuiteCreate(EvaluationModel):
|
||||
project_id: uuid.UUID
|
||||
key: str
|
||||
name: str = Field(min_length=1, max_length=255)
|
||||
description: str = Field(min_length=1, max_length=4000)
|
||||
revision: EvaluationRevisionCreate
|
||||
|
||||
@field_validator("key")
|
||||
@classmethod
|
||||
def valid_key(cls, value: str) -> str:
|
||||
if not _KEY.fullmatch(value):
|
||||
raise ValueError("suite key must be a safe identifier")
|
||||
return value
|
||||
|
||||
|
||||
class EvaluationSuiteResponse(EvaluationModel):
|
||||
id: uuid.UUID
|
||||
project_id: uuid.UUID
|
||||
key: str
|
||||
name: str
|
||||
description: str
|
||||
latest_revision_id: uuid.UUID
|
||||
latest_revision: str
|
||||
case_count: int
|
||||
critical_case_count: int
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class EvaluationCaseDefinitionResponse(EvaluationModel):
|
||||
id: uuid.UUID
|
||||
suite_revision_id: uuid.UUID
|
||||
case_key: str
|
||||
query: str
|
||||
relevant_chunk_ids: list[str]
|
||||
relevant_document_ids: list[str]
|
||||
relevance_grades: dict[str, int]
|
||||
label_provenance: dict[str, Any]
|
||||
critical: bool
|
||||
review_status: str
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class EvaluationRunCreate(EvaluationModel):
|
||||
project_id: uuid.UUID
|
||||
suite_revision_id: uuid.UUID
|
||||
target_kind: TargetKind
|
||||
target_index_ref: str = Field(min_length=1, max_length=512)
|
||||
embedding_space_ref: str = Field(min_length=1, max_length=255)
|
||||
capability_deployment_id: uuid.UUID | None = None
|
||||
corpus_revision: str = Field(min_length=1, max_length=128)
|
||||
retrieval_config_digest: str = Field(pattern=r"^[0-9a-f]{64}$")
|
||||
environment_fingerprint: dict[str, Any]
|
||||
|
||||
|
||||
class RankedResult(EvaluationModel):
|
||||
chunk_id: uuid.UUID
|
||||
document_id: uuid.UUID | None = None
|
||||
score: float
|
||||
|
||||
@field_validator("score")
|
||||
@classmethod
|
||||
def finite_score(cls, value: float) -> float:
|
||||
if not math.isfinite(value):
|
||||
raise ValueError("score must be finite")
|
||||
return value
|
||||
|
||||
|
||||
class CandidatePoolEntry(EvaluationModel):
|
||||
id: str = Field(min_length=1, max_length=255)
|
||||
document_id: str | None = Field(default=None, max_length=255)
|
||||
score: float
|
||||
content_sha256: str = Field(pattern=r"^[0-9a-f]{64}$")
|
||||
|
||||
@field_validator("score")
|
||||
@classmethod
|
||||
def finite_score(cls, value: float) -> float:
|
||||
if not math.isfinite(value):
|
||||
raise ValueError("score must be finite")
|
||||
return value
|
||||
|
||||
|
||||
class RetrievalCandidatePoolCreate(EvaluationModel):
|
||||
project_id: uuid.UUID
|
||||
suite_revision_id: uuid.UUID
|
||||
evaluation_case_id: uuid.UUID
|
||||
source_embedding_space: str = Field(min_length=1, max_length=255)
|
||||
source_index_ref: str = Field(min_length=1, max_length=512)
|
||||
corpus_revision: str = Field(min_length=1, max_length=128)
|
||||
retrieval_config_digest: str = Field(pattern=r"^[0-9a-f]{64}$")
|
||||
ordered_candidates: list[CandidatePoolEntry] = Field(min_length=1, max_length=40)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def unique_ordered_ids(self) -> RetrievalCandidatePoolCreate:
|
||||
ids = [item.id for item in self.ordered_candidates]
|
||||
if len(set(ids)) != len(ids):
|
||||
raise ValueError("candidate pool ids must be unique")
|
||||
return self
|
||||
|
||||
|
||||
class RetrievalCandidatePoolResponse(EvaluationModel):
|
||||
id: uuid.UUID
|
||||
project_id: uuid.UUID
|
||||
suite_revision_id: uuid.UUID
|
||||
evaluation_case_id: uuid.UUID
|
||||
source_embedding_space: str
|
||||
source_index_ref: str
|
||||
corpus_revision: str
|
||||
retrieval_config_digest: str
|
||||
candidate_count: int
|
||||
ordered_candidates: list[dict[str, Any]]
|
||||
fingerprint: str
|
||||
created_at: datetime
|
||||
immutable_at: datetime
|
||||
|
||||
|
||||
class RetrievalPipelineIdentityCreate(EvaluationModel):
|
||||
project_id: uuid.UUID
|
||||
embedding_space_ref: str = Field(min_length=1, max_length=255)
|
||||
sparse_config_digest: str = Field(pattern=r"^[0-9a-f]{64}$")
|
||||
fusion_config_digest: str = Field(pattern=r"^[0-9a-f]{64}$")
|
||||
reranker_deployment_id: uuid.UUID | None = None
|
||||
reranker_config_digest: str | None = Field(default=None, pattern=r"^[0-9a-f]{64}$")
|
||||
candidate_k: Literal[40] = 40
|
||||
output_k: int = Field(default=10, ge=1, le=10)
|
||||
configuration: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def reranker_identity_is_complete(self) -> RetrievalPipelineIdentityCreate:
|
||||
if (self.reranker_deployment_id is None) != (self.reranker_config_digest is None):
|
||||
raise ValueError("reranker deployment and config digest must be supplied together")
|
||||
return self
|
||||
|
||||
|
||||
class RetrievalPipelineIdentityResponse(EvaluationModel):
|
||||
id: uuid.UUID
|
||||
project_id: uuid.UUID
|
||||
embedding_space_ref: str
|
||||
sparse_config_digest: str
|
||||
fusion_config_digest: str
|
||||
reranker_deployment_id: uuid.UUID | None
|
||||
reranker_config_digest: str | None
|
||||
candidate_k: int
|
||||
output_k: int
|
||||
identity_digest: str
|
||||
configuration: dict[str, Any]
|
||||
migration_class: str
|
||||
created_at: datetime
|
||||
immutable_at: datetime
|
||||
|
||||
|
||||
class RerankingRunCreate(EvaluationModel):
|
||||
project_id: uuid.UUID
|
||||
suite_revision_id: uuid.UUID
|
||||
pipeline_identity_id: uuid.UUID
|
||||
control_pipeline_identity_id: uuid.UUID
|
||||
candidate_pool_ids: list[uuid.UUID] = Field(min_length=1, max_length=500)
|
||||
corpus_revision: str = Field(min_length=1, max_length=128)
|
||||
environment_fingerprint: dict[str, Any]
|
||||
|
||||
|
||||
class RerankingCaseResultCreate(EvaluationModel):
|
||||
case_id: uuid.UUID
|
||||
candidate_pool_id: uuid.UUID
|
||||
ranked_results: list[RankedResult] = Field(min_length=1, max_length=10)
|
||||
retrieval_latency_ms: float = Field(ge=0, le=3_600_000)
|
||||
rerank_latency_ms: float = Field(ge=0, le=3_600_000)
|
||||
total_latency_ms: float = Field(ge=0, le=3_600_000)
|
||||
error_code: str | None = Field(default=None, max_length=64)
|
||||
|
||||
|
||||
class RerankingRunComplete(EvaluationModel):
|
||||
results: list[RerankingCaseResultCreate] = Field(min_length=1, max_length=500)
|
||||
|
||||
|
||||
class RerankingRunResponse(EvaluationModel):
|
||||
id: uuid.UUID
|
||||
project_id: uuid.UUID
|
||||
suite_revision_id: uuid.UUID
|
||||
pipeline_identity_id: uuid.UUID
|
||||
control_pipeline_identity_id: uuid.UUID
|
||||
reranker_deployment_id: uuid.UUID | None
|
||||
status: str
|
||||
corpus_revision: str
|
||||
candidate_pool_set_fingerprint: str
|
||||
environment_fingerprint: dict[str, Any]
|
||||
environment_digest: str
|
||||
expected_cases: int
|
||||
completed_cases: int
|
||||
error_count: int
|
||||
aggregate_metrics: dict[str, float]
|
||||
latency_metrics: dict[str, float]
|
||||
started_at: datetime | None
|
||||
completed_at: datetime | None
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class RerankingCaseResultResponse(EvaluationModel):
|
||||
id: uuid.UUID
|
||||
run_id: uuid.UUID
|
||||
case_id: uuid.UUID
|
||||
case_key: str
|
||||
critical: bool
|
||||
candidate_pool_id: uuid.UUID
|
||||
ranked_results: list[dict[str, Any]]
|
||||
relevant_results: list[str]
|
||||
first_relevant_rank: int | None
|
||||
metrics: dict[str, float]
|
||||
retrieval_latency_ms: float
|
||||
rerank_latency_ms: float
|
||||
total_latency_ms: float
|
||||
error_code: str | None
|
||||
|
||||
|
||||
class DiscoveryCandidateAssessmentCreate(EvaluationModel):
|
||||
model_id: uuid.UUID | None = None
|
||||
upstream_snapshot_id: uuid.UUID
|
||||
candidate_key: str
|
||||
repository_id: str = Field(min_length=3, max_length=255)
|
||||
resolved_commit_sha: str = Field(pattern=r"^[0-9a-f]{40,64}$")
|
||||
artifact_evidence: dict[str, Any]
|
||||
security_state: dict[str, Any]
|
||||
license_state: dict[str, Any]
|
||||
gpu_fit: dict[str, Any]
|
||||
status: Literal[
|
||||
"shortlisted",
|
||||
"preflight_passed",
|
||||
"evaluated",
|
||||
"discovery_security_blocked",
|
||||
"discovery_gpu_fit_blocked",
|
||||
"license_blocked",
|
||||
"not_selected",
|
||||
]
|
||||
rationale: str = Field(min_length=10, max_length=4000)
|
||||
|
||||
@field_validator("candidate_key")
|
||||
@classmethod
|
||||
def valid_candidate_key(cls, value: str) -> str:
|
||||
if not _KEY.fullmatch(value):
|
||||
raise ValueError("candidate_key must be a safe identifier")
|
||||
return value
|
||||
|
||||
|
||||
class DiscoveryCandidateAssessmentResponse(DiscoveryCandidateAssessmentCreate):
|
||||
id: uuid.UUID
|
||||
evidence_fingerprint: str
|
||||
created_at: datetime
|
||||
immutable_at: datetime
|
||||
|
||||
|
||||
class EvaluationCaseResultCreate(EvaluationModel):
|
||||
case_id: uuid.UUID
|
||||
ranked_results: list[RankedResult] = Field(max_length=100)
|
||||
latency_ms: float = Field(ge=0, le=3_600_000)
|
||||
error_code: str | None = Field(default=None, max_length=64)
|
||||
|
||||
|
||||
class EvaluationRunComplete(EvaluationModel):
|
||||
results: list[EvaluationCaseResultCreate] = Field(min_length=1, max_length=500)
|
||||
|
||||
|
||||
class EvaluationRunResponse(EvaluationModel):
|
||||
id: uuid.UUID
|
||||
project_id: uuid.UUID
|
||||
suite_revision_id: uuid.UUID
|
||||
target_kind: str
|
||||
target_index_ref: str
|
||||
embedding_space_ref: str
|
||||
capability_deployment_id: uuid.UUID | None
|
||||
status: str
|
||||
corpus_revision: str
|
||||
retrieval_config_digest: str
|
||||
environment_fingerprint: dict[str, Any]
|
||||
environment_digest: str
|
||||
expected_cases: int
|
||||
completed_cases: int
|
||||
error_count: int
|
||||
aggregate_metrics: dict[str, float]
|
||||
started_at: datetime | None
|
||||
completed_at: datetime | None
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class EvaluationCaseResultResponse(EvaluationModel):
|
||||
id: uuid.UUID
|
||||
run_id: uuid.UUID
|
||||
case_id: uuid.UUID
|
||||
case_key: str
|
||||
critical: bool
|
||||
ranked_results: list[dict[str, Any]]
|
||||
relevant_results: list[str]
|
||||
first_relevant_rank: int | None
|
||||
metrics: dict[str, float]
|
||||
latency_ms: float
|
||||
error_code: str | None
|
||||
|
||||
|
||||
class EvaluationComparisonCreate(EvaluationModel):
|
||||
baseline_run_id: uuid.UUID
|
||||
candidate_run_id: uuid.UUID
|
||||
require_no_critical_regressions: bool = True
|
||||
maximum_error_rate: float = Field(default=0.0, ge=0, le=1)
|
||||
minimum_recall_at_10_delta: float = Field(default=0.0, ge=-1, le=1)
|
||||
|
||||
|
||||
class EvaluationComparisonResponse(EvaluationModel):
|
||||
id: uuid.UUID
|
||||
project_id: uuid.UUID
|
||||
baseline_run_id: uuid.UUID
|
||||
candidate_run_id: uuid.UUID
|
||||
comparability: str
|
||||
comparability_evidence: dict[str, Any]
|
||||
metric_deltas: dict[str, float]
|
||||
improved_cases: int
|
||||
unchanged_cases: int
|
||||
regressed_cases: int
|
||||
critical_regressions: int
|
||||
case_comparisons: list[dict[str, Any]]
|
||||
promotion_eligibility: str
|
||||
eligibility_evidence: dict[str, Any]
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class ModelComparisonCandidateCreate(EvaluationModel):
|
||||
candidate_key: str
|
||||
label: str = Field(min_length=1, max_length=255)
|
||||
status: Literal["evaluated", "blocked", "unknown"]
|
||||
evaluation_run_id: uuid.UUID | None = None
|
||||
candidate_deployment_id: uuid.UUID | None = None
|
||||
embedding_space: str | None = Field(default=None, max_length=255)
|
||||
artifact_size_bytes: int | None = Field(default=None, ge=0)
|
||||
latency_ms: dict[str, float] = Field(default_factory=dict)
|
||||
resource_evidence: dict[str, Any] = Field(default_factory=dict)
|
||||
migration_impact: dict[str, Any] = Field(default_factory=dict)
|
||||
security_state: dict[str, Any] = Field(default_factory=dict)
|
||||
provenance: dict[str, Any] = Field(default_factory=dict)
|
||||
blockers: list[str] = Field(default_factory=list, max_length=32)
|
||||
candidate_kind: Literal["embedding_deployment", "retrieval_pipeline"] = "embedding_deployment"
|
||||
pipeline_identity_id: uuid.UUID | None = None
|
||||
|
||||
@field_validator("candidate_key")
|
||||
@classmethod
|
||||
def valid_candidate_key(cls, value: str) -> str:
|
||||
if not _KEY.fullmatch(value):
|
||||
raise ValueError("candidate_key must be a safe identifier")
|
||||
return value
|
||||
|
||||
@model_validator(mode="after")
|
||||
def evaluated_run_required(self) -> ModelComparisonCandidateCreate:
|
||||
if (
|
||||
self.status == "evaluated"
|
||||
and self.candidate_kind == "embedding_deployment"
|
||||
and self.evaluation_run_id is None
|
||||
):
|
||||
raise ValueError("evaluated embedding candidate requires evaluation_run_id")
|
||||
if self.candidate_kind == "retrieval_pipeline" and self.evaluation_run_id is not None:
|
||||
raise ValueError("retrieval pipeline evidence must come from a fixed-pool run")
|
||||
if self.status != "evaluated" and self.evaluation_run_id is not None:
|
||||
raise ValueError("blocked or unknown candidate cannot claim an evaluation run")
|
||||
if self.candidate_kind == "retrieval_pipeline" and self.pipeline_identity_id is None:
|
||||
raise ValueError("retrieval pipeline candidate requires pipeline_identity_id")
|
||||
return self
|
||||
|
||||
|
||||
class ModelComparisonCreate(EvaluationModel):
|
||||
project_id: uuid.UUID
|
||||
capability_contract_id: uuid.UUID
|
||||
suite_revision_id: uuid.UUID
|
||||
current_run_id: uuid.UUID
|
||||
title: str = Field(min_length=1, max_length=255)
|
||||
candidates: list[ModelComparisonCandidateCreate] = Field(min_length=1, max_length=20)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def unique_candidates(self) -> ModelComparisonCreate:
|
||||
if len({item.candidate_key for item in self.candidates}) != len(self.candidates):
|
||||
raise ValueError("candidate keys must be unique")
|
||||
return self
|
||||
|
||||
|
||||
class ModelComparisonResponse(EvaluationModel):
|
||||
id: uuid.UUID
|
||||
project_id: uuid.UUID
|
||||
capability_contract_id: uuid.UUID
|
||||
suite_revision_id: uuid.UUID
|
||||
current_run_id: uuid.UUID
|
||||
title: str
|
||||
candidates: list[dict[str, Any]]
|
||||
comparability: str
|
||||
evidence_fingerprint: str
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class AdvisorPolicyUpdate(EvaluationModel):
|
||||
maximum_latency_p95_regression_ratio: float = Field(ge=0, le=10)
|
||||
minimum_metric_deltas: dict[MetricName, float]
|
||||
rationale: str = Field(min_length=20, max_length=4000)
|
||||
|
||||
|
||||
class AdvisorPolicyResponse(EvaluationModel):
|
||||
id: uuid.UUID
|
||||
key: str
|
||||
required_evidence_level: str
|
||||
critical_regression_hard_block: bool
|
||||
maximum_latency_p95_regression_ratio: float
|
||||
minimum_metric_deltas: dict[str, float]
|
||||
require_verified_supply_chain: bool
|
||||
require_runtime_fit: bool
|
||||
rationale: str
|
||||
version: int
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
|
||||
|
||||
class AdvisorRecommendationCreate(EvaluationModel):
|
||||
candidate_key: str
|
||||
|
||||
@field_validator("candidate_key")
|
||||
@classmethod
|
||||
def valid_candidate_key(cls, value: str) -> str:
|
||||
if not _KEY.fullmatch(value):
|
||||
raise ValueError("candidate_key must be a safe identifier")
|
||||
return value
|
||||
|
||||
|
||||
class AdvisorRecommendationDismiss(EvaluationModel):
|
||||
dismissed_by: str = Field(min_length=1, max_length=255)
|
||||
reason: str = Field(min_length=10, max_length=2000)
|
||||
|
||||
|
||||
class AdvisorRecommendationResponse(EvaluationModel):
|
||||
id: uuid.UUID
|
||||
comparison_id: uuid.UUID
|
||||
project_id: uuid.UUID
|
||||
capability_contract_id: uuid.UUID
|
||||
policy_id: uuid.UUID
|
||||
candidate_key: str
|
||||
current_deployment_id: uuid.UUID | None
|
||||
candidate_deployment_id: uuid.UUID | None
|
||||
current_embedding_space: str
|
||||
candidate_embedding_space: str | None
|
||||
target_kind: Literal["embedding_deployment", "retrieval_pipeline"]
|
||||
current_pipeline_identity_id: uuid.UUID | None
|
||||
candidate_pipeline_identity_id: uuid.UUID | None
|
||||
verdict: Literal[
|
||||
"KEEP_CURRENT",
|
||||
"KEEP_CURRENT_EMBEDDING_ADD_RERANKER_CANDIDATE",
|
||||
"PROMOTION_ELIGIBLE",
|
||||
"PROMOTION_NOT_RECOMMENDED",
|
||||
"REQUIRES_MORE_EVIDENCE",
|
||||
]
|
||||
confidence: str
|
||||
evidence_level: str
|
||||
quality_deltas: dict[str, float]
|
||||
latency_deltas: dict[str, Any]
|
||||
resource_deltas: dict[str, Any]
|
||||
migration_impact: dict[str, Any]
|
||||
security_state: dict[str, Any]
|
||||
key_improvements: list[str]
|
||||
blockers: list[str]
|
||||
policy_snapshot: dict[str, Any]
|
||||
evidence_fingerprint: str
|
||||
status: str
|
||||
generated_at: datetime
|
||||
dismissed_at: datetime | None
|
||||
dismissed_by: str | None
|
||||
dismissal_reason: str | None
|
||||
|
||||
|
||||
class EmbeddingMigrationCreate(EvaluationModel):
|
||||
source_embedding_space: str = Field(min_length=1, max_length=255)
|
||||
target_embedding_space_id: uuid.UUID
|
||||
source_index_ref: str = Field(min_length=1, max_length=512)
|
||||
target_index_ref: str = Field(min_length=1, max_length=512)
|
||||
corpus_revision: str = Field(min_length=1, max_length=128)
|
||||
total_chunks: int = Field(ge=1, le=10_000_000)
|
||||
batch_size: int = Field(default=32, ge=1, le=256)
|
||||
concurrency: int = Field(default=1, ge=1, le=8)
|
||||
|
||||
|
||||
class MigrationUpdate(EvaluationModel):
|
||||
status: Literal[
|
||||
"preflight",
|
||||
"backfilling",
|
||||
"validating",
|
||||
"ready_for_evaluation",
|
||||
"evaluated",
|
||||
"promotion_eligible",
|
||||
"not_eligible",
|
||||
"failed",
|
||||
"cancelled",
|
||||
]
|
||||
completed_chunks: int = Field(ge=0)
|
||||
failed_chunks: int = Field(ge=0)
|
||||
retried_chunks: int = Field(ge=0)
|
||||
preflight_evidence: dict[str, Any] = Field(default_factory=dict)
|
||||
progress_evidence: dict[str, Any] = Field(default_factory=dict)
|
||||
validation_evidence: dict[str, Any] = Field(default_factory=dict)
|
||||
operational_metrics: dict[str, Any] = Field(default_factory=dict)
|
||||
failure_code: str | None = Field(default=None, max_length=64)
|
||||
failure_message: str | None = Field(default=None, max_length=2000)
|
||||
|
||||
|
||||
class EmbeddingMigrationResponse(EvaluationModel):
|
||||
id: uuid.UUID
|
||||
project_id: uuid.UUID
|
||||
source_embedding_space: str
|
||||
target_embedding_space_id: uuid.UUID
|
||||
source_index_ref: str
|
||||
target_index_ref: str
|
||||
corpus_revision: str
|
||||
status: str
|
||||
total_chunks: int
|
||||
completed_chunks: int
|
||||
failed_chunks: int
|
||||
retried_chunks: int
|
||||
batch_size: int
|
||||
concurrency: int
|
||||
priority: str
|
||||
preflight_evidence: dict[str, Any]
|
||||
progress_evidence: dict[str, Any]
|
||||
validation_evidence: dict[str, Any]
|
||||
operational_metrics: dict[str, Any]
|
||||
evaluation_eligibility: bool
|
||||
cancel_requested: bool
|
||||
failure_code: str | None
|
||||
failure_message: str | None
|
||||
started_at: datetime | None
|
||||
finished_at: datetime | None
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
@@ -0,0 +1,246 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from datetime import UTC, datetime
|
||||
from typing import Protocol
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
||||
|
||||
from modelforge_api.domain.enums import (
|
||||
AgentHealth,
|
||||
Availability,
|
||||
HardwareStatus,
|
||||
InventorySource,
|
||||
NodeLiveness,
|
||||
ObservationSource,
|
||||
)
|
||||
|
||||
|
||||
class HardwareModel(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class ObservedValue[T](HardwareModel):
|
||||
value: T | None = None
|
||||
availability: Availability
|
||||
reason: str | None = None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def value_matches_availability(self) -> ObservedValue[T]:
|
||||
if self.availability is Availability.KNOWN and self.value is None:
|
||||
raise ValueError("known observations require a value")
|
||||
if self.availability is not Availability.KNOWN and self.value is not None:
|
||||
raise ValueError("non-known observations cannot carry a value")
|
||||
return self
|
||||
|
||||
@classmethod
|
||||
def known(cls, value: T) -> ObservedValue[T]:
|
||||
return cls(value=value, availability=Availability.KNOWN)
|
||||
|
||||
@classmethod
|
||||
def absent(cls, availability: Availability, reason: str | None = None) -> ObservedValue[T]:
|
||||
return cls(availability=availability, reason=reason)
|
||||
|
||||
|
||||
class StorageObservation(HardwareModel):
|
||||
purpose: str
|
||||
path: str
|
||||
total_bytes: ObservedValue[int]
|
||||
used_bytes: ObservedValue[int]
|
||||
free_bytes: ObservedValue[int]
|
||||
|
||||
|
||||
class HostInventory(HardwareModel):
|
||||
identity_key: str
|
||||
identity_source: str
|
||||
hostname: str
|
||||
display_name: str
|
||||
os_name: str
|
||||
os_version: ObservedValue[str]
|
||||
architecture: str
|
||||
kernel_version: ObservedValue[str]
|
||||
cpu_model: ObservedValue[str]
|
||||
logical_cpu_count: ObservedValue[int]
|
||||
physical_core_count: ObservedValue[int]
|
||||
total_ram_bytes: ObservedValue[int]
|
||||
available_ram_bytes: ObservedValue[int]
|
||||
agent_version: str
|
||||
inventory_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
||||
storage: list[StorageObservation] = Field(default_factory=list)
|
||||
metadata: dict[str, str] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class AcceleratorInventory(HardwareModel):
|
||||
device_index: int
|
||||
device_uuid: str
|
||||
pci_bus_id: ObservedValue[str]
|
||||
name: str
|
||||
vendor: str = "NVIDIA"
|
||||
architecture: ObservedValue[str]
|
||||
compute_capability_major: ObservedValue[int]
|
||||
compute_capability_minor: ObservedValue[int]
|
||||
total_vram_bytes: ObservedValue[int]
|
||||
driver_version: ObservedValue[str]
|
||||
cuda_driver_version: ObservedValue[str]
|
||||
mig_mode_current: ObservedValue[bool]
|
||||
inventory_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
||||
source: InventorySource = InventorySource.NVIDIA_NVML
|
||||
|
||||
|
||||
class AcceleratorTelemetry(HardwareModel):
|
||||
device_uuid: str
|
||||
observed_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
||||
used_vram_bytes: ObservedValue[int]
|
||||
free_vram_bytes: ObservedValue[int]
|
||||
gpu_utilization_percent: ObservedValue[int]
|
||||
memory_utilization_percent: ObservedValue[int]
|
||||
temperature_c: ObservedValue[int]
|
||||
power_draw_w: ObservedValue[float]
|
||||
power_limit_w: ObservedValue[float]
|
||||
graphics_clock_mhz: ObservedValue[int]
|
||||
memory_clock_mhz: ObservedValue[int]
|
||||
fan_speed_percent: ObservedValue[int]
|
||||
performance_state: ObservedValue[str]
|
||||
|
||||
|
||||
class NvidiaCollection(HardwareModel):
|
||||
availability: Availability
|
||||
reason: str | None = None
|
||||
inventory: list[AcceleratorInventory] = Field(default_factory=list)
|
||||
telemetry: list[AcceleratorTelemetry] = Field(default_factory=list)
|
||||
observed_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class HardwareSnapshot(HardwareModel):
|
||||
host: HostInventory
|
||||
nvidia: NvidiaCollection
|
||||
|
||||
@property
|
||||
def fingerprint(self) -> str:
|
||||
facts = {
|
||||
"node_identity": self.host.identity_key,
|
||||
"os": self.host.os_name,
|
||||
"os_version": self.host.os_version.model_dump(mode="json"),
|
||||
"architecture": self.host.architecture,
|
||||
"kernel": self.host.kernel_version.model_dump(mode="json"),
|
||||
"accelerators": [
|
||||
{
|
||||
"uuid": item.device_uuid,
|
||||
"name": item.name,
|
||||
"pci": item.pci_bus_id.model_dump(mode="json"),
|
||||
"vram": item.total_vram_bytes.model_dump(mode="json"),
|
||||
"compute_major": item.compute_capability_major.model_dump(mode="json"),
|
||||
"compute_minor": item.compute_capability_minor.model_dump(mode="json"),
|
||||
"driver": item.driver_version.model_dump(mode="json"),
|
||||
"cuda_driver": item.cuda_driver_version.model_dump(mode="json"),
|
||||
}
|
||||
for item in sorted(self.nvidia.inventory, key=lambda value: value.device_uuid)
|
||||
],
|
||||
}
|
||||
canonical = json.dumps(facts, sort_keys=True, separators=(",", ":"))
|
||||
return hashlib.sha256(canonical.encode()).hexdigest()
|
||||
|
||||
|
||||
class HostCollector(Protocol):
|
||||
def collect(self) -> HostInventory: ...
|
||||
|
||||
|
||||
class AcceleratorCollector(Protocol):
|
||||
def collect(self) -> NvidiaCollection: ...
|
||||
|
||||
|
||||
class HardwareOverview(HardwareModel):
|
||||
status: HardwareStatus
|
||||
inventory_state: HardwareStatus
|
||||
node_count: int
|
||||
accelerator_count: int
|
||||
last_inventory_at: datetime | None
|
||||
reason: str | None = None
|
||||
|
||||
|
||||
class StorageState(HardwareModel):
|
||||
id: str
|
||||
purpose: str
|
||||
path: str
|
||||
total_bytes: ObservedValue[int]
|
||||
used_bytes: ObservedValue[int]
|
||||
free_bytes: ObservedValue[int]
|
||||
observed_at: datetime
|
||||
|
||||
|
||||
class AcceleratorState(HardwareModel):
|
||||
id: str
|
||||
node_id: str
|
||||
status: HardwareStatus
|
||||
status_reason: str | None
|
||||
device_index: int
|
||||
device_uuid: str
|
||||
pci_bus_id: ObservedValue[str]
|
||||
name: str
|
||||
vendor: str
|
||||
architecture: ObservedValue[str]
|
||||
compute_capability_major: ObservedValue[int]
|
||||
compute_capability_minor: ObservedValue[int]
|
||||
total_vram_bytes: ObservedValue[int]
|
||||
driver_version: ObservedValue[str]
|
||||
cuda_driver_version: ObservedValue[str]
|
||||
mig_mode_current: ObservedValue[bool]
|
||||
first_seen_at: datetime
|
||||
last_seen_at: datetime | None
|
||||
inventory_at: datetime | None
|
||||
telemetry: AcceleratorTelemetry | None
|
||||
|
||||
|
||||
class NodeState(HardwareModel):
|
||||
id: str
|
||||
identity_key: str
|
||||
identity_source: str
|
||||
hostname: str
|
||||
display_name: str
|
||||
status: HardwareStatus
|
||||
status_reason: str | None
|
||||
os_name: str | None
|
||||
os_version: ObservedValue[str]
|
||||
architecture: str | None
|
||||
kernel_version: ObservedValue[str]
|
||||
cpu_model: ObservedValue[str]
|
||||
logical_cpu_count: ObservedValue[int]
|
||||
physical_core_count: ObservedValue[int]
|
||||
total_ram_bytes: ObservedValue[int]
|
||||
available_ram_bytes: ObservedValue[int]
|
||||
agent_version: str | None
|
||||
first_seen_at: datetime
|
||||
last_seen_at: datetime | None
|
||||
inventory_at: datetime | None
|
||||
hardware_fingerprint: str | None
|
||||
enabled: bool = True
|
||||
liveness: NodeLiveness = NodeLiveness.OFFLINE
|
||||
agent_health: AgentHealth = AgentHealth.UNKNOWN
|
||||
observation_source: ObservationSource = ObservationSource.LOCAL_CONTROL_PLANE
|
||||
protocol_version: int | None = None
|
||||
supported_capabilities: list[str] = Field(default_factory=list)
|
||||
agent_started_at: datetime | None = None
|
||||
last_heartbeat_at: datetime | None = None
|
||||
last_inventory_received_at: datetime | None = None
|
||||
last_telemetry_received_at: datetime | None = None
|
||||
inventory_age_seconds: int | None = None
|
||||
telemetry_age_seconds: int | None = None
|
||||
last_connection_error: str | None = None
|
||||
role: str | None = None
|
||||
labels: dict[str, str | bool] = Field(default_factory=dict)
|
||||
production_eligible: bool = False
|
||||
lab_eligible: bool = True
|
||||
benchmark_eligible: bool = False
|
||||
generation: int = 1
|
||||
decommissioned_at: datetime | None = None
|
||||
decommission_reason: str | None = None
|
||||
decommissioned_by: str | None = None
|
||||
environment: str | None = None
|
||||
storage: list[StorageState]
|
||||
accelerators: list[AcceleratorState]
|
||||
|
||||
|
||||
class HardwareState(HardwareModel):
|
||||
overview: HardwareOverview
|
||||
nodes: list[NodeState]
|
||||
@@ -0,0 +1,75 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from .enums import MigrationStatus, ModelLifecycle, UpgradeClass, VerificationStatus
|
||||
|
||||
|
||||
class InvalidTransition(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
MODEL_TRANSITIONS: dict[ModelLifecycle, frozenset[ModelLifecycle]] = {
|
||||
ModelLifecycle.DISCOVERED: frozenset(
|
||||
{ModelLifecycle.CANDIDATE, ModelLifecycle.REJECTED, ModelLifecycle.DEPRECATED}
|
||||
),
|
||||
ModelLifecycle.CANDIDATE: frozenset(
|
||||
{ModelLifecycle.DOWNLOADING, ModelLifecycle.REJECTED, ModelLifecycle.DEPRECATED}
|
||||
),
|
||||
ModelLifecycle.DOWNLOADING: frozenset({ModelLifecycle.QUARANTINED, ModelLifecycle.REJECTED}),
|
||||
ModelLifecycle.QUARANTINED: frozenset(
|
||||
{ModelLifecycle.VERIFIED, ModelLifecycle.SECURITY_BLOCKED, ModelLifecycle.LICENSE_BLOCKED}
|
||||
),
|
||||
ModelLifecycle.VERIFIED: frozenset({ModelLifecycle.TESTING, ModelLifecycle.INCOMPATIBLE}),
|
||||
ModelLifecycle.TESTING: frozenset(
|
||||
{ModelLifecycle.APPROVED, ModelLifecycle.INCOMPATIBLE, ModelLifecycle.REJECTED}
|
||||
),
|
||||
ModelLifecycle.APPROVED: frozenset(
|
||||
{ModelLifecycle.STANDBY, ModelLifecycle.ACTIVE, ModelLifecycle.DEPRECATED}
|
||||
),
|
||||
ModelLifecycle.STANDBY: frozenset({ModelLifecycle.ACTIVE, ModelLifecycle.DEPRECATED}),
|
||||
ModelLifecycle.ACTIVE: frozenset({ModelLifecycle.STANDBY, ModelLifecycle.DEPRECATED}),
|
||||
ModelLifecycle.DEPRECATED: frozenset({ModelLifecycle.ARCHIVED, ModelLifecycle.ACTIVE}),
|
||||
ModelLifecycle.ARCHIVED: frozenset(),
|
||||
ModelLifecycle.REJECTED: frozenset(),
|
||||
ModelLifecycle.INCOMPATIBLE: frozenset({ModelLifecycle.CANDIDATE}),
|
||||
ModelLifecycle.SECURITY_BLOCKED: frozenset(
|
||||
{ModelLifecycle.QUARANTINED, ModelLifecycle.REJECTED}
|
||||
),
|
||||
ModelLifecycle.LICENSE_BLOCKED: frozenset(
|
||||
{ModelLifecycle.QUARANTINED, ModelLifecycle.REJECTED}
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def assert_model_transition(current: ModelLifecycle, target: ModelLifecycle) -> None:
|
||||
if target not in MODEL_TRANSITIONS[current]:
|
||||
raise InvalidTransition(f"model lifecycle transition {current} -> {target} is not allowed")
|
||||
|
||||
|
||||
def assert_promotion_allowed(
|
||||
*,
|
||||
upgrade_class: UpgradeClass,
|
||||
verification_status: VerificationStatus,
|
||||
local_benchmark_ids: list[str],
|
||||
project_benchmark_ids: list[str],
|
||||
operator_approval_id: str | None,
|
||||
rollback_deployment_id: str | None,
|
||||
migration_status: MigrationStatus | None = None,
|
||||
) -> None:
|
||||
blockers: list[str] = []
|
||||
if verification_status is not VerificationStatus.VERIFIED:
|
||||
blockers.append("artifact is not verified")
|
||||
if not local_benchmark_ids:
|
||||
blockers.append("local benchmark evidence is missing")
|
||||
if not project_benchmark_ids:
|
||||
blockers.append("project benchmark evidence is missing")
|
||||
if not operator_approval_id:
|
||||
blockers.append("operator approval is missing")
|
||||
if not rollback_deployment_id:
|
||||
blockers.append("rollback target is missing")
|
||||
if (
|
||||
upgrade_class is UpgradeClass.REQUIRES_REINDEX
|
||||
and migration_status is not MigrationStatus.READY
|
||||
):
|
||||
blockers.append("ready migration is required for reindexing change")
|
||||
if blockers:
|
||||
raise InvalidTransition("promotion blocked: " + "; ".join(blockers))
|
||||
@@ -0,0 +1,412 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from enum import StrEnum
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
||||
|
||||
|
||||
class LifecycleModel(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", from_attributes=True)
|
||||
|
||||
|
||||
class LifecycleEnvironment(StrEnum):
|
||||
LAB = "LAB"
|
||||
PRODUCTION = "PRODUCTION"
|
||||
|
||||
|
||||
class DeploymentLifecycleState(StrEnum):
|
||||
CANDIDATE = "CANDIDATE"
|
||||
LAB_READY = "LAB_READY"
|
||||
PROMOTION_ELIGIBLE = "PROMOTION_ELIGIBLE"
|
||||
CANARY = "CANARY"
|
||||
LAB_STABLE = "LAB_STABLE"
|
||||
STABLE = "STABLE"
|
||||
DRAINING = "DRAINING"
|
||||
DEPRECATED = "DEPRECATED"
|
||||
ARCHIVED = "ARCHIVED"
|
||||
MANUAL_INTERVENTION_REQUIRED = "MANUAL_INTERVENTION_REQUIRED"
|
||||
|
||||
|
||||
class ApprovalStatus(StrEnum):
|
||||
PENDING = "PENDING"
|
||||
APPROVED = "APPROVED"
|
||||
REJECTED = "REJECTED"
|
||||
BLOCKED = "BLOCKED"
|
||||
EXPIRED = "EXPIRED"
|
||||
STALE = "STALE"
|
||||
REVOKED = "REVOKED"
|
||||
|
||||
|
||||
class PromotionPlanStatus(StrEnum):
|
||||
DRAFT = "DRAFT"
|
||||
APPROVED = "APPROVED"
|
||||
EXECUTING = "EXECUTING"
|
||||
COMMITTED = "COMMITTED"
|
||||
ROLLED_BACK = "ROLLED_BACK"
|
||||
FAILED = "FAILED"
|
||||
|
||||
|
||||
class OperationStage(StrEnum):
|
||||
PLANNED = "PLANNED"
|
||||
APPROVED = "APPROVED"
|
||||
PREPARING = "PREPARING"
|
||||
ACTIVATING = "ACTIVATING"
|
||||
VERIFYING = "VERIFYING"
|
||||
COMMITTED = "COMMITTED"
|
||||
ROLLING_BACK = "ROLLING_BACK"
|
||||
ROLLED_BACK = "ROLLED_BACK"
|
||||
FAILED = "FAILED"
|
||||
|
||||
|
||||
class CanaryStatus(StrEnum):
|
||||
PLANNED = "PLANNED"
|
||||
RUNNING = "RUNNING"
|
||||
READY_FOR_PROMOTION_REVIEW = "READY_FOR_PROMOTION_REVIEW"
|
||||
ABORTED = "ABORTED"
|
||||
|
||||
|
||||
class RetentionState(StrEnum):
|
||||
ACTIVE = "ACTIVE"
|
||||
ROLLBACK_RETAINED = "ROLLBACK_RETAINED"
|
||||
DEPRECATED = "DEPRECATED"
|
||||
ARCHIVABLE = "ARCHIVABLE"
|
||||
ARCHIVED = "ARCHIVED"
|
||||
|
||||
|
||||
class CleanupStatus(StrEnum):
|
||||
READY = "READY"
|
||||
BLOCKED = "BLOCKED"
|
||||
STALE = "STALE"
|
||||
EXECUTED = "EXECUTED"
|
||||
|
||||
|
||||
ALLOWED_DEPLOYMENT_TRANSITIONS: dict[
|
||||
DeploymentLifecycleState, frozenset[DeploymentLifecycleState]
|
||||
] = {
|
||||
DeploymentLifecycleState.CANDIDATE: frozenset(
|
||||
{DeploymentLifecycleState.LAB_READY, DeploymentLifecycleState.DEPRECATED}
|
||||
),
|
||||
DeploymentLifecycleState.LAB_READY: frozenset(
|
||||
{
|
||||
DeploymentLifecycleState.PROMOTION_ELIGIBLE,
|
||||
DeploymentLifecycleState.CANARY,
|
||||
DeploymentLifecycleState.DEPRECATED,
|
||||
}
|
||||
),
|
||||
DeploymentLifecycleState.PROMOTION_ELIGIBLE: frozenset(
|
||||
{DeploymentLifecycleState.CANARY, DeploymentLifecycleState.DEPRECATED}
|
||||
),
|
||||
DeploymentLifecycleState.CANARY: frozenset(
|
||||
{
|
||||
DeploymentLifecycleState.LAB_READY,
|
||||
DeploymentLifecycleState.LAB_STABLE,
|
||||
DeploymentLifecycleState.STABLE,
|
||||
}
|
||||
),
|
||||
DeploymentLifecycleState.LAB_STABLE: frozenset(
|
||||
{
|
||||
DeploymentLifecycleState.CANARY,
|
||||
DeploymentLifecycleState.LAB_READY,
|
||||
DeploymentLifecycleState.DEPRECATED,
|
||||
}
|
||||
),
|
||||
DeploymentLifecycleState.STABLE: frozenset(
|
||||
{DeploymentLifecycleState.CANARY, DeploymentLifecycleState.DRAINING}
|
||||
),
|
||||
DeploymentLifecycleState.DRAINING: frozenset(
|
||||
{DeploymentLifecycleState.STABLE, DeploymentLifecycleState.DEPRECATED}
|
||||
),
|
||||
DeploymentLifecycleState.DEPRECATED: frozenset({DeploymentLifecycleState.ARCHIVED}),
|
||||
DeploymentLifecycleState.ARCHIVED: frozenset(),
|
||||
DeploymentLifecycleState.MANUAL_INTERVENTION_REQUIRED: frozenset(),
|
||||
}
|
||||
|
||||
|
||||
def assert_deployment_transition(
|
||||
current: DeploymentLifecycleState,
|
||||
target: DeploymentLifecycleState,
|
||||
) -> None:
|
||||
if target not in ALLOWED_DEPLOYMENT_TRANSITIONS[current]:
|
||||
raise ValueError(f"lifecycle transition {current} -> {target} is not allowed")
|
||||
|
||||
|
||||
class LifecycleEvidenceBundle(LifecycleModel):
|
||||
artifact_set_id: uuid.UUID | None = None
|
||||
model_revision_id: uuid.UUID | None = None
|
||||
runtime_profile_id: uuid.UUID | None = None
|
||||
runtime_probe_id: uuid.UUID | None = None
|
||||
production_execution_approval_id: uuid.UUID | None = None
|
||||
runtime_image_digest: str | None = Field(default=None, max_length=128)
|
||||
artifact_digests: list[str] = Field(default_factory=list, max_length=1000)
|
||||
capability_deployment_id: uuid.UUID | None = None
|
||||
project_binding_id: uuid.UUID | None = None
|
||||
project_fit_evidence_id: uuid.UUID | None = None
|
||||
evaluation_run_ids: list[uuid.UUID] = Field(default_factory=list, max_length=1000)
|
||||
resource_envelope_id: uuid.UUID | None = None
|
||||
embedding_space_id: uuid.UUID | None = None
|
||||
retrieval_pipeline_id: uuid.UUID | None = None
|
||||
migration_id: uuid.UUID | None = None
|
||||
rollback_target_ref: str | None = Field(default=None, max_length=255)
|
||||
integrity: Literal["UNKNOWN", "VERIFIED", "CORRUPT"] = "UNKNOWN"
|
||||
security: Literal["UNREVIEWED", "APPROVED", "BLOCKED"] = "UNREVIEWED"
|
||||
license: Literal["UNKNOWN", "APPROVED", "BLOCKED"] = "UNKNOWN"
|
||||
runtime: Literal["UNPROBED", "PROVEN", "INCOMPATIBLE"] = "UNPROBED"
|
||||
evaluation: Literal["NOT_EVALUATED", "EVALUATED", "REGRESSED", "PROMOTION_ELIGIBLE"] = (
|
||||
"NOT_EVALUATED"
|
||||
)
|
||||
project_fit: Literal[
|
||||
"UNKNOWN",
|
||||
"REQUIRES_MORE_EVIDENCE",
|
||||
"ELIGIBLE",
|
||||
"BLOCKED",
|
||||
"DEFERRED_EXTERNAL_VALIDATION",
|
||||
"KEEP_LAB",
|
||||
] = "UNKNOWN"
|
||||
engineering_integration: Literal["UNKNOWN", "PASS", "INCOMPLETE", "BLOCKED"] = "UNKNOWN"
|
||||
platform_readiness: Literal["UNKNOWN", "LAB_READY", "PROMOTION_ELIGIBLE", "STABLE"] = (
|
||||
"UNKNOWN"
|
||||
)
|
||||
production_validation: Literal[
|
||||
"NOT_REQUIRED", "REQUIRED", "DEFERRED_EXTERNAL_VALIDATION", "SATISFIED"
|
||||
] = "REQUIRED"
|
||||
scheduler_readiness: Literal["UNKNOWN", "READY", "BLOCKED"] = "UNKNOWN"
|
||||
critical_regressions: int = Field(default=0, ge=0)
|
||||
evidence_revisions: dict[str, str] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class ApprovalPolicyCreate(LifecycleModel):
|
||||
key: str = Field(pattern=r"^[a-z][a-z0-9-]+$", max_length=128)
|
||||
scope: Literal["LAB_PROMOTION", "CAPABILITY_PRODUCTION", "PROJECT_PRODUCTION"]
|
||||
requirements: dict[str, Any]
|
||||
created_by: str = Field(min_length=1, max_length=255)
|
||||
|
||||
|
||||
class ApprovalPolicyResponse(LifecycleModel):
|
||||
id: uuid.UUID
|
||||
key: str
|
||||
revision: int
|
||||
scope: str
|
||||
requirements: dict[str, Any]
|
||||
fingerprint: str
|
||||
active: bool
|
||||
created_by: str
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class LifecycleSubjectCreate(LifecycleModel):
|
||||
target_type: Literal[
|
||||
"CAPABILITY_DEPLOYMENT", "PROJECT_BINDING", "ARTIFACT_SET", "LAB_REHEARSAL"
|
||||
]
|
||||
target_ref: str = Field(min_length=1, max_length=255)
|
||||
environment: LifecycleEnvironment
|
||||
state: DeploymentLifecycleState
|
||||
|
||||
|
||||
class LifecycleSubjectResponse(LifecycleModel):
|
||||
id: uuid.UUID
|
||||
target_type: str
|
||||
target_ref: str
|
||||
environment: str
|
||||
state: str
|
||||
version: int
|
||||
superseded_by_ref: str | None
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
|
||||
|
||||
class ApprovalRequestCreate(LifecycleModel):
|
||||
policy_key: str = Field(pattern=r"^[a-z][a-z0-9-]+$", max_length=128)
|
||||
subject_id: uuid.UUID | None = None
|
||||
target_type: Literal[
|
||||
"CAPABILITY_DEPLOYMENT", "PROJECT_BINDING", "CAPABILITY", "ARTIFACT_SET", "LAB_REHEARSAL"
|
||||
]
|
||||
target_ref: str = Field(min_length=1, max_length=255)
|
||||
environment: LifecycleEnvironment
|
||||
requested_transition: DeploymentLifecycleState
|
||||
evidence: LifecycleEvidenceBundle
|
||||
requested_by: str = Field(min_length=1, max_length=255)
|
||||
reason: str = Field(min_length=8, max_length=4000)
|
||||
expires_at: datetime | None = None
|
||||
|
||||
|
||||
class ApprovalDecision(LifecycleModel):
|
||||
actor: str = Field(min_length=1, max_length=255)
|
||||
reason: str = Field(min_length=8, max_length=4000)
|
||||
|
||||
|
||||
class ApprovalRequestResponse(LifecycleModel):
|
||||
id: uuid.UUID
|
||||
policy_revision_id: uuid.UUID
|
||||
policy_key: str
|
||||
policy_revision: int
|
||||
version: int
|
||||
subject_id: uuid.UUID | None
|
||||
target_type: str
|
||||
target_ref: str
|
||||
environment: str
|
||||
requested_transition: str
|
||||
evidence_snapshot: dict[str, Any]
|
||||
evidence_fingerprint: str
|
||||
status: str
|
||||
blockers: list[str]
|
||||
warnings: list[str]
|
||||
requested_by: str
|
||||
approved_by: str | None
|
||||
reason: str
|
||||
expires_at: datetime | None
|
||||
stale_at: datetime | None
|
||||
decided_at: datetime | None
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class PromotionPlanCreate(LifecycleModel):
|
||||
approval_request_id: uuid.UUID
|
||||
subject_id: uuid.UUID
|
||||
desired_state: DeploymentLifecycleState
|
||||
migration_class: Literal["transparent", "behavioral", "requires_reindex", "schema_breaking"]
|
||||
candidate_deployment_id: uuid.UUID | None = None
|
||||
rollback_target_ref: str = Field(min_length=1, max_length=255)
|
||||
project_consumers: list[str] = Field(default_factory=list, max_length=1000)
|
||||
affected_identities: dict[str, Any] = Field(default_factory=dict)
|
||||
migration_id: uuid.UUID | None = None
|
||||
canary_strategy: dict[str, Any] = Field(default_factory=dict)
|
||||
drain_strategy: dict[str, Any] = Field(default_factory=dict)
|
||||
health_gates: dict[str, Any] = Field(default_factory=dict)
|
||||
automatic_abort_conditions: list[str] = Field(default_factory=list)
|
||||
created_by: str = Field(min_length=1, max_length=255)
|
||||
|
||||
|
||||
class PromotionPlanResponse(LifecycleModel):
|
||||
id: uuid.UUID
|
||||
approval_request_id: uuid.UUID
|
||||
subject_id: uuid.UUID
|
||||
current_state: str
|
||||
desired_state: str
|
||||
migration_class: str
|
||||
candidate_deployment_id: uuid.UUID | None
|
||||
rollback_target_ref: str
|
||||
project_consumers: list[str]
|
||||
affected_identities: dict[str, Any]
|
||||
impact_analysis: dict[str, Any]
|
||||
migration_id: uuid.UUID | None
|
||||
canary_strategy: dict[str, Any]
|
||||
drain_strategy: dict[str, Any]
|
||||
health_gates: dict[str, Any]
|
||||
automatic_abort_conditions: list[str]
|
||||
plan_fingerprint: str
|
||||
status: str
|
||||
version: int
|
||||
created_by: str
|
||||
approved_by: str | None
|
||||
immutable_at: datetime | None
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class PlanExecutionCreate(LifecycleModel):
|
||||
executor: str = Field(min_length=1, max_length=255)
|
||||
idempotency_key: str = Field(min_length=8, max_length=128)
|
||||
expected_subject_version: int = Field(ge=1)
|
||||
rehearsal_pause_stage: Literal["PREPARING"] | None = None
|
||||
|
||||
|
||||
class CanaryObservation(LifecycleModel):
|
||||
request_count: int = Field(ge=0)
|
||||
error_count: int = Field(ge=0)
|
||||
latency_p95_ms: float = Field(ge=0)
|
||||
capability_health: Literal["HEALTHY", "UNHEALTHY"]
|
||||
scheduler_ready: bool
|
||||
critical_project_failures: int = Field(default=0, ge=0)
|
||||
external_pressure: bool = False
|
||||
worker_healthy: bool = True
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_counts(self) -> CanaryObservation:
|
||||
if self.error_count > self.request_count:
|
||||
raise ValueError("error_count cannot exceed request_count")
|
||||
return self
|
||||
|
||||
|
||||
class LifecycleOperationResponse(LifecycleModel):
|
||||
id: uuid.UUID
|
||||
promotion_plan_id: uuid.UUID
|
||||
version: int
|
||||
stage: str
|
||||
idempotency_key: str
|
||||
expected_subject_version: int
|
||||
requester: str
|
||||
approver: str
|
||||
executor: str
|
||||
failure_code: str | None
|
||||
failure_details: dict[str, Any]
|
||||
rollback_duration_ms: float | None
|
||||
started_at: datetime
|
||||
finished_at: datetime | None
|
||||
canary: dict[str, Any] | None = None
|
||||
|
||||
|
||||
class RetentionPolicyCreate(LifecycleModel):
|
||||
key: str = Field(pattern=r"^[a-z][a-z0-9-]+$", max_length=128)
|
||||
minimum_rollback_days: int = Field(ge=1, le=3650)
|
||||
requirements: dict[str, Any] = Field(default_factory=dict)
|
||||
created_by: str = Field(min_length=1, max_length=255)
|
||||
|
||||
|
||||
class RetentionPolicyResponse(LifecycleModel):
|
||||
id: uuid.UUID
|
||||
key: str
|
||||
revision: int
|
||||
minimum_rollback_days: int
|
||||
requirements: dict[str, Any]
|
||||
fingerprint: str
|
||||
active: bool
|
||||
created_by: str
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class CleanupPlanCreate(LifecycleModel):
|
||||
target_type: Literal["ARTIFACT_SET", "ARTIFACT_LOCATION"]
|
||||
target_ref: uuid.UUID
|
||||
action: Literal["ARCHIVE_METADATA", "RECORD_LOCATION_REMOVAL"]
|
||||
created_by: str = Field(min_length=1, max_length=255)
|
||||
|
||||
|
||||
class CleanupPlanResponse(LifecycleModel):
|
||||
id: uuid.UUID
|
||||
target_type: str
|
||||
target_ref: str
|
||||
action: str
|
||||
dependencies: list[dict[str, Any]]
|
||||
dependency_digest: str
|
||||
reclaimable_bytes: int
|
||||
retention_state: str
|
||||
blockers: list[str]
|
||||
status: str
|
||||
created_by: str
|
||||
created_at: datetime
|
||||
executed_at: datetime | None
|
||||
|
||||
|
||||
class CleanupExecutionCreate(LifecycleModel):
|
||||
executor: str = Field(min_length=1, max_length=255)
|
||||
confirm: Literal[True]
|
||||
physical_removal_confirmed: bool = False
|
||||
|
||||
|
||||
class LifecycleEventResponse(LifecycleModel):
|
||||
id: uuid.UUID
|
||||
event_type: str
|
||||
object_type: str
|
||||
object_ref: str
|
||||
from_state: str | None
|
||||
to_state: str | None
|
||||
actor: str
|
||||
actor_role: str
|
||||
policy_revision_id: uuid.UUID | None
|
||||
evidence_ids: list[str]
|
||||
reason: str
|
||||
change_id: str
|
||||
details: dict[str, Any]
|
||||
occurred_at: datetime
|
||||
@@ -0,0 +1,543 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from enum import StrEnum
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
|
||||
|
||||
class MigrationModel(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", from_attributes=True)
|
||||
|
||||
|
||||
class MigrationClass(StrEnum):
|
||||
TRANSPARENT = "TRANSPARENT"
|
||||
BEHAVIORAL = "BEHAVIORAL"
|
||||
REQUIRES_REINDEX = "REQUIRES_REINDEX"
|
||||
SCHEMA_BREAKING = "SCHEMA_BREAKING"
|
||||
|
||||
|
||||
class MigrationState(StrEnum):
|
||||
PLANNED = "PLANNED"
|
||||
PREFLIGHT = "PREFLIGHT"
|
||||
READY = "READY"
|
||||
BACKFILLING = "BACKFILLING"
|
||||
BACKFILL_PAUSED = "BACKFILL_PAUSED"
|
||||
BACKFILL_COMPLETE = "BACKFILL_COMPLETE"
|
||||
VALIDATING = "VALIDATING"
|
||||
VALIDATION_FAILED = "VALIDATION_FAILED"
|
||||
READY_FOR_SHADOW = "READY_FOR_SHADOW"
|
||||
SHADOWING = "SHADOWING"
|
||||
READY_FOR_CUTOVER = "READY_FOR_CUTOVER"
|
||||
CUTOVER_PREPARING = "CUTOVER_PREPARING"
|
||||
CUTTING_OVER = "CUTTING_OVER"
|
||||
VERIFYING_CUTOVER = "VERIFYING_CUTOVER"
|
||||
CUTOVER_COMMITTED = "CUTOVER_COMMITTED"
|
||||
ROLLING_BACK = "ROLLING_BACK"
|
||||
ROLLED_BACK = "ROLLED_BACK"
|
||||
FAILED = "FAILED"
|
||||
CANCELLED = "CANCELLED"
|
||||
MANUAL_INTERVENTION_REQUIRED = "MANUAL_INTERVENTION_REQUIRED"
|
||||
|
||||
|
||||
class CutoverStage(StrEnum):
|
||||
PREPARING = "PREPARING"
|
||||
LOCKING = "LOCKING"
|
||||
SWITCHING = "SWITCHING"
|
||||
VERIFYING = "VERIFYING"
|
||||
COMMITTING = "COMMITTING"
|
||||
COMMITTED = "COMMITTED"
|
||||
ROLLING_BACK = "ROLLING_BACK"
|
||||
ROLLED_BACK = "ROLLED_BACK"
|
||||
FAILED = "FAILED"
|
||||
|
||||
|
||||
class BatchState(StrEnum):
|
||||
PLANNED = "PLANNED"
|
||||
RUNNING = "RUNNING"
|
||||
COMPLETED = "COMPLETED"
|
||||
RETRYABLE_FAILED = "RETRYABLE_FAILED"
|
||||
PERMANENTLY_FAILED = "PERMANENTLY_FAILED"
|
||||
|
||||
|
||||
class MigrationFailureCode(StrEnum):
|
||||
SOURCE_CHANGED = "SOURCE_CHANGED"
|
||||
STALE_SOURCE = "STALE_SOURCE"
|
||||
TARGET_CONFLICT = "TARGET_CONFLICT"
|
||||
CAPABILITY_UNAVAILABLE = "CAPABILITY_UNAVAILABLE"
|
||||
BATCH_FAILED = "BATCH_FAILED"
|
||||
VECTOR_INVALID = "VECTOR_INVALID"
|
||||
WRITE_FAILED = "WRITE_FAILED"
|
||||
CHECKPOINT_FAILED = "CHECKPOINT_FAILED"
|
||||
CONTENT_HASH_MISMATCH = "CONTENT_HASH_MISMATCH"
|
||||
TARGET_SCHEMA_MISMATCH = "TARGET_SCHEMA_MISMATCH"
|
||||
VALIDATION_FAILED = "VALIDATION_FAILED"
|
||||
HEALTH_CHECK_FAILED = "HEALTH_CHECK_FAILED"
|
||||
ROLLBACK_FAILED = "ROLLBACK_FAILED"
|
||||
STALE_APPROVAL = "STALE_APPROVAL"
|
||||
|
||||
|
||||
ALLOWED_TRANSITIONS: dict[MigrationState, frozenset[MigrationState]] = {
|
||||
MigrationState.PLANNED: frozenset(
|
||||
{MigrationState.PREFLIGHT, MigrationState.CANCELLED, MigrationState.FAILED}
|
||||
),
|
||||
MigrationState.PREFLIGHT: frozenset(
|
||||
{MigrationState.READY, MigrationState.FAILED, MigrationState.CANCELLED}
|
||||
),
|
||||
MigrationState.READY: frozenset(
|
||||
{MigrationState.BACKFILLING, MigrationState.CANCELLED, MigrationState.FAILED}
|
||||
),
|
||||
MigrationState.BACKFILLING: frozenset(
|
||||
{
|
||||
MigrationState.BACKFILL_PAUSED,
|
||||
MigrationState.BACKFILL_COMPLETE,
|
||||
MigrationState.CANCELLED,
|
||||
MigrationState.FAILED,
|
||||
}
|
||||
),
|
||||
MigrationState.BACKFILL_PAUSED: frozenset(
|
||||
{MigrationState.BACKFILLING, MigrationState.CANCELLED, MigrationState.FAILED}
|
||||
),
|
||||
MigrationState.BACKFILL_COMPLETE: frozenset(
|
||||
{MigrationState.VALIDATING, MigrationState.CANCELLED, MigrationState.FAILED}
|
||||
),
|
||||
MigrationState.VALIDATING: frozenset(
|
||||
{
|
||||
MigrationState.VALIDATION_FAILED,
|
||||
MigrationState.READY_FOR_SHADOW,
|
||||
MigrationState.FAILED,
|
||||
MigrationState.CANCELLED,
|
||||
}
|
||||
),
|
||||
MigrationState.VALIDATION_FAILED: frozenset(
|
||||
{MigrationState.VALIDATING, MigrationState.CANCELLED, MigrationState.FAILED}
|
||||
),
|
||||
MigrationState.READY_FOR_SHADOW: frozenset(
|
||||
{MigrationState.SHADOWING, MigrationState.CANCELLED, MigrationState.FAILED}
|
||||
),
|
||||
MigrationState.SHADOWING: frozenset(
|
||||
{MigrationState.READY_FOR_CUTOVER, MigrationState.CANCELLED, MigrationState.FAILED}
|
||||
),
|
||||
MigrationState.READY_FOR_CUTOVER: frozenset(
|
||||
{MigrationState.CUTOVER_PREPARING, MigrationState.CANCELLED, MigrationState.FAILED}
|
||||
),
|
||||
MigrationState.CUTOVER_PREPARING: frozenset(
|
||||
{MigrationState.CUTTING_OVER, MigrationState.FAILED, MigrationState.CANCELLED}
|
||||
),
|
||||
MigrationState.CUTTING_OVER: frozenset(
|
||||
{
|
||||
MigrationState.VERIFYING_CUTOVER,
|
||||
MigrationState.ROLLING_BACK,
|
||||
MigrationState.MANUAL_INTERVENTION_REQUIRED,
|
||||
}
|
||||
),
|
||||
MigrationState.VERIFYING_CUTOVER: frozenset(
|
||||
{
|
||||
MigrationState.CUTOVER_COMMITTED,
|
||||
MigrationState.ROLLING_BACK,
|
||||
MigrationState.MANUAL_INTERVENTION_REQUIRED,
|
||||
}
|
||||
),
|
||||
MigrationState.CUTOVER_COMMITTED: frozenset({MigrationState.ROLLING_BACK}),
|
||||
MigrationState.ROLLING_BACK: frozenset(
|
||||
{MigrationState.ROLLED_BACK, MigrationState.MANUAL_INTERVENTION_REQUIRED}
|
||||
),
|
||||
MigrationState.ROLLED_BACK: frozenset(),
|
||||
MigrationState.FAILED: frozenset(),
|
||||
MigrationState.CANCELLED: frozenset(),
|
||||
MigrationState.MANUAL_INTERVENTION_REQUIRED: frozenset(),
|
||||
}
|
||||
|
||||
|
||||
def assert_migration_transition(before: str, after: str) -> None:
|
||||
current = MigrationState(before)
|
||||
target = MigrationState(after)
|
||||
if target not in ALLOWED_TRANSITIONS[current]:
|
||||
raise ValueError(f"invalid migration transition {current.value} -> {target.value}")
|
||||
|
||||
|
||||
class AdapterContract(MigrationModel):
|
||||
key: str = Field(pattern=r"^[a-z0-9][a-z0-9._-]{1,127}$")
|
||||
version: str = Field(min_length=1, max_length=64)
|
||||
operations: frozenset[
|
||||
Literal[
|
||||
"preflight",
|
||||
"create_shadow_target",
|
||||
"enumerate_source_items",
|
||||
"transform_batch",
|
||||
"write_batch",
|
||||
"validate_batch",
|
||||
"finalize_backfill",
|
||||
"validate_target",
|
||||
"shadow_compare",
|
||||
"cutover",
|
||||
"rollback",
|
||||
"inspect_external_state",
|
||||
]
|
||||
]
|
||||
fingerprint: str = Field(pattern=r"^[0-9a-f]{64}$")
|
||||
schema_operations: frozenset[str] = Field(default_factory=frozenset, max_length=32)
|
||||
|
||||
@field_validator("schema_operations")
|
||||
@classmethod
|
||||
def validate_schema_operations(cls, value: frozenset[str]) -> frozenset[str]:
|
||||
if any(not re.fullmatch(r"[a-z0-9][a-z0-9._-]{1,127}", item) for item in value):
|
||||
raise ValueError("schema operations must be typed adapter keys")
|
||||
return value
|
||||
|
||||
|
||||
class SchemaMigrationStep(MigrationModel):
|
||||
operation: str = Field(min_length=1, max_length=64)
|
||||
adapter_step: str = Field(default="manual-boundary", min_length=1, max_length=128)
|
||||
preconditions: list[str] = Field(default_factory=list, max_length=32)
|
||||
required_application_versions: dict[str, str] = Field(default_factory=dict)
|
||||
compatibility_window: str = Field(default="unspecified", min_length=1, max_length=255)
|
||||
rollback_feasible: bool = False
|
||||
irreversible: bool = False
|
||||
|
||||
|
||||
class MigrationPlanCreate(MigrationModel):
|
||||
project_id: uuid.UUID
|
||||
project_binding_id: uuid.UUID
|
||||
capability_contract_id: uuid.UUID
|
||||
migration_class: MigrationClass
|
||||
environment: Literal["LAB", "PRODUCTION"]
|
||||
adapter: AdapterContract
|
||||
source_identity: dict[str, Any]
|
||||
target_identity: dict[str, Any]
|
||||
source_data_target: str = Field(min_length=1, max_length=512)
|
||||
target_shadow_target: str = Field(min_length=1, max_length=512)
|
||||
source_space_ref: str = Field(min_length=1, max_length=255)
|
||||
target_space_id: uuid.UUID
|
||||
corpus_revision: str = Field(min_length=1, max_length=128)
|
||||
migration_policy_revision: str = Field(min_length=1, max_length=128)
|
||||
validation_policy_revision_id: uuid.UUID
|
||||
lifecycle_approval_id: uuid.UUID
|
||||
promotion_plan_id: uuid.UUID | None = None
|
||||
rollback_target_ref: str = Field(min_length=1, max_length=512)
|
||||
total_expected_items: int = Field(ge=1, le=10_000_000)
|
||||
batch_size: int = Field(default=32, ge=1, le=256)
|
||||
max_in_flight_batches: int = Field(default=1, ge=1, le=8)
|
||||
concurrency: int = Field(default=1, ge=1, le=8)
|
||||
priority: Literal["BACKGROUND"] = "BACKGROUND"
|
||||
target_storage: dict[str, Any]
|
||||
shadow_policy: dict[str, Any]
|
||||
cutover_policy: dict[str, Any]
|
||||
rollback_retention_days: int = Field(default=30, ge=30, le=3650)
|
||||
environment_fingerprint: str = Field(pattern=r"^[0-9a-f]{64}$")
|
||||
idempotency_key: str = Field(min_length=8, max_length=128)
|
||||
created_by: str = Field(min_length=1, max_length=255)
|
||||
irreversible: bool = False
|
||||
schema_steps: list[SchemaMigrationStep] = Field(default_factory=list, max_length=32)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def identities_are_isolated(self) -> MigrationPlanCreate:
|
||||
if self.migration_class is MigrationClass.REQUIRES_REINDEX:
|
||||
if self.source_data_target == self.target_shadow_target:
|
||||
raise ValueError("reindex target must be separate from source")
|
||||
if self.source_space_ref == str(self.target_space_id):
|
||||
raise ValueError("reindex requires a distinct target embedding space")
|
||||
if self.migration_class is MigrationClass.SCHEMA_BREAKING and not self.schema_steps:
|
||||
raise ValueError("schema-breaking plans require typed adapter steps")
|
||||
return self
|
||||
|
||||
|
||||
class MigrationPlanResponse(MigrationPlanCreate):
|
||||
id: uuid.UUID
|
||||
state: MigrationState
|
||||
version: int
|
||||
generation: int
|
||||
plan_fingerprint: str
|
||||
approval_fingerprint: str
|
||||
completed_items: int
|
||||
failed_items: int
|
||||
retryable_items: int
|
||||
permanent_failed_items: int
|
||||
last_cursor: str | None
|
||||
cancel_requested: bool
|
||||
failure_code: str | None
|
||||
failure_details: dict[str, Any]
|
||||
started_at: datetime | None
|
||||
completed_at: datetime | None
|
||||
immutable_at: datetime
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
|
||||
|
||||
class MigrationValidationPolicyCreate(MigrationModel):
|
||||
key: str = Field(min_length=1, max_length=128)
|
||||
revision: int = Field(ge=1)
|
||||
required_completeness: float = Field(default=1.0, ge=0, le=1)
|
||||
allowed_failures: int = Field(default=0, ge=0)
|
||||
required_evaluation: bool = True
|
||||
critical_regressions_allowed: int = Field(default=0, ge=0)
|
||||
maximum_latency_regression_ratio: float | None = Field(default=None, ge=0)
|
||||
require_project_fit: bool = True
|
||||
require_external_validation: bool = True
|
||||
require_security_approved: bool = True
|
||||
allow_isolated_lab_cutover: bool = False
|
||||
created_by: str = Field(min_length=1, max_length=255)
|
||||
|
||||
|
||||
class MigrationValidationPolicyResponse(MigrationValidationPolicyCreate):
|
||||
id: uuid.UUID
|
||||
fingerprint: str
|
||||
active: bool
|
||||
created_at: datetime
|
||||
immutable_at: datetime
|
||||
|
||||
|
||||
class PreflightReport(MigrationModel):
|
||||
expected_version: int = Field(ge=1)
|
||||
adapter_fingerprint: str = Field(pattern=r"^[0-9a-f]{64}$")
|
||||
source_fingerprint: str = Field(pattern=r"^[0-9a-f]{64}$")
|
||||
source_exists: bool
|
||||
source_healthy: bool
|
||||
source_count: int = Field(ge=0)
|
||||
target_conflict_free: bool
|
||||
target_space_valid: bool
|
||||
capability_healthy: bool
|
||||
project_credential_valid: bool
|
||||
storage_sufficient: bool
|
||||
scheduler_capacity: bool
|
||||
adapter_available: bool
|
||||
rollback_source_retained: bool
|
||||
evaluation_suite_available: bool
|
||||
lifecycle_approval_current: bool
|
||||
evidence: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
@property
|
||||
def passed(self) -> bool:
|
||||
return all(
|
||||
(
|
||||
self.source_exists,
|
||||
self.source_healthy,
|
||||
self.target_conflict_free,
|
||||
self.target_space_valid,
|
||||
self.capability_healthy,
|
||||
self.project_credential_valid,
|
||||
self.storage_sufficient,
|
||||
self.scheduler_capacity,
|
||||
self.adapter_available,
|
||||
self.rollback_source_retained,
|
||||
self.evaluation_suite_available,
|
||||
self.lifecycle_approval_current,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class BatchReport(MigrationModel):
|
||||
expected_version: int = Field(ge=1)
|
||||
generation: int = Field(ge=1)
|
||||
batch_number: int = Field(ge=0)
|
||||
cursor_start: str = Field(min_length=1, max_length=255)
|
||||
cursor_end: str = Field(min_length=1, max_length=255)
|
||||
item_count: int = Field(ge=1, le=256)
|
||||
completed_items: int = Field(ge=0, le=256)
|
||||
failed_items: int = Field(ge=0, le=256)
|
||||
retryable_items: int = Field(ge=0, le=256)
|
||||
permanent_failed_items: int = Field(ge=0, le=256)
|
||||
item_fingerprint: str = Field(pattern=r"^[0-9a-f]{64}$")
|
||||
result_fingerprint: str = Field(pattern=r"^[0-9a-f]{64}$")
|
||||
output_shape_valid: bool
|
||||
finite: bool
|
||||
target_space_matches: bool
|
||||
destination_committed: bool
|
||||
content_hashes_match: bool
|
||||
duration_ms: float = Field(ge=0)
|
||||
retries: int = Field(default=0, ge=0, le=20)
|
||||
error_code: MigrationFailureCode | None = None
|
||||
bounded_errors: list[dict[str, Any]] = Field(default_factory=list, max_length=20)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def totals_are_consistent(self) -> BatchReport:
|
||||
if self.completed_items + self.failed_items != self.item_count:
|
||||
raise ValueError("batch outcome count must equal item count")
|
||||
if self.retryable_items + self.permanent_failed_items != self.failed_items:
|
||||
raise ValueError("failed item taxonomy is incomplete")
|
||||
return self
|
||||
|
||||
|
||||
class StateAction(MigrationModel):
|
||||
expected_version: int = Field(ge=1)
|
||||
actor: str = Field(min_length=1, max_length=255)
|
||||
reason: str = Field(min_length=1, max_length=2000)
|
||||
|
||||
|
||||
class ValidationReport(MigrationModel):
|
||||
expected_version: int = Field(ge=1)
|
||||
generation: int = Field(ge=1)
|
||||
expected_count: int = Field(ge=0)
|
||||
actual_count: int = Field(ge=0)
|
||||
missing_count: int = Field(ge=0)
|
||||
duplicate_count: int = Field(ge=0)
|
||||
malformed_count: int = Field(ge=0)
|
||||
non_finite_count: int = Field(ge=0)
|
||||
wrong_dimension_count: int = Field(ge=0)
|
||||
content_hash_mismatch_count: int = Field(ge=0)
|
||||
wrong_space_count: int = Field(ge=0)
|
||||
index_schema_matches: bool
|
||||
distance_metric_matches: bool
|
||||
payload_integrity: bool
|
||||
target_fingerprint: str = Field(pattern=r"^[0-9a-f]{64}$")
|
||||
evaluation_run_ids: list[uuid.UUID] = Field(default_factory=list, max_length=32)
|
||||
comparable: bool
|
||||
critical_regressions: int = Field(ge=0)
|
||||
latency_regression_ratio: float | None = Field(default=None, ge=0)
|
||||
project_fit_eligible: bool
|
||||
external_validation_satisfied: bool
|
||||
security_approved: bool
|
||||
evidence: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class ValidationSnapshotResponse(ValidationReport):
|
||||
id: uuid.UUID
|
||||
migration_plan_id: uuid.UUID
|
||||
validation_policy_revision_id: uuid.UUID
|
||||
snapshot_fingerprint: str
|
||||
passed: bool
|
||||
technical_cutover_eligible: bool
|
||||
project_promotion_eligible: bool
|
||||
blockers: list[str]
|
||||
created_at: datetime
|
||||
immutable_at: datetime
|
||||
|
||||
|
||||
class ShadowReport(MigrationModel):
|
||||
expected_version: int = Field(ge=1)
|
||||
generation: int = Field(ge=1)
|
||||
request_count: int = Field(ge=1)
|
||||
source_error_count: int = Field(ge=0)
|
||||
target_error_count: int = Field(ge=0)
|
||||
source_latency_p95_ms: float = Field(ge=0)
|
||||
target_latency_p95_ms: float = Field(ge=0)
|
||||
critical_regressions: int = Field(ge=0)
|
||||
metrics: dict[str, float]
|
||||
evidence_refs: list[str] = Field(default_factory=list, max_length=64)
|
||||
result_fingerprint: str = Field(pattern=r"^[0-9a-f]{64}$")
|
||||
|
||||
|
||||
class CutoverPrepare(MigrationModel):
|
||||
expected_version: int = Field(ge=1)
|
||||
idempotency_key: str = Field(min_length=8, max_length=128)
|
||||
actor: str = Field(min_length=1, max_length=255)
|
||||
expected_external_source: str = Field(min_length=1, max_length=512)
|
||||
observed_external_source: str = Field(min_length=1, max_length=512)
|
||||
external_state_fingerprint: str = Field(pattern=r"^[0-9a-f]{64}$")
|
||||
configuration_version: str = Field(min_length=1, max_length=128)
|
||||
|
||||
|
||||
class CutoverReport(MigrationModel):
|
||||
expected_version: int = Field(ge=1)
|
||||
operation_id: uuid.UUID
|
||||
generation: int = Field(ge=1)
|
||||
external_source_before: str = Field(min_length=1, max_length=512)
|
||||
external_target_after: str = Field(min_length=1, max_length=512)
|
||||
external_state_fingerprint: str = Field(pattern=r"^[0-9a-f]{64}$")
|
||||
switch_duration_ms: float = Field(ge=0)
|
||||
target_reachable: bool
|
||||
expected_identity: bool
|
||||
capability_healthy: bool
|
||||
project_read_path_healthy: bool
|
||||
error_rate: float = Field(ge=0, le=1)
|
||||
smoke_query_count: int = Field(ge=0, le=100)
|
||||
smoke_error_count: int = Field(ge=0, le=100)
|
||||
evidence: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class RollbackReport(MigrationModel):
|
||||
expected_version: int = Field(ge=1)
|
||||
operation_id: uuid.UUID
|
||||
generation: int = Field(ge=1)
|
||||
restored_external_target: str = Field(min_length=1, max_length=512)
|
||||
external_state_fingerprint: str = Field(pattern=r"^[0-9a-f]{64}$")
|
||||
elapsed_ms: float = Field(ge=0)
|
||||
source_reachable: bool
|
||||
exact_identity_restored: bool
|
||||
capability_healthy: bool
|
||||
project_read_path_healthy: bool
|
||||
evidence: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class ReconciliationReport(MigrationModel):
|
||||
operation_id: uuid.UUID
|
||||
generation: int = Field(ge=1)
|
||||
observed_external_target: str = Field(min_length=1, max_length=512)
|
||||
external_state_fingerprint: str = Field(pattern=r"^[0-9a-f]{64}$")
|
||||
target_healthy: bool
|
||||
source_healthy: bool
|
||||
switch_duration_ms: float | None = Field(default=None, ge=0)
|
||||
smoke_query_count: int = Field(default=0, ge=0, le=100)
|
||||
smoke_error_count: int = Field(default=0, ge=0, le=100)
|
||||
evidence: dict[str, Any] = Field(default_factory=dict)
|
||||
actor: Literal["migration-reconciler"] = "migration-reconciler"
|
||||
|
||||
@model_validator(mode="after")
|
||||
def smoke_totals_are_consistent(self) -> ReconciliationReport:
|
||||
if self.smoke_error_count > self.smoke_query_count:
|
||||
raise ValueError("smoke errors cannot exceed smoke queries")
|
||||
return self
|
||||
|
||||
|
||||
class MigrationBatchResponse(MigrationModel):
|
||||
id: uuid.UUID
|
||||
migration_plan_id: uuid.UUID
|
||||
batch_number: int
|
||||
generation: int
|
||||
cursor_start: str
|
||||
cursor_end: str
|
||||
item_count: int
|
||||
completed_items: int
|
||||
failed_items: int
|
||||
retryable_items: int
|
||||
permanent_failed_items: int
|
||||
item_fingerprint: str
|
||||
result_fingerprint: str | None
|
||||
status: BatchState
|
||||
attempts: int
|
||||
duration_ms: float | None
|
||||
error_code: str | None
|
||||
bounded_errors: list[dict[str, Any]]
|
||||
started_at: datetime | None
|
||||
completed_at: datetime | None
|
||||
|
||||
|
||||
class CutoverOperationResponse(MigrationModel):
|
||||
id: uuid.UUID
|
||||
migration_plan_id: uuid.UUID
|
||||
stage: CutoverStage
|
||||
idempotency_key: str
|
||||
generation: int
|
||||
expected_plan_version: int
|
||||
source_before: str
|
||||
target_after: str
|
||||
external_state_fingerprint: str
|
||||
health_evidence: dict[str, Any]
|
||||
failure_code: str | None
|
||||
failure_details: dict[str, Any]
|
||||
switch_duration_ms: float | None
|
||||
rollback_duration_ms: float | None
|
||||
started_at: datetime
|
||||
finished_at: datetime | None
|
||||
|
||||
|
||||
class MigrationEventResponse(MigrationModel):
|
||||
id: uuid.UUID
|
||||
migration_plan_id: uuid.UUID
|
||||
operation_id: uuid.UUID | None
|
||||
event_type: str
|
||||
before_state: str | None
|
||||
after_state: str | None
|
||||
actor: str
|
||||
policy_revision: str
|
||||
evidence_refs: list[str]
|
||||
reason: str
|
||||
source_identity: dict[str, Any]
|
||||
target_identity: dict[str, Any]
|
||||
generation: int
|
||||
change_id: str
|
||||
details: dict[str, Any]
|
||||
occurred_at: datetime
|
||||
@@ -0,0 +1,60 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
|
||||
class NodeDecommissionModel(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class DecommissionBlocker(NodeDecommissionModel):
|
||||
code: str
|
||||
message: str
|
||||
record_type: str
|
||||
count: int = Field(ge=1)
|
||||
resource_ids: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
class DecommissionRecordCount(NodeDecommissionModel):
|
||||
record_type: str
|
||||
count: int = Field(ge=0)
|
||||
action: str
|
||||
|
||||
|
||||
class NodeDecommissionPreview(NodeDecommissionModel):
|
||||
node_id: uuid.UUID
|
||||
persisted_identity: str
|
||||
hostname: str
|
||||
display_name: str
|
||||
current_state: dict[str, object]
|
||||
node_generation: int = Field(ge=1)
|
||||
dependency_digest: str = Field(pattern=r"^[0-9a-f]{64}$")
|
||||
safe: bool
|
||||
blockers: list[DecommissionBlocker]
|
||||
cleanup: list[DecommissionRecordCount]
|
||||
preserved: list[DecommissionRecordCount]
|
||||
dependent_records: list[DecommissionRecordCount]
|
||||
|
||||
|
||||
class NodeDecommissionExecute(NodeDecommissionModel):
|
||||
expected_generation: int = Field(ge=1)
|
||||
preview_digest: str = Field(pattern=r"^[0-9a-f]{64}$")
|
||||
idempotency_key: str = Field(min_length=8, max_length=128)
|
||||
operator: str = Field(min_length=1, max_length=255)
|
||||
reason: str = Field(min_length=10, max_length=2000)
|
||||
confirmation: str = Field(min_length=1, max_length=255)
|
||||
|
||||
|
||||
class NodeDecommissionResult(NodeDecommissionModel):
|
||||
operation_id: uuid.UUID
|
||||
node_id: uuid.UUID
|
||||
persisted_identity: str
|
||||
status: str
|
||||
decommissioned_at: datetime
|
||||
cleanup_summary: dict[str, int]
|
||||
previous_state: dict[str, object]
|
||||
credential_revocations: int = Field(ge=0)
|
||||
idempotent_replay: bool = False
|
||||
@@ -0,0 +1,399 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from collections import defaultdict
|
||||
from datetime import datetime
|
||||
from enum import StrEnum
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
|
||||
|
||||
class TelemetryType(StrEnum):
|
||||
COUNTER = "COUNTER"
|
||||
GAUGE = "GAUGE"
|
||||
HISTOGRAM = "HISTOGRAM"
|
||||
EVENT = "EVENT"
|
||||
STATE = "STATE"
|
||||
|
||||
|
||||
class SLOState(StrEnum):
|
||||
HEALTHY = "HEALTHY"
|
||||
AT_RISK = "AT_RISK"
|
||||
BREACHED = "BREACHED"
|
||||
INSUFFICIENT_DATA = "INSUFFICIENT_DATA"
|
||||
STALE = "STALE"
|
||||
DISABLED = "DISABLED"
|
||||
|
||||
|
||||
class AlertState(StrEnum):
|
||||
PENDING = "PENDING"
|
||||
FIRING = "FIRING"
|
||||
ACKNOWLEDGED = "ACKNOWLEDGED"
|
||||
RESOLVED = "RESOLVED"
|
||||
SUPPRESSED = "SUPPRESSED"
|
||||
|
||||
|
||||
class AlertSeverity(StrEnum):
|
||||
INFO = "INFO"
|
||||
WARNING = "WARNING"
|
||||
CRITICAL = "CRITICAL"
|
||||
|
||||
|
||||
class OperationalModel(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", from_attributes=True)
|
||||
|
||||
|
||||
class MetricDefinition(OperationalModel):
|
||||
name: str = Field(pattern=r"^modelforge_[a-z][a-z0-9_]*$")
|
||||
type: TelemetryType
|
||||
help: str = Field(min_length=3, max_length=500)
|
||||
labels: tuple[str, ...] = ()
|
||||
|
||||
@field_validator("labels")
|
||||
@classmethod
|
||||
def bounded_labels(cls, value: tuple[str, ...]) -> tuple[str, ...]:
|
||||
forbidden = {
|
||||
"request_id",
|
||||
"artifact_sha",
|
||||
"query",
|
||||
"project_uuid",
|
||||
"filename",
|
||||
"error_text",
|
||||
}
|
||||
if len(value) > 6 or len(set(value)) != len(value) or forbidden.intersection(value):
|
||||
raise ValueError("metric labels must be unique, bounded and non-sensitive")
|
||||
if any(not re.fullmatch(r"[a-z][a-z0-9_]*", item) for item in value):
|
||||
raise ValueError("metric label names must use snake_case")
|
||||
return value
|
||||
|
||||
|
||||
class MetricRegistry:
|
||||
"""Process-local Prometheus registry; DB history is deliberately a separate concern."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._definitions: dict[str, MetricDefinition] = {}
|
||||
self._values: dict[tuple[str, tuple[tuple[str, str], ...]], float] = defaultdict(float)
|
||||
self._histograms: dict[tuple[str, tuple[tuple[str, str], ...]], list[float]] = defaultdict(
|
||||
list
|
||||
)
|
||||
self._lock = threading.Lock()
|
||||
self.started_at = time.time()
|
||||
|
||||
def define(self, definition: MetricDefinition) -> None:
|
||||
current = self._definitions.get(definition.name)
|
||||
if current and current != definition:
|
||||
raise ValueError(f"metric {definition.name} is already defined differently")
|
||||
self._definitions[definition.name] = definition
|
||||
|
||||
def _key(self, name: str, labels: dict[str, str]) -> tuple[str, tuple[tuple[str, str], ...]]:
|
||||
definition = self._definitions[name]
|
||||
if set(labels) != set(definition.labels):
|
||||
raise ValueError(f"metric {name} requires labels {definition.labels}")
|
||||
if any(len(value) > 128 or "\n" in value for value in labels.values()):
|
||||
raise ValueError("metric label values must be bounded single-line values")
|
||||
return name, tuple(sorted(labels.items()))
|
||||
|
||||
def increment(self, name: str, labels: dict[str, str], value: float = 1.0) -> None:
|
||||
if self._definitions[name].type is not TelemetryType.COUNTER or value < 0:
|
||||
raise ValueError("only counters accept non-negative increments")
|
||||
with self._lock:
|
||||
self._values[self._key(name, labels)] += value
|
||||
|
||||
def gauge(self, name: str, labels: dict[str, str], value: float) -> None:
|
||||
if self._definitions[name].type is not TelemetryType.GAUGE:
|
||||
raise ValueError("only gauges accept current values")
|
||||
with self._lock:
|
||||
self._values[self._key(name, labels)] = value
|
||||
|
||||
def observe(self, name: str, labels: dict[str, str], value: float) -> None:
|
||||
if self._definitions[name].type is not TelemetryType.HISTOGRAM or value < 0:
|
||||
raise ValueError("only histograms accept non-negative observations")
|
||||
with self._lock:
|
||||
samples = self._histograms[self._key(name, labels)]
|
||||
samples.append(value)
|
||||
if len(samples) > 10_000:
|
||||
del samples[: len(samples) - 10_000]
|
||||
|
||||
def histogram(self, name: str, labels: dict[str, str]) -> list[float]:
|
||||
with self._lock:
|
||||
return list(self._histograms.get(self._key(name, labels), []))
|
||||
|
||||
def samples(self, name: str) -> list[tuple[dict[str, str], float]]:
|
||||
with self._lock:
|
||||
return [
|
||||
(dict(labels), value)
|
||||
for (metric, labels), value in self._values.items()
|
||||
if metric == name
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _label_text(labels: tuple[tuple[str, str], ...]) -> str:
|
||||
if not labels:
|
||||
return ""
|
||||
escaped = [f'{key}="{value.replace(chr(92), chr(92) * 2).replace(chr(34), chr(92) + chr(34))}"' for key, value in labels]
|
||||
return "{" + ",".join(escaped) + "}"
|
||||
|
||||
def render(self) -> str:
|
||||
buckets = (0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0)
|
||||
lines: list[str] = []
|
||||
with self._lock:
|
||||
for name, definition in sorted(self._definitions.items()):
|
||||
lines.extend((f"# HELP {name} {definition.help}", f"# TYPE {name} {definition.type.value.lower()}"))
|
||||
for (metric, labels), value in sorted(self._values.items()):
|
||||
if metric == name:
|
||||
lines.append(f"{name}{self._label_text(labels)} {value}")
|
||||
for (metric, labels), values in sorted(self._histograms.items()):
|
||||
if metric != name:
|
||||
continue
|
||||
label_text = self._label_text(labels)
|
||||
for upper_bound in buckets:
|
||||
bucket_labels = tuple(sorted((*labels, ("le", str(upper_bound)))))
|
||||
count = sum(value <= upper_bound for value in values)
|
||||
lines.append(f"{name}_bucket{self._label_text(bucket_labels)} {count}")
|
||||
infinite_labels = tuple(sorted((*labels, ("le", "+Inf"))))
|
||||
lines.append(
|
||||
f"{name}_bucket{self._label_text(infinite_labels)} {len(values)}"
|
||||
)
|
||||
lines.append(f"{name}_count{label_text} {len(values)}")
|
||||
lines.append(f"{name}_sum{label_text} {sum(values)}")
|
||||
lines.append(f"modelforge_process_uptime_seconds {max(0.0, time.time() - self.started_at)}")
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
metrics = MetricRegistry()
|
||||
for _definition in (
|
||||
MetricDefinition(name="modelforge_api_requests_total", type=TelemetryType.COUNTER, help="Bounded API request outcomes", labels=("method", "route_class", "status_class")),
|
||||
MetricDefinition(name="modelforge_api_request_duration_seconds", type=TelemetryType.HISTOGRAM, help="API request duration", labels=("method", "route_class")),
|
||||
MetricDefinition(name="modelforge_observability_degraded", type=TelemetryType.GAUGE, help="Historical observability persistence is degraded", labels=()),
|
||||
):
|
||||
metrics.define(_definition)
|
||||
|
||||
|
||||
class SLIDefinitionCreate(OperationalModel):
|
||||
key: str = Field(pattern=r"^[a-z][a-z0-9_.-]{2,127}$")
|
||||
name: str = Field(min_length=3, max_length=255)
|
||||
service: str = Field(min_length=2, max_length=128)
|
||||
capability: str | None = Field(default=None, max_length=128)
|
||||
measurement: Literal["SUCCESS_RATIO", "LATENCY_P95", "FRESHNESS", "CORRECTNESS_RATIO"]
|
||||
valid_population: dict[str, Any]
|
||||
success_condition: dict[str, Any]
|
||||
default_window_seconds: int = Field(ge=60, le=2_592_000)
|
||||
|
||||
|
||||
class SLOPolicyCreate(OperationalModel):
|
||||
key: str = Field(pattern=r"^[a-z][a-z0-9_.-]{2,127}$")
|
||||
sli_definition_id: uuid.UUID
|
||||
objective: float = Field(gt=0, le=1)
|
||||
threshold_ms: float | None = Field(default=None, gt=0)
|
||||
rolling_window_seconds: int = Field(ge=60, le=2_592_000)
|
||||
minimum_sample_count: int = Field(ge=1, le=1_000_000)
|
||||
severity: AlertSeverity
|
||||
environment: Literal["PRODUCTION", "LAB", "BACKGROUND"]
|
||||
effective_from: datetime
|
||||
rationale: str = Field(min_length=10, max_length=4000)
|
||||
created_by: str = Field(default="operator", min_length=1, max_length=255)
|
||||
|
||||
|
||||
class AlertRuleCreate(OperationalModel):
|
||||
key: str = Field(pattern=r"^[a-z][a-z0-9_.-]{2,127}$")
|
||||
alert_type: str = Field(pattern=r"^[A-Z][A-Z0-9_]{2,63}$")
|
||||
signal: str = Field(min_length=3, max_length=128)
|
||||
slo_policy_id: uuid.UUID | None = None
|
||||
condition: dict[str, Any]
|
||||
pending_seconds: int = Field(ge=0, le=86_400)
|
||||
severity: AlertSeverity
|
||||
labels: dict[str, str] = Field(default_factory=dict)
|
||||
cooldown_seconds: int = Field(ge=0, le=604_800)
|
||||
recovery_condition: dict[str, Any]
|
||||
created_by: str = Field(default="operator", min_length=1, max_length=255)
|
||||
|
||||
@field_validator("labels")
|
||||
@classmethod
|
||||
def validate_labels(cls, value: dict[str, str]) -> dict[str, str]:
|
||||
if len(value) > 8 or any(len(key) > 64 or len(item) > 128 for key, item in value.items()):
|
||||
raise ValueError("alert labels must be bounded")
|
||||
return value
|
||||
|
||||
|
||||
class MaintenanceWindowCreate(OperationalModel):
|
||||
name: str = Field(min_length=3, max_length=255)
|
||||
starts_at: datetime
|
||||
ends_at: datetime
|
||||
matcher: dict[str, str]
|
||||
reason: str = Field(min_length=5, max_length=2000)
|
||||
created_by: str = Field(default="operator", min_length=1, max_length=255)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def ordered(self) -> MaintenanceWindowCreate:
|
||||
if self.ends_at <= self.starts_at:
|
||||
raise ValueError("maintenance window end must follow start")
|
||||
return self
|
||||
|
||||
|
||||
class AlertAction(OperationalModel):
|
||||
actor: str = Field(min_length=1, max_length=255)
|
||||
reason: str = Field(min_length=3, max_length=2000)
|
||||
|
||||
|
||||
class SLIDefinitionResponse(SLIDefinitionCreate):
|
||||
id: uuid.UUID
|
||||
enabled: bool
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class SLOPolicyResponse(SLOPolicyCreate):
|
||||
id: uuid.UUID
|
||||
revision: int
|
||||
active: bool
|
||||
fingerprint: str
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class AlertRuleResponse(AlertRuleCreate):
|
||||
id: uuid.UUID
|
||||
revision: int
|
||||
active: bool
|
||||
fingerprint: str
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class MaintenanceWindowResponse(MaintenanceWindowCreate):
|
||||
id: uuid.UUID
|
||||
active: bool
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class AlertHistoryResponse(OperationalModel):
|
||||
id: uuid.UUID
|
||||
alert_id: uuid.UUID
|
||||
from_state: str | None
|
||||
to_state: str
|
||||
actor: str
|
||||
reason: str
|
||||
evidence: dict[str, Any]
|
||||
occurred_at: datetime
|
||||
|
||||
|
||||
class IncidentResponse(OperationalModel):
|
||||
id: uuid.UUID
|
||||
fingerprint: str
|
||||
title: str
|
||||
state: str
|
||||
severity: str
|
||||
root_subject_type: str
|
||||
root_subject_ref: str
|
||||
correlation: Literal["RELATED", "LIKELY_ROOT", "DOWNSTREAM", "UNKNOWN"]
|
||||
first_seen_at: datetime
|
||||
last_seen_at: datetime
|
||||
resolved_at: datetime | None
|
||||
|
||||
|
||||
class SLOEvaluationResponse(OperationalModel):
|
||||
id: uuid.UUID
|
||||
policy_id: uuid.UUID
|
||||
policy_key: str
|
||||
policy_revision: int
|
||||
sli_key: str
|
||||
environment: str
|
||||
objective: float
|
||||
observed_value: float | None
|
||||
threshold_ms: float | None
|
||||
sample_count: int
|
||||
good_count: int
|
||||
bad_count: int
|
||||
state: SLOState
|
||||
window_start: datetime
|
||||
window_end: datetime
|
||||
observed_at: datetime
|
||||
freshness_seconds: float | None
|
||||
allowed_bad: float | None
|
||||
consumed_bad: int | None
|
||||
remaining_bad: float | None
|
||||
short_burn_rate: float | None
|
||||
long_burn_rate: float | None
|
||||
evidence: dict[str, Any]
|
||||
|
||||
|
||||
class AlertResponse(OperationalModel):
|
||||
id: uuid.UUID
|
||||
rule_id: uuid.UUID
|
||||
fingerprint: str
|
||||
alert_type: str
|
||||
severity: str
|
||||
state: AlertState
|
||||
source: str
|
||||
subject_type: str
|
||||
subject_ref: str
|
||||
summary: str
|
||||
details: dict[str, Any]
|
||||
first_seen_at: datetime
|
||||
last_seen_at: datetime
|
||||
firing_at: datetime | None
|
||||
acknowledged_at: datetime | None
|
||||
acknowledged_by: str | None
|
||||
resolved_at: datetime | None
|
||||
suppressed_until: datetime | None
|
||||
occurrence_count: int
|
||||
|
||||
|
||||
class CapacitySnapshotResponse(OperationalModel):
|
||||
id: uuid.UUID
|
||||
observed_at: datetime
|
||||
received_at: datetime
|
||||
node_id: uuid.UUID
|
||||
node_name: str
|
||||
accelerator_id: uuid.UUID | None
|
||||
gpu_total_bytes: int | None
|
||||
gpu_observed_bytes: int | None
|
||||
gpu_external_bytes: int | None
|
||||
gpu_managed_resident_bytes: int | None
|
||||
gpu_leased_bytes: int | None
|
||||
gpu_reserve_bytes: int | None
|
||||
gpu_schedulable_bytes: int | None
|
||||
pressure_state: str
|
||||
system_ram_total_bytes: int | None
|
||||
system_ram_available_bytes: int | None
|
||||
storage_total_bytes: int | None
|
||||
storage_free_bytes: int | None
|
||||
availability: str
|
||||
freshness_seconds: float
|
||||
|
||||
|
||||
class TrendResponse(OperationalModel):
|
||||
subject: str
|
||||
sample_count: int
|
||||
period_start: datetime | None
|
||||
period_end: datetime | None
|
||||
status: Literal["AVAILABLE", "INSUFFICIENT_DATA", "STALE"]
|
||||
metrics: dict[str, float | int | None]
|
||||
forecast: dict[str, Any]
|
||||
|
||||
|
||||
class OperationsOverview(OperationalModel):
|
||||
status: Literal["HEALTHY", "DEGRADED", "OBSERVABILITY_DEGRADED"]
|
||||
observed_at: datetime
|
||||
active_alerts: list[AlertResponse]
|
||||
slo_evaluations: list[SLOEvaluationResponse]
|
||||
capacity: list[CapacitySnapshotResponse]
|
||||
capability_health: list[dict[str, Any]]
|
||||
project_health: list[dict[str, Any]]
|
||||
recent_failures: list[dict[str, Any]]
|
||||
history_available: bool
|
||||
|
||||
|
||||
def route_class(path: str) -> str:
|
||||
if path.startswith("/api/v1/capabilities/") or path == "/v1/embeddings":
|
||||
return "gateway"
|
||||
if path.startswith("/api/v1/agent/"):
|
||||
return "agent"
|
||||
if path.startswith("/api/v1/admin/"):
|
||||
return "admin"
|
||||
if path.startswith("/api/v1/health/"):
|
||||
return "health"
|
||||
if path == "/metrics":
|
||||
return "metrics"
|
||||
return "control_plane"
|
||||
@@ -0,0 +1,471 @@
|
||||
"""M15 backup, restore and disaster-recovery contracts.
|
||||
|
||||
Recovery is a first-class control-plane concern: what ModelForge owns authoritatively,
|
||||
what it can rebuild from an exact upstream identity, what is deliberately discarded and
|
||||
what is never exportable are separate classifications with separate guarantees.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from enum import StrEnum
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
|
||||
BACKUP_ID_PATTERN = r"^[a-z0-9][a-z0-9-]{6,62}$"
|
||||
SHA256_PATTERN = r"^[0-9a-f]{64}$"
|
||||
|
||||
# ModelForge stores verified logical snapshots. WAL archiving and continuous point-in-time
|
||||
# recovery are deliberately out of scope for v1; the value is exported so operators and the
|
||||
# recovery dashboard never have to infer it.
|
||||
POINT_IN_TIME_SUPPORT: Literal["SUPPORTED", "NOT_SUPPORTED"] = "NOT_SUPPORTED"
|
||||
|
||||
|
||||
class RecoveryAssetClass(StrEnum):
|
||||
"""Top-level classification that decides whether state must be copied at all."""
|
||||
|
||||
AUTHORITATIVE = "AUTHORITATIVE"
|
||||
REBUILDABLE = "REBUILDABLE"
|
||||
EPHEMERAL = "EPHEMERAL"
|
||||
EXTERNAL = "EXTERNAL"
|
||||
SECRET = "SECRET" # noqa: S105 - asset classification, not a credential
|
||||
|
||||
|
||||
class SecretRecoveryClass(StrEnum):
|
||||
RESTORABLE_SECRET = "RESTORABLE_SECRET" # noqa: S105 - asset classification, not a credential
|
||||
ROTATABLE_SECRET = "ROTATABLE_SECRET" # noqa: S105 - asset classification, not a credential
|
||||
NON_EXPORTABLE_SECRET = "NON_EXPORTABLE_SECRET" # noqa: S105 - asset classification, not a credential
|
||||
|
||||
|
||||
class ArtifactRecoveryClass(StrEnum):
|
||||
REHYDRATABLE = "REHYDRATABLE"
|
||||
NON_REHYDRATABLE = "NON_REHYDRATABLE"
|
||||
DERIVED = "DERIVED"
|
||||
LOCAL_ONLY = "LOCAL_ONLY"
|
||||
|
||||
|
||||
class ConfigurationClass(StrEnum):
|
||||
SOURCE_CONTROLLED = "SOURCE_CONTROLLED"
|
||||
SECRET = "SECRET" # noqa: S105 - asset classification, not a credential
|
||||
GENERATED = "GENERATED"
|
||||
HOST_LOCAL = "HOST_LOCAL"
|
||||
|
||||
|
||||
class BackupState(StrEnum):
|
||||
PLANNED = "PLANNED"
|
||||
CREATING = "CREATING"
|
||||
CREATED = "CREATED"
|
||||
VERIFYING = "VERIFYING"
|
||||
VERIFIED = "VERIFIED"
|
||||
FAILED = "FAILED"
|
||||
EXPIRED = "EXPIRED"
|
||||
DELETED = "DELETED"
|
||||
|
||||
|
||||
RESTORE_ELIGIBLE_STATES = frozenset({BackupState.VERIFIED})
|
||||
BACKUP_TERMINAL_STATES = frozenset({BackupState.FAILED, BackupState.EXPIRED, BackupState.DELETED})
|
||||
|
||||
|
||||
class BackupMethod(StrEnum):
|
||||
POSTGRES_LOGICAL_CUSTOM = "POSTGRES_LOGICAL_CUSTOM"
|
||||
MANIFEST_ONLY = "MANIFEST_ONLY"
|
||||
FILE_COPY = "FILE_COPY"
|
||||
SOURCE_CONTROL_REFERENCE = "SOURCE_CONTROL_REFERENCE"
|
||||
NOT_BACKED_UP = "NOT_BACKED_UP"
|
||||
|
||||
|
||||
class RestoreMode(StrEnum):
|
||||
"""VALIDATION never touches an active target; the destructive modes are operator-gated."""
|
||||
|
||||
VALIDATION = "VALIDATION"
|
||||
REPLACEMENT = "REPLACEMENT"
|
||||
DISASTER_RECOVERY = "DISASTER_RECOVERY"
|
||||
|
||||
|
||||
class RestorePlanState(StrEnum):
|
||||
DRAFT = "DRAFT"
|
||||
PREFLIGHT_PASSED = "PREFLIGHT_PASSED"
|
||||
PREFLIGHT_FAILED = "PREFLIGHT_FAILED"
|
||||
CONSUMED = "CONSUMED"
|
||||
|
||||
|
||||
class RestoreState(StrEnum):
|
||||
PLANNED = "PLANNED"
|
||||
PREFLIGHT = "PREFLIGHT"
|
||||
RESTORING_DATABASE = "RESTORING_DATABASE"
|
||||
RESTORING_CONFIGURATION = "RESTORING_CONFIGURATION"
|
||||
REHYDRATING_ARTIFACTS = "REHYDRATING_ARTIFACTS"
|
||||
RECONCILING = "RECONCILING"
|
||||
VALIDATING = "VALIDATING"
|
||||
READY = "READY"
|
||||
FAILED = "FAILED"
|
||||
MANUAL_INTERVENTION_REQUIRED = "MANUAL_INTERVENTION_REQUIRED"
|
||||
|
||||
|
||||
RESTORE_TERMINAL_STATES = frozenset(
|
||||
{RestoreState.READY, RestoreState.FAILED, RestoreState.MANUAL_INTERVENTION_REQUIRED}
|
||||
)
|
||||
|
||||
RESTORE_PHASE_ORDER: tuple[RestoreState, ...] = (
|
||||
RestoreState.PREFLIGHT,
|
||||
RestoreState.RESTORING_DATABASE,
|
||||
RestoreState.RESTORING_CONFIGURATION,
|
||||
RestoreState.REHYDRATING_ARTIFACTS,
|
||||
RestoreState.RECONCILING,
|
||||
RestoreState.VALIDATING,
|
||||
RestoreState.READY,
|
||||
)
|
||||
|
||||
|
||||
class ArtifactRecoveryState(StrEnum):
|
||||
PLANNED = "PLANNED"
|
||||
REHYDRATING = "REHYDRATING"
|
||||
VERIFYING = "VERIFYING"
|
||||
RECOVERED = "RECOVERED"
|
||||
BLOCKED = "BLOCKED"
|
||||
FAILED = "FAILED"
|
||||
|
||||
|
||||
class RecoveryReadiness(StrEnum):
|
||||
PROTECTED = "PROTECTED"
|
||||
REHYDRATABLE = "REHYDRATABLE"
|
||||
ROTATION_REQUIRED = "ROTATION_REQUIRED"
|
||||
EXTERNAL_DEPENDENCY = "EXTERNAL_DEPENDENCY"
|
||||
UNPROTECTED = "UNPROTECTED"
|
||||
|
||||
|
||||
class RecoveryFailureCode(StrEnum):
|
||||
HASH_MISMATCH = "HASH_MISMATCH"
|
||||
MANIFEST_HASH_MISMATCH = "MANIFEST_HASH_MISMATCH"
|
||||
MANIFEST_INCOMPLETE = "MANIFEST_INCOMPLETE"
|
||||
PAYLOAD_MISSING = "PAYLOAD_MISSING"
|
||||
BACKUP_NOT_RESTORE_ELIGIBLE = "BACKUP_NOT_RESTORE_ELIGIBLE"
|
||||
ENCRYPTION_KEY_UNAVAILABLE = "ENCRYPTION_KEY_UNAVAILABLE"
|
||||
DECRYPTION_FAILED = "DECRYPTION_FAILED"
|
||||
SCHEMA_TOO_NEW = "SCHEMA_TOO_NEW"
|
||||
SCHEMA_UNKNOWN = "SCHEMA_UNKNOWN"
|
||||
POSTGRES_VERSION_INCOMPATIBLE = "POSTGRES_VERSION_INCOMPATIBLE"
|
||||
DESTINATION_NOT_ISOLATED = "DESTINATION_NOT_ISOLATED"
|
||||
DESTINATION_NOT_EMPTY = "DESTINATION_NOT_EMPTY"
|
||||
INSUFFICIENT_CAPACITY = "INSUFFICIENT_CAPACITY"
|
||||
DESTINATION_UNAVAILABLE = "DESTINATION_UNAVAILABLE"
|
||||
BACKUP_TOOL_UNAVAILABLE = "BACKUP_TOOL_UNAVAILABLE"
|
||||
BACKUP_TOOL_FAILED = "BACKUP_TOOL_FAILED"
|
||||
CONCURRENT_OPERATION = "CONCURRENT_OPERATION"
|
||||
PATH_NOT_ALLOWED = "PATH_NOT_ALLOWED"
|
||||
ARCHIVE_UNSAFE = "ARCHIVE_UNSAFE"
|
||||
ARTIFACT_REHYDRATION_BLOCKED = "ARTIFACT_REHYDRATION_BLOCKED"
|
||||
ARTIFACT_HASH_MISMATCH = "ARTIFACT_HASH_MISMATCH"
|
||||
ARTIFACT_NOT_REHYDRATABLE = "ARTIFACT_NOT_REHYDRATABLE"
|
||||
AUDIT_CHAIN_CORRUPT = "AUDIT_CHAIN_CORRUPT"
|
||||
RECOVERY_RECONCILIATION_REQUIRED = "RECOVERY_RECONCILIATION_REQUIRED"
|
||||
SOURCE_CONTROL_REVISION_UNAVAILABLE = "SOURCE_CONTROL_REVISION_UNAVAILABLE"
|
||||
|
||||
|
||||
class RecoveryModel(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", from_attributes=True)
|
||||
|
||||
|
||||
class RecoveryPolicyCreate(RecoveryModel):
|
||||
"""A versioned recovery contract per asset class; never scattered inline rules."""
|
||||
|
||||
key: str = Field(pattern=r"^[a-z][a-z0-9_.-]{2,127}$")
|
||||
name: str = Field(min_length=3, max_length=255)
|
||||
asset_class: RecoveryAssetClass
|
||||
backup_method: BackupMethod
|
||||
retention_days: int = Field(ge=1, le=3650)
|
||||
minimum_verified_backups: int = Field(ge=1, le=100)
|
||||
rpo_seconds: int | None = Field(default=None, ge=0, le=2_592_000)
|
||||
rto_target_seconds: int | None = Field(default=None, ge=0, le=2_592_000)
|
||||
restore_verification: Literal["FULL_RESTORE", "HASH_ONLY", "MANIFEST_ONLY", "NOT_APPLICABLE"]
|
||||
encryption_required: bool
|
||||
external_dependency: bool = False
|
||||
rehydration_allowed: bool = False
|
||||
secret_class: SecretRecoveryClass | None = None
|
||||
rationale: str = Field(min_length=10, max_length=4000)
|
||||
created_by: str = Field(default="operator", min_length=1, max_length=255)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def coherent(self) -> RecoveryPolicyCreate:
|
||||
if self.asset_class is RecoveryAssetClass.AUTHORITATIVE:
|
||||
if self.backup_method in {BackupMethod.NOT_BACKED_UP, BackupMethod.MANIFEST_ONLY}:
|
||||
raise ValueError("authoritative state requires a payload-bearing backup method")
|
||||
if self.rpo_seconds is None:
|
||||
raise ValueError("authoritative state requires an explicit RPO target")
|
||||
if self.asset_class is RecoveryAssetClass.EPHEMERAL and self.backup_method not in {
|
||||
BackupMethod.NOT_BACKED_UP,
|
||||
BackupMethod.MANIFEST_ONLY,
|
||||
}:
|
||||
raise ValueError("ephemeral state must not claim a payload backup")
|
||||
if self.asset_class is RecoveryAssetClass.EXTERNAL and not self.external_dependency:
|
||||
raise ValueError("external state must be marked as an external dependency")
|
||||
if self.asset_class is RecoveryAssetClass.SECRET and self.secret_class is None:
|
||||
raise ValueError("secret state requires an explicit secret recovery class")
|
||||
if self.rehydration_allowed and self.asset_class is not RecoveryAssetClass.REBUILDABLE:
|
||||
raise ValueError("only rebuildable state may be rehydrated instead of copied")
|
||||
return self
|
||||
|
||||
|
||||
class RecoveryPolicyResponse(RecoveryPolicyCreate):
|
||||
id: uuid.UUID
|
||||
revision: int
|
||||
active: bool
|
||||
fingerprint: str
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class RecoveryAssetResponse(RecoveryModel):
|
||||
id: uuid.UUID
|
||||
key: str
|
||||
name: str
|
||||
asset_class: RecoveryAssetClass
|
||||
owner: str
|
||||
location: str
|
||||
backup_method: BackupMethod
|
||||
restore_method: str
|
||||
rebuild_method: str | None
|
||||
rpo_seconds: int | None
|
||||
readiness: RecoveryReadiness
|
||||
dependencies: list[str]
|
||||
notes: str
|
||||
policy_key: str
|
||||
updated_at: datetime
|
||||
|
||||
|
||||
class BackupSetCreate(RecoveryModel):
|
||||
backup_id: str = Field(pattern=BACKUP_ID_PATTERN)
|
||||
reason: str = Field(min_length=5, max_length=2000)
|
||||
milestone: str | None = Field(default=None, max_length=64)
|
||||
legal_hold: bool = False
|
||||
include_artifact_manifest: bool = True
|
||||
created_by: str = Field(default="operator", min_length=1, max_length=255)
|
||||
|
||||
|
||||
class BackupManifestEntryResponse(RecoveryModel):
|
||||
id: uuid.UUID
|
||||
logical_asset_type: str
|
||||
object_name: str
|
||||
relative_path: str
|
||||
size_bytes: int
|
||||
sha256: str
|
||||
source_generation: str
|
||||
schema_version: str | None
|
||||
dependency_refs: dict[str, Any]
|
||||
|
||||
|
||||
class BackupSetResponse(RecoveryModel):
|
||||
id: uuid.UUID
|
||||
backup_id: str
|
||||
state: BackupState
|
||||
policy_key: str
|
||||
policy_revision: int
|
||||
modelforge_version: str
|
||||
modelforge_commit: str | None
|
||||
schema_revision: str | None
|
||||
environment_fingerprint: dict[str, Any]
|
||||
database_identity: dict[str, Any]
|
||||
destination_root: str
|
||||
manifest_relative_path: str | None
|
||||
manifest_sha256: str | None
|
||||
included_asset_classes: list[str]
|
||||
excluded_asset_classes: list[str]
|
||||
payload_bytes: int
|
||||
encrypted: bool
|
||||
encryption_algorithm: str | None
|
||||
encryption_key_id: str | None
|
||||
verification_details: dict[str, Any]
|
||||
verified_at: datetime | None
|
||||
failure_code: str | None
|
||||
failure_reason: str | None
|
||||
milestone: str | None
|
||||
legal_hold: bool
|
||||
restore_eligible: bool
|
||||
reason: str
|
||||
created_by: str
|
||||
started_at: datetime | None
|
||||
completed_at: datetime | None
|
||||
expires_at: datetime | None
|
||||
created_at: datetime
|
||||
entries: list[BackupManifestEntryResponse] = Field(default_factory=list)
|
||||
|
||||
|
||||
class RestorePlanCreate(RecoveryModel):
|
||||
backup_set_id: uuid.UUID
|
||||
mode: RestoreMode
|
||||
target_environment: Literal["ISOLATED", "STAGING", "PRODUCTION"]
|
||||
target_label: str = Field(min_length=3, max_length=128)
|
||||
database_destination: str = Field(min_length=8, max_length=1024)
|
||||
artifact_strategy: Literal["NONE", "MANIFEST_ONLY", "REHYDRATE_MISSING", "RESTORE_LOCAL"]
|
||||
secret_strategy: Literal["ROTATE", "RESTORE_HASHES", "MANUAL"]
|
||||
node_strategy: Literal["REUSE_CREDENTIAL", "RE_ENROLL", "NONE"]
|
||||
expected_modelforge_version: str | None = Field(default=None, max_length=64)
|
||||
validation_requirements: dict[str, Any] = Field(default_factory=dict)
|
||||
reason: str = Field(min_length=10, max_length=4000)
|
||||
created_by: str = Field(default="operator", min_length=1, max_length=255)
|
||||
|
||||
@field_validator("database_destination")
|
||||
@classmethod
|
||||
def supported_destination(cls, value: str) -> str:
|
||||
if not value.startswith("postgresql+psycopg://"):
|
||||
raise ValueError("restore destinations must be PostgreSQL SQLAlchemy URLs")
|
||||
return value
|
||||
|
||||
@model_validator(mode="after")
|
||||
def guarded(self) -> RestorePlanCreate:
|
||||
if self.mode is RestoreMode.VALIDATION and self.target_environment == "PRODUCTION":
|
||||
raise ValueError("validation restores must never target a production environment")
|
||||
if self.target_environment == "PRODUCTION" and self.mode is not RestoreMode.DISASTER_RECOVERY:
|
||||
raise ValueError("production targets require an explicit disaster-recovery restore")
|
||||
return self
|
||||
|
||||
|
||||
class RestorePlanResponse(RecoveryModel):
|
||||
id: uuid.UUID
|
||||
backup_set_id: uuid.UUID
|
||||
backup_id: str
|
||||
mode: RestoreMode
|
||||
state: RestorePlanState
|
||||
target_environment: str
|
||||
target_label: str
|
||||
database_destination_redacted: str
|
||||
artifact_strategy: str
|
||||
secret_strategy: str
|
||||
node_strategy: str
|
||||
expected_modelforge_version: str | None
|
||||
preflight: dict[str, Any]
|
||||
validation_requirements: dict[str, Any]
|
||||
fingerprint: str
|
||||
reason: str
|
||||
created_by: str
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class RestoreOperationResponse(RecoveryModel):
|
||||
id: uuid.UUID
|
||||
plan_id: uuid.UUID
|
||||
backup_set_id: uuid.UUID
|
||||
backup_id: str
|
||||
state: RestoreState
|
||||
mode: RestoreMode
|
||||
attempt: int
|
||||
idempotency_key: str
|
||||
preflight_result: dict[str, Any]
|
||||
phase_durations: dict[str, float]
|
||||
source_fingerprint: dict[str, Any]
|
||||
restored_fingerprint: dict[str, Any]
|
||||
fingerprint_diff: dict[str, Any]
|
||||
validation_result: dict[str, Any]
|
||||
rpo_seconds: float | None
|
||||
rto_seconds: float | None
|
||||
failure_code: str | None
|
||||
failure_reason: str | None
|
||||
started_at: datetime
|
||||
updated_at: datetime
|
||||
ready_at: datetime | None
|
||||
|
||||
|
||||
class RestoreOperationEventResponse(RecoveryModel):
|
||||
id: uuid.UUID
|
||||
restore_operation_id: uuid.UUID
|
||||
from_state: str | None
|
||||
to_state: str
|
||||
phase: str
|
||||
actor: str
|
||||
reason: str
|
||||
evidence: dict[str, Any]
|
||||
occurred_at: datetime
|
||||
|
||||
|
||||
class RestoreAdvanceRequest(RecoveryModel):
|
||||
actor: str = Field(default="operator", min_length=1, max_length=255)
|
||||
reason: str = Field(min_length=5, max_length=2000)
|
||||
stop_after: RestoreState | None = None
|
||||
simulate_interruption_after: RestoreState | None = None
|
||||
|
||||
|
||||
class ArtifactRecoveryCreate(RecoveryModel):
|
||||
artifact_set_id: uuid.UUID
|
||||
target_storage_root_id: uuid.UUID
|
||||
reason: str = Field(min_length=10, max_length=2000)
|
||||
restore_operation_id: uuid.UUID | None = None
|
||||
created_by: str = Field(default="operator", min_length=1, max_length=255)
|
||||
|
||||
|
||||
class ArtifactRecoveryResponse(RecoveryModel):
|
||||
id: uuid.UUID
|
||||
restore_operation_id: uuid.UUID | None
|
||||
artifact_set_id: uuid.UUID
|
||||
model_revision_id: uuid.UUID
|
||||
recovery_class: ArtifactRecoveryClass
|
||||
state: ArtifactRecoveryState
|
||||
upstream_repository: str | None
|
||||
upstream_commit_sha: str | None
|
||||
target_storage_root_id: uuid.UUID
|
||||
expected_files: list[dict[str, Any]]
|
||||
verified_files: list[dict[str, Any]]
|
||||
bytes_total: int
|
||||
bytes_recovered: int
|
||||
download_plan_id: uuid.UUID | None
|
||||
artifact_job_id: uuid.UUID | None
|
||||
lineage: dict[str, Any]
|
||||
duration_seconds: float | None
|
||||
failure_code: str | None
|
||||
failure_reason: str | None
|
||||
started_at: datetime
|
||||
completed_at: datetime | None
|
||||
|
||||
|
||||
class RecoveryReadinessEntry(RecoveryModel):
|
||||
asset_key: str
|
||||
asset_name: str
|
||||
asset_class: RecoveryAssetClass
|
||||
readiness: RecoveryReadiness
|
||||
policy_key: str
|
||||
rpo_seconds: int | None
|
||||
detail: str
|
||||
|
||||
|
||||
class RecoveryDashboard(RecoveryModel):
|
||||
"""Every field is measured; unknown values stay null instead of becoming a fiction."""
|
||||
|
||||
observed_at: datetime
|
||||
point_in_time_support: Literal["SUPPORTED", "NOT_SUPPORTED"]
|
||||
latest_verified_backup_id: str | None
|
||||
latest_verified_backup_at: datetime | None
|
||||
latest_verified_backup_age_seconds: float | None
|
||||
latest_verified_schema_revision: str | None
|
||||
backup_states: dict[str, int]
|
||||
verified_backup_count: int
|
||||
stale_backup: bool
|
||||
backup_staleness_threshold_seconds: int
|
||||
last_restore_rehearsal_at: datetime | None
|
||||
last_restore_rehearsal_state: str | None
|
||||
observed_restore_seconds: float | None
|
||||
observed_rpo_seconds: float | None
|
||||
protected_asset_count: int
|
||||
unprotected_assets: list[str]
|
||||
readiness: list[RecoveryReadinessEntry]
|
||||
coverage_ratio: float
|
||||
estimated_protected_bytes: int
|
||||
estimated_rehydratable_bytes: int
|
||||
destination_capacity_bytes: int | None
|
||||
destination_free_bytes: int | None
|
||||
|
||||
|
||||
class BackupCapacityEstimate(RecoveryModel):
|
||||
bytes_to_copy: int
|
||||
bytes_manifest_only: int
|
||||
estimated_protected_bytes: int
|
||||
available_bytes: int | None
|
||||
capacity_bytes: int | None
|
||||
sufficient: bool
|
||||
detail: str
|
||||
|
||||
|
||||
def redact_database_url(url: str) -> str:
|
||||
"""Never journal or return a restore DSN that still carries its password."""
|
||||
|
||||
return re.sub(r"://([^:/@]+):[^@]*@", r"://\1:***@", url)
|
||||
@@ -0,0 +1,239 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator
|
||||
|
||||
from modelforge_api.domain.enums import ArtifactStatus, ModelLifecycle, StorageRootStatus
|
||||
|
||||
|
||||
class RegistryModel(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", from_attributes=True)
|
||||
|
||||
|
||||
class Page[Item](RegistryModel):
|
||||
items: list[Item]
|
||||
page: int
|
||||
page_size: int
|
||||
total: int
|
||||
pages: int
|
||||
|
||||
|
||||
class ModelCreate(RegistryModel):
|
||||
key: str = Field(min_length=1, max_length=128, pattern=r"^[a-z0-9][a-z0-9._-]*$")
|
||||
display_name: str = Field(min_length=1, max_length=255)
|
||||
description: str | None = None
|
||||
source_type: Literal["huggingface", "local", "custom"] = "huggingface"
|
||||
upstream_provider: str = Field(min_length=1, max_length=64)
|
||||
upstream_source: str = Field(min_length=1, max_length=255)
|
||||
upstream_metadata: dict[str, Any] = Field(default_factory=dict)
|
||||
local_metadata: dict[str, Any] = Field(default_factory=dict)
|
||||
interpretation_metadata: dict[str, Any] = Field(default_factory=dict)
|
||||
family: str | None = None
|
||||
modalities: list[str] = Field(default_factory=list)
|
||||
parameter_metadata: dict[str, Any] = Field(default_factory=dict)
|
||||
license_metadata: dict[str, Any] = Field(default_factory=lambda: {"status": "unknown"})
|
||||
lifecycle: ModelLifecycle = ModelLifecycle.CANDIDATE
|
||||
|
||||
|
||||
class ModelUpdate(RegistryModel):
|
||||
display_name: str | None = Field(default=None, min_length=1, max_length=255)
|
||||
description: str | None = None
|
||||
local_metadata: dict[str, Any] | None = None
|
||||
interpretation_metadata: dict[str, Any] | None = None
|
||||
lifecycle: ModelLifecycle | None = None
|
||||
|
||||
|
||||
class ModelResponse(ModelCreate):
|
||||
id: uuid.UUID
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
deprecated_at: datetime | None = None
|
||||
revision_count: int = 0
|
||||
artifact_count: int = 0
|
||||
verification_status: str = "unverified"
|
||||
deployment_status: str = "not_deployed"
|
||||
|
||||
|
||||
class RevisionCreate(RegistryModel):
|
||||
upstream_revision: str = Field(min_length=1, max_length=255)
|
||||
resolved_commit_sha: str = Field(pattern=r"^[0-9a-f]{40,64}$")
|
||||
metadata_snapshot: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class RevisionResponse(RevisionCreate):
|
||||
id: uuid.UUID
|
||||
model_id: uuid.UUID
|
||||
discovered_at: datetime
|
||||
approved_at: datetime | None = None
|
||||
immutable_at: datetime
|
||||
deprecated_at: datetime | None = None
|
||||
archived_at: datetime | None = None
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
|
||||
|
||||
class ArtifactLocationCreate(RegistryModel):
|
||||
storage_root_id: uuid.UUID
|
||||
relative_path: str = Field(min_length=1)
|
||||
status: ArtifactStatus = ArtifactStatus.LOCAL
|
||||
|
||||
@field_validator("relative_path")
|
||||
@classmethod
|
||||
def safe_relative_path(cls, value: str) -> str:
|
||||
normalized = value.replace("\\", "/")
|
||||
if normalized.startswith("/") or ".." in normalized.split("/"):
|
||||
raise ValueError("relative_path must remain within the storage root")
|
||||
return normalized
|
||||
|
||||
|
||||
class ArtifactLocationResponse(ArtifactLocationCreate):
|
||||
id: uuid.UUID
|
||||
artifact_id: uuid.UUID | None = None
|
||||
derived_artifact_id: uuid.UUID | None = None
|
||||
size_bytes: int | None = None
|
||||
observed_sha256: str | None = None
|
||||
last_checked_at: datetime | None = None
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
|
||||
|
||||
class ArtifactCreate(RegistryModel):
|
||||
filename: str = Field(min_length=1, max_length=512)
|
||||
artifact_type: str = Field(min_length=1, max_length=64)
|
||||
serialization_format: str = Field(min_length=1, max_length=64)
|
||||
sha256: str = Field(pattern=r"^[0-9a-f]{64}$")
|
||||
size_bytes: int = Field(ge=0)
|
||||
status: ArtifactStatus = ArtifactStatus.REMOTE
|
||||
security_status: str = "unverified"
|
||||
license_status: str = "unknown"
|
||||
locations: list[ArtifactLocationCreate] = Field(default_factory=list)
|
||||
|
||||
|
||||
class ArtifactResponse(RegistryModel):
|
||||
id: uuid.UUID
|
||||
revision_id: uuid.UUID
|
||||
filename: str
|
||||
artifact_type: str
|
||||
serialization_format: str
|
||||
sha256: str
|
||||
size_bytes: int
|
||||
status: ArtifactStatus
|
||||
security_status: str
|
||||
license_status: str
|
||||
quarantined: bool
|
||||
verification_details: dict[str, Any]
|
||||
verified_at: datetime | None = None
|
||||
immutable_at: datetime | None = None
|
||||
deprecated_at: datetime | None = None
|
||||
archived_at: datetime | None = None
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
locations: list[ArtifactLocationResponse] = Field(default_factory=list)
|
||||
|
||||
|
||||
class DerivedArtifactCreate(RegistryModel):
|
||||
revision_id: uuid.UUID
|
||||
source_artifact_ids: list[uuid.UUID] = Field(min_length=1)
|
||||
filename: str = Field(min_length=1, max_length=512)
|
||||
artifact_type: str = Field(min_length=1, max_length=64)
|
||||
sha256: str = Field(pattern=r"^[0-9a-f]{64}$")
|
||||
size_bytes: int = Field(ge=0)
|
||||
transformation_type: str = Field(min_length=1, max_length=64)
|
||||
tool: str = Field(min_length=1, max_length=128)
|
||||
tool_version: str = Field(min_length=1, max_length=128)
|
||||
configuration: dict[str, Any] = Field(default_factory=dict)
|
||||
environment_snapshot: dict[str, Any] = Field(default_factory=dict)
|
||||
status: ArtifactStatus = ArtifactStatus.REMOTE
|
||||
locations: list[ArtifactLocationCreate] = Field(default_factory=list)
|
||||
|
||||
|
||||
class DerivedSourceResponse(RegistryModel):
|
||||
artifact_id: uuid.UUID
|
||||
sha256: str
|
||||
ordinal: int
|
||||
|
||||
|
||||
class DerivedArtifactResponse(RegistryModel):
|
||||
id: uuid.UUID
|
||||
revision_id: uuid.UUID
|
||||
filename: str
|
||||
artifact_type: str
|
||||
sha256: str
|
||||
size_bytes: int
|
||||
transformation_type: str
|
||||
tool: str
|
||||
tool_version: str
|
||||
configuration: dict[str, Any]
|
||||
environment_snapshot: dict[str, Any]
|
||||
status: ArtifactStatus
|
||||
immutable_at: datetime | None = None
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
sources: list[DerivedSourceResponse]
|
||||
locations: list[ArtifactLocationResponse]
|
||||
|
||||
|
||||
class StorageRootCreate(RegistryModel):
|
||||
compute_node_id: uuid.UUID
|
||||
name: str = Field(min_length=1, max_length=128)
|
||||
purpose: str = Field(default="model_artifacts", min_length=1, max_length=64)
|
||||
path: str = Field(min_length=1)
|
||||
agent_path: str | None = Field(default=None, min_length=1)
|
||||
reserve_bytes: int = Field(default=0, ge=0)
|
||||
reserve_percent: int = Field(default=10, ge=0, le=100)
|
||||
|
||||
|
||||
class StorageRootObservation(RegistryModel):
|
||||
writable: bool
|
||||
capacity_bytes: int | None = Field(default=None, ge=0)
|
||||
free_bytes: int | None = Field(default=None, ge=0)
|
||||
details: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class StorageRootUpdate(RegistryModel):
|
||||
agent_path: str = Field(min_length=1)
|
||||
|
||||
|
||||
class StorageRootResponse(StorageRootCreate):
|
||||
id: uuid.UUID
|
||||
status: StorageRootStatus
|
||||
writable: bool
|
||||
capacity_bytes: int | None = None
|
||||
free_bytes: int | None = None
|
||||
capacity_observed_at: datetime | None = None
|
||||
validation_details: dict[str, Any]
|
||||
deprecated_at: datetime | None = None
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
|
||||
|
||||
class CapacityDecision(RegistryModel):
|
||||
allowed: bool
|
||||
status: StorageRootStatus
|
||||
requested_bytes: int
|
||||
usable_bytes: int | None
|
||||
reason: str
|
||||
|
||||
|
||||
class VerifyResponse(RegistryModel):
|
||||
artifact_id: uuid.UUID
|
||||
status: ArtifactStatus
|
||||
expected_sha256: str
|
||||
observed_sha256: str | None
|
||||
observed_size_bytes: int | None
|
||||
checked_at: datetime
|
||||
location_id: uuid.UUID
|
||||
|
||||
|
||||
class DependencyReference(RegistryModel):
|
||||
resource_type: str
|
||||
resource_id: uuid.UUID
|
||||
relation: str
|
||||
|
||||
|
||||
class DependencyConflict(RegistryModel):
|
||||
message: str
|
||||
dependencies: list[DependencyReference]
|
||||
@@ -0,0 +1,223 @@
|
||||
"""The v1 release contract.
|
||||
|
||||
One authoritative product version, and the compatibility ranges that version promises. Before M17
|
||||
the version existed four times — backend, Node Agent, Runtime Worker and the console each declared
|
||||
``0.1.0`` independently — which is three opportunities for a release to describe itself wrongly.
|
||||
|
||||
The version lives in the repository's ``VERSION`` file. Every packaged manifest is checked against
|
||||
it rather than trusted to agree, because a manifest that has drifted looks exactly like one that has
|
||||
not.
|
||||
|
||||
Compatibility is stated, not implied. An application refusing to start against a schema it does not
|
||||
support is a far better outcome than one that starts and writes rows the next version cannot read.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from enum import StrEnum
|
||||
from pathlib import Path
|
||||
|
||||
# --------------------------------------------------------------------------- version
|
||||
|
||||
#: Schema revisions this application version can run against, oldest first. The application refuses
|
||||
#: to serve against anything outside this list rather than guessing.
|
||||
SUPPORTED_SCHEMA_REVISIONS: tuple[str, ...] = ("20260830_0024",)
|
||||
|
||||
#: The revision a clean installation and a completed upgrade must both arrive at.
|
||||
TARGET_SCHEMA_REVISION = SUPPORTED_SCHEMA_REVISIONS[-1]
|
||||
|
||||
#: Schema revisions from which the upgrade tool may migrate to the target. These are deliberately
|
||||
#: separate from runtime compatibility: the node-decommission code reads columns that do not exist
|
||||
#: at 0021, so serving on 0021 would be unsafe even though migrating from it is supported.
|
||||
SUPPORTED_UPGRADE_SOURCE_SCHEMA_REVISIONS: tuple[str, ...] = (
|
||||
"20260827_0021",
|
||||
"20260828_0022",
|
||||
"20260830_0023",
|
||||
TARGET_SCHEMA_REVISION,
|
||||
)
|
||||
|
||||
#: The oldest release an in-place upgrade to this version is supported from.
|
||||
MINIMUM_UPGRADE_SOURCE = "v1.0.0"
|
||||
|
||||
#: Agent protocol versions this control plane accepts.
|
||||
SUPPORTED_AGENT_PROTOCOL_VERSIONS: tuple[int, ...] = (1,)
|
||||
|
||||
#: The protocol version this control plane itself speaks.
|
||||
CURRENT_AGENT_PROTOCOL_VERSION = SUPPORTED_AGENT_PROTOCOL_VERSIONS[-1]
|
||||
|
||||
#: Release channels. v1 ships one; the enum exists so adding another is a typed change.
|
||||
RELEASE_CHANNEL = "stable"
|
||||
|
||||
#: Minimum PostgreSQL major version. The schema uses partial unique indexes and generated columns
|
||||
#: that older majors either lack or plan differently.
|
||||
MINIMUM_POSTGRES_MAJOR = 16
|
||||
|
||||
|
||||
def _read_version_file() -> str:
|
||||
"""Read the repository's VERSION file, falling back to the packaged distribution metadata."""
|
||||
|
||||
here = Path(__file__).resolve()
|
||||
for parent in here.parents:
|
||||
candidate = parent / "VERSION"
|
||||
if candidate.is_file():
|
||||
return candidate.read_text("utf-8").strip()
|
||||
# An installed wheel has no VERSION file beside it; the distribution metadata is authoritative
|
||||
# there, and the packaging test proves the two agree at build time.
|
||||
from importlib.metadata import PackageNotFoundError, version
|
||||
|
||||
try:
|
||||
return version("modelforge-api")
|
||||
except PackageNotFoundError: # pragma: no cover - only in a broken install
|
||||
return "0.0.0"
|
||||
|
||||
|
||||
PRODUCT_VERSION = _read_version_file()
|
||||
PRODUCT_NAME = "ITWorx ModelForge"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- semver
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SemanticVersion:
|
||||
major: int
|
||||
minor: int
|
||||
patch: int
|
||||
prerelease: str | None = None
|
||||
|
||||
@classmethod
|
||||
def parse(cls, value: str) -> SemanticVersion:
|
||||
core, _, prerelease = value.partition("-")
|
||||
parts = core.split(".")
|
||||
if len(parts) != 3 or not all(part.isdigit() for part in parts):
|
||||
raise ValueError(f"{value!r} is not a MAJOR.MINOR.PATCH version")
|
||||
major, minor, patch = (int(part) for part in parts)
|
||||
return cls(major, minor, patch, prerelease or None)
|
||||
|
||||
def __str__(self) -> str:
|
||||
core = f"{self.major}.{self.minor}.{self.patch}"
|
||||
return f"{core}-{self.prerelease}" if self.prerelease else core
|
||||
|
||||
@property
|
||||
def is_prerelease(self) -> bool:
|
||||
return self.prerelease is not None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- compatibility
|
||||
|
||||
|
||||
class Compatibility(StrEnum):
|
||||
"""The four answers a compatibility question can have. There is no fifth, and no silent pass."""
|
||||
|
||||
COMPATIBLE = "COMPATIBLE"
|
||||
TOO_OLD = "TOO_OLD"
|
||||
TOO_NEW = "TOO_NEW"
|
||||
UNKNOWN = "UNKNOWN"
|
||||
|
||||
|
||||
def schema_compatibility(revision: str | None) -> Compatibility:
|
||||
"""Is this application version able to run against that schema revision?"""
|
||||
|
||||
if revision is None:
|
||||
return Compatibility.UNKNOWN
|
||||
if revision in SUPPORTED_SCHEMA_REVISIONS:
|
||||
return Compatibility.COMPATIBLE
|
||||
# Revisions are date-ordered identifiers, so a straight comparison against the oldest and newest
|
||||
# supported revision tells old from new without a migration graph walk.
|
||||
if revision < SUPPORTED_SCHEMA_REVISIONS[0]:
|
||||
return Compatibility.TOO_OLD
|
||||
if revision > SUPPORTED_SCHEMA_REVISIONS[-1]:
|
||||
return Compatibility.TOO_NEW
|
||||
return Compatibility.UNKNOWN
|
||||
|
||||
|
||||
def agent_protocol_compatibility(protocol_version: int | None) -> Compatibility:
|
||||
"""Can an agent speaking that protocol version talk to this control plane?"""
|
||||
|
||||
if protocol_version is None:
|
||||
return Compatibility.UNKNOWN
|
||||
if protocol_version in SUPPORTED_AGENT_PROTOCOL_VERSIONS:
|
||||
return Compatibility.COMPATIBLE
|
||||
if protocol_version < SUPPORTED_AGENT_PROTOCOL_VERSIONS[0]:
|
||||
return Compatibility.TOO_OLD
|
||||
return Compatibility.TOO_NEW
|
||||
|
||||
|
||||
def upgrade_required(compatibility: Compatibility) -> str | None:
|
||||
"""What the operator has to do, in one sentence, or None when nothing is required."""
|
||||
|
||||
match compatibility:
|
||||
case Compatibility.COMPATIBLE:
|
||||
return None
|
||||
case Compatibility.TOO_OLD:
|
||||
return "upgrade the component to a release that speaks the current protocol"
|
||||
case Compatibility.TOO_NEW:
|
||||
return "upgrade the control plane, which is older than the component reporting to it"
|
||||
case Compatibility.UNKNOWN:
|
||||
return "the version could not be determined and is refused rather than assumed"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- build identity
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class BuildIdentity:
|
||||
"""What a running binary can say about where it came from.
|
||||
|
||||
Everything here is either compiled in at build time or read from the environment the image was
|
||||
built with. Nothing is inferred at runtime, because a build identity that a running process can
|
||||
talk itself into is worth nothing during an incident.
|
||||
"""
|
||||
|
||||
version: str
|
||||
source_commit: str | None
|
||||
built_at: str | None
|
||||
image_digest: str | None
|
||||
channel: str
|
||||
schema_revision: str
|
||||
agent_protocol_version: int
|
||||
|
||||
def as_dict(self) -> dict[str, object]:
|
||||
return {
|
||||
"version": self.version,
|
||||
"source_commit": self.source_commit,
|
||||
"built_at": self.built_at,
|
||||
"image_digest": self.image_digest,
|
||||
"channel": self.channel,
|
||||
"schema_revision": self.schema_revision,
|
||||
"agent_protocol_version": self.agent_protocol_version,
|
||||
}
|
||||
|
||||
|
||||
def _clean(value: str | None) -> str | None:
|
||||
"""Treat an unsubstituted build argument as absent rather than reporting it as a fact."""
|
||||
|
||||
if value is None:
|
||||
return None
|
||||
stripped = value.strip()
|
||||
if not stripped or stripped.lower() in {"unknown", "none", "null"}:
|
||||
return None
|
||||
return stripped
|
||||
|
||||
|
||||
def build_identity(
|
||||
*,
|
||||
source_commit: str | None = None,
|
||||
built_at: str | None = None,
|
||||
image_digest: str | None = None,
|
||||
) -> BuildIdentity:
|
||||
return BuildIdentity(
|
||||
version=PRODUCT_VERSION,
|
||||
source_commit=_clean(source_commit),
|
||||
built_at=_clean(built_at),
|
||||
image_digest=_clean(image_digest),
|
||||
channel=RELEASE_CHANNEL,
|
||||
schema_revision=TARGET_SCHEMA_REVISION,
|
||||
agent_protocol_version=CURRENT_AGENT_PROTOCOL_VERSION,
|
||||
)
|
||||
|
||||
|
||||
def utc_now_iso() -> str:
|
||||
return datetime.now(UTC).isoformat()
|
||||
@@ -0,0 +1,249 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
|
||||
RuntimeAdapterName = Literal[
|
||||
"sentence_transformers",
|
||||
"qwen3_reranker",
|
||||
"transformers_trocr",
|
||||
"transformers_siglip2",
|
||||
"transformers_whisper",
|
||||
"transformers",
|
||||
"vllm",
|
||||
"llama_cpp",
|
||||
"diffusers",
|
||||
"custom",
|
||||
]
|
||||
CompatibilityStatus = Literal["compatible", "incompatible", "unknown", "requires_probe", "blocked"]
|
||||
ProbeStatus = Literal[
|
||||
"queued",
|
||||
"preparing",
|
||||
"loading",
|
||||
"healthchecking",
|
||||
"ready",
|
||||
"unloading",
|
||||
"completed",
|
||||
"failed",
|
||||
"cancelled",
|
||||
]
|
||||
|
||||
|
||||
class RuntimeModel(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", from_attributes=True)
|
||||
|
||||
|
||||
class RuntimeEnvironmentCreate(RuntimeModel):
|
||||
name: str = Field(min_length=1, max_length=255)
|
||||
adapter: RuntimeAdapterName
|
||||
runtime_version: str = Field(min_length=1, max_length=128)
|
||||
image_repository: str = Field(min_length=1, max_length=255)
|
||||
image_digest: str = Field(pattern=r"^sha256:[a-f0-9]{64}$")
|
||||
python_version: str = Field(min_length=1, max_length=64)
|
||||
cuda_runtime_version: str | None = Field(default=None, max_length=64)
|
||||
package_versions: dict[str, str] = Field(default_factory=dict)
|
||||
supported_model_types: list[str] = Field(default_factory=list)
|
||||
supported_formats: list[str] = Field(default_factory=list)
|
||||
supported_modalities: list[str] = Field(default_factory=list)
|
||||
network_policy: Literal["offline_control_plane_only"] = "offline_control_plane_only"
|
||||
|
||||
|
||||
class RuntimeEnvironmentResponse(RuntimeEnvironmentCreate):
|
||||
id: uuid.UUID
|
||||
fingerprint: str
|
||||
immutable_at: datetime
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class RuntimeProfileCreate(RuntimeModel):
|
||||
name: str = Field(min_length=1, max_length=255)
|
||||
runtime_environment_id: uuid.UUID
|
||||
artifact_set_id: uuid.UUID
|
||||
dtype: Literal["float32", "float16", "bfloat16"] = "bfloat16"
|
||||
quantization: str | None = Field(default=None, max_length=64)
|
||||
modality: Literal[
|
||||
"embedding", "reranking", "text_generation", "vision", "document", "audio", "diffusion"
|
||||
]
|
||||
max_sequence_length: int = Field(default=128, ge=1, le=131072)
|
||||
batch_size: int = Field(default=1, ge=1, le=128)
|
||||
concurrency: int = Field(default=1, ge=1, le=128)
|
||||
device_policy: Literal["cuda_required", "cuda_preferred", "cpu_only"] = "cuda_required"
|
||||
gpu_memory_policy: dict[str, Any] = Field(default_factory=dict)
|
||||
launch_parameters: dict[str, Any] = Field(default_factory=dict)
|
||||
environment_variables: dict[str, str] = Field(default_factory=dict)
|
||||
trust_remote_code: Literal[False] = False
|
||||
network_egress: Literal[False] = False
|
||||
|
||||
@field_validator("environment_variables")
|
||||
@classmethod
|
||||
def reject_secrets(cls, value: dict[str, str]) -> dict[str, str]:
|
||||
forbidden = {"TOKEN", "SECRET", "PASSWORD", "KEY", "CREDENTIAL"}
|
||||
if any(
|
||||
forbidden.intersection(filter(None, re.split(r"[^A-Z0-9]+", key.upper())))
|
||||
for key in value
|
||||
):
|
||||
raise ValueError("runtime profile environment cannot contain secrets")
|
||||
return value
|
||||
|
||||
@model_validator(mode="after")
|
||||
def fixed_probe_shape(self) -> RuntimeProfileCreate:
|
||||
if self.modality == "embedding" and self.batch_size != 1:
|
||||
raise ValueError("M4 embedding probes require batch_size=1")
|
||||
return self
|
||||
|
||||
|
||||
class RuntimeProfileResponse(RuntimeProfileCreate):
|
||||
id: uuid.UUID
|
||||
adapter: str
|
||||
runtime_version: str
|
||||
image_digest: str
|
||||
version: int
|
||||
fingerprint: str
|
||||
health_contract: dict[str, Any]
|
||||
immutable_at: datetime
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class CompatibilityAssessmentCreate(RuntimeModel):
|
||||
runtime_profile_id: uuid.UUID
|
||||
compute_node_id: uuid.UUID
|
||||
|
||||
|
||||
class CompatibilityAssessmentResponse(RuntimeModel):
|
||||
id: uuid.UUID
|
||||
artifact_set_id: uuid.UUID
|
||||
runtime_profile_id: uuid.UUID
|
||||
compute_node_id: uuid.UUID
|
||||
adapter: str
|
||||
runtime_version: str
|
||||
status: CompatibilityStatus
|
||||
static_result: dict[str, Any]
|
||||
evidence: dict[str, Any]
|
||||
blockers: list[str]
|
||||
warnings: list[str]
|
||||
required_approvals: list[str]
|
||||
hardware_facts: dict[str, Any]
|
||||
artifact_facts: dict[str, Any]
|
||||
environment_fingerprint: str
|
||||
stale: bool
|
||||
stale_reason: str | None
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class ExecutionApprovalCreate(RuntimeModel):
|
||||
scope: Literal["lab_execution"] = "lab_execution"
|
||||
reason: str = Field(min_length=8, max_length=2000)
|
||||
approved_by: str = Field(min_length=1, max_length=255)
|
||||
expires_at: datetime | None = None
|
||||
|
||||
|
||||
class ExecutionApprovalResponse(RuntimeModel):
|
||||
id: uuid.UUID
|
||||
artifact_set_id: uuid.UUID
|
||||
scope: str
|
||||
status: str
|
||||
evidence_fingerprint: str
|
||||
reason: str
|
||||
approved_by: str
|
||||
approved_at: datetime
|
||||
expires_at: datetime | None
|
||||
revoked_at: datetime | None
|
||||
stale: bool
|
||||
|
||||
|
||||
class RuntimeProbeCreate(RuntimeModel):
|
||||
compatibility_assessment_id: uuid.UUID
|
||||
execution_approval_id: uuid.UUID
|
||||
input_text: Literal["ModelForge runtime compatibility probe"] = (
|
||||
"ModelForge runtime compatibility probe"
|
||||
)
|
||||
|
||||
|
||||
class RuntimeProbeResponse(RuntimeModel):
|
||||
id: uuid.UUID
|
||||
artifact_set_id: uuid.UUID
|
||||
runtime_profile_id: uuid.UUID
|
||||
compute_node_id: uuid.UUID
|
||||
compatibility_assessment_id: uuid.UUID
|
||||
execution_approval_id: uuid.UUID
|
||||
status: ProbeStatus
|
||||
phase: str | None
|
||||
attempt_count: int
|
||||
cancel_requested: bool
|
||||
load_result: dict[str, Any]
|
||||
health_result: dict[str, Any]
|
||||
inference_result: dict[str, Any]
|
||||
unload_result: dict[str, Any]
|
||||
measured_resources: dict[str, Any]
|
||||
runtime_facts: dict[str, Any]
|
||||
environment_fingerprint: str
|
||||
failure_code: str | None
|
||||
failure_message: str | None
|
||||
logs_reference: str | None
|
||||
started_at: datetime | None
|
||||
finished_at: datetime | None
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
|
||||
|
||||
class DeploymentCandidateResponse(RuntimeModel):
|
||||
id: uuid.UUID
|
||||
artifact_set_id: uuid.UUID
|
||||
runtime_profile_id: uuid.UUID
|
||||
compute_node_id: uuid.UUID
|
||||
compatibility_assessment_id: uuid.UUID
|
||||
runtime_probe_id: uuid.UUID
|
||||
channel: Literal["lab"]
|
||||
status: Literal["lab_ready"]
|
||||
production: Literal[False]
|
||||
health_contract: dict[str, Any]
|
||||
measured_resources: dict[str, Any]
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class AgentRuntimeProbeLease(RuntimeModel):
|
||||
probe_id: uuid.UUID
|
||||
lease_token: str
|
||||
lease_expires_at: datetime
|
||||
artifact_set_id: uuid.UUID
|
||||
revision_sha: str = Field(pattern=r"^[a-f0-9]{40,64}$")
|
||||
artifact_root: str
|
||||
artifact_relative_path: str
|
||||
expected_manifest: dict[str, Any]
|
||||
runtime_profile: dict[str, Any]
|
||||
runtime_environment: dict[str, Any]
|
||||
probe_input: Literal["ModelForge runtime compatibility probe"]
|
||||
|
||||
|
||||
class AgentRuntimeProbeProgress(RuntimeModel):
|
||||
lease_token: str = Field(min_length=32, max_length=512)
|
||||
status: Literal["preparing", "loading", "healthchecking", "ready", "unloading"]
|
||||
details: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class AgentRuntimeProbeControl(RuntimeModel):
|
||||
accepted: bool
|
||||
cancel_requested: bool
|
||||
lease_expires_at: datetime
|
||||
|
||||
|
||||
class AgentRuntimeProbeComplete(RuntimeModel):
|
||||
lease_token: str = Field(min_length=32, max_length=512)
|
||||
load_result: dict[str, Any]
|
||||
health_result: dict[str, Any]
|
||||
inference_result: dict[str, Any]
|
||||
unload_result: dict[str, Any]
|
||||
measured_resources: dict[str, Any]
|
||||
runtime_facts: dict[str, Any]
|
||||
environment_fingerprint: str = Field(pattern=r"^[a-f0-9]{64}$")
|
||||
|
||||
|
||||
class AgentRuntimeProbeFailure(RuntimeModel):
|
||||
lease_token: str = Field(min_length=32, max_length=512)
|
||||
failure_code: str = Field(min_length=1, max_length=64)
|
||||
failure_message: str = Field(min_length=1, max_length=2000)
|
||||
details: dict[str, Any] = Field(default_factory=dict)
|
||||
@@ -0,0 +1,172 @@
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from .contracts import (
|
||||
CapabilityCategory,
|
||||
CapabilityContractManifest,
|
||||
CapabilityStability,
|
||||
EvaluationType,
|
||||
ProjectBindingManifest,
|
||||
ResourceClass,
|
||||
)
|
||||
from .enums import (
|
||||
HealthStatus,
|
||||
ModelLifecycle,
|
||||
VerificationStatus,
|
||||
)
|
||||
|
||||
|
||||
class HealthResponse(BaseModel):
|
||||
status: str = "ok"
|
||||
service: str = "modelforge-api"
|
||||
version: str
|
||||
timestamp: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class ReadinessResponse(BaseModel):
|
||||
status: HealthStatus
|
||||
checks: dict[str, HealthStatus]
|
||||
version: str
|
||||
timestamp: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class SystemMetadata(BaseModel):
|
||||
name: str
|
||||
version: str
|
||||
environment: str
|
||||
api_version: str = "v1"
|
||||
release_channel: str = "stable"
|
||||
production_inference_available: bool = False
|
||||
|
||||
|
||||
class ReleaseCompatibility(BaseModel):
|
||||
"""What this build will and will not talk to, stated rather than implied."""
|
||||
|
||||
schema_revision: str
|
||||
supported_schema_revisions: list[str]
|
||||
agent_protocol_version: int
|
||||
supported_agent_protocol_versions: list[int]
|
||||
minimum_upgrade_source: str
|
||||
minimum_postgres_major: int
|
||||
|
||||
|
||||
class ReleaseInfo(BaseModel):
|
||||
"""Build identity for the running process.
|
||||
|
||||
Deliberately free of anything sensitive: no paths, no configuration values, no credentials —
|
||||
only what an operator needs to answer "which build is this, and what does it support?".
|
||||
"""
|
||||
|
||||
name: str
|
||||
version: str
|
||||
release_channel: str
|
||||
source_commit: str | None = None
|
||||
built_at: str | None = None
|
||||
image_digest: str | None = None
|
||||
compatibility: ReleaseCompatibility
|
||||
|
||||
|
||||
class ErrorDetail(BaseModel):
|
||||
code: str
|
||||
message: str
|
||||
correlation_id: str
|
||||
details: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class ErrorEnvelope(BaseModel):
|
||||
error: ErrorDetail
|
||||
|
||||
|
||||
class GPUInfo(BaseModel):
|
||||
index: int
|
||||
name: str
|
||||
uuid: str | None = None
|
||||
memory_total_mb: int | None = None
|
||||
memory_used_mb: int | None = None
|
||||
utilization_gpu_percent: int | None = None
|
||||
utilization_memory_percent: int | None = None
|
||||
temperature_c: int | None = None
|
||||
power_draw_w: float | None = None
|
||||
telemetry_available: bool = True
|
||||
error: str | None = None
|
||||
|
||||
|
||||
class CapabilityContractResponse(BaseModel):
|
||||
key: str
|
||||
version: int
|
||||
description: str
|
||||
contract: CapabilityContractManifest
|
||||
stable_deployment: dict[str, Any] | None = None
|
||||
|
||||
|
||||
class CapabilityEstateResponse(BaseModel):
|
||||
capability: str
|
||||
version: int
|
||||
category: CapabilityCategory
|
||||
purpose: str
|
||||
declared_stability: CapabilityStability
|
||||
operational_state: str
|
||||
current_deployment_id: uuid.UUID | None = None
|
||||
model: str | None = None
|
||||
revision: str | None = None
|
||||
runtime: str | None = None
|
||||
node: str | None = None
|
||||
resource_class: ResourceClass
|
||||
measured_required_vram_bytes: int | None = None
|
||||
consumers: list[str]
|
||||
privacy_class: str
|
||||
evaluation_type: EvaluationType
|
||||
evaluation_state: str
|
||||
|
||||
|
||||
class InstallationDependencyResponse(BaseModel):
|
||||
capability: str
|
||||
version: int
|
||||
deployment_id: uuid.UUID
|
||||
channel: str
|
||||
production: bool
|
||||
project_consumers: list[str]
|
||||
active_project_consumers: list[str]
|
||||
project_fit_evidence_ids: list[uuid.UUID]
|
||||
evaluation_run_ids: list[uuid.UUID]
|
||||
last_used_at: datetime | None
|
||||
|
||||
|
||||
class ModelInstallationRationaleResponse(BaseModel):
|
||||
model_id: uuid.UUID
|
||||
display_name: str
|
||||
upstream_source: str
|
||||
installed: bool
|
||||
installed_bytes: int
|
||||
dependencies: list[InstallationDependencyResponse]
|
||||
can_delete: bool
|
||||
deletion_blockers: list[str]
|
||||
|
||||
|
||||
class CandidateSummary(BaseModel):
|
||||
id: str
|
||||
display_name: str
|
||||
source: str
|
||||
intended_capabilities: list[str]
|
||||
proposed_role: str
|
||||
preferred_runtime: str | None = None
|
||||
lifecycle: ModelLifecycle = ModelLifecycle.CANDIDATE
|
||||
verification_status: VerificationStatus = VerificationStatus.UNVERIFIED
|
||||
deployment_status: str = "not_deployed"
|
||||
|
||||
|
||||
class ProjectBindingResponse(BaseModel):
|
||||
capability: str
|
||||
contract_version: int
|
||||
binding: ProjectBindingManifest
|
||||
|
||||
|
||||
class ProjectResponse(BaseModel):
|
||||
id: str
|
||||
name: str
|
||||
description: str
|
||||
bindings: list[ProjectBindingResponse]
|
||||
notes: list[str] = Field(default_factory=list)
|
||||
@@ -0,0 +1,764 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import math
|
||||
import re
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
|
||||
CAPABILITY_VERSION_KEY = re.compile(r"^[a-z][a-z0-9_]*(?:\.[a-z][a-z0-9_]*)+@[1-9][0-9]*$")
|
||||
ServingOperation = Literal["load", "invoke", "health", "drain", "unload"]
|
||||
|
||||
|
||||
class ServingModel(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", from_attributes=True)
|
||||
|
||||
|
||||
class SupplyChainReview(ServingModel):
|
||||
exact_revision_reviewed: Literal[True]
|
||||
artifact_hashes_reviewed: Literal[True]
|
||||
safetensors_only: Literal[True]
|
||||
remote_code_required: Literal[False]
|
||||
pickle_present: Literal[False]
|
||||
scanner_evidence_reviewed: Literal[True]
|
||||
provenance_complete: Literal[True]
|
||||
runtime_offline_reviewed: Literal[True]
|
||||
dependency_provenance_reviewed: Literal[True]
|
||||
license_reviewed: Literal[True]
|
||||
license_identifier: str = Field(min_length=1, max_length=255)
|
||||
|
||||
|
||||
class ProductionApprovalCreate(ServingModel):
|
||||
capability: Literal["rag.embedding"] = "rag.embedding"
|
||||
contract_version: Literal[1] = 1
|
||||
approved_by: str = Field(min_length=1, max_length=255)
|
||||
reason: str = Field(min_length=16, max_length=4000)
|
||||
supply_chain_review: SupplyChainReview
|
||||
deployment_config: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class ProductionApprovalResponse(ServingModel):
|
||||
id: uuid.UUID
|
||||
deployment_candidate_id: uuid.UUID
|
||||
capability_contract_id: uuid.UUID
|
||||
artifact_set_id: uuid.UUID
|
||||
runtime_profile_id: uuid.UUID
|
||||
compute_node_id: uuid.UUID
|
||||
deployment_config: dict[str, Any]
|
||||
supply_chain_evidence: dict[str, Any]
|
||||
evidence_fingerprint: str
|
||||
status: str
|
||||
approved_by: str
|
||||
reason: str
|
||||
approved_at: datetime
|
||||
revoked_at: datetime | None
|
||||
stale: bool = False
|
||||
|
||||
|
||||
class CapabilityPromotionCreate(ServingModel):
|
||||
production_approval_id: uuid.UUID
|
||||
residency_policy: Literal["always_warm", "keep_warm", "load_on_demand"] = "keep_warm"
|
||||
keep_warm_seconds: int = Field(default=900, ge=5, le=86400)
|
||||
max_concurrency: Literal[1] = 1
|
||||
max_queue_depth: int = Field(default=16, ge=1, le=1024)
|
||||
routing_weight: int = Field(default=100, ge=0, le=100)
|
||||
rollback_policy: dict[str, Any] = Field(
|
||||
default_factory=lambda: {"mode": "drain_to_unavailable", "previous_deployment_id": None}
|
||||
)
|
||||
|
||||
|
||||
class EmbeddingSpaceResponse(ServingModel):
|
||||
id: uuid.UUID
|
||||
identity_digest: str
|
||||
dimension: int
|
||||
normalized: bool
|
||||
migration_class: Literal["requires_reindex"]
|
||||
identity_facts: dict[str, Any]
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class ResourceEnvelopeResponse(ServingModel):
|
||||
id: uuid.UUID
|
||||
runtime_probe_id: uuid.UUID
|
||||
accelerator_kind: str
|
||||
accelerator_uuid: str
|
||||
environment_fingerprint: str
|
||||
concurrency: int
|
||||
batch_size: int
|
||||
max_sequence_length: int
|
||||
baseline_vram_bytes: int
|
||||
resident_vram_bytes: int
|
||||
peak_vram_bytes: int
|
||||
required_vram_bytes: int
|
||||
cold_load_time_ms: float
|
||||
inference_latency_ms: float
|
||||
stale: bool
|
||||
stale_reason: str | None
|
||||
|
||||
|
||||
class ResidencyResponse(ServingModel):
|
||||
id: uuid.UUID
|
||||
state: str
|
||||
worker_instance_id: str | None
|
||||
load_count: int
|
||||
active_requests: int
|
||||
measured_resident_vram_bytes: int
|
||||
external_baseline_vram_bytes: int
|
||||
health: dict[str, Any]
|
||||
resident_since: datetime | None
|
||||
last_used_at: datetime | None
|
||||
failure_code: str | None
|
||||
failure_message: str | None
|
||||
generation: int = 1
|
||||
transition_reason: str | None = None
|
||||
|
||||
|
||||
class CapabilityDeploymentResponse(ServingModel):
|
||||
id: uuid.UUID
|
||||
capability: str
|
||||
contract_version: int
|
||||
deployment_candidate_id: uuid.UUID
|
||||
production_approval_id: uuid.UUID | None
|
||||
execution_approval_id: uuid.UUID | None
|
||||
artifact_set_id: uuid.UUID
|
||||
runtime_profile_id: uuid.UUID
|
||||
compute_node_id: uuid.UUID
|
||||
accelerator_id: uuid.UUID
|
||||
embedding_space: EmbeddingSpaceResponse | None
|
||||
resource_envelope: ResourceEnvelopeResponse
|
||||
residency: ResidencyResponse
|
||||
channel: str
|
||||
status: str
|
||||
production: bool
|
||||
health_status: str
|
||||
routing_weight: int
|
||||
fallback_policy: dict[str, Any]
|
||||
residency_policy: str
|
||||
keep_warm_seconds: int
|
||||
max_concurrency: int
|
||||
max_queue_depth: int
|
||||
config_fingerprint: str
|
||||
provenance: dict[str, Any]
|
||||
rollback_policy: dict[str, Any]
|
||||
promoted_at: datetime | None
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class ServiceClientCreate(ServingModel):
|
||||
name: str = Field(min_length=1, max_length=255, pattern=r"^[a-z][a-z0-9-]*$")
|
||||
allowed_capabilities: list[str] = Field(min_length=1, max_length=32)
|
||||
requests_per_minute: int = Field(default=60, ge=1, le=10000)
|
||||
max_concurrent_requests: int = Field(default=1, ge=1, le=64)
|
||||
workload_priority: Literal["production", "interactive", "background", "benchmark"] = (
|
||||
"production"
|
||||
)
|
||||
project_key: str | None = Field(default=None, pattern=r"^[a-z][a-z0-9-]*$")
|
||||
integration_environment: Literal["production", "shadow", "evaluation", "lab"] | None = None
|
||||
purpose: str | None = Field(default=None, min_length=3, max_length=500)
|
||||
credential_expires_at: datetime | None = None
|
||||
|
||||
@field_validator("allowed_capabilities")
|
||||
@classmethod
|
||||
def validate_capability_scopes(cls, value: list[str]) -> list[str]:
|
||||
if len(set(value)) != len(value) or any(
|
||||
not CAPABILITY_VERSION_KEY.fullmatch(v) for v in value
|
||||
):
|
||||
raise ValueError("capability scopes must be unique versioned capability keys")
|
||||
return value
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_project_scope(self) -> ServiceClientCreate:
|
||||
project_fields = (self.project_key, self.integration_environment, self.purpose)
|
||||
if any(value is not None for value in project_fields) and not all(
|
||||
value is not None for value in project_fields
|
||||
):
|
||||
raise ValueError("project_key, integration_environment and purpose are required together")
|
||||
if self.project_key is not None and len(self.allowed_capabilities) != 1:
|
||||
raise ValueError("a project service identity must bind exactly one capability")
|
||||
return self
|
||||
|
||||
|
||||
class ServiceClientResponse(ServingModel):
|
||||
id: uuid.UUID
|
||||
name: str
|
||||
status: str
|
||||
allowed_capabilities: list[str]
|
||||
requests_per_minute: int
|
||||
max_concurrent_requests: int
|
||||
workload_priority: str
|
||||
project_key: str | None
|
||||
project_binding_id: uuid.UUID | None
|
||||
integration_environment: str | None
|
||||
purpose: str | None
|
||||
credential_prefix: str | None
|
||||
credential_created_at: datetime | None
|
||||
credential_expires_at: datetime | None
|
||||
credential_revoked_at: datetime | None
|
||||
last_used_at: datetime | None
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class ServiceClientCreated(ServiceClientResponse):
|
||||
credential: str
|
||||
|
||||
|
||||
class ProjectFitEvidenceCreate(ServingModel):
|
||||
project_key: str = Field(pattern=r"^[a-z][a-z0-9-]*$")
|
||||
capability: str = Field(pattern=r"^[a-z][a-z0-9.]*@[1-9][0-9]*$")
|
||||
environment: Literal["production", "shadow", "evaluation", "lab"]
|
||||
recommendation: Literal[
|
||||
"PROMOTION_ELIGIBLE", "KEEP_LAB", "BLOCKED", "REQUIRES_MORE_EVIDENCE"
|
||||
]
|
||||
evidence_class: Literal[
|
||||
"OWNER_PHOTO", "PUBLIC_PHYSICAL_CAPTURE", "CATALOG_REFERENCE"
|
||||
]
|
||||
engineering_integration: Literal["PASS", "INCOMPLETE", "BLOCKED"]
|
||||
production_validation: Literal[
|
||||
"DEFERRED_EXTERNAL_VALIDATION", "REQUIRED", "SATISFIED"
|
||||
]
|
||||
production_action: Literal["NONE"] = "NONE"
|
||||
case_count: int = Field(ge=0, le=1_000_000)
|
||||
metric_values: dict[str, float | int | str | bool | None]
|
||||
critical_errors: int = Field(ge=0)
|
||||
blockers: list[str] = Field(default_factory=list, max_length=100)
|
||||
evidence: dict[str, Any] = Field(default_factory=dict)
|
||||
capability_deployment_id: uuid.UUID | None = None
|
||||
|
||||
|
||||
class ProjectFitEvidenceResponse(ServingModel):
|
||||
id: uuid.UUID
|
||||
project_key: str
|
||||
capability: str
|
||||
environment: str
|
||||
recommendation: str
|
||||
evidence_class: str
|
||||
engineering_integration: str
|
||||
production_validation: str
|
||||
production_action: str
|
||||
case_count: int
|
||||
metric_values: dict[str, Any]
|
||||
critical_errors: int
|
||||
blockers: list[str]
|
||||
evidence_digest: str
|
||||
evidence: dict[str, Any]
|
||||
capability_deployment_id: uuid.UUID | None
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class ProjectIntegrationUsage(ServingModel):
|
||||
request_volume: int
|
||||
successful_requests: int
|
||||
error_count: int
|
||||
last_used_at: datetime | None
|
||||
latency_p50_ms: float | None
|
||||
latency_p95_ms: float | None
|
||||
|
||||
|
||||
class ProjectIntegrationResponse(ServingModel):
|
||||
project_key: str
|
||||
project_name: str
|
||||
capability: str
|
||||
environment: str
|
||||
state: str
|
||||
purpose: str
|
||||
client_id: uuid.UUID
|
||||
client_name: str
|
||||
deployment_id: uuid.UUID | None
|
||||
resource_impact_bytes: int | None
|
||||
usage: ProjectIntegrationUsage
|
||||
project_fit: ProjectFitEvidenceResponse | None
|
||||
|
||||
|
||||
class EmbeddingInvokeRequest(ServingModel):
|
||||
input: str | list[str]
|
||||
input_type: Literal["raw", "query", "document"] = "raw"
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_input(self) -> EmbeddingInvokeRequest:
|
||||
values = [self.input] if isinstance(self.input, str) else self.input
|
||||
if not values or any(not item.strip() for item in values):
|
||||
raise ValueError("input must contain at least one non-empty string")
|
||||
return self
|
||||
|
||||
def inputs(self) -> list[str]:
|
||||
return [self.input] if isinstance(self.input, str) else self.input
|
||||
|
||||
|
||||
class CapabilityExperimentCreate(ServingModel):
|
||||
route_key: str = Field(min_length=3, max_length=128, pattern=r"^[a-z][a-z0-9-]+$")
|
||||
execution_approval_id: uuid.UUID
|
||||
capability: Literal[
|
||||
"rag.embedding",
|
||||
"rag.reranking",
|
||||
"document.ocr",
|
||||
"vision.embedding",
|
||||
"speech.transcription",
|
||||
] = "rag.embedding"
|
||||
purpose: str = Field(min_length=16, max_length=2000)
|
||||
expected_dimension: int = Field(default=1024, ge=32, le=8192)
|
||||
residency_policy: Literal["keep_warm", "load_on_demand"] = "keep_warm"
|
||||
keep_warm_seconds: int = Field(default=300, ge=5, le=86400)
|
||||
max_queue_depth: int = Field(default=16, ge=1, le=1024)
|
||||
|
||||
|
||||
class CapabilityExperimentResponse(ServingModel):
|
||||
id: uuid.UUID
|
||||
route_key: str
|
||||
capability: str
|
||||
contract_version: int
|
||||
capability_deployment_id: uuid.UUID
|
||||
status: str
|
||||
purpose: str
|
||||
evidence: dict[str, Any]
|
||||
deployment: CapabilityDeploymentResponse
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class GatewayTiming(ServingModel):
|
||||
validation_ms: float = 0.0
|
||||
resolution_ms: float = 0.0
|
||||
scheduling_ms: float = 0.0
|
||||
payload_ms: float = 0.0
|
||||
dispatch_ms: float = 0.0
|
||||
worker_preprocess_ms: float = 0.0
|
||||
worker_serialize_ms: float = 0.0
|
||||
worker_total_ms: float = 0.0
|
||||
completion_transport_ms: float = 0.0
|
||||
result_ms: float = 0.0
|
||||
gateway_ms: float
|
||||
# Backwards-compatible alias for the scheduler/lease wait used by M5/M6.
|
||||
queue_ms: float
|
||||
load_ms: float
|
||||
inference_ms: float
|
||||
total_ms: float
|
||||
|
||||
|
||||
class GatewayExecution(ServingModel):
|
||||
cold: bool
|
||||
node: str
|
||||
residency: str
|
||||
load_count: int
|
||||
timings: GatewayTiming
|
||||
|
||||
|
||||
class GatewayUsage(ServingModel):
|
||||
input_count: int
|
||||
input_tokens: int
|
||||
|
||||
|
||||
class EmbeddingInvokeResponse(ServingModel):
|
||||
capability: Literal["rag.embedding@1"] = "rag.embedding@1"
|
||||
dimension: int = Field(ge=1, le=65_536)
|
||||
normalized: Literal[True] = True
|
||||
embedding_space_id: uuid.UUID
|
||||
data: list[list[float]]
|
||||
request_id: uuid.UUID
|
||||
execution: GatewayExecution
|
||||
usage: GatewayUsage
|
||||
|
||||
|
||||
class StableEmbeddingInvokeResponse(EmbeddingInvokeResponse):
|
||||
"""Public stable contract; lab experiment routes may use another dimension."""
|
||||
|
||||
dimension: Literal[1024] = 1024
|
||||
|
||||
|
||||
class RerankDocument(ServingModel):
|
||||
id: str = Field(min_length=1, max_length=255)
|
||||
text: str = Field(min_length=1, max_length=16_384)
|
||||
|
||||
|
||||
class RerankingInvokeRequest(ServingModel):
|
||||
query: str = Field(min_length=1, max_length=4_000)
|
||||
documents: list[RerankDocument] = Field(min_length=1, max_length=40)
|
||||
top_n: int = Field(default=10, ge=1, le=10)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_reranking_input(self) -> RerankingInvokeRequest:
|
||||
if not self.query.strip() or any(not document.text.strip() for document in self.documents):
|
||||
raise ValueError("query and document text must be non-empty")
|
||||
if len({document.id for document in self.documents}) != len(self.documents):
|
||||
raise ValueError("document ids must be unique")
|
||||
if self.top_n > len(self.documents):
|
||||
raise ValueError("top_n cannot exceed the document count")
|
||||
if len(self.query) + sum(len(document.text) for document in self.documents) > 262_144:
|
||||
raise ValueError("rerank request is too large")
|
||||
return self
|
||||
|
||||
|
||||
class RerankingResult(ServingModel):
|
||||
id: str
|
||||
score: float
|
||||
rank: int = Field(ge=1, le=10)
|
||||
|
||||
@field_validator("score")
|
||||
@classmethod
|
||||
def finite_score(cls, value: float) -> float:
|
||||
if not math.isfinite(value):
|
||||
raise ValueError("reranking score must be finite")
|
||||
return value
|
||||
|
||||
|
||||
class RerankingInvokeResponse(ServingModel):
|
||||
capability: Literal["rag.reranking@1"] = "rag.reranking@1"
|
||||
results: list[RerankingResult]
|
||||
request_id: uuid.UUID
|
||||
execution: GatewayExecution
|
||||
usage: GatewayUsage
|
||||
|
||||
|
||||
def _bounded_base64(value: str, maximum_bytes: int) -> str:
|
||||
try:
|
||||
decoded = base64.b64decode(value, validate=True)
|
||||
except (binascii.Error, ValueError) as exc:
|
||||
raise ValueError("content must be canonical base64") from exc
|
||||
if not decoded or len(decoded) > maximum_bytes:
|
||||
raise ValueError(f"decoded content must be between 1 and {maximum_bytes} bytes")
|
||||
return value
|
||||
|
||||
|
||||
class OCRInvokeRequest(ServingModel):
|
||||
content_base64: str
|
||||
media_type: Literal["image/png", "image/jpeg"]
|
||||
language_hint: Literal["nl", "en", "auto"] = "auto"
|
||||
|
||||
@field_validator("content_base64")
|
||||
@classmethod
|
||||
def bounded_content(cls, value: str) -> str:
|
||||
return _bounded_base64(value, 8 * 1024 * 1024)
|
||||
|
||||
|
||||
class OCRBlock(ServingModel):
|
||||
text: str
|
||||
order: int = Field(ge=0)
|
||||
bbox: list[int] | None = Field(default=None, min_length=4, max_length=4)
|
||||
confidence: float | None = Field(default=None, ge=0, le=1)
|
||||
|
||||
|
||||
class OCRPage(ServingModel):
|
||||
page: Literal[1] = 1
|
||||
width: int = Field(ge=1, le=4096)
|
||||
height: int = Field(ge=1, le=4096)
|
||||
blocks: list[OCRBlock]
|
||||
|
||||
|
||||
class OCRInvokeResponse(ServingModel):
|
||||
capability: Literal["document.ocr@1"] = "document.ocr@1"
|
||||
text: str
|
||||
pages: list[OCRPage] = Field(min_length=1, max_length=1)
|
||||
confidence: float | None = Field(default=None, ge=0, le=1)
|
||||
request_id: uuid.UUID
|
||||
execution: GatewayExecution
|
||||
|
||||
|
||||
class VisionEmbeddingItem(ServingModel):
|
||||
image_base64: str | None = None
|
||||
media_type: Literal["image/png", "image/jpeg"] | None = None
|
||||
text: str | None = Field(default=None, min_length=1, max_length=4096)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def exactly_one_modality(self) -> VisionEmbeddingItem:
|
||||
image = self.image_base64 is not None
|
||||
text = self.text is not None
|
||||
if image == text or image != (self.media_type is not None):
|
||||
raise ValueError("each item must contain exactly one typed image or text")
|
||||
if self.image_base64 is not None:
|
||||
_bounded_base64(self.image_base64, 8 * 1024 * 1024)
|
||||
return self
|
||||
|
||||
|
||||
class VisionEmbeddingInvokeRequest(ServingModel):
|
||||
items: list[VisionEmbeddingItem] = Field(min_length=1, max_length=4)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def bounded_total(self) -> VisionEmbeddingInvokeRequest:
|
||||
encoded = sum(len(item.image_base64 or "") for item in self.items)
|
||||
if encoded > 11_184_812:
|
||||
raise ValueError("combined image payload exceeds 8 MiB")
|
||||
return self
|
||||
|
||||
|
||||
class VisionEmbeddingInvokeResponse(ServingModel):
|
||||
capability: Literal["vision.embedding@1"] = "vision.embedding@1"
|
||||
dimension: int = Field(ge=32, le=8192)
|
||||
normalized: Literal[True] = True
|
||||
embedding_space_id: uuid.UUID
|
||||
data: list[list[float]] = Field(min_length=1, max_length=4)
|
||||
request_id: uuid.UUID
|
||||
execution: GatewayExecution
|
||||
|
||||
|
||||
class SpeechTranscriptionInvokeRequest(ServingModel):
|
||||
audio_base64: str
|
||||
media_type: Literal["audio/wav"] = "audio/wav"
|
||||
language: Literal["nl", "en", "fr", "auto"] = "auto"
|
||||
|
||||
@field_validator("audio_base64")
|
||||
@classmethod
|
||||
def bounded_audio(cls, value: str) -> str:
|
||||
return _bounded_base64(value, 16 * 1024 * 1024)
|
||||
|
||||
|
||||
class TranscriptionSegment(ServingModel):
|
||||
start_seconds: float = Field(ge=0)
|
||||
end_seconds: float = Field(ge=0)
|
||||
text: str
|
||||
|
||||
|
||||
class SpeechTranscriptionInvokeResponse(ServingModel):
|
||||
capability: Literal["speech.transcription@1"] = "speech.transcription@1"
|
||||
text: str
|
||||
language: str
|
||||
duration_seconds: float = Field(gt=0, le=120)
|
||||
segments: list[TranscriptionSegment] = Field(default_factory=list)
|
||||
request_id: uuid.UUID
|
||||
execution: GatewayExecution
|
||||
|
||||
|
||||
class OpenAIEmbeddingRequest(ServingModel):
|
||||
model: Literal["rag.embedding", "rag.embedding@1"]
|
||||
input: str | list[str]
|
||||
|
||||
|
||||
class OpenAIEmbeddingItem(ServingModel):
|
||||
object: Literal["embedding"] = "embedding"
|
||||
index: int
|
||||
embedding: list[float]
|
||||
|
||||
|
||||
class OpenAIUsage(ServingModel):
|
||||
prompt_tokens: int
|
||||
total_tokens: int
|
||||
|
||||
|
||||
class OpenAIEmbeddingResponse(ServingModel):
|
||||
object: Literal["list"] = "list"
|
||||
data: list[OpenAIEmbeddingItem]
|
||||
model: Literal["rag.embedding"] = "rag.embedding"
|
||||
usage: OpenAIUsage
|
||||
|
||||
|
||||
class SchedulerBudgetResponse(ServingModel):
|
||||
compute_node_id: uuid.UUID
|
||||
node_name: str
|
||||
accelerator_id: uuid.UUID
|
||||
accelerator_uuid: str
|
||||
accelerator_name: str
|
||||
total_vram_bytes: int
|
||||
observed_used_vram_bytes: int
|
||||
external_vram_bytes: int
|
||||
resident_vram_bytes: int
|
||||
leased_vram_bytes: int
|
||||
safety_reserve_bytes: int
|
||||
schedulable_free_vram_bytes: int
|
||||
pressure: bool
|
||||
pressure_state: Literal["NORMAL", "ELEVATED", "HIGH", "CRITICAL"] = "NORMAL"
|
||||
attribution_confidence: Literal["KNOWN", "ESTIMATED", "UNKNOWN"] = "UNKNOWN"
|
||||
invariant_delta_bytes: int = 0
|
||||
policy_revision: str = "m10-v1"
|
||||
observed_at: datetime | None
|
||||
|
||||
|
||||
class PlacementPlanRequest(ServingModel):
|
||||
priority: Literal["production", "interactive", "background", "lab"] = "interactive"
|
||||
deadline_remaining_ms: float | None = Field(default=None, gt=0, le=900_000)
|
||||
|
||||
|
||||
class PlacementEvictionResponse(ServingModel):
|
||||
deployment_id: uuid.UUID
|
||||
capability: str
|
||||
expected_reclaimed_bytes: int
|
||||
reason: str
|
||||
|
||||
|
||||
class PlacementPlanResponse(ServingModel):
|
||||
id: uuid.UUID | None = None
|
||||
deployment_id: uuid.UUID
|
||||
capability: str
|
||||
node_id: uuid.UUID
|
||||
accelerator_id: uuid.UUID
|
||||
policy_revision: str
|
||||
verdict: Literal[
|
||||
"ADMIT",
|
||||
"ADMIT_AFTER_EVICTION",
|
||||
"QUEUE",
|
||||
"REJECT_CAPACITY",
|
||||
"REJECT_HEALTH",
|
||||
"REJECT_POLICY",
|
||||
]
|
||||
reason_codes: list[str]
|
||||
required_vram_bytes: int
|
||||
headroom_before_bytes: int
|
||||
headroom_after_bytes: int
|
||||
expected_cold_load_ms: float
|
||||
evictions: list[PlacementEvictionResponse] = Field(default_factory=list)
|
||||
decision_fingerprint: str
|
||||
dry_run: bool
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class SchedulerPolicyResponse(ServingModel):
|
||||
revision: str
|
||||
active: bool
|
||||
configuration: dict[str, Any]
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class SchedulerMetricsResponse(ServingModel):
|
||||
counters: dict[str, int]
|
||||
gauges: dict[str, int | str]
|
||||
queue_seconds: dict[str, float]
|
||||
|
||||
|
||||
class SchedulerPolicyUpdate(ServingModel):
|
||||
lab_paused: bool
|
||||
|
||||
|
||||
class CoResidencyEvidenceResponse(ServingModel):
|
||||
left_deployment_id: uuid.UUID
|
||||
left_capability: str
|
||||
right_deployment_id: uuid.UUID
|
||||
right_capability: str
|
||||
left_alone_bytes: int
|
||||
right_alone_bytes: int
|
||||
expected_combined_bytes: int
|
||||
measured_combined_bytes: int | None
|
||||
status: Literal["PROVEN_SAFE", "EXPECTED_SAFE", "NOT_SAFE", "UNKNOWN"]
|
||||
evidence: dict[str, Any] = Field(default_factory=dict)
|
||||
measured_at: datetime | None
|
||||
|
||||
|
||||
class CoResidencyEvidenceCreate(ServingModel):
|
||||
left_deployment_id: uuid.UUID
|
||||
right_deployment_id: uuid.UUID
|
||||
|
||||
@model_validator(mode="after")
|
||||
def distinct_deployments(self) -> CoResidencyEvidenceCreate:
|
||||
if self.left_deployment_id == self.right_deployment_id:
|
||||
raise ValueError("co-residency evidence requires two distinct deployments")
|
||||
return self
|
||||
|
||||
|
||||
class ResidencyPolicyUpdate(ServingModel):
|
||||
residency_policy: Literal["always_warm", "keep_warm", "load_on_demand", "lab_only"]
|
||||
keep_warm_seconds: int = Field(default=900, ge=5, le=86400)
|
||||
|
||||
|
||||
class GatewayRequestResponse(ServingModel):
|
||||
request_id: uuid.UUID
|
||||
capability: str
|
||||
client: str | None
|
||||
status: str
|
||||
cold: bool | None
|
||||
queue_time_ms: float | None
|
||||
load_time_ms: float | None
|
||||
inference_time_ms: float | None
|
||||
total_latency_ms: float | None
|
||||
failure_code: str | None
|
||||
latency_breakdown: dict[str, float] = Field(default_factory=dict)
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class AgentServingJobLease(ServingModel):
|
||||
job_id: uuid.UUID
|
||||
lease_token: str = Field(min_length=32, max_length=512)
|
||||
lease_expires_at: datetime
|
||||
operation: ServingOperation
|
||||
deployment_id: uuid.UUID
|
||||
artifact_set_id: uuid.UUID
|
||||
revision_sha: str = Field(pattern=r"^[a-f0-9]{40,64}$")
|
||||
artifact_root: str
|
||||
artifact_relative_path: str
|
||||
expected_manifest: dict[str, Any]
|
||||
runtime_profile: dict[str, Any]
|
||||
runtime_environment: dict[str, Any]
|
||||
capability: Literal[
|
||||
"rag.embedding",
|
||||
"rag.reranking",
|
||||
"document.ocr",
|
||||
"vision.embedding",
|
||||
"speech.transcription",
|
||||
] = "rag.embedding"
|
||||
embedding_space_id: uuid.UUID | None = None
|
||||
expected_dimension: int | None = Field(default=None, ge=32, le=8192)
|
||||
normalize: Literal[True] | None = True
|
||||
input: list[str] | None = None
|
||||
input_type: Literal["raw", "query", "document"] = "raw"
|
||||
rerank_query: str | None = None
|
||||
rerank_documents: list[RerankDocument] | None = None
|
||||
top_n: int | None = Field(default=None, ge=1, le=10)
|
||||
modality_payload: dict[str, Any] | None = None
|
||||
request_id: uuid.UUID | None = None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_capability_payload(self) -> AgentServingJobLease:
|
||||
if self.operation != "invoke":
|
||||
return self
|
||||
if self.capability == "rag.embedding":
|
||||
if self.embedding_space_id is None or self.expected_dimension is None:
|
||||
raise ValueError("embedding invoke requires embedding identity")
|
||||
if self.input is None or self.rerank_query is not None:
|
||||
raise ValueError("embedding invoke payload is invalid")
|
||||
elif self.capability == "rag.reranking" and (
|
||||
self.rerank_query is None
|
||||
or self.rerank_documents is None
|
||||
or self.top_n is None
|
||||
or self.input is not None
|
||||
):
|
||||
raise ValueError("reranking invoke payload is invalid")
|
||||
elif self.capability not in {"rag.embedding", "rag.reranking"} and (
|
||||
self.modality_payload is None or self.input is not None or self.rerank_query is not None
|
||||
):
|
||||
raise ValueError("modality invoke requires a typed modality payload")
|
||||
return self
|
||||
|
||||
|
||||
class AgentServingJobComplete(ServingModel):
|
||||
lease_token: str = Field(min_length=32, max_length=512)
|
||||
worker_instance_id: str = Field(min_length=1, max_length=255)
|
||||
state: str
|
||||
result: dict[str, Any] = Field(default_factory=dict)
|
||||
metrics: dict[str, Any] = Field(default_factory=dict)
|
||||
health: dict[str, Any] = Field(default_factory=dict)
|
||||
runtime_facts: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class AgentServingJobFailure(ServingModel):
|
||||
lease_token: str = Field(min_length=32, max_length=512)
|
||||
worker_instance_id: str = Field(min_length=1, max_length=255)
|
||||
failure_code: str = Field(min_length=1, max_length=64)
|
||||
failure_message: str = Field(min_length=1, max_length=2000)
|
||||
state: str = "failed"
|
||||
details: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class AgentResidentState(ServingModel):
|
||||
deployment_id: uuid.UUID
|
||||
artifact_set_id: uuid.UUID
|
||||
runtime_profile_id: uuid.UUID
|
||||
runtime_environment_fingerprint: str = Field(pattern=r"^[a-f0-9]{64}$")
|
||||
revision_sha: str = Field(pattern=r"^[a-f0-9]{40,64}$")
|
||||
state: Literal["loading", "warm", "busy", "draining", "unloading", "failed"]
|
||||
load_count: int = Field(ge=1)
|
||||
resident_vram_bytes: int = Field(ge=0)
|
||||
health: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class AgentServingStateReport(ServingModel):
|
||||
worker_instance_id: str = Field(min_length=1, max_length=255)
|
||||
deployment_id: uuid.UUID | None
|
||||
state: Literal["cold", "loading", "warm", "busy", "draining", "unloading", "failed"]
|
||||
load_count: int = Field(ge=0)
|
||||
health: dict[str, Any] = Field(default_factory=dict)
|
||||
observed_at: datetime
|
||||
generation: int = Field(default=1, ge=1)
|
||||
residencies: list[AgentResidentState] = Field(default_factory=list, max_length=32)
|
||||
|
||||
|
||||
class AgentServingStateAck(ServingModel):
|
||||
accepted: bool = True
|
||||
desired_operation: Literal["none", "unload"] = "none"
|
||||
@@ -0,0 +1,3 @@
|
||||
from .collectors import NvidiaNvmlCollector, SystemHostCollector, build_nvml_collector
|
||||
|
||||
__all__ = ["NvidiaNvmlCollector", "SystemHostCollector", "build_nvml_collector"]
|
||||
@@ -0,0 +1,339 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import platform
|
||||
import socket
|
||||
import sys
|
||||
import uuid
|
||||
from collections.abc import Callable
|
||||
from contextlib import suppress
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, TypeVar
|
||||
|
||||
import psutil # type: ignore[import-untyped]
|
||||
|
||||
from modelforge_api import __version__
|
||||
from modelforge_api.domain.enums import Availability
|
||||
from modelforge_api.domain.hardware import (
|
||||
AcceleratorInventory,
|
||||
AcceleratorTelemetry,
|
||||
HostInventory,
|
||||
NvidiaCollection,
|
||||
ObservedValue,
|
||||
StorageObservation,
|
||||
)
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
class NodeIdentityProvider:
|
||||
"""Stable node identity: explicit override, OS machine ID, then persisted UUID."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
identity_file: Path,
|
||||
explicit_identity: str | None = None,
|
||||
force_persisted: bool = False,
|
||||
) -> None:
|
||||
self.identity_file = identity_file
|
||||
self.explicit_identity = explicit_identity
|
||||
self.force_persisted = force_persisted
|
||||
|
||||
def resolve(self) -> tuple[str, str]:
|
||||
if self.explicit_identity:
|
||||
return self.explicit_identity, "configured"
|
||||
if not self.force_persisted:
|
||||
system_id = self._system_machine_id()
|
||||
if system_id:
|
||||
return str(uuid.uuid5(uuid.NAMESPACE_OID, system_id)), "os_machine_id"
|
||||
try:
|
||||
if self.identity_file.exists():
|
||||
return self.identity_file.read_text(encoding="utf-8").strip(), "persisted_uuid"
|
||||
value = str(uuid.uuid4())
|
||||
self.identity_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
self.identity_file.write_text(value, encoding="utf-8")
|
||||
return value, "persisted_uuid"
|
||||
except OSError as exc:
|
||||
raise RuntimeError("unable to establish stable node identity") from exc
|
||||
|
||||
@staticmethod
|
||||
def _system_machine_id() -> str | None:
|
||||
# `sys.platform`, not `os.name`: a type checker narrows on the former and not on the latter,
|
||||
# so under `os.name` this block was analysed on Linux too — where winreg has no attributes.
|
||||
# The images run on Linux, so the platform the release actually ships on was the one the
|
||||
# type check could not see. They mean the same thing at runtime.
|
||||
if sys.platform == "win32":
|
||||
try:
|
||||
import winreg
|
||||
|
||||
with winreg.OpenKey(
|
||||
winreg.HKEY_LOCAL_MACHINE,
|
||||
r"SOFTWARE\Microsoft\Cryptography",
|
||||
) as key:
|
||||
return str(winreg.QueryValueEx(key, "MachineGuid")[0])
|
||||
except (OSError, ImportError):
|
||||
return None
|
||||
for path in (Path("/etc/machine-id"), Path("/var/lib/dbus/machine-id")):
|
||||
try:
|
||||
value = path.read_text(encoding="utf-8").strip()
|
||||
if value:
|
||||
return value
|
||||
except OSError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
class SystemHostCollector:
|
||||
def __init__(
|
||||
self,
|
||||
identity_provider: NodeIdentityProvider,
|
||||
storage_paths: dict[str, Path],
|
||||
) -> None:
|
||||
self.identity_provider = identity_provider
|
||||
self.storage_paths = storage_paths
|
||||
|
||||
@staticmethod
|
||||
def _known_or_unknown(value: T | None, reason: str) -> ObservedValue[T]:
|
||||
return (
|
||||
ObservedValue.known(value)
|
||||
if value is not None
|
||||
else ObservedValue.absent(Availability.UNKNOWN, reason)
|
||||
)
|
||||
|
||||
def collect(self) -> HostInventory:
|
||||
identity_key, identity_source = self.identity_provider.resolve()
|
||||
memory = psutil.virtual_memory()
|
||||
cpu_model = platform.processor().strip() or None
|
||||
physical = psutil.cpu_count(logical=False)
|
||||
logical = psutil.cpu_count(logical=True)
|
||||
storage: list[StorageObservation] = []
|
||||
for purpose, path in self.storage_paths.items():
|
||||
try:
|
||||
usage = psutil.disk_usage(str(path))
|
||||
storage.append(
|
||||
StorageObservation(
|
||||
purpose=purpose,
|
||||
path=str(path.resolve()),
|
||||
total_bytes=ObservedValue.known(usage.total),
|
||||
used_bytes=ObservedValue.known(usage.used),
|
||||
free_bytes=ObservedValue.known(usage.free),
|
||||
)
|
||||
)
|
||||
except OSError as exc:
|
||||
reason = f"{type(exc).__name__}: path unavailable"
|
||||
unavailable = ObservedValue[int].absent(Availability.UNAVAILABLE, reason)
|
||||
storage.append(
|
||||
StorageObservation(
|
||||
purpose=purpose,
|
||||
path=str(path),
|
||||
total_bytes=unavailable,
|
||||
used_bytes=unavailable,
|
||||
free_bytes=unavailable,
|
||||
)
|
||||
)
|
||||
release = platform.release()
|
||||
environment = "wsl2" if "microsoft" in release.lower() else "native"
|
||||
return HostInventory(
|
||||
identity_key=identity_key,
|
||||
identity_source=identity_source,
|
||||
hostname=socket.gethostname(),
|
||||
display_name=socket.gethostname(),
|
||||
os_name=platform.system() or "unknown",
|
||||
os_version=self._known_or_unknown(platform.version() or None, "OS version unavailable"),
|
||||
architecture=platform.machine() or "unknown",
|
||||
kernel_version=self._known_or_unknown(release or None, "kernel unavailable"),
|
||||
cpu_model=self._known_or_unknown(cpu_model, "CPU model unavailable"),
|
||||
logical_cpu_count=self._known_or_unknown(logical, "logical CPU count unavailable"),
|
||||
physical_core_count=self._known_or_unknown(physical, "physical core count unavailable"),
|
||||
total_ram_bytes=ObservedValue.known(memory.total),
|
||||
available_ram_bytes=ObservedValue.known(memory.available),
|
||||
agent_version=__version__,
|
||||
storage=storage,
|
||||
metadata={"environment": environment},
|
||||
)
|
||||
|
||||
|
||||
class NvidiaNvmlCollector:
|
||||
def __init__(self, api: Any) -> None:
|
||||
self.api = api
|
||||
self.nvml_error = getattr(api, "NVMLError", Exception)
|
||||
self.not_supported = getattr(api, "NVMLError_NotSupported", ())
|
||||
|
||||
def _optional(
|
||||
self, call: Callable[[], T], transform: Callable[[T], Any] | None = None
|
||||
) -> ObservedValue[Any]:
|
||||
try:
|
||||
value = call()
|
||||
return ObservedValue.known(transform(value) if transform else value)
|
||||
except self.not_supported:
|
||||
return ObservedValue.absent(Availability.UNSUPPORTED, "NVML metric not supported")
|
||||
except self.nvml_error as exc:
|
||||
return ObservedValue.absent(Availability.TEMPORARILY_FAILED, type(exc).__name__)
|
||||
|
||||
@staticmethod
|
||||
def _text(value: Any) -> str:
|
||||
return value.decode("utf-8", errors="replace") if isinstance(value, bytes) else str(value)
|
||||
|
||||
@staticmethod
|
||||
def _cuda_version(value: int) -> str:
|
||||
return f"{value // 1000}.{(value % 1000) // 10}"
|
||||
|
||||
def _architecture(self, handle: Any) -> ObservedValue[str]:
|
||||
def transform(value: Any) -> str:
|
||||
for name in dir(self.api):
|
||||
if name.startswith("NVML_DEVICE_ARCH_") and getattr(self.api, name) == value:
|
||||
return name.removeprefix("NVML_DEVICE_ARCH_").lower()
|
||||
return f"nvml_arch_{value}"
|
||||
|
||||
function = getattr(self.api, "nvmlDeviceGetArchitecture", None)
|
||||
if function is None:
|
||||
return ObservedValue.absent(Availability.UNSUPPORTED, "architecture API unavailable")
|
||||
return self._optional(lambda: function(handle), transform)
|
||||
|
||||
def collect(self) -> NvidiaCollection:
|
||||
initialized = False
|
||||
try:
|
||||
self.api.nvmlInit()
|
||||
initialized = True
|
||||
count = self.api.nvmlDeviceGetCount()
|
||||
driver = self._optional(self.api.nvmlSystemGetDriverVersion, self._text)
|
||||
cuda_function = getattr(self.api, "nvmlSystemGetCudaDriverVersion_v2", None) or getattr(
|
||||
self.api, "nvmlSystemGetCudaDriverVersion", None
|
||||
)
|
||||
cuda_driver = (
|
||||
self._optional(cuda_function, self._cuda_version)
|
||||
if cuda_function
|
||||
else ObservedValue.absent(Availability.UNSUPPORTED, "CUDA driver API unavailable")
|
||||
)
|
||||
inventory: list[AcceleratorInventory] = []
|
||||
telemetry: list[AcceleratorTelemetry] = []
|
||||
failures: list[str] = []
|
||||
observed_at = datetime.now(UTC)
|
||||
for index in range(count):
|
||||
try:
|
||||
handle = self.api.nvmlDeviceGetHandleByIndex(index)
|
||||
device_uuid = self._text(self.api.nvmlDeviceGetUUID(handle))
|
||||
name = self._text(self.api.nvmlDeviceGetName(handle))
|
||||
memory = self.api.nvmlDeviceGetMemoryInfo(handle)
|
||||
pci = self._optional(
|
||||
lambda: self.api.nvmlDeviceGetPciInfo(handle),
|
||||
lambda value: self._text(value.busId),
|
||||
)
|
||||
compute = self._optional(
|
||||
lambda: self.api.nvmlDeviceGetCudaComputeCapability(handle)
|
||||
)
|
||||
major = (
|
||||
ObservedValue.known(int(compute.value[0]))
|
||||
if compute.availability is Availability.KNOWN and compute.value is not None
|
||||
else ObservedValue.absent(compute.availability, compute.reason)
|
||||
)
|
||||
minor = (
|
||||
ObservedValue.known(int(compute.value[1]))
|
||||
if compute.availability is Availability.KNOWN and compute.value is not None
|
||||
else ObservedValue.absent(compute.availability, compute.reason)
|
||||
)
|
||||
mig_function = getattr(self.api, "nvmlDeviceGetMigMode", None)
|
||||
mig = (
|
||||
self._optional(lambda: mig_function(handle), lambda value: bool(value[0]))
|
||||
if mig_function
|
||||
else ObservedValue.absent(Availability.UNSUPPORTED, "MIG API unavailable")
|
||||
)
|
||||
inventory.append(
|
||||
AcceleratorInventory(
|
||||
device_index=index,
|
||||
device_uuid=device_uuid,
|
||||
pci_bus_id=pci,
|
||||
name=name,
|
||||
architecture=self._architecture(handle),
|
||||
compute_capability_major=major,
|
||||
compute_capability_minor=minor,
|
||||
total_vram_bytes=ObservedValue.known(int(memory.total)),
|
||||
driver_version=driver,
|
||||
cuda_driver_version=cuda_driver,
|
||||
mig_mode_current=mig,
|
||||
inventory_at=observed_at,
|
||||
)
|
||||
)
|
||||
utilization = self._optional(
|
||||
lambda: self.api.nvmlDeviceGetUtilizationRates(handle)
|
||||
)
|
||||
gpu_util = (
|
||||
ObservedValue.known(int(utilization.value.gpu))
|
||||
if utilization.availability is Availability.KNOWN
|
||||
and utilization.value is not None
|
||||
else ObservedValue.absent(utilization.availability, utilization.reason)
|
||||
)
|
||||
mem_util = (
|
||||
ObservedValue.known(int(utilization.value.memory))
|
||||
if utilization.availability is Availability.KNOWN
|
||||
and utilization.value is not None
|
||||
else ObservedValue.absent(utilization.availability, utilization.reason)
|
||||
)
|
||||
telemetry.append(
|
||||
AcceleratorTelemetry(
|
||||
device_uuid=device_uuid,
|
||||
observed_at=observed_at,
|
||||
used_vram_bytes=ObservedValue.known(int(memory.used)),
|
||||
free_vram_bytes=ObservedValue.known(int(memory.free)),
|
||||
gpu_utilization_percent=gpu_util,
|
||||
memory_utilization_percent=mem_util,
|
||||
temperature_c=self._optional(
|
||||
lambda: self.api.nvmlDeviceGetTemperature(
|
||||
handle, self.api.NVML_TEMPERATURE_GPU
|
||||
),
|
||||
int,
|
||||
),
|
||||
power_draw_w=self._optional(
|
||||
lambda: self.api.nvmlDeviceGetPowerUsage(handle),
|
||||
lambda value: round(value / 1000, 3),
|
||||
),
|
||||
power_limit_w=self._optional(
|
||||
lambda: self.api.nvmlDeviceGetEnforcedPowerLimit(handle),
|
||||
lambda value: round(value / 1000, 3),
|
||||
),
|
||||
graphics_clock_mhz=self._optional(
|
||||
lambda: self.api.nvmlDeviceGetClockInfo(
|
||||
handle, self.api.NVML_CLOCK_GRAPHICS
|
||||
),
|
||||
int,
|
||||
),
|
||||
memory_clock_mhz=self._optional(
|
||||
lambda: self.api.nvmlDeviceGetClockInfo(
|
||||
handle, self.api.NVML_CLOCK_MEM
|
||||
),
|
||||
int,
|
||||
),
|
||||
fan_speed_percent=self._optional(
|
||||
lambda: self.api.nvmlDeviceGetFanSpeed(handle), int
|
||||
),
|
||||
performance_state=self._optional(
|
||||
lambda: self.api.nvmlDeviceGetPerformanceState(handle),
|
||||
lambda value: f"P{value}",
|
||||
),
|
||||
)
|
||||
)
|
||||
except self.nvml_error as exc:
|
||||
failures.append(f"device {index}: {type(exc).__name__}")
|
||||
availability = Availability.KNOWN if not failures else Availability.TEMPORARILY_FAILED
|
||||
return NvidiaCollection(
|
||||
availability=availability,
|
||||
reason="; ".join(failures)
|
||||
or ("no NVIDIA devices detected" if count == 0 else None),
|
||||
inventory=inventory,
|
||||
telemetry=telemetry,
|
||||
observed_at=observed_at,
|
||||
)
|
||||
except self.nvml_error as exc:
|
||||
return NvidiaCollection(
|
||||
availability=Availability.UNAVAILABLE, reason=type(exc).__name__
|
||||
)
|
||||
finally:
|
||||
if initialized:
|
||||
with suppress(self.nvml_error):
|
||||
self.api.nvmlShutdown()
|
||||
|
||||
|
||||
def build_nvml_collector() -> NvidiaNvmlCollector:
|
||||
import pynvml # type: ignore[import-untyped]
|
||||
|
||||
return NvidiaNvmlCollector(pynvml)
|
||||
@@ -0,0 +1,660 @@
|
||||
import asyncio
|
||||
import math
|
||||
import time
|
||||
import uuid
|
||||
from collections.abc import AsyncIterator, Awaitable, Callable, Iterator
|
||||
from contextlib import asynccontextmanager, contextmanager, suppress
|
||||
from typing import Any, cast
|
||||
|
||||
import structlog
|
||||
from fastapi import FastAPI, HTTPException, Request, Response
|
||||
from fastapi.concurrency import run_in_threadpool
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import JSONResponse
|
||||
from sqlalchemy.exc import SQLAlchemyError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from modelforge_api import __version__
|
||||
from modelforge_api.api.authorization import (
|
||||
AccessBoundary,
|
||||
access_boundary_for_request,
|
||||
authenticate_operator_token,
|
||||
request_body_limit_bytes,
|
||||
required_capability_for_request,
|
||||
)
|
||||
from modelforge_api.api.request_limits import RequestBodyLimitMiddleware
|
||||
from modelforge_api.api.routes import (
|
||||
acquisition,
|
||||
agent,
|
||||
catalog,
|
||||
evaluation,
|
||||
hardware,
|
||||
health,
|
||||
lifecycle,
|
||||
migrations,
|
||||
observability,
|
||||
recovery,
|
||||
registry,
|
||||
runtime,
|
||||
serving,
|
||||
)
|
||||
from modelforge_api.db import engine, get_session
|
||||
from modelforge_api.domain.observability import metrics, route_class
|
||||
from modelforge_api.domain.release import build_identity
|
||||
from modelforge_api.services.evaluation import EvaluationError
|
||||
from modelforge_api.services.hardware_polling import HardwarePollingService
|
||||
from modelforge_api.services.lifecycle import LifecycleError, LifecycleService
|
||||
from modelforge_api.services.manifest_registry import ManifestRegistry
|
||||
from modelforge_api.services.migration_engine import (
|
||||
MigrationEngineError,
|
||||
MigrationEngineService,
|
||||
)
|
||||
from modelforge_api.services.node_agent import (
|
||||
AgentProtocolError,
|
||||
NodeAgentService,
|
||||
NodeAuthenticationEvidence,
|
||||
)
|
||||
from modelforge_api.services.node_decommission import NodeDecommissionError
|
||||
from modelforge_api.services.node_liveness import NodeLivenessPollingService
|
||||
from modelforge_api.services.observability import ObservabilityError, ObservabilityService
|
||||
from modelforge_api.services.observability_polling import ObservabilityPollingService
|
||||
from modelforge_api.services.project_registry import sync_project_registry
|
||||
from modelforge_api.services.recovery import RecoveryError, RecoveryService
|
||||
from modelforge_api.services.registry import RegistryError, seed_candidate_registry
|
||||
from modelforge_api.services.serving import (
|
||||
CapabilityAuthenticationEvidence,
|
||||
ServingError,
|
||||
ServingService,
|
||||
)
|
||||
from modelforge_api.services.serving_reconciliation import ServingReconciliationService
|
||||
from modelforge_api.services.startup_validation import enforce_startup
|
||||
from modelforge_api.settings import Settings, get_settings
|
||||
|
||||
settings = get_settings()
|
||||
structlog.configure(processors=[structlog.processors.JSONRenderer()])
|
||||
logger = structlog.get_logger()
|
||||
|
||||
|
||||
def interactive_api_enabled_for(environment: str) -> bool:
|
||||
"""Keep schema explorers off deployed/test surfaces unless development is explicit."""
|
||||
|
||||
return environment == "development"
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(_app: FastAPI) -> AsyncIterator[None]:
|
||||
# Configuration and schema compatibility are checked before anything else runs. In production a
|
||||
# problem here stops the process; elsewhere it is logged, because finding out at startup rather
|
||||
# than at first use is the point either way.
|
||||
identity = build_identity(
|
||||
source_commit=settings.build_commit,
|
||||
built_at=settings.build_timestamp,
|
||||
image_digest=settings.build_image_digest,
|
||||
)
|
||||
logger.info("modelforge_starting", **identity.as_dict(), environment=settings.env)
|
||||
validation = enforce_startup(settings, engine)
|
||||
for problem in validation.problems:
|
||||
logger.error(
|
||||
"startup_configuration_problem",
|
||||
code=str(problem.code),
|
||||
setting=problem.setting,
|
||||
detail=problem.message,
|
||||
)
|
||||
for problem in validation.warnings:
|
||||
logger.warning(
|
||||
"startup_configuration_warning",
|
||||
code=str(problem.code),
|
||||
setting=problem.setting,
|
||||
detail=problem.message,
|
||||
)
|
||||
|
||||
if settings.lifecycle_reconciliation_enabled:
|
||||
with Session(engine) as session:
|
||||
lifecycle_service = LifecycleService(session)
|
||||
lifecycle_service.ensure_defaults()
|
||||
reconciled = lifecycle_service.reconcile_incomplete()
|
||||
if reconciled:
|
||||
logger.warning("lifecycle_operations_reconciled", count=reconciled)
|
||||
if settings.migration_reconciliation_enabled:
|
||||
with Session(engine) as session:
|
||||
migration_service = MigrationEngineService(session)
|
||||
migration_service.ensure_defaults()
|
||||
pending_migrations = migration_service.pending_reconciliation_count()
|
||||
if pending_migrations:
|
||||
# External state is deliberately never guessed during startup. An operator or
|
||||
# adapter must report observed truth through the reconciliation endpoint.
|
||||
logger.warning(
|
||||
"migration_cutovers_require_reconciliation", count=pending_migrations
|
||||
)
|
||||
if settings.recovery_reconciliation_enabled:
|
||||
with Session(engine) as session:
|
||||
recovery_service = RecoveryService(session, settings, "control_plane", "startup")
|
||||
recovery_service.ensure_defaults()
|
||||
interrupted_backups = recovery_service.reconcile_interrupted_backups()
|
||||
interrupted_restores = recovery_service.reconcile_interrupted_restores()
|
||||
if interrupted_backups or interrupted_restores:
|
||||
# An interrupted backup is never restore eligible and an interrupted restore is
|
||||
# never silently resumed; both surface for explicit operator review.
|
||||
logger.warning(
|
||||
"recovery_operations_reconciled",
|
||||
interrupted_backups=interrupted_backups,
|
||||
interrupted_restores=interrupted_restores,
|
||||
)
|
||||
if settings.registry_seed_on_startup:
|
||||
with Session(engine) as session:
|
||||
manifests = ManifestRegistry(settings.config_root)
|
||||
seed_candidate_registry(session, manifests)
|
||||
sync_result = sync_project_registry(session, manifests)
|
||||
if sync_result.unavailable_contracts:
|
||||
logger.warning(
|
||||
"project_bindings_waiting_for_contracts",
|
||||
bindings=sync_result.unavailable_contracts,
|
||||
)
|
||||
poller = HardwarePollingService(settings)
|
||||
hardware_task = (
|
||||
asyncio.create_task(poller.run()) if settings.hardware_refresh_on_startup else None
|
||||
)
|
||||
liveness = NodeLivenessPollingService(settings)
|
||||
liveness_task = (
|
||||
asyncio.create_task(liveness.run()) if settings.node_liveness_monitor_enabled else None
|
||||
)
|
||||
serving_reconciliation = ServingReconciliationService(settings, engine)
|
||||
serving_task = (
|
||||
asyncio.create_task(serving_reconciliation.run())
|
||||
if settings.serving_reconciliation_enabled
|
||||
else None
|
||||
)
|
||||
observability_poller = ObservabilityPollingService(settings, engine)
|
||||
if settings.observability_monitor_enabled:
|
||||
with Session(engine) as session:
|
||||
ObservabilityService(session, settings).ensure_defaults()
|
||||
observability_task = (
|
||||
asyncio.create_task(observability_poller.run())
|
||||
if settings.observability_monitor_enabled
|
||||
else None
|
||||
)
|
||||
yield
|
||||
if observability_task:
|
||||
observability_poller.stop()
|
||||
await observability_task
|
||||
if serving_task:
|
||||
serving_reconciliation.stop()
|
||||
await serving_task
|
||||
if liveness_task:
|
||||
liveness.stop()
|
||||
await liveness_task
|
||||
if hardware_task:
|
||||
poller.stop()
|
||||
await hardware_task
|
||||
|
||||
|
||||
interactive_api_enabled = interactive_api_enabled_for(settings.env)
|
||||
app = FastAPI(
|
||||
title="ITWorx ModelForge API",
|
||||
version=__version__,
|
||||
description="Local AI ModelOps and GPU control-plane API",
|
||||
lifespan=lifespan,
|
||||
docs_url="/docs" if interactive_api_enabled else None,
|
||||
redoc_url="/redoc" if interactive_api_enabled else None,
|
||||
openapi_url="/openapi.json" if interactive_api_enabled else None,
|
||||
)
|
||||
app.include_router(health.router)
|
||||
app.include_router(lifecycle.router)
|
||||
app.include_router(migrations.router)
|
||||
app.include_router(observability.router)
|
||||
app.include_router(recovery.router)
|
||||
app.include_router(agent.router)
|
||||
app.include_router(hardware.router)
|
||||
app.include_router(catalog.router)
|
||||
app.include_router(registry.router)
|
||||
app.include_router(acquisition.router)
|
||||
app.include_router(runtime.router)
|
||||
app.include_router(serving.router)
|
||||
app.include_router(evaluation.router)
|
||||
|
||||
|
||||
@app.exception_handler(LifecycleError)
|
||||
async def lifecycle_error(request: Request, exc: LifecycleError) -> JSONResponse:
|
||||
correlation_id = getattr(request.state, "correlation_id", "unknown")
|
||||
return JSONResponse(
|
||||
status_code=exc.status_code,
|
||||
content={
|
||||
"error": {
|
||||
"code": exc.code,
|
||||
"message": exc.message,
|
||||
"correlation_id": correlation_id,
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@app.exception_handler(MigrationEngineError)
|
||||
async def migration_engine_error(request: Request, exc: MigrationEngineError) -> JSONResponse:
|
||||
correlation_id = getattr(request.state, "correlation_id", "unknown")
|
||||
return JSONResponse(
|
||||
status_code=exc.status_code,
|
||||
content={
|
||||
"error": {
|
||||
"code": exc.code,
|
||||
"message": exc.message,
|
||||
"correlation_id": correlation_id,
|
||||
"details": {},
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@app.exception_handler(RecoveryError)
|
||||
async def recovery_error(request: Request, exc: RecoveryError) -> JSONResponse:
|
||||
correlation_id = getattr(request.state, "correlation_id", "unknown")
|
||||
return JSONResponse(
|
||||
status_code=exc.status_code,
|
||||
content={
|
||||
"error": {
|
||||
"code": exc.code,
|
||||
"message": exc.message,
|
||||
"correlation_id": correlation_id,
|
||||
"details": exc.details,
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@app.exception_handler(ObservabilityError)
|
||||
async def observability_error(request: Request, exc: ObservabilityError) -> JSONResponse:
|
||||
correlation_id = getattr(request.state, "correlation_id", "unknown")
|
||||
return JSONResponse(
|
||||
status_code=exc.status_code,
|
||||
content={
|
||||
"error": {
|
||||
"code": exc.code,
|
||||
"message": exc.message,
|
||||
"correlation_id": correlation_id,
|
||||
"details": {},
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _authentication_session(application: FastAPI) -> Iterator[Session]:
|
||||
"""Use a test/application session override when present, otherwise a short owned session."""
|
||||
|
||||
override = application.dependency_overrides.get(get_session)
|
||||
if override is None:
|
||||
with Session(engine) as session:
|
||||
yield session
|
||||
return
|
||||
|
||||
provided = override()
|
||||
if isinstance(provided, Session):
|
||||
yield provided
|
||||
return
|
||||
iterator = iter(cast(Any, provided))
|
||||
session = next(iterator)
|
||||
if not isinstance(session, Session):
|
||||
raise TypeError("get_session override did not provide a SQLAlchemy Session")
|
||||
try:
|
||||
yield session
|
||||
finally:
|
||||
with suppress(StopIteration):
|
||||
next(iterator)
|
||||
|
||||
|
||||
def _authenticate_node_before_body(
|
||||
application: FastAPI,
|
||||
request_settings: Settings,
|
||||
authorization: str | None,
|
||||
) -> NodeAuthenticationEvidence:
|
||||
service_override = application.dependency_overrides.get(agent.get_agent_service)
|
||||
if service_override is not None:
|
||||
service = service_override()
|
||||
if not isinstance(service, NodeAgentService):
|
||||
raise TypeError("node authentication service override has an invalid type")
|
||||
_identity, evidence = service.authenticate_with_evidence(authorization)
|
||||
service.session.commit()
|
||||
return evidence
|
||||
|
||||
with _authentication_session(application) as session:
|
||||
service = NodeAgentService(session, request_settings)
|
||||
_identity, evidence = service.authenticate_with_evidence(authorization)
|
||||
session.commit()
|
||||
return evidence
|
||||
|
||||
|
||||
def _authenticate_capability_before_body(
|
||||
application: FastAPI,
|
||||
request_settings: Settings,
|
||||
authorization: str | None,
|
||||
capability: str,
|
||||
) -> CapabilityAuthenticationEvidence:
|
||||
service_override = application.dependency_overrides.get(serving.get_serving_service)
|
||||
if service_override is not None:
|
||||
service = service_override()
|
||||
if not isinstance(service, ServingService):
|
||||
raise TypeError("capability authentication service override has an invalid type")
|
||||
_client, evidence = service.authenticate_with_evidence(authorization, capability)
|
||||
return evidence
|
||||
|
||||
with _authentication_session(application) as session:
|
||||
service = ServingService(
|
||||
session,
|
||||
request_settings,
|
||||
ManifestRegistry(request_settings.config_root),
|
||||
None,
|
||||
)
|
||||
_client, evidence = service.authenticate_with_evidence(authorization, capability)
|
||||
return evidence
|
||||
|
||||
|
||||
def _authentication_error(
|
||||
status_code: int,
|
||||
code: str,
|
||||
message: str,
|
||||
correlation_id: str,
|
||||
details: dict[str, Any] | None = None,
|
||||
) -> JSONResponse:
|
||||
return JSONResponse(
|
||||
status_code=status_code,
|
||||
content={
|
||||
"error": {
|
||||
"code": code,
|
||||
"message": message,
|
||||
"correlation_id": correlation_id,
|
||||
"details": details or {},
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
async def correlation_middleware(
|
||||
request: Request, call_next: Callable[[Request], Awaitable[Response]]
|
||||
) -> Response:
|
||||
started = time.perf_counter()
|
||||
correlation_id = request.headers.get("x-correlation-id") or str(uuid.uuid4())
|
||||
request.state.correlation_id = correlation_id
|
||||
boundary = access_boundary_for_request(
|
||||
request.method,
|
||||
request.url.path,
|
||||
)
|
||||
settings_provider = cast(
|
||||
Callable[[], Settings],
|
||||
request.app.dependency_overrides.get(get_settings, get_settings),
|
||||
)
|
||||
request_settings = settings_provider()
|
||||
request.state.request_body_limit_bytes = request_body_limit_bytes(
|
||||
request_settings,
|
||||
boundary,
|
||||
)
|
||||
response: Response | None = None
|
||||
if boundary is AccessBoundary.CONTROL_PLANE:
|
||||
try:
|
||||
request.state.principal = authenticate_operator_token(
|
||||
request_settings,
|
||||
request.headers.get("x-modelforge-admin-token"),
|
||||
)
|
||||
except HTTPException as exc:
|
||||
response = _authentication_error(
|
||||
exc.status_code,
|
||||
f"http_{exc.status_code}",
|
||||
str(exc.detail),
|
||||
correlation_id,
|
||||
)
|
||||
elif boundary is AccessBoundary.NODE:
|
||||
try:
|
||||
request.state.node_authentication = await run_in_threadpool(
|
||||
_authenticate_node_before_body,
|
||||
request.app,
|
||||
request_settings,
|
||||
request.headers.get("authorization"),
|
||||
)
|
||||
except AgentProtocolError as exc:
|
||||
response = _authentication_error(
|
||||
exc.status_code,
|
||||
exc.code,
|
||||
str(exc),
|
||||
correlation_id,
|
||||
)
|
||||
except SQLAlchemyError:
|
||||
response = _authentication_error(
|
||||
503,
|
||||
"machine_authentication_unavailable",
|
||||
"node authentication backend is unavailable",
|
||||
correlation_id,
|
||||
)
|
||||
elif boundary is AccessBoundary.CAPABILITY_CLIENT:
|
||||
capability = required_capability_for_request(request.method, request.url.path)
|
||||
if capability is None:
|
||||
response = _authentication_error(
|
||||
503,
|
||||
"machine_authentication_unavailable",
|
||||
"capability authentication policy is unavailable",
|
||||
correlation_id,
|
||||
)
|
||||
else:
|
||||
try:
|
||||
request.state.capability_authentication = await run_in_threadpool(
|
||||
_authenticate_capability_before_body,
|
||||
request.app,
|
||||
request_settings,
|
||||
request.headers.get("authorization"),
|
||||
capability,
|
||||
)
|
||||
except ServingError as exc:
|
||||
response = _authentication_error(
|
||||
exc.status_code,
|
||||
exc.code,
|
||||
str(exc),
|
||||
correlation_id,
|
||||
exc.details,
|
||||
)
|
||||
except SQLAlchemyError:
|
||||
response = _authentication_error(
|
||||
503,
|
||||
"machine_authentication_unavailable",
|
||||
"capability authentication backend is unavailable",
|
||||
correlation_id,
|
||||
)
|
||||
if response is None:
|
||||
response = await call_next(request)
|
||||
duration_seconds = time.perf_counter() - started
|
||||
response.headers["x-correlation-id"] = correlation_id
|
||||
body_error_status_code = getattr(
|
||||
request.state,
|
||||
"request_body_error_status_code",
|
||||
None,
|
||||
)
|
||||
effective_status_code = (
|
||||
body_error_status_code
|
||||
if isinstance(body_error_status_code, int)
|
||||
else response.status_code
|
||||
)
|
||||
labels = {
|
||||
"method": request.method,
|
||||
"route_class": route_class(request.url.path),
|
||||
}
|
||||
metrics.increment(
|
||||
"modelforge_api_requests_total",
|
||||
{**labels, "status_class": f"{effective_status_code // 100}xx"},
|
||||
)
|
||||
metrics.observe("modelforge_api_request_duration_seconds", labels, duration_seconds)
|
||||
logger.info(
|
||||
"http_request",
|
||||
correlation_id=correlation_id,
|
||||
method=request.method,
|
||||
path=request.url.path,
|
||||
status_code=effective_status_code,
|
||||
duration_ms=duration_seconds * 1000,
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
# Starlette wraps user middleware in reverse registration order. Register CORS after the
|
||||
# request limiter and correlation/authentication middleware so it remains outermost and decorates
|
||||
# pre-body errors as well as responses produced by routing and exception handlers. The pure ASGI
|
||||
# limiter stays outside BaseHTTPMiddleware so empty request frames cannot be normalized away before
|
||||
# its progress counters observe them; authentication still runs before the first wrapped receive.
|
||||
app.add_middleware(RequestBodyLimitMiddleware)
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=settings.cors_origin_list,
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
|
||||
MAX_ECHOED_INPUT_CHARACTERS = 200
|
||||
|
||||
|
||||
def _renderable(value: object) -> object:
|
||||
"""Render a rejected value so the error response can always be serialised.
|
||||
|
||||
A body may legally contain values JSON cannot round-trip — `NaN` and `Infinity` are accepted by
|
||||
Python's parser but rejected by the serialiser — and echoing one raw made the error handler
|
||||
itself fail, turning a 422 into a server error. Rejected input is also unbounded by definition,
|
||||
so it is truncated rather than mirrored back in full.
|
||||
"""
|
||||
|
||||
if isinstance(value, float) and not math.isfinite(value):
|
||||
return repr(value)
|
||||
if isinstance(value, str):
|
||||
return value[:MAX_ECHOED_INPUT_CHARACTERS]
|
||||
if isinstance(value, bytes):
|
||||
return value[:MAX_ECHOED_INPUT_CHARACTERS].decode("utf-8", "replace")
|
||||
if isinstance(value, list | tuple):
|
||||
return [_renderable(item) for item in value[:20]]
|
||||
if isinstance(value, dict):
|
||||
return {str(key): _renderable(item) for key, item in list(value.items())[:20]}
|
||||
if isinstance(value, bool | int | float | type(None)):
|
||||
return value
|
||||
return repr(value)[:MAX_ECHOED_INPUT_CHARACTERS]
|
||||
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def validation_error(request: Request, exc: RequestValidationError) -> JSONResponse:
|
||||
correlation_id = getattr(request.state, "correlation_id", "unknown")
|
||||
return JSONResponse(
|
||||
status_code=422,
|
||||
content={
|
||||
"error": {
|
||||
"code": "request_validation_failed",
|
||||
"message": "Request validation failed",
|
||||
"correlation_id": correlation_id,
|
||||
"details": {
|
||||
"errors": [
|
||||
{key: _renderable(value) for key, value in error.items() if key != "ctx"}
|
||||
for error in exc.errors()
|
||||
]
|
||||
},
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@app.exception_handler(HTTPException)
|
||||
async def http_error(request: Request, exc: HTTPException) -> JSONResponse:
|
||||
correlation_id = getattr(request.state, "correlation_id", "unknown")
|
||||
return JSONResponse(
|
||||
status_code=exc.status_code,
|
||||
content={
|
||||
"error": {
|
||||
"code": f"http_{exc.status_code}",
|
||||
"message": str(exc.detail),
|
||||
"correlation_id": correlation_id,
|
||||
"details": {},
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@app.exception_handler(AgentProtocolError)
|
||||
async def agent_protocol_error(request: Request, exc: AgentProtocolError) -> JSONResponse:
|
||||
correlation_id = getattr(request.state, "correlation_id", "unknown")
|
||||
return JSONResponse(
|
||||
status_code=exc.status_code,
|
||||
content={
|
||||
"error": {
|
||||
"code": exc.code,
|
||||
"message": str(exc),
|
||||
"correlation_id": correlation_id,
|
||||
"details": {},
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@app.exception_handler(NodeDecommissionError)
|
||||
async def node_decommission_error(request: Request, exc: NodeDecommissionError) -> JSONResponse:
|
||||
correlation_id = getattr(request.state, "correlation_id", "unknown")
|
||||
return JSONResponse(
|
||||
status_code=exc.status_code,
|
||||
content={
|
||||
"error": {
|
||||
"code": exc.code,
|
||||
"message": exc.message,
|
||||
"correlation_id": correlation_id,
|
||||
"details": exc.details,
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@app.exception_handler(RegistryError)
|
||||
async def registry_error(request: Request, exc: RegistryError) -> JSONResponse:
|
||||
correlation_id = getattr(request.state, "correlation_id", "unknown")
|
||||
return JSONResponse(
|
||||
status_code=exc.status_code,
|
||||
content={
|
||||
"error": {
|
||||
"code": exc.code,
|
||||
"message": str(exc),
|
||||
"correlation_id": correlation_id,
|
||||
"details": exc.details,
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@app.exception_handler(ServingError)
|
||||
async def serving_error(request: Request, exc: ServingError) -> JSONResponse:
|
||||
correlation_id = getattr(request.state, "correlation_id", "unknown")
|
||||
return JSONResponse(
|
||||
status_code=exc.status_code,
|
||||
content={
|
||||
"error": {
|
||||
"code": exc.code,
|
||||
"message": str(exc),
|
||||
"correlation_id": correlation_id,
|
||||
"details": exc.details,
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@app.exception_handler(EvaluationError)
|
||||
async def evaluation_error(request: Request, exc: EvaluationError) -> JSONResponse:
|
||||
correlation_id = getattr(request.state, "correlation_id", "unknown")
|
||||
return JSONResponse(
|
||||
status_code=exc.status_code,
|
||||
content={
|
||||
"error": {
|
||||
"code": exc.code,
|
||||
"message": str(exc),
|
||||
"correlation_id": correlation_id,
|
||||
"details": exc.details,
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@app.get("/")
|
||||
def root() -> dict[str, str]:
|
||||
return {
|
||||
"name": "ITWorx ModelForge",
|
||||
"version": __version__,
|
||||
"docs": "/docs" if interactive_api_enabled else "disabled",
|
||||
"api": "/api/v1",
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
from .models import Base
|
||||
|
||||
__all__ = ["Base"]
|
||||
@@ -0,0 +1,123 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from typing import TypeVar
|
||||
|
||||
from sqlalchemy import or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from modelforge_api.persistence.models import (
|
||||
ArtifactInspection,
|
||||
ArtifactJob,
|
||||
ArtifactSet,
|
||||
DownloadPlan,
|
||||
DownloadPlanFile,
|
||||
UpstreamFile,
|
||||
UpstreamSnapshot,
|
||||
)
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
class AcquisitionRepository:
|
||||
def __init__(self, session: Session) -> None:
|
||||
self.session = session
|
||||
|
||||
def add(self, entity: T) -> T:
|
||||
self.session.add(entity)
|
||||
self.session.flush()
|
||||
return entity
|
||||
|
||||
def latest_snapshot(self, model_id: uuid.UUID) -> UpstreamSnapshot | None:
|
||||
return self.session.scalar(
|
||||
select(UpstreamSnapshot)
|
||||
.where(UpstreamSnapshot.model_id == model_id)
|
||||
.order_by(UpstreamSnapshot.observed_at.desc(), UpstreamSnapshot.id.desc())
|
||||
.limit(1)
|
||||
)
|
||||
|
||||
def snapshot(self, snapshot_id: uuid.UUID) -> UpstreamSnapshot | None:
|
||||
return self.session.get(UpstreamSnapshot, snapshot_id)
|
||||
|
||||
def files(self, snapshot_id: uuid.UUID) -> list[UpstreamFile]:
|
||||
return list(
|
||||
self.session.scalars(
|
||||
select(UpstreamFile)
|
||||
.where(UpstreamFile.snapshot_id == snapshot_id)
|
||||
.order_by(UpstreamFile.path)
|
||||
)
|
||||
)
|
||||
|
||||
def artifact_set(self, artifact_set_id: uuid.UUID) -> ArtifactSet | None:
|
||||
return self.session.get(ArtifactSet, artifact_set_id)
|
||||
|
||||
def artifact_set_for_variant(self, revision_id: uuid.UUID, variant: str) -> ArtifactSet | None:
|
||||
return self.session.scalar(
|
||||
select(ArtifactSet).where(
|
||||
ArtifactSet.revision_id == revision_id, ArtifactSet.variant_key == variant
|
||||
)
|
||||
)
|
||||
|
||||
def artifact_sets(self, revision_id: uuid.UUID) -> list[ArtifactSet]:
|
||||
return list(
|
||||
self.session.scalars(
|
||||
select(ArtifactSet)
|
||||
.where(ArtifactSet.revision_id == revision_id)
|
||||
.order_by(ArtifactSet.variant_key)
|
||||
)
|
||||
)
|
||||
|
||||
def plan(self, plan_id: uuid.UUID) -> DownloadPlan | None:
|
||||
return self.session.get(DownloadPlan, plan_id)
|
||||
|
||||
def plan_by_key(self, key: str) -> DownloadPlan | None:
|
||||
return self.session.scalar(select(DownloadPlan).where(DownloadPlan.idempotency_key == key))
|
||||
|
||||
def plan_files(self, plan_id: uuid.UUID) -> list[DownloadPlanFile]:
|
||||
return list(
|
||||
self.session.scalars(
|
||||
select(DownloadPlanFile)
|
||||
.where(DownloadPlanFile.plan_id == plan_id)
|
||||
.order_by(DownloadPlanFile.ordinal)
|
||||
)
|
||||
)
|
||||
|
||||
def job(self, job_id: uuid.UUID) -> ArtifactJob | None:
|
||||
return self.session.get(ArtifactJob, job_id)
|
||||
|
||||
def job_for_plan(self, plan_id: uuid.UUID) -> ArtifactJob | None:
|
||||
return self.session.scalar(select(ArtifactJob).where(ArtifactJob.plan_id == plan_id))
|
||||
|
||||
def jobs(self, limit: int = 100) -> list[ArtifactJob]:
|
||||
return list(
|
||||
self.session.scalars(
|
||||
select(ArtifactJob).order_by(ArtifactJob.created_at.desc()).limit(limit)
|
||||
)
|
||||
)
|
||||
|
||||
def claimable_job(self, node_id: uuid.UUID, now: datetime) -> ArtifactJob | None:
|
||||
return self.session.scalar(
|
||||
select(ArtifactJob)
|
||||
.where(
|
||||
ArtifactJob.compute_node_id == node_id,
|
||||
ArtifactJob.cancel_requested.is_(False),
|
||||
or_(
|
||||
ArtifactJob.status == "queued",
|
||||
(ArtifactJob.status.in_(("claimed", "downloading", "verifying", "promoting")))
|
||||
& (ArtifactJob.lease_expires_at < now),
|
||||
),
|
||||
)
|
||||
.order_by(ArtifactJob.created_at)
|
||||
.with_for_update(skip_locked=True)
|
||||
.limit(1)
|
||||
)
|
||||
|
||||
def inspections(self, job_id: uuid.UUID) -> list[ArtifactInspection]:
|
||||
return list(
|
||||
self.session.scalars(
|
||||
select(ArtifactInspection)
|
||||
.where(ArtifactInspection.job_id == job_id)
|
||||
.order_by(ArtifactInspection.file_path, ArtifactInspection.inspection_type)
|
||||
)
|
||||
)
|
||||
@@ -0,0 +1,37 @@
|
||||
"""Immutable names for the PostgreSQL audit privilege boundary.
|
||||
|
||||
Migration 0024 owns the DDL. Runtime code keeps only the callable contract and catalog policy
|
||||
names here; it never receives the owner/migration credential or an arbitrary privileged SQL path.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
AUDIT_OWNER_ROLE = "modelforge"
|
||||
AUDIT_RUNTIME_ROLE = "modelforge_runtime"
|
||||
AUDIT_FUNCTION_SCHEMA = "modelforge_audit"
|
||||
AUDIT_APPEND_FUNCTION = "append_event_v2"
|
||||
AUDIT_MUTATION_TRIGGER_FUNCTION = "enforce_owner_mutation"
|
||||
AUDIT_EVENT_MUTATION_TRIGGER = "trg_modelforge_audit_events_owner"
|
||||
AUDIT_EVENT_TRUNCATE_TRIGGER = "trg_modelforge_audit_events_truncate_owner"
|
||||
AUDIT_HEAD_MUTATION_TRIGGER = "trg_modelforge_audit_head_owner"
|
||||
AUDIT_HEAD_TRUNCATE_TRIGGER = "trg_modelforge_audit_head_truncate_owner"
|
||||
AUDIT_FUNCTION_SEARCH_PATH = "pg_catalog"
|
||||
|
||||
# ``pg_proc.prosrc`` hashes for the exact migration-0024 function bodies. Production startup
|
||||
# attests these as well as owner/grants/search_path: a same-signature replacement is not the
|
||||
# canonical boundary. The static migration test derives both values from its immutable SQL so a
|
||||
# body edit cannot silently drift this copy.
|
||||
AUDIT_APPEND_BODY_SHA256 = "c9154911f1b1b70f77fe2642de156c98358cd0423f0c3e2a75bc4f1ee4409a86"
|
||||
AUDIT_GUARD_BODY_SHA256 = "8af5fc31f3b2c729445baaae902eccd9cb3d70b257d0a19e6399f538ee43c6bf"
|
||||
|
||||
POSTGRES_APPEND_AUDIT_SQL = """
|
||||
select event_id, sequence, event_hash, occurred_at
|
||||
from modelforge_audit.append_event_v2(
|
||||
cast(:event_id as uuid), cast(:occurred_at as timestamptz),
|
||||
:correlation_id, :actor_type, :actor_id, :action, :resource_type,
|
||||
:resource_id, :outcome, cast(:details_json as jsonb),
|
||||
:expected_event_count, :expected_last_sequence, :expected_last_event_hash,
|
||||
:expected_hash_format, :expected_v2_start_sequence,
|
||||
:expected_legacy_prefix_count, :expected_legacy_prefix_seal
|
||||
)
|
||||
""".strip()
|
||||
@@ -0,0 +1,126 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from typing import TypeVar
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from modelforge_api.persistence.models import (
|
||||
Accelerator,
|
||||
AcceleratorTelemetryLatest,
|
||||
ComputeNode,
|
||||
HardwareInventoryRun,
|
||||
HostTelemetryLatest,
|
||||
NodeCredential,
|
||||
NodeEnrollment,
|
||||
StorageVolumeState,
|
||||
)
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
class HardwareRepository:
|
||||
def __init__(self, session: Session) -> None:
|
||||
self.session = session
|
||||
|
||||
def node_by_key(self, key: str) -> ComputeNode | None:
|
||||
return self.session.scalar(select(ComputeNode).where(ComputeNode.key == key))
|
||||
|
||||
def node_by_key_for_update(self, key: str) -> ComputeNode | None:
|
||||
return self.session.scalar(
|
||||
select(ComputeNode).where(ComputeNode.key == key).with_for_update()
|
||||
)
|
||||
|
||||
def nodes(self) -> list[ComputeNode]:
|
||||
return list(self.session.scalars(select(ComputeNode).order_by(ComputeNode.hostname)))
|
||||
|
||||
def node(self, node_id: uuid.UUID) -> ComputeNode | None:
|
||||
return self.session.get(ComputeNode, node_id)
|
||||
|
||||
def node_for_update(self, node_id: uuid.UUID) -> ComputeNode | None:
|
||||
return self.session.scalar(
|
||||
select(ComputeNode).where(ComputeNode.id == node_id).with_for_update()
|
||||
)
|
||||
|
||||
def accelerators(self, node_id: uuid.UUID | None = None) -> list[Accelerator]:
|
||||
query = select(Accelerator).order_by(Accelerator.device_index)
|
||||
if node_id is not None:
|
||||
query = query.where(Accelerator.compute_node_id == node_id)
|
||||
return list(self.session.scalars(query))
|
||||
|
||||
def accelerator(self, accelerator_id: uuid.UUID) -> Accelerator | None:
|
||||
return self.session.get(Accelerator, accelerator_id)
|
||||
|
||||
def accelerator_by_uuid(self, node_id: uuid.UUID, device_uuid: str) -> Accelerator | None:
|
||||
return self.session.scalar(
|
||||
select(Accelerator).where(
|
||||
Accelerator.compute_node_id == node_id, Accelerator.device_uuid == device_uuid
|
||||
)
|
||||
)
|
||||
|
||||
def host_telemetry(self, node_id: uuid.UUID) -> HostTelemetryLatest | None:
|
||||
return self.session.scalar(
|
||||
select(HostTelemetryLatest).where(HostTelemetryLatest.compute_node_id == node_id)
|
||||
)
|
||||
|
||||
def accelerator_telemetry(self, accelerator_id: uuid.UUID) -> AcceleratorTelemetryLatest | None:
|
||||
return self.session.scalar(
|
||||
select(AcceleratorTelemetryLatest).where(
|
||||
AcceleratorTelemetryLatest.accelerator_id == accelerator_id
|
||||
)
|
||||
)
|
||||
|
||||
def storage(self, node_id: uuid.UUID) -> list[StorageVolumeState]:
|
||||
return list(
|
||||
self.session.scalars(
|
||||
select(StorageVolumeState)
|
||||
.where(StorageVolumeState.compute_node_id == node_id)
|
||||
.order_by(StorageVolumeState.purpose)
|
||||
)
|
||||
)
|
||||
|
||||
def storage_by_key(
|
||||
self, node_id: uuid.UUID, purpose: str, path: str
|
||||
) -> StorageVolumeState | None:
|
||||
return self.session.scalar(
|
||||
select(StorageVolumeState).where(
|
||||
StorageVolumeState.compute_node_id == node_id,
|
||||
StorageVolumeState.purpose == purpose,
|
||||
StorageVolumeState.path == path,
|
||||
)
|
||||
)
|
||||
|
||||
def latest_run(self) -> HardwareInventoryRun | None:
|
||||
return self.session.scalar(
|
||||
select(HardwareInventoryRun).order_by(HardwareInventoryRun.started_at.desc()).limit(1)
|
||||
)
|
||||
|
||||
def enrollment(self, enrollment_id: uuid.UUID) -> NodeEnrollment | None:
|
||||
return self.session.get(NodeEnrollment, enrollment_id)
|
||||
|
||||
def enrollment_by_hash(self, token_hash: str) -> NodeEnrollment | None:
|
||||
return self.session.scalar(
|
||||
select(NodeEnrollment).where(NodeEnrollment.token_hash == token_hash)
|
||||
)
|
||||
|
||||
def enrollments(self) -> list[NodeEnrollment]:
|
||||
return list(
|
||||
self.session.scalars(select(NodeEnrollment).order_by(NodeEnrollment.created_at.desc()))
|
||||
)
|
||||
|
||||
def credential(self, credential_id: uuid.UUID) -> NodeCredential | None:
|
||||
return self.session.get(NodeCredential, credential_id)
|
||||
|
||||
def active_credential_for_node(self, node_id: uuid.UUID) -> NodeCredential | None:
|
||||
return self.session.scalar(
|
||||
select(NodeCredential).where(
|
||||
NodeCredential.compute_node_id == node_id,
|
||||
NodeCredential.revoked_at.is_(None),
|
||||
)
|
||||
)
|
||||
|
||||
def add(self, entity: T) -> T:
|
||||
self.session.add(entity)
|
||||
self.session.flush()
|
||||
return entity
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,288 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from typing import Any, TypeVar
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session, selectinload
|
||||
|
||||
from modelforge_api.persistence.models import (
|
||||
ArtifactLocation,
|
||||
DerivedArtifact,
|
||||
DerivedArtifactSource,
|
||||
Model,
|
||||
ModelArtifact,
|
||||
ModelRevision,
|
||||
RuntimeProfile,
|
||||
StorageRoot,
|
||||
)
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
class RegistryRepository:
|
||||
"""Persistence-only operations for the M2 registry aggregate."""
|
||||
|
||||
def __init__(self, session: Session) -> None:
|
||||
self.session = session
|
||||
|
||||
def page_models(
|
||||
self,
|
||||
*,
|
||||
page: int,
|
||||
page_size: int,
|
||||
search: str | None = None,
|
||||
lifecycle: str | None = None,
|
||||
source_type: str | None = None,
|
||||
) -> tuple[list[Model], int]:
|
||||
query = select(Model)
|
||||
count_query = select(func.count()).select_from(Model)
|
||||
filters: list[Any] = []
|
||||
if search:
|
||||
pattern = f"%{search.lower()}%"
|
||||
filters.append(
|
||||
func.lower(Model.display_name).like(pattern)
|
||||
| func.lower(Model.key).like(pattern)
|
||||
| func.lower(Model.upstream_source).like(pattern)
|
||||
)
|
||||
if lifecycle:
|
||||
filters.append(Model.lifecycle == lifecycle)
|
||||
if source_type:
|
||||
filters.append(Model.source_type == source_type)
|
||||
if filters:
|
||||
query = query.where(*filters)
|
||||
count_query = count_query.where(*filters)
|
||||
total = int(self.session.scalar(count_query) or 0)
|
||||
items = list(
|
||||
self.session.scalars(
|
||||
query.options(selectinload(Model.revisions).selectinload(ModelRevision.artifacts))
|
||||
.order_by(Model.display_name, Model.id)
|
||||
.offset((page - 1) * page_size)
|
||||
.limit(page_size)
|
||||
)
|
||||
)
|
||||
return items, total
|
||||
|
||||
def model(self, model_id: uuid.UUID) -> Model | None:
|
||||
return self.session.scalar(
|
||||
select(Model)
|
||||
.where(Model.id == model_id)
|
||||
.options(selectinload(Model.revisions).selectinload(ModelRevision.artifacts))
|
||||
)
|
||||
|
||||
def model_by_key(self, key: str) -> Model | None:
|
||||
return self.session.scalar(select(Model).where(Model.key == key))
|
||||
|
||||
def model_by_source(self, source: str) -> Model | None:
|
||||
return self.session.scalar(select(Model).where(Model.upstream_source == source))
|
||||
|
||||
def revision(self, revision_id: uuid.UUID) -> ModelRevision | None:
|
||||
return self.session.get(ModelRevision, revision_id)
|
||||
|
||||
def revision_by_commit(self, model_id: uuid.UUID, commit: str) -> ModelRevision | None:
|
||||
return self.session.scalar(
|
||||
select(ModelRevision).where(
|
||||
ModelRevision.model_id == model_id,
|
||||
ModelRevision.resolved_commit_sha == commit,
|
||||
)
|
||||
)
|
||||
|
||||
def revisions(self, model_id: uuid.UUID) -> list[ModelRevision]:
|
||||
return list(
|
||||
self.session.scalars(
|
||||
select(ModelRevision)
|
||||
.where(ModelRevision.model_id == model_id)
|
||||
.order_by(ModelRevision.discovered_at.desc())
|
||||
)
|
||||
)
|
||||
|
||||
def artifact(self, artifact_id: uuid.UUID) -> ModelArtifact | None:
|
||||
return self.session.scalar(
|
||||
select(ModelArtifact)
|
||||
.where(ModelArtifact.id == artifact_id)
|
||||
.options(selectinload(ModelArtifact.locations))
|
||||
)
|
||||
|
||||
def artifact_by_digest(self, revision_id: uuid.UUID, digest: str) -> ModelArtifact | None:
|
||||
return self.session.scalar(
|
||||
select(ModelArtifact).where(
|
||||
ModelArtifact.revision_id == revision_id, ModelArtifact.sha256 == digest
|
||||
)
|
||||
)
|
||||
|
||||
def artifacts(self, revision_id: uuid.UUID) -> list[ModelArtifact]:
|
||||
return list(
|
||||
self.session.scalars(
|
||||
select(ModelArtifact)
|
||||
.where(ModelArtifact.revision_id == revision_id)
|
||||
.options(selectinload(ModelArtifact.locations))
|
||||
.order_by(ModelArtifact.filename)
|
||||
)
|
||||
)
|
||||
|
||||
def derived(self, derived_id: uuid.UUID) -> DerivedArtifact | None:
|
||||
return self.session.scalar(
|
||||
select(DerivedArtifact)
|
||||
.where(DerivedArtifact.id == derived_id)
|
||||
.options(
|
||||
selectinload(DerivedArtifact.source_artifacts),
|
||||
selectinload(DerivedArtifact.locations),
|
||||
)
|
||||
)
|
||||
|
||||
def derived_for_revision(self, revision_id: uuid.UUID) -> list[DerivedArtifact]:
|
||||
return list(
|
||||
self.session.scalars(
|
||||
select(DerivedArtifact)
|
||||
.where(DerivedArtifact.revision_id == revision_id)
|
||||
.options(
|
||||
selectinload(DerivedArtifact.source_artifacts),
|
||||
selectinload(DerivedArtifact.locations),
|
||||
)
|
||||
.order_by(DerivedArtifact.filename)
|
||||
)
|
||||
)
|
||||
|
||||
def derived_sources(self, derived_id: uuid.UUID) -> list[DerivedArtifactSource]:
|
||||
return list(
|
||||
self.session.scalars(
|
||||
select(DerivedArtifactSource)
|
||||
.where(DerivedArtifactSource.derived_artifact_id == derived_id)
|
||||
.order_by(DerivedArtifactSource.ordinal)
|
||||
)
|
||||
)
|
||||
|
||||
def storage_root(self, root_id: uuid.UUID) -> StorageRoot | None:
|
||||
return self.session.get(StorageRoot, root_id)
|
||||
|
||||
def storage_roots(self, node_id: uuid.UUID | None = None) -> list[StorageRoot]:
|
||||
query = select(StorageRoot).order_by(StorageRoot.name)
|
||||
if node_id:
|
||||
query = query.where(StorageRoot.compute_node_id == node_id)
|
||||
return list(self.session.scalars(query))
|
||||
|
||||
def location(self, location_id: uuid.UUID) -> ArtifactLocation | None:
|
||||
return self.session.get(ArtifactLocation, location_id)
|
||||
|
||||
def add(self, entity: T) -> T:
|
||||
self.session.add(entity)
|
||||
self.session.flush()
|
||||
return entity
|
||||
|
||||
def dependencies(self, resource_type: str, resource_id: uuid.UUID) -> list[dict[str, str]]:
|
||||
dependencies: list[dict[str, str]] = []
|
||||
if resource_type == "model":
|
||||
ids = self.session.scalars(
|
||||
select(ModelRevision.id).where(ModelRevision.model_id == resource_id)
|
||||
)
|
||||
dependencies.extend(
|
||||
{
|
||||
"resource_type": "model_revision",
|
||||
"resource_id": str(item),
|
||||
"relation": "revision",
|
||||
}
|
||||
for item in ids
|
||||
)
|
||||
elif resource_type == "model_revision":
|
||||
ids = self.session.scalars(
|
||||
select(ModelArtifact.id).where(ModelArtifact.revision_id == resource_id)
|
||||
)
|
||||
dependencies.extend(
|
||||
{
|
||||
"resource_type": "model_artifact",
|
||||
"resource_id": str(item),
|
||||
"relation": "artifact",
|
||||
}
|
||||
for item in ids
|
||||
)
|
||||
derived_ids = self.session.scalars(
|
||||
select(DerivedArtifact.id).where(DerivedArtifact.revision_id == resource_id)
|
||||
)
|
||||
dependencies.extend(
|
||||
{
|
||||
"resource_type": "derived_artifact",
|
||||
"resource_id": str(item),
|
||||
"relation": "derived",
|
||||
}
|
||||
for item in derived_ids
|
||||
)
|
||||
elif resource_type == "model_artifact":
|
||||
links = self.session.scalars(
|
||||
select(DerivedArtifactSource.derived_artifact_id).where(
|
||||
DerivedArtifactSource.source_artifact_id == resource_id
|
||||
)
|
||||
)
|
||||
dependencies.extend(
|
||||
{
|
||||
"resource_type": "derived_artifact",
|
||||
"resource_id": str(item),
|
||||
"relation": "source",
|
||||
}
|
||||
for item in links
|
||||
)
|
||||
locations = self.session.scalars(
|
||||
select(ArtifactLocation.id).where(ArtifactLocation.artifact_id == resource_id)
|
||||
)
|
||||
dependencies.extend(
|
||||
{
|
||||
"resource_type": "artifact_location",
|
||||
"resource_id": str(item),
|
||||
"relation": "location",
|
||||
}
|
||||
for item in locations
|
||||
)
|
||||
artifact = self.session.get(ModelArtifact, resource_id)
|
||||
if artifact:
|
||||
profiles = self.session.scalars(
|
||||
select(RuntimeProfile.id).where(
|
||||
RuntimeProfile.artifact_sha256 == artifact.sha256
|
||||
)
|
||||
)
|
||||
dependencies.extend(
|
||||
{
|
||||
"resource_type": "runtime_profile",
|
||||
"resource_id": str(item),
|
||||
"relation": "digest",
|
||||
}
|
||||
for item in profiles
|
||||
)
|
||||
elif resource_type == "derived_artifact":
|
||||
sources = self.session.scalars(
|
||||
select(DerivedArtifactSource.source_artifact_id).where(
|
||||
DerivedArtifactSource.derived_artifact_id == resource_id
|
||||
)
|
||||
)
|
||||
dependencies.extend(
|
||||
{
|
||||
"resource_type": "model_artifact",
|
||||
"resource_id": str(item),
|
||||
"relation": "source_lineage",
|
||||
}
|
||||
for item in sources
|
||||
)
|
||||
locations = self.session.scalars(
|
||||
select(ArtifactLocation.id).where(
|
||||
ArtifactLocation.derived_artifact_id == resource_id
|
||||
)
|
||||
)
|
||||
dependencies.extend(
|
||||
{
|
||||
"resource_type": "artifact_location",
|
||||
"resource_id": str(item),
|
||||
"relation": "location",
|
||||
}
|
||||
for item in locations
|
||||
)
|
||||
elif resource_type == "storage_root":
|
||||
locations = self.session.scalars(
|
||||
select(ArtifactLocation.id).where(ArtifactLocation.storage_root_id == resource_id)
|
||||
)
|
||||
dependencies.extend(
|
||||
{
|
||||
"resource_type": "artifact_location",
|
||||
"resource_id": str(item),
|
||||
"relation": "location",
|
||||
}
|
||||
for item in locations
|
||||
)
|
||||
return dependencies
|
||||
@@ -0,0 +1,64 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from modelforge_api.services.audit import AuditContext, AuditWriter
|
||||
|
||||
from .models import AuditEvent, Base, Model
|
||||
|
||||
|
||||
class Repository[Entity: Base]:
|
||||
def __init__(self, session: Session, entity_type: type[Entity]) -> None:
|
||||
self.session = session
|
||||
self.entity_type = entity_type
|
||||
|
||||
def get(self, entity_id: Any) -> Entity | None:
|
||||
return self.session.get(self.entity_type, entity_id)
|
||||
|
||||
def list(self) -> list[Entity]:
|
||||
return list(self.session.scalars(select(self.entity_type)))
|
||||
|
||||
def add(self, entity: Entity) -> Entity:
|
||||
self.session.add(entity)
|
||||
self.session.flush()
|
||||
return entity
|
||||
|
||||
|
||||
class ModelRepository(Repository[Model]):
|
||||
def __init__(self, session: Session) -> None:
|
||||
super().__init__(session, Model)
|
||||
|
||||
|
||||
class AuditRepository:
|
||||
def __init__(self, session: Session) -> None:
|
||||
self.session = session
|
||||
|
||||
def append(
|
||||
self,
|
||||
*,
|
||||
correlation_id: str,
|
||||
actor_type: str,
|
||||
actor_id: str,
|
||||
action: str,
|
||||
resource_type: str,
|
||||
resource_id: str | None,
|
||||
outcome: str,
|
||||
details: dict[str, Any] | None = None,
|
||||
) -> AuditEvent:
|
||||
return AuditWriter(
|
||||
self.session,
|
||||
context=AuditContext(
|
||||
correlation_id=correlation_id,
|
||||
actor_type=actor_type,
|
||||
actor_id=actor_id,
|
||||
),
|
||||
).write(
|
||||
action=action,
|
||||
resource_type=resource_type,
|
||||
resource_id=resource_id,
|
||||
outcome=outcome,
|
||||
details=details or {},
|
||||
)
|
||||
@@ -0,0 +1,23 @@
|
||||
"""Identity registry for Engines constructed by the application process.
|
||||
|
||||
Alembic creates its own Engine and is intentionally absent. This registry is only a SQLite/test
|
||||
and accidental-misuse defense; PostgreSQL permissions and triggers are the production boundary.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from weakref import WeakSet
|
||||
|
||||
from sqlalchemy.engine import Connection, Engine
|
||||
|
||||
_APPLICATION_ENGINES: WeakSet[Engine] = WeakSet()
|
||||
|
||||
|
||||
def register_application_engine(engine: Engine) -> Engine:
|
||||
_APPLICATION_ENGINES.add(engine)
|
||||
return engine
|
||||
|
||||
|
||||
def is_application_connection(connection: Connection) -> bool:
|
||||
return connection.engine in _APPLICATION_ENGINES
|
||||
|
||||
@@ -0,0 +1,187 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from typing import TypeVar
|
||||
|
||||
from sqlalchemy import or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from modelforge_api.persistence.models import (
|
||||
Accelerator,
|
||||
ArtifactLocation,
|
||||
ArtifactSet,
|
||||
ArtifactSetMember,
|
||||
ComputeNode,
|
||||
DeploymentCandidate,
|
||||
ExecutionApproval,
|
||||
Model,
|
||||
ModelArtifact,
|
||||
ModelRevision,
|
||||
RuntimeCompatibilityAssessment,
|
||||
RuntimeEnvironment,
|
||||
RuntimeProbe,
|
||||
RuntimeProbeMetric,
|
||||
RuntimeProfile,
|
||||
StorageRoot,
|
||||
)
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
class RuntimeRepository:
|
||||
def __init__(self, session: Session) -> None:
|
||||
self.session = session
|
||||
|
||||
def add(self, entity: T) -> T:
|
||||
self.session.add(entity)
|
||||
self.session.flush()
|
||||
return entity
|
||||
|
||||
def environment(self, environment_id: uuid.UUID) -> RuntimeEnvironment | None:
|
||||
return self.session.get(RuntimeEnvironment, environment_id)
|
||||
|
||||
def environment_by_fingerprint(self, fingerprint: str) -> RuntimeEnvironment | None:
|
||||
return self.session.scalar(
|
||||
select(RuntimeEnvironment).where(RuntimeEnvironment.fingerprint == fingerprint)
|
||||
)
|
||||
|
||||
def environments(self) -> list[RuntimeEnvironment]:
|
||||
return list(
|
||||
self.session.scalars(
|
||||
select(RuntimeEnvironment).order_by(RuntimeEnvironment.created_at.desc())
|
||||
)
|
||||
)
|
||||
|
||||
def profile(self, profile_id: uuid.UUID) -> RuntimeProfile | None:
|
||||
return self.session.get(RuntimeProfile, profile_id)
|
||||
|
||||
def profile_by_fingerprint(self, fingerprint: str) -> RuntimeProfile | None:
|
||||
return self.session.scalar(
|
||||
select(RuntimeProfile).where(RuntimeProfile.fingerprint == fingerprint)
|
||||
)
|
||||
|
||||
def profiles(self) -> list[RuntimeProfile]:
|
||||
return list(
|
||||
self.session.scalars(select(RuntimeProfile).order_by(RuntimeProfile.created_at.desc()))
|
||||
)
|
||||
|
||||
def artifact_set(self, artifact_set_id: uuid.UUID) -> ArtifactSet | None:
|
||||
return self.session.get(ArtifactSet, artifact_set_id)
|
||||
|
||||
def revision(self, revision_id: uuid.UUID) -> ModelRevision | None:
|
||||
return self.session.get(ModelRevision, revision_id)
|
||||
|
||||
def model(self, model_id: uuid.UUID) -> Model | None:
|
||||
return self.session.get(Model, model_id)
|
||||
|
||||
def set_artifacts(
|
||||
self, artifact_set_id: uuid.UUID
|
||||
) -> list[tuple[ArtifactSetMember, ModelArtifact]]:
|
||||
rows = self.session.execute(
|
||||
select(ArtifactSetMember, ModelArtifact)
|
||||
.join(ModelArtifact, ModelArtifact.id == ArtifactSetMember.artifact_id)
|
||||
.where(ArtifactSetMember.artifact_set_id == artifact_set_id)
|
||||
.order_by(ArtifactSetMember.ordinal)
|
||||
)
|
||||
return [(member, artifact) for member, artifact in rows]
|
||||
|
||||
def artifact_locations(self, artifact_ids: list[uuid.UUID]) -> list[ArtifactLocation]:
|
||||
if not artifact_ids:
|
||||
return []
|
||||
return list(
|
||||
self.session.scalars(
|
||||
select(ArtifactLocation).where(ArtifactLocation.artifact_id.in_(artifact_ids))
|
||||
)
|
||||
)
|
||||
|
||||
def storage_root(self, root_id: uuid.UUID) -> StorageRoot | None:
|
||||
return self.session.get(StorageRoot, root_id)
|
||||
|
||||
def node(self, node_id: uuid.UUID) -> ComputeNode | None:
|
||||
return self.session.get(ComputeNode, node_id)
|
||||
|
||||
def accelerators(self, node_id: uuid.UUID) -> list[Accelerator]:
|
||||
return list(
|
||||
self.session.scalars(
|
||||
select(Accelerator).where(
|
||||
Accelerator.compute_node_id == node_id,
|
||||
Accelerator.status == "active",
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
def assessment(self, assessment_id: uuid.UUID) -> RuntimeCompatibilityAssessment | None:
|
||||
return self.session.get(RuntimeCompatibilityAssessment, assessment_id)
|
||||
|
||||
def assessments(self) -> list[RuntimeCompatibilityAssessment]:
|
||||
return list(
|
||||
self.session.scalars(
|
||||
select(RuntimeCompatibilityAssessment).order_by(
|
||||
RuntimeCompatibilityAssessment.created_at.desc()
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
def approval(self, approval_id: uuid.UUID) -> ExecutionApproval | None:
|
||||
return self.session.get(ExecutionApproval, approval_id)
|
||||
|
||||
def approvals(self, artifact_set_id: uuid.UUID | None = None) -> list[ExecutionApproval]:
|
||||
query = select(ExecutionApproval).order_by(ExecutionApproval.approved_at.desc())
|
||||
if artifact_set_id:
|
||||
query = query.where(ExecutionApproval.artifact_set_id == artifact_set_id)
|
||||
return list(self.session.scalars(query))
|
||||
|
||||
def probe(self, probe_id: uuid.UUID) -> RuntimeProbe | None:
|
||||
return self.session.get(RuntimeProbe, probe_id)
|
||||
|
||||
def probe_by_key(self, key: str) -> RuntimeProbe | None:
|
||||
return self.session.scalar(select(RuntimeProbe).where(RuntimeProbe.idempotency_key == key))
|
||||
|
||||
def probes(self) -> list[RuntimeProbe]:
|
||||
return list(
|
||||
self.session.scalars(select(RuntimeProbe).order_by(RuntimeProbe.created_at.desc()))
|
||||
)
|
||||
|
||||
def claimable_probe(self, node_id: uuid.UUID, now: datetime) -> RuntimeProbe | None:
|
||||
return self.session.scalar(
|
||||
select(RuntimeProbe)
|
||||
.where(
|
||||
RuntimeProbe.compute_node_id == node_id,
|
||||
RuntimeProbe.cancel_requested.is_(False),
|
||||
or_(
|
||||
RuntimeProbe.status == "queued",
|
||||
RuntimeProbe.status.in_(
|
||||
("preparing", "loading", "healthchecking", "ready", "unloading")
|
||||
)
|
||||
& (RuntimeProbe.lease_expires_at < now),
|
||||
),
|
||||
)
|
||||
.order_by(RuntimeProbe.created_at)
|
||||
.with_for_update(skip_locked=True)
|
||||
.limit(1)
|
||||
)
|
||||
|
||||
def candidate_for_probe(self, probe_id: uuid.UUID) -> DeploymentCandidate | None:
|
||||
return self.session.scalar(
|
||||
select(DeploymentCandidate).where(DeploymentCandidate.runtime_probe_id == probe_id)
|
||||
)
|
||||
|
||||
def candidate(self, candidate_id: uuid.UUID) -> DeploymentCandidate | None:
|
||||
return self.session.get(DeploymentCandidate, candidate_id)
|
||||
|
||||
def candidates(self) -> list[DeploymentCandidate]:
|
||||
return list(
|
||||
self.session.scalars(
|
||||
select(DeploymentCandidate).order_by(DeploymentCandidate.created_at.desc())
|
||||
)
|
||||
)
|
||||
|
||||
def metrics(self, probe_id: uuid.UUID) -> list[RuntimeProbeMetric]:
|
||||
return list(
|
||||
self.session.scalars(
|
||||
select(RuntimeProbeMetric)
|
||||
.where(RuntimeProbeMetric.runtime_probe_id == probe_id)
|
||||
.order_by(RuntimeProbeMetric.observed_at)
|
||||
)
|
||||
)
|
||||
@@ -0,0 +1,424 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import datetime, timedelta
|
||||
from typing import TypeVar
|
||||
|
||||
from sqlalchemy import case, delete, func, or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from modelforge_api.persistence.models import (
|
||||
Accelerator,
|
||||
AcceleratorTelemetryLatest,
|
||||
ArtifactLocation,
|
||||
ArtifactSet,
|
||||
ArtifactSetMember,
|
||||
Capability,
|
||||
CapabilityContract,
|
||||
CapabilityDeployment,
|
||||
CapabilityExperimentRoute,
|
||||
CapabilityResourceEnvelope,
|
||||
ComputeNode,
|
||||
CoResidencyEvidence,
|
||||
DeploymentCandidate,
|
||||
EmbeddingSpace,
|
||||
ExecutionApproval,
|
||||
GatewayRequest,
|
||||
ModelArtifact,
|
||||
ModelRevision,
|
||||
PlacementPlanRecord,
|
||||
ProductionExecutionApproval,
|
||||
ResidencyAllocation,
|
||||
RuntimeEnvironment,
|
||||
RuntimeProbe,
|
||||
RuntimeProfile,
|
||||
SchedulerAcceleratorState,
|
||||
SchedulerPolicyRevision,
|
||||
ServiceClient,
|
||||
ServiceCredential,
|
||||
ServingGpuLease,
|
||||
ServingJob,
|
||||
StorageRoot,
|
||||
)
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
class ServingRepository:
|
||||
def __init__(self, session: Session) -> None:
|
||||
self.session = session
|
||||
|
||||
def add(self, entity: T) -> T:
|
||||
self.session.add(entity)
|
||||
self.session.flush()
|
||||
return entity
|
||||
|
||||
def capability(self, key: str) -> Capability | None:
|
||||
return self.session.scalar(select(Capability).where(Capability.key == key))
|
||||
|
||||
def contract(self, key: str, version: int) -> CapabilityContract | None:
|
||||
return self.session.scalar(
|
||||
select(CapabilityContract)
|
||||
.join(Capability, Capability.id == CapabilityContract.capability_id)
|
||||
.where(Capability.key == key, CapabilityContract.version == version)
|
||||
)
|
||||
|
||||
def candidate(self, candidate_id: uuid.UUID) -> DeploymentCandidate | None:
|
||||
return self.session.get(DeploymentCandidate, candidate_id)
|
||||
|
||||
def probe(self, probe_id: uuid.UUID) -> RuntimeProbe | None:
|
||||
return self.session.get(RuntimeProbe, probe_id)
|
||||
|
||||
def profile(self, profile_id: uuid.UUID) -> RuntimeProfile | None:
|
||||
return self.session.get(RuntimeProfile, profile_id)
|
||||
|
||||
def environment(self, environment_id: uuid.UUID) -> RuntimeEnvironment | None:
|
||||
return self.session.get(RuntimeEnvironment, environment_id)
|
||||
|
||||
def artifact_set(self, artifact_set_id: uuid.UUID) -> ArtifactSet | None:
|
||||
return self.session.get(ArtifactSet, artifact_set_id)
|
||||
|
||||
def revision(self, revision_id: uuid.UUID) -> ModelRevision | None:
|
||||
return self.session.get(ModelRevision, revision_id)
|
||||
|
||||
def node(self, node_id: uuid.UUID) -> ComputeNode | None:
|
||||
return self.session.get(ComputeNode, node_id)
|
||||
|
||||
def accelerator(self, accelerator_id: uuid.UUID) -> Accelerator | None:
|
||||
return self.session.get(Accelerator, accelerator_id)
|
||||
|
||||
def node_accelerators(self, node_id: uuid.UUID) -> list[Accelerator]:
|
||||
return list(
|
||||
self.session.scalars(
|
||||
select(Accelerator).where(
|
||||
Accelerator.compute_node_id == node_id,
|
||||
Accelerator.status == "active",
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
def lock_accelerator(self, accelerator_id: uuid.UUID) -> Accelerator | None:
|
||||
return self.session.scalar(
|
||||
select(Accelerator).where(Accelerator.id == accelerator_id).with_for_update()
|
||||
)
|
||||
|
||||
def telemetry(self, accelerator_id: uuid.UUID) -> AcceleratorTelemetryLatest | None:
|
||||
return self.session.scalar(
|
||||
select(AcceleratorTelemetryLatest).where(
|
||||
AcceleratorTelemetryLatest.accelerator_id == accelerator_id
|
||||
)
|
||||
)
|
||||
|
||||
def set_artifacts(self, artifact_set_id: uuid.UUID) -> list[ModelArtifact]:
|
||||
return list(
|
||||
self.session.scalars(
|
||||
select(ModelArtifact)
|
||||
.join(ArtifactSetMember, ArtifactSetMember.artifact_id == ModelArtifact.id)
|
||||
.where(ArtifactSetMember.artifact_set_id == artifact_set_id)
|
||||
.order_by(ArtifactSetMember.ordinal)
|
||||
)
|
||||
)
|
||||
|
||||
def artifact_locations(self, artifact_ids: list[uuid.UUID]) -> list[ArtifactLocation]:
|
||||
return list(
|
||||
self.session.scalars(
|
||||
select(ArtifactLocation).where(ArtifactLocation.artifact_id.in_(artifact_ids))
|
||||
)
|
||||
)
|
||||
|
||||
def storage_root(self, root_id: uuid.UUID) -> StorageRoot | None:
|
||||
return self.session.get(StorageRoot, root_id)
|
||||
|
||||
def approval_by_fingerprint(self, fingerprint: str) -> ProductionExecutionApproval | None:
|
||||
return self.session.scalar(
|
||||
select(ProductionExecutionApproval).where(
|
||||
ProductionExecutionApproval.evidence_fingerprint == fingerprint
|
||||
)
|
||||
)
|
||||
|
||||
def approval(self, approval_id: uuid.UUID) -> ProductionExecutionApproval | None:
|
||||
return self.session.get(ProductionExecutionApproval, approval_id)
|
||||
|
||||
def execution_approval(self, approval_id: uuid.UUID) -> ExecutionApproval | None:
|
||||
return self.session.get(ExecutionApproval, approval_id)
|
||||
|
||||
def approvals(self) -> list[ProductionExecutionApproval]:
|
||||
return list(
|
||||
self.session.scalars(
|
||||
select(ProductionExecutionApproval).order_by(
|
||||
ProductionExecutionApproval.approved_at.desc()
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
def embedding_space_by_digest(self, digest: str) -> EmbeddingSpace | None:
|
||||
return self.session.scalar(
|
||||
select(EmbeddingSpace).where(EmbeddingSpace.identity_digest == digest)
|
||||
)
|
||||
|
||||
def embedding_space(self, space_id: uuid.UUID | None) -> EmbeddingSpace | None:
|
||||
if space_id is None:
|
||||
return None
|
||||
return self.session.get(EmbeddingSpace, space_id)
|
||||
|
||||
def deployment_by_fingerprint(self, fingerprint: str) -> CapabilityDeployment | None:
|
||||
return self.session.scalar(
|
||||
select(CapabilityDeployment).where(
|
||||
CapabilityDeployment.config_fingerprint == fingerprint
|
||||
)
|
||||
)
|
||||
|
||||
def deployment(self, deployment_id: uuid.UUID) -> CapabilityDeployment | None:
|
||||
return self.session.get(CapabilityDeployment, deployment_id)
|
||||
|
||||
def deployments(self) -> list[CapabilityDeployment]:
|
||||
return list(
|
||||
self.session.scalars(
|
||||
select(CapabilityDeployment).order_by(CapabilityDeployment.created_at.desc())
|
||||
)
|
||||
)
|
||||
|
||||
def experiment_route(self, route_key: str) -> CapabilityExperimentRoute | None:
|
||||
return self.session.scalar(
|
||||
select(CapabilityExperimentRoute).where(
|
||||
CapabilityExperimentRoute.route_key == route_key
|
||||
)
|
||||
)
|
||||
|
||||
def experiment_route_by_id(self, route_id: uuid.UUID) -> CapabilityExperimentRoute | None:
|
||||
return self.session.get(CapabilityExperimentRoute, route_id)
|
||||
|
||||
def experiment_routes(self) -> list[CapabilityExperimentRoute]:
|
||||
return list(
|
||||
self.session.scalars(
|
||||
select(CapabilityExperimentRoute).order_by(
|
||||
CapabilityExperimentRoute.created_at.desc()
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
def stable_deployment(self, contract_id: uuid.UUID) -> CapabilityDeployment | None:
|
||||
return self.session.scalar(
|
||||
select(CapabilityDeployment).where(
|
||||
CapabilityDeployment.capability_contract_id == contract_id,
|
||||
CapabilityDeployment.channel == "stable",
|
||||
CapabilityDeployment.status == "stable",
|
||||
CapabilityDeployment.production.is_(True),
|
||||
)
|
||||
)
|
||||
|
||||
def envelope(self, deployment_id: uuid.UUID) -> CapabilityResourceEnvelope | None:
|
||||
return self.session.scalar(
|
||||
select(CapabilityResourceEnvelope).where(
|
||||
CapabilityResourceEnvelope.capability_deployment_id == deployment_id
|
||||
)
|
||||
)
|
||||
|
||||
def residency(
|
||||
self, deployment_id: uuid.UUID, *, lock: bool = False
|
||||
) -> ResidencyAllocation | None:
|
||||
query = select(ResidencyAllocation).where(
|
||||
ResidencyAllocation.capability_deployment_id == deployment_id
|
||||
)
|
||||
if lock:
|
||||
query = query.with_for_update()
|
||||
return self.session.scalar(query)
|
||||
|
||||
def accelerator_residencies(self, accelerator_id: uuid.UUID) -> list[ResidencyAllocation]:
|
||||
return list(
|
||||
self.session.scalars(
|
||||
select(ResidencyAllocation)
|
||||
.join(
|
||||
CapabilityDeployment,
|
||||
CapabilityDeployment.id == ResidencyAllocation.capability_deployment_id,
|
||||
)
|
||||
.where(CapabilityDeployment.accelerator_id == accelerator_id)
|
||||
.order_by(ResidencyAllocation.created_at)
|
||||
)
|
||||
)
|
||||
|
||||
def client_by_name(self, name: str) -> ServiceClient | None:
|
||||
return self.session.scalar(select(ServiceClient).where(ServiceClient.name == name))
|
||||
|
||||
def client(self, client_id: uuid.UUID) -> ServiceClient | None:
|
||||
return self.session.get(ServiceClient, client_id)
|
||||
|
||||
def clients(self) -> list[ServiceClient]:
|
||||
return list(self.session.scalars(select(ServiceClient).order_by(ServiceClient.created_at)))
|
||||
|
||||
def active_credential(self, client_id: uuid.UUID) -> ServiceCredential | None:
|
||||
return self.session.scalar(
|
||||
select(ServiceCredential).where(
|
||||
ServiceCredential.service_client_id == client_id,
|
||||
ServiceCredential.revoked_at.is_(None),
|
||||
)
|
||||
)
|
||||
|
||||
def credential_by_hash(
|
||||
self, secret_hash: str
|
||||
) -> tuple[ServiceCredential, ServiceClient] | None:
|
||||
row = self.session.execute(
|
||||
select(ServiceCredential, ServiceClient)
|
||||
.join(ServiceClient, ServiceClient.id == ServiceCredential.service_client_id)
|
||||
.where(ServiceCredential.secret_hash == secret_hash)
|
||||
).first()
|
||||
return (row[0], row[1]) if row else None
|
||||
|
||||
def recent_request_count(self, client_id: uuid.UUID, since: datetime) -> int:
|
||||
return int(
|
||||
self.session.scalar(
|
||||
select(func.count(GatewayRequest.id)).where(
|
||||
GatewayRequest.service_client_id == client_id,
|
||||
GatewayRequest.created_at >= since,
|
||||
)
|
||||
)
|
||||
or 0
|
||||
)
|
||||
|
||||
def active_client_requests(self, client_id: uuid.UUID) -> int:
|
||||
return int(
|
||||
self.session.scalar(
|
||||
select(func.count(GatewayRequest.id)).where(
|
||||
GatewayRequest.service_client_id == client_id,
|
||||
GatewayRequest.status.in_(("queued", "running")),
|
||||
)
|
||||
)
|
||||
or 0
|
||||
)
|
||||
|
||||
def active_gateway_requests(self) -> int:
|
||||
return int(
|
||||
self.session.scalar(
|
||||
select(func.count(GatewayRequest.id)).where(
|
||||
GatewayRequest.status.in_(("queued", "running"))
|
||||
)
|
||||
)
|
||||
or 0
|
||||
)
|
||||
|
||||
def gateway_requests(self, limit: int = 100) -> list[GatewayRequest]:
|
||||
return list(
|
||||
self.session.scalars(
|
||||
select(GatewayRequest).order_by(GatewayRequest.created_at.desc()).limit(limit)
|
||||
)
|
||||
)
|
||||
|
||||
def queued_job_count(self, deployment_id: uuid.UUID) -> int:
|
||||
return int(
|
||||
self.session.scalar(
|
||||
select(func.count(ServingJob.id)).where(
|
||||
ServingJob.capability_deployment_id == deployment_id,
|
||||
ServingJob.operation == "invoke",
|
||||
ServingJob.status.in_(("queued", "running")),
|
||||
)
|
||||
)
|
||||
or 0
|
||||
)
|
||||
|
||||
def job(self, job_id: uuid.UUID) -> ServingJob | None:
|
||||
return self.session.get(ServingJob, job_id)
|
||||
|
||||
def job_by_key(self, key: str) -> ServingJob | None:
|
||||
return self.session.scalar(select(ServingJob).where(ServingJob.idempotency_key == key))
|
||||
|
||||
def claimable_job(self, node_id: uuid.UUID, now: datetime) -> ServingJob | None:
|
||||
aging_cutoff = now - timedelta(minutes=5)
|
||||
priority_order = case(
|
||||
(ServingJob.priority == "production", 0),
|
||||
((ServingJob.priority == "background") & (ServingJob.created_at < aging_cutoff), 1),
|
||||
(
|
||||
(ServingJob.priority.in_(("lab", "benchmark")))
|
||||
& (ServingJob.created_at < aging_cutoff),
|
||||
2,
|
||||
),
|
||||
(ServingJob.priority == "interactive", 1),
|
||||
(ServingJob.priority == "background", 2),
|
||||
(ServingJob.priority.in_(("lab", "benchmark")), 3),
|
||||
else_=4,
|
||||
)
|
||||
return self.session.scalar(
|
||||
select(ServingJob)
|
||||
.where(
|
||||
ServingJob.compute_node_id == node_id,
|
||||
or_(
|
||||
ServingJob.status == "queued",
|
||||
(ServingJob.status == "running") & (ServingJob.lease_expires_at < now),
|
||||
),
|
||||
)
|
||||
.order_by(priority_order, ServingJob.created_at)
|
||||
.with_for_update(skip_locked=True)
|
||||
.limit(1)
|
||||
)
|
||||
|
||||
def active_leases(self, accelerator_id: uuid.UUID, now: datetime) -> list[ServingGpuLease]:
|
||||
return list(
|
||||
self.session.scalars(
|
||||
select(ServingGpuLease).where(
|
||||
ServingGpuLease.accelerator_id == accelerator_id,
|
||||
ServingGpuLease.state.in_(("pending", "granted", "active", "releasing")),
|
||||
ServingGpuLease.expires_at > now,
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
def lease_for_request(self, request_id: uuid.UUID) -> ServingGpuLease | None:
|
||||
return self.session.scalar(
|
||||
select(ServingGpuLease).where(ServingGpuLease.request_id == request_id)
|
||||
)
|
||||
|
||||
def expired_leases(self, now: datetime) -> list[ServingGpuLease]:
|
||||
return list(
|
||||
self.session.scalars(
|
||||
select(ServingGpuLease).where(
|
||||
ServingGpuLease.state.in_(("pending", "granted", "active", "releasing")),
|
||||
ServingGpuLease.expires_at <= now,
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
def active_scheduler_policy(self) -> SchedulerPolicyRevision | None:
|
||||
return self.session.scalar(
|
||||
select(SchedulerPolicyRevision)
|
||||
.where(SchedulerPolicyRevision.active.is_(True))
|
||||
.order_by(SchedulerPolicyRevision.created_at.desc())
|
||||
.limit(1)
|
||||
)
|
||||
|
||||
def scheduler_accelerator_state(
|
||||
self, accelerator_id: uuid.UUID
|
||||
) -> SchedulerAcceleratorState | None:
|
||||
return self.session.scalar(
|
||||
select(SchedulerAcceleratorState).where(
|
||||
SchedulerAcceleratorState.accelerator_id == accelerator_id
|
||||
)
|
||||
)
|
||||
|
||||
def placement_plans(self, limit: int = 100) -> list[PlacementPlanRecord]:
|
||||
return list(
|
||||
self.session.scalars(
|
||||
select(PlacementPlanRecord)
|
||||
.order_by(PlacementPlanRecord.created_at.desc())
|
||||
.limit(limit)
|
||||
)
|
||||
)
|
||||
|
||||
def trim_placement_plans(self, keep: int) -> None:
|
||||
retained = (
|
||||
select(PlacementPlanRecord.id)
|
||||
.order_by(PlacementPlanRecord.created_at.desc())
|
||||
.limit(keep)
|
||||
)
|
||||
self.session.execute(
|
||||
delete(PlacementPlanRecord).where(PlacementPlanRecord.id.not_in(retained))
|
||||
)
|
||||
|
||||
def co_residency_evidence(
|
||||
self, left_id: uuid.UUID, right_id: uuid.UUID
|
||||
) -> CoResidencyEvidence | None:
|
||||
low, high = sorted((left_id, right_id), key=str)
|
||||
return self.session.scalar(
|
||||
select(CoResidencyEvidence).where(
|
||||
CoResidencyEvidence.left_deployment_id == low,
|
||||
CoResidencyEvidence.right_deployment_id == high,
|
||||
)
|
||||
)
|
||||
@@ -0,0 +1 @@
|
||||
"""External provider boundaries."""
|
||||
@@ -0,0 +1,238 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from pathlib import PurePosixPath
|
||||
from typing import Any, Protocol
|
||||
|
||||
import httpx
|
||||
from huggingface_hub import HfApi
|
||||
from huggingface_hub.errors import (
|
||||
GatedRepoError,
|
||||
HfHubHTTPError,
|
||||
RepositoryNotFoundError,
|
||||
RevisionNotFoundError,
|
||||
)
|
||||
|
||||
|
||||
class HuggingFaceProviderError(RuntimeError):
|
||||
code = "huggingface_error"
|
||||
access_state = "error"
|
||||
|
||||
|
||||
class HuggingFaceNotFound(HuggingFaceProviderError):
|
||||
code = "repository_not_found"
|
||||
access_state = "not_found"
|
||||
|
||||
|
||||
class HuggingFaceGated(HuggingFaceProviderError):
|
||||
code = "repository_gated"
|
||||
access_state = "gated"
|
||||
|
||||
|
||||
class HuggingFaceRevisionNotFound(HuggingFaceProviderError):
|
||||
code = "revision_not_found"
|
||||
access_state = "revision_not_found"
|
||||
|
||||
|
||||
class HuggingFaceUnavailable(HuggingFaceProviderError):
|
||||
code = "provider_unavailable"
|
||||
access_state = "unavailable"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ProviderFile:
|
||||
path: str
|
||||
size_bytes: int | None
|
||||
blob_id: str | None
|
||||
upstream_sha256: str | None
|
||||
file_format: str
|
||||
role: str
|
||||
risk_flags: tuple[str, ...] = ()
|
||||
metadata: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ProviderSnapshot:
|
||||
repository_id: str
|
||||
requested_revision: str
|
||||
resolved_commit_sha: str
|
||||
access_state: str
|
||||
metadata: dict[str, Any]
|
||||
card_metadata: dict[str, Any]
|
||||
security_metadata: dict[str, Any]
|
||||
source_updated_at: datetime | None
|
||||
files: tuple[ProviderFile, ...]
|
||||
|
||||
|
||||
class HuggingFaceProvider(Protocol):
|
||||
def search(
|
||||
self, query: str, *, limit: int, sort: str, pipeline_tag: str | None
|
||||
) -> list[dict[str, Any]]: ...
|
||||
|
||||
def snapshot(self, repository_id: str, revision: str) -> ProviderSnapshot: ...
|
||||
|
||||
|
||||
def classify_file(path: str) -> tuple[str, str, tuple[str, ...]]:
|
||||
suffix = PurePosixPath(path.lower()).suffix
|
||||
name = PurePosixPath(path.lower()).name
|
||||
risks: list[str] = []
|
||||
if suffix == ".safetensors":
|
||||
return "safetensors", "weights", ()
|
||||
if suffix == ".gguf":
|
||||
return "gguf", "weights", ()
|
||||
if suffix in {".bin", ".pt", ".pth", ".ckpt", ".pkl", ".pickle"}:
|
||||
risks.append("pickle_or_executable_serialization")
|
||||
return suffix.lstrip("."), "weights", tuple(risks)
|
||||
if suffix == ".py":
|
||||
return "python", "repository_code", ("remote_code",)
|
||||
if name.endswith(".safetensors.index.json"):
|
||||
return "json", "weight_index", ()
|
||||
if suffix in {".json", ".model", ".txt", ".tiktoken"}:
|
||||
role = "configuration" if "config" in name else "tokenizer"
|
||||
return suffix.lstrip("."), role, ()
|
||||
if suffix in {".md", ".rst"}:
|
||||
return suffix.lstrip("."), "documentation", ()
|
||||
return suffix.lstrip(".") or "unknown", "other", ()
|
||||
|
||||
|
||||
def _card_dict(value: Any) -> dict[str, Any]:
|
||||
if value is None:
|
||||
return {}
|
||||
if hasattr(value, "to_dict"):
|
||||
result = value.to_dict()
|
||||
return dict(result) if isinstance(result, dict) else {}
|
||||
return dict(value) if isinstance(value, dict) else {}
|
||||
|
||||
|
||||
def _lfs_value(lfs: Any, name: str) -> Any:
|
||||
if lfs is None:
|
||||
return None
|
||||
return lfs.get(name) if isinstance(lfs, dict) else getattr(lfs, name, None)
|
||||
|
||||
|
||||
class OfficialHuggingFaceProvider:
|
||||
"""Narrow adapter over the official huggingface_hub client; no HTML scraping."""
|
||||
|
||||
def __init__(self, *, token: str | None = None, timeout: float = 30) -> None:
|
||||
self.api = HfApi(token=token)
|
||||
self.token = token
|
||||
self.timeout = timeout
|
||||
|
||||
@staticmethod
|
||||
def _search_item(item: Any) -> dict[str, Any]:
|
||||
return {
|
||||
"repository_id": item.id,
|
||||
"resolved_commit_sha": getattr(item, "sha", None),
|
||||
"access_state": "gated"
|
||||
if getattr(item, "gated", False)
|
||||
else "private"
|
||||
if getattr(item, "private", False)
|
||||
else "public",
|
||||
"pipeline_tag": getattr(item, "pipeline_tag", None),
|
||||
"library_name": getattr(item, "library_name", None),
|
||||
"tags": list(getattr(item, "tags", None) or []),
|
||||
"downloads": getattr(item, "downloads", None),
|
||||
"likes": getattr(item, "likes", None),
|
||||
"last_modified": getattr(item, "last_modified", None),
|
||||
}
|
||||
|
||||
def search(
|
||||
self, query: str, *, limit: int, sort: str, pipeline_tag: str | None
|
||||
) -> list[dict[str, Any]]:
|
||||
try:
|
||||
items: Iterable[Any] = self.api.list_models(
|
||||
search=query,
|
||||
pipeline_tag=pipeline_tag,
|
||||
sort=sort,
|
||||
limit=limit,
|
||||
full=True,
|
||||
cardData=True,
|
||||
token=self.token,
|
||||
)
|
||||
return [self._search_item(item) for item in items]
|
||||
except (HfHubHTTPError, httpx.TimeoutException) as exc:
|
||||
raise HuggingFaceUnavailable("Hugging Face search is unavailable") from exc
|
||||
|
||||
def snapshot(self, repository_id: str, revision: str) -> ProviderSnapshot:
|
||||
try:
|
||||
info = self.api.model_info(
|
||||
repository_id,
|
||||
revision=revision,
|
||||
timeout=self.timeout,
|
||||
securityStatus=True,
|
||||
files_metadata=True,
|
||||
token=self.token,
|
||||
)
|
||||
except GatedRepoError as exc:
|
||||
raise HuggingFaceGated(f"repository {repository_id} is gated") from exc
|
||||
except RevisionNotFoundError as exc:
|
||||
raise HuggingFaceRevisionNotFound(
|
||||
f"revision {revision} was not found for {repository_id}"
|
||||
) from exc
|
||||
except RepositoryNotFoundError as exc:
|
||||
raise HuggingFaceNotFound(
|
||||
f"repository {repository_id} was not found or is private"
|
||||
) from exc
|
||||
except (HfHubHTTPError, httpx.TimeoutException) as exc:
|
||||
raise HuggingFaceUnavailable(
|
||||
f"Hugging Face metadata is unavailable for {repository_id}"
|
||||
) from exc
|
||||
files: list[ProviderFile] = []
|
||||
for sibling in info.siblings or []:
|
||||
file_format, role, risks = classify_file(sibling.rfilename)
|
||||
lfs = getattr(sibling, "lfs", None)
|
||||
files.append(
|
||||
ProviderFile(
|
||||
path=sibling.rfilename,
|
||||
size_bytes=getattr(sibling, "size", None) or _lfs_value(lfs, "size"),
|
||||
blob_id=getattr(sibling, "blob_id", None),
|
||||
upstream_sha256=_lfs_value(lfs, "sha256"),
|
||||
file_format=file_format,
|
||||
role=role,
|
||||
risk_flags=risks,
|
||||
metadata={
|
||||
"lfs_pointer_size": _lfs_value(lfs, "pointer_size"),
|
||||
"source": "huggingface_hub.model_info(files_metadata=true)",
|
||||
},
|
||||
)
|
||||
)
|
||||
card = _card_dict(getattr(info, "card_data", None))
|
||||
if not info.sha:
|
||||
raise HuggingFaceUnavailable(
|
||||
f"Hugging Face did not resolve an exact commit for {repository_id}"
|
||||
)
|
||||
access = (
|
||||
"gated"
|
||||
if getattr(info, "gated", False)
|
||||
else "private"
|
||||
if getattr(info, "private", False)
|
||||
else "public"
|
||||
)
|
||||
return ProviderSnapshot(
|
||||
repository_id=info.id,
|
||||
requested_revision=revision,
|
||||
resolved_commit_sha=info.sha,
|
||||
access_state=access,
|
||||
metadata={
|
||||
"repository_id": info.id,
|
||||
"author": getattr(info, "author", None),
|
||||
"pipeline_tag": getattr(info, "pipeline_tag", None),
|
||||
"library_name": getattr(info, "library_name", None),
|
||||
"tags": list(getattr(info, "tags", None) or []),
|
||||
"downloads": getattr(info, "downloads", None),
|
||||
"likes": getattr(info, "likes", None),
|
||||
"gated": getattr(info, "gated", False),
|
||||
"private": getattr(info, "private", False),
|
||||
"source": "huggingface_hub.model_info",
|
||||
},
|
||||
card_metadata=card,
|
||||
security_metadata={
|
||||
"upstream_scanner": getattr(info, "security_repo_status", None),
|
||||
"evidence_only": True,
|
||||
"local_static_inspection": "not_run",
|
||||
},
|
||||
source_updated_at=getattr(info, "last_modified", None),
|
||||
files=tuple(sorted(files, key=lambda item: item.path)),
|
||||
)
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,948 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import secrets
|
||||
import uuid
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from modelforge_api.domain.acquisition import (
|
||||
AgentArtifactJobFile,
|
||||
AgentArtifactJobLease,
|
||||
AgentJobComplete,
|
||||
AgentJobControl,
|
||||
AgentJobFailure,
|
||||
AgentJobProgress,
|
||||
ArtifactJobResponse,
|
||||
ArtifactSetResponse,
|
||||
DiscoveryCandidate,
|
||||
DiscoverySearchRequest,
|
||||
DownloadPlanCreate,
|
||||
DownloadPlanFileResponse,
|
||||
DownloadPlanResponse,
|
||||
UpstreamFileResponse,
|
||||
UpstreamSnapshotResponse,
|
||||
)
|
||||
from modelforge_api.domain.enums import ArtifactStatus, NodeLiveness
|
||||
from modelforge_api.persistence.acquisition_repository import AcquisitionRepository
|
||||
from modelforge_api.persistence.models import (
|
||||
ArtifactInspection,
|
||||
ArtifactJob,
|
||||
ArtifactJobAttempt,
|
||||
ArtifactLocation,
|
||||
ArtifactSet,
|
||||
ArtifactSetMember,
|
||||
ComputeNode,
|
||||
DownloadPlan,
|
||||
DownloadPlanFile,
|
||||
Model,
|
||||
ModelArtifact,
|
||||
ModelRevision,
|
||||
StorageRoot,
|
||||
UpstreamFile,
|
||||
UpstreamSnapshot,
|
||||
)
|
||||
from modelforge_api.providers.huggingface import (
|
||||
HuggingFaceProvider,
|
||||
HuggingFaceProviderError,
|
||||
ProviderFile,
|
||||
)
|
||||
from modelforge_api.services.audit import AuditWriter
|
||||
from modelforge_api.services.registry import RegistryConflict, RegistryNotFound, RegistryService
|
||||
from modelforge_api.settings import Settings
|
||||
|
||||
|
||||
class AcquisitionError(RegistryConflict):
|
||||
code = "acquisition_error"
|
||||
|
||||
|
||||
class ProviderBoundaryError(AcquisitionError):
|
||||
status_code = 502
|
||||
|
||||
|
||||
def _utcnow() -> datetime:
|
||||
return datetime.now(UTC)
|
||||
|
||||
|
||||
def _aware(value: datetime) -> datetime:
|
||||
return value if value.tzinfo else value.replace(tzinfo=UTC)
|
||||
|
||||
|
||||
def _canonical_hash(payload: dict[str, Any]) -> str:
|
||||
return hashlib.sha256(
|
||||
json.dumps(payload, sort_keys=True, separators=(",", ":")).encode()
|
||||
).hexdigest()
|
||||
|
||||
|
||||
def _safe_key(repository_id: str) -> str:
|
||||
return repository_id.lower().replace("/", "--").replace("_", "-")[:120]
|
||||
|
||||
|
||||
def _select_artifact_files(files: list[ProviderFile]) -> tuple[str, str, list[ProviderFile]]:
|
||||
safe_weights = [
|
||||
item for item in files if item.role == "weights" and item.file_format == "safetensors"
|
||||
]
|
||||
gguf_weights = [item for item in files if item.role == "weights" and item.file_format == "gguf"]
|
||||
other_weights = [
|
||||
item
|
||||
for item in files
|
||||
if item.role == "weights" and item not in safe_weights and item not in gguf_weights
|
||||
]
|
||||
support = [
|
||||
item
|
||||
for item in files
|
||||
if item.role in {"configuration", "tokenizer", "weight_index"}
|
||||
and "remote_code" not in item.risk_flags
|
||||
]
|
||||
if safe_weights:
|
||||
return (
|
||||
"safetensors-default",
|
||||
"Safetensors files were preferred; duplicate pickle/GGUF weight variants were excluded.",
|
||||
sorted(safe_weights + support, key=lambda item: item.path),
|
||||
)
|
||||
if gguf_weights:
|
||||
return (
|
||||
"gguf-default",
|
||||
"No safetensors were advertised; the GGUF variant was selected without pickle weights.",
|
||||
sorted(gguf_weights + support, key=lambda item: item.path),
|
||||
)
|
||||
return (
|
||||
"upstream-default",
|
||||
"No safe tensor variant was advertised; upstream weights retain explicit risk flags.",
|
||||
sorted(other_weights + support, key=lambda item: item.path),
|
||||
)
|
||||
|
||||
|
||||
class AcquisitionService:
|
||||
def __init__(
|
||||
self,
|
||||
session: Session,
|
||||
settings: Settings,
|
||||
provider: HuggingFaceProvider,
|
||||
*,
|
||||
actor_type: str = "operator",
|
||||
actor_id: str = "local-operator",
|
||||
) -> None:
|
||||
self.session = session
|
||||
self.settings = settings
|
||||
self.provider = provider
|
||||
self.repo = AcquisitionRepository(session)
|
||||
self.audit = AuditWriter(session, actor_type, actor_id)
|
||||
|
||||
def _commit(self) -> None:
|
||||
self.session.commit()
|
||||
|
||||
def search(self, request: DiscoverySearchRequest) -> list[DiscoveryCandidate]:
|
||||
try:
|
||||
results = self.provider.search(
|
||||
request.query,
|
||||
limit=request.limit,
|
||||
sort=request.sort,
|
||||
pipeline_tag=request.pipeline_tag,
|
||||
)
|
||||
except HuggingFaceProviderError as exc:
|
||||
raise ProviderBoundaryError(str(exc), {"provider_code": exc.code}) from exc
|
||||
sources = {
|
||||
item.upstream_source: item.id
|
||||
for item in self.session.scalars(
|
||||
select(Model).where(Model.source_type == "huggingface")
|
||||
)
|
||||
}
|
||||
response: list[DiscoveryCandidate] = []
|
||||
for item in results:
|
||||
repository_id = str(item["repository_id"])
|
||||
response.append(
|
||||
DiscoveryCandidate(
|
||||
**item,
|
||||
matched_model_id=sources.get(repository_id),
|
||||
upstream_facts={
|
||||
"source": "huggingface_hub.list_models",
|
||||
"pipeline_tag": item.get("pipeline_tag"),
|
||||
"tags": item.get("tags", []),
|
||||
},
|
||||
local_interpretation={
|
||||
"candidate_only": True,
|
||||
"approval": "not_evaluated",
|
||||
"hardware_compatibility": "not_measured",
|
||||
},
|
||||
)
|
||||
)
|
||||
self.audit.write(
|
||||
"HF_DISCOVERY_SEARCHED",
|
||||
"upstream_provider",
|
||||
"huggingface",
|
||||
{"query": request.query, "result_count": len(response)},
|
||||
)
|
||||
self._commit()
|
||||
return response
|
||||
|
||||
def refresh_model(self, model_id: uuid.UUID, revision: str) -> UpstreamSnapshotResponse:
|
||||
model = self.session.get(Model, model_id)
|
||||
if not model:
|
||||
raise RegistryNotFound("model not found")
|
||||
if model.source_type != "huggingface":
|
||||
raise AcquisitionError("only Hugging Face models can use this provider boundary")
|
||||
try:
|
||||
source = self.provider.snapshot(model.upstream_source, revision)
|
||||
except HuggingFaceProviderError as exc:
|
||||
self.audit.write(
|
||||
"HF_METADATA_REFRESH_FAILED",
|
||||
"model",
|
||||
str(model.id),
|
||||
{"provider_code": exc.code, "access_state": exc.access_state},
|
||||
outcome="blocked",
|
||||
)
|
||||
self._commit()
|
||||
raise ProviderBoundaryError(
|
||||
str(exc), {"provider_code": exc.code, "access_state": exc.access_state}
|
||||
) from exc
|
||||
now = _utcnow()
|
||||
snapshot = self.repo.add(
|
||||
UpstreamSnapshot(
|
||||
model_id=model.id,
|
||||
provider="huggingface",
|
||||
repository_id=source.repository_id,
|
||||
requested_revision=revision,
|
||||
resolved_commit_sha=source.resolved_commit_sha,
|
||||
access_state=source.access_state,
|
||||
metadata_snapshot=source.metadata,
|
||||
card_metadata=source.card_metadata,
|
||||
security_metadata=source.security_metadata,
|
||||
source_updated_at=source.source_updated_at,
|
||||
observed_at=now,
|
||||
stale_after=now + timedelta(seconds=self.settings.hf_snapshot_ttl_seconds),
|
||||
)
|
||||
)
|
||||
for item in source.files:
|
||||
self.repo.add(
|
||||
UpstreamFile(
|
||||
snapshot_id=snapshot.id,
|
||||
path=item.path,
|
||||
size_bytes=item.size_bytes,
|
||||
blob_id=item.blob_id,
|
||||
upstream_sha256=item.upstream_sha256,
|
||||
file_format=item.file_format,
|
||||
role=item.role,
|
||||
risk_flags=list(item.risk_flags),
|
||||
metadata_snapshot=item.metadata,
|
||||
)
|
||||
)
|
||||
existing_revision = self.session.scalar(
|
||||
select(ModelRevision).where(
|
||||
ModelRevision.model_id == model.id,
|
||||
ModelRevision.resolved_commit_sha == source.resolved_commit_sha,
|
||||
)
|
||||
)
|
||||
exact_revision = existing_revision or self.repo.add(
|
||||
ModelRevision(
|
||||
model_id=model.id,
|
||||
upstream_revision=revision,
|
||||
resolved_commit_sha=source.resolved_commit_sha,
|
||||
metadata_snapshot={
|
||||
"upstream_snapshot_id": str(snapshot.id),
|
||||
"provider": "huggingface",
|
||||
"observed_at": now.isoformat(),
|
||||
},
|
||||
discovered_at=now,
|
||||
immutable_at=now,
|
||||
)
|
||||
)
|
||||
variant, reason, selected = _select_artifact_files(list(source.files))
|
||||
artifact_set = self.repo.artifact_set_for_variant(exact_revision.id, variant)
|
||||
if artifact_set is None and selected:
|
||||
missing_sizes = [item.path for item in selected if item.size_bytes is None]
|
||||
artifact_set = self.repo.add(
|
||||
ArtifactSet(
|
||||
revision_id=exact_revision.id,
|
||||
snapshot_id=snapshot.id,
|
||||
variant_key=variant,
|
||||
label=variant.replace("-", " ").title(),
|
||||
selection_reason=reason,
|
||||
selected_paths=[item.path for item in selected],
|
||||
total_size_bytes=sum(item.size_bytes or 0 for item in selected),
|
||||
file_count=len(selected),
|
||||
availability="remote",
|
||||
status="remote",
|
||||
completeness="incomplete_metadata" if missing_sizes else "planned",
|
||||
security_status="risk_detected"
|
||||
if any(item.risk_flags for item in selected)
|
||||
else "unverified",
|
||||
license_status="captured_unreviewed"
|
||||
if source.card_metadata.get("license")
|
||||
else "unknown",
|
||||
immutable_at=now,
|
||||
)
|
||||
)
|
||||
model.upstream_metadata = source.metadata | {
|
||||
"resolved_commit_sha": source.resolved_commit_sha,
|
||||
"latest_snapshot_id": str(snapshot.id),
|
||||
"metadata_observed_at": now.isoformat(),
|
||||
}
|
||||
model.license_metadata = {
|
||||
"status": "captured_unreviewed" if source.card_metadata.get("license") else "unknown",
|
||||
"declared": source.card_metadata.get("license"),
|
||||
"source": "huggingface_model_card",
|
||||
"approval": "not_evaluated",
|
||||
}
|
||||
model.interpretation_metadata = model.interpretation_metadata | {
|
||||
"discovery_interpretation": {
|
||||
"pipeline_tag": source.metadata.get("pipeline_tag"),
|
||||
"safe_default_variant": variant if selected else None,
|
||||
"hardware_compatibility": "not_measured",
|
||||
"evidence_status": "upstream_metadata_only",
|
||||
}
|
||||
}
|
||||
self.audit.write(
|
||||
"HF_METADATA_REFRESHED",
|
||||
"model",
|
||||
str(model.id),
|
||||
{
|
||||
"snapshot_id": str(snapshot.id),
|
||||
"resolved_commit_sha": source.resolved_commit_sha,
|
||||
"file_count": len(source.files),
|
||||
"artifact_set_id": str(artifact_set.id) if artifact_set else None,
|
||||
},
|
||||
)
|
||||
self.audit.write(
|
||||
"REVISION_RESOLVED",
|
||||
"model_revision",
|
||||
str(exact_revision.id),
|
||||
{
|
||||
"requested_revision": revision,
|
||||
"resolved_commit_sha": source.resolved_commit_sha,
|
||||
"new_revision": existing_revision is None,
|
||||
},
|
||||
)
|
||||
self._commit()
|
||||
return self.snapshot_response(snapshot.id)
|
||||
|
||||
def snapshot_response(self, snapshot_id: uuid.UUID) -> UpstreamSnapshotResponse:
|
||||
snapshot = self.repo.snapshot(snapshot_id)
|
||||
if not snapshot:
|
||||
raise RegistryNotFound("upstream snapshot not found")
|
||||
return UpstreamSnapshotResponse(
|
||||
id=snapshot.id,
|
||||
model_id=snapshot.model_id,
|
||||
provider=snapshot.provider,
|
||||
repository_id=snapshot.repository_id,
|
||||
requested_revision=snapshot.requested_revision,
|
||||
resolved_commit_sha=snapshot.resolved_commit_sha,
|
||||
access_state=snapshot.access_state,
|
||||
metadata_snapshot=snapshot.metadata_snapshot,
|
||||
card_metadata=snapshot.card_metadata,
|
||||
security_metadata=snapshot.security_metadata,
|
||||
source_updated_at=snapshot.source_updated_at,
|
||||
observed_at=snapshot.observed_at,
|
||||
stale=_aware(snapshot.stale_after) <= _utcnow(),
|
||||
stale_after=snapshot.stale_after,
|
||||
files=[
|
||||
UpstreamFileResponse.model_validate(item) for item in self.repo.files(snapshot.id)
|
||||
],
|
||||
)
|
||||
|
||||
def latest_snapshot(self, model_id: uuid.UUID) -> UpstreamSnapshotResponse:
|
||||
if not self.session.get(Model, model_id):
|
||||
raise RegistryNotFound("model not found")
|
||||
snapshot = self.repo.latest_snapshot(model_id)
|
||||
if not snapshot:
|
||||
raise RegistryNotFound("model has no upstream metadata snapshot")
|
||||
return self.snapshot_response(snapshot.id)
|
||||
|
||||
def artifact_sets(self, revision_id: uuid.UUID) -> list[ArtifactSetResponse]:
|
||||
if not self.session.get(ModelRevision, revision_id):
|
||||
raise RegistryNotFound("model revision not found")
|
||||
return [
|
||||
ArtifactSetResponse.model_validate(item)
|
||||
for item in self.repo.artifact_sets(revision_id)
|
||||
]
|
||||
|
||||
def create_plan(self, request: DownloadPlanCreate) -> DownloadPlanResponse:
|
||||
artifact_set = self.repo.artifact_set(request.artifact_set_id)
|
||||
root = self.session.get(StorageRoot, request.storage_root_id)
|
||||
node = self.session.get(ComputeNode, request.compute_node_id)
|
||||
if not artifact_set:
|
||||
raise RegistryNotFound("artifact set not found")
|
||||
if not root:
|
||||
raise RegistryNotFound("storage root not found")
|
||||
if not node:
|
||||
raise RegistryNotFound("compute node not found")
|
||||
if root.compute_node_id != node.id:
|
||||
raise AcquisitionError("storage root belongs to a different compute node")
|
||||
if not root.agent_path:
|
||||
raise AcquisitionError("storage root has no node-agent path mapping")
|
||||
if (
|
||||
node.decommissioned_at is not None
|
||||
or not node.enabled
|
||||
or node.liveness_state != NodeLiveness.ONLINE
|
||||
):
|
||||
raise AcquisitionError("target compute node is not online and enabled")
|
||||
if "artifact.acquire.v1" not in node.agent_capabilities:
|
||||
raise AcquisitionError("target agent does not advertise artifact.acquire.v1")
|
||||
snapshot = self.repo.snapshot(artifact_set.snapshot_id)
|
||||
revision = self.session.get(ModelRevision, artifact_set.revision_id)
|
||||
if not snapshot or not revision:
|
||||
raise AcquisitionError("artifact set provenance is incomplete")
|
||||
if snapshot.access_state != "public" and not self.settings.hf_token:
|
||||
raise AcquisitionError(
|
||||
f"repository access is {snapshot.access_state}; Hugging Face authentication is required"
|
||||
)
|
||||
source_files = {item.path: item for item in self.repo.files(snapshot.id)}
|
||||
files = [source_files[path] for path in artifact_set.selected_paths if path in source_files]
|
||||
if len(files) != artifact_set.file_count or any(item.size_bytes is None for item in files):
|
||||
raise AcquisitionError("artifact set has incomplete file-size metadata")
|
||||
decision = RegistryService.capacity_decision(root, artifact_set.total_size_bytes)
|
||||
if not decision.allowed:
|
||||
self.audit.write(
|
||||
"DOWNLOAD_PLAN_BLOCKED",
|
||||
"artifact_set",
|
||||
str(artifact_set.id),
|
||||
decision.model_dump(mode="json"),
|
||||
outcome="blocked",
|
||||
)
|
||||
self._commit()
|
||||
raise AcquisitionError(decision.reason, decision.model_dump(mode="json"))
|
||||
now = _utcnow()
|
||||
payload = {
|
||||
"artifact_set_id": str(artifact_set.id),
|
||||
"repository_id": snapshot.repository_id,
|
||||
"resolved_commit_sha": revision.resolved_commit_sha,
|
||||
"compute_node_id": str(node.id),
|
||||
"storage_root_id": str(root.id),
|
||||
"target_root": root.agent_path,
|
||||
"files": [
|
||||
{
|
||||
"path": item.path,
|
||||
"size_bytes": item.size_bytes,
|
||||
"upstream_sha256": item.upstream_sha256,
|
||||
"file_format": item.file_format,
|
||||
"role": item.role,
|
||||
"risk_flags": item.risk_flags,
|
||||
}
|
||||
for item in files
|
||||
],
|
||||
}
|
||||
key = _canonical_hash(payload)
|
||||
existing = self.repo.plan_by_key(key)
|
||||
if existing:
|
||||
return self.plan_response(existing.id)
|
||||
preflight = {
|
||||
"capacity": decision.model_dump(mode="json"),
|
||||
"node_liveness": node.liveness_state,
|
||||
"agent_capability": "artifact.acquire.v1",
|
||||
"storage_agent_path": root.agent_path,
|
||||
"exact_revision": True,
|
||||
"trust_remote_code": False,
|
||||
}
|
||||
plan = self.repo.add(
|
||||
DownloadPlan(
|
||||
artifact_set_id=artifact_set.id,
|
||||
compute_node_id=node.id,
|
||||
storage_root_id=root.id,
|
||||
repository_id=snapshot.repository_id,
|
||||
resolved_commit_sha=revision.resolved_commit_sha,
|
||||
total_size_bytes=artifact_set.total_size_bytes,
|
||||
file_count=len(files),
|
||||
status="planned",
|
||||
idempotency_key=key,
|
||||
preflight=preflight,
|
||||
immutable_payload=payload,
|
||||
planned_at=now,
|
||||
expires_at=now + timedelta(seconds=request.expires_in_seconds),
|
||||
immutable_at=now,
|
||||
)
|
||||
)
|
||||
for ordinal, item in enumerate(files):
|
||||
self.repo.add(
|
||||
DownloadPlanFile(
|
||||
plan_id=plan.id,
|
||||
ordinal=ordinal,
|
||||
path=item.path,
|
||||
size_bytes=int(item.size_bytes or 0),
|
||||
upstream_sha256=item.upstream_sha256,
|
||||
file_format=item.file_format,
|
||||
role=item.role,
|
||||
risk_flags=item.risk_flags,
|
||||
)
|
||||
)
|
||||
self.audit.write(
|
||||
"DOWNLOAD_PLAN_CREATED",
|
||||
"download_plan",
|
||||
str(plan.id),
|
||||
{"idempotency_key": key, "total_size_bytes": plan.total_size_bytes},
|
||||
)
|
||||
self._commit()
|
||||
return self.plan_response(plan.id)
|
||||
|
||||
def approve_plan(self, plan_id: uuid.UUID) -> DownloadPlanResponse:
|
||||
plan = self.repo.plan(plan_id)
|
||||
if not plan:
|
||||
raise RegistryNotFound("download plan not found")
|
||||
if plan.status == "ready":
|
||||
return self.plan_response(plan.id)
|
||||
if plan.status != "planned":
|
||||
raise AcquisitionError("only a planned download plan can be approved")
|
||||
if _aware(plan.expires_at) <= _utcnow():
|
||||
plan.status = "stale"
|
||||
self._commit()
|
||||
raise AcquisitionError("download plan is stale; create a new immutable plan")
|
||||
artifact_set = self.repo.artifact_set(plan.artifact_set_id)
|
||||
if not artifact_set:
|
||||
raise AcquisitionError("download plan lost its artifact set")
|
||||
if artifact_set.security_status == "risk_detected":
|
||||
self.audit.write(
|
||||
"DOWNLOAD_PLAN_APPROVAL_BLOCKED",
|
||||
"download_plan",
|
||||
str(plan.id),
|
||||
{"reason": "artifact selection contains blocked security risks"},
|
||||
outcome="blocked",
|
||||
)
|
||||
self._commit()
|
||||
raise AcquisitionError("artifact selection contains blocked security risks")
|
||||
plan.status = "ready"
|
||||
artifact_set.status = "ready"
|
||||
self.audit.write(
|
||||
"DOWNLOAD_PLAN_APPROVED",
|
||||
"download_plan",
|
||||
str(plan.id),
|
||||
{
|
||||
"resolved_commit_sha": plan.resolved_commit_sha,
|
||||
"file_count": plan.file_count,
|
||||
"security_preflight": "passed",
|
||||
},
|
||||
)
|
||||
self._commit()
|
||||
return self.plan_response(plan.id)
|
||||
|
||||
def plan_response(self, plan_id: uuid.UUID) -> DownloadPlanResponse:
|
||||
plan = self.repo.plan(plan_id)
|
||||
if not plan:
|
||||
raise RegistryNotFound("download plan not found")
|
||||
return DownloadPlanResponse(
|
||||
**{
|
||||
name: getattr(plan, name)
|
||||
for name in DownloadPlanResponse.model_fields
|
||||
if name not in {"files", "stale"}
|
||||
},
|
||||
stale=_aware(plan.expires_at) <= _utcnow(),
|
||||
files=[
|
||||
DownloadPlanFileResponse.model_validate(item)
|
||||
for item in self.repo.plan_files(plan.id)
|
||||
],
|
||||
)
|
||||
|
||||
def execute_plan(self, plan_id: uuid.UUID) -> ArtifactJobResponse:
|
||||
plan = self.repo.plan(plan_id)
|
||||
if not plan:
|
||||
raise RegistryNotFound("download plan not found")
|
||||
existing = self.repo.job_for_plan(plan.id)
|
||||
if existing:
|
||||
return ArtifactJobResponse.model_validate(existing)
|
||||
if plan.status != "ready":
|
||||
raise AcquisitionError("download plan requires explicit approval before execution")
|
||||
if _aware(plan.expires_at) <= _utcnow():
|
||||
plan.status = "stale"
|
||||
self._commit()
|
||||
raise AcquisitionError("download plan is stale; create a new immutable plan")
|
||||
root = self.session.get(StorageRoot, plan.storage_root_id)
|
||||
node = self.session.get(ComputeNode, plan.compute_node_id)
|
||||
if not root or not node:
|
||||
raise AcquisitionError("download target no longer exists")
|
||||
decision = RegistryService.capacity_decision(root, plan.total_size_bytes)
|
||||
if not decision.allowed:
|
||||
self.audit.write(
|
||||
"ARTIFACT_DOWNLOAD_BLOCKED",
|
||||
"download_plan",
|
||||
str(plan.id),
|
||||
decision.model_dump(mode="json"),
|
||||
outcome="blocked",
|
||||
)
|
||||
self._commit()
|
||||
raise AcquisitionError(
|
||||
"execution capacity preflight failed", decision.model_dump(mode="json")
|
||||
)
|
||||
if node.liveness_state != NodeLiveness.ONLINE or not node.enabled:
|
||||
raise AcquisitionError("target compute node is not online and enabled")
|
||||
job = self.repo.add(
|
||||
ArtifactJob(
|
||||
plan_id=plan.id,
|
||||
compute_node_id=plan.compute_node_id,
|
||||
storage_root_id=plan.storage_root_id,
|
||||
status="queued",
|
||||
idempotency_key=_canonical_hash({"job_for_plan": str(plan.id)}),
|
||||
total_bytes=plan.total_size_bytes,
|
||||
)
|
||||
)
|
||||
plan.status = "queued"
|
||||
artifact_set = self.repo.artifact_set(plan.artifact_set_id)
|
||||
if artifact_set:
|
||||
artifact_set.status = "queued"
|
||||
self.audit.write(
|
||||
"ARTIFACT_DOWNLOAD_QUEUED",
|
||||
"artifact_job",
|
||||
str(job.id),
|
||||
{"plan_id": str(plan.id), "compute_node_id": str(job.compute_node_id)},
|
||||
)
|
||||
self._commit()
|
||||
return ArtifactJobResponse.model_validate(job)
|
||||
|
||||
def jobs(self) -> list[ArtifactJobResponse]:
|
||||
return [ArtifactJobResponse.model_validate(item) for item in self.repo.jobs()]
|
||||
|
||||
def job(self, job_id: uuid.UUID) -> ArtifactJobResponse:
|
||||
job = self.repo.job(job_id)
|
||||
if not job:
|
||||
raise RegistryNotFound("artifact job not found")
|
||||
return ArtifactJobResponse.model_validate(job)
|
||||
|
||||
def cancel(self, job_id: uuid.UUID) -> ArtifactJobResponse:
|
||||
job = self.repo.job(job_id)
|
||||
if not job:
|
||||
raise RegistryNotFound("artifact job not found")
|
||||
if job.status in {"completed", "failed", "cancelled"}:
|
||||
raise AcquisitionError("terminal artifact jobs cannot be cancelled")
|
||||
job.cancel_requested = True
|
||||
if job.status == "queued":
|
||||
job.status = "cancelled"
|
||||
job.completed_at = _utcnow()
|
||||
self.audit.write(
|
||||
"DOWNLOAD_CANCELLED" if job.status == "cancelled" else "ARTIFACT_JOB_CANCEL_REQUESTED",
|
||||
"artifact_job",
|
||||
str(job.id),
|
||||
{},
|
||||
)
|
||||
self._commit()
|
||||
return ArtifactJobResponse.model_validate(job)
|
||||
|
||||
def retry(self, job_id: uuid.UUID) -> ArtifactJobResponse:
|
||||
job = self.repo.job(job_id)
|
||||
if not job:
|
||||
raise RegistryNotFound("artifact job not found")
|
||||
if job.status != "failed":
|
||||
raise AcquisitionError("only failed artifact jobs can be retried explicitly")
|
||||
plan = self.repo.plan(job.plan_id)
|
||||
node = self.session.get(ComputeNode, job.compute_node_id)
|
||||
root = self.session.get(StorageRoot, job.storage_root_id)
|
||||
if not plan or not node or not root:
|
||||
raise AcquisitionError("artifact job target or immutable plan no longer exists")
|
||||
if not node.enabled or node.liveness_state != NodeLiveness.ONLINE:
|
||||
raise AcquisitionError("target compute node is not online and enabled")
|
||||
decision = RegistryService.capacity_decision(root, plan.total_size_bytes)
|
||||
if not decision.allowed:
|
||||
raise AcquisitionError(
|
||||
"retry capacity preflight failed", decision.model_dump(mode="json")
|
||||
)
|
||||
previous_error = {"code": job.error_code, "message": job.error_message}
|
||||
job.status = "queued"
|
||||
job.cancel_requested = False
|
||||
job.progress_bytes = 0
|
||||
job.current_file = None
|
||||
job.error_code = None
|
||||
job.error_message = None
|
||||
job.completed_at = None
|
||||
job.lease_token_hash = None
|
||||
job.lease_expires_at = None
|
||||
job.result = {
|
||||
"operator_retry": True,
|
||||
"previous_error": previous_error,
|
||||
"resume_from_quarantine": bool(job.quarantine_relative_path),
|
||||
}
|
||||
plan.status = "queued"
|
||||
artifact_set = self.repo.artifact_set(plan.artifact_set_id)
|
||||
if artifact_set:
|
||||
artifact_set.status = "queued"
|
||||
self.audit.write(
|
||||
"ARTIFACT_JOB_OPERATOR_RETRY",
|
||||
"artifact_job",
|
||||
str(job.id),
|
||||
{
|
||||
"previous_attempts": job.attempt_count,
|
||||
"resume_from_quarantine": bool(job.quarantine_relative_path),
|
||||
},
|
||||
)
|
||||
self._commit()
|
||||
return ArtifactJobResponse.model_validate(job)
|
||||
|
||||
@staticmethod
|
||||
def _lease_hash(token: str) -> str:
|
||||
return hashlib.sha256(token.encode()).hexdigest()
|
||||
|
||||
def claim_next(self, node: ComputeNode) -> AgentArtifactJobLease | None:
|
||||
now = _utcnow()
|
||||
job = self.repo.claimable_job(node.id, now)
|
||||
if not job:
|
||||
return None
|
||||
plan = self.repo.plan(job.plan_id)
|
||||
root = self.session.get(StorageRoot, job.storage_root_id)
|
||||
if not plan or not root or not root.agent_path:
|
||||
raise AcquisitionError("claimable job has an invalid target")
|
||||
token = f"mflease_{secrets.token_urlsafe(48)}"
|
||||
job.lease_token_hash = self._lease_hash(token)
|
||||
job.lease_expires_at = now + timedelta(seconds=120)
|
||||
job.status = "claimed"
|
||||
job.attempt_count += 1
|
||||
job.started_at = job.started_at or now
|
||||
self.repo.add(
|
||||
ArtifactJobAttempt(job_id=job.id, attempt=job.attempt_count, status="claimed")
|
||||
)
|
||||
self.audit.write(
|
||||
"ARTIFACT_DOWNLOAD_STARTED",
|
||||
"artifact_job",
|
||||
str(job.id),
|
||||
{"attempt": job.attempt_count, "compute_node_id": str(node.id)},
|
||||
)
|
||||
self._commit()
|
||||
return AgentArtifactJobLease(
|
||||
job_id=job.id,
|
||||
lease_token=token,
|
||||
lease_expires_at=job.lease_expires_at,
|
||||
repository_id=plan.repository_id,
|
||||
resolved_commit_sha=plan.resolved_commit_sha,
|
||||
storage_root_id=root.id,
|
||||
target_root=root.agent_path,
|
||||
total_size_bytes=plan.total_size_bytes,
|
||||
reserve_bytes=root.reserve_bytes,
|
||||
reserve_percent=root.reserve_percent,
|
||||
files=[
|
||||
AgentArtifactJobFile.model_validate(item) for item in self.repo.plan_files(plan.id)
|
||||
],
|
||||
)
|
||||
|
||||
def _leased_job(self, job_id: uuid.UUID, node: ComputeNode, token: str) -> ArtifactJob:
|
||||
job = self.repo.job(job_id)
|
||||
if not job:
|
||||
raise RegistryNotFound("artifact job not found")
|
||||
if job.compute_node_id != node.id:
|
||||
raise AcquisitionError("artifact job belongs to a different node")
|
||||
if not job.lease_token_hash or not secrets.compare_digest(
|
||||
job.lease_token_hash, self._lease_hash(token)
|
||||
):
|
||||
raise AcquisitionError("invalid artifact-job lease")
|
||||
if not job.lease_expires_at or _aware(job.lease_expires_at) <= _utcnow():
|
||||
raise AcquisitionError("artifact-job lease expired")
|
||||
return job
|
||||
|
||||
def progress(
|
||||
self, job_id: uuid.UUID, node: ComputeNode, request: AgentJobProgress
|
||||
) -> AgentJobControl:
|
||||
job = self._leased_job(job_id, node, request.lease_token)
|
||||
if request.progress_bytes < job.progress_bytes or request.progress_bytes > job.total_bytes:
|
||||
raise AcquisitionError("artifact-job progress is out of bounds")
|
||||
job.status = request.status
|
||||
job.progress_bytes = request.progress_bytes
|
||||
job.current_file = request.current_file
|
||||
first_quarantine_observation = (
|
||||
job.quarantine_relative_path is None and request.quarantine_relative_path is not None
|
||||
)
|
||||
job.quarantine_relative_path = request.quarantine_relative_path
|
||||
job.lease_expires_at = _utcnow() + timedelta(seconds=120)
|
||||
if first_quarantine_observation:
|
||||
self.audit.write(
|
||||
"ARTIFACT_QUARANTINED",
|
||||
"artifact_job",
|
||||
str(job.id),
|
||||
{"quarantine_relative_path": request.quarantine_relative_path},
|
||||
)
|
||||
self._commit()
|
||||
return AgentJobControl(
|
||||
accepted=True,
|
||||
cancel_requested=job.cancel_requested,
|
||||
lease_expires_at=job.lease_expires_at,
|
||||
)
|
||||
|
||||
def complete(
|
||||
self, job_id: uuid.UUID, node: ComputeNode, request: AgentJobComplete
|
||||
) -> ArtifactJobResponse:
|
||||
job = self._leased_job(job_id, node, request.lease_token)
|
||||
if job.cancel_requested:
|
||||
raise AcquisitionError("artifact job was cancelled")
|
||||
plan = self.repo.plan(job.plan_id)
|
||||
if not plan:
|
||||
raise AcquisitionError("artifact job lost its immutable plan")
|
||||
expected = {item.path: item for item in self.repo.plan_files(plan.id)}
|
||||
if set(expected) != {item.path for item in request.files}:
|
||||
raise AcquisitionError("completed file inventory does not match immutable plan")
|
||||
for item in request.files:
|
||||
planned = expected[item.path]
|
||||
if planned.size_bytes != item.size_bytes:
|
||||
raise AcquisitionError(f"size mismatch for {item.path}")
|
||||
if planned.upstream_sha256 and planned.upstream_sha256 != item.sha256:
|
||||
raise AcquisitionError(f"upstream checksum mismatch for {item.path}")
|
||||
for finding in item.inspections:
|
||||
self.repo.add(
|
||||
ArtifactInspection(
|
||||
job_id=job.id,
|
||||
file_path=item.path,
|
||||
inspection_type=str(finding.get("type", "static"))[:64],
|
||||
status=str(finding.get("status", "observed"))[:32],
|
||||
severity=str(finding.get("severity", "info"))[:32],
|
||||
evidence=dict(finding.get("evidence", {})),
|
||||
)
|
||||
)
|
||||
if finding.get("severity") == "block":
|
||||
raise AcquisitionError(f"blocking static inspection for {item.path}")
|
||||
artifact_set = self.repo.artifact_set(plan.artifact_set_id)
|
||||
if not artifact_set:
|
||||
raise AcquisitionError("artifact set not found")
|
||||
for ordinal, item in enumerate(request.files):
|
||||
planned = expected[item.path]
|
||||
artifact = self.session.scalar(
|
||||
select(ModelArtifact).where(
|
||||
ModelArtifact.revision_id == artifact_set.revision_id,
|
||||
ModelArtifact.sha256 == item.sha256,
|
||||
)
|
||||
)
|
||||
if artifact is None:
|
||||
artifact = self.repo.add(
|
||||
ModelArtifact(
|
||||
revision_id=artifact_set.revision_id,
|
||||
filename=item.path,
|
||||
artifact_type=planned.role,
|
||||
serialization_format=planned.file_format,
|
||||
sha256=item.sha256,
|
||||
size_bytes=item.size_bytes,
|
||||
security_status="static_checks_passed_unapproved",
|
||||
license_status=artifact_set.license_status,
|
||||
status=ArtifactStatus.VERIFIED,
|
||||
verification_details={
|
||||
"job_id": str(job.id),
|
||||
"integrity": "streaming_sha256",
|
||||
"upstream_checksum": planned.upstream_sha256,
|
||||
"supply_chain_approval": "not_evaluated_m4",
|
||||
},
|
||||
quarantined=False,
|
||||
verified_at=_utcnow(),
|
||||
immutable_at=_utcnow(),
|
||||
)
|
||||
)
|
||||
location = self.session.scalar(
|
||||
select(ArtifactLocation).where(
|
||||
ArtifactLocation.storage_root_id == job.storage_root_id,
|
||||
ArtifactLocation.relative_path == item.relative_path,
|
||||
)
|
||||
)
|
||||
if location is None:
|
||||
self.repo.add(
|
||||
ArtifactLocation(
|
||||
artifact_id=artifact.id,
|
||||
storage_root_id=job.storage_root_id,
|
||||
relative_path=item.relative_path,
|
||||
status=ArtifactStatus.VERIFIED,
|
||||
size_bytes=item.size_bytes,
|
||||
observed_sha256=item.sha256,
|
||||
last_checked_at=_utcnow(),
|
||||
)
|
||||
)
|
||||
if self.session.get(ArtifactSetMember, (artifact_set.id, artifact.id)) is None:
|
||||
self.repo.add(
|
||||
ArtifactSetMember(
|
||||
artifact_set_id=artifact_set.id,
|
||||
artifact_id=artifact.id,
|
||||
ordinal=ordinal,
|
||||
required=True,
|
||||
)
|
||||
)
|
||||
artifact_set.status = "verified"
|
||||
artifact_set.availability = "local"
|
||||
artifact_set.completeness = "complete"
|
||||
artifact_set.security_status = "static_checks_passed_unapproved"
|
||||
plan.status = "completed"
|
||||
job.status = "completed"
|
||||
job.progress_bytes = job.total_bytes
|
||||
job.current_file = None
|
||||
job.promoted_relative_path = request.promoted_relative_path
|
||||
job.completed_at = _utcnow()
|
||||
job.lease_token_hash = None
|
||||
job.lease_expires_at = None
|
||||
job.result = {
|
||||
"files": len(request.files),
|
||||
"capacity_observation": request.capacity_observation,
|
||||
"integrity_verified": True,
|
||||
"supply_chain_approved": False,
|
||||
}
|
||||
root = self.session.get(StorageRoot, job.storage_root_id)
|
||||
capacity = request.capacity_observation
|
||||
if root:
|
||||
observed_capacity = capacity.get("capacity_bytes")
|
||||
observed_free = capacity.get("free_bytes")
|
||||
if isinstance(observed_capacity, int) and isinstance(observed_free, int):
|
||||
root.capacity_bytes = observed_capacity
|
||||
root.free_bytes = observed_free
|
||||
root.capacity_observed_at = _utcnow()
|
||||
root.status = RegistryService.capacity_decision(root, 0).status
|
||||
root.validation_details = {
|
||||
"source": "artifact_job_completion",
|
||||
"job_id": str(job.id),
|
||||
"agent_path": capacity.get("agent_path"),
|
||||
}
|
||||
attempt = self.session.scalar(
|
||||
select(ArtifactJobAttempt)
|
||||
.where(ArtifactJobAttempt.job_id == job.id)
|
||||
.order_by(ArtifactJobAttempt.attempt.desc(), ArtifactJobAttempt.started_at.desc())
|
||||
.limit(1)
|
||||
)
|
||||
if attempt:
|
||||
attempt.status = "completed"
|
||||
attempt.completed_at = _utcnow()
|
||||
self.audit.write(
|
||||
"ARTIFACT_SET_ACQUIRED",
|
||||
"artifact_set",
|
||||
str(artifact_set.id),
|
||||
{"job_id": str(job.id), "file_count": len(request.files), "approved": False},
|
||||
)
|
||||
for action in ("ARTIFACT_DOWNLOADED", "ARTIFACT_VERIFIED", "ARTIFACT_PROMOTED"):
|
||||
self.audit.write(
|
||||
action,
|
||||
"artifact_set",
|
||||
str(artifact_set.id),
|
||||
{"job_id": str(job.id), "file_count": len(request.files)},
|
||||
)
|
||||
self._commit()
|
||||
return ArtifactJobResponse.model_validate(job)
|
||||
|
||||
def fail(
|
||||
self, job_id: uuid.UUID, node: ComputeNode, request: AgentJobFailure
|
||||
) -> ArtifactJobResponse:
|
||||
job = self._leased_job(job_id, node, request.lease_token)
|
||||
retry = request.retryable and job.attempt_count < 3 and not job.cancel_requested
|
||||
job.status = "queued" if retry else "cancelled" if job.cancel_requested else "failed"
|
||||
job.error_code = request.error_code
|
||||
job.error_message = request.error_message
|
||||
job.result = {"failure_details": request.details, "retry_scheduled": retry}
|
||||
job.lease_token_hash = None
|
||||
job.lease_expires_at = None
|
||||
if not retry:
|
||||
job.completed_at = _utcnow()
|
||||
attempt = self.session.scalar(
|
||||
select(ArtifactJobAttempt)
|
||||
.where(ArtifactJobAttempt.job_id == job.id)
|
||||
.order_by(ArtifactJobAttempt.attempt.desc(), ArtifactJobAttempt.started_at.desc())
|
||||
.limit(1)
|
||||
)
|
||||
if attempt:
|
||||
attempt.status = "retry_scheduled" if retry else job.status
|
||||
attempt.completed_at = _utcnow()
|
||||
plan = self.repo.plan(job.plan_id)
|
||||
if plan:
|
||||
plan.status = job.status
|
||||
action = (
|
||||
"DOWNLOAD_CANCELLED"
|
||||
if job.status == "cancelled"
|
||||
else "ARTIFACT_JOB_RETRY_SCHEDULED"
|
||||
if retry
|
||||
else "ARTIFACT_REJECTED"
|
||||
if request.error_code in {"static_inspection_blocked", "verification_digest_mismatch"}
|
||||
else "ARTIFACT_DOWNLOAD_FAILED"
|
||||
)
|
||||
self.audit.write(
|
||||
action,
|
||||
"artifact_job",
|
||||
str(job.id),
|
||||
{"error_code": request.error_code, "retryable": request.retryable},
|
||||
outcome="blocked",
|
||||
)
|
||||
self._commit()
|
||||
return ArtifactJobResponse.model_validate(job)
|
||||
@@ -0,0 +1,558 @@
|
||||
"""Canonical, append-only audit writes, checkpoints and verification.
|
||||
|
||||
The chain head is a single serialization point. PostgreSQL writers use a transaction-scoped
|
||||
advisory lock; SQLite test databases use an engine-scoped process lock held until the owning
|
||||
transaction ends. An event and the durable singleton checkpoint advance in the same transaction.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import threading
|
||||
import uuid
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any, Protocol
|
||||
from weakref import WeakKeyDictionary
|
||||
|
||||
from sqlalchemy import event, select, text
|
||||
from sqlalchemy.engine import Connection, Engine
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from modelforge_api.domain.audit import (
|
||||
AUDIT_CHAIN_SINGLETON_ID,
|
||||
AUDIT_CURRENT_HASH_FORMAT,
|
||||
AUDIT_EMPTY_LEGACY_PREFIX_SEAL,
|
||||
AUDIT_HASH_FORMAT_V1,
|
||||
AUDIT_HASH_FORMAT_V2,
|
||||
AUDIT_LEGACY_PREFIX_DOMAIN,
|
||||
canonical_audit_payload_and_hash,
|
||||
canonical_audit_payload_text_and_hash,
|
||||
normalise_audit_event_id,
|
||||
normalise_audit_timestamp,
|
||||
)
|
||||
from modelforge_api.persistence.models import (
|
||||
AuditChainHead,
|
||||
AuditEvent,
|
||||
_append_canonical_audit_event,
|
||||
)
|
||||
|
||||
AUDIT_CHAIN_POSTGRES_LOCK_KEY = int.from_bytes(b"MF_AUDIT", byteorder="big", signed=False)
|
||||
_SQLITE_LOCK_INFO_KEY = "modelforge_audit_chain_lock"
|
||||
_SQLITE_LOCKS: WeakKeyDictionary[Engine, threading.Lock] = WeakKeyDictionary()
|
||||
_SQLITE_LOCKS_GUARD = threading.Lock()
|
||||
|
||||
|
||||
class AuditChainIntegrityError(RuntimeError):
|
||||
"""The writer cannot safely append to the authoritative audit chain."""
|
||||
|
||||
|
||||
class AuditEventLike(Protocol):
|
||||
id: Any
|
||||
sequence: int
|
||||
occurred_at: Any
|
||||
correlation_id: str
|
||||
actor_type: str
|
||||
actor_id: str
|
||||
action: str
|
||||
resource_type: str
|
||||
resource_id: str | None
|
||||
outcome: str
|
||||
details: dict[str, Any]
|
||||
previous_event_hash: str | None
|
||||
event_hash: str
|
||||
hash_format: str
|
||||
canonical_payload: str | None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class AuditContext:
|
||||
"""Authenticated/request identity seam for audit-producing services.
|
||||
|
||||
Existing callers can keep passing their actor fields directly. The API authentication boundary
|
||||
can instead construct this context with its request correlation id without changing chain code.
|
||||
"""
|
||||
|
||||
actor_type: str
|
||||
actor_id: str
|
||||
correlation_id: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class AuditChainCheckpoint:
|
||||
"""Transport-neutral checkpoint used by local and restored-database verification."""
|
||||
|
||||
singleton_id: int
|
||||
event_count: int
|
||||
last_sequence: int
|
||||
last_event_hash: str | None
|
||||
hash_format: str
|
||||
v2_start_sequence: int
|
||||
legacy_prefix_count: int
|
||||
legacy_prefix_seal: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class AuditEventRecord:
|
||||
"""Transport-neutral event used when recovery reads a database through psql."""
|
||||
|
||||
id: Any
|
||||
sequence: int
|
||||
occurred_at: Any
|
||||
correlation_id: str
|
||||
actor_type: str
|
||||
actor_id: str
|
||||
action: str
|
||||
resource_type: str
|
||||
resource_id: str | None
|
||||
outcome: str
|
||||
details: dict[str, Any]
|
||||
previous_event_hash: str | None
|
||||
event_hash: str
|
||||
hash_format: str
|
||||
canonical_payload: str | None = None
|
||||
|
||||
|
||||
def _decode_canonical_audit_payload(value: str) -> dict[str, Any]:
|
||||
def reject_duplicate_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
|
||||
result: dict[str, Any] = {}
|
||||
for key, item in pairs:
|
||||
if key in result:
|
||||
raise ValueError(f"duplicate canonical audit payload key {key!r}")
|
||||
result[key] = item
|
||||
return result
|
||||
|
||||
decoded = json.loads(value, object_pairs_hook=reject_duplicate_keys)
|
||||
if not isinstance(decoded, dict):
|
||||
raise ValueError("canonical audit payload is not an object")
|
||||
return decoded
|
||||
|
||||
|
||||
def _event_payload_violations(audit_event: AuditEventLike) -> list[str]:
|
||||
"""Verify one event's semantic payload and exact stored hash bytes."""
|
||||
|
||||
identity = str(audit_event.id)
|
||||
if audit_event.hash_format == AUDIT_HASH_FORMAT_V1:
|
||||
violations: list[str] = []
|
||||
if audit_event.canonical_payload is not None:
|
||||
violations.append(f"legacy audit event {identity} unexpectedly stores a v2 payload")
|
||||
try:
|
||||
_, expected_hash = canonical_audit_payload_and_hash(
|
||||
correlation_id=audit_event.correlation_id,
|
||||
actor_type=audit_event.actor_type,
|
||||
actor_id=audit_event.actor_id,
|
||||
action=audit_event.action,
|
||||
resource_type=audit_event.resource_type,
|
||||
resource_id=audit_event.resource_id,
|
||||
outcome=audit_event.outcome,
|
||||
details=audit_event.details,
|
||||
previous_event_hash=audit_event.previous_event_hash,
|
||||
hash_format=audit_event.hash_format,
|
||||
event_id=audit_event.id,
|
||||
occurred_at=audit_event.occurred_at,
|
||||
)
|
||||
except (TypeError, ValueError):
|
||||
violations.append(f"audit event {identity} has a non-canonical payload")
|
||||
else:
|
||||
if audit_event.event_hash != expected_hash:
|
||||
violations.append(
|
||||
f"audit event {identity} content hash does not match its payload"
|
||||
)
|
||||
return violations
|
||||
|
||||
if audit_event.hash_format != AUDIT_HASH_FORMAT_V2:
|
||||
return [f"audit event {identity} has an unsupported hash format"]
|
||||
canonical_payload = audit_event.canonical_payload
|
||||
if not isinstance(canonical_payload, str):
|
||||
return [f"audit event {identity} has no exact v2 canonical payload"]
|
||||
violations = []
|
||||
if hashlib.sha256(canonical_payload.encode("utf-8")).hexdigest() != audit_event.event_hash:
|
||||
violations.append(f"audit event {identity} content hash does not match stored bytes")
|
||||
try:
|
||||
decoded_payload = _decode_canonical_audit_payload(canonical_payload)
|
||||
_, expected_text, _expected_hash = canonical_audit_payload_text_and_hash(
|
||||
correlation_id=audit_event.correlation_id,
|
||||
actor_type=audit_event.actor_type,
|
||||
actor_id=audit_event.actor_id,
|
||||
action=audit_event.action,
|
||||
resource_type=audit_event.resource_type,
|
||||
resource_id=audit_event.resource_id,
|
||||
outcome=audit_event.outcome,
|
||||
details=audit_event.details,
|
||||
previous_event_hash=audit_event.previous_event_hash,
|
||||
hash_format=audit_event.hash_format,
|
||||
event_id=audit_event.id,
|
||||
occurred_at=audit_event.occurred_at,
|
||||
)
|
||||
expected_payload = _decode_canonical_audit_payload(expected_text)
|
||||
except (json.JSONDecodeError, TypeError, ValueError):
|
||||
violations.append(f"audit event {identity} has a non-canonical payload")
|
||||
else:
|
||||
canonical_decoded = json.dumps(
|
||||
decoded_payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False
|
||||
)
|
||||
canonical_expected = json.dumps(
|
||||
expected_payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False
|
||||
)
|
||||
if canonical_decoded != canonical_expected:
|
||||
violations.append(
|
||||
f"audit event {identity} content hash bytes do not describe its event columns"
|
||||
)
|
||||
return violations
|
||||
|
||||
|
||||
def _legacy_prefix_entry(event_row: AuditEventLike) -> bytes:
|
||||
payload = {
|
||||
"sequence": int(event_row.sequence),
|
||||
"id": normalise_audit_event_id(event_row.id),
|
||||
"occurred_at": normalise_audit_timestamp(event_row.occurred_at),
|
||||
"event_hash": str(event_row.event_hash),
|
||||
}
|
||||
return json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8") + b"\n"
|
||||
|
||||
|
||||
def legacy_audit_prefix_seal(events: Iterable[AuditEventLike]) -> str:
|
||||
"""Seal immutable v1 identities/timestamps without rewriting approved legacy events."""
|
||||
|
||||
digest = hashlib.sha256()
|
||||
digest.update(AUDIT_LEGACY_PREFIX_DOMAIN)
|
||||
for audit_event in events:
|
||||
digest.update(_legacy_prefix_entry(audit_event))
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
EMPTY_LEGACY_PREFIX_SEAL = AUDIT_EMPTY_LEGACY_PREFIX_SEAL
|
||||
|
||||
|
||||
def audit_chain_violations(
|
||||
events: Iterable[AuditEventLike], checkpoint: AuditChainHead | AuditChainCheckpoint | None
|
||||
) -> list[str]:
|
||||
"""Verify sequence, links, versioned hashes, the prefix seal and durable checkpoint."""
|
||||
|
||||
violations: list[str] = []
|
||||
if checkpoint is None:
|
||||
violations.append("audit chain checkpoint is missing")
|
||||
cutover = 1
|
||||
prefix_count = 0
|
||||
else:
|
||||
cutover = int(checkpoint.v2_start_sequence)
|
||||
prefix_count = int(checkpoint.legacy_prefix_count)
|
||||
if checkpoint.singleton_id != AUDIT_CHAIN_SINGLETON_ID:
|
||||
violations.append("audit chain checkpoint has an invalid singleton id")
|
||||
if checkpoint.hash_format != AUDIT_CURRENT_HASH_FORMAT:
|
||||
violations.append("audit chain checkpoint names an unsupported current hash format")
|
||||
if cutover < 1 or prefix_count != cutover - 1:
|
||||
violations.append("audit chain checkpoint has an invalid v2 cutover")
|
||||
|
||||
previous: AuditEventLike | None = None
|
||||
seen_sequences: set[int] = set()
|
||||
seen_event_ids: set[str] = set()
|
||||
observed = 0
|
||||
prefix_observed = 0
|
||||
prefix_digest = hashlib.sha256()
|
||||
prefix_digest.update(AUDIT_LEGACY_PREFIX_DOMAIN)
|
||||
for expected_sequence, audit_event in enumerate(events, start=1):
|
||||
observed += 1
|
||||
identity = str(audit_event.id)
|
||||
try:
|
||||
normalised_identity = normalise_audit_event_id(audit_event.id)
|
||||
except ValueError:
|
||||
violations.append(f"audit event {identity} has a non-canonical UUID")
|
||||
else:
|
||||
if normalised_identity in seen_event_ids:
|
||||
violations.append(f"audit event UUID {normalised_identity} is duplicated")
|
||||
seen_event_ids.add(normalised_identity)
|
||||
sequence = int(audit_event.sequence)
|
||||
if sequence in seen_sequences:
|
||||
violations.append(f"audit sequence {sequence} is duplicated at event {identity}")
|
||||
seen_sequences.add(sequence)
|
||||
if sequence != expected_sequence:
|
||||
violations.append(
|
||||
f"audit event {identity} has sequence {sequence}; expected {expected_sequence}"
|
||||
)
|
||||
|
||||
expected_link = previous.event_hash if previous is not None else None
|
||||
if audit_event.previous_event_hash != expected_link:
|
||||
position = "first event" if previous is None else f"event after {previous.id}"
|
||||
violations.append(
|
||||
f"audit event {identity} has an invalid previous hash for the {position}"
|
||||
)
|
||||
|
||||
expected_format = AUDIT_HASH_FORMAT_V1 if sequence < cutover else AUDIT_HASH_FORMAT_V2
|
||||
if audit_event.hash_format != expected_format:
|
||||
violations.append(
|
||||
f"audit event {identity} uses {audit_event.hash_format!r}; "
|
||||
f"expected {expected_format!r} at sequence {sequence}"
|
||||
)
|
||||
violations.extend(_event_payload_violations(audit_event))
|
||||
|
||||
if sequence <= prefix_count:
|
||||
prefix_observed += 1
|
||||
try:
|
||||
prefix_digest.update(_legacy_prefix_entry(audit_event))
|
||||
except (TypeError, ValueError):
|
||||
violations.append(
|
||||
f"legacy audit event {identity} has a non-canonical identity or timestamp"
|
||||
)
|
||||
previous = audit_event
|
||||
|
||||
if checkpoint is not None:
|
||||
observed_last_sequence = int(previous.sequence) if previous is not None else 0
|
||||
observed_last_hash = previous.event_hash if previous is not None else None
|
||||
if int(checkpoint.event_count) != observed:
|
||||
violations.append(
|
||||
f"audit checkpoint records {checkpoint.event_count} events; observed {observed}"
|
||||
)
|
||||
if int(checkpoint.last_sequence) != observed_last_sequence:
|
||||
violations.append(
|
||||
"audit checkpoint last sequence does not match the retained event suffix"
|
||||
)
|
||||
if checkpoint.last_event_hash != observed_last_hash:
|
||||
violations.append("audit checkpoint last hash does not match the retained event suffix")
|
||||
if prefix_observed != prefix_count:
|
||||
violations.append(
|
||||
f"audit checkpoint seals {prefix_count} legacy events; observed {prefix_observed}"
|
||||
)
|
||||
if checkpoint.legacy_prefix_seal != prefix_digest.hexdigest():
|
||||
violations.append("audit legacy-prefix seal does not match immutable legacy history")
|
||||
return violations
|
||||
|
||||
|
||||
def _audit_engine(session: Session) -> Engine:
|
||||
bind = session.get_bind()
|
||||
if isinstance(bind, Connection):
|
||||
return bind.engine
|
||||
return bind
|
||||
|
||||
|
||||
def _acquire_audit_write_lock(session: Session) -> None:
|
||||
"""Hold the chain-head lock until the current root transaction completes."""
|
||||
|
||||
engine = _audit_engine(session)
|
||||
dialect = engine.dialect.name
|
||||
if dialect == "postgresql":
|
||||
session.execute(
|
||||
text("SELECT pg_advisory_xact_lock(:lock_key)"),
|
||||
{"lock_key": AUDIT_CHAIN_POSTGRES_LOCK_KEY},
|
||||
)
|
||||
return
|
||||
if dialect != "sqlite":
|
||||
raise RuntimeError(f"audit-chain writes do not support the {dialect!r} database dialect")
|
||||
|
||||
if _SQLITE_LOCK_INFO_KEY in session.info:
|
||||
return
|
||||
if not session.in_transaction():
|
||||
session.begin()
|
||||
with _SQLITE_LOCKS_GUARD:
|
||||
lock = _SQLITE_LOCKS.setdefault(engine, threading.Lock())
|
||||
lock.acquire()
|
||||
session.info[_SQLITE_LOCK_INFO_KEY] = lock
|
||||
|
||||
|
||||
@event.listens_for(Session, "after_transaction_end")
|
||||
def _release_sqlite_audit_write_lock(session: Session, transaction: Any) -> None:
|
||||
if transaction.parent is not None:
|
||||
return
|
||||
lock = session.info.pop(_SQLITE_LOCK_INFO_KEY, None)
|
||||
if lock is not None:
|
||||
lock.release()
|
||||
|
||||
|
||||
def _checkpoint_from_mapping(row: Any) -> AuditChainCheckpoint:
|
||||
return AuditChainCheckpoint(
|
||||
singleton_id=int(row.singleton_id),
|
||||
event_count=int(row.event_count),
|
||||
last_sequence=int(row.last_sequence),
|
||||
last_event_hash=row.last_event_hash,
|
||||
hash_format=str(row.hash_format),
|
||||
v2_start_sequence=int(row.v2_start_sequence),
|
||||
legacy_prefix_count=int(row.legacy_prefix_count),
|
||||
legacy_prefix_seal=str(row.legacy_prefix_seal),
|
||||
)
|
||||
|
||||
|
||||
def _load_checkpoint(session: Session) -> AuditChainCheckpoint:
|
||||
row = session.execute(
|
||||
select(
|
||||
AuditChainHead.singleton_id,
|
||||
AuditChainHead.event_count,
|
||||
AuditChainHead.last_sequence,
|
||||
AuditChainHead.last_event_hash,
|
||||
AuditChainHead.hash_format,
|
||||
AuditChainHead.v2_start_sequence,
|
||||
AuditChainHead.legacy_prefix_count,
|
||||
AuditChainHead.legacy_prefix_seal,
|
||||
).where(AuditChainHead.singleton_id == AUDIT_CHAIN_SINGLETON_ID)
|
||||
).one_or_none()
|
||||
if row is not None:
|
||||
return _checkpoint_from_mapping(row)
|
||||
raise AuditChainIntegrityError(
|
||||
"audit chain checkpoint is missing; only schema creation or the audited migration may seed it"
|
||||
)
|
||||
|
||||
|
||||
def _assert_checkpoint_is_appendable(
|
||||
session: Session, checkpoint: AuditChainCheckpoint
|
||||
) -> AuditEvent | None:
|
||||
"""Validate the locked checkpoint and constant-size retained tail before append.
|
||||
|
||||
Migration and recovery seal/verify the complete immutable prefix. Runtime append therefore
|
||||
proves the checkpoint shape, current tail hash/link and compare-and-set predecessor in O(1).
|
||||
A privileged edit in older middle history is intentionally the responsibility of the explicit
|
||||
strict invariant/recovery gates; ordinary SQLAlchemy audit DML is blocked separately.
|
||||
"""
|
||||
|
||||
violations: list[str] = []
|
||||
if checkpoint.singleton_id != AUDIT_CHAIN_SINGLETON_ID:
|
||||
violations.append("checkpoint singleton id is invalid")
|
||||
if checkpoint.hash_format != AUDIT_CURRENT_HASH_FORMAT:
|
||||
violations.append("checkpoint hash format is unsupported")
|
||||
if checkpoint.event_count < 0 or checkpoint.last_sequence < 0:
|
||||
violations.append("checkpoint counts cannot be negative")
|
||||
if checkpoint.event_count != checkpoint.last_sequence:
|
||||
violations.append("checkpoint event count and last sequence disagree")
|
||||
if checkpoint.v2_start_sequence < 1:
|
||||
violations.append("checkpoint v2 cutover is invalid")
|
||||
if checkpoint.legacy_prefix_count != checkpoint.v2_start_sequence - 1:
|
||||
violations.append("checkpoint legacy-prefix count and v2 cutover disagree")
|
||||
if checkpoint.legacy_prefix_count > checkpoint.event_count:
|
||||
violations.append("checkpoint legacy prefix exceeds the retained event count")
|
||||
if (
|
||||
len(checkpoint.legacy_prefix_seal) != 64
|
||||
or any(character not in "0123456789abcdef" for character in checkpoint.legacy_prefix_seal)
|
||||
):
|
||||
violations.append("checkpoint legacy-prefix seal is malformed")
|
||||
if (
|
||||
checkpoint.legacy_prefix_count == 0
|
||||
and checkpoint.legacy_prefix_seal != EMPTY_LEGACY_PREFIX_SEAL
|
||||
):
|
||||
violations.append("empty legacy-prefix checkpoint has the wrong seal")
|
||||
|
||||
tail = list(
|
||||
session.scalars(
|
||||
select(AuditEvent)
|
||||
.order_by(AuditEvent.sequence.desc(), AuditEvent.id.desc())
|
||||
.limit(2)
|
||||
.execution_options(populate_existing=True)
|
||||
)
|
||||
)
|
||||
latest = tail[0] if tail else None
|
||||
if checkpoint.event_count == 0:
|
||||
if latest is not None:
|
||||
violations.append("empty checkpoint has a retained audit tail")
|
||||
if checkpoint.last_event_hash is not None:
|
||||
violations.append("empty checkpoint carries a last-event hash")
|
||||
else:
|
||||
if latest is None:
|
||||
violations.append("non-empty checkpoint has no retained audit tail")
|
||||
elif (
|
||||
latest.sequence != checkpoint.last_sequence
|
||||
or latest.event_hash != checkpoint.last_event_hash
|
||||
):
|
||||
violations.append("checkpoint does not identify the current retained audit tail")
|
||||
if checkpoint.last_event_hash is None:
|
||||
violations.append("non-empty checkpoint has no last-event hash")
|
||||
elif len(checkpoint.last_event_hash) != 64 or any(
|
||||
character not in "0123456789abcdef"
|
||||
for character in checkpoint.last_event_hash
|
||||
):
|
||||
violations.append("checkpoint last-event hash is malformed")
|
||||
|
||||
if latest is not None:
|
||||
expected_format = (
|
||||
AUDIT_HASH_FORMAT_V1
|
||||
if latest.sequence < checkpoint.v2_start_sequence
|
||||
else AUDIT_HASH_FORMAT_V2
|
||||
)
|
||||
if latest.hash_format != expected_format:
|
||||
violations.append("retained audit tail uses the wrong hash format")
|
||||
payload_violations = _event_payload_violations(latest)
|
||||
violations.extend(
|
||||
f"retained audit tail: {violation}" for violation in payload_violations
|
||||
)
|
||||
|
||||
if latest.sequence == 1:
|
||||
if latest.previous_event_hash is not None:
|
||||
violations.append("first retained audit event has a previous hash")
|
||||
if len(tail) != 1:
|
||||
violations.append("checkpoint count one has more than one retained event")
|
||||
elif latest.sequence > 1:
|
||||
if len(tail) != 2:
|
||||
violations.append("retained audit tail has no predecessor")
|
||||
else:
|
||||
predecessor = tail[1]
|
||||
if predecessor.sequence != latest.sequence - 1:
|
||||
violations.append("retained audit tail predecessor is not contiguous")
|
||||
if latest.previous_event_hash != predecessor.event_hash:
|
||||
violations.append("retained audit tail link does not match its predecessor")
|
||||
|
||||
if violations:
|
||||
raise AuditChainIntegrityError(
|
||||
"audit checkpoint/tail failed pre-append verification: "
|
||||
+ "; ".join(violations[:5])
|
||||
)
|
||||
return latest
|
||||
|
||||
|
||||
class AuditWriter:
|
||||
def __init__(
|
||||
self,
|
||||
session: Session,
|
||||
actor_type: str | None = None,
|
||||
actor_id: str | None = None,
|
||||
*,
|
||||
context: AuditContext | None = None,
|
||||
) -> None:
|
||||
if context is not None:
|
||||
if actor_type is not None or actor_id is not None:
|
||||
raise ValueError("pass either an audit context or actor fields, not both")
|
||||
resolved = context
|
||||
else:
|
||||
if actor_type is None or actor_id is None:
|
||||
raise ValueError("audit actor type and id are required")
|
||||
resolved = AuditContext(actor_type=actor_type, actor_id=actor_id)
|
||||
self.session = session
|
||||
self.context = resolved
|
||||
|
||||
def write(
|
||||
self,
|
||||
action: str,
|
||||
resource_type: str,
|
||||
resource_id: str | None,
|
||||
details: dict[str, Any],
|
||||
outcome: str = "success",
|
||||
) -> AuditEvent:
|
||||
_acquire_audit_write_lock(self.session)
|
||||
try:
|
||||
checkpoint = _load_checkpoint(self.session)
|
||||
previous = _assert_checkpoint_is_appendable(self.session, checkpoint)
|
||||
event_id = uuid.uuid4()
|
||||
occurred_at = datetime.now(UTC)
|
||||
return _append_canonical_audit_event(
|
||||
self.session,
|
||||
event_id=event_id,
|
||||
occurred_at=occurred_at,
|
||||
correlation_id=self.context.correlation_id or str(uuid.uuid4()),
|
||||
actor_type=self.context.actor_type,
|
||||
actor_id=self.context.actor_id,
|
||||
action=action,
|
||||
resource_type=resource_type,
|
||||
resource_id=resource_id,
|
||||
outcome=outcome,
|
||||
details=details,
|
||||
previous_event_hash=previous.event_hash if previous else None,
|
||||
expected_event_count=checkpoint.event_count,
|
||||
expected_last_sequence=checkpoint.last_sequence,
|
||||
expected_last_event_hash=checkpoint.last_event_hash,
|
||||
expected_hash_format=checkpoint.hash_format,
|
||||
expected_v2_start_sequence=checkpoint.v2_start_sequence,
|
||||
expected_legacy_prefix_count=checkpoint.legacy_prefix_count,
|
||||
expected_legacy_prefix_seal=checkpoint.legacy_prefix_seal,
|
||||
)
|
||||
except Exception:
|
||||
# A caller must never be able to catch an audit failure and commit an unaudited domain
|
||||
# mutation or a detached event. Roll back the complete owning transaction fail-closed.
|
||||
self.session.rollback()
|
||||
raise
|
||||
@@ -0,0 +1,267 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from typing import Any, cast
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from modelforge_api.domain.capability_evaluation import (
|
||||
ALLOWED_METRICS,
|
||||
CapabilityAdvisorResponse,
|
||||
CapabilityCaseResult,
|
||||
CapabilityEvaluationCase,
|
||||
CapabilityEvaluationRunCreate,
|
||||
CapabilityEvaluationRunResponse,
|
||||
CapabilityEvaluationSuiteCreate,
|
||||
CapabilityEvaluationSuiteResponse,
|
||||
CapabilityRecommendationState,
|
||||
EvaluationType,
|
||||
MetricDefinition,
|
||||
)
|
||||
from modelforge_api.persistence.models import (
|
||||
Capability,
|
||||
CapabilityContract,
|
||||
CapabilityDeployment,
|
||||
CapabilityEvaluationRun,
|
||||
CapabilityEvaluationSuite,
|
||||
)
|
||||
from modelforge_api.services.serving import ServingError
|
||||
|
||||
|
||||
def _digest(value: Any) -> str:
|
||||
raw = json.dumps(value, sort_keys=True, separators=(",", ":"), default=str)
|
||||
return hashlib.sha256(raw.encode()).hexdigest()
|
||||
|
||||
|
||||
class CapabilityEvaluationService:
|
||||
def __init__(self, session: Session) -> None:
|
||||
self.session = session
|
||||
|
||||
def _contract(self, key: str, version: int) -> CapabilityContract | None:
|
||||
return self.session.scalar(
|
||||
select(CapabilityContract)
|
||||
.join(Capability, Capability.id == CapabilityContract.capability_id)
|
||||
.where(Capability.key == key, CapabilityContract.version == version)
|
||||
)
|
||||
|
||||
def create_suite(
|
||||
self, request: CapabilityEvaluationSuiteCreate
|
||||
) -> CapabilityEvaluationSuiteResponse:
|
||||
contract = self._contract(request.capability, request.contract_version)
|
||||
if not contract:
|
||||
raise ServingError(404, "CAPABILITY_NOT_FOUND", "capability contract was not found")
|
||||
contract_type = str(contract.contract.get("estate", {}).get("evaluation_type", ""))
|
||||
if contract_type != request.evaluation_type:
|
||||
raise ServingError(409, "EVALUATION_TYPE_MISMATCH", "suite type differs from capability contract")
|
||||
definition = request.model_dump(mode="json")
|
||||
digest = _digest(definition)
|
||||
existing = self.session.scalar(
|
||||
select(CapabilityEvaluationSuite).where(
|
||||
CapabilityEvaluationSuite.definition_digest == digest
|
||||
)
|
||||
)
|
||||
if existing:
|
||||
return self.suite_response(existing)
|
||||
suite = CapabilityEvaluationSuite(
|
||||
capability_contract_id=contract.id,
|
||||
key=request.key,
|
||||
evaluation_type=request.evaluation_type,
|
||||
revision=request.revision,
|
||||
dataset_revision=request.dataset_revision,
|
||||
definition_digest=digest,
|
||||
metric_definitions={item.name: item.model_dump(mode="json") for item in request.metrics},
|
||||
case_definitions=[item.model_dump(mode="json") for item in request.cases],
|
||||
thresholds=request.thresholds,
|
||||
)
|
||||
self.session.add(suite)
|
||||
self.session.commit()
|
||||
self.session.refresh(suite)
|
||||
return self.suite_response(suite)
|
||||
|
||||
def suite_response(self, suite: CapabilityEvaluationSuite) -> CapabilityEvaluationSuiteResponse:
|
||||
contract = self.session.get(CapabilityContract, suite.capability_contract_id)
|
||||
capability = self.session.get(Capability, contract.capability_id) if contract else None
|
||||
if not contract or not capability:
|
||||
raise ServingError(409, "EVALUATION_PROVENANCE_INCOMPLETE", "suite contract is missing")
|
||||
return CapabilityEvaluationSuiteResponse(
|
||||
id=suite.id,
|
||||
capability=capability.key,
|
||||
contract_version=contract.version,
|
||||
key=suite.key,
|
||||
evaluation_type=cast(EvaluationType, suite.evaluation_type),
|
||||
revision=suite.revision,
|
||||
dataset_revision=suite.dataset_revision,
|
||||
definition_digest=suite.definition_digest,
|
||||
metrics=[
|
||||
MetricDefinition.model_validate(item)
|
||||
for item in suite.metric_definitions.values()
|
||||
],
|
||||
cases=[
|
||||
CapabilityEvaluationCase.model_validate(item)
|
||||
for item in suite.case_definitions
|
||||
],
|
||||
thresholds={key: float(value) for key, value in suite.thresholds.items()},
|
||||
created_at=suite.created_at,
|
||||
)
|
||||
|
||||
def suites(self) -> list[CapabilityEvaluationSuiteResponse]:
|
||||
rows = self.session.scalars(
|
||||
select(CapabilityEvaluationSuite).order_by(CapabilityEvaluationSuite.created_at)
|
||||
).all()
|
||||
return [self.suite_response(item) for item in rows]
|
||||
|
||||
def create_run(self, request: CapabilityEvaluationRunCreate) -> CapabilityEvaluationRunResponse:
|
||||
suite = self.session.get(CapabilityEvaluationSuite, request.suite_id)
|
||||
deployment = self.session.get(CapabilityDeployment, request.capability_deployment_id)
|
||||
if not suite or not deployment:
|
||||
raise ServingError(404, "EVALUATION_TARGET_NOT_FOUND", "suite or deployment was not found")
|
||||
if suite.capability_contract_id != deployment.capability_contract_id:
|
||||
raise ServingError(409, "EVALUATION_TARGET_MISMATCH", "suite and deployment capabilities differ")
|
||||
declared = set(suite.metric_definitions)
|
||||
if set(request.metrics) != declared or declared - ALLOWED_METRICS[suite.evaluation_type]:
|
||||
raise ServingError(422, "INVALID_METRICS", "aggregate metrics do not match the suite")
|
||||
expected_cases = {str(item["key"]) for item in suite.case_definitions}
|
||||
actual_cases = {item.case_key for item in request.cases}
|
||||
if actual_cases != expected_cases:
|
||||
raise ServingError(422, "INVALID_CASE_SET", "run must contain every immutable suite case exactly once")
|
||||
evidence_payload = request.model_dump(mode="json")
|
||||
digest = _digest(evidence_payload)
|
||||
existing = self.session.scalar(
|
||||
select(CapabilityEvaluationRun).where(CapabilityEvaluationRun.evidence_digest == digest)
|
||||
)
|
||||
if existing:
|
||||
return self.run_response(existing)
|
||||
run = CapabilityEvaluationRun(
|
||||
suite_id=suite.id,
|
||||
capability_deployment_id=deployment.id,
|
||||
status=request.status,
|
||||
metric_values=request.metrics,
|
||||
case_results=[item.model_dump(mode="json") for item in request.cases],
|
||||
resource_metrics=request.resource_metrics,
|
||||
environment_fingerprint=request.environment_fingerprint,
|
||||
evidence_digest=digest,
|
||||
evidence={**request.evidence, "payload_persisted": False, "content_logged": False},
|
||||
started_at=request.started_at,
|
||||
completed_at=request.completed_at,
|
||||
)
|
||||
self.session.add(run)
|
||||
self.session.commit()
|
||||
self.session.refresh(run)
|
||||
return self.run_response(run)
|
||||
|
||||
def run_response(self, run: CapabilityEvaluationRun) -> CapabilityEvaluationRunResponse:
|
||||
suite = self.session.get(CapabilityEvaluationSuite, run.suite_id)
|
||||
if not suite:
|
||||
raise ServingError(409, "EVALUATION_PROVENANCE_INCOMPLETE", "run suite is missing")
|
||||
return CapabilityEvaluationRunResponse(
|
||||
id=run.id,
|
||||
suite_id=run.suite_id,
|
||||
capability_deployment_id=run.capability_deployment_id,
|
||||
evaluation_type=cast(EvaluationType, suite.evaluation_type),
|
||||
status=run.status,
|
||||
metrics=run.metric_values,
|
||||
cases=[CapabilityCaseResult.model_validate(item) for item in run.case_results],
|
||||
resource_metrics=run.resource_metrics,
|
||||
environment_fingerprint=run.environment_fingerprint,
|
||||
evidence_digest=run.evidence_digest,
|
||||
evidence=run.evidence,
|
||||
started_at=run.started_at,
|
||||
completed_at=run.completed_at,
|
||||
created_at=run.created_at,
|
||||
)
|
||||
|
||||
def runs(self) -> list[CapabilityEvaluationRunResponse]:
|
||||
rows = self.session.scalars(
|
||||
select(CapabilityEvaluationRun).order_by(CapabilityEvaluationRun.created_at)
|
||||
).all()
|
||||
return [self.run_response(item) for item in rows]
|
||||
|
||||
def advisor(self) -> list[CapabilityAdvisorResponse]:
|
||||
contracts = self.session.execute(
|
||||
select(CapabilityContract, Capability)
|
||||
.join(Capability, Capability.id == CapabilityContract.capability_id)
|
||||
.order_by(Capability.key, CapabilityContract.version)
|
||||
).all()
|
||||
result: list[CapabilityAdvisorResponse] = []
|
||||
for contract, capability in contracts:
|
||||
deployment = self.session.scalar(
|
||||
select(CapabilityDeployment)
|
||||
.where(
|
||||
CapabilityDeployment.capability_contract_id == contract.id,
|
||||
CapabilityDeployment.status.in_(["candidate", "approved", "active", "stable"]),
|
||||
)
|
||||
.order_by(CapabilityDeployment.production.desc(), CapabilityDeployment.created_at.desc())
|
||||
)
|
||||
if not deployment:
|
||||
result.append(CapabilityAdvisorResponse(
|
||||
capability=capability.key,
|
||||
contract_version=contract.version,
|
||||
state="REQUIRES_MORE_EVIDENCE",
|
||||
reasons=["no active capability deployment has proven this contract"],
|
||||
))
|
||||
continue
|
||||
run = self.session.scalar(
|
||||
select(CapabilityEvaluationRun)
|
||||
.join(CapabilityEvaluationSuite, CapabilityEvaluationSuite.id == CapabilityEvaluationRun.suite_id)
|
||||
.where(
|
||||
CapabilityEvaluationRun.capability_deployment_id == deployment.id,
|
||||
CapabilityEvaluationSuite.capability_contract_id == contract.id,
|
||||
)
|
||||
.order_by(CapabilityEvaluationRun.completed_at.desc())
|
||||
)
|
||||
if not run:
|
||||
if deployment.production:
|
||||
result.append(CapabilityAdvisorResponse(
|
||||
capability=capability.key,
|
||||
contract_version=contract.version,
|
||||
state="KEEP_CURRENT",
|
||||
deployment_id=deployment.id,
|
||||
reasons=["current production deployment remains active; no comparable new typed run supersedes it"],
|
||||
))
|
||||
continue
|
||||
result.append(CapabilityAdvisorResponse(
|
||||
capability=capability.key,
|
||||
contract_version=contract.version,
|
||||
state="LAB_READY",
|
||||
deployment_id=deployment.id,
|
||||
reasons=["runtime deployment is available; modality evaluation is still required"],
|
||||
))
|
||||
continue
|
||||
suite = self.session.get(CapabilityEvaluationSuite, run.suite_id)
|
||||
if suite is None:
|
||||
raise ServingError(
|
||||
409, "EVALUATION_PROVENANCE_INCOMPLETE", "run suite is missing"
|
||||
)
|
||||
failures = [item["case_key"] for item in run.case_results if item["status"] != "passed"]
|
||||
threshold_failures: list[str] = []
|
||||
for name, threshold in suite.thresholds.items():
|
||||
definition = suite.metric_definitions[name]
|
||||
value = float(run.metric_values[name])
|
||||
direction = definition["direction"]
|
||||
if (direction == "higher_is_better" and value < float(threshold)) or (
|
||||
direction == "lower_is_better" and value > float(threshold)
|
||||
):
|
||||
threshold_failures.append(name)
|
||||
if run.status != "completed" or failures or threshold_failures:
|
||||
reasons = []
|
||||
if run.status != "completed":
|
||||
reasons.append("latest evaluation run did not complete")
|
||||
if failures:
|
||||
reasons.append("case failures: " + ",".join(failures))
|
||||
if threshold_failures:
|
||||
reasons.append("threshold failures: " + ",".join(threshold_failures))
|
||||
state: CapabilityRecommendationState = "BLOCKED"
|
||||
else:
|
||||
reasons = ["completed typed evaluation satisfies every declared threshold"]
|
||||
state = "PROMOTION_ELIGIBLE"
|
||||
result.append(CapabilityAdvisorResponse(
|
||||
capability=capability.key,
|
||||
contract_version=contract.version,
|
||||
state=state,
|
||||
deployment_id=deployment.id,
|
||||
evaluation_run_id=run.id,
|
||||
reasons=reasons,
|
||||
))
|
||||
return result
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,395 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from enum import StrEnum
|
||||
from typing import Any
|
||||
|
||||
|
||||
class AttributionConfidence(StrEnum):
|
||||
KNOWN = "KNOWN"
|
||||
ESTIMATED = "ESTIMATED"
|
||||
UNKNOWN = "UNKNOWN"
|
||||
|
||||
|
||||
class PressureState(StrEnum):
|
||||
NORMAL = "NORMAL"
|
||||
ELEVATED = "ELEVATED"
|
||||
HIGH = "HIGH"
|
||||
CRITICAL = "CRITICAL"
|
||||
|
||||
|
||||
class PlacementVerdict(StrEnum):
|
||||
ADMIT = "ADMIT"
|
||||
ADMIT_AFTER_EVICTION = "ADMIT_AFTER_EVICTION"
|
||||
QUEUE = "QUEUE"
|
||||
REJECT_CAPACITY = "REJECT_CAPACITY"
|
||||
REJECT_HEALTH = "REJECT_HEALTH"
|
||||
REJECT_POLICY = "REJECT_POLICY"
|
||||
|
||||
|
||||
class ResidencyPolicy(StrEnum):
|
||||
ALWAYS_WARM = "always_warm"
|
||||
KEEP_WARM = "keep_warm"
|
||||
LOAD_ON_DEMAND = "load_on_demand"
|
||||
LAB_ONLY = "lab_only"
|
||||
|
||||
|
||||
PRIORITY_RANK = {"production": 0, "interactive": 1, "background": 2, "lab": 3, "benchmark": 3}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SchedulerPolicy:
|
||||
revision: str = "m10-v1"
|
||||
reserve_minimum_bytes: int = 1_073_741_824
|
||||
reserve_percentage: float = 0.05
|
||||
runtime_margin_bytes: int = 268_435_456
|
||||
deployment_margin_minimum_bytes: int = 134_217_728
|
||||
deployment_margin_percentage: float = 0.10
|
||||
request_execution_floor_bytes: int = 67_108_864
|
||||
elevated_headroom_percentage: float = 0.20
|
||||
high_headroom_percentage: float = 0.10
|
||||
critical_headroom_percentage: float = 0.03
|
||||
recovery_extra_percentage: float = 0.03
|
||||
pressure_stable_seconds: int = 30
|
||||
eviction_cooldown_seconds: int = 60
|
||||
global_queue_limit: int = 128
|
||||
placement_history_limit: int = 500
|
||||
lab_paused: bool = False
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LeaseAllocation:
|
||||
reserved_bytes: int
|
||||
materialized_bytes: int = 0
|
||||
|
||||
@property
|
||||
def future_bytes(self) -> int:
|
||||
return max(0, self.reserved_bytes - self.materialized_bytes)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AccountingSnapshot:
|
||||
total_bytes: int
|
||||
observed_used_bytes: int | None
|
||||
resident_bytes: int
|
||||
future_lease_bytes: int
|
||||
reserve_bytes: int
|
||||
external_bytes: int
|
||||
schedulable_bytes: int
|
||||
attribution: AttributionConfidence
|
||||
invariant_delta_bytes: int
|
||||
pressure: PressureState
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ResidentCandidate:
|
||||
deployment_id: str
|
||||
capability: str
|
||||
resident_bytes: int
|
||||
active_requests: int
|
||||
priority: str
|
||||
policy: str
|
||||
idle_since: datetime | None
|
||||
cold_load_ms: float
|
||||
last_used_at: datetime | None
|
||||
pinned: bool = False
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PlacementInput:
|
||||
deployment_id: str
|
||||
capability: str
|
||||
node_id: str
|
||||
accelerator_id: str
|
||||
priority: str
|
||||
required_bytes: int
|
||||
cold_load_ms: float
|
||||
is_resident: bool
|
||||
envelope_stale: bool
|
||||
runtime_healthy: bool
|
||||
node_eligible: bool
|
||||
deadline_remaining_ms: float | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PlannedEviction:
|
||||
deployment_id: str
|
||||
capability: str
|
||||
expected_reclaimed_bytes: int
|
||||
reason: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PlacementDecision:
|
||||
verdict: PlacementVerdict
|
||||
reason_codes: tuple[str, ...]
|
||||
required_bytes: int
|
||||
headroom_before_bytes: int
|
||||
headroom_after_bytes: int
|
||||
evictions: tuple[PlannedEviction, ...] = field(default_factory=tuple)
|
||||
fingerprint: str = ""
|
||||
|
||||
def evidence(self) -> dict[str, Any]:
|
||||
result = asdict(self)
|
||||
result["verdict"] = self.verdict.value
|
||||
result["evictions"] = [asdict(item) for item in self.evictions]
|
||||
return result
|
||||
|
||||
|
||||
def dynamic_reserve(total_bytes: int, policy: SchedulerPolicy) -> int:
|
||||
return max(
|
||||
policy.reserve_minimum_bytes,
|
||||
int(total_bytes * policy.reserve_percentage),
|
||||
policy.runtime_margin_bytes,
|
||||
)
|
||||
|
||||
|
||||
def required_envelope(peak_bytes: int, policy: SchedulerPolicy) -> int:
|
||||
margin = max(
|
||||
policy.deployment_margin_minimum_bytes,
|
||||
int(peak_bytes * policy.deployment_margin_percentage),
|
||||
)
|
||||
return max(0, peak_bytes) + margin
|
||||
|
||||
|
||||
def pressure_for(
|
||||
headroom: int, total: int, previous: PressureState = PressureState.NORMAL
|
||||
) -> PressureState:
|
||||
ratio = headroom / total if total > 0 else 0.0
|
||||
thresholds = {
|
||||
PressureState.CRITICAL: 0.03,
|
||||
PressureState.HIGH: 0.10,
|
||||
PressureState.ELEVATED: 0.20,
|
||||
}
|
||||
if ratio <= thresholds[PressureState.CRITICAL]:
|
||||
return PressureState.CRITICAL
|
||||
if ratio <= thresholds[PressureState.HIGH]:
|
||||
return PressureState.HIGH
|
||||
if ratio <= thresholds[PressureState.ELEVATED]:
|
||||
return PressureState.ELEVATED
|
||||
# Recovery has a 3%-of-total gap. Persistence of the candidate state is
|
||||
# handled by the accelerator pressure record in the service.
|
||||
if previous == PressureState.CRITICAL and ratio <= 0.06:
|
||||
return previous
|
||||
if previous == PressureState.HIGH and ratio <= 0.13:
|
||||
return previous
|
||||
if previous == PressureState.ELEVATED and ratio <= 0.23:
|
||||
return previous
|
||||
return PressureState.NORMAL
|
||||
|
||||
|
||||
def calculate_accounting(
|
||||
*,
|
||||
total_bytes: int,
|
||||
observed_used_bytes: int | None,
|
||||
resident_bytes: int,
|
||||
leases: list[LeaseAllocation],
|
||||
policy: SchedulerPolicy,
|
||||
telemetry_fresh: bool = True,
|
||||
previous_pressure: PressureState = PressureState.NORMAL,
|
||||
) -> AccountingSnapshot:
|
||||
total = max(0, total_bytes)
|
||||
resident = max(0, resident_bytes)
|
||||
reserve = dynamic_reserve(total, policy)
|
||||
future = sum(item.future_bytes for item in leases)
|
||||
if observed_used_bytes is None or not telemetry_fresh:
|
||||
return AccountingSnapshot(
|
||||
total,
|
||||
observed_used_bytes,
|
||||
resident,
|
||||
future,
|
||||
reserve,
|
||||
max(0, total - resident),
|
||||
0,
|
||||
AttributionConfidence.UNKNOWN,
|
||||
0,
|
||||
PressureState.CRITICAL,
|
||||
)
|
||||
observed = min(total, max(0, observed_used_bytes))
|
||||
external = max(0, observed - resident)
|
||||
attributed = external + resident
|
||||
delta = observed - attributed
|
||||
tolerance = max(67_108_864, int(total * 0.01))
|
||||
confidence = (
|
||||
AttributionConfidence.KNOWN if abs(delta) <= tolerance else AttributionConfidence.ESTIMATED
|
||||
)
|
||||
# NVML used already contains resident allocations. Only the unmaterialized
|
||||
# portion of leases is additionally reserved, preventing double counting.
|
||||
schedulable = max(0, total - max(observed, resident) - future - reserve)
|
||||
pressure = pressure_for(schedulable, total, previous_pressure)
|
||||
return AccountingSnapshot(
|
||||
total,
|
||||
observed,
|
||||
resident,
|
||||
future,
|
||||
reserve,
|
||||
external,
|
||||
schedulable,
|
||||
confidence,
|
||||
delta,
|
||||
pressure,
|
||||
)
|
||||
|
||||
|
||||
def _eviction_key(candidate: ResidentCandidate) -> tuple[int, int, float, float]:
|
||||
policy_penalty = {
|
||||
ResidencyPolicy.LAB_ONLY.value: 0,
|
||||
ResidencyPolicy.LOAD_ON_DEMAND.value: 1,
|
||||
ResidencyPolicy.KEEP_WARM.value: 2,
|
||||
ResidencyPolicy.ALWAYS_WARM.value: 10,
|
||||
}.get(candidate.policy, 5)
|
||||
last_use = candidate.last_used_at or candidate.idle_since or datetime.min.replace(tzinfo=UTC)
|
||||
# Safety/priority precede reload cost; among equivalent candidates evict
|
||||
# the cheaper and least-recently-used residency first.
|
||||
return (
|
||||
policy_penalty,
|
||||
-PRIORITY_RANK.get(candidate.priority, 4),
|
||||
candidate.cold_load_ms,
|
||||
last_use.timestamp(),
|
||||
)
|
||||
|
||||
|
||||
def plan_placement(
|
||||
request: PlacementInput,
|
||||
snapshot: AccountingSnapshot,
|
||||
residents: list[ResidentCandidate],
|
||||
policy: SchedulerPolicy,
|
||||
*,
|
||||
global_queue_depth: int = 0,
|
||||
) -> PlacementDecision:
|
||||
reasons: list[str] = []
|
||||
verdict: PlacementVerdict
|
||||
evictions: list[PlannedEviction] = []
|
||||
needed = max(policy.request_execution_floor_bytes, request.required_bytes)
|
||||
if not request.node_eligible or not request.runtime_healthy:
|
||||
verdict = PlacementVerdict.REJECT_HEALTH
|
||||
reasons.append("NODE_UNAVAILABLE" if not request.node_eligible else "RUNTIME_UNHEALTHY")
|
||||
elif request.envelope_stale or snapshot.attribution is AttributionConfidence.UNKNOWN:
|
||||
verdict = PlacementVerdict.REJECT_HEALTH
|
||||
reasons.append("SCHEDULER_STATE_STALE")
|
||||
elif policy.lab_paused and PRIORITY_RANK.get(request.priority, 4) >= PRIORITY_RANK["lab"]:
|
||||
verdict = PlacementVerdict.REJECT_POLICY
|
||||
reasons.append("POLICY_BLOCKED")
|
||||
elif (
|
||||
request.deadline_remaining_ms is not None
|
||||
and not request.is_resident
|
||||
and request.cold_load_ms > request.deadline_remaining_ms
|
||||
):
|
||||
verdict = PlacementVerdict.REJECT_POLICY
|
||||
reasons.append("DEADLINE_CANNOT_BE_MET")
|
||||
elif snapshot.pressure is PressureState.CRITICAL:
|
||||
verdict = PlacementVerdict.REJECT_CAPACITY
|
||||
reasons.append("EXTERNAL_GPU_PRESSURE")
|
||||
elif (
|
||||
snapshot.pressure in {PressureState.HIGH, PressureState.ELEVATED}
|
||||
and PRIORITY_RANK.get(request.priority, 4) >= PRIORITY_RANK["background"]
|
||||
and not request.is_resident
|
||||
):
|
||||
verdict = (
|
||||
PlacementVerdict.QUEUE
|
||||
if global_queue_depth < policy.global_queue_limit
|
||||
else PlacementVerdict.REJECT_CAPACITY
|
||||
)
|
||||
reasons.append(
|
||||
"EXTERNAL_GPU_PRESSURE" if verdict is PlacementVerdict.QUEUE else "QUEUE_FULL"
|
||||
)
|
||||
elif needed <= snapshot.schedulable_bytes:
|
||||
verdict = PlacementVerdict.ADMIT
|
||||
reasons.append(
|
||||
"RESIDENT_WARM_HIT" if request.is_resident else "MEASURED_CAPACITY_AVAILABLE"
|
||||
)
|
||||
else:
|
||||
reclaim = 0
|
||||
for candidate in sorted(residents, key=_eviction_key):
|
||||
if (
|
||||
candidate.deployment_id == request.deployment_id
|
||||
or candidate.active_requests > 0
|
||||
or candidate.pinned
|
||||
or candidate.policy == ResidencyPolicy.ALWAYS_WARM.value
|
||||
or PRIORITY_RANK.get(candidate.priority, 4) < PRIORITY_RANK.get(request.priority, 4)
|
||||
):
|
||||
continue
|
||||
reason = (
|
||||
"IDLE_LAB_EVICTED_FOR_PRODUCTION"
|
||||
if PRIORITY_RANK.get(request.priority, 4) <= PRIORITY_RANK["interactive"]
|
||||
and PRIORITY_RANK.get(candidate.priority, 4) >= PRIORITY_RANK["lab"]
|
||||
else "CAPACITY_REBALANCE"
|
||||
)
|
||||
evictions.append(
|
||||
PlannedEviction(
|
||||
candidate.deployment_id, candidate.capability, candidate.resident_bytes, reason
|
||||
)
|
||||
)
|
||||
reclaim += candidate.resident_bytes
|
||||
if needed <= snapshot.schedulable_bytes + reclaim:
|
||||
break
|
||||
if needed <= snapshot.schedulable_bytes + reclaim:
|
||||
verdict = PlacementVerdict.ADMIT_AFTER_EVICTION
|
||||
reasons.append("MANAGED_IDLE_EVICTION_REQUIRED")
|
||||
elif any(item.active_requests > 0 for item in residents):
|
||||
verdict = PlacementVerdict.QUEUE
|
||||
reasons.append("RESIDENCY_CONFLICT")
|
||||
evictions = []
|
||||
else:
|
||||
verdict = PlacementVerdict.REJECT_CAPACITY
|
||||
reasons.append("INSUFFICIENT_SCHEDULABLE_VRAM")
|
||||
evictions = []
|
||||
after = max(
|
||||
0,
|
||||
snapshot.schedulable_bytes
|
||||
+ sum(item.expected_reclaimed_bytes for item in evictions)
|
||||
- (
|
||||
needed
|
||||
if verdict in {PlacementVerdict.ADMIT, PlacementVerdict.ADMIT_AFTER_EVICTION}
|
||||
else 0
|
||||
),
|
||||
)
|
||||
evidence = {
|
||||
"policy_revision": policy.revision,
|
||||
"request": asdict(request),
|
||||
"snapshot": asdict(snapshot),
|
||||
"verdict": verdict.value,
|
||||
"reasons": reasons,
|
||||
"evictions": [asdict(item) for item in evictions],
|
||||
}
|
||||
fingerprint = hashlib.sha256(
|
||||
json.dumps(evidence, sort_keys=True, default=str, separators=(",", ":")).encode()
|
||||
).hexdigest()
|
||||
return PlacementDecision(
|
||||
verdict,
|
||||
tuple(reasons),
|
||||
needed,
|
||||
snapshot.schedulable_bytes,
|
||||
after,
|
||||
tuple(evictions),
|
||||
fingerprint,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class PressureHysteresis:
|
||||
state: PressureState = PressureState.NORMAL
|
||||
candidate: PressureState | None = None
|
||||
candidate_since: datetime | None = None
|
||||
changed_at: datetime = field(default_factory=lambda: datetime.now(UTC))
|
||||
|
||||
def observe(
|
||||
self, candidate: PressureState, now: datetime, stable_seconds: int = 30
|
||||
) -> PressureState:
|
||||
if candidate == self.state:
|
||||
self.candidate = None
|
||||
self.candidate_since = None
|
||||
return self.state
|
||||
if self.candidate != candidate:
|
||||
self.candidate = candidate
|
||||
self.candidate_since = now
|
||||
return self.state
|
||||
if self.candidate_since and now - self.candidate_since >= timedelta(seconds=stable_seconds):
|
||||
self.state = candidate
|
||||
self.changed_at = now
|
||||
self.candidate = None
|
||||
self.candidate_since = None
|
||||
return self.state
|
||||
@@ -0,0 +1,27 @@
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from modelforge_api.hardware.collectors import (
|
||||
NodeIdentityProvider,
|
||||
SystemHostCollector,
|
||||
build_nvml_collector,
|
||||
)
|
||||
from modelforge_api.services.hardware_inventory import HardwareInventoryService
|
||||
from modelforge_api.settings import Settings
|
||||
|
||||
|
||||
def build_hardware_service(session: Session, settings: Settings) -> HardwareInventoryService:
|
||||
host = SystemHostCollector(
|
||||
NodeIdentityProvider(
|
||||
settings.node_identity_file,
|
||||
settings.node_identity,
|
||||
force_persisted=settings.node_identity_mode == "persisted",
|
||||
),
|
||||
{
|
||||
"model_cache": Path(settings.hf_home),
|
||||
"artifacts": Path(settings.artifact_root),
|
||||
"quarantine": Path(settings.quarantine_root),
|
||||
},
|
||||
)
|
||||
return HardwareInventoryService(session, host, build_nvml_collector())
|
||||
@@ -0,0 +1,557 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any, TypeVar
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from modelforge_api.domain.agent_protocol import AGENT_PROTOCOL_VERSION
|
||||
from modelforge_api.domain.enums import (
|
||||
AgentHealth,
|
||||
Availability,
|
||||
HardwareStatus,
|
||||
InventoryRunStatus,
|
||||
NodeLiveness,
|
||||
ObservationSource,
|
||||
)
|
||||
from modelforge_api.domain.hardware import (
|
||||
AcceleratorCollector,
|
||||
AcceleratorState,
|
||||
AcceleratorTelemetry,
|
||||
HardwareOverview,
|
||||
HardwareSnapshot,
|
||||
HardwareState,
|
||||
HostCollector,
|
||||
NodeState,
|
||||
ObservedValue,
|
||||
StorageState,
|
||||
)
|
||||
from modelforge_api.persistence.hardware_repository import HardwareRepository
|
||||
from modelforge_api.persistence.models import (
|
||||
Accelerator,
|
||||
AcceleratorTelemetryLatest,
|
||||
ComputeNode,
|
||||
HardwareInventoryRun,
|
||||
HostTelemetryLatest,
|
||||
StorageVolumeState,
|
||||
)
|
||||
from modelforge_api.services.audit import AuditWriter
|
||||
|
||||
|
||||
class HardwareRefreshBusy(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
def _aware(value: datetime) -> datetime:
|
||||
return value if value.tzinfo else value.replace(tzinfo=UTC)
|
||||
|
||||
|
||||
class HardwareInventoryService:
|
||||
_refresh_lock = threading.Lock()
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
session: Session,
|
||||
host_collector: HostCollector,
|
||||
accelerator_collector: AcceleratorCollector,
|
||||
source: ObservationSource = ObservationSource.LOCAL_CONTROL_PLANE,
|
||||
received_at: datetime | None = None,
|
||||
observation_sequence: int | None = None,
|
||||
) -> None:
|
||||
self.session = session
|
||||
self.repository = HardwareRepository(session)
|
||||
self.host_collector = host_collector
|
||||
self.accelerator_collector = accelerator_collector
|
||||
self.source = source
|
||||
self.received_at = received_at
|
||||
self.observation_sequence = observation_sequence
|
||||
|
||||
def _audit(
|
||||
self,
|
||||
action: str,
|
||||
resource_type: str,
|
||||
resource_id: str | None,
|
||||
details: dict[str, Any],
|
||||
) -> None:
|
||||
AuditWriter(self.session, "system", "hardware-inventory").write(
|
||||
action, resource_type, resource_id, details
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _value(observation: ObservedValue[T]) -> T | None:
|
||||
return observation.value if observation.availability is Availability.KNOWN else None
|
||||
|
||||
def refresh(self) -> HardwareState:
|
||||
if not self._refresh_lock.acquire(blocking=False):
|
||||
raise HardwareRefreshBusy("hardware refresh already in progress")
|
||||
started = datetime.now(UTC)
|
||||
received_at = self.received_at or datetime.now(UTC)
|
||||
run = HardwareInventoryRun(
|
||||
status=InventoryRunStatus.RUNNING,
|
||||
source=self.source,
|
||||
summary={},
|
||||
started_at=started,
|
||||
observation_sequence=self.observation_sequence,
|
||||
received_at=received_at,
|
||||
)
|
||||
self.repository.add(run)
|
||||
try:
|
||||
host = self.host_collector.collect()
|
||||
nvidia = self.accelerator_collector.collect()
|
||||
snapshot = HardwareSnapshot(host=host, nvidia=nvidia)
|
||||
node = self.repository.node_by_key(host.identity_key)
|
||||
created = node is None
|
||||
if node is None:
|
||||
node = ComputeNode(key=host.identity_key, hostname=host.hostname)
|
||||
self.repository.add(node)
|
||||
elif node.decommissioned_at is not None:
|
||||
raise RuntimeError(
|
||||
"decommissioned compute-node identity requires explicit recovery enrollment"
|
||||
)
|
||||
previous_fingerprint = node.hardware_fingerprint
|
||||
node.hostname = host.hostname
|
||||
if created or not node.display_name:
|
||||
node.display_name = host.display_name
|
||||
node.identity_source = host.identity_source
|
||||
node.os_name = host.os_name
|
||||
node.os_version = self._value(host.os_version)
|
||||
node.architecture = host.architecture
|
||||
node.kernel_version = self._value(host.kernel_version)
|
||||
node.cpu_model = self._value(host.cpu_model)
|
||||
node.logical_cpu_count = self._value(host.logical_cpu_count)
|
||||
node.physical_core_count = self._value(host.physical_core_count)
|
||||
node.total_ram_bytes = self._value(host.total_ram_bytes)
|
||||
node.agent_version = host.agent_version
|
||||
node.status = HardwareStatus.ACTIVE
|
||||
node.status_reason = None
|
||||
node.hardware_fingerprint = snapshot.fingerprint
|
||||
node.inventory = host.model_dump(
|
||||
mode="json", exclude={"storage", "available_ram_bytes"}
|
||||
)
|
||||
node.last_seen_at = host.inventory_at
|
||||
node.inventory_at = host.inventory_at
|
||||
node.observation_source = self.source
|
||||
node.last_inventory_observed_at = host.inventory_at
|
||||
node.last_inventory_received_at = received_at
|
||||
node.inventory_sequence = self.observation_sequence or node.inventory_sequence
|
||||
if self.source is ObservationSource.LOCAL_CONTROL_PLANE:
|
||||
node.liveness_state = NodeLiveness.ONLINE
|
||||
node.last_heartbeat_at = received_at
|
||||
self.session.flush()
|
||||
if created:
|
||||
self._audit(
|
||||
"NODE_DISCOVERED", "compute_node", str(node.id), {"hostname": node.hostname}
|
||||
)
|
||||
elif previous_fingerprint and previous_fingerprint != snapshot.fingerprint:
|
||||
self._audit(
|
||||
"NODE_INVENTORY_CHANGED"
|
||||
if self.source is ObservationSource.REMOTE_AGENT
|
||||
else "HARDWARE_CHANGED",
|
||||
"compute_node",
|
||||
str(node.id),
|
||||
{
|
||||
"previous_fingerprint": previous_fingerprint,
|
||||
"fingerprint": snapshot.fingerprint,
|
||||
},
|
||||
)
|
||||
|
||||
host_latest = self.repository.host_telemetry(node.id)
|
||||
if host_latest is None:
|
||||
host_latest = HostTelemetryLatest(
|
||||
compute_node_id=node.id, observed_at=host.inventory_at
|
||||
)
|
||||
self.repository.add(host_latest)
|
||||
host_latest.available_ram_bytes = self._value(host.available_ram_bytes)
|
||||
host_latest.availability = {
|
||||
"available_ram_bytes": host.available_ram_bytes.model_dump(mode="json")
|
||||
}
|
||||
host_latest.observed_at = host.inventory_at
|
||||
host_latest.received_at = received_at
|
||||
|
||||
for storage in host.storage:
|
||||
state = self.repository.storage_by_key(node.id, storage.purpose, storage.path)
|
||||
if state is None:
|
||||
state = StorageVolumeState(
|
||||
compute_node_id=node.id,
|
||||
purpose=storage.purpose,
|
||||
path=storage.path,
|
||||
observed_at=host.inventory_at,
|
||||
)
|
||||
self.repository.add(state)
|
||||
state.total_bytes = self._value(storage.total_bytes)
|
||||
state.used_bytes = self._value(storage.used_bytes)
|
||||
state.free_bytes = self._value(storage.free_bytes)
|
||||
state.availability = {
|
||||
key: getattr(storage, key).model_dump(mode="json")
|
||||
for key in ("total_bytes", "used_bytes", "free_bytes")
|
||||
}
|
||||
state.observed_at = host.inventory_at
|
||||
state.received_at = received_at
|
||||
|
||||
seen: set[str] = set()
|
||||
telemetry_by_uuid = {item.device_uuid: item for item in nvidia.telemetry}
|
||||
for observed in nvidia.inventory:
|
||||
seen.add(observed.device_uuid)
|
||||
accelerator = self.repository.accelerator_by_uuid(node.id, observed.device_uuid)
|
||||
new_accelerator = accelerator is None
|
||||
if accelerator is None:
|
||||
accelerator = Accelerator(
|
||||
compute_node_id=node.id,
|
||||
device_index=observed.device_index,
|
||||
device_uuid=observed.device_uuid,
|
||||
name=observed.name,
|
||||
)
|
||||
self.repository.add(accelerator)
|
||||
accelerator.device_index = observed.device_index
|
||||
accelerator.pci_bus_id = self._value(observed.pci_bus_id)
|
||||
accelerator.name = observed.name
|
||||
accelerator.vendor = observed.vendor
|
||||
accelerator.architecture = self._value(observed.architecture)
|
||||
accelerator.compute_capability_major = self._value(
|
||||
observed.compute_capability_major
|
||||
)
|
||||
accelerator.compute_capability_minor = self._value(
|
||||
observed.compute_capability_minor
|
||||
)
|
||||
accelerator.total_vram_bytes = self._value(observed.total_vram_bytes)
|
||||
accelerator.memory_total_mb = (
|
||||
round(accelerator.total_vram_bytes / 1024 / 1024)
|
||||
if accelerator.total_vram_bytes is not None
|
||||
else None
|
||||
)
|
||||
accelerator.driver_version = self._value(observed.driver_version)
|
||||
accelerator.cuda_version = self._value(observed.cuda_driver_version)
|
||||
accelerator.mig_mode_current = self._value(observed.mig_mode_current)
|
||||
accelerator.status = HardwareStatus.ACTIVE
|
||||
accelerator.status_reason = None
|
||||
accelerator.inventory_source = observed.source
|
||||
accelerator.last_seen_at = observed.inventory_at
|
||||
accelerator.inventory_at = observed.inventory_at
|
||||
accelerator.capabilities = observed.model_dump(mode="json")
|
||||
self.session.flush()
|
||||
if new_accelerator:
|
||||
self._audit(
|
||||
"ACCELERATOR_DISCOVERED",
|
||||
"accelerator",
|
||||
str(accelerator.id),
|
||||
{"device_uuid": accelerator.device_uuid, "name": accelerator.name},
|
||||
)
|
||||
telemetry = telemetry_by_uuid.get(observed.device_uuid)
|
||||
if telemetry:
|
||||
latest = self.repository.accelerator_telemetry(accelerator.id)
|
||||
if latest is None:
|
||||
latest = AcceleratorTelemetryLatest(
|
||||
accelerator_id=accelerator.id, observed_at=telemetry.observed_at
|
||||
)
|
||||
self.repository.add(latest)
|
||||
for field in (
|
||||
"used_vram_bytes",
|
||||
"free_vram_bytes",
|
||||
"gpu_utilization_percent",
|
||||
"memory_utilization_percent",
|
||||
"temperature_c",
|
||||
"power_draw_w",
|
||||
"power_limit_w",
|
||||
"graphics_clock_mhz",
|
||||
"memory_clock_mhz",
|
||||
"fan_speed_percent",
|
||||
"performance_state",
|
||||
):
|
||||
setattr(latest, field, self._value(getattr(telemetry, field)))
|
||||
latest.availability = {
|
||||
field: getattr(telemetry, field).model_dump(mode="json")
|
||||
for field in (
|
||||
"used_vram_bytes",
|
||||
"free_vram_bytes",
|
||||
"gpu_utilization_percent",
|
||||
"memory_utilization_percent",
|
||||
"temperature_c",
|
||||
"power_draw_w",
|
||||
"power_limit_w",
|
||||
"graphics_clock_mhz",
|
||||
"memory_clock_mhz",
|
||||
"fan_speed_percent",
|
||||
"performance_state",
|
||||
)
|
||||
}
|
||||
latest.observed_at = telemetry.observed_at
|
||||
latest.received_at = received_at
|
||||
|
||||
if nvidia.availability is Availability.KNOWN:
|
||||
for accelerator in self.repository.accelerators(node.id):
|
||||
if (
|
||||
accelerator.device_uuid not in seen
|
||||
and accelerator.status != HardwareStatus.MISSING
|
||||
):
|
||||
accelerator.status = HardwareStatus.MISSING
|
||||
accelerator.status_reason = (
|
||||
"not observed in latest successful NVML enumeration"
|
||||
)
|
||||
self._audit(
|
||||
"ACCELERATOR_MISSING",
|
||||
"accelerator",
|
||||
str(accelerator.id),
|
||||
{"device_uuid": accelerator.device_uuid},
|
||||
)
|
||||
|
||||
run.compute_node_id = node.id
|
||||
run.status = (
|
||||
InventoryRunStatus.SUCCEEDED
|
||||
if nvidia.availability is Availability.KNOWN
|
||||
else InventoryRunStatus.DEGRADED
|
||||
)
|
||||
run.fingerprint = snapshot.fingerprint
|
||||
run.summary = {
|
||||
"accelerator_count": len(nvidia.inventory),
|
||||
"nvidia_availability": nvidia.availability,
|
||||
"nvidia_reason": nvidia.reason,
|
||||
}
|
||||
run.completed_at = datetime.now(UTC)
|
||||
run.observed_at = host.inventory_at
|
||||
self.session.commit()
|
||||
return self.state()
|
||||
except Exception as exc:
|
||||
self.session.rollback()
|
||||
failed = HardwareInventoryRun(
|
||||
status=InventoryRunStatus.FAILED,
|
||||
source="local_collectors",
|
||||
summary={},
|
||||
error=f"{type(exc).__name__}: {exc}",
|
||||
started_at=started,
|
||||
completed_at=datetime.now(UTC),
|
||||
observation_sequence=self.observation_sequence,
|
||||
received_at=received_at,
|
||||
)
|
||||
self.session.add(failed)
|
||||
self.session.flush()
|
||||
self._audit(
|
||||
"INVENTORY_FAILED",
|
||||
"hardware_inventory_run",
|
||||
str(failed.id),
|
||||
{"error": failed.error},
|
||||
)
|
||||
self.session.commit()
|
||||
raise
|
||||
finally:
|
||||
self._refresh_lock.release()
|
||||
|
||||
@staticmethod
|
||||
def _observed(
|
||||
payload: dict[str, Any],
|
||||
key: str,
|
||||
value: Any = None,
|
||||
default: Availability = Availability.UNKNOWN,
|
||||
) -> ObservedValue[Any]:
|
||||
data = payload.get(key)
|
||||
if isinstance(data, dict) and "availability" in data:
|
||||
return ObservedValue.model_validate(data)
|
||||
return ObservedValue.known(value) if value is not None else ObservedValue.absent(default)
|
||||
|
||||
def _accelerator_state(self, accelerator: Accelerator) -> AcceleratorState:
|
||||
facts = accelerator.capabilities or {}
|
||||
latest = self.repository.accelerator_telemetry(accelerator.id)
|
||||
telemetry = None
|
||||
if latest:
|
||||
telemetry = AcceleratorTelemetry(
|
||||
device_uuid=accelerator.device_uuid,
|
||||
observed_at=latest.observed_at,
|
||||
**{
|
||||
field: self._observed(latest.availability, field, getattr(latest, field))
|
||||
for field in (
|
||||
"used_vram_bytes",
|
||||
"free_vram_bytes",
|
||||
"gpu_utilization_percent",
|
||||
"memory_utilization_percent",
|
||||
"temperature_c",
|
||||
"power_draw_w",
|
||||
"power_limit_w",
|
||||
"graphics_clock_mhz",
|
||||
"memory_clock_mhz",
|
||||
"fan_speed_percent",
|
||||
"performance_state",
|
||||
)
|
||||
},
|
||||
)
|
||||
return AcceleratorState(
|
||||
id=str(accelerator.id),
|
||||
node_id=str(accelerator.compute_node_id),
|
||||
status=HardwareStatus(accelerator.status),
|
||||
status_reason=accelerator.status_reason,
|
||||
device_index=accelerator.device_index,
|
||||
device_uuid=accelerator.device_uuid,
|
||||
pci_bus_id=self._observed(facts, "pci_bus_id", accelerator.pci_bus_id),
|
||||
name=accelerator.name,
|
||||
vendor=accelerator.vendor,
|
||||
architecture=self._observed(facts, "architecture", accelerator.architecture),
|
||||
compute_capability_major=self._observed(
|
||||
facts, "compute_capability_major", accelerator.compute_capability_major
|
||||
),
|
||||
compute_capability_minor=self._observed(
|
||||
facts, "compute_capability_minor", accelerator.compute_capability_minor
|
||||
),
|
||||
total_vram_bytes=self._observed(
|
||||
facts, "total_vram_bytes", accelerator.total_vram_bytes
|
||||
),
|
||||
driver_version=self._observed(facts, "driver_version", accelerator.driver_version),
|
||||
cuda_driver_version=self._observed(
|
||||
facts, "cuda_driver_version", accelerator.cuda_version
|
||||
),
|
||||
mig_mode_current=self._observed(
|
||||
facts, "mig_mode_current", accelerator.mig_mode_current
|
||||
),
|
||||
first_seen_at=accelerator.first_seen_at,
|
||||
last_seen_at=accelerator.last_seen_at,
|
||||
inventory_at=accelerator.inventory_at,
|
||||
telemetry=telemetry,
|
||||
)
|
||||
|
||||
def state(self) -> HardwareState:
|
||||
nodes: list[NodeState] = []
|
||||
for node in self.repository.nodes():
|
||||
active_credential = self.repository.active_credential_for_node(node.id)
|
||||
facts = node.inventory or {}
|
||||
host_latest = self.repository.host_telemetry(node.id)
|
||||
available = self._observed(
|
||||
host_latest.availability if host_latest else {},
|
||||
"available_ram_bytes",
|
||||
host_latest.available_ram_bytes if host_latest else None,
|
||||
)
|
||||
storage = [
|
||||
StorageState(
|
||||
id=str(item.id),
|
||||
purpose=item.purpose,
|
||||
path=item.path,
|
||||
total_bytes=self._observed(item.availability, "total_bytes", item.total_bytes),
|
||||
used_bytes=self._observed(item.availability, "used_bytes", item.used_bytes),
|
||||
free_bytes=self._observed(item.availability, "free_bytes", item.free_bytes),
|
||||
observed_at=item.observed_at,
|
||||
)
|
||||
for item in self.repository.storage(node.id)
|
||||
]
|
||||
accelerators = [
|
||||
self._accelerator_state(item) for item in self.repository.accelerators(node.id)
|
||||
]
|
||||
nodes.append(
|
||||
NodeState(
|
||||
id=str(node.id),
|
||||
identity_key=node.key,
|
||||
identity_source=node.identity_source,
|
||||
hostname=node.hostname,
|
||||
display_name=node.display_name or node.hostname,
|
||||
status=HardwareStatus(node.status),
|
||||
status_reason=node.status_reason,
|
||||
os_name=node.os_name,
|
||||
os_version=self._observed(facts, "os_version", node.os_version),
|
||||
architecture=node.architecture,
|
||||
kernel_version=self._observed(facts, "kernel_version", node.kernel_version),
|
||||
cpu_model=self._observed(facts, "cpu_model", node.cpu_model),
|
||||
logical_cpu_count=self._observed(
|
||||
facts, "logical_cpu_count", node.logical_cpu_count
|
||||
),
|
||||
physical_core_count=self._observed(
|
||||
facts, "physical_core_count", node.physical_core_count
|
||||
),
|
||||
total_ram_bytes=self._observed(facts, "total_ram_bytes", node.total_ram_bytes),
|
||||
available_ram_bytes=available,
|
||||
agent_version=node.agent_version,
|
||||
first_seen_at=node.first_seen_at,
|
||||
last_seen_at=node.last_seen_at,
|
||||
inventory_at=node.inventory_at,
|
||||
hardware_fingerprint=node.hardware_fingerprint,
|
||||
enabled=node.enabled,
|
||||
liveness=NodeLiveness(node.liveness_state),
|
||||
agent_health=(
|
||||
AgentHealth.REVOKED
|
||||
if node.observation_source == ObservationSource.REMOTE_AGENT
|
||||
and active_credential is None
|
||||
else (
|
||||
AgentHealth.HEALTHY
|
||||
if node.agent_protocol_version == AGENT_PROTOCOL_VERSION
|
||||
and node.enabled
|
||||
else (
|
||||
AgentHealth.INCOMPATIBLE
|
||||
if node.agent_protocol_version is not None
|
||||
and node.agent_protocol_version != AGENT_PROTOCOL_VERSION
|
||||
else AgentHealth.UNKNOWN
|
||||
)
|
||||
)
|
||||
),
|
||||
observation_source=ObservationSource(node.observation_source),
|
||||
protocol_version=node.agent_protocol_version,
|
||||
supported_capabilities=node.agent_capabilities or [],
|
||||
agent_started_at=node.agent_started_at,
|
||||
last_heartbeat_at=node.last_heartbeat_at,
|
||||
last_inventory_received_at=node.last_inventory_received_at,
|
||||
last_telemetry_received_at=node.last_telemetry_received_at,
|
||||
inventory_age_seconds=(
|
||||
max(
|
||||
0,
|
||||
int(
|
||||
(
|
||||
datetime.now(UTC) - _aware(node.last_inventory_received_at)
|
||||
).total_seconds()
|
||||
),
|
||||
)
|
||||
if node.last_inventory_received_at
|
||||
else None
|
||||
),
|
||||
telemetry_age_seconds=(
|
||||
max(
|
||||
0,
|
||||
int(
|
||||
(
|
||||
datetime.now(UTC) - _aware(node.last_telemetry_received_at)
|
||||
).total_seconds()
|
||||
),
|
||||
)
|
||||
if node.last_telemetry_received_at
|
||||
else None
|
||||
),
|
||||
last_connection_error=node.last_connection_error,
|
||||
role=node.role,
|
||||
labels=node.labels or {},
|
||||
production_eligible=node.production_eligible,
|
||||
lab_eligible=node.lab_eligible,
|
||||
benchmark_eligible=node.benchmark_eligible,
|
||||
generation=node.generation,
|
||||
decommissioned_at=node.decommissioned_at,
|
||||
decommission_reason=node.decommission_reason,
|
||||
decommissioned_by=node.decommissioned_by,
|
||||
environment=(facts.get("metadata") or {}).get("environment"),
|
||||
storage=storage,
|
||||
accelerators=accelerators,
|
||||
)
|
||||
)
|
||||
latest = self.repository.latest_run()
|
||||
accelerators = [item for node in nodes for item in node.accelerators]
|
||||
inventory_state = HardwareStatus.PENDING
|
||||
if latest is not None:
|
||||
inventory_state = {
|
||||
InventoryRunStatus.RUNNING: HardwareStatus.PENDING,
|
||||
InventoryRunStatus.SUCCEEDED: HardwareStatus.ACTIVE,
|
||||
InventoryRunStatus.DEGRADED: HardwareStatus.DEGRADED,
|
||||
InventoryRunStatus.FAILED: HardwareStatus.UNAVAILABLE,
|
||||
}[InventoryRunStatus(latest.status)]
|
||||
status = HardwareStatus.PENDING if not nodes else HardwareStatus.ACTIVE
|
||||
reason = (
|
||||
latest.error
|
||||
if latest and latest.status == InventoryRunStatus.FAILED
|
||||
else (latest.summary.get("nvidia_reason") if latest else "inventory has not run")
|
||||
)
|
||||
return HardwareState(
|
||||
overview=HardwareOverview(
|
||||
status=status,
|
||||
inventory_state=inventory_state,
|
||||
node_count=len(nodes),
|
||||
accelerator_count=len(
|
||||
[item for item in accelerators if item.status is HardwareStatus.ACTIVE]
|
||||
),
|
||||
last_inventory_at=latest.completed_at if latest else None,
|
||||
reason=reason,
|
||||
),
|
||||
nodes=nodes,
|
||||
)
|
||||
@@ -0,0 +1,38 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from modelforge_api.db import engine
|
||||
from modelforge_api.services.hardware_factory import build_hardware_service
|
||||
from modelforge_api.settings import Settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class HardwarePollingService:
|
||||
def __init__(self, settings: Settings) -> None:
|
||||
self.settings = settings
|
||||
self._stop = asyncio.Event()
|
||||
|
||||
def _refresh_once(self) -> None:
|
||||
with Session(engine) as session:
|
||||
build_hardware_service(session, self.settings).refresh()
|
||||
|
||||
async def run(self) -> None:
|
||||
while not self._stop.is_set():
|
||||
try:
|
||||
await asyncio.to_thread(self._refresh_once)
|
||||
except Exception:
|
||||
logger.exception("hardware inventory poll failed")
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
self._stop.wait(), self.settings.hardware_poll_interval_seconds
|
||||
)
|
||||
except TimeoutError:
|
||||
continue
|
||||
|
||||
def stop(self) -> None:
|
||||
self._stop.set()
|
||||
@@ -0,0 +1,810 @@
|
||||
"""M16 executable platform invariants.
|
||||
|
||||
An invariant is a property that must hold no matter what fault was injected. Writing them down in
|
||||
prose is not enough for a release gate: each one here is a query the platform can run against its
|
||||
own authoritative state, before a chaos scenario, after it, and after recovery.
|
||||
|
||||
These checks are read-only. They never mutate state, never call an external system, and never
|
||||
depend on the observability database being healthy, so they stay usable exactly when monitoring is
|
||||
the thing that failed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from enum import StrEnum
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from modelforge_api.persistence.models import (
|
||||
Accelerator,
|
||||
AcceleratorTelemetryLatest,
|
||||
ArtifactJob,
|
||||
ArtifactLocation,
|
||||
AuditChainHead,
|
||||
AuditEvent,
|
||||
BackupSet,
|
||||
CapabilityContract,
|
||||
CapabilityDeployment,
|
||||
ComputeNode,
|
||||
Deployment,
|
||||
EmbeddingSpace,
|
||||
GatewayRequest,
|
||||
GpuLease,
|
||||
HostTelemetryLatest,
|
||||
LifecycleOperation,
|
||||
MigrationCutoverOperation,
|
||||
ModelArtifact,
|
||||
NodeCredential,
|
||||
ResidencyAllocation,
|
||||
RestoreOperation,
|
||||
RestorePlan,
|
||||
RuntimeProbe,
|
||||
SchedulerAcceleratorState,
|
||||
ServiceClient,
|
||||
ServiceCredential,
|
||||
ServingGpuLease,
|
||||
ServingJob,
|
||||
StorageRoot,
|
||||
StorageVolumeState,
|
||||
)
|
||||
from modelforge_api.services.audit import audit_chain_violations
|
||||
|
||||
# Lease and residency states that claim a live GPU reservation right now.
|
||||
ACTIVE_LEASE_STATES = ("pending", "granted", "active", "held")
|
||||
ACTIVE_RESIDENCY_STATES = ("loading", "resident", "unloading")
|
||||
TERMINAL_LIFECYCLE_STAGES = ("COMMITTED", "ROLLED_BACK", "FAILED")
|
||||
TERMINAL_CUTOVER_STAGES = ("COMMITTED", "ROLLED_BACK", "FAILED")
|
||||
|
||||
|
||||
class InvariantSeverity(StrEnum):
|
||||
CRITICAL = "CRITICAL"
|
||||
HIGH = "HIGH"
|
||||
|
||||
|
||||
class InvariantStatus(StrEnum):
|
||||
HOLDS = "HOLDS"
|
||||
VIOLATED = "VIOLATED"
|
||||
NOT_APPLICABLE = "NOT_APPLICABLE"
|
||||
|
||||
|
||||
class InvariantResult(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
key: str
|
||||
name: str
|
||||
severity: InvariantSeverity
|
||||
status: InvariantStatus
|
||||
observed: int
|
||||
detail: str
|
||||
violations: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
class InvariantReport(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
observed_at: datetime
|
||||
checked: int
|
||||
holding: int
|
||||
violated: int
|
||||
results: list[InvariantResult]
|
||||
|
||||
@property
|
||||
def ok(self) -> bool:
|
||||
return self.violated == 0
|
||||
|
||||
|
||||
def _result(
|
||||
key: str,
|
||||
name: str,
|
||||
severity: InvariantSeverity,
|
||||
violations: list[str],
|
||||
holds_detail: str,
|
||||
violated_detail: str,
|
||||
) -> InvariantResult:
|
||||
if violations:
|
||||
return InvariantResult(
|
||||
key=key,
|
||||
name=name,
|
||||
severity=severity,
|
||||
status=InvariantStatus.VIOLATED,
|
||||
observed=len(violations),
|
||||
detail=violated_detail,
|
||||
violations=violations[:25],
|
||||
)
|
||||
return InvariantResult(
|
||||
key=key,
|
||||
name=name,
|
||||
severity=severity,
|
||||
status=InvariantStatus.HOLDS,
|
||||
observed=0,
|
||||
detail=holds_detail,
|
||||
)
|
||||
|
||||
|
||||
def _no_duplicate_production_stable(session: Session) -> InvariantResult:
|
||||
rows = session.execute(
|
||||
select(CapabilityDeployment.capability_contract_id, func.count())
|
||||
.where(
|
||||
CapabilityDeployment.production.is_(True),
|
||||
CapabilityDeployment.status == "stable",
|
||||
)
|
||||
.group_by(CapabilityDeployment.capability_contract_id)
|
||||
.having(func.count() > 1)
|
||||
).all()
|
||||
return _result(
|
||||
"single_production_stable",
|
||||
"At most one stable production deployment per capability contract",
|
||||
InvariantSeverity.CRITICAL,
|
||||
[
|
||||
f"contract {contract} has {count} stable production deployments"
|
||||
for contract, count in rows
|
||||
],
|
||||
"every capability contract has at most one stable production deployment",
|
||||
"a capability contract has more than one stable production deployment",
|
||||
)
|
||||
|
||||
|
||||
def _no_duplicate_node_identity(session: Session) -> InvariantResult:
|
||||
rows = session.execute(
|
||||
select(ComputeNode.key, func.count())
|
||||
.where(ComputeNode.enabled.is_(True))
|
||||
.group_by(ComputeNode.key)
|
||||
.having(func.count() > 1)
|
||||
).all()
|
||||
hardware = session.execute(
|
||||
select(ComputeNode.hardware_fingerprint, func.count())
|
||||
.where(
|
||||
ComputeNode.enabled.is_(True),
|
||||
ComputeNode.hardware_fingerprint.is_not(None),
|
||||
)
|
||||
.group_by(ComputeNode.hardware_fingerprint)
|
||||
.having(func.count() > 1)
|
||||
).all()
|
||||
violations = [f"identity key {key} is claimed by {count} enabled nodes" for key, count in rows]
|
||||
violations += [
|
||||
f"hardware fingerprint {fingerprint} is claimed by {count} enabled nodes"
|
||||
for fingerprint, count in hardware
|
||||
]
|
||||
return _result(
|
||||
"single_node_identity",
|
||||
"No two enabled nodes claim the same identity or hardware",
|
||||
InvariantSeverity.CRITICAL,
|
||||
violations,
|
||||
"every enabled node holds a distinct identity and hardware fingerprint",
|
||||
"more than one enabled node claims the same hardware",
|
||||
)
|
||||
|
||||
|
||||
def _single_active_node_credential(session: Session) -> InvariantResult:
|
||||
rows = session.execute(
|
||||
select(NodeCredential.compute_node_id, func.count())
|
||||
.where(NodeCredential.revoked_at.is_(None))
|
||||
.group_by(NodeCredential.compute_node_id)
|
||||
.having(func.count() > 1)
|
||||
).all()
|
||||
return _result(
|
||||
"single_active_node_credential",
|
||||
"A node has at most one unrevoked credential",
|
||||
InvariantSeverity.CRITICAL,
|
||||
[f"node {node} has {count} active credentials" for node, count in rows],
|
||||
"no node holds more than one active credential",
|
||||
"a node holds more than one active credential",
|
||||
)
|
||||
|
||||
|
||||
def _no_stale_gpu_lease(session: Session, now: datetime | None = None) -> InvariantResult:
|
||||
moment = now or datetime.now(UTC)
|
||||
rows = session.scalars(
|
||||
select(ServingGpuLease).where(
|
||||
ServingGpuLease.state.in_(ACTIVE_LEASE_STATES),
|
||||
ServingGpuLease.released_at.is_(None),
|
||||
ServingGpuLease.expires_at.is_not(None),
|
||||
)
|
||||
)
|
||||
violations = []
|
||||
for lease in rows:
|
||||
expires = lease.expires_at
|
||||
if expires is None:
|
||||
continue
|
||||
aware = expires if expires.tzinfo else expires.replace(tzinfo=UTC)
|
||||
if aware < moment:
|
||||
violations.append(
|
||||
f"lease {lease.id} is {lease.state} but expired at {aware.isoformat()}"
|
||||
)
|
||||
return _result(
|
||||
"no_stale_gpu_lease",
|
||||
"No expired GPU lease is still presented as active",
|
||||
InvariantSeverity.CRITICAL,
|
||||
violations,
|
||||
"every active GPU lease is within its expiry",
|
||||
"an expired GPU lease is still held as current truth",
|
||||
)
|
||||
|
||||
|
||||
def _no_mixed_embedding_space(session: Session) -> InvariantResult:
|
||||
rows = session.execute(
|
||||
select(
|
||||
EmbeddingSpace.capability_contract_id,
|
||||
func.count(func.distinct(EmbeddingSpace.identity_digest)),
|
||||
).group_by(EmbeddingSpace.capability_contract_id)
|
||||
).all()
|
||||
deployments = session.execute(
|
||||
select(
|
||||
CapabilityDeployment.capability_contract_id,
|
||||
func.count(func.distinct(CapabilityDeployment.embedding_space_id)),
|
||||
)
|
||||
.where(
|
||||
CapabilityDeployment.production.is_(True),
|
||||
CapabilityDeployment.status == "stable",
|
||||
CapabilityDeployment.embedding_space_id.is_not(None),
|
||||
)
|
||||
.group_by(CapabilityDeployment.capability_contract_id)
|
||||
.having(func.count(func.distinct(CapabilityDeployment.embedding_space_id)) > 1)
|
||||
).all()
|
||||
violations = [
|
||||
f"contract {contract} serves {count} embedding spaces from stable production"
|
||||
for contract, count in deployments
|
||||
]
|
||||
detail = (
|
||||
f"{len(rows)} contracts carry embedding-space identities; stable production serves one each"
|
||||
)
|
||||
return _result(
|
||||
"no_mixed_embedding_space",
|
||||
"Stable production never serves two embedding spaces for one contract",
|
||||
InvariantSeverity.CRITICAL,
|
||||
violations,
|
||||
detail,
|
||||
"stable production serves more than one embedding space for a capability contract",
|
||||
)
|
||||
|
||||
|
||||
def _no_lifecycle_commit_without_evidence(session: Session) -> InvariantResult:
|
||||
rows = session.scalars(
|
||||
select(LifecycleOperation).where(LifecycleOperation.stage == "COMMITTED")
|
||||
)
|
||||
violations = []
|
||||
for operation in rows:
|
||||
plan = operation.promotion_plan_id
|
||||
if plan is None:
|
||||
violations.append(f"operation {operation.id} committed without a promotion plan")
|
||||
continue
|
||||
if not operation.approver or not operation.executor:
|
||||
violations.append(
|
||||
f"operation {operation.id} committed without recorded approver and executor"
|
||||
)
|
||||
return _result(
|
||||
"lifecycle_commit_has_evidence",
|
||||
"No lifecycle operation commits without its plan, approver and executor",
|
||||
InvariantSeverity.CRITICAL,
|
||||
violations,
|
||||
"every committed lifecycle operation carries its approval evidence",
|
||||
"a lifecycle operation committed without approval evidence",
|
||||
)
|
||||
|
||||
|
||||
def _no_cutover_commit_without_validation(session: Session) -> InvariantResult:
|
||||
rows = session.scalars(
|
||||
select(MigrationCutoverOperation).where(MigrationCutoverOperation.stage == "COMMITTED")
|
||||
)
|
||||
violations = [
|
||||
f"cutover {item.id} committed without an external state fingerprint"
|
||||
for item in rows
|
||||
if not item.external_state_fingerprint
|
||||
]
|
||||
return _result(
|
||||
"cutover_commit_has_validation",
|
||||
"No migration cutover commits without an external-state fingerprint",
|
||||
InvariantSeverity.CRITICAL,
|
||||
violations,
|
||||
"every committed cutover records the external state it observed",
|
||||
"a cutover committed without observing external truth",
|
||||
)
|
||||
|
||||
|
||||
def _no_restore_from_unverified_backup(session: Session) -> InvariantResult:
|
||||
rows = session.execute(
|
||||
select(RestoreOperation.id, BackupSet.backup_id, BackupSet.state)
|
||||
.join(BackupSet, BackupSet.id == RestoreOperation.backup_set_id)
|
||||
.where(RestoreOperation.state != "PLANNED")
|
||||
).all()
|
||||
violations = [
|
||||
f"restore {operation} ran against backup {backup} in state {state}"
|
||||
for operation, backup, state in rows
|
||||
if state not in ("VERIFIED", "EXPIRED")
|
||||
]
|
||||
plans = session.execute(
|
||||
select(RestorePlan.id, BackupSet.backup_id, BackupSet.state).join(
|
||||
BackupSet, BackupSet.id == RestorePlan.backup_set_id
|
||||
)
|
||||
).all()
|
||||
violations += [
|
||||
f"restore plan {plan} targets backup {backup} that was never verified"
|
||||
for plan, backup, state in plans
|
||||
if state in ("PLANNED", "CREATING", "CREATED", "FAILED")
|
||||
]
|
||||
return _result(
|
||||
"restore_requires_verified_backup",
|
||||
"No restore is planned or executed from an unverified backup",
|
||||
InvariantSeverity.CRITICAL,
|
||||
violations,
|
||||
"every restore plan and operation references a verified backup",
|
||||
"a restore referenced a backup that was never verified",
|
||||
)
|
||||
|
||||
|
||||
def _revoked_credentials_stay_revoked(session: Session) -> InvariantResult:
|
||||
violations: list[str] = []
|
||||
for label, revoked_rows in (
|
||||
(
|
||||
"node",
|
||||
session.scalars(
|
||||
select(NodeCredential).where(NodeCredential.revoked_at.is_not(None))
|
||||
).all(),
|
||||
),
|
||||
(
|
||||
"service",
|
||||
session.scalars(
|
||||
select(ServiceCredential).where(ServiceCredential.revoked_at.is_not(None))
|
||||
).all(),
|
||||
),
|
||||
):
|
||||
for credential in revoked_rows:
|
||||
used = credential.last_used_at
|
||||
revoked = credential.revoked_at
|
||||
if used is None or revoked is None:
|
||||
continue
|
||||
used_aware = used if used.tzinfo else used.replace(tzinfo=UTC)
|
||||
revoked_aware = revoked if revoked.tzinfo else revoked.replace(tzinfo=UTC)
|
||||
if used_aware > revoked_aware:
|
||||
violations.append(
|
||||
f"{label} credential {credential.id} was used after it was revoked"
|
||||
)
|
||||
return _result(
|
||||
"revoked_credentials_stay_revoked",
|
||||
"No revoked credential is ever accepted again",
|
||||
InvariantSeverity.CRITICAL,
|
||||
violations,
|
||||
"no revoked node or service credential records a later successful use",
|
||||
"a revoked credential was accepted after revocation",
|
||||
)
|
||||
|
||||
|
||||
def _capability_clients_are_not_operators(session: Session) -> InvariantResult:
|
||||
"""A gateway client may only name capabilities; operator and node scopes are a different plane."""
|
||||
|
||||
violations: list[str] = []
|
||||
for client in session.scalars(select(ServiceClient)):
|
||||
allowed = client.allowed_capabilities
|
||||
capabilities = allowed if isinstance(allowed, list) else []
|
||||
forbidden = [
|
||||
capability
|
||||
for capability in capabilities
|
||||
if isinstance(capability, str)
|
||||
and (
|
||||
capability.startswith(("admin", "operator", "node."))
|
||||
or capability in {"recovery", "lifecycle", "migration"}
|
||||
)
|
||||
]
|
||||
if forbidden:
|
||||
violations.append(
|
||||
f"service client {client.id} claims non-capability scopes {forbidden}"
|
||||
)
|
||||
return _result(
|
||||
"capability_clients_are_not_operators",
|
||||
"No capability client holds operator or node scopes",
|
||||
InvariantSeverity.CRITICAL,
|
||||
violations,
|
||||
"every service client is scoped to capabilities only",
|
||||
"a capability client carries operator or node scopes",
|
||||
)
|
||||
|
||||
|
||||
def _no_unsafe_artifact_promoted(session: Session) -> InvariantResult:
|
||||
rows = session.execute(
|
||||
select(ModelArtifact.id, ModelArtifact.filename, ModelArtifact.security_status)
|
||||
.join(ArtifactLocation, ArtifactLocation.artifact_id == ModelArtifact.id)
|
||||
.where(
|
||||
ArtifactLocation.status == "verified",
|
||||
ModelArtifact.security_status.in_(("blocked", "quarantined", "unverified")),
|
||||
)
|
||||
.distinct()
|
||||
).all()
|
||||
return _result(
|
||||
"no_unsafe_artifact_promoted",
|
||||
"No artifact is promoted to a verified location while its security status is unsafe",
|
||||
InvariantSeverity.CRITICAL,
|
||||
[
|
||||
f"artifact {artifact} ({filename}) is verified on disk but {status}"
|
||||
for artifact, filename, status in rows
|
||||
],
|
||||
"every verified artifact location holds an artifact that passed its security checks",
|
||||
"an unsafe artifact occupies a verified location",
|
||||
)
|
||||
|
||||
|
||||
def _no_orphan_serving_work(session: Session, now: datetime | None = None) -> InvariantResult:
|
||||
moment = now or datetime.now(UTC)
|
||||
jobs = session.scalars(
|
||||
select(ServingJob).where(ServingJob.status.in_(("queued", "leased", "running")))
|
||||
)
|
||||
violations = []
|
||||
for job in jobs:
|
||||
expires = job.lease_expires_at
|
||||
if expires is None:
|
||||
continue
|
||||
aware = expires if expires.tzinfo else expires.replace(tzinfo=UTC)
|
||||
if aware < moment:
|
||||
violations.append(
|
||||
f"serving job {job.id} is {job.status} with a lease expired at {aware.isoformat()}"
|
||||
)
|
||||
residency = session.scalars(
|
||||
select(ResidencyAllocation).where(ResidencyAllocation.state.in_(ACTIVE_RESIDENCY_STATES))
|
||||
)
|
||||
for allocation in residency:
|
||||
node = session.get(ComputeNode, allocation.compute_node_id)
|
||||
if node is not None and not node.enabled:
|
||||
violations.append(
|
||||
f"residency {allocation.id} is {allocation.state} on disabled node {node.id}"
|
||||
)
|
||||
return _result(
|
||||
"no_orphan_serving_work",
|
||||
"No queued or leased serving work survives its lease, and no residency sits on a disabled node",
|
||||
InvariantSeverity.CRITICAL,
|
||||
violations,
|
||||
"no orphaned serving job or residency allocation exists",
|
||||
"orphaned serving work survived recovery",
|
||||
)
|
||||
|
||||
|
||||
def _no_hidden_auto_promotion(session: Session) -> InvariantResult:
|
||||
rows = session.scalars(
|
||||
select(CapabilityDeployment).where(
|
||||
CapabilityDeployment.production.is_(True),
|
||||
CapabilityDeployment.status.in_(("stable", "draining")),
|
||||
)
|
||||
)
|
||||
violations = [
|
||||
f"production deployment {item.id} has no recorded production approval"
|
||||
for item in rows
|
||||
if item.production_approval_id is None
|
||||
]
|
||||
return _result(
|
||||
"no_hidden_auto_promotion",
|
||||
"Every production deployment carries an explicit production approval",
|
||||
InvariantSeverity.CRITICAL,
|
||||
violations,
|
||||
"every production deployment references its production approval",
|
||||
"a production deployment exists without an approval",
|
||||
)
|
||||
|
||||
|
||||
def _audit_chain_is_intact(session: Session) -> InvariantResult:
|
||||
events = list(
|
||||
session.scalars(select(AuditEvent).order_by(AuditEvent.sequence, AuditEvent.id))
|
||||
)
|
||||
checkpoint = session.get(AuditChainHead, 1)
|
||||
violations = audit_chain_violations(events, checkpoint)
|
||||
return _result(
|
||||
"audit_chain_intact",
|
||||
"The audit trail has a strict sequence and canonical content-hash chain",
|
||||
InvariantSeverity.CRITICAL,
|
||||
violations,
|
||||
f"{len(events)} audit events form one contiguous, content-verified chain",
|
||||
"the audit trail contains a gap, fork, invalid link or content-hash mismatch",
|
||||
)
|
||||
|
||||
|
||||
def _decommissioned_nodes_are_terminal(session: Session) -> InvariantResult:
|
||||
violations: list[str] = []
|
||||
nodes = list(
|
||||
session.scalars(select(ComputeNode).where(ComputeNode.decommissioned_at.is_not(None)))
|
||||
)
|
||||
for node in nodes:
|
||||
if (
|
||||
node.enabled
|
||||
or node.status != "decommissioned"
|
||||
or node.liveness_state != "decommissioned"
|
||||
):
|
||||
violations.append(
|
||||
f"node {node.id} is decommissioned but enabled={node.enabled}, "
|
||||
f"status={node.status} and liveness={node.liveness_state}"
|
||||
)
|
||||
if node.production_eligible or node.lab_eligible or node.benchmark_eligible:
|
||||
violations.append(f"node {node.id} is decommissioned but retains scheduler eligibility")
|
||||
if node.inventory or node.agent_capabilities:
|
||||
violations.append(f"node {node.id} is decommissioned but retains current inventory")
|
||||
active_credentials = (
|
||||
session.scalar(
|
||||
select(func.count())
|
||||
.select_from(NodeCredential)
|
||||
.where(
|
||||
NodeCredential.compute_node_id == node.id,
|
||||
NodeCredential.revoked_at.is_(None),
|
||||
)
|
||||
)
|
||||
or 0
|
||||
)
|
||||
if active_credentials:
|
||||
violations.append(f"node {node.id} retains {active_credentials} active credentials")
|
||||
for model, status_column, terminal in (
|
||||
(ArtifactJob, ArtifactJob.status, ("completed", "failed", "cancelled")),
|
||||
(RuntimeProbe, RuntimeProbe.status, ("completed", "failed", "cancelled")),
|
||||
(ServingJob, ServingJob.status, ("completed", "failed", "cancelled")),
|
||||
):
|
||||
active = (
|
||||
session.scalar(
|
||||
select(func.count())
|
||||
.select_from(model)
|
||||
.where(
|
||||
model.compute_node_id == node.id,
|
||||
status_column.not_in(terminal),
|
||||
)
|
||||
)
|
||||
or 0
|
||||
)
|
||||
if active:
|
||||
violations.append(f"node {node.id} retains {active} active {model.__tablename__}")
|
||||
residency = (
|
||||
session.scalar(
|
||||
select(func.count())
|
||||
.select_from(ResidencyAllocation)
|
||||
.where(ResidencyAllocation.compute_node_id == node.id)
|
||||
)
|
||||
or 0
|
||||
)
|
||||
if residency:
|
||||
violations.append(f"node {node.id} retains {residency} active residencies")
|
||||
accelerator_ids = list(
|
||||
session.scalars(select(Accelerator.id).where(Accelerator.compute_node_id == node.id))
|
||||
)
|
||||
if accelerator_ids:
|
||||
active_serving_leases = (
|
||||
session.scalar(
|
||||
select(func.count())
|
||||
.select_from(ServingGpuLease)
|
||||
.where(
|
||||
ServingGpuLease.accelerator_id.in_(accelerator_ids),
|
||||
ServingGpuLease.state.not_in(
|
||||
("released", "expired", "failed", "cancelled")
|
||||
),
|
||||
)
|
||||
)
|
||||
or 0
|
||||
)
|
||||
active_legacy_leases = (
|
||||
session.scalar(
|
||||
select(func.count())
|
||||
.select_from(GpuLease)
|
||||
.where(
|
||||
GpuLease.accelerator_id.in_(accelerator_ids),
|
||||
GpuLease.state.not_in(("released", "expired", "failed", "cancelled")),
|
||||
)
|
||||
)
|
||||
or 0
|
||||
)
|
||||
current_accelerator_state = sum(
|
||||
int(
|
||||
session.scalar(
|
||||
select(func.count()).select_from(model).where(column.in_(accelerator_ids))
|
||||
)
|
||||
or 0
|
||||
)
|
||||
for model, column in (
|
||||
(AcceleratorTelemetryLatest, AcceleratorTelemetryLatest.accelerator_id),
|
||||
(SchedulerAcceleratorState, SchedulerAcceleratorState.accelerator_id),
|
||||
)
|
||||
)
|
||||
if active_serving_leases + active_legacy_leases:
|
||||
violations.append(
|
||||
f"node {node.id} retains {active_serving_leases + active_legacy_leases} "
|
||||
"active GPU leases"
|
||||
)
|
||||
if current_accelerator_state:
|
||||
violations.append(
|
||||
f"node {node.id} retains {current_accelerator_state} current accelerator rows"
|
||||
)
|
||||
active_accelerators = (
|
||||
session.scalar(
|
||||
select(func.count())
|
||||
.select_from(Accelerator)
|
||||
.where(
|
||||
Accelerator.id.in_(accelerator_ids),
|
||||
Accelerator.status != "decommissioned",
|
||||
)
|
||||
)
|
||||
or 0
|
||||
)
|
||||
if active_accelerators:
|
||||
violations.append(
|
||||
f"node {node.id} retains {active_accelerators} non-decommissioned accelerators"
|
||||
)
|
||||
active_deployments = sum(
|
||||
int(
|
||||
session.scalar(
|
||||
select(func.count())
|
||||
.select_from(model)
|
||||
.where(
|
||||
model.compute_node_id == node.id,
|
||||
status.not_in(("retired", "deprecated", "failed", "archived")),
|
||||
)
|
||||
)
|
||||
or 0
|
||||
)
|
||||
for model, status in (
|
||||
(CapabilityDeployment, CapabilityDeployment.status),
|
||||
(Deployment, Deployment.status),
|
||||
)
|
||||
)
|
||||
active_gateway_requests = (
|
||||
session.scalar(
|
||||
select(func.count())
|
||||
.select_from(GatewayRequest)
|
||||
.where(
|
||||
GatewayRequest.compute_node_id == node.id,
|
||||
GatewayRequest.status.in_(("queued", "running")),
|
||||
)
|
||||
)
|
||||
or 0
|
||||
)
|
||||
if active_deployments or active_gateway_requests:
|
||||
violations.append(
|
||||
f"node {node.id} retains {active_deployments} active deployments and "
|
||||
f"{active_gateway_requests} gateway requests"
|
||||
)
|
||||
unsafe_roots = (
|
||||
session.scalar(
|
||||
select(func.count())
|
||||
.select_from(StorageRoot)
|
||||
.where(
|
||||
StorageRoot.compute_node_id == node.id,
|
||||
(StorageRoot.writable.is_(True) | (StorageRoot.status != "unavailable")),
|
||||
)
|
||||
)
|
||||
or 0
|
||||
)
|
||||
if unsafe_roots:
|
||||
violations.append(f"node {node.id} retains {unsafe_roots} writable storage roots")
|
||||
current_node_state = sum(
|
||||
int(
|
||||
session.scalar(select(func.count()).select_from(model).where(column == node.id))
|
||||
or 0
|
||||
)
|
||||
for model, column in (
|
||||
(HostTelemetryLatest, HostTelemetryLatest.compute_node_id),
|
||||
(StorageVolumeState, StorageVolumeState.compute_node_id),
|
||||
)
|
||||
)
|
||||
if current_node_state:
|
||||
violations.append(f"node {node.id} retains {current_node_state} current telemetry rows")
|
||||
return _result(
|
||||
"decommissioned_nodes_are_terminal",
|
||||
"Decommissioned nodes cannot authenticate, schedule, or retain current runtime truth",
|
||||
InvariantSeverity.CRITICAL,
|
||||
violations,
|
||||
"every decommissioned node is a terminal tombstone with no active work or current state",
|
||||
"a decommissioned node retained mutable current state",
|
||||
)
|
||||
|
||||
|
||||
def _contracts_have_one_stable_identity(session: Session) -> InvariantResult:
|
||||
rows = session.execute(
|
||||
select(
|
||||
CapabilityContract.id,
|
||||
func.count(func.distinct(CapabilityDeployment.artifact_set_id)),
|
||||
)
|
||||
.join(
|
||||
CapabilityDeployment,
|
||||
CapabilityDeployment.capability_contract_id == CapabilityContract.id,
|
||||
)
|
||||
.where(
|
||||
CapabilityDeployment.production.is_(True),
|
||||
CapabilityDeployment.status == "stable",
|
||||
)
|
||||
.group_by(CapabilityContract.id)
|
||||
.having(func.count(func.distinct(CapabilityDeployment.artifact_set_id)) > 1)
|
||||
).all()
|
||||
return _result(
|
||||
"stable_identity_is_singular",
|
||||
"Stable production serves exactly one artifact set per contract",
|
||||
InvariantSeverity.CRITICAL,
|
||||
[
|
||||
f"contract {contract} serves {count} artifact sets from stable production"
|
||||
for contract, count in rows
|
||||
],
|
||||
"every contract serves a single stable production artifact set",
|
||||
"a contract serves more than one stable production artifact set",
|
||||
)
|
||||
|
||||
|
||||
CHECKS = (
|
||||
_no_duplicate_production_stable,
|
||||
_contracts_have_one_stable_identity,
|
||||
_no_duplicate_node_identity,
|
||||
_single_active_node_credential,
|
||||
_no_stale_gpu_lease,
|
||||
_no_mixed_embedding_space,
|
||||
_no_lifecycle_commit_without_evidence,
|
||||
_no_cutover_commit_without_validation,
|
||||
_no_restore_from_unverified_backup,
|
||||
_revoked_credentials_stay_revoked,
|
||||
_capability_clients_are_not_operators,
|
||||
_no_unsafe_artifact_promoted,
|
||||
_no_orphan_serving_work,
|
||||
_no_hidden_auto_promotion,
|
||||
_audit_chain_is_intact,
|
||||
_decommissioned_nodes_are_terminal,
|
||||
)
|
||||
|
||||
|
||||
def check_invariants(session: Session, now: datetime | None = None) -> InvariantReport:
|
||||
"""Run every platform invariant against authoritative state and report each outcome."""
|
||||
|
||||
moment = now or datetime.now(UTC)
|
||||
results: list[InvariantResult] = []
|
||||
for check in CHECKS:
|
||||
try:
|
||||
results.append(check(session, moment)) # type: ignore[call-arg]
|
||||
except TypeError:
|
||||
results.append(check(session))
|
||||
violated = sum(item.status is InvariantStatus.VIOLATED for item in results)
|
||||
return InvariantReport(
|
||||
observed_at=moment,
|
||||
checked=len(results),
|
||||
holding=sum(item.status is InvariantStatus.HOLDS for item in results),
|
||||
violated=violated,
|
||||
results=results,
|
||||
)
|
||||
|
||||
|
||||
def invariant_keys() -> tuple[str, ...]:
|
||||
"""Stable ordering used by the chaos harness and the release gate."""
|
||||
|
||||
return tuple(
|
||||
result.key
|
||||
for result in (
|
||||
InvariantResult(
|
||||
key=key,
|
||||
name=key,
|
||||
severity=InvariantSeverity.CRITICAL,
|
||||
status=InvariantStatus.HOLDS,
|
||||
observed=0,
|
||||
detail="",
|
||||
)
|
||||
for key in (
|
||||
"single_production_stable",
|
||||
"stable_identity_is_singular",
|
||||
"single_node_identity",
|
||||
"single_active_node_credential",
|
||||
"no_stale_gpu_lease",
|
||||
"no_mixed_embedding_space",
|
||||
"lifecycle_commit_has_evidence",
|
||||
"cutover_commit_has_validation",
|
||||
"restore_requires_verified_backup",
|
||||
"revoked_credentials_stay_revoked",
|
||||
"capability_clients_are_not_operators",
|
||||
"no_unsafe_artifact_promoted",
|
||||
"no_orphan_serving_work",
|
||||
"no_hidden_auto_promotion",
|
||||
"audit_chain_intact",
|
||||
"decommissioned_nodes_are_terminal",
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def summarise(report: InvariantReport) -> dict[str, Any]:
|
||||
return {
|
||||
"observed_at": report.observed_at.isoformat(),
|
||||
"checked": report.checked,
|
||||
"holding": report.holding,
|
||||
"violated": report.violated,
|
||||
"violations": [
|
||||
{"key": item.key, "detail": item.detail, "examples": item.violations}
|
||||
for item in report.results
|
||||
if item.status is InvariantStatus.VIOLATED
|
||||
],
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,122 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import Any, cast
|
||||
|
||||
import yaml # type: ignore[import-untyped]
|
||||
|
||||
from modelforge_api.domain.contracts import (
|
||||
BenchmarkSuiteManifest,
|
||||
CandidateRegistryManifest,
|
||||
CapabilityContractManifest,
|
||||
PolicyDefaults,
|
||||
ProjectManifest,
|
||||
)
|
||||
from modelforge_api.settings import get_settings
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[4]
|
||||
|
||||
|
||||
class ManifestRegistry:
|
||||
"""Validated, read-only M0 registry backed by version-controlled manifests.
|
||||
|
||||
Parsed manifests are memoised for the lifetime of the instance. They are read-only
|
||||
configuration mounted into the container and cannot change without restarting the process, so
|
||||
re-reading them per call bought nothing and cost a great deal: one readiness probe used to
|
||||
perform 77 YAML reads across 14 files — one file ten times — and take 528 ms. That was the whole
|
||||
of the 638 ms p50 M16 measured on /api/v1/health/ready under load, which is a real operational
|
||||
risk for an orchestrator with a tight probe timeout.
|
||||
|
||||
The cache is per-instance rather than global so a test pointing at a temporary directory still
|
||||
gets what is on disk when it builds its own registry.
|
||||
"""
|
||||
|
||||
def __init__(self, config_root: Path | None = None) -> None:
|
||||
self.config_root = config_root or get_settings().config_root
|
||||
self._cache: dict[str, Any] = {}
|
||||
|
||||
def _memoised(self, key: str, produce: Callable[[], Any]) -> Any:
|
||||
if key not in self._cache:
|
||||
self._cache[key] = produce()
|
||||
return self._cache[key]
|
||||
|
||||
@staticmethod
|
||||
def _read_yaml(path: Path) -> dict[str, Any]:
|
||||
with path.open("r", encoding="utf-8") as handle:
|
||||
payload = yaml.safe_load(handle)
|
||||
if not isinstance(payload, dict):
|
||||
raise TypeError(f"manifest must be an object: {path}")
|
||||
return payload
|
||||
|
||||
def capabilities(self) -> list[CapabilityContractManifest]:
|
||||
return cast(
|
||||
"list[CapabilityContractManifest]", self._memoised("capabilities", self._capabilities)
|
||||
)
|
||||
|
||||
def _capabilities(self) -> list[CapabilityContractManifest]:
|
||||
manifests = [
|
||||
CapabilityContractManifest.model_validate(self._read_yaml(path))
|
||||
for path in sorted((self.config_root / "capabilities").glob("*.yaml"))
|
||||
]
|
||||
identities = [(item.capability, item.version) for item in manifests]
|
||||
if len(identities) != len(set(identities)):
|
||||
raise ValueError("duplicate capability contract identity")
|
||||
return manifests
|
||||
|
||||
def projects(self) -> list[ProjectManifest]:
|
||||
return cast("list[ProjectManifest]", self._memoised("projects", self._projects))
|
||||
|
||||
def _projects(self) -> list[ProjectManifest]:
|
||||
manifests = [
|
||||
ProjectManifest.model_validate(self._read_yaml(path))
|
||||
for path in sorted((self.config_root / "projects").glob("*.yaml"))
|
||||
]
|
||||
capabilities = self.capabilities()
|
||||
by_identity = {(item.capability, item.version): item for item in capabilities}
|
||||
capability_versions = set(by_identity)
|
||||
for project in manifests:
|
||||
for key, binding in project.bindings.items():
|
||||
if (key, binding.contract_version) not in capability_versions:
|
||||
raise ValueError(
|
||||
f"{project.project.id} binds missing capability contract {key}@{binding.contract_version}"
|
||||
)
|
||||
contract = by_identity[key, binding.contract_version]
|
||||
if (
|
||||
contract.upgrade_class.value == "requires_reindex"
|
||||
and binding.migration_support != "reindex"
|
||||
):
|
||||
raise ValueError(
|
||||
f"{project.project.id}/{key} must declare reindex migration support"
|
||||
)
|
||||
return manifests
|
||||
|
||||
def candidates(self) -> CandidateRegistryManifest:
|
||||
return cast("CandidateRegistryManifest", self._memoised("candidates", self._candidates))
|
||||
|
||||
def _candidates(self) -> CandidateRegistryManifest:
|
||||
path = self.config_root / "models" / "initial-candidates.yaml"
|
||||
return CandidateRegistryManifest.model_validate(self._read_yaml(path))
|
||||
|
||||
def benchmarks(self) -> list[BenchmarkSuiteManifest]:
|
||||
return cast("list[BenchmarkSuiteManifest]", self._memoised("benchmarks", self._benchmarks))
|
||||
|
||||
def _benchmarks(self) -> list[BenchmarkSuiteManifest]:
|
||||
return [
|
||||
BenchmarkSuiteManifest.model_validate(self._read_yaml(path))
|
||||
for path in sorted((self.config_root / "benchmarks").glob("*.yaml"))
|
||||
]
|
||||
|
||||
def policies(self) -> PolicyDefaults:
|
||||
return cast("PolicyDefaults", self._memoised("policies", self._policies))
|
||||
|
||||
def _policies(self) -> PolicyDefaults:
|
||||
return PolicyDefaults.model_validate(
|
||||
self._read_yaml(self.config_root / "policies" / "defaults.yaml")
|
||||
)
|
||||
|
||||
|
||||
@lru_cache
|
||||
def get_manifest_registry() -> ManifestRegistry:
|
||||
return ManifestRegistry()
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user