211 lines
12 KiB
Python
211 lines
12 KiB
Python
"""M3 Hugging Face discovery and node-local artifact acquisition.
|
|
|
|
Revision ID: 20260825_0005
|
|
Revises: 20260825_0004
|
|
"""
|
|
|
|
import sqlalchemy as sa
|
|
|
|
from alembic import op
|
|
|
|
revision = "20260825_0005"
|
|
down_revision = "20260825_0004"
|
|
branch_labels = None
|
|
depends_on = None
|
|
|
|
|
|
def _identity() -> list[sa.Column]:
|
|
return [
|
|
sa.Column("id", sa.Uuid(), primary_key=True),
|
|
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
|
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
|
]
|
|
|
|
|
|
def upgrade() -> None:
|
|
op.add_column("storage_roots", sa.Column("agent_path", sa.Text()))
|
|
op.create_table(
|
|
"upstream_snapshots",
|
|
sa.Column("model_id", sa.Uuid()),
|
|
sa.Column("provider", sa.String(64), nullable=False, server_default="huggingface"),
|
|
sa.Column("repository_id", sa.String(255), nullable=False),
|
|
sa.Column("requested_revision", sa.String(255), nullable=False, server_default="main"),
|
|
sa.Column("resolved_commit_sha", sa.String(64), nullable=False),
|
|
sa.Column("access_state", sa.String(32), nullable=False, server_default="public"),
|
|
sa.Column("metadata_snapshot", sa.JSON(), nullable=False, server_default=sa.text("'{}'")),
|
|
sa.Column("card_metadata", sa.JSON(), nullable=False, server_default=sa.text("'{}'")),
|
|
sa.Column("security_metadata", sa.JSON(), nullable=False, server_default=sa.text("'{}'")),
|
|
sa.Column("source_updated_at", sa.DateTime(timezone=True)),
|
|
sa.Column("observed_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
|
|
sa.Column("stale_after", sa.DateTime(timezone=True), nullable=False),
|
|
sa.Column("id", sa.Uuid(), primary_key=True),
|
|
sa.ForeignKeyConstraint(["model_id"], ["models.id"], ondelete="RESTRICT"),
|
|
sa.CheckConstraint("length(resolved_commit_sha) >= 40", name="ck_snapshot_commit_length"),
|
|
)
|
|
op.create_index("ix_upstream_snapshots_model_id", "upstream_snapshots", ["model_id"])
|
|
op.create_index("ix_upstream_snapshots_repository_id", "upstream_snapshots", ["repository_id"])
|
|
op.create_index("ix_upstream_snapshots_resolved_commit_sha", "upstream_snapshots", ["resolved_commit_sha"])
|
|
op.create_table(
|
|
"upstream_files",
|
|
sa.Column("snapshot_id", sa.Uuid(), nullable=False),
|
|
sa.Column("path", sa.Text(), nullable=False),
|
|
sa.Column("size_bytes", sa.BigInteger()),
|
|
sa.Column("blob_id", sa.String(128)),
|
|
sa.Column("upstream_sha256", sa.String(64)),
|
|
sa.Column("file_format", sa.String(64), nullable=False, server_default="unknown"),
|
|
sa.Column("role", sa.String(64), nullable=False, server_default="other"),
|
|
sa.Column("risk_flags", sa.JSON(), nullable=False, server_default=sa.text("'[]'")),
|
|
sa.Column("metadata_snapshot", sa.JSON(), nullable=False, server_default=sa.text("'{}'")),
|
|
sa.Column("id", sa.Uuid(), primary_key=True),
|
|
sa.ForeignKeyConstraint(["snapshot_id"], ["upstream_snapshots.id"], ondelete="RESTRICT"),
|
|
sa.UniqueConstraint("snapshot_id", "path", name="uq_upstream_file_snapshot_path"),
|
|
sa.CheckConstraint("size_bytes IS NULL OR size_bytes >= 0", name="ck_upstream_file_size"),
|
|
)
|
|
op.create_index("ix_upstream_files_snapshot_id", "upstream_files", ["snapshot_id"])
|
|
op.create_table(
|
|
"artifact_sets",
|
|
sa.Column("revision_id", sa.Uuid(), nullable=False),
|
|
sa.Column("snapshot_id", sa.Uuid(), nullable=False),
|
|
sa.Column("variant_key", sa.String(128), nullable=False),
|
|
sa.Column("label", sa.String(255), nullable=False),
|
|
sa.Column("selection_reason", sa.Text(), nullable=False),
|
|
sa.Column("selected_paths", sa.JSON(), nullable=False, server_default=sa.text("'[]'")),
|
|
sa.Column("total_size_bytes", sa.BigInteger(), nullable=False, server_default="0"),
|
|
sa.Column("file_count", sa.Integer(), nullable=False, server_default="0"),
|
|
sa.Column("availability", sa.String(32), nullable=False, server_default="remote"),
|
|
sa.Column("status", sa.String(32), nullable=False, server_default="remote"),
|
|
sa.Column("completeness", sa.String(32), nullable=False, server_default="planned"),
|
|
sa.Column("security_status", sa.String(32), nullable=False, server_default="unverified"),
|
|
sa.Column("license_status", sa.String(32), nullable=False, server_default="unknown"),
|
|
sa.Column("immutable_at", sa.DateTime(timezone=True), nullable=False),
|
|
*_identity(),
|
|
sa.ForeignKeyConstraint(["revision_id"], ["model_revisions.id"], ondelete="RESTRICT"),
|
|
sa.ForeignKeyConstraint(["snapshot_id"], ["upstream_snapshots.id"], ondelete="RESTRICT"),
|
|
sa.UniqueConstraint("revision_id", "variant_key", name="uq_artifact_set_revision_variant"),
|
|
sa.CheckConstraint("total_size_bytes >= 0", name="ck_artifact_set_size"),
|
|
)
|
|
op.create_index("ix_artifact_sets_revision_id", "artifact_sets", ["revision_id"])
|
|
op.create_index("ix_artifact_sets_snapshot_id", "artifact_sets", ["snapshot_id"])
|
|
op.create_table(
|
|
"download_plans",
|
|
sa.Column("artifact_set_id", sa.Uuid(), nullable=False),
|
|
sa.Column("compute_node_id", sa.Uuid(), nullable=False),
|
|
sa.Column("storage_root_id", sa.Uuid(), nullable=False),
|
|
sa.Column("repository_id", sa.String(255), nullable=False),
|
|
sa.Column("resolved_commit_sha", sa.String(64), nullable=False),
|
|
sa.Column("total_size_bytes", sa.BigInteger(), nullable=False),
|
|
sa.Column("file_count", sa.Integer(), nullable=False),
|
|
sa.Column("status", sa.String(32), nullable=False, server_default="ready"),
|
|
sa.Column("idempotency_key", sa.String(64), nullable=False),
|
|
sa.Column("preflight", sa.JSON(), nullable=False, server_default=sa.text("'{}'")),
|
|
sa.Column("immutable_payload", sa.JSON(), nullable=False),
|
|
sa.Column("planned_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
|
|
sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False),
|
|
sa.Column("immutable_at", sa.DateTime(timezone=True), nullable=False),
|
|
*_identity(),
|
|
sa.ForeignKeyConstraint(["artifact_set_id"], ["artifact_sets.id"], ondelete="RESTRICT"),
|
|
sa.ForeignKeyConstraint(["compute_node_id"], ["compute_nodes.id"], ondelete="RESTRICT"),
|
|
sa.ForeignKeyConstraint(["storage_root_id"], ["storage_roots.id"], ondelete="RESTRICT"),
|
|
sa.UniqueConstraint("idempotency_key", name="uq_download_plan_idempotency"),
|
|
sa.CheckConstraint("total_size_bytes >= 0", name="ck_download_plan_size"),
|
|
)
|
|
for col in ("artifact_set_id", "compute_node_id", "storage_root_id"):
|
|
op.create_index(f"ix_download_plans_{col}", "download_plans", [col])
|
|
op.create_table(
|
|
"download_plan_files",
|
|
sa.Column("plan_id", sa.Uuid(), nullable=False),
|
|
sa.Column("ordinal", sa.Integer(), nullable=False),
|
|
sa.Column("path", sa.Text(), nullable=False),
|
|
sa.Column("size_bytes", sa.BigInteger(), nullable=False),
|
|
sa.Column("upstream_sha256", sa.String(64)),
|
|
sa.Column("file_format", sa.String(64), nullable=False),
|
|
sa.Column("role", sa.String(64), nullable=False),
|
|
sa.Column("risk_flags", sa.JSON(), nullable=False, server_default=sa.text("'[]'")),
|
|
sa.Column("id", sa.Uuid(), primary_key=True),
|
|
sa.ForeignKeyConstraint(["plan_id"], ["download_plans.id"], ondelete="RESTRICT"),
|
|
sa.UniqueConstraint("plan_id", "path", name="uq_download_plan_file_path"),
|
|
sa.UniqueConstraint("plan_id", "ordinal", name="uq_download_plan_file_ordinal"),
|
|
)
|
|
op.create_index("ix_download_plan_files_plan_id", "download_plan_files", ["plan_id"])
|
|
op.create_table(
|
|
"artifact_jobs",
|
|
sa.Column("plan_id", sa.Uuid(), nullable=False),
|
|
sa.Column("compute_node_id", sa.Uuid(), nullable=False),
|
|
sa.Column("storage_root_id", sa.Uuid(), nullable=False),
|
|
sa.Column("status", sa.String(32), nullable=False, server_default="queued"),
|
|
sa.Column("idempotency_key", sa.String(64), nullable=False),
|
|
sa.Column("attempt_count", sa.Integer(), nullable=False, server_default="0"),
|
|
sa.Column("lease_token_hash", sa.String(64)),
|
|
sa.Column("lease_expires_at", sa.DateTime(timezone=True)),
|
|
sa.Column("progress_bytes", sa.BigInteger(), nullable=False, server_default="0"),
|
|
sa.Column("total_bytes", sa.BigInteger(), nullable=False),
|
|
sa.Column("current_file", sa.Text()),
|
|
sa.Column("cancel_requested", sa.Boolean(), nullable=False, server_default=sa.false()),
|
|
sa.Column("quarantine_relative_path", sa.Text()),
|
|
sa.Column("promoted_relative_path", sa.Text()),
|
|
sa.Column("error_code", sa.String(64)),
|
|
sa.Column("error_message", sa.Text()),
|
|
sa.Column("result", sa.JSON(), nullable=False, server_default=sa.text("'{}'")),
|
|
sa.Column("started_at", sa.DateTime(timezone=True)),
|
|
sa.Column("completed_at", sa.DateTime(timezone=True)),
|
|
*_identity(),
|
|
sa.ForeignKeyConstraint(["plan_id"], ["download_plans.id"], ondelete="RESTRICT"),
|
|
sa.ForeignKeyConstraint(["compute_node_id"], ["compute_nodes.id"], ondelete="RESTRICT"),
|
|
sa.ForeignKeyConstraint(["storage_root_id"], ["storage_roots.id"], ondelete="RESTRICT"),
|
|
sa.UniqueConstraint("plan_id", name="uq_artifact_job_plan"),
|
|
sa.UniqueConstraint("idempotency_key", name="uq_artifact_job_idempotency"),
|
|
)
|
|
for col in ("plan_id", "compute_node_id", "storage_root_id", "status"):
|
|
op.create_index(f"ix_artifact_jobs_{col}", "artifact_jobs", [col])
|
|
op.create_table(
|
|
"artifact_job_attempts",
|
|
sa.Column("job_id", sa.Uuid(), nullable=False),
|
|
sa.Column("attempt", sa.Integer(), nullable=False),
|
|
sa.Column("status", sa.String(32), nullable=False),
|
|
sa.Column("started_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
|
|
sa.Column("completed_at", sa.DateTime(timezone=True)),
|
|
sa.Column("details", sa.JSON(), nullable=False, server_default=sa.text("'{}'")),
|
|
sa.Column("id", sa.Uuid(), primary_key=True),
|
|
sa.ForeignKeyConstraint(["job_id"], ["artifact_jobs.id"], ondelete="RESTRICT"),
|
|
)
|
|
op.create_index("ix_artifact_job_attempts_job_id", "artifact_job_attempts", ["job_id"])
|
|
op.create_table(
|
|
"artifact_inspections",
|
|
sa.Column("job_id", sa.Uuid(), nullable=False),
|
|
sa.Column("file_path", sa.Text(), nullable=False),
|
|
sa.Column("inspection_type", sa.String(64), nullable=False),
|
|
sa.Column("status", sa.String(32), nullable=False),
|
|
sa.Column("severity", sa.String(32), nullable=False),
|
|
sa.Column("evidence", sa.JSON(), nullable=False, server_default=sa.text("'{}'")),
|
|
sa.Column("observed_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
|
|
sa.Column("id", sa.Uuid(), primary_key=True),
|
|
sa.ForeignKeyConstraint(["job_id"], ["artifact_jobs.id"], ondelete="RESTRICT"),
|
|
)
|
|
op.create_index("ix_artifact_inspections_job_id", "artifact_inspections", ["job_id"])
|
|
op.create_table(
|
|
"artifact_set_members",
|
|
sa.Column("artifact_set_id", sa.Uuid(), nullable=False),
|
|
sa.Column("artifact_id", sa.Uuid(), nullable=False),
|
|
sa.Column("ordinal", sa.Integer(), nullable=False),
|
|
sa.Column("required", sa.Boolean(), nullable=False, server_default=sa.true()),
|
|
sa.ForeignKeyConstraint(["artifact_set_id"], ["artifact_sets.id"], ondelete="RESTRICT"),
|
|
sa.ForeignKeyConstraint(["artifact_id"], ["model_artifacts.id"], ondelete="RESTRICT"),
|
|
sa.PrimaryKeyConstraint("artifact_set_id", "artifact_id"),
|
|
)
|
|
|
|
|
|
def downgrade() -> None:
|
|
for table in (
|
|
"artifact_set_members",
|
|
"artifact_inspections",
|
|
"artifact_job_attempts",
|
|
"artifact_jobs",
|
|
"download_plan_files",
|
|
"download_plans",
|
|
"artifact_sets",
|
|
"upstream_files",
|
|
"upstream_snapshots",
|
|
):
|
|
op.drop_table(table)
|
|
op.drop_column("storage_roots", "agent_path")
|