171 lines
7.6 KiB
Python
171 lines
7.6 KiB
Python
"""M2 operational model registry and artifact provenance.
|
|
|
|
Revision ID: 20260825_0004
|
|
Revises: 20260825_0003
|
|
"""
|
|
|
|
import sqlalchemy as sa
|
|
|
|
from alembic import op
|
|
|
|
revision = "20260825_0004"
|
|
down_revision = "20260825_0003"
|
|
branch_labels = None
|
|
depends_on = None
|
|
|
|
|
|
def upgrade() -> None:
|
|
for column in (
|
|
sa.Column("source_type", sa.String(32), nullable=False, server_default="huggingface"),
|
|
sa.Column("upstream_metadata", sa.JSON(), nullable=False, server_default=sa.text("'{}'")),
|
|
sa.Column("local_metadata", sa.JSON(), nullable=False, server_default=sa.text("'{}'")),
|
|
sa.Column(
|
|
"interpretation_metadata", sa.JSON(), nullable=False, server_default=sa.text("'{}'")
|
|
),
|
|
sa.Column("description", sa.Text()),
|
|
):
|
|
op.add_column("models", column)
|
|
op.add_column("model_revisions", sa.Column("archived_at", sa.DateTime(timezone=True)))
|
|
for column in (
|
|
sa.Column("status", sa.String(32), nullable=False, server_default="remote"),
|
|
sa.Column(
|
|
"verification_details", sa.JSON(), nullable=False, server_default=sa.text("'{}'")
|
|
),
|
|
sa.Column("archived_at", sa.DateTime(timezone=True)),
|
|
):
|
|
op.add_column("model_artifacts", column)
|
|
op.alter_column("model_artifacts", "storage_uri", existing_type=sa.Text(), nullable=True)
|
|
for column in (
|
|
sa.Column(
|
|
"transformation_type", sa.String(64), nullable=False, server_default="conversion"
|
|
),
|
|
sa.Column(
|
|
"environment_snapshot", sa.JSON(), nullable=False, server_default=sa.text("'{}'")
|
|
),
|
|
sa.Column("status", sa.String(32), nullable=False, server_default="remote"),
|
|
sa.Column(
|
|
"verification_details", sa.JSON(), nullable=False, server_default=sa.text("'{}'")
|
|
),
|
|
):
|
|
op.add_column("derived_artifacts", column)
|
|
op.alter_column("derived_artifacts", "storage_uri", existing_type=sa.Text(), nullable=True)
|
|
op.alter_column(
|
|
"derived_artifacts", "source_artifact_id", existing_type=sa.Uuid(), nullable=True
|
|
)
|
|
|
|
op.create_table(
|
|
"storage_roots",
|
|
sa.Column("compute_node_id", sa.Uuid(), nullable=False),
|
|
sa.Column("name", sa.String(128), nullable=False),
|
|
sa.Column("purpose", sa.String(64), nullable=False, server_default="model_artifacts"),
|
|
sa.Column("path", sa.Text(), nullable=False),
|
|
sa.Column("status", sa.String(32), nullable=False, server_default="unknown"),
|
|
sa.Column("writable", sa.Boolean(), nullable=False, server_default=sa.false()),
|
|
sa.Column("capacity_bytes", sa.BigInteger()),
|
|
sa.Column("free_bytes", sa.BigInteger()),
|
|
sa.Column("reserve_bytes", sa.BigInteger(), nullable=False, server_default="0"),
|
|
sa.Column("reserve_percent", sa.Integer(), nullable=False, server_default="10"),
|
|
sa.Column("capacity_observed_at", sa.DateTime(timezone=True)),
|
|
sa.Column("validation_details", sa.JSON(), nullable=False, server_default=sa.text("'{}'")),
|
|
sa.Column("deprecated_at", sa.DateTime(timezone=True)),
|
|
sa.Column("id", sa.Uuid(), primary_key=True),
|
|
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
|
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
|
sa.ForeignKeyConstraint(["compute_node_id"], ["compute_nodes.id"], ondelete="RESTRICT"),
|
|
sa.UniqueConstraint("compute_node_id", "path", name="uq_storage_root_node_path"),
|
|
sa.CheckConstraint("reserve_bytes >= 0", name="ck_storage_root_reserve_bytes"),
|
|
sa.CheckConstraint(
|
|
"reserve_percent >= 0 AND reserve_percent <= 100",
|
|
name="ck_storage_root_reserve_percent",
|
|
),
|
|
)
|
|
op.create_index("ix_storage_roots_compute_node_id", "storage_roots", ["compute_node_id"])
|
|
op.create_table(
|
|
"derived_artifact_sources",
|
|
sa.Column("derived_artifact_id", sa.Uuid(), nullable=False),
|
|
sa.Column("source_artifact_id", sa.Uuid(), nullable=False),
|
|
sa.Column("ordinal", sa.Integer(), nullable=False, server_default="0"),
|
|
sa.Column("source_sha256", sa.String(64), nullable=False),
|
|
sa.ForeignKeyConstraint(
|
|
["derived_artifact_id"], ["derived_artifacts.id"], ondelete="RESTRICT"
|
|
),
|
|
sa.ForeignKeyConstraint(
|
|
["source_artifact_id"], ["model_artifacts.id"], ondelete="RESTRICT"
|
|
),
|
|
sa.PrimaryKeyConstraint("derived_artifact_id", "source_artifact_id"),
|
|
sa.UniqueConstraint(
|
|
"derived_artifact_id", "source_artifact_id", name="uq_derived_artifact_source"
|
|
),
|
|
)
|
|
op.create_table(
|
|
"artifact_locations",
|
|
sa.Column("artifact_id", sa.Uuid()),
|
|
sa.Column("derived_artifact_id", sa.Uuid()),
|
|
sa.Column("storage_root_id", sa.Uuid(), nullable=False),
|
|
sa.Column("relative_path", sa.Text(), nullable=False),
|
|
sa.Column("status", sa.String(32), nullable=False, server_default="unknown"),
|
|
sa.Column("size_bytes", sa.BigInteger()),
|
|
sa.Column("observed_sha256", sa.String(64)),
|
|
sa.Column("last_checked_at", sa.DateTime(timezone=True)),
|
|
sa.Column("id", sa.Uuid(), primary_key=True),
|
|
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
|
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
|
sa.ForeignKeyConstraint(["artifact_id"], ["model_artifacts.id"], ondelete="RESTRICT"),
|
|
sa.ForeignKeyConstraint(
|
|
["derived_artifact_id"], ["derived_artifacts.id"], ondelete="RESTRICT"
|
|
),
|
|
sa.ForeignKeyConstraint(["storage_root_id"], ["storage_roots.id"], ondelete="RESTRICT"),
|
|
sa.CheckConstraint(
|
|
"(artifact_id IS NOT NULL AND derived_artifact_id IS NULL) OR "
|
|
"(artifact_id IS NULL AND derived_artifact_id IS NOT NULL)",
|
|
name="ck_artifact_location_one_owner",
|
|
),
|
|
sa.UniqueConstraint(
|
|
"storage_root_id", "relative_path", name="uq_storage_root_relative_path"
|
|
),
|
|
)
|
|
op.create_index("ix_artifact_locations_artifact_id", "artifact_locations", ["artifact_id"])
|
|
op.create_index(
|
|
"ix_artifact_locations_derived_artifact_id",
|
|
"artifact_locations",
|
|
["derived_artifact_id"],
|
|
)
|
|
op.create_index(
|
|
"ix_artifact_locations_storage_root_id", "artifact_locations", ["storage_root_id"]
|
|
)
|
|
|
|
|
|
def downgrade() -> None:
|
|
op.drop_table("artifact_locations")
|
|
op.drop_table("derived_artifact_sources")
|
|
op.drop_table("storage_roots")
|
|
op.execute(
|
|
"UPDATE model_artifacts SET storage_uri = 'registry://artifact/' || id::text "
|
|
"WHERE storage_uri IS NULL"
|
|
)
|
|
op.execute(
|
|
"UPDATE derived_artifacts SET storage_uri = 'registry://derived/' || id::text "
|
|
"WHERE storage_uri IS NULL"
|
|
)
|
|
op.alter_column("derived_artifacts", "source_artifact_id", existing_type=sa.Uuid(), nullable=False)
|
|
op.alter_column("derived_artifacts", "storage_uri", existing_type=sa.Text(), nullable=False)
|
|
for column in (
|
|
"verification_details",
|
|
"status",
|
|
"environment_snapshot",
|
|
"transformation_type",
|
|
):
|
|
op.drop_column("derived_artifacts", column)
|
|
op.alter_column("model_artifacts", "storage_uri", existing_type=sa.Text(), nullable=False)
|
|
for column in ("archived_at", "verification_details", "status"):
|
|
op.drop_column("model_artifacts", column)
|
|
op.drop_column("model_revisions", "archived_at")
|
|
for column in (
|
|
"description",
|
|
"interpretation_metadata",
|
|
"local_metadata",
|
|
"upstream_metadata",
|
|
"source_type",
|
|
):
|
|
op.drop_column("models", column)
|