334 lines
14 KiB
Python
334 lines
14 KiB
Python
"""Add versioned recovery policies, immutable backup sets, restore journals and artifact recovery.
|
|
|
|
Revision ID: 20260827_0021
|
|
Revises: 20260827_0020
|
|
"""
|
|
|
|
from collections.abc import Sequence
|
|
|
|
import sqlalchemy as sa
|
|
|
|
from alembic import op
|
|
|
|
revision: str = "20260827_0021"
|
|
down_revision: str | None = "20260827_0020"
|
|
branch_labels: str | Sequence[str] | None = None
|
|
depends_on: str | Sequence[str] | None = None
|
|
|
|
|
|
def _id() -> sa.Column[object]:
|
|
return sa.Column("id", sa.Uuid(), primary_key=True, nullable=False)
|
|
|
|
|
|
def _created() -> sa.Column[object]:
|
|
return sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now())
|
|
|
|
|
|
def upgrade() -> None:
|
|
op.create_table(
|
|
"recovery_policy_revisions",
|
|
_id(),
|
|
sa.Column("key", sa.String(128), nullable=False),
|
|
sa.Column("revision", sa.Integer(), nullable=False),
|
|
sa.Column("name", sa.String(255), nullable=False),
|
|
sa.Column("asset_class", sa.String(32), nullable=False),
|
|
sa.Column("backup_method", sa.String(48), nullable=False),
|
|
sa.Column("retention_days", sa.Integer(), nullable=False),
|
|
sa.Column("minimum_verified_backups", sa.Integer(), nullable=False),
|
|
sa.Column("rpo_seconds", sa.Integer()),
|
|
sa.Column("rto_target_seconds", sa.Integer()),
|
|
sa.Column("restore_verification", sa.String(32), nullable=False),
|
|
sa.Column("encryption_required", sa.Boolean(), nullable=False, server_default=sa.false()),
|
|
sa.Column("external_dependency", sa.Boolean(), nullable=False, server_default=sa.false()),
|
|
sa.Column("rehydration_allowed", sa.Boolean(), nullable=False, server_default=sa.false()),
|
|
sa.Column("secret_class", sa.String(32)),
|
|
sa.Column("rationale", sa.Text(), nullable=False),
|
|
sa.Column("active", sa.Boolean(), nullable=False, server_default=sa.true()),
|
|
sa.Column("fingerprint", sa.String(64), nullable=False, unique=True),
|
|
sa.Column("created_by", sa.String(255), nullable=False),
|
|
_created(),
|
|
sa.Column("immutable_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
|
sa.UniqueConstraint("key", "revision", name="uq_recovery_policy_key_revision"),
|
|
sa.CheckConstraint("retention_days >= 1", name="ck_recovery_policy_retention"),
|
|
sa.CheckConstraint(
|
|
"minimum_verified_backups >= 1", name="ck_recovery_policy_minimum_verified"
|
|
),
|
|
)
|
|
for column in ("key", "asset_class", "active"):
|
|
op.create_index(
|
|
f"ix_recovery_policy_revisions_{column}", "recovery_policy_revisions", [column]
|
|
)
|
|
|
|
op.create_table(
|
|
"recovery_asset_records",
|
|
_id(),
|
|
sa.Column("key", sa.String(128), nullable=False, unique=True),
|
|
sa.Column("name", sa.String(255), nullable=False),
|
|
sa.Column("asset_class", sa.String(32), nullable=False),
|
|
sa.Column("owner", sa.String(128), nullable=False),
|
|
sa.Column("location", sa.Text(), nullable=False),
|
|
sa.Column("backup_method", sa.String(48), nullable=False),
|
|
sa.Column("restore_method", sa.Text(), nullable=False),
|
|
sa.Column("rebuild_method", sa.Text()),
|
|
sa.Column("rpo_seconds", sa.Integer()),
|
|
sa.Column("readiness", sa.String(32), nullable=False),
|
|
sa.Column("dependencies", sa.JSON(), nullable=False),
|
|
sa.Column("notes", sa.Text(), nullable=False, server_default=""),
|
|
sa.Column(
|
|
"policy_revision_id",
|
|
sa.Uuid(),
|
|
sa.ForeignKey("recovery_policy_revisions.id", ondelete="RESTRICT"),
|
|
nullable=False,
|
|
),
|
|
_created(),
|
|
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
|
)
|
|
for column in ("key", "asset_class", "readiness", "policy_revision_id"):
|
|
op.create_index(f"ix_recovery_asset_records_{column}", "recovery_asset_records", [column])
|
|
|
|
op.create_table(
|
|
"backup_sets",
|
|
_id(),
|
|
sa.Column("backup_id", sa.String(64), nullable=False),
|
|
sa.Column("state", sa.String(32), nullable=False, server_default="PLANNED"),
|
|
sa.Column(
|
|
"policy_revision_id",
|
|
sa.Uuid(),
|
|
sa.ForeignKey("recovery_policy_revisions.id", ondelete="RESTRICT"),
|
|
nullable=False,
|
|
),
|
|
sa.Column("modelforge_version", sa.String(64), nullable=False),
|
|
sa.Column("modelforge_commit", sa.String(64)),
|
|
sa.Column("source_repository", sa.Text()),
|
|
sa.Column("source_reference", sa.String(255)),
|
|
sa.Column("schema_revision", sa.String(64)),
|
|
sa.Column("environment_fingerprint", sa.JSON(), nullable=False),
|
|
sa.Column("database_identity", sa.JSON(), nullable=False),
|
|
sa.Column("destination_root", sa.Text(), nullable=False),
|
|
sa.Column("payload_relative_path", sa.Text()),
|
|
sa.Column("payload_sha256", sa.String(64)),
|
|
sa.Column("manifest_relative_path", sa.Text()),
|
|
sa.Column("manifest_sha256", sa.String(64)),
|
|
sa.Column("included_asset_classes", sa.JSON(), nullable=False),
|
|
sa.Column("excluded_asset_classes", sa.JSON(), nullable=False),
|
|
sa.Column("payload_bytes", sa.BigInteger(), nullable=False, server_default="0"),
|
|
sa.Column("encrypted", sa.Boolean(), nullable=False, server_default=sa.false()),
|
|
sa.Column("encryption_algorithm", sa.String(64)),
|
|
sa.Column("encryption_key_id", sa.String(128)),
|
|
sa.Column("verification_details", sa.JSON(), nullable=False),
|
|
sa.Column("verified_at", sa.DateTime(timezone=True)),
|
|
sa.Column("failure_code", sa.String(64)),
|
|
sa.Column("failure_reason", sa.Text()),
|
|
sa.Column("milestone", sa.String(64)),
|
|
sa.Column("legal_hold", sa.Boolean(), nullable=False, server_default=sa.false()),
|
|
sa.Column("reason", sa.Text(), nullable=False),
|
|
sa.Column("created_by", sa.String(255), nullable=False),
|
|
sa.Column("started_at", sa.DateTime(timezone=True)),
|
|
sa.Column("completed_at", sa.DateTime(timezone=True)),
|
|
sa.Column("expires_at", sa.DateTime(timezone=True)),
|
|
_created(),
|
|
sa.Column("immutable_at", sa.DateTime(timezone=True)),
|
|
sa.UniqueConstraint("backup_id", name="uq_backup_set_backup_id"),
|
|
sa.CheckConstraint("payload_bytes >= 0", name="ck_backup_set_payload_bytes"),
|
|
)
|
|
for column in (
|
|
"backup_id",
|
|
"state",
|
|
"policy_revision_id",
|
|
"schema_revision",
|
|
"manifest_sha256",
|
|
"verified_at",
|
|
"milestone",
|
|
"legal_hold",
|
|
"expires_at",
|
|
):
|
|
op.create_index(f"ix_backup_sets_{column}", "backup_sets", [column])
|
|
|
|
op.create_table(
|
|
"backup_manifest_entries",
|
|
_id(),
|
|
sa.Column(
|
|
"backup_set_id",
|
|
sa.Uuid(),
|
|
sa.ForeignKey("backup_sets.id", ondelete="CASCADE"),
|
|
nullable=False,
|
|
),
|
|
sa.Column("logical_asset_type", sa.String(64), nullable=False),
|
|
sa.Column("object_name", sa.String(255), nullable=False),
|
|
sa.Column("relative_path", sa.Text(), nullable=False),
|
|
sa.Column("size_bytes", sa.BigInteger(), nullable=False),
|
|
sa.Column("sha256", sa.String(64), nullable=False),
|
|
sa.Column("source_generation", sa.String(128), nullable=False),
|
|
sa.Column("schema_version", sa.String(64)),
|
|
sa.Column("dependency_refs", sa.JSON(), nullable=False),
|
|
_created(),
|
|
sa.UniqueConstraint("backup_set_id", "object_name", name="uq_backup_manifest_object"),
|
|
sa.CheckConstraint("length(sha256) = 64", name="ck_backup_manifest_sha256_length"),
|
|
sa.CheckConstraint("size_bytes >= 0", name="ck_backup_manifest_size"),
|
|
)
|
|
for column in ("backup_set_id", "logical_asset_type"):
|
|
op.create_index(
|
|
f"ix_backup_manifest_entries_{column}", "backup_manifest_entries", [column]
|
|
)
|
|
|
|
op.create_table(
|
|
"restore_plans",
|
|
_id(),
|
|
sa.Column(
|
|
"backup_set_id",
|
|
sa.Uuid(),
|
|
sa.ForeignKey("backup_sets.id", ondelete="RESTRICT"),
|
|
nullable=False,
|
|
),
|
|
sa.Column("mode", sa.String(32), nullable=False),
|
|
sa.Column("state", sa.String(32), nullable=False, server_default="DRAFT"),
|
|
sa.Column("target_environment", sa.String(32), nullable=False),
|
|
sa.Column("target_label", sa.String(128), nullable=False),
|
|
sa.Column("database_destination", sa.Text(), nullable=False),
|
|
sa.Column("artifact_strategy", sa.String(32), nullable=False),
|
|
sa.Column("secret_strategy", sa.String(32), nullable=False),
|
|
sa.Column("node_strategy", sa.String(32), nullable=False),
|
|
sa.Column("expected_modelforge_version", sa.String(64)),
|
|
sa.Column("preflight", sa.JSON(), nullable=False),
|
|
sa.Column("validation_requirements", sa.JSON(), nullable=False),
|
|
sa.Column("fingerprint", sa.String(64), nullable=False),
|
|
sa.Column("reason", sa.Text(), nullable=False),
|
|
sa.Column("created_by", sa.String(255), nullable=False),
|
|
_created(),
|
|
sa.UniqueConstraint("fingerprint", name="uq_restore_plan_fingerprint"),
|
|
)
|
|
for column in ("backup_set_id", "mode", "state", "target_environment"):
|
|
op.create_index(f"ix_restore_plans_{column}", "restore_plans", [column])
|
|
|
|
op.create_table(
|
|
"restore_operations",
|
|
_id(),
|
|
sa.Column(
|
|
"plan_id",
|
|
sa.Uuid(),
|
|
sa.ForeignKey("restore_plans.id", ondelete="RESTRICT"),
|
|
nullable=False,
|
|
),
|
|
sa.Column(
|
|
"backup_set_id",
|
|
sa.Uuid(),
|
|
sa.ForeignKey("backup_sets.id", ondelete="RESTRICT"),
|
|
nullable=False,
|
|
),
|
|
sa.Column("state", sa.String(48), nullable=False, server_default="PLANNED"),
|
|
sa.Column("attempt", sa.Integer(), nullable=False, server_default="1"),
|
|
sa.Column("idempotency_key", sa.String(64), nullable=False),
|
|
sa.Column("preflight_result", sa.JSON(), nullable=False),
|
|
sa.Column("phase_durations", sa.JSON(), nullable=False),
|
|
sa.Column("source_fingerprint", sa.JSON(), nullable=False),
|
|
sa.Column("restored_fingerprint", sa.JSON(), nullable=False),
|
|
sa.Column("fingerprint_diff", sa.JSON(), nullable=False),
|
|
sa.Column("validation_result", sa.JSON(), nullable=False),
|
|
sa.Column("rpo_seconds", sa.Float()),
|
|
sa.Column("rto_seconds", sa.Float()),
|
|
sa.Column("failure_code", sa.String(64)),
|
|
sa.Column("failure_reason", sa.Text()),
|
|
sa.Column("started_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
|
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
|
sa.Column("ready_at", sa.DateTime(timezone=True)),
|
|
sa.UniqueConstraint("idempotency_key", name="uq_restore_operation_idempotency"),
|
|
)
|
|
for column in ("plan_id", "backup_set_id", "state"):
|
|
op.create_index(f"ix_restore_operations_{column}", "restore_operations", [column])
|
|
|
|
op.create_table(
|
|
"restore_operation_events",
|
|
_id(),
|
|
sa.Column(
|
|
"restore_operation_id",
|
|
sa.Uuid(),
|
|
sa.ForeignKey("restore_operations.id", ondelete="CASCADE"),
|
|
nullable=False,
|
|
),
|
|
sa.Column("from_state", sa.String(48)),
|
|
sa.Column("to_state", sa.String(48), nullable=False),
|
|
sa.Column("phase", sa.String(48), nullable=False),
|
|
sa.Column("actor", sa.String(255), nullable=False),
|
|
sa.Column("reason", sa.Text(), nullable=False),
|
|
sa.Column("evidence", sa.JSON(), nullable=False),
|
|
sa.Column("occurred_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
|
)
|
|
for column in ("restore_operation_id", "to_state", "occurred_at"):
|
|
op.create_index(
|
|
f"ix_restore_operation_events_{column}", "restore_operation_events", [column]
|
|
)
|
|
|
|
op.create_table(
|
|
"artifact_recovery_operations",
|
|
_id(),
|
|
sa.Column(
|
|
"restore_operation_id",
|
|
sa.Uuid(),
|
|
sa.ForeignKey("restore_operations.id", ondelete="SET NULL"),
|
|
),
|
|
sa.Column(
|
|
"artifact_set_id",
|
|
sa.Uuid(),
|
|
sa.ForeignKey("artifact_sets.id", ondelete="RESTRICT"),
|
|
nullable=False,
|
|
),
|
|
sa.Column(
|
|
"model_revision_id",
|
|
sa.Uuid(),
|
|
sa.ForeignKey("model_revisions.id", ondelete="RESTRICT"),
|
|
nullable=False,
|
|
),
|
|
sa.Column("recovery_class", sa.String(32), nullable=False),
|
|
sa.Column("state", sa.String(32), nullable=False, server_default="PLANNED"),
|
|
sa.Column("upstream_repository", sa.Text()),
|
|
sa.Column("upstream_commit_sha", sa.String(64)),
|
|
sa.Column(
|
|
"target_storage_root_id",
|
|
sa.Uuid(),
|
|
sa.ForeignKey("storage_roots.id", ondelete="RESTRICT"),
|
|
nullable=False,
|
|
),
|
|
sa.Column("expected_files", sa.JSON(), nullable=False),
|
|
sa.Column("verified_files", sa.JSON(), nullable=False),
|
|
sa.Column("bytes_total", sa.BigInteger(), nullable=False, server_default="0"),
|
|
sa.Column("bytes_recovered", sa.BigInteger(), nullable=False, server_default="0"),
|
|
sa.Column(
|
|
"download_plan_id", sa.Uuid(), sa.ForeignKey("download_plans.id", ondelete="SET NULL")
|
|
),
|
|
sa.Column(
|
|
"artifact_job_id", sa.Uuid(), sa.ForeignKey("artifact_jobs.id", ondelete="SET NULL")
|
|
),
|
|
sa.Column("lineage", sa.JSON(), nullable=False),
|
|
sa.Column("duration_seconds", sa.Float()),
|
|
sa.Column("failure_code", sa.String(64)),
|
|
sa.Column("failure_reason", sa.Text()),
|
|
sa.Column("reason", sa.Text(), nullable=False),
|
|
sa.Column("created_by", sa.String(255), nullable=False),
|
|
sa.Column("started_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
|
sa.Column("completed_at", sa.DateTime(timezone=True)),
|
|
)
|
|
for column in (
|
|
"restore_operation_id",
|
|
"artifact_set_id",
|
|
"model_revision_id",
|
|
"recovery_class",
|
|
"state",
|
|
"target_storage_root_id",
|
|
"download_plan_id",
|
|
"artifact_job_id",
|
|
):
|
|
op.create_index(
|
|
f"ix_artifact_recovery_operations_{column}", "artifact_recovery_operations", [column]
|
|
)
|
|
|
|
|
|
def downgrade() -> None:
|
|
op.drop_table("artifact_recovery_operations")
|
|
op.drop_table("restore_operation_events")
|
|
op.drop_table("restore_operations")
|
|
op.drop_table("restore_plans")
|
|
op.drop_table("backup_manifest_entries")
|
|
op.drop_table("backup_sets")
|
|
op.drop_table("recovery_asset_records")
|
|
op.drop_table("recovery_policy_revisions")
|