Initial public ModelForge release
This commit is contained in:
Jens
@@ -0,0 +1,215 @@
|
||||
"""Fail-closed checks for the production delivery workflow.
|
||||
|
||||
This test intentionally uses only the Python standard library so it can run before
|
||||
project dependencies are installed. The small helpers parse indentation-delimited
|
||||
YAML blocks that are relevant to the policy; they are not a general YAML parser.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import subprocess
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[2]
|
||||
DEPLOY_WORKFLOW = REPOSITORY_ROOT / ".gitea" / "workflows" / "unraid-deploy.yml"
|
||||
VALIDATION_WORKFLOW = REPOSITORY_ROOT / ".gitea" / "workflows" / "managed-validation.yml"
|
||||
|
||||
|
||||
def _indented_block(text: str, header: str) -> str:
|
||||
"""Return a YAML block beginning at an exact header line."""
|
||||
|
||||
lines = text.splitlines()
|
||||
try:
|
||||
start = lines.index(header)
|
||||
except ValueError as exc:
|
||||
raise AssertionError(f"Missing YAML header: {header!r}") from exc
|
||||
indent = len(header) - len(header.lstrip())
|
||||
end = len(lines)
|
||||
for index in range(start + 1, len(lines)):
|
||||
line = lines[index]
|
||||
if not line.strip() or line.lstrip().startswith("#"):
|
||||
continue
|
||||
current_indent = len(line) - len(line.lstrip())
|
||||
if current_indent <= indent:
|
||||
end = index
|
||||
break
|
||||
return "\n".join(lines[start:end])
|
||||
|
||||
|
||||
def _named_step(text: str, name: str) -> str:
|
||||
return _indented_block(text, f" - name: {name}")
|
||||
|
||||
|
||||
def _run_script(step: str) -> str:
|
||||
"""Extract one inline or literal run body without normalizing shell content."""
|
||||
|
||||
lines = step.splitlines()
|
||||
if " run: |" not in lines:
|
||||
inline = next((line for line in lines if line.startswith(" run: ")), None)
|
||||
if inline is None:
|
||||
raise AssertionError("Named step does not contain a run command")
|
||||
return inline.removeprefix(" run: ") + "\n"
|
||||
start = lines.index(" run: |") + 1
|
||||
script_lines = lines[start:]
|
||||
if any(line and not line.startswith(" ") for line in script_lines):
|
||||
raise AssertionError("Unexpected indentation in literal run block")
|
||||
return "\n".join(line[10:] if line else "" for line in script_lines) + "\n"
|
||||
|
||||
|
||||
class ProductionDeployPolicyTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
cls.deploy = DEPLOY_WORKFLOW.read_text(encoding="utf-8")
|
||||
cls.validation = VALIDATION_WORKFLOW.read_text(encoding="utf-8")
|
||||
|
||||
def test_deployment_has_no_automatic_trigger(self) -> None:
|
||||
trigger = _indented_block(self.deploy, "on:")
|
||||
trigger_keys = re.findall(r"^ ([a-z_]+):", trigger, flags=re.MULTILINE)
|
||||
self.assertEqual(trigger_keys, ["workflow_dispatch"])
|
||||
self.assertNotRegex(trigger, r"(?m)^\s+push:")
|
||||
|
||||
def test_dispatch_inputs_are_required_and_fail_safe(self) -> None:
|
||||
inputs = _indented_block(self.deploy, " inputs:")
|
||||
input_names = re.findall(r"^ ([a-z_]+):", inputs, flags=re.MULTILINE)
|
||||
self.assertEqual(input_names, ["release_tag", "release_commit", "action"])
|
||||
|
||||
for input_name in ("release_tag", "release_commit"):
|
||||
block = _indented_block(self.deploy, f" {input_name}:")
|
||||
self.assertRegex(block, r"(?m)^ required: true$")
|
||||
self.assertRegex(block, r"(?m)^ type: string$")
|
||||
|
||||
action = _indented_block(self.deploy, " action:")
|
||||
self.assertRegex(action, r"(?m)^ required: true$")
|
||||
self.assertRegex(action, r"(?m)^ default: VERIFY_ONLY$")
|
||||
self.assertRegex(action, r"(?m)^ type: choice$")
|
||||
options = re.findall(r"^ - (\S+)$", action, flags=re.MULTILINE)
|
||||
self.assertEqual(options, ["VERIFY_ONLY", "DEPLOY_STABLE_TO_PRODUCTION"])
|
||||
|
||||
def test_inputs_are_validated_before_checkout(self) -> None:
|
||||
validation = _named_step(self.deploy, "Validate immutable dispatch inputs")
|
||||
checkout = _named_step(self.deploy, "Check out the exact release commit with full history")
|
||||
self.assertLess(self.deploy.index(validation), self.deploy.index(checkout))
|
||||
self.assertIn('"refs/heads/master"', validation)
|
||||
self.assertIn(
|
||||
r"^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$",
|
||||
validation,
|
||||
)
|
||||
self.assertIn(r"^[0-9a-f]{40}$", validation)
|
||||
self.assertIn("VERIFY_ONLY|DEPLOY_STABLE_TO_PRODUCTION", validation)
|
||||
|
||||
tag_pattern = re.search(r'RELEASE_TAG\}" =~ (\^.*\$) \]\]', validation)
|
||||
commit_pattern = re.search(r'RELEASE_COMMIT\}" =~ (\^.*\$) \]\]', validation)
|
||||
self.assertIsNotNone(tag_pattern)
|
||||
self.assertIsNotNone(commit_pattern)
|
||||
assert tag_pattern is not None and commit_pattern is not None
|
||||
for valid_tag in ("v0.0.0", "v1.2.1", "v10.20.30"):
|
||||
self.assertRegex(valid_tag, tag_pattern.group(1))
|
||||
for invalid_tag in ("1.2.1", "v01.2.1", "v1.2.1-rc.1", "v1.2.1+build", "master"):
|
||||
self.assertNotRegex(invalid_tag, tag_pattern.group(1))
|
||||
self.assertRegex("0" * 40, commit_pattern.group(1))
|
||||
for invalid_commit in ("0" * 39, "A" * 40, "g" * 40, "master"):
|
||||
self.assertNotRegex(invalid_commit, commit_pattern.group(1))
|
||||
|
||||
def test_checkout_is_pinned_to_the_validated_commit_with_full_history(self) -> None:
|
||||
checkout = _named_step(self.deploy, "Check out the exact release commit with full history")
|
||||
self.assertRegex(checkout, r"actions/checkout@[0-9a-f]{40}")
|
||||
self.assertIn("ref: ${{ inputs.release_commit }}", checkout)
|
||||
self.assertIn("fetch-depth: 0", checkout)
|
||||
self.assertIn("persist-credentials: true", checkout)
|
||||
|
||||
def test_release_provenance_is_verified_and_exported(self) -> None:
|
||||
provenance = _named_step(self.deploy, "Verify stable release provenance")
|
||||
required_fragments = (
|
||||
"git fetch --force --no-recurse-submodules origin",
|
||||
'git cat-file -t "${tag_ref}"',
|
||||
'git rev-parse "${tag_ref}^{tag}"',
|
||||
'git rev-parse "${tag_ref}^{commit}"',
|
||||
"git rev-parse HEAD",
|
||||
"< VERSION",
|
||||
"git merge-base --is-ancestor",
|
||||
"refs/remotes/origin/master",
|
||||
"DEPLOY_COMMIT=%s",
|
||||
'>> "${GITHUB_ENV}"',
|
||||
)
|
||||
for fragment in required_fragments:
|
||||
with self.subTest(fragment=fragment):
|
||||
self.assertIn(fragment, provenance)
|
||||
self.assertNotIn("GITHUB_SHA", self.deploy)
|
||||
|
||||
def test_all_production_shell_blocks_parse(self) -> None:
|
||||
names = (
|
||||
"Validate immutable dispatch inputs",
|
||||
"Verify stable release provenance",
|
||||
"Verification-only result",
|
||||
"Deploy verified stable release to production",
|
||||
)
|
||||
for name in names:
|
||||
script = _run_script(_named_step(self.deploy, name))
|
||||
result = subprocess.run(
|
||||
["bash", "-n"],
|
||||
input=script.encode("utf-8"),
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
with self.subTest(step=name):
|
||||
self.assertEqual(result.returncode, 0, result.stderr.decode("utf-8", "replace"))
|
||||
|
||||
def test_deployment_requires_the_explicit_action_and_verified_commit(self) -> None:
|
||||
deploy = _named_step(self.deploy, "Deploy verified stable release to production")
|
||||
self.assertIn("inputs.action == 'DEPLOY_STABLE_TO_PRODUCTION'", deploy)
|
||||
self.assertIn('test -n "${DEPLOY_COMMIT:-}"', deploy)
|
||||
self.assertIn('"${GITHUB_REPOSITORY}" "${DEPLOY_COMMIT}"', deploy)
|
||||
self.assertEqual(self.deploy.count("/opt/gitea-deploy/deploy.py deploy"), 1)
|
||||
|
||||
verify_only = _named_step(self.deploy, "Verification-only result")
|
||||
self.assertIn("inputs.action == 'VERIFY_ONLY'", verify_only)
|
||||
|
||||
def test_managed_validation_runs_policy_before_profile_work(self) -> None:
|
||||
policy_step = _named_step(self.validation, "Production delivery policy")
|
||||
self.assertIn("python3 -m unittest discover -s .gitea/tests", policy_step)
|
||||
self.assertNotRegex(policy_step, r"(?m)^ if:")
|
||||
self.assertLess(
|
||||
self.validation.index(policy_step),
|
||||
self.validation.index(" - name: Validate the requested profile"),
|
||||
)
|
||||
|
||||
def test_security_profile_runs_fail_closed_scanners(self) -> None:
|
||||
step = _named_step(self.validation, "Security — secrets and vulnerable dependencies")
|
||||
self.assertIn("env.PROFILE == 'security'", step)
|
||||
self.assertIn("env.PROFILE == 'full'", step)
|
||||
script = _run_script(step)
|
||||
self.assertIn("gitleaks git .", script)
|
||||
self.assertIn("pip_audit --strict --skip-editable", script)
|
||||
self.assertIn("npm audit --audit-level=high --omit=dev", script)
|
||||
self.assertNotRegex(script, r"(?m)(?:\|\|\s*true|continue-on-error)")
|
||||
|
||||
toolchain = _named_step(self.validation, "Toolchain")
|
||||
toolchain_script = _run_script(toolchain)
|
||||
self.assertIn("command -v gitleaks", toolchain_script)
|
||||
self.assertIn("exit 1", toolchain_script)
|
||||
|
||||
def test_compose_validation_renders_the_production_projection(self) -> None:
|
||||
step = _named_step(self.validation, "Compose projections")
|
||||
script = _run_script(step)
|
||||
self.assertIn(
|
||||
"for overlay in backup dr gpu node-agent node-recovery production runtime-worker",
|
||||
script,
|
||||
)
|
||||
for variable in (
|
||||
"MODELFORGE_POSTGRES_ADMIN_PASSWORD",
|
||||
"MODELFORGE_MIGRATION_DATABASE_URL",
|
||||
"MODELFORGE_RUNTIME_DATABASE_URL",
|
||||
"MODELFORGE_OPERATOR_API_KEY",
|
||||
"MODELFORGE_BACKUP_ENCRYPTION_KEY",
|
||||
"MODELFORGE_CORS_ORIGINS",
|
||||
"VITE_API_BASE_URL",
|
||||
):
|
||||
with self.subTest(variable=variable):
|
||||
self.assertIn(f"export {variable}=", script)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,199 @@
|
||||
"""Fail-closed integration tests for the parentless public source boundary."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[2]
|
||||
EXPORTER = REPOSITORY_ROOT / "scripts" / "export-public-source.mjs"
|
||||
VALIDATOR = REPOSITORY_ROOT / "scripts" / "validate-public-source.mjs"
|
||||
|
||||
|
||||
class PublicSourceExportTests(unittest.TestCase):
|
||||
def _repository(
|
||||
self,
|
||||
root: Path,
|
||||
*,
|
||||
license_present: bool = True,
|
||||
managed_validation_workflow: str | None = None,
|
||||
) -> Path:
|
||||
repository = root / "private-source"
|
||||
repository.mkdir()
|
||||
files: dict[str, str | bytes] = {
|
||||
"README.md": "# Product\n",
|
||||
"SECURITY.md": "# Security\n",
|
||||
"CONTRIBUTING.md": "# Contributing\n",
|
||||
"docker-compose.yml": "services: {}\n",
|
||||
"VERSION": "1.2.1\n",
|
||||
"backend/pyproject.toml": "[project]\nname = 'fixture-api'\n",
|
||||
"frontend/package.json": '{"name":"fixture-web"}\n',
|
||||
"node-agent/pyproject.toml": "[project]\nname = 'fixture-agent'\n",
|
||||
"runtime-worker/pyproject.toml": "[project]\nname = 'fixture-worker'\n",
|
||||
}
|
||||
if license_present:
|
||||
files["LICENSE"] = (REPOSITORY_ROOT / "LICENSE").read_bytes()
|
||||
if managed_validation_workflow is not None:
|
||||
files[".gitea/workflows/managed-validation.yml"] = managed_validation_workflow
|
||||
for relative_path, contents in files.items():
|
||||
destination = repository / relative_path
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
if isinstance(contents, bytes):
|
||||
destination.write_bytes(contents)
|
||||
else:
|
||||
destination.write_text(contents, encoding="utf-8")
|
||||
(repository / "public-source.allowlist").write_text(
|
||||
"\n".join(sorted(files)) + "\n", encoding="utf-8"
|
||||
)
|
||||
subprocess.run(["git", "init", "-q"], cwd=repository, check=True)
|
||||
subprocess.run(["git", "add", "."], cwd=repository, check=True)
|
||||
subprocess.run(
|
||||
[
|
||||
"git",
|
||||
"-c",
|
||||
"user.name=ModelForge test",
|
||||
"-c",
|
||||
"user.email=test@example.invalid",
|
||||
"commit",
|
||||
"-qm",
|
||||
"fixture",
|
||||
],
|
||||
cwd=repository,
|
||||
check=True,
|
||||
)
|
||||
return repository
|
||||
|
||||
def _export(self, repository: Path, output: Path) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(
|
||||
["node", str(EXPORTER), "--repository", str(repository), "--output", str(output)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
check=False,
|
||||
)
|
||||
|
||||
def _validate(self, output: Path) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(
|
||||
["node", str(VALIDATOR)],
|
||||
cwd=output,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
check=False,
|
||||
)
|
||||
|
||||
def test_export_requires_an_explicit_tracked_license(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
repository = self._repository(root, license_present=False)
|
||||
result = self._export(repository, root / "public")
|
||||
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("canonical LICENSE is missing", result.stderr)
|
||||
|
||||
def test_manifest_detects_content_tampering_and_unexpected_files(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
output = root / "public"
|
||||
result = self._export(self._repository(root), output)
|
||||
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
|
||||
validation = self._validate(output)
|
||||
self.assertEqual(validation.returncode, 0, validation.stdout + validation.stderr)
|
||||
|
||||
original_readme = (output / "README.md").read_bytes()
|
||||
(output / "README.md").write_text("tampered\n", encoding="utf-8")
|
||||
tampered = self._validate(output)
|
||||
self.assertNotEqual(tampered.returncode, 0)
|
||||
self.assertIn("Manifest", tampered.stderr)
|
||||
|
||||
(output / "README.md").write_bytes(original_readme)
|
||||
(output / "not-reviewed.txt").write_text("extra\n", encoding="utf-8")
|
||||
unexpected = self._validate(output)
|
||||
self.assertNotEqual(unexpected.returncode, 0)
|
||||
self.assertIn("Unexpected public source files", unexpected.stderr)
|
||||
|
||||
def test_dirty_source_tree_cannot_claim_the_head_revision(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
repository = self._repository(root)
|
||||
(repository / "README.md").write_text("changed after commit\n", encoding="utf-8")
|
||||
result = self._export(repository, root / "public")
|
||||
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("commit the exact source tree", result.stderr)
|
||||
|
||||
def test_sanitized_destination_collisions_fail_before_writing(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
repository = self._repository(root)
|
||||
private_name = "Tow" + "er.txt"
|
||||
public_name = "GPU " + "Node.txt"
|
||||
(repository / private_name).write_text("one\n", encoding="utf-8")
|
||||
(repository / public_name).write_text("two\n", encoding="utf-8")
|
||||
with (repository / "public-source.allowlist").open("a", encoding="utf-8") as allowlist:
|
||||
allowlist.write(f"{private_name}\n{public_name}\n")
|
||||
subprocess.run(["git", "add", "."], cwd=repository, check=True)
|
||||
subprocess.run(
|
||||
[
|
||||
"git",
|
||||
"-c",
|
||||
"user.name=ModelForge test",
|
||||
"-c",
|
||||
"user.email=test@example.invalid",
|
||||
"commit",
|
||||
"-qm",
|
||||
"collision fixture",
|
||||
],
|
||||
cwd=repository,
|
||||
check=True,
|
||||
)
|
||||
result = self._export(repository, root / "public")
|
||||
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("finding", result.stderr)
|
||||
|
||||
def test_public_managed_validation_requires_explicit_owner_dispatch(self) -> None:
|
||||
workflow = "name: Managed validation\n\non:\n pull_request:\n workflow_dispatch:\n"
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
repository = self._repository(
|
||||
root,
|
||||
managed_validation_workflow=workflow,
|
||||
)
|
||||
output = root / "public"
|
||||
result = self._export(repository, output)
|
||||
|
||||
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
|
||||
canonical = (
|
||||
repository / ".gitea/workflows/managed-validation.yml"
|
||||
).read_text(encoding="utf-8")
|
||||
exported = (
|
||||
output / ".gitea/workflows/managed-validation.yml"
|
||||
).read_text(encoding="utf-8")
|
||||
|
||||
self.assertIn(" pull_request:\n", canonical)
|
||||
self.assertNotIn(" pull_request:\n", exported)
|
||||
self.assertIn(" workflow_dispatch:\n", exported)
|
||||
self.assertIn("fork PRs never reach private runners", exported)
|
||||
|
||||
def test_changed_managed_validation_trigger_fails_closed(self) -> None:
|
||||
workflow = "name: Managed validation\n\non:\n workflow_dispatch:\n"
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
repository = self._repository(
|
||||
root,
|
||||
managed_validation_workflow=workflow,
|
||||
)
|
||||
result = self._export(repository, root / "public")
|
||||
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("has no expected pull_request trigger", result.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,280 @@
|
||||
"""Structural safety checks for public-candidate server acceptance."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import importlib.util
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
WORKFLOW = ROOT / ".gitea" / "workflows" / "public-candidate-acceptance.yml"
|
||||
MANAGED_WORKFLOW = ROOT / ".gitea" / "workflows" / "managed-validation.yml"
|
||||
ACCEPTANCE = ROOT / "scripts" / "rc_server_acceptance.py"
|
||||
TRIVY_VALIDATOR = ROOT / "scripts" / "validate_trivy_report.py"
|
||||
RUNTIME_DOCKERFILE = ROOT / "runtime-worker" / "Dockerfile"
|
||||
RUNTIME_PYPROJECT = ROOT / "runtime-worker" / "pyproject.toml"
|
||||
|
||||
|
||||
class RcAcceptancePolicyTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
cls.workflow = WORKFLOW.read_text(encoding="utf-8")
|
||||
cls.managed = MANAGED_WORKFLOW.read_text(encoding="utf-8")
|
||||
cls.acceptance = ACCEPTANCE.read_text(encoding="utf-8")
|
||||
cls.trivy = TRIVY_VALIDATOR.read_text(encoding="utf-8")
|
||||
cls.runtime_dockerfile = RUNTIME_DOCKERFILE.read_text(encoding="utf-8")
|
||||
cls.runtime_pyproject = RUNTIME_PYPROJECT.read_text(encoding="utf-8")
|
||||
|
||||
def test_workflow_is_manual_and_never_calls_production_deployment(self) -> None:
|
||||
self.assertIn("workflow_dispatch:", self.workflow)
|
||||
self.assertNotIn("\n push:", self.workflow)
|
||||
self.assertNotIn("pull_request:", self.workflow)
|
||||
self.assertNotIn("deploy.py", self.workflow)
|
||||
self.assertNotIn("DEPLOY_STABLE_TO_PRODUCTION", self.workflow)
|
||||
|
||||
def test_exact_source_and_docker_are_required(self) -> None:
|
||||
self.assertIn(r"^[0-9a-f]{40}$", self.workflow)
|
||||
self.assertIn("ref: ${{ inputs.source_commit }}", self.workflow)
|
||||
self.assertIn("docker version", self.workflow)
|
||||
self.assertNotIn("SKIPPED: no docker", self.workflow)
|
||||
|
||||
def test_security_tools_are_downloaded_with_exact_checksums(self) -> None:
|
||||
self.assertIn("GITLEAKS_VERSION: 8.30.1", self.workflow)
|
||||
self.assertIn(
|
||||
"GITLEAKS_SHA256: 551f6fc83ea457d62a0d98237cbad105af8d557003051f41f3e7ca7b3f2470eb",
|
||||
self.workflow,
|
||||
)
|
||||
self.assertIn('printf \'GITLEAKS=%s\\n\'', self.workflow)
|
||||
self.assertIn('\"${GITLEAKS}\" dir \"${public_source}\"', self.workflow)
|
||||
self.assertIn("sha256sum --check --strict", self.workflow)
|
||||
self.assertNotIn("command -v gitleaks", self.workflow)
|
||||
|
||||
def test_unmerged_branch_can_use_the_existing_managed_dispatch_entry(self) -> None:
|
||||
self.assertIn("workflow_call:", self.workflow)
|
||||
self.assertIn("uses: ./.gitea/workflows/public-candidate-acceptance.yml", self.managed)
|
||||
self.assertIn("inputs.profile == 'build'", self.managed)
|
||||
self.assertIn("source_commit: ${{ gitea.sha }}", self.managed)
|
||||
|
||||
def test_all_four_images_are_built_and_scanned(self) -> None:
|
||||
for image in (
|
||||
"modelforge-api",
|
||||
"modelforge-web",
|
||||
"modelforge-node-agent",
|
||||
"modelforge-runtime-worker",
|
||||
):
|
||||
self.assertIn(image, self.acceptance)
|
||||
self.assertIn('"Metadata"', self.trivy)
|
||||
self.assertIn('metadata.get("ImageID") != expected_image_id', self.trivy)
|
||||
self.assertIn("reject_duplicate_keys", self.trivy)
|
||||
self.assertIn("fixable_high_critical + unreviewed_unfixed_high_critical", self.trivy)
|
||||
self.assertIn("public-candidate-unfixed-vulnerabilities.json", self.acceptance)
|
||||
|
||||
def test_runtime_image_applies_security_updates_and_exact_inference_pins(self) -> None:
|
||||
self.assertIn("apt-get upgrade --yes", self.runtime_dockerfile)
|
||||
self.assertIn("apt-get purge --yes linux-libc-dev", self.runtime_dockerfile)
|
||||
self.assertNotIn("apt-get autoremove", self.runtime_dockerfile)
|
||||
self.assertIn("pip check", self.runtime_dockerfile)
|
||||
for dependency in (
|
||||
"Pillow==12.3.0",
|
||||
"protobuf==5.29.6",
|
||||
"sentencepiece==0.2.2",
|
||||
"sentence-transformers==6.0.1",
|
||||
"transformers==5.16.1",
|
||||
"urllib3==2.7.0",
|
||||
):
|
||||
self.assertIn(dependency, self.runtime_pyproject)
|
||||
|
||||
def test_compose_project_is_isolated_and_always_removed(self) -> None:
|
||||
self.assertIn('project = f"modelforge-rc-', self.acceptance)
|
||||
self.assertIn('"MODELFORGE_API_BIND": "127.0.0.1"', self.acceptance)
|
||||
self.assertIn('"MODELFORGE_API_PUBLISHED_PORT": "0"', self.acceptance)
|
||||
self.assertIn('"production_changed": False', self.acceptance)
|
||||
self.assertIn('"compute_identity_created": False', self.acceptance)
|
||||
self.assertIn('"down", "--volumes", "--remove-orphans"', self.acceptance)
|
||||
self.assertIn("atexit.register(cleanup_candidate_images", self.acceptance)
|
||||
self.assertIn('"docker", "image", "rm", "--force", tag', self.acceptance)
|
||||
self.assertNotIn("docker-compose.runtime-worker.yml", self.acceptance)
|
||||
self.assertNotIn("docker-compose.node-agent.yml", self.acceptance)
|
||||
|
||||
def test_clean_install_bootstraps_roles_across_a_remote_docker_daemon(self) -> None:
|
||||
self.assertIn("def provision_database_roles(", self.acceptance)
|
||||
self.assertIn('run("docker", "cp", str(bootstrap)', self.acceptance)
|
||||
self.assertIn('"psql",', self.acceptance)
|
||||
self.assertIn('"--file",', self.acceptance)
|
||||
self.assertIn(
|
||||
'"--wait", "postgres", "redis", env=env', self.acceptance
|
||||
)
|
||||
self.assertLess(
|
||||
self.acceptance.index("provision_database_roles(compose, env, database)"),
|
||||
self.acceptance.index(
|
||||
'"--wait", "api", "web", env=env'
|
||||
),
|
||||
)
|
||||
|
||||
def test_clean_install_seeds_config_across_a_remote_docker_daemon(self) -> None:
|
||||
self.assertIn("def provision_config_volume(project: str)", self.acceptance)
|
||||
self.assertIn('f"{project}_acceptance-config"', self.acceptance)
|
||||
self.assertIn('f"com.docker.compose.project={project}"', self.acceptance)
|
||||
self.assertIn('run("docker", "cp", f"{ROOT / \'config\'}/."', self.acceptance)
|
||||
self.assertIn(
|
||||
'run("docker", "container", "rm", "--force", seed, check=False)',
|
||||
self.acceptance,
|
||||
)
|
||||
self.assertIn('" read_only: true\\n"', self.acceptance)
|
||||
self.assertLess(
|
||||
self.acceptance.index("provision_config_volume(project)"),
|
||||
self.acceptance.index('"--wait", "api", "web", env=env'),
|
||||
)
|
||||
|
||||
def test_http_contract_runs_inside_the_isolated_compose_network(self) -> None:
|
||||
self.assertIn("def container_http_status(", self.acceptance)
|
||||
self.assertIn("def wait_for_container_status(", self.acceptance)
|
||||
self.assertIn('headers["X-ModelForge-Admin-Token"]', self.acceptance)
|
||||
self.assertIn('os.environ["MODELFORGE_OPERATOR_API_KEY"]', self.acceptance)
|
||||
self.assertIn('wait_for_container_status(api_container, "http://web:3000/", 200)', self.acceptance)
|
||||
self.assertNotIn("urllib.request.urlopen(request, timeout=timeout)", self.acceptance)
|
||||
|
||||
def test_workflow_has_an_always_run_exact_project_cleanup_fallback(self) -> None:
|
||||
self.assertIn("- name: Always remove acceptance Docker resources", self.workflow)
|
||||
cleanup = self.workflow.split(
|
||||
" - name: Always remove acceptance Docker resources", 1
|
||||
)[1].split(" - name: Upload acceptance evidence", 1)[0]
|
||||
self.assertIn("if: always()", cleanup)
|
||||
self.assertIn('label=com.docker.compose.project=${project}', cleanup)
|
||||
self.assertIn("docker container rm --force", cleanup)
|
||||
self.assertIn("docker volume rm --force", cleanup)
|
||||
self.assertIn("docker network rm", cleanup)
|
||||
self.assertIn("docker image rm --force", cleanup)
|
||||
self.assertNotIn("docker system prune", cleanup)
|
||||
|
||||
def test_python_entrypoints_compile(self) -> None:
|
||||
for path in (ACCEPTANCE, TRIVY_VALIDATOR):
|
||||
result = subprocess.run(
|
||||
["python", "-m", "py_compile", str(path)],
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
with self.subTest(path=path.name):
|
||||
self.assertEqual(result.returncode, 0, result.stderr.decode("utf-8", "replace"))
|
||||
|
||||
def test_trivy_validator_binds_image_and_rejects_blockers(self) -> None:
|
||||
spec = importlib.util.spec_from_file_location("trivy_report", TRIVY_VALIDATOR)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
image_id = "sha256:" + "a" * 64
|
||||
report = {
|
||||
"ArtifactName": image_id,
|
||||
"ArtifactType": "container_image",
|
||||
"Metadata": {"ImageID": image_id},
|
||||
"Results": [
|
||||
{
|
||||
"Target": "debian",
|
||||
"Class": "os-pkgs",
|
||||
"Type": "debian",
|
||||
"Vulnerabilities": [
|
||||
{
|
||||
"VulnerabilityID": "CVE-2099-9999",
|
||||
"PkgName": "blocked-package",
|
||||
"InstalledVersion": "1.0.0",
|
||||
"FixedVersion": "1.0.1",
|
||||
"Severity": "HIGH",
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
path = Path(directory) / "report.json"
|
||||
path.write_text(json.dumps(report), encoding="utf-8")
|
||||
result = module.validate(path, image_id)
|
||||
self.assertEqual(result["release_blockers"], 1)
|
||||
with self.assertRaisesRegex(ValueError, "does not match"):
|
||||
module.validate(path, "sha256:" + "b" * 64)
|
||||
|
||||
def test_trivy_validator_rejects_duplicate_keys_and_empty_coverage(self) -> None:
|
||||
spec = importlib.util.spec_from_file_location("trivy_report_negative", TRIVY_VALIDATOR)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
image_id = "sha256:" + "a" * 64
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
path = Path(directory) / "report.json"
|
||||
path.write_text('{"ArtifactType":"container_image","ArtifactType":"container_image"}', encoding="utf-8")
|
||||
with self.assertRaisesRegex(ValueError, "duplicate JSON key"):
|
||||
module.validate(path, image_id)
|
||||
path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"ArtifactType": "container_image",
|
||||
"Metadata": {"ImageID": image_id},
|
||||
"Results": [],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
with self.assertRaisesRegex(ValueError, "no package result coverage"):
|
||||
module.validate(path, image_id)
|
||||
|
||||
def test_trivy_validator_allows_only_exact_reviewed_upstream_unfixed_findings(self) -> None:
|
||||
spec = importlib.util.spec_from_file_location("trivy_report_reviewed", TRIVY_VALIDATOR)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
image_id = "sha256:" + "a" * 64
|
||||
finding = {
|
||||
"VulnerabilityID": "CVE-2099-0001",
|
||||
"PkgName": "example-package",
|
||||
"InstalledVersion": "1.0.0",
|
||||
"FixedVersion": "",
|
||||
"Severity": "HIGH",
|
||||
}
|
||||
report = {
|
||||
"ArtifactType": "container_image",
|
||||
"Metadata": {"ImageID": image_id},
|
||||
"Results": [
|
||||
{
|
||||
"Target": "debian",
|
||||
"Class": "os-pkgs",
|
||||
"Vulnerabilities": [finding],
|
||||
}
|
||||
],
|
||||
}
|
||||
baseline = {
|
||||
"schema_version": 1,
|
||||
"vulnerabilities": [
|
||||
{
|
||||
"vulnerability_id": "CVE-2099-0001",
|
||||
"package": "example-package",
|
||||
"installed_version": "1.0.0",
|
||||
"severity": "HIGH",
|
||||
}
|
||||
],
|
||||
}
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
report_path = Path(directory) / "report.json"
|
||||
baseline_path = Path(directory) / "baseline.json"
|
||||
report_path.write_text(json.dumps(report), encoding="utf-8")
|
||||
baseline_path.write_text(json.dumps(baseline), encoding="utf-8")
|
||||
summary = module.validate(report_path, image_id, baseline_path)
|
||||
self.assertEqual(summary["reviewed_unfixed_high_critical"], 1)
|
||||
self.assertEqual(summary["release_blockers"], 0)
|
||||
|
||||
finding["InstalledVersion"] = "1.0.1"
|
||||
report_path.write_text(json.dumps(report), encoding="utf-8")
|
||||
self.assertEqual(module.validate(report_path, image_id, baseline_path)["release_blockers"], 1)
|
||||
|
||||
finding["InstalledVersion"] = "1.0.0"
|
||||
finding["FixedVersion"] = "1.0.2"
|
||||
report_path.write_text(json.dumps(report), encoding="utf-8")
|
||||
summary = module.validate(report_path, image_id, baseline_path)
|
||||
self.assertEqual(summary["fixable_high_critical"], 1)
|
||||
self.assertEqual(summary["release_blockers"], 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user