Initial public ModelForge release
This commit is contained in:
Jens
@@ -0,0 +1,218 @@
|
||||
param(
|
||||
[int]$Cycles = 4,
|
||||
[int]$IdleSeconds = 45
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
if ($Cycles -lt 2 -or $Cycles -gt 12) {
|
||||
throw "Cycles must be between 2 and 12"
|
||||
}
|
||||
if ($IdleSeconds -lt 10 -or $IdleSeconds -gt 300) {
|
||||
throw "IdleSeconds must be between 10 and 300"
|
||||
}
|
||||
|
||||
$baseUrl = "http://127.0.0.1:8000"
|
||||
$operatorLine = Get-Content .env | Where-Object {
|
||||
$_ -match '^MODELFORGE_OPERATOR_API_KEY='
|
||||
} | Select-Object -First 1
|
||||
if (-not $operatorLine) {
|
||||
throw "MODELFORGE_OPERATOR_API_KEY is unavailable"
|
||||
}
|
||||
$operatorToken = $operatorLine.Substring($operatorLine.IndexOf('=') + 1).Trim().Trim('"').Trim("'")
|
||||
$operatorHeaders = @{ "X-ModelForge-Admin-Token" = $operatorToken }
|
||||
$runId = [guid]::NewGuid().ToString("N").Substring(0, 12)
|
||||
$clientId = $null
|
||||
$results = [System.Collections.Generic.List[object]]::new()
|
||||
$startedAt = [datetime]::UtcNow
|
||||
|
||||
function New-SilenceWavBase64 {
|
||||
$sampleRate = 16000
|
||||
$sampleCount = $sampleRate
|
||||
$dataLength = $sampleCount * 2
|
||||
$stream = [System.IO.MemoryStream]::new()
|
||||
$writer = [System.IO.BinaryWriter]::new($stream)
|
||||
try {
|
||||
$writer.Write([System.Text.Encoding]::ASCII.GetBytes("RIFF"))
|
||||
$writer.Write([int](36 + $dataLength))
|
||||
$writer.Write([System.Text.Encoding]::ASCII.GetBytes("WAVE"))
|
||||
$writer.Write([System.Text.Encoding]::ASCII.GetBytes("fmt "))
|
||||
$writer.Write([int]16)
|
||||
$writer.Write([int16]1)
|
||||
$writer.Write([int16]1)
|
||||
$writer.Write([int]$sampleRate)
|
||||
$writer.Write([int]($sampleRate * 2))
|
||||
$writer.Write([int16]2)
|
||||
$writer.Write([int16]16)
|
||||
$writer.Write([System.Text.Encoding]::ASCII.GetBytes("data"))
|
||||
$writer.Write([int]$dataLength)
|
||||
$writer.Write([byte[]]::new($dataLength))
|
||||
$writer.Flush()
|
||||
return [Convert]::ToBase64String($stream.ToArray())
|
||||
}
|
||||
finally {
|
||||
$writer.Dispose()
|
||||
$stream.Dispose()
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-SoakCapability {
|
||||
param(
|
||||
[string]$Capability,
|
||||
[string]$Path,
|
||||
[hashtable]$Body,
|
||||
[int]$Cycle,
|
||||
[hashtable]$Headers
|
||||
)
|
||||
$watch = [System.Diagnostics.Stopwatch]::StartNew()
|
||||
try {
|
||||
$response = Invoke-RestMethod -Method Post -Uri "$baseUrl$Path" -Headers $Headers `
|
||||
-ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 8 -Compress) `
|
||||
-TimeoutSec 180
|
||||
$watch.Stop()
|
||||
$results.Add([pscustomobject]@{
|
||||
cycle = $Cycle
|
||||
capability = $Capability
|
||||
success = $true
|
||||
request_id = [string]$response.request_id
|
||||
cold = [bool]$response.execution.cold
|
||||
residency = [string]$response.execution.residency
|
||||
node = [string]$response.execution.node
|
||||
queue_ms = [double]$response.execution.timings.queue_ms
|
||||
load_ms = [double]$response.execution.timings.load_ms
|
||||
inference_ms = [double]$response.execution.timings.inference_ms
|
||||
total_ms = [double]$response.execution.timings.total_ms
|
||||
wall_ms = [double]$watch.Elapsed.TotalMilliseconds
|
||||
error_class = $null
|
||||
error_code = $null
|
||||
})
|
||||
}
|
||||
catch {
|
||||
$watch.Stop()
|
||||
$errorCode = $null
|
||||
if ($_.ErrorDetails.Message) {
|
||||
try {
|
||||
$errorBody = $_.ErrorDetails.Message | ConvertFrom-Json
|
||||
$errorCode = [string]$errorBody.error.code
|
||||
}
|
||||
catch {
|
||||
$errorCode = "UNPARSEABLE_HTTP_ERROR"
|
||||
}
|
||||
}
|
||||
$results.Add([pscustomobject]@{
|
||||
cycle = $Cycle
|
||||
capability = $Capability
|
||||
success = $false
|
||||
request_id = $null
|
||||
cold = $null
|
||||
residency = $null
|
||||
node = $null
|
||||
queue_ms = $null
|
||||
load_ms = $null
|
||||
inference_ms = $null
|
||||
total_ms = $null
|
||||
wall_ms = [double]$watch.Elapsed.TotalMilliseconds
|
||||
error_class = $_.Exception.GetType().Name
|
||||
error_code = $errorCode
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
function Get-Percentile {
|
||||
param([double[]]$Values, [double]$Percentile)
|
||||
if (-not $Values -or $Values.Count -eq 0) {
|
||||
return $null
|
||||
}
|
||||
$ordered = @($Values | Sort-Object)
|
||||
$index = [math]::Ceiling($Percentile * $ordered.Count) - 1
|
||||
return [double]$ordered[[math]::Max(0, $index)]
|
||||
}
|
||||
|
||||
try {
|
||||
$clientBody = @{
|
||||
name = "m14-soak-$runId"
|
||||
allowed_capabilities = @(
|
||||
"rag.embedding@1",
|
||||
"vision.embedding@1",
|
||||
"speech.transcription@1"
|
||||
)
|
||||
requests_per_minute = 60
|
||||
max_concurrent_requests = 1
|
||||
workload_priority = "interactive"
|
||||
credential_expires_at = [datetime]::UtcNow.AddMinutes(30).ToString("o")
|
||||
}
|
||||
$client = Invoke-RestMethod -Method Post -Uri "$baseUrl/api/v1/admin/service-clients" `
|
||||
-Headers $operatorHeaders -ContentType "application/json" `
|
||||
-Body ($clientBody | ConvertTo-Json -Depth 6 -Compress)
|
||||
$clientId = [string]$client.id
|
||||
$serviceHeaders = @{ Authorization = "Bearer $($client.credential)" }
|
||||
$audio = New-SilenceWavBase64
|
||||
|
||||
for ($cycle = 1; $cycle -le $Cycles; $cycle++) {
|
||||
Invoke-RestMethod -Method Post -Uri "$baseUrl/api/v1/admin/operations/capacity/collect" `
|
||||
-Headers $operatorHeaders | Out-Null
|
||||
|
||||
Invoke-SoakCapability -Capability "rag.embedding@1" `
|
||||
-Path "/api/v1/capabilities/rag.embedding@1/invoke" `
|
||||
-Body @{ input = @("M14 bounded operational soak cycle $cycle") } `
|
||||
-Cycle $cycle -Headers $serviceHeaders
|
||||
Invoke-SoakCapability -Capability "vision.embedding@1 LAB" `
|
||||
-Path "/api/v1/capabilities/vision.embedding@1/invoke" `
|
||||
-Body @{ items = @(@{ text = "M14 visual control sample cycle $cycle" }) } `
|
||||
-Cycle $cycle -Headers $serviceHeaders
|
||||
Invoke-SoakCapability -Capability "speech.transcription@1 LAB" `
|
||||
-Path "/api/v1/capabilities/speech.transcription@1/invoke" `
|
||||
-Body @{ audio_base64 = $audio; media_type = "audio/wav"; language = "en" } `
|
||||
-Cycle $cycle -Headers $serviceHeaders
|
||||
|
||||
Invoke-RestMethod -Method Post -Uri "$baseUrl/api/v1/admin/operations/capacity/collect" `
|
||||
-Headers $operatorHeaders | Out-Null
|
||||
Invoke-RestMethod -Method Post -Uri "$baseUrl/api/v1/admin/operations/slo-evaluations/run" `
|
||||
-Headers $operatorHeaders | Out-Null
|
||||
Invoke-RestMethod -Method Post -Uri "$baseUrl/api/v1/admin/operations/alerts/evaluate" `
|
||||
-Headers $operatorHeaders | Out-Null
|
||||
|
||||
if ($cycle -lt $Cycles) {
|
||||
Start-Sleep -Seconds $IdleSeconds
|
||||
}
|
||||
}
|
||||
}
|
||||
finally {
|
||||
if ($clientId) {
|
||||
Invoke-RestMethod -Method Delete `
|
||||
-Uri "$baseUrl/api/v1/admin/service-clients/$clientId/credential" `
|
||||
-Headers $operatorHeaders | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
$finishedAt = [datetime]::UtcNow
|
||||
$summary = foreach ($group in ($results | Group-Object capability)) {
|
||||
$successful = @($group.Group | Where-Object success)
|
||||
$latencies = @($successful | ForEach-Object { [double]$_.total_ms })
|
||||
[pscustomobject]@{
|
||||
capability = $group.Name
|
||||
requests = $group.Count
|
||||
failures = @($group.Group | Where-Object { -not $_.success }).Count
|
||||
cold = @($successful | Where-Object cold).Count
|
||||
warm = @($successful | Where-Object { -not $_.cold }).Count
|
||||
p50_ms = Get-Percentile -Values $latencies -Percentile 0.50
|
||||
p95_ms = Get-Percentile -Values $latencies -Percentile 0.95
|
||||
max_queue_ms = if ($successful) {
|
||||
[double](($successful | Measure-Object queue_ms -Maximum).Maximum)
|
||||
} else { $null }
|
||||
max_load_ms = if ($successful) {
|
||||
[double](($successful | Measure-Object load_ms -Maximum).Maximum)
|
||||
} else { $null }
|
||||
}
|
||||
}
|
||||
|
||||
[pscustomobject]@{
|
||||
run_id = $runId
|
||||
started_at = $startedAt.ToString("o")
|
||||
finished_at = $finishedAt.ToString("o")
|
||||
duration_seconds = ($finishedAt - $startedAt).TotalSeconds
|
||||
cycles = $Cycles
|
||||
idle_seconds = $IdleSeconds
|
||||
credential_revoked = [bool]$clientId
|
||||
summary = @($summary)
|
||||
requests = @($results)
|
||||
} | ConvertTo-Json -Depth 8
|
||||
Reference in New Issue
Block a user