param( [int]$Cycles = 4, [int]$IdleSeconds = 45 ) $ErrorActionPreference = "Stop" if ($Cycles -lt 2 -or $Cycles -gt 12) { throw "Cycles must be between 2 and 12" } if ($IdleSeconds -lt 10 -or $IdleSeconds -gt 300) { throw "IdleSeconds must be between 10 and 300" } $baseUrl = "http://127.0.0.1:8000" $operatorLine = Get-Content .env | Where-Object { $_ -match '^MODELFORGE_OPERATOR_API_KEY=' } | Select-Object -First 1 if (-not $operatorLine) { throw "MODELFORGE_OPERATOR_API_KEY is unavailable" } $operatorToken = $operatorLine.Substring($operatorLine.IndexOf('=') + 1).Trim().Trim('"').Trim("'") $operatorHeaders = @{ "X-ModelForge-Admin-Token" = $operatorToken } $runId = [guid]::NewGuid().ToString("N").Substring(0, 12) $clientId = $null $results = [System.Collections.Generic.List[object]]::new() $startedAt = [datetime]::UtcNow function New-SilenceWavBase64 { $sampleRate = 16000 $sampleCount = $sampleRate $dataLength = $sampleCount * 2 $stream = [System.IO.MemoryStream]::new() $writer = [System.IO.BinaryWriter]::new($stream) try { $writer.Write([System.Text.Encoding]::ASCII.GetBytes("RIFF")) $writer.Write([int](36 + $dataLength)) $writer.Write([System.Text.Encoding]::ASCII.GetBytes("WAVE")) $writer.Write([System.Text.Encoding]::ASCII.GetBytes("fmt ")) $writer.Write([int]16) $writer.Write([int16]1) $writer.Write([int16]1) $writer.Write([int]$sampleRate) $writer.Write([int]($sampleRate * 2)) $writer.Write([int16]2) $writer.Write([int16]16) $writer.Write([System.Text.Encoding]::ASCII.GetBytes("data")) $writer.Write([int]$dataLength) $writer.Write([byte[]]::new($dataLength)) $writer.Flush() return [Convert]::ToBase64String($stream.ToArray()) } finally { $writer.Dispose() $stream.Dispose() } } function Invoke-SoakCapability { param( [string]$Capability, [string]$Path, [hashtable]$Body, [int]$Cycle, [hashtable]$Headers ) $watch = [System.Diagnostics.Stopwatch]::StartNew() try { $response = Invoke-RestMethod -Method Post -Uri "$baseUrl$Path" -Headers $Headers ` -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 8 -Compress) ` -TimeoutSec 180 $watch.Stop() $results.Add([pscustomobject]@{ cycle = $Cycle capability = $Capability success = $true request_id = [string]$response.request_id cold = [bool]$response.execution.cold residency = [string]$response.execution.residency node = [string]$response.execution.node queue_ms = [double]$response.execution.timings.queue_ms load_ms = [double]$response.execution.timings.load_ms inference_ms = [double]$response.execution.timings.inference_ms total_ms = [double]$response.execution.timings.total_ms wall_ms = [double]$watch.Elapsed.TotalMilliseconds error_class = $null error_code = $null }) } catch { $watch.Stop() $errorCode = $null if ($_.ErrorDetails.Message) { try { $errorBody = $_.ErrorDetails.Message | ConvertFrom-Json $errorCode = [string]$errorBody.error.code } catch { $errorCode = "UNPARSEABLE_HTTP_ERROR" } } $results.Add([pscustomobject]@{ cycle = $Cycle capability = $Capability success = $false request_id = $null cold = $null residency = $null node = $null queue_ms = $null load_ms = $null inference_ms = $null total_ms = $null wall_ms = [double]$watch.Elapsed.TotalMilliseconds error_class = $_.Exception.GetType().Name error_code = $errorCode }) } } function Get-Percentile { param([double[]]$Values, [double]$Percentile) if (-not $Values -or $Values.Count -eq 0) { return $null } $ordered = @($Values | Sort-Object) $index = [math]::Ceiling($Percentile * $ordered.Count) - 1 return [double]$ordered[[math]::Max(0, $index)] } try { $clientBody = @{ name = "m14-soak-$runId" allowed_capabilities = @( "rag.embedding@1", "vision.embedding@1", "speech.transcription@1" ) requests_per_minute = 60 max_concurrent_requests = 1 workload_priority = "interactive" credential_expires_at = [datetime]::UtcNow.AddMinutes(30).ToString("o") } $client = Invoke-RestMethod -Method Post -Uri "$baseUrl/api/v1/admin/service-clients" ` -Headers $operatorHeaders -ContentType "application/json" ` -Body ($clientBody | ConvertTo-Json -Depth 6 -Compress) $clientId = [string]$client.id $serviceHeaders = @{ Authorization = "Bearer $($client.credential)" } $audio = New-SilenceWavBase64 for ($cycle = 1; $cycle -le $Cycles; $cycle++) { Invoke-RestMethod -Method Post -Uri "$baseUrl/api/v1/admin/operations/capacity/collect" ` -Headers $operatorHeaders | Out-Null Invoke-SoakCapability -Capability "rag.embedding@1" ` -Path "/api/v1/capabilities/rag.embedding@1/invoke" ` -Body @{ input = @("M14 bounded operational soak cycle $cycle") } ` -Cycle $cycle -Headers $serviceHeaders Invoke-SoakCapability -Capability "vision.embedding@1 LAB" ` -Path "/api/v1/capabilities/vision.embedding@1/invoke" ` -Body @{ items = @(@{ text = "M14 visual control sample cycle $cycle" }) } ` -Cycle $cycle -Headers $serviceHeaders Invoke-SoakCapability -Capability "speech.transcription@1 LAB" ` -Path "/api/v1/capabilities/speech.transcription@1/invoke" ` -Body @{ audio_base64 = $audio; media_type = "audio/wav"; language = "en" } ` -Cycle $cycle -Headers $serviceHeaders Invoke-RestMethod -Method Post -Uri "$baseUrl/api/v1/admin/operations/capacity/collect" ` -Headers $operatorHeaders | Out-Null Invoke-RestMethod -Method Post -Uri "$baseUrl/api/v1/admin/operations/slo-evaluations/run" ` -Headers $operatorHeaders | Out-Null Invoke-RestMethod -Method Post -Uri "$baseUrl/api/v1/admin/operations/alerts/evaluate" ` -Headers $operatorHeaders | Out-Null if ($cycle -lt $Cycles) { Start-Sleep -Seconds $IdleSeconds } } } finally { if ($clientId) { Invoke-RestMethod -Method Delete ` -Uri "$baseUrl/api/v1/admin/service-clients/$clientId/credential" ` -Headers $operatorHeaders | Out-Null } } $finishedAt = [datetime]::UtcNow $summary = foreach ($group in ($results | Group-Object capability)) { $successful = @($group.Group | Where-Object success) $latencies = @($successful | ForEach-Object { [double]$_.total_ms }) [pscustomobject]@{ capability = $group.Name requests = $group.Count failures = @($group.Group | Where-Object { -not $_.success }).Count cold = @($successful | Where-Object cold).Count warm = @($successful | Where-Object { -not $_.cold }).Count p50_ms = Get-Percentile -Values $latencies -Percentile 0.50 p95_ms = Get-Percentile -Values $latencies -Percentile 0.95 max_queue_ms = if ($successful) { [double](($successful | Measure-Object queue_ms -Maximum).Maximum) } else { $null } max_load_ms = if ($successful) { [double](($successful | Measure-Object load_ms -Maximum).Maximum) } else { $null } } } [pscustomobject]@{ run_id = $runId started_at = $startedAt.ToString("o") finished_at = $finishedAt.ToString("o") duration_seconds = ($finishedAt - $startedAt).TotalSeconds cycles = $Cycles idle_seconds = $IdleSeconds credential_revoked = [bool]$clientId summary = @($summary) requests = @($results) } | ConvertTo-Json -Depth 8