Ran a real AI Operations Brief via the live ITWorx MCP Hub connector's own MobilityOpsClient against production Fleet Ops: real operations summary, real most-pressing vehicle, real grounded knowledge answer with citations, real correlation IDs verified end-to-end in Fleet Ops's own audit log. No write actions performed. Runbook and full output in docs/final-integrations/ai-operations-brief-runbook.md. Ran the full Playwright e2e suite against the live deployed instance and fixed two pre-existing fragile locators unrelated to this session's feature work (both broke because Automation now legitimately has two tables sharing the same generic selectors, exposed by running the full suite rather than individual files) plus one pre-existing untranslated-loanword false positive. All specs pass. artifacts/final-integrations/final-summary.md has the complete evidence write-up: repository/deployment state, what was fixed vs. handed off, test results, and known limitations stated plainly. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
394 lines
18 KiB
TypeScript
394 lines
18 KiB
TypeScript
import { expect, test, type APIRequestContext, type Page } from "@playwright/test";
|
|
|
|
// Targeted end-to-end coverage for the Fleet Ops correction brief (docs/fleet-ops-
|
|
// correction/): branding, the redesigned status-recommendation flow (preview/apply/
|
|
// manual-review/stale-token), MO-016 order independence, trilingual knowledge
|
|
// grounding, and localized audit/automation content. See also i18n-coverage.spec.ts
|
|
// (key parity, brand invariant, translation-quality) and responsive-i18n.spec.ts
|
|
// (breakpoint matrix) for the complementary static-content checks.
|
|
|
|
async function resetDemoData(request: APIRequestContext) {
|
|
const login = await request.post("/api/v1/demo/login", { data: { role: "operations_manager" } });
|
|
expect(login.ok()).toBeTruthy();
|
|
const reset = await request.post("/api/v1/demo/reset");
|
|
expect(reset.ok()).toBeTruthy();
|
|
// /api/v1/demo/reset deletes the session cookie (it recreates the users table), so any
|
|
// further authenticated call through this same request context needs a fresh login.
|
|
const relogin = await request.post("/api/v1/demo/login", { data: { role: "operations_manager" } });
|
|
expect(relogin.ok()).toBeTruthy();
|
|
}
|
|
|
|
const EXPLORE_OPS_MANAGER: Record<string, string> = {
|
|
"nl-BE": "Verken als Operationsmanager",
|
|
"en-GB": "Explore as Operations Manager",
|
|
"fr-BE": "Explorer en tant que Responsable des opérations",
|
|
};
|
|
|
|
const REVIEW_RECOMMENDATION: Record<string, string> = {
|
|
"nl-BE": "Aanbeveling bekijken",
|
|
"en-GB": "Review recommendation",
|
|
"fr-BE": "Voir la recommandation",
|
|
};
|
|
|
|
const CHANGE_STATUS_PREFIX: Record<string, RegExp> = {
|
|
"nl-BE": /^Status wijzigen naar/,
|
|
"en-GB": /^Change status to/,
|
|
"fr-BE": /^Changer le statut vers/,
|
|
};
|
|
|
|
const MANUAL_REVIEW_HEADING: Record<string, string> = {
|
|
"nl-BE": "Handmatige beoordeling vereist",
|
|
"en-GB": "Manual review required",
|
|
"fr-BE": "Évaluation manuelle requise",
|
|
};
|
|
|
|
async function loginAsOpsManager(page: Page, lang: string) {
|
|
await page.addInitScript((l) => localStorage.setItem("fleetops.language", l), lang);
|
|
await page.goto("/login");
|
|
await page.getByRole("button", { name: EXPLORE_OPS_MANAGER[lang] }).click();
|
|
await expect(page).toHaveURL(/\/dashboard$/);
|
|
}
|
|
|
|
test.describe.configure({ mode: "serial" });
|
|
|
|
test.describe("branding", () => {
|
|
for (const lang of ["nl-BE", "en-GB", "fr-BE"]) {
|
|
test(`Fleet Ops is the visible brand and no MobilityOps/PoC leaks through (${lang})`, async ({
|
|
page,
|
|
request,
|
|
}) => {
|
|
await resetDemoData(request);
|
|
await loginAsOpsManager(page, lang);
|
|
await expect(page.locator(".brand-mark").first()).toBeVisible();
|
|
await expect(page.getByText("Fleet Ops", { exact: true }).first()).toBeVisible();
|
|
await expect(page.locator(".app-footer")).toContainText("Fleet Ops");
|
|
await expect(page.locator("html")).toHaveAttribute("lang", lang);
|
|
|
|
for (const path of ["/dashboard", "/vehicles", "/data-quality", "/audit", "/automation", "/knowledge"]) {
|
|
await page.goto(path);
|
|
const text = await page.locator("body").innerText();
|
|
expect(text, `${path} (${lang})`).not.toContain("MobilityOps");
|
|
expect(text, `${path} (${lang})`).not.toMatch(/\bPoC\b/);
|
|
}
|
|
});
|
|
}
|
|
});
|
|
|
|
test("the Fleet Ops favicon is linked and resolves (not the browser's blank-tab default)", async ({ page, request }) => {
|
|
await resetDemoData(request);
|
|
await page.goto("/login");
|
|
const href = await page.locator('link[rel="icon"]').getAttribute("href");
|
|
expect(href).toBe("/favicon.svg");
|
|
const response = await page.request.get(href as string);
|
|
expect(response.ok()).toBeTruthy();
|
|
expect(response.headers()["content-type"]).toContain("svg");
|
|
});
|
|
|
|
test("language switcher control changes the UI and persists across a reload", async ({ page, request }) => {
|
|
await resetDemoData(request);
|
|
// Deliberately not using loginAsOpsManager here: its addInitScript would re-force
|
|
// nl-BE on every reload, defeating exactly the persistence behaviour under test.
|
|
await page.goto("/login");
|
|
await page.getByRole("button", { name: EXPLORE_OPS_MANAGER["nl-BE"] }).click();
|
|
await expect(page).toHaveURL(/\/dashboard$/);
|
|
await expect(page.getByRole("heading", { name: "Aandachtspunten" })).toBeVisible();
|
|
|
|
await page.getByRole("combobox", { name: "Taal" }).selectOption("fr-BE");
|
|
await expect(page.getByRole("heading", { name: "File d'attention" })).toBeVisible();
|
|
await expect(page.locator("html")).toHaveAttribute("lang", "fr-BE");
|
|
|
|
await page.reload();
|
|
await expect(page.getByRole("heading", { name: "File d'attention" })).toBeVisible();
|
|
await expect(page.locator("html")).toHaveAttribute("lang", "fr-BE");
|
|
});
|
|
|
|
test.describe("status-recommendation flow", () => {
|
|
test("preview does not mutate anything, apply names the exact target status", async ({ page, request }) => {
|
|
await resetDemoData(request);
|
|
await loginAsOpsManager(page, "nl-BE");
|
|
await page.goto("/data-quality/DQ-DEMO-STATUS");
|
|
await expect(page.getByRole("heading", { name: "DQ-DEMO-STATUS" })).toBeVisible();
|
|
|
|
await page.getByRole("button", { name: REVIEW_RECOMMENDATION["nl-BE"] }).click();
|
|
await expect(page.getByText("Aanbevolen status")).toBeVisible();
|
|
await expect(page.getByRole("heading", { name: "Waarom" })).toBeVisible();
|
|
await expect(page.getByRole("heading", { name: "Gevolg" })).toBeVisible();
|
|
|
|
// Previewing must not have resolved the issue -- still open, using the page's own
|
|
// authenticated session (page.request shares cookies with the browser context).
|
|
const issue = await page.request.get("/api/v1/data-quality/issues/DQ-DEMO-STATUS");
|
|
expect((await issue.json()).status).toBe("open");
|
|
|
|
const confirmButton = page.getByRole("button", { name: CHANGE_STATUS_PREFIX["nl-BE"] });
|
|
await expect(confirmButton).toHaveText(/Geblokkeerd/);
|
|
await confirmButton.click();
|
|
await expect(page.getByText("Toegepast", { exact: false })).toBeVisible();
|
|
|
|
const resolved = await page.request.get("/api/v1/data-quality/issues/DQ-DEMO-STATUS");
|
|
expect((await resolved.json()).status).toBe("resolved");
|
|
});
|
|
|
|
test("manual review state offers no generic apply button for a genuine fact contradiction", async ({
|
|
page,
|
|
request,
|
|
}) => {
|
|
await resetDemoData(request);
|
|
const issues = await (
|
|
await request.get("/api/v1/data-quality/issues", {
|
|
params: { rule_type: "vehicle_status_conflict", status: "open" },
|
|
})
|
|
).json();
|
|
const conflicted = issues.find((i: { entity_ref: string }) => i.entity_ref === "MO-024");
|
|
expect(conflicted, "expected MO-024's vehicle_status_conflict issue to exist after reset").toBeTruthy();
|
|
|
|
await loginAsOpsManager(page, "en-GB");
|
|
await page.goto(`/data-quality/${conflicted.public_ref}`);
|
|
await page.getByRole("button", { name: REVIEW_RECOMMENDATION["en-GB"] }).click();
|
|
|
|
await expect(page.getByRole("heading", { name: MANUAL_REVIEW_HEADING["en-GB"] })).toBeVisible();
|
|
await expect(page.getByRole("button", { name: /^Change status to/ })).toHaveCount(0);
|
|
});
|
|
|
|
test("a stale recommendation is rejected and the user must review again before applying", async ({
|
|
page,
|
|
request,
|
|
}) => {
|
|
await resetDemoData(request);
|
|
await loginAsOpsManager(page, "en-GB");
|
|
await page.goto("/data-quality/DQ-DEMO-STATUS");
|
|
await page.getByRole("button", { name: REVIEW_RECOMMENDATION["en-GB"] }).click();
|
|
await expect(page.getByRole("button", { name: /^Change status to/ })).toBeVisible();
|
|
|
|
// Simulate the underlying facts changing after the preview was shown (the same
|
|
// session resolves the booking overlap in the meantime) -- the previously-fetched
|
|
// recommendation token must no longer be accepted.
|
|
await page.evaluate(async () => {
|
|
await fetch("/api/v1/data-quality/issues/DQ-DEMO-OVERLAP/resolve-overlap", {
|
|
method: "POST",
|
|
headers: { "Content-Type": "application/json" },
|
|
body: JSON.stringify({ booking_ref: "BK-DEMO-OVERLAP-B" }),
|
|
});
|
|
});
|
|
|
|
await page.getByRole("button", { name: /^Change status to/ }).click();
|
|
await expect(page.getByText(/situation has changed/i)).toBeVisible();
|
|
await expect(page.getByRole("button", { name: REVIEW_RECOMMENDATION["en-GB"] })).toBeVisible();
|
|
});
|
|
});
|
|
|
|
test.describe("MO-016 status conflict is order-independent", () => {
|
|
// Order independence does NOT mean "the same final vehicle status regardless of
|
|
// order" -- resolving the booking overlap first genuinely removes the conflict, so
|
|
// there is correctly nothing left to apply afterwards. What must hold in either
|
|
// order: the recommendation always reflects the real, current facts (never a stale
|
|
// "was some other issue open" proxy), and nothing unsafe is ever applied (never
|
|
// "rented").
|
|
|
|
test("resolving the booking overlap first correctly leaves nothing to apply", async ({ page, request }) => {
|
|
await resetDemoData(request);
|
|
await loginAsOpsManager(page, "nl-BE");
|
|
await page.goto("/data-quality/DQ-DEMO-OVERLAP");
|
|
await page.getByRole("radio", { name: /BK-DEMO-OVERLAP-B blokkeren/ }).check();
|
|
await page.getByRole("button", { name: /BK-DEMO-OVERLAP-B blokkeren/ }).click();
|
|
await expect(page.getByText("Opgelost").first()).toBeVisible();
|
|
|
|
await page.goto("/data-quality/DQ-DEMO-STATUS");
|
|
await page.getByRole("button", { name: REVIEW_RECOMMENDATION["nl-BE"] }).click();
|
|
await expect(page.getByRole("heading", { name: "Geen wijziging nodig" })).toBeVisible();
|
|
await expect(page.getByRole("button", { name: CHANGE_STATUS_PREFIX["nl-BE"] })).toHaveCount(0);
|
|
|
|
const vehicle = await request.get("/api/v1/vehicles/MO-016");
|
|
expect((await vehicle.json()).operational_status).toBe("available");
|
|
});
|
|
|
|
test("resolving the status conflict first safely blocks the vehicle, unaffected by the later overlap fix", async ({
|
|
page,
|
|
request,
|
|
}) => {
|
|
await resetDemoData(request);
|
|
await loginAsOpsManager(page, "nl-BE");
|
|
await page.goto("/data-quality/DQ-DEMO-STATUS");
|
|
await page.getByRole("button", { name: REVIEW_RECOMMENDATION["nl-BE"] }).click();
|
|
await page.getByRole("button", { name: CHANGE_STATUS_PREFIX["nl-BE"] }).click();
|
|
await expect(page.getByText("Toegepast", { exact: false })).toBeVisible();
|
|
|
|
const vehicleMid = await request.get("/api/v1/vehicles/MO-016");
|
|
const statusAfterApply = (await vehicleMid.json()).operational_status;
|
|
expect(statusAfterApply).not.toBe("rented");
|
|
|
|
await page.goto("/data-quality/DQ-DEMO-OVERLAP");
|
|
await page.getByRole("radio", { name: /BK-DEMO-OVERLAP-B blokkeren/ }).check();
|
|
await page.getByRole("button", { name: /BK-DEMO-OVERLAP-B blokkeren/ }).click();
|
|
await expect(page.getByText("Opgelost").first()).toBeVisible();
|
|
|
|
// Resolving the now-redundant overlap afterwards must not itself change the
|
|
// vehicle's status as a side effect.
|
|
const vehicleFinal = await request.get("/api/v1/vehicles/MO-016");
|
|
const statusFinal = (await vehicleFinal.json()).operational_status;
|
|
expect(statusFinal).toBe(statusAfterApply);
|
|
expect(statusFinal).not.toBe("rented");
|
|
|
|
await resetDemoData(request);
|
|
});
|
|
});
|
|
|
|
test.describe("knowledge base is grounded in the operator's own language", () => {
|
|
const cases: { lang: string; question: string; sourceHint: RegExp }[] = [
|
|
{
|
|
lang: "nl-BE",
|
|
question: "Wat moet ik doen wanneer een voertuig beschadigd terugkomt?",
|
|
sourceHint: /schadeafhandeling/i,
|
|
},
|
|
{
|
|
lang: "en-GB",
|
|
question: "What should I do when a vehicle returns with damage?",
|
|
sourceHint: /damage/i,
|
|
},
|
|
{
|
|
lang: "fr-BE",
|
|
question: "Que dois-je faire lorsqu'un véhicule revient endommagé ?",
|
|
sourceHint: /dommages/i,
|
|
},
|
|
];
|
|
|
|
for (const { lang, question, sourceHint } of cases) {
|
|
test(`grounded ${lang} answer cites a ${lang} source about damage`, async ({ page, request }) => {
|
|
await resetDemoData(request);
|
|
await loginAsOpsManager(page, lang);
|
|
await page.goto("/knowledge");
|
|
await page.locator("#knowledge-question").fill(question);
|
|
await page.getByRole("button", { name: /^(Vraag stellen|Ask|Demander)$/ }).click();
|
|
await expect(page.getByText(sourceHint).first()).toBeVisible({ timeout: 10_000 });
|
|
});
|
|
}
|
|
});
|
|
|
|
test("audit trail shows localized action and field labels with raw codes only in technical details", async ({
|
|
page,
|
|
request,
|
|
}) => {
|
|
await resetDemoData(request);
|
|
await loginAsOpsManager(page, "nl-BE");
|
|
await page.goto("/data-quality/DQ-DEMO-STATUS");
|
|
await page.getByRole("button", { name: REVIEW_RECOMMENDATION["nl-BE"] }).click();
|
|
await page.getByRole("button", { name: CHANGE_STATUS_PREFIX["nl-BE"] }).click();
|
|
await expect(page.getByText("Toegepast", { exact: false })).toBeVisible();
|
|
|
|
await page.goto("/audit");
|
|
|
|
// Applying resolves both the vehicle status and the issue in one correlated action --
|
|
// expand the group's "technical events" toggle so the other audit row's diff renders
|
|
// too, regardless of which one the grouping picked as primary.
|
|
const toggle = page.getByRole("button", { name: /technische gebeurtenis/ }).first();
|
|
await expect(toggle).toBeVisible();
|
|
await toggle.click();
|
|
await expect(page.getByRole("button", { name: "Technische gebeurtenissen verbergen" })).toBeVisible();
|
|
|
|
await expect(page.getByText("Aanbevolen status toegepast").first()).toBeVisible();
|
|
const diffs = page.locator(".change-diff");
|
|
await expect(diffs.first()).toBeVisible();
|
|
const combinedDiffText = (await diffs.allInnerTexts()).join(" ");
|
|
expect(combinedDiffText).toContain("Operationele status");
|
|
expect(combinedDiffText).not.toContain("operational_status");
|
|
});
|
|
|
|
test("automation shows a localized error explanation with the raw error only under technical details", async ({
|
|
page,
|
|
request,
|
|
}) => {
|
|
await resetDemoData(request);
|
|
await loginAsOpsManager(page, "nl-BE");
|
|
await page.goto("/automation");
|
|
|
|
await expect(page.getByText(/tijdelijk niet bereikbaar/).first()).toBeVisible();
|
|
await expect(page.getByText("Synthetic connection timeout to n8n")).not.toBeVisible();
|
|
// Scope to the failed job's own row -- the workflow-evidence table above it also has
|
|
// "Technische details" toggles (one per workflow), so an unscoped .first() can open
|
|
// the wrong one.
|
|
const failedJobRow = page.locator("tr", { has: page.getByText(/tijdelijk niet bereikbaar/) });
|
|
await failedJobRow.getByText("Technische details").click();
|
|
await expect(page.getByText("Synthetic connection timeout to n8n")).toBeVisible();
|
|
});
|
|
|
|
test.describe("route matrix (section 11F)", () => {
|
|
// Opens every main route in all 3 languages: no console errors, correct html[lang],
|
|
// and a real, non-empty page heading (proving the route actually rendered content
|
|
// instead of silently falling back to a raw i18next key or a blank screen). Key
|
|
// parity across locale files is already proven structurally by i18n-coverage.spec.ts
|
|
// (every key that exists in nl-BE also exists, non-empty, in en-GB/fr-BE), so this
|
|
// matrix focuses on what only a live render can catch.
|
|
const routes = [
|
|
"/dashboard",
|
|
"/vehicles",
|
|
"/vehicles/MO-001",
|
|
"/bookings",
|
|
"/bookings/BK-DEMO-RETURN",
|
|
"/data-quality",
|
|
"/data-quality/DQ-DEMO-STATUS",
|
|
"/automation",
|
|
"/knowledge",
|
|
"/audit",
|
|
"/scenarios",
|
|
"/about",
|
|
];
|
|
|
|
for (const lang of ["nl-BE", "en-GB", "fr-BE"]) {
|
|
test(`every main route renders correctly with no console errors (${lang})`, async ({ page, request }) => {
|
|
await resetDemoData(request);
|
|
|
|
const errors: string[] = [];
|
|
page.on("console", (msg) => {
|
|
if (msg.type() !== "error") return;
|
|
if (msg.text().includes("401") && msg.text().includes("Unauthorized")) return;
|
|
errors.push(msg.text());
|
|
});
|
|
page.on("pageerror", (err) => errors.push(err.message));
|
|
|
|
await loginAsOpsManager(page, lang);
|
|
|
|
for (const route of routes) {
|
|
await page.goto(route);
|
|
await expect(page.locator("html")).toHaveAttribute("lang", lang);
|
|
const heading = page.getByRole("heading", { level: 1 });
|
|
await expect(heading, `${route} (${lang})`).toBeVisible();
|
|
const headingText = (await heading.first().textContent())?.trim() ?? "";
|
|
expect(headingText, `${route} (${lang}) heading text`).not.toBe("");
|
|
// A raw, unresolved i18next key looks like "namespace:some.key.path" -- real
|
|
// page headings never contain a colon followed by a dotted identifier.
|
|
expect(headingText, `${route} (${lang}) heading looks like a raw i18n key`).not.toMatch(
|
|
/^[a-zA-Z]+:[\w.]+$/,
|
|
);
|
|
}
|
|
|
|
expect(errors, `Console errors across the route matrix (${lang}):\n${errors.join("\n")}`).toEqual([]);
|
|
});
|
|
}
|
|
});
|
|
|
|
test.describe("data-quality evidence summary is localized, not raw English (section 6)", () => {
|
|
// The primary evidence line at the top of every issue's detail page must render the
|
|
// structured `evidence.signals` in the operator's language; the legacy English
|
|
// `evidence.summary` string is a technical fallback only, visible solely inside
|
|
// "Technical details". Live-caught: this line was unconditionally showing raw
|
|
// English ("vehicle marked available while reserved bookings conflict") in every
|
|
// language until fixed.
|
|
const cases: { lang: string; expectedText: RegExp }[] = [
|
|
{ lang: "nl-BE", expectedText: /overlappende reserveringen/i },
|
|
{ lang: "en-GB", expectedText: /overlapping bookings/i },
|
|
{ lang: "fr-BE", expectedText: /chevauchent|chevauchement/i },
|
|
];
|
|
|
|
for (const { lang, expectedText } of cases) {
|
|
test(`vehicle_status_conflict evidence is localized (${lang})`, async ({ page, request }) => {
|
|
await resetDemoData(request);
|
|
await loginAsOpsManager(page, lang);
|
|
await page.goto("/data-quality/DQ-DEMO-STATUS");
|
|
|
|
const summarySection = page.locator(".record-surface-evidence");
|
|
await expect(summarySection).toBeVisible();
|
|
await expect(summarySection).toContainText(expectedText);
|
|
await expect(summarySection).not.toContainText("vehicle marked available while reserved bookings conflict");
|
|
});
|
|
}
|
|
});
|