mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-09-10 18:45:53 +00:00
test(assistant): add portable identity browser regression
Release-line could reproduce tool identity and shared evidence regressions but could not replay browser evidence tied to private capture paths. Supply a synthetic standalone real reducer/transcript journey with blocked backend and external requests, desktop and narrow assertions, and content-hashed receipts. This verifies renderer behaviour without claiming provider, production or release admission qualification. Change-source: pulse-maintainer
This commit is contained in:
pulse-triage[bot]
@@ -0,0 +1,37 @@
|
||||
# Assistant identity browser regression
|
||||
|
||||
From the repository root (Node 24):
|
||||
|
||||
```sh
|
||||
npm ci --ignore-scripts
|
||||
npm ci --ignore-scripts --prefix frontend-modern
|
||||
npx playwright install chromium
|
||||
pulse-heavy-run -- node scripts/check-assistant-identity.mjs
|
||||
```
|
||||
|
||||
This standalone Vite fixture imports the current checkout's real `useChat` and
|
||||
`ChatMessages`, including `MessageItem`, approval cards and tool detail rows.
|
||||
It is not a production entry point and needs no backend, credentials, provider,
|
||||
customer captures or live infrastructure. The only replaced API is the chat
|
||||
stream callback; all browser API/external HTTP requests are blocked and fail the
|
||||
check. Approval completion is synthetic, not evidence of an authorised action.
|
||||
|
||||
Checks at 1440, 900 and 390 pixels cover concurrent same-name invocation IDs,
|
||||
progress/completion isolation, retaining a sibling approval after completion,
|
||||
settled rendered results, and shared input/output evidence after removal of a
|
||||
workflow row. Both evidence rows are expanded and document overflow is checked.
|
||||
The shared-object journey deliberately retains references, matching the original
|
||||
regression rather than concealing it with cloned fixtures.
|
||||
|
||||
Output defaults to `tmp/assistant-identity`; set `ASSISTANT_IDENTITY_OUTPUT` to
|
||||
an absolute path to retain a candidate-specific run. A successful run writes
|
||||
screenshots and `receipt.json` with browser version and SHA-256 source hashes.
|
||||
Run on the actual backport checkout: a main receipt does not qualify release
|
||||
content. This is a component/reducer browser proof, not the release admission
|
||||
attestation, a production-build check, SSE transport qualification, provider
|
||||
reasoning proof, or installed end-to-end action verification.
|
||||
|
||||
Regression sensitivity can be checked by temporarily substituting either
|
||||
`ChatMessages.tsx` or `hooks/useChat.ts` from the parent of fix `33b852f66b`:
|
||||
each must fail independently. Restore files afterwards; never admit those
|
||||
experimental substitutions as candidate content.
|
||||
@@ -0,0 +1,72 @@
|
||||
// Synthetic component journey: real reducer and transcript, no application server.
|
||||
import { createSignal } from 'solid-js';
|
||||
import { render } from 'solid-js/web';
|
||||
import { AIChatAPI, type StreamEvent } from '../../src/api/aiChat';
|
||||
import { useChat } from '../../src/components/AI/Chat/hooks/useChat';
|
||||
import { ChatMessages } from '../../src/components/AI/Chat/ChatMessages';
|
||||
import type { ChatMessage } from '../../src/components/AI/Chat/types';
|
||||
import '../../src/index.css';
|
||||
|
||||
let dispatch: (event: StreamEvent) => void;
|
||||
AIChatAPI.chat = async (_prompt, _session, _model, onEvent) => {
|
||||
dispatch = onEvent;
|
||||
await new Promise(() => {}); // Keep stream open until the fixture is disposed.
|
||||
};
|
||||
const noOp = () => {};
|
||||
function Fixture() {
|
||||
const chat = useChat({ sessionId: 'synthetic-identity' });
|
||||
const [override, setOverride] = createSignal<ChatMessage[] | null>(null);
|
||||
const a = { name: 'pulse_query', input: 'client', output: 'client evidence', success: true };
|
||||
const b = { name: 'pulse_alerts', input: 'alerts', output: 'alert evidence', success: true };
|
||||
const pendingA = { id: 'a', name: a.name, input: a.input };
|
||||
const pendingB = { id: 'b', name: b.name, input: b.input };
|
||||
const workflow = {
|
||||
type: 'workflow_status' as const,
|
||||
workflowStatus: { phase: 'provider_start', message: 'Starting' },
|
||||
};
|
||||
const message = (extra: Partial<ChatMessage>): ChatMessage => ({
|
||||
id: 'shared',
|
||||
role: 'assistant',
|
||||
content: '',
|
||||
timestamp: new Date('2026-01-01T00:00:00Z'),
|
||||
...extra,
|
||||
});
|
||||
Object.assign(window, {
|
||||
identityFixture: {
|
||||
start: () => {
|
||||
void chat.sendMessage('Synthetic identity check');
|
||||
},
|
||||
fire: (event: StreamEvent) => dispatch(event),
|
||||
snapshot: () => chat.messages().find((m) => m.role === 'assistant'),
|
||||
evidence: () => [a, b],
|
||||
shared: (stage: number) =>
|
||||
setOverride([
|
||||
message({
|
||||
toolCalls: stage === 0 ? [] : stage === 1 ? [a] : [a, b],
|
||||
pendingTools: stage === 0 ? [pendingA, pendingB] : stage === 1 ? [pendingB] : [],
|
||||
streamEvents: [
|
||||
...(stage < 3 ? [workflow] : []),
|
||||
stage === 0
|
||||
? { type: 'pending_tool', toolId: 'a', pendingTool: pendingA }
|
||||
: { type: 'tool', toolId: 'a', tool: a },
|
||||
stage < 2
|
||||
? { type: 'pending_tool', toolId: 'b', pendingTool: pendingB }
|
||||
: { type: 'tool', toolId: 'b', tool: b },
|
||||
],
|
||||
}),
|
||||
]),
|
||||
},
|
||||
});
|
||||
return (
|
||||
<main style={{ height: '100vh', display: 'flex', 'flex-direction': 'column' }}>
|
||||
<ChatMessages
|
||||
messages={override() ?? chat.messages()}
|
||||
onApprove={(id, approval) => chat.updateApproval(id, approval.toolId, { removed: true })}
|
||||
onSkip={(id, toolId) => chat.updateApproval(id, toolId, { removed: true })}
|
||||
onAnswerQuestion={noOp}
|
||||
onSkipQuestion={noOp}
|
||||
/>
|
||||
</main>
|
||||
);
|
||||
}
|
||||
render(() => <Fixture />, document.getElementById('root')!);
|
||||
@@ -0,0 +1,10 @@
|
||||
<!doctype html>
|
||||
<html>
|
||||
<head>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
</head>
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
<script type="module" src="./fixture.tsx"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,232 @@
|
||||
// Run with pulse-heavy-run -- node scripts/check-assistant-identity.mjs
|
||||
import { chromium, expect } from "../node_modules/@playwright/test/index.mjs";
|
||||
import { createServer } from "../frontend-modern/node_modules/vite/dist/node/index.js";
|
||||
import solid from "../frontend-modern/node_modules/vite-plugin-solid/dist/esm/index.mjs";
|
||||
import { resolve } from "node:path";
|
||||
import { mkdir, readFile, writeFile, readdir, rm } from "node:fs/promises";
|
||||
import { createHash } from "node:crypto";
|
||||
const root = resolve(import.meta.dirname, "../frontend-modern");
|
||||
const output = resolve(
|
||||
process.env.ASSISTANT_IDENTITY_OUTPUT || `${root}/../tmp/assistant-identity`,
|
||||
);
|
||||
await mkdir(output, { recursive: true });
|
||||
await rm(`${output}/receipt.json`, { force: true });
|
||||
process.chdir(root);
|
||||
const server = await createServer({
|
||||
configFile: false,
|
||||
root,
|
||||
optimizeDeps: { esbuildOptions: { target: "esnext" } },
|
||||
plugins: [solid()],
|
||||
resolve: { alias: { "@": `${root}/src` } },
|
||||
server: { host: "127.0.0.1", port: 0 },
|
||||
});
|
||||
let browser;
|
||||
const results = [];
|
||||
try {
|
||||
await server.listen();
|
||||
const base = server.resolvedUrls.local[0];
|
||||
browser = await chromium.launch({ headless: true });
|
||||
for (const width of [1440, 900, 390]) {
|
||||
const context = await browser.newContext({
|
||||
viewport: { width, height: 1000 },
|
||||
serviceWorkers: "block",
|
||||
});
|
||||
const page = await context.newPage();
|
||||
const errors = [],
|
||||
blocked = [];
|
||||
page.on("pageerror", (e) => errors.push(e.message));
|
||||
await context.route("**/*", (route) => {
|
||||
const url = new URL(route.request().url());
|
||||
if (
|
||||
url.origin === new URL(base).origin &&
|
||||
!url.pathname.startsWith("/api/")
|
||||
)
|
||||
return route.continue();
|
||||
blocked.push(url.pathname);
|
||||
return route.abort();
|
||||
});
|
||||
await page.goto(`${base}qualification/assistant-identity/`);
|
||||
await page.waitForFunction(() => !!window.identityFixture);
|
||||
const call = (method, arg) =>
|
||||
page.evaluate(
|
||||
([method, arg]) => window.identityFixture[method](arg),
|
||||
[method, arg],
|
||||
);
|
||||
const fire = (type, data) => call("fire", { type, data });
|
||||
await call("start");
|
||||
for (const id of ["a", "b"])
|
||||
await fire("tool_start", {
|
||||
id,
|
||||
name: "pulse_query",
|
||||
input: JSON.stringify({ action: "search", query: `fixture-${id}` }),
|
||||
});
|
||||
expect((await call("snapshot")).pendingTools.map((t) => t.id)).toEqual([
|
||||
"a",
|
||||
"b",
|
||||
]);
|
||||
await fire("tool_progress", {
|
||||
id: "a",
|
||||
name: "pulse_query",
|
||||
message: "Reading A",
|
||||
});
|
||||
await fire("tool_end", {
|
||||
id: "a",
|
||||
name: "pulse_query",
|
||||
output: "identity evidence A",
|
||||
success: true,
|
||||
});
|
||||
expect((await call("snapshot")).pendingTools.map((t) => t.id)).toEqual([
|
||||
"b",
|
||||
]);
|
||||
await fire("tool_end", {
|
||||
id: "b",
|
||||
name: "pulse_query",
|
||||
output: "identity evidence B",
|
||||
success: false,
|
||||
});
|
||||
expect((await call("snapshot")).toolCalls.map((t) => t.output)).toEqual([
|
||||
"identity evidence A",
|
||||
"identity evidence B",
|
||||
]);
|
||||
for (const id of ["c", "d"]) {
|
||||
await fire("tool_start", {
|
||||
id,
|
||||
name: "pulse_control",
|
||||
input: JSON.stringify({ resource_id: id }),
|
||||
});
|
||||
await fire("approval_needed", {
|
||||
tool_id: id,
|
||||
tool_name: "pulse_control",
|
||||
approval_id: `approval-${id}`,
|
||||
command: `synthetic-${id}`,
|
||||
});
|
||||
}
|
||||
await expect(
|
||||
page.getByText("Approval Required", { exact: true }),
|
||||
).toHaveCount(2);
|
||||
await fire("tool_end", {
|
||||
id: "c",
|
||||
name: "pulse_control",
|
||||
output: "synthetic c completed",
|
||||
success: true,
|
||||
});
|
||||
await expect(
|
||||
page.getByText("Approval Required", { exact: true }),
|
||||
).toHaveCount(1);
|
||||
expect(
|
||||
(await call("snapshot")).pendingApprovals.map((a) => a.toolId),
|
||||
).toEqual(["d"]);
|
||||
await expect(page.getByText("synthetic-d", { exact: true })).toBeVisible();
|
||||
await page.screenshot({
|
||||
path: `${output}/approvals-${width}.png`,
|
||||
fullPage: true,
|
||||
});
|
||||
await fire("done", {});
|
||||
while (await page.locator('[aria-expanded="false"]').count())
|
||||
await page.locator('[aria-expanded="false"]').first().click();
|
||||
await expect(
|
||||
page.getByText("identity evidence A", { exact: true }).last(),
|
||||
).toBeVisible();
|
||||
await expect(
|
||||
page.getByText("identity evidence B", { exact: true }).last(),
|
||||
).toBeVisible();
|
||||
for (let stage = 0; stage < 4; stage++) {
|
||||
await call("shared", stage);
|
||||
// Allow Solid's render effects to run between immutable snapshots.
|
||||
await page.evaluate(() => new Promise(requestAnimationFrame));
|
||||
}
|
||||
expect(await call("evidence")).toEqual([
|
||||
{
|
||||
name: "pulse_query",
|
||||
input: "client",
|
||||
output: "client evidence",
|
||||
success: true,
|
||||
},
|
||||
{
|
||||
name: "pulse_alerts",
|
||||
input: "alerts",
|
||||
output: "alert evidence",
|
||||
success: true,
|
||||
},
|
||||
]);
|
||||
const expand = page.locator('[role="button"][aria-expanded="false"]');
|
||||
while (await expand.count()) await expand.first().click();
|
||||
await expect(
|
||||
page.getByText("client evidence", { exact: true }).last(),
|
||||
).toBeVisible();
|
||||
await expect(
|
||||
page.getByText("alert evidence", { exact: true }).last(),
|
||||
).toBeVisible();
|
||||
await expect(
|
||||
page.getByText("client", { exact: true }).last(),
|
||||
).toBeVisible();
|
||||
await expect(
|
||||
page.getByText("alerts", { exact: true }).last(),
|
||||
).toBeVisible();
|
||||
expect(
|
||||
await page.evaluate(
|
||||
() => document.documentElement.scrollWidth <= innerWidth,
|
||||
),
|
||||
).toBe(true);
|
||||
await page.screenshot({
|
||||
path: `${output}/expanded-${width}.png`,
|
||||
fullPage: true,
|
||||
});
|
||||
expect(errors).toEqual([]);
|
||||
expect(blocked).toEqual([]);
|
||||
results.push({ width, passed: true });
|
||||
await context.close();
|
||||
}
|
||||
const hashes = {};
|
||||
for (const file of await readdir(output)) {
|
||||
if (file.endsWith(".png"))
|
||||
hashes[`artifacts/${file}`] = createHash("sha256")
|
||||
.update(await readFile(`${output}/${file}`))
|
||||
.digest("hex");
|
||||
}
|
||||
hashes["../package-lock.json"] = createHash("sha256")
|
||||
.update(await readFile(`${root}/../package-lock.json`))
|
||||
.digest("hex");
|
||||
async function hashTree(dir) {
|
||||
for (const entry of await readdir(dir, { withFileTypes: true })) {
|
||||
const path = `${dir}/${entry.name}`;
|
||||
if (entry.isDirectory()) await hashTree(path);
|
||||
else
|
||||
hashes[path.slice(root.length + 1)] = createHash("sha256")
|
||||
.update(await readFile(path))
|
||||
.digest("hex");
|
||||
}
|
||||
}
|
||||
await hashTree(`${root}/src`);
|
||||
await hashTree(`${root}/qualification/assistant-identity`);
|
||||
for (const path of [
|
||||
"package-lock.json",
|
||||
"tailwind.config.js",
|
||||
"postcss.config.js",
|
||||
])
|
||||
hashes[path] = createHash("sha256")
|
||||
.update(await readFile(`${root}/${path}`))
|
||||
.digest("hex");
|
||||
hashes["../scripts/check-assistant-identity.mjs"] = createHash("sha256")
|
||||
.update(await readFile(import.meta.filename))
|
||||
.digest("hex");
|
||||
await writeFile(
|
||||
`${output}/receipt.json`,
|
||||
JSON.stringify(
|
||||
{
|
||||
recordedAt: new Date().toISOString(),
|
||||
scope:
|
||||
"Synthetic reducer and renderer only; no provider, backend or real actions qualified",
|
||||
browser: browser.version(),
|
||||
results,
|
||||
hashes,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
console.log(JSON.stringify({ output, results }));
|
||||
} finally {
|
||||
await browser?.close();
|
||||
await server.close();
|
||||
}
|
||||
Reference in New Issue
Block a user