From b93c92a3e2bedf2019d74a9bb453450a185bf61b Mon Sep 17 00:00:00 2001 From: john Date: Sat, 22 Aug 2026 10:50:23 +0800 Subject: [PATCH] test(scenario): cover multi-image page generation end to end The read_image poisoning fix had no end-to-end guard because scenarios could only send text. Add an upload_images step that renders and uploads visually distinct images, let chat steps attach them the same way the WeChat channel does, and assert the generated page actually embeds every upload. Co-authored-by: Cursor --- .claude/skills/memind-scenario-test/SKILL.md | 8 ++ .cursor/skills/memind-scenario-test/SKILL.md | 8 ++ .../vision-turn-read-image-isolation.md | 10 ++ scenarios/image-theme-page.json | 45 +++++++ scripts/run-scenario-test.mjs | 21 +++ scripts/scenario-test-lib.mjs | 122 +++++++++++++++++- 6 files changed, 212 insertions(+), 2 deletions(-) create mode 100644 scenarios/image-theme-page.json diff --git a/.claude/skills/memind-scenario-test/SKILL.md b/.claude/skills/memind-scenario-test/SKILL.md index fcc9e58..c135c6d 100644 --- a/.claude/skills/memind-scenario-test/SKILL.md +++ b/.claude/skills/memind-scenario-test/SKILL.md @@ -67,12 +67,20 @@ node scripts/run-scenario-test.mjs --scenario suzhou-page |--------|------| | `login` | HTTP 登录，取 cookie + userId | | `chat` | 发消息（同 session 多轮），等 run 终态 + assistant 新回复 | +| `upload_images` | 生成并上传测试图片到 MindSpace，供后续 `chat` 步骤引用 | + +`upload_images` 字段：`categoryCode`（默认 `oa`）、`images[]` +（每项 `filename` / `background` / `shape` / `accent`，shape 支持 +`circle` `square` `triangle` `diamond`）。后续 `chat` 步骤设 +`useUploadedAssets: true` 即可把这些图作为本轮附件发出。 `chat.expect` 可选断言： - `assistantMinChars` / `timeoutMs` - `replyKeywords`：回复文本关键词 +- `forbidReplyPatterns`：回复中不允许出现的字符串 - `page.keywords` / `requirePublicLink` / `requireHttp200` / `requireMindspaceCover` +- `page.requireUploadedImages` / `page.minUploadedImages`：页面必须嵌入 `upload_images` 上传的资产 新增更复杂步骤（截图、微信、多账户）时：先扩展 `scripts/scenario-test-lib.mjs`，再加场景 JSON，**不要**在 skill 里写死一次性命令。 diff --git a/.cursor/skills/memind-scenario-test/SKILL.md b/.cursor/skills/memind-scenario-test/SKILL.md index fcc9e58..c135c6d 100644 --- a/.cursor/skills/memind-scenario-test/SKILL.md +++ b/.cursor/skills/memind-scenario-test/SKILL.md @@ -67,12 +67,20 @@ node scripts/run-scenario-test.mjs --scenario suzhou-page |--------|------| | `login` | HTTP 登录，取 cookie + userId | | `chat` | 发消息（同 session 多轮），等 run 终态 + assistant 新回复 | +| `upload_images` | 生成并上传测试图片到 MindSpace，供后续 `chat` 步骤引用 | + +`upload_images` 字段：`categoryCode`（默认 `oa`）、`images[]` +（每项 `filename` / `background` / `shape` / `accent`，shape 支持 +`circle` `square` `triangle` `diamond`）。后续 `chat` 步骤设 +`useUploadedAssets: true` 即可把这些图作为本轮附件发出。 `chat.expect` 可选断言： - `assistantMinChars` / `timeoutMs` - `replyKeywords`：回复文本关键词 +- `forbidReplyPatterns`：回复中不允许出现的字符串 - `page.keywords` / `requirePublicLink` / `requireHttp200` / `requireMindspaceCover` +- `page.requireUploadedImages` / `page.minUploadedImages`：页面必须嵌入 `upload_images` 上传的资产 新增更复杂步骤（截图、微信、多账户）时：先扩展 `scripts/scenario-test-lib.mjs`，再加场景 JSON，**不要**在 skill 里写死一次性命令。 diff --git a/docs/regression-guards/vision-turn-read-image-isolation.md b/docs/regression-guards/vision-turn-read-image-isolation.md index dbb7685..6225828 100644 --- a/docs/regression-guards/vision-turn-read-image-isolation.md +++ b/docs/regression-guards/vision-turn-read-image-isolation.md @@ -50,6 +50,16 @@ node --test tkmind-proxy.test.mjs chat-image-turn-scope.test.mjs wechat-mp.test. npm run verify:h5-session-patches ``` +端到端（需本地 Portal 8081 + goosed）： + +```bash +node scripts/run-scenario-test.mjs --scenario image-theme-page +``` + +该场景上传 4 张几何图、要求做成主题页面，断言页面生成、公网 200、4 张图全部嵌入， +并禁止回复里出现 `unknown variant`。2026-08-22 首次通过时会话内 `read_image` 调用为 0、 +持久化工具图片块为 0，工具序列是 `load_skill → write_file → read_file`。 + ## 相关代码与用例 | 位置 | 作用 | diff --git a/scenarios/image-theme-page.json b/scenarios/image-theme-page.json new file mode 100644 index 0000000..85300bf --- /dev/null +++ b/scenarios/image-theme-page.json @@ -0,0 +1,45 @@ +{ + "id": "image-theme-page", + "name": "多图主题页面（视觉链路回归）", + "description": "上传 4 张图 → 要求做成主题页面 → 验证页面生成、公网可访问、4 张图都嵌进页面；复现唐用户 read_image 污染文本模型会话导致页面不生成的故障", + "account": { + "username": "john2", + "password": "888888" + }, + "steps": [ + { + "action": "login", + "label": "登录 john2 账户" + }, + { + "action": "upload_images", + "label": "上传 4 张图片", + "categoryCode": "oa", + "images": [ + { "filename": "theme-1.jpg", "background": "#1e3a8a", "shape": "circle", "accent": "#fbbf24" }, + { "filename": "theme-2.jpg", "background": "#7f1d1d", "shape": "square", "accent": "#fca5a5" }, + { "filename": "theme-3.jpg", "background": "#064e3b", "shape": "triangle", "accent": "#6ee7b7" }, + { "filename": "theme-4.jpg", "background": "#3b0764", "shape": "diamond", "accent": "#e9d5ff" } + ] + }, + { + "action": "chat", + "label": "要求把 4 张图做成主题页面", + "message": "把我刚才发的这 4 张图片做成一个主题页面，每张图配一段文字", + "useUploadedAssets": true, + "expect": { + "assistantMinChars": 20, + "timeoutMs": 420000, + "page": { + "requirePublicLink": true, + "requireHttp200": true, + "requireUploadedImages": true + }, + "forbidReplyPatterns": [ + "unknown variant", + "没有按服务号页面技能真正生成成功" + ] + } + } + ] +} diff --git a/scripts/run-scenario-test.mjs b/scripts/run-scenario-test.mjs index 7f7842c..48c93b3 100644 --- a/scripts/run-scenario-test.mjs +++ b/scripts/run-scenario-test.mjs @@ -9,6 +9,7 @@ */ import { loadH5Environment } from './load-env.mjs'; import { + buildAssetImageUrl, createAgentRun, createReporter, listScenarios, @@ -17,6 +18,7 @@ import { logoutViaApi, resolvePortalBase, snapshotPublicHtml, + uploadImageAssets, verifyPageAccess, verifySurveyDelivery, verifyChildrenHobbyDietSurvey, @@ -95,6 +97,7 @@ async function runScenario(scenario, port) { let assistantCombinedLength = 0; let publishKey = null; let htmlBefore = []; + let uploadedAssetIds = []; for (const step of scenario.steps ?? []) { const label = step.label ?? step.action; @@ -119,6 +122,20 @@ async function runScenario(scenario, port) { continue; } + if (step.action === 'upload_images') { + if (!auth) { + throw new Error('upload_images 步骤前必须先 login'); + } + const assets = await uploadImageAssets( + baseUrl, + auth.cookie, + { images: step.images ?? [], categoryCode: step.categoryCode ?? 'oa' }, + reporter, + ); + uploadedAssetIds = assets.map((asset) => asset.id); + continue; + } + if (step.action === 'chat') { if (!auth) { throw new Error('chat 步骤前必须先 login'); @@ -133,6 +150,9 @@ async function runScenario(scenario, port) { sessionId, selectedChatSkill: step.selectedChatSkill ?? null, selectedAssetIds: step.selectedAssetIds ?? null, + imageUrls: step.useUploadedAssets + ? uploadedAssetIds.map(buildAssetImageUrl) + : null, }); reporter.pass('提交消息', `"${step.message}" → run ${run.runId}`); @@ -191,6 +211,7 @@ async function runScenario(scenario, port) { replyText: reply.combined, htmlBefore, expect: step.expect.page, + uploadedAssetIds, reporter, }); } diff --git a/scripts/scenario-test-lib.mjs b/scripts/scenario-test-lib.mjs index ae8b221..8a51dda 100644 --- a/scripts/scenario-test-lib.mjs +++ b/scripts/scenario-test-lib.mjs @@ -121,11 +121,16 @@ export async function logoutViaApi(baseUrl, cookie, reporter) { return body; } -function buildUserMessage(text, { selectedChatSkill = null } = {}) { +function buildUserMessage(text, { selectedChatSkill = null, imageUrls = null } = {}) { const metadata = { userVisible: true, displayText: text }; if (selectedChatSkill) { metadata.memindRun = { selectedChatSkill }; } + // Same shape the WeChat channel attaches for recent uploads, so scenarios can + // exercise the vision turn without the browser upload widget. + if (Array.isArray(imageUrls) && imageUrls.length > 0) { + metadata.imageUrls = imageUrls; + } return { id: crypto.randomUUID(), role: 'user', @@ -134,16 +139,21 @@ function buildUserMessage(text, { selectedChatSkill = null } = {}) { }; } +export function buildAssetImageUrl(assetId) { + return `/api/mindspace/v1/assets/${assetId}/download?inline=1`; +} + export async function createAgentRun(baseUrl, cookie, { message, sessionId = null, selectedChatSkill = null, selectedAssetIds = null, + imageUrls = null, }) { const requestId = crypto.randomUUID(); const body = { request_id: requestId, - user_message: buildUserMessage(message, { selectedChatSkill }), + user_message: buildUserMessage(message, { selectedChatSkill, imageUrls }), }; if (sessionId) body.session_id = sessionId; if (Array.isArray(selectedAssetIds) && selectedAssetIds.length > 0) { @@ -296,6 +306,100 @@ export async function waitForAssistantGrowth(baseUrl, cookie, sessionId, { return null; } +const UPLOADABLE_CATEGORY_CODES = ['oa', 'public']; + +async function resolveUploadCategoryId(baseUrl, cookie, preferredCode) { + const response = await fetch(`${baseUrl}/api/mindspace/v1/space`, { + headers: { Cookie: cookie }, + }); + if (!response.ok) { + throw new Error(`读取 MindSpace 空间失败 ${response.status}`); + } + const payload = await response.json(); + const categories = payload?.data?.categories ?? payload?.categories ?? []; + const uploadable = categories.filter((item) => UPLOADABLE_CATEGORY_CODES.includes(item?.code)); + const picked = uploadable.find((item) => item.code === preferredCode) ?? uploadable[0]; + if (!picked?.id) { + throw new Error(`MindSpace 没有可上传的分类（需要 ${UPLOADABLE_CATEGORY_CODES.join(' / ')}）`); + } + return picked.id; +} + +/** + * Renders a deterministic, visually distinct JPEG so a vision model can tell the + * uploads apart. Shapes and colors only — no text, because the rasterizer has no + * guaranteed CJK font. + */ +async function renderScenarioImage({ background = '#1e3a8a', shape = 'circle', accent = '#fbbf24' }) { + const { default: sharp } = await import('sharp'); + const shapes = { + circle: '', + square: '', + triangle: '', + diamond: '', + }; + const figure = (shapes[shape] ?? shapes.circle).replace('ACCENT', accent); + const svg = `` + + `${figure}`; + return sharp(Buffer.from(svg)).jpeg({ quality: 90 }).toBuffer(); +} + +export async function uploadImageAssets( + baseUrl, + cookie, + { images = [], categoryCode = 'oa' } = {}, + reporter, +) { + const categoryId = await resolveUploadCategoryId(baseUrl, cookie, categoryCode); + const assets = []; + + for (const [index, spec] of images.entries()) { + const buffer = await renderScenarioImage(spec ?? {}); + const filename = spec?.filename ?? `scenario-image-${index + 1}.jpg`; + + const createResponse = await fetch(`${baseUrl}/api/mindspace/v1/uploads`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', Cookie: cookie }, + body: JSON.stringify({ + category_id: categoryId, + filename, + size_bytes: buffer.length, + declared_mime_type: 'image/jpeg', + }), + }); + if (!createResponse.ok) { + throw new Error(`创建上传失败 ${createResponse.status}: ${await createResponse.text()}`); + } + const upload = (await createResponse.json())?.data; + const uploadUrl = new URL(upload.uploadUrl, baseUrl).href; + + const putResponse = await fetch(uploadUrl, { + method: 'PUT', + headers: { 'Content-Type': 'application/octet-stream', Cookie: cookie }, + body: buffer, + }); + if (!putResponse.ok) { + throw new Error(`上传内容失败 ${putResponse.status}: ${await putResponse.text()}`); + } + + const completeResponse = await fetch( + `${baseUrl}/api/mindspace/v1/uploads/${encodeURIComponent(upload.id)}/complete`, + { + method: 'POST', + headers: { 'Content-Type': 'application/json', Cookie: cookie }, + body: JSON.stringify({}), + }, + ); + if (!completeResponse.ok) { + throw new Error(`完成上传失败 ${completeResponse.status}: ${await completeResponse.text()}`); + } + assets.push((await completeResponse.json())?.data); + } + + reporter?.pass('上传图片', `${assets.length} 张 → ${assets.map((a) => a.id).join(', ')}`); + return assets; +} + export async function verifyPageAccess({ baseUrl, cookie, @@ -303,6 +407,7 @@ export async function verifyPageAccess({ replyText, htmlBefore = [], expect = {}, + uploadedAssetIds = [], reporter, }) { const keywords = expect.keywords ?? []; @@ -354,6 +459,19 @@ export async function verifyPageAccess({ reporter.fail('页面元数据', '缺少 mindspace-cover'); return false; } + + if (expect.requireUploadedImages) { + const embedded = uploadedAssetIds.filter((assetId) => assetId && html.includes(assetId)); + const required = expect.minUploadedImages ?? uploadedAssetIds.length; + if (embedded.length < required) { + reporter.fail( + '页面嵌入上传图片', + `命中 ${embedded.length}/${uploadedAssetIds.length} 张，要求至少 ${required} 张`, + ); + return false; + } + reporter.pass('页面嵌入上传图片', `${embedded.length}/${uploadedAssetIds.length} 张`); + } } return true;