{ "evalId": "eval-m02-2026-06-23T06:59:33", "results": { "version": 3, "timestamp": "2026-06-23T06:59:33.793Z", "prompts": [ { "id": "7e756b499a1da155962cf88585b671e3ddacc5edf410c782ea1dbff2b92d0a70", "raw": "{{customerInput}}", "label": "{{customerInput}}", "provider": "levyo-benchmark-v0-artifact-replay", "metrics": { "score": 50, "testPassCount": 50, "testFailCount": 0, "testErrorCount": 0, "assertPassCount": 50, "assertFailCount": 0, "totalLatencyMs": 16, "tokenUsage": { "prompt": 0, "completion": 0, "cached": 0, "total": 0, "numRequests": 50, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 }, "assertions": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 0, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 } } }, "namedScores": {}, "namedScoresCount": {}, "namedScoreWeights": {}, "cost": 0 } } ], "results": [ { "cost": 0, "gradingResult": { "pass": true, "score": 1, "reason": "All assertions passed", "namedScores": {}, "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 0 }, "componentResults": [ { "pass": true, "score": 1, "reason": "Assertion passed", "assertion": { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_001\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } } ] }, "id": "cd3a83b6-df30-4d95-9915-b8cfbcaf81ef", "latencyMs": 2, "namedScores": {}, "prompt": { "raw": "Need a stag do shirt for Ben's last ride, Prague 2026, make it look chaotic but not too cringe", "label": "{{customerInput}}", "config": {} }, "promptId": "7e756b499a1da155962cf88585b671e3ddacc5edf410c782ea1dbff2b92d0a70", "promptIdx": 0, "provider": { "id": "file://./providers/benchmark-v0-replay-provider.mjs", "label": "levyo-benchmark-v0-artifact-replay" }, "response": { "output": "{\"ok\":true,\"schemaVersion\":\"levyo_promptfoo_benchmark_v0_replay_result_v1\",\"caseId\":\"lvyo_bv0_candidate_001\",\"promptEcho\":\"Need a stag do shirt for Ben's last ride, Prague 2026, make it look chaotic but not too cringe\",\"status\":\"success\",\"modality\":\"text_only\",\"caseRole\":\"text_core\",\"cluster\":\"stag weekend\",\"runtimeProfile\":\"internal_codex\",\"imageProvider\":\"codex_account_ui\",\"lockRef\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_001/artifact-lock.json\",\"lockSha256\":\"5ae07bb36e59cfbd4a347871855d90e91232966463d160cc5ff556512879e3f9\",\"final\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_001/artifact/final.png\",\"sha256\":\"984c26c5c2829d294f5b7ed1d461ee05f60931b7b69e341363cb7e2c6a4071ff\"},\"mockup\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_001/artifact/mockup.png\",\"sha256\":\"bfc654fdfab199b7bb107ec752f0ae8924b2d429aa2fcb08b09642bd482076b9\",\"publicSrc\":\"assets/lvyo_bv0_candidate_001_m4_mockup.webp\"},\"referenceImage\":null,\"noSpendReplay\":true,\"customerLiveFlowChanged\":false}", "images": [ { "mimeType": "image/webp", "blobRef": { "uri": "promptfoo://blob/4e5975fd08962e773825b3216bb5412a0d4d19ac1ffdde6396dfffd2fe526d1d", "hash": "4e5975fd08962e773825b3216bb5412a0d4d19ac1ffdde6396dfffd2fe526d1d", "mimeType": "image/webp", "sizeBytes": 39966, "provider": "filesystem" } } ], "metadata": { "caseId": "lvyo_bv0_candidate_001", "modality": "text_only", "cluster": "stag weekend", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_001/artifact-lock.json", "finalSha256": "984c26c5c2829d294f5b7ed1d461ee05f60931b7b69e341363cb7e2c6a4071ff", "mockupSha256": "bfc654fdfab199b7bb107ec752f0ae8924b2d429aa2fcb08b09642bd482076b9", "publicMockup": "assets/lvyo_bv0_candidate_001_m4_mockup.webp", "noSpendReplay": true } }, "score": 1, "success": true, "testCase": { "description": "01 text_only stag weekend lvyo_bv0_candidate_001", "vars": { "caseId": "lvyo_bv0_candidate_001", "customerInput": "Need a stag do shirt for Ben's last ride, Prague 2026, make it look chaotic but not too cringe", "modality": "text_only", "cluster": "stag weekend" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_001\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ], "options": {}, "metadata": {} }, "testIdx": 0, "tokenUsage": { "prompt": 0, "completion": 0, "cached": 0, "total": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 }, "assertions": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 } } }, "vars": { "caseId": "lvyo_bv0_candidate_001", "customerInput": "Need a stag do shirt for Ben's last ride, Prague 2026, make it look chaotic but not too cringe", "modality": "text_only", "cluster": "stag weekend" }, "metadata": { "caseId": "lvyo_bv0_candidate_001", "modality": "text_only", "cluster": "stag weekend", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_001/artifact-lock.json", "finalSha256": "984c26c5c2829d294f5b7ed1d461ee05f60931b7b69e341363cb7e2c6a4071ff", "mockupSha256": "bfc654fdfab199b7bb107ec752f0ae8924b2d429aa2fcb08b09642bd482076b9", "publicMockup": "assets/lvyo_bv0_candidate_001_m4_mockup.webp", "noSpendReplay": true, "_promptfooFileMetadata": {} }, "failureReason": 0 }, { "cost": 0, "gradingResult": { "pass": true, "score": 1, "reason": "All assertions passed", "namedScores": {}, "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 0 }, "componentResults": [ { "pass": true, "score": 1, "reason": "Assertion passed", "assertion": { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_002\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } } ] }, "id": "28758845-547b-4ee8-b4e0-0ecee05e7012", "latencyMs": 0, "namedScores": {}, "prompt": { "raw": "Bride tribe tee for Molly's hen in Brighton. Pink, sparkly, a bit cheeky. Names are Molly, Abi, Rach and Tilly", "label": "{{customerInput}}", "config": {} }, "promptId": "7e756b499a1da155962cf88585b671e3ddacc5edf410c782ea1dbff2b92d0a70", "promptIdx": 0, "provider": { "id": "file://./providers/benchmark-v0-replay-provider.mjs", "label": "levyo-benchmark-v0-artifact-replay" }, "response": { "output": "{\"ok\":true,\"schemaVersion\":\"levyo_promptfoo_benchmark_v0_replay_result_v1\",\"caseId\":\"lvyo_bv0_candidate_002\",\"promptEcho\":\"Bride tribe tee for Molly's hen in Brighton. Pink, sparkly, a bit cheeky. Names are Molly, Abi, Rach and Tilly\",\"status\":\"success\",\"modality\":\"text_only\",\"caseRole\":\"text_core\",\"cluster\":\"hen party\",\"runtimeProfile\":\"internal_codex\",\"imageProvider\":\"codex_account_ui\",\"lockRef\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_002/artifact-lock.json\",\"lockSha256\":\"8f8dd2eda3bc7f394e27c1a454ce6ddafe9bd7e5e71ed7ff565caed46d4898e2\",\"final\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_002/artifact/final.png\",\"sha256\":\"472e835775f6c36f0cad4aefa922924bce8ca641e3aba7b2f2b7a69cb4a19ebb\"},\"mockup\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_002/artifact/mockup.png\",\"sha256\":\"0981393fe0dd46af6fe51766ea0c58fa14513704977c372ab80b4f54b49e82a2\",\"publicSrc\":\"assets/lvyo_bv0_candidate_002_m4_mockup.webp\"},\"referenceImage\":null,\"noSpendReplay\":true,\"customerLiveFlowChanged\":false}", "images": [ { "mimeType": "image/webp", "blobRef": { "uri": "promptfoo://blob/5a06f224bc5893fdd0e6cfc5f3b41319333c241355a611afee1eaef99e2dccf5", "hash": "5a06f224bc5893fdd0e6cfc5f3b41319333c241355a611afee1eaef99e2dccf5", "mimeType": "image/webp", "sizeBytes": 42602, "provider": "filesystem" } } ], "metadata": { "caseId": "lvyo_bv0_candidate_002", "modality": "text_only", "cluster": "hen party", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_002/artifact-lock.json", "finalSha256": "472e835775f6c36f0cad4aefa922924bce8ca641e3aba7b2f2b7a69cb4a19ebb", "mockupSha256": "0981393fe0dd46af6fe51766ea0c58fa14513704977c372ab80b4f54b49e82a2", "publicMockup": "assets/lvyo_bv0_candidate_002_m4_mockup.webp", "noSpendReplay": true } }, "score": 1, "success": true, "testCase": { "description": "02 text_only hen party lvyo_bv0_candidate_002", "vars": { "caseId": "lvyo_bv0_candidate_002", "customerInput": "Bride tribe tee for Molly's hen in Brighton. Pink, sparkly, a bit cheeky. Names are Molly, Abi, Rach and Tilly", "modality": "text_only", "cluster": "hen party" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_002\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ], "options": {}, "metadata": {} }, "testIdx": 1, "tokenUsage": { "prompt": 0, "completion": 0, "cached": 0, "total": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 }, "assertions": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 } } }, "vars": { "caseId": "lvyo_bv0_candidate_002", "customerInput": "Bride tribe tee for Molly's hen in Brighton. Pink, sparkly, a bit cheeky. Names are Molly, Abi, Rach and Tilly", "modality": "text_only", "cluster": "hen party" }, "metadata": { "caseId": "lvyo_bv0_candidate_002", "modality": "text_only", "cluster": "hen party", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_002/artifact-lock.json", "finalSha256": "472e835775f6c36f0cad4aefa922924bce8ca641e3aba7b2f2b7a69cb4a19ebb", "mockupSha256": "0981393fe0dd46af6fe51766ea0c58fa14513704977c372ab80b4f54b49e82a2", "publicMockup": "assets/lvyo_bv0_candidate_002_m4_mockup.webp", "noSpendReplay": true, "_promptfooFileMetadata": {} }, "failureReason": 0 }, { "cost": 0, "gradingResult": { "pass": true, "score": 1, "reason": "All assertions passed", "namedScores": {}, "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 0 }, "componentResults": [ { "pass": true, "score": 1, "reason": "Assertion passed", "assertion": { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_003\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } } ] }, "id": "d261f159-70e5-465e-a315-b75d51db6a9a", "latencyMs": 0, "namedScores": {}, "prompt": { "raw": "30th birthday shirt that says Still ID'd, Unfortunately Thriving. For a night out in Manchester", "label": "{{customerInput}}", "config": {} }, "promptId": "7e756b499a1da155962cf88585b671e3ddacc5edf410c782ea1dbff2b92d0a70", "promptIdx": 0, "provider": { "id": "file://./providers/benchmark-v0-replay-provider.mjs", "label": "levyo-benchmark-v0-artifact-replay" }, "response": { "output": "{\"ok\":true,\"schemaVersion\":\"levyo_promptfoo_benchmark_v0_replay_result_v1\",\"caseId\":\"lvyo_bv0_candidate_003\",\"promptEcho\":\"30th birthday shirt that says Still ID'd, Unfortunately Thriving. For a night out in Manchester\",\"status\":\"success\",\"modality\":\"text_only\",\"caseRole\":\"text_core\",\"cluster\":\"birthday party\",\"runtimeProfile\":\"internal_codex\",\"imageProvider\":\"codex_account_ui\",\"lockRef\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_003/artifact-lock.json\",\"lockSha256\":\"ac08f343f889d2a377b6786fa18943e44f9b3b64770dd7ab4a1c8504717109b3\",\"final\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_003/artifact/final.png\",\"sha256\":\"2f42447c01998ba9d04c978b0bb8cf2cd072de7894e9d1c367ef2464c495dcfc\"},\"mockup\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_003/artifact/mockup.png\",\"sha256\":\"586d823f4bea4bc1c1cb46df96e4075214f6893e7b83821943e63b2dcfd3e856\",\"publicSrc\":\"assets/lvyo_bv0_candidate_003_m4_mockup.webp\"},\"referenceImage\":null,\"noSpendReplay\":true,\"customerLiveFlowChanged\":false}", "images": [ { "mimeType": "image/webp", "blobRef": { "uri": "promptfoo://blob/34ef8869752048521f01bcde99d32f4fe970184d2d8647e35a3c1ceaf09e2ce9", "hash": "34ef8869752048521f01bcde99d32f4fe970184d2d8647e35a3c1ceaf09e2ce9", "mimeType": "image/webp", "sizeBytes": 43356, "provider": "filesystem" } } ], "metadata": { "caseId": "lvyo_bv0_candidate_003", "modality": "text_only", "cluster": "birthday party", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_003/artifact-lock.json", "finalSha256": "2f42447c01998ba9d04c978b0bb8cf2cd072de7894e9d1c367ef2464c495dcfc", "mockupSha256": "586d823f4bea4bc1c1cb46df96e4075214f6893e7b83821943e63b2dcfd3e856", "publicMockup": "assets/lvyo_bv0_candidate_003_m4_mockup.webp", "noSpendReplay": true } }, "score": 1, "success": true, "testCase": { "description": "03 text_only birthday party lvyo_bv0_candidate_003", "vars": { "caseId": "lvyo_bv0_candidate_003", "customerInput": "30th birthday shirt that says Still ID'd, Unfortunately Thriving. For a night out in Manchester", "modality": "text_only", "cluster": "birthday party" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_003\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ], "options": {}, "metadata": {} }, "testIdx": 2, "tokenUsage": { "prompt": 0, "completion": 0, "cached": 0, "total": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 }, "assertions": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 } } }, "vars": { "caseId": "lvyo_bv0_candidate_003", "customerInput": "30th birthday shirt that says Still ID'd, Unfortunately Thriving. For a night out in Manchester", "modality": "text_only", "cluster": "birthday party" }, "metadata": { "caseId": "lvyo_bv0_candidate_003", "modality": "text_only", "cluster": "birthday party", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_003/artifact-lock.json", "finalSha256": "2f42447c01998ba9d04c978b0bb8cf2cd072de7894e9d1c367ef2464c495dcfc", "mockupSha256": "586d823f4bea4bc1c1cb46df96e4075214f6893e7b83821943e63b2dcfd3e856", "publicMockup": "assets/lvyo_bv0_candidate_003_m4_mockup.webp", "noSpendReplay": true, "_promptfooFileMetadata": {} }, "failureReason": 0 }, { "cost": 0, "gradingResult": { "pass": true, "score": 1, "reason": "All assertions passed", "namedScores": {}, "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 0 }, "componentResults": [ { "pass": true, "score": 1, "reason": "Assertion passed", "assertion": { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_004\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } } ] }, "id": "36c67414-f53a-4e2c-8a9b-419a62728dab", "latencyMs": 0, "namedScores": {}, "prompt": { "raw": "Make a funny graduation T shirt for my flatmates: survived uni, barely. Leeds 2026", "label": "{{customerInput}}", "config": {} }, "promptId": "7e756b499a1da155962cf88585b671e3ddacc5edf410c782ea1dbff2b92d0a70", "promptIdx": 0, "provider": { "id": "file://./providers/benchmark-v0-replay-provider.mjs", "label": "levyo-benchmark-v0-artifact-replay" }, "response": { "output": "{\"ok\":true,\"schemaVersion\":\"levyo_promptfoo_benchmark_v0_replay_result_v1\",\"caseId\":\"lvyo_bv0_candidate_004\",\"promptEcho\":\"Make a funny graduation T shirt for my flatmates: survived uni, barely. Leeds 2026\",\"status\":\"success\",\"modality\":\"text_only\",\"caseRole\":\"text_core\",\"cluster\":\"graduation\",\"runtimeProfile\":\"internal_codex\",\"imageProvider\":\"codex_account_ui\",\"lockRef\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_004/artifact-lock.json\",\"lockSha256\":\"e88175a78980cc1c64ab0fde67acfe24ea9fb10b14a4d69dc4b789eec3e78e1b\",\"final\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_004/artifact/final.png\",\"sha256\":\"5f0fbd4c34be96fda2fe4b62c2de1d46ced339a1801d76f7f61d5274cdc710c8\"},\"mockup\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_004/artifact/mockup.png\",\"sha256\":\"aac59e64a79fe62b8025611d226cf1ba85f3dd50071dab0f19b66d4305204136\",\"publicSrc\":\"assets/lvyo_bv0_candidate_004_m4_mockup.webp\"},\"referenceImage\":null,\"noSpendReplay\":true,\"customerLiveFlowChanged\":false}", "images": [ { "mimeType": "image/webp", "blobRef": { "uri": "promptfoo://blob/621e29b79ab3b7c2471cd11f93737abececbc0ff672d5fb7f0e9007e43961b62", "hash": "621e29b79ab3b7c2471cd11f93737abececbc0ff672d5fb7f0e9007e43961b62", "mimeType": "image/webp", "sizeBytes": 40302, "provider": "filesystem" } } ], "metadata": { "caseId": "lvyo_bv0_candidate_004", "modality": "text_only", "cluster": "graduation", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_004/artifact-lock.json", "finalSha256": "5f0fbd4c34be96fda2fe4b62c2de1d46ced339a1801d76f7f61d5274cdc710c8", "mockupSha256": "aac59e64a79fe62b8025611d226cf1ba85f3dd50071dab0f19b66d4305204136", "publicMockup": "assets/lvyo_bv0_candidate_004_m4_mockup.webp", "noSpendReplay": true } }, "score": 1, "success": true, "testCase": { "description": "04 text_only graduation lvyo_bv0_candidate_004", "vars": { "caseId": "lvyo_bv0_candidate_004", "customerInput": "Make a funny graduation T shirt for my flatmates: survived uni, barely. Leeds 2026", "modality": "text_only", "cluster": "graduation" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_004\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ], "options": {}, "metadata": {} }, "testIdx": 3, "tokenUsage": { "prompt": 0, "completion": 0, "cached": 0, "total": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 }, "assertions": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 } } }, "vars": { "caseId": "lvyo_bv0_candidate_004", "customerInput": "Make a funny graduation T shirt for my flatmates: survived uni, barely. Leeds 2026", "modality": "text_only", "cluster": "graduation" }, "metadata": { "caseId": "lvyo_bv0_candidate_004", "modality": "text_only", "cluster": "graduation", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_004/artifact-lock.json", "finalSha256": "5f0fbd4c34be96fda2fe4b62c2de1d46ced339a1801d76f7f61d5274cdc710c8", "mockupSha256": "aac59e64a79fe62b8025611d226cf1ba85f3dd50071dab0f19b66d4305204136", "publicMockup": "assets/lvyo_bv0_candidate_004_m4_mockup.webp", "noSpendReplay": true, "_promptfooFileMetadata": {} }, "failureReason": 0 }, { "cost": 0, "gradingResult": { "pass": true, "score": 1, "reason": "All assertions passed", "namedScores": {}, "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 0 }, "componentResults": [ { "pass": true, "score": 1, "reason": "Assertion passed", "assertion": { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_005\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } } ] }, "id": "95db1633-0b51-4759-95ec-0a54886f9866", "latencyMs": 1, "namedScores": {}, "prompt": { "raw": "Ibiza girls trip top, four of us, classy front but silly back, something about lost sunglasses and bad decisions", "label": "{{customerInput}}", "config": {} }, "promptId": "7e756b499a1da155962cf88585b671e3ddacc5edf410c782ea1dbff2b92d0a70", "promptIdx": 0, "provider": { "id": "file://./providers/benchmark-v0-replay-provider.mjs", "label": "levyo-benchmark-v0-artifact-replay" }, "response": { "output": "{\"ok\":true,\"schemaVersion\":\"levyo_promptfoo_benchmark_v0_replay_result_v1\",\"caseId\":\"lvyo_bv0_candidate_005\",\"promptEcho\":\"Ibiza girls trip top, four of us, classy front but silly back, something about lost sunglasses and bad decisions\",\"status\":\"success\",\"modality\":\"text_only\",\"caseRole\":\"text_core\",\"cluster\":\"girls trip\",\"runtimeProfile\":\"internal_codex\",\"imageProvider\":\"codex_account_ui\",\"lockRef\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_005/artifact-lock.json\",\"lockSha256\":\"7bf7c1167879c52e84d293727c4929b2a2d3c24e9cc99497d3893da079955fe4\",\"final\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_005/artifact/final.png\",\"sha256\":\"2382d9f0f3b4d3f617e7c1c3e5ce5ec1f4f2a3a574124d68816b1fa3afc708e4\"},\"mockup\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_005/artifact/mockup.png\",\"sha256\":\"e5c6e1d53f4f648d7aaa8a630cc093150796502c9078232aa79694ed36480145\",\"publicSrc\":\"assets/lvyo_bv0_candidate_005_m4_mockup.webp\"},\"referenceImage\":null,\"noSpendReplay\":true,\"customerLiveFlowChanged\":false}", "images": [ { "mimeType": "image/webp", "blobRef": { "uri": "promptfoo://blob/85f668bb1b17d44634217f3272ea4b906216e1aa870fdd09d1b8038b3ab6096e", "hash": "85f668bb1b17d44634217f3272ea4b906216e1aa870fdd09d1b8038b3ab6096e", "mimeType": "image/webp", "sizeBytes": 39944, "provider": "filesystem" } } ], "metadata": { "caseId": "lvyo_bv0_candidate_005", "modality": "text_only", "cluster": "girls trip", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_005/artifact-lock.json", "finalSha256": "2382d9f0f3b4d3f617e7c1c3e5ce5ec1f4f2a3a574124d68816b1fa3afc708e4", "mockupSha256": "e5c6e1d53f4f648d7aaa8a630cc093150796502c9078232aa79694ed36480145", "publicMockup": "assets/lvyo_bv0_candidate_005_m4_mockup.webp", "noSpendReplay": true } }, "score": 1, "success": true, "testCase": { "description": "05 text_only girls trip lvyo_bv0_candidate_005", "vars": { "caseId": "lvyo_bv0_candidate_005", "customerInput": "Ibiza girls trip top, four of us, classy front but silly back, something about lost sunglasses and bad decisions", "modality": "text_only", "cluster": "girls trip" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_005\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ], "options": {}, "metadata": {} }, "testIdx": 4, "tokenUsage": { "prompt": 0, "completion": 0, "cached": 0, "total": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 }, "assertions": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 } } }, "vars": { "caseId": "lvyo_bv0_candidate_005", "customerInput": "Ibiza girls trip top, four of us, classy front but silly back, something about lost sunglasses and bad decisions", "modality": "text_only", "cluster": "girls trip" }, "metadata": { "caseId": "lvyo_bv0_candidate_005", "modality": "text_only", "cluster": "girls trip", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_005/artifact-lock.json", "finalSha256": "2382d9f0f3b4d3f617e7c1c3e5ce5ec1f4f2a3a574124d68816b1fa3afc708e4", "mockupSha256": "e5c6e1d53f4f648d7aaa8a630cc093150796502c9078232aa79694ed36480145", "publicMockup": "assets/lvyo_bv0_candidate_005_m4_mockup.webp", "noSpendReplay": true, "_promptfooFileMetadata": {} }, "failureReason": 0 }, { "cost": 0, "gradingResult": { "pass": true, "score": 1, "reason": "All assertions passed", "namedScores": {}, "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 0 }, "componentResults": [ { "pass": true, "score": 1, "reason": "Assertion passed", "assertion": { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_006\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } } ] }, "id": "c746134c-0b38-4dbb-8d06-b8f8e0c3b2b3", "latencyMs": 0, "namedScores": {}, "prompt": { "raw": "Lads holiday tees for Ayia Napa. Front: Team Terrible Ideas. Back can have fake tour dates please", "label": "{{customerInput}}", "config": {} }, "promptId": "7e756b499a1da155962cf88585b671e3ddacc5edf410c782ea1dbff2b92d0a70", "promptIdx": 0, "provider": { "id": "file://./providers/benchmark-v0-replay-provider.mjs", "label": "levyo-benchmark-v0-artifact-replay" }, "response": { "output": "{\"ok\":true,\"schemaVersion\":\"levyo_promptfoo_benchmark_v0_replay_result_v1\",\"caseId\":\"lvyo_bv0_candidate_006\",\"promptEcho\":\"Lads holiday tees for Ayia Napa. Front: Team Terrible Ideas. Back can have fake tour dates please\",\"status\":\"success\",\"modality\":\"text_only\",\"caseRole\":\"text_core\",\"cluster\":\"lads holiday\",\"runtimeProfile\":\"internal_codex\",\"imageProvider\":\"codex_account_ui\",\"lockRef\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_006/artifact-lock.json\",\"lockSha256\":\"64eb432524b525c8bf182fa9daec6954c3e58a6013f53f420f29d3938718063c\",\"final\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_006/artifact/final.png\",\"sha256\":\"aee57f031e2baa8a317f3d2794b8af1d943719d94973ff63e756d5978596571c\"},\"mockup\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_006/artifact/mockup.png\",\"sha256\":\"160ffadbfd184a8c7c3c9b7c4d8d581bea35532f671a404813d6a7d15983e1e9\",\"publicSrc\":\"assets/lvyo_bv0_candidate_006_m4_mockup.webp\"},\"referenceImage\":null,\"noSpendReplay\":true,\"customerLiveFlowChanged\":false}", "images": [ { "mimeType": "image/webp", "blobRef": { "uri": "promptfoo://blob/b40193589b97007ae2bb7f2811145d40adad9158f4ed9f4289683ec454459c61", "hash": "b40193589b97007ae2bb7f2811145d40adad9158f4ed9f4289683ec454459c61", "mimeType": "image/webp", "sizeBytes": 45972, "provider": "filesystem" } } ], "metadata": { "caseId": "lvyo_bv0_candidate_006", "modality": "text_only", "cluster": "lads holiday", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_006/artifact-lock.json", "finalSha256": "aee57f031e2baa8a317f3d2794b8af1d943719d94973ff63e756d5978596571c", "mockupSha256": "160ffadbfd184a8c7c3c9b7c4d8d581bea35532f671a404813d6a7d15983e1e9", "publicMockup": "assets/lvyo_bv0_candidate_006_m4_mockup.webp", "noSpendReplay": true } }, "score": 1, "success": true, "testCase": { "description": "06 text_only lads holiday lvyo_bv0_candidate_006", "vars": { "caseId": "lvyo_bv0_candidate_006", "customerInput": "Lads holiday tees for Ayia Napa. Front: Team Terrible Ideas. Back can have fake tour dates please", "modality": "text_only", "cluster": "lads holiday" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_006\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ], "options": {}, "metadata": {} }, "testIdx": 5, "tokenUsage": { "prompt": 0, "completion": 0, "cached": 0, "total": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 }, "assertions": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 } } }, "vars": { "caseId": "lvyo_bv0_candidate_006", "customerInput": "Lads holiday tees for Ayia Napa. Front: Team Terrible Ideas. Back can have fake tour dates please", "modality": "text_only", "cluster": "lads holiday" }, "metadata": { "caseId": "lvyo_bv0_candidate_006", "modality": "text_only", "cluster": "lads holiday", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_006/artifact-lock.json", "finalSha256": "aee57f031e2baa8a317f3d2794b8af1d943719d94973ff63e756d5978596571c", "mockupSha256": "160ffadbfd184a8c7c3c9b7c4d8d581bea35532f671a404813d6a7d15983e1e9", "publicMockup": "assets/lvyo_bv0_candidate_006_m4_mockup.webp", "noSpendReplay": true, "_promptfooFileMetadata": {} }, "failureReason": 0 }, { "cost": 0, "gradingResult": { "pass": true, "score": 1, "reason": "All assertions passed", "namedScores": {}, "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 0 }, "componentResults": [ { "pass": true, "score": 1, "reason": "Assertion passed", "assertion": { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_007\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } } ] }, "id": "0bf784e7-65f8-4fc7-88f2-4a4410e12d5d", "latencyMs": 0, "namedScores": {}, "prompt": { "raw": "Road trip shirt for Scotland NC500, two mates and one tiny car, please make it outdoorsy but funny", "label": "{{customerInput}}", "config": {} }, "promptId": "7e756b499a1da155962cf88585b671e3ddacc5edf410c782ea1dbff2b92d0a70", "promptIdx": 0, "provider": { "id": "file://./providers/benchmark-v0-replay-provider.mjs", "label": "levyo-benchmark-v0-artifact-replay" }, "response": { "output": "{\"ok\":true,\"schemaVersion\":\"levyo_promptfoo_benchmark_v0_replay_result_v1\",\"caseId\":\"lvyo_bv0_candidate_007\",\"promptEcho\":\"Road trip shirt for Scotland NC500, two mates and one tiny car, please make it outdoorsy but funny\",\"status\":\"success\",\"modality\":\"text_only\",\"caseRole\":\"text_core\",\"cluster\":\"road trip\",\"runtimeProfile\":\"internal_codex\",\"imageProvider\":\"codex_account_ui\",\"lockRef\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_007/artifact-lock.json\",\"lockSha256\":\"78774e55bde7dba9c8062585be9f3ea63d85ddc10797cd18737aacd6d72429b2\",\"final\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_007/artifact/final.png\",\"sha256\":\"768ce746b98f54f742b79a8801e3402218ae1df0c1b1c4f358d9c16e23ec244c\"},\"mockup\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_007/artifact/mockup.png\",\"sha256\":\"ccd5960a94ab90b15e90e64acd7a68bf8bc8c9568c590fac52831798b27f1cc4\",\"publicSrc\":\"assets/lvyo_bv0_candidate_007_m4_mockup.webp\"},\"referenceImage\":null,\"noSpendReplay\":true,\"customerLiveFlowChanged\":false}", "images": [ { "mimeType": "image/webp", "blobRef": { "uri": "promptfoo://blob/dfcd02f491337dfcc81defa6c4d6b332072c99ba2b19586fd96585f83d3c6959", "hash": "dfcd02f491337dfcc81defa6c4d6b332072c99ba2b19586fd96585f83d3c6959", "mimeType": "image/webp", "sizeBytes": 49448, "provider": "filesystem" } } ], "metadata": { "caseId": "lvyo_bv0_candidate_007", "modality": "text_only", "cluster": "road trip", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_007/artifact-lock.json", "finalSha256": "768ce746b98f54f742b79a8801e3402218ae1df0c1b1c4f358d9c16e23ec244c", "mockupSha256": "ccd5960a94ab90b15e90e64acd7a68bf8bc8c9568c590fac52831798b27f1cc4", "publicMockup": "assets/lvyo_bv0_candidate_007_m4_mockup.webp", "noSpendReplay": true } }, "score": 1, "success": true, "testCase": { "description": "07 text_only road trip lvyo_bv0_candidate_007", "vars": { "caseId": "lvyo_bv0_candidate_007", "customerInput": "Road trip shirt for Scotland NC500, two mates and one tiny car, please make it outdoorsy but funny", "modality": "text_only", "cluster": "road trip" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_007\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ], "options": {}, "metadata": {} }, "testIdx": 6, "tokenUsage": { "prompt": 0, "completion": 0, "cached": 0, "total": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 }, "assertions": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 } } }, "vars": { "caseId": "lvyo_bv0_candidate_007", "customerInput": "Road trip shirt for Scotland NC500, two mates and one tiny car, please make it outdoorsy but funny", "modality": "text_only", "cluster": "road trip" }, "metadata": { "caseId": "lvyo_bv0_candidate_007", "modality": "text_only", "cluster": "road trip", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_007/artifact-lock.json", "finalSha256": "768ce746b98f54f742b79a8801e3402218ae1df0c1b1c4f358d9c16e23ec244c", "mockupSha256": "ccd5960a94ab90b15e90e64acd7a68bf8bc8c9568c590fac52831798b27f1cc4", "publicMockup": "assets/lvyo_bv0_candidate_007_m4_mockup.webp", "noSpendReplay": true, "_promptfooFileMetadata": {} }, "failureReason": 0 }, { "cost": 0, "gradingResult": { "pass": true, "score": 1, "reason": "All assertions passed", "namedScores": {}, "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 0 }, "componentResults": [ { "pass": true, "score": 1, "reason": "Assertion passed", "assertion": { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_008\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } } ] }, "id": "ab4a9d02-f475-4cba-a49f-1c114911b71f", "latencyMs": 1, "namedScores": {}, "prompt": { "raw": "Ski trip hoodie design for Val Thorens, slogan: Apres First, Questions Later. Add mountains and beers", "label": "{{customerInput}}", "config": {} }, "promptId": "7e756b499a1da155962cf88585b671e3ddacc5edf410c782ea1dbff2b92d0a70", "promptIdx": 0, "provider": { "id": "file://./providers/benchmark-v0-replay-provider.mjs", "label": "levyo-benchmark-v0-artifact-replay" }, "response": { "output": "{\"ok\":true,\"schemaVersion\":\"levyo_promptfoo_benchmark_v0_replay_result_v1\",\"caseId\":\"lvyo_bv0_candidate_008\",\"promptEcho\":\"Ski trip hoodie design for Val Thorens, slogan: Apres First, Questions Later. Add mountains and beers\",\"status\":\"success\",\"modality\":\"text_only\",\"caseRole\":\"text_core\",\"cluster\":\"ski trip\",\"runtimeProfile\":\"internal_codex\",\"imageProvider\":\"codex_account_ui\",\"lockRef\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_008/artifact-lock.json\",\"lockSha256\":\"f74fed5ebb263cd730b18f59f96622354e69560a41fb95b6eb0f602e16a9a2ae\",\"final\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_008/artifact/final.png\",\"sha256\":\"bf5e9529c4f6257c804fec1db67868fca5eb27d68fadda33d23abfd3c8d357e4\"},\"mockup\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_008/artifact/mockup.png\",\"sha256\":\"109d739cd3b203a5abb8dcb68cee5eef1803f1fbbf1beff9087981724b91d07b\",\"publicSrc\":\"assets/lvyo_bv0_candidate_008_m4_mockup.webp\"},\"referenceImage\":null,\"noSpendReplay\":true,\"customerLiveFlowChanged\":false}", "images": [ { "mimeType": "image/webp", "blobRef": { "uri": "promptfoo://blob/0e4806ed3bbbc33a9d0ed3007060ae387b2ca148b8d354786b86682f6a160f9d", "hash": "0e4806ed3bbbc33a9d0ed3007060ae387b2ca148b8d354786b86682f6a160f9d", "mimeType": "image/webp", "sizeBytes": 52862, "provider": "filesystem" } } ], "metadata": { "caseId": "lvyo_bv0_candidate_008", "modality": "text_only", "cluster": "ski trip", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_008/artifact-lock.json", "finalSha256": "bf5e9529c4f6257c804fec1db67868fca5eb27d68fadda33d23abfd3c8d357e4", "mockupSha256": "109d739cd3b203a5abb8dcb68cee5eef1803f1fbbf1beff9087981724b91d07b", "publicMockup": "assets/lvyo_bv0_candidate_008_m4_mockup.webp", "noSpendReplay": true } }, "score": 1, "success": true, "testCase": { "description": "08 text_only ski trip lvyo_bv0_candidate_008", "vars": { "caseId": "lvyo_bv0_candidate_008", "customerInput": "Ski trip hoodie design for Val Thorens, slogan: Apres First, Questions Later. Add mountains and beers", "modality": "text_only", "cluster": "ski trip" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_008\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ], "options": {}, "metadata": {} }, "testIdx": 7, "tokenUsage": { "prompt": 0, "completion": 0, "cached": 0, "total": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 }, "assertions": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 } } }, "vars": { "caseId": "lvyo_bv0_candidate_008", "customerInput": "Ski trip hoodie design for Val Thorens, slogan: Apres First, Questions Later. Add mountains and beers", "modality": "text_only", "cluster": "ski trip" }, "metadata": { "caseId": "lvyo_bv0_candidate_008", "modality": "text_only", "cluster": "ski trip", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_008/artifact-lock.json", "finalSha256": "bf5e9529c4f6257c804fec1db67868fca5eb27d68fadda33d23abfd3c8d357e4", "mockupSha256": "109d739cd3b203a5abb8dcb68cee5eef1803f1fbbf1beff9087981724b91d07b", "publicMockup": "assets/lvyo_bv0_candidate_008_m4_mockup.webp", "noSpendReplay": true, "_promptfooFileMetadata": {} }, "failureReason": 0 }, { "cost": 0, "gradingResult": { "pass": true, "score": 1, "reason": "All assertions passed", "namedScores": {}, "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 0 }, "componentResults": [ { "pass": true, "score": 1, "reason": "Assertion passed", "assertion": { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_009\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } } ] }, "id": "efd22056-163e-4d5c-ab81-56aa2dad3886", "latencyMs": 0, "namedScores": {}, "prompt": { "raw": "Golf weekend shirt for Dad's 60th. Bad backs, worse handicaps, Portugal tour 2026", "label": "{{customerInput}}", "config": {} }, "promptId": "7e756b499a1da155962cf88585b671e3ddacc5edf410c782ea1dbff2b92d0a70", "promptIdx": 0, "provider": { "id": "file://./providers/benchmark-v0-replay-provider.mjs", "label": "levyo-benchmark-v0-artifact-replay" }, "response": { "output": "{\"ok\":true,\"schemaVersion\":\"levyo_promptfoo_benchmark_v0_replay_result_v1\",\"caseId\":\"lvyo_bv0_candidate_009\",\"promptEcho\":\"Golf weekend shirt for Dad's 60th. Bad backs, worse handicaps, Portugal tour 2026\",\"status\":\"success\",\"modality\":\"text_only\",\"caseRole\":\"text_core\",\"cluster\":\"golf trip\",\"runtimeProfile\":\"internal_codex\",\"imageProvider\":\"codex_account_ui\",\"lockRef\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_009/artifact-lock.json\",\"lockSha256\":\"6bb7970dc860180fba127ec5d742da146699404207631cb654b6cb7bc42da0ee\",\"final\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_009/artifact/final.png\",\"sha256\":\"343e7dc503ac58823c143f2b3d038d3b0a3e0dcbd1cd1ea488c17d366e39d061\"},\"mockup\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_009/artifact/mockup.png\",\"sha256\":\"d78c5d404e60335ba6068636411c45536928a75100176e08a3cec7b3bf56ceca\",\"publicSrc\":\"assets/lvyo_bv0_candidate_009_m4_mockup.webp\"},\"referenceImage\":null,\"noSpendReplay\":true,\"customerLiveFlowChanged\":false}", "images": [ { "mimeType": "image/webp", "blobRef": { "uri": "promptfoo://blob/e5d510990675066c460a6136dcad7552850f6949ad1e452b343c880d599e76e7", "hash": "e5d510990675066c460a6136dcad7552850f6949ad1e452b343c880d599e76e7", "mimeType": "image/webp", "sizeBytes": 43320, "provider": "filesystem" } } ], "metadata": { "caseId": "lvyo_bv0_candidate_009", "modality": "text_only", "cluster": "golf trip", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_009/artifact-lock.json", "finalSha256": "343e7dc503ac58823c143f2b3d038d3b0a3e0dcbd1cd1ea488c17d366e39d061", "mockupSha256": "d78c5d404e60335ba6068636411c45536928a75100176e08a3cec7b3bf56ceca", "publicMockup": "assets/lvyo_bv0_candidate_009_m4_mockup.webp", "noSpendReplay": true } }, "score": 1, "success": true, "testCase": { "description": "09 text_only golf trip lvyo_bv0_candidate_009", "vars": { "caseId": "lvyo_bv0_candidate_009", "customerInput": "Golf weekend shirt for Dad's 60th. Bad backs, worse handicaps, Portugal tour 2026", "modality": "text_only", "cluster": "golf trip" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_009\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ], "options": {}, "metadata": {} }, "testIdx": 8, "tokenUsage": { "prompt": 0, "completion": 0, "cached": 0, "total": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 }, "assertions": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 } } }, "vars": { "caseId": "lvyo_bv0_candidate_009", "customerInput": "Golf weekend shirt for Dad's 60th. Bad backs, worse handicaps, Portugal tour 2026", "modality": "text_only", "cluster": "golf trip" }, "metadata": { "caseId": "lvyo_bv0_candidate_009", "modality": "text_only", "cluster": "golf trip", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_009/artifact-lock.json", "finalSha256": "343e7dc503ac58823c143f2b3d038d3b0a3e0dcbd1cd1ea488c17d366e39d061", "mockupSha256": "d78c5d404e60335ba6068636411c45536928a75100176e08a3cec7b3bf56ceca", "publicMockup": "assets/lvyo_bv0_candidate_009_m4_mockup.webp", "noSpendReplay": true, "_promptfooFileMetadata": {} }, "failureReason": 0 }, { "cost": 0, "gradingResult": { "pass": true, "score": 1, "reason": "All assertions passed", "namedScores": {}, "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 0 }, "componentResults": [ { "pass": true, "score": 1, "reason": "Assertion passed", "assertion": { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_010\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } } ] }, "id": "0fc5f75f-e181-42a7-8911-d61c64453c89", "latencyMs": 0, "namedScores": {}, "prompt": { "raw": "Drama society tour shirt. Needs to say Main Character Energy Dept and have a stage curtain vibe", "label": "{{customerInput}}", "config": {} }, "promptId": "7e756b499a1da155962cf88585b671e3ddacc5edf410c782ea1dbff2b92d0a70", "promptIdx": 0, "provider": { "id": "file://./providers/benchmark-v0-replay-provider.mjs", "label": "levyo-benchmark-v0-artifact-replay" }, "response": { "output": "{\"ok\":true,\"schemaVersion\":\"levyo_promptfoo_benchmark_v0_replay_result_v1\",\"caseId\":\"lvyo_bv0_candidate_010\",\"promptEcho\":\"Drama society tour shirt. Needs to say Main Character Energy Dept and have a stage curtain vibe\",\"status\":\"success\",\"modality\":\"text_only\",\"caseRole\":\"text_core\",\"cluster\":\"student society\",\"runtimeProfile\":\"internal_codex\",\"imageProvider\":\"codex_account_ui\",\"lockRef\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_010/artifact-lock.json\",\"lockSha256\":\"ba63a1916b07a4c16a5f5c400bda82b8eacef7f4ae4e5d474637bc31017b52fa\",\"final\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_010/artifact/final.png\",\"sha256\":\"d9efe1a78fe5827e187776f6351fc55d80a86c19f7716f9ff04c9ef6e45b7d02\"},\"mockup\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_010/artifact/mockup.png\",\"sha256\":\"2ad6c7e16f59120e2bfeef27351b19f0c5c62cf393e43438832eebdd0195f443\",\"publicSrc\":\"assets/lvyo_bv0_candidate_010_m4_mockup.webp\"},\"referenceImage\":null,\"noSpendReplay\":true,\"customerLiveFlowChanged\":false}", "images": [ { "mimeType": "image/webp", "blobRef": { "uri": "promptfoo://blob/ed4347ec512a8648aa55ac51df24e2ca23b68556341c88887c3ed62651e085c9", "hash": "ed4347ec512a8648aa55ac51df24e2ca23b68556341c88887c3ed62651e085c9", "mimeType": "image/webp", "sizeBytes": 49768, "provider": "filesystem" } } ], "metadata": { "caseId": "lvyo_bv0_candidate_010", "modality": "text_only", "cluster": "student society", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_010/artifact-lock.json", "finalSha256": "d9efe1a78fe5827e187776f6351fc55d80a86c19f7716f9ff04c9ef6e45b7d02", "mockupSha256": "2ad6c7e16f59120e2bfeef27351b19f0c5c62cf393e43438832eebdd0195f443", "publicMockup": "assets/lvyo_bv0_candidate_010_m4_mockup.webp", "noSpendReplay": true } }, "score": 1, "success": true, "testCase": { "description": "10 text_only student society lvyo_bv0_candidate_010", "vars": { "caseId": "lvyo_bv0_candidate_010", "customerInput": "Drama society tour shirt. Needs to say Main Character Energy Dept and have a stage curtain vibe", "modality": "text_only", "cluster": "student society" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_010\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ], "options": {}, "metadata": {} }, "testIdx": 9, "tokenUsage": { "prompt": 0, "completion": 0, "cached": 0, "total": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 }, "assertions": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 } } }, "vars": { "caseId": "lvyo_bv0_candidate_010", "customerInput": "Drama society tour shirt. Needs to say Main Character Energy Dept and have a stage curtain vibe", "modality": "text_only", "cluster": "student society" }, "metadata": { "caseId": "lvyo_bv0_candidate_010", "modality": "text_only", "cluster": "student society", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_010/artifact-lock.json", "finalSha256": "d9efe1a78fe5827e187776f6351fc55d80a86c19f7716f9ff04c9ef6e45b7d02", "mockupSha256": "2ad6c7e16f59120e2bfeef27351b19f0c5c62cf393e43438832eebdd0195f443", "publicMockup": "assets/lvyo_bv0_candidate_010_m4_mockup.webp", "noSpendReplay": true, "_promptfooFileMetadata": {} }, "failureReason": 0 }, { "cost": 0, "gradingResult": { "pass": true, "score": 1, "reason": "All assertions passed", "namedScores": {}, "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 0 }, "componentResults": [ { "pass": true, "score": 1, "reason": "Assertion passed", "assertion": { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_011\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } } ] }, "id": "130e3688-a92d-4470-bd37-cb07ec7475f2", "latencyMs": 0, "namedScores": {}, "prompt": { "raw": "Freshers halls social tee, Block C Chaos Committee, make it look like a fake university crest", "label": "{{customerInput}}", "config": {} }, "promptId": "7e756b499a1da155962cf88585b671e3ddacc5edf410c782ea1dbff2b92d0a70", "promptIdx": 0, "provider": { "id": "file://./providers/benchmark-v0-replay-provider.mjs", "label": "levyo-benchmark-v0-artifact-replay" }, "response": { "output": "{\"ok\":true,\"schemaVersion\":\"levyo_promptfoo_benchmark_v0_replay_result_v1\",\"caseId\":\"lvyo_bv0_candidate_011\",\"promptEcho\":\"Freshers halls social tee, Block C Chaos Committee, make it look like a fake university crest\",\"status\":\"success\",\"modality\":\"text_only\",\"caseRole\":\"text_core\",\"cluster\":\"freshers week\",\"runtimeProfile\":\"internal_codex\",\"imageProvider\":\"codex_account_ui\",\"lockRef\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_011/artifact-lock.json\",\"lockSha256\":\"3de6d46cef52d8d5946083e9cd7095dc4091520f2f56d206dd8aa5a236ae4095\",\"final\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_011/artifact/final.png\",\"sha256\":\"de82c67ff674cc2b9ce8302e5d760a81cbf9df6abf6b45b87f3f237fe9cc96e4\"},\"mockup\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_011/artifact/mockup.png\",\"sha256\":\"c44a9fada8b6262fbd856ae5d5bc5e4ed7b4c9fbf60d2ba6fe632c151f4c9e59\",\"publicSrc\":\"assets/lvyo_bv0_candidate_011_m4_mockup.webp\"},\"referenceImage\":null,\"noSpendReplay\":true,\"customerLiveFlowChanged\":false}", "images": [ { "mimeType": "image/webp", "blobRef": { "uri": "promptfoo://blob/89f724f97bc0db43be201fb78efc76f6cd6ff78778c6766065b97f3c9c536565", "hash": "89f724f97bc0db43be201fb78efc76f6cd6ff78778c6766065b97f3c9c536565", "mimeType": "image/webp", "sizeBytes": 39718, "provider": "filesystem" } } ], "metadata": { "caseId": "lvyo_bv0_candidate_011", "modality": "text_only", "cluster": "freshers week", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_011/artifact-lock.json", "finalSha256": "de82c67ff674cc2b9ce8302e5d760a81cbf9df6abf6b45b87f3f237fe9cc96e4", "mockupSha256": "c44a9fada8b6262fbd856ae5d5bc5e4ed7b4c9fbf60d2ba6fe632c151f4c9e59", "publicMockup": "assets/lvyo_bv0_candidate_011_m4_mockup.webp", "noSpendReplay": true } }, "score": 1, "success": true, "testCase": { "description": "11 text_only freshers week lvyo_bv0_candidate_011", "vars": { "caseId": "lvyo_bv0_candidate_011", "customerInput": "Freshers halls social tee, Block C Chaos Committee, make it look like a fake university crest", "modality": "text_only", "cluster": "freshers week" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_011\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ], "options": {}, "metadata": {} }, "testIdx": 10, "tokenUsage": { "prompt": 0, "completion": 0, "cached": 0, "total": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 }, "assertions": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 } } }, "vars": { "caseId": "lvyo_bv0_candidate_011", "customerInput": "Freshers halls social tee, Block C Chaos Committee, make it look like a fake university crest", "modality": "text_only", "cluster": "freshers week" }, "metadata": { "caseId": "lvyo_bv0_candidate_011", "modality": "text_only", "cluster": "freshers week", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_011/artifact-lock.json", "finalSha256": "de82c67ff674cc2b9ce8302e5d760a81cbf9df6abf6b45b87f3f237fe9cc96e4", "mockupSha256": "c44a9fada8b6262fbd856ae5d5bc5e4ed7b4c9fbf60d2ba6fe632c151f4c9e59", "publicMockup": "assets/lvyo_bv0_candidate_011_m4_mockup.webp", "noSpendReplay": true, "_promptfooFileMetadata": {} }, "failureReason": 0 }, { "cost": 0, "gradingResult": { "pass": true, "score": 1, "reason": "All assertions passed", "namedScores": {}, "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 0 }, "componentResults": [ { "pass": true, "score": 1, "reason": "Assertion passed", "assertion": { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_012\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } } ] }, "id": "743920ab-361f-48a2-babe-a811857c6d33", "latencyMs": 0, "namedScores": {}, "prompt": { "raw": "Rugby tour shirt for the front row lads. Mud, pints, terrible moustaches. Keep it printable", "label": "{{customerInput}}", "config": {} }, "promptId": "7e756b499a1da155962cf88585b671e3ddacc5edf410c782ea1dbff2b92d0a70", "promptIdx": 0, "provider": { "id": "file://./providers/benchmark-v0-replay-provider.mjs", "label": "levyo-benchmark-v0-artifact-replay" }, "response": { "output": "{\"ok\":true,\"schemaVersion\":\"levyo_promptfoo_benchmark_v0_replay_result_v1\",\"caseId\":\"lvyo_bv0_candidate_012\",\"promptEcho\":\"Rugby tour shirt for the front row lads. Mud, pints, terrible moustaches. Keep it printable\",\"status\":\"success\",\"modality\":\"text_only\",\"caseRole\":\"text_core\",\"cluster\":\"rugby social\",\"runtimeProfile\":\"internal_codex\",\"imageProvider\":\"codex_account_ui\",\"lockRef\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_012/artifact-lock.json\",\"lockSha256\":\"9970dd1c261a8dee3b4708bdb615dc16567a2161daa8d26ddd453f116c493490\",\"final\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_012/artifact/final.png\",\"sha256\":\"c73342e4e5783016324619cd413cebbccd7cf91c5b0c54038c851da104603fc5\"},\"mockup\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_012/artifact/mockup.png\",\"sha256\":\"052df8503e4372c48b2e1c2199bb20accf94b033801cfb42bd67e87f2b6065aa\",\"publicSrc\":\"assets/lvyo_bv0_candidate_012_m4_mockup.webp\"},\"referenceImage\":null,\"noSpendReplay\":true,\"customerLiveFlowChanged\":false}", "images": [ { "mimeType": "image/webp", "blobRef": { "uri": "promptfoo://blob/a41771d7159070f9e5741f8aba52364f1bc554d9dbf4c1ee0a3bca9917361742", "hash": "a41771d7159070f9e5741f8aba52364f1bc554d9dbf4c1ee0a3bca9917361742", "mimeType": "image/webp", "sizeBytes": 40708, "provider": "filesystem" } } ], "metadata": { "caseId": "lvyo_bv0_candidate_012", "modality": "text_only", "cluster": "rugby social", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_012/artifact-lock.json", "finalSha256": "c73342e4e5783016324619cd413cebbccd7cf91c5b0c54038c851da104603fc5", "mockupSha256": "052df8503e4372c48b2e1c2199bb20accf94b033801cfb42bd67e87f2b6065aa", "publicMockup": "assets/lvyo_bv0_candidate_012_m4_mockup.webp", "noSpendReplay": true } }, "score": 1, "success": true, "testCase": { "description": "12 text_only rugby social lvyo_bv0_candidate_012", "vars": { "caseId": "lvyo_bv0_candidate_012", "customerInput": "Rugby tour shirt for the front row lads. Mud, pints, terrible moustaches. Keep it printable", "modality": "text_only", "cluster": "rugby social" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_012\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ], "options": {}, "metadata": {} }, "testIdx": 11, "tokenUsage": { "prompt": 0, "completion": 0, "cached": 0, "total": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 }, "assertions": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 } } }, "vars": { "caseId": "lvyo_bv0_candidate_012", "customerInput": "Rugby tour shirt for the front row lads. Mud, pints, terrible moustaches. Keep it printable", "modality": "text_only", "cluster": "rugby social" }, "metadata": { "caseId": "lvyo_bv0_candidate_012", "modality": "text_only", "cluster": "rugby social", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_012/artifact-lock.json", "finalSha256": "c73342e4e5783016324619cd413cebbccd7cf91c5b0c54038c851da104603fc5", "mockupSha256": "052df8503e4372c48b2e1c2199bb20accf94b033801cfb42bd67e87f2b6065aa", "publicMockup": "assets/lvyo_bv0_candidate_012_m4_mockup.webp", "noSpendReplay": true, "_promptfooFileMetadata": {} }, "failureReason": 0 }, { "cost": 0, "gradingResult": { "pass": true, "score": 1, "reason": "All assertions passed", "namedScores": {}, "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 0 }, "componentResults": [ { "pass": true, "score": 1, "reason": "Assertion passed", "assertion": { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_013\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } } ] }, "id": "47ffa9a9-bcb9-40e0-a302-cd6baf079ea0", "latencyMs": 1, "namedScores": {}, "prompt": { "raw": "5 a side team top for The Unathletico Madrid, please make it like a retro football badge", "label": "{{customerInput}}", "config": {} }, "promptId": "7e756b499a1da155962cf88585b671e3ddacc5edf410c782ea1dbff2b92d0a70", "promptIdx": 0, "provider": { "id": "file://./providers/benchmark-v0-replay-provider.mjs", "label": "levyo-benchmark-v0-artifact-replay" }, "response": { "output": "{\"ok\":true,\"schemaVersion\":\"levyo_promptfoo_benchmark_v0_replay_result_v1\",\"caseId\":\"lvyo_bv0_candidate_013\",\"promptEcho\":\"5 a side team top for The Unathletico Madrid, please make it like a retro football badge\",\"status\":\"success\",\"modality\":\"text_only\",\"caseRole\":\"text_core\",\"cluster\":\"football social\",\"runtimeProfile\":\"internal_codex\",\"imageProvider\":\"codex_account_ui\",\"lockRef\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_013/artifact-lock.json\",\"lockSha256\":\"1a38b42d71505d0b9bdf67514a7387490c5d5b52a6e4c7bf89176d1f091c8a29\",\"final\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_013/artifact/final.png\",\"sha256\":\"ef9bb403182d0f226ef49b7b4323cba66f9c4928a33dee9206dbb49874483c36\"},\"mockup\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_013/artifact/mockup.png\",\"sha256\":\"6fccc166efa46dba053b31a308cfc79a9f01bb53f7e82d2549ddbc38dce3a090\",\"publicSrc\":\"assets/lvyo_bv0_candidate_013_m4_mockup.webp\"},\"referenceImage\":null,\"noSpendReplay\":true,\"customerLiveFlowChanged\":false}", "images": [ { "mimeType": "image/webp", "blobRef": { "uri": "promptfoo://blob/a0466a4c98403a1374d849188096a3007d7bbc5f31d9ecb4ab6128279e8176d0", "hash": "a0466a4c98403a1374d849188096a3007d7bbc5f31d9ecb4ab6128279e8176d0", "mimeType": "image/webp", "sizeBytes": 39584, "provider": "filesystem" } } ], "metadata": { "caseId": "lvyo_bv0_candidate_013", "modality": "text_only", "cluster": "football social", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_013/artifact-lock.json", "finalSha256": "ef9bb403182d0f226ef49b7b4323cba66f9c4928a33dee9206dbb49874483c36", "mockupSha256": "6fccc166efa46dba053b31a308cfc79a9f01bb53f7e82d2549ddbc38dce3a090", "publicMockup": "assets/lvyo_bv0_candidate_013_m4_mockup.webp", "noSpendReplay": true } }, "score": 1, "success": true, "testCase": { "description": "13 text_only football social lvyo_bv0_candidate_013", "vars": { "caseId": "lvyo_bv0_candidate_013", "customerInput": "5 a side team top for The Unathletico Madrid, please make it like a retro football badge", "modality": "text_only", "cluster": "football social" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_013\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ], "options": {}, "metadata": {} }, "testIdx": 12, "tokenUsage": { "prompt": 0, "completion": 0, "cached": 0, "total": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 }, "assertions": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 } } }, "vars": { "caseId": "lvyo_bv0_candidate_013", "customerInput": "5 a side team top for The Unathletico Madrid, please make it like a retro football badge", "modality": "text_only", "cluster": "football social" }, "metadata": { "caseId": "lvyo_bv0_candidate_013", "modality": "text_only", "cluster": "football social", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_013/artifact-lock.json", "finalSha256": "ef9bb403182d0f226ef49b7b4323cba66f9c4928a33dee9206dbb49874483c36", "mockupSha256": "6fccc166efa46dba053b31a308cfc79a9f01bb53f7e82d2549ddbc38dce3a090", "publicMockup": "assets/lvyo_bv0_candidate_013_m4_mockup.webp", "noSpendReplay": true, "_promptfooFileMetadata": {} }, "failureReason": 0 }, { "cost": 0, "gradingResult": { "pass": true, "score": 1, "reason": "All assertions passed", "namedScores": {}, "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 0 }, "componentResults": [ { "pass": true, "score": 1, "reason": "Assertion passed", "assertion": { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_014\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } } ] }, "id": "cba05a2f-78ee-4a88-a210-587c135c9568", "latencyMs": 0, "namedScores": {}, "prompt": { "raw": "Running club shirt for our first half marathon. Slow But Extremely Dramatic. Maybe trainers and a coffee cup", "label": "{{customerInput}}", "config": {} }, "promptId": "7e756b499a1da155962cf88585b671e3ddacc5edf410c782ea1dbff2b92d0a70", "promptIdx": 0, "provider": { "id": "file://./providers/benchmark-v0-replay-provider.mjs", "label": "levyo-benchmark-v0-artifact-replay" }, "response": { "output": "{\"ok\":true,\"schemaVersion\":\"levyo_promptfoo_benchmark_v0_replay_result_v1\",\"caseId\":\"lvyo_bv0_candidate_014\",\"promptEcho\":\"Running club shirt for our first half marathon. Slow But Extremely Dramatic. Maybe trainers and a coffee cup\",\"status\":\"success\",\"modality\":\"text_only\",\"caseRole\":\"text_core\",\"cluster\":\"running club\",\"runtimeProfile\":\"internal_codex\",\"imageProvider\":\"codex_account_ui\",\"lockRef\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_014/artifact-lock.json\",\"lockSha256\":\"314f0b5612835ff7dbbca9e7c9256d02e297473bcf88e0ca2584f02991a78a8d\",\"final\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_014/artifact/final.png\",\"sha256\":\"b44017dcd93cb6ca39b6535169a8ab53e6f1f186969327805dbc196cfbb8a176\"},\"mockup\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_014/artifact/mockup.png\",\"sha256\":\"c3cd88cdfb3c24817cfbfd87a765d53bdd074617b2d139e09567eeeb03c0a3c3\",\"publicSrc\":\"assets/lvyo_bv0_candidate_014_m4_mockup.webp\"},\"referenceImage\":null,\"noSpendReplay\":true,\"customerLiveFlowChanged\":false}", "images": [ { "mimeType": "image/webp", "blobRef": { "uri": "promptfoo://blob/192296a2f3cf4eb696652915b7b620fa4835e8a221e3ba1996ed1871fb04d850", "hash": "192296a2f3cf4eb696652915b7b620fa4835e8a221e3ba1996ed1871fb04d850", "mimeType": "image/webp", "sizeBytes": 39646, "provider": "filesystem" } } ], "metadata": { "caseId": "lvyo_bv0_candidate_014", "modality": "text_only", "cluster": "running club", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_014/artifact-lock.json", "finalSha256": "b44017dcd93cb6ca39b6535169a8ab53e6f1f186969327805dbc196cfbb8a176", "mockupSha256": "c3cd88cdfb3c24817cfbfd87a765d53bdd074617b2d139e09567eeeb03c0a3c3", "publicMockup": "assets/lvyo_bv0_candidate_014_m4_mockup.webp", "noSpendReplay": true } }, "score": 1, "success": true, "testCase": { "description": "14 text_only running club lvyo_bv0_candidate_014", "vars": { "caseId": "lvyo_bv0_candidate_014", "customerInput": "Running club shirt for our first half marathon. Slow But Extremely Dramatic. Maybe trainers and a coffee cup", "modality": "text_only", "cluster": "running club" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_014\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ], "options": {}, "metadata": {} }, "testIdx": 13, "tokenUsage": { "prompt": 0, "completion": 0, "cached": 0, "total": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 }, "assertions": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 } } }, "vars": { "caseId": "lvyo_bv0_candidate_014", "customerInput": "Running club shirt for our first half marathon. Slow But Extremely Dramatic. Maybe trainers and a coffee cup", "modality": "text_only", "cluster": "running club" }, "metadata": { "caseId": "lvyo_bv0_candidate_014", "modality": "text_only", "cluster": "running club", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_014/artifact-lock.json", "finalSha256": "b44017dcd93cb6ca39b6535169a8ab53e6f1f186969327805dbc196cfbb8a176", "mockupSha256": "c3cd88cdfb3c24817cfbfd87a765d53bdd074617b2d139e09567eeeb03c0a3c3", "publicMockup": "assets/lvyo_bv0_candidate_014_m4_mockup.webp", "noSpendReplay": true, "_promptfooFileMetadata": {} }, "failureReason": 0 }, { "cost": 0, "gradingResult": { "pass": true, "score": 1, "reason": "All assertions passed", "namedScores": {}, "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 0 }, "componentResults": [ { "pass": true, "score": 1, "reason": "Assertion passed", "assertion": { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_015\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } } ] }, "id": "1feef844-848c-490c-871d-988ce7af8d17", "latencyMs": 0, "namedScores": {}, "prompt": { "raw": "Festival crew tee, Glasto-ish but don't use the actual logo, saying Field Rats 2026 and make it colourful", "label": "{{customerInput}}", "config": {} }, "promptId": "7e756b499a1da155962cf88585b671e3ddacc5edf410c782ea1dbff2b92d0a70", "promptIdx": 0, "provider": { "id": "file://./providers/benchmark-v0-replay-provider.mjs", "label": "levyo-benchmark-v0-artifact-replay" }, "response": { "output": "{\"ok\":true,\"schemaVersion\":\"levyo_promptfoo_benchmark_v0_replay_result_v1\",\"caseId\":\"lvyo_bv0_candidate_015\",\"promptEcho\":\"Festival crew tee, Glasto-ish but don't use the actual logo, saying Field Rats 2026 and make it colourful\",\"status\":\"success\",\"modality\":\"text_only\",\"caseRole\":\"text_core\",\"cluster\":\"festival crew\",\"runtimeProfile\":\"internal_codex\",\"imageProvider\":\"codex_account_ui\",\"lockRef\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_015/artifact-lock.json\",\"lockSha256\":\"96de0ce9fca4031def8aac930e7cb0ce3c003689bf9dc1cb7cc2e25e6030e07c\",\"final\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_015/artifact/final.png\",\"sha256\":\"cfbc74889687364ebf09ed4f511e7fa0600a0d1262d789583a21041b9c61e42d\"},\"mockup\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_015/artifact/mockup.png\",\"sha256\":\"b747130d74bacb9b499ec927c7f4a6e3bb819564b45dd7975a82bfe187bdf739\",\"publicSrc\":\"assets/lvyo_bv0_candidate_015_m4_mockup.webp\"},\"referenceImage\":null,\"noSpendReplay\":true,\"customerLiveFlowChanged\":false}", "images": [ { "mimeType": "image/webp", "blobRef": { "uri": "promptfoo://blob/dd38d1e94b05bfa828f88084cb78a1a687b253174c96112893b5ef8a501bb5c8", "hash": "dd38d1e94b05bfa828f88084cb78a1a687b253174c96112893b5ef8a501bb5c8", "mimeType": "image/webp", "sizeBytes": 40906, "provider": "filesystem" } } ], "metadata": { "caseId": "lvyo_bv0_candidate_015", "modality": "text_only", "cluster": "festival crew", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_015/artifact-lock.json", "finalSha256": "cfbc74889687364ebf09ed4f511e7fa0600a0d1262d789583a21041b9c61e42d", "mockupSha256": "b747130d74bacb9b499ec927c7f4a6e3bb819564b45dd7975a82bfe187bdf739", "publicMockup": "assets/lvyo_bv0_candidate_015_m4_mockup.webp", "noSpendReplay": true } }, "score": 1, "success": true, "testCase": { "description": "15 text_only festival crew lvyo_bv0_candidate_015", "vars": { "caseId": "lvyo_bv0_candidate_015", "customerInput": "Festival crew tee, Glasto-ish but don't use the actual logo, saying Field Rats 2026 and make it colourful", "modality": "text_only", "cluster": "festival crew" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_015\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ], "options": {}, "metadata": {} }, "testIdx": 14, "tokenUsage": { "prompt": 0, "completion": 0, "cached": 0, "total": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 }, "assertions": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 } } }, "vars": { "caseId": "lvyo_bv0_candidate_015", "customerInput": "Festival crew tee, Glasto-ish but don't use the actual logo, saying Field Rats 2026 and make it colourful", "modality": "text_only", "cluster": "festival crew" }, "metadata": { "caseId": "lvyo_bv0_candidate_015", "modality": "text_only", "cluster": "festival crew", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_015/artifact-lock.json", "finalSha256": "cfbc74889687364ebf09ed4f511e7fa0600a0d1262d789583a21041b9c61e42d", "mockupSha256": "b747130d74bacb9b499ec927c7f4a6e3bb819564b45dd7975a82bfe187bdf739", "publicMockup": "assets/lvyo_bv0_candidate_015_m4_mockup.webp", "noSpendReplay": true, "_promptfooFileMetadata": {} }, "failureReason": 0 }, { "cost": 0, "gradingResult": { "pass": true, "score": 1, "reason": "All assertions passed", "namedScores": {}, "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 0 }, "componentResults": [ { "pass": true, "score": 1, "reason": "Assertion passed", "assertion": { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_016\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } } ] }, "id": "8ff32ab7-d64b-4acc-acd2-68c8266bb724", "latencyMs": 0, "namedScores": {}, "prompt": { "raw": "Warehouse rave shirt for my birthday. Black tee, neon green text: I Came For The Bass And Lost My Friends", "label": "{{customerInput}}", "config": {} }, "promptId": "7e756b499a1da155962cf88585b671e3ddacc5edf410c782ea1dbff2b92d0a70", "promptIdx": 0, "provider": { "id": "file://./providers/benchmark-v0-replay-provider.mjs", "label": "levyo-benchmark-v0-artifact-replay" }, "response": { "output": "{\"ok\":true,\"schemaVersion\":\"levyo_promptfoo_benchmark_v0_replay_result_v1\",\"caseId\":\"lvyo_bv0_candidate_016\",\"promptEcho\":\"Warehouse rave shirt for my birthday. Black tee, neon green text: I Came For The Bass And Lost My Friends\",\"status\":\"success\",\"modality\":\"text_only\",\"caseRole\":\"text_core\",\"cluster\":\"rave weekend\",\"runtimeProfile\":\"internal_codex\",\"imageProvider\":\"codex_account_ui\",\"lockRef\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_016/artifact-lock.json\",\"lockSha256\":\"6d84b11732aeb2c822a866a55639a2c421b38b7784fec5154f310946fd4adbe5\",\"final\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_016/artifact/final.png\",\"sha256\":\"e622c69a564b99b4ffc0045d495ea3d17fb57889bb3e2d879bf9c32abcbcfe23\"},\"mockup\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_016/artifact/mockup.png\",\"sha256\":\"b4b54aa7a75feaa3886796d3c5ca1638944813a7551801e77a387aac30217080\",\"publicSrc\":\"assets/lvyo_bv0_candidate_016_m4_mockup.webp\"},\"referenceImage\":null,\"noSpendReplay\":true,\"customerLiveFlowChanged\":false}", "images": [ { "mimeType": "image/webp", "blobRef": { "uri": "promptfoo://blob/77760130f84d1394398141ade8116769fabc5602ba39d1d6dc9e1b6dcf78dbd1", "hash": "77760130f84d1394398141ade8116769fabc5602ba39d1d6dc9e1b6dcf78dbd1", "mimeType": "image/webp", "sizeBytes": 51038, "provider": "filesystem" } } ], "metadata": { "caseId": "lvyo_bv0_candidate_016", "modality": "text_only", "cluster": "rave weekend", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_016/artifact-lock.json", "finalSha256": "e622c69a564b99b4ffc0045d495ea3d17fb57889bb3e2d879bf9c32abcbcfe23", "mockupSha256": "b4b54aa7a75feaa3886796d3c5ca1638944813a7551801e77a387aac30217080", "publicMockup": "assets/lvyo_bv0_candidate_016_m4_mockup.webp", "noSpendReplay": true } }, "score": 1, "success": true, "testCase": { "description": "16 text_only rave weekend lvyo_bv0_candidate_016", "vars": { "caseId": "lvyo_bv0_candidate_016", "customerInput": "Warehouse rave shirt for my birthday. Black tee, neon green text: I Came For The Bass And Lost My Friends", "modality": "text_only", "cluster": "rave weekend" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_016\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ], "options": {}, "metadata": {} }, "testIdx": 15, "tokenUsage": { "prompt": 0, "completion": 0, "cached": 0, "total": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 }, "assertions": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 } } }, "vars": { "caseId": "lvyo_bv0_candidate_016", "customerInput": "Warehouse rave shirt for my birthday. Black tee, neon green text: I Came For The Bass And Lost My Friends", "modality": "text_only", "cluster": "rave weekend" }, "metadata": { "caseId": "lvyo_bv0_candidate_016", "modality": "text_only", "cluster": "rave weekend", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_016/artifact-lock.json", "finalSha256": "e622c69a564b99b4ffc0045d495ea3d17fb57889bb3e2d879bf9c32abcbcfe23", "mockupSha256": "b4b54aa7a75feaa3886796d3c5ca1638944813a7551801e77a387aac30217080", "publicMockup": "assets/lvyo_bv0_candidate_016_m4_mockup.webp", "noSpendReplay": true, "_promptfooFileMetadata": {} }, "failureReason": 0 }, { "cost": 0, "gradingResult": { "pass": true, "score": 1, "reason": "All assertions passed", "namedScores": {}, "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 0 }, "componentResults": [ { "pass": true, "score": 1, "reason": "Assertion passed", "assertion": { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_017\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } } ] }, "id": "55cb3ae5-10d5-4e26-9262-5040f6279ffd", "latencyMs": 0, "namedScores": {}, "prompt": { "raw": "Pub crawl shirt for Cambridge, 12 pubs, one bad idea. Need checkboxes on the back", "label": "{{customerInput}}", "config": {} }, "promptId": "7e756b499a1da155962cf88585b671e3ddacc5edf410c782ea1dbff2b92d0a70", "promptIdx": 0, "provider": { "id": "file://./providers/benchmark-v0-replay-provider.mjs", "label": "levyo-benchmark-v0-artifact-replay" }, "response": { "output": "{\"ok\":true,\"schemaVersion\":\"levyo_promptfoo_benchmark_v0_replay_result_v1\",\"caseId\":\"lvyo_bv0_candidate_017\",\"promptEcho\":\"Pub crawl shirt for Cambridge, 12 pubs, one bad idea. Need checkboxes on the back\",\"status\":\"success\",\"modality\":\"text_only\",\"caseRole\":\"text_core\",\"cluster\":\"pub crawl\",\"runtimeProfile\":\"internal_codex\",\"imageProvider\":\"codex_account_ui\",\"lockRef\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_017/artifact-lock.json\",\"lockSha256\":\"b47279b88362bc3c0e47146527f637d4043c0c5c6b4e7e76dc6dc532d672541d\",\"final\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_017/artifact/final.png\",\"sha256\":\"44b854682a7a1a80268def2124930ead014ae2916842c0f574d270708560d7db\"},\"mockup\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_017/artifact/mockup.png\",\"sha256\":\"4806ba154261b27d60aa3d1fc99e2f4707ca7f9ce3aa58f2803393b9ee2e50a3\",\"publicSrc\":\"assets/lvyo_bv0_candidate_017_m4_mockup.webp\"},\"referenceImage\":null,\"noSpendReplay\":true,\"customerLiveFlowChanged\":false}", "images": [ { "mimeType": "image/webp", "blobRef": { "uri": "promptfoo://blob/9c4c70a9d4f5ed488ada5c1a783329800b7664f01d15d5a40df282ad707aaed5", "hash": "9c4c70a9d4f5ed488ada5c1a783329800b7664f01d15d5a40df282ad707aaed5", "mimeType": "image/webp", "sizeBytes": 48554, "provider": "filesystem" } } ], "metadata": { "caseId": "lvyo_bv0_candidate_017", "modality": "text_only", "cluster": "pub crawl", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_017/artifact-lock.json", "finalSha256": "44b854682a7a1a80268def2124930ead014ae2916842c0f574d270708560d7db", "mockupSha256": "4806ba154261b27d60aa3d1fc99e2f4707ca7f9ce3aa58f2803393b9ee2e50a3", "publicMockup": "assets/lvyo_bv0_candidate_017_m4_mockup.webp", "noSpendReplay": true } }, "score": 1, "success": true, "testCase": { "description": "17 text_only pub crawl lvyo_bv0_candidate_017", "vars": { "caseId": "lvyo_bv0_candidate_017", "customerInput": "Pub crawl shirt for Cambridge, 12 pubs, one bad idea. Need checkboxes on the back", "modality": "text_only", "cluster": "pub crawl" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_017\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ], "options": {}, "metadata": {} }, "testIdx": 16, "tokenUsage": { "prompt": 0, "completion": 0, "cached": 0, "total": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 }, "assertions": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 } } }, "vars": { "caseId": "lvyo_bv0_candidate_017", "customerInput": "Pub crawl shirt for Cambridge, 12 pubs, one bad idea. Need checkboxes on the back", "modality": "text_only", "cluster": "pub crawl" }, "metadata": { "caseId": "lvyo_bv0_candidate_017", "modality": "text_only", "cluster": "pub crawl", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_017/artifact-lock.json", "finalSha256": "44b854682a7a1a80268def2124930ead014ae2916842c0f574d270708560d7db", "mockupSha256": "4806ba154261b27d60aa3d1fc99e2f4707ca7f9ce3aa58f2803393b9ee2e50a3", "publicMockup": "assets/lvyo_bv0_candidate_017_m4_mockup.webp", "noSpendReplay": true, "_promptfooFileMetadata": {} }, "failureReason": 0 }, { "cost": 0, "gradingResult": { "pass": true, "score": 1, "reason": "All assertions passed", "namedScores": {}, "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 0 }, "componentResults": [ { "pass": true, "score": 1, "reason": "Assertion passed", "assertion": { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_018\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } } ] }, "id": "86ac5785-79c7-439f-8d94-2a3845b9faab", "latencyMs": 0, "namedScores": {}, "prompt": { "raw": "Beer Olympics team shirt. Team name: The Chuggernauts. Red white and blue but not too American", "label": "{{customerInput}}", "config": {} }, "promptId": "7e756b499a1da155962cf88585b671e3ddacc5edf410c782ea1dbff2b92d0a70", "promptIdx": 0, "provider": { "id": "file://./providers/benchmark-v0-replay-provider.mjs", "label": "levyo-benchmark-v0-artifact-replay" }, "response": { "output": "{\"ok\":true,\"schemaVersion\":\"levyo_promptfoo_benchmark_v0_replay_result_v1\",\"caseId\":\"lvyo_bv0_candidate_018\",\"promptEcho\":\"Beer Olympics team shirt. Team name: The Chuggernauts. Red white and blue but not too American\",\"status\":\"success\",\"modality\":\"text_only\",\"caseRole\":\"text_core\",\"cluster\":\"beer olympics\",\"runtimeProfile\":\"internal_codex\",\"imageProvider\":\"codex_account_ui\",\"lockRef\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_018/artifact-lock.json\",\"lockSha256\":\"d36591c2946c662380b13d967119c9af39f16f0b7ead8c8718d7b266cfed3089\",\"final\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_018/artifact/final.png\",\"sha256\":\"f24b1082fa2b6666ff397139e8e0ad854ddb6f7597115a6005459102de832335\"},\"mockup\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_018/artifact/mockup.png\",\"sha256\":\"551ebc1fb0962c7731a7e7742202c1b90697195eca04b7662cac15e757244fed\",\"publicSrc\":\"assets/lvyo_bv0_candidate_018_m4_mockup.webp\"},\"referenceImage\":null,\"noSpendReplay\":true,\"customerLiveFlowChanged\":false}", "images": [ { "mimeType": "image/webp", "blobRef": { "uri": "promptfoo://blob/33786d21e2dfe3be844706db2d363d5a9f0bc386fe493c56e61a9ca68ee75fb0", "hash": "33786d21e2dfe3be844706db2d363d5a9f0bc386fe493c56e61a9ca68ee75fb0", "mimeType": "image/webp", "sizeBytes": 36966, "provider": "filesystem" } } ], "metadata": { "caseId": "lvyo_bv0_candidate_018", "modality": "text_only", "cluster": "beer olympics", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_018/artifact-lock.json", "finalSha256": "f24b1082fa2b6666ff397139e8e0ad854ddb6f7597115a6005459102de832335", "mockupSha256": "551ebc1fb0962c7731a7e7742202c1b90697195eca04b7662cac15e757244fed", "publicMockup": "assets/lvyo_bv0_candidate_018_m4_mockup.webp", "noSpendReplay": true } }, "score": 1, "success": true, "testCase": { "description": "18 text_only beer olympics lvyo_bv0_candidate_018", "vars": { "caseId": "lvyo_bv0_candidate_018", "customerInput": "Beer Olympics team shirt. Team name: The Chuggernauts. Red white and blue but not too American", "modality": "text_only", "cluster": "beer olympics" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_018\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ], "options": {}, "metadata": {} }, "testIdx": 17, "tokenUsage": { "prompt": 0, "completion": 0, "cached": 0, "total": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 }, "assertions": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 } } }, "vars": { "caseId": "lvyo_bv0_candidate_018", "customerInput": "Beer Olympics team shirt. Team name: The Chuggernauts. Red white and blue but not too American", "modality": "text_only", "cluster": "beer olympics" }, "metadata": { "caseId": "lvyo_bv0_candidate_018", "modality": "text_only", "cluster": "beer olympics", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_018/artifact-lock.json", "finalSha256": "f24b1082fa2b6666ff397139e8e0ad854ddb6f7597115a6005459102de832335", "mockupSha256": "551ebc1fb0962c7731a7e7742202c1b90697195eca04b7662cac15e757244fed", "publicMockup": "assets/lvyo_bv0_candidate_018_m4_mockup.webp", "noSpendReplay": true, "_promptfooFileMetadata": {} }, "failureReason": 0 }, { "cost": 0, "gradingResult": { "pass": true, "score": 1, "reason": "All assertions passed", "namedScores": {}, "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 0 }, "componentResults": [ { "pass": true, "score": 1, "reason": "Assertion passed", "assertion": { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_019\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } } ] }, "id": "5f242d35-238e-4571-a72f-bd57e3f3341f", "latencyMs": 1, "namedScores": {}, "prompt": { "raw": "White lie party T-shirt, mine should say I only came for one drink, make it bold and a bit messy", "label": "{{customerInput}}", "config": {} }, "promptId": "7e756b499a1da155962cf88585b671e3ddacc5edf410c782ea1dbff2b92d0a70", "promptIdx": 0, "provider": { "id": "file://./providers/benchmark-v0-replay-provider.mjs", "label": "levyo-benchmark-v0-artifact-replay" }, "response": { "output": "{\"ok\":true,\"schemaVersion\":\"levyo_promptfoo_benchmark_v0_replay_result_v1\",\"caseId\":\"lvyo_bv0_candidate_019\",\"promptEcho\":\"White lie party T-shirt, mine should say I only came for one drink, make it bold and a bit messy\",\"status\":\"success\",\"modality\":\"text_only\",\"caseRole\":\"text_core\",\"cluster\":\"white lie party\",\"runtimeProfile\":\"internal_codex\",\"imageProvider\":\"codex_account_ui\",\"lockRef\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_019/artifact-lock.json\",\"lockSha256\":\"f7f0f744d99df87c3b2cc3b36ca0ec05173e3fca059f14874b213d54c3c7de0e\",\"final\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_019/artifact/final.png\",\"sha256\":\"80af23056a1970ee4f9b23dc381f7bba757fa920c763c6bd155a7d0056db7773\"},\"mockup\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_019/artifact/mockup.png\",\"sha256\":\"fa5e72f9f42fd209835e595581ee27292e10ac400e924e9f2f43b07c81860c6b\",\"publicSrc\":\"assets/lvyo_bv0_candidate_019_m4_mockup.webp\"},\"referenceImage\":null,\"noSpendReplay\":true,\"customerLiveFlowChanged\":false}", "images": [ { "mimeType": "image/webp", "blobRef": { "uri": "promptfoo://blob/61a9949c4d72675607772ce92c02989021cf648b533f333db49154b8714c1d56", "hash": "61a9949c4d72675607772ce92c02989021cf648b533f333db49154b8714c1d56", "mimeType": "image/webp", "sizeBytes": 52350, "provider": "filesystem" } } ], "metadata": { "caseId": "lvyo_bv0_candidate_019", "modality": "text_only", "cluster": "white lie party", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_019/artifact-lock.json", "finalSha256": "80af23056a1970ee4f9b23dc381f7bba757fa920c763c6bd155a7d0056db7773", "mockupSha256": "fa5e72f9f42fd209835e595581ee27292e10ac400e924e9f2f43b07c81860c6b", "publicMockup": "assets/lvyo_bv0_candidate_019_m4_mockup.webp", "noSpendReplay": true } }, "score": 1, "success": true, "testCase": { "description": "19 text_only white lie party lvyo_bv0_candidate_019", "vars": { "caseId": "lvyo_bv0_candidate_019", "customerInput": "White lie party T-shirt, mine should say I only came for one drink, make it bold and a bit messy", "modality": "text_only", "cluster": "white lie party" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_019\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ], "options": {}, "metadata": {} }, "testIdx": 18, "tokenUsage": { "prompt": 0, "completion": 0, "cached": 0, "total": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 }, "assertions": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 } } }, "vars": { "caseId": "lvyo_bv0_candidate_019", "customerInput": "White lie party T-shirt, mine should say I only came for one drink, make it bold and a bit messy", "modality": "text_only", "cluster": "white lie party" }, "metadata": { "caseId": "lvyo_bv0_candidate_019", "modality": "text_only", "cluster": "white lie party", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_019/artifact-lock.json", "finalSha256": "80af23056a1970ee4f9b23dc381f7bba757fa920c763c6bd155a7d0056db7773", "mockupSha256": "fa5e72f9f42fd209835e595581ee27292e10ac400e924e9f2f43b07c81860c6b", "publicMockup": "assets/lvyo_bv0_candidate_019_m4_mockup.webp", "noSpendReplay": true, "_promptfooFileMetadata": {} }, "failureReason": 0 }, { "cost": 0, "gradingResult": { "pass": true, "score": 1, "reason": "All assertions passed", "namedScores": {}, "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 0 }, "componentResults": [ { "pass": true, "score": 1, "reason": "Assertion passed", "assertion": { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_020\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } } ] }, "id": "02757307-a9c6-4691-9e87-7b755ebea160", "latencyMs": 0, "namedScores": {}, "prompt": { "raw": "Bottomless brunch top for Laura's birthday, prosecco tower, sunglasses, 'Booked for 12, emotionally there at 3'", "label": "{{customerInput}}", "config": {} }, "promptId": "7e756b499a1da155962cf88585b671e3ddacc5edf410c782ea1dbff2b92d0a70", "promptIdx": 0, "provider": { "id": "file://./providers/benchmark-v0-replay-provider.mjs", "label": "levyo-benchmark-v0-artifact-replay" }, "response": { "output": "{\"ok\":true,\"schemaVersion\":\"levyo_promptfoo_benchmark_v0_replay_result_v1\",\"caseId\":\"lvyo_bv0_candidate_020\",\"promptEcho\":\"Bottomless brunch top for Laura's birthday, prosecco tower, sunglasses, 'Booked for 12, emotionally there at 3'\",\"status\":\"success\",\"modality\":\"text_only\",\"caseRole\":\"text_core\",\"cluster\":\"bottomless brunch\",\"runtimeProfile\":\"internal_codex\",\"imageProvider\":\"codex_account_ui\",\"lockRef\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_020/artifact-lock.json\",\"lockSha256\":\"dd7e6c5728f9e570700fc188fd8b02e2c335774db00d43fa538d21c732c7bca7\",\"final\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_020/artifact/final.png\",\"sha256\":\"86fcdd2d6abd9a6466e9dea8c6458d8ee6b5ec0045778545f43cd779b12bcd34\"},\"mockup\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_020/artifact/mockup.png\",\"sha256\":\"175d5063a645211a5a50e925d83a08f631179d64561c056fda3fb646caef6fcd\",\"publicSrc\":\"assets/lvyo_bv0_candidate_020_m4_mockup.webp\"},\"referenceImage\":null,\"noSpendReplay\":true,\"customerLiveFlowChanged\":false}", "images": [ { "mimeType": "image/webp", "blobRef": { "uri": "promptfoo://blob/2f8f6a4ba276a01cb2167516502ff8bb587903e3d6bf47d305586510c573c69e", "hash": "2f8f6a4ba276a01cb2167516502ff8bb587903e3d6bf47d305586510c573c69e", "mimeType": "image/webp", "sizeBytes": 49206, "provider": "filesystem" } } ], "metadata": { "caseId": "lvyo_bv0_candidate_020", "modality": "text_only", "cluster": "bottomless brunch", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_020/artifact-lock.json", "finalSha256": "86fcdd2d6abd9a6466e9dea8c6458d8ee6b5ec0045778545f43cd779b12bcd34", "mockupSha256": "175d5063a645211a5a50e925d83a08f631179d64561c056fda3fb646caef6fcd", "publicMockup": "assets/lvyo_bv0_candidate_020_m4_mockup.webp", "noSpendReplay": true } }, "score": 1, "success": true, "testCase": { "description": "20 text_only bottomless brunch lvyo_bv0_candidate_020", "vars": { "caseId": "lvyo_bv0_candidate_020", "customerInput": "Bottomless brunch top for Laura's birthday, prosecco tower, sunglasses, 'Booked for 12, emotionally there at 3'", "modality": "text_only", "cluster": "bottomless brunch" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_020\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ], "options": {}, "metadata": {} }, "testIdx": 19, "tokenUsage": { "prompt": 0, "completion": 0, "cached": 0, "total": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 }, "assertions": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 } } }, "vars": { "caseId": "lvyo_bv0_candidate_020", "customerInput": "Bottomless brunch top for Laura's birthday, prosecco tower, sunglasses, 'Booked for 12, emotionally there at 3'", "modality": "text_only", "cluster": "bottomless brunch" }, "metadata": { "caseId": "lvyo_bv0_candidate_020", "modality": "text_only", "cluster": "bottomless brunch", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_020/artifact-lock.json", "finalSha256": "86fcdd2d6abd9a6466e9dea8c6458d8ee6b5ec0045778545f43cd779b12bcd34", "mockupSha256": "175d5063a645211a5a50e925d83a08f631179d64561c056fda3fb646caef6fcd", "publicMockup": "assets/lvyo_bv0_candidate_020_m4_mockup.webp", "noSpendReplay": true, "_promptfooFileMetadata": {} }, "failureReason": 0 }, { "cost": 0, "gradingResult": { "pass": true, "score": 1, "reason": "All assertions passed", "namedScores": {}, "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 0 }, "componentResults": [ { "pass": true, "score": 1, "reason": "Assertion passed", "assertion": { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_021\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } } ] }, "id": "b72b2276-e893-4deb-94f2-bf97567764c0", "latencyMs": 0, "namedScores": {}, "prompt": { "raw": "Garden party tee for my mum's 50th, floral but funny, says Rosie's Gin Garden Est. 1976", "label": "{{customerInput}}", "config": {} }, "promptId": "7e756b499a1da155962cf88585b671e3ddacc5edf410c782ea1dbff2b92d0a70", "promptIdx": 0, "provider": { "id": "file://./providers/benchmark-v0-replay-provider.mjs", "label": "levyo-benchmark-v0-artifact-replay" }, "response": { "output": "{\"ok\":true,\"schemaVersion\":\"levyo_promptfoo_benchmark_v0_replay_result_v1\",\"caseId\":\"lvyo_bv0_candidate_021\",\"promptEcho\":\"Garden party tee for my mum's 50th, floral but funny, says Rosie's Gin Garden Est. 1976\",\"status\":\"success\",\"modality\":\"text_only\",\"caseRole\":\"text_core\",\"cluster\":\"garden party\",\"runtimeProfile\":\"internal_codex\",\"imageProvider\":\"codex_account_ui\",\"lockRef\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_021/artifact-lock.json\",\"lockSha256\":\"9704af14afc330efb60a374348b52358ad47a3a3e176e21ced63f939a0432659\",\"final\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_021/artifact/final.png\",\"sha256\":\"e2953cfcd106288beb7df6d83a8959c55110cfe2dcefef5a4e66000b611ce5b0\"},\"mockup\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_021/artifact/mockup.png\",\"sha256\":\"f7557f5732a91451730ea13be5b79039aad13617ce4aaea29f0999c9781fa111\",\"publicSrc\":\"assets/lvyo_bv0_candidate_021_m4_mockup.webp\"},\"referenceImage\":null,\"noSpendReplay\":true,\"customerLiveFlowChanged\":false}", "images": [ { "mimeType": "image/webp", "blobRef": { "uri": "promptfoo://blob/fc79b81edbce55d887efe5df3e29ec9652a0ab93f8f4ac95274073728d44acdb", "hash": "fc79b81edbce55d887efe5df3e29ec9652a0ab93f8f4ac95274073728d44acdb", "mimeType": "image/webp", "sizeBytes": 58176, "provider": "filesystem" } } ], "metadata": { "caseId": "lvyo_bv0_candidate_021", "modality": "text_only", "cluster": "garden party", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_021/artifact-lock.json", "finalSha256": "e2953cfcd106288beb7df6d83a8959c55110cfe2dcefef5a4e66000b611ce5b0", "mockupSha256": "f7557f5732a91451730ea13be5b79039aad13617ce4aaea29f0999c9781fa111", "publicMockup": "assets/lvyo_bv0_candidate_021_m4_mockup.webp", "noSpendReplay": true } }, "score": 1, "success": true, "testCase": { "description": "21 text_only garden party lvyo_bv0_candidate_021", "vars": { "caseId": "lvyo_bv0_candidate_021", "customerInput": "Garden party tee for my mum's 50th, floral but funny, says Rosie's Gin Garden Est. 1976", "modality": "text_only", "cluster": "garden party" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_021\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ], "options": {}, "metadata": {} }, "testIdx": 20, "tokenUsage": { "prompt": 0, "completion": 0, "cached": 0, "total": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 }, "assertions": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 } } }, "vars": { "caseId": "lvyo_bv0_candidate_021", "customerInput": "Garden party tee for my mum's 50th, floral but funny, says Rosie's Gin Garden Est. 1976", "modality": "text_only", "cluster": "garden party" }, "metadata": { "caseId": "lvyo_bv0_candidate_021", "modality": "text_only", "cluster": "garden party", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_021/artifact-lock.json", "finalSha256": "e2953cfcd106288beb7df6d83a8959c55110cfe2dcefef5a4e66000b611ce5b0", "mockupSha256": "f7557f5732a91451730ea13be5b79039aad13617ce4aaea29f0999c9781fa111", "publicMockup": "assets/lvyo_bv0_candidate_021_m4_mockup.webp", "noSpendReplay": true, "_promptfooFileMetadata": {} }, "failureReason": 0 }, { "cost": 0, "gradingResult": { "pass": true, "score": 1, "reason": "All assertions passed", "namedScores": {}, "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 0 }, "componentResults": [ { "pass": true, "score": 1, "reason": "Assertion passed", "assertion": { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_022\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } } ] }, "id": "a60bb600-d097-46ab-a027-b99f3dda6203", "latencyMs": 0, "namedScores": {}, "prompt": { "raw": "Gaming squad shirt for our Discord group, Midnight Lobby Survivors, cyber but cosy, no skulls", "label": "{{customerInput}}", "config": {} }, "promptId": "7e756b499a1da155962cf88585b671e3ddacc5edf410c782ea1dbff2b92d0a70", "promptIdx": 0, "provider": { "id": "file://./providers/benchmark-v0-replay-provider.mjs", "label": "levyo-benchmark-v0-artifact-replay" }, "response": { "output": "{\"ok\":true,\"schemaVersion\":\"levyo_promptfoo_benchmark_v0_replay_result_v1\",\"caseId\":\"lvyo_bv0_candidate_022\",\"promptEcho\":\"Gaming squad shirt for our Discord group, Midnight Lobby Survivors, cyber but cosy, no skulls\",\"status\":\"success\",\"modality\":\"text_only\",\"caseRole\":\"text_core\",\"cluster\":\"gaming squad\",\"runtimeProfile\":\"internal_codex\",\"imageProvider\":\"codex_account_ui\",\"lockRef\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_022/artifact-lock.json\",\"lockSha256\":\"a9ed16a8855778095a65f6d57d533dd0165e9a4ef5adccba834066418188d38f\",\"final\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_022/artifact/final.png\",\"sha256\":\"b5b027e773570745dec914ab08447f1cab3ce0ed1ae0a4c5092cc4e1ab17a196\"},\"mockup\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_022/artifact/mockup.png\",\"sha256\":\"333476e0e5651d385f98597d04d8a7770f4dd9abffd5845584a021452084c332\",\"publicSrc\":\"assets/lvyo_bv0_candidate_022_m4_mockup.webp\"},\"referenceImage\":null,\"noSpendReplay\":true,\"customerLiveFlowChanged\":false}", "images": [ { "mimeType": "image/webp", "blobRef": { "uri": "promptfoo://blob/23a05fc04b3bd30e5ef71e15956ded188565726aa2276b8b8c8d8cc4da1dfc44", "hash": "23a05fc04b3bd30e5ef71e15956ded188565726aa2276b8b8c8d8cc4da1dfc44", "mimeType": "image/webp", "sizeBytes": 47502, "provider": "filesystem" } } ], "metadata": { "caseId": "lvyo_bv0_candidate_022", "modality": "text_only", "cluster": "gaming squad", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_022/artifact-lock.json", "finalSha256": "b5b027e773570745dec914ab08447f1cab3ce0ed1ae0a4c5092cc4e1ab17a196", "mockupSha256": "333476e0e5651d385f98597d04d8a7770f4dd9abffd5845584a021452084c332", "publicMockup": "assets/lvyo_bv0_candidate_022_m4_mockup.webp", "noSpendReplay": true } }, "score": 1, "success": true, "testCase": { "description": "22 text_only gaming squad lvyo_bv0_candidate_022", "vars": { "caseId": "lvyo_bv0_candidate_022", "customerInput": "Gaming squad shirt for our Discord group, Midnight Lobby Survivors, cyber but cosy, no skulls", "modality": "text_only", "cluster": "gaming squad" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_022\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ], "options": {}, "metadata": {} }, "testIdx": 21, "tokenUsage": { "prompt": 0, "completion": 0, "cached": 0, "total": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 }, "assertions": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 } } }, "vars": { "caseId": "lvyo_bv0_candidate_022", "customerInput": "Gaming squad shirt for our Discord group, Midnight Lobby Survivors, cyber but cosy, no skulls", "modality": "text_only", "cluster": "gaming squad" }, "metadata": { "caseId": "lvyo_bv0_candidate_022", "modality": "text_only", "cluster": "gaming squad", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_022/artifact-lock.json", "finalSha256": "b5b027e773570745dec914ab08447f1cab3ce0ed1ae0a4c5092cc4e1ab17a196", "mockupSha256": "333476e0e5651d385f98597d04d8a7770f4dd9abffd5845584a021452084c332", "publicMockup": "assets/lvyo_bv0_candidate_022_m4_mockup.webp", "noSpendReplay": true, "_promptfooFileMetadata": {} }, "failureReason": 0 }, { "cost": 0, "gradingResult": { "pass": true, "score": 1, "reason": "All assertions passed", "namedScores": {}, "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 0 }, "componentResults": [ { "pass": true, "score": 1, "reason": "Assertion passed", "assertion": { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_023\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } } ] }, "id": "3f96f0ff-e901-4e94-aad6-8ff2305320f7", "latencyMs": 0, "namedScores": {}, "prompt": { "raw": "Fantasy football draft night shirt. League name: Spreadsheet Sadness. Need a fake trophy and tiny crying chart", "label": "{{customerInput}}", "config": {} }, "promptId": "7e756b499a1da155962cf88585b671e3ddacc5edf410c782ea1dbff2b92d0a70", "promptIdx": 0, "provider": { "id": "file://./providers/benchmark-v0-replay-provider.mjs", "label": "levyo-benchmark-v0-artifact-replay" }, "response": { "output": "{\"ok\":true,\"schemaVersion\":\"levyo_promptfoo_benchmark_v0_replay_result_v1\",\"caseId\":\"lvyo_bv0_candidate_023\",\"promptEcho\":\"Fantasy football draft night shirt. League name: Spreadsheet Sadness. Need a fake trophy and tiny crying chart\",\"status\":\"success\",\"modality\":\"text_only\",\"caseRole\":\"text_core\",\"cluster\":\"fantasy draft\",\"runtimeProfile\":\"internal_codex\",\"imageProvider\":\"codex_account_ui\",\"lockRef\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_023/artifact-lock.json\",\"lockSha256\":\"4186845f6ff63482cbd0e1a91201375ee95badbdda0a5decef7071826aee0326\",\"final\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_023/artifact/final.png\",\"sha256\":\"d89bf3a8fb501695d72a9a64d4db318cc1b341b51254b63b8b5afd7316223db0\"},\"mockup\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_023/artifact/mockup.png\",\"sha256\":\"025933b275b0a0a4ee716c93dc385718c01c5c80dea97d846f6f4bf03df147fb\",\"publicSrc\":\"assets/lvyo_bv0_candidate_023_m4_mockup.webp\"},\"referenceImage\":null,\"noSpendReplay\":true,\"customerLiveFlowChanged\":false}", "images": [ { "mimeType": "image/webp", "blobRef": { "uri": "promptfoo://blob/4610f457230d799924d4220ba5c8cb61ad34667b03458384e0d145e4a67230c6", "hash": "4610f457230d799924d4220ba5c8cb61ad34667b03458384e0d145e4a67230c6", "mimeType": "image/webp", "sizeBytes": 42586, "provider": "filesystem" } } ], "metadata": { "caseId": "lvyo_bv0_candidate_023", "modality": "text_only", "cluster": "fantasy draft", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_023/artifact-lock.json", "finalSha256": "d89bf3a8fb501695d72a9a64d4db318cc1b341b51254b63b8b5afd7316223db0", "mockupSha256": "025933b275b0a0a4ee716c93dc385718c01c5c80dea97d846f6f4bf03df147fb", "publicMockup": "assets/lvyo_bv0_candidate_023_m4_mockup.webp", "noSpendReplay": true } }, "score": 1, "success": true, "testCase": { "description": "23 text_only fantasy draft lvyo_bv0_candidate_023", "vars": { "caseId": "lvyo_bv0_candidate_023", "customerInput": "Fantasy football draft night shirt. League name: Spreadsheet Sadness. Need a fake trophy and tiny crying chart", "modality": "text_only", "cluster": "fantasy draft" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_023\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ], "options": {}, "metadata": {} }, "testIdx": 22, "tokenUsage": { "prompt": 0, "completion": 0, "cached": 0, "total": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 }, "assertions": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 } } }, "vars": { "caseId": "lvyo_bv0_candidate_023", "customerInput": "Fantasy football draft night shirt. League name: Spreadsheet Sadness. Need a fake trophy and tiny crying chart", "modality": "text_only", "cluster": "fantasy draft" }, "metadata": { "caseId": "lvyo_bv0_candidate_023", "modality": "text_only", "cluster": "fantasy draft", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_023/artifact-lock.json", "finalSha256": "d89bf3a8fb501695d72a9a64d4db318cc1b341b51254b63b8b5afd7316223db0", "mockupSha256": "025933b275b0a0a4ee716c93dc385718c01c5c80dea97d846f6f4bf03df147fb", "publicMockup": "assets/lvyo_bv0_candidate_023_m4_mockup.webp", "noSpendReplay": true, "_promptfooFileMetadata": {} }, "failureReason": 0 }, { "cost": 0, "gradingResult": { "pass": true, "score": 1, "reason": "All assertions passed", "namedScores": {}, "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 0 }, "componentResults": [ { "pass": true, "score": 1, "reason": "Assertion passed", "assertion": { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_024\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } } ] }, "id": "35db7b56-d821-426a-849e-58804755e60d", "latencyMs": 0, "namedScores": {}, "prompt": { "raw": "Pub quiz team tee, name is Agatha Quiztie. Make it clever, vintage detective style", "label": "{{customerInput}}", "config": {} }, "promptId": "7e756b499a1da155962cf88585b671e3ddacc5edf410c782ea1dbff2b92d0a70", "promptIdx": 0, "provider": { "id": "file://./providers/benchmark-v0-replay-provider.mjs", "label": "levyo-benchmark-v0-artifact-replay" }, "response": { "output": "{\"ok\":true,\"schemaVersion\":\"levyo_promptfoo_benchmark_v0_replay_result_v1\",\"caseId\":\"lvyo_bv0_candidate_024\",\"promptEcho\":\"Pub quiz team tee, name is Agatha Quiztie. Make it clever, vintage detective style\",\"status\":\"success\",\"modality\":\"text_only\",\"caseRole\":\"text_core\",\"cluster\":\"pub quiz team\",\"runtimeProfile\":\"internal_codex\",\"imageProvider\":\"codex_account_ui\",\"lockRef\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_024/artifact-lock.json\",\"lockSha256\":\"024deb5b679bcdfb64e233936fcde7c04da4d58ccad930f8a8ca159229a4ac70\",\"final\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_024/artifact/final.png\",\"sha256\":\"5dd836938e6f3f3181b02b9f47af8e194e8bd8783032b17a95d3f4268babf00b\"},\"mockup\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_024/artifact/mockup.png\",\"sha256\":\"aed13fcc3db9b5a011931c186ca2d9783eaecbbce493e0aa974b9bcdd3a00a74\",\"publicSrc\":\"assets/lvyo_bv0_candidate_024_m4_mockup.webp\"},\"referenceImage\":null,\"noSpendReplay\":true,\"customerLiveFlowChanged\":false}", "images": [ { "mimeType": "image/webp", "blobRef": { "uri": "promptfoo://blob/88ab911a5b9b3217b2776af2a582f9ed79cbaa56640962919b261d906477e6e3", "hash": "88ab911a5b9b3217b2776af2a582f9ed79cbaa56640962919b261d906477e6e3", "mimeType": "image/webp", "sizeBytes": 38574, "provider": "filesystem" } } ], "metadata": { "caseId": "lvyo_bv0_candidate_024", "modality": "text_only", "cluster": "pub quiz team", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_024/artifact-lock.json", "finalSha256": "5dd836938e6f3f3181b02b9f47af8e194e8bd8783032b17a95d3f4268babf00b", "mockupSha256": "aed13fcc3db9b5a011931c186ca2d9783eaecbbce493e0aa974b9bcdd3a00a74", "publicMockup": "assets/lvyo_bv0_candidate_024_m4_mockup.webp", "noSpendReplay": true } }, "score": 1, "success": true, "testCase": { "description": "24 text_only pub quiz team lvyo_bv0_candidate_024", "vars": { "caseId": "lvyo_bv0_candidate_024", "customerInput": "Pub quiz team tee, name is Agatha Quiztie. Make it clever, vintage detective style", "modality": "text_only", "cluster": "pub quiz team" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_024\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ], "options": {}, "metadata": {} }, "testIdx": 23, "tokenUsage": { "prompt": 0, "completion": 0, "cached": 0, "total": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 }, "assertions": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 } } }, "vars": { "caseId": "lvyo_bv0_candidate_024", "customerInput": "Pub quiz team tee, name is Agatha Quiztie. Make it clever, vintage detective style", "modality": "text_only", "cluster": "pub quiz team" }, "metadata": { "caseId": "lvyo_bv0_candidate_024", "modality": "text_only", "cluster": "pub quiz team", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_024/artifact-lock.json", "finalSha256": "5dd836938e6f3f3181b02b9f47af8e194e8bd8783032b17a95d3f4268babf00b", "mockupSha256": "aed13fcc3db9b5a011931c186ca2d9783eaecbbce493e0aa974b9bcdd3a00a74", "publicMockup": "assets/lvyo_bv0_candidate_024_m4_mockup.webp", "noSpendReplay": true, "_promptfooFileMetadata": {} }, "failureReason": 0 }, { "cost": 0, "gradingResult": { "pass": true, "score": 1, "reason": "All assertions passed", "namedScores": {}, "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 0 }, "componentResults": [ { "pass": true, "score": 1, "reason": "Assertion passed", "assertion": { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_025\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } } ] }, "id": "4d50bdb1-0e59-449a-9806-0971c602aeac", "latencyMs": 1, "namedScores": {}, "prompt": { "raw": "Team offsite shirt for a startup retreat in Lisbon. Less corporate, more inside-joke: ship it then sip it", "label": "{{customerInput}}", "config": {} }, "promptId": "7e756b499a1da155962cf88585b671e3ddacc5edf410c782ea1dbff2b92d0a70", "promptIdx": 0, "provider": { "id": "file://./providers/benchmark-v0-replay-provider.mjs", "label": "levyo-benchmark-v0-artifact-replay" }, "response": { "output": "{\"ok\":true,\"schemaVersion\":\"levyo_promptfoo_benchmark_v0_replay_result_v1\",\"caseId\":\"lvyo_bv0_candidate_025\",\"promptEcho\":\"Team offsite shirt for a startup retreat in Lisbon. Less corporate, more inside-joke: ship it then sip it\",\"status\":\"success\",\"modality\":\"text_only\",\"caseRole\":\"text_core\",\"cluster\":\"company retreat\",\"runtimeProfile\":\"internal_codex\",\"imageProvider\":\"codex_account_ui\",\"lockRef\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_025/artifact-lock.json\",\"lockSha256\":\"ff214346a6b5847d654d431b7f171a1c2954e0168d229d60130f4c13be5ee008\",\"final\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_025/artifact/final.png\",\"sha256\":\"ba58e453a0626e4916de6bcb2f4df7ddcbcab863758a6f3d62ffa9564d0d2992\"},\"mockup\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_025/artifact/mockup.png\",\"sha256\":\"711dc0eda2c0fa509d036bb088f62806f944a0250049df8d019edfe7fedc61ba\",\"publicSrc\":\"assets/lvyo_bv0_candidate_025_m4_mockup.webp\"},\"referenceImage\":null,\"noSpendReplay\":true,\"customerLiveFlowChanged\":false}", "images": [ { "mimeType": "image/webp", "blobRef": { "uri": "promptfoo://blob/12647a632588ee62069faabadfcf38ceb141aa5cb83a19ddf7be986c0447e2fc", "hash": "12647a632588ee62069faabadfcf38ceb141aa5cb83a19ddf7be986c0447e2fc", "mimeType": "image/webp", "sizeBytes": 37846, "provider": "filesystem" } } ], "metadata": { "caseId": "lvyo_bv0_candidate_025", "modality": "text_only", "cluster": "company retreat", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_025/artifact-lock.json", "finalSha256": "ba58e453a0626e4916de6bcb2f4df7ddcbcab863758a6f3d62ffa9564d0d2992", "mockupSha256": "711dc0eda2c0fa509d036bb088f62806f944a0250049df8d019edfe7fedc61ba", "publicMockup": "assets/lvyo_bv0_candidate_025_m4_mockup.webp", "noSpendReplay": true } }, "score": 1, "success": true, "testCase": { "description": "25 text_only company retreat lvyo_bv0_candidate_025", "vars": { "caseId": "lvyo_bv0_candidate_025", "customerInput": "Team offsite shirt for a startup retreat in Lisbon. Less corporate, more inside-joke: ship it then sip it", "modality": "text_only", "cluster": "company retreat" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_025\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ], "options": {}, "metadata": {} }, "testIdx": 24, "tokenUsage": { "prompt": 0, "completion": 0, "cached": 0, "total": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 }, "assertions": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 } } }, "vars": { "caseId": "lvyo_bv0_candidate_025", "customerInput": "Team offsite shirt for a startup retreat in Lisbon. Less corporate, more inside-joke: ship it then sip it", "modality": "text_only", "cluster": "company retreat" }, "metadata": { "caseId": "lvyo_bv0_candidate_025", "modality": "text_only", "cluster": "company retreat", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_025/artifact-lock.json", "finalSha256": "ba58e453a0626e4916de6bcb2f4df7ddcbcab863758a6f3d62ffa9564d0d2992", "mockupSha256": "711dc0eda2c0fa509d036bb088f62806f944a0250049df8d019edfe7fedc61ba", "publicMockup": "assets/lvyo_bv0_candidate_025_m4_mockup.webp", "noSpendReplay": true, "_promptfooFileMetadata": {} }, "failureReason": 0 }, { "cost": 0, "gradingResult": { "pass": true, "score": 1, "reason": "All assertions passed", "namedScores": {}, "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 0 }, "componentResults": [ { "pass": true, "score": 1, "reason": "Assertion passed", "assertion": { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_026\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } } ] }, "id": "f6de1bdb-2dbf-44c6-8a59-80d40cd5bc4f", "latencyMs": 1, "namedScores": {}, "prompt": { "raw": "Hackathon team tee for 48 hours no sleep, project name Patch Panic, make it look like an error screen but wearable", "label": "{{customerInput}}", "config": {} }, "promptId": "7e756b499a1da155962cf88585b671e3ddacc5edf410c782ea1dbff2b92d0a70", "promptIdx": 0, "provider": { "id": "file://./providers/benchmark-v0-replay-provider.mjs", "label": "levyo-benchmark-v0-artifact-replay" }, "response": { "output": "{\"ok\":true,\"schemaVersion\":\"levyo_promptfoo_benchmark_v0_replay_result_v1\",\"caseId\":\"lvyo_bv0_candidate_026\",\"promptEcho\":\"Hackathon team tee for 48 hours no sleep, project name Patch Panic, make it look like an error screen but wearable\",\"status\":\"success\",\"modality\":\"text_only\",\"caseRole\":\"text_core\",\"cluster\":\"hackathon\",\"runtimeProfile\":\"internal_codex\",\"imageProvider\":\"codex_account_ui\",\"lockRef\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_026/artifact-lock.json\",\"lockSha256\":\"a0172125bae705981e9bec13aebc4af2bfea7f02b8856bd150bc304a88415081\",\"final\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_026/artifact/final.png\",\"sha256\":\"5811659b84ef5faf2efab9ab185c5d7adc03f5b3037a2d52b2dd40c84a4bc6a0\"},\"mockup\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_026/artifact/mockup.png\",\"sha256\":\"1a1fb45d9f6f1ff06b21ac5745f8d750131289f8810cc6a726fdda46d44b32be\",\"publicSrc\":\"assets/lvyo_bv0_candidate_026_m4_mockup.webp\"},\"referenceImage\":null,\"noSpendReplay\":true,\"customerLiveFlowChanged\":false}", "images": [ { "mimeType": "image/webp", "blobRef": { "uri": "promptfoo://blob/5aa5b0fda602fa0411d624a6faa06c3c80e38e79a1de6fe0bc423d83db068c59", "hash": "5aa5b0fda602fa0411d624a6faa06c3c80e38e79a1de6fe0bc423d83db068c59", "mimeType": "image/webp", "sizeBytes": 39352, "provider": "filesystem" } } ], "metadata": { "caseId": "lvyo_bv0_candidate_026", "modality": "text_only", "cluster": "hackathon", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_026/artifact-lock.json", "finalSha256": "5811659b84ef5faf2efab9ab185c5d7adc03f5b3037a2d52b2dd40c84a4bc6a0", "mockupSha256": "1a1fb45d9f6f1ff06b21ac5745f8d750131289f8810cc6a726fdda46d44b32be", "publicMockup": "assets/lvyo_bv0_candidate_026_m4_mockup.webp", "noSpendReplay": true } }, "score": 1, "success": true, "testCase": { "description": "26 text_only hackathon lvyo_bv0_candidate_026", "vars": { "caseId": "lvyo_bv0_candidate_026", "customerInput": "Hackathon team tee for 48 hours no sleep, project name Patch Panic, make it look like an error screen but wearable", "modality": "text_only", "cluster": "hackathon" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_026\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ], "options": {}, "metadata": {} }, "testIdx": 25, "tokenUsage": { "prompt": 0, "completion": 0, "cached": 0, "total": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 }, "assertions": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 } } }, "vars": { "caseId": "lvyo_bv0_candidate_026", "customerInput": "Hackathon team tee for 48 hours no sleep, project name Patch Panic, make it look like an error screen but wearable", "modality": "text_only", "cluster": "hackathon" }, "metadata": { "caseId": "lvyo_bv0_candidate_026", "modality": "text_only", "cluster": "hackathon", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_026/artifact-lock.json", "finalSha256": "5811659b84ef5faf2efab9ab185c5d7adc03f5b3037a2d52b2dd40c84a4bc6a0", "mockupSha256": "1a1fb45d9f6f1ff06b21ac5745f8d750131289f8810cc6a726fdda46d44b32be", "publicMockup": "assets/lvyo_bv0_candidate_026_m4_mockup.webp", "noSpendReplay": true, "_promptfooFileMetadata": {} }, "failureReason": 0 }, { "cost": 0, "gradingResult": { "pass": true, "score": 1, "reason": "All assertions passed", "namedScores": {}, "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 0 }, "componentResults": [ { "pass": true, "score": 1, "reason": "Assertion passed", "assertion": { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_027\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } } ] }, "id": "33c59d35-a3cb-4e3b-8ba4-8eeb77698b6c", "latencyMs": 1, "namedScores": {}, "prompt": { "raw": "Small launch party T-shirt for our bakery app, 'crumbs deployed successfully', cute but not childish", "label": "{{customerInput}}", "config": {} }, "promptId": "7e756b499a1da155962cf88585b671e3ddacc5edf410c782ea1dbff2b92d0a70", "promptIdx": 0, "provider": { "id": "file://./providers/benchmark-v0-replay-provider.mjs", "label": "levyo-benchmark-v0-artifact-replay" }, "response": { "output": "{\"ok\":true,\"schemaVersion\":\"levyo_promptfoo_benchmark_v0_replay_result_v1\",\"caseId\":\"lvyo_bv0_candidate_027\",\"promptEcho\":\"Small launch party T-shirt for our bakery app, 'crumbs deployed successfully', cute but not childish\",\"status\":\"success\",\"modality\":\"text_only\",\"caseRole\":\"text_core\",\"cluster\":\"launch party\",\"runtimeProfile\":\"internal_codex\",\"imageProvider\":\"codex_account_ui\",\"lockRef\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_027/artifact-lock.json\",\"lockSha256\":\"07e07c62eb607d07a52e53ca155641e89144bc5db0048d85bd85a1de3dd54764\",\"final\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_027/artifact/final.png\",\"sha256\":\"e1067d5937b214918ae7e1ff1ba8bf3791a149aa47412335fdda916dd37db531\"},\"mockup\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_027/artifact/mockup.png\",\"sha256\":\"d5af81e4b782e5e719b3259cec4fefe5fe1d3d33feff18e24ed6fe48e806e420\",\"publicSrc\":\"assets/lvyo_bv0_candidate_027_m4_mockup.webp\"},\"referenceImage\":null,\"noSpendReplay\":true,\"customerLiveFlowChanged\":false}", "images": [ { "mimeType": "image/webp", "blobRef": { "uri": "promptfoo://blob/b8a0bedb46ac45db4be6f386de55904e1705fbdedb77d7130726143b7b4c40e0", "hash": "b8a0bedb46ac45db4be6f386de55904e1705fbdedb77d7130726143b7b4c40e0", "mimeType": "image/webp", "sizeBytes": 44954, "provider": "filesystem" } } ], "metadata": { "caseId": "lvyo_bv0_candidate_027", "modality": "text_only", "cluster": "launch party", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_027/artifact-lock.json", "finalSha256": "e1067d5937b214918ae7e1ff1ba8bf3791a149aa47412335fdda916dd37db531", "mockupSha256": "d5af81e4b782e5e719b3259cec4fefe5fe1d3d33feff18e24ed6fe48e806e420", "publicMockup": "assets/lvyo_bv0_candidate_027_m4_mockup.webp", "noSpendReplay": true } }, "score": 1, "success": true, "testCase": { "description": "27 text_only launch party lvyo_bv0_candidate_027", "vars": { "caseId": "lvyo_bv0_candidate_027", "customerInput": "Small launch party T-shirt for our bakery app, 'crumbs deployed successfully', cute but not childish", "modality": "text_only", "cluster": "launch party" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_027\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ], "options": {}, "metadata": {} }, "testIdx": 26, "tokenUsage": { "prompt": 0, "completion": 0, "cached": 0, "total": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 }, "assertions": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 } } }, "vars": { "caseId": "lvyo_bv0_candidate_027", "customerInput": "Small launch party T-shirt for our bakery app, 'crumbs deployed successfully', cute but not childish", "modality": "text_only", "cluster": "launch party" }, "metadata": { "caseId": "lvyo_bv0_candidate_027", "modality": "text_only", "cluster": "launch party", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_027/artifact-lock.json", "finalSha256": "e1067d5937b214918ae7e1ff1ba8bf3791a149aa47412335fdda916dd37db531", "mockupSha256": "d5af81e4b782e5e719b3259cec4fefe5fe1d3d33feff18e24ed6fe48e806e420", "publicMockup": "assets/lvyo_bv0_candidate_027_m4_mockup.webp", "noSpendReplay": true, "_promptfooFileMetadata": {} }, "failureReason": 0 }, { "cost": 0, "gradingResult": { "pass": true, "score": 1, "reason": "All assertions passed", "namedScores": {}, "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 0 }, "componentResults": [ { "pass": true, "score": 1, "reason": "Assertion passed", "assertion": { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_028\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } } ] }, "id": "84fad967-7063-467b-b6a4-97d81234502c", "latencyMs": 1, "namedScores": {}, "prompt": { "raw": "Charity 10k tops for Team Nora. Warm, hopeful, with trainers and a sunrise, not too sad", "label": "{{customerInput}}", "config": {} }, "promptId": "7e756b499a1da155962cf88585b671e3ddacc5edf410c782ea1dbff2b92d0a70", "promptIdx": 0, "provider": { "id": "file://./providers/benchmark-v0-replay-provider.mjs", "label": "levyo-benchmark-v0-artifact-replay" }, "response": { "output": "{\"ok\":true,\"schemaVersion\":\"levyo_promptfoo_benchmark_v0_replay_result_v1\",\"caseId\":\"lvyo_bv0_candidate_028\",\"promptEcho\":\"Charity 10k tops for Team Nora. Warm, hopeful, with trainers and a sunrise, not too sad\",\"status\":\"success\",\"modality\":\"text_only\",\"caseRole\":\"text_core\",\"cluster\":\"charity run\",\"runtimeProfile\":\"internal_codex\",\"imageProvider\":\"codex_account_ui\",\"lockRef\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_028/artifact-lock.json\",\"lockSha256\":\"fba05f0883b8eafb88e7dc8cb76540a4f81dca391310caa5be74be9e902a4cac\",\"final\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_028/artifact/final.png\",\"sha256\":\"662740ed775404118e3ad9e096344732715478f9a942945b4b3ae6be5461a73c\"},\"mockup\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_028/artifact/mockup.png\",\"sha256\":\"8c98b73998d1024e8990d6f860c3b5d3b36301e18d9819c065bdd24b967df012\",\"publicSrc\":\"assets/lvyo_bv0_candidate_028_m4_mockup.webp\"},\"referenceImage\":null,\"noSpendReplay\":true,\"customerLiveFlowChanged\":false}", "images": [ { "mimeType": "image/webp", "blobRef": { "uri": "promptfoo://blob/5942333f2dc4f1823aab4abce888a71072324eb9f53d05bdebb7f090e1c197af", "hash": "5942333f2dc4f1823aab4abce888a71072324eb9f53d05bdebb7f090e1c197af", "mimeType": "image/webp", "sizeBytes": 44296, "provider": "filesystem" } } ], "metadata": { "caseId": "lvyo_bv0_candidate_028", "modality": "text_only", "cluster": "charity run", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_028/artifact-lock.json", "finalSha256": "662740ed775404118e3ad9e096344732715478f9a942945b4b3ae6be5461a73c", "mockupSha256": "8c98b73998d1024e8990d6f860c3b5d3b36301e18d9819c065bdd24b967df012", "publicMockup": "assets/lvyo_bv0_candidate_028_m4_mockup.webp", "noSpendReplay": true } }, "score": 1, "success": true, "testCase": { "description": "28 text_only charity run lvyo_bv0_candidate_028", "vars": { "caseId": "lvyo_bv0_candidate_028", "customerInput": "Charity 10k tops for Team Nora. Warm, hopeful, with trainers and a sunrise, not too sad", "modality": "text_only", "cluster": "charity run" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_028\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ], "options": {}, "metadata": {} }, "testIdx": 27, "tokenUsage": { "prompt": 0, "completion": 0, "cached": 0, "total": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 }, "assertions": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 } } }, "vars": { "caseId": "lvyo_bv0_candidate_028", "customerInput": "Charity 10k tops for Team Nora. Warm, hopeful, with trainers and a sunrise, not too sad", "modality": "text_only", "cluster": "charity run" }, "metadata": { "caseId": "lvyo_bv0_candidate_028", "modality": "text_only", "cluster": "charity run", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_028/artifact-lock.json", "finalSha256": "662740ed775404118e3ad9e096344732715478f9a942945b4b3ae6be5461a73c", "mockupSha256": "8c98b73998d1024e8990d6f860c3b5d3b36301e18d9819c065bdd24b967df012", "publicMockup": "assets/lvyo_bv0_candidate_028_m4_mockup.webp", "noSpendReplay": true, "_promptfooFileMetadata": {} }, "failureReason": 0 }, { "cost": 0, "gradingResult": { "pass": true, "score": 1, "reason": "All assertions passed", "namedScores": {}, "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 0 }, "componentResults": [ { "pass": true, "score": 1, "reason": "Assertion passed", "assertion": { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_029\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } } ] }, "id": "41b9d8ff-933e-48f7-b002-33b8ddca8ffd", "latencyMs": 1, "namedScores": {}, "prompt": { "raw": "Volunteer crew shirts for community clean up day. Make it say Litter Quitters and include gloves/bags somehow", "label": "{{customerInput}}", "config": {} }, "promptId": "7e756b499a1da155962cf88585b671e3ddacc5edf410c782ea1dbff2b92d0a70", "promptIdx": 0, "provider": { "id": "file://./providers/benchmark-v0-replay-provider.mjs", "label": "levyo-benchmark-v0-artifact-replay" }, "response": { "output": "{\"ok\":true,\"schemaVersion\":\"levyo_promptfoo_benchmark_v0_replay_result_v1\",\"caseId\":\"lvyo_bv0_candidate_029\",\"promptEcho\":\"Volunteer crew shirts for community clean up day. Make it say Litter Quitters and include gloves/bags somehow\",\"status\":\"success\",\"modality\":\"text_only\",\"caseRole\":\"text_core\",\"cluster\":\"volunteer crew\",\"runtimeProfile\":\"internal_codex\",\"imageProvider\":\"codex_account_ui\",\"lockRef\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_029/artifact-lock.json\",\"lockSha256\":\"20cd5920066b8c33bb63f2c168ee35fbb32471f6dbd02380c709f24883fcd51f\",\"final\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_029/artifact/final.png\",\"sha256\":\"5afbd6a045bb07946e2dada3c406da0973ab4065874bf32f32a38e31d53d44d5\"},\"mockup\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_029/artifact/mockup.png\",\"sha256\":\"475a89189e70215db3c5b1b645bed6234d6aec1aadbefe5bbade5c95a5555d9b\",\"publicSrc\":\"assets/lvyo_bv0_candidate_029_m4_mockup.webp\"},\"referenceImage\":null,\"noSpendReplay\":true,\"customerLiveFlowChanged\":false}", "images": [ { "mimeType": "image/webp", "blobRef": { "uri": "promptfoo://blob/c9660ade52f3e0310b8ea4248acbdcff31d88fbd9ec569bf53035624d4b1ffdd", "hash": "c9660ade52f3e0310b8ea4248acbdcff31d88fbd9ec569bf53035624d4b1ffdd", "mimeType": "image/webp", "sizeBytes": 45810, "provider": "filesystem" } } ], "metadata": { "caseId": "lvyo_bv0_candidate_029", "modality": "text_only", "cluster": "volunteer crew", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_029/artifact-lock.json", "finalSha256": "5afbd6a045bb07946e2dada3c406da0973ab4065874bf32f32a38e31d53d44d5", "mockupSha256": "475a89189e70215db3c5b1b645bed6234d6aec1aadbefe5bbade5c95a5555d9b", "publicMockup": "assets/lvyo_bv0_candidate_029_m4_mockup.webp", "noSpendReplay": true } }, "score": 1, "success": true, "testCase": { "description": "29 text_only volunteer crew lvyo_bv0_candidate_029", "vars": { "caseId": "lvyo_bv0_candidate_029", "customerInput": "Volunteer crew shirts for community clean up day. Make it say Litter Quitters and include gloves/bags somehow", "modality": "text_only", "cluster": "volunteer crew" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_029\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ], "options": {}, "metadata": {} }, "testIdx": 28, "tokenUsage": { "prompt": 0, "completion": 0, "cached": 0, "total": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 }, "assertions": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 } } }, "vars": { "caseId": "lvyo_bv0_candidate_029", "customerInput": "Volunteer crew shirts for community clean up day. Make it say Litter Quitters and include gloves/bags somehow", "modality": "text_only", "cluster": "volunteer crew" }, "metadata": { "caseId": "lvyo_bv0_candidate_029", "modality": "text_only", "cluster": "volunteer crew", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_029/artifact-lock.json", "finalSha256": "5afbd6a045bb07946e2dada3c406da0973ab4065874bf32f32a38e31d53d44d5", "mockupSha256": "475a89189e70215db3c5b1b645bed6234d6aec1aadbefe5bbade5c95a5555d9b", "publicMockup": "assets/lvyo_bv0_candidate_029_m4_mockup.webp", "noSpendReplay": true, "_promptfooFileMetadata": {} }, "failureReason": 0 }, { "cost": 0, "gradingResult": { "pass": true, "score": 1, "reason": "All assertions passed", "namedScores": {}, "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 0 }, "componentResults": [ { "pass": true, "score": 1, "reason": "Assertion passed", "assertion": { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_030\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } } ] }, "id": "0edca230-162b-4bb8-a1c0-b2839c0739f5", "latencyMs": 1, "namedScores": {}, "prompt": { "raw": "Wedding weekend shirt for the guests, Max & Priya, Cornwall 2026, subtle seaside illustration please", "label": "{{customerInput}}", "config": {} }, "promptId": "7e756b499a1da155962cf88585b671e3ddacc5edf410c782ea1dbff2b92d0a70", "promptIdx": 0, "provider": { "id": "file://./providers/benchmark-v0-replay-provider.mjs", "label": "levyo-benchmark-v0-artifact-replay" }, "response": { "output": "{\"ok\":true,\"schemaVersion\":\"levyo_promptfoo_benchmark_v0_replay_result_v1\",\"caseId\":\"lvyo_bv0_candidate_030\",\"promptEcho\":\"Wedding weekend shirt for the guests, Max & Priya, Cornwall 2026, subtle seaside illustration please\",\"status\":\"success\",\"modality\":\"text_only\",\"caseRole\":\"text_core\",\"cluster\":\"wedding weekend\",\"runtimeProfile\":\"internal_codex\",\"imageProvider\":\"codex_account_ui\",\"lockRef\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_030/artifact-lock.json\",\"lockSha256\":\"b6afe7fb98c1c7d5a0bca9d23d13826d2d67d30954a8e92e559d3ac90c75f3c1\",\"final\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_030/artifact/final.png\",\"sha256\":\"2b412eb776a72e07af63700dbcd144e155614ce1a7db11c17b6d22ae91b2ba07\"},\"mockup\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_030/artifact/mockup.png\",\"sha256\":\"cccc4097a95a9a2949863d8d8b70c8d1592a404f63926aea107ee5069a3db5d8\",\"publicSrc\":\"assets/lvyo_bv0_candidate_030_m4_mockup.webp\"},\"referenceImage\":null,\"noSpendReplay\":true,\"customerLiveFlowChanged\":false}", "images": [ { "mimeType": "image/webp", "blobRef": { "uri": "promptfoo://blob/cc60f517fbe7e5c34879fb6af5fdfdea1c02efeacd069b4e5eb9cfb69efb116c", "hash": "cc60f517fbe7e5c34879fb6af5fdfdea1c02efeacd069b4e5eb9cfb69efb116c", "mimeType": "image/webp", "sizeBytes": 43060, "provider": "filesystem" } } ], "metadata": { "caseId": "lvyo_bv0_candidate_030", "modality": "text_only", "cluster": "wedding weekend", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_030/artifact-lock.json", "finalSha256": "2b412eb776a72e07af63700dbcd144e155614ce1a7db11c17b6d22ae91b2ba07", "mockupSha256": "cccc4097a95a9a2949863d8d8b70c8d1592a404f63926aea107ee5069a3db5d8", "publicMockup": "assets/lvyo_bv0_candidate_030_m4_mockup.webp", "noSpendReplay": true } }, "score": 1, "success": true, "testCase": { "description": "30 text_only wedding weekend lvyo_bv0_candidate_030", "vars": { "caseId": "lvyo_bv0_candidate_030", "customerInput": "Wedding weekend shirt for the guests, Max & Priya, Cornwall 2026, subtle seaside illustration please", "modality": "text_only", "cluster": "wedding weekend" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_030\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ], "options": {}, "metadata": {} }, "testIdx": 29, "tokenUsage": { "prompt": 0, "completion": 0, "cached": 0, "total": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 }, "assertions": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 } } }, "vars": { "caseId": "lvyo_bv0_candidate_030", "customerInput": "Wedding weekend shirt for the guests, Max & Priya, Cornwall 2026, subtle seaside illustration please", "modality": "text_only", "cluster": "wedding weekend" }, "metadata": { "caseId": "lvyo_bv0_candidate_030", "modality": "text_only", "cluster": "wedding weekend", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_030/artifact-lock.json", "finalSha256": "2b412eb776a72e07af63700dbcd144e155614ce1a7db11c17b6d22ae91b2ba07", "mockupSha256": "cccc4097a95a9a2949863d8d8b70c8d1592a404f63926aea107ee5069a3db5d8", "publicMockup": "assets/lvyo_bv0_candidate_030_m4_mockup.webp", "noSpendReplay": true, "_promptfooFileMetadata": {} }, "failureReason": 0 }, { "cost": 0, "gradingResult": { "pass": true, "score": 1, "reason": "All assertions passed", "namedScores": {}, "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 0 }, "componentResults": [ { "pass": true, "score": 1, "reason": "Assertion passed", "assertion": { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_037\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } } ] }, "id": "9a696dbb-2a82-409e-82ae-f37427c4e879", "latencyMs": 0, "namedScores": {}, "prompt": { "raw": "Use the attached photo of Tom in the awful inflatable crown and make a stag shirt that says King of Bad Choices, Dublin 2026", "label": "{{customerInput}}", "config": {} }, "promptId": "7e756b499a1da155962cf88585b671e3ddacc5edf410c782ea1dbff2b92d0a70", "promptIdx": 0, "provider": { "id": "file://./providers/benchmark-v0-replay-provider.mjs", "label": "levyo-benchmark-v0-artifact-replay" }, "response": { "output": "{\"ok\":true,\"schemaVersion\":\"levyo_promptfoo_benchmark_v0_replay_result_v1\",\"caseId\":\"lvyo_bv0_candidate_037\",\"promptEcho\":\"Use the attached photo of Tom in the awful inflatable crown and make a stag shirt that says King of Bad Choices, Dublin 2026\",\"status\":\"success\",\"modality\":\"image_attached\",\"caseRole\":\"image_pack\",\"cluster\":\"stag weekend\",\"runtimeProfile\":\"internal_codex\",\"imageProvider\":\"codex_account_ui\",\"lockRef\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_037/artifact-lock.json\",\"lockSha256\":\"4de9a316f0b22f574c1a3e3e685020fd4e28993a6ddeecb8d31c16fd5a60ffa8\",\"final\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_037/artifact/final.png\",\"sha256\":\"f454b196bf244e005fba8a6b54715c84eaa049dc7e881b25cb96f06c41d081ff\"},\"mockup\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_037/artifact/mockup.png\",\"sha256\":\"279fe7a4c1c3c63a2502613896181a0f7431495c9747594ef4a276097d82797d\",\"publicSrc\":\"assets/lvyo_bv0_candidate_037_m4_mockup.webp\"},\"referenceImage\":{\"ref\":\"data/reference_inputs/references/lvyo_bv0_candidate_037_reference.png\",\"sha256\":\"3e2be3c29c07d0e2cff13cff2564bd47fe9f759e8a22e659b0cb09dec0819fce\"},\"noSpendReplay\":true,\"customerLiveFlowChanged\":false}", "images": [ { "mimeType": "image/webp", "blobRef": { "uri": "promptfoo://blob/f3cdbe8707eec08c6b4aedbc474696f076a43fc154dcc93dc343d9feb7878595", "hash": "f3cdbe8707eec08c6b4aedbc474696f076a43fc154dcc93dc343d9feb7878595", "mimeType": "image/webp", "sizeBytes": 38872, "provider": "filesystem" } } ], "metadata": { "caseId": "lvyo_bv0_candidate_037", "modality": "image_attached", "cluster": "stag weekend", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_037/artifact-lock.json", "finalSha256": "f454b196bf244e005fba8a6b54715c84eaa049dc7e881b25cb96f06c41d081ff", "mockupSha256": "279fe7a4c1c3c63a2502613896181a0f7431495c9747594ef4a276097d82797d", "publicMockup": "assets/lvyo_bv0_candidate_037_m4_mockup.webp", "noSpendReplay": true } }, "score": 1, "success": true, "testCase": { "description": "37 image_attached stag weekend lvyo_bv0_candidate_037", "vars": { "caseId": "lvyo_bv0_candidate_037", "customerInput": "Use the attached photo of Tom in the awful inflatable crown and make a stag shirt that says King of Bad Choices, Dublin 2026", "modality": "image_attached", "cluster": "stag weekend" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_037\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } ], "options": {}, "metadata": {} }, "testIdx": 30, "tokenUsage": { "prompt": 0, "completion": 0, "cached": 0, "total": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 }, "assertions": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 } } }, "vars": { "caseId": "lvyo_bv0_candidate_037", "customerInput": "Use the attached photo of Tom in the awful inflatable crown and make a stag shirt that says King of Bad Choices, Dublin 2026", "modality": "image_attached", "cluster": "stag weekend" }, "metadata": { "caseId": "lvyo_bv0_candidate_037", "modality": "image_attached", "cluster": "stag weekend", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_037/artifact-lock.json", "finalSha256": "f454b196bf244e005fba8a6b54715c84eaa049dc7e881b25cb96f06c41d081ff", "mockupSha256": "279fe7a4c1c3c63a2502613896181a0f7431495c9747594ef4a276097d82797d", "publicMockup": "assets/lvyo_bv0_candidate_037_m4_mockup.webp", "noSpendReplay": true, "_promptfooFileMetadata": {} }, "failureReason": 0 }, { "cost": 0, "gradingResult": { "pass": true, "score": 1, "reason": "All assertions passed", "namedScores": {}, "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 0 }, "componentResults": [ { "pass": true, "score": 1, "reason": "Assertion passed", "assertion": { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_038\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } } ] }, "id": "5b08f6e6-eac1-4806-b9a6-7688ca564554", "latencyMs": 0, "namedScores": {}, "prompt": { "raw": "Can you turn this group selfie into a classy hen weekend tee for Sophie's Last Fling, keep the sunglasses and cocktails vibe", "label": "{{customerInput}}", "config": {} }, "promptId": "7e756b499a1da155962cf88585b671e3ddacc5edf410c782ea1dbff2b92d0a70", "promptIdx": 0, "provider": { "id": "file://./providers/benchmark-v0-replay-provider.mjs", "label": "levyo-benchmark-v0-artifact-replay" }, "response": { "output": "{\"ok\":true,\"schemaVersion\":\"levyo_promptfoo_benchmark_v0_replay_result_v1\",\"caseId\":\"lvyo_bv0_candidate_038\",\"promptEcho\":\"Can you turn this group selfie into a classy hen weekend tee for Sophie's Last Fling, keep the sunglasses and cocktails vibe\",\"status\":\"success\",\"modality\":\"image_attached\",\"caseRole\":\"image_pack\",\"cluster\":\"hen party\",\"runtimeProfile\":\"internal_codex\",\"imageProvider\":\"codex_account_ui\",\"lockRef\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_038/artifact-lock.json\",\"lockSha256\":\"3fb0f14e1b15e772c051edebefc977f7120d17048059744742bc5f672d82fe8b\",\"final\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_038/artifact/final.png\",\"sha256\":\"145ddedc4a2a5e3bef99f1b20b2c86e5b80e0992a6afbcc3f437e3a05d39c801\"},\"mockup\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_038/artifact/mockup.png\",\"sha256\":\"8176ab5a4bb4aa361180d81723b16108b81d0a743ab7fa6d201b518d1539df24\",\"publicSrc\":\"assets/lvyo_bv0_candidate_038_m4_mockup.webp\"},\"referenceImage\":{\"ref\":\"data/reference_inputs/references/lvyo_bv0_candidate_038_reference.png\",\"sha256\":\"b413719bc7e359f3976c8ce768111f2c4d9ebd1a707c543468c1375e2b9819b0\"},\"noSpendReplay\":true,\"customerLiveFlowChanged\":false}", "images": [ { "mimeType": "image/webp", "blobRef": { "uri": "promptfoo://blob/7ca47b9a8808eafcdc0fa8677f3abb298c26bb2a6f412f882c87cab6a8522d1f", "hash": "7ca47b9a8808eafcdc0fa8677f3abb298c26bb2a6f412f882c87cab6a8522d1f", "mimeType": "image/webp", "sizeBytes": 45742, "provider": "filesystem" } } ], "metadata": { "caseId": "lvyo_bv0_candidate_038", "modality": "image_attached", "cluster": "hen party", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_038/artifact-lock.json", "finalSha256": "145ddedc4a2a5e3bef99f1b20b2c86e5b80e0992a6afbcc3f437e3a05d39c801", "mockupSha256": "8176ab5a4bb4aa361180d81723b16108b81d0a743ab7fa6d201b518d1539df24", "publicMockup": "assets/lvyo_bv0_candidate_038_m4_mockup.webp", "noSpendReplay": true } }, "score": 1, "success": true, "testCase": { "description": "38 image_attached hen party lvyo_bv0_candidate_038", "vars": { "caseId": "lvyo_bv0_candidate_038", "customerInput": "Can you turn this group selfie into a classy hen weekend tee for Sophie's Last Fling, keep the sunglasses and cocktails vibe", "modality": "image_attached", "cluster": "hen party" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_038\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } ], "options": {}, "metadata": {} }, "testIdx": 31, "tokenUsage": { "prompt": 0, "completion": 0, "cached": 0, "total": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 }, "assertions": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 } } }, "vars": { "caseId": "lvyo_bv0_candidate_038", "customerInput": "Can you turn this group selfie into a classy hen weekend tee for Sophie's Last Fling, keep the sunglasses and cocktails vibe", "modality": "image_attached", "cluster": "hen party" }, "metadata": { "caseId": "lvyo_bv0_candidate_038", "modality": "image_attached", "cluster": "hen party", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_038/artifact-lock.json", "finalSha256": "145ddedc4a2a5e3bef99f1b20b2c86e5b80e0992a6afbcc3f437e3a05d39c801", "mockupSha256": "8176ab5a4bb4aa361180d81723b16108b81d0a743ab7fa6d201b518d1539df24", "publicMockup": "assets/lvyo_bv0_candidate_038_m4_mockup.webp", "noSpendReplay": true, "_promptfooFileMetadata": {} }, "failureReason": 0 }, { "cost": 0, "gradingResult": { "pass": true, "score": 1, "reason": "All assertions passed", "namedScores": {}, "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 0 }, "componentResults": [ { "pass": true, "score": 1, "reason": "Assertion passed", "assertion": { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_039\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } } ] }, "id": "98d924ee-95fa-48f3-a0d3-5444ba199cb8", "latencyMs": 0, "namedScores": {}, "prompt": { "raw": "Make a 40th birthday shirt based on this pic of Dan asleep on the sofa after the party. Text: Still Got It Apparently", "label": "{{customerInput}}", "config": {} }, "promptId": "7e756b499a1da155962cf88585b671e3ddacc5edf410c782ea1dbff2b92d0a70", "promptIdx": 0, "provider": { "id": "file://./providers/benchmark-v0-replay-provider.mjs", "label": "levyo-benchmark-v0-artifact-replay" }, "response": { "output": "{\"ok\":true,\"schemaVersion\":\"levyo_promptfoo_benchmark_v0_replay_result_v1\",\"caseId\":\"lvyo_bv0_candidate_039\",\"promptEcho\":\"Make a 40th birthday shirt based on this pic of Dan asleep on the sofa after the party. Text: Still Got It Apparently\",\"status\":\"success\",\"modality\":\"image_attached\",\"caseRole\":\"image_pack\",\"cluster\":\"milestone birthday\",\"runtimeProfile\":\"internal_codex\",\"imageProvider\":\"codex_account_ui\",\"lockRef\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_039/artifact-lock.json\",\"lockSha256\":\"21f3993e3d433479f86f27233796220a93feaeebe012853475a095fb6b23401f\",\"final\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_039/artifact/final.png\",\"sha256\":\"fe44e0bb158f8356327f63c8216594455bf06bf0b5bcc9fefff0a03cbeb85562\"},\"mockup\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_039/artifact/mockup.png\",\"sha256\":\"d9720733936a0cf4a68f0699058c827f5c4b8413e2457266df37e80247a94573\",\"publicSrc\":\"assets/lvyo_bv0_candidate_039_m4_mockup.webp\"},\"referenceImage\":{\"ref\":\"data/reference_inputs/references/lvyo_bv0_candidate_039_reference.png\",\"sha256\":\"97ec75bcb13b2edded0ebffc35c8ffb26953c9e25e3eebcb8d36f3ff240c7586\"},\"noSpendReplay\":true,\"customerLiveFlowChanged\":false}", "images": [ { "mimeType": "image/webp", "blobRef": { "uri": "promptfoo://blob/ba0187727f153679caf986873776ae426f9d3fa43630776e64b9ffb40bb8ed85", "hash": "ba0187727f153679caf986873776ae426f9d3fa43630776e64b9ffb40bb8ed85", "mimeType": "image/webp", "sizeBytes": 49296, "provider": "filesystem" } } ], "metadata": { "caseId": "lvyo_bv0_candidate_039", "modality": "image_attached", "cluster": "milestone birthday", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_039/artifact-lock.json", "finalSha256": "fe44e0bb158f8356327f63c8216594455bf06bf0b5bcc9fefff0a03cbeb85562", "mockupSha256": "d9720733936a0cf4a68f0699058c827f5c4b8413e2457266df37e80247a94573", "publicMockup": "assets/lvyo_bv0_candidate_039_m4_mockup.webp", "noSpendReplay": true } }, "score": 1, "success": true, "testCase": { "description": "39 image_attached milestone birthday lvyo_bv0_candidate_039", "vars": { "caseId": "lvyo_bv0_candidate_039", "customerInput": "Make a 40th birthday shirt based on this pic of Dan asleep on the sofa after the party. Text: Still Got It Apparently", "modality": "image_attached", "cluster": "milestone birthday" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_039\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } ], "options": {}, "metadata": {} }, "testIdx": 32, "tokenUsage": { "prompt": 0, "completion": 0, "cached": 0, "total": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 }, "assertions": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 } } }, "vars": { "caseId": "lvyo_bv0_candidate_039", "customerInput": "Make a 40th birthday shirt based on this pic of Dan asleep on the sofa after the party. Text: Still Got It Apparently", "modality": "image_attached", "cluster": "milestone birthday" }, "metadata": { "caseId": "lvyo_bv0_candidate_039", "modality": "image_attached", "cluster": "milestone birthday", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_039/artifact-lock.json", "finalSha256": "fe44e0bb158f8356327f63c8216594455bf06bf0b5bcc9fefff0a03cbeb85562", "mockupSha256": "d9720733936a0cf4a68f0699058c827f5c4b8413e2457266df37e80247a94573", "publicMockup": "assets/lvyo_bv0_candidate_039_m4_mockup.webp", "noSpendReplay": true, "_promptfooFileMetadata": {} }, "failureReason": 0 }, { "cost": 0, "gradingResult": { "pass": true, "score": 1, "reason": "All assertions passed", "namedScores": {}, "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 0 }, "componentResults": [ { "pass": true, "score": 1, "reason": "Assertion passed", "assertion": { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_040\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } } ] }, "id": "46329c26-48e4-4916-a14a-8c25444ada1b", "latencyMs": 0, "namedScores": {}, "prompt": { "raw": "Use this airport photo for our girls trip shirts. Maybe cartoon us with suitcases and the words Gate Closing? Never Heard Of Her", "label": "{{customerInput}}", "config": {} }, "promptId": "7e756b499a1da155962cf88585b671e3ddacc5edf410c782ea1dbff2b92d0a70", "promptIdx": 0, "provider": { "id": "file://./providers/benchmark-v0-replay-provider.mjs", "label": "levyo-benchmark-v0-artifact-replay" }, "response": { "output": "{\"ok\":true,\"schemaVersion\":\"levyo_promptfoo_benchmark_v0_replay_result_v1\",\"caseId\":\"lvyo_bv0_candidate_040\",\"promptEcho\":\"Use this airport photo for our girls trip shirts. Maybe cartoon us with suitcases and the words Gate Closing? Never Heard Of Her\",\"status\":\"success\",\"modality\":\"image_attached\",\"caseRole\":\"image_pack\",\"cluster\":\"girls trip\",\"runtimeProfile\":\"internal_codex\",\"imageProvider\":\"codex_account_ui\",\"lockRef\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_040/artifact-lock.json\",\"lockSha256\":\"ceecdf0f806761b2dfe578b351b0738ce67f81bb8d0a91e35828874e7813e516\",\"final\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_040/artifact/final.png\",\"sha256\":\"f77ec38a96ac5e93c73a4dfbbee76723b8f7f3c4e091b6295697f33fae1c1745\"},\"mockup\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_040/artifact/mockup.png\",\"sha256\":\"59eb3420a828df30bc832f7db4981f0406568137e189e6184fea16299c46c8de\",\"publicSrc\":\"assets/lvyo_bv0_candidate_040_m4_mockup.webp\"},\"referenceImage\":{\"ref\":\"data/reference_inputs/references/lvyo_bv0_candidate_040_reference.png\",\"sha256\":\"55a28f7deb0c6f17d8d4148d20a5c520140ac47f9ac9a16f1ff31cc7516132ac\"},\"noSpendReplay\":true,\"customerLiveFlowChanged\":false}", "images": [ { "mimeType": "image/webp", "blobRef": { "uri": "promptfoo://blob/df419aa6df54f50e70c42cf3a92dd9853a3b9c1aacde05f74f179abc0608ce0a", "hash": "df419aa6df54f50e70c42cf3a92dd9853a3b9c1aacde05f74f179abc0608ce0a", "mimeType": "image/webp", "sizeBytes": 49478, "provider": "filesystem" } } ], "metadata": { "caseId": "lvyo_bv0_candidate_040", "modality": "image_attached", "cluster": "girls trip", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_040/artifact-lock.json", "finalSha256": "f77ec38a96ac5e93c73a4dfbbee76723b8f7f3c4e091b6295697f33fae1c1745", "mockupSha256": "59eb3420a828df30bc832f7db4981f0406568137e189e6184fea16299c46c8de", "publicMockup": "assets/lvyo_bv0_candidate_040_m4_mockup.webp", "noSpendReplay": true } }, "score": 1, "success": true, "testCase": { "description": "40 image_attached girls trip lvyo_bv0_candidate_040", "vars": { "caseId": "lvyo_bv0_candidate_040", "customerInput": "Use this airport photo for our girls trip shirts. Maybe cartoon us with suitcases and the words Gate Closing? Never Heard Of Her", "modality": "image_attached", "cluster": "girls trip" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_040\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } ], "options": {}, "metadata": {} }, "testIdx": 33, "tokenUsage": { "prompt": 0, "completion": 0, "cached": 0, "total": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 }, "assertions": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 } } }, "vars": { "caseId": "lvyo_bv0_candidate_040", "customerInput": "Use this airport photo for our girls trip shirts. Maybe cartoon us with suitcases and the words Gate Closing? Never Heard Of Her", "modality": "image_attached", "cluster": "girls trip" }, "metadata": { "caseId": "lvyo_bv0_candidate_040", "modality": "image_attached", "cluster": "girls trip", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_040/artifact-lock.json", "finalSha256": "f77ec38a96ac5e93c73a4dfbbee76723b8f7f3c4e091b6295697f33fae1c1745", "mockupSha256": "59eb3420a828df30bc832f7db4981f0406568137e189e6184fea16299c46c8de", "publicMockup": "assets/lvyo_bv0_candidate_040_m4_mockup.webp", "noSpendReplay": true, "_promptfooFileMetadata": {} }, "failureReason": 0 }, { "cost": 0, "gradingResult": { "pass": true, "score": 1, "reason": "All assertions passed", "namedScores": {}, "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 0 }, "componentResults": [ { "pass": true, "score": 1, "reason": "Assertion passed", "assertion": { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_041\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } } ] }, "id": "916cd30e-514a-4c31-a22d-21a80d8d1dfd", "latencyMs": 0, "namedScores": {}, "prompt": { "raw": "This balcony photo is the vibe. Need lads holiday tops saying Balcony Board Meeting 2026, make us look like a terrible committee", "label": "{{customerInput}}", "config": {} }, "promptId": "7e756b499a1da155962cf88585b671e3ddacc5edf410c782ea1dbff2b92d0a70", "promptIdx": 0, "provider": { "id": "file://./providers/benchmark-v0-replay-provider.mjs", "label": "levyo-benchmark-v0-artifact-replay" }, "response": { "output": "{\"ok\":true,\"schemaVersion\":\"levyo_promptfoo_benchmark_v0_replay_result_v1\",\"caseId\":\"lvyo_bv0_candidate_041\",\"promptEcho\":\"This balcony photo is the vibe. Need lads holiday tops saying Balcony Board Meeting 2026, make us look like a terrible committee\",\"status\":\"success\",\"modality\":\"image_attached\",\"caseRole\":\"image_pack\",\"cluster\":\"lads holiday\",\"runtimeProfile\":\"internal_codex\",\"imageProvider\":\"codex_account_ui\",\"lockRef\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_041/artifact-lock.json\",\"lockSha256\":\"499daeb265c4d6037817169f44a084f5904e77ed113514357e254e35f3626a08\",\"final\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_041/artifact/final.png\",\"sha256\":\"1ac553de381cdc1fa84b1f9facc5f9afb03979b23595fa053853c8cc96aa6835\"},\"mockup\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_041/artifact/mockup.png\",\"sha256\":\"4d23c8d0952d22709d7909f34eebeeeb53c7f57aa654b73057055dca6bb7a186\",\"publicSrc\":\"assets/lvyo_bv0_candidate_041_m4_mockup.webp\"},\"referenceImage\":{\"ref\":\"data/reference_inputs/references/lvyo_bv0_candidate_041_reference.png\",\"sha256\":\"f81a4f47d30af98e8e95a3eedc1d8f0ec5341b817053d28a42de6ea6d200c5e9\"},\"noSpendReplay\":true,\"customerLiveFlowChanged\":false}", "images": [ { "mimeType": "image/webp", "blobRef": { "uri": "promptfoo://blob/dd8f4aff56c7a89acac99106e7ad63fcdd5a3e7518b2d4c09e5de9cd4719c6e3", "hash": "dd8f4aff56c7a89acac99106e7ad63fcdd5a3e7518b2d4c09e5de9cd4719c6e3", "mimeType": "image/webp", "sizeBytes": 39170, "provider": "filesystem" } } ], "metadata": { "caseId": "lvyo_bv0_candidate_041", "modality": "image_attached", "cluster": "lads holiday", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_041/artifact-lock.json", "finalSha256": "1ac553de381cdc1fa84b1f9facc5f9afb03979b23595fa053853c8cc96aa6835", "mockupSha256": "4d23c8d0952d22709d7909f34eebeeeb53c7f57aa654b73057055dca6bb7a186", "publicMockup": "assets/lvyo_bv0_candidate_041_m4_mockup.webp", "noSpendReplay": true } }, "score": 1, "success": true, "testCase": { "description": "41 image_attached lads holiday lvyo_bv0_candidate_041", "vars": { "caseId": "lvyo_bv0_candidate_041", "customerInput": "This balcony photo is the vibe. Need lads holiday tops saying Balcony Board Meeting 2026, make us look like a terrible committee", "modality": "image_attached", "cluster": "lads holiday" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_041\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } ], "options": {}, "metadata": {} }, "testIdx": 34, "tokenUsage": { "prompt": 0, "completion": 0, "cached": 0, "total": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 }, "assertions": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 } } }, "vars": { "caseId": "lvyo_bv0_candidate_041", "customerInput": "This balcony photo is the vibe. Need lads holiday tops saying Balcony Board Meeting 2026, make us look like a terrible committee", "modality": "image_attached", "cluster": "lads holiday" }, "metadata": { "caseId": "lvyo_bv0_candidate_041", "modality": "image_attached", "cluster": "lads holiday", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_041/artifact-lock.json", "finalSha256": "1ac553de381cdc1fa84b1f9facc5f9afb03979b23595fa053853c8cc96aa6835", "mockupSha256": "4d23c8d0952d22709d7909f34eebeeeb53c7f57aa654b73057055dca6bb7a186", "publicMockup": "assets/lvyo_bv0_candidate_041_m4_mockup.webp", "noSpendReplay": true, "_promptfooFileMetadata": {} }, "failureReason": 0 }, { "cost": 0, "gradingResult": { "pass": true, "score": 1, "reason": "All assertions passed", "namedScores": {}, "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 0 }, "componentResults": [ { "pass": true, "score": 1, "reason": "Assertion passed", "assertion": { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_042\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } } ] }, "id": "1a1d3ce6-a438-4aaa-b52d-f69f596c8ec2", "latencyMs": 0, "namedScores": {}, "prompt": { "raw": "Turn this ski lift photo into a retro ski club shirt, say Powder Panic Crew and include the stupid orange helmet", "label": "{{customerInput}}", "config": {} }, "promptId": "7e756b499a1da155962cf88585b671e3ddacc5edf410c782ea1dbff2b92d0a70", "promptIdx": 0, "provider": { "id": "file://./providers/benchmark-v0-replay-provider.mjs", "label": "levyo-benchmark-v0-artifact-replay" }, "response": { "output": "{\"ok\":true,\"schemaVersion\":\"levyo_promptfoo_benchmark_v0_replay_result_v1\",\"caseId\":\"lvyo_bv0_candidate_042\",\"promptEcho\":\"Turn this ski lift photo into a retro ski club shirt, say Powder Panic Crew and include the stupid orange helmet\",\"status\":\"success\",\"modality\":\"image_attached\",\"caseRole\":\"image_pack\",\"cluster\":\"ski trip\",\"runtimeProfile\":\"internal_codex\",\"imageProvider\":\"codex_account_ui\",\"lockRef\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_042/artifact-lock.json\",\"lockSha256\":\"8f70344e5b930ad3189528058b4757739e1e8d72b4ebf651d7b83025295c4eca\",\"final\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_042/artifact/final.png\",\"sha256\":\"70761cec0fbf73f80a336194d9c55650a814baff280e1f5667ddc29d88962473\"},\"mockup\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_042/artifact/mockup.png\",\"sha256\":\"dc65627116751b9befc72aa2eef9b729ce54fa711548242f6c4b70797ed6c8b5\",\"publicSrc\":\"assets/lvyo_bv0_candidate_042_m4_mockup.webp\"},\"referenceImage\":{\"ref\":\"data/reference_inputs/references/lvyo_bv0_candidate_042_reference.png\",\"sha256\":\"e59b992cfec62bd722145bfce457224711fe5b5ec5c0096d7989051d6f54c4a1\"},\"noSpendReplay\":true,\"customerLiveFlowChanged\":false}", "images": [ { "mimeType": "image/webp", "blobRef": { "uri": "promptfoo://blob/425d5a5af66ff5b07bd4b74b78ed9107381c4c258f6b3ad94115d4c91898a602", "hash": "425d5a5af66ff5b07bd4b74b78ed9107381c4c258f6b3ad94115d4c91898a602", "mimeType": "image/webp", "sizeBytes": 53736, "provider": "filesystem" } } ], "metadata": { "caseId": "lvyo_bv0_candidate_042", "modality": "image_attached", "cluster": "ski trip", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_042/artifact-lock.json", "finalSha256": "70761cec0fbf73f80a336194d9c55650a814baff280e1f5667ddc29d88962473", "mockupSha256": "dc65627116751b9befc72aa2eef9b729ce54fa711548242f6c4b70797ed6c8b5", "publicMockup": "assets/lvyo_bv0_candidate_042_m4_mockup.webp", "noSpendReplay": true } }, "score": 1, "success": true, "testCase": { "description": "42 image_attached ski trip lvyo_bv0_candidate_042", "vars": { "caseId": "lvyo_bv0_candidate_042", "customerInput": "Turn this ski lift photo into a retro ski club shirt, say Powder Panic Crew and include the stupid orange helmet", "modality": "image_attached", "cluster": "ski trip" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_042\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } ], "options": {}, "metadata": {} }, "testIdx": 35, "tokenUsage": { "prompt": 0, "completion": 0, "cached": 0, "total": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 }, "assertions": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 } } }, "vars": { "caseId": "lvyo_bv0_candidate_042", "customerInput": "Turn this ski lift photo into a retro ski club shirt, say Powder Panic Crew and include the stupid orange helmet", "modality": "image_attached", "cluster": "ski trip" }, "metadata": { "caseId": "lvyo_bv0_candidate_042", "modality": "image_attached", "cluster": "ski trip", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_042/artifact-lock.json", "finalSha256": "70761cec0fbf73f80a336194d9c55650a814baff280e1f5667ddc29d88962473", "mockupSha256": "dc65627116751b9befc72aa2eef9b729ce54fa711548242f6c4b70797ed6c8b5", "publicMockup": "assets/lvyo_bv0_candidate_042_m4_mockup.webp", "noSpendReplay": true, "_promptfooFileMetadata": {} }, "failureReason": 0 }, { "cost": 0, "gradingResult": { "pass": true, "score": 1, "reason": "All assertions passed", "namedScores": {}, "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 0 }, "componentResults": [ { "pass": true, "score": 1, "reason": "Assertion passed", "assertion": { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_043\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } } ] }, "id": "2b916f72-11e8-4d27-8ed0-0289a71a0441", "latencyMs": 0, "namedScores": {}, "prompt": { "raw": "Use this pic of my dad missing the putt for his birthday golf shirt. Can it say Three Putts to Glory", "label": "{{customerInput}}", "config": {} }, "promptId": "7e756b499a1da155962cf88585b671e3ddacc5edf410c782ea1dbff2b92d0a70", "promptIdx": 0, "provider": { "id": "file://./providers/benchmark-v0-replay-provider.mjs", "label": "levyo-benchmark-v0-artifact-replay" }, "response": { "output": "{\"ok\":true,\"schemaVersion\":\"levyo_promptfoo_benchmark_v0_replay_result_v1\",\"caseId\":\"lvyo_bv0_candidate_043\",\"promptEcho\":\"Use this pic of my dad missing the putt for his birthday golf shirt. Can it say Three Putts to Glory\",\"status\":\"success\",\"modality\":\"image_attached\",\"caseRole\":\"image_pack\",\"cluster\":\"golf trip\",\"runtimeProfile\":\"internal_codex\",\"imageProvider\":\"codex_account_ui\",\"lockRef\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_043/artifact-lock.json\",\"lockSha256\":\"f3453a375563dd55009d933c00263f8059c5a42591df63e0c07f0317c6fbedc7\",\"final\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_043/artifact/final.png\",\"sha256\":\"26d9d03cc90b703d1b554649dd8f309732c50533fa93978f5fcd2be140125d72\"},\"mockup\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_043/artifact/mockup.png\",\"sha256\":\"6a0b9eb507ef1d634bd50c43909f9aacb3f2859dddd90e87d7eac71e74b5cc36\",\"publicSrc\":\"assets/lvyo_bv0_candidate_043_m4_mockup.webp\"},\"referenceImage\":{\"ref\":\"data/reference_inputs/references/lvyo_bv0_candidate_043_reference.png\",\"sha256\":\"41a1f1bd11d0e720cf6a837cbf7257cd98bf174c3e6a837436c09755987fc080\"},\"noSpendReplay\":true,\"customerLiveFlowChanged\":false}", "images": [ { "mimeType": "image/webp", "blobRef": { "uri": "promptfoo://blob/39fd369033001254e00f217fdebc3a41367a88a3496973a8de35c312e248d6d0", "hash": "39fd369033001254e00f217fdebc3a41367a88a3496973a8de35c312e248d6d0", "mimeType": "image/webp", "sizeBytes": 39472, "provider": "filesystem" } } ], "metadata": { "caseId": "lvyo_bv0_candidate_043", "modality": "image_attached", "cluster": "golf trip", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_043/artifact-lock.json", "finalSha256": "26d9d03cc90b703d1b554649dd8f309732c50533fa93978f5fcd2be140125d72", "mockupSha256": "6a0b9eb507ef1d634bd50c43909f9aacb3f2859dddd90e87d7eac71e74b5cc36", "publicMockup": "assets/lvyo_bv0_candidate_043_m4_mockup.webp", "noSpendReplay": true } }, "score": 1, "success": true, "testCase": { "description": "43 image_attached golf trip lvyo_bv0_candidate_043", "vars": { "caseId": "lvyo_bv0_candidate_043", "customerInput": "Use this pic of my dad missing the putt for his birthday golf shirt. Can it say Three Putts to Glory", "modality": "image_attached", "cluster": "golf trip" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_043\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } ], "options": {}, "metadata": {} }, "testIdx": 36, "tokenUsage": { "prompt": 0, "completion": 0, "cached": 0, "total": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 }, "assertions": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 } } }, "vars": { "caseId": "lvyo_bv0_candidate_043", "customerInput": "Use this pic of my dad missing the putt for his birthday golf shirt. Can it say Three Putts to Glory", "modality": "image_attached", "cluster": "golf trip" }, "metadata": { "caseId": "lvyo_bv0_candidate_043", "modality": "image_attached", "cluster": "golf trip", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_043/artifact-lock.json", "finalSha256": "26d9d03cc90b703d1b554649dd8f309732c50533fa93978f5fcd2be140125d72", "mockupSha256": "6a0b9eb507ef1d634bd50c43909f9aacb3f2859dddd90e87d7eac71e74b5cc36", "publicMockup": "assets/lvyo_bv0_candidate_043_m4_mockup.webp", "noSpendReplay": true, "_promptfooFileMetadata": {} }, "failureReason": 0 }, { "cost": 0, "gradingResult": { "pass": true, "score": 1, "reason": "All assertions passed", "namedScores": {}, "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 0 }, "componentResults": [ { "pass": true, "score": 1, "reason": "Assertion passed", "assertion": { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_044\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } } ] }, "id": "3779634d-8a87-4594-a2fc-119e350afec2", "latencyMs": 0, "namedScores": {}, "prompt": { "raw": "Make a tour shirt from this photo of our dance squad in the rehearsal room, but more bold and stagey", "label": "{{customerInput}}", "config": {} }, "promptId": "7e756b499a1da155962cf88585b671e3ddacc5edf410c782ea1dbff2b92d0a70", "promptIdx": 0, "provider": { "id": "file://./providers/benchmark-v0-replay-provider.mjs", "label": "levyo-benchmark-v0-artifact-replay" }, "response": { "output": "{\"ok\":true,\"schemaVersion\":\"levyo_promptfoo_benchmark_v0_replay_result_v1\",\"caseId\":\"lvyo_bv0_candidate_044\",\"promptEcho\":\"Make a tour shirt from this photo of our dance squad in the rehearsal room, but more bold and stagey\",\"status\":\"success\",\"modality\":\"image_attached\",\"caseRole\":\"image_pack\",\"cluster\":\"student society\",\"runtimeProfile\":\"internal_codex\",\"imageProvider\":\"codex_account_ui\",\"lockRef\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_044/artifact-lock.json\",\"lockSha256\":\"37ed0f15ae5bb6410b6d1175a94c3bb20c3f1b21d32d30751ee6653bdd9875d7\",\"final\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_044/artifact/final.png\",\"sha256\":\"7000caea6b3fc72fd05e338929c16b244ab51cc5f7353d4b209a143b5096f2ca\"},\"mockup\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_044/artifact/mockup.png\",\"sha256\":\"0581d0fc7078c492dfb486722f318b89ed10b06294a225c4a7274e8336c77993\",\"publicSrc\":\"assets/lvyo_bv0_candidate_044_m4_mockup.webp\"},\"referenceImage\":{\"ref\":\"data/reference_inputs/references/lvyo_bv0_candidate_044_reference.png\",\"sha256\":\"cf474c95488a70e562fe14627eb7cc839f84bb8503803ab95074a149e1e8d8a2\"},\"noSpendReplay\":true,\"customerLiveFlowChanged\":false}", "images": [ { "mimeType": "image/webp", "blobRef": { "uri": "promptfoo://blob/ee1abb0e9074ca237e761c6c38ed549537cb8d17420e5b43580b92d400c4e065", "hash": "ee1abb0e9074ca237e761c6c38ed549537cb8d17420e5b43580b92d400c4e065", "mimeType": "image/webp", "sizeBytes": 47300, "provider": "filesystem" } } ], "metadata": { "caseId": "lvyo_bv0_candidate_044", "modality": "image_attached", "cluster": "student society", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_044/artifact-lock.json", "finalSha256": "7000caea6b3fc72fd05e338929c16b244ab51cc5f7353d4b209a143b5096f2ca", "mockupSha256": "0581d0fc7078c492dfb486722f318b89ed10b06294a225c4a7274e8336c77993", "publicMockup": "assets/lvyo_bv0_candidate_044_m4_mockup.webp", "noSpendReplay": true } }, "score": 1, "success": true, "testCase": { "description": "44 image_attached student society lvyo_bv0_candidate_044", "vars": { "caseId": "lvyo_bv0_candidate_044", "customerInput": "Make a tour shirt from this photo of our dance squad in the rehearsal room, but more bold and stagey", "modality": "image_attached", "cluster": "student society" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_044\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } ], "options": {}, "metadata": {} }, "testIdx": 37, "tokenUsage": { "prompt": 0, "completion": 0, "cached": 0, "total": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 }, "assertions": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 } } }, "vars": { "caseId": "lvyo_bv0_candidate_044", "customerInput": "Make a tour shirt from this photo of our dance squad in the rehearsal room, but more bold and stagey", "modality": "image_attached", "cluster": "student society" }, "metadata": { "caseId": "lvyo_bv0_candidate_044", "modality": "image_attached", "cluster": "student society", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_044/artifact-lock.json", "finalSha256": "7000caea6b3fc72fd05e338929c16b244ab51cc5f7353d4b209a143b5096f2ca", "mockupSha256": "0581d0fc7078c492dfb486722f318b89ed10b06294a225c4a7274e8336c77993", "publicMockup": "assets/lvyo_bv0_candidate_044_m4_mockup.webp", "noSpendReplay": true, "_promptfooFileMetadata": {} }, "failureReason": 0 }, { "cost": 0, "gradingResult": { "pass": true, "score": 1, "reason": "All assertions passed", "namedScores": {}, "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 0 }, "componentResults": [ { "pass": true, "score": 1, "reason": "Assertion passed", "assertion": { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_045\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } } ] }, "id": "2ced4fbe-1c02-4892-9a32-bd798c673ee0", "latencyMs": 0, "namedScores": {}, "prompt": { "raw": "This muddy team pic should become our rugby social shirt. Text: Scrum And Tonic, make it less disgusting please", "label": "{{customerInput}}", "config": {} }, "promptId": "7e756b499a1da155962cf88585b671e3ddacc5edf410c782ea1dbff2b92d0a70", "promptIdx": 0, "provider": { "id": "file://./providers/benchmark-v0-replay-provider.mjs", "label": "levyo-benchmark-v0-artifact-replay" }, "response": { "output": "{\"ok\":true,\"schemaVersion\":\"levyo_promptfoo_benchmark_v0_replay_result_v1\",\"caseId\":\"lvyo_bv0_candidate_045\",\"promptEcho\":\"This muddy team pic should become our rugby social shirt. Text: Scrum And Tonic, make it less disgusting please\",\"status\":\"success\",\"modality\":\"image_attached\",\"caseRole\":\"image_pack\",\"cluster\":\"rugby social\",\"runtimeProfile\":\"internal_codex\",\"imageProvider\":\"codex_account_ui\",\"lockRef\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_045/artifact-lock.json\",\"lockSha256\":\"6c5a26466436cde96fc680c4e384f3f88f0efd490b4b1bc4812a204717a3d8b7\",\"final\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_045/artifact/final.png\",\"sha256\":\"b596da11fd316186f77cd76978cf85b33e2a7cd4e32e58c7b7fa8e240012aabd\"},\"mockup\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_045/artifact/mockup.png\",\"sha256\":\"f9bc497915535715820bc7c81c0e96efc307540b40e30c0887e8c6bb36088ef6\",\"publicSrc\":\"assets/lvyo_bv0_candidate_045_m4_mockup.webp\"},\"referenceImage\":{\"ref\":\"data/reference_inputs/references/lvyo_bv0_candidate_045_reference.png\",\"sha256\":\"633c638068b862ac1c8ceedf7ea61d7d14d4de759b845029449088ac60e59f65\"},\"noSpendReplay\":true,\"customerLiveFlowChanged\":false}", "images": [ { "mimeType": "image/webp", "blobRef": { "uri": "promptfoo://blob/76ff210836146c172b49b338a7c9db620c59ff37ef6d8ecdd67a4c2ea47a1884", "hash": "76ff210836146c172b49b338a7c9db620c59ff37ef6d8ecdd67a4c2ea47a1884", "mimeType": "image/webp", "sizeBytes": 42294, "provider": "filesystem" } } ], "metadata": { "caseId": "lvyo_bv0_candidate_045", "modality": "image_attached", "cluster": "rugby social", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_045/artifact-lock.json", "finalSha256": "b596da11fd316186f77cd76978cf85b33e2a7cd4e32e58c7b7fa8e240012aabd", "mockupSha256": "f9bc497915535715820bc7c81c0e96efc307540b40e30c0887e8c6bb36088ef6", "publicMockup": "assets/lvyo_bv0_candidate_045_m4_mockup.webp", "noSpendReplay": true } }, "score": 1, "success": true, "testCase": { "description": "45 image_attached rugby social lvyo_bv0_candidate_045", "vars": { "caseId": "lvyo_bv0_candidate_045", "customerInput": "This muddy team pic should become our rugby social shirt. Text: Scrum And Tonic, make it less disgusting please", "modality": "image_attached", "cluster": "rugby social" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_045\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } ], "options": {}, "metadata": {} }, "testIdx": 38, "tokenUsage": { "prompt": 0, "completion": 0, "cached": 0, "total": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 }, "assertions": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 } } }, "vars": { "caseId": "lvyo_bv0_candidate_045", "customerInput": "This muddy team pic should become our rugby social shirt. Text: Scrum And Tonic, make it less disgusting please", "modality": "image_attached", "cluster": "rugby social" }, "metadata": { "caseId": "lvyo_bv0_candidate_045", "modality": "image_attached", "cluster": "rugby social", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_045/artifact-lock.json", "finalSha256": "b596da11fd316186f77cd76978cf85b33e2a7cd4e32e58c7b7fa8e240012aabd", "mockupSha256": "f9bc497915535715820bc7c81c0e96efc307540b40e30c0887e8c6bb36088ef6", "publicMockup": "assets/lvyo_bv0_candidate_045_m4_mockup.webp", "noSpendReplay": true, "_promptfooFileMetadata": {} }, "failureReason": 0 }, { "cost": 0, "gradingResult": { "pass": true, "score": 1, "reason": "All assertions passed", "namedScores": {}, "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 0 }, "componentResults": [ { "pass": true, "score": 1, "reason": "Assertion passed", "assertion": { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_046\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } } ] }, "id": "9749cded-3484-4eee-a9a2-89206621279f", "latencyMs": 1, "namedScores": {}, "prompt": { "raw": "Use the tent photo for our festival shirts. Need something like Camp Questionable Decisions with the wonky flag in it", "label": "{{customerInput}}", "config": {} }, "promptId": "7e756b499a1da155962cf88585b671e3ddacc5edf410c782ea1dbff2b92d0a70", "promptIdx": 0, "provider": { "id": "file://./providers/benchmark-v0-replay-provider.mjs", "label": "levyo-benchmark-v0-artifact-replay" }, "response": { "output": "{\"ok\":true,\"schemaVersion\":\"levyo_promptfoo_benchmark_v0_replay_result_v1\",\"caseId\":\"lvyo_bv0_candidate_046\",\"promptEcho\":\"Use the tent photo for our festival shirts. Need something like Camp Questionable Decisions with the wonky flag in it\",\"status\":\"success\",\"modality\":\"image_attached\",\"caseRole\":\"image_pack\",\"cluster\":\"festival crew\",\"runtimeProfile\":\"internal_codex\",\"imageProvider\":\"codex_account_ui\",\"lockRef\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_046/artifact-lock.json\",\"lockSha256\":\"dc4fd6792b7d4171c877a9fdb3ed67d048a8f4aee64749a3f235f5fa591ab47d\",\"final\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_046/artifact/final.png\",\"sha256\":\"07dd911a76c4dff8b998d02127505614a5748208e2515ecbe4c953c59cc4bb6c\"},\"mockup\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_046/artifact/mockup.png\",\"sha256\":\"acbe04d5ead42f0dea8434aed33de835a126c552e6f3bdaa025e272b0b001e61\",\"publicSrc\":\"assets/lvyo_bv0_candidate_046_m4_mockup.webp\"},\"referenceImage\":{\"ref\":\"data/reference_inputs/references/lvyo_bv0_candidate_046_reference.png\",\"sha256\":\"7145c1219a45f6b3a1eb80d88b175d0ccac2336ae5c02593bdd5d6f29c8d734c\"},\"noSpendReplay\":true,\"customerLiveFlowChanged\":false}", "images": [ { "mimeType": "image/webp", "blobRef": { "uri": "promptfoo://blob/6a4ebad5810b4bb8c5b1f795f1374ae5738d617ad890235ead7545d3b99f0d9a", "hash": "6a4ebad5810b4bb8c5b1f795f1374ae5738d617ad890235ead7545d3b99f0d9a", "mimeType": "image/webp", "sizeBytes": 50982, "provider": "filesystem" } } ], "metadata": { "caseId": "lvyo_bv0_candidate_046", "modality": "image_attached", "cluster": "festival crew", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_046/artifact-lock.json", "finalSha256": "07dd911a76c4dff8b998d02127505614a5748208e2515ecbe4c953c59cc4bb6c", "mockupSha256": "acbe04d5ead42f0dea8434aed33de835a126c552e6f3bdaa025e272b0b001e61", "publicMockup": "assets/lvyo_bv0_candidate_046_m4_mockup.webp", "noSpendReplay": true } }, "score": 1, "success": true, "testCase": { "description": "46 image_attached festival crew lvyo_bv0_candidate_046", "vars": { "caseId": "lvyo_bv0_candidate_046", "customerInput": "Use the tent photo for our festival shirts. Need something like Camp Questionable Decisions with the wonky flag in it", "modality": "image_attached", "cluster": "festival crew" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_046\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } ], "options": {}, "metadata": {} }, "testIdx": 39, "tokenUsage": { "prompt": 0, "completion": 0, "cached": 0, "total": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 }, "assertions": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 } } }, "vars": { "caseId": "lvyo_bv0_candidate_046", "customerInput": "Use the tent photo for our festival shirts. Need something like Camp Questionable Decisions with the wonky flag in it", "modality": "image_attached", "cluster": "festival crew" }, "metadata": { "caseId": "lvyo_bv0_candidate_046", "modality": "image_attached", "cluster": "festival crew", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_046/artifact-lock.json", "finalSha256": "07dd911a76c4dff8b998d02127505614a5748208e2515ecbe4c953c59cc4bb6c", "mockupSha256": "acbe04d5ead42f0dea8434aed33de835a126c552e6f3bdaa025e272b0b001e61", "publicMockup": "assets/lvyo_bv0_candidate_046_m4_mockup.webp", "noSpendReplay": true, "_promptfooFileMetadata": {} }, "failureReason": 0 }, { "cost": 0, "gradingResult": { "pass": true, "score": 1, "reason": "All assertions passed", "namedScores": {}, "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 0 }, "componentResults": [ { "pass": true, "score": 1, "reason": "Assertion passed", "assertion": { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_047\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } } ] }, "id": "c1ba5bbd-2405-4a49-9ce5-836dee532ac6", "latencyMs": 1, "namedScores": {}, "prompt": { "raw": "Can you use this photo of the first pub sign and make a crawl tee, but don't copy the pub name, just the old sign style", "label": "{{customerInput}}", "config": {} }, "promptId": "7e756b499a1da155962cf88585b671e3ddacc5edf410c782ea1dbff2b92d0a70", "promptIdx": 0, "provider": { "id": "file://./providers/benchmark-v0-replay-provider.mjs", "label": "levyo-benchmark-v0-artifact-replay" }, "response": { "output": "{\"ok\":true,\"schemaVersion\":\"levyo_promptfoo_benchmark_v0_replay_result_v1\",\"caseId\":\"lvyo_bv0_candidate_047\",\"promptEcho\":\"Can you use this photo of the first pub sign and make a crawl tee, but don't copy the pub name, just the old sign style\",\"status\":\"success\",\"modality\":\"image_attached\",\"caseRole\":\"image_pack\",\"cluster\":\"pub crawl\",\"runtimeProfile\":\"internal_codex\",\"imageProvider\":\"codex_account_ui\",\"lockRef\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_047/artifact-lock.json\",\"lockSha256\":\"8a64211f9450e159cb8340edcf8dff3783433bda2725aa1e21796d3fb720f44b\",\"final\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_047/artifact/final.png\",\"sha256\":\"7d0a6a08a712826048d9a3dd81594ed65396f91326e20abfb943e094cb28eb89\"},\"mockup\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_047/artifact/mockup.png\",\"sha256\":\"828bbaa5ee618f7153f74b3a956c64bb4c0829e72b3eeebb30ed9e031c3ae991\",\"publicSrc\":\"assets/lvyo_bv0_candidate_047_m4_mockup.webp\"},\"referenceImage\":{\"ref\":\"data/reference_inputs/references/lvyo_bv0_candidate_047_reference.png\",\"sha256\":\"c669c88b5d61fa04ca462922ff26c4abec896d9fbc3d52d956963a052e03e24e\"},\"noSpendReplay\":true,\"customerLiveFlowChanged\":false}", "images": [ { "mimeType": "image/webp", "blobRef": { "uri": "promptfoo://blob/e626f5a4be424b521f99763b1a1f93147e1aee1ef1540eb265de0b94daa702d5", "hash": "e626f5a4be424b521f99763b1a1f93147e1aee1ef1540eb265de0b94daa702d5", "mimeType": "image/webp", "sizeBytes": 45494, "provider": "filesystem" } } ], "metadata": { "caseId": "lvyo_bv0_candidate_047", "modality": "image_attached", "cluster": "pub crawl", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_047/artifact-lock.json", "finalSha256": "7d0a6a08a712826048d9a3dd81594ed65396f91326e20abfb943e094cb28eb89", "mockupSha256": "828bbaa5ee618f7153f74b3a956c64bb4c0829e72b3eeebb30ed9e031c3ae991", "publicMockup": "assets/lvyo_bv0_candidate_047_m4_mockup.webp", "noSpendReplay": true } }, "score": 1, "success": true, "testCase": { "description": "47 image_attached pub crawl lvyo_bv0_candidate_047", "vars": { "caseId": "lvyo_bv0_candidate_047", "customerInput": "Can you use this photo of the first pub sign and make a crawl tee, but don't copy the pub name, just the old sign style", "modality": "image_attached", "cluster": "pub crawl" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_047\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } ], "options": {}, "metadata": {} }, "testIdx": 40, "tokenUsage": { "prompt": 0, "completion": 0, "cached": 0, "total": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 }, "assertions": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 } } }, "vars": { "caseId": "lvyo_bv0_candidate_047", "customerInput": "Can you use this photo of the first pub sign and make a crawl tee, but don't copy the pub name, just the old sign style", "modality": "image_attached", "cluster": "pub crawl" }, "metadata": { "caseId": "lvyo_bv0_candidate_047", "modality": "image_attached", "cluster": "pub crawl", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_047/artifact-lock.json", "finalSha256": "7d0a6a08a712826048d9a3dd81594ed65396f91326e20abfb943e094cb28eb89", "mockupSha256": "828bbaa5ee618f7153f74b3a956c64bb4c0829e72b3eeebb30ed9e031c3ae991", "publicMockup": "assets/lvyo_bv0_candidate_047_m4_mockup.webp", "noSpendReplay": true, "_promptfooFileMetadata": {} }, "failureReason": 0 }, { "cost": 0, "gradingResult": { "pass": true, "score": 1, "reason": "All assertions passed", "namedScores": {}, "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 0 }, "componentResults": [ { "pass": true, "score": 1, "reason": "Assertion passed", "assertion": { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_048\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } } ] }, "id": "31a5d85a-cb55-41ca-a521-53f9114502f8", "latencyMs": 1, "namedScores": {}, "prompt": { "raw": "Make this brunch table photo into a cute birthday tee for Jess, please include the pancakes and prosecco glasses", "label": "{{customerInput}}", "config": {} }, "promptId": "7e756b499a1da155962cf88585b671e3ddacc5edf410c782ea1dbff2b92d0a70", "promptIdx": 0, "provider": { "id": "file://./providers/benchmark-v0-replay-provider.mjs", "label": "levyo-benchmark-v0-artifact-replay" }, "response": { "output": "{\"ok\":true,\"schemaVersion\":\"levyo_promptfoo_benchmark_v0_replay_result_v1\",\"caseId\":\"lvyo_bv0_candidate_048\",\"promptEcho\":\"Make this brunch table photo into a cute birthday tee for Jess, please include the pancakes and prosecco glasses\",\"status\":\"success\",\"modality\":\"image_attached\",\"caseRole\":\"image_pack\",\"cluster\":\"bottomless brunch\",\"runtimeProfile\":\"internal_codex\",\"imageProvider\":\"codex_account_ui\",\"lockRef\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_048/artifact-lock.json\",\"lockSha256\":\"40a4c1ccdc03c403525d6d80e3e34755cc7286e52bea89c80d84dc84ffa9d459\",\"final\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_048/artifact/final.png\",\"sha256\":\"6e57572f97601818b5d6c8f9b1e9e096ea4a7334e39987b9129597a4df02ed71\"},\"mockup\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_048/artifact/mockup.png\",\"sha256\":\"6490f9ffd15cbfcf4dde8840b133093a26f1fa0ff8592aae30d806e298fd4773\",\"publicSrc\":\"assets/lvyo_bv0_candidate_048_m4_mockup.webp\"},\"referenceImage\":{\"ref\":\"data/reference_inputs/references/lvyo_bv0_candidate_048_reference.png\",\"sha256\":\"6b4b4477a63901652e401dbc932a5b6063854045f277f9b7c454bbb95f4aee95\"},\"noSpendReplay\":true,\"customerLiveFlowChanged\":false}", "images": [ { "mimeType": "image/webp", "blobRef": { "uri": "promptfoo://blob/736ff3a7ee85610f65fbf908cdd39ce743fb81dd4b8ae368201b6589632546fd", "hash": "736ff3a7ee85610f65fbf908cdd39ce743fb81dd4b8ae368201b6589632546fd", "mimeType": "image/webp", "sizeBytes": 42542, "provider": "filesystem" } } ], "metadata": { "caseId": "lvyo_bv0_candidate_048", "modality": "image_attached", "cluster": "bottomless brunch", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_048/artifact-lock.json", "finalSha256": "6e57572f97601818b5d6c8f9b1e9e096ea4a7334e39987b9129597a4df02ed71", "mockupSha256": "6490f9ffd15cbfcf4dde8840b133093a26f1fa0ff8592aae30d806e298fd4773", "publicMockup": "assets/lvyo_bv0_candidate_048_m4_mockup.webp", "noSpendReplay": true } }, "score": 1, "success": true, "testCase": { "description": "48 image_attached bottomless brunch lvyo_bv0_candidate_048", "vars": { "caseId": "lvyo_bv0_candidate_048", "customerInput": "Make this brunch table photo into a cute birthday tee for Jess, please include the pancakes and prosecco glasses", "modality": "image_attached", "cluster": "bottomless brunch" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_048\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } ], "options": {}, "metadata": {} }, "testIdx": 41, "tokenUsage": { "prompt": 0, "completion": 0, "cached": 0, "total": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 }, "assertions": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 } } }, "vars": { "caseId": "lvyo_bv0_candidate_048", "customerInput": "Make this brunch table photo into a cute birthday tee for Jess, please include the pancakes and prosecco glasses", "modality": "image_attached", "cluster": "bottomless brunch" }, "metadata": { "caseId": "lvyo_bv0_candidate_048", "modality": "image_attached", "cluster": "bottomless brunch", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_048/artifact-lock.json", "finalSha256": "6e57572f97601818b5d6c8f9b1e9e096ea4a7334e39987b9129597a4df02ed71", "mockupSha256": "6490f9ffd15cbfcf4dde8840b133093a26f1fa0ff8592aae30d806e298fd4773", "publicMockup": "assets/lvyo_bv0_candidate_048_m4_mockup.webp", "noSpendReplay": true, "_promptfooFileMetadata": {} }, "failureReason": 0 }, { "cost": 0, "gradingResult": { "pass": true, "score": 1, "reason": "All assertions passed", "namedScores": {}, "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 0 }, "componentResults": [ { "pass": true, "score": 1, "reason": "Assertion passed", "assertion": { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_049\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } } ] }, "id": "2527447b-e887-42cb-bab8-785ac6cf907d", "latencyMs": 0, "namedScores": {}, "prompt": { "raw": "Use my setup photo for a gaming squad shirt. Purple lights, messy desk, title should be Respawn Roommates", "label": "{{customerInput}}", "config": {} }, "promptId": "7e756b499a1da155962cf88585b671e3ddacc5edf410c782ea1dbff2b92d0a70", "promptIdx": 0, "provider": { "id": "file://./providers/benchmark-v0-replay-provider.mjs", "label": "levyo-benchmark-v0-artifact-replay" }, "response": { "output": "{\"ok\":true,\"schemaVersion\":\"levyo_promptfoo_benchmark_v0_replay_result_v1\",\"caseId\":\"lvyo_bv0_candidate_049\",\"promptEcho\":\"Use my setup photo for a gaming squad shirt. Purple lights, messy desk, title should be Respawn Roommates\",\"status\":\"success\",\"modality\":\"image_attached\",\"caseRole\":\"image_pack\",\"cluster\":\"gaming squad\",\"runtimeProfile\":\"internal_codex\",\"imageProvider\":\"codex_account_ui\",\"lockRef\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_049/artifact-lock.json\",\"lockSha256\":\"37a4309eb66d673543f073d5ce00c3a57d2c3c70179b0eb3364340791930083f\",\"final\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_049/artifact/final.png\",\"sha256\":\"84757b069e88f55e68283c8f3cd988b0976fa69a67857118ed4336329ca10e52\"},\"mockup\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_049/artifact/mockup.png\",\"sha256\":\"c23e4bbce6053931cd64169d86b3638f3c7c011bc6cabccbf5a1c79b8e160fd2\",\"publicSrc\":\"assets/lvyo_bv0_candidate_049_m4_mockup.webp\"},\"referenceImage\":{\"ref\":\"data/reference_inputs/references/lvyo_bv0_candidate_049_reference.png\",\"sha256\":\"adc5b0bea9d1ad7720cc91e6821f6eae3e3ff63ecf6b9a909a4ec0c11dff51e8\"},\"noSpendReplay\":true,\"customerLiveFlowChanged\":false}", "images": [ { "mimeType": "image/webp", "blobRef": { "uri": "promptfoo://blob/a1c116551669050ce8924e0bab7519514014f7d88840ffa32c787fc840941b82", "hash": "a1c116551669050ce8924e0bab7519514014f7d88840ffa32c787fc840941b82", "mimeType": "image/webp", "sizeBytes": 44566, "provider": "filesystem" } } ], "metadata": { "caseId": "lvyo_bv0_candidate_049", "modality": "image_attached", "cluster": "gaming squad", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_049/artifact-lock.json", "finalSha256": "84757b069e88f55e68283c8f3cd988b0976fa69a67857118ed4336329ca10e52", "mockupSha256": "c23e4bbce6053931cd64169d86b3638f3c7c011bc6cabccbf5a1c79b8e160fd2", "publicMockup": "assets/lvyo_bv0_candidate_049_m4_mockup.webp", "noSpendReplay": true } }, "score": 1, "success": true, "testCase": { "description": "49 image_attached gaming squad lvyo_bv0_candidate_049", "vars": { "caseId": "lvyo_bv0_candidate_049", "customerInput": "Use my setup photo for a gaming squad shirt. Purple lights, messy desk, title should be Respawn Roommates", "modality": "image_attached", "cluster": "gaming squad" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_049\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } ], "options": {}, "metadata": {} }, "testIdx": 42, "tokenUsage": { "prompt": 0, "completion": 0, "cached": 0, "total": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 }, "assertions": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 } } }, "vars": { "caseId": "lvyo_bv0_candidate_049", "customerInput": "Use my setup photo for a gaming squad shirt. Purple lights, messy desk, title should be Respawn Roommates", "modality": "image_attached", "cluster": "gaming squad" }, "metadata": { "caseId": "lvyo_bv0_candidate_049", "modality": "image_attached", "cluster": "gaming squad", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_049/artifact-lock.json", "finalSha256": "84757b069e88f55e68283c8f3cd988b0976fa69a67857118ed4336329ca10e52", "mockupSha256": "c23e4bbce6053931cd64169d86b3638f3c7c011bc6cabccbf5a1c79b8e160fd2", "publicMockup": "assets/lvyo_bv0_candidate_049_m4_mockup.webp", "noSpendReplay": true, "_promptfooFileMetadata": {} }, "failureReason": 0 }, { "cost": 0, "gradingResult": { "pass": true, "score": 1, "reason": "All assertions passed", "namedScores": {}, "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 0 }, "componentResults": [ { "pass": true, "score": 1, "reason": "Assertion passed", "assertion": { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_050\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } } ] }, "id": "94956f80-a828-4a56-8960-fdc94aef42c4", "latencyMs": 0, "namedScores": {}, "prompt": { "raw": "Turn this awkward offsite photo into a nicer team retreat shirt. Keep the mountain cabin and coffee cups, lose the corporate cringe", "label": "{{customerInput}}", "config": {} }, "promptId": "7e756b499a1da155962cf88585b671e3ddacc5edf410c782ea1dbff2b92d0a70", "promptIdx": 0, "provider": { "id": "file://./providers/benchmark-v0-replay-provider.mjs", "label": "levyo-benchmark-v0-artifact-replay" }, "response": { "output": "{\"ok\":true,\"schemaVersion\":\"levyo_promptfoo_benchmark_v0_replay_result_v1\",\"caseId\":\"lvyo_bv0_candidate_050\",\"promptEcho\":\"Turn this awkward offsite photo into a nicer team retreat shirt. Keep the mountain cabin and coffee cups, lose the corporate cringe\",\"status\":\"success\",\"modality\":\"image_attached\",\"caseRole\":\"image_pack\",\"cluster\":\"company retreat\",\"runtimeProfile\":\"internal_codex\",\"imageProvider\":\"codex_account_ui\",\"lockRef\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_050/artifact-lock.json\",\"lockSha256\":\"33bbef4007f58cd0b53751dad801c07d178486a76d855b31d112167bf8c0073e\",\"final\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_050/artifact/final.png\",\"sha256\":\"1d545dd740c571dc62ac459619278f70bcb60c32cf73ce631886561fe5d00c02\"},\"mockup\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_050/artifact/mockup.png\",\"sha256\":\"4218b44bc0313dabff30bb9223612a8f4c493fefb6ff7279d3a44c15d1ae11ec\",\"publicSrc\":\"assets/lvyo_bv0_candidate_050_m4_mockup.webp\"},\"referenceImage\":{\"ref\":\"data/reference_inputs/references/lvyo_bv0_candidate_050_reference.png\",\"sha256\":\"664e517af7c8edf96ecf5f0e88cb3da00b3a3d64fab6c5b64f5752302fbff539\"},\"noSpendReplay\":true,\"customerLiveFlowChanged\":false}", "images": [ { "mimeType": "image/webp", "blobRef": { "uri": "promptfoo://blob/646370a411e117dc259c65ba61005bcb96fc5ae22fdf2ff9427fd61a9e27e1e7", "hash": "646370a411e117dc259c65ba61005bcb96fc5ae22fdf2ff9427fd61a9e27e1e7", "mimeType": "image/webp", "sizeBytes": 43448, "provider": "filesystem" } } ], "metadata": { "caseId": "lvyo_bv0_candidate_050", "modality": "image_attached", "cluster": "company retreat", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_050/artifact-lock.json", "finalSha256": "1d545dd740c571dc62ac459619278f70bcb60c32cf73ce631886561fe5d00c02", "mockupSha256": "4218b44bc0313dabff30bb9223612a8f4c493fefb6ff7279d3a44c15d1ae11ec", "publicMockup": "assets/lvyo_bv0_candidate_050_m4_mockup.webp", "noSpendReplay": true } }, "score": 1, "success": true, "testCase": { "description": "50 image_attached company retreat lvyo_bv0_candidate_050", "vars": { "caseId": "lvyo_bv0_candidate_050", "customerInput": "Turn this awkward offsite photo into a nicer team retreat shirt. Keep the mountain cabin and coffee cups, lose the corporate cringe", "modality": "image_attached", "cluster": "company retreat" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_050\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } ], "options": {}, "metadata": {} }, "testIdx": 43, "tokenUsage": { "prompt": 0, "completion": 0, "cached": 0, "total": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 }, "assertions": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 } } }, "vars": { "caseId": "lvyo_bv0_candidate_050", "customerInput": "Turn this awkward offsite photo into a nicer team retreat shirt. Keep the mountain cabin and coffee cups, lose the corporate cringe", "modality": "image_attached", "cluster": "company retreat" }, "metadata": { "caseId": "lvyo_bv0_candidate_050", "modality": "image_attached", "cluster": "company retreat", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_050/artifact-lock.json", "finalSha256": "1d545dd740c571dc62ac459619278f70bcb60c32cf73ce631886561fe5d00c02", "mockupSha256": "4218b44bc0313dabff30bb9223612a8f4c493fefb6ff7279d3a44c15d1ae11ec", "publicMockup": "assets/lvyo_bv0_candidate_050_m4_mockup.webp", "noSpendReplay": true, "_promptfooFileMetadata": {} }, "failureReason": 0 }, { "cost": 0, "gradingResult": { "pass": true, "score": 1, "reason": "All assertions passed", "namedScores": {}, "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 0 }, "componentResults": [ { "pass": true, "score": 1, "reason": "Assertion passed", "assertion": { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_051\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } } ] }, "id": "ef011211-987f-45ed-b474-8648569f45bd", "latencyMs": 0, "namedScores": {}, "prompt": { "raw": "Use this laptop graveyard photo for our hackathon shirts. Text idea: Merge Conflict Survivors", "label": "{{customerInput}}", "config": {} }, "promptId": "7e756b499a1da155962cf88585b671e3ddacc5edf410c782ea1dbff2b92d0a70", "promptIdx": 0, "provider": { "id": "file://./providers/benchmark-v0-replay-provider.mjs", "label": "levyo-benchmark-v0-artifact-replay" }, "response": { "output": "{\"ok\":true,\"schemaVersion\":\"levyo_promptfoo_benchmark_v0_replay_result_v1\",\"caseId\":\"lvyo_bv0_candidate_051\",\"promptEcho\":\"Use this laptop graveyard photo for our hackathon shirts. Text idea: Merge Conflict Survivors\",\"status\":\"success\",\"modality\":\"image_attached\",\"caseRole\":\"image_pack\",\"cluster\":\"hackathon\",\"runtimeProfile\":\"internal_codex\",\"imageProvider\":\"codex_account_ui\",\"lockRef\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_051/artifact-lock.json\",\"lockSha256\":\"54d132d1c8835c7c61707a441a8fca336cc38ac7323cf60a0afbbf2da831f69d\",\"final\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_051/artifact/final.png\",\"sha256\":\"de2bcb33af81e39a7fc947d6acdf7cdf360ac3eec9cf189d1d5629b3285de1ce\"},\"mockup\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_051/artifact/mockup.png\",\"sha256\":\"a3909123dfdb0f0e34d88c53ea127e6903494f46e5f401162fa5b400fc0e19cd\",\"publicSrc\":\"assets/lvyo_bv0_candidate_051_m4_mockup.webp\"},\"referenceImage\":{\"ref\":\"data/reference_inputs/references/lvyo_bv0_candidate_051_reference.png\",\"sha256\":\"128f4fb8452126c07ef6347db1bb78a85f9fc2ec2e61e31cc1984e950ad8a506\"},\"noSpendReplay\":true,\"customerLiveFlowChanged\":false}", "images": [ { "mimeType": "image/webp", "blobRef": { "uri": "promptfoo://blob/a03252e928611fce645cf4d5a7b41ebbc977ba2b7e5f733f09b365c332c11524", "hash": "a03252e928611fce645cf4d5a7b41ebbc977ba2b7e5f733f09b365c332c11524", "mimeType": "image/webp", "sizeBytes": 49352, "provider": "filesystem" } } ], "metadata": { "caseId": "lvyo_bv0_candidate_051", "modality": "image_attached", "cluster": "hackathon", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_051/artifact-lock.json", "finalSha256": "de2bcb33af81e39a7fc947d6acdf7cdf360ac3eec9cf189d1d5629b3285de1ce", "mockupSha256": "a3909123dfdb0f0e34d88c53ea127e6903494f46e5f401162fa5b400fc0e19cd", "publicMockup": "assets/lvyo_bv0_candidate_051_m4_mockup.webp", "noSpendReplay": true } }, "score": 1, "success": true, "testCase": { "description": "51 image_attached hackathon lvyo_bv0_candidate_051", "vars": { "caseId": "lvyo_bv0_candidate_051", "customerInput": "Use this laptop graveyard photo for our hackathon shirts. Text idea: Merge Conflict Survivors", "modality": "image_attached", "cluster": "hackathon" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_051\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } ], "options": {}, "metadata": {} }, "testIdx": 44, "tokenUsage": { "prompt": 0, "completion": 0, "cached": 0, "total": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 }, "assertions": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 } } }, "vars": { "caseId": "lvyo_bv0_candidate_051", "customerInput": "Use this laptop graveyard photo for our hackathon shirts. Text idea: Merge Conflict Survivors", "modality": "image_attached", "cluster": "hackathon" }, "metadata": { "caseId": "lvyo_bv0_candidate_051", "modality": "image_attached", "cluster": "hackathon", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_051/artifact-lock.json", "finalSha256": "de2bcb33af81e39a7fc947d6acdf7cdf360ac3eec9cf189d1d5629b3285de1ce", "mockupSha256": "a3909123dfdb0f0e34d88c53ea127e6903494f46e5f401162fa5b400fc0e19cd", "publicMockup": "assets/lvyo_bv0_candidate_051_m4_mockup.webp", "noSpendReplay": true, "_promptfooFileMetadata": {} }, "failureReason": 0 }, { "cost": 0, "gradingResult": { "pass": true, "score": 1, "reason": "All assertions passed", "namedScores": {}, "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 0 }, "componentResults": [ { "pass": true, "score": 1, "reason": "Assertion passed", "assertion": { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_052\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } } ] }, "id": "5c76e17f-e3a3-420a-88b7-b1871133a74c", "latencyMs": 0, "namedScores": {}, "prompt": { "raw": "Please base the charity run shirt on this trainer photo, with Team Nora and a sunrise, make it hopeful", "label": "{{customerInput}}", "config": {} }, "promptId": "7e756b499a1da155962cf88585b671e3ddacc5edf410c782ea1dbff2b92d0a70", "promptIdx": 0, "provider": { "id": "file://./providers/benchmark-v0-replay-provider.mjs", "label": "levyo-benchmark-v0-artifact-replay" }, "response": { "output": "{\"ok\":true,\"schemaVersion\":\"levyo_promptfoo_benchmark_v0_replay_result_v1\",\"caseId\":\"lvyo_bv0_candidate_052\",\"promptEcho\":\"Please base the charity run shirt on this trainer photo, with Team Nora and a sunrise, make it hopeful\",\"status\":\"success\",\"modality\":\"image_attached\",\"caseRole\":\"image_pack\",\"cluster\":\"charity run\",\"runtimeProfile\":\"internal_codex\",\"imageProvider\":\"codex_account_ui\",\"lockRef\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_052/artifact-lock.json\",\"lockSha256\":\"de21bfc65f9d7f8e46c600536cde1b8005f5783b3b8c02f0a840d87e9f55d793\",\"final\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_052/artifact/final.png\",\"sha256\":\"bc9282c251241bc970ce9b6fc432f950250fbe2fd7d693ac7032ca4b3da00dd7\"},\"mockup\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_052/artifact/mockup.png\",\"sha256\":\"dd32843a0499bd0c857091bb5c58ab6eb82397a84486684bdd756897ec0426ae\",\"publicSrc\":\"assets/lvyo_bv0_candidate_052_m4_mockup.webp\"},\"referenceImage\":{\"ref\":\"data/reference_inputs/references/lvyo_bv0_candidate_052_reference.png\",\"sha256\":\"360534fe30dd72cac7e37d2680643b8717a80f1beb4a12373c8a12bd47d885c1\"},\"noSpendReplay\":true,\"customerLiveFlowChanged\":false}", "images": [ { "mimeType": "image/webp", "blobRef": { "uri": "promptfoo://blob/439e4a9e6fdd905424f942fd414b4aad57d1015b49b5539a6df5c324d2c25679", "hash": "439e4a9e6fdd905424f942fd414b4aad57d1015b49b5539a6df5c324d2c25679", "mimeType": "image/webp", "sizeBytes": 44590, "provider": "filesystem" } } ], "metadata": { "caseId": "lvyo_bv0_candidate_052", "modality": "image_attached", "cluster": "charity run", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_052/artifact-lock.json", "finalSha256": "bc9282c251241bc970ce9b6fc432f950250fbe2fd7d693ac7032ca4b3da00dd7", "mockupSha256": "dd32843a0499bd0c857091bb5c58ab6eb82397a84486684bdd756897ec0426ae", "publicMockup": "assets/lvyo_bv0_candidate_052_m4_mockup.webp", "noSpendReplay": true } }, "score": 1, "success": true, "testCase": { "description": "52 image_attached charity run lvyo_bv0_candidate_052", "vars": { "caseId": "lvyo_bv0_candidate_052", "customerInput": "Please base the charity run shirt on this trainer photo, with Team Nora and a sunrise, make it hopeful", "modality": "image_attached", "cluster": "charity run" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_052\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } ], "options": {}, "metadata": {} }, "testIdx": 45, "tokenUsage": { "prompt": 0, "completion": 0, "cached": 0, "total": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 }, "assertions": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 } } }, "vars": { "caseId": "lvyo_bv0_candidate_052", "customerInput": "Please base the charity run shirt on this trainer photo, with Team Nora and a sunrise, make it hopeful", "modality": "image_attached", "cluster": "charity run" }, "metadata": { "caseId": "lvyo_bv0_candidate_052", "modality": "image_attached", "cluster": "charity run", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_052/artifact-lock.json", "finalSha256": "bc9282c251241bc970ce9b6fc432f950250fbe2fd7d693ac7032ca4b3da00dd7", "mockupSha256": "dd32843a0499bd0c857091bb5c58ab6eb82397a84486684bdd756897ec0426ae", "publicMockup": "assets/lvyo_bv0_candidate_052_m4_mockup.webp", "noSpendReplay": true, "_promptfooFileMetadata": {} }, "failureReason": 0 }, { "cost": 0, "gradingResult": { "pass": true, "score": 1, "reason": "All assertions passed", "namedScores": {}, "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 0 }, "componentResults": [ { "pass": true, "score": 1, "reason": "Assertion passed", "assertion": { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_053\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } } ] }, "id": "b9bc5262-fd17-4589-9093-2c54c9159f12", "latencyMs": 0, "namedScores": {}, "prompt": { "raw": "Can you make a welcome weekend tee from this beach photo where Alex and Noor got engaged, subtle and not cheesy", "label": "{{customerInput}}", "config": {} }, "promptId": "7e756b499a1da155962cf88585b671e3ddacc5edf410c782ea1dbff2b92d0a70", "promptIdx": 0, "provider": { "id": "file://./providers/benchmark-v0-replay-provider.mjs", "label": "levyo-benchmark-v0-artifact-replay" }, "response": { "output": "{\"ok\":true,\"schemaVersion\":\"levyo_promptfoo_benchmark_v0_replay_result_v1\",\"caseId\":\"lvyo_bv0_candidate_053\",\"promptEcho\":\"Can you make a welcome weekend tee from this beach photo where Alex and Noor got engaged, subtle and not cheesy\",\"status\":\"success\",\"modality\":\"image_attached\",\"caseRole\":\"image_pack\",\"cluster\":\"wedding weekend\",\"runtimeProfile\":\"internal_codex\",\"imageProvider\":\"codex_account_ui\",\"lockRef\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_053/artifact-lock.json\",\"lockSha256\":\"bd0a55253043f0737f9d2221817bbc9701153859d40a54ca09931585070ded18\",\"final\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_053/artifact/final.png\",\"sha256\":\"52adb42e09ab6a794649b4dd61e6c9691ed85d1f56bfd4f3402700d0c8876163\"},\"mockup\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_053/artifact/mockup.png\",\"sha256\":\"3a76e77e109b05da8efec539c444e675ce5ccffa93bf988ee6b758dddf21273a\",\"publicSrc\":\"assets/lvyo_bv0_candidate_053_m4_mockup.webp\"},\"referenceImage\":{\"ref\":\"data/reference_inputs/references/lvyo_bv0_candidate_053_reference.png\",\"sha256\":\"9ff12e0c25b69dd87cd5e187b3b20ccd2ecdeee7198787ccec1dd2fbcc2bfd99\"},\"noSpendReplay\":true,\"customerLiveFlowChanged\":false}", "images": [ { "mimeType": "image/webp", "blobRef": { "uri": "promptfoo://blob/b4b9cf685d1a08c750cb582ab6691498a56a79c02ba8ede1cd2be2c4acac8e4c", "hash": "b4b9cf685d1a08c750cb582ab6691498a56a79c02ba8ede1cd2be2c4acac8e4c", "mimeType": "image/webp", "sizeBytes": 41612, "provider": "filesystem" } } ], "metadata": { "caseId": "lvyo_bv0_candidate_053", "modality": "image_attached", "cluster": "wedding weekend", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_053/artifact-lock.json", "finalSha256": "52adb42e09ab6a794649b4dd61e6c9691ed85d1f56bfd4f3402700d0c8876163", "mockupSha256": "3a76e77e109b05da8efec539c444e675ce5ccffa93bf988ee6b758dddf21273a", "publicMockup": "assets/lvyo_bv0_candidate_053_m4_mockup.webp", "noSpendReplay": true } }, "score": 1, "success": true, "testCase": { "description": "53 image_attached wedding weekend lvyo_bv0_candidate_053", "vars": { "caseId": "lvyo_bv0_candidate_053", "customerInput": "Can you make a welcome weekend tee from this beach photo where Alex and Noor got engaged, subtle and not cheesy", "modality": "image_attached", "cluster": "wedding weekend" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_053\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } ], "options": {}, "metadata": {} }, "testIdx": 46, "tokenUsage": { "prompt": 0, "completion": 0, "cached": 0, "total": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 }, "assertions": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 } } }, "vars": { "caseId": "lvyo_bv0_candidate_053", "customerInput": "Can you make a welcome weekend tee from this beach photo where Alex and Noor got engaged, subtle and not cheesy", "modality": "image_attached", "cluster": "wedding weekend" }, "metadata": { "caseId": "lvyo_bv0_candidate_053", "modality": "image_attached", "cluster": "wedding weekend", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_053/artifact-lock.json", "finalSha256": "52adb42e09ab6a794649b4dd61e6c9691ed85d1f56bfd4f3402700d0c8876163", "mockupSha256": "3a76e77e109b05da8efec539c444e675ce5ccffa93bf988ee6b758dddf21273a", "publicMockup": "assets/lvyo_bv0_candidate_053_m4_mockup.webp", "noSpendReplay": true, "_promptfooFileMetadata": {} }, "failureReason": 0 }, { "cost": 0, "gradingResult": { "pass": true, "score": 1, "reason": "All assertions passed", "namedScores": {}, "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 0 }, "componentResults": [ { "pass": true, "score": 1, "reason": "Assertion passed", "assertion": { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_054\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } } ] }, "id": "cc86dd4d-6bcd-4f37-9010-74d9b01abd5f", "latencyMs": 0, "namedScores": {}, "prompt": { "raw": "Use the locker photo for leavers shirts, class of 2026, doodles and signatures kind of look", "label": "{{customerInput}}", "config": {} }, "promptId": "7e756b499a1da155962cf88585b671e3ddacc5edf410c782ea1dbff2b92d0a70", "promptIdx": 0, "provider": { "id": "file://./providers/benchmark-v0-replay-provider.mjs", "label": "levyo-benchmark-v0-artifact-replay" }, "response": { "output": "{\"ok\":true,\"schemaVersion\":\"levyo_promptfoo_benchmark_v0_replay_result_v1\",\"caseId\":\"lvyo_bv0_candidate_054\",\"promptEcho\":\"Use the locker photo for leavers shirts, class of 2026, doodles and signatures kind of look\",\"status\":\"success\",\"modality\":\"image_attached\",\"caseRole\":\"image_pack\",\"cluster\":\"school leavers\",\"runtimeProfile\":\"internal_codex\",\"imageProvider\":\"codex_account_ui\",\"lockRef\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_054/artifact-lock.json\",\"lockSha256\":\"2551aa9acf77daf8de604af03a55d03d5d9e899562d8194c4ab75b0ecc3a977a\",\"final\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_054/artifact/final.png\",\"sha256\":\"adff8ddd0cbb8b6b0e796f6fb362e1557860b7bc5cb5384a052d8dd033a2f829\"},\"mockup\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_054/artifact/mockup.png\",\"sha256\":\"155bb30ae93710a21067fc2150f284ef9a925cff2db03d4a42fcf81923b23f3b\",\"publicSrc\":\"assets/lvyo_bv0_candidate_054_m4_mockup.webp\"},\"referenceImage\":{\"ref\":\"data/reference_inputs/references/lvyo_bv0_candidate_054_reference.png\",\"sha256\":\"418486e1c891726a458a31567ec541bb11b7a1bf3079969ebd6ebe4177ea3eac\"},\"noSpendReplay\":true,\"customerLiveFlowChanged\":false}", "images": [ { "mimeType": "image/webp", "blobRef": { "uri": "promptfoo://blob/9b18ed72b449f5b72495e581f724d2ef9ee4ae67b8df16008c1eb4a64ea98e14", "hash": "9b18ed72b449f5b72495e581f724d2ef9ee4ae67b8df16008c1eb4a64ea98e14", "mimeType": "image/webp", "sizeBytes": 49046, "provider": "filesystem" } } ], "metadata": { "caseId": "lvyo_bv0_candidate_054", "modality": "image_attached", "cluster": "school leavers", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_054/artifact-lock.json", "finalSha256": "adff8ddd0cbb8b6b0e796f6fb362e1557860b7bc5cb5384a052d8dd033a2f829", "mockupSha256": "155bb30ae93710a21067fc2150f284ef9a925cff2db03d4a42fcf81923b23f3b", "publicMockup": "assets/lvyo_bv0_candidate_054_m4_mockup.webp", "noSpendReplay": true } }, "score": 1, "success": true, "testCase": { "description": "54 image_attached school leavers lvyo_bv0_candidate_054", "vars": { "caseId": "lvyo_bv0_candidate_054", "customerInput": "Use the locker photo for leavers shirts, class of 2026, doodles and signatures kind of look", "modality": "image_attached", "cluster": "school leavers" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_054\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } ], "options": {}, "metadata": {} }, "testIdx": 47, "tokenUsage": { "prompt": 0, "completion": 0, "cached": 0, "total": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 }, "assertions": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 } } }, "vars": { "caseId": "lvyo_bv0_candidate_054", "customerInput": "Use the locker photo for leavers shirts, class of 2026, doodles and signatures kind of look", "modality": "image_attached", "cluster": "school leavers" }, "metadata": { "caseId": "lvyo_bv0_candidate_054", "modality": "image_attached", "cluster": "school leavers", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_054/artifact-lock.json", "finalSha256": "adff8ddd0cbb8b6b0e796f6fb362e1557860b7bc5cb5384a052d8dd033a2f829", "mockupSha256": "155bb30ae93710a21067fc2150f284ef9a925cff2db03d4a42fcf81923b23f3b", "publicMockup": "assets/lvyo_bv0_candidate_054_m4_mockup.webp", "noSpendReplay": true, "_promptfooFileMetadata": {} }, "failureReason": 0 }, { "cost": 0, "gradingResult": { "pass": true, "score": 1, "reason": "All assertions passed", "namedScores": {}, "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 0 }, "componentResults": [ { "pass": true, "score": 1, "reason": "Assertion passed", "assertion": { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_055\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } } ] }, "id": "716acc96-a68e-4268-8c8b-5034d956908d", "latencyMs": 1, "namedScores": {}, "prompt": { "raw": "Make our surf trip tee from this pic of the boards on the van. Text: Wipeout Witnesses", "label": "{{customerInput}}", "config": {} }, "promptId": "7e756b499a1da155962cf88585b671e3ddacc5edf410c782ea1dbff2b92d0a70", "promptIdx": 0, "provider": { "id": "file://./providers/benchmark-v0-replay-provider.mjs", "label": "levyo-benchmark-v0-artifact-replay" }, "response": { "output": "{\"ok\":true,\"schemaVersion\":\"levyo_promptfoo_benchmark_v0_replay_result_v1\",\"caseId\":\"lvyo_bv0_candidate_055\",\"promptEcho\":\"Make our surf trip tee from this pic of the boards on the van. Text: Wipeout Witnesses\",\"status\":\"success\",\"modality\":\"image_attached\",\"caseRole\":\"image_pack\",\"cluster\":\"surf trip\",\"runtimeProfile\":\"internal_codex\",\"imageProvider\":\"codex_account_ui\",\"lockRef\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_055/artifact-lock.json\",\"lockSha256\":\"187881fb3d4fac3778ad40fad6bbe97bb5c318c9c45366f77850b6e5423a6b47\",\"final\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_055/artifact/final.png\",\"sha256\":\"29d8aac339305c03e8ef7401b376cd49f60e8cd7ea2bbba7c69b3d63c344996f\"},\"mockup\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_055/artifact/mockup.png\",\"sha256\":\"f21dcfe33b20866fbc40eb808122620498f5186c492ed50fab9cd1db9eb99e8b\",\"publicSrc\":\"assets/lvyo_bv0_candidate_055_m4_mockup.webp\"},\"referenceImage\":{\"ref\":\"data/reference_inputs/references/lvyo_bv0_candidate_055_reference.png\",\"sha256\":\"39112d9b6a9b2f4db85f55eaed36474262fb79093e51add6837dd7951b774075\"},\"noSpendReplay\":true,\"customerLiveFlowChanged\":false}", "images": [ { "mimeType": "image/webp", "blobRef": { "uri": "promptfoo://blob/09bb0215ff0b38dc9d6c99cfa9007ee0cdd5583790b6c57628008d612d96f6d2", "hash": "09bb0215ff0b38dc9d6c99cfa9007ee0cdd5583790b6c57628008d612d96f6d2", "mimeType": "image/webp", "sizeBytes": 42898, "provider": "filesystem" } } ], "metadata": { "caseId": "lvyo_bv0_candidate_055", "modality": "image_attached", "cluster": "surf trip", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_055/artifact-lock.json", "finalSha256": "29d8aac339305c03e8ef7401b376cd49f60e8cd7ea2bbba7c69b3d63c344996f", "mockupSha256": "f21dcfe33b20866fbc40eb808122620498f5186c492ed50fab9cd1db9eb99e8b", "publicMockup": "assets/lvyo_bv0_candidate_055_m4_mockup.webp", "noSpendReplay": true } }, "score": 1, "success": true, "testCase": { "description": "55 image_attached surf trip lvyo_bv0_candidate_055", "vars": { "caseId": "lvyo_bv0_candidate_055", "customerInput": "Make our surf trip tee from this pic of the boards on the van. Text: Wipeout Witnesses", "modality": "image_attached", "cluster": "surf trip" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_055\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } ], "options": {}, "metadata": {} }, "testIdx": 48, "tokenUsage": { "prompt": 0, "completion": 0, "cached": 0, "total": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 }, "assertions": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 } } }, "vars": { "caseId": "lvyo_bv0_candidate_055", "customerInput": "Make our surf trip tee from this pic of the boards on the van. Text: Wipeout Witnesses", "modality": "image_attached", "cluster": "surf trip" }, "metadata": { "caseId": "lvyo_bv0_candidate_055", "modality": "image_attached", "cluster": "surf trip", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_055/artifact-lock.json", "finalSha256": "29d8aac339305c03e8ef7401b376cd49f60e8cd7ea2bbba7c69b3d63c344996f", "mockupSha256": "f21dcfe33b20866fbc40eb808122620498f5186c492ed50fab9cd1db9eb99e8b", "publicMockup": "assets/lvyo_bv0_candidate_055_m4_mockup.webp", "noSpendReplay": true, "_promptfooFileMetadata": {} }, "failureReason": 0 }, { "cost": 0, "gradingResult": { "pass": true, "score": 1, "reason": "All assertions passed", "namedScores": {}, "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 0 }, "componentResults": [ { "pass": true, "score": 1, "reason": "Assertion passed", "assertion": { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_056\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } } ] }, "id": "ea0296fd-34e6-4c38-ab91-0bf287f73904", "latencyMs": 0, "namedScores": {}, "prompt": { "raw": "Use this blurry late night metro photo as inspo for our Paris shirts, make it look intentional not like we can't take photos", "label": "{{customerInput}}", "config": {} }, "promptId": "7e756b499a1da155962cf88585b671e3ddacc5edf410c782ea1dbff2b92d0a70", "promptIdx": 0, "provider": { "id": "file://./providers/benchmark-v0-replay-provider.mjs", "label": "levyo-benchmark-v0-artifact-replay" }, "response": { "output": "{\"ok\":true,\"schemaVersion\":\"levyo_promptfoo_benchmark_v0_replay_result_v1\",\"caseId\":\"lvyo_bv0_candidate_056\",\"promptEcho\":\"Use this blurry late night metro photo as inspo for our Paris shirts, make it look intentional not like we can't take photos\",\"status\":\"success\",\"modality\":\"image_attached\",\"caseRole\":\"image_pack\",\"cluster\":\"city break\",\"runtimeProfile\":\"internal_codex\",\"imageProvider\":\"codex_account_ui\",\"lockRef\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_056/artifact-lock.json\",\"lockSha256\":\"be635fd5bbf99239389657cdfc2d38e2579411ebd7396a5c5c5bd0f0415d5eee\",\"final\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_056/artifact/final.png\",\"sha256\":\"4a26aa681090d392a750dab91ef78f107a23510c024584897871258c05f6601f\"},\"mockup\":{\"ref\":\"PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_056/artifact/mockup.png\",\"sha256\":\"1d6caa0a271f8bb44a9a239f30ad9220e47fbb325f2718708f94404eccd58075\",\"publicSrc\":\"assets/lvyo_bv0_candidate_056_m4_mockup.webp\"},\"referenceImage\":{\"ref\":\"data/reference_inputs/references/lvyo_bv0_candidate_056_reference.png\",\"sha256\":\"6c04ecf342cd58a1e5daaaecae08031a37468a460ce43ef4af20807f7716a389\"},\"noSpendReplay\":true,\"customerLiveFlowChanged\":false}", "images": [ { "mimeType": "image/webp", "blobRef": { "uri": "promptfoo://blob/1c41a861e01da3a712696c2cbbb825ef60a652bec130f609a1fa8ea6dd837774", "hash": "1c41a861e01da3a712696c2cbbb825ef60a652bec130f609a1fa8ea6dd837774", "mimeType": "image/webp", "sizeBytes": 38460, "provider": "filesystem" } } ], "metadata": { "caseId": "lvyo_bv0_candidate_056", "modality": "image_attached", "cluster": "city break", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_056/artifact-lock.json", "finalSha256": "4a26aa681090d392a750dab91ef78f107a23510c024584897871258c05f6601f", "mockupSha256": "1d6caa0a271f8bb44a9a239f30ad9220e47fbb325f2718708f94404eccd58075", "publicMockup": "assets/lvyo_bv0_candidate_056_m4_mockup.webp", "noSpendReplay": true } }, "score": 1, "success": true, "testCase": { "description": "56 image_attached city break lvyo_bv0_candidate_056", "vars": { "caseId": "lvyo_bv0_candidate_056", "customerInput": "Use this blurry late night metro photo as inspo for our Paris shirts, make it look intentional not like we can't take photos", "modality": "image_attached", "cluster": "city break" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_056\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } ], "options": {}, "metadata": {} }, "testIdx": 49, "tokenUsage": { "prompt": 0, "completion": 0, "cached": 0, "total": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 }, "assertions": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 1, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 } } }, "vars": { "caseId": "lvyo_bv0_candidate_056", "customerInput": "Use this blurry late night metro photo as inspo for our Paris shirts, make it look intentional not like we can't take photos", "modality": "image_attached", "cluster": "city break" }, "metadata": { "caseId": "lvyo_bv0_candidate_056", "modality": "image_attached", "cluster": "city break", "status": "success", "lockRef": "PLANS_WORK/levyo-benchmark-v0/data/baseline_champion/m4/cases/lvyo_bv0_candidate_056/artifact-lock.json", "finalSha256": "4a26aa681090d392a750dab91ef78f107a23510c024584897871258c05f6601f", "mockupSha256": "1d6caa0a271f8bb44a9a239f30ad9220e47fbb325f2718708f94404eccd58075", "publicMockup": "assets/lvyo_bv0_candidate_056_m4_mockup.webp", "noSpendReplay": true, "_promptfooFileMetadata": {} }, "failureReason": 0 } ], "stats": { "successes": 50, "failures": 0, "errors": 0, "tokenUsage": { "prompt": 0, "completion": 0, "cached": 0, "total": 0, "numRequests": 50, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 }, "assertions": { "total": 0, "prompt": 0, "completion": 0, "cached": 0, "numRequests": 0, "completionDetails": { "reasoning": 0, "acceptedPrediction": 0, "rejectedPrediction": 0, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0 } } }, "durationMs": 203, "evaluationDurationMs": 203 } }, "config": { "tags": {}, "description": "Levyo Benchmark v0 locked artifact replay - 50 cases", "prompts": [ "{{customerInput}}" ], "providers": [ { "id": "file://./providers/benchmark-v0-replay-provider.mjs", "label": "levyo-benchmark-v0-artifact-replay", "config": { "dataPath": "../../public/admin/benchmark-v0/data.json" } } ], "tests": [ { "description": "01 text_only stag weekend lvyo_bv0_candidate_001", "vars": { "caseId": "lvyo_bv0_candidate_001", "customerInput": "Need a stag do shirt for Ben's last ride, Prague 2026, make it look chaotic but not too cringe", "modality": "text_only", "cluster": "stag weekend" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_001\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ] }, { "description": "02 text_only hen party lvyo_bv0_candidate_002", "vars": { "caseId": "lvyo_bv0_candidate_002", "customerInput": "Bride tribe tee for Molly's hen in Brighton. Pink, sparkly, a bit cheeky. Names are Molly, Abi, Rach and Tilly", "modality": "text_only", "cluster": "hen party" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_002\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ] }, { "description": "03 text_only birthday party lvyo_bv0_candidate_003", "vars": { "caseId": "lvyo_bv0_candidate_003", "customerInput": "30th birthday shirt that says Still ID'd, Unfortunately Thriving. For a night out in Manchester", "modality": "text_only", "cluster": "birthday party" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_003\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ] }, { "description": "04 text_only graduation lvyo_bv0_candidate_004", "vars": { "caseId": "lvyo_bv0_candidate_004", "customerInput": "Make a funny graduation T shirt for my flatmates: survived uni, barely. Leeds 2026", "modality": "text_only", "cluster": "graduation" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_004\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ] }, { "description": "05 text_only girls trip lvyo_bv0_candidate_005", "vars": { "caseId": "lvyo_bv0_candidate_005", "customerInput": "Ibiza girls trip top, four of us, classy front but silly back, something about lost sunglasses and bad decisions", "modality": "text_only", "cluster": "girls trip" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_005\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ] }, { "description": "06 text_only lads holiday lvyo_bv0_candidate_006", "vars": { "caseId": "lvyo_bv0_candidate_006", "customerInput": "Lads holiday tees for Ayia Napa. Front: Team Terrible Ideas. Back can have fake tour dates please", "modality": "text_only", "cluster": "lads holiday" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_006\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ] }, { "description": "07 text_only road trip lvyo_bv0_candidate_007", "vars": { "caseId": "lvyo_bv0_candidate_007", "customerInput": "Road trip shirt for Scotland NC500, two mates and one tiny car, please make it outdoorsy but funny", "modality": "text_only", "cluster": "road trip" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_007\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ] }, { "description": "08 text_only ski trip lvyo_bv0_candidate_008", "vars": { "caseId": "lvyo_bv0_candidate_008", "customerInput": "Ski trip hoodie design for Val Thorens, slogan: Apres First, Questions Later. Add mountains and beers", "modality": "text_only", "cluster": "ski trip" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_008\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ] }, { "description": "09 text_only golf trip lvyo_bv0_candidate_009", "vars": { "caseId": "lvyo_bv0_candidate_009", "customerInput": "Golf weekend shirt for Dad's 60th. Bad backs, worse handicaps, Portugal tour 2026", "modality": "text_only", "cluster": "golf trip" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_009\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ] }, { "description": "10 text_only student society lvyo_bv0_candidate_010", "vars": { "caseId": "lvyo_bv0_candidate_010", "customerInput": "Drama society tour shirt. Needs to say Main Character Energy Dept and have a stage curtain vibe", "modality": "text_only", "cluster": "student society" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_010\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ] }, { "description": "11 text_only freshers week lvyo_bv0_candidate_011", "vars": { "caseId": "lvyo_bv0_candidate_011", "customerInput": "Freshers halls social tee, Block C Chaos Committee, make it look like a fake university crest", "modality": "text_only", "cluster": "freshers week" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_011\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ] }, { "description": "12 text_only rugby social lvyo_bv0_candidate_012", "vars": { "caseId": "lvyo_bv0_candidate_012", "customerInput": "Rugby tour shirt for the front row lads. Mud, pints, terrible moustaches. Keep it printable", "modality": "text_only", "cluster": "rugby social" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_012\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ] }, { "description": "13 text_only football social lvyo_bv0_candidate_013", "vars": { "caseId": "lvyo_bv0_candidate_013", "customerInput": "5 a side team top for The Unathletico Madrid, please make it like a retro football badge", "modality": "text_only", "cluster": "football social" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_013\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ] }, { "description": "14 text_only running club lvyo_bv0_candidate_014", "vars": { "caseId": "lvyo_bv0_candidate_014", "customerInput": "Running club shirt for our first half marathon. Slow But Extremely Dramatic. Maybe trainers and a coffee cup", "modality": "text_only", "cluster": "running club" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_014\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ] }, { "description": "15 text_only festival crew lvyo_bv0_candidate_015", "vars": { "caseId": "lvyo_bv0_candidate_015", "customerInput": "Festival crew tee, Glasto-ish but don't use the actual logo, saying Field Rats 2026 and make it colourful", "modality": "text_only", "cluster": "festival crew" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_015\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ] }, { "description": "16 text_only rave weekend lvyo_bv0_candidate_016", "vars": { "caseId": "lvyo_bv0_candidate_016", "customerInput": "Warehouse rave shirt for my birthday. Black tee, neon green text: I Came For The Bass And Lost My Friends", "modality": "text_only", "cluster": "rave weekend" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_016\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ] }, { "description": "17 text_only pub crawl lvyo_bv0_candidate_017", "vars": { "caseId": "lvyo_bv0_candidate_017", "customerInput": "Pub crawl shirt for Cambridge, 12 pubs, one bad idea. Need checkboxes on the back", "modality": "text_only", "cluster": "pub crawl" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_017\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ] }, { "description": "18 text_only beer olympics lvyo_bv0_candidate_018", "vars": { "caseId": "lvyo_bv0_candidate_018", "customerInput": "Beer Olympics team shirt. Team name: The Chuggernauts. Red white and blue but not too American", "modality": "text_only", "cluster": "beer olympics" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_018\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ] }, { "description": "19 text_only white lie party lvyo_bv0_candidate_019", "vars": { "caseId": "lvyo_bv0_candidate_019", "customerInput": "White lie party T-shirt, mine should say I only came for one drink, make it bold and a bit messy", "modality": "text_only", "cluster": "white lie party" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_019\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ] }, { "description": "20 text_only bottomless brunch lvyo_bv0_candidate_020", "vars": { "caseId": "lvyo_bv0_candidate_020", "customerInput": "Bottomless brunch top for Laura's birthday, prosecco tower, sunglasses, 'Booked for 12, emotionally there at 3'", "modality": "text_only", "cluster": "bottomless brunch" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_020\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ] }, { "description": "21 text_only garden party lvyo_bv0_candidate_021", "vars": { "caseId": "lvyo_bv0_candidate_021", "customerInput": "Garden party tee for my mum's 50th, floral but funny, says Rosie's Gin Garden Est. 1976", "modality": "text_only", "cluster": "garden party" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_021\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ] }, { "description": "22 text_only gaming squad lvyo_bv0_candidate_022", "vars": { "caseId": "lvyo_bv0_candidate_022", "customerInput": "Gaming squad shirt for our Discord group, Midnight Lobby Survivors, cyber but cosy, no skulls", "modality": "text_only", "cluster": "gaming squad" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_022\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ] }, { "description": "23 text_only fantasy draft lvyo_bv0_candidate_023", "vars": { "caseId": "lvyo_bv0_candidate_023", "customerInput": "Fantasy football draft night shirt. League name: Spreadsheet Sadness. Need a fake trophy and tiny crying chart", "modality": "text_only", "cluster": "fantasy draft" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_023\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ] }, { "description": "24 text_only pub quiz team lvyo_bv0_candidate_024", "vars": { "caseId": "lvyo_bv0_candidate_024", "customerInput": "Pub quiz team tee, name is Agatha Quiztie. Make it clever, vintage detective style", "modality": "text_only", "cluster": "pub quiz team" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_024\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ] }, { "description": "25 text_only company retreat lvyo_bv0_candidate_025", "vars": { "caseId": "lvyo_bv0_candidate_025", "customerInput": "Team offsite shirt for a startup retreat in Lisbon. Less corporate, more inside-joke: ship it then sip it", "modality": "text_only", "cluster": "company retreat" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_025\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ] }, { "description": "26 text_only hackathon lvyo_bv0_candidate_026", "vars": { "caseId": "lvyo_bv0_candidate_026", "customerInput": "Hackathon team tee for 48 hours no sleep, project name Patch Panic, make it look like an error screen but wearable", "modality": "text_only", "cluster": "hackathon" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_026\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ] }, { "description": "27 text_only launch party lvyo_bv0_candidate_027", "vars": { "caseId": "lvyo_bv0_candidate_027", "customerInput": "Small launch party T-shirt for our bakery app, 'crumbs deployed successfully', cute but not childish", "modality": "text_only", "cluster": "launch party" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_027\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ] }, { "description": "28 text_only charity run lvyo_bv0_candidate_028", "vars": { "caseId": "lvyo_bv0_candidate_028", "customerInput": "Charity 10k tops for Team Nora. Warm, hopeful, with trainers and a sunrise, not too sad", "modality": "text_only", "cluster": "charity run" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_028\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ] }, { "description": "29 text_only volunteer crew lvyo_bv0_candidate_029", "vars": { "caseId": "lvyo_bv0_candidate_029", "customerInput": "Volunteer crew shirts for community clean up day. Make it say Litter Quitters and include gloves/bags somehow", "modality": "text_only", "cluster": "volunteer crew" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_029\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ] }, { "description": "30 text_only wedding weekend lvyo_bv0_candidate_030", "vars": { "caseId": "lvyo_bv0_candidate_030", "customerInput": "Wedding weekend shirt for the guests, Max & Priya, Cornwall 2026, subtle seaside illustration please", "modality": "text_only", "cluster": "wedding weekend" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_030\" &&\n body.modality === \"text_only\" &&\n body.caseRole === \"text_core\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n body.referenceImage === null;\n" } ] }, { "description": "37 image_attached stag weekend lvyo_bv0_candidate_037", "vars": { "caseId": "lvyo_bv0_candidate_037", "customerInput": "Use the attached photo of Tom in the awful inflatable crown and make a stag shirt that says King of Bad Choices, Dublin 2026", "modality": "image_attached", "cluster": "stag weekend" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_037\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } ] }, { "description": "38 image_attached hen party lvyo_bv0_candidate_038", "vars": { "caseId": "lvyo_bv0_candidate_038", "customerInput": "Can you turn this group selfie into a classy hen weekend tee for Sophie's Last Fling, keep the sunglasses and cocktails vibe", "modality": "image_attached", "cluster": "hen party" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_038\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } ] }, { "description": "39 image_attached milestone birthday lvyo_bv0_candidate_039", "vars": { "caseId": "lvyo_bv0_candidate_039", "customerInput": "Make a 40th birthday shirt based on this pic of Dan asleep on the sofa after the party. Text: Still Got It Apparently", "modality": "image_attached", "cluster": "milestone birthday" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_039\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } ] }, { "description": "40 image_attached girls trip lvyo_bv0_candidate_040", "vars": { "caseId": "lvyo_bv0_candidate_040", "customerInput": "Use this airport photo for our girls trip shirts. Maybe cartoon us with suitcases and the words Gate Closing? Never Heard Of Her", "modality": "image_attached", "cluster": "girls trip" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_040\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } ] }, { "description": "41 image_attached lads holiday lvyo_bv0_candidate_041", "vars": { "caseId": "lvyo_bv0_candidate_041", "customerInput": "This balcony photo is the vibe. Need lads holiday tops saying Balcony Board Meeting 2026, make us look like a terrible committee", "modality": "image_attached", "cluster": "lads holiday" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_041\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } ] }, { "description": "42 image_attached ski trip lvyo_bv0_candidate_042", "vars": { "caseId": "lvyo_bv0_candidate_042", "customerInput": "Turn this ski lift photo into a retro ski club shirt, say Powder Panic Crew and include the stupid orange helmet", "modality": "image_attached", "cluster": "ski trip" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_042\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } ] }, { "description": "43 image_attached golf trip lvyo_bv0_candidate_043", "vars": { "caseId": "lvyo_bv0_candidate_043", "customerInput": "Use this pic of my dad missing the putt for his birthday golf shirt. Can it say Three Putts to Glory", "modality": "image_attached", "cluster": "golf trip" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_043\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } ] }, { "description": "44 image_attached student society lvyo_bv0_candidate_044", "vars": { "caseId": "lvyo_bv0_candidate_044", "customerInput": "Make a tour shirt from this photo of our dance squad in the rehearsal room, but more bold and stagey", "modality": "image_attached", "cluster": "student society" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_044\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } ] }, { "description": "45 image_attached rugby social lvyo_bv0_candidate_045", "vars": { "caseId": "lvyo_bv0_candidate_045", "customerInput": "This muddy team pic should become our rugby social shirt. Text: Scrum And Tonic, make it less disgusting please", "modality": "image_attached", "cluster": "rugby social" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_045\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } ] }, { "description": "46 image_attached festival crew lvyo_bv0_candidate_046", "vars": { "caseId": "lvyo_bv0_candidate_046", "customerInput": "Use the tent photo for our festival shirts. Need something like Camp Questionable Decisions with the wonky flag in it", "modality": "image_attached", "cluster": "festival crew" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_046\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } ] }, { "description": "47 image_attached pub crawl lvyo_bv0_candidate_047", "vars": { "caseId": "lvyo_bv0_candidate_047", "customerInput": "Can you use this photo of the first pub sign and make a crawl tee, but don't copy the pub name, just the old sign style", "modality": "image_attached", "cluster": "pub crawl" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_047\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } ] }, { "description": "48 image_attached bottomless brunch lvyo_bv0_candidate_048", "vars": { "caseId": "lvyo_bv0_candidate_048", "customerInput": "Make this brunch table photo into a cute birthday tee for Jess, please include the pancakes and prosecco glasses", "modality": "image_attached", "cluster": "bottomless brunch" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_048\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } ] }, { "description": "49 image_attached gaming squad lvyo_bv0_candidate_049", "vars": { "caseId": "lvyo_bv0_candidate_049", "customerInput": "Use my setup photo for a gaming squad shirt. Purple lights, messy desk, title should be Respawn Roommates", "modality": "image_attached", "cluster": "gaming squad" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_049\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } ] }, { "description": "50 image_attached company retreat lvyo_bv0_candidate_050", "vars": { "caseId": "lvyo_bv0_candidate_050", "customerInput": "Turn this awkward offsite photo into a nicer team retreat shirt. Keep the mountain cabin and coffee cups, lose the corporate cringe", "modality": "image_attached", "cluster": "company retreat" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_050\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } ] }, { "description": "51 image_attached hackathon lvyo_bv0_candidate_051", "vars": { "caseId": "lvyo_bv0_candidate_051", "customerInput": "Use this laptop graveyard photo for our hackathon shirts. Text idea: Merge Conflict Survivors", "modality": "image_attached", "cluster": "hackathon" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_051\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } ] }, { "description": "52 image_attached charity run lvyo_bv0_candidate_052", "vars": { "caseId": "lvyo_bv0_candidate_052", "customerInput": "Please base the charity run shirt on this trainer photo, with Team Nora and a sunrise, make it hopeful", "modality": "image_attached", "cluster": "charity run" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_052\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } ] }, { "description": "53 image_attached wedding weekend lvyo_bv0_candidate_053", "vars": { "caseId": "lvyo_bv0_candidate_053", "customerInput": "Can you make a welcome weekend tee from this beach photo where Alex and Noor got engaged, subtle and not cheesy", "modality": "image_attached", "cluster": "wedding weekend" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_053\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } ] }, { "description": "54 image_attached school leavers lvyo_bv0_candidate_054", "vars": { "caseId": "lvyo_bv0_candidate_054", "customerInput": "Use the locker photo for leavers shirts, class of 2026, doodles and signatures kind of look", "modality": "image_attached", "cluster": "school leavers" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_054\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } ] }, { "description": "55 image_attached surf trip lvyo_bv0_candidate_055", "vars": { "caseId": "lvyo_bv0_candidate_055", "customerInput": "Make our surf trip tee from this pic of the boards on the van. Text: Wipeout Witnesses", "modality": "image_attached", "cluster": "surf trip" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_055\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } ] }, { "description": "56 image_attached city break lvyo_bv0_candidate_056", "vars": { "caseId": "lvyo_bv0_candidate_056", "customerInput": "Use this blurry late night metro photo as inspo for our Paris shirts, make it look intentional not like we can't take photos", "modality": "image_attached", "cluster": "city break" }, "assert": [ { "type": "javascript", "value": "const body = JSON.parse(output);\nreturn body.ok === true &&\n body.caseId === \"lvyo_bv0_candidate_056\" &&\n body.modality === \"image_attached\" &&\n body.caseRole === \"image_pack\" &&\n body.status === 'success' &&\n body.runtimeProfile === 'internal_codex' &&\n body.imageProvider === 'codex_account_ui' &&\n body.noSpendReplay === true &&\n body.customerLiveFlowChanged === false &&\n Boolean(body.final?.sha256) &&\n Boolean(body.mockup?.sha256) &&\n Boolean(body.referenceImage?.sha256);\n" } ] } ], "env": {}, "outputPath": [ "public/admin/promptfoo/promptfoo-benchmark-v0.html", "public/admin/promptfoo/promptfoo-benchmark-v0.json" ], "extensions": [], "metadata": {}, "evaluateOptions": {} }, "shareableUrl": null, "metadata": { "promptfooVersion": "0.121.17", "nodeVersion": "v25.2.1", "platform": "darwin", "arch": "arm64", "exportedAt": "2026-06-23T06:59:34.042Z", "evaluationCreatedAt": "2026-06-23T06:59:33.793Z", "author": "dominik@levyo.co.uk" }, "vars": [ "caseId", "customerInput", "modality", "cluster" ], "runtimeOptions": { "eventSource": "cli", "showProgressBar": false, "repeat": 1, "maxConcurrency": 1, "cache": false } }