Inital commit
This commit is contained in:
@@ -0,0 +1,248 @@
|
||||
// Microbenchmarks for the functions on the agentic-thread streaming hot path.
|
||||
//
|
||||
// Every function here is invoked from a `$derived` that is invalidated on each
|
||||
// streamed token, for every tool-call section in the message. These numbers give
|
||||
// the per-call cost; multiply by (tokens x sections) for the real damage.
|
||||
//
|
||||
// Run: npx vitest bench --project=unit tests/unit/agentic-hotpath.bench.ts
|
||||
|
||||
import { bench, describe } from 'vitest';
|
||||
|
||||
import { remark } from 'remark';
|
||||
import remarkBreaks from 'remark-breaks';
|
||||
import remarkGfm from 'remark-gfm';
|
||||
import remarkMath from 'remark-math';
|
||||
import remarkRehype from 'remark-rehype';
|
||||
import rehypeKatex from 'rehype-katex';
|
||||
import rehypeHighlight from 'rehype-highlight';
|
||||
import rehypeStringify from 'rehype-stringify';
|
||||
import { all as lowlightAll } from 'lowlight';
|
||||
|
||||
import { computeLineDiff } from '$lib/utils/compute-line-diff';
|
||||
import { extractSearchResults, extractSearchQuery } from '$lib/utils/search-results';
|
||||
import { parsePartialJsonArgs } from '$lib/utils/parse-partial-json-args';
|
||||
import { highlightCode, detectIncompleteCodeBlock } from '$lib/utils/code';
|
||||
import { classifyToolResult, parseToolResultWithImages } from '$lib/utils/agentic';
|
||||
import { preprocessLaTeX } from '$lib/utils/latex-protection';
|
||||
|
||||
// --- fixtures -------------------------------------------------------------
|
||||
|
||||
function lines(n: number, seed: string): string {
|
||||
const out: string[] = [];
|
||||
for (let i = 0; i < n; i++) out.push(`${seed} line ${i} const value_${i} = compute(${i});`);
|
||||
return out.join('\n');
|
||||
}
|
||||
|
||||
const SHELL_OUTPUT_1KB = lines(12, 'out');
|
||||
const SHELL_OUTPUT_200KB = lines(2600, 'out');
|
||||
const SHELL_OUTPUT_2MB = lines(26000, 'out');
|
||||
|
||||
// Realistic exec_shell_command result: the exit-code marker is the final line,
|
||||
// which is what the un-anchored EXIT_CODE regex has to scan the whole blob for.
|
||||
const SHELL_2MB_WITH_EXIT = `${SHELL_OUTPUT_2MB}\n[exit code: 0]`;
|
||||
|
||||
const EDIT_OLD_400 = lines(400, 'old');
|
||||
const EDIT_NEW_400 = lines(400, 'new');
|
||||
const EDIT_OLD_50 = lines(50, 'old');
|
||||
const EDIT_NEW_50 = lines(50, 'new');
|
||||
|
||||
const WRITE_FILE_ARGS = JSON.stringify({
|
||||
path: '/src/lib/thing.ts',
|
||||
content: lines(1500, 'src')
|
||||
});
|
||||
|
||||
const MARKDOWN_50KB = Array.from(
|
||||
{ length: 400 },
|
||||
(_, i) =>
|
||||
`## Section ${i}\n\nSome **bold** prose with a [link](https://example.com) and \`inline\` code.\n\n- bullet one\n- bullet two\n\n\`\`\`ts\nconst x${i} = ${i};\n\`\`\`\n`
|
||||
).join('\n');
|
||||
|
||||
const CODE_BLOCK_5KB = lines(60, 'code');
|
||||
|
||||
// --- A: computeLineDiff (O(m*n) LCS, allocates a full matrix) --------------
|
||||
|
||||
describe('computeLineDiff', () => {
|
||||
bench('50 x 50 lines', () => {
|
||||
computeLineDiff(EDIT_OLD_50, EDIT_NEW_50);
|
||||
});
|
||||
|
||||
bench('400 x 400 lines', () => {
|
||||
computeLineDiff(EDIT_OLD_400, EDIT_NEW_400);
|
||||
});
|
||||
});
|
||||
|
||||
// --- B: the unified processor, rebuilt per call ---------------------------
|
||||
// Mirrors MarkdownContent.svelte:144-176. The local rehype/remark plugins are
|
||||
// omitted (they pull in browser-only modules); rehypeHighlight + lowlightAll is
|
||||
// the dominant term, so this is a lower bound on the real cost.
|
||||
|
||||
function buildProcessor() {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
let proc: any = remark().use(remarkGfm);
|
||||
proc = proc.use(remarkMath).use(remarkBreaks).use(remarkRehype).use(rehypeKatex);
|
||||
return proc.use(rehypeHighlight, { languages: lowlightAll }).use(rehypeStringify, {
|
||||
allowDangerousHtml: true
|
||||
});
|
||||
}
|
||||
|
||||
describe('markdown processor', () => {
|
||||
bench('build processor (per processMarkdown call, x2)', () => {
|
||||
buildProcessor();
|
||||
});
|
||||
|
||||
const prebuilt = buildProcessor();
|
||||
|
||||
bench('parse 50KB markdown with a prebuilt processor', () => {
|
||||
prebuilt.parse(MARKDOWN_50KB);
|
||||
});
|
||||
});
|
||||
|
||||
// Isolates the O(n^2) term measured in Tier 1: processMarkdown re-parses the
|
||||
// WHOLE accumulated string every frame, while only the last block can have
|
||||
// changed. If parse() dominates at these sizes, incremental parsing (re-parsing
|
||||
// only the tail after the last stable block) is the fix; if not, the cost is
|
||||
// downstream in transform/stringify/DOM.
|
||||
describe('markdown parse scaling (whole-string reparse per frame)', () => {
|
||||
const proc = buildProcessor();
|
||||
const prose = (kb: number) =>
|
||||
Array.from(
|
||||
{ length: Math.ceil((kb * 1024) / 64) },
|
||||
(_, i) => `The quick brown fox jumps over the lazy dog. Sentence ${i}.`
|
||||
).join(' ');
|
||||
|
||||
const MD_3KB = prose(3);
|
||||
const MD_11KB = prose(11);
|
||||
const MD_26KB = prose(26);
|
||||
|
||||
bench('parse 3KB', () => {
|
||||
proc.parse(MD_3KB);
|
||||
});
|
||||
|
||||
bench('parse 11KB', () => {
|
||||
proc.parse(MD_11KB);
|
||||
});
|
||||
|
||||
bench('parse 26KB', () => {
|
||||
proc.parse(MD_26KB);
|
||||
});
|
||||
|
||||
bench('parse only a 200-char tail (proposed incremental)', () => {
|
||||
proc.parse(MD_26KB.slice(-200));
|
||||
});
|
||||
});
|
||||
|
||||
// processMarkdown runs these over the WHOLE accumulated string every frame too,
|
||||
// before parse() even starts. Measure them before assuming parse is the target.
|
||||
describe('other whole-string passes per frame', () => {
|
||||
const prose = (kb: number) =>
|
||||
Array.from(
|
||||
{ length: Math.ceil((kb * 1024) / 64) },
|
||||
(_, i) => `The quick brown fox jumps over the lazy dog. Sentence ${i}.`
|
||||
).join(' ');
|
||||
|
||||
const MD_3KB = prose(3);
|
||||
const MD_11KB = prose(11);
|
||||
const MD_26KB = prose(26);
|
||||
const MD_26KB_LATEX = `${MD_26KB} and some math $x^2 + y^2 = z^2$ inline.`;
|
||||
|
||||
bench('preprocessLaTeX 3KB (no latex present)', () => {
|
||||
preprocessLaTeX(MD_3KB);
|
||||
});
|
||||
|
||||
bench('preprocessLaTeX 11KB (no latex present)', () => {
|
||||
preprocessLaTeX(MD_11KB);
|
||||
});
|
||||
|
||||
bench('preprocessLaTeX 26KB (no latex present)', () => {
|
||||
preprocessLaTeX(MD_26KB);
|
||||
});
|
||||
|
||||
bench('preprocessLaTeX 26KB (latex present)', () => {
|
||||
preprocessLaTeX(MD_26KB_LATEX);
|
||||
});
|
||||
|
||||
bench('detectIncompleteCodeBlock 26KB', () => {
|
||||
detectIncompleteCodeBlock(MD_26KB);
|
||||
});
|
||||
});
|
||||
|
||||
// --- C: extractSearchResults, run for EVERY tool of every type ------------
|
||||
|
||||
describe('extractSearchResults (only .length > 0 is consumed)', () => {
|
||||
bench('1KB non-search tool result', () => {
|
||||
extractSearchResults(SHELL_OUTPUT_1KB);
|
||||
});
|
||||
|
||||
bench('200KB non-search tool result', () => {
|
||||
extractSearchResults(SHELL_OUTPUT_200KB);
|
||||
});
|
||||
|
||||
bench('2MB non-search tool result', () => {
|
||||
extractSearchResults(SHELL_OUTPUT_2MB);
|
||||
});
|
||||
});
|
||||
|
||||
describe('extractSearchQuery', () => {
|
||||
bench('write_file args (throws on the JSON.parse path)', () => {
|
||||
extractSearchQuery(WRITE_FILE_ARGS);
|
||||
});
|
||||
});
|
||||
|
||||
// --- E: the un-anchored exit-code regex -----------------------------------
|
||||
// Reproduced inline so the bench is independent of the current call site.
|
||||
|
||||
const EXIT_CODE_TAIL = /\[exit code: (-?\d+)\]\s*$/;
|
||||
|
||||
describe('exit-code regex', () => {
|
||||
bench('whole 2MB blob (current behaviour)', () => {
|
||||
SHELL_2MB_WITH_EXIT.match(EXIT_CODE_TAIL);
|
||||
});
|
||||
|
||||
bench('last 64 chars only (proposed)', () => {
|
||||
SHELL_2MB_WITH_EXIT.slice(-64).match(EXIT_CODE_TAIL);
|
||||
});
|
||||
});
|
||||
|
||||
// --- per-line result parsers ----------------------------------------------
|
||||
|
||||
describe('parseToolResultWithImages', () => {
|
||||
bench('1KB', () => {
|
||||
parseToolResultWithImages(SHELL_OUTPUT_1KB, []);
|
||||
});
|
||||
|
||||
bench('200KB', () => {
|
||||
parseToolResultWithImages(SHELL_OUTPUT_200KB, []);
|
||||
});
|
||||
|
||||
bench('2MB', () => {
|
||||
parseToolResultWithImages(SHELL_OUTPUT_2MB, []);
|
||||
});
|
||||
});
|
||||
|
||||
describe('classifyToolResult', () => {
|
||||
bench('200KB plain text (falls through to looksLikeMarkdown)', () => {
|
||||
classifyToolResult(SHELL_OUTPUT_200KB);
|
||||
});
|
||||
});
|
||||
|
||||
describe('parsePartialJsonArgs', () => {
|
||||
bench('write_file args, ~60KB (char-by-char scan)', () => {
|
||||
parsePartialJsonArgs(WRITE_FILE_ARGS);
|
||||
});
|
||||
});
|
||||
|
||||
// --- F / I: highlight.js ---------------------------------------------------
|
||||
|
||||
describe('highlightCode', () => {
|
||||
bench('short bash command (title row, per token)', () => {
|
||||
highlightCode('grep -rn "foo" src/ | head -50', 'bash');
|
||||
});
|
||||
|
||||
bench('5KB known language', () => {
|
||||
highlightCode(CODE_BLOCK_5KB, 'typescript');
|
||||
});
|
||||
|
||||
bench('5KB unknown language -> highlightAuto', () => {
|
||||
highlightCode(CODE_BLOCK_5KB, 'not-a-language');
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user