adding data collection
This commit is contained in:
@@ -0,0 +1,47 @@
|
||||
import { describe, expect, it, vi } from 'vitest';
|
||||
import { createRequestLogger, ensureSchema, insertRequest } from '../src/db.js';
|
||||
|
||||
describe('request database', () => {
|
||||
it('creates the requests table, compatible columns, and recent-request index', async () => {
|
||||
const pool = { query: vi.fn().mockResolvedValue({}) };
|
||||
|
||||
await ensureSchema(pool);
|
||||
|
||||
expect(pool.query).toHaveBeenCalledTimes(3);
|
||||
expect(pool.query.mock.calls[0][0]).toContain('CREATE TABLE IF NOT EXISTS requests');
|
||||
expect(pool.query.mock.calls[1][0]).toContain('ADD COLUMN IF NOT EXISTS request_headers');
|
||||
expect(pool.query.mock.calls[1][0]).toContain('ADD COLUMN IF NOT EXISTS duration_ms');
|
||||
expect(pool.query.mock.calls[1][0]).toContain('ADD COLUMN IF NOT EXISTS training_data');
|
||||
expect(pool.query.mock.calls[2][0]).toContain('created_at DESC, id DESC');
|
||||
});
|
||||
|
||||
it('stores request and response data as one row', async () => {
|
||||
const pool = { query: vi.fn().mockResolvedValue({}) };
|
||||
const entry = {
|
||||
method: 'POST', endpoint: '/oai/v1/responses',
|
||||
requestHeaders: { host: 'proxy.test' }, requestBody: { model: 'm', input: 'Hi' },
|
||||
responseHeaders: { 'content-type': 'application/json' }, responseBody: '{"id":"r1"}',
|
||||
responseStatus: 200, durationMs: 123.5, model: 'm', clientIp: '198.51.100.2', stream: false,
|
||||
trainingData: { messages: [{ role: 'user', content: 'Hi' }, { role: 'assistant', content: 'Hello' }] },
|
||||
};
|
||||
|
||||
await insertRequest(pool, entry);
|
||||
|
||||
expect(pool.query).toHaveBeenCalledOnce();
|
||||
expect(pool.query.mock.calls[0][0]).toContain('INSERT INTO requests');
|
||||
expect(pool.query.mock.calls[0][1]).toEqual([
|
||||
'POST', '/oai/v1/responses', '{"host":"proxy.test"}',
|
||||
'{"model":"m","input":"Hi"}', '{"content-type":"application/json"}',
|
||||
'{"id":"r1"}', 200, 123.5, 'm', '198.51.100.2', false,
|
||||
'{"messages":[{"role":"user","content":"Hi"},{"role":"assistant","content":"Hello"}]}',
|
||||
]);
|
||||
});
|
||||
|
||||
it('reports storage failures without rejecting request handling', async () => {
|
||||
const onError = vi.fn();
|
||||
const logger = createRequestLogger({ query: vi.fn().mockRejectedValue(new Error('database offline')) }, { onError });
|
||||
|
||||
await expect(logger.log({ method: 'GET', endpoint: '/', responseStatus: 200 })).resolves.toBeUndefined();
|
||||
expect(onError).toHaveBeenCalledWith('Failed to store request: database offline');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,30 @@
|
||||
import fs from 'node:fs/promises';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { afterEach, describe, expect, it } from 'vitest';
|
||||
import { loadEnv } from '../src/env.js';
|
||||
|
||||
const loadedKey = 'OPENCODE_PROXY_TEST_ENV_LOADED';
|
||||
const existingKey = 'OPENCODE_PROXY_TEST_ENV_EXISTING';
|
||||
|
||||
afterEach(() => {
|
||||
delete process.env[loadedKey];
|
||||
delete process.env[existingKey];
|
||||
});
|
||||
|
||||
describe('.env loading', () => {
|
||||
it('loads values without overriding the process environment', async () => {
|
||||
const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'opencode-proxy-env-'));
|
||||
const envPath = path.join(directory, '.env');
|
||||
await fs.writeFile(envPath, `${loadedKey}=from-file\n${existingKey}=from-file\n`);
|
||||
process.env[existingKey] = 'from-process';
|
||||
|
||||
try {
|
||||
loadEnv(envPath);
|
||||
expect(process.env[loadedKey]).toBe('from-file');
|
||||
expect(process.env[existingKey]).toBe('from-process');
|
||||
} finally {
|
||||
await fs.rm(directory, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -87,6 +87,58 @@ describe('proxy', () => {
|
||||
expect(fetchImpl.mock.calls[0][1].headers.accept).toBe('text/event-stream');
|
||||
});
|
||||
|
||||
it('collects the request, chat history, and complete streamed response', async () => {
|
||||
const stream = 'data: {"choices":[{"delta":{"reasoning_content":"Think. ","content":"Hi"}}]}\n\ndata: {"choices":[{"delta":{},"finish_reason":"stop"}]}\n\ndata: [DONE]\n\n';
|
||||
const fetchImpl = vi.fn().mockResolvedValue(response(stream, {
|
||||
headers: { 'content-type': 'text/event-stream' },
|
||||
}));
|
||||
const requestLogger = { log: vi.fn() };
|
||||
const app = createProxyApp({ keys: ['key'], fetchImpl, requestLogger });
|
||||
const body = { model: 'm', messages: [{ role: 'user', content: 'Hi' }], stream: true };
|
||||
|
||||
await request(app)
|
||||
.post('/oai/v1/chat/completions?source=test')
|
||||
.set('X-Real-IP', '198.51.100.10')
|
||||
.send(body)
|
||||
.expect(200);
|
||||
|
||||
expect(requestLogger.log).toHaveBeenCalledTimes(1);
|
||||
expect(requestLogger.log.mock.calls[0][0]).toMatchObject({
|
||||
method: 'POST',
|
||||
endpoint: '/oai/v1/chat/completions?source=test',
|
||||
requestBody: body,
|
||||
responseBody: stream,
|
||||
responseStatus: 200,
|
||||
model: 'm',
|
||||
clientIp: '198.51.100.10',
|
||||
stream: true,
|
||||
trainingData: { messages: [
|
||||
{ role: 'user', content: 'Hi' },
|
||||
{ role: 'assistant', content: 'Hi', reasoning_content: 'Think. ' },
|
||||
] },
|
||||
});
|
||||
expect(requestLogger.log.mock.calls[0][0].requestHeaders['content-type']).toMatch('application/json');
|
||||
expect(requestLogger.log.mock.calls[0][0].responseHeaders['content-type']).toMatch('text/event-stream');
|
||||
expect(requestLogger.log.mock.calls[0][0].durationMs).toBeGreaterThanOrEqual(0);
|
||||
expect(requestLogger.log.mock.calls[0][0].trainingData.metadata).toMatchObject({
|
||||
schema_version: 1, api: 'openai_chat_completions', model: 'm', stream: true,
|
||||
tools: [], response_status: 200, response: { finish_reason: 'stop' },
|
||||
});
|
||||
});
|
||||
|
||||
it('collects error and not-found responses too', async () => {
|
||||
const requestLogger = { log: vi.fn() };
|
||||
const app = createProxyApp({ keys: ['key'], requestLogger });
|
||||
|
||||
await request(app).get('/missing').expect(404);
|
||||
|
||||
expect(requestLogger.log).toHaveBeenCalledOnce();
|
||||
expect(requestLogger.log.mock.calls[0][0]).toMatchObject({
|
||||
method: 'GET', endpoint: '/missing', requestBody: null, responseStatus: 404,
|
||||
});
|
||||
expect(requestLogger.log.mock.calls[0][0].responseBody).toContain('Cannot GET /missing');
|
||||
});
|
||||
|
||||
it('translates Responses requests and returns a Responses object', async () => {
|
||||
const fetchImpl = vi.fn().mockResolvedValue(response(JSON.stringify({
|
||||
id: 'chatcmpl-response', model: 'm',
|
||||
|
||||
@@ -0,0 +1,168 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { createTrainingExample } from '../src/training-data.js';
|
||||
|
||||
describe('training data normalization', () => {
|
||||
it('exports Chat Completions histories, reasoning, tool calls, and tool results', () => {
|
||||
const example = createTrainingExample({
|
||||
endpoint: '/oai/v1/chat/completions', responseStatus: 200,
|
||||
requestBody: {
|
||||
model: 'model-a', temperature: 0.2, max_tokens: 200, tool_choice: 'auto', metadata: { dataset: 'test' },
|
||||
tools: [{ type: 'function', function: { name: 'weather', description: 'Get weather', parameters: { type: 'object' } } }],
|
||||
messages: [
|
||||
{ role: 'system', content: 'Be useful' },
|
||||
{ role: 'user', content: 'Find the weather' },
|
||||
{ role: 'assistant', content: null, tool_calls: [{ id: 'call_1', type: 'function', function: { name: 'weather', arguments: '{"city":"Oslo"}' } }] },
|
||||
{ role: 'tool', tool_call_id: 'call_1', content: '{"temp":12}' },
|
||||
{ role: 'user', content: 'Summarize it' },
|
||||
],
|
||||
},
|
||||
responseBody: JSON.stringify({ id: 'chatcmpl_1', model: 'model-a', usage: { prompt_tokens: 20, completion_tokens: 8 }, choices: [{ finish_reason: 'tool_calls', message: {
|
||||
role: 'assistant', reasoning_content: 'The result says 12 degrees.', content: 'It is 12°C in Oslo.',
|
||||
tool_calls: [{ id: 'call_2', type: 'function', function: { name: 'log', arguments: '{"temp":12}' } }],
|
||||
} }] }),
|
||||
durationMs: 123.5,
|
||||
});
|
||||
|
||||
expect(example.messages).toHaveLength(6);
|
||||
expect(example.messages[2].tool_calls[0]).toMatchObject({ id: 'call_1', function: { name: 'weather' } });
|
||||
expect(example.messages[3]).toEqual({ role: 'tool', tool_call_id: 'call_1', content: '{"temp":12}' });
|
||||
expect(example.messages.at(-1)).toEqual({
|
||||
role: 'assistant', content: 'It is 12°C in Oslo.', reasoning_content: 'The result says 12 degrees.',
|
||||
tool_calls: [{ id: 'call_2', type: 'function', function: { name: 'log', arguments: '{"temp":12}' } }],
|
||||
});
|
||||
expect(example.metadata).toEqual({
|
||||
schema_version: 1,
|
||||
api: 'openai_chat_completions', endpoint: '/oai/v1/chat/completions', model: 'model-a', stream: false,
|
||||
tools: [{ type: 'function', function: { name: 'weather', description: 'Get weather', parameters: { type: 'object' } } }],
|
||||
tool_choice: 'auto', request_metadata: { dataset: 'test' }, parameters: { temperature: 0.2, max_tokens: 200 }, response_status: 200, duration_ms: 123.5,
|
||||
response: { id: 'chatcmpl_1', model: 'model-a', finish_reason: 'tool_calls', usage: { prompt_tokens: 20, completion_tokens: 8 } },
|
||||
});
|
||||
});
|
||||
|
||||
it('reconstructs streamed Chat Completions output', () => {
|
||||
const responseBody = [
|
||||
'data: {"choices":[{"delta":{"reasoning_content":"check "}}]}',
|
||||
'data: {"choices":[{"delta":{"content":"Done","tool_calls":[{"index":0,"id":"call_1","function":{"name":"save","arguments":"{\\"ok\\":"}}]}}]}',
|
||||
'data: {"choices":[{"delta":{"tool_calls":[{"index":0,"function":{"arguments":"true}"}}]},"finish_reason":"tool_calls"}]}',
|
||||
'data: [DONE]', '',
|
||||
].join('\n\n');
|
||||
const example = createTrainingExample({
|
||||
endpoint: '/oai/v1/chat/completions', responseStatus: 200, stream: true,
|
||||
requestBody: { messages: [{ role: 'user', content: 'Do it' }] }, responseBody,
|
||||
});
|
||||
|
||||
expect(example.messages.at(-1)).toEqual({
|
||||
role: 'assistant', content: 'Done', reasoning_content: 'check ',
|
||||
tool_calls: [{ id: 'call_1', type: 'function', function: { name: 'save', arguments: '{"ok":true}' } }],
|
||||
});
|
||||
});
|
||||
|
||||
it('normalizes Responses input and output items', () => {
|
||||
const example = createTrainingExample({
|
||||
endpoint: '/oai/v1/responses', responseStatus: 200,
|
||||
requestBody: {
|
||||
model: 'model-r', instructions: 'Use tools',
|
||||
tools: [{ type: 'function', name: 'read', parameters: { type: 'object' } }, { type: 'web_search_preview' }],
|
||||
input: [
|
||||
{ type: 'message', role: 'user', content: [{ type: 'input_text', text: 'Read file' }] },
|
||||
{ type: 'reasoning', summary: [{ type: 'summary_text', text: 'Need the file.' }] },
|
||||
{ type: 'function_call', call_id: 'call_read', name: 'read', arguments: '{"path":"a.txt"}' },
|
||||
{ type: 'function_call_output', call_id: 'call_read', output: 'hello' },
|
||||
],
|
||||
},
|
||||
responseBody: JSON.stringify({ output: [
|
||||
{ type: 'reasoning', summary: [{ type: 'summary_text', text: 'The file says hello.' }] },
|
||||
{ type: 'message', role: 'assistant', content: [{ type: 'output_text', text: 'It says hello.' }] },
|
||||
{ type: 'function_call', call_id: 'call_log', name: 'log', arguments: '{"value":"hello"}' },
|
||||
] }),
|
||||
});
|
||||
|
||||
expect(example.messages).toEqual([
|
||||
{ role: 'system', content: 'Use tools' },
|
||||
{ role: 'user', content: [{ type: 'text', text: 'Read file' }] },
|
||||
{ role: 'assistant', content: null, reasoning_content: 'Need the file.', tool_calls: [{ id: 'call_read', type: 'function', function: { name: 'read', arguments: '{"path":"a.txt"}' } }] },
|
||||
{ role: 'tool', tool_call_id: 'call_read', content: 'hello' },
|
||||
{ role: 'assistant', content: 'It says hello.', reasoning_content: 'The file says hello.', tool_calls: [{ id: 'call_log', type: 'function', function: { name: 'log', arguments: '{"value":"hello"}' } }] },
|
||||
]);
|
||||
expect(example.metadata.tools).toEqual([
|
||||
{ type: 'function', name: 'read', parameters: { type: 'object' } },
|
||||
{ type: 'web_search_preview' },
|
||||
]);
|
||||
expect(example.metadata).toMatchObject({ api: 'openai_responses', model: 'model-r', stream: false, response_status: 200 });
|
||||
});
|
||||
|
||||
it('uses the completed Responses streaming event as the assistant output', () => {
|
||||
const responseBody = [
|
||||
'event: response.completed',
|
||||
'data: {"type":"response.completed","response":{"output":[{"type":"reasoning","summary":[{"type":"summary_text","text":"Need a lookup."}]},{"type":"message","role":"assistant","content":[{"type":"output_text","text":"Found it."}]},{"type":"function_call","call_id":"call_1","name":"lookup","arguments":"{\\"q\\":\\"x\\"}"}]}}',
|
||||
'', '',
|
||||
].join('\n');
|
||||
const example = createTrainingExample({
|
||||
endpoint: '/oai/v1/responses', responseStatus: 200, stream: true,
|
||||
requestBody: { input: 'Find x' }, responseBody,
|
||||
});
|
||||
|
||||
expect(example.messages.at(-1)).toEqual({
|
||||
role: 'assistant', content: 'Found it.', reasoning_content: 'Need a lookup.',
|
||||
tool_calls: [{ id: 'call_1', type: 'function', function: { name: 'lookup', arguments: '{"q":"x"}' } }],
|
||||
});
|
||||
});
|
||||
|
||||
it('normalizes Anthropic thinking, tool use, and tool results', () => {
|
||||
const example = createTrainingExample({
|
||||
endpoint: '/ant/v1/messages', responseStatus: 200,
|
||||
requestBody: {
|
||||
model: 'model-ant', system: 'Be useful',
|
||||
tools: [{ name: 'calculator', description: 'Calculate', input_schema: { type: 'object' } }],
|
||||
messages: [
|
||||
{ role: 'user', content: 'Calculate' },
|
||||
{ role: 'assistant', content: [{ type: 'thinking', thinking: 'Need calculator.' }, { type: 'tool_use', id: 'tool_1', name: 'calculator', input: { expression: '2+2' } }] },
|
||||
{ role: 'user', content: [{ type: 'tool_result', tool_use_id: 'tool_1', content: '4' }] },
|
||||
],
|
||||
},
|
||||
responseBody: JSON.stringify({ content: [
|
||||
{ type: 'thinking', thinking: 'The result is four.' },
|
||||
{ type: 'text', text: 'The answer is 4.' },
|
||||
{ type: 'tool_use', id: 'tool_2', name: 'record', input: { answer: 4 } },
|
||||
] }),
|
||||
});
|
||||
|
||||
expect(example.messages).toEqual([
|
||||
{ role: 'system', content: 'Be useful' },
|
||||
{ role: 'user', content: 'Calculate' },
|
||||
{ role: 'assistant', content: null, reasoning_content: 'Need calculator.', tool_calls: [{ id: 'tool_1', type: 'function', function: { name: 'calculator', arguments: '{"expression":"2+2"}' } }] },
|
||||
{ role: 'tool', tool_call_id: 'tool_1', content: '4' },
|
||||
{ role: 'assistant', content: 'The answer is 4.', reasoning_content: 'The result is four.', tool_calls: [{ id: 'tool_2', type: 'function', function: { name: 'record', arguments: '{"answer":4}' } }] },
|
||||
]);
|
||||
expect(example.metadata.tools).toEqual([
|
||||
{ name: 'calculator', description: 'Calculate', input_schema: { type: 'object' } },
|
||||
]);
|
||||
expect(example.metadata).toMatchObject({ api: 'anthropic_messages', model: 'model-ant', stream: false, response_status: 200 });
|
||||
});
|
||||
|
||||
it('reconstructs streamed Anthropic content blocks', () => {
|
||||
const responseBody = [
|
||||
'event: content_block_start\ndata: {"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":""}}',
|
||||
'event: content_block_delta\ndata: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"Need tool."}}',
|
||||
'event: content_block_start\ndata: {"type":"content_block_start","index":1,"content_block":{"type":"text","text":""}}',
|
||||
'event: content_block_delta\ndata: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"Working"}}',
|
||||
'event: content_block_start\ndata: {"type":"content_block_start","index":2,"content_block":{"type":"tool_use","id":"tool_1","name":"run","input":{}}}',
|
||||
'event: content_block_delta\ndata: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":"{\\"x\\":1}"}}',
|
||||
'',
|
||||
].join('\n\n');
|
||||
const example = createTrainingExample({
|
||||
endpoint: '/ant/v1/messages', responseStatus: 200, stream: true,
|
||||
requestBody: { messages: [{ role: 'user', content: 'Work' }] }, responseBody,
|
||||
});
|
||||
|
||||
expect(example.messages.at(-1)).toEqual({
|
||||
role: 'assistant', content: 'Working', reasoning_content: 'Need tool.',
|
||||
tool_calls: [{ id: 'tool_1', type: 'function', function: { name: 'run', arguments: '{"x":1}' } }],
|
||||
});
|
||||
});
|
||||
|
||||
it('excludes failed and non-chat requests', () => {
|
||||
expect(createTrainingExample({ endpoint: '/oai/v1/models', responseStatus: 200, requestBody: {} })).toBeNull();
|
||||
expect(createTrainingExample({ endpoint: '/oai/v1/chat/completions', responseStatus: 500, requestBody: { messages: [] } })).toBeNull();
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user