Writing an LLM Eval with Vercel's AI SDK and Vitest

async function llm(prompt: string): string {
  // 1 Trillion parameter LLM model no human understands
  ...
}

./eval-run-output/
├── evalResults.json
├── eval_id_1
│   ├── evalResult.json
│   ├── human.txt
│   ├── judgeResponse.txt
│   └── response.json
├── eval_id_2
│   ├── evalResult.json
│   ├── human.txt
│   ├── judgeResponse.txt
│   └── response.json

System Prompt:
You are an AI assistant expert in PostgreSQL and database administration.
Your name is Xata Agent.
...

--------
User Prompt: What tables do I have in my db?
--------
Step: 1

I'll help you get an overview of the tables in your database. I'll use the getTablesAndInstanceInfo tool to retrieve this information.

getTablesAndInstanceInfo with args: {}

Tool Result:
Here are the tables, their sizes, and usage counts:

[{"name":"dogs","schema":"public","rows":150,"size":"24 kB","seqScans":45,"idxScans":120,"nTupIns":200,"nTupUpd":50,"nTupDel":10}]
...

--------

Step: 2

Based on the results, you have one table in your database: `dogs`

export default defineConfig({
  test: {
    globalSetup: './src/evals/global-setup.ts'
    ...

import { randomUUID } from 'crypto';

export default async function globalSetup() {
  process.env.TEST_RUN_ID = randomUUID();
}

import { describe, it } from 'vitest';

describe.concurrent('judge', () => {
  it.test('eval_id_1', ({ expect }) => {
    // note: we must use a local version of expect when running tests concurrently
    const fullEvalId = getEvalId(expect);
  });
});

export const getEvalId = (expect: ExpectStatic) => {
  const testName = expect.getState().currentTestName;
  return testNameToEvalId(testName);
};

export const testNameToEvalId = (testName: string | undefined) => {
  if (!testName) {
    throw new Error('Expected testName to be defined');
  }
  return testName?.replace(' > ', '_');
};

import fs from 'fs';
import path from 'path';
import { TestCase } from 'vitest/node';
import { Reporter } from 'vitest/reporters';
import { testNameToEvalId } from './lib/test-id';

export default class EvalReporter implements Reporter {
  async onTestRunEnd() {
    const traceFolder = path.join('/tmp/eval-runs/', process.env.TEST_RUN_ID);
    const folders = fs.readdirSync(evalTraceFolder);

    // post run processing goes here

    console.log(`View eval results: http://localhost:4001/evals?folder=${evalTraceFolder}`);
  }

  onTestCaseResult(testCase: TestCase) {
    if (['skipped', 'pending'].includes(testCase.result().state)) {
      return;
    }

    const evalId = testNameToEvalId(testCase.fullName);

    const testCaseResult = {
      id: evalId,
      result: testCase.result().state as 'passed' | 'failed'
      // other stuff..
    };

    const traceFolder = path.join('/tmp/eval-runs/', process.env.TEST_RUN_ID, testNameToEvalId(expect));
    fs.writeFileSync(path.join(traceFolder, 'result.json'), JSON.stringify(testCaseResult, null, 2));
  }
}