EvalTest

import { EvalTest } from "@mcpjam/sdk";

new EvalTest(options: EvalTestOptions)

type TestFunction = (agent: TestAgent) => Promise<boolean>

const test = new EvalTest({
  name: "addition-accuracy",
  test: async (agent) => {
    const result = await agent.prompt("Add 2 and 3");
    return result.hasToolCall("add");
  },
});

run(agent: TestAgent, options: RunOptions): Promise<void>

type ProgressCallback = (completed: number, total: number) => void

await test.run(agent, {
  iterations: 30,
  concurrency: 5,
  retries: 2,
  timeoutMs: 30000,
  onProgress: (done, total) => {
    console.log(`${done}/${total}`);
  },
});

accuracy(): number

console.log(`Accuracy: ${(test.accuracy() * 100).toFixed(1)}%`);
// "Accuracy: 96.7%"

precision(): number

recall(): number

truePositiveRate(): number

falsePositiveRate(): number

averageTokenUse(): number

console.log(`Avg tokens: ${test.averageTokenUse()}`);

getResults(): TestResult[]

const results = test.getResults();

const failures = results.filter(r => !r.passed);
console.log(`Failures: ${failures.length}`);

for (const fail of failures) {
  console.log(`  Error: ${fail.error}`);
}

test.name // "addition-accuracy"

test: async (agent) => {
  const result = await agent.prompt("Add 5 and 3");
  return result.hasToolCall("add");
}

test: async (agent) => {
  const result = await agent.prompt("Add 10 and 20");
  const args = result.getToolArguments("add");
  return args?.a === 10 && args?.b === 20;
}

test: async (agent) => {
  const result = await agent.prompt("What is 5 + 5?");
  return result.getText().includes("10");
}

test: async (agent) => {
  const result = await agent.prompt("Calculate 5 * 3");
  return (
    result.hasToolCall("multiply") &&
    !result.hasError() &&
    result.getText().length > 0
  );
}

test: async (agent) => {
  const r1 = await agent.prompt("Create a project");
  const r2 = await agent.prompt("Add a task to it", { context: r1 });
  return r1.hasToolCall("createProject") && r2.hasToolCall("createTask");
}

import { matchToolCallWithArgs } from "@mcpjam/sdk";

test: async (agent) => {
  const result = await agent.prompt("Add 2 and 3");
  return matchToolCallWithArgs("add", { a: 2, b: 3 }, result.getToolCalls());
}

import { MCPClientManager, TestAgent, EvalTest } from "@mcpjam/sdk";

async function main() {
  const manager = new MCPClientManager({
    everything: {
      command: "npx",
      args: ["-y", "@modelcontextprotocol/server-everything"],
    },
  });
  await manager.connectToServer("everything");

  const agent = new TestAgent({
    tools: await manager.getTools(),
    model: "anthropic/claude-sonnet-4-20250514",
    apiKey: process.env.ANTHROPIC_API_KEY,
    temperature: 0.1,
  });

  const test = new EvalTest({
    name: "addition",
    test: async (agent) => {
      const r = await agent.prompt("Add 2 and 3");
      return r.hasToolCall("add");
    },
  });

  console.log("Running evaluation...");

  await test.run(agent, {
    iterations: 30,
    concurrency: 5,
    onProgress: (done, total) => {
      process.stdout.write(`\r${done}/${total}`);
    },
  });

  console.log("\n\nResults:");
  console.log(`  Accuracy: ${(test.accuracy() * 100).toFixed(1)}%`);
  console.log(`  Precision: ${(test.precision() * 100).toFixed(1)}%`);
  console.log(`  Recall: ${(test.recall() * 100).toFixed(1)}%`);
  console.log(`  Avg tokens: ${test.averageTokenUse()}`);

  await manager.disconnectServer("everything");
}