EvalSuite

import { EvalSuite, EvalTest } from "@mcpjam/sdk";

new EvalSuite(options?: EvalSuiteConfig)

const suite = new EvalSuite({ name: "Math Operations" });

const reportingSuite = new EvalSuite({
  name: "Math Operations",
  mcpjam: {
    suiteName: "Math Eval",
    passCriteria: { minimumPassRate: 90 },
  },
});

import { EvalSuite, EvalTest, MCPClientManager } from "@mcpjam/sdk";

const manager = new MCPClientManager({
  local: { url: "http://localhost:3000/mcp" },
});
const suite = new EvalSuite({ name: "Saved client checks" });
suite.add(new EvalTest({
  id: "list_tools",
  name: "Can use my server",
  test: async (client) => {
    const result = await client.run("Use the server to list the available items.");
    return result.text.length > 0;
  },
}));

try {
  await manager.connectToServer("local");
  await suite.runWithClient({
    client: "My client", // A saved client name or ID
    projectId: process.env.MCPJAM_PROJECT_ID!,
    apiKey: process.env.MCPJAM_API_KEY!,
    manager,
  }, { iterations: 3, runTimeoutMs: 120_000 });
} finally {
  await manager.disconnectAllServers();
}

add(test: EvalTest): void

suite.add(new EvalTest({
  id: "c_addition",
  name: "addition",
  test: async (agent) => {
    const r = await agent.run("Add 2 and 3");
    return r.hasToolCall("add");
  },
}));

suite.add(new EvalTest({
  id: "c_multiplication",
  name: "multiplication",
  test: async (agent) => {
    const r = await agent.run("Multiply 4 by 5");
    return r.hasToolCall("multiply");
  },
}));

run(executor: HostExecutor, options: EvalTestRunOptions): Promise<EvalSuiteResult>

const result = await suite.run(agent, {
  iterations: 30,
  concurrency: 5,
});

console.log(`Overall: ${(result.aggregate.accuracy * 100).toFixed(1)}%`);

accuracy(): number

console.log(`Suite accuracy: ${(suite.accuracy() * 100).toFixed(1)}%`);

get(name: string): EvalTest | undefined

const addTest = suite.get("addition");
if (addTest) {
  console.log(`Addition: ${(addTest.accuracy() * 100).toFixed(1)}%`);
}

getAll(): EvalTest[]

for (const test of suite.getAll()) {
  console.log(`${test.getName()}: ${(test.accuracy() * 100).toFixed(1)}%`);
}

getName(): string

size(): number

getResults(): EvalSuiteResult | null

recall(): number

precision(): number

truePositiveRate(): number

unexpectedToolCallRate(): number

falsePositiveRate(): number

averageTokenUse(): number

suite.getName() // "Math Operations"

import { MCPClientManager, HostRunner, EvalSuite, EvalTest } from "@mcpjam/sdk";

async function main() {
  // Setup
  const manager = new MCPClientManager({
    everything: {
      command: "npx",
      args: ["-y", "@modelcontextprotocol/server-everything"],
    },
  });
  await manager.connectToServer("everything");

  const agent = new HostRunner({
    tools: await manager.getTools(),
    model: "anthropic/claude-sonnet-5",
    apiKey: process.env.ANTHROPIC_API_KEY,
    temperature: 0.1,
  });

  // Build suite (with auto-save to MCPJam)
  const suite = new EvalSuite({
    name: "Everything Server Tests",
    mcpjam: {
      suiteName: "Everything Server Eval",
      passCriteria: { minimumPassRate: 90 },
    },
  });

  suite.add(new EvalTest({
    id: "c_add",
    name: "add",
    test: async (a) => (await a.run("Add 2+3")).hasToolCall("add"),
  }));

  suite.add(new EvalTest({
    id: "c_echo",
    name: "echo",
    test: async (a) => (await a.run("Echo 'test'")).hasToolCall("echo"),
  }));

  suite.add(new EvalTest({
    id: "c_longrunningoperation",
    name: "longRunningOperation",
    test: async (a) => (await a.run("Run a long operation")).hasToolCall("longRunningOperation"),
  }));

  // Run
  console.log(`Running ${suite.getName()}...\n`);

  const result = await suite.run(agent, {
    iterations: 20,
    concurrency: 3,
    onProgress: (done, total) => {
      process.stdout.write(`\r  Progress: ${done}/${total}`);
    },
  });

  // Report
  console.log(`\n\nOverall: ${(suite.accuracy() * 100).toFixed(1)}%`);
  console.log(`Total iterations: ${result.aggregate.iterations}\n`);

  console.log("Per-test breakdown:");
  for (const test of suite.getAll()) {
    const pct = (test.accuracy() * 100).toFixed(1);
    console.log(`  ${test.getName()}: ${pct}%`);
  }

  // Access individual test
  const echoTest = suite.get("echo");
  if (echoTest) {
    console.log(`\nEcho test details:`);
    console.log(`  Precision: ${(echoTest.precision() * 100).toFixed(1)}%`);
    console.log(`  Recall: ${(echoTest.recall() * 100).toFixed(1)}%`);
    console.log(`  Avg tokens: ${echoTest.averageTokenUse()}`);
  }

  // Cleanup
  await manager.disconnectServer("everything");
}

await suite.run(agent, { iterations: 30 });

if (suite.accuracy() < 0.90) {
  console.error(`❌ Suite accuracy ${(suite.accuracy() * 100).toFixed(1)}% below 90% threshold`);
  process.exit(1);
}

console.log("✅ All quality gates passed");

await suite.run(agent, { iterations: 30 });

// Every test must clear its own bar, whatever the suite-wide rate says.
const belowThreshold = suite
  .getAll()
  .filter((test) => test.accuracy() < 0.9)
  .map((test) => test.getName());

if (belowThreshold.length > 0) {
  console.error(`❌ Below 90% per case: ${belowThreshold.join(", ")}`);
  process.exit(1);
}

await suite.run(agent, { iterations: 30 });

const criticalTests = ["createOrder", "processPayment"];
let failed = false;

for (const name of criticalTests) {
  const test = suite.get(name);
  if (test && test.accuracy() < 0.95) {
    console.error(`❌ Critical test "${name}" below 95%`);
    failed = true;
  }
}

if (failed) process.exit(1);

const providers = [
  { model: "anthropic/claude-sonnet-5", key: "ANTHROPIC_API_KEY" },
  { model: "openai/gpt-5.6-sol", key: "OPENAI_API_KEY" },
];

for (const { model, key } of providers) {
  const agent = new HostRunner({
    tools,
    model,
    apiKey: process.env[key],
  });

  await suite.run(agent, { iterations: 20 });
  console.log(`${model}: ${(suite.accuracy() * 100).toFixed(1)}%`);
}