Eval runs

Create an eval run (async)

Creates a suite run from an existing suiteId (rerun) and/or inline tests, then detaches execution and responds 202 immediately with the runId. Validation and quota errors surface on this request; poll GET /eval-runs/{runId} for progress. The run appears live in the hosted UI Runs tab, tagged source: "api".

A bare suiteId with no inline tests reruns the suite as configured. Per-organization concurrency is capped (default 2 concurrent runs); exceeding it returns 429 with details.reason: "CONCURRENT_RUN_LIMIT".

POST

projects

{projectId}

eval-runs

Create an eval run (async)

curl --request POST \
  --url https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "suiteName": "smoke",
  "serverIds": [
    "srv_abc123"
  ],
  "tests": [
    {
      "title": "echo works",
      "runs": 1,
      "model": "anthropic/claude-haiku-4.5",
      "provider": "anthropic",
      "steps": [
        {
          "id": "s1",
          "kind": "prompt",
          "prompt": "Use the echo tool to say hi"
        },
        {
          "id": "s2",
          "kind": "assert",
          "assertion": {
            "type": "toolCalledWith",
            "toolName": "echo",
            "args": {
              "args": {
                "text": "hi"
              }
            }
          }
        }
      ]
    }
  ]
}
'

import requests

url = "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs"

payload = {
    "suiteName": "smoke",
    "serverIds": ["srv_abc123"],
    "tests": [
        {
            "title": "echo works",
            "runs": 1,
            "model": "anthropic/claude-haiku-4.5",
            "provider": "anthropic",
            "steps": [
                {
                    "id": "s1",
                    "kind": "prompt",
                    "prompt": "Use the echo tool to say hi"
                },
                {
                    "id": "s2",
                    "kind": "assert",
                    "assertion": {
                        "type": "toolCalledWith",
                        "toolName": "echo",
                        "args": { "args": { "text": "hi" } }
                    }
                }
            ]
        }
    ]
}
headers = {
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    suiteName: 'smoke',
    serverIds: ['srv_abc123'],
    tests: [
      {
        title: 'echo works',
        runs: 1,
        model: 'anthropic/claude-haiku-4.5',
        provider: 'anthropic',
        steps: [
          {id: 's1', kind: 'prompt', prompt: 'Use the echo tool to say hi'},
          {
            id: 's2',
            kind: 'assert',
            assertion: {type: 'toolCalledWith', toolName: 'echo', args: {args: {text: 'hi'}}}
          }
        ]
      }
    ]
  })
};

fetch('https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'suiteName' => 'smoke',
    'serverIds' => [
        'srv_abc123'
    ],
    'tests' => [
        [
                'title' => 'echo works',
                'runs' => 1,
                'model' => 'anthropic/claude-haiku-4.5',
                'provider' => 'anthropic',
                'steps' => [
                                [
                                                                'id' => 's1',
                                                                'kind' => 'prompt',
                                                                'prompt' => 'Use the echo tool to say hi'
                                ],
                                [
                                                                'id' => 's2',
                                                                'kind' => 'assert',
                                                                'assertion' => [
                                                                                                                                'type' => 'toolCalledWith',
                                                                                                                                'toolName' => 'echo',
                                                                                                                                'args' => [
                                                                                                                                                                                                                                                                'args' => [
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                'text' => 'hi'
                                                                                                                                                                                                                                                                ]
                                                                                                                                ]
                                                                ]
                                ]
                ]
        ]
    ]
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs"

	payload := strings.NewReader("{\n  \"suiteName\": \"smoke\",\n  \"serverIds\": [\n    \"srv_abc123\"\n  ],\n  \"tests\": [\n    {\n      \"title\": \"echo works\",\n      \"runs\": 1,\n      \"model\": \"anthropic/claude-haiku-4.5\",\n      \"provider\": \"anthropic\",\n      \"steps\": [\n        {\n          \"id\": \"s1\",\n          \"kind\": \"prompt\",\n          \"prompt\": \"Use the echo tool to say hi\"\n        },\n        {\n          \"id\": \"s2\",\n          \"kind\": \"assert\",\n          \"assertion\": {\n            \"type\": \"toolCalledWith\",\n            \"toolName\": \"echo\",\n            \"args\": {\n              \"args\": {\n                \"text\": \"hi\"\n              }\n            }\n          }\n        }\n      ]\n    }\n  ]\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"suiteName\": \"smoke\",\n  \"serverIds\": [\n    \"srv_abc123\"\n  ],\n  \"tests\": [\n    {\n      \"title\": \"echo works\",\n      \"runs\": 1,\n      \"model\": \"anthropic/claude-haiku-4.5\",\n      \"provider\": \"anthropic\",\n      \"steps\": [\n        {\n          \"id\": \"s1\",\n          \"kind\": \"prompt\",\n          \"prompt\": \"Use the echo tool to say hi\"\n        },\n        {\n          \"id\": \"s2\",\n          \"kind\": \"assert\",\n          \"assertion\": {\n            \"type\": \"toolCalledWith\",\n            \"toolName\": \"echo\",\n            \"args\": {\n              \"args\": {\n                \"text\": \"hi\"\n              }\n            }\n          }\n        }\n      ]\n    }\n  ]\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"suiteName\": \"smoke\",\n  \"serverIds\": [\n    \"srv_abc123\"\n  ],\n  \"tests\": [\n    {\n      \"title\": \"echo works\",\n      \"runs\": 1,\n      \"model\": \"anthropic/claude-haiku-4.5\",\n      \"provider\": \"anthropic\",\n      \"steps\": [\n        {\n          \"id\": \"s1\",\n          \"kind\": \"prompt\",\n          \"prompt\": \"Use the echo tool to say hi\"\n        },\n        {\n          \"id\": \"s2\",\n          \"kind\": \"assert\",\n          \"assertion\": {\n            \"type\": \"toolCalledWith\",\n            \"toolName\": \"echo\",\n            \"args\": {\n              \"args\": {\n                \"text\": \"hi\"\n              }\n            }\n          }\n        }\n      ]\n    }\n  ]\n}"

response = http.request(request)
puts response.read_body

{
  "runId": "run_abc123",
  "suiteId": "suite_def456",
  "status": "running",
  "caseUpsert": {
    "committed": [
      {
        "name": "echo works"
      }
    ],
    "failed": []
  }
}

Authorizations

Authorization

string

header

required

MCPJam API key (sk_…). Create one at Settings → API keys. Guest sessions cannot use the API, and API keys cannot manage other API keys.

Path Parameters

projectId

string

required

ID of the hosted project that contains the server.

Body

application/json

Option 1
Option 2
Option 3

Three valid shapes: suiteId (rerun an existing suite, optionally upserting inline tests into it), suiteName + tests + serverIds (create a new suite and run it), or suiteName + tests + environmentId (create a new suite and run it against a project environment, which supplies the servers). Inline tests alone — without a suiteId or a suiteName — are rejected with VALIDATION_ERROR.

suiteId

string

required

Existing suite to rerun. A bare suiteId with no tests reruns the suite exactly as configured.

suiteName

string

Name for a new suite. Required (non-empty) when no suiteId is given.

suiteDescription

string

tests

object[]

Inline test cases to upsert into the suite before running.

Maximum array length: 100

Show child attributes

serverIds

string[]

Servers (by ID) the run connects to. Required when creating a new suite unless environmentId is given (the environment supplies the closed server set, and any serverIds sent alongside it are ignored); optional on reruns — when omitted, the run connects the suite's saved server selection (the set its snapshot references). A rerun of a suite with no saved selection is rejected with VALIDATION_ERROR (details.reason: "NO_SAVED_SERVER_SELECTION").

Minimum array length: 1

modelApiKeys

object

Optional per-provider model API keys (e.g. { "anthropic": "sk-ant-…" }). Falls back to your organization's configured providers when omitted.

Show child attributes

notes

string

passCriteria

object

Show child attributes

iterationOverride

integer

Override the per-case runs count for this run only.

Required range: 1 <= x <= 10

environmentId

string

Run against a project environment. The environment supplies the closed server set (so serverIds is not required and is ignored if sent), and the run is pinned to the revision resolved at launch — if the environment changes in between, the run is rejected with 409 rather than executing against a different configuration.

Response

Run created; execution continues in the background.

runId

string

required

suiteId

string

required

status

string

required

Allowed value: "running"

caseUpsert

object

required

Per-case upsert outcomes for inline tests. Partial failures don't abort the run.

Show child attributes

servers

object[]

The servers the run connects to — explicit or derived from the suite's saved selection. name is present when known (always, on the derived path).

Show child attributes

Get run statusRun status, result, and summary. Poll until `status` is terminal (`completed`, `failed`, or `cancelled`).

⌘I

Create an eval run (async)

curl --request POST \
  --url https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "suiteName": "smoke",
  "serverIds": [
    "srv_abc123"
  ],
  "tests": [
    {
      "title": "echo works",
      "runs": 1,
      "model": "anthropic/claude-haiku-4.5",
      "provider": "anthropic",
      "steps": [
        {
          "id": "s1",
          "kind": "prompt",
          "prompt": "Use the echo tool to say hi"
        },
        {
          "id": "s2",
          "kind": "assert",
          "assertion": {
            "type": "toolCalledWith",
            "toolName": "echo",
            "args": {
              "args": {
                "text": "hi"
              }
            }
          }
        }
      ]
    }
  ]
}
'

import requests

url = "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs"

payload = {
    "suiteName": "smoke",
    "serverIds": ["srv_abc123"],
    "tests": [
        {
            "title": "echo works",
            "runs": 1,
            "model": "anthropic/claude-haiku-4.5",
            "provider": "anthropic",
            "steps": [
                {
                    "id": "s1",
                    "kind": "prompt",
                    "prompt": "Use the echo tool to say hi"
                },
                {
                    "id": "s2",
                    "kind": "assert",
                    "assertion": {
                        "type": "toolCalledWith",
                        "toolName": "echo",
                        "args": { "args": { "text": "hi" } }
                    }
                }
            ]
        }
    ]
}
headers = {
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    suiteName: 'smoke',
    serverIds: ['srv_abc123'],
    tests: [
      {
        title: 'echo works',
        runs: 1,
        model: 'anthropic/claude-haiku-4.5',
        provider: 'anthropic',
        steps: [
          {id: 's1', kind: 'prompt', prompt: 'Use the echo tool to say hi'},
          {
            id: 's2',
            kind: 'assert',
            assertion: {type: 'toolCalledWith', toolName: 'echo', args: {args: {text: 'hi'}}}
          }
        ]
      }
    ]
  })
};

fetch('https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'suiteName' => 'smoke',
    'serverIds' => [
        'srv_abc123'
    ],
    'tests' => [
        [
                'title' => 'echo works',
                'runs' => 1,
                'model' => 'anthropic/claude-haiku-4.5',
                'provider' => 'anthropic',
                'steps' => [
                                [
                                                                'id' => 's1',
                                                                'kind' => 'prompt',
                                                                'prompt' => 'Use the echo tool to say hi'
                                ],
                                [
                                                                'id' => 's2',
                                                                'kind' => 'assert',
                                                                'assertion' => [
                                                                                                                                'type' => 'toolCalledWith',
                                                                                                                                'toolName' => 'echo',
                                                                                                                                'args' => [
                                                                                                                                                                                                                                                                'args' => [
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                'text' => 'hi'
                                                                                                                                                                                                                                                                ]
                                                                                                                                ]
                                                                ]
                                ]
                ]
        ]
    ]
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs"

	payload := strings.NewReader("{\n  \"suiteName\": \"smoke\",\n  \"serverIds\": [\n    \"srv_abc123\"\n  ],\n  \"tests\": [\n    {\n      \"title\": \"echo works\",\n      \"runs\": 1,\n      \"model\": \"anthropic/claude-haiku-4.5\",\n      \"provider\": \"anthropic\",\n      \"steps\": [\n        {\n          \"id\": \"s1\",\n          \"kind\": \"prompt\",\n          \"prompt\": \"Use the echo tool to say hi\"\n        },\n        {\n          \"id\": \"s2\",\n          \"kind\": \"assert\",\n          \"assertion\": {\n            \"type\": \"toolCalledWith\",\n            \"toolName\": \"echo\",\n            \"args\": {\n              \"args\": {\n                \"text\": \"hi\"\n              }\n            }\n          }\n        }\n      ]\n    }\n  ]\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"suiteName\": \"smoke\",\n  \"serverIds\": [\n    \"srv_abc123\"\n  ],\n  \"tests\": [\n    {\n      \"title\": \"echo works\",\n      \"runs\": 1,\n      \"model\": \"anthropic/claude-haiku-4.5\",\n      \"provider\": \"anthropic\",\n      \"steps\": [\n        {\n          \"id\": \"s1\",\n          \"kind\": \"prompt\",\n          \"prompt\": \"Use the echo tool to say hi\"\n        },\n        {\n          \"id\": \"s2\",\n          \"kind\": \"assert\",\n          \"assertion\": {\n            \"type\": \"toolCalledWith\",\n            \"toolName\": \"echo\",\n            \"args\": {\n              \"args\": {\n                \"text\": \"hi\"\n              }\n            }\n          }\n        }\n      ]\n    }\n  ]\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"suiteName\": \"smoke\",\n  \"serverIds\": [\n    \"srv_abc123\"\n  ],\n  \"tests\": [\n    {\n      \"title\": \"echo works\",\n      \"runs\": 1,\n      \"model\": \"anthropic/claude-haiku-4.5\",\n      \"provider\": \"anthropic\",\n      \"steps\": [\n        {\n          \"id\": \"s1\",\n          \"kind\": \"prompt\",\n          \"prompt\": \"Use the echo tool to say hi\"\n        },\n        {\n          \"id\": \"s2\",\n          \"kind\": \"assert\",\n          \"assertion\": {\n            \"type\": \"toolCalledWith\",\n            \"toolName\": \"echo\",\n            \"args\": {\n              \"args\": {\n                \"text\": \"hi\"\n              }\n            }\n          }\n        }\n      ]\n    }\n  ]\n}"

response = http.request(request)
puts response.read_body

{
  "runId": "run_abc123",
  "suiteId": "suite_def456",
  "status": "running",
  "caseUpsert": {
    "committed": [
      {
        "name": "echo works"
      }
    ],
    "failed": []
  }
}

API Guides

Endpoints

Create an eval run (async)

Authorizations

Path Parameters

Body

Response