Eval runs

Create an eval run (async)

Creates a suite run from an existing suiteId (rerun) and/or inline tests, then detaches execution and responds 202 immediately with the runId. Validation and quota errors surface on this request; poll GET /eval-runs/{runId} for progress. The run appears live in the hosted UI Runs tab, tagged source: "api".

A bare suiteId with no inline tests reruns the suite as configured. Per-organization concurrency is capped (default 2 concurrent runs); exceeding it returns 429 with details.reason: "CONCURRENT_RUN_LIMIT".

For a suite with attached project environments, pass environmentId to choose which one the run uses; the 202 echoes the resolved environment triple, and GET /eval-runs/{runId} reports the same triple for the life of the run.

To make the run readable as something more specific than API in the Runs table, send x-mcpjam-launcher (and x-mcpjam-ci from a CI job). Both are labels: source is stamped api either way.

POST

projects

{projectId}

eval-runs

Create an eval run (async)

curl --request POST \
  --url https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "suiteName": "smoke",
  "serverIds": [
    "srv_abc123"
  ],
  "tests": [
    {
      "title": "echo works",
      "runs": 1,
      "model": "anthropic/claude-haiku-4.5",
      "provider": "anthropic",
      "steps": [
        {
          "id": "s1",
          "kind": "prompt",
          "prompt": "Use the echo tool to say hi"
        },
        {
          "id": "s2",
          "kind": "assert",
          "assertion": {
            "type": "toolCalledWith",
            "toolName": "echo",
            "args": {
              "args": {
                "text": "hi"
              }
            }
          }
        }
      ]
    }
  ]
}
'

import requests

url = "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs"

payload = {
    "suiteName": "smoke",
    "serverIds": ["srv_abc123"],
    "tests": [
        {
            "title": "echo works",
            "runs": 1,
            "model": "anthropic/claude-haiku-4.5",
            "provider": "anthropic",
            "steps": [
                {
                    "id": "s1",
                    "kind": "prompt",
                    "prompt": "Use the echo tool to say hi"
                },
                {
                    "id": "s2",
                    "kind": "assert",
                    "assertion": {
                        "type": "toolCalledWith",
                        "toolName": "echo",
                        "args": { "args": { "text": "hi" } }
                    }
                }
            ]
        }
    ]
}
headers = {
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    suiteName: 'smoke',
    serverIds: ['srv_abc123'],
    tests: [
      {
        title: 'echo works',
        runs: 1,
        model: 'anthropic/claude-haiku-4.5',
        provider: 'anthropic',
        steps: [
          {id: 's1', kind: 'prompt', prompt: 'Use the echo tool to say hi'},
          {
            id: 's2',
            kind: 'assert',
            assertion: {type: 'toolCalledWith', toolName: 'echo', args: {args: {text: 'hi'}}}
          }
        ]
      }
    ]
  })
};

fetch('https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'suiteName' => 'smoke',
    'serverIds' => [
        'srv_abc123'
    ],
    'tests' => [
        [
                'title' => 'echo works',
                'runs' => 1,
                'model' => 'anthropic/claude-haiku-4.5',
                'provider' => 'anthropic',
                'steps' => [
                                [
                                                                'id' => 's1',
                                                                'kind' => 'prompt',
                                                                'prompt' => 'Use the echo tool to say hi'
                                ],
                                [
                                                                'id' => 's2',
                                                                'kind' => 'assert',
                                                                'assertion' => [
                                                                                                                                'type' => 'toolCalledWith',
                                                                                                                                'toolName' => 'echo',
                                                                                                                                'args' => [
                                                                                                                                                                                                                                                                'args' => [
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                'text' => 'hi'
                                                                                                                                                                                                                                                                ]
                                                                                                                                ]
                                                                ]
                                ]
                ]
        ]
    ]
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs"

	payload := strings.NewReader("{\n  \"suiteName\": \"smoke\",\n  \"serverIds\": [\n    \"srv_abc123\"\n  ],\n  \"tests\": [\n    {\n      \"title\": \"echo works\",\n      \"runs\": 1,\n      \"model\": \"anthropic/claude-haiku-4.5\",\n      \"provider\": \"anthropic\",\n      \"steps\": [\n        {\n          \"id\": \"s1\",\n          \"kind\": \"prompt\",\n          \"prompt\": \"Use the echo tool to say hi\"\n        },\n        {\n          \"id\": \"s2\",\n          \"kind\": \"assert\",\n          \"assertion\": {\n            \"type\": \"toolCalledWith\",\n            \"toolName\": \"echo\",\n            \"args\": {\n              \"args\": {\n                \"text\": \"hi\"\n              }\n            }\n          }\n        }\n      ]\n    }\n  ]\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"suiteName\": \"smoke\",\n  \"serverIds\": [\n    \"srv_abc123\"\n  ],\n  \"tests\": [\n    {\n      \"title\": \"echo works\",\n      \"runs\": 1,\n      \"model\": \"anthropic/claude-haiku-4.5\",\n      \"provider\": \"anthropic\",\n      \"steps\": [\n        {\n          \"id\": \"s1\",\n          \"kind\": \"prompt\",\n          \"prompt\": \"Use the echo tool to say hi\"\n        },\n        {\n          \"id\": \"s2\",\n          \"kind\": \"assert\",\n          \"assertion\": {\n            \"type\": \"toolCalledWith\",\n            \"toolName\": \"echo\",\n            \"args\": {\n              \"args\": {\n                \"text\": \"hi\"\n              }\n            }\n          }\n        }\n      ]\n    }\n  ]\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"suiteName\": \"smoke\",\n  \"serverIds\": [\n    \"srv_abc123\"\n  ],\n  \"tests\": [\n    {\n      \"title\": \"echo works\",\n      \"runs\": 1,\n      \"model\": \"anthropic/claude-haiku-4.5\",\n      \"provider\": \"anthropic\",\n      \"steps\": [\n        {\n          \"id\": \"s1\",\n          \"kind\": \"prompt\",\n          \"prompt\": \"Use the echo tool to say hi\"\n        },\n        {\n          \"id\": \"s2\",\n          \"kind\": \"assert\",\n          \"assertion\": {\n            \"type\": \"toolCalledWith\",\n            \"toolName\": \"echo\",\n            \"args\": {\n              \"args\": {\n                \"text\": \"hi\"\n              }\n            }\n          }\n        }\n      ]\n    }\n  ]\n}"

response = http.request(request)
puts response.read_body

{
  "runId": "run_abc123",
  "suiteId": "suite_def456",
  "status": "running",
  "caseUpsert": {
    "committed": [
      {
        "name": "echo works"
      }
    ],
    "failed": []
  },
  "environment": null
}

Authorizations

Authorization

string

header

required

MCPJam API key (sk_…). Create one at Settings → API keys. Guest sessions cannot use the API, and API keys cannot manage other API keys.

Headers

x-mcpjam-launcher

string

The launching process's own DECLARED identity, as compact JSON: {"kind":"cli"|"mcp"|"github_action","client":"…","version":"…"}. Stored on the run as launcher and used only as a display label — source stays api and the audit trail reads the platform-minted attribution instead.

A header rather than a body field so that sending it never breaks against an older deployment: both eval-run bodies reject unknown properties, while an unknown header is ignored everywhere. For the same reason a malformed value, or a kind outside the three above, is DROPPED rather than rejected — a label must never fail a launch. Max 512 bytes.

x-mcpjam-ci

string

The CI job this run was launched from, as compact JSON. Recognized keys: provider, pipelineId (or runId), jobId (or job), runUrl, branch, commitSha. Stored on the run as ciMetadata, which is what makes a run resolvable by commit for baseline comparison. Unrecognized keys are dropped.

A header for the same compatibility reason as x-mcpjam-launcher, and dropped rather than rejected when malformed. Max 2048 bytes.

Path Parameters

projectId

string

required

ID of the hosted project that contains the server.

Body

application/json

Option 1
Option 2

Two valid shapes: suiteId (rerun an existing suite, optionally upserting inline tests into it) or suiteName + tests + serverIds (create a new suite and run it). Inline tests alone — without a suiteId or a suiteName — are rejected with VALIDATION_ERROR.

environmentId requires suiteId: an environment is launchable only through a suite that has it attached (environmentIds, set via PATCH /eval-suites/{suiteId}), so an environment run on a not-yet-created suite could never be satisfied. environmentId and serverIds are mutually exclusive.

suiteId

string

required

Existing suite to rerun. A bare suiteId with no tests reruns the suite exactly as configured.

suiteName

string

Name for a new suite. Required (non-empty) when no suiteId is given.

suiteDescription

string

tests

object[]

Inline test cases to upsert into the suite before running.

Maximum array length: 100

Inline eval test case. The case body is an ordered steps array (the unified test model). Exactly one of runs / iterations is required — an inline test on a run has no suite default to inherit. The object is CLOSED: an unknown key is a VALIDATION_ERROR, never a silently dropped field. repetitions, passThreshold and kind are per-case policy fields this surface cannot author — set them through POST /v1/projects/{projectId}/eval-suites/{suiteId}/cases.

Option 1
Option 2

Show child attributes

serverIds

string[]

Servers (by ID) the run connects to. Required when creating a new suite; optional on reruns — when omitted, the run connects the suite's saved server selection (the set its snapshot references). A rerun of a suite with no saved selection is rejected with VALIDATION_ERROR (details.reason: "NO_SAVED_SERVER_SELECTION"). Rejected outright for a suite with attached environments (details.reason: "ENVIRONMENT_SERVERS_NOT_OVERRIDABLE"): the environment supplies a closed set that a server override cannot change, so accepting one would connect a different set than the run is stamped with.

Minimum array length: 1

serverNames

string[]

Optional display names, parallel to serverIds.

suiteRerun

boolean

When true, skip per-case upsert and rerun the persisted suite. A bare suiteId with no tests is always treated as a rerun.

modelApiKeys

object

Optional per-provider model API keys (e.g. { "anthropic": "sk-ant-…" }). Falls back to your organization's configured providers when omitted.

Show child attributes

notes

string

passCriteria

object

The floor a run must clear, as a PERCENT in [0, 100] — 80 means 80%. minimumPassRatePercent is the canonical spelling (the unit is in the name); minimumPassRate is the deprecated alias for it. Send exactly one. THE NAME DISAMBIGUATES THE UNIT: a value in (0, 1) is accepted on minimumPassRatePercent, where it unambiguously means a sub-1% floor the evaluator can act on, and REJECTED on the bare minimumPassRate, where 0.8 cannot be told apart from a fraction sent by mistake and would silently make the gate unfailable. Neither field ever reinterprets a value. A per-case passThreshold IS a fraction; this suite/run floor is not.

Option 1
Option 2

Show child attributes

iterationOverride

integer

Override the per-case runs count for this run only.

Required range: 1 <= x <= 10

environmentId

string

Run against one of the suite's attached project environments. Requires suiteId, and must be a member of that suite's environmentIds — otherwise 400 with details.reason: "ENVIRONMENT_NOT_ATTACHED", raised before any case is authored or any server connected.

Omission is meaningful: a suite with no attached environments runs legacy; a suite with exactly ONE attached environment runs against it automatically (the response's environment says which); a suite with several returns 400 with details.reason: "ENVIRONMENT_REQUIRED", naming the candidates.

The environment supplies the closed server set (so serverIds is not required, and is rejected alongside it), and the run is pinned to the revision resolved at launch — if the environment changes in between, the run is rejected with 409 rather than executing against a different configuration.

ephemeralEnvironment

boolean

When true, environmentId may be a project-scoped, non-archived environment that is NOT attached to the suite. The launch never mutates the suite. Absent / false keeps the membership check. Probe GET /environments/capabilities (ephemeralEnvironmentLaunch) before sending — older servers reject the unknown field.

namedHostId

string

Run against ONE host attached to the suite. The platform snapshots that host's current config onto the run and derives the run's server set from it, so a host launch needs no serverIds. Without this, a suite with host attachments runs under the suite's own default host config — the run executes, but the result is attributed to the wrong host.

To run SEVERAL attached hosts, use POST /eval-run-groups rather than N calls here: it is the surface that bounds the fan-out and meters it as one launch.

caseIds

string[]

Narrow the run to these suite cases. The persisted suite is untouched — this filters the run's snapshot only. Every id must belong to the suite; none matching returns 404.

Minimum array length: 1

matchOptionsOverride

object

One-off tool-call match options for this run only, layered over suite defaults and per-case overrides. Does NOT mutate the suite or its cases.

Accepts EITHER the public vocabulary (toolCallOrder: any|in-order|exact, extraToolCalls, arguments) or the internal one (toolCallOrder: ignore|superset|strict, maxExtraToolCalls, argumentMatching). The two are disjoint, so a body can only be one of them; public bodies are normalized server-side.

Show child attributes

skillsOverride

enum<string>

The "without skills" arm of an A/B comparison: the run pins NO skills from any channel and is marked skillsExcluded, so the arm is labelled rather than merely empty. Scoped to skill DELIVERY — a pinned plugin's MCP servers stay connected, because which servers an arm connects is the one variable a skills A/B has to hold fixed.

Available options:

exclude

refreshSnapshot

boolean

PERSISTS A SUITE MUTATION. Re-derives and stores the suite's host-config snapshot from this request's server list, so future runs of the suite use it too. Without it a rerun leaves the snapshot frozen, which is what stops newly connected servers from silently contaminating an existing suite. Single-target launches only — it is not accepted on POST /eval-run-groups, where last-writer-wins on a frozen snapshot is never what a fan-out means.

runGroupId

string

A LABEL that groups sibling run rows for display. It has NO quota or launch semantics here: N calls carrying one id are still N independent launches, each metered separately. Grouped-launch behaviour lives on POST /eval-run-groups, which mints the id itself. Echoed back on the 202.

idempotencyKey

string

Write-idempotency key. A repeat call with the same key (same actor and suite) returns the EXISTING run instead of creating and billing a second one. The Idempotency-Key header carries the same value and WINS over this field — it is the transport-level channel unattended clients control, whereas a body key could be shaped by model output.

Maximum string length: 256

sourceHash

string

SHA-256 hex of the suite-file bytes that launched this run. Lowercase, 64 characters. Set by eval run --file; a UI or API launch that did not come from a file omits it.

Pattern: ^[a-f0-9]{64}$

importApprovals

object[]

Approve approximated imported cases for THIS RUN ONLY. The caller supplies a hosted test-case id and a reason and nothing else: the approver's identity and the approval time are derived server-side and frozen into the run's snapshot, because a caller-supplied approver would file one person's approval under another's name and a caller-supplied timestamp could be backdated past the edit that invalidated the claim. Nothing persists on the case — the next run of the same approximation needs a new approval.

Minimum array length: 1

Show child attributes

Response

Run created; execution continues in the background.

runId

string

required

suiteId

string

required

status

string

required

The run's status. running on a fresh launch; on a replay (see deduped), the existing run's own status, which may already be terminal.

caseUpsert

object

required

Per-case upsert outcomes for inline tests. Partial failures don't abort the run.

Show child attributes

deduped

boolean

Present and true when this request REPLAYED an existing run instead of starting one (an idempotency-key hit, or the short keyless dedupe window). A replayed run is not executed again, so no further credits are spent; read status for what that run actually is. Absent on a fresh launch.

runGroupId

string

Echo of the request's runGroupId, when one was sent. A LABEL only — it groups sibling rows for display and carries no quota or launch semantics.

servers

object[]

The servers the run connects to — explicit or derived from the suite's saved selection. name is present when known (always, on the derived path).

Show child attributes

environment

object · null · null

The environment revision this run is pinned to. null on a legacy run that recorded none — always present, so a caller never has to distinguish absent from unpinned.

Show child attributes

⌘I

Create an eval run (async)

curl --request POST \
  --url https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "suiteName": "smoke",
  "serverIds": [
    "srv_abc123"
  ],
  "tests": [
    {
      "title": "echo works",
      "runs": 1,
      "model": "anthropic/claude-haiku-4.5",
      "provider": "anthropic",
      "steps": [
        {
          "id": "s1",
          "kind": "prompt",
          "prompt": "Use the echo tool to say hi"
        },
        {
          "id": "s2",
          "kind": "assert",
          "assertion": {
            "type": "toolCalledWith",
            "toolName": "echo",
            "args": {
              "args": {
                "text": "hi"
              }
            }
          }
        }
      ]
    }
  ]
}
'

import requests

url = "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs"

payload = {
    "suiteName": "smoke",
    "serverIds": ["srv_abc123"],
    "tests": [
        {
            "title": "echo works",
            "runs": 1,
            "model": "anthropic/claude-haiku-4.5",
            "provider": "anthropic",
            "steps": [
                {
                    "id": "s1",
                    "kind": "prompt",
                    "prompt": "Use the echo tool to say hi"
                },
                {
                    "id": "s2",
                    "kind": "assert",
                    "assertion": {
                        "type": "toolCalledWith",
                        "toolName": "echo",
                        "args": { "args": { "text": "hi" } }
                    }
                }
            ]
        }
    ]
}
headers = {
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    suiteName: 'smoke',
    serverIds: ['srv_abc123'],
    tests: [
      {
        title: 'echo works',
        runs: 1,
        model: 'anthropic/claude-haiku-4.5',
        provider: 'anthropic',
        steps: [
          {id: 's1', kind: 'prompt', prompt: 'Use the echo tool to say hi'},
          {
            id: 's2',
            kind: 'assert',
            assertion: {type: 'toolCalledWith', toolName: 'echo', args: {args: {text: 'hi'}}}
          }
        ]
      }
    ]
  })
};

fetch('https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'suiteName' => 'smoke',
    'serverIds' => [
        'srv_abc123'
    ],
    'tests' => [
        [
                'title' => 'echo works',
                'runs' => 1,
                'model' => 'anthropic/claude-haiku-4.5',
                'provider' => 'anthropic',
                'steps' => [
                                [
                                                                'id' => 's1',
                                                                'kind' => 'prompt',
                                                                'prompt' => 'Use the echo tool to say hi'
                                ],
                                [
                                                                'id' => 's2',
                                                                'kind' => 'assert',
                                                                'assertion' => [
                                                                                                                                'type' => 'toolCalledWith',
                                                                                                                                'toolName' => 'echo',
                                                                                                                                'args' => [
                                                                                                                                                                                                                                                                'args' => [
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                'text' => 'hi'
                                                                                                                                                                                                                                                                ]
                                                                                                                                ]
                                                                ]
                                ]
                ]
        ]
    ]
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs"

	payload := strings.NewReader("{\n  \"suiteName\": \"smoke\",\n  \"serverIds\": [\n    \"srv_abc123\"\n  ],\n  \"tests\": [\n    {\n      \"title\": \"echo works\",\n      \"runs\": 1,\n      \"model\": \"anthropic/claude-haiku-4.5\",\n      \"provider\": \"anthropic\",\n      \"steps\": [\n        {\n          \"id\": \"s1\",\n          \"kind\": \"prompt\",\n          \"prompt\": \"Use the echo tool to say hi\"\n        },\n        {\n          \"id\": \"s2\",\n          \"kind\": \"assert\",\n          \"assertion\": {\n            \"type\": \"toolCalledWith\",\n            \"toolName\": \"echo\",\n            \"args\": {\n              \"args\": {\n                \"text\": \"hi\"\n              }\n            }\n          }\n        }\n      ]\n    }\n  ]\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"suiteName\": \"smoke\",\n  \"serverIds\": [\n    \"srv_abc123\"\n  ],\n  \"tests\": [\n    {\n      \"title\": \"echo works\",\n      \"runs\": 1,\n      \"model\": \"anthropic/claude-haiku-4.5\",\n      \"provider\": \"anthropic\",\n      \"steps\": [\n        {\n          \"id\": \"s1\",\n          \"kind\": \"prompt\",\n          \"prompt\": \"Use the echo tool to say hi\"\n        },\n        {\n          \"id\": \"s2\",\n          \"kind\": \"assert\",\n          \"assertion\": {\n            \"type\": \"toolCalledWith\",\n            \"toolName\": \"echo\",\n            \"args\": {\n              \"args\": {\n                \"text\": \"hi\"\n              }\n            }\n          }\n        }\n      ]\n    }\n  ]\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"suiteName\": \"smoke\",\n  \"serverIds\": [\n    \"srv_abc123\"\n  ],\n  \"tests\": [\n    {\n      \"title\": \"echo works\",\n      \"runs\": 1,\n      \"model\": \"anthropic/claude-haiku-4.5\",\n      \"provider\": \"anthropic\",\n      \"steps\": [\n        {\n          \"id\": \"s1\",\n          \"kind\": \"prompt\",\n          \"prompt\": \"Use the echo tool to say hi\"\n        },\n        {\n          \"id\": \"s2\",\n          \"kind\": \"assert\",\n          \"assertion\": {\n            \"type\": \"toolCalledWith\",\n            \"toolName\": \"echo\",\n            \"args\": {\n              \"args\": {\n                \"text\": \"hi\"\n              }\n            }\n          }\n        }\n      ]\n    }\n  ]\n}"

response = http.request(request)
puts response.read_body

{
  "runId": "run_abc123",
  "suiteId": "suite_def456",
  "status": "running",
  "caseUpsert": {
    "committed": [
      {
        "name": "echo works"
      }
    ],
    "failed": []
  },
  "environment": null
}

API Guides

Endpoints

Create an eval run (async)

Authorizations

Headers

Path Parameters

Body

Response