Eval runs

Launch one run per target as a group (async)

Launches ONE run per target — attached project environments, or attached named hosts — under a single server-minted runGroupId, then detaches execution and responds 202 with a per-target receipt.

This is the only endpoint with grouped-launch semantics. POST /eval-runs also accepts a runGroupId, but purely as a display label: it groups sibling rows in the UI and gives N separate launches no group treatment. Here the server bounds the fan-out (max 10 targets), validates EVERY target before launching any of them (an unattached target, a heterogeneous target list, or a target whose host selects an unavailable harness returns 400 with zero runs started), and holds exactly ONE organization concurrency slot for the whole group — released only when the last sibling finishes.

Targets fan out along ONE axis: all environmentId or all namedHostId, never a mix. Duplicates are deduplicated by id, preserving order.

A per-target failure does NOT abort its siblings, so read outcome rather than treating the 202 as “everything started”: started means every target launched, partial means some did, failed means none did.

Send Idempotency-Key (or idempotencyKey) to make the launch replayable: the group id is DERIVED from the key, and each target carries a run key derived from that, so a retry after a crash mid-launch returns the original run ids instead of double-launching the targets that already started.

POST

projects

{projectId}

eval-run-groups

Launch one run per target as a group (async)

curl --request POST \
  --url https://app.mcpjam.com/api/v1/projects/{projectId}/eval-run-groups \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "suiteId": "suite_def456",
  "targets": [
    {
      "namedHostId": "host_claude"
    },
    {
      "namedHostId": "host_chatgpt"
    }
  ],
  "iterationOverride": 3
}
'

import requests

url = "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-run-groups"

payload = {
    "suiteId": "suite_def456",
    "targets": [{ "namedHostId": "host_claude" }, { "namedHostId": "host_chatgpt" }],
    "iterationOverride": 3
}
headers = {
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    suiteId: 'suite_def456',
    targets: [{namedHostId: 'host_claude'}, {namedHostId: 'host_chatgpt'}],
    iterationOverride: 3
  })
};

fetch('https://app.mcpjam.com/api/v1/projects/{projectId}/eval-run-groups', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-run-groups",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'suiteId' => 'suite_def456',
    'targets' => [
        [
                'namedHostId' => 'host_claude'
        ],
        [
                'namedHostId' => 'host_chatgpt'
        ]
    ],
    'iterationOverride' => 3
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-run-groups"

	payload := strings.NewReader("{\n  \"suiteId\": \"suite_def456\",\n  \"targets\": [\n    {\n      \"namedHostId\": \"host_claude\"\n    },\n    {\n      \"namedHostId\": \"host_chatgpt\"\n    }\n  ],\n  \"iterationOverride\": 3\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-run-groups")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"suiteId\": \"suite_def456\",\n  \"targets\": [\n    {\n      \"namedHostId\": \"host_claude\"\n    },\n    {\n      \"namedHostId\": \"host_chatgpt\"\n    }\n  ],\n  \"iterationOverride\": 3\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-run-groups")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"suiteId\": \"suite_def456\",\n  \"targets\": [\n    {\n      \"namedHostId\": \"host_claude\"\n    },\n    {\n      \"namedHostId\": \"host_chatgpt\"\n    }\n  ],\n  \"iterationOverride\": 3\n}"

response = http.request(request)
puts response.read_body

{
  "runGroupId": "6f1a2b3c4d5e6f708192a3b4c5d6e7f8",
  "suiteId": "suite_def456",
  "outcome": "partial",
  "startedCount": 1,
  "failedCount": 1,
  "targets": [
    {
      "target": {
        "namedHostId": "host_claude",
        "name": "Claude"
      },
      "status": "started",
      "runId": "run_abc123",
      "runStatus": "running"
    },
    {
      "target": {
        "namedHostId": "host_chatgpt",
        "name": "ChatGPT"
      },
      "status": "failed",
      "error": {
        "code": "VALIDATION_ERROR",
        "message": "Suite has no saved server selection to rerun against."
      }
    }
  ],
  "runId": "run_abc123",
  "status": "running"
}

Authorizations

Authorization

string

header

required

MCPJam API key (sk_…). Create one at Settings → API keys. Guest sessions cannot use the API, and API keys cannot manage other API keys.

Path Parameters

projectId

string

required

ID of the hosted project that contains the server.

Body

application/json

Launch one run per target under a single server-minted group id.

targets fan out along ONE axis: every entry is an environmentId, or every entry is a namedHostId. A mixed list returns 400 with details.reason: "HETEROGENEOUS_TARGETS" — mixing them would have to answer whether a host runs inside an environment or beside it, and every answer to that is a cross product nobody asked for. Duplicates are deduplicated by id, preserving order.

The knobs below apply UNIFORMLY to every run in the group. serverIds and refreshSnapshot are deliberately absent: an environment supplies its own closed server set, a host derives one from its config, and last-writer-wins on a frozen suite snapshot is never what a fan-out means. Inline tests are absent for the same reason — a group runs a persisted suite.

suiteId

string

required

The suite every target runs.

targets

object[]

required

One entry per run. Every target must be ATTACHED to the suite; an unattached one returns 400 with zero runs started.

Required array length: 1 - 10 elements

Option 1
Option 2

Show child attributes

iterationOverride

integer

Override the per-case runs count for every run in the group.

Required range: 1 <= x <= 10

caseIds

string[]

Narrow every run in the group to these suite cases.

Minimum array length: 1

matchOptionsOverride

object

Match options for every run in the group. Same public-or-internal vocabulary as POST /eval-runs.

skillsOverride

enum<string>

Run every target as the "without skills" arm.

Available options:

exclude

notes

string

passCriteria

object

Show child attributes

idempotencyKey

string

Makes the whole launch replayable. The runGroupId is DERIVED from this key and each target's run key is derived from that, so a retry after a crash mid-launch returns the original run ids rather than double-launching the targets that already started. The Idempotency-Key header carries the same value and wins over this field.

Maximum string length: 256

Response

Group accepted. Each target's outcome is in targets; execution continues in the background.

One entry per target. A per-target failure does NOT abort its siblings, so read outcome rather than treating the 202 as "everything started".

runGroupId

string

required

Server-minted. Every sibling run carries it, and the hosted UI collapses them into one row.

suiteId

string

required

outcome

enum<string>

required

started — every target launched; partial — some did and some did not; failed — none did (still a 202: the group itself was valid, and the per-target reasons are in targets).

Available options:

started,

partial,

failed

startedCount

integer

required

failedCount

integer

required

targets

object[]

required

Discriminated on status, so a target that failed can never be mistaken for one that started with an unread runId.

Show child attributes

runId

string

deprecated

Mirror of the FIRST started run, so readers written against the single-run receipt keep working. Absent when nothing started. Read targets instead — this describes one run out of several.

status

string

deprecated

See runId.

servers

object[]

deprecated

See runId.

Show child attributes

environment

object · null · null

deprecated

See runId.

Show child attributes

caseUpsert

object

deprecated

See runId.

⌘I

Launch one run per target as a group (async)

curl --request POST \
  --url https://app.mcpjam.com/api/v1/projects/{projectId}/eval-run-groups \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "suiteId": "suite_def456",
  "targets": [
    {
      "namedHostId": "host_claude"
    },
    {
      "namedHostId": "host_chatgpt"
    }
  ],
  "iterationOverride": 3
}
'

import requests

url = "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-run-groups"

payload = {
    "suiteId": "suite_def456",
    "targets": [{ "namedHostId": "host_claude" }, { "namedHostId": "host_chatgpt" }],
    "iterationOverride": 3
}
headers = {
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    suiteId: 'suite_def456',
    targets: [{namedHostId: 'host_claude'}, {namedHostId: 'host_chatgpt'}],
    iterationOverride: 3
  })
};

fetch('https://app.mcpjam.com/api/v1/projects/{projectId}/eval-run-groups', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-run-groups",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'suiteId' => 'suite_def456',
    'targets' => [
        [
                'namedHostId' => 'host_claude'
        ],
        [
                'namedHostId' => 'host_chatgpt'
        ]
    ],
    'iterationOverride' => 3
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-run-groups"

	payload := strings.NewReader("{\n  \"suiteId\": \"suite_def456\",\n  \"targets\": [\n    {\n      \"namedHostId\": \"host_claude\"\n    },\n    {\n      \"namedHostId\": \"host_chatgpt\"\n    }\n  ],\n  \"iterationOverride\": 3\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-run-groups")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"suiteId\": \"suite_def456\",\n  \"targets\": [\n    {\n      \"namedHostId\": \"host_claude\"\n    },\n    {\n      \"namedHostId\": \"host_chatgpt\"\n    }\n  ],\n  \"iterationOverride\": 3\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-run-groups")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"suiteId\": \"suite_def456\",\n  \"targets\": [\n    {\n      \"namedHostId\": \"host_claude\"\n    },\n    {\n      \"namedHostId\": \"host_chatgpt\"\n    }\n  ],\n  \"iterationOverride\": 3\n}"

response = http.request(request)
puts response.read_body

{
  "runGroupId": "6f1a2b3c4d5e6f708192a3b4c5d6e7f8",
  "suiteId": "suite_def456",
  "outcome": "partial",
  "startedCount": 1,
  "failedCount": 1,
  "targets": [
    {
      "target": {
        "namedHostId": "host_claude",
        "name": "Claude"
      },
      "status": "started",
      "runId": "run_abc123",
      "runStatus": "running"
    },
    {
      "target": {
        "namedHostId": "host_chatgpt",
        "name": "ChatGPT"
      },
      "status": "failed",
      "error": {
        "code": "VALIDATION_ERROR",
        "message": "Suite has no saved server selection to rerun against."
      }
    }
  ],
  "runId": "run_abc123",
  "status": "running"
}

API Guides

Endpoints

Launch one run per target as a group (async)

Authorizations

Path Parameters

Body

Response