Create several eval cases
Adds up to 100 cases to the suite in one call: the bulk form of the single create, and the way to import a suite or convert a repo’s test files without a round trip per case. Every entry needs a title and a non-empty steps array. Cases are validated together and reported individually: a failed entry does not roll back its committed siblings, so the response is 201 even when some entries were refused. Read failed rather than branching on the status alone. Send an Idempotency-Key header to make a retry land on the same rows.
curl --request POST \
--url https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/cases/batch \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"cases": [
{
"id": "c_lists_files",
"title": "lists files",
"steps": [
{
"id": "s1",
"kind": "prompt",
"prompt": "List the files in the project root."
},
{
"id": "s2",
"kind": "assert",
"assertion": {
"type": "toolCalledWith",
"toolName": "list_files",
"args": {
"args": {}
}
}
}
]
},
{
"title": "reads a file",
"steps": [
{
"id": "s1",
"kind": "prompt",
"prompt": "Read README.md."
},
{
"id": "s2",
"kind": "assert",
"assertion": {
"type": "toolCalledWith",
"toolName": "read_file",
"args": {
"args": {}
}
}
}
]
}
]
}
'import requests
url = "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/cases/batch"
payload = { "cases": [
{
"id": "c_lists_files",
"title": "lists files",
"steps": [
{
"id": "s1",
"kind": "prompt",
"prompt": "List the files in the project root."
},
{
"id": "s2",
"kind": "assert",
"assertion": {
"type": "toolCalledWith",
"toolName": "list_files",
"args": { "args": {} }
}
}
]
},
{
"title": "reads a file",
"steps": [
{
"id": "s1",
"kind": "prompt",
"prompt": "Read README.md."
},
{
"id": "s2",
"kind": "assert",
"assertion": {
"type": "toolCalledWith",
"toolName": "read_file",
"args": { "args": {} }
}
}
]
}
] }
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
cases: [
{
id: 'c_lists_files',
title: 'lists files',
steps: [
{id: 's1', kind: 'prompt', prompt: 'List the files in the project root.'},
{
id: 's2',
kind: 'assert',
assertion: {type: 'toolCalledWith', toolName: 'list_files', args: {args: {}}}
}
]
},
{
title: 'reads a file',
steps: [
{id: 's1', kind: 'prompt', prompt: 'Read README.md.'},
{
id: 's2',
kind: 'assert',
assertion: {type: 'toolCalledWith', toolName: 'read_file', args: {args: {}}}
}
]
}
]
})
};
fetch('https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/cases/batch', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/cases/batch",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'cases' => [
[
'id' => 'c_lists_files',
'title' => 'lists files',
'steps' => [
[
'id' => 's1',
'kind' => 'prompt',
'prompt' => 'List the files in the project root.'
],
[
'id' => 's2',
'kind' => 'assert',
'assertion' => [
'type' => 'toolCalledWith',
'toolName' => 'list_files',
'args' => [
'args' => [
]
]
]
]
]
],
[
'title' => 'reads a file',
'steps' => [
[
'id' => 's1',
'kind' => 'prompt',
'prompt' => 'Read README.md.'
],
[
'id' => 's2',
'kind' => 'assert',
'assertion' => [
'type' => 'toolCalledWith',
'toolName' => 'read_file',
'args' => [
'args' => [
]
]
]
]
]
]
]
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/cases/batch"
payload := strings.NewReader("{\n \"cases\": [\n {\n \"id\": \"c_lists_files\",\n \"title\": \"lists files\",\n \"steps\": [\n {\n \"id\": \"s1\",\n \"kind\": \"prompt\",\n \"prompt\": \"List the files in the project root.\"\n },\n {\n \"id\": \"s2\",\n \"kind\": \"assert\",\n \"assertion\": {\n \"type\": \"toolCalledWith\",\n \"toolName\": \"list_files\",\n \"args\": {\n \"args\": {}\n }\n }\n }\n ]\n },\n {\n \"title\": \"reads a file\",\n \"steps\": [\n {\n \"id\": \"s1\",\n \"kind\": \"prompt\",\n \"prompt\": \"Read README.md.\"\n },\n {\n \"id\": \"s2\",\n \"kind\": \"assert\",\n \"assertion\": {\n \"type\": \"toolCalledWith\",\n \"toolName\": \"read_file\",\n \"args\": {\n \"args\": {}\n }\n }\n }\n ]\n }\n ]\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/cases/batch")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"cases\": [\n {\n \"id\": \"c_lists_files\",\n \"title\": \"lists files\",\n \"steps\": [\n {\n \"id\": \"s1\",\n \"kind\": \"prompt\",\n \"prompt\": \"List the files in the project root.\"\n },\n {\n \"id\": \"s2\",\n \"kind\": \"assert\",\n \"assertion\": {\n \"type\": \"toolCalledWith\",\n \"toolName\": \"list_files\",\n \"args\": {\n \"args\": {}\n }\n }\n }\n ]\n },\n {\n \"title\": \"reads a file\",\n \"steps\": [\n {\n \"id\": \"s1\",\n \"kind\": \"prompt\",\n \"prompt\": \"Read README.md.\"\n },\n {\n \"id\": \"s2\",\n \"kind\": \"assert\",\n \"assertion\": {\n \"type\": \"toolCalledWith\",\n \"toolName\": \"read_file\",\n \"args\": {\n \"args\": {}\n }\n }\n }\n ]\n }\n ]\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/cases/batch")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"cases\": [\n {\n \"id\": \"c_lists_files\",\n \"title\": \"lists files\",\n \"steps\": [\n {\n \"id\": \"s1\",\n \"kind\": \"prompt\",\n \"prompt\": \"List the files in the project root.\"\n },\n {\n \"id\": \"s2\",\n \"kind\": \"assert\",\n \"assertion\": {\n \"type\": \"toolCalledWith\",\n \"toolName\": \"list_files\",\n \"args\": {\n \"args\": {}\n }\n }\n }\n ]\n },\n {\n \"title\": \"reads a file\",\n \"steps\": [\n {\n \"id\": \"s1\",\n \"kind\": \"prompt\",\n \"prompt\": \"Read README.md.\"\n },\n {\n \"id\": \"s2\",\n \"kind\": \"assert\",\n \"assertion\": {\n \"type\": \"toolCalledWith\",\n \"toolName\": \"read_file\",\n \"args\": {\n \"args\": {}\n }\n }\n }\n ]\n }\n ]\n}"
response = http.request(request)
puts response.read_bodyAuthorizations
MCPJam API key (sk_…). Create one at Settings → API keys. Guest sessions cannot use the API, and API keys cannot manage other API keys.
Headers
Which vocabulary this request and its response speak. Absent means 1, which is byte-for-byte today's contract: the same request fields, the same refusals, the same response projection. 2 is the canonical vocabulary. Any other value is a 400 with code: "VALIDATION_ERROR".
Today it decides one thing: the spelling of an evaluator's policy role. Vocabulary 1 accepts and returns gating; vocabulary 2 accepts both spellings and returns the canonical required. Sending required without the header is a 400, deliberately: vocabulary 1 is not widened to meet vocabulary 2 half way, because a boundary that accepts a spelling it does not announce is one two implementations can disagree about.
A response that varies by vocabulary sends Vary: x-mcpjam-eval-vocabulary.
1, 2 Path Parameters
Query Parameters
The suite file's own suite.id, when this request is that file syncing itself. A CI-owned suite (see EvalSuiteDetail.managedBy) refuses configuration writes with 409 and details.reason: "CI_OWNED_SUITE_READ_ONLY"; naming the suite's own declared id is how the file writing itself is allowed through. Naming any other id refuses exactly as loudly as naming none, so it is not a capability. Omit it for an ordinary edit.
A QUERY PARAMETER on every route that takes it, never a body field. An older deployment rejects an unknown body field with 400 but ignores an unknown query parameter.
128Body
Author several cases in one call. Each entry takes the same fields as a single create. The 100-case cap is deliberately smaller than a suite file's 500-case limit, so a maximal file uploads in several calls.
The suite file's own suite.id, when THIS request is a suite-file sync.
A CI-owned suite (see EvalSuiteDetail.managedBy) refuses configuration writes with 409 and details.reason: "CI_OWNED_SUITE_READ_ONLY". Naming the suite's own declared id is how the file writing itself is allowed through: the platform permits the write only when the id matches, so naming any other id refuses exactly as loudly as naming none. Omit it for ordinary edits.
PREFER THE declaredSuiteId QUERY PARAMETER on this route. This body field still works here, but these request bodies are strict on every Inspector that predates the CI-owned lock, so a body field is a 400 against an older deployment while an unknown query parameter is simply ignored. The MCPJam SDK and CLI send the query parameter.
128Response
Per-case outcomes. Some entries may have failed; see failed.
A partial outcome by design: the cases in created were written even when failed is non-empty. Both arrays carry the index of the request entry they describe.
Was this page helpful?
curl --request POST \
--url https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/cases/batch \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"cases": [
{
"id": "c_lists_files",
"title": "lists files",
"steps": [
{
"id": "s1",
"kind": "prompt",
"prompt": "List the files in the project root."
},
{
"id": "s2",
"kind": "assert",
"assertion": {
"type": "toolCalledWith",
"toolName": "list_files",
"args": {
"args": {}
}
}
}
]
},
{
"title": "reads a file",
"steps": [
{
"id": "s1",
"kind": "prompt",
"prompt": "Read README.md."
},
{
"id": "s2",
"kind": "assert",
"assertion": {
"type": "toolCalledWith",
"toolName": "read_file",
"args": {
"args": {}
}
}
}
]
}
]
}
'import requests
url = "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/cases/batch"
payload = { "cases": [
{
"id": "c_lists_files",
"title": "lists files",
"steps": [
{
"id": "s1",
"kind": "prompt",
"prompt": "List the files in the project root."
},
{
"id": "s2",
"kind": "assert",
"assertion": {
"type": "toolCalledWith",
"toolName": "list_files",
"args": { "args": {} }
}
}
]
},
{
"title": "reads a file",
"steps": [
{
"id": "s1",
"kind": "prompt",
"prompt": "Read README.md."
},
{
"id": "s2",
"kind": "assert",
"assertion": {
"type": "toolCalledWith",
"toolName": "read_file",
"args": { "args": {} }
}
}
]
}
] }
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
cases: [
{
id: 'c_lists_files',
title: 'lists files',
steps: [
{id: 's1', kind: 'prompt', prompt: 'List the files in the project root.'},
{
id: 's2',
kind: 'assert',
assertion: {type: 'toolCalledWith', toolName: 'list_files', args: {args: {}}}
}
]
},
{
title: 'reads a file',
steps: [
{id: 's1', kind: 'prompt', prompt: 'Read README.md.'},
{
id: 's2',
kind: 'assert',
assertion: {type: 'toolCalledWith', toolName: 'read_file', args: {args: {}}}
}
]
}
]
})
};
fetch('https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/cases/batch', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/cases/batch",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'cases' => [
[
'id' => 'c_lists_files',
'title' => 'lists files',
'steps' => [
[
'id' => 's1',
'kind' => 'prompt',
'prompt' => 'List the files in the project root.'
],
[
'id' => 's2',
'kind' => 'assert',
'assertion' => [
'type' => 'toolCalledWith',
'toolName' => 'list_files',
'args' => [
'args' => [
]
]
]
]
]
],
[
'title' => 'reads a file',
'steps' => [
[
'id' => 's1',
'kind' => 'prompt',
'prompt' => 'Read README.md.'
],
[
'id' => 's2',
'kind' => 'assert',
'assertion' => [
'type' => 'toolCalledWith',
'toolName' => 'read_file',
'args' => [
'args' => [
]
]
]
]
]
]
]
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/cases/batch"
payload := strings.NewReader("{\n \"cases\": [\n {\n \"id\": \"c_lists_files\",\n \"title\": \"lists files\",\n \"steps\": [\n {\n \"id\": \"s1\",\n \"kind\": \"prompt\",\n \"prompt\": \"List the files in the project root.\"\n },\n {\n \"id\": \"s2\",\n \"kind\": \"assert\",\n \"assertion\": {\n \"type\": \"toolCalledWith\",\n \"toolName\": \"list_files\",\n \"args\": {\n \"args\": {}\n }\n }\n }\n ]\n },\n {\n \"title\": \"reads a file\",\n \"steps\": [\n {\n \"id\": \"s1\",\n \"kind\": \"prompt\",\n \"prompt\": \"Read README.md.\"\n },\n {\n \"id\": \"s2\",\n \"kind\": \"assert\",\n \"assertion\": {\n \"type\": \"toolCalledWith\",\n \"toolName\": \"read_file\",\n \"args\": {\n \"args\": {}\n }\n }\n }\n ]\n }\n ]\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/cases/batch")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"cases\": [\n {\n \"id\": \"c_lists_files\",\n \"title\": \"lists files\",\n \"steps\": [\n {\n \"id\": \"s1\",\n \"kind\": \"prompt\",\n \"prompt\": \"List the files in the project root.\"\n },\n {\n \"id\": \"s2\",\n \"kind\": \"assert\",\n \"assertion\": {\n \"type\": \"toolCalledWith\",\n \"toolName\": \"list_files\",\n \"args\": {\n \"args\": {}\n }\n }\n }\n ]\n },\n {\n \"title\": \"reads a file\",\n \"steps\": [\n {\n \"id\": \"s1\",\n \"kind\": \"prompt\",\n \"prompt\": \"Read README.md.\"\n },\n {\n \"id\": \"s2\",\n \"kind\": \"assert\",\n \"assertion\": {\n \"type\": \"toolCalledWith\",\n \"toolName\": \"read_file\",\n \"args\": {\n \"args\": {}\n }\n }\n }\n ]\n }\n ]\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/cases/batch")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"cases\": [\n {\n \"id\": \"c_lists_files\",\n \"title\": \"lists files\",\n \"steps\": [\n {\n \"id\": \"s1\",\n \"kind\": \"prompt\",\n \"prompt\": \"List the files in the project root.\"\n },\n {\n \"id\": \"s2\",\n \"kind\": \"assert\",\n \"assertion\": {\n \"type\": \"toolCalledWith\",\n \"toolName\": \"list_files\",\n \"args\": {\n \"args\": {}\n }\n }\n }\n ]\n },\n {\n \"title\": \"reads a file\",\n \"steps\": [\n {\n \"id\": \"s1\",\n \"kind\": \"prompt\",\n \"prompt\": \"Read README.md.\"\n },\n {\n \"id\": \"s2\",\n \"kind\": \"assert\",\n \"assertion\": {\n \"type\": \"toolCalledWith\",\n \"toolName\": \"read_file\",\n \"args\": {\n \"args\": {}\n }\n }\n }\n ]\n }\n ]\n}"
response = http.request(request)
puts response.read_body
