Eval runs
Compare a run against a baseline
Compare this run against a baseline run: per-case status (regressed, fixed, new_case, removed_case, changed), per-scorer pass-rate and mean deltas from the evaluation contract, and whether the evaluation config changed. Omit baseRunId to compare against the nearest earlier completed run in the same suite. Returns 404 with details.reason = BASELINE_NOT_FOUND when there is no comparable predecessor — that is an incomplete comparison, not a failing one. A scorer whose definitionChanged is true was graded by a different definition on each side, so its delta is not a regression.
GET
/
projects
/
{projectId}
/
eval-runs
/
{runId}
/
compare
Compare a run against a baseline
curl --request GET \
--url https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs/{runId}/compare \
--header 'Authorization: Bearer <token>'import requests
url = "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs/{runId}/compare"
headers = {"Authorization": "Bearer <token>"}
response = requests.get(url, headers=headers)
print(response.text)const options = {method: 'GET', headers: {Authorization: 'Bearer <token>'}};
fetch('https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs/{runId}/compare', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs/{runId}/compare",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "GET",
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"net/http"
"io"
)
func main() {
url := "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs/{runId}/compare"
req, _ := http.NewRequest("GET", url, nil)
req.Header.Add("Authorization", "Bearer <token>")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.get("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs/{runId}/compare")
.header("Authorization", "Bearer <token>")
.asString();require 'uri'
require 'net/http'
url = URI("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs/{runId}/compare")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Get.new(url)
request["Authorization"] = 'Bearer <token>'
response = http.request(request)
puts response.read_body{
"suite": {
"id": "<string>",
"name": "<string>"
},
"baseline": {
"baseRunId": "<string>"
},
"baseRun": {
"id": "<string>",
"runNumber": 123,
"result": "<string>",
"createdAt": 123,
"completedAt": 123,
"summary": {
"total": 123,
"passed": 123,
"failed": 123,
"passRate": 123
}
},
"compareRun": {
"id": "<string>",
"runNumber": 123,
"result": "<string>",
"createdAt": 123,
"completedAt": 123,
"summary": {
"total": 123,
"passed": 123,
"failed": 123,
"passRate": 123
}
},
"passSummary": {
"passRatePercent": {
"base": 123,
"compare": 123,
"delta": 123,
"percentDelta": 123
},
"total": {
"base": 123,
"compare": 123,
"delta": 123,
"percentDelta": 123
},
"passed": {
"base": 123,
"compare": 123,
"delta": 123,
"percentDelta": 123
},
"failed": {
"base": 123,
"compare": 123,
"delta": 123,
"percentDelta": 123
}
},
"metrics": {
"wallDurationMs": {
"base": 123,
"compare": 123,
"delta": 123,
"percentDelta": 123
},
"totalTokens": {
"base": 123,
"compare": 123,
"delta": 123,
"percentDelta": 123
},
"estimatedCostUsd": {
"base": 123,
"compare": 123,
"delta": 123,
"percentDelta": 123
}
},
"scoreContract": {
"base": {
"evaluationConfigHash": "<string>",
"scoredIterations": 123,
"quarantinedIterations": 123
},
"compare": {
"evaluationConfigHash": "<string>",
"scoredIterations": 123,
"quarantinedIterations": 123
},
"evaluationConfigChanged": true,
"scorers": [
{
"scorerId": "<string>",
"gating": true,
"deterministic": true,
"definitionChanged": true,
"passRate": {
"base": 123,
"compare": 123,
"delta": 123,
"percentDelta": 123
},
"meanValue": {
"base": 123,
"compare": 123,
"delta": 123,
"percentDelta": 123
},
"errorCount": {
"base": 123,
"compare": 123
}
}
]
},
"cases": [
{
"caseKey": "<string>",
"title": "<string>",
"configChanged": true,
"evaluationConfigChanged": true,
"scoreDeltas": [
{
"scorerId": "<string>",
"gating": true,
"deterministic": true,
"definitionChanged": true,
"base": {
"value": 123,
"passed": true
},
"compare": {
"value": 123,
"passed": true
},
"value": {
"base": 123,
"compare": 123,
"delta": 123,
"percentDelta": 123
}
}
],
"base": {
"iterationIds": [
"<string>"
],
"representativeIterationId": "<string>",
"error": "<string>"
},
"compare": {
"iterationIds": [
"<string>"
],
"representativeIterationId": "<string>",
"error": "<string>"
}
}
]
}
Authorizations
MCPJam API key (sk_…). Create one at Settings → API keys. Guest sessions cannot use the API, and API keys cannot manage other API keys.
Path Parameters
ID of the hosted project that contains the server.
Eval run ID, as returned by POST /eval-runs.
Query Parameters
Run ID to compare against. Omit to use the nearest earlier completed run in the same suite.
Response
The comparison.
Show child attributes
Show child attributes
Show child attributes
Show child attributes
Show child attributes
Show child attributes
Show child attributes
Show child attributes
Run-summary counters. Named passSummary, not scores, so it cannot be confused with scoreContract — the two answer different questions.
Show child attributes
Show child attributes
Show child attributes
Show child attributes
Show child attributes
Show child attributes
Show child attributes
Show child attributes
Was this page helpful?
⌘I
Compare a run against a baseline
curl --request GET \
--url https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs/{runId}/compare \
--header 'Authorization: Bearer <token>'import requests
url = "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs/{runId}/compare"
headers = {"Authorization": "Bearer <token>"}
response = requests.get(url, headers=headers)
print(response.text)const options = {method: 'GET', headers: {Authorization: 'Bearer <token>'}};
fetch('https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs/{runId}/compare', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs/{runId}/compare",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "GET",
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"net/http"
"io"
)
func main() {
url := "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs/{runId}/compare"
req, _ := http.NewRequest("GET", url, nil)
req.Header.Add("Authorization", "Bearer <token>")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.get("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs/{runId}/compare")
.header("Authorization", "Bearer <token>")
.asString();require 'uri'
require 'net/http'
url = URI("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs/{runId}/compare")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Get.new(url)
request["Authorization"] = 'Bearer <token>'
response = http.request(request)
puts response.read_body{
"suite": {
"id": "<string>",
"name": "<string>"
},
"baseline": {
"baseRunId": "<string>"
},
"baseRun": {
"id": "<string>",
"runNumber": 123,
"result": "<string>",
"createdAt": 123,
"completedAt": 123,
"summary": {
"total": 123,
"passed": 123,
"failed": 123,
"passRate": 123
}
},
"compareRun": {
"id": "<string>",
"runNumber": 123,
"result": "<string>",
"createdAt": 123,
"completedAt": 123,
"summary": {
"total": 123,
"passed": 123,
"failed": 123,
"passRate": 123
}
},
"passSummary": {
"passRatePercent": {
"base": 123,
"compare": 123,
"delta": 123,
"percentDelta": 123
},
"total": {
"base": 123,
"compare": 123,
"delta": 123,
"percentDelta": 123
},
"passed": {
"base": 123,
"compare": 123,
"delta": 123,
"percentDelta": 123
},
"failed": {
"base": 123,
"compare": 123,
"delta": 123,
"percentDelta": 123
}
},
"metrics": {
"wallDurationMs": {
"base": 123,
"compare": 123,
"delta": 123,
"percentDelta": 123
},
"totalTokens": {
"base": 123,
"compare": 123,
"delta": 123,
"percentDelta": 123
},
"estimatedCostUsd": {
"base": 123,
"compare": 123,
"delta": 123,
"percentDelta": 123
}
},
"scoreContract": {
"base": {
"evaluationConfigHash": "<string>",
"scoredIterations": 123,
"quarantinedIterations": 123
},
"compare": {
"evaluationConfigHash": "<string>",
"scoredIterations": 123,
"quarantinedIterations": 123
},
"evaluationConfigChanged": true,
"scorers": [
{
"scorerId": "<string>",
"gating": true,
"deterministic": true,
"definitionChanged": true,
"passRate": {
"base": 123,
"compare": 123,
"delta": 123,
"percentDelta": 123
},
"meanValue": {
"base": 123,
"compare": 123,
"delta": 123,
"percentDelta": 123
},
"errorCount": {
"base": 123,
"compare": 123
}
}
]
},
"cases": [
{
"caseKey": "<string>",
"title": "<string>",
"configChanged": true,
"evaluationConfigChanged": true,
"scoreDeltas": [
{
"scorerId": "<string>",
"gating": true,
"deterministic": true,
"definitionChanged": true,
"base": {
"value": 123,
"passed": true
},
"compare": {
"value": 123,
"passed": true
},
"value": {
"base": 123,
"compare": 123,
"delta": 123,
"percentDelta": 123
}
}
],
"base": {
"iterationIds": [
"<string>"
],
"representativeIterationId": "<string>",
"error": "<string>"
},
"compare": {
"iterationIds": [
"<string>"
],
"representativeIterationId": "<string>",
"error": "<string>"
}
}
]
}