Evals

Grade an eval run with LLM as Judge

Spends. Runs the goal-completion judge over the finished run, scoring each case’s final answer against its expected output.

202: scheduled, not done. Read the grades from the run detail’s judges.goalCompletion rather than re-requesting — a second POST only spends again.

A run’s grading config is pinned when the run is created, so turning the judge on for the suite does not reach an already-recorded run: enable: true is what grades one, and it changes nothing beyond that run. Omitting model and threshold clears any override a previous request left on the run.

POST

projects

{projectId}

eval-runs

{runId}

judge

Grade an eval run with LLM as Judge

curl --request POST \
  --url https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs/{runId}/judge \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "force": true,
  "enable": true,
  "model": "<string>",
  "threshold": 0.5
}
'

import requests

url = "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs/{runId}/judge"

payload = {
    "force": True,
    "enable": True,
    "model": "<string>",
    "threshold": 0.5
}
headers = {
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: JSON.stringify({force: true, enable: true, model: '<string>', threshold: 0.5})
};

fetch('https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs/{runId}/judge', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs/{runId}/judge",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'force' => true,
    'enable' => true,
    'model' => '<string>',
    'threshold' => 0.5
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs/{runId}/judge"

	payload := strings.NewReader("{\n  \"force\": true,\n  \"enable\": true,\n  \"model\": \"<string>\",\n  \"threshold\": 0.5\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs/{runId}/judge")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"force\": true,\n  \"enable\": true,\n  \"model\": \"<string>\",\n  \"threshold\": 0.5\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs/{runId}/judge")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"force\": true,\n  \"enable\": true,\n  \"model\": \"<string>\",\n  \"threshold\": 0.5\n}"

response = http.request(request)
puts response.read_body

{
  "runId": "<string>",
  "projectId": "<string>",
  "status": "pending"
}

Authorizations

Authorization

string

header

required

MCPJam API key (sk_…). Create one at Settings → API keys. Guest sessions cannot use the API, and API keys cannot manage other API keys.

Path Parameters

projectId

string

required

ID of the hosted project that contains the server.

runId

string

required

Eval run ID, as returned by POST /eval-runs.

Body

application/json

Optional. A bodyless POST grades with the suite's own config.

force

boolean

Re-grade a run that already has a result. SPENDS AGAIN. Must be a real boolean — "false" is rejected rather than read as consent.

enable

boolean

Grade this run even though the judge was off when it ran. A per-RUN answer, not a suite edit: grading reads the config pinned when the run was created, so enabling the judge on the suite does not reach an already-recorded run.

model

string

Judge model for this run only.

threshold

number

Pass threshold for this run only.

Required range: 0 <= x <= 1

Response

Scheduled.

runId

string

required

projectId

string

required

status

enum<string>

required

Available options:

pending

⌘I

Grade an eval run with LLM as Judge

curl --request POST \
  --url https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs/{runId}/judge \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "force": true,
  "enable": true,
  "model": "<string>",
  "threshold": 0.5
}
'

import requests

url = "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs/{runId}/judge"

payload = {
    "force": True,
    "enable": True,
    "model": "<string>",
    "threshold": 0.5
}
headers = {
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: JSON.stringify({force: true, enable: true, model: '<string>', threshold: 0.5})
};

fetch('https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs/{runId}/judge', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs/{runId}/judge",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'force' => true,
    'enable' => true,
    'model' => '<string>',
    'threshold' => 0.5
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs/{runId}/judge"

	payload := strings.NewReader("{\n  \"force\": true,\n  \"enable\": true,\n  \"model\": \"<string>\",\n  \"threshold\": 0.5\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs/{runId}/judge")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"force\": true,\n  \"enable\": true,\n  \"model\": \"<string>\",\n  \"threshold\": 0.5\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs/{runId}/judge")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"force\": true,\n  \"enable\": true,\n  \"model\": \"<string>\",\n  \"threshold\": 0.5\n}"

response = http.request(request)
puts response.read_body

{
  "runId": "<string>",
  "projectId": "<string>",
  "status": "pending"
}

API Guides

Endpoints

Grade an eval run with LLM as Judge

Authorizations

Path Parameters

Body

Response