Eval runs

Get a run's decision summary

The canonical, versioned run decision contract: the verdict and where it came from, the counts with the population they count, the run’s own verdict decision when it has one, and one page of per-trial diagnostics carrying the user-value chain, the first failed stage, the failure category, evidence scoped to that stage, and one next action.

ADDITIVE and composed: it is the same two reads a caller could make by hand (GET …/eval-runs/{runId} and GET …/iterations), assembled once here so every client shares one reading of a run instead of inventing its own. cursor and limit page the DIAGNOSTICS using the same cursors the iterations endpoint issues, and diagnostics.complete says honestly whether the page you got is the run’s whole non-passing set.

GET

projects

{projectId}

eval-runs

{runId}

decision-summary

Get a run's decision summary

curl --request GET \
  --url https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs/{runId}/decision-summary \
  --header 'Authorization: Bearer <token>'

import requests

url = "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs/{runId}/decision-summary"

headers = {"Authorization": "Bearer <token>"}

response = requests.get(url, headers=headers)

print(response.text)

const options = {method: 'GET', headers: {Authorization: 'Bearer <token>'}};

fetch('https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs/{runId}/decision-summary', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs/{runId}/decision-summary",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "GET",
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"net/http"
	"io"
)

func main() {

	url := "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs/{runId}/decision-summary"

	req, _ := http.NewRequest("GET", url, nil)

	req.Header.Add("Authorization", "Bearer <token>")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

require 'uri'
require 'net/http'

url = URI("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs/{runId}/decision-summary")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Get.new(url)
request["Authorization"] = 'Bearer <token>'

response = http.request(request)
puts response.read_body

{
  "schemaVersion": 1,
  "runId": "<string>",
  "runStatus": "<string>",
  "verdict": "passed",
  "verdictSource": "policyV2",
  "diagnostics": {
    "items": [
      {
        "iterationId": "<string>",
        "iterationNumber": 1,
        "status": "<string>",
        "chain": {
          "status": "verified",
          "stages": [
            {
              "stage": "connection",
              "state": "passed",
              "reason": "noSpanChannel",
              "evidence": {
                "spanIds": [
                  "<string>"
                ],
                "promptIndexes": [
                  123
                ],
                "predicateReasons": [
                  "<string>"
                ]
              }
            }
          ],
          "analyzerVersion": 1,
          "firstFailedStage": "connection",
          "failureCategory": "setup",
          "analyzerVersionAhead": {
            "reported": 1,
            "known": 1
          }
        },
        "evidence": {
          "runId": "<string>",
          "iterationId": "<string>",
          "tracePath": "<string>",
          "stage": "connection",
          "spanIds": [
            "<string>"
          ],
          "promptIndexes": [
            1
          ],
          "reasons": [
            "<string>"
          ]
        },
        "nextAction": "<string>",
        "caseId": "<string>",
        "testCaseId": "<string>",
        "title": "<string>",
        "result": "passed",
        "expected": {
          "toolNames": [
            "<string>"
          ]
        },
        "observed": {
          "toolNames": [
            "<string>"
          ],
          "failure": "<string>"
        }
      }
    ],
    "complete": true,
    "scannedIterations": 1,
    "nextCursor": "<string>"
  },
  "counts": {
    "measurementUnit": "caseVariant",
    "total": 1,
    "passed": 1,
    "failed": 1,
    "inconclusive": 1
  },
  "decision": {},
  "undecided": {
    "reason": "runNotTerminal",
    "detail": "<string>"
  }
}

Authorizations

Authorization

string

header

required

MCPJam API key (sk_…). Create one at Settings → API keys. Guest sessions cannot use the API, and API keys cannot manage other API keys.

Path Parameters

projectId

string

required

ID of the hosted project that contains the server.

runId

string

required

Eval run ID, as returned by POST /eval-runs.

Query Parameters

limit

integer

default:50

Iterations examined per page, 1–200. Defaults to 50.

Required range: 1 <= x <= 200

cursor

string

Opaque cursor from a previous response's diagnostics.nextCursor. A page fetched with one is never reported as complete.

Response

The run's decision summary.

The canonical run decision contract (schema version 1): what the run decided, in what unit it counted, and what evidence sits under the non-passing trials. START HERE when a run did not pass.

It EXPLAINS the verdict and never DECIDES it. Under verdict policy v2 the authority is the run's own decision — its verdict, rates, validity phase, reasons, per-case stability and mixed-verdict flags are copied after validation and never recomputed from the trials; counts is a tally of decision.cases[].verdict and nothing more.

verdict has a fourth value the run verdict vocabulary does not: notEstablished means no verdict exists at all (the run is unfinished, stopped early, or its decision could not be read). It is not a failure and not inconclusive — inconclusive is a decision the validity phase reached, and this is the absence of any decision.

schemaVersion

enum<integer>

required

Available options:

1

runId

string

required

runStatus

string

required

The run's lifecycle status, verbatim. Not a verdict.

verdict

enum<string>

required

Available options:

passed,

failed,

inconclusive,

notEstablished

verdictSource

enum<string>

required

policyV2 — the run's own decision, carried on decision; verdict is its verdict. legacy — a percent-threshold run predating policy v2; there is no decision object and any counts are trials. none — no verdict; verdict is notEstablished and undecided says why.

Available options:

policyV2,

legacy,

none

diagnostics

object

required

One page of diagnostics, with its completeness stated rather than implied. complete is true ONLY when items is the whole non-passing set for the run: a page reached through a cursor is never complete however few rows follow it. scannedIterations is how many trials this page examined, which is what separates "we looked at 50 and none failed" from "we did not look" — both otherwise render as an empty items.

Show child attributes

counts

object

A tally WITH the population it counted. measurementUnit is never omitted and never inferred: under verdict policy v2 the counts are case-execution VARIANTS (repetitions are trials inside one, not members of it), and on a legacy run they are trials. A 3-case suite with 5 repetitions is legitimately 3 under one unit and 15 under the other, so a count reported without its unit is not a fact. On a caseVariant count every field is present and the three buckets sum to total; on a trial count each field is present only if the legacy run recorded it — absence stays absence rather than defaulting to zero.

Option 1
Option 2

Show child attributes

decision

object

The run's authoritative v2 verdict decision, copied verbatim after validation — the same shape as EvalRun.verdictSummary, published at https://mcpjam.com/schemas/eval-verdict-policy/v2.json. Present exactly when verdictSource is policyV2.

undecided

object

Why no verdict was established. Present exactly when verdictSource is none.

Show child attributes

⌘I

Get a run's decision summary

curl --request GET \
  --url https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs/{runId}/decision-summary \
  --header 'Authorization: Bearer <token>'

import requests

url = "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs/{runId}/decision-summary"

headers = {"Authorization": "Bearer <token>"}

response = requests.get(url, headers=headers)

print(response.text)

const options = {method: 'GET', headers: {Authorization: 'Bearer <token>'}};

fetch('https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs/{runId}/decision-summary', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs/{runId}/decision-summary",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "GET",
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"net/http"
	"io"
)

func main() {

	url := "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs/{runId}/decision-summary"

	req, _ := http.NewRequest("GET", url, nil)

	req.Header.Add("Authorization", "Bearer <token>")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

require 'uri'
require 'net/http'

url = URI("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs/{runId}/decision-summary")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Get.new(url)
request["Authorization"] = 'Bearer <token>'

response = http.request(request)
puts response.read_body

{
  "schemaVersion": 1,
  "runId": "<string>",
  "runStatus": "<string>",
  "verdict": "passed",
  "verdictSource": "policyV2",
  "diagnostics": {
    "items": [
      {
        "iterationId": "<string>",
        "iterationNumber": 1,
        "status": "<string>",
        "chain": {
          "status": "verified",
          "stages": [
            {
              "stage": "connection",
              "state": "passed",
              "reason": "noSpanChannel",
              "evidence": {
                "spanIds": [
                  "<string>"
                ],
                "promptIndexes": [
                  123
                ],
                "predicateReasons": [
                  "<string>"
                ]
              }
            }
          ],
          "analyzerVersion": 1,
          "firstFailedStage": "connection",
          "failureCategory": "setup",
          "analyzerVersionAhead": {
            "reported": 1,
            "known": 1
          }
        },
        "evidence": {
          "runId": "<string>",
          "iterationId": "<string>",
          "tracePath": "<string>",
          "stage": "connection",
          "spanIds": [
            "<string>"
          ],
          "promptIndexes": [
            1
          ],
          "reasons": [
            "<string>"
          ]
        },
        "nextAction": "<string>",
        "caseId": "<string>",
        "testCaseId": "<string>",
        "title": "<string>",
        "result": "passed",
        "expected": {
          "toolNames": [
            "<string>"
          ]
        },
        "observed": {
          "toolNames": [
            "<string>"
          ],
          "failure": "<string>"
        }
      }
    ],
    "complete": true,
    "scannedIterations": 1,
    "nextCursor": "<string>"
  },
  "counts": {
    "measurementUnit": "caseVariant",
    "total": 1,
    "passed": 1,
    "failed": 1,
    "inconclusive": 1
  },
  "decision": {},
  "undecided": {
    "reason": "runNotTerminal",
    "detail": "<string>"
  }
}

API Guides

Endpoints

Get a run's decision summary

Authorizations

Path Parameters

Query Parameters

Response