Eval runs

Write a finished authoring job's cases into the suite

Writes the job’s drafts into the suite. A draft carrying unresolved blocking issues or proposed additions is skipped rather than written, and comes back under skipped with a reviewUrl for finishing it in the app.

A job whose source is markdown — the in-app upload — is not committable here and answers 404: those drafts exist so a person decides on them, and an API commit would decide on their behalf.

POST

projects

{projectId}

eval-suites

{suiteId}

authoring

{jobId}

commit

Write a finished authoring job's cases into the suite

curl --request POST \
  --url https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/authoring/{jobId}/commit \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "overrideReason": "<string>"
}
'

import requests

url = "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/authoring/{jobId}/commit"

payload = { "overrideReason": "<string>" }
headers = {
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: JSON.stringify({overrideReason: '<string>'})
};

fetch('https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/authoring/{jobId}/commit', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/authoring/{jobId}/commit",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'overrideReason' => '<string>'
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/authoring/{jobId}/commit"

	payload := strings.NewReader("{\n  \"overrideReason\": \"<string>\"\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/authoring/{jobId}/commit")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"overrideReason\": \"<string>\"\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/authoring/{jobId}/commit")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"overrideReason\": \"<string>\"\n}"

response = http.request(request)
puts response.read_body

{
  "generationModel": "<string>",
  "created": [
    {
      "id": "<string>",
      "title": "<string>",
      "steps": [
        {
          "id": "<string>",
          "kind": "prompt",
          "prompt": "<string>",
          "serverName": "<string>",
          "toolName": "<string>",
          "arguments": {},
          "action": {},
          "assertion": {}
        }
      ],
      "iterations": 5,
      "isNegative": true,
      "models": [
        {
          "model": "<string>",
          "provider": "<string>"
        }
      ],
      "judge": {
        "enabled": true
      },
      "declaredId": "<string>",
      "expectedOutput": "<string>",
      "repetitions": 2,
      "passThreshold": 0.5,
      "scenario": "<string>",
      "intent": "<string>",
      "kind": "capability",
      "matchOptions": {},
      "checks": {
        "mode": "inherit",
        "list": [
          {
            "type": "toolDescriptionsPresent",
            "minLength": 123,
            "role": "gating",
            "severity": "warn"
          }
        ]
      },
      "import": {
        "status": "exact",
        "sourceCaseKey": "<string>",
        "note": "<string>"
      },
      "source": {
        "format": "markdown",
        "method": "ai",
        "fileName": "<string>",
        "fileHash": "<string>",
        "excerpt": "<string>",
        "startLine": 2,
        "endLine": 2,
        "extractorVersion": "<string>"
      },
      "createdAt": 123,
      "updatedAt": 123,
      "suppressedSuiteStandardCheckIds": [
        "<string>"
      ]
    }
  ],
  "counts": {
    "normal": 123,
    "negative": 123
  },
  "skipped": [
    {}
  ],
  "reviewUrl": "<string>"
}

Authorizations

Authorization

string

header

required

MCPJam API key (sk_…). Create one at Settings → API keys. Guest sessions cannot use the API, and API keys cannot manage other API keys.

Path Parameters

projectId

string

required

ID of the hosted project that contains the server.

suiteId

string

required

Eval suite ID, as returned by POST /eval-runs.

jobId

string

required

Body

application/json

duplicatePolicy

enum<string>

How to treat a case that duplicates one already in the suite. Anything but block requires overrideReason.

Available options:

block,

warn,

create_anyway

overrideReason

string

Minimum string length: 1

Response

The cases written and any the commit skipped, or the job's own status when it finished without committing (failed, cancelled, or still pending at the end of the wait).

Option 1
Option 2

generationModel

string

required

created

object[]

required

Show child attributes

counts

object

required

Show child attributes

skipped

object[]

Cases the commit could not write, each with its reason. They stay on the job rather than being lost.

reviewUrl

string<uri>

Opens the app on exactly the skipped drafts. Present only when something was skipped — finishing them there costs nothing, whereas re-sending the document re-authors and re-bills every case in it.

⌘I

Write a finished authoring job's cases into the suite

curl --request POST \
  --url https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/authoring/{jobId}/commit \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "overrideReason": "<string>"
}
'

import requests

url = "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/authoring/{jobId}/commit"

payload = { "overrideReason": "<string>" }
headers = {
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: JSON.stringify({overrideReason: '<string>'})
};

fetch('https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/authoring/{jobId}/commit', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/authoring/{jobId}/commit",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'overrideReason' => '<string>'
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/authoring/{jobId}/commit"

	payload := strings.NewReader("{\n  \"overrideReason\": \"<string>\"\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/authoring/{jobId}/commit")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"overrideReason\": \"<string>\"\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/authoring/{jobId}/commit")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"overrideReason\": \"<string>\"\n}"

response = http.request(request)
puts response.read_body

{
  "generationModel": "<string>",
  "created": [
    {
      "id": "<string>",
      "title": "<string>",
      "steps": [
        {
          "id": "<string>",
          "kind": "prompt",
          "prompt": "<string>",
          "serverName": "<string>",
          "toolName": "<string>",
          "arguments": {},
          "action": {},
          "assertion": {}
        }
      ],
      "iterations": 5,
      "isNegative": true,
      "models": [
        {
          "model": "<string>",
          "provider": "<string>"
        }
      ],
      "judge": {
        "enabled": true
      },
      "declaredId": "<string>",
      "expectedOutput": "<string>",
      "repetitions": 2,
      "passThreshold": 0.5,
      "scenario": "<string>",
      "intent": "<string>",
      "kind": "capability",
      "matchOptions": {},
      "checks": {
        "mode": "inherit",
        "list": [
          {
            "type": "toolDescriptionsPresent",
            "minLength": 123,
            "role": "gating",
            "severity": "warn"
          }
        ]
      },
      "import": {
        "status": "exact",
        "sourceCaseKey": "<string>",
        "note": "<string>"
      },
      "source": {
        "format": "markdown",
        "method": "ai",
        "fileName": "<string>",
        "fileHash": "<string>",
        "excerpt": "<string>",
        "startLine": 2,
        "endLine": 2,
        "extractorVersion": "<string>"
      },
      "createdAt": 123,
      "updatedAt": 123,
      "suppressedSuiteStandardCheckIds": [
        "<string>"
      ]
    }
  ],
  "counts": {
    "normal": 123,
    "negative": 123
  },
  "skipped": [
    {}
  ],
  "reviewUrl": "<string>"
}

API Guides

Endpoints

Write a finished authoring job's cases into the suite

Authorizations

Path Parameters

Body

Response