HTTP API

Arbiter

POST/api/v1/arbiter/evaluate
POST
/api/v1/arbiter/evaluate

Authorization

bearer_token
AuthorizationBearer <token>

In: header

Request Body

application/json

TypeScript Definitions

Use the request body type in TypeScript.

Response Body

application/json

curl -X POST "https://example.com/api/v1/arbiter/evaluate" \  -H "Content-Type: application/json" \  -d '{    "content": null,    "criteria": "string"  }'
{  "aggregate_score": 0.1,  "confidence": 0.1,  "evaluation_id": "string",  "model": "string",  "scores": [    {      "criteria": "string",      "reasoning": "string",      "score": null    }  ],  "status": "pending",  "usage": null}
GET/api/v1/arbiter/evaluations/{evaluation_id}
GET
/api/v1/arbiter/evaluations/{evaluation_id}

Authorization

bearer_token
AuthorizationBearer <token>

In: header

Path Parameters

evaluation_id*string

Response Body

application/json

curl -X GET "https://example.com/api/v1/arbiter/evaluations/string"
{  "aggregate_score": 0.1,  "completed_at": "2019-08-24T14:15:22Z",  "confidence": 0.1,  "content": null,  "created_at": "2019-08-24T14:15:22Z",  "evaluation_id": "string",  "ground_truth": null,  "ground_truth_hash": "string",  "judge_metadata": null,  "model": "string",  "scores": [    {      "criteria": "string",      "reasoning": "string",      "score": null    }  ],  "session_id": "string",  "status": "pending",  "usage": null}
POST/api/v1/arbiter/evaluations/{evaluation_id}/appeal
POST
/api/v1/arbiter/evaluations/{evaluation_id}/appeal

Authorization

bearer_token
AuthorizationBearer <token>

In: header

Path Parameters

evaluation_id*string

Request Body

application/json

TypeScript Definitions

Use the request body type in TypeScript.

Response Body

application/json

curl -X POST "https://example.com/api/v1/arbiter/evaluations/string/appeal" \  -H "Content-Type: application/json" \  -d '{    "reason": "string"  }'
{  "appeal_id": "string",  "evaluation_id": "string",  "review_method": "string",  "status": "pending_review"}
POST/api/v1/arbiter/evaluations/{evaluation_id}/override
POST
/api/v1/arbiter/evaluations/{evaluation_id}/override

Authorization

bearer_token
AuthorizationBearer <token>

In: header

Path Parameters

evaluation_id*string

Request Body

application/json

TypeScript Definitions

Use the request body type in TypeScript.

Response Body

application/json

curl -X POST "https://example.com/api/v1/arbiter/evaluations/string/override" \  -H "Content-Type: application/json" \  -d '{    "reason": "string",    "scores": [      {        "criteria": "string",        "score": null      }    ]  }'
{  "evaluation_id": "string",  "new_aggregate": 0.1,  "overridden_at": "2019-08-24T14:15:22Z",  "overridden_by": "string",  "previous_aggregate": 0.1,  "status": "pending"}
GET/api/v1/arbiter/experiments
GET
/api/v1/arbiter/experiments

Authorization

bearer_token
AuthorizationBearer <token>

In: header

Query Parameters

status?string

Filter by status

page?integer

1-indexed page number

Formatint64
Range0 <= value
page_size?integer

Page size

Formatint64
Range0 <= value

Response Body

application/json

curl -X GET "https://example.com/api/v1/arbiter/experiments"
{  "experiments": [    {      "created_at": "2019-08-24T14:15:22Z",      "experiment_id": "string",      "name": "string",      "results": null,      "sample_size": 0,      "session_a_id": "string",      "session_b_id": "string",      "status": "created"    }  ],  "pagination": {    "has_more": true,    "page": 0,    "page_size": 0,    "total": 0  }}
POST/api/v1/arbiter/experiments
POST
/api/v1/arbiter/experiments

Authorization

bearer_token
AuthorizationBearer <token>

In: header

Request Body

application/json

TypeScript Definitions

Use the request body type in TypeScript.

Response Body

application/json

curl -X POST "https://example.com/api/v1/arbiter/experiments" \  -H "Content-Type: application/json" \  -d '{    "name": "string",    "sample_size": 0,    "session_a": {},    "session_b": {}  }'
{  "experiment_id": "string",  "sample_size": 0,  "session_a_id": "string",  "session_b_id": "string",  "status": "created"}
GET/api/v1/arbiter/experiments/{experiment_id}
GET
/api/v1/arbiter/experiments/{experiment_id}

Authorization

bearer_token
AuthorizationBearer <token>

In: header

Path Parameters

experiment_id*string

Response Body

application/json

curl -X GET "https://example.com/api/v1/arbiter/experiments/string"
{  "created_at": "2019-08-24T14:15:22Z",  "experiment_id": "string",  "name": "string",  "results": null,  "sample_size": 0,  "session_a_id": "string",  "session_b_id": "string",  "status": "created"}
GET/api/v1/arbiter/health
GET
/api/v1/arbiter/health

Authorization

bearer_token
AuthorizationBearer <token>

In: header

Response Body

application/json

curl -X GET "https://example.com/api/v1/arbiter/health"
{  "active_sessions": 0,  "axl_reachable": true,  "pending_evaluations": 0,  "redis_connected": true,  "status": "string"}
GET/api/v1/arbiter/rubrics
GET
/api/v1/arbiter/rubrics

Authorization

bearer_token
AuthorizationBearer <token>

In: header

Query Parameters

search?string

Full-text search query

page?integer

1-indexed page number

Formatint64
Range0 <= value
page_size?integer

Page size

Formatint64
Range0 <= value
sort_by?string

Sort field

sort_order?string

Sort order (asc or desc)

Value in

  • "asc"
  • "desc"

Response Body

application/json

curl -X GET "https://example.com/api/v1/arbiter/rubrics"
{  "pagination": {    "has_more": true,    "page": 0,    "page_size": 0,    "total": 0  },  "rubrics": [    {      "aggregation": "weighted_mean",      "created_at": "2019-08-24T14:15:22Z",      "criteria": [        {          "categories": [            "string"          ],          "description": "string",          "name": "string",          "range_max": 0,          "range_min": 0,          "score_type": "numeric01",          "weight": 0.1        }      ],      "custom_formula": "string",      "description": "string",      "name": "string",      "owner_id": "string",      "rubric_id": "string",      "updated_at": "2019-08-24T14:15:22Z",      "version": 0    }  ]}
POST/api/v1/arbiter/rubrics
POST
/api/v1/arbiter/rubrics

Authorization

bearer_token
AuthorizationBearer <token>

In: header

Request Body

application/json

TypeScript Definitions

Use the request body type in TypeScript.

Response Body

application/json

curl -X POST "https://example.com/api/v1/arbiter/rubrics" \  -H "Content-Type: application/json" \  -d '{    "criteria": [      {        "description": "string",        "name": "string",        "score_type": "numeric01",        "weight": 0.1      }    ],    "name": "string"  }'
{  "created_at": "2019-08-24T14:15:22Z",  "name": "string",  "rubric_id": "string",  "version": 0}
GET/api/v1/arbiter/rubrics/{rubric_id}
GET
/api/v1/arbiter/rubrics/{rubric_id}

Authorization

bearer_token
AuthorizationBearer <token>

In: header

Path Parameters

rubric_id*string

Response Body

application/json

curl -X GET "https://example.com/api/v1/arbiter/rubrics/string"
{  "aggregation": "weighted_mean",  "created_at": "2019-08-24T14:15:22Z",  "criteria": [    {      "categories": [        "string"      ],      "description": "string",      "name": "string",      "range_max": 0,      "range_min": 0,      "score_type": "numeric01",      "weight": 0.1    }  ],  "custom_formula": "string",  "description": "string",  "name": "string",  "owner_id": "string",  "rubric_id": "string",  "updated_at": "2019-08-24T14:15:22Z",  "version": 0}
PATCH/api/v1/arbiter/rubrics/{rubric_id}
PATCH
/api/v1/arbiter/rubrics/{rubric_id}

Authorization

bearer_token
AuthorizationBearer <token>

In: header

Path Parameters

rubric_id*string

Request Body

application/json

TypeScript Definitions

Use the request body type in TypeScript.

Response Body

application/json

curl -X PATCH "https://example.com/api/v1/arbiter/rubrics/string" \  -H "Content-Type: application/json" \  -d '{}'
{  "aggregation": "weighted_mean",  "created_at": "2019-08-24T14:15:22Z",  "criteria": [    {      "categories": [        "string"      ],      "description": "string",      "name": "string",      "range_max": 0,      "range_min": 0,      "score_type": "numeric01",      "weight": 0.1    }  ],  "custom_formula": "string",  "description": "string",  "name": "string",  "owner_id": "string",  "rubric_id": "string",  "updated_at": "2019-08-24T14:15:22Z",  "version": 0}
DELETE/api/v1/arbiter/rubrics/{rubric_id}
DELETE
/api/v1/arbiter/rubrics/{rubric_id}

Authorization

bearer_token
AuthorizationBearer <token>

In: header

Path Parameters

rubric_id*string

Response Body

application/json

curl -X DELETE "https://example.com/api/v1/arbiter/rubrics/string"
{  "id": "string",  "message": "string"}
GET/api/v1/arbiter/sessions
GET
/api/v1/arbiter/sessions

Authorization

bearer_token
AuthorizationBearer <token>

In: header

Query Parameters

status?string

Filter by status

page?integer

1-indexed page number

Formatint64
Range0 <= value
page_size?integer

Page size

Formatint64
Range0 <= value
sort_by?string

Sort field

sort_order?string

Sort order (asc or desc)

Value in

  • "asc"
  • "desc"

Response Body

application/json

curl -X GET "https://example.com/api/v1/arbiter/sessions"
{  "pagination": {    "has_more": true,    "page": 0,    "page_size": 0,    "total": 0  },  "sessions": [    {      "content_source": null,      "created_at": "2019-08-24T14:15:22Z",      "evaluate_on_gt_change_only": true,      "expires_at": "2019-08-24T14:15:22Z",      "ground_truth_source": null,      "judge": {        "endpoint": "string",        "mode": {          "kind": "single"        },        "model": "string",        "provider": null      },      "key_provider_url": "string",      "max_evaluations": 0,      "name": "string",      "owner_id": "string",      "result_webhook_url": "string",      "rubric_id": "string",      "rubric_version": 0,      "session_id": "string",      "started_at": "2019-08-24T14:15:22Z",      "stats": null,      "status": "created",      "triggers": null    }  ]}
POST/api/v1/arbiter/sessions
POST
/api/v1/arbiter/sessions

Authorization

bearer_token
AuthorizationBearer <token>

In: header

Request Body

application/json

TypeScript Definitions

Use the request body type in TypeScript.

Response Body

application/json

curl -X POST "https://example.com/api/v1/arbiter/sessions" \  -H "Content-Type: application/json" \  -d '{    "name": "string"  }'
{  "created_at": "2019-08-24T14:15:22Z",  "name": "string",  "rubric_id": "string",  "rubric_version": 0,  "session_id": "string",  "status": "created"}
GET/api/v1/arbiter/sessions/{session_id}
GET
/api/v1/arbiter/sessions/{session_id}

Authorization

bearer_token
AuthorizationBearer <token>

In: header

Path Parameters

session_id*string

Response Body

application/json

curl -X GET "https://example.com/api/v1/arbiter/sessions/string"
{  "content_source": null,  "created_at": "2019-08-24T14:15:22Z",  "evaluate_on_gt_change_only": true,  "expires_at": "2019-08-24T14:15:22Z",  "ground_truth_source": null,  "judge": {    "endpoint": "string",    "mode": {      "kind": "single"    },    "model": "string",    "provider": null  },  "key_provider_url": "string",  "max_evaluations": 0,  "name": "string",  "owner_id": "string",  "result_webhook_url": "string",  "rubric_id": "string",  "rubric_version": 0,  "session_id": "string",  "started_at": "2019-08-24T14:15:22Z",  "stats": null,  "status": "created",  "triggers": null}
PATCH/api/v1/arbiter/sessions/{session_id}
PATCH
/api/v1/arbiter/sessions/{session_id}

Authorization

bearer_token
AuthorizationBearer <token>

In: header

Path Parameters

session_id*string

Request Body

application/json

TypeScript Definitions

Use the request body type in TypeScript.

Response Body

application/json

curl -X PATCH "https://example.com/api/v1/arbiter/sessions/string" \  -H "Content-Type: application/json" \  -d '{}'
{  "content_source": null,  "created_at": "2019-08-24T14:15:22Z",  "evaluate_on_gt_change_only": true,  "expires_at": "2019-08-24T14:15:22Z",  "ground_truth_source": null,  "judge": {    "endpoint": "string",    "mode": {      "kind": "single"    },    "model": "string",    "provider": null  },  "key_provider_url": "string",  "max_evaluations": 0,  "name": "string",  "owner_id": "string",  "result_webhook_url": "string",  "rubric_id": "string",  "rubric_version": 0,  "session_id": "string",  "started_at": "2019-08-24T14:15:22Z",  "stats": null,  "status": "created",  "triggers": null}
DELETE/api/v1/arbiter/sessions/{session_id}
DELETE
/api/v1/arbiter/sessions/{session_id}

Authorization

bearer_token
AuthorizationBearer <token>

In: header

Path Parameters

session_id*string

Response Body

application/json

curl -X DELETE "https://example.com/api/v1/arbiter/sessions/string"
{  "id": "string",  "message": "string"}
POST/api/v1/arbiter/sessions/{session_id}/calibrate
POST
/api/v1/arbiter/sessions/{session_id}/calibrate

Authorization

bearer_token
AuthorizationBearer <token>

In: header

Path Parameters

session_id*string

Request Body

application/json

TypeScript Definitions

Use the request body type in TypeScript.

Response Body

application/json

curl -X POST "https://example.com/api/v1/arbiter/sessions/string/calibrate" \  -H "Content-Type: application/json" \  -d '{    "content": null,    "known_aggregate": 0.1,    "known_scores": [      {        "criteria": "string",        "score": null      }    ]  }'
{  "aggregate_deviation": 0.1,  "calibration_id": "string",  "cumulative_accuracy": 0.1,  "judge_aggregate": 0.1,  "judge_scores": [    {      "criteria": "string",      "deviation": 0.1,      "score": null    }  ],  "total_calibrations": 0}
GET/api/v1/arbiter/sessions/{session_id}/calibration
GET
/api/v1/arbiter/sessions/{session_id}/calibration

Authorization

bearer_token
AuthorizationBearer <token>

In: header

Path Parameters

session_id*string

Response Body

application/json

curl -X GET "https://example.com/api/v1/arbiter/sessions/string/calibration"
{  "cumulative_accuracy": 0.1,  "max_deviation": 0.1,  "mean_deviation": 0.1,  "session_id": "string",  "total_calibrations": 0,  "trend": "string"}
POST/api/v1/arbiter/sessions/{session_id}/compare
POST
/api/v1/arbiter/sessions/{session_id}/compare

Authorization

bearer_token
AuthorizationBearer <token>

In: header

Path Parameters

session_id*string

Request Body

application/json

TypeScript Definitions

Use the request body type in TypeScript.

Response Body

application/json

curl -X POST "https://example.com/api/v1/arbiter/sessions/string/compare" \  -H "Content-Type: application/json" \  -d '{    "content_a": null,    "content_b": null  }'
{  "a": {    "aggregate": 0.1,    "criteria": [      {        "criteria": "string",        "reasoning": "string",        "score": null      }    ]  },  "b": {    "aggregate": 0.1,    "criteria": [      {        "criteria": "string",        "reasoning": "string",        "score": null      }    ]  },  "comparison_id": "string",  "confidence": 0.1,  "margin": 0.1,  "reasoning": "string",  "winner": "string"}
POST/api/v1/arbiter/sessions/{session_id}/content
POST
/api/v1/arbiter/sessions/{session_id}/content

Authorization

bearer_token
AuthorizationBearer <token>

In: header

Path Parameters

session_id*string

Request Body

application/json

TypeScript Definitions

Use the request body type in TypeScript.

Response Body

application/json

curl -X POST "https://example.com/api/v1/arbiter/sessions/string/content" \  -H "Content-Type: application/json" \  -d '{    "data": null  }'
{  "accepted": true,  "fetched_at": "2019-08-24T14:15:22Z"}
POST/api/v1/arbiter/sessions/{session_id}/evaluate
POST
/api/v1/arbiter/sessions/{session_id}/evaluate

Authorization

bearer_token
AuthorizationBearer <token>

In: header

Path Parameters

session_id*string

Request Body

application/json

TypeScript Definitions

Use the request body type in TypeScript.

Response Body

application/json

curl -X POST "https://example.com/api/v1/arbiter/sessions/string/evaluate" \  -H "Content-Type: application/json" \  -d '{}'
{  "aggregate_score": 0.1,  "confidence": 0.1,  "evaluation_id": "string",  "model": "string",  "scores": [    {      "criteria": "string",      "reasoning": "string",      "score": null    }  ],  "status": "pending",  "usage": null}
GET/api/v1/arbiter/sessions/{session_id}/evaluations
GET
/api/v1/arbiter/sessions/{session_id}/evaluations

Authorization

bearer_token
AuthorizationBearer <token>

In: header

Path Parameters

session_id*string

Query Parameters

status?string

Filter by status

page?integer

1-indexed page number

Formatint64
Range0 <= value
page_size?integer

Page size

Formatint64
Range0 <= value
sort_by?string

Sort field

sort_order?string

Sort order (asc or desc)

Value in

  • "asc"
  • "desc"

Response Body

application/json

curl -X GET "https://example.com/api/v1/arbiter/sessions/string/evaluations"
{  "evaluations": [    {      "aggregate_score": 0.1,      "completed_at": "2019-08-24T14:15:22Z",      "confidence": 0.1,      "content": null,      "created_at": "2019-08-24T14:15:22Z",      "evaluation_id": "string",      "ground_truth": null,      "ground_truth_hash": "string",      "judge_metadata": null,      "model": "string",      "scores": [        {          "criteria": "string",          "reasoning": "string",          "score": null        }      ],      "session_id": "string",      "status": "pending",      "usage": null    }  ],  "pagination": {    "has_more": true,    "page": 0,    "page_size": 0,    "total": 0  }}
POST/api/v1/arbiter/sessions/{session_id}/ground-truth
POST
/api/v1/arbiter/sessions/{session_id}/ground-truth

Authorization

bearer_token
AuthorizationBearer <token>

In: header

Path Parameters

session_id*string

Request Body

application/json

TypeScript Definitions

Use the request body type in TypeScript.

Response Body

application/json

curl -X POST "https://example.com/api/v1/arbiter/sessions/string/ground-truth" \  -H "Content-Type: application/json" \  -d '{    "data": null  }'
{  "accepted": true,  "changed": true,  "hash": "string",  "previous_hash": "string"}
POST/api/v1/arbiter/sessions/{session_id}/pause
POST
/api/v1/arbiter/sessions/{session_id}/pause

Authorization

bearer_token
AuthorizationBearer <token>

In: header

Path Parameters

session_id*string

Response Body

application/json

curl -X POST "https://example.com/api/v1/arbiter/sessions/string/pause"
{  "content_source": null,  "created_at": "2019-08-24T14:15:22Z",  "evaluate_on_gt_change_only": true,  "expires_at": "2019-08-24T14:15:22Z",  "ground_truth_source": null,  "judge": {    "endpoint": "string",    "mode": {      "kind": "single"    },    "model": "string",    "provider": null  },  "key_provider_url": "string",  "max_evaluations": 0,  "name": "string",  "owner_id": "string",  "result_webhook_url": "string",  "rubric_id": "string",  "rubric_version": 0,  "session_id": "string",  "started_at": "2019-08-24T14:15:22Z",  "stats": null,  "status": "created",  "triggers": null}
POST/api/v1/arbiter/sessions/{session_id}/resume
POST
/api/v1/arbiter/sessions/{session_id}/resume

Authorization

bearer_token
AuthorizationBearer <token>

In: header

Path Parameters

session_id*string

Response Body

application/json

curl -X POST "https://example.com/api/v1/arbiter/sessions/string/resume"
{  "content_source": null,  "created_at": "2019-08-24T14:15:22Z",  "evaluate_on_gt_change_only": true,  "expires_at": "2019-08-24T14:15:22Z",  "ground_truth_source": null,  "judge": {    "endpoint": "string",    "mode": {      "kind": "single"    },    "model": "string",    "provider": null  },  "key_provider_url": "string",  "max_evaluations": 0,  "name": "string",  "owner_id": "string",  "result_webhook_url": "string",  "rubric_id": "string",  "rubric_version": 0,  "session_id": "string",  "started_at": "2019-08-24T14:15:22Z",  "stats": null,  "status": "created",  "triggers": null}
POST/api/v1/arbiter/sessions/{session_id}/start
POST
/api/v1/arbiter/sessions/{session_id}/start

Authorization

bearer_token
AuthorizationBearer <token>

In: header

Path Parameters

session_id*string

Response Body

application/json

curl -X POST "https://example.com/api/v1/arbiter/sessions/string/start"
{  "content_source": null,  "created_at": "2019-08-24T14:15:22Z",  "evaluate_on_gt_change_only": true,  "expires_at": "2019-08-24T14:15:22Z",  "ground_truth_source": null,  "judge": {    "endpoint": "string",    "mode": {      "kind": "single"    },    "model": "string",    "provider": null  },  "key_provider_url": "string",  "max_evaluations": 0,  "name": "string",  "owner_id": "string",  "result_webhook_url": "string",  "rubric_id": "string",  "rubric_version": 0,  "session_id": "string",  "started_at": "2019-08-24T14:15:22Z",  "stats": null,  "status": "created",  "triggers": null}
GET/api/v1/arbiter/sessions/{session_id}/stats
GET
/api/v1/arbiter/sessions/{session_id}/stats

Authorization

bearer_token
AuthorizationBearer <token>

In: header

Path Parameters

session_id*string

Response Body

application/json

curl -X GET "https://example.com/api/v1/arbiter/sessions/string/stats"
{  "drift": {    "alert": true,    "baseline_mean": 0.1,    "current_window_mean": 0.1,    "message": "string",    "shift": 0.1  },  "score_stats": {    "max": 0.1,    "mean": 0.1,    "median": 0.1,    "min": 0.1,    "p25": 0.1,    "p5": 0.1,    "p75": 0.1,    "p95": 0.1,    "stddev": 0.1  },  "session_id": "string",  "status_breakdown": null,  "total_evaluations": 0,  "usage": null}
POST/api/v1/arbiter/sessions/{session_id}/stop
POST
/api/v1/arbiter/sessions/{session_id}/stop

Authorization

bearer_token
AuthorizationBearer <token>

In: header

Path Parameters

session_id*string

Response Body

application/json

curl -X POST "https://example.com/api/v1/arbiter/sessions/string/stop"
{  "content_source": null,  "created_at": "2019-08-24T14:15:22Z",  "evaluate_on_gt_change_only": true,  "expires_at": "2019-08-24T14:15:22Z",  "ground_truth_source": null,  "judge": {    "endpoint": "string",    "mode": {      "kind": "single"    },    "model": "string",    "provider": null  },  "key_provider_url": "string",  "max_evaluations": 0,  "name": "string",  "owner_id": "string",  "result_webhook_url": "string",  "rubric_id": "string",  "rubric_version": 0,  "session_id": "string",  "started_at": "2019-08-24T14:15:22Z",  "stats": null,  "status": "created",  "triggers": null}

On this page