Evaluators
Create an evaluator
Creates the evaluator and its version 1. An llm_judge evaluator must name a deployment in the caller’s workspace as its judge.
POST
/
v1
/
evaluators
Create an evaluator
curl --request POST \
--url https://api.veri.studio/v1/evaluators \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"name": "<string>",
"method": "<string>",
"output_type": "<string>",
"description": "<string>",
"scope": "<string>",
"output_config": "<unknown>",
"method_config": "<unknown>",
"bindings": "<unknown>",
"requires_reference": true
}
'import requests
url = "https://api.veri.studio/v1/evaluators"
payload = {
"name": "<string>",
"method": "<string>",
"output_type": "<string>",
"description": "<string>",
"scope": "<string>",
"output_config": "<unknown>",
"method_config": "<unknown>",
"bindings": "<unknown>",
"requires_reference": True
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
name: '<string>',
method: '<string>',
output_type: '<string>',
description: '<string>',
scope: '<string>',
output_config: '<unknown>',
method_config: '<unknown>',
bindings: '<unknown>',
requires_reference: true
})
};
fetch('https://api.veri.studio/v1/evaluators', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));{
"object": "<string>",
"id": "<string>",
"name": "<string>",
"latest_version": 123,
"created_at": "2023-11-07T05:31:56Z",
"updated_at": "2023-11-07T05:31:56Z",
"version": {
"object": "<string>",
"evaluator_id": "<string>",
"version": 123,
"method": "<string>",
"scope": "<string>",
"output_type": "<string>",
"output_config": "<unknown>",
"method_config": "<unknown>",
"bindings": "<unknown>",
"requires_reference": true,
"created_at": "2023-11-07T05:31:56Z",
"created_by": "<string>"
},
"description": "<string>",
"archived_at": "2023-11-07T05:31:56Z",
"created_by": "<string>",
"versions": [
{
"version": 123,
"created_at": "2023-11-07T05:31:56Z",
"created_by": "<string>"
}
]
}Authorizations
API key with the vk_ prefix. Create one from the dashboard.
Body
application/json
1-64 lowercase letters, digits, '.', '_' or '-'. Also the Langfuse score name.
llm_judge | code | human | jev
boolean | categorical | numeric
single_turn (default) | thread
See EvaluatorVersionResponse.output_config. Optional for boolean (pass_when defaults to true).
See EvaluatorVersionResponse.method_config. runtime: cloud requires
javascript or typescript.
Variable name -> JSONPath ($, .field, ['field'], [index]).
Response
The evaluator with version 1
Always "evaluator".
evl_...
Unique per workspace; usable in place of the id in every path.
The latest version, or the one asked for with ?version=N.
Show child attributes
Show child attributes
Set when archived: no new versions; hidden from the default list.
Version history, oldest first. Only on GET /v1/evaluators/{id_or_name}.
Show child attributes
Show child attributes
Create an evaluator
curl --request POST \
--url https://api.veri.studio/v1/evaluators \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"name": "<string>",
"method": "<string>",
"output_type": "<string>",
"description": "<string>",
"scope": "<string>",
"output_config": "<unknown>",
"method_config": "<unknown>",
"bindings": "<unknown>",
"requires_reference": true
}
'import requests
url = "https://api.veri.studio/v1/evaluators"
payload = {
"name": "<string>",
"method": "<string>",
"output_type": "<string>",
"description": "<string>",
"scope": "<string>",
"output_config": "<unknown>",
"method_config": "<unknown>",
"bindings": "<unknown>",
"requires_reference": True
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
name: '<string>',
method: '<string>',
output_type: '<string>',
description: '<string>',
scope: '<string>',
output_config: '<unknown>',
method_config: '<unknown>',
bindings: '<unknown>',
requires_reference: true
})
};
fetch('https://api.veri.studio/v1/evaluators', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));{
"object": "<string>",
"id": "<string>",
"name": "<string>",
"latest_version": 123,
"created_at": "2023-11-07T05:31:56Z",
"updated_at": "2023-11-07T05:31:56Z",
"version": {
"object": "<string>",
"evaluator_id": "<string>",
"version": 123,
"method": "<string>",
"scope": "<string>",
"output_type": "<string>",
"output_config": "<unknown>",
"method_config": "<unknown>",
"bindings": "<unknown>",
"requires_reference": true,
"created_at": "2023-11-07T05:31:56Z",
"created_by": "<string>"
},
"description": "<string>",
"archived_at": "2023-11-07T05:31:56Z",
"created_by": "<string>",
"versions": [
{
"version": 123,
"created_at": "2023-11-07T05:31:56Z",
"created_by": "<string>"
}
]
}