Create an annotation queue
Creates a human review queue for one target type, pinning the evaluator versions reviewers score with.
curl --request POST \
--url https://api.veri.studio/v1/annotation-queues \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"name": "<string>",
"target_type": "<string>",
"evaluators": [
"<string>"
],
"description": "<string>",
"instructions": "<string>",
"reservation_minutes": 123
}
'import requests
url = "https://api.veri.studio/v1/annotation-queues"
payload = {
"name": "<string>",
"target_type": "<string>",
"evaluators": ["<string>"],
"description": "<string>",
"instructions": "<string>",
"reservation_minutes": 123
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
name: '<string>',
target_type: '<string>',
evaluators: ['<string>'],
description: '<string>',
instructions: '<string>',
reservation_minutes: 123
})
};
fetch('https://api.veri.studio/v1/annotation-queues', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));{
"object": "<string>",
"id": "<string>",
"name": "<string>",
"target_type": "<string>",
"evaluators": [
{
"evaluator_id": "<string>",
"version": 123,
"name": "<string>",
"method": "<string>",
"scope": "<string>",
"output_type": "<string>"
}
],
"reservation_minutes": 123,
"counts": {
"pending": 123,
"reserved": 123,
"completed": 123,
"skipped": 123
},
"created_at": "2023-11-07T05:31:56Z",
"updated_at": "2023-11-07T05:31:56Z",
"description": "<string>",
"instructions": "<string>",
"archived_at": "2023-11-07T05:31:56Z",
"created_by": "<string>",
"evaluator_specs": [
{
"evaluator_id": "<string>",
"name": "<string>",
"version": 123,
"method": "<string>",
"scope": "<string>",
"output_type": "<string>",
"output_config": {},
"description": "<string>",
"instructions": "<string>"
}
]
}Authorizations
API key with the vk_ prefix. Create one from the dashboard.
Body
1-64 lowercase letters, digits, '.', '_' or '-'; usable in place of the id in every path.
trace | thread | experiment_item: what the queue reviews.
Evaluator ids or names, each optionally @N (default: the latest
version). Any method: human rubrics, or an llm_judge / code evaluator
to collect human labels on it (the judge-vs-human agreement check).
All share one scope: single_turn for trace queues, thread for thread
queues.
Shown to reviewers with every item (markdown).
How long next holds an item for one reviewer, 1-1440 (default 15).
Response
The queue
Always "annotation_queue".
aq_...
trace | thread | experiment_item
The pinned evaluator versions.
Show child attributes
Show child attributes
Items per status; an expired reservation counts as pending.
Show child attributes
Show child attributes
Set when archived: no new items or reservations.
The pinned versions in full. Only on GET /v1/annotation-queues/{id_or_name}.
Show child attributes
Show child attributes
curl --request POST \
--url https://api.veri.studio/v1/annotation-queues \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"name": "<string>",
"target_type": "<string>",
"evaluators": [
"<string>"
],
"description": "<string>",
"instructions": "<string>",
"reservation_minutes": 123
}
'import requests
url = "https://api.veri.studio/v1/annotation-queues"
payload = {
"name": "<string>",
"target_type": "<string>",
"evaluators": ["<string>"],
"description": "<string>",
"instructions": "<string>",
"reservation_minutes": 123
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
name: '<string>',
target_type: '<string>',
evaluators: ['<string>'],
description: '<string>',
instructions: '<string>',
reservation_minutes: 123
})
};
fetch('https://api.veri.studio/v1/annotation-queues', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));{
"object": "<string>",
"id": "<string>",
"name": "<string>",
"target_type": "<string>",
"evaluators": [
{
"evaluator_id": "<string>",
"version": 123,
"name": "<string>",
"method": "<string>",
"scope": "<string>",
"output_type": "<string>"
}
],
"reservation_minutes": 123,
"counts": {
"pending": 123,
"reserved": 123,
"completed": 123,
"skipped": 123
},
"created_at": "2023-11-07T05:31:56Z",
"updated_at": "2023-11-07T05:31:56Z",
"description": "<string>",
"instructions": "<string>",
"archived_at": "2023-11-07T05:31:56Z",
"created_by": "<string>",
"evaluator_specs": [
{
"evaluator_id": "<string>",
"name": "<string>",
"version": 123,
"method": "<string>",
"scope": "<string>",
"output_type": "<string>",
"output_config": {},
"description": "<string>",
"instructions": "<string>"
}
]
}