Annotation queues
Complete a reserved item
The reviewer holding a live reservation writes one human score per pinned evaluator (source human, author = the caller, on the item’s exact target keys, target_revision included), optionally stores a correction and appends it to an eval stream as a row, and closes the item. Everything commits together, with the annotation.completed webhook.
POST
/
v1
/
annotation-queues
/
{id_or_name}
/
items
/
{item_id}
/
complete
Complete a reserved item
curl --request POST \
--url https://api.veri.studio/v1/annotation-queues/{id_or_name}/items/{item_id}/complete \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"scores": [
{
"evaluator_id": "<string>",
"value": "<unknown>"
}
],
"correction": "<unknown>",
"note": "<string>",
"append_to_dataset": "<string>"
}
'import requests
url = "https://api.veri.studio/v1/annotation-queues/{id_or_name}/items/{item_id}/complete"
payload = {
"scores": [
{
"evaluator_id": "<string>",
"value": "<unknown>"
}
],
"correction": "<unknown>",
"note": "<string>",
"append_to_dataset": "<string>"
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
scores: [{evaluator_id: '<string>', value: '<unknown>'}],
correction: '<unknown>',
note: '<string>',
append_to_dataset: '<string>'
})
};
fetch('https://api.veri.studio/v1/annotation-queues/{id_or_name}/items/{item_id}/complete', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));{
"object": "<string>",
"item": {
"object": "<string>",
"id": "<string>",
"queue_id": "<string>",
"target_type": "<string>",
"status": "<string>",
"created_at": "2023-11-07T05:31:56Z",
"request_id": "<string>",
"deployment_id": "<string>",
"thread_id": "<string>",
"target_revision": "<string>",
"experiment_id": "<string>",
"item_id": "<string>",
"reserved_by": "<string>",
"reserved_until": "2023-11-07T05:31:56Z",
"completed_by": "<string>",
"completed_at": "2023-11-07T05:31:56Z",
"correction": "<unknown>",
"note": "<string>"
},
"scores": [
{
"object": "<string>",
"id": "<string>",
"evaluator_id": "<string>",
"evaluator_version": 123,
"name": "<string>",
"target_type": "<string>",
"value": "<unknown>",
"source": "<string>",
"status": "<string>",
"created_at": "2023-11-07T05:31:56Z",
"request_id": "<string>",
"deployment_id": "<string>",
"thread_id": "<string>",
"target_revision": "<string>",
"experiment_id": "<string>",
"item_id": "<string>",
"passed": true,
"reasoning": "<string>",
"error": "<string>",
"author": "<string>",
"model_version_id": "<string>",
"monitor_id": "<string>",
"confidence": 123,
"probabilities": {}
}
],
"dataset_row": {
"dataset_id": "<string>",
"row_id": 123
}
}Authorizations
API key with the vk_ prefix. Create one from the dashboard.
Path Parameters
Queue id (aq_...) or name
Item id (ai_...)
Body
application/json
Exactly one score per pinned evaluator.
Show child attributes
Show child attributes
The corrected answer (any JSON, at most 64 KB).
At most 8 KB.
Dataset id or name: append the correction as an eval row
({input, output?, expected: correction, metadata: {queue, reviewer}},
or {turns, expected, metadata} for a thread) to this stream, whose
format must be unset or eval. Needs correction.
Response
The completed item, its scores and the appended row
Always "annotation_completion".
Show child attributes
Show child attributes
The human scores written (source "human", author = the reviewer).
Show child attributes
Show child attributes
The appended dataset row, when append_to_dataset was set.
Show child attributes
Show child attributes
Complete a reserved item
curl --request POST \
--url https://api.veri.studio/v1/annotation-queues/{id_or_name}/items/{item_id}/complete \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"scores": [
{
"evaluator_id": "<string>",
"value": "<unknown>"
}
],
"correction": "<unknown>",
"note": "<string>",
"append_to_dataset": "<string>"
}
'import requests
url = "https://api.veri.studio/v1/annotation-queues/{id_or_name}/items/{item_id}/complete"
payload = {
"scores": [
{
"evaluator_id": "<string>",
"value": "<unknown>"
}
],
"correction": "<unknown>",
"note": "<string>",
"append_to_dataset": "<string>"
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
scores: [{evaluator_id: '<string>', value: '<unknown>'}],
correction: '<unknown>',
note: '<string>',
append_to_dataset: '<string>'
})
};
fetch('https://api.veri.studio/v1/annotation-queues/{id_or_name}/items/{item_id}/complete', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));{
"object": "<string>",
"item": {
"object": "<string>",
"id": "<string>",
"queue_id": "<string>",
"target_type": "<string>",
"status": "<string>",
"created_at": "2023-11-07T05:31:56Z",
"request_id": "<string>",
"deployment_id": "<string>",
"thread_id": "<string>",
"target_revision": "<string>",
"experiment_id": "<string>",
"item_id": "<string>",
"reserved_by": "<string>",
"reserved_until": "2023-11-07T05:31:56Z",
"completed_by": "<string>",
"completed_at": "2023-11-07T05:31:56Z",
"correction": "<unknown>",
"note": "<string>"
},
"scores": [
{
"object": "<string>",
"id": "<string>",
"evaluator_id": "<string>",
"evaluator_version": 123,
"name": "<string>",
"target_type": "<string>",
"value": "<unknown>",
"source": "<string>",
"status": "<string>",
"created_at": "2023-11-07T05:31:56Z",
"request_id": "<string>",
"deployment_id": "<string>",
"thread_id": "<string>",
"target_revision": "<string>",
"experiment_id": "<string>",
"item_id": "<string>",
"passed": true,
"reasoning": "<string>",
"error": "<string>",
"author": "<string>",
"model_version_id": "<string>",
"monitor_id": "<string>",
"confidence": 123,
"probabilities": {}
}
],
"dataset_row": {
"dataset_id": "<string>",
"row_id": 123
}
}