Reserve the next item
Returns the caller’s own live reservation in this queue if they hold one, else reserves the oldest pending item (or an expired reservation) for the queue’s reservation_minutes; two reviewers never hold the same item. The response carries what a reviewer needs: the item, the queue instructions, the pinned evaluator specs, and the context (trace items: request metadata, the trace body and transcript; thread items: the thread’s request ids and the latest trace’s transcript; experiment items: input, output, expected and transcript). 204 when nothing is left to review.
curl --request POST \
--url https://api.veri.studio/v1/annotation-queues/{id_or_name}/items/next \
--header 'Authorization: Bearer <token>'import requests
url = "https://api.veri.studio/v1/annotation-queues/{id_or_name}/items/next"
headers = {"Authorization": "Bearer <token>"}
response = requests.post(url, headers=headers)
print(response.text)const options = {method: 'POST', headers: {Authorization: 'Bearer <token>'}};
fetch('https://api.veri.studio/v1/annotation-queues/{id_or_name}/items/next', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));{
"object": "<string>",
"item": {
"object": "<string>",
"id": "<string>",
"queue_id": "<string>",
"target_type": "<string>",
"status": "<string>",
"created_at": "2023-11-07T05:31:56Z",
"request_id": "<string>",
"deployment_id": "<string>",
"thread_id": "<string>",
"target_revision": "<string>",
"experiment_id": "<string>",
"item_id": "<string>",
"reserved_by": "<string>",
"reserved_until": "2023-11-07T05:31:56Z",
"completed_by": "<string>",
"completed_at": "2023-11-07T05:31:56Z",
"correction": "<unknown>",
"note": "<string>"
},
"queue": {
"id": "<string>",
"name": "<string>",
"target_type": "<string>",
"reservation_minutes": 123,
"instructions": "<string>"
},
"evaluators": [
{
"evaluator_id": "<string>",
"name": "<string>",
"version": 123,
"method": "<string>",
"scope": "<string>",
"output_type": "<string>",
"output_config": {},
"description": "<string>",
"instructions": "<string>"
}
],
"context": {
"request": {
"id": "<string>",
"deployment_id": "<string>",
"created_at": "2023-11-07T05:31:56Z",
"status_code": 123,
"latency_ms": 123,
"prompt_tokens": 123,
"completion_tokens": 123,
"model_version_id": "<string>",
"thread_id": "<string>",
"finish_reason": "<string>"
},
"request_ids": [
"<string>"
],
"trace": {},
"trace_error": "<string>",
"transcript": [
{}
],
"experiment_item": {
"experiment_id": "<string>",
"experiment_name": "<string>",
"row_id": 123,
"trial": 123,
"kind": "<string>",
"status": "<string>",
"turn": 123,
"turns": 123,
"input": "<unknown>",
"output": "<unknown>",
"expected": "<unknown>",
"metadata": "<unknown>",
"transcript": "<unknown>"
}
}
}Authorizations
API key with the vk_ prefix. Create one from the dashboard.
Path Parameters
Queue id (aq_...) or name
Response
The reserved item and its review context
A reserved item and everything needed to review it.
Always "annotation_task".
Show child attributes
Show child attributes
The queue facts a reviewer needs next to an item.
Show child attributes
Show child attributes
Score each of these (POST .../complete takes one score per entry).
Show child attributes
Show child attributes
Everything a reviewer needs to judge one item. Trace items carry
request, trace and transcript; thread items add the thread's
request_ids (the trace and request are the revision's, whose messages
hold the whole conversation); experiment items carry experiment_item.
Show child attributes
Show child attributes
curl --request POST \
--url https://api.veri.studio/v1/annotation-queues/{id_or_name}/items/next \
--header 'Authorization: Bearer <token>'import requests
url = "https://api.veri.studio/v1/annotation-queues/{id_or_name}/items/next"
headers = {"Authorization": "Bearer <token>"}
response = requests.post(url, headers=headers)
print(response.text)const options = {method: 'POST', headers: {Authorization: 'Bearer <token>'}};
fetch('https://api.veri.studio/v1/annotation-queues/{id_or_name}/items/next', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));{
"object": "<string>",
"item": {
"object": "<string>",
"id": "<string>",
"queue_id": "<string>",
"target_type": "<string>",
"status": "<string>",
"created_at": "2023-11-07T05:31:56Z",
"request_id": "<string>",
"deployment_id": "<string>",
"thread_id": "<string>",
"target_revision": "<string>",
"experiment_id": "<string>",
"item_id": "<string>",
"reserved_by": "<string>",
"reserved_until": "2023-11-07T05:31:56Z",
"completed_by": "<string>",
"completed_at": "2023-11-07T05:31:56Z",
"correction": "<unknown>",
"note": "<string>"
},
"queue": {
"id": "<string>",
"name": "<string>",
"target_type": "<string>",
"reservation_minutes": 123,
"instructions": "<string>"
},
"evaluators": [
{
"evaluator_id": "<string>",
"name": "<string>",
"version": 123,
"method": "<string>",
"scope": "<string>",
"output_type": "<string>",
"output_config": {},
"description": "<string>",
"instructions": "<string>"
}
],
"context": {
"request": {
"id": "<string>",
"deployment_id": "<string>",
"created_at": "2023-11-07T05:31:56Z",
"status_code": 123,
"latency_ms": 123,
"prompt_tokens": 123,
"completion_tokens": 123,
"model_version_id": "<string>",
"thread_id": "<string>",
"finish_reason": "<string>"
},
"request_ids": [
"<string>"
],
"trace": {},
"trace_error": "<string>",
"transcript": [
{}
],
"experiment_item": {
"experiment_id": "<string>",
"experiment_name": "<string>",
"row_id": 123,
"trial": 123,
"kind": "<string>",
"status": "<string>",
"turn": 123,
"turns": 123,
"input": "<unknown>",
"output": "<unknown>",
"expected": "<unknown>",
"metadata": "<unknown>",
"transcript": "<unknown>"
}
}
}