Examples

dep = client.deployments.create(
    model="Qwen/Qwen2.5-0.5B-Instruct",
    source="huggingface",
    name="hello-deploy",
    gpu={"gpu_type": "L4-24GB", "gpu_count": 1},
)

try:
    dep.wait()
    print(f"Endpoint: {dep.endpoint_url}")

    response = dep.chat([{"role": "user", "content": "What is 12 + 7?"}])
    print(response["choices"][0]["message"]["content"])
finally:
    dep.stop()
    final = client.deployments.get(dep.id)
    print(f"Requests: {final.total_requests}")

# Assume job is a completed training_job from /training
job = client.training_jobs.get("<your-completed-job-id>")

assert job.status == "completed", f"Job not done: {job.status}"

dep = client.deployments.create(
    model=job.id,                # the training job ID
    source="training_job",       # not huggingface
    name=f"ft-{job.id[:8]}",
    gpu={"gpu_type": "L4-24GB", "gpu_count": 1},  # often smaller than training GPU
)

dep.wait()

# Test it
response = dep.chat([
    {"role": "user", "content": "Solve this: A train travels 60mph for 2 hours. How far?"},
])
print(response["choices"][0]["message"]["content"])

# Inspect the request log
recent = dep.requests(limit=5)
for req in recent:
    print(f"{req['latency_ms']:.0f}ms  {req['prompt_tokens']}→{req['completion_tokens']} tokens")

dep.stop()

from openai import OpenAI

# Spin up the deployment as usual
dep = client.deployments.create(
    model="meta-llama/Llama-3.1-8B-Instruct",
    source="huggingface",
    name="llama-8b",
    gpu={"gpu_type": "L4-24GB", "gpu_count": 4},  # tensor parallel; room for the 128K context
)
dep.wait()

# Swap in the OpenAI client — same shape, different base_url
oai = OpenAI(
    base_url=f"https://api.veri.studio/v1/deployments/{dep.id}",
    api_key="vk_your_api_key",
)

# All your existing OpenAI-shaped code works unchanged
def chat(prompt: str) -> str:
    r = oai.chat.completions.create(
        model=dep.name,
        messages=[{"role": "user", "content": prompt}],
        temperature=0.7,
        max_tokens=512,
    )
    return r.choices[0].message.content

print(chat("What's the capital of France?"))
print(chat("Write me a haiku about debugging."))

dep.stop()

prompts = ["What is 12 + 7?", "Write a haiku about coffee.", "Explain RLHF in one sentence."]

dep_small = client.deployments.create(
    model="Qwen/Qwen2.5-0.5B-Instruct",
    source="huggingface",
    name="qwen-small",
    gpu={"gpu_type": "L4-24GB", "gpu_count": 1},
)
dep_big = client.deployments.create(
    model="Qwen/Qwen2.5-7B-Instruct",
    source="huggingface",
    name="qwen-big",
    gpu={"gpu_type": "L4-24GB", "gpu_count": 4},
)

dep_small.wait()
dep_big.wait()

try:
    for p in prompts:
        small = dep_small.chat([{"role": "user", "content": p}])["choices"][0]["message"]["content"]
        big = dep_big.chat([{"role": "user", "content": p}])["choices"][0]["message"]["content"]
        print(f"\nPROMPT: {p}\n  SMALL: {small[:100]}\n  BIG:   {big[:100]}")
finally:
    dep_small.stop()
    dep_big.stop()