Local inference basics
This migrates the original local notebook: load one cached model, run the same text questions in Decisions and SystemOne, then run image and direct video inference. It preserves the original video's separate color and action questions.
Valen.local loads an ONNX bundle. Native Transformers inference is a separate branch in the shared runner. Both branches require an existing local directory, use CPU FP32, and disable implicit model downloads. Download the pinned model on your intended compute machine using the deployment guide before running. Native loading executes the model repository's Python code (trust_remote_code=True); use the revision recorded with the results.
Decisions returns an ordered answer list, a predicate probability, and labeled score levels. SystemOne returns an answer mapping, noul, and ordered score criteria. String candidate keys and their order are preserved. Scores are probability-weighted zero-based indices, not the text of the level label. The adapter call and both raw responses are visible below.
The direct video path uses the model's media configuration (16 uniform frames in the qualified 0.8B bundle); it differs from the one-frame HTTP recipe. It can answer an action question, but one curated clip is not a video accuracy benchmark.
Install and prepare inputs
Use Python 3.11+. Download the small source bundle, extract it, then install its dependencies:
curl -fLO https://valen-team.github.io/cookbook/valen-cookbook.zip
python -m zipfile -e valen-cookbook.zip .
cd valen-cookbook
python -m pip install -r requirements.txt
For ONNX CPU, also install the local extra. The native branch separately needs Torch/Transformers. These commands run only when you execute them, not when viewing this page.
python -m pip install 'valen-sdk[onnx,video]==0.2.0'
# Only for --backend native:
python -m pip install torch torchvision transformers
Fetch only these small assets when you choose to run (image ≈58 KB, clip ≈395 KB), or provide authorized files with the same names through --asset-dir.
curl -fL https://raw.githubusercontent.com/Valen-Team/Valen/main/assets/cookbook/sheep.jpg -o sheep.jpg
curl -fL https://raw.githubusercontent.com/Valen-Team/Valen/main/assets/cookbook/momos.mp4 -o momos.mp4
Run
python local_inference.py --backend onnx --model /path/to/Valen-0.8B-ONNX --asset-dir . --output local.json
# Alternatively, an already downloaded native model:
python local_inference.py --backend native --model /path/to/Valen-0.8B --asset-dir . --output native.json
Complete executable code
"""Local basics: both protocols and direct image/video inference. Apache-2.0."""
from common import client, emit, parser, request, samples
def main():
p = parser(__doc__)
p.set_defaults(backend="onnx")
args = p.parse_args()
if args.backend == "remote":
p.error("Use api_quickstart.py for HTTP; this tutorial requires an offline local model")
data = samples()
from valen_sdk.protocol import to_internal
body = {"input": data["ticket"]["text"], "questions": data["ticket"]["questions"], "temperature": 1.0}
# The SDK adapter preserves candidate order and maps predicate to noul.
internal = to_internal(body)
with client(args) as predict:
public_text = predict(body)
system_one_text = predict(internal)
image = request(predict, [{"role": "user", "content": [
{"type": "input_image", "image_url": str((args.asset_dir / data["product"]["image"]).resolve())},
]}], [data["product"]["question"]])
video = request(predict, [{"role": "user", "content": [
{"type": "input_video", "video_url": str((args.asset_dir / data["video"]["path"]).resolve())},
]}], [data["video"]["question"], data["video"]["action_question"]])
emit({"decisions": public_text, "system_one": system_one_text, "image": image, "video": video}, args)
if __name__ == "__main__":
main()
The shared runner and samples.json are included in the download bundle, from the same maintained source as this page.
common.py
"""Shared cookbook runner. Apache-2.0. No implicit model or media downloads."""
import argparse
import base64
from contextlib import contextmanager
import json
import math
import os
from pathlib import Path
HERE = Path(__file__).resolve().parent
def samples():
return json.loads((HERE / "samples.json").read_text())
def parser(description):
p = argparse.ArgumentParser(description=description)
p.add_argument("--backend", choices=["remote", "onnx", "native"], default="remote")
p.add_argument("--base-url", default=os.environ.get("VALEN_BASE_URL", "https://llm-proxy.openxxlab.com/v1"))
p.add_argument("--model", default=os.environ.get("VALEN_MODEL"))
p.add_argument("--threads", type=int, default=4)
p.add_argument("--asset-dir", type=Path, default=Path("."))
p.add_argument("--output", type=Path)
return p
@contextmanager
def client(args):
if args.threads < 1:
raise ValueError("--threads must be positive")
if args.backend == "native":
if not args.model or not Path(args.model).is_dir():
raise ValueError("native requires --model pointing to an existing local Valen model directory")
import torch
from transformers import AutoModel, AutoProcessor
torch.set_num_threads(args.threads)
model = AutoModel.from_pretrained(args.model, trust_remote_code=True, local_files_only=True,
dtype=torch.float32, attn_implementation="eager").float().cpu().eval()
model.backbone_autocast_dtype = None
processor = AutoProcessor.from_pretrained(args.model, trust_remote_code=True, local_files_only=True)
def predict(body):
with torch.inference_mode():
return model.predict(body, processor=processor, execution="shared_state",
temperature=body.get("temperature", 1.0))
yield predict
else:
from valen_sdk import Valen
if args.backend == "onnx":
if not args.model or not Path(args.model).is_dir():
raise ValueError("onnx requires --model pointing to an existing local ONNX bundle")
instance = Valen.local(args.model, variant="fp32", local_files_only=True, num_threads=args.threads)
else:
instance = Valen.remote(args.base_url, model=args.model or "valen-2b",
api_key=os.environ.get("VALEN_API_KEY"), max_retries=0)
with instance:
yield instance.predict
def image_input(payload):
"""Inline media works across HTTP services; never send a client-local path."""
return [{"role": "user", "content": [
{"type": "input_text", "text": "Answer using only the visible image."},
{"type": "input_image", "image_url": "data:image/jpeg;base64," + base64.b64encode(payload).decode("ascii")},
]}]
def image_bytes(path):
import io
from PIL import Image, ImageOps
with Image.open(path) as image:
image = ImageOps.exif_transpose(image).convert("RGB")
image.thumbnail((512, 512))
buffer = io.BytesIO()
image.save(buffer, format="JPEG", quality=95)
return buffer.getvalue()
def middle_frame(path):
"""Decode a bounded clip twice: count first, then keep only its middle frame."""
import av
import io
from PIL import ImageOps
with av.open(str(path)) as video:
stream = video.streams.video[0]
if stream.width * stream.height > 4096 * 2160:
raise ValueError("Clip exceeds the cookbook's 4K frame limit")
count = 0
for _ in video.decode(video=0):
count += 1
if count > 1800:
raise ValueError("Clip exceeds the cookbook's 1800-frame limit")
if not count:
raise ValueError("Clip contains no decodable frames")
target = count // 2
with av.open(str(path)) as video:
for index, frame in enumerate(video.decode(video=0)):
if index == target:
image = ImageOps.exif_transpose(frame.to_image()).convert("RGB")
image.thumbnail((512, 512))
buffer = io.BytesIO()
image.save(buffer, format="JPEG", quality=95)
return buffer.getvalue(), {"frame_index": target, "total_frames": count,
"timestamp_seconds": float(frame.time) if frame.time is not None else None}
raise ValueError("Clip changed while decoding")
def request(predict, source, questions):
response = predict({"input": source, "questions": questions, "temperature": 1.0})
answers = response["answers"]
if [a["name"] for a in answers] != [q["name"] for q in questions]:
raise ValueError("Service returned missing or reordered answers")
return {a["name"]: a for a in answers}
def choice_status(answer, *, threshold=0.8, margin=0.2):
probabilities = [float(x["probability"]) for x in answer["probabilities"]]
if (not probabilities or any(not math.isfinite(x) or not 0 <= x <= 1 for x in probabilities)
or abs(sum(probabilities) - 1) > 1e-4):
raise ValueError("Invalid choice probability distribution")
selected = next(x for x in answer["probabilities"]
if type(x["value"]) is type(answer["choice"]) and x["value"] == answer["choice"])
ordered = sorted(probabilities, reverse=True)
confidence = float(selected["probability"])
gap = ordered[0] - (ordered[1] if len(ordered) > 1 else 0)
return confidence, confidence >= threshold and gap >= margin
def emit(value, args):
text = json.dumps(value, indent=2, ensure_ascii=False, allow_nan=False) + "\n"
if args.output:
args.output.write_text(text)
print(text, end="")
Real validation results
Real-model verification: 2026-10-26 · Valen-0.8B / ONNX FP32 · SDK 0.2.0 · ONNX Runtime 1.31.0 / CPU direct local.
Source model revision: a0a10112716f54457bad6f60b792ad334f9e89fd. Machine-readable results
{
"decisions": {
"model": "Valen-0.8B",
"answers": [
{
"name": "intent",
"type": "choice",
"probabilities": [
{
"value": "billing_question",
"probability": 0.0000014444915728188846
},
{
"value": "cancel_order",
"probability": 0.00013284262147016516
},
{
"value": "change_address",
"probability": 0.9997590278141875
},
{
"value": "report_damage",
"probability": 0.000004776970793180753
},
{
"value": "track_order",
"probability": 0.00010190810197642957
}
],
"confidence": 0.9996987847677343,
"choice": "change_address"
},
{
"name": "address_change",
"type": "predicate",
"probability": 0.9862781119951404
},
{
"name": "explicitness",
"type": "score",
"probabilities": [
{
"label": "none",
"value": 0,
"probability": 0.015131870978155534
},
{
"label": "possible",
"value": 1,
"probability": 0.1913538155432454
},
{
"label": "explicit",
"value": 2,
"probability": 0.7935143134785991
}
],
"confidence": 0.6675736637506653,
"score": 1.7783824425004437
}
],
"usage": {
"input_tokens": 225,
"output_tokens": 0
},
"internal_usage": {
"compute_tokens": 225
}
},
"system_one": {
"model": "Valen-0.8B",
"answers": {
"intent": {
"type": "choice",
"probabilities": {
"billing_question": 0.0000014444915728188846,
"cancel_order": 0.00013284262147016516,
"change_address": 0.9997590278141875,
"report_damage": 0.000004776970793180753,
"track_order": 0.00010190810197642957
},
"choice": "change_address",
"confidence": 0.9996987847677343
},
"address_change": {
"type": "noul",
"noul": 0.9862781119951404
},
"explicitness": {
"type": "score",
"probabilities": {
"0": 0.015131870978155534,
"1": 0.1913538155432454,
"2": 0.7935143134785991
},
"score": 1.7783824425004437,
"legend": {
"0": "No request",
"1": "Asks whether a change is possible",
"2": "Explicitly requests a change"
},
"confidence": 0.6675736637506653
}
},
"usage": {
"input_tokens": 225,
"output_tokens": 0
},
"internal_usage": {
"compute_tokens": 225
}
},
"image": {
"group": {
"name": "group",
"type": "choice",
"probabilities": [
{
"value": "pride",
"probability": 0.0006759448829023258
},
{
"value": "flock",
"probability": 0.9923460443802665
},
{
"value": "clowder",
"probability": 0.0058704688188683804
},
{
"value": "gang",
"probability": 0.001107541917962753
}
],
"confidence": 0.9897947258403553,
"choice": "flock"
}
},
"video": {
"sauce_color": {
"name": "sauce_color",
"type": "choice",
"probabilities": [
{
"value": "brown",
"probability": 0.039694426657567713
},
{
"value": "white",
"probability": 0.9325758148359371
},
{
"value": "red",
"probability": 0.01581158949841234
},
{
"value": "orange",
"probability": 0.011918169008082881
}
],
"confidence": 0.9101010864479161,
"choice": "white"
},
"action": {
"name": "action",
"type": "choice",
"probabilities": [
{
"value": "A",
"probability": 0.9996676220259049
},
{
"value": "B",
"probability": 0.00006066860086765039
},
{
"value": "C",
"probability": 0.0002527517367929761
},
{
"value": "D",
"probability": 0.000018957636434528354
}
],
"confidence": 0.9995568293678732,
"choice": "A"
}
}
}
Results belong to the recorded model and backend; they do not imply identical versions, scores or speed on the default public service. Policy unit tests use constructed answers. Real-model execution is recorded separately and is not replaced by those tests.
Sources, licensing and related documentation
Code and authored JSON text samples use Apache-2.0. The ticket is fictional. sheep.jpg and momos.mp4 reuse the original notebooks' evaluation excerpts and stable asset URLs. The old directory did not record their original dataset attribution or a separate media license; this page does not grant redistribution or commercial media rights. Use your own authorized media for production or redistribution. The code license does not relicense the media, and this migration creates no additional media copies.
Decisions API · SystemOne API · ONNX / SDK · Native deployment · GGUF deployment · Evaluation · Validation and performance