本地推理基础

本教程迁移旧本地 Notebook：加载一个缓存模型，用 Decisions 和 SystemOne 运行相同文本问题，再执行图片与直接视频推理。保留原视频独有的颜色和动作两道题。

Valen.local 加载的是 ONNX 模型包。原版 Transformers 使用共享运行器中的另一个分支。两个分支均要求已有本地模型目录，采用 CPU FP32，并禁止隐式下载模型。请先按部署文档在实际计算机器上下载固定 revision。原版加载会执行模型仓库 Python 代码（trust_remote_code=True），应使用结果记录中的 revision。

Decisions 返回有序答案列表、predicate 概率与带标签的评分等级；SystemOne 返回答案字典、noul 与有序评分标准。字符串候选键及顺序保持一致。score 是从零开始的等级索引的概率加权平均，不是 label 文本中的数字。下方代码及实际结果展示了映射与两种原始响应。

直接视频使用模型的媒体配置，已验证的 0.8B 包采用 16 帧均匀采样，与单帧 HTTP 教程不同。它可以回答动作问题，但一段精选视频不能作为视频准确率基准。

安装与输入

Python 3.11+。先下载小型源文件包并解压，然后安装所需依赖：

Bash

curl -fLO https://valen-team.github.io/cookbook/valen-cookbook.zip
python -m zipfile -e valen-cookbook.zip .
cd valen-cookbook
python -m pip install -r requirements.txt

ONNX CPU 还需安装本地 extra；原版分支另需 Torch/Transformers。这里只展示安装命令，不会在网页访问时下载任何依赖或模型。

Bash

python -m pip install 'valen-sdk[onnx,video]==0.2.0'
# Only for --backend native:
python -m pip install torch torchvision transformers

明确执行后才获取以下小素材（图片约 58 KB，视频约 395 KB）；也可将有权使用的同名文件放在 --asset-dir 指定目录。

Bash

curl -fL https://raw.githubusercontent.com/Valen-Team/Valen/main/assets/cookbook/sheep.jpg -o sheep.jpg
curl -fL https://raw.githubusercontent.com/Valen-Team/Valen/main/assets/cookbook/momos.mp4 -o momos.mp4

运行

Bash

python local_inference.py --backend onnx --model /path/to/Valen-0.8B-ONNX --asset-dir . --output local.json
# Alternatively, an already downloaded native model:
python local_inference.py --backend native --model /path/to/Valen-0.8B --asset-dir . --output native.json

完整可执行代码

local_inference.py

"""Local basics: both protocols and direct image/video inference. Apache-2.0."""
from common import client, emit, parser, request, samples


def main():
    p = parser(__doc__)
    p.set_defaults(backend="onnx")
    args = p.parse_args()
    if args.backend == "remote":
        p.error("Use api_quickstart.py for HTTP; this tutorial requires an offline local model")
    data = samples()
    from valen_sdk.protocol import to_internal
    body = {"input": data["ticket"]["text"], "questions": data["ticket"]["questions"], "temperature": 1.0}
    # The SDK adapter preserves candidate order and maps predicate to noul.
    internal = to_internal(body)
    with client(args) as predict:
        public_text = predict(body)
        system_one_text = predict(internal)
        image = request(predict, [{"role": "user", "content": [
            {"type": "input_image", "image_url": str((args.asset_dir / data["product"]["image"]).resolve())},
        ]}], [data["product"]["question"]])
        video = request(predict, [{"role": "user", "content": [
            {"type": "input_video", "video_url": str((args.asset_dir / data["video"]["path"]).resolve())},
        ]}], [data["video"]["question"], data["video"]["action_question"]])
        emit({"decisions": public_text, "system_one": system_one_text, "image": image, "video": video}, args)


if __name__ == "__main__":
    main()

共享运行器与 samples.json 已包含在下载包中，与网页展示内容使用同一个维护源。

common.py

common.py

"""Shared cookbook runner. Apache-2.0. No implicit model or media downloads."""
import argparse
import base64
from contextlib import contextmanager
import json
import math
import os
from pathlib import Path

HERE = Path(__file__).resolve().parent


def samples():
    return json.loads((HERE / "samples.json").read_text())


def parser(description):
    p = argparse.ArgumentParser(description=description)
    p.add_argument("--backend", choices=["remote", "onnx", "native"], default="remote")
    p.add_argument("--base-url", default=os.environ.get("VALEN_BASE_URL", "https://llm-proxy.openxxlab.com/v1"))
    p.add_argument("--model", default=os.environ.get("VALEN_MODEL"))
    p.add_argument("--threads", type=int, default=4)
    p.add_argument("--asset-dir", type=Path, default=Path("."))
    p.add_argument("--output", type=Path)
    return p


@contextmanager
def client(args):
    if args.threads < 1:
        raise ValueError("--threads must be positive")
    if args.backend == "native":
        if not args.model or not Path(args.model).is_dir():
            raise ValueError("native requires --model pointing to an existing local Valen model directory")
        import torch
        from transformers import AutoModel, AutoProcessor
        torch.set_num_threads(args.threads)
        model = AutoModel.from_pretrained(args.model, trust_remote_code=True, local_files_only=True,
                                          dtype=torch.float32, attn_implementation="eager").float().cpu().eval()
        model.backbone_autocast_dtype = None
        processor = AutoProcessor.from_pretrained(args.model, trust_remote_code=True, local_files_only=True)

        def predict(body):
            with torch.inference_mode():
                return model.predict(body, processor=processor, execution="shared_state",
                                     temperature=body.get("temperature", 1.0))
        yield predict
    else:
        from valen_sdk import Valen
        if args.backend == "onnx":
            if not args.model or not Path(args.model).is_dir():
                raise ValueError("onnx requires --model pointing to an existing local ONNX bundle")
            instance = Valen.local(args.model, variant="fp32", local_files_only=True, num_threads=args.threads)
        else:
            instance = Valen.remote(args.base_url, model=args.model or "valen-2b",
                                    api_key=os.environ.get("VALEN_API_KEY"), max_retries=0)
        with instance:
            yield instance.predict


def image_input(payload):
    """Inline media works across HTTP services; never send a client-local path."""
    return [{"role": "user", "content": [
        {"type": "input_text", "text": "Answer using only the visible image."},
        {"type": "input_image", "image_url": "data:image/jpeg;base64," + base64.b64encode(payload).decode("ascii")},
    ]}]


def image_bytes(path):
    import io
    from PIL import Image, ImageOps
    with Image.open(path) as image:
        image = ImageOps.exif_transpose(image).convert("RGB")
        image.thumbnail((512, 512))
        buffer = io.BytesIO()
        image.save(buffer, format="JPEG", quality=95)
        return buffer.getvalue()


def middle_frame(path):
    """Decode a bounded clip twice: count first, then keep only its middle frame."""
    import av
    import io
    from PIL import ImageOps
    with av.open(str(path)) as video:
        stream = video.streams.video[0]
        if stream.width * stream.height > 4096 * 2160:
            raise ValueError("Clip exceeds the cookbook's 4K frame limit")
        count = 0
        for _ in video.decode(video=0):
            count += 1
            if count > 1800:
                raise ValueError("Clip exceeds the cookbook's 1800-frame limit")
    if not count:
        raise ValueError("Clip contains no decodable frames")
    target = count // 2
    with av.open(str(path)) as video:
        for index, frame in enumerate(video.decode(video=0)):
            if index == target:
                image = ImageOps.exif_transpose(frame.to_image()).convert("RGB")
                image.thumbnail((512, 512))
                buffer = io.BytesIO()
                image.save(buffer, format="JPEG", quality=95)
                return buffer.getvalue(), {"frame_index": target, "total_frames": count,
                                            "timestamp_seconds": float(frame.time) if frame.time is not None else None}
    raise ValueError("Clip changed while decoding")


def request(predict, source, questions):
    response = predict({"input": source, "questions": questions, "temperature": 1.0})
    answers = response["answers"]
    if [a["name"] for a in answers] != [q["name"] for q in questions]:
        raise ValueError("Service returned missing or reordered answers")
    return {a["name"]: a for a in answers}


def choice_status(answer, *, threshold=0.8, margin=0.2):
    probabilities = [float(x["probability"]) for x in answer["probabilities"]]
    if (not probabilities or any(not math.isfinite(x) or not 0 <= x <= 1 for x in probabilities)
            or abs(sum(probabilities) - 1) > 1e-4):
        raise ValueError("Invalid choice probability distribution")
    selected = next(x for x in answer["probabilities"]
                    if type(x["value"]) is type(answer["choice"]) and x["value"] == answer["choice"])
    ordered = sorted(probabilities, reverse=True)
    confidence = float(selected["probability"])
    gap = ordered[0] - (ordered[1] if len(ordered) > 1 else 0)
    return confidence, confidence >= threshold and gap >= margin


def emit(value, args):
    text = json.dumps(value, indent=2, ensure_ascii=False, allow_nan=False) + "\n"
    if args.output:
        args.output.write_text(text)
    print(text, end="")

真实验证结果

真实模型验证: 2026-10-26 · Valen-0.8B / ONNX FP32 · SDK 0.2.0 · ONNX Runtime 1.31.0 / CPU direct local.

源模型 revision: a0a10112716f54457bad6f60b792ad334f9e89fd. 机器可读结果

实际业务输出

{
  "decisions": {
    "model": "Valen-0.8B",
    "answers": [
      {
        "name": "intent",
        "type": "choice",
        "probabilities": [
          {
            "value": "billing_question",
            "probability": 0.0000014444915728188846
          },
          {
            "value": "cancel_order",
            "probability": 0.00013284262147016516
          },
          {
            "value": "change_address",
            "probability": 0.9997590278141875
          },
          {
            "value": "report_damage",
            "probability": 0.000004776970793180753
          },
          {
            "value": "track_order",
            "probability": 0.00010190810197642957
          }
        ],
        "confidence": 0.9996987847677343,
        "choice": "change_address"
      },
      {
        "name": "address_change",
        "type": "predicate",
        "probability": 0.9862781119951404
      },
      {
        "name": "explicitness",
        "type": "score",
        "probabilities": [
          {
            "label": "none",
            "value": 0,
            "probability": 0.015131870978155534
          },
          {
            "label": "possible",
            "value": 1,
            "probability": 0.1913538155432454
          },
          {
            "label": "explicit",
            "value": 2,
            "probability": 0.7935143134785991
          }
        ],
        "confidence": 0.6675736637506653,
        "score": 1.7783824425004437
      }
    ],
    "usage": {
      "input_tokens": 225,
      "output_tokens": 0
    },
    "internal_usage": {
      "compute_tokens": 225
    }
  },
  "system_one": {
    "model": "Valen-0.8B",
    "answers": {
      "intent": {
        "type": "choice",
        "probabilities": {
          "billing_question": 0.0000014444915728188846,
          "cancel_order": 0.00013284262147016516,
          "change_address": 0.9997590278141875,
          "report_damage": 0.000004776970793180753,
          "track_order": 0.00010190810197642957
        },
        "choice": "change_address",
        "confidence": 0.9996987847677343
      },
      "address_change": {
        "type": "noul",
        "noul": 0.9862781119951404
      },
      "explicitness": {
        "type": "score",
        "probabilities": {
          "0": 0.015131870978155534,
          "1": 0.1913538155432454,
          "2": 0.7935143134785991
        },
        "score": 1.7783824425004437,
        "legend": {
          "0": "No request",
          "1": "Asks whether a change is possible",
          "2": "Explicitly requests a change"
        },
        "confidence": 0.6675736637506653
      }
    },
    "usage": {
      "input_tokens": 225,
      "output_tokens": 0
    },
    "internal_usage": {
      "compute_tokens": 225
    }
  },
  "image": {
    "group": {
      "name": "group",
      "type": "choice",
      "probabilities": [
        {
          "value": "pride",
          "probability": 0.0006759448829023258
        },
        {
          "value": "flock",
          "probability": 0.9923460443802665
        },
        {
          "value": "clowder",
          "probability": 0.0058704688188683804
        },
        {
          "value": "gang",
          "probability": 0.001107541917962753
        }
      ],
      "confidence": 0.9897947258403553,
      "choice": "flock"
    }
  },
  "video": {
    "sauce_color": {
      "name": "sauce_color",
      "type": "choice",
      "probabilities": [
        {
          "value": "brown",
          "probability": 0.039694426657567713
        },
        {
          "value": "white",
          "probability": 0.9325758148359371
        },
        {
          "value": "red",
          "probability": 0.01581158949841234
        },
        {
          "value": "orange",
          "probability": 0.011918169008082881
        }
      ],
      "confidence": 0.9101010864479161,
      "choice": "white"
    },
    "action": {
      "name": "action",
      "type": "choice",
      "probabilities": [
        {
          "value": "A",
          "probability": 0.9996676220259049
        },
        {
          "value": "B",
          "probability": 0.00006066860086765039
        },
        {
          "value": "C",
          "probability": 0.0002527517367929761
        },
        {
          "value": "D",
          "probability": 0.000018957636434528354
        }
      ],
      "confidence": 0.9995568293678732,
      "choice": "A"
    }
  }
}

这里记录的是指定版本与后端的实测，不意味着默认公共服务的模型版本、分数或速度完全相同。业务阈值单元测试使用构造响应，真实模型执行单独记录，二者不互相替代。

素材、许可与相关文档

代码与作者编写的 JSON 文本样例采用 Apache-2.0。工单为虚构示例。sheep.jpg 与 momos.mp4 沿用旧 Notebook 的评测片段和稳定素材 URL；旧目录未提供原始数据集归属及单独的媒体许可记录，因此这里不授予媒体再分发或商业使用许可。生产或再分发时请替换为有权使用的素材，勿把代码许可视为媒体许可。官网不额外复制这些媒体文件。

Decisions API · SystemOne API · ONNX / SDK · Native deployment · GGUF deployment · Evaluation · Validation and performance