OMLX 面试题库

graph LR
    subgraph OLLAMA["原生 Ollama（单节点）"]
        OC["Ollama Client"]
        OAPI["Ollama API Server<br/>+ Runner<br/>+ Model Manager"]
        OBACK["llama.cpp Backend"]
        OGPU["单 GPU / CPU"]
        OC --> OAPI --> OBACK --> OGPU
    end

    subgraph OMLX["OMLX（企业级平台）"]
        MC["Ollama Client / SDK"]
        MS["OMLX Server（API 兼容层）"]
        MG["OMLX Gateway（路由 / 负载均衡）"]
        MCTRL["Controller（调度 / 编排）"]
        MW["Worker x N（多后端推理节点）"]
        MBACK1["llama.cpp"]
        MBACK2["vLLM"]
        MBACK3["SGLang"]
        MBACK4["TensorRT-LLM"]
        MGPU["多 GPU / 多节点 / K8s 集群"]
        MC --> MS --> MG --> MCTRL --> MW
        MW --> MBACK1
        MW --> MBACK2
        MW --> MBACK3
        MW --> MBACK4
        MW --> MGPU
    end

graph TD
    Client["Client Request<br/>（POST /api/generate）"]
    Server["Server（API 兼容层）<br/>解析请求 → 提取 model / prompt / parameters<br/>验证 API Key → 查询 Gateway 路由表"]
    Gateway["Gateway（路由层）<br/>根据 model 名查找可用 Worker 列表<br/>负载均衡选择一个 Worker<br/>会话保持：将同一会话路由至同一 Worker"]
    Controller["Controller（调度层）<br/>Worker 健康状态跟踪<br/>模型加载 / 卸载决策<br/>将路由决策下发至 Gateway"]
    Worker["Worker（推理层）<br/>检查模型是否已加载到显存<br/>若未加载：从模型仓库拉取并加载<br/>执行推理 → 返回 token 流<br/>管理显存：Keep-Alive 计时 / Idle Unload"]
    Response["Server ← Gateway → Client<br/>（流式返回 SSE）"]

    Client --> Server
    Server --> Gateway
    Gateway --> Controller
    Controller --> Worker
    Worker --> Response

# Generate 请求（完全兼容 Ollama 客户端）
curl http://omlx-server:11434/api/generate -d '{
  "model": "qwen2.5:7b",
  "prompt": "Why is the sky blue?",
  "stream": false,
  "options": {
    "temperature": 0.7,
    "num_predict": 256
  }
}'

# Chat 请求
curl http://omlx-server:11434/api/chat -d '{
  "model": "qwen2.5:7b",
  "messages": [
    {"role": "system", "content": "You are a helpful assistant."},
    {"role": "user", "content": "Hello!"}
  ],
  "stream": true
}'

# Embedding 请求
curl http://omlx-server:11434/api/embeddings -d '{
  "model": "nomic-embed-text",
  "prompt": "Hello world"
}'

graph TD
    GW["Gateway<br/>请求路由器<br/>Model -> Worker 映射表"]
    GW --> WA["Worker A"]
    GW --> WB["Worker B"]
    subgraph WA["Worker A"]
        QW1["qwen2.5:7b (显存 6GB)"]
        NE1["nomic-embed (显存 2GB)"]
    end
    subgraph WB["Worker B"]
        LL["llama3:8b (显存 5GB)"]
        CL["codellama:7b (显存 5GB)"]
    end

graph TD
    Gateway["Gateway 收到 /api/chat model=qwen2.5:7b"]
    Query["查询 Model → Worker 映射表"]
    List["qwen2.5:7b → [Worker-A, Worker-B]"]
    LB["负载均衡选择 Worker-A"]
    Check["Worker-A 检查显存中是否已加载 qwen2.5:7b"]
    Loaded["已加载：直接推理，重置 Keep-Alive 计时器"]
    NotLoaded{"未加载"}
    VRAM_OK["显存充足：加载模型 → 推理"]
    VRAM_FULL["显存不足：LRU 卸载最久未使用模型 → 加载 → 推理"]

    Gateway --> Query
    Query --> List
    List --> LB
    LB --> Check
    Check -->|"已加载"| Loaded
    Check -->|"未加载"| NotLoaded
    NotLoaded -->|"显存充足"| VRAM_OK
    NotLoaded -->|"显存不足"| VRAM_FULL

OMLX 模型管理
├── 模型存储层
│   ├── 本地存储：/var/lib/omlx/models/
│   │   ├── blobs/         # 模型层文件（GGUF / safetensors）
│   │   └── manifests/     # OCI 清单
│   └── 远程存储：OCI Registry（Harbor / Docker Hub / GHCR）
│       ├── 模型层（Layer）
│       └── 模型配置（Config）
├── 分发协议
│   ├── OCI Distribution Spec v1.1
│   ├── 支持 Registry 认证（Bearer Token / Basic Auth）
│   └── 支持分层拉取（断点续传 / 并行下载）
└── 模型索引
    ├── 模型名 → 标签 → Digest 映射
    └── GGUF 文件 → 参数映射（量化级别 / 上下文长度）

graph TD
    Client["Client → POST /api/pull model: qwen2.5:7b"]
    Server["OMLX Server → 解析模型名 → 查询 Registry"]
    Manifest["Registry 返回 Manifest（Layer Digest 列表）"]
    Cache{"OMLX Server → 检查本地缓存（blobs 目录）"}
    Cached["已缓存：跳过下载"]
    NotCached["未缓存：并行下载 Layer → 校验 SHA256 Digest"]
    Extract["解压 / 放置 GGUF 文件到模型目录"]
    Done["更新本地模型索引 → 返回成功"]

    Client --> Server
    Server --> Manifest
    Manifest --> Cache
    Cache -->|"已缓存"| Cached
    Cache -->|"未缓存"| NotCached
    Cached --> Extract
    NotCached --> Extract
    Extract --> Done

{registry}/{namespace}/{model}:{tag}
├── library/qwen2.5:7b          → 默认 7B-Q4_K_M 量化
├── library/qwen2.5:7b-q8_0     → Q8_0 量化版本
├── library/qwen2.5:7b-fp16     → FP16 全精度
└── library/qwen2.5:7b-instruct → 指令微调版本

# Pull 指定量化版本
curl http://omlx:11434/api/pull -d '{"model": "qwen2.5:7b-q4_k_m"}'

# 或在 Modelfile 中指定
FROM qwen2.5:7b-q8_0

# OMLX 扩展：列出模型的可用量化版本
curl http://omlx:11434/api/show -d '{"model": "qwen2.5:7b", "verbose": true}'

graph TD
    Remote["远程层（OCI Registry）<br/>模型层文件（GGUF / safetensors）<br/>OCI Manifest"]
    Shared["共享缓存层（可选，如 MinIO / S3 / CephFS）<br/>集群内共享模型文件<br/>避免每个 Worker 重复下载相同模型"]
    Local["本地缓存层（Worker 节点本地）<br/>blobs：OCI Layer 缓存<br/>models：已解压模型<br/>metadata：模型元数据"]
    VRAM["显存层（GPU VRAM）<br/>已加载模型（活跃服务中）<br/>Keep-Alive 计时器中"]

    Remote --> Shared
    Shared --> Local
    Local --> VRAM

# 列出模型占用空间
ollama list --omlx-stats
# 输出示例：
# qwen2.5:7b-q4_k_m    4.5 GB  (磁盘)  +  5.8 GB  (显存)  Worker: node-gpu-01

# 清理未使用模型（仅删除磁盘缓存，不影响显存中的活跃模型）
curl -X DELETE http://omlx:11434/api/delete -d '{"model": "unused-model"}'

# OMLX 扩展：批量清理过期缓存
curl -X POST http://omlx:11434/api/omlx/cache/prune -d '{"older_than": "7d"}'

Worker-A 首次加载 qwen2.5:7b
  → 检查本地缓存：未命中
  → 检查共享缓存（MinIO）：命中
  → 从共享缓存读取 GGUF 文件
  → 加载到显存
  → 注册到 Controller

Worker-B 请求相同模型
  → 检查本地缓存：未命中
  → 检查共享缓存（MinIO）：命中
  → 无需重复 Pull，直接加载
  → 加载到显存

# Worker 配置
gpu:
  devices: [0, 1, 2, 3]         # 使用的 GPU 设备列表
  tensor_parallel_size: 2        # 张量并行度（每 2 张 GPU 一组）
  pipeline_parallel_size: 1      # 流水线并行度
  data_parallel_size: 2          # 数据并行副本数（2 组张量并行组）
  # 以上配置对 4 张 GPU：TP=2 × DP=2 → 可同时处理 2 个独立请求

Worker（4 × GPU）
├── GPU 0：qwen2.5:7b 完整副本 → 处理 Req-1, Req-4
├── GPU 1：qwen2.5:7b 完整副本 → 处理 Req-2, Req-5
├── GPU 2：qwen2.5:7b 完整副本 → 处理 Req-3, Req-6
└── GPU 3：qwen2.5:7b 完整副本 → 处理 Req-7

Worker（2 × GPU，TP=2）
├── GPU 0：qwen2.5:72b Layer 0-39（前半）
└── GPU 1：qwen2.5:72b Layer 40-79（后半）
        
单请求 → 同时使用 GPU 0 + GPU 1
GPU 0 计算前半层输出 → NVLink/PCIe → GPU 1 继续计算

worker:
  gpu_selector:
    strategy: "topology"         # 就近 NUMA 节点匹配
    prefer_nvlink: true          # 优先选择 NVLink 互联的 GPU 对
    max_gpu_per_model: 4         # 单模型最大使用 GPU 数量

模型加载到显存
  │
  ├── Active（活跃）：有请求正在推理
  │     └── 每个请求完成后重置 Keep-Alive 计时器
  │
  ├── Keep-Alive（保持期）：最近一次请求完成起计时
  │     ├── 计时器未到期：模型保持显存，可立即响应新请求
  │     └── 计时器到期 → Idle 状态
  │
  └── Idle（空闲）：Keep-Alive 到期，尚未卸载
        ├── 有新请求到达 → 无需重新加载，响应延迟 ≈ 0
        └── 显存不足，需要加载新模型 → 卸载此模型 → 释放显存

graph TD
    Request["新模型加载请求到达 Worker"]
    Calc["计算所需显存 = 模型权重 + KV Cache 预分配"]
    Check{"当前可用显存 >= 所需显存？"}
    Load["直接加载"]
    NeedFree{"需要释放显存"}
    HasIdle{"有 Idle 模型？"}
    LRU["LRU 卸载 Idle 模型 → 加载新模型"]
    OOM["返回错误（GPU OOM）"]
    HasKA{"有 Keep-Alive 模型且紧急（priority=10）？"}
    UnloadKA["卸载 Keep-Alive 模型 → 加载新模型"]

    Request --> Calc
    Calc --> Check
    Check -->|"是"| Load
    Check -->|"否"| NeedFree
    NeedFree --> HasIdle
    HasIdle -->|"是"| LRU
    HasIdle -->|"否"| OOM
    HasIdle -->|"否"| HasKA
    HasKA -->|"是"| UnloadKA
    HasKA -->|"否"| OOM
    LRU --> Load
    UnloadKA --> Load

# 请求中指定 Keep-Alive
curl http://omlx:11434/api/generate -d '{
  "model": "qwen2.5:7b",
  "prompt": "Hello",
  "keep_alive": "30m"           # 请求完成后保留 30 分钟
}'

# OMLX 扩展：手动卸载模型
curl -X POST http://omlx:11434/api/omlx/unload -d '{"model": "qwen2.5:7b"}'

# OMLX 扩展：查询显存使用状态
curl http://omlx:11434/api/omlx/vram/status
# 响应：
# {
#   "worker": "node-gpu-01",
#   "total_vram_gb": 80,
#   "used_vram_gb": 54.2,
#   "loaded_models": [
#     {"name": "qwen2.5:72b", "vram_gb": 45.5, "state": "active"},
#     {"name": "qwen2.5:7b", "vram_gb": 5.8, "state": "keep-alive", "ttl": "3m"},
#     {"name": "nomic-embed", "vram_gb": 1.5, "state": "idle"}
#   ]
# }

后端选择决策树：

模型是否为 GGUF 格式？
├── 是 → llama.cpp（天然兼容）
└── 否（HuggingFace safetensors）
      ├── 追求最高吞吐？ → vLLM
      ├── 需要结构化生成（JSON / Regex）？ → SGLang
      ├── 追求最低延迟？ → TensorRT-LLM
      ├── 长上下文多轮对话？ → SGLang（RadixAttention 优势）
      └── 通用在线推理？ → vLLM（生态最成熟）

# Modelfile 中指定后端
FROM qwen2.5:7b
BACKEND vllm                  # 指定推理后端
PARAMETER tensor_parallel_size 2

# API 请求中覆盖后端
curl http://omlx:11434/api/chat -d '{
  "model": "qwen2.5:7b",
  "messages": [{"role": "user", "content": "Hello"}],
  "options": {"omlx_backend": "sglang"}
}'

# Worker 配置中声明支持的后端
worker:
  backends:
    - name: vllm
      enabled: true
      models_path: /models/huggingface/
    - name: llama.cpp
      enabled: true
      models_path: /models/gguf/
    - name: sglang
      enabled: false

OMLX Worker 推理抽象层
  │
  ├── Backend Interface（统一接口）
  │   ├── load(model_path, config) → ModelHandle
  │   ├── generate(handle, prompt, params) → Response
  │   ├── chat(handle, messages, params) → Response
  │   ├── embeddings(handle, texts) → []float32
  │   └── unload(handle)
  │
  └── Backend Implementation（各后端实现）
      ├── LlamaCppBackend
      ├── VLLMBackend
      ├── SGLangBackend
      └── TensorRTLLMBackend

graph TD
    INGRESS["Ingress / Gateway<br/>(Nginx / Traefik / Istio)<br/>:11434 (HTTP)"]

    subgraph CONTROL["控制平面"]
        S1["Server Deployment<br/>(Replicas: 2-3)<br/>HPA: CPU > 70%"]
        S2["Server Deployment<br/>(多副本水平扩展)"]
    end

    GW["Gateway Deployment<br/>(Replicas: 2-3)<br/>HPA: RPS > 100"]

    CTRL["Controller Deployment<br/>(Replicas: 1)<br/>单实例主节点"]

    subgraph WORKERS["Worker StatefulSet (Replicas: N)"]
        W0["Worker-0<br/>GPU: A100"]
        W1["Worker-1<br/>GPU: A10"]
        W2["Worker-2<br/>GPU: A10"]
        W3["Worker-3<br/>GPU: A10"]
    end

    subgraph STORAGE["Shared Storage"]
        MINIO["MinIO / S3 / CephFS (模型缓存)"]
        HARBOR["Harbor (模型 Registry)"]
    end

    INGRESS --> CONTROL
    CONTROL --> GW
    GW --> CTRL
    GW --> WORKERS
    WORKERS --> STORAGE

# GPU Worker Pod 配置示例
apiVersion: v1
kind: Pod
spec:
  nodeSelector:
    node.kubernetes.io/instance-type: gpu-worker
  tolerations:
    - key: "nvidia.com/gpu"
      operator: "Exists"
      effect: "NoSchedule"
  containers:
    - name: worker
      resources:
        limits:
          nvidia.com/gpu: 1

# 添加 OMLX Helm 仓库
helm repo add omlx https://charts.omlx.dev/stable
helm repo update

# 查看可用版本
helm search repo omlx

# 安装 OMLX Stack
helm install omlx omlx/omlx-stack \
  --namespace omlx-system \
  --create-namespace \
  --values custom-values.yaml

# 升级
helm upgrade omlx omlx/omlx-stack \
  --namespace omlx-system \
  --values custom-values.yaml

# 全局配置
global:
  imageRegistry: "docker.io/omlx"
  imageTag: "v0.4.0"
  storageClass: "gp3"
  modelRegistry: "harbor.example.com/models"

# Server 组件
server:
  replicas: 3
  resources:
    requests:
      cpu: "2"
      memory: "4Gi"
    limits:
      cpu: "4"
      memory: "8Gi"
  hpa:
    enabled: true
    minReplicas: 2
    maxReplicas: 10
    targetCPUUtilizationPercentage: 70
  ingress:
    enabled: true
    className: "nginx"
    hosts:
      - omlx.example.com
    tls:
      - secretName: omlx-tls
        hosts:
          - omlx.example.com

# Gateway 组件
gateway:
  replicas: 3
  resources:
    requests:
      cpu: "1"
      memory: "2Gi"
  config:
    routeStrategy: "least-loaded"      # round-robin / least-loaded / random
    rateLimit:
      enabled: true
      requestsPerSecond: 100
    queueMaxSize: 1000

# Controller 组件
controller:
  replicas: 1
  config:
    leaderElection: true
    healthCheckInterval: 10s
    modelLoadTimeout: 300s

# Worker 组件
worker:
  replicas: 4
  resources:
    limits:
      nvidia.com/gpu: 1
    requests:
      cpu: "8"
      memory: "32Gi"
  config:
    keepAlive: "15m"
    maxLoadedModels: 4
    vramLimit: "38Gi"                # 单 Worker 显存上限
    backends:
      - name: "llama.cpp"
        enabled: true
      - name: "vllm"
        enabled: true
  nodeSelector:
    node.kubernetes.io/instance-type: "gpu-worker"
  persistence:
    enabled: true
    size: "200Gi"
    storageClass: "gp3"

# 模型预加载
models:
  preload:
    - "qwen2.5:7b-q4_k_m"
    - "nomic-embed-text"
    - "llama3.1:8b-q4_k_m"

# 监控集成
monitoring:
  enabled: true
  serviceMonitor:
    enabled: true
    interval: "30s"
  grafanaDashboard:
    enabled: true
    labels:
      grafana_dashboard: "1"

# 认证配置
auth:
  enabled: true
  apiKeys:
    - name: "admin-key"
      key: "omlx-admin-xxxxx"
      role: "admin"
  oidc:
    enabled: false

# 目录结构
omlx-deploy/
├── base-values.yaml          # 公共基础配置
├── dev-values.yaml           # 开发环境覆盖
├── staging-values.yaml       # 预发环境覆盖
└── prod-values.yaml          # 生产环境覆盖

# 部署命令（Helm 合并多 values 文件）
helm install omlx omlx/omlx-stack \
  -f base-values.yaml \
  -f prod-values.yaml \
  --namespace omlx-system

# OMLX Modelfile - Qwen2.5 代码助手
FROM qwen2.5:7b-instruct-q8_0

# 推理参数
PARAMETER temperature 0.3
PARAMETER top_p 0.9
PARAMETER num_predict 2048
PARAMETER repeat_penalty 1.1
PARAMETER stop "</s>"
PARAMETER stop "User:"
PARAMETER stop "Assistant:"

# OMLX 扩展：指定推理后端
BACKEND vllm
PARAMETER tensor_parallel_size 1

# OMLX 扩展：显存管理
PARAMETER omlx_keep_alive "30m"

# 系统提示词
SYSTEM """You are an expert software engineer assistant specializing in Go and Python.
Follow these rules:
1. Write clean, idiomatic code
2. Include error handling
3. Add brief comments for complex logic
4. Prefer standard library over third-party packages"""

# 对话模板
TEMPLATE """XQOPEN if .System XQCLOSE<|system|>
XQOPEN .System XQCLOSE<|end|>
XQOPEN end XQCLOSEXQOPEN if .Prompt XQCLOSE<|user|>
XQOPEN .Prompt XQCLOSE<|end|>
XQOPEN end XQCLOSE<|assistant|>
XQOPEN .Response XQCLOSE<|end|>"""

# 对话示例
MESSAGE user "Write a Go function to read a YAML config file"
MESSAGE assistant "```go\npackage config\n\nimport (\n    \"os\"\n    \"gopkg.in/yaml.v3\"\n)\n\ntype Config struct {\n    Server   ServerConfig   `yaml:\"server\"`\n    Database DatabaseConfig `yaml:\"database\"`\n}\n\nfunc Load(path string) (*Config, error) {\n    data, err := os.ReadFile(path)\n    if err != nil {\n        return nil, fmt.Errorf(\"read config: %w\", err)\n    }\n    var cfg Config\n    if err := yaml.Unmarshal(data, &cfg); err != nil {\n        return nil, fmt.Errorf(\"parse config: %w\", err)\n    }\n    return &cfg, nil\n}\n```"

# 1. 创建 Modelfile
cat > Modelfile.qwen-code << 'EOF'
FROM qwen2.5:7b-instruct-q8_0
SYSTEM "You are a code assistant."
PARAMETER temperature 0.3
EOF

# 2. 构建模型
curl http://omlx:11434/api/create -d '{
  "name": "qwen-code-assistant",
  "modelfile": "FROM qwen2.5:7b-instruct-q8_0\nSYSTEM \"You are a code assistant.\"\nPARAMETER temperature 0.3"
}'

# 或者用 ollama CLI
ollama create qwen-code-assistant -f Modelfile.qwen-code

# 3. 本地测试
ollama run qwen-code-assistant "Write a hello world in Rust"

# 4. 推送到 Registry
ollama push qwen-code-assistant

# 5. 在其他节点拉取
ollama pull qwen-code-assistant

graph TD
    CLIENT["Client Request"] --> S["Server"] --> GW["Gateway"]
    GW --> CTRL["Controller<br/>调度分布式推理任务"]
    CTRL --> WA["Worker-A<br/>Node-1<br/>GPU: x8<br/>Layers 0-26"]
    CTRL --> WB["Worker-B<br/>Node-2<br/>GPU: x8<br/>Layers 27-53"]
    CTRL --> WC["Worker-C<br/>Node-3<br/>GPU: x8<br/>Layers 54-79"]
    WA <-->|"RDMA"| WB
    WB <-->|"RDMA"| WC
    WA & WB & WC --> NET["高速网络（InfiniBand / RoCE）<br/>Pipeline Parallel 跨节点"]

Step 1: Worker-A (Node-1) 计算 Layers 0-26
  → 输出激活值 → RDMA 传输 → Worker-B (Node-2)
Step 2: Worker-B (Node-2) 计算 Layers 27-53
  → 输出激活值 → RDMA 传输 → Worker-C (Node-3)
Step 3: Worker-C (Node-3) 计算 Layers 54-79
  → 输出 Logits → 采样 → Token → 返回 Gateway

gantt
    title Micro-Batch 流水线调度
    dateFormat X
    axisFormat %s
    section Node-1
    B1_L0-26    : 0, 1
    B2_L0-26    : 1, 2
    B3_L0-26    : 2, 3
    B4_L0-26    : 3, 4
    section Node-2
    B1_L27-53   : 1, 2
    B2_L27-53   : 2, 3
    B3_L27-53   : 3, 4
    B4_L27-53   : 4, 5
    section Node-3
    B1_L54-79   : 2, 3
    B2_L54-79   : 3, 4
    B3_L54-79   : 4, 5
    B4_L54-79   : 5, 6

**OMLX 分布式推理配置**

```yaml
# 分布式推理 Worker 配置
worker:
  distributed:
    enabled: true
    mode: "pipeline_parallel"       # pipeline_parallel / expert_parallel
    world_size: 8                   # 总 GPU 数（跨节点）
    pp_size: 4                      # Pipeline Parallel 大小（4 个节点）
    node_rank: 0                    # 当前节点在 pipeline 中的序号
    master_addr: "worker-0.omlx-headless.omlx-system.svc.cluster.local"
    master_port: 29500
    transport: "nccl"               # NCCL / Gloo
    network:
      interface: "eth0"
      protocol: "RoCE"              # InfiniBand / RoCE / TCP

  # 网络要求
  # - InfiniBand HDR (200 Gbps) 或 RoCE v2 (100 Gbps)
  # - 节点间延迟 < 10µs
  # - 启用 GPU Direct RDMA (GDR)

Gateway 收到请求
  │
  ├── 1. 解析 model 名称 → 查 Model-Worker 映射表（从 Controller 同步）
  │
  ├── 2. 过滤候选 Worker 列表
  │     ├── 模型已加载到显存 → 优先级最高
  │     ├── 模型已在本地缓存（未加载到显存）→ 优先级中
  │     └── 模型未缓存（需 Pull）→ 优先级低
  │
  ├── 3. 应用会话亲和性（Session Affinity）
  │     └── 同一 Session ID 的请求路由至同一 Worker（复用 KV Cache）
  │
  ├── 4. 应用标签过滤（Label Selector）
  │     └── 按请求指定的 omlx_worker_label 过滤 Worker
  │
  ├── 5. 负载均衡算法选择目标 Worker
  │
  ├── 6. 检查目标 Worker 并发上限
  │     ├── 未达上限 → 转发请求
  │     └── 已达上限 → 排队或回退至次优 Worker
  │
  └── 7. 转发请求 → 等待 Worker 响应

gateway:
  sessionAffinity:
    enabled: true
    mode: "header"                  # header / cookie
    headerName: "X-OMLX-Session-Id"
    ttl: "30m"                      # 亲和性保持时长
    fallbackOnUnavailable: true     # 目标 Worker 不可用时的回退策略

gateway:
  queue:
    enabled: true
    maxSize: 1000                   # 全局最大排队请求数
    perWorkerMaxSize: 100           # 单 Worker 最大排队请求数
    timeout: "60s"                  # 排队超时
    overflowPolicy: "reject"        # reject / drop-head / drop-tail

  rateLimit:
    enabled: true
    requestsPerSecond: 500          # 全局 QPS 限制
    burstSize: 100                  # 突发容量
    perModelRateLimit:              # 按模型粒度限制
      "qwen2.5:72b":
        requestsPerSecond: 50
      "qwen2.5:7b":
        requestsPerSecond: 200

Controller 维护集群拓扑

Worker-0 (node-gpu-01)：[qwen2.5:72b(active), nomic-embed(idle)]
Worker-1 (node-gpu-02)：[qwen2.5:7b(active), codellama:7b(active)]
Worker-2 (node-gpu-03)：[qwen2.5:7b(active)]
Worker-3 (node-gpu-04)：[llama3:8b(keep-alive)]

↓ Controller 发生变更事件（模型加载 / 卸载 / Worker 上下线）→ 推送更新

Gateway 路由表更新：
  qwen2.5:72b → [Worker-0]
  qwen2.5:7b  → [Worker-1, Worker-2]  (least-loaded 选择)
  codellama:7b → [Worker-1]
  llama3:8b   → [Worker-3]
  nomic-embed → [Worker-0]

# 单文本向量化
curl http://omlx:11434/api/embeddings -d '{
  "model": "nomic-embed-text",
  "prompt": "The sky is blue because of Rayleigh scattering."
}'

# 响应
{
  "embedding": [0.0123, -0.0456, 0.0789, ...],  // 768/1024/4096 维向量
  "model": "nomic-embed-text"
}

# 批量向量化（OMLX 扩展）
curl http://omlx:11434/api/omlx/embeddings/batch -d '{
  "model": "nomic-embed-text",
  "input": [
    "What is Kubernetes?",
    "How to deploy a Pod?",
    "Explain container orchestration."
  ]
}'

graph LR
    subgraph EMB["Embedding 部署架构"]
        APP["Application<br/>(RAG / Search)"]
        OMLX["OMLX Server<br/>/api/embeddings<br/>nomic-embed<br/>(独立 Worker)"]
        VDB["向量数据库<br/>(Milvus/Qdrant/Weaviate)"]
        LOGIC["应用逻辑<br/>Query -> Embedding<br/>-> Search -> LLM"]
        APP --> OMLX
        APP --> VDB
        APP --> LOGIC
    end

# 专用 Embedding Worker
worker:
  embedding:
    dedicated: true
    replicas: 2
    models:
      - "nomic-embed-text"
      - "bge-m3"
    resources:
      limits:
        nvidia.com/gpu: 1       # 仅需少量显存（1-2GB）
    config:
      keepAlive: "24h"           # Embedding 模型始终驻留显存

import ollama
import numpy as np
from qdrant_client import QdrantClient

client = ollama.Client(host="http://omlx:11434")

# 1. 文档向量化
def embed_documents(docs: list[str]) -> list[list[float]]:
    embeddings = []
    for doc in docs:
        resp = client.embeddings(model="nomic-embed-text", prompt=doc)
        embeddings.append(resp["embedding"])
    return embeddings

# 2. 存入 Qdrant 向量数据库
qdrant = QdrantClient(host="qdrant.example.com")
qdrant.upsert(
    collection_name="knowledge_base",
    points=[
        {"id": i, "vector": emb, "payload": {"text": doc}}
        for i, (doc, emb) in enumerate(zip(docs, embeddings))
    ]
)

# 3. 检索增强生成（RAG）
def rag_query(question: str) -> str:
    # 查询向量化
    q_emb = client.embeddings(model="nomic-embed-text", prompt=question)["embedding"]
    # 向量检索
    results = qdrant.search(collection_name="knowledge_base", query_vector=q_emb, limit=3)
    # 拼接上下文
    context = "\n".join([r.payload["text"] for r in results])
    # LLM 生成
    resp = client.chat(model="qwen2.5:7b", messages=[
        {"role": "system", "content": f"Answer based on context:\n{context}"},
        {"role": "user", "content": question}
    ])
    return resp["message"]["content"]

# 单张图片推理
curl http://omlx:11434/api/chat -d '{
  "model": "llava:13b",
  "messages": [
    {
      "role": "user",
      "content": "Describe this image in detail.",
      "images": ["'$(base64 -i image.jpg)'"]
    }
  ],
  "stream": false
}'

# 多张图片推理
curl http://omlx:11434/api/generate -d '{
  "model": "minicpm-v:8b",
  "prompt": "Compare these two charts. Which one shows better performance?",
  "images": ["'$(base64 -i chart1.png)'", "'$(base64 -i chart2.png)'"]
}'

# OMLX 扩展：图片 URL 直接传入
curl http://omlx:11434/api/omlx/chat -d '{
  "model": "qwen2-vl:7b",
  "messages": [
    {
      "role": "user",
      "content": "What is in this image?",
      "image_urls": ["https://example.com/image.jpg"]
    }
  ]
}'

graph TD
    Input["图片输入"]
    Preprocess["图片预处理<br/>Resize 到模型要求的输入尺寸<br/>Normalize 归一化<br/>转换为 Tensor"]
    VisionEncoder["视觉编码器<br/>（Vision Encoder / ViT）<br/>提取视觉特征（Visual Tokens / Image Embeddings）"]
    Projection["特征投影层<br/>（Projection Layer）<br/>将视觉特征映射到 LLM Embedding 空间"]
    LLM["大语言模型<br/>（LLM Backbone）<br/>文本 Token Embeddings + Image Embeddings<br/>Transformer 层处理多模态输入"]
    Output["文本解码输出"]

    Input --> Preprocess
    Preprocess --> VisionEncoder
    VisionEncoder --> Projection
    Projection --> LLM
    LLM --> Output

graph TD
    Input["用户输入 + Tools 定义"]
    Analysis["OMLX → 模型分析 → 输出 Tool Call（JSON）"]
    Execute["调用方解析 Tool Call → 执行函数 → 获取结果"]
    Result["Tool Result 回传 OMLX"]
    Final["模型根据结果继续生成最终回复"]

    Input --> Analysis
    Analysis --> Execute
    Execute --> Result
    Result --> Final

# 定义 Tools 并发起请求
curl http://omlx:11434/api/chat -d '{
  "model": "qwen2.5:14b",
  "messages": [
    {"role": "user", "content": "What is the weather in Beijing?"}
  ],
  "stream": false,
  "tools": [
    {
      "type": "function",
      "function": {
        "name": "get_weather",
        "description": "Get current weather for a given city",
        "parameters": {
          "type": "object",
          "properties": {
            "city": {
              "type": "string",
              "description": "City name, e.g. Beijing"
            },
            "unit": {
              "type": "string",
              "enum": ["celsius", "fahrenheit"],
              "default": "celsius"
            }
          },
          "required": ["city"]
        }
      }
    },
    {
      "type": "function",
      "function": {
        "name": "get_stock_price",
        "description": "Get current stock price for a ticker",
        "parameters": {
          "type": "object",
          "properties": {
            "ticker": {
              "type": "string",
              "description": "Stock ticker symbol"
            }
          },
          "required": ["ticker"]
        }
      }
    }
  ]
}'

# 模型返回 Tool Call
{
  "message": {
    "role": "assistant",
    "content": "",
    "tool_calls": [
      {
        "function": {
          "name": "get_weather",
          "arguments": {
            "city": "Beijing",
            "unit": "celsius"
          }
        }
      }
    ]
  }
}

# 调用方执行函数后，将结果回传
curl http://omlx:11434/api/chat -d '{
  "model": "qwen2.5:14b",
  "messages": [
    {"role": "user", "content": "What is the weather in Beijing?"},
    {
      "role": "assistant",
      "content": "",
      "tool_calls": [{"function": {"name": "get_weather", "arguments": {"city": "Beijing"}}}]
    },
    {"role": "tool", "content": "Beijing: 25°C, sunny, humidity 45%"}
  ],
  "stream": false
}'

// 模型可能返回多个独立 Tool Call
{
  "tool_calls": [
    {"function": {"name": "get_weather", "arguments": {"city": "Beijing"}}},
    {"function": {"name": "get_weather", "arguments": {"city": "Shanghai"}}}
  ]
}
// 调用方可并行执行这两个函数，减少总延迟

# OMLX Gateway 配置
gateway:
  toolCalling:
    maxIterations: 10              # 最大 Tool Call 轮次
    toolExecutionTimeout: "30s"    # 单次工具执行超时
    maxToolResultTokens: 4096      # 工具返回结果最大 Token

# 请求（stream: true）
curl http://omlx:11434/api/generate -d '{
  "model": "qwen2.5:7b",
  "prompt": "Explain quantum computing in simple terms.",
  "stream": true
}'

# SSE 响应流
data: {"model":"qwen2.5:7b","created_at":"2025-01-01T00:00:00Z","response":"Quantum","done":false}
data: {"model":"qwen2.5:7b","created_at":"2025-01-01T00:00:00Z","response":" computing","done":false}
data: {"model":"qwen2.5:7b","created_at":"2025-01-01T00:00:00Z","response":" uses","done":false}
# ... 持续流式返回 ...
data: {"model":"qwen2.5:7b","created_at":"2025-01-01T00:00:00Z","response":"","done":true,"total_duration":3250000000,"load_duration":1200000000,"prompt_eval_count":12,"prompt_eval_duration":150000000,"eval_count":156,"eval_duration":2930000000}

curl http://omlx:11434/api/chat -d '{
  "model": "qwen2.5:7b",
  "messages": [{"role": "user", "content": "Hello"}],
  "stream": true
}'

# Chat 流式响应格式
data: {"message":{"role":"assistant","content":"Hello"},"done":false}
data: {"message":{"role":"assistant","content":"!"},"done":false}
data: {"message":{"role":"assistant","content":" How"},"done":false}
data: {"message":{"role":"assistant","content":" can"},"done":false}
data: {"message":{"role":"assistant","content":" I"},"done":false}
data: {"message":{"role":"assistant","content":" help"},"done":false}
data: {"message":{"role":"assistant","content":" you"},"done":false}
data: {"message":{"role":"assistant","content":" today"},"done":false}
data: {"message":{"role":"assistant","content":"?"},"done":false}
data: {"done":true,"total_duration":1850000000,"prompt_eval_count":10,"eval_count":9}

graph TD
    Client["Client"]
    SSE["SSE Connection<br/>（HTTP Long-Poll）"]
    Server["OMLX Server"]
    GRPC1["gRPC Stream"]
    Gateway["OMLX Gateway"]
    GRPC2["gRPC Stream"]
    Worker["OMLX Worker"]
    Backend["推理后端（vLLM / llama.cpp）<br/>逐 Token 生成"]

    Client --> SSE
    SSE --> Server
    Server --> GRPC1
    GRPC1 --> Gateway
    Gateway --> GRPC2
    GRPC2 --> Worker
    Worker --> Backend

graph TD
    L1["层级 1：Gateway 速率限制（Rate Limiting）<br/>全局 QPS 限制 + 单模型 QPS 限制<br/>超过限制 → HTTP 429 Too Many Requests"]
    L2["层级 2：Gateway 请求队列（Request Queuing）<br/>Worker 并发满时排队等待<br/>超过队列长度 / 超时 → HTTP 503 Service Unavailable"]
    L3["层级 3：Worker 并发上限（Concurrency Limit）<br/>单 Worker 最大并发请求数<br/>到达上限 → 回退至队列或拒绝"]
    L4["层级 4：推理后端批处理（Dynamic Batching）<br/>vLLM / SGLang 将并发请求合并为 Batch<br/>提升 GPU 利用率和吞吐量"]
    GPU["GPU 执行推理"]

    L1 --> L2
    L2 --> L3
    L3 --> L4
    L4 --> GPU

gateway:
  concurrency:
    maxGlobalConcurrency: 500          # 全局最大并发请求
    defaultModelMaxConcurrency: 100    # 单模型默认最大并发
    modelOverrides:                    # 按模型粒度覆盖
      "qwen2.5:72b":
        maxConcurrency: 50
      "qwen2.5:7b":
        maxConcurrency: 200
    maxQueueSize: 2000
    queueTimeout: "120s"

worker:
  concurrency:
    maxRequestsPerWorker: 50           # 单 Worker 最大并发请求
    maxBatchSize: 32                   # 最大批处理大小
    maxWaitingRequests: 200            # 单 Worker 最大等待请求
    batchTimeout: "50ms"               # 批处理等待窗口（动态 batching 最大等待时间）

时间轴（每个槽 = 10ms）
────────────────────────────────────────────────────────────→

Request Arrival:
Req-1 ───[B1][B1][B1][B1][B1]────────────────────→
Req-2 ──────[B1][B1][B1][B1][B1]─────────────────→
Req-3 ──────────[B2][B2][B2][B2][B2]─────────────→
Req-4 ──────────[B2][B2][B2][B2][B2]─────────────→
Req-5 ───────────────────[B3][B3][B3][B3]────────→

Batch 合并：
Batch-1 = [Req-1, Req-2]    （在 batchTimeout 50ms 窗口内到达的请求合并）
Batch-2 = [Req-3, Req-4]    （下一批）
Batch-3 = [Req-5]            （窗口内只有一个请求）

理论吞吐量(tokens/s) = 
  min(
    模型推理速度(tokens/s/GPU) × GPU 数量 / TP_SIZE,
    Batch Size × Avg Output Tokens / (Prefill Time + Decode Time per Token × Avg Output Tokens)
  ) × GPU 利用率

实例（vLLM, qwen2.5:7b, 4×A10, TP=1, FIX=Batch 32）：
  推理速度 ≈ 150 tokens/s/GPU
  Throughput ≈ 150 × 4 × 0.85(利用率) = ~510 tokens/s

实例（vLLM, qwen2.5:72b, 2×A100-80G, TP=2）：
  推理速度 ≈ 45 tokens/s（2 卡合并为 1 组 TP）
  Throughput ≈ 45 × 0.85 = ~38 tokens/s

monitoring:
  enabled: true
  serviceMonitor:
    enabled: true
    interval: "30s"
    labels:
      release: "prometheus"
  grafanaDashboard:
    enabled: true
    labels:
      grafana_dashboard: "1"

  # Alert Rules
  alerting:
    enabled: true
    rules:
      - alert: OMLXHighErrorRate
        expr: |
          sum(rate(omlx_server_requests_total{status=~"5.."}[5m]))
          / sum(rate(omlx_server_requests_total[5m])) > 0.05          
        for: 5m
        severity: critical
        annotations:
          summary: "OMLX error rate exceeds 5%"

      - alert: OMLXWorkerDown
        expr: omlx_controller_workers_healthy < omlx_controller_workers_total
        for: 2m
        severity: critical
        annotations:
          summary: "OMLX Worker node is down"

      - alert: OMLXVRAMHigh
        expr: omlx_worker_vram_used_bytes / omlx_worker_vram_total_bytes > 0.95
        for: 5m
        severity: warning
        annotations:
          summary: "OMLX VRAM usage exceeds 95%"

      - alert: OMLXQueueHigh
        expr: omlx_gateway_queue_length > 100
        for: 5m
        severity: warning
        annotations:
          summary: "OMLX request queue depth exceeds 100"

graph TD
    Client["Client Request<br/>携带认证凭证<br/>Header: Authorization: Bearer omlx-xxx<br/>Header: Authorization: Bearer jwt_token<br/>Header: X-OMLX-API-Key: omlx-xxx"]
    Server["OMLX Server（认证中间件）<br/>验证 API Key → 查询本地 Key Store<br/>验证 JWT Token → 调用 OIDC Provider（Keycloak / Auth0）"]
    RBAC["RBAC 鉴权<br/>提取用户角色（admin / developer / viewer）<br/>检查角色是否有权限执行当前操作"]
    Gateway["转发至 Gateway"]

    Client --> Server
    Server --> RBAC
    RBAC --> Gateway

auth:
  enabled: true
  apiKeys:
    - name: "admin-key"
      key: "omlx-admin-xxxxxxxxxxxx"
      role: "admin"
      description: "Full access key for automation"
    - name: "developer-key"
      key: "omlx-dev-xxxxxxxxxxxx"
      role: "developer"
      description: "Developer team shared key"
    - name: "viewer-key"
      key: "omlx-viewer-xxxxxxxxxxxx"
      role: "viewer"
      description: "Read-only access for monitoring"

  oidc:
    enabled: false
    issuer: "https://keycloak.example.com/realms/omlx"
    clientId: "omlx-server"
    clientSecret: "xxxxxxxx"
    redirectUri: "https://omlx.example.com/oauth/callback"
    scopes: ["openid", "profile", "email"]
    groupsClaim: "groups"

auth:
  modelACLs:
    - model: "qwen2.5:72b"
      allowedRoles: ["admin", "developer"]
      rateLimit: 50                    # 该模型的额外 QPS 限制
    - model: "internal-fine-tuned"
      allowedRoles: ["admin"]          # 仅管理员可访问
    - model: "public-model"
      allowedRoles: ["admin", "developer", "viewer"]
      rateLimit: 200

1. Client 通过 OIDC Provider 获取 JWT Token
2. 请求中携带 Authorization: Bearer <jwt_token>
3. OMLX Server 验证 JWT：
   ├── 验证签名（jwks_uri）
   ├── 验证 iss / aud / exp
   └── 提取 claims：sub, groups, email
4. 根据 groups 映射到 RBAC 角色
5. 检查请求操作权限

{
  "timestamp": "2025-06-15T10:30:00.123Z",
  "level": "INFO",
  "component": "worker",
  "worker_id": "worker-2",
  "trace_id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890",
  "session_id": "sess-xyz",
  "message": "Inference completed",
  "fields": {
    "model": "qwen2.5:7b",
    "prompt_tokens": 128,
    "completion_tokens": 256,
    "duration_ms": 1250,
    "ttft_ms": 180,
    "tpot_ms": 45,
    "backend": "vllm",
    "gpu_id": 0
  }
}

logging:
  format: "json"                    # json / text
  level: "info"                     # debug / info / warn / error / fatal
  output: "stdout"                  # stdout / file
  filePath: "/var/log/omlx/"

  # 请求日志
  accessLog:
    enabled: true
    excludePaths: ["/health", "/metrics"]

  # 审计日志
  auditLog:
    enabled: true
    output: "stdout"                # 单独输出到不同位置便于收集
    filePath: "/var/log/omlx/audit/"

  # Trace 集成（OpenTelemetry）
  tracing:
    enabled: true
    endpoint: "http://tempo-distributor.tempo.svc:4317"
    protocol: "grpc"                # grpc / http
    sampleRate: 0.1

# 查询特定模型的推理耗时
{component="worker", model="qwen2.5:72b"}
  | json
  | duration_ms > 5000

# 查询 Worker 故障
{component="controller"} |= "worker.leave"

# 查询错误请求
{component="server"} | json | level="ERROR"
  | logfmt
  | status_code >= 500

# 按模型统计 QPS 趋势
sum by (model) (
  rate({component="server"} | json | message="Inference completed" [5m])
)

# Modelfile 中设置
FROM qwen2.5:7b
PARAMETER num_ctx 8192           # 上下文窗口 Token 数

# API 请求中动态设置（不得超过模型原生上限）
curl http://omlx:11434/api/generate -d '{
  "model": "qwen2.5:7b",
  "prompt": "...",
  "options": {
    "num_ctx": 4096              # 此请求使用 4096 Token 上下文
  }
}'

graph TD
    Calc["请求 Token 数 = system_prompt_tokens + messages_tokens + max_tokens"]
    Check{"检查：请求 Token 数 > num_ctx？"}
    Normal["正常推理"]
    Truncate{"触发截断策略"}
    S1["策略 1：Truncate Head（截断最早的消息）<br/>保留 System Prompt + 最近 N 条消息，丢弃较早消息<br/>适用：多轮对话（历史不重要）"]
    S2["策略 2：Truncate Middle（截断中间消息）<br/>保留 System Prompt + 最早 2 条 + 最近 N 条<br/>适用：需保留上下文起始信息"]
    S3["策略 3：Summarize（摘要压缩）<br/>对历史消息自动生成摘要替代原文<br/>适用：需保留全部上下文语义"]
    S4["策略 4：Reject（拒绝）<br/>返回错误：context length exceeded<br/>适用：严格场景"]

    Calc --> Check
    Check -->|"否"| Normal
    Check -->|"是"| Truncate
    Truncate -->|"策略 1"| S1
    Truncate -->|"策略 2"| S2
    Truncate -->|"策略 3"| S3
    Truncate -->|"策略 4"| S4

worker:
  context:
    defaultNumCtx: 4096              # 默认上下文长度
    maxNumCtx: 32768                 # 允许的最大上下文（受显存限制）
    truncation:
      strategy: "truncate-head"      # truncate-head / truncate-middle / summarize / reject
      keepSystemPrompt: true         # 始终保留 System Prompt
      minKeepMessages: 4             # 至少保留最近几条消息
      summarizationModel: "qwen2.5:7b"  # 摘要压缩使用的模型

KV Cache 大小 ≈
  2 × num_layers × num_ctx × num_key_value_heads × head_dim × dtype_size

实例（llama3.1:8b, num_ctx=8192, FP16）：
  num_layers = 32
  num_key_value_heads = 8
  head_dim = 128
  dtype_size = 2 (FP16)

  KV Cache ≈ 2 × 32 × 8192 × 8 × 128 × 2
           ≈ 1,073,741,824 bytes
           ≈ 1.0 GB

实例（qwen2.5:72b, num_ctx=32768, FP16）：
  num_layers = 80
  num_key_value_heads = 8
  head_dim = 128

  KV Cache ≈ 2 × 80 × 32768 × 8 × 128 × 2
           ≈ 10,737,418,240 bytes
           ≈ 10.0 GB

OMLX 平台层
├── OMLX Server（Ollama API 兼容）
├── OMLX Gateway（路由 / 负载均衡）
├── OMLX Controller（调度 / 编排）
└── OMLX Worker（推理执行层）
      ├── vLLM Backend         ← 可将 vLLM 作为推理引擎
      ├── llama.cpp Backend
      ├── SGLang Backend
      └── TensorRT-LLM Backend

单独使用 vLLM：
  vLLM OpenAI Server → 需要额外搭建网关、负载均衡、认证、监控

使用 OMLX + vLLM：
  OMLX 管理运维层 + vLLM 提供推理能力 = 完整平台体验

graph TD
    subgraph GS["GPUStack"]
        GSS["Server<br/>(OpenAI API)"]
        GSW["Worker<br/>(llama.cpp)"]
        GSS --> GSW
    end

    subgraph OMLX["OMLX"]
        OMLXS["Server<br/>(Ollama API)"]
        OMLXG["Gateway<br/>路由 / 负载均衡"]
        OMLXC["Controller<br/>调度 / 编排"]
        OMLXW["Worker<br/>(多后端)"]
        OMLXS --> OMLXG --> OMLXC --> OMLXW
    end

层级                    优化手段                    关键参数
────────────────────────────────────────────────────────────
推理引擎层    PagedAttention / KV Cache 管理     max_num_batched_tokens
              Prefix Caching / Chunked Prefill   enable_prefix_caching
              Continuous Batching                max_num_seqs
                                                 
调度层        路由亲和性 / 会话保持               session_affinity_ttl
              请求优先级队列                      omlx_priority
              模型预加载                           precache_models
                                                 
资源管理层    显存池化 / 多 Worker 调度             vram_limit
              GPU 拓扑感知                          gpu_selector.topology
              动态 Batch Size                      max_batch_size

worker:
  kv_cache:
    block_size: 16                  # KV Cache 块大小（Token 数），越大显存利用率越高
    gpu_memory_utilization: 0.90    # GPU 显存用于 KV Cache 的最大比例
    swap_space: 4                   # CPU 内存作为 KV Cache 换出空间（GB）
    max_num_seqs: 256               # 最大并发序列数（PagedAttention 管理）
    enable_prefix_caching: true     # 启用前缀缓存

graph TD
    Prefix["相同前缀 System Prompt（200 tokens）"]
    Cache["Prefix Cache 命中 → 复用已计算的 KV Cache"]
    A["请求 A：Prefix(200t) 命中 → 仅计算 User Q1(50t) + 生成 A1"]
    B["请求 B：Prefix(200t) 命中 → 仅计算 User Q2(50t) + 生成 A2"]
    C["请求 C：Prefix(200t) 命中 → 仅计算 User Q3(50t) + 生成"]

    Prefix --> Cache
    Cache --> A
    Cache --> B
    Cache --> C

Latency vs Throughput 权衡：

低延迟模式（交互式对话）：
  max_num_seqs: 32
  max_num_batched_tokens: 4096
  → 每个 Batch 处理少量短请求，TTFT 低

高吞吐模式（批量 API）：
  max_num_seqs: 256
  max_num_batched_tokens: 16384
  → 每个 Batch 合并大量请求，GPU 利用率高，但单个请求 TTFT 可能升高

自适应模式（OMLX 扩展）：
  adaptive_batching: true
  target_ttft_ms: 300
  → OMLX 根据 TTFT 目标自动调节 batch 大小

graph TD
    subgraph LB["负载均衡层"]
        INGRESS["Ingress Controller (Nginx / Traefik) x 2<br/>+ cert-manager 自动 TLS 证书"]
    end

    subgraph CONTROL["控制平面节点"]
        S["Server Deployment (Replicas: 3)"]
        GW["Gateway Deployment (Replicas: 2-3)"]
        CTRL["Controller Deployment (Replicas: 1 + standby)"]
    end

    subgraph WORKERS["推理 Worker 节点"]
        APOOL["A100 Pool: qwen2.5:72b"]
        A10POOL["A10 Pool: qwen2.5:7b, llava:13b"]
        EPool["Embedding Pool: nomic-embed-text"]
    end

    subgraph MID["中间件与存储"]
        HARBOR["Harbor（模型 Registry）"]
        MINIO["MinIO / CephFS（共享模型缓存）"]
        PROM["Prometheus + Grafana（监控）"]
        LOKI["Loki / Elasticsearch（日志）"]
    end

    LB --> CONTROL --> WORKERS --> MID

# 1. 日常巡检
curl http://omlx:11434/api/omlx/health       # 整体健康状态
curl http://omlx:11434/api/omlx/workers      # Worker 列表
curl http://omlx:11434/api/omlx/vram/status  # 显存分布

# 2. 模型上线
ollama pull qwen2.5:14b                       # 拉取新模型
ollama run qwen2.5:14b --test                 # 功能验证
# 更新 Helm values 添加模型到 preload 列表
helm upgrade omlx omlx/omlx-stack -f values.yaml

# 3. Worker 扩容
helm upgrade omlx omlx/omlx-stack \
  --set worker.replicas=6
# 或使用 kubectl scale（依赖 StatefulSet）
kubectl scale statefulset omlx-worker --replicas=6 -n omlx-system

# 4. 模型下线
curl -X DELETE http://omlx:11434/api/delete \
  -d '{"model": "deprecated-model"}'
# 从 preload 列表中移除
helm upgrade ...

# 5. 灰度发布（Canary）
# 部署新版 Worker StatefulSet（新版本标签）
# Gateway 按标签逐步将流量切至新版 Worker
# 验证无异常后全量切换

# Server - 建议 Request = Limit 避免被驱逐
server:
  resources:
    requests:
      cpu: "2"
      memory: "4Gi"
    limits:
      cpu: "2"
      memory: "4Gi"

# Worker - GPU 任务不要设置 CPU limit（避免 throttling）
worker:
  resources:
    requests:
      cpu: "8"
      memory: "32Gi"
    limits:
      nvidia.com/gpu: 1     # 仅限制 GPU，不设 CPU limit
      memory: "32Gi"        # 内存设置 limit 防止 OOM 波及节点

# Gateway - 适度超卖
gateway:
  resources:
    requests:
      cpu: "1"
      memory: "2Gi"
    limits:
      cpu: "4"
      memory: "4Gi"