基于 Kubernetes + Triton 的 LLM 推理服务：部署、调优与弹性伸缩全流程

helm repo add nvidia https://nvidia.github.io/gpu-operator
helm install gpu-operator nvidia/gpu-operator \
  --set devicePlugin.enabled=true \
  --set mig.strategy=single \
  --set mig.deviceSplit=1g.10gb,2g.20gb

kubectl get node -o json | jq '.items[].status.capacity | with_entries(select(.key | contains("nvidia")))'

apiVersion: resource.k8s.io/v1alpha2
kind: ResourceClass
name: gpu-mem
driverName: nvidia.com
---
apiVersion: resource.k8s.io/v1alpha2
kind: ResourceClaimTemplate
metadata:
  name: gpu-40gb-template
spec:
  spec:
    resourceClassName: gpu-mem
    parameters:
      apiVersion: nvidia.com/v1
      kind: GpuClaimParameters
      memory: 40Gi

spec:
  template:
    spec:
      containers:
      - name: triton
        resources:
          claims:
          - name: gpu-resource
      resourceClaims:
      - name: gpu-resource
        source:
          resourceClaimTemplateName: gpu-40gb-template

/model-repository/
├── llama/
│   ├── config.pbtxt
│   └── 1/
│       ├── model.py          # vLLM Python backend 脚本
│       └── model.pt          # 权重软链接（实际存于 PVC）

name: "llama"
platform: "python"
max_batch_size: 128

dynamic_batching {
  preferred_batch_size: [1, 2, 4, 8, 16, 32, 64, 128]
  max_queue_delay_microseconds: 500   # 最大攒批等待 500μs
  preserve_ordering: false            # 允许乱序返回，提高吞吐
}

instance_group [
  {
    count: 2                          # 启动 2 个 Python 实例，利用多线程
    kind: KIND_GPU
    gpus: [0]                         # 绑定到同一 GPU（配合 MIG）
  }
]

parameters {
  key: "vllm_max_num_seqs"
  value: { string_value: "256" }
}
parameters {
  key: "vllm_max_num_batched_tokens"
  value: { string_value: "8192" }
}
parameters {
  key: "vllm_gpu_memory_utilization"
  value: { string_value: "0.85" }
}

# model.py
import asyncio
from vllm import AsyncLLMEngine, SamplingParams

class TritonPythonModel:
    def initialize(self, args):
        self.engine = AsyncLLMEngine.from_engine_args(
            engine_args=["--model", "/model-repository/llama/1",
                         "--tensor-parallel-size", "1",
                         "--dtype", "float16",
                         "--max-num-seqs", "256"]
        )
        self.sampling_params = SamplingParams(temperature=0.7, max_tokens=128)

    async def execute(self, requests):
        coros = [self.engine.generate(req.input_text, self.sampling_params) 
                 for req in requests]
        outputs = await asyncio.gather(*coros)
        return [output.outputs[0].text for output in outputs]

istioctl install --set profile=default -y
kubectl label namespace inference istio-injection=enabled

apiVersion: networking.istio.io/v1beta1
kind: DestinationRule
metadata: name: triton-dr
spec:
  host: triton-svc
  subsets:
  - name: v1
    labels: { version: "v1" }
  - name: v2
    labels: { version: "v2" }
---
apiVersion: networking.istio.io/v1beta1
kind: VirtualService
metadata: name: triton-vs
spec:
  hosts: ["triton-svc"]
  http:
  - match:
    - headers: { model-version: { exact: "v2" } }
    route: [{ destination: { host: triton-svc, subset: v2 }, weight: 100 }]
  - route:
    - destination: { host: triton-svc, subset: v1, weight: 90 }
    - destination: { host: triton-svc, subset: v2, weight: 10 }

apiVersion: networking.istio.io/v1beta1
kind: DestinationRule
metadata: name: triton-dr
spec:
  trafficPolicy:
    connectionPool:
      tcp: { maxConnections: 200 }
    loadBalancer: { simple: LEAST_REQUEST }
    outlierDetection:
      consecutive5xxErrors: 5
      interval: 10s
      baseEjectionTime: 30s
      maxEjectionPercent: 50
    retry:
      attempts: 3
      perTryTimeout: 2s
      retryOn: "5xx,reset,connect-failure"

apiVersion: tekton.dev/v1beta1
kind: Pipeline
spec:
  tasks:
  - name: validate-model
    taskSpec:
      steps:
      - image: nvcr.io/nvidia/tritonserver:24.05-py3
        script: |
          model-analyzer -m /workspace/model-repository --run-profiling
  - name: build-oci
    taskSpec:
      steps:
      - image: oras:1.1.0
        script: |
          oras push registry.internal/models/llama:v2.1 \
            --manifest-config /dev/null:application/vnd.unknown.config.v1+json \
            ./model-repository/ --media-type application/vnd.model-repository.v1.tar
  - name: trigger-rollout
    taskSpec:
      steps:
      - image: bitnami/kubectl:latest
        script: |
          kubectl patch rollout -n inference triton-rollout \
            -p '{"spec":{"template":{"metadata":{"annotations":{"model-version":"v2.1"}}}}}'

apiVersion: argoproj.io/v1alpha1
kind: Rollout
metadata: name: triton-rollout
spec:
  replicas: 5
  strategy:
    canary:
      steps:
      - setWeight: 10
      - pause: { duration: 5m }   # 人工观察
      - setWeight: 50
      - pause: { duration: 10m }
      - setWeight: 100
      trafficRouting:
        istio:
          virtualService: { name: triton-vs, routes: [primary] }
  selector: { matchLabels: { app: triton } }
  template:
    metadata: { labels: { app: triton } }
    spec:
      containers:
      - name: triton
        image: triton-server:latest
        env:
        - name: MODEL_REPO
          value: "oci://registry.internal/models/llama:v2.1"  # 由镜像拉取模型

apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata: name: triton-monitor
spec:
  selector: { matchLabels: { app: triton } }
  endpoints:
  - port: metrics
    path: /metrics
    interval: 10s

function filter(tag, timestamp, record)
    if math.random() < 0.01 or record.latency_ms > 500 then
        return 1, timestamp, record
    end
    return -1, timestamp, record
end

filters:
- Name: lua
  Match: triton.*
  script: /etc/fluent-bit/sample.lua
  call: filter

spec:
  template:
    metadata:
      annotations:
        sidecar.opentelemetry.io/inject: "true"
        instrumentation.opentelemetry.io/inject-python: "true"

from opentelemetry import trace
tracer = trace.get_tracer("vllm.backend")
with tracer.start_as_current_span("generate") as span:
    span.set_attribute("input_length", len(prompt))
    output = await engine.generate(...)
    span.set_attribute("output_length", len(output))

helm repo add kedacore https://kedacore.github.io/charts
helm install keda kedacore/keda

apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata: name: triton-scaler
spec:
  scaleTargetRef:
    apiVersion: argoproj.io/v1alpha1
    kind: Rollout
    name: triton-rollout
  minReplicaCount: 2   # 至少 2 个实例预热
  maxReplicaCount: 12
  triggers:
  - type: prometheus
    metadata:
      serverAddress: http://prometheus.monitoring:9090
      metricName: gpu_util
      threshold: '65'
      query: |
        avg(avg_over_time(nv_gpu_utilization{job="triton"}[2m])) by (pod)
  - type: prometheus
    metadata:
      metricName: queue_latency_p99
      threshold: '300'  # 单位 ms
      query: |
        histogram_quantile(0.99, sum(rate(triton_request_queue_duration_ms_bucket[1m])) by (le))
  advanced:
    horizontalPodAutoscalerConfig:
      behavior:
        scaleDown:
          stabilizationWindowSeconds: 300   # 5 分钟稳定窗口
          policies:
          - type: Percent
            value: 10
            periodSeconds: 60

apiVersion: apps/v1
kind: Deployment
metadata: name: model-warmup
spec:
  replicas: 2
  template:
    spec:
      containers:
      - name: warm
        image: registry.k8s.io/pause:3.9
        volumeMounts:
        - name: model-pvc
          mountPath: /models
      volumes:
      - name: model-pvc
        persistentVolumeClaim: { claimName: llama-model-pvc }