基于腾讯云TKE与vLLM的AI写作服务弹性伸缩与推理加速实践

# 在Deployment中增加COSFS挂载
volumes:
- name: model-storage
  flexVolume:
    driver: "cosfs"
    options:
      bucket: "ai-models-125xxxxxx"
      url: "https://cos.ap-guangzhou.myqcloud.com"
      path: "/lora"

python -m vllm.entrypoints.openai.api_server \
  --model Qwen/Qwen2-7B-Instruct \
  --enable-lora \
  --lora-modules marketing=/models/lora/marketing-v2 \
  --max-num-seqs 64 \
  --max-model-len 8192 \
  --gpu-memory-utilization 0.92 \
  --block-size 16 \
  --max-num-batched-tokens 16384 \
  --disable-log-requests  # 减少日志IO

from fastapi import FastAPI, BackgroundTasks
import redis.asyncio as redis
import time
import hashlib

app = FastAPI()
redis_client = redis.Redis(host=os.getenv("REDIS_HOST"), port=6379, decode_responses=True)

# 预先初始化vLLM引擎（使用AsyncLLMEngine）
from vllm import AsyncLLMEngine, SamplingParams
from vllm.lora.request import LoRARequest

engine = AsyncLLMEngine.from_engine_args(...)  # 同上

@app.post("/v1/writing")
async def writing(request: WritingRequest):
    # 缓存键
    cache_key = hashlib.md5(f"{request.prompt}_{request.temperature}".encode()).hexdigest()
    cached = await redis_client.get(cache_key)
    if cached:
        return {"result": cached, "source": "cache"}
    
    # 构造Qwen2聊天模板（使用transformers tokenizer）
    tokenizer = get_tokenizer()  # 全局单例
    messages = [{"role": "system", "content": "你是专业文案助手"}, {"role": "user", "content": request.prompt}]
    formatted = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
    
    sampling_params = SamplingParams(
        max_tokens=request.max_tokens,
        temperature=request.temperature,
        top_p=0.9,
        presence_penalty=0.1
    )
    lora_req = LoRARequest("marketing", 1, "/models/lora/marketing-v2")
    
    # 异步生成，设置超时
    try:
        result = await asyncio.wait_for(
            engine.generate(formatted, sampling_params, lora_request=lora_req),
            timeout=10.0
        )
        text = result.outputs[0].text
    except asyncio.TimeoutError:
        raise HTTPException(408, "Generation timeout")
    
    # 异步写入缓存（TTL=1小时）
    background_tasks.add_task(redis_client.setex, cache_key, 3600, text)
    return {"result": text, "source": "model"}

apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: writing-hpa
  namespace: prod
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: writing-deploy
  minReplicas: 2
  maxReplicas: 20
  metrics:
  - type: Pods
    pods:
      metric:
        name: qps_per_pod   # 由Prometheus adapter暴露
      target:
        type: AverageValue
        averageValue: 50    # 每个Pod目标QPS=50

nodeSelector:
  node.kubernetes.io/instance-type: gn7.2xlarge
tolerations:  # 允许调度到竞价节点
- key: "spot"
  operator: "Equal"
  value: "true"
  effect: "NoSchedule"

import structlog
logger = structlog.get_logger()
logger.info("generation_success", prompt_length=len(prompt), tokens=len(text), latency=latency_ms)

apiVersion: apps/v1
kind: Deployment
metadata:
  name: writing-deploy
spec:
  replicas: 3
  strategy:
    rollingUpdate:
      maxSurge: 1
      maxUnavailable: 0
  selector:
    matchLabels:
      app: writing
  template:
    metadata:
      labels:
        app: writing
    spec:
      containers:
      - name: vllm
        image: ccr.ccs.tencentyun.com/ai/writing:v2
        resources:
          limits:
            nvidia.com/gpu: 1
            memory: 64Gi
            cpu: 16
          requests:
            nvidia.com/gpu: 1
            memory: 32Gi
            cpu: 8
        ports:
        - containerPort: 8000
        env:
        - name: REDIS_HOST
          value: "redis.prod.svc.cluster.local"
        - name: COS_MOUNT
          value: "/models"
        volumeMounts:
        - name: model-storage
          mountPath: /models
        readinessProbe:
          httpGet:
            path: /health
            port: 8000
          initialDelaySeconds: 120
        lifecycle:
          preStop:
            exec:
              command: ["sh", "-c", "sleep 30"]  # 等待已有请求
      volumes:
      - name: model-storage
        flexVolume:
          driver: "cosfs"
          options:
            bucket: "ai-models-125xxxxxx"
            url: "https://cos.ap-guangzhou.myqcloud.com"
            path: "/"
      nodeSelector:
        node.kubernetes.io/instance-type: gn7.2xlarge
      tolerations:
      - key: "spot"
        operator: "Equal"
        value: "true"
        effect: "NoSchedule"