从 0 到 1 开发轻量化私有大模型：选型、微调、量化、部署

pip install torch transformers peft bitsandbytes datasets accelerate trl
# 量化与部署
pip install llama-cpp-python vllm fastapi uvicorn

from datasets import load_dataset

def to_chat(example, tokenizer):
    msgs = [
        {"role": "system", "content": "你是公司内部技术助手，只依据给定资料回答。"},
        {"role": "user", "content": example["instruction"] + "\n" + example.get("input", "")},
        {"role": "assistant", "content": example["output"]},
    ]
    text = tokenizer.apply_chat_template(msgs, tokenize=False)
    ids = tokenizer(text, truncation=True, max_length=2048)
    # 只对 assistant 部分算 loss：把 prompt 段 label 置 -100
    prompt_ids = tokenizer(
        tokenizer.apply_chat_template(msgs[:-1], tokenize=False, add_generation_prompt=True)
    )["input_ids"]
    ids["labels"] = [-100] * len(prompt_ids) + ids["input_ids"][len(prompt_ids):]
    return ids

import torch
from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training
from trl import SFTTrainer, SFTConfig

BASE = "Qwen/Qwen2.5-1.5B-Instruct"
tok = AutoTokenizer.from_pretrained(BASE, trust_remote_code=True)

bnb = BitsAndBytesConfig(
    load_in_4bit=True, bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype=torch.bfloat16, bnb_4bit_use_double_quant=True,
)
model = AutoModelForCausalLM.from_pretrained(
    BASE, quantization_config=bnb, device_map="auto", trust_remote_code=True,
)
model = prepare_model_for_kbit_training(model)

lora = LoraConfig(
    r=16, lora_alpha=32, lora_dropout=0.05, bias="none", task_type="CAUSAL_LM",
    target_modules=["q_proj", "k_proj", "v_proj", "o_proj",
                    "gate_proj", "up_proj", "down_proj"],
)
model = get_peft_model(model, lora)
model.print_trainable_parameters()   # 约 1%

ds = load_dataset("json", data_files="data/train.jsonl")["train"]
ds = ds.map(lambda x: to_chat(x, tok), remove_columns=ds.column_names)

trainer = SFTTrainer(
    model=model, train_dataset=ds,
    args=SFTConfig(
        output_dir="out/lora", num_train_epochs=3,
        per_device_train_batch_size=2, gradient_accumulation_steps=8,
        learning_rate=2e-4, lr_scheduler_type="cosine", warmup_ratio=0.03,
        logging_steps=10, save_strategy="epoch", bf16=True,
        gradient_checkpointing=True, max_length=2048,
    ),
)
trainer.train()
model.save_pretrained("out/lora"); tok.save_pretrained("out/lora")

from peft import PeftModel
from transformers import AutoModelForCausalLM

base = AutoModelForCausalLM.from_pretrained(BASE, torch_dtype=torch.bfloat16)
merged = PeftModel.from_pretrained(base, "out/lora").merge_and_unload()
merged.save_pretrained("out/merged"); tok.save_pretrained("out/merged")

# 转 GGUF 并量化（llama.cpp）
python convert_hf_to_gguf.py out/merged --outfile out/model-f16.gguf
./llama-quantize out/model-f16.gguf out/model-q4_k_m.gguf Q4_K_M

python -m llama_cpp.server --model out/model-q4_k_m.gguf \
  --host 0.0.0.0 --port 8000 --n_ctx 4096 --n_gpu_layers -1 \
  --chat_format qwen

from openai import OpenAI
cli = OpenAI(base_url="http://127.0.0.1:8000/v1", api_key="none")

def ask(q: str) -> str:
    r = cli.chat.completions.create(
        model="local",
        messages=[{"role": "system", "content": "你是公司内部技术助手。"},
                  {"role": "user", "content": q}],
        temperature=0.2, max_tokens=512,
    )
    return r.choices[0].message.content

def score(pred, ref):
    # 简易指标：关键词命中 + 长度合理 + 无复读
    hit = sum(k in pred for k in ref["keys"]) / max(len(ref["keys"]), 1)
    rep = 1 - pred.count(pred[:20]) / max(len(pred) // 20, 1)  # 复读惩罚
    return 0.7 * hit + 0.3 * max(rep, 0)

base_avg = sum(score(ask_base(c["q"]), c["ref"]) for c in cases) / len(cases)
ft_avg   = sum(score(ask(c["q"]),      c["ref"]) for c in cases) / len(cases)
print(f"base={base_avg:.3f} finetuned={ft_avg:.3f}")