Kubernetes高可用集群部署实战：基于kubeadm + 腾讯云CLB的生产级方案

# 关闭swap（kubelet强制要求）
swapoff -a && sed -i '/swap/d' /etc/fstab

# 关闭防火墙（或放行必要端口：6443,2379-2380,10250,10259,10257等）
systemctl stop firewalld && systemctl disable firewalld

# 设置内核参数
cat <<EOF | tee /etc/sysctl.d/k8s.conf
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
vm.swappiness = 0
EOF
sysctl --system

# 加载br_netfilter模块
modprobe br_netfilter && lsmod | grep br_netfilter

# 安装基础工具
yum install -y wget curl vim git conntrack-tools

# 安装containerd
yum install -y yum-utils
yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
yum install -y containerd.io

# 生成默认配置并修改cgroup驱动为systemd（与kubelet一致）
mkdir -p /etc/containerd
containerd config default > /etc/containerd/config.toml
sed -i 's/SystemdCgroup = false/SystemdCgroup = true/g' /etc/containerd/config.toml

# 配置pause镜像加速（腾讯云内网镜像）
sed -i 's/sandbox_image = .*/sandbox_image = "ccr.ccs.tencentyun.com/k8s/pause:3.9"/' /etc/containerd/config.toml

# 启动containerd
systemctl daemon-reload && systemctl enable containerd --now

cat <<EOF | tee /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.tencent.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://mirrors.tencent.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.tencent.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

yum install -y kubeadm-1.28.8 kubelet-1.28.8 kubectl-1.28.8 --disableexcludes=kubernetes
systemctl enable kubelet --now

10.0.1.100 apiserver.cluster.local

# init-config.yaml
apiVersion: kubeadm.k8s.io/v1beta3
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: 10.0.1.10   # 本机内网IP
  bindPort: 6443
nodeRegistration:
  name: k8s-master-1
  criSocket: unix:///var/run/containerd/containerd.sock
  taints:
  - effect: NoSchedule
    key: node-role.kubernetes.io/control-plane
---
apiVersion: kubeadm.k8s.io/v1beta3
kind: ClusterConfiguration
kubernetesVersion: v1.28.8
controlPlaneEndpoint: "apiserver.cluster.local:6443"  # CLB VIP域名
apiServer:
  extraArgs:
    bind-address: 0.0.0.0
    # 启用etcd健康检查等
  certSANs:
  - apiserver.cluster.local
  - 10.0.1.100   # CLB VIP
  - 127.0.0.1
  - 10.0.1.10
  - 10.0.1.11
  - 10.0.1.12
controllerManager:
  extraArgs:
    bind-address: 0.0.0.0
scheduler:
  extraArgs:
    bind-address: 0.0.0.0
etcd:
  local:
    dataDir: /var/lib/etcd
    extraArgs:
      listen-client-urls: https://0.0.0.0:2379
      advertise-client-urls: https://10.0.1.10:2379
      listen-peer-urls: https://0.0.0.0:2380
      initial-advertise-peer-urls: https://10.0.1.10:2380
      initial-cluster: k8s-master-1=https://10.0.1.10:2380,k8s-master-2=https://10.0.1.11:2380,k8s-master-3=https://10.0.1.12:2380
      initial-cluster-state: new
      initial-cluster-token: etcd-cluster-1
---
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
cgroupDriver: systemd

# 查看所需镜像
kubeadm config images list --config=init-config.yaml
# 使用腾讯云镜像仓库拉取（替换registry）
kubeadm config images pull --config=init-config.yaml --image-repository=ccr.ccs.tencentyun.com/k8s

kubeadm init --config=init-config.yaml --upload-certs

mkdir -p $HOME/.kube
cp /etc/kubernetes/admin.conf $HOME/.kube/config
chown $(id -u):$(id -g) $HOME/.kube/config

kubectl get nodes  # 仅master-1，状态NotReady（未安装CNI）
kubectl get pods -n kube-system  # etcd、apiserver等均Running

kubeadm join apiserver.cluster.local:6443 --token <token> \
    --discovery-token-ca-cert-hash sha256:<hash> \
    --control-plane --certificate-key <cert-key> \
    --node-name k8s-master-2

# 在任一控制平面节点执行
kubectl exec -n kube-system etcd-k8s-master-1 -- etcdctl --endpoints=https://10.0.1.10:2379,https://10.0.1.11:2379,https://10.0.1.12:2379 \
    --cacert=/etc/kubernetes/pki/etcd/ca.crt \
    --cert=/etc/kubernetes/pki/etcd/server.crt \
    --key=/etc/kubernetes/pki/etcd/server.key \
    endpoint health

kubectl get pods -n kube-system -o wide | grep -E 'apiserver|controller-manager|scheduler'

# 安装Calico operator和资源
kubectl apply -f https://raw.githubusercontent.com/projectcalico/calico/v3.27.0/manifests/calico.yaml

# 等待所有pod Running
kubectl wait --for=condition=Ready pods --all -n calico-system --timeout=300s

kubectl get nodes  # 全部Ready

kubeadm join apiserver.cluster.local:6443 --token <token> \
    --discovery-token-ca-cert-hash sha256:<hash> \
    --node-name k8s-worker-1

kubectl get nodes -o wide
NAME            STATUS   ROLES           AGE   VERSION   INTERNAL-IP
k8s-master-1    Ready    control-plane   ...   v1.28.8   10.0.1.10
k8s-master-2    Ready    control-plane   ...   v1.28.8   10.0.1.11
k8s-master-3    Ready    control-plane   ...   v1.28.8   10.0.1.12
k8s-worker-1    Ready    <none>          ...   v1.28.8   10.0.1.20
k8s-worker-2    Ready    <none>          ...   v1.28.8   10.0.1.21

kubectl get nodes --kubeconfig=/etc/kubernetes/admin.conf  # 依然正常返回

kubectl exec -n kube-system etcd-k8s-master-2 -- etcdctl member list

kubectl delete pod -n kube-system kube-controller-manager-k8s-master-1
kubectl get lease -n kube-system kube-controller-manager -o yaml  # 查看holderIdentity变化

apiServer:
  extraArgs:
    audit-policy-file: /etc/kubernetes/audit-policy.yaml
    audit-log-path: /var/log/audit/kube-apiserver-audit.log
    audit-log-maxage: "30"
    audit-log-maxbackup: "10"
    audit-log-maxsize: "100"