安装Kubernetes1.31.0多Master集群

准备工作

修改HostName

hostnamectl set-hostname k8s-master01

hostnamectl set-hostname k8s-master02

hostnamectl set-hostname k8s-master03

hostnamectl set-hostname k8s-node01

hostnamectl set-hostname k8s-node02

hostnamectl set-hostname k8s-node03

所有节点添加Host解析

cat >>/etc/hosts<<EOF
192.168.6.254 k8s-master01
192.168.7.2 k8s-master02
192.168.7.247 k8s-master03
192.168.6.86 k8s-node01
192.168.7.154 k8s-node02
192.168.6.164 k8s-node03
EOF

关闭防火墙

systemctl stop firewalld && systemctl disable firewalld

关闭SELinux

sed -i 's/^SELINUX=enforcing$/SELINUX=permissive/' /etc/selinux/config
setenforce 0

禁用交换分区

swapoff -a
sed -i '/swap/s/^/#/' /etc/fstab

设置内核参数

tee /etc/sysctl.d/k8s.conf <<EOF
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward = 1
EOF

modprobe overlay
modprobe br_netfilter

sysctl --system

更新系统并安装基础工具

apt update && sudo apt upgrade -y
apt install -y apt-transport-https ca-certificates curl gnupg2 software-properties-common

安装Containerd

apt install -y containerd

初始化Containerd配置文件

mkdir -p /etc/containerd
containerd config default | tee /etc/containerd/config.toml > /dev/null
sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml

启动Containerd

systemctl start containerd
systemctl enable containerd

改变默认容器运行时并增加环境变量

export CONTAINER_RUNTIME_ENDPOINT=unix:///run/containerd/containerd.sock

vim ~/.bashrc

export CONTAINER_RUNTIME_ENDPOINT=unix:///run/containerd/containerd.sock

安装Kubeadm, Kubelet, Kubectl

curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.31/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.31/deb/ /' | sudo tee /etc/apt/sources.list.d/kubernetes.list
apt-get update
apt-get install -y kubelet kubeadm kubectl
apt-mark hold kubelet kubeadm kubectl

配置Keepalived

注意

只要涉及到VIP操作，都需要看云平台文档或者问虚拟网络管理员需不需要配置VIP注册和绑定.

在Master节点外的服务器安装Keepavlied

apt install -y keepalived

Keepalived01配置

vim /etc/keepalived/keepalived.conf

! Configuration File for keepalived
global_defs {
   router_id Chair01
}
vrrp_instance VI_1 {
    state MASTER
    interface eth0
    virtual_router_id 50
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.1.200
    }
}

启动服务

systemctl enable keepalived
systemctl start keepalived

Keepalived02配置

vim /etc/keepalived/keepalived.conf

! Configuration File for keepalived
global_defs {
   router_id Chair02
}
vrrp_instance VI_1 {
    state BACKUP
    interface eth0
    virtual_router_id 50
    priority 90
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.1.200
    }
}

启动服务

systemctl enable keepalived
systemctl start keepalived

Keepalived03配置

vim /etc/keepalived/keepalived.conf

! Configuration File for keepalived
global_defs {
   router_id Chair03
}
vrrp_instance VI_1 {
    state BACKUP
    interface eth0
    virtual_router_id 50
    priority 80
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.1.200
    }
}

启动服务

systemctl enable keepalived
systemctl start keepalived

配置HAproxy

在Keepalived服务器上做HAproxy安装

apt install keepalived haproxy psmisc -y
cp -p /etc/haproxy/haproxy.cfg /etc/haproxy/haproxy.cfg.bak

HAProxy01配置

vim /etc/haproxy/haproxy.cfg

global
    log /dev/log    local0
    log /dev/log    local1 notice
    chroot /var/lib/haproxy
    stats socket /run/haproxy/admin.sock mode 660 level admin
    stats timeout 30s
    user haproxy
    group haproxy
    daemon

defaults
    log     global
    mode    tcp
    option  tcplog
    option  dontlognull
    timeout connect 5000
    timeout client  50000
    timeout server  50000

frontend kubernetes-frontend
    bind 192.168.1.200:6443
    default_backend kubernetes-backend

backend kubernetes-backend
    balance roundrobin
    option tcp-check
    option httpchk GET /healthz  # 健康检查
    server k8s-master01 192.168.1.8:6443 check fall 3 rise 2 ssl verify none
    server k8s-master02 192.168.1.10:6443 check fall 3 rise 2 ssl verify none
    server k8s-master03 192.168.1.9:6443 check fall 3 rise 2 ssl verify none

确保/run/haproxy/目录存在，并且具有正确的权限

mkdir -p /run/haproxy
chown haproxy:haproxy /run/haproxy
chmod 755 /run/haproxy

启动HAProxy服务

systemctl daemon-reload
systemctl enable haproxy
systemctl start haproxy

HAProxy02配置

vim /etc/haproxy/haproxy.cfg

global
    log /dev/log    local0
    log /dev/log    local1 notice
    chroot /var/lib/haproxy
    stats socket /run/haproxy/admin.sock mode 660 level admin
    stats timeout 30s
    user haproxy
    group haproxy
    daemon

defaults
    log     global
    mode    tcp
    option  tcplog
    option  dontlognull
    timeout connect 5000
    timeout client  50000
    timeout server  50000

frontend kubernetes-frontend
    bind 192.168.1.200:6443
    default_backend kubernetes-backend

backend kubernetes-backend
    balance roundrobin
    option tcp-check
    option httpchk GET /healthz  # 健康检查
    server k8s-master01 192.168.1.8:6443 check fall 3 rise 2 ssl verify none
    server k8s-master02 192.168.1.10:6443 check fall 3 rise 2 ssl verify none
    server k8s-master03 192.168.1.9:6443 check fall 3 rise 2 ssl verify none

确保/run/haproxy/目录存在，并且具有正确的权限

mkdir -p /run/haproxy
chown haproxy:haproxy /run/haproxy
chmod 755 /run/haproxy

启动HAProxy服务

systemctl daemon-reload
systemctl enable haproxy
systemctl start haproxy

HAproxy03配置

vim /etc/haproxy/haproxy.cfg

global
    log /dev/log    local0
    log /dev/log    local1 notice
    chroot /var/lib/haproxy
    stats socket /run/haproxy/admin.sock mode 660 level admin
    stats timeout 30s
    user haproxy
    group haproxy
    daemon

defaults
    log     global
    mode    tcp
    option  tcplog
    option  dontlognull
    timeout connect 5000
    timeout client  50000
    timeout server  50000

frontend kubernetes-frontend
    bind 192.168.1.200:6444
    default_backend kubernetes-backend

backend kubernetes-backend
    balance roundrobin
    option tcp-check
    option httpchk GET /healthz  # 健康检查
    server k8s-master01 192.168.1.8:6443 check fall 3 rise 2 ssl verify none
    server k8s-master02 192.168.1.10:6443 check fall 3 rise 2 ssl verify none
    server k8s-master03 192.168.1.9:6443 check fall 3 rise 2 ssl verify none

确保/run/haproxy/目录存在，并且具有正确的权限

mkdir -p /run/haproxy
chown haproxy:haproxy /run/haproxy
chmod 755 /run/haproxy

启动haproxy

systemctl daemon-reload
systemctl enable haproxy
systemctl start haproxy

检查haproxy状态

systemctl status haproxy -l

初始化Kubernetes集群

根据kubeadm-config文件启动集群

vim kubeadm-config.yaml

apiVersion: kubeadm.k8s.io/v1beta4
kind: ClusterConfiguration
kubernetesVersion: 1.31.0
caCertificateValidityPeriod: 87600h0m0s      # 证书有效期
certificateValidityPeriod: 87600h0m0s        # 证书有效期
controlPlaneEndpoint: "192.168.88.129:6443"  # 控制平面 API 端点，可以用负载均衡的地址
networking:
  podSubnet: "10.244.0.0/16"                  # Flannel
  serviceSubnet: "10.96.0.0/12"

kubeadm init --config kubeadm-config.yaml

根据提示操作

export KUBECONFIG=/etc/kubernetes/admin.conf
vim .bashrc
export KUBECONFIG=/etc/kubernetes/admin.conf

在所有需要以Master身份加入机器的服务器上创建文件夹

mkdir /etc/kubernetes/pki
mkdir /etc/kubernetes/pki/etcd

从Master01复制密钥文件到Master02,03

scp /etc/kubernetes/admin.conf root@172.17.32.6:/etc/kubernetes
scp /etc/kubernetes/pki/{ca.*,sa.*,front-proxy-ca.*} root@172.17.32.6:/etc/kubernetes/pki
scp /etc/kubernetes/pki/etcd/ca.* root@172.17.32.6:/etc/kubernetes/pki/etcd
scp /etc/kubernetes/admin.conf root@172.17.32.16:/etc/kubernetes
scp /etc/kubernetes/pki/{ca.*,sa.*,front-proxy-ca.*} root@172.17.32.16:/etc/kubernetes/pki
scp /etc/kubernetes/pki/etcd/ca.* root@172.17.32.16:/etc/kubernetes/pki/etcd

以Master加入集群

kubeadm join x.x.x.x:6443 --token z0z4uk.39ag2d6z66itsct5 --discovery-token-ca-cert-hash sha256:cf3204ecf3e39eec4d19263ce666f8e266abb5cffbc89ac1edc84e07216d7ff4 --control-plane

以Node加入集群

kubeadm join 192.168.1.200:6443 --token z0z4uk.39ag2d6z66itsct5 --discovery-token-ca-cert-hash sha256:cf3204ecf3e39eec4d19263ce666f8e266abb5cffbc89ac1edc84e07216d7ff4

创建Flannel Pod

kubectl apply -f https://github.com/flannel-io/flannel/releases/latest/download/kube-flannel.yml

稍等一会，使用kubeget get node获取Node状态

Centos7 配置NFS

安装服务（服务端和客户端都要安装）

yum -y install nfs-utils  rpcbind

服务端配置

vim /etc/exports

=======================================================================================================================================================
/JOJO *(rw,no_root_squash,no_all_squash,sync)

提示

权限说明： rw：read-write，可读写； ro：read-only，只读； sync：文件同时写入硬盘和内存；由于我们的日志需要同步写，所有用sync选项。 async：文件暂存于内存，而不是直接写入内存； no_root_squash：NFS客户端连接服务端时如果使用的是root的话，那么对服务端分享的目录来说，也拥有root权限。显然开启这项是不安全的。 root_squash：NFS客户端连接服务端时如果使用的是root的话，那么对服务端分享的目录来说，拥有匿名用户权限，通常他将使用nobody或nfsnobody身份； all_squash：不论NFS客户端连接服务端时使用什么用户，对服务端分享的目录来说都是拥有匿名用户权限； anonuid：匿名用户的UID值，通常是nobody或nfsnobody，可以在此处自行设定； anongid：匿名用户的GID值。

使配置生效

exportfs  -r

启动服务

systemctl start rpcbind && systemctl enable rpcbind
systemctl start nfs && systemctl enable nfs

查看服务端共享配置

showmount -e x.x.x.x

Kubernetes常用指令

配置kubernetes命令自动补全

yum install -y bash-completion  
source /usr/share/bash-completion/bash_completion  
source <(kubectl completion bash)  
echo "source <(kubectl completion bash)" >> ~/.bashrc

查看命名空间

kubectl get namespaces

查看deployment

kubectl get deployment -A

查看所有pod

kubectl get pods -A

可以指定命令空间

kubectl get pods -n pod所在的namespaces

使用yml文件创建一个资源，可以是Services，Configmaps，Pod等

kubectl apply -f 资源yml文件

如果pod的Kind是Deployment的话，可以通过delete的方式来重启pod

kubectl delete pod -n pod所在的命名空间 pod名字

删除pod

kubectl delete pod -n pod所在的命名空间 pod名字

提示

如果pod的Kind是Deployment的话，要删除pod就要先删除Deployment

kubectl delete -f Deployment资源yml文件

查看一个pod的详情信息

kubectl describe pod -n namespaces podname

查看一个pod的yaml文件

kubectl get pod -n Namespase PodName -o yaml

停止一个pod

kubectl scale --replicas=0 -n pod所在的命名空间 deployment/要更改的deployment名称

相关信息

--replicas=0：指定副本数为0，同理，要恢复的话就改为N

创建Configmaps-从文件创建

kubectl create cm -n NameSpace CmName --from-file=test

配置Master节点可以运行Pod

kubectl taint nodes <master-node-name> node-role.kubernetes.io/control-plane:NoSchedule-

配置Master节点不允许调度Pod

kubectl taint nodes <node-name> node-role.kubernetes.io/control-plane:NoSchedule

新节点加入集群

相关信息

以下操作在新增节点操作

关闭防火墙

systemctl stop firewalld
systemctl disable firewalld

修改主机名

hostnamectl set-hostname xxxxx

关闭selinux

sed -i 's/enforcing/disabled/' /etc/selinux/config
setenforce 0

关闭swap分区

swapoff -a
sed -i 's/.\\*swap.\\*/#&/' /etc/fstab

修改内核参数

cat > /etc/sysctl.d/k8s.conf << EOF
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward=1
vm.max_map_count=262144
EOF

modprobe br_netfilter

sysctl -p /etc/sysctl.d/k8s.conf

修改yum源

curl -o /etc/yum.repos.d/CentOS-7.repo http://mirrors.aliyun.com/repo/Centos-7.repo
curl -o /etc/yum.repos.d/docker-ce.repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo

cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

yum clean all && yum makecache

安装docker

curl -sSL https://get.daocloud.io/docker | sh

修改docker daemon配置文件

mkdir -p /etc/docker

tee /etc/docker/daemon.json <<-'EOF'
{
  "registry-mirrors": ["https://sb2pbegg.mirror.aliyuncs.com"]
}
EOF

启动docker

systemctl enable docker && systemctl start docker

安装kubectl,kubelet,kubeadm

yum install kubelet-x.xx.x kubeadm-x.xx.x kubectl-x.xx.x -y

设置kubelet开机启动

systemctl enable kubelet

相关信息

以下操作在Master节点操作

查看证书

kubeadm token list

如果没有输出，可能是证书过期了，重新生成证书

kubeadm token create
kubeadm token list

获取证书的hash值

openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //'

拼接

kubeadm join x.x.x.x:6443 --token {查看证书} --discovery-token-ca-cert-hash sha256:{获取证书的hash值}

提示

追加参数"--control-plane"实现以master身份加入k8s集群

Kubernetes 1.19.0 更新集群证书

手动更新证书，在集群证书未过期的情况下

查看证书过期时间

kubeadm certs check-expiration

生成集群Yaml文件

kubectl -n kube-system get configmap kubeadm-config -o jsonpath='{.downloads.ClusterConfiguration}' > kubeadm-config.yaml

备份原有证书

cp -r /etc/kubernetes /root/kubernetes.bak

删除原有证书

rm -rf /etc/kubernetes/pki/{apiserver.crt,apiserver.key,apiserver-kubelet-client.crt,apiserver-kubelet-client.key,front-proxy-client.crt,front-proxy-client.key}

生成新的证书

kubeadm init phase certs all --config kubeadm-config.yaml

删除所有Master机器上的ETCD相关证书

rm -rf /etc/kubernetes/pki/apiserver-etcd-client.*
rm -rf /etc/kubernetes/pki/etcd/peer.*
rm -rf /etc/kubernetes/pki/etcd/server.*
rm -rf /etc/kubernetes/pki/etcd/healthcheck-client.*

重新生成所有Mster机器上ETCD相关证书

kubeadm init phase certs apiserver-etcd-client --config=/root/kubeadm-config.yaml
kubeadm init phase certs etcd-healthcheck-client --config=/root/kubeadm-config.yaml
kubeadm init phase certs etcd-peer --config=/root/kubeadm-config.yaml
kubeadm init phase certs etcd-server --config=/root/kubeadm-config.yaml

删除所有Master节点上的scheduler.conf controller-manager.conf admin.conf kubelet.conf

rm -rf /etc/kubernetes/admin.conf 
rm -rf /etc/kubernetes/controller-manager.conf 
rm -rf /etc/kubernetes/scheduler.conf
rm -rf /etc/kubernetes/kubelet.conf

在所有Master节点上配置新的KubeConfig

kubeadm init phase kubeconfig all --config=/root/kubeadm-config.yaml

cp -i /etc/kubernetes/admin.conf $HOME/.kube/config

重启所有Master节点上的Kubelet

systemctl daemon-reload
systemctl restart kubelet

检查证书过期时间

kubeadm certs check-expiration

相关信息

以上操作需要在所有Master节点操作

复制Master机器上的ca证书到所有Node节点上

scp /etc/kubernetes/pki/ca.* user@ipaddress:/root/

使用以下命令生成新的kubelet.conf

kubeadm init phase kubeconfig kubelet --apiserver-advertise-address=<control-plane-host> --kubeconfig-dir=/root/ --cert-dir=/root/

备份原有kubelet.conf

cp /etc/kubernetes/kubelet.conf /etc/kubernetes/kubelet.conf.bak

使用新生成的kubelet.conf覆盖旧的kubelet.conf

cp /root/kubelet.conf /etc/kubernetes/kubelet.conf

重启kubelet服务

systemctl daemon-reload
systemctl restart kubelet

重启所有节点上的kube-proxy

kubectl get pods -n kube-system -l k8s-app=kube-proxy -o jsonpath='{.items[*].metadownloads.name}' | xargs -n 1 kubectl delete pod -n kube-system

相关信息

到这里就更新完成了，可能更新后要稍等几分钟，如果还是不行的话建议看看组件容器的日志

手动更新证书，在集群证书已经过期的情况下

相关信息

下列操作是基于多Master集群

备份原有证书

cp -r /etc/kubernetes/pki/ /etc/kubernetes/pki_backup

删除原有证书

rm -rf /etc/kubernetes/pki/{apiserver.crt,apiserver.key,apiserver-kubelet-client.crt,apiserver-kubelet-client.key,front-proxy-client.crt,front-proxy-client.key}

使用kubeadm-config.yaml文件生成新的证书

kubeadm init phase certs apiserver --config=/root/kubeadm-config.yaml
kubeadm init phase certs front-proxy-client --config=/root/kubeadm-config.yaml
kubeadm init phase certs apiserver-kubelet-client --config=/root/kubeadm-config.yaml

删除原有组件conf

rm -rf /etc/kubernetes/scheduler.conf
rm -rf /etc/kubernetes/controller-manager.conf
rm -rf /etc/kubernetes/kubelet.conf
rm -rf /etc/kubernetes/admin.conf

根据kubeadm-config.yaml生成新的证书

kubeadm init phase kubeconfig all --config=/root/kubeadm-config.yaml

替换新生成的admin.conf

cp -i /etc/kubernetes/admin.conf $HOME/.kube/config

重启Kubelet

systemctl daemon-reload
systemctl restart kubelet

相关信息

上述操作需要在每个Master执行

更新ETCD证书

kubeadm init phase certs etcd-ca --config /root/kubeadm-config.yaml

复制证书到其他Master节点

scp /etc/kubernetes/pki/etcd/ca.* user@ipaddress:/etc/kubernetes/pki/etcd/

创建sign文件，在每个Master节点操作

vim /etc/kubernetes/pki/etcd/sign.conf

subjectAltName = DNS.1:k8s-master01,DNS.2:localhost,IP.1:192.168.1.11,IP.2:127.0.0.1,IP.3:0:0:0:0:0:0:0:1

注意

每个Master节点都是不一样的，按自己的配置填

生成相关证书，在每个Master节点操作

openssl req -new -key healthcheck-client.key -out healthcheck-client.csr

openssl req -new -key peer.key -out peer.csr

openssl req -new -key server.key -out server.csr

openssl x509 -in /etc/kubernetes/pki/etcd/server.crt -text -noout

openssl x509 -req -days 3650 -in healthcheck-client.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out healthcheck-client.crt -extfile sign.conf
openssl x509 -req -days 3650 -in peer.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out peer.crt -extfile sign.conf
openssl x509 -req -days 3650 -in server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out server.crt -extfile sign.conf
openssl req -new -key /etc/kubernetes/pki/apiserver-etcd-client.key -subj "/CN=kube-apiserver-etcd-client" -out /etc/kubernetes/pki/apiserver-etcd-client.csr
openssl x509 -req -in /etc/kubernetes/pki/apiserver-etcd-client.csr -CA /etc/kubernetes/pki/etcd/ca.crt -CAkey /etc/kubernetes/pki/etcd/ca.key -CAcreateserial -out /etc/kubernetes/pki/apiserver-etcd-client.crt -days 365

查看生成的证书信息

openssl x509 -in /etc/kubernetes/pki/etcd/server.crt -text -noout

重启Etcd和ApiServer容器

相关信息

此时应该可以使用kubectl命令了，但是还无法调度pod

配置Scheduler和Controller-Manager证书，所有Master节点操作

kubeadm alpha certs renew scheduler.conf --config=/root/kubeadm-config.yaml
kubeadm alpha certs renew controller-manager.conf --config=/root/kubeadm-config.yaml

重启Scheduler和Controller-Manager容器

提示

如果有Node处于NotReady，就要尝试重新生成Node的Kubelet.conf

首先要把Master节点上的/etc/kubernetes/pki/ca.crt复制到问题Node上

scp /etc/kubernetes/pki/ca.* user@ipaddress:/root/

然后使用下列命令生成为Node生成新的conf

kubeadm init phase kubeconfig kubelet --apiserver-advertise-address=<control-plane-host> --kubeconfig-dir=/root/ --cert-dir=/root/

将新生成的 kubelet.conf 文件复制到默认位置 /etc/kubernetes。

cp /root/kubelet.conf /etc/kubernetes/

重启Kubelete

systemctl daemon-reload
systemctl restart kubelet