基於docker和cri-dockerd部署kubernetes v1.25.3

• 2022 年 11 月 5 日
• 筆記
• Kubernetes

基於docker和cri-dockerd部署kubernetes v1.25.3

1、環境準備

1-1、主機清單

1-2、軟體清單

 docker-ce    20.10.21
 cri-dockerd  0.2.6
 kubeadm      1.25.3
 flannel      0.20.1

1-3、系統基礎環境

1-3-1、關閉防火牆

 #所有節點執行：
 [root@k8s-master01 ~]# ufw disable
 [root@k8s-master01 ~]# ufw status

1-3-2、時間同步

 #所有節點執行：
 [root@k8s-master01 ~]# apt install -y chrony
 [root@k8s-master01 ~]# systemctl restart chrony
 [root@k8s-master01 ~]# systemctl status chrony
 [root@k8s-master01 ~]# chronyc sources

1-3-3、主機名互相解析

 #所有節點執行：
 [root@k8s-master01 ~]# vim /etc/hosts
 10.0.0.101 k8s-master01 k8s-master01.wang.org kubeapi.wang.org kubeapi
 10.0.0.102 k8s-master02 k8s-master02.wang.org
 10.0.0.103 k8s-master03 k8s-master03.wang.org
 10.0.0.111 k8s-node01 k8s-node01.wang.org
 10.0.0.112 k8s-node02 k8s-node02.wang.org
 10.0.0.113 k8s-node03 k8s-node03.wang.org
 ​
 [root@k8s-master01 ~]# cat /etc/hosts        

1-3-4、禁用swap

 #所有節點執行：
 [root@k8s-master01 ~]# sed -r -i '/\/swap/s@^@#@' /etc/fstab
 [root@k8s-master01 ~]# swapoff -a
 [root@k8s-master01 ~]# systemctl --type swap
 ​
 #若不禁用Swap設備，需要在後續編輯kubelet的配置文件/etc/default/kubelet，設置其忽略Swap啟用的狀態錯誤，內容：KUBELET_EXTRA_ARGS="--fail-swap-on=false"

2、安裝docker

 #所有節點執行：
 ​
 #安裝必要的一些系統工具
 [root@k8s-master01 ~]# apt update
 [root@k8s-master01 ~]# apt -y install apt-transport-https ca-certificates curl software-properties-common
 ​
 #安裝GPG證書
 [root@k8s-master01 ~]# curl -fsSL //mirrors.aliyun.com/docker-ce/linux/ubuntu/gpg | apt-key add -
 OK
 #寫入軟體源資訊
 [root@k8s-master01 ~]# add-apt-repository "deb [arch=amd64] //mirrors.aliyun.com/docker-ce/linux/ubuntu $(lsb_release -cs) stable"
 ​
 #更新並安裝Docker-CE
 [root@k8s-master01 ~]# apt update
 [root@k8s-master01 ~]# apt install -y docker-ce

 #所有節點執行：
 kubelet需要讓docker容器引擎使用systemd作為CGroup的驅動，其默認值為cgroupfs，因而，我們還需要編輯docker的配置文件/etc/docker/daemon.json，添加如下內容，其中的registry-mirrors用於指明使用的鏡像加速服務。
 ​
 [root@k8s-master01 ~]# vim /etc/docker/daemon.json
 {
     "registry-mirrors": [
         "//docker.mirrors.ustc.edu.cn",
         "//hub-mirror.c.163.com",
         "//reg-mirror.qiniu.com",
         "//registry.docker-cn.com"
 ],
     "exec-opts": ["native.cgroupdriver=systemd"],
     "log-driver": "json-file",
     "log-opts": {
     "max-size": "200m"
 },
     "storage-driver": "overlay2"
 }
 ​
 [root@k8s-master01 ~]# systemctl daemon-reload
 [root@k8s-master01 ~]# systemctl start docker
 [root@k8s-master01 ~]# systemctl enable docker
 [root@k8s-master01 ~]# docker version
 Client: Docker Engine - Community
  Version:           20.10.21
 #註：kubeadm部署Kubernetes集群的過程中，默認使用Google的Registry服務k8s.gcr.io上的鏡像,由於2022年倉庫已經改為registry.k8s.io，中國可以直接訪問，所以現在不需要鏡像加速或者綠色上網就可以拉鏡像了，如果使用中國鏡像請參考//blog.51cto.com/dayu/5811307

3、安裝cri-dockerd

 #所有節點執行：
 #下載地址：//github.com/Mirantis/cri-dockerd
 [root@k8s-master01 ~]# apt install ./cri-dockerd_0.2.6.3-0.ubuntu-focal_amd64.deb -y
 ​
 #完成安裝後，相應的服務cri-dockerd.service便會自動啟動
 [root@k8s-master01 ~]#systemctl status cri-docker.service

4、安裝kubeadm、kubelet和kubectl

 #所有節點執行：
 #在各主機上生成kubelet和kubeadm等相關程式包的倉庫，可參考阿里雲官網
 [root@k8s-master01 ~]# apt update
 [root@k8s-master01 ~]# apt install -y apt-transport-https curl
 [root@k8s-master01 ~]# curl //mirrors.aliyun.com/kubernetes/apt/doc/apt-key.gpg | apt-key add -
 [root@k8s-master01 ~]#cat <<EOF >/etc/apt/sources.list.d/kubernetes.list
 > deb //mirrors.aliyun.com/kubernetes/apt/ kubernetes-xenial main
 > EOF
 ​
 ​
 #更新倉庫並安裝
 [root@k8s-master01 ~]# apt update 
 [root@k8s-master01 ~]# apt install -y kubelet kubeadm kubectl
 ​
 #注意：先不要啟動，只是設置開機自啟動
 [root@k8s-master01 ~]# systemctl enable kubelet
 ​
 #確定kubeadm等程式文件的版本
 [root@k8s-master01 ~]# kubeadm version
 kubeadm version: &version.Info{Major:"1", Minor:"25", GitVersion:"v1.25.3", GitCommit:"434bfd82814af038ad94d62ebe59b133fcb50506", GitTreeState:"clean", BuildDate:"2022-10-12T10:55:36Z", GoVersion:"go1.19.2", Compiler:"gc", Platform:"linux/amd64"}
 ​

5、整合kubelet和cri-dockerd

5-1、配置cri-dockerd

 #所有節點執行：
 ​
 [root@k8s-master01 ~]# vim /usr/lib/systemd/system/cri-docker.service
 ​
 #ExecStart=/usr/bin/cri-dockerd --container-runtime-endpoint fd://
 ExecStart=/usr/bin/cri-dockerd --pod-infra-container-image=registry.aliyuncs.com/google_containers/pause:3.8 --container-runtime-endpoint fd:// --network-plugin=cni --cni-bin-dir=/opt/cni/bin --cni-cache-dir=/var/lib/cni/cache --cni-conf-dir=/etc/cni/net.d
 ​
 ​
 ​
 #說明：
 需要添加的各配置參數（各參數的值要與系統部署的CNI插件的實際路徑相對應）：
     --network-plugin：指定網路插件規範的類型，這裡要使用CNI；
     --cni-bin-dir：指定CNI插件二進位程式文件的搜索目錄；
     --cni-cache-dir：CNI插件使用的快取目錄；
     --cni-conf-dir：CNI插件載入配置文件的目錄；
 配置完成後，重載並重啟cri-docker.service服務。
 ​
 [root@k8s-master01 ~]# systemctl daemon-reload && systemctl restart cri-docker.service
 [root@k8s-master01 ~]# systemctl status cri-docker
 ​

5-2、配置kubelet

 #所有節點執行：
 ​
 #配置kubelet，為其指定cri-dockerd在本地打開的Unix Sock文件的路徑，該路徑一般默認為「/run/cri-dockerd.sock「
 [root@k8s-master01 ~]# mkdir /etc/sysconfig
 [root@k8s-master01 ~]# vim /etc/sysconfig/kubelet
 KUBELET_KUBEADM_ARGS="--container-runtime=remote --container-runtime-endpoint=/run/cri-dockerd.sock"
 [root@k8s-master01 ~]# cat /etc/sysconfig/kubelet
 KUBELET_KUBEADM_ARGS="--container-runtime=remote --container-runtime-endpoint=/run/cri-dockerd.sock"
 ​
 #說明：該配置也可不進行，而是直接在後面的各kubeadm命令上使用「--cri-socket unix:///run/cri-dockerd.sock」選項

6、初始化第一個主節點

 #第一個主節點執行：
 ​
 #列出k8s所需要的鏡像
 [root@k8s-master01 ~]# kubeadm config images list
 registry.k8s.io/kube-apiserver:v1.25.3
 registry.k8s.io/kube-controller-manager:v1.25.3
 registry.k8s.io/kube-scheduler:v1.25.3
 registry.k8s.io/kube-proxy:v1.25.3
 registry.k8s.io/pause:3.8
 registry.k8s.io/etcd:3.5.4-0
 registry.k8s.io/coredns/coredns:v1.9.3
 ​
 #使用阿里雲拉取所需鏡像
 [root@k8s-master01 ~]# kubeadm config images pull --image-repository=registry.aliyuncs.com/google_containers --cri-socket unix:///run/cri-dockerd.sock
 ​
 [root@k8s-master01 ~]# docker images
 REPOSITORY                                                        TAG       IMAGE ID       CREATED     
 registry.aliyuncs.com/google_containers/kube-apiserver            v1.25.3   0346dbd74bcb   3 weeks ago 
 registry.aliyuncs.com/google_containers/kube-scheduler            v1.25.3   6d23ec0e8b87   3 weeks ago 
 registry.aliyuncs.com/google_containers/kube-controller-manager   v1.25.3   603999231275   3 weeks ago 
 registry.aliyuncs.com/google_containers/kube-proxy                v1.25.3   beaaf00edd38   3 weeks ago 
 registry.aliyuncs.com/google_containers/pause                     3.8       4873874c08ef   4 months ago
 registry.aliyuncs.com/google_containers/etcd                      3.5.4-0   a8a176a5d5d6   5 months ago
 registry.aliyuncs.com/google_containers/coredns                   v1.9.3    5185b96f0bec   5 months ago
 ​
 [root@k8s-master01 ~]# kubeadm init --control-plane-endpoint="kubeapi.wang.org" --kubernetes-version=v1.25.3 --pod-network-cidr=10.244.0.0/16 --service-cidr=10.96.0.0/12 --token-ttl=0 --cri-socket unix:///run/cri-dockerd.sock --upload-certs --image-repository registry.aliyuncs.com/google_containers
 ​
 #如提示以下資訊，代表初始化完成，請記錄資訊，以便後續使用：
 .....
 ​
 Your Kubernetes control-plane has initialized successfully!
 ​
 To start using your cluster, you need to run the following as a regular user:
 ​
   mkdir -p $HOME/.kube
   sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
   sudo chown $(id -u):$(id -g) $HOME/.kube/config
 ​
 Alternatively, if you are the root user, you can run:
 ​
   export KUBECONFIG=/etc/kubernetes/admin.conf
 ​
 You should now deploy a pod network to the cluster.
 Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
   //kubernetes.io/docs/concepts/cluster-administration/addons/
 ​
 You can now join any number of the control-plane node running the following command on each as root:
 ​
   kubeadm join kubeapi.wang.org:6443 --token s800nl.dtylo6tpgghpre7p \
     --discovery-token-ca-cert-hash sha256:a13ee9d0212edbd255fe0c5929186725b217a650f0b04ba75c6a1d6e67576aea \
     --control-plane --certificate-key 185090182962d3e322ff37a902734bfdd769e8e6e82d796bc882d0b1967c9886
 ​
 Please note that the certificate-key gives access to cluster sensitive data, keep it secret!
 As a safeguard, uploaded-certs will be deleted in two hours; If necessary, you can use
 "kubeadm init phase upload-certs --upload-certs" to reload certs afterward.
 ​
 Then you can join any number of worker nodes by running the following on each as root:
 ​
 kubeadm join kubeapi.wang.org:6443 --token s800nl.dtylo6tpgghpre7p \
     --discovery-token-ca-cert-hash sha256:a13ee9d0212edbd255fe0c5929186725b217a650f0b04ba75c6a1d6e67576aea 
 ​
 [root@k8s-master01 ~]#  mkdir -p $HOME/.kube
 [root@k8s-master01 ~]#  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
 [root@k8s-master01 ~]#  sudo chown $(id -u):$(id -g) $HOME/.kube/config
 ​

 #如果初始化報如下錯誤：
 Error getting node" err="node \"k8s-master01\" not found
 ​
 #1、在cri-docker.service文件指定下pause版本：
 [root@k8s-master01 ~]# vim /usr/lib/systemd/system/cri-docker.service
 ExecStart=/usr/bin/cri-dockerd --pod-infra-container-image=registry.aliyuncs.com/google_containers/pause:3.8 --container-runtime-endpoint fd:// --network-plugin=cni --cni-bin-dir=/opt/cni/bin --cni-cache-dir=/var/lib/cni/cache --cni-conf-dir=/etc/cni/net.d
 #2、重啟服務：
 systemctl daemon-reload
 systemctl restart cri-docker.service
 ​
 #3、重置集群：
 kubeadm reset --cri-socket unix:///run/cri-dockerd.sock && rm -rf /etc/kubernetes/ /var/lib/kubelet /var/lib/dockershim /var/run/kubernetes /var/lib/cni /etc/cni/net.d

7、部署網路插件

 #所有節點執行：
 #下載鏈接：
 //github.com/flannel-io/flannel/releases
 ​
 [root@k8s-master01 ~]# cp flanneld-amd64 /opt/bin/flanneld
 [root@k8s-master01 ~]# chmod +x /opt/bin/flanneld
 [root@k8s-master01 ~]# ll /opt/bin/flanneld
 -rwxr-xr-x 1 root root 39358256 11月  4 22:46 /opt/bin/flanneld*
 ​
 ​

 #第一個主節點執行：
 #部署kube-flannel
 [root@k8s-master01 ~]# kubectl apply -f //raw.githubusercontent.com/flannel-io/flannel/master/Documentation/kube-flannel.yml
 namespace/kube-flannel created
 clusterrole.rbac.authorization.k8s.io/flannel created
 clusterrolebinding.rbac.authorization.k8s.io/flannel created
 serviceaccount/flannel created
 configmap/kube-flannel-cfg created
 daemonset.apps/kube-flannel-ds created
 ​
 #確認Pod的狀態為「Running」
 [root@k8s-master01 ~]# kubectl get pods -n kube-flannel
 NAME                    READY   STATUS    RESTARTS   AGE
 kube-flannel-ds-9bkgl   1/1     Running   0          50s
 ​
 #此時，k8s-master01已經就緒
 [root@k8s-master01 ~]# kubectl get nodes
 NAME           STATUS   ROLES           AGE   VERSION
 k8s-master01   Ready    control-plane   20m   v1.25.3

8、添加其他節點到集群中

 #k8s-master02和k8s-master03執行：
 #k8s-master02和k8s-master03加入集群
 [root@k8s-master02 ~]# kubeadm join kubeapi.wang.org:6443 --token s800nl.dtylo6tpgghpre7p --discovery-token-ca-cert-hash sha256:a13ee9d0212edbd255fe0c5929186725b217a650f0b04ba75c6a1d6e67576aea --control-plane --certificate-key 185090182962d3e322ff37a902734bfdd769e8e6e82d796bc882d0b1967c9886 --cri-socket unix:///run/cri-dockerd.sock
 ​
 #注意，命令需要加上--cri-socket unix:///run/cri-dockerd.sock

 #k8s-node01、k8s-node02、k8s-node03執行
 #node節點加入集群
 ​
 [root@k8s-node01 ~]# kubeadm join kubeapi.wang.org:6443 --token s800nl.dtylo6tpgghpre7p --discovery-token-ca-cert-hash sha256:a13ee9d0212edbd255fe0c5929186725b217a650f0b04ba75c6a1d6e67576aea --cri-socket unix:///run/cri-dockerd.sock
 ​
 #注意，命令需要加上--cri-socket unix:///run/cri-dockerd.sock

 #第一節點驗證節點添加結果
 ​
 [root@k8s-master01 ~]#kubectl get nodes
 NAME           STATUS   ROLES           AGE     VERSION
 k8s-master01   Ready    control-plane   25m     v1.25.3
 k8s-master02   Ready    control-plane   10m     v1.25.3
 k8s-master03   Ready    control-plane   8m41s   v1.25.3
 k8s-node01     Ready    <none>          6m54s   v1.25.3
 k8s-node02     Ready    <none>          6m31s   v1.25.3
 k8s-node03     Ready    <none>          6m5s    v1.25.3
 ​

9、部署nginx測試

 [root@k8s-master01 ~]#kubectl create deployment nginx --image nginx:alpine --replicas=3
 deployment.apps/nginx created
 ​
 [root@k8s-master01 ~]#kubectl get pods
 NAME                     READY   STATUS    RESTARTS   AGE
 nginx-55f494c486-4js9n   1/1     Running   0          58s
 nginx-55f494c486-fsgxq   1/1     Running   0          58s
 nginx-55f494c486-z2gzv   1/1     Running   0          58s
 ​
 [root@k8s-master01 ~]#kubectl get pods -o wide
 NAME                     READY   STATUS    RESTARTS   AGE     IP           NODE         NOMINATED NODE   READINESS GATES
 nginx-55f494c486-4js9n   1/1     Running   0          4m31s   10.244.3.2   k8s-node01   <none>           <none>
 nginx-55f494c486-fsgxq   1/1     Running   0          4m31s   10.244.4.2   k8s-node02   <none>           <none>
 nginx-55f494c486-z2gzv   1/1     Running   0          4m31s   10.244.5.2   k8s-node03   <none>           <none>
 ​
 [root@k8s-master01 ~]#curl 10.244.4.2
 <!DOCTYPE html>
 <html>
 <head>
 <title>Welcome to nginx!</title>
 <style>
 html { color-scheme: light dark; }
 body { width: 35em; margin: 0 auto;
 font-family: Tahoma, Verdana, Arial, sans-serif; }
 </style>
 </head>
 <body>
 <h1>Welcome to nginx!</h1>
 <p>If you see this page, the nginx web server is successfully installed and
 working. Further configuration is required.</p>
 ​
 <p>For online documentation and support please refer to
 <a href="//nginx.org/">nginx.org</a>.<br/>
 Commercial support is available at
 <a href="//nginx.com/">nginx.com</a>.</p>
 ​
 <p><em>Thank you for using nginx.</em></p>
 </body>
 </html>
 ​
 ​
 #刪除pod，自動拉起
 [root@k8s-master01 ~]#kubectl delete pods nginx-55f494c486-fsgxq
 pod "nginx-55f494c486-fsgxq" deleted
 ​
 [root@k8s-master01 ~]#kubectl get pods -owide
 NAME                     READY   STATUS    RESTARTS   AGE   IP           NODE         NOMINATED NODE   READINESS GATES
 nginx-55f494c486-4js9n   1/1     Running   0          11m   10.244.3.2   k8s-node01   <none>           <none>
 nginx-55f494c486-xqcph   1/1     Running   0          16s   10.244.4.3   k8s-node02   <none>           <none>
 nginx-55f494c486-z2gzv   1/1     Running   0          11m   10.244.5.2   k8s-node03   <none>           <none>
 ​

 ​
 [root@k8s-master01 ~]#kubectl create service nodeport nginx --tcp=80:80
 service/nginx created
 ​
 [root@k8s-master01 ~]#kubectl get svc 
 NAME         TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)        AGE
 kubernetes   ClusterIP   10.96.0.1       <none>        443/TCP        89m
 nginx        NodePort    10.103.151.47   <none>        80:31901/TCP   18s
 [root@k8s-master01 ~]#curl 10.103.151.47
 <!DOCTYPE html>
 <html>
 <head>
 <title>Welcome to nginx!</title>
 <style>
 html { color-scheme: light dark; }
 body { width: 35em; margin: 0 auto;
 font-family: Tahoma, Verdana, Arial, sans-serif; }
 </style>
 </head>
 <body>
 <h1>Welcome to nginx!</h1>
 <p>If you see this page, the nginx web server is successfully installed and
 working. Further configuration is required.</p>
 ​
 <p>For online documentation and support please refer to
 <a href="//nginx.org/">nginx.org</a>.<br/>
 Commercial support is available at
 <a href="//nginx.com/">nginx.com</a>.</p>
 ​
 <p><em>Thank you for using nginx.</em></p>
 </body>
 </html>
 ​
 #外部訪問任意node幾點ip加埠（注意，隨機產生）都可以訪問