AWS EKS 創建k8s生產環境實例

• 2022 年 8 月 3 日
• 筆記
• AWS, k8s, 公有雲

#AWS EKS 創建k8s生產環境實例

• 在AWS部署海外節點, 圖簡單使用web控制台創建VPC和k8s集群出錯(k8s), 使用cli命令行工具創建成功
• 本實例為復盤, 記錄aws命令行工具創建eks, 安裝efs驅動、LBS、ingress-nginx,使用ECR鏡像儲存等

#安裝命令行工具

#安裝aws cli
cd /tmp 
curl -kL "//awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
unzip awscliv2.zip
sudo ./aws/install
aws --version
#配置aws key
aws configure
#查看配置
aws configure list

#安裝kubectl
curl -o kubectl //s3.us-west-2.amazonaws.com/amazon-eks/1.22.6/2022-03-09/bin/linux/amd64/kubectl
chmod +x ./kubectl
mv kubectl /usr/local/bin
kubectl version --short --client

#安裝eksctl
curl --silent --location "//github.com/weaveworks/eksctl/releases/latest/download/eksctl_$(uname -s)_amd64.tar.gz" | tar xz -C /tmp
sudo mv /tmp/eksctl /usr/local/bin
eksctl version

#創建VPC網絡和子網

#創建VPC網絡和子網已單獨發帖
//www.cnblogs.com/elvi/p/16542406.html

#創建k8s集群

#env
k8s_name=aws-k8s
Region=ap-southeast-1 #新加坡 
#獲取aws賬戶id
OwnerId=$(aws ec2 describe-vpcs --region ${Region} |jq -r ".Vpcs[0].OwnerId")
#使用已有子網 
private-subnets-id="subnet-lan-a-xxx,subnet-lan-b-xxx"
public-subnets-id="subnet-public-a-xxx,subnet-public-b-xxx"

# k8s cluster 
eksctl create cluster \
 --region ${Region} \
 --name ${k8s_name} \
 --version 1.22 \
 --vpc-private-subnets ${private-subnets-id} \
 --vpc-public-subnets  ${public-subnets-id} \
 --managed \
 --without-nodegroup \
 --dry-run

# 查看
eksctl get cluster --name ${k8s_name} --region ${Region}

# 出錯或不要了,可刪除
# eksctl delete cluster --name=${k8s_name}

# --dry-run 試運行,正式創建時去掉
# --without-nodegroup 不創建node節點
# --vpc-xx 添加已有網絡,若不指定會自動創建
# 建議使用多個可用區網絡,k8s集群創建後無法更改
# eksctl create cluster --help #查看幫助

#創建k8s計算節點組

#創建b區k8s節點
#k8s nodegroup  test
eksctl create nodegroup \
 --region ${Region} \
 --cluster ${k8s_name} \
 --name k8s-work-test  \
 --node-type m5.large \
 --nodes 1 \
 --nodes-min 1 \
 --nodes-max 10 \
 --instance-name test-node-b \
 --node-ami-family Ubuntu2004 \
 --node-private-networking \
 --node-zones ${Region}b \
 --node-security-groups sg-xxxxxxx \
 --ssh-access \
 --ssh-public-key aws-bastion \
 --full-ecr-access \
 --managed \
 --dry-run

# --nodes 1 創建1個node節點, 規格 m5.large 2核8G
# --node-ami-family Ubuntu2004 操作系統Ubuntu20.04
# --node-private-networking 使用私有子網
# --node-zones 可用區
# --node-security-groups 使用已創建的安全組
# --full-ecr-access ECR鏡像倉庫權限,一定要
# eksctl create nodegroup --help #查看幫助

#節點擴容
eksctl scale nodegroup --region ${Region} \
 --cluster ${k8s_name} --nodes=2 --name k8s-work-test

# 測試正常就可以刪除, 創建配置更高的正式節點
# delete node
# eksctl delete nodegroup --cluster=${k8s_name} --name=k8s-work-test

#創建b區正式節點組 
eksctl create nodegroup \
 --region ${Region} \
 --cluster ${k8s_name} \
 --name k8s-work-b  \
 --node-type m5.4xlarge \
 --nodes 2 \
 --nodes-min 1 \
 --nodes-max 10 \
 --instance-name k8s-node-b \
 --max-pods-per-node 110 \
 --node-ami-family Ubuntu2004 \
 --node-private-networking \
 --node-zones ${Region}b \
 --node-security-groups sg-xxxxxxx \
 --ssh-access \
 --ssh-public-key aws-bastion \
 --full-ecr-access \
 --external-dns-access \
 --managed \
 --dry-run

#規格m5.4xlarge 16核64G
#node-zones創建多區,可用於高可用

#為k8s集群創建IAM OIDC提供商

# IAM OIDC即 AWS Identity and Access Management (IAM) OpenID Connect (OIDC)
# 創建IMA權限角色時,需要此功能開啟  

#查看是否有OIDC,沒有則創建
oidc_id=$(aws eks describe-cluster --name ${k8s_name} --query "cluster.identity.oidc.issuer" --output text |cut -d'/' -f 5)
if [ $(aws iam list-open-id-connect-providers | grep $oidc_id | wc -l ) -eq 0 ]; then
  eksctl utils associate-iam-oidc-provider --cluster ${k8s_name} --approve
fi

#eks安裝efs csi驅動

• k8s使用AWS EFS儲存時用到csi驅動
• efs可使用nfs協議掛載,但k8s節點默認沒安裝nfs客戶端

#創建IAM policy和角色
curl -o iam-policy-efs.json \
 //raw.githubusercontent.com/kubernetes-sigs/aws-efs-csi-driver/master/docs/iam-policy-example.json

aws iam create-policy \
    --policy-name EKS_EFS_CSI_Driver_Policy \
    --policy-document file://iam-policy-efs.json

#創建權限
eksctl create iamserviceaccount \
  --cluster ${k8s_name} \
  --namespace kube-system \
  --name efs-csi-controller-sa \
  --attach-policy-arn arn:aws:iam::${OwnerId}:policy/EKS_EFS_CSI_Driver_Policy \
  --approve \
  --region ${Region}

# 更新kubeconfig  ~/.kube/config  
aws eks update-kubeconfig --region ${Region} --name ${k8s_name}

#下載yaml文件
kubectl kustomize \
  "github.com/kubernetes-sigs/aws-efs-csi-driver/deploy/kubernetes/overlays/stable/?ref=release-1.4" > aws-eks-efs-csi.1.4.yaml

# vim aws-eks-efs-csi.1.4.yaml
# 手動刪除如下部分  
apiVersion: v1
kind: ServiceAccount
metadata:
  labels:
    app.kubernetes.io/name: aws-efs-csi-driver
  name: efs-csi-controller-sa
  namespace: kube-system
---

#部署efs csi
kubectl apply -f aws-eks-efs-csi.1.4.yaml

#使用efs創建pvc實例

apiVersion: v1
kind: PersistentVolume
metadata:
  name: aws-efs-test
spec:
  capacity:
    storage: 2000Gi
  accessModes: 
    - ReadWriteMany
  persistentVolumeReclaimPolicy: Retain
  csi:
    driver: efs.csi.aws.com
    volumeHandle: fs-xxx:/data
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: aws-efs-test
spec:
  accessModes: 
    - ReadWriteMany
  resources:
    requests:
      storage: 2000Gi

# fs-xxx 為efs實例id,需要單獨創建  
# 創建efs後需添加子網和安全組,否則無法訪問  

#安裝AWS LB Controller

• AWS LoadBalancer默認使用Classic Load Balancer模式
• 使用NLB、ALB模式的負載均衡器,和綁定EIP(綁定固定IP),必須安裝LB controller

#創建IAM角色
curl -o iam_lbs_v2.4.2.json \
  //raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.4.2/docs/install/iam_policy.json

aws iam create-policy \
  --policy-name iam_lbs_v2.4.2 \
  --policy-document file://iam_lbs_v2.4.2.json

eksctl create iamserviceaccount \
  --cluster=${k8s_name} \
  --namespace=kube-system \
  --name=aws-load-balancer-controller \
  --role-name "AmazonEKSLoadBalancerControllerRole" \
  --attach-policy-arn=arn:aws:iam::${OwnerId}:policy/iam_lbs_v2.4.2 \
  --approve

#安裝cert-manager
kubectl apply \
  --validate=false \
  -f //github.com/jetstack/cert-manager/releases/download/v1.5.4/cert-manager.yaml

#下載yaml
curl -Lo aws-load-balancer-controller_2.4.2.yaml \
  //github.com/kubernetes-sigs/aws-load-balancer-controller/releases/download/v2.4.2/v2_4_2_full.yaml

#更改k8s集群名稱
sed -i.bak -e "s|your-cluster-name|${k8s_name}|" aws-load-balancer-controller_2.4.2.yaml

#手動刪除如下部分
apiVersion: v1
kind: ServiceAccount
metadata:
  labels:
    app.kubernetes.io/component: controller
    app.kubernetes.io/name: aws-load-balancer-controller
  name: aws-load-balancer-controller
  namespace: kube-system
---

#部署lbs
kubectl apply -f aws-load-balancer-controller_2.4.2.yaml

#查看
kubectl get deployment -n kube-system aws-load-balancer-controller

#安裝ingress-nginx-controller

#下載yaml
curl -o aws-ingress-nginx.nlb.v1.3.0.yml \
  //raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.3.0/deploy/static/provider/aws/deploy.yaml

#增加spec.ipFamilyPolicy: SingleStack

#修改LoadBalancer部分的Service如下

---
apiVersion: v1
kind: Service
metadata:
  annotations:
    #負載均衡器自定義名稱
    service.beta.kubernetes.io/aws-load-balancer-name: k8s-ingress-slb
    #負載均衡 NLB模式
    service.beta.kubernetes.io/aws-load-balancer-type: "external"
    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: "ip"
    #使用EIP,互聯網模式
    service.beta.kubernetes.io/aws-load-balancer-scheme: "internet-facing"
    #public子網
    service.beta.kubernetes.io/aws-load-balancer-subnets: subnet-axxx, subnet-bxxx
    #彈性IP地址
    service.beta.kubernetes.io/aws-load-balancer-eip-allocations: eipalloc-axxx, eipalloc-bxxx
    #獲取客戶端真事IP
    service.beta.kubernetes.io/aws-load-balancer-target-group-attributes: preserve_client_ip.enabled=true
  labels:
    app.kubernetes.io/component: controller
    app.kubernetes.io/instance: ingress-nginx
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
    app.kubernetes.io/version: 1.3.0
  name: ingress-nginx-controller
  namespace: ingress-nginx
spec:
  type: LoadBalancer
  # externalTrafficPolicy: Local
  ipFamilyPolicy: SingleStack
  ipFamilies:
    - IPv4
  ports:
  - appProtocol: http
    name: http
    port: 80
    protocol: TCP
    targetPort: http
  - appProtocol: https
    name: https
    port: 443
    protocol: TCP
    targetPort: https
  selector:
    app.kubernetes.io/component: controller
    app.kubernetes.io/instance: ingress-nginx
    app.kubernetes.io/name: ingress-nginx

#部署
kubectl apply -f aws-ingress-nginx.nlb.v1.3.0.yml

#查看,獲得得到EXTERNAL-IP地址 
kubectl get svc ingress-nginx-controller -n ingress-nginx

#ping測試EXTERNAL-IP地址ip是否為自己的EIP地址
ping k8s-ingress-slb-xxx.elb.${Region}.amazonaws.com

#訪問測試
curl -I k8s-ingress-slb-xxx.elb.${Region}.amazonaws.com

#使用私有鏡像倉庫,並部署服務測試

#創建存儲庫nginx
aws ecr create-repository \
    --repository-name nginx \
    --region $Region

#登錄儲存庫(緩存的登錄憑證有效期12小時)
aws ecr get-login-password --region $Region \
 | docker login --username AWS --password-stdin ${OwnerId}.dkr.ecr.${Region}.amazonaws.com

#下載公共鏡像, 改tag為私有儲存庫地址
docker pull public.ecr.aws/nginx/nginx:alpine
docker tag  public.ecr.aws/nginx/nginx:alpine \
  ${OwnerId}.dkr.ecr.${Region}.amazonaws.com/nginx:alpine

#push鏡像到新建的儲存庫
docker push ${OwnerId}.dkr.ecr.${Region}.amazonaws.com/nginx:alpine

#deploy test
kubectl create deployment nginx --port=80 \
 --image=${OwnerId}.dkr.ecr.${Region}.amazonaws.com/nginx:alpine

#查看
kubectl get pod

#生命周期策略示例，保持5個鏡像版本(tag)
cat >aws-ecr-policy.json <<EOF
{
  "rules": [
    {
      "rulePriority": 1,
      "description": "Keep only 3 image",
      "selection": {
        "tagStatus": "any",
        "countType": "imageCountMoreThan",
        "countNumber": 3
      },
      "action": {
        "type": "expire"
      }
    }
  ]
}
EOF
#創建策略
aws ecr put-lifecycle-policy --region $Region \
  --repository-name nginx \
  --lifecycle-policy-text file://aws-ecr-policy.json 

#刪除清理pod
kubectl delete deploy/nginx

#刪除存儲庫
aws ecr delete-repository \
  --region $Region --force \
  --repository-name nginx

• k8s有pull私有鏡像倉庫權限,是因為創建參數–full-ecr-access
• AWS ECR鏡像儲存服務不支持目錄,只能分別給每個鏡像創建儲存庫
• aws ecr get-login-password生成的憑證有效期12小時,可使用定時任務每天登錄2次解決

參考文檔

• eks //docs.aws.amazon.com/zh_cn/eks/latest/userguide/create-cluster.html
• efs //docs.aws.amazon.com/zh_cn/eks/latest/userguide/efs-csi.html
• lbs //docs.aws.amazon.com/zh_cn/eks/latest/userguide/network-load-balancing.html
• ingress //kubernetes.github.io/ingress-nginx/deploy/#aws
• ECR //docs.aws.amazon.com/zh_cn/AmazonECR/latest/userguide/getting-started-cli.html