1、概述

本文檔主要介紹如何在k8s集群中部署prometheus server用來作為監控的數據採集服務器，這樣做可以很方便的對k8s集群中的指標、pod的、節點的指標進行採集和監控。

2、下載鏡像並且上傳到本地的habor

可以先將prometheus server的鏡像下載並且導入到本地的鏡像倉庫中。

鏡像版本信息

docker pull prom/prometheus:v2.33.1

3、節點添加標籤

為了保證prometheus的數據可以在重啟或者重新調度之後依然在一個主機上，可以考慮設置節點標籤，然後通過標籤的方式將pod綁定到該節點上，當然也可以用pvc的方式。

3.1、查看節點的標籤

[root@nccztsjb-node-23 ~]# kubectl get nodes --show-labels
NAME               STATUS   ROLES                  AGE   VERSION   LABELS
nccztsjb-node-23   Ready    control-plane,master   23d   v1.23.2   beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=nccztsjb-node-23,kubernetes.io/os=linux,node-role.kubernetes.io/control-plane=,node-role.kubernetes.io/master=,node.kubernetes.io/exclude-from-external-load-balancers=
nccztsjb-node-24   Ready    <none>                 23d   v1.23.2   beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=nccztsjb-node-24,kubernetes.io/os=linux
nccztsjb-node-25   Ready    ingress                23d   v1.23.2   beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=nccztsjb-node-25,kubernetes.io/os=linux,node-role.kubernetes.io/ingress=
[root@nccztsjb-node-23 ~]# 

3.2、在nccztsjb-node-25加入標籤

[root@nccztsjb-node-23 ~]# kubectl label node nccztsjb-node-25 node-role.kubernetes.io/prometheus-server=
node/nccztsjb-node-25 labeled
[root@nccztsjb-node-23 ~]# kubectl get nodes
NAME               STATUS   ROLES                       AGE   VERSION
nccztsjb-node-23   Ready    control-plane,master        23d   v1.23.2
nccztsjb-node-24   Ready    <none>                      23d   v1.23.2
nccztsjb-node-25   Ready    ingress,prometheus-server   23d   v1.23.2

標籤已經增加。

4、創建yaml配置文件

cat >prometheus.yaml <<EOF
---
# 創建命名空間
apiVersion: v1
kind: Namespace
metadata:
  name: monitoring
---
# 創建serviceaccount
apiVersion: v1
kind: ServiceAccount
metadata:
  name: prometheus
  namespace: monitoring
---
# 對serviceaccount進行授權綁定,給cluster-admin角色權限
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: prometheus
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
subjects:
- kind: ServiceAccount
  name: prometheus
  namespace: monitoring
---
# 創建configmap用來指定配置，本例主要是獲取cadvisor數據即容器數據
apiVersion: v1
kind: ConfigMap
metadata:
  name: prometheus-config
  labels:
    name: prometheus-config
  namespace: monitoring
data:
  prometheus.yml: |-
    global:
      scrape_interval: 5s
      evaluation_interval: 5s
    rule_files:
      - /etc/prometheus/prometheus.rules
    alerting:
      alertmanagers:
      - scheme: http
        static_configs:
        - targets:
          - "alertmanager.monitoring.svc:9093"

    scrape_configs:
      - job_name: "kubernetes-cadvisor"
        
        scrape_interval: 10s
        scrape_timeout: 10s
        
        scheme: https  #要加，否則默認使用http的協議
        
        metrics_path: /metrics/cadvisor
        
        tls_config:
          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
          insecure_skip_verify: true
        
        authorization:
          credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token
        
        kubernetes_sd_configs:
          - role: node
        
        relabel_configs:
          - action: labelmap
            regex: __meta_kubernetes_node_label_(.+)
---
#創建deployment
apiVersion: apps/v1
kind: Deployment
metadata:
  name: prometheus
  namespace: monitoring
spec:
  selector:
    matchLabels:
      app: prometheus
  replicas: 1
  template:
    metadata:
      labels:
        app: prometheus
    spec:
      securityContext:
        runAsUser: 0
        runAsGroup: 0
        fsGroup: 0
      serviceAccountName: prometheus
      containers:
      - name: prometheus
        image: 172.20.58.152/prom/prometheus:v2.33.1
        ports:
        - containerPort: 9090
          name: default
        resources: 
          requests: 
            cpu: 500m 
            memory: 500M 
          limits: 
            cpu: 1 
            memory: 1Gi
        volumeMounts:
        - name: config-volume
          mountPath: /etc/prometheus
        - name: prometheus-data
          mountPath: /prometheus
      volumes:
      - name: config-volume
        configMap:
         name: prometheus-config  
      - name: prometheus-data  #設置數據存儲盤在本地
        hostPath:
          path: /data/prometheus/data
          type: ""
      nodeSelector:
        node-role.kubernetes.io/prometheus-server: "" #使用3中創建的標籤    
---
# 創建service可以用於外部的訪問，端口30909
kind: Service
apiVersion: v1
metadata:
  name: prometheus
  namespace: monitoring
spec:
  selector:
    app: prometheus
  type: LoadBalancer
  ports:
  - protocol: TCP
    port: 9090
    targetPort: 9090
    nodePort: 30909
EOF

5、安裝delployment

基於4中創建的yaml配置文件來部署deploymetn

[root@nccztsjb-node-23 ~]# kubectl apply -f prometheus.yaml 
namespace/monitoring created
serviceaccount/prometheus created
clusterrolebinding.rbac.authorization.k8s.io/prometheus created
configmap/prometheus-config created
deployment.apps/prometheus created
service/prometheus created

查看創建的pod

[root@nccztsjb-node-23 ~]# kubectl get pod -n monitoring -o wide
NAME                          READY   STATUS    RESTARTS   AGE   IP             NODE               NOMINATED NODE   READINESS GATES
prometheus-55b9769449-2tq4l   1/1     Running   0          51s   172.39.21.81   nccztsjb-node-25   <none>           <none>
[root@nccztsjb-node-23 ~]# 

已經運行並且是運行在節點nccztsjb-node-25上的。

6、訪問prometheus

prometheus本身是包含了一個web界面的，可以通過service直接訪問。

查看service

[root@nccztsjb-node-23 ~]# kubectl get svc -n monitoring 
NAME         TYPE           CLUSTER-IP     EXTERNAL-IP   PORT(S)          AGE
prometheus   LoadBalancer   10.98.172.41   <pending>     9090:30909/TCP   2m28s
[root@nccztsjb-node-23 ~]# 

使用任意的節點IP:30909進行訪問

status->targets可以看到target的信息

OK，可以看到配置的job的信息。

以上就是在k8s中部署prometheus server的過程，如果想要更改prometheus.yml的配置，修改configmap就可以。

後面會有部署node export和通過grafana顯示的部署過程。