Attachment 014.Kubernetes Prometheus+Grafana+EFK+Kibana+Glusterfs Integration Scheme

A glusterfs storage cluster deployment

Note: The following are brief steps for detailed reference to GlusterFS Independent Deployment with 009.Kubernetes Permanent Storage.

1.1 schema

slightly

1.2 Related Planning

Host
IP
disk
Remarks
k8smaster01
172.24.8.71
-
Kubernetes Master Node
Heketi Host
k8smaster02
172.24.8.72
-
Kubernetes Master Node
Heketi Host
k8smaster03
172.24.8.73
-
Kubernetes Master Node
Heketi Host
k8snode01
172.24.8.74
sdb
Kubernetes Worker Node
glusterfs 01 node
k8snode02
172.24.8.75
sdb
Kubernetes Worker Node
glusterfs 02 Node
k8snode03
172.24.8.76
sdb
Kubernetes Worker Node
glusterfs 03 Node
Tip: This plan is completed directly with bare disks.

1.3 Install glusterfs

# yum -y install centos-release-gluster
# yum -y install glusterfs-server
# systemctl start glusterd
# systemctl enable glusterd
Tip: All nodes are recommended for installation.

1.4 Add Trust Pool

[root@k8snode01 ~]# gluster peer probe k8snode02
[root@k8snode01 ~]# gluster peer probe k8snode03
[root@k8snode01 ~]# gluster peer status #View trust pool status
[root@k8snode01 ~]# gluster pool list #View list of trust pools
Tip: It only needs to be executed once at any node of glusterfs.

1.5 Install heketi

[root@k8smaster01 ~]# yum -y install heketi heketi-client

1.6 Configure heketi

[root@k8smaster01 ~]# vi /etc/heketi/heketi.json
  1 {
  2   "_port_comment": "Heketi Server Port Number",
  3   "port": "8080",
  4 
  5   "_use_auth": "Enable JWT authorization. Please enable for deployment",
  6   "use_auth": true,
  7 
  8   "_jwt": "Private keys for access",
  9   "jwt": {
 10     "_admin": "Admin has access to all APIs",
 11     "admin": {
 12       "key": "admin123"
 13     },
 14     "_user": "User only has access to /volumes endpoint",
 15     "user": {
 16       "key": "xianghy"
 17     }
 18   },
 19 
 20   "_glusterfs_comment": "GlusterFS Configuration",
 21   "glusterfs": {
 22     "_executor_comment": [
 23       "Execute plugin. Possible choices: mock, ssh",
 24       "mock: This setting is used for testing and development.",
 25       "      It will not send commands to any node.",
 26       "ssh:  This setting will notify Heketi to ssh to the nodes.",
 27       "      It will need the values in sshexec to be configured.",
 28       "kubernetes: Communicate with GlusterFS containers over",
 29       "            Kubernetes exec api."
 30     ],
 31     "executor": "ssh",
 32 
 33     "_sshexec_comment": "SSH username and private key file information",
 34     "sshexec": {
 35       "keyfile": "/etc/heketi/heketi_key",
 36       "user": "root",
 37       "port": "22",
 38       "fstab": "/etc/fstab"
 39     },
 40 
 41     "_db_comment": "Database file name",
 42     "db": "/var/lib/heketi/heketi.db",
 43 
 44     "_loglevel_comment": [
 45       "Set log level. Choices are:",
 46       "  none, critical, error, warning, info, debug",
 47       "Default is warning"
 48     ],
 49     "loglevel" : "warning"
 50   }
 51 }
 

1.7 Configure Secret-Free Keys

[root@k8smaster01 ~]# ssh-keygen -t rsa -q -f /etc/heketi/heketi_key -N ""
[root@k8smaster01 ~]# chown heketi:heketi /etc/heketi/heketi_key
[root@k8smaster01 ~]# ssh-copy-id -i /etc/heketi/heketi_key.pub root@k8snode01
[root@k8smaster01 ~]# ssh-copy-id -i /etc/heketi/heketi_key.pub root@k8snode02
[root@k8smaster01 ~]# ssh-copy-id -i /etc/heketi/heketi_key.pub root@k8snode03

1.8 Start heketi

[root@k8smaster01 ~]# systemctl enable heketi.service
[root@k8smaster01 ~]# systemctl start heketi.service
[root@k8smaster01 ~]# systemctl status heketi.service
[root@k8smaster01 ~]# curl http://localhost:8080/hello #Test access

1.9 Configure Heketi Topology

[root@k8smaster01 ~]# vi /etc/heketi/topology.json
  1 {
  2   "clusters": [
  3     {
  4       "nodes": [
  5         {
  6           "node": {
  7             "hostnames": {
  8               "manage": [
  9                 "k8snode01"
 10               ],
 11               "storage": [
 12                 "172.24.8.74"
 13               ]
 14             },
 15             "zone": 1
 16           },
 17           "devices": [
 18             "/dev/sdb"
 19           ]
 20         },
 21         {
 22           "node": {
 23             "hostnames": {
 24               "manage": [
 25                 "k8snode02"
 26               ],
 27               "storage": [
 28                 "172.24.8.75"
 29               ]
 30             },
 31             "zone": 1
 32           },
 33           "devices": [
 34             "/dev/sdb"
 35           ]
 36         },
 37         {
 38           "node": {
 39             "hostnames": {
 40               "manage": [
 41                 "k8snode03"
 42               ],
 43               "storage": [
 44                 "172.24.8.76"
 45               ]
 46             },
 47             "zone": 1
 48           },
 49           "devices": [
 50             "/dev/sdb"
 51           ]
 52         }
 53       ]
 54     }
 55   ]
 56 }
 
[root@k8smaster01 ~]# echo "export HEKETI_CLI_SERVER=http://k8smaster01:8080" >> /etc/profile.d/heketi.sh
[root@k8smaster01 ~]# echo "alias heketi-cli='heketi-cli --user admin --secret admin123'" >> .bashrc
[root@k8smaster01 ~]# source /etc/profile.d/heketi.sh
[root@k8smaster01 ~]# source .bashrc
[root@k8smaster01 ~]# echo $HEKETI_CLI_SERVER
http://k8smaster01:8080
[root@k8smaster01 ~]# heketi-cli --server $HEKETI_CLI_SERVER --user admin --secret admin123 topology load --json=/etc/heketi/topology.json

1.10 Cluster Management and Testing

[root@heketi ~]# heketi-cli cluster list #cluster list
[root@heketi ~]# heketi-cli node list #volume information
[root@heketi ~]# heketi-cli volume list #volume information
[root@k8snode01 ~]# gluster volume info # Viewed through the glusterfs node

1.11 Create StorageClass

[root@k8smaster01 study]# vi heketi-secret.yaml
  1 apiVersion: v1
  2 kind: Secret
  3 metadata:
  4   name: heketi-secret
  5   namespace: heketi
  6 data:
  7   key: YWRtaW4xMjM=
  8 type: kubernetes.io/glusterfs
 
[root@k8smaster01 study]# kubectl create ns heketi
[root@k8smaster01 study]# kubectl create-f heketi-secret.yaml #create heketi
[root@k8smaster01 study]# kubectl get secrets -n heketi
[root@k8smaster01 study]# vim gluster-heketi-storageclass.yaml #Formally create StorageClass
  1 apiVersion: storage.k8s.io/v1
  2 kind: StorageClass
  3 metadata:
  4   name: ghstorageclass
  5 parameters:
  6   resturl: "http://172.24.8.71:8080"
  7   clusterid: "ad0f81f75f01d01ebd6a21834a2caa30"
  8   restauthenabled: "true"
  9   restuser: "admin"
 10   secretName: "heketi-secret"
 11   secretNamespace: "heketi"
 12   volumetype: "replicate:3"
 13 provisioner: kubernetes.io/glusterfs
 14 reclaimPolicy: Delete
 
[root@k8smaster01 study]# kubectl create -f gluster-heketi-storageclass.yaml
Note: storageclass resources cannot be changed after they are created. Modifications can only be deleted and rebuilt.
[root@k8smaster01 heketi]# kubectl get storageclasses #View confirmation
NAME PROVISIONER AGE
gluster-heketi-storageclass kubernetes.io/glusterfs 85s
[root@k8smaster01 heketi]# kubectl describe storageclasses ghstorageclass
 

2. Cluster Monitoring Metrics

Note: The following steps are brief and refer to "049. Cluster Management - Cluster Monitoring Metrics" for details.

2.1 Open Aggregation Layer

Turn on the Aggregation Layer feature, which is turned on by default using kubeadm, to view the validation below.
[root@k8smaster01 ~]# cat /etc/kubernetes/manifests/kube-apiserver.yaml

2.2 Getting deployment files

[root@k8smaster01 ~]# git clone https://github.com/kubernetes-incubator/metrics-server.git
[root@k8smaster01 ~]# cd metrics-server/deploy/1.8+/
[root@k8smaster01 1.8+]# vi metrics-server-deployment.yaml
  1 ......
  2         image: mirrorgooglecontainers/metrics-server-amd64:v0.3.6	#Modify to domestic source
  3         command:
  4         - /metrics-server
  5         - --metric-resolution=30s
  6         - --kubelet-insecure-tls
  7         - --kubelet-preferred-address-types=InternalIP,Hostname,InternalDNS,ExternalDNS,ExternalIP	#Add command as above
  8 ......
 

2.3 Official Deployment

[root@k8smaster01 1.8+]# kubectl apply -f .
[root@k8smaster01 1.8+]# kubectl -n kube-system get pods -l k8s-app=metrics-server
[root@k8smaster01 1.8+]# kubectl-n kube-system logs-l k8s-app=metrics-server-f #View deployment logs

2.4 Confirmation Verification

[root@k8smaster01 ~]# kubectl top nodes
[root@k8smaster01 ~]# kubectl top pods --all-namespaces
 

3. Prometheus deployment

Note: The following steps are brief and refer to the "050.Cluster Management-Prometheus+Grafana Monitoring Scheme" for details.

3.1 Get deployment files

[root@k8smaster01 ~]# git clone https://github.com/prometheus/prometheus

3.2 Create Namespace

[root@k8smaster01 ~]# cd prometheus/documentation/examples/
[root@k8smaster01 examples]# vi monitor-namespace.yaml
  1 apiVersion: v1
  2 kind: Namespace
  3 metadata:
  4   name: monitoring
[root@k8smaster01 examples]# kubectl create -f monitor-namespace.yaml

3.3 Create RBAC

[root@k8smaster01 examples]# vi rbac-setup.yml
  1 apiVersion: rbac.authorization.k8s.io/v1beta1
  2 kind: ClusterRole
  3 metadata:
  4   name: prometheus
  5 rules:
  6 - apiGroups: [""]
  7   resources:
  8   - nodes
  9   - nodes/proxy
 10   - services
 11   - endpoints
 12   - pods
 13   verbs: ["get", "list", "watch"]
 14 - apiGroups:
 15   - extensions
 16   resources:
 17   - ingresses
 18   verbs: ["get", "list", "watch"]
 19 - nonResourceURLs: ["/metrics"]
 20   verbs: ["get"]
 21 ---
 22 apiVersion: v1
 23 kind: ServiceAccount
 24 metadata:
 25   name: prometheus
 26   namespace: monitoring               #Just modify the namespace
 27 ---
 28 apiVersion: rbac.authorization.k8s.io/v1beta1
 29 kind: ClusterRoleBinding
 30 metadata:
 31   name: prometheus
 32 roleRef:
 33   apiGroup: rbac.authorization.k8s.io
 34   kind: ClusterRole
 35   name: prometheus
 36 subjects:
 37 - kind: ServiceAccount
 38   name: prometheus
 39   namespace: monitoring              #Just modify the namespace
[root@k8smaster01 examples]# kubectl create -f rbac-setup.yml

3.4 Create Prometheus ConfigMap

[root@k8smaster01 examples]# cat prometheus-kubernetes.yml | grep -v ^$ | grep -v "#" >> prometheus-config.yaml
[root@k8smaster01 examples]# vi prometheus-config.yaml
  1 apiVersion: v1
  2 kind: ConfigMap
  3 metadata:
  4   name: prometheus-server-conf
  5   labels:
  6     name: prometheus-server-conf
  7   namespace: monitoring               #Modify Namespace
  8 ......
[root@k8smaster01 examples]# kubectl create -f prometheus-config.yaml

3.5 Create durable PVC

[root@k8smaster01 examples]# vi prometheus-pvc.yaml
  1 apiVersion: v1
  2 kind: PersistentVolumeClaim
  3 metadata:
  4   name: prometheus-pvc
  5   namespace: monitoring
  6   annotations:
  7     volume.beta.kubernetes.io/storage-class: ghstorageclass
  8 spec:
  9   accessModes:
 10   - ReadWriteMany
 11   resources:
 12     requests:
 13       storage: 5Gi
[root@k8smaster01 examples]# kubectl create -f prometheus-pvc.yaml

3.6 Prometheus Deployment

[root@k8smaster01 examples]# vi prometheus-deployment.yml
  1 apiVersion: apps/v1beta2
  2 kind: Deployment
  3 metadata:
  4   labels:
  5     name: prometheus-deployment
  6   name: prometheus-server
  7   namespace: monitoring
  8 spec:
  9   replicas: 1
 10   selector:
 11     matchLabels:
 12       app: prometheus-server
 13   template:
 14     metadata:
 15       labels:
 16         app: prometheus-server
 17     spec:
 18       containers:
 19         - name: prometheus-server
 20           image: prom/prometheus:v2.14.0
 21           command:
 22           - "/bin/prometheus"
 23           args:
 24             - "--config.file=/etc/prometheus/prometheus.yml"
 25             - "--storage.tsdb.path=/prometheus/"
 26             - "--storage.tsdb.retention=72h"
 27           ports:
 28             - containerPort: 9090
 29               protocol: TCP
 30           volumeMounts:
 31             - name: prometheus-config-volume
 32               mountPath: /etc/prometheus/
 33             - name: prometheus-storage-volume
 34               mountPath: /prometheus/
 35       serviceAccountName: prometheus
 36       imagePullSecrets:
 37         - name: regsecret
 38       volumes:
 39         - name: prometheus-config-volume
 40           configMap:
 41             defaultMode: 420
 42             name: prometheus-server-conf
 43         - name: prometheus-storage-volume
 44           persistentVolumeClaim:
 45             claimName: prometheus-pvc
[root@k8smaster01 examples]# kubectl create -f prometheus-deployment.yml

3.7 Create Prometheus Service

[root@k8smaster01 examples]# vi prometheus-service.yaml
  1 apiVersion: v1
  2 kind: Service
  3 metadata:
  4   labels:
  5     app: prometheus-service
  6   name: prometheus-service
  7   namespace: monitoring
  8 spec:
  9   type: NodePort
 10   selector:
 11     app: prometheus-server
 12   ports:
 13     - port: 9090
 14       targetPort: 9090
 15       nodePort: 30001
[root@k8smaster01 examples]# kubectl create -f prometheus-service.yaml
[root@k8smaster01 examples]# kubectl get all -n monitoring

3.8 Verify Prometheus

Direct browser access: http://172.24.8.100:30001/
 

IV. Deployment of grafana

Note: The following steps are brief and refer to the "050.Cluster Management-Prometheus+Grafana Monitoring Scheme" for details.

4.1 Getting deployment files

[root@k8smaster01 ~]# git clone https://github.com/liukuan73/kubernetes-addons
[root@k8smaster01 ~]# cd /root/kubernetes-addons/monitor/prometheus+grafana

4.2 Creating durable PVC

[root@k8smaster01 prometheus+grafana]# vi grafana-data-pvc.yaml
  1 apiVersion: v1
  2 kind: PersistentVolumeClaim
  3 metadata:
  4   name: grafana-data-pvc
  5   namespace: monitoring
  6   annotations:
  7     volume.beta.kubernetes.io/storage-class: ghstorageclass
  8 spec:
  9   accessModes:
 10   - ReadWriteOnce
 11   resources:
 12     requests:
 13       storage: 5Gi
[root@k8smaster01 prometheus+grafana]# kubectl create -f grafana-data-pvc.yaml

4.3 grafana deployment

[root@k8smaster01 prometheus+grafana]# vi grafana.yaml
  1 apiVersion: extensions/v1beta1
  2 kind: Deployment
  3 metadata:
  4   name: monitoring-grafana
  5   namespace: monitoring
  6 spec:
  7   replicas: 1
  8   template:
  9     metadata:
 10       labels:
 11         task: monitoring
 12         k8s-app: grafana
 13     spec:
 14       containers:
 15       - name: grafana
 16         image: grafana/grafana:6.5.0
 17         imagePullPolicy: IfNotPresent
 18         ports:
 19         - containerPort: 3000
 20           protocol: TCP
 21         volumeMounts:
 22         - mountPath: /var/lib/grafana
 23           name: grafana-storage
 24         env:
 25           - name: INFLUXDB_HOST
 26             value: monitoring-influxdb
 27           - name: GF_SERVER_HTTP_PORT
 28             value: "3000"
 29           - name: GF_AUTH_BASIC_ENABLED
 30             value: "false"
 31           - name: GF_AUTH_ANONYMOUS_ENABLED
 32             value: "true"
 33           - name: GF_AUTH_ANONYMOUS_ORG_ROLE
 34             value: Admin
 35           - name: GF_SERVER_ROOT_URL
 36             value: /
 37         readinessProbe:
 38           httpGet:
 39             path: /login
 40             port: 3000
 41       volumes:
 42       - name: grafana-storage
 43         persistentVolumeClaim:
 44           claimName: grafana-data-pvc
 45       nodeSelector:
 46         node-role.kubernetes.io/master: "true"
 47       tolerations:
 48       - key: "node-role.kubernetes.io/master"
 49         effect: "NoSchedule"
 50 ---
 51 apiVersion: v1
 52 kind: Service
 53 metadata:
 54   labels:
 55     kubernetes.io/cluster-service: 'true'
 56     kubernetes.io/name: monitoring-grafana
 57   annotations:
 58     prometheus.io/scrape: 'true'
 59     prometheus.io/tcp-probe: 'true'
 60     prometheus.io/tcp-probe-port: '80'
 61   name: monitoring-grafana
 62   namespace: monitoring
 63 spec:
 64   type: NodePort
 65   ports:
 66   - port: 80
 67     targetPort: 3000
 68     nodePort: 30002
 69   selector:
 70     k8s-app: grafana
[root@k8smaster01 prometheus+grafana]# kubectl label nodes k8smaster01 node-role.kubernetes.io/master=true
[root@k8smaster01 prometheus+grafana]# kubectl label nodes k8smaster02 node-role.kubernetes.io/master=true
[root@k8smaster01 prometheus+grafana]# kubectl label nodes k8smaster03 node-role.kubernetes.io/master=true
[root@k8smaster01 prometheus+grafana]# kubectl create -f grafana.yaml
[root@k8smaster01 examples]# kubectl get all -n monitoring

4.4 Verify Prometheus

Direct browser access: http://172.24.8.100:30002/

4.4 grafana Configuration

  • Add data source: omit
  • Create user: omit
Tip: All grafana configurations are configurable reference: https://grafana.com/docs/grafana/latest/installation/configuration/.

4.5 View Monitoring

Browser Access Again: http://172.24.8.100:30002/

Tags: Linux Kubernetes ssh JSON git

Posted on Fri, 20 Mar 2020 03:13:05 -0400 by yanti