Kubernetes cluster deployment based on binary

1. Cluster deployment architecture

2. Configure the basic environment

2.1 system configuration

  • host name
  • iptables
  • firewall
  • Configuration of kernel parameters and resource restriction policies

2.2 installing docker

  • Install docker on the master, etcd and node nodes. Refer to 2.5.1.1 for installation and configuration

2.3 installing ansible

  • Install ansible on the deployment host. Take master1 as the deployment host
root@master1:~# apt install python3-pip git 
root@master1:~# pip3 install ansible -i https://mirrors.aliyun.com/pypi/simple 
root@master1:~# ansible --version
ansible [core 2.11.5]
  config file = None
  configured module search path = ['/root/.ansible/plugins/modules', '/usr/share/ansible/plugins/modules']
  ansible python module location = /usr/local/lib/python3.8/dist-packages/ansible
  ansible collection location = /root/.ansible/collections:/usr/share/ansible/collections
  executable location = /usr/local/bin/ansible
  python version = 3.8.10 (default, Jun  2 2021, 10:49:15) [GCC 9.4.0]
  jinja version = 2.10.1
  libyaml = True

2.4 configure node password free and docker login

2.4.1 log in to docker on master1

root@master1:~# mkdir -p  /etc/docker/certs.d/harbor.k8s.local/
root@master1:~# scp root@192.168.6.87:/data/certs/harbor-ca.crt /etc/docker/certs.d/harbor.k8s.local/
root@master1:~# docker login harbor.k8s.local
Username: admin
Password:
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store

Login Succeeded

2.4.2 use scripts on master 1 node to configure secret free login and docker login for master, etcd and node nodes

  • The script is as follows:
root@master1:~# ssh-keygen
root@master1:~# apt install sshpass
root@master1:~# vim scp-key.sh
#!/bin/bash
IP="
192.168.6.79
192.168.6.80
192.168.6.81
192.168.6.84
192.168.6.85
192.168.6.86
192.168.6.89
192.168.6.90
192.168.6.91
"
for node in ${IP};do
   sshpass -p 123456 ssh-copy-id ${node} -o StrictHostKeyChecking=no
   if [ $? -eq 0 ];then
      echo "${node} ssh-key copy completed"
      echo "copy ssl crt for docker"
      ssh ${node} "mkdir -p  /etc/docker/certs.d/harbor.k8s.local/"
      echo "create harbor ssl path sucess"
      scp  /etc/docker/certs.d/harbor.k8s.local/harbor-ca.crt ${node}:/etc/docker/certs.d/harbor.k8s.local/harbor-ca.crt
      echo "copy harbor ssl key sucess"
      ssh ${node} "echo "192.168.6.201 harbor.k8s.local" >> /etc/hosts"
      echo "copy hosts file complete"
      scp -r /root/.docker ${node}:/root/
      echo "harbor ssl copy complete"
   else
     echo "${node} ssh-key copy failed"
   fi
done


2.5. Deploy Harbor and HA nodes

2.5.1 deploy Harbor

2.5.1.1 installing docker and docker compose

  • Install with this installation script
#!/bin/bash
User=$1
DIR=$(pwd)
PACKAGE_NAME=$2
DOCKER_FILE=${DIR}/${PACKAGE_NAME}
centos_install_docker(){
 grep "Kernel" /etc/issue &> /dev/null
 if [ $? -eq 0 ];then
   /bin/echo "Current system is $(cat /etc/redhat-release),Ready to start system initialization, configuration docker-compose And installation docker" && sleep 1
   systemctl stop firewalld && systemctl disable firewalld && echo "The firewall has been turned off" && sleep 1
   systemctl stop NetworkManager && systemctl disable NetworkManager && echo "NetworkManager Stopped" && sleep 1
   sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/sysconfig/selinux && setenforce 0 && echo "Selinux Closed" && sleep 1
   \cp ${DIR}/limits.conf /etc/security/limits.conf
   \cp ${DIR}/sysctl.conf /etc/sysctl.conf

   /bin/tar xvf ${DOCKER_FILE}
   \cp -rf docker/* /usr/bin/
   \cp containerd.service /lib/systemd/system/containerd.service
   \cp docker.service /lib/systemd/system/docker.service
   \cp docker.socket /lib/systemd/system/docker.socket

   \cp ${DIR}/docker-compose-Linux-x86_64 /usr/bin/docker-compose
   chmod u+x /usr/bin/docker-compose

   id -u docker &>/dev/null
   if [ $? -ne 0 ];then
     groupadd docker && useradd docker -g docker
   fi

   id -u ${User} &> /dev/null
   if [ $? -ne 0 ];then
     useradd ${User}
     useradd ${User} -G docker
   fi
   /usr/bin/ls /data &>/dev/null
   if [ $? -ne 0 ];then
      mkdir /data
   fi
   \cp ${DIR}/daemon.json /etc/docker/
   systemctl enable containerd.service && systemctl restart containerd.service
   systemctl enable docker.service && systemctl restart docker.service
   systemctl enable docker.socket && systemctl restart docker.socket
 fi
}
ubuntu_install_docker(){
   grep "Ubuntu" /etc/issue &> /dev/null
 if [ $? -eq 0 ];then
   /bin/echo "Current system is $(cat /etc/issue),Ready to start system initialization, configuration docker-compose And installation docker" && sleep 1
   \cp ${DIR}/limits.conf /etc/security/limits.conf
   \cp ${DIR}/sysctl.conf /etc/sysctl.conf

   /bin/tar xvf ${DOCKER_FILE}

   \cp -rf docker/* /usr/bin/
   \cp containerd.service /lib/systemd/system/containerd.service
   \cp docker.service /lib/systemd/system/docker.service
   \cp docker.socket /lib/systemd/system/docker.socket

   \cp ${DIR}/docker-compose-Linux-x86_64 /usr/bin/docker-compose
   chmod u+x /usr/bin/docker-compose

   id -u ${User} &> /dev/null
   if [ $? -ne 0 ];then
     /bin/grep -q docker /etc/group
       if [ $? -ne 0 ];then
         groupadd -r docker
       fi
     useradd -r -m -g ${User} ${User}
   fi

   /usr/bin/ls /data/docker &>/dev/null
   if [ $? -ne 0 ];then
      mkdir -p /data/docker
   fi
   \cp ${DIR}/daemon.json /etc/docker/

   systemctl enable containerd.service && systemctl restart containerd.service
   systemctl enable docker.service && systemctl restart docker.service
   systemctl enable docker.socket && systemctl restart docker.socket
 fi
}
main(){
 centos_install_docker
 ubuntu_install_docker
}
main
root@harbor1:~# bash docker_install.sh docker docker-19.03.15.tgz
  • Relevant scripts and files:
https://codeup.aliyun.com/614c9d1b3a631c643ae62857/Scripts.git

2.5.1.2 installing Harbor

  • After copying the harbor binary to the corresponding host:
root@harbor1:~# tar xf harbor-offline-installer-v2.3.2.tgz
root@harbor1:~# mkdir /data/harbor/
root@harbor1:~# cd harbor
root@harbor1:~# cp harbor.yml.tmpl harbor.yml
root@harbor1:~# vim harbor.yml
# Configuration file of Harbor

# The IP address or hostname to access admin UI and registry service.
# DO NOT use localhost or 127.0.0.1, because Harbor needs to be accessed by external clients.
hostname: 192.168.6.82

# http related config
http:
  # port for http, default is 80. If https enabled, this port will redirect to https port
  port: 80

# https related config
#https:
  # https port for harbor, default is 443
#  port: 443
  # The path of cert and key files for nginx
#  certificate: /your/certificate/path
#  private_key: /your/private/key/path

# # Uncomment following will enable tls communication between all harbor components
# internal_tls:
#   # set enabled to true means internal tls is enabled
#   enabled: true
#   # put your cert and key files on dir
#   dir: /etc/harbor/tls/internal

# Uncomment external_url if you want to enable external proxy
# And when it enabled the hostname will no longer used
# external_url: https://reg.mydomain.com:8433

# The initial password of Harbor admin
# It only works in first time to install harbor
# Remember Change the admin password from UI after launching Harbor.
harbor_admin_password: 123456

# Harbor DB configuration
database:
  # The password for the root user of Harbor DB. Change this before any production use.
  password: root123
  # The maximum number of connections in the idle connection pool. If it <=0, no idle connections are retained.
  max_idle_conns: 100
  # The maximum number of open connections to the database. If it <= 0, then there is no limit on the number of open connections.
  # Note: the default number of connections is 1024 for postgres of harbor.
  max_open_conns: 900

# The default data volume
data_volume: /data/harbor

# Harbor Storage settings by default is using /data dir on local filesystem
# Uncomment storage_service setting If you want to using external storage
# storage_service:
#   # ca_bundle is the path to the custom root ca certificate, which will be injected into the truststore
#   # of registry's and chart repository's containers.  This is usually needed when the user hosts a internal storage with self signed certificate.
#   ca_bundle:

#   # storage backend, default is filesystem, options include filesystem, azure, gcs, s3, swift and oss
#   # for more info about this configuration please refer https://docs.docker.com/registry/configuration/
#   filesystem:
#     maxthreads: 100
#   # set disable to true when you want to disable registry redirect
#   redirect:
#     disabled: false

# Trivy configuration
#
# Trivy DB contains vulnerability information from NVD, Red Hat, and many other upstream vulnerability databases.
# It is downloaded by Trivy from the GitHub release page https://github.com/aquasecurity/trivy-db/releases and cached
# in the local file system. In addition, the database contains the update timestamp so Trivy can detect whether it
# should download a newer version from the Internet or use the cached one. Currently, the database is updated every
# 12 hours and published as a new release to GitHub.
trivy:
  # ignoreUnfixed The flag to display only fixed vulnerabilities
  ignore_unfixed: false
  # skipUpdate The flag to enable or disable Trivy DB downloads from GitHub
  #
  # You might want to enable this flag in test or CI/CD environments to avoid GitHub rate limiting issues.
  # If the flag is enabled you have to download the `trivy-offline.tar.gz` archive manually, extract `trivy.db` and
  # `metadata.json` files and mount them in the `/home/scanner/.cache/trivy/db` path.
  skip_update: false
  #
  # insecure The flag to skip verifying registry certificate
  insecure: false
  # github_token The GitHub access token to download Trivy DB
  #
  # Anonymous downloads from GitHub are subject to the limit of 60 requests per hour. Normally such rate limit is enough
  # for production operations. If, for any reason, it's not enough, you could increase the rate limit to 5000
  # requests per hour by specifying the GitHub access token. For more details on GitHub rate limiting please consult
  # https://developer.github.com/v3/#rate-limiting
  #
  # You can create a GitHub token by following the instructions in
  # https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line
  #
  # github_token: xxx

jobservice:
  # Maximum number of job workers in job service
  max_job_workers: 10

notification:
  # Maximum retry count for webhook job
  webhook_job_max_retry: 10

chart:
  # Change the value of absolute_url to enabled can enable absolute url in chart
  absolute_url: disabled

# Log configurations
log:
  # options are debug, info, warning, error, fatal
  level: info
  # configs for logs in local storage
  local:
    # Log files are rotated log_rotate_count times before being removed. If count is 0, old versions are removed rather than rotated.
    rotate_count: 50
    # Log files are rotated only if they grow bigger than log_rotate_size bytes. If size is followed by k, the size is assumed to be in kilobytes.
    # If the M is used, the size is in megabytes, and if G is used, the size is in gigabytes. So size 100, size 100k, size 100M and size 100G
    # are all valid.
    rotate_size: 200M
    # The directory on your host that store log
    location: /var/log/harbor

  # Uncomment following lines to enable external syslog endpoint.
  # external_endpoint:
  #   # protocol used to transmit log to external endpoint, options is tcp or udp
  #   protocol: tcp
  #   # The host of external endpoint
  #   host: localhost
  #   # Port of external endpoint
  #   port: 5140

#This attribute is for migrator to detect the version of the .cfg file, DO NOT MODIFY!
_version: 2.3.0

# Uncomment external_database if using external database.
# external_database:
#   harbor:
#     host: harbor_db_host
#     port: harbor_db_port
#     db_name: harbor_db_name
#     username: harbor_db_username
#     password: harbor_db_password
#     ssl_mode: disable
#     max_idle_conns: 2
#     max_open_conns: 0
#   notary_signer:
#     host: notary_signer_db_host
#     port: notary_signer_db_port
#     db_name: notary_signer_db_name
#     username: notary_signer_db_username
#     password: notary_signer_db_password
#     ssl_mode: disable
#   notary_server:
#     host: notary_server_db_host
#     port: notary_server_db_port
#     db_name: notary_server_db_name
#     username: notary_server_db_username
#     password: notary_server_db_password
#     ssl_mode: disable

# Uncomment external_redis if using external Redis server
# external_redis:
#   # support redis, redis+sentinel
#   # host for redis: <host_redis>:<port_redis>
#   # host for redis+sentinel:
#   #  <host_sentinel1>:<port_sentinel1>,<host_sentinel2>:<port_sentinel2>,<host_sentinel3>:<port_sentinel3>
#   host: redis:6379
#   password:
#   # sentinel_master_set must be set to support redis+sentinel
#   #sentinel_master_set:
#   # db_index 0 is for core, it's unchangeable
#   registry_db_index: 1
#   jobservice_db_index: 2
#   chartmuseum_db_index: 3
#   trivy_db_index: 5
#   idle_timeout_seconds: 30

# Uncomment uaa for trusting the certificate of uaa instance that is hosted via self-signed cert.
# uaa:
#   ca_file: /path/to/ca

# Global proxy
# Config http proxy for components, e.g. http://my.proxy.com:3128
# Components doesn't need to connect to each others via http proxy.
# Remove component from `components` array if want disable proxy
# for it. If you want use proxy for replication, MUST enable proxy
# for core and jobservice, and set `http_proxy` and `https_proxy`.
# Add domain to the `no_proxy` field, when you want disable proxy
# for some special registry.
proxy:
  http_proxy:
  https_proxy:
  no_proxy:
  components:
    - core
    - jobservice
    - trivy

# metric:
#   enabled: false
#   port: 9090
#   path: /metrics

root@harbor1:~# ./install.sh --with-trivy
root@harbor1:~# docker-compose start
  • Perform the above configuration on another harbor node
  • Configure harbor dual node replication

2.5.2 deploy HA nodes

2.5.2.1 generating ssl certificate using openssl

  • Generate ssl Certificate in HA1 node
root@ha1:~# mkdir -p /data/certs/
root@ha1:/data/certs#  openssl genrsa -out harbor-ca.key
root@ha1:/data/certs#  openssl req -x509 -new -nodes -key harbor-ca.key -subj "/CN=harbor.k8s.local" -days 7120 -out harbor-ca.crt
root@ha1:/data/certs#  cat harbor-ca.crt  harbor-ca.key | tee harbor-ca.pem
  • Copy certificate to HA2 node
root@ha2:~# mkdir -p /data/certs/
root@ha1:/data/certs# scp -r * root@192.168.6.83:/data/certs

2.5.2.2 installation and configuration kept

  • This is configured on both HA nodes
root@ha1:~# apt update
root@ha1:~# apt install -y keepalived
root@ha1:~# vim /etc/keepalived/keepalived.conf
global_defs {
   notification_email {
     acassen
   }
   notification_email_from Alexandre.Cassen@firewall.loc
   smtp_server 192.168.200.1
   smtp_connect_timeout 30
   router_id LVS_DEVEL
}

vrrp_instance VI_1 {
    state MASTER
    interface eth0
    garp_master_delay 10
    smtp_alert
    virtual_router_id 56
    priority 100    #90 on HA2 node
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.6.201 dev eth0 label eth0:0
        192.168.6.202 dev eth0 label eth0:1
        192.168.6.203 dev eth0 label eth0:2
        192.168.6.204 dev eth0 label eth0:3
        192.168.6.205 dev eth0 label eth0:4
    }
}
root@ha1:~# vim /etc/sysctl.conf 
...
net.ipv4.ip_nonlocal_bind = 1

root@ha1:~# sysctl -p
root@ha1:~# systemctl start keepalived
root@ha1:~# systemctl enable keepalived

2.5.2.3 installation and configuration of Haproxy

  • For the high availability of Harbor, the method of haproxy ssl termination is adopted, that is, ssl connection is adopted between the client and haproxy, ssl connection is terminated in the load balancer, and non encrypted connection is adopted between haproxy and back-end applications.

  • This is configured on both HA nodes

root@ha1:~# apt install haproxy
root@ha1:~# vim /etc/haproxy/haproxy.cfg
...
listen harbor-443           #Configure harbor listening
  bind 192.168.6.201:80
  bind 192.168.6.201:443 ssl crt /data/certs/harbor-ca.pem
  reqadd X-Forwarded-Proto:\ https
  option forwardfor
  mode http
  balance source
  redirect scheme https if !{ ssl_fc }
  server harbor1 192.168.6.82:80 check inter 3s fall 3 rise 5
  server harbor2 192.168.6.83:80 check inter 3s fall 3 rise 5
listen k8s-6443            #Configure Kube API listening
  bind 192.168.6.202:6443
  mode tcp
  server k8s1 192.168.6.79:6443 check inter 3s fall 3 rise 5
  server k8s2 192.168.6.80:6443 check inter 3s fall 3 rise 5
  server k8s3 192.168.6.81:6443 check inter 3s fall 3 rise 5 
root@ha1:~# systemctl restart haproxy
root@ha1:~# systemctl enable haproxy

2.5.2.4 verify landing Harbor

#Login test
root@harbor2:~# docker login harbor.k8s.local
Username: admin
Password:
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store

Login Succeeded

#image upload test
root@harbor2:~# docker pull calico/cni:v3.15.3
v3.15.3: Pulling from calico/cni
1ff8efc68ede: Pull complete
dbf74493f8ac: Pull complete
6a02335af7ae: Pull complete
a9d90ecd95cb: Pull complete
269efe44f16b: Pull complete
d94997f3700d: Pull complete
8c7602656f2e: Pull complete
34fcbf8be9e7: Pull complete
Digest: sha256:519e5c74c3c801ee337ca49b95b47153e01fd02b7d2797c601aeda48dc6367ff
Status: Downloaded newer image for calico/cni:v3.15.3
docker.io/calico/cni:v3.15.3

root@harbor2:~# docker tag  calico/cni:v3.15.3 harbor.k8s.local/k8s/calico-cni:v3.15.3

root@harbor2:~# docker push harbor.k8s.local/k8s/calico-cni:v3.15.3
The push refers to repository [harbor.k8s.local/k8s/calico-cni]
3fbbefdb79e9: Pushed
0cab240caf05: Pushed
ed578a6324e2: Pushed
e00099f6d86f: Pushed
0b954fdf8c09: Pushed
fa6b2b5f2343: Pushed
710ad749cc2d: Pushed
3e05a70dae71: Pushed
v3.15.3: digest: sha256:a559d264c7a75a7528560d11778dba2d3b55c588228aed99be401fd2baa9b607 size: 1997

2.5 download deployment items and components at the deployment node

root@master1:~# export release=3.1.0
root@master1:~# curl -C- -fLO --retry 3 https://github.com/easzlab/kubeasz/releases/download/${release}/ezdown
root@master1:~# chmod +x ./ezdown
root@master1:~# ./ezdown -D
root@master1:~# ll /etc/kubeasz/down/
total 1239424
drwxr-xr-x  2 root root      4096 Sep 29 15:41 ./
drwxrwxr-x 11 root root       209 Sep 29 15:36 ../
-rw-------  1 root root 451969024 Sep 29 15:38 calico_v3.15.3.tar
-rw-------  1 root root  42592768 Sep 29 15:38 coredns_1.8.0.tar
-rw-------  1 root root 227933696 Sep 29 15:39 dashboard_v2.2.0.tar
-rw-r--r--  1 root root  62436240 Sep 29 15:34 docker-19.03.15.tgz
-rw-------  1 root root  58150912 Sep 29 15:39 flannel_v0.13.0-amd64.tar
-rw-------  1 root root 124833792 Sep 29 15:38 k8s-dns-node-cache_1.17.0.tar
-rw-------  1 root root 179014144 Sep 29 15:41 kubeasz_3.1.0.tar
-rw-------  1 root root  34566656 Sep 29 15:40 metrics-scraper_v1.0.6.tar
-rw-------  1 root root  41199616 Sep 29 15:40 metrics-server_v0.3.6.tar
-rw-------  1 root root  45063680 Sep 29 15:40 nfs-provisioner_v4.0.1.tar
-rw-------  1 root root    692736 Sep 29 15:40 pause.tar
-rw-------  1 root root    692736 Sep 29 15:40 pause_3.4.1.tar                                              

3. Configure deployment tools and K8S cluster configuration

3.1 generate a new K8S cluster configuration file

root@master1:~# cd /etc/kubeasz/
root@master1:/etc/kubeasz# ./ezctl new k8s-01

3.2 configuring ansible files

root@master1:/etc/kubeasz# vim clusters/k8s-01/hosts 
# 'etcd' cluster should have odd member(s) (1,3,5,...)
[etcd]
192.168.6.84
192.168.6.85
192.168.6.86

# master node(s)
[kube_master]
192.168.6.79
192.168.6.80

# work node(s)
[kube_node]
192.168.6.89
192.168.6.90

# [optional] harbor server, a private docker registry
# 'NEW_INSTALL': 'true' to install a harbor server; 'false' to integrate with existed one
[harbor]
# 192.168.6.201 NEW_INSTALL=false

# [optional] loadbalance for accessing k8s from outside
[ex_lb]                     #The deployment tool needs to be opened. An error will be reported if the deployment tool is not opened
192.168.6.88 LB_ROLE=backup EX_APISERVER_VIP=192.168.6.202 EX_APISERVER_PORT=8443
192.168.6.87 LB_ROLE=master EX_APISERVER_VIP=192.168.6.202 EX_APISERVER_PORT=8443

# [optional] ntp server for the cluster
[chrony]
# 192.168.6.79

[all:vars]
# --------- Main Variables ---------------
# Secure port for apiservers
SECURE_PORT="6443"

# Cluster container-runtime supported: docker, containerd
CONTAINER_RUNTIME="docker"

# Network plugins supported: calico, flannel, kube-router, cilium, kube-ovn
CLUSTER_NETWORK="calico"         #Modify to the required network plug-in as needed

# Service proxy mode of kube-proxy: 'iptables' or 'ipvs'
PROXY_MODE="ipvs"

# K8S Service CIDR, not overlap with node(host) networking
SERVICE_CIDR="10.100.0.0/16"     #Modify as needed

# Cluster CIDR (Pod CIDR), not overlap with node(host) networking
CLUSTER_CIDR="10.200.0.0/16"     #Modify as needed

# NodePort Range
NODE_PORT_RANGE="30000-60000"    #It is recommended to set the port range larger as needed

# Cluster DNS Domain
CLUSTER_DNS_DOMAIN="k8s.local"   #Modify as needed

# -------- Additional Variables (don't change the default value right now) ---
# Binaries Directory
bin_dir="/usr/local/bin"          #It is recommended to change to common directory

# Deploy Directory (kubeasz workspace)
base_dir="/etc/kubeasz"

# Directory for a specific cluster
cluster_dir="{{ base_dir }}/clusters/k8s-01"

# CA and other components cert/key Directory
ca_dir="/etc/kubernetes/ssl"

3.3 config file

root@master1:/etc/kubeasz# vim clusters/k8s-01/config.yml
############################
# prepare
############################
# Optional offline installation system software package (offline|online)
INSTALL_SOURCE: "online"

# Optional system security reinforcement github.com/dev-sec/ansible-collection-hardening
OS_HARDEN: false

# Set the time source server [important: the machine time in the cluster must be synchronized]
ntp_servers:
  - "ntp1.aliyun.com"
  - "time1.cloud.tencent.com"
  - "0.cn.pool.ntp.org"

# Set the network segments that allow internal time synchronization, such as "10.0.0.0/8". All are allowed by default
local_network: "0.0.0.0/0"


############################
# role:deploy
############################
# default: ca will expire in 100 years
# default: certs issued by the ca will expire in 50 years
CA_EXPIRY: "876000h"
CERT_EXPIRY: "438000h"

# kubeconfig configuration parameters
CLUSTER_NAME: "cluster1"
CONTEXT_NAME: "context-{{ CLUSTER_NAME }}"


############################
# role:etcd
############################
# Setting different wal directories can avoid disk io competition and improve performance
ETCD_DATA_DIR: "/var/lib/etcd"
ETCD_WAL_DIR: ""


############################
# role:runtime [containerd,docker]
############################
# ------------------------------------------- containerd
# [.] enable container warehouse mirroring
ENABLE_MIRROR_REGISTRY: true

# [containerd] base container image
SANDBOX_IMAGE: "harbor.k8s.local/k8s/easzlab-pause-amd64:3.4.1"  #Change to local harbor image address

# [containerd] container persistent storage directory
CONTAINERD_STORAGE_DIR: "/var/lib/containerd"

# ------------------------------------------- docker
# [docker] container storage directory
DOCKER_STORAGE_DIR: "/var/lib/docker"

# [docker] open Restful API
ENABLE_REMOTE_API: false

# [docker] trusted HTTP repository
INSECURE_REG: '["127.0.0.1/8"]'


############################
# role:kube-master
############################
# k8s cluster master node certificate configuration, you can add multiple ip and domain names (such as adding public ip and domain names)
MASTER_CERT_HOSTS:
  - "10.1.1.1"
  - "k8s.test.io"
  #- "www.test.com"

# The length of the pod segment mask on the node (determines the maximum pod ip address that each node can allocate)
# If flannel uses the -- Kube subnet Mgr parameter, it will read this setting and assign a pod segment to each node
# https://github.com/coreos/flannel/issues/847
NODE_CIDR_LEN: 24


############################
# role:kube-node
############################
# Kubelet root directory
KUBELET_ROOT_DIR: "/var/lib/kubelet"

# Maximum number of pod s on node
MAX_PODS: 300

# Configure the amount of resources reserved for kube components (kubelet, kube proxy, dockerd, etc.)
# See templates/kubelet-config.yaml.j2 for numerical settings
KUBE_RESERVED_ENABLED: "yes"

# k8s officials do not recommend starting system reserved hastily, unless you know the resource occupation of the system based on long-term monitoring;
# In addition, with the running time of the system, resource reservation needs to be increased appropriately. See templates/kubelet-config.yaml.j2 for numerical settings
# The system reservation settings are based on 4c/8g virtual machines to minimize the installation of system services. If high-performance physical machines are used, the reservation can be increased appropriately
# In addition, apiserver and other resources will occupy a large amount of time during cluster installation. It is recommended to reserve at least 1g of memory
SYS_RESERVED_ENABLED: "no"

# haproxy balance mode
BALANCE_ALG: "roundrobin"


############################
# role:network [flannel,calico,cilium,kube-ovn,kube-router]
############################
# ------------------------------------------- flannel
# [flannel] set the flannel backend "host GW", "vxlan", etc
FLANNEL_BACKEND: "vxlan"
DIRECT_ROUTING: false

# [flannel] flanneld_image: "quay.io/coreos/flannel:v0.10.0-amd64"
flannelVer: "v0.13.0-amd64"
flanneld_image: "easzlab/flannel:{{ flannelVer }}"

# [flannel] offline image tar package
flannel_offline: "flannel_{{ flannelVer }}.tar"

# ------------------------------------------- calico
# [calico] set CALICO_IPV4POOL_IPIP = "off", which can improve network performance. See docs/setup/calico.md for conditions and restrictions
CALICO_IPV4POOL_IPIP: "Always"

# [calico] set the host IP used by calico node. bgp neighbors are established through this address, which can be specified manually or found automatically
IP_AUTODETECTION_METHOD: "can-reach={{ groups['kube_master'][0] }}"

# [calico] set calico network backend: brid, vxlan, none
CALICO_NETWORKING_BACKEND: "brid"

# [calico] update supports calico version: [v3.3.x] [v3.4.x] [v3.8.x] [v3.15.x]
calico_ver: "v3.15.3"

# [calico]calico major version
calico_ver_main: "{{ calico_ver.split('.')[0] }}.{{ calico_ver.split('.')[1] }}"

# [calico] offline image tar package
calico_offline: "calico_{{ calico_ver }}.tar"

# ------------------------------------------- cilium
# [cilium]CILIUM_ ETCD_ Number of etcd cluster nodes created by operator 1,3,5,7
ETCD_CLUSTER_SIZE: 1

# [cilium] mirrored version
cilium_ver: "v1.4.1"

# [cilium] offline image tar package
cilium_offline: "cilium_{{ cilium_ver }}.tar"

# ------------------------------------------- kube-ovn
# [Kube ovn] select the OVN DB and OVN Control Plane node, which is the first master node by default
OVN_DB_NODE: "{{ groups['kube_master'][0] }}"

# [Kube ovn] offline image tar package
kube_ovn_ver: "v1.5.3"
kube_ovn_offline: "kube_ovn_{{ kube_ovn_ver }}.tar"

# ------------------------------------------- kube-router
# [Kube router] there are restrictions on the public cloud. Generally, you need to always turn on ipinip; The self owned environment can be set to "subnet"
OVERLAY_TYPE: "full"

# [Kube router] networkpolicy support switch
FIREWALL_ENABLE: "true"

# [Kube router] Kube router image version
kube_router_ver: "v0.3.1"
busybox_ver: "1.28.4"

# [Kube router] Kube router offline image tar package
kuberouter_offline: "kube-router_{{ kube_router_ver }}.tar"
busybox_offline: "busybox_{{ busybox_ver }}.tar"


############################
# role:cluster-addon
############################
# coredns automatic installation
dns_install: "no"               #No installation, separate installation in the later stage
corednsVer: "1.8.0"
ENABLE_LOCAL_DNS_CACHE: false   #The test environment is not installed. It is recommended to build the environment installation
dnsNodeCacheVer: "1.17.0"
# Set local dns cache address
LOCAL_DNS_CACHE: "169.254.20.10"

# Automatic installation of metric server
metricsserver_install: "no"     #Not installed
metricsVer: "v0.3.6"

# dashboard Automatic installation             #No installation, separate installation in the later stage
dashboard_install: "no"
dashboardVer: "v2.2.0"
dashboardMetricsScraperVer: "v1.0.6"

# ingress Automatic installation               #Not installed
ingress_install: "no"
ingress_backend: "traefik"
traefik_chart_ver: "9.12.3"

# prometheus Automatic installation            #Not installed
prom_install: "no"
prom_namespace: "monitor"
prom_chart_ver: "12.10.6"

# nfs-provisioner Automatic installation       #Not installed
nfs_provisioner_install: "no"
nfs_provisioner_namespace: "kube-system"
nfs_provisioner_ver: "v4.0.1"
nfs_storage_class: "managed-nfs-storage"
nfs_server: "192.168.1.10"
nfs_path: "/data/nfs"

############################
# role:harbor
############################
# harbor version, full version number
HARBOR_VER: "v2.1.3"
HARBOR_DOMAIN: "harbor.yourdomain.com"
HARBOR_TLS_PORT: 8443

# if set 'false', you need to put certs named harbor.pem and harbor-key.pem in directory 'down'
HARBOR_SELF_SIGNED_CERT: true

# install extra component
HARBOR_WITH_NOTARY: false
HARBOR_WITH_TRIVY: false
HARBOR_WITH_CLAIR: false
HARBOR_WITH_CHARTMUSEUM: true

3.4 initialize cluster environment

root@master1:/etc/kubeasz# ./ezctl help setup
Usage: ezctl setup <cluster> <step>
available steps:
    01  prepare            to prepare CA/certs & kubeconfig & other system settings
    02  etcd               to setup the etcd cluster
    03  container-runtime  to setup the container runtime(docker or containerd)
    04  kube-master        to setup the master nodes
    05  kube-node          to setup the worker nodes
    06  network            to setup the network plugin
    07  cluster-addon      to setup other useful plugins
    90  all                to run 01~07 all at once
    10  ex-lb              to install external loadbalance for accessing k8s from outside
    11  harbor             to install a new harbor server or to integrate with an existed one

examples: ./ezctl setup test-k8s 01  (or ./ezctl setup test-k8s prepare)
          ./ezctl setup test-k8s 02  (or ./ezctl setup test-k8s etcd)
          ./ezctl setup test-k8s all
          ./ezctl setup test-k8s 04 -t restart_master
root@master1:/etc/kubeasz# vim playbooks/01.prepare.yml
root@master1:/etc/kubeasz# ./ezctl setup k8s-01 01

3.5 upload pause image

root@master1:/etc/kubeasz# grep SANDBOX_IMAGE ./clusters/* -R
./clusters/k8s-01/config.yml:SANDBOX_IMAGE: "harbor.k8s.local/k8s/pause-amd64:3.4.1"

root@master1:/etc/kubeasz# docker pull easzlab/pause-amd64:3.4.1
3.4.1: Pulling from easzlab/pause-amd64
Digest: sha256:9ec1e780f5c0196af7b28f135ffc0533eddcb0a54a0ba8b32943303ce76fe70d
Status: Image is up to date for easzlab/pause-amd64:3.4.1
docker.io/easzlab/pause-amd64:3.4.1

root@master1:/etc/kubeasz# docker tag easzlab/pause-amd64:3.4.1 harbor.k8s.local/k8s/easzlab-pause-amd64:3.4.1

root@master1:/etc/kubeasz# docker push harbor.k8s.local/k8s/easzlab-pause-amd64:3.4.1
The push refers to repository [harbor.k8s.local/k8s/easzlab-pause-amd64]
915e8870f7d1: Pushed
3.4.1: digest: sha256:9ec1e780f5c0196af7b28f135ffc0533eddcb0a54a0ba8b32943303ce76fe70d size: 526

4. Deploy etcd node

4.1 deployment node

root@master1:/etc/kubeasz# ./ezctl setup k8s-01 02

4.2 verify the service status of each node

root@etcd1:~# export node_ip='192.168.6.84 192.168.6.85 192.168.6.86'
root@etcd1:~# for i in ${node_ip}; do ETCDCTL_API=3 /usr/local/bin/etcdctl --endpoints=https://${i}:2379 --cacert=/etc/kubernetes/ssl/ca.pem --cert=/etc/kubernetes/ssl/etcd.pem --key=/etc/kubernetes/ssl/etcd-key.pem  endpoint health; done
https://192.168.6.84:2379 is healthy: successfully committed proposal: took = 31.318624ms
https://192.168.6.85:2379 is healthy: successfully committed proposal: took = 16.456257ms
https://192.168.6.86:2379 is healthy: successfully committed proposal: took = 14.769783ms

5. Deploy container runtime

  • The node node must be installed with docker, which can be installed by itself. This step is optional
root@master1:/etc/kubeasz# ./ezctl setup k8s-01 03

6. Deploy the Master node

root@master1:/etc/kubeasz# ./ezctl setup k8s-01 04 

#Validation cluster
root@master1:/etc/kubeasz# kubectl get node
NAME           STATUS                     ROLES    AGE   VERSION
192.168.6.79   Ready,SchedulingDisabled   master   59s   v1.21.0
192.168.6.80   Ready,SchedulingDisabled   master   60s   v1.21.0

7. Deploy Node

7.1 modify the configuration by modifying the template (take modifying the configuration of Kube proxy as an example)

root@master1:/etc/kubeasz# vim /etc/kubeasz/roles/kube-node/templates/kube-proxy-config.yaml.j2 
kind: KubeProxyConfiguration
apiVersion: kubeproxy.config.k8s.io/v1alpha1
bindAddress: {{ inventory_hostname }}
clientConnection:
  kubeconfig: "/etc/kubernetes/kube-proxy.kubeconfig"
clusterCIDR: "{{ CLUSTER_CIDR }}"
conntrack:
  maxPerCore: 32768
  min: 131072
  tcpCloseWaitTimeout: 1h0m0s
  tcpEstablishedTimeout: 24h0m0s
healthzBindAddress: {{ inventory_hostname }}:10256
hostnameOverride: "{{ inventory_hostname }}"
metricsBindAddress: {{ inventory_hostname }}:10249
mode: "{{ PROXY_MODE }}"
ipvs:                  #Newly added, change the scheduling algorithm to sh, and the default is rr
  scheduler: sh 

7.2 deploying Node

root@master1:/etc/kubeasz# ./ezctl setup k8s-01 05

7.3 verification

7.3.1 viewing cluster status on the deployment machine

root@master1:/etc/kubeasz# kubectl get node
NAME           STATUS                     ROLES    AGE     VERSION
192.168.6.79   Ready,SchedulingDisabled   master   2d16h   v1.21.0
192.168.6.80   Ready,SchedulingDisabled   master   2d17h   v1.21.0
192.168.6.89   Ready                      node     2d16h   v1.21.0
192.168.6.90   Ready                      node     2d16h   v1.21.0


7.3.2 view the ipvs rules on the Node and confirm that the modification takes effect

root@node1:~# ipvsadm --list
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  node1.k8s.local:https sh                               #The scheduling algorithm is source hashing
  -> 192.168.6.79:6443            Masq    1      0          0
  -> 192.168.6.80:6443            Masq    1      0          0   

8. Deploy cluster network

8.1 modify the configuration template according to the selected network components

Take calico component as an example, modify the image address in the template (calico has four images that need to be changed) to the address of the local image warehouse to speed up the deployment. The image can be downloaded in advance and uploaded to the local Harbor warehouse

root@master1:/etc/kubeasz# vim roles/calico/templates/calico-v3.15.yaml.j2 
...
 serviceAccountName: calico-node
      # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force
      # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods.
      terminationGracePeriodSeconds: 0
      priorityClassName: system-node-critical
      initContainers:
        # This container installs the CNI binaries
        # and CNI network config file on each node.
        - name: install-cni
          image: harbor.k8s.local/k8s/calico-cni:v3.15.3   #Change to local harbor image address
          command: ["/install-cni.sh"]
          env:
            # Name of the CNI config file to create.
            - name: CNI_CONF_NAME
              value: "10-calico.conflist"
            # The CNI network config to install on each node.
            - name: CNI_NETWORK_CONFIG
              valueFrom:
                configMapKeyRef:
                  name: calico-config
                  key: cni_network_config
            # The location of the etcd cluster.   
...
...
 # Adds a Flex Volume Driver that creates a per-pod Unix Domain Socket to allow Dikastes
        # to communicate with Felix over the Policy Sync API.
        - name: flexvol-driver
          image: harbor.k8s.local/k8s/calico-pod2daemon-flexvol:v3.15.3  #Change to local harbor image address
          volumeMounts:
          - name: flexvol-driver-host
            mountPath: /host/driver
          securityContext:
            privileged: true  
...
...
containers:
        # Runs calico-node container on each Kubernetes node. This
        # container programs network policy and routes on each
        # host.
        - name: calico-node
          image: harbor.k8s.local/k8s/calico-node:v3.15.3   #Change to local harbor image address
          env:
            # The location of the etcd cluster.
            - name: ETCD_ENDPOINTS
              valueFrom:
                configMapKeyRef:
                  name: calico-config
                  key: etcd_endpoints  
...
...
 priorityClassName: system-cluster-critical
      # The controllers must run in the host network namespace so that
      # it isn't governed by policy that would prevent it from working.
      hostNetwork: true
      containers:
        - name: calico-kube-controllers
          image: harbor.k8s.local/k8s/calico-kube-controllers:v3.15.3  #Change to local harbor image address
          env:
            # The location of the etcd cluster.
            - name: ETCD_ENDPOINTS
              valueFrom:
                configMapKeyRef:
                  name: calico-config
                  key: etcd_endpoints                
...

######
#Note: in this configuration file:
...
 # Set MTU for the Wireguard tunnel device.
            - name: FELIX_WIREGUARDMTU
              valueFrom:
                configMapKeyRef:
                  name: calico-config
                  key: veth_mtu
            # The default IPv4 pool to create on startup if none exists. Pod IPs will be
            # chosen from this range. Changing this value after installation will have
            # no effect. This should fall within `--cluster-cidr`.
            - name: CALICO_IPV4POOL_CIDR
              value: "{{ CLUSTER_CIDR }}"       # No matter what deployment tool is used, if the cluster address segment is automatically defined, it must be modified to reflect the new customized address segment here
            # Disable file logging so `kubectl logs` works.
            - name: CALICO_DISABLE_FILE_LOGGING
              value: "true"
            # Set Felix endpoint to host default action to ACCEPT.
            - name: FELIX_DEFAULTENDPOINTTOHOSTACTION
              value: "ACCEPT"    
...

8.2 deploying cluster network plug-ins

root@master1:/etc/kubeasz# ./ezctl setup k8s-01 06

8.3 verify the deployment of network plug-ins

root@node1:~# calicoctl node status
Calico process is running.

IPv4 BGP status
+--------------+-------------------+-------+----------+-------------+
| PEER ADDRESS |     PEER TYPE     | STATE |  SINCE   |    INFO     |
+--------------+-------------------+-------+----------+-------------+
| 192.168.6.79 | node-to-node mesh | up    | 03:49:14 | Established |
| 192.168.6.80 | node-to-node mesh | up    | 03:49:14 | Established |
| 192.168.6.90 | node-to-node mesh | up    | 03:49:14 | Established |
+--------------+-------------------+-------+----------+-------------+

IPv6 BGP status
No IPv6 peers found.

root@node1:~# route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         192.168.6.1     0.0.0.0         UG    100    0        0 eth0
10.200.67.0     192.168.6.80    255.255.255.192 UG    0      0        0 tunl0
10.200.99.64    192.168.6.90    255.255.255.192 UG    0      0        0 tunl0
10.200.213.128  192.168.6.79    255.255.255.192 UG    0      0        0 tunl0
10.200.255.128  0.0.0.0         255.255.255.192 U     0      0        0 *
172.17.0.0      0.0.0.0         255.255.0.0     U     0      0        0 docker0
192.168.6.0     0.0.0.0         255.255.255.0   U     0      0        0 eth0
192.168.6.1     0.0.0.0         255.255.255.255 UH    100    0        0 eth0

8.4 create a test pod to verify the network

root@master1:/etc/kubeasz# kubectl run net-test --image=harbor.k8s.local/k8s/alpine:latest sleep 300000
root@master1:/etc/kubeasz# kubectl run net-test1 --image=harbor.k8s.local/k8s/alpine:latest sleep 300000 
root@master1:/etc/kubeasz# kubectl run net-test2 --image=harbor.k8s.local/k8s/alpine:latest sleep 300000 
root@master1:/etc/kubeasz# kubectl get pod -A -o wide
NAMESPACE     NAME                                       READY   STATUS    RESTARTS   AGE   IP               NODE           NOMINATED NODE   READINESS GATES
default       net-test                                   1/1     Running   0          41s   10.200.255.129   192.168.6.89   <none>           <none>
default       net-test1                                  1/1     Running   0          17s   10.200.99.65     192.168.6.90   <none>           <none>
default       net-test2                                  1/1     Running   0          8s    10.200.255.130   192.168.6.89   <none>           <none>
kube-system   calico-kube-controllers-759545cb9c-dr8gz   1/1     Running   0          71m   192.168.6.90     192.168.6.90   <none>           <none>
kube-system   calico-node-7zlwc                          1/1     Running   0          71m   192.168.6.79     192.168.6.79   <none>           <none>
kube-system   calico-node-k77p7                          1/1     Running   0          71m   192.168.6.89     192.168.6.89   <none>           <none>
kube-system   calico-node-sv98g                          1/1     Running   0          71m   192.168.6.90     192.168.6.90   <none>           <none>
kube-system   calico-node-x9qbn                          1/1     Running   0          71m   192.168.6.80     192.168.6.80   <none>           <none>

root@node1:~# kubectl exec -it net-test sh
/ # ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
2: tunl0@NONE: <NOARP> mtu 1480 qdisc noop state DOWN qlen 1000
    link/ipip 0.0.0.0 brd 0.0.0.0
4: eth0@if8: <BROADCAST,MULTICAST,UP,LOWER_UP,M-DOWN> mtu 1500 qdisc noqueue state UP
    link/ether 72:0c:91:dd:a3:6c brd ff:ff:ff:ff:ff:ff
    inet 10.200.255.129/32 scope global eth0
       valid_lft forever preferred_lft forever
/ # ping 10.200.99.65     #Enter the test container and ping whether the cross host address is connected
PING 10.200.99.65 (10.200.99.65): 56 data bytes
64 bytes from 10.200.99.65: seq=0 ttl=62 time=1.627 ms
64 bytes from 10.200.99.65: seq=1 ttl=62 time=0.856 ms
64 bytes from 10.200.99.65: seq=2 ttl=62 time=0.914 ms
^C
/ # ping 223.6.6.6        #ping whether the Internet address is connected
PING 223.6.6.6 (223.6.6.6): 56 data bytes
64 bytes from 223.6.6.6: seq=0 ttl=116 time=14.000 ms
64 bytes from 223.6.6.6: seq=1 ttl=116 time=7.925 ms
64 bytes from 223.6.6.6: seq=2 ttl=116 time=16.233 ms
^C
--- 223.6.6.6 ping statistics ---
3 packets transmitted, 3 packets received, 0% packet loss
round-trip min/avg/max = 7.925/12.719/16.233 ms

9. Deploy cluster plug-ins

9.1 core DNS deployment

9.1.1 main configuration parameters

  • Error: output the error log to stdout

  • Health: the running status of Coredns is reported as http://localhost:8080/health

  • cache: enable dns caching

  • Reload: automatically reload the configuration file. If the ConfigMap configuration is modified, it will take effect in two minutes

  • loadbalance: multiple records of a domain name will be polled and resolved

  • cache 30: cache time

  • Kubernetes: coredns will perform domain name resolution in kubernetes SVC according to the specified service domain name

  • forward: not all domain name queries in the Kubernetes cluster are forwarded to the specified server (/ etc/resolv.conf)

  • prometheus: the index data of coredns can be accessed by configuring prometheus http://coredns SVC: 9154/metrics for collection

  • Ready: after the coredns service is started, the status will be monitored, there will be a URL, the path is / ready, and the 200 status code will be returned; otherwise, an error will be returned

9.1.2 deploying coredns

9.1.2.1 get the yaml file of coredns

  • github: https://github.com/coredns/deployment/blob/master/kubernetes/coredns.yaml.sed

  • kubernetes official: the official deployment package is adopted, which contains relevant yaml files and scripts, including four packages. Select the corresponding version. Take version 1.21.4 as an example:

    • Corresponding version address of GitHub: https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.21.md#downloads-for-v1214
    • 4 packages:
      • kubernetes-node-linux-amd64.tar.gz
      • kubernetes-server-linux-amd64.tar.gz
      • kubernetes-client-linux-amd64.tar.gz
      • kubernetes.tar.gz

Upload to the server, extract and get the relevant yaml file, and use this file as a template for modification

root@master1:/usr/local/src# ls
kubernetes-client-linux-amd64.tar.gz  kubernetes-server-linux-amd64.tar.gz
kubernetes-node-linux-amd64.tar.gz    kubernetes.tar.gz
root@master1:/usr/local/src# tar xf kubernetes-client-linux-amd64.tar.gz
root@master1:/usr/local/src# tar xf kubernetes-server-linux-amd64.tar.gz
root@master1:/usr/local/src# tar xf kubernetes-node-linux-amd64.tar.gz
root@master1:/usr/local/src# tar xf kubernetes.tar.gz
root@master1:/usr/local/src# ls
kubernetes                            kubernetes-node-linux-amd64.tar.gz    kubernetes.tar.gz
kubernetes-client-linux-amd64.tar.gz  kubernetes-server-linux-amd64.tar.gz
root@master1:/usr/local/src/kubernetes/cluster/addons/dns/coredns# ls
Makefile  coredns.yaml.base  coredns.yaml.in  coredns.yaml.sed  transforms2salt.sed  transforms2sed.sed
# coredns.yaml.base can be modified as a template file (note that the template yaml obtained by this method deploys coredns. Note that the version of coredns corresponds to the yaml file

9.1.2.2 modify yaml file

Refer to 9.1.2.1 for the main parameters to be modified. Here, the deployment yaml file on coredns github is used as the template for modification.

root@master1:~# vim coredns.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  name: coredns
  namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  labels:
    kubernetes.io/bootstrapping: rbac-defaults
  name: system:coredns
rules:
  - apiGroups:
    - ""
    resources:
    - endpoints
    - services
    - pods
    - namespaces
    verbs:
    - list
    - watch
  - apiGroups:
    - discovery.k8s.io
    resources:
    - endpointslices
    verbs:
    - list
    - watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  annotations:
    rbac.authorization.kubernetes.io/autoupdate: "true"
  labels:
    kubernetes.io/bootstrapping: rbac-defaults
  name: system:coredns
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:coredns
subjects:
- kind: ServiceAccount
  name: coredns
  namespace: kube-system
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: coredns
  namespace: kube-system
data:
  Corefile: |
    .:53 {
        errors
        health {
          lameduck 5s
        }
        #bind 0.0.0.0    #I don't know the effect
        ready
        kubernetes k8s.local in-addr.arpa ip6.arpa {  ##Fill in the CLUSTER_DNS_DOMAIN configuration in kubeasz's cluster host file, that is, the domain name of the cluster
          fallthrough in-addr.arpa ip6.arpa
        }
        prometheus :9153
        forward . /etc/resolv.conf {  #Configure the parent dns of the cluster dns
          max_concurrent 1000
        }
        cache 30
        loop
        reload
        loadbalance
    }
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: coredns
  namespace: kube-system
  labels:
    k8s-app: kube-dns
    kubernetes.io/name: "CoreDNS"
spec:
  # replicas: not specified here:
  # 1. Default is 1.
  # 2. Will be tuned in real time if DNS horizontal auto-scaling is turned on.
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxUnavailable: 1
  selector:
    matchLabels:
      k8s-app: kube-dns
  template:
    metadata:
      labels:
        k8s-app: kube-dns
    spec:
      priorityClassName: system-cluster-critical
      serviceAccountName: coredns
      tolerations:
        - key: "CriticalAddonsOnly"
          operator: "Exists"
      nodeSelector:
        kubernetes.io/os: linux
      affinity:
         podAntiAffinity:
           preferredDuringSchedulingIgnoredDuringExecution:
           - weight: 100
             podAffinityTerm:
               labelSelector:
                 matchExpressions:
                   - key: k8s-app
                     operator: In
                     values: ["kube-dns"]
               topologyKey: kubernetes.io/hostname
      containers:
      - name: coredns
        image: harbor.k8s.local/k8s/coredns:1.8.3  #Mirror as a local mirror warehouse
        imagePullPolicy: IfNotPresent
        resources:
          limits:
            memory: 170Mi     #Here, you can modify the memory limit as needed. Generally, the production environment is limited to 4G at most, and more copies will be opened for load
          requests:
            cpu: 100m
            memory: 70Mi
        args: [ "-conf", "/etc/coredns/Corefile" ]
        volumeMounts:
        - name: config-volume
          mountPath: /etc/coredns
          readOnly: true
        ports:
        - containerPort: 53
          name: dns
          protocol: UDP
        - containerPort: 53
          name: dns-tcp
          protocol: TCP
        - containerPort: 9153
          name: metrics
          protocol: TCP
        securityContext:
          allowPrivilegeEscalation: false
          capabilities:
            add:
            - NET_BIND_SERVICE
            drop:
            - all
          readOnlyRootFilesystem: true
        livenessProbe:
          httpGet:
            path: /health
            port: 8080
            scheme: HTTP
          initialDelaySeconds: 60
          timeoutSeconds: 5
          successThreshold: 1
          failureThreshold: 5
        readinessProbe:
          httpGet:
            path: /ready
            port: 8181
            scheme: HTTP
      dnsPolicy: Default
      volumes:
        - name: config-volume
          configMap:
            name: coredns
            items:
            - key: Corefile
              path: Corefile
---
apiVersion: v1
kind: Service
metadata:
  name: kube-dns
  namespace: kube-system
  annotations:
    prometheus.io/port: "9153"
    prometheus.io/scrape: "true"
  labels:
    k8s-app: kube-dns
    kubernetes.io/cluster-service: "true"
    kubernetes.io/name: "CoreDNS"
spec:
  type: NodePort #It is actually added to expose metrics to prometheus
  selector:
    k8s-app: kube-dns
  clusterIP: 10.100.0.2  #In fact, it is the SERVICE_CIDR address, which is generally the second address of the IP segment. Here, you can enter the cluster test container and check the nameserver address in the / etc/resolv.conf file for confirmation.
  ports:
  - name: dns
    port: 53
    protocol: UDP
  - name: dns-tcp
    port: 53
    protocol: TCP
  - name: metrics
    port: 9153
    protocol: TCP
    targetPort: 9153   #Added, exposed metrics
    nodePort: 30009    #Added to expose metrics to port 30009 of the host

9.1.2.3 deploying coredns

root@master1:~# kubectl apply -f coredns.yaml

9.1.2.4 verification

Enter the test container and ping the extranet

root@master1:~# kubectl exec -it net-test1 sh
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
/ # ping www.baidu.com
PING www.baidu.com (110.242.68.4): 56 data bytes
64 bytes from 110.242.68.4: seq=0 ttl=51 time=10.841 ms
64 bytes from 110.242.68.4: seq=1 ttl=51 time=11.695 ms
64 bytes from 110.242.68.4: seq=2 ttl=51 time=21.819 ms
64 bytes from 110.242.68.4: seq=3 ttl=51 time=12.340 ms
^C
--- www.baidu.com ping statistics ---
4 packets transmitted, 4 packets received, 0% packet loss
round-trip min/avg/max = 10.841/14.173/21.819 ms

9.2 deploying Dashboard

9.2.1 obtaining basic images and yaml files

  • yaml file:

Github kubernetes dashboard -- determine the corresponding dashboard version according to the version selection of kubernetes cluster

https://github.com/kubernetes/dashboard/releases

wget https://raw.githubusercontent.com/kubernetes/dashboard/v2.3.1/aio/deploy/recommended.yaml
  • Basic image:
kubernetesui/dashboard:v2.3.1
kubernetesui/metrics-scraper:v1.0.6
#Download the image and push it to the private image warehouse
root@harbor1:~# docker tag kubernetesui/metrics-scraper:v1.0.6 harbor.k8s.local/k8s/kubernetesui-metrics-scraper:v1.0.6
root@harbor1:~# docker tag kubernetesui/dashboard:v2.3.1 harbor.k8s.local/k8s/kubernetesui-dashboard:v2.3.1               
root@harbor1:~# docker push harbor.k8s.local/k8s/kubernetesui-metrics-scraper:v1.0.6
root@harbor1:~# docker push harbor.k8s.local/k8s/kubernetesui-dashboard:v2.3.1

9.2.2 modify yaml file

root@master1:~# cp recommended.yaml k8s-dashboard.yaml
root@master1:~# vim k8s-dashboard.yaml
# Copyright 2017 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

apiVersion: v1
kind: Namespace
metadata:
  name: kubernetes-dashboard

---

apiVersion: v1
kind: ServiceAccount
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard
  namespace: kubernetes-dashboard

---

kind: Service
apiVersion: v1
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard
  namespace: kubernetes-dashboard
spec:
  type: NodePort          #Add and expose dashboard ports
  ports:
    - port: 443
      targetPort: 8443
      nodePort: 38443     #Add and expose dashboard ports
  selector:
    k8s-app: kubernetes-dashboard

---

apiVersion: v1
kind: Secret
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard-certs
  namespace: kubernetes-dashboard
type: Opaque

---

apiVersion: v1
kind: Secret
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard-csrf
  namespace: kubernetes-dashboard
type: Opaque
data:
  csrf: ""

---

apiVersion: v1
kind: Secret
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard-key-holder
  namespace: kubernetes-dashboard
type: Opaque

---

kind: ConfigMap
apiVersion: v1
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard-settings
  namespace: kubernetes-dashboard

---

kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard
  namespace: kubernetes-dashboard
rules:
  # Allow Dashboard to get, update and delete Dashboard exclusive secrets.
  - apiGroups: [""]
    resources: ["secrets"]
    resourceNames: ["kubernetes-dashboard-key-holder", "kubernetes-dashboard-certs", "kubernetes-dashboard-csrf"]
    verbs: ["get", "update", "delete"]
    # Allow Dashboard to get and update 'kubernetes-dashboard-settings' config map.
  - apiGroups: [""]
    resources: ["configmaps"]
    resourceNames: ["kubernetes-dashboard-settings"]
    verbs: ["get", "update"]
    # Allow Dashboard to get metrics.
  - apiGroups: [""]
    resources: ["services"]
    resourceNames: ["heapster", "dashboard-metrics-scraper"]
    verbs: ["proxy"]
  - apiGroups: [""]
    resources: ["services/proxy"]
    resourceNames: ["heapster", "http:heapster:", "https:heapster:", "dashboard-metrics-scraper", "http:dashboard-metrics-scraper"]
    verbs: ["get"]

---

kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard
rules:
  # Allow Metrics Scraper to get metrics from the Metrics server
  - apiGroups: ["metrics.k8s.io"]
    resources: ["pods", "nodes"]
    verbs: ["get", "list", "watch"]

---

apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard
  namespace: kubernetes-dashboard
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: kubernetes-dashboard
subjects:
  - kind: ServiceAccount
    name: kubernetes-dashboard
    namespace: kubernetes-dashboard

---

apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: kubernetes-dashboard
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: kubernetes-dashboard
subjects:
  - kind: ServiceAccount
    name: kubernetes-dashboard
    namespace: kubernetes-dashboard

---

kind: Deployment
apiVersion: apps/v1
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard
  namespace: kubernetes-dashboard
spec:
  replicas: 1
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      k8s-app: kubernetes-dashboard
  template:
    metadata:
      labels:
        k8s-app: kubernetes-dashboard
    spec:
      containers:
        - name: kubernetes-dashboard
          image: harbor.k8s.local/k8s/kubernetesui-dashboard:v2.3.1  #Change to private image warehouse
          imagePullPolicy: Always
          ports:
            - containerPort: 8443
              protocol: TCP
          args:
            - --auto-generate-certificates
            - --namespace=kubernetes-dashboard
            # Uncomment the following line to manually specify Kubernetes API server Host
            # If not specified, Dashboard will attempt to auto discover the API server and connect
            # to it. Uncomment only if the default does not work.
            # - --apiserver-host=http://my-address:port
          volumeMounts:
            - name: kubernetes-dashboard-certs
              mountPath: /certs
              # Create on-disk volume to store exec logs
            - mountPath: /tmp
              name: tmp-volume
          livenessProbe:
            httpGet:
              scheme: HTTPS
              path: /
              port: 8443
            initialDelaySeconds: 30
            timeoutSeconds: 30
          securityContext:
            allowPrivilegeEscalation: false
            readOnlyRootFilesystem: true
            runAsUser: 1001
            runAsGroup: 2001
      volumes:
        - name: kubernetes-dashboard-certs
          secret:
            secretName: kubernetes-dashboard-certs
        - name: tmp-volume
          emptyDir: {}
      serviceAccountName: kubernetes-dashboard
      nodeSelector:
        "kubernetes.io/os": linux
      # Comment the following tolerations if Dashboard must not be deployed on master
      tolerations:
        - key: node-role.kubernetes.io/master
          effect: NoSchedule

---

kind: Service
apiVersion: v1
metadata:
  labels:
    k8s-app: dashboard-metrics-scraper
  name: dashboard-metrics-scraper
  namespace: kubernetes-dashboard
spec:
  ports:
    - port: 8000
      targetPort: 8000
  selector:
    k8s-app: dashboard-metrics-scraper

---

kind: Deployment
apiVersion: apps/v1
metadata:
  labels:
    k8s-app: dashboard-metrics-scraper
  name: dashboard-metrics-scraper
  namespace: kubernetes-dashboard
spec:
  replicas: 1
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      k8s-app: dashboard-metrics-scraper
  template:
    metadata:
      labels:
        k8s-app: dashboard-metrics-scraper
      annotations:
        seccomp.security.alpha.kubernetes.io/pod: 'runtime/default'
    spec:
      containers:
        - name: dashboard-metrics-scraper
          image: harbor.k8s.local/k8s/kubernetesui-metrics-scraper:v1.0.6  #Change to private image warehouse
          ports:
            - containerPort: 8000
              protocol: TCP
          livenessProbe:
            httpGet:
              scheme: HTTP
              path: /
              port: 8000
            initialDelaySeconds: 30
            timeoutSeconds: 30
          volumeMounts:
          - mountPath: /tmp
            name: tmp-volume
          securityContext:
            allowPrivilegeEscalation: false
            readOnlyRootFilesystem: true
            runAsUser: 1001
            runAsGroup: 2001
      serviceAccountName: kubernetes-dashboard
      nodeSelector:
        "kubernetes.io/os": linux
      # Comment the following tolerations if Dashboard must not be deployed on master
      tolerations:
        - key: node-role.kubernetes.io/master
          effect: NoSchedule
      volumes:
        - name: tmp-volume
          emptyDir: {}

9.2.3 deploying Dashboard

kubectl apply -f k8s-dashboard.yaml

9.2.4 verification and use

9.2.4.1 create account

root@master1:~# vim admin-user.yml
apiVersion: v1
kind: ServiceAccount
metadata:
  name: admin-user
  namespace: kubernetes-dashboard

---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: admin-user
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
subjects:
- kind: ServiceAccount
  name: admin-user
  namespace: kubernetes-dashboard

9.2.4.1 obtain the token and log in for verification

root@master1:~# kubectl get secrets -A | grep admin
kubernetes-dashboard   admin-user-token-xwrjl                           kubernetes.io/service-account-token   3      74s

root@master1:~# kubectl describe secrets  admin-user-token-xwrjl  -n kubernetes-dashboard
Name:         admin-user-token-xwrjl
Namespace:    kubernetes-dashboard
Labels:       <none>
Annotations:  kubernetes.io/service-account.name: admin-user
              kubernetes.io/service-account.uid: d0286365-2a5e-4955-adcc-f4ec78d13f65

Type:  kubernetes.io/service-account-token

Data
====
ca.crt:     1350 bytes
namespace:  20 bytes
token:      eyJhbGciOiJSUzI1NiIsImtpZCI6InZhRkxvMGNjbHFPQ1A5ak1vd0hfVjM4bTRfNnRHYmt5SmhiR2ZYaDVWM2cifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlcm5ldGVzLWRhc2hib2FyZCIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJhZG1pbi11c2VyLXRva2VuLXh3cmpsIiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQubmFtZSI6ImFkbWluLXVzZXIiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC51aWQiOiJkMDI4NjM2NS0yYTVlLTQ5NTUtYWRjYy1mNGVjNzhkMTNmNjUiLCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6a3ViZXJuZXRlcy1kYXNoYm9hcmQ6YWRtaW4tdXNlciJ9.GE0fZh7VOvl0yJa6moWP546OEZV_HlA8wn2nlpiDSKIo97UHvPfd6I8hhe-LiPOo9eNWhnTmZqnA3c3_4yvqUMVASUAqykFISs4aOKZFpZ_RpW2QLTTJra04p0vXSOlvprNaxVAQnrCA2oRQYSV_5Y5rS49Lj_YEx9IqOb1PXJSS4Bl5y9YOYjAlVmE31oh2ynSd2G06x9-TUil3vx_8a0UEz74zzpejesen9cF_A9bZwywymw9HU7BWL5EzoSdpN_rHcrXAKzxPne4v_J7320jrghAciPFFRSpgApqmuc9jG-8msy24g-oljmfHg--obkl3y7oNLe1b8RYkbp9qEw

10. Adding cluster nodes

10.1 adding a Master node

root@master1:~# cd /etc/kubeasz/
root@master1:/etc/kubeasz# ./ezctl add-master k8s-01 192.168.6.81 

10.2 add node

root@master1:/etc/kubeasz# ./ezctl add-node k8s-01 192.168.6.91

Tags: Kubernetes

Posted on Thu, 07 Oct 2021 19:35:07 -0400 by shortysbest