docker builds a massive real-time log analysis system

Server directory

/home/log-script/lib //Contain elasticsearch-6.3.0.tar.gz filebeat-6.3.0-linux-x86_64.tar.gz jdk-8u171-linux-x64.tar.gz kafka_2.11-1.1.0.tgz kibana-6.3.0-linux-x86_64.tar.gz lc-centos7-ssh.tar logstash-6.3.0.tar.gz zookeeper-3.4.12.tar.gz

Decompression jdk
tar -zxvf jdk-8u171-linux-x64.tar.gz -C /home/log-script/lib

Create docker network

]# docker network create --subnet=172.172.0.0/24 elknet

tips

docker network rm elknet method of deleting network docker network ls how to view the existing network Other servers access docker on 192.168.62.133 to add routes route add -net 172.17.0.0 netmask 255.255.0.0 gw 192.168.62.133

This step is the precondition to fix the ip address of docker, otherwise, every time the docker is started, it will assign other ip addresses

Build start es

]# docker run --name=elasticsearch --net elknet --ip 172.172.0.9 --privileged=true -e \ TZ=Asia/Shanghai -v /home/log-script/lib/:/home/lib -itd lc-centos7-ssh bin/bash ]# docker exec -it elasticsearch bin/bash /]# tar -zxvf /home/lib/elasticsearch-6.3.0.tar.gz -C /home /]# mkdir /home/es_data /]# mkdir /home/es_logs /]# vi /home/elasticsearch-6.3.0/config/elasticsearch.yml //Add and save node.name: es path.data: /home/es_data path.logs: /home/es_logs network.host: 172.172.0.9 http.port: 9200

Increase the memory and modify the file handle. Otherwise, it will prompt Max virtual memory areas vm.max map count [65530] is too low, increase to at least [262144]

/]# vi /etc/security/limits.conf Add and save * soft nofile 65536 * hard nofile 131072 * soft nproc 4096 * hard nproc 4096 /]# vi /etc/security/limits.d/90-nproc.conf Add and save * soft nproc 4096 /]# vi /etc/sysctl.conf Add and save vm.max_map_count=655360 /]# sysctl -p

Continue. es cannot be started with root. You need to create a user

/]# yum install -y which /]# adduser es /]# passwd es /]# chown -R es /home/es_* /]# vi /home/startes.sh //Add and save #!/bin/bash export JAVA_HOME=/home/lib/jdk1.8.0_171/ export PATH=$JAVA_HOME/bin:$PATH /home/elasticsearch-6.3.0/bin/elasticsearch -d /]# chmod 777 /home/startes.sh

Exit container and start

]# docker exec -i -u es elasticsearch /home/startes.sh

Browser open, see success page
http://172.172.0.9:9200/

Start kibana

]# docker run --name=kibana --net elknet --ip 172.172.0.10 --privileged=true -e TZ=Asia/Shanghai \ -v /home/log-script/lib/:/home/lib -itd lc-centos7-ssh bin/bash ]# docker exec -it kibana bin/bash /]# tar -zxvf /home/lib/kibana-6.3.0-linux-x86_64.tar.gz -C /home /]# vi /home/kibana-6.3.0-linux-x86_64/config/kibana.yml //Add and save server.port: 5601 server.host: "172.172.0.10" elasticsearch.url: "http://172.172.0.9:9200" /]# nohup /home/kibana-6.3.0-linux-x86_64/bin/kibana > /dev/null &

Open web page
http://172.172.0.10:5601/

Start zookeeper

Configure zookeeper 1

]# docker run --name=zookeeper1 --net elknet --ip 172.172.0.5 --privileged=true -e\ TZ=Asia/Shanghai -v /home/log-script/lib/:/home/lib -itd lc-centos7-ssh bin/bash ]# docker exec -it zookeeper1 bin/bash /]# tar -zxvf /home/lib/zookeeper-3.4.12.tar.gz -C /home /]# cp /home/zookeeper-3.4.12/conf/zoo_sample.cfg /home/zookeeper-3.4.12/conf/zoo.cfg /]# mkdir /home/zoodata /]# vi /home/zookeeper-3.4.12/conf/zoo.cfg //Add and save (Note: dataDir has default value) dataDir=/home/zoodata server.1=172.172.0.5:12888:13888 server.2=172.172.0.6:12888:13888 server.3=172.172.0.7:12888:13888 /]# echo 1 >/home/zoodata/myid /]# vi /home/zookeeper-3.4.12/bin/zkServer.sh //Add and save export JAVA_HOME=/home/lib/jdk1.8.0_171/ export PATH=$JAVA_HOME/bin:$PATH ctrl+d Exit container

Configure zookeeper 2

]# docker run --name=zookeeper2 --net elknet --ip 172.172.0.6 --privileged=true -e\ TZ=Asia/Shanghai -v /home/log-script/lib/:/home/lib -itd lc-centos7-ssh bin/bash ]# docker exec -it zookeeper2 bin/bash /]# tar -zxvf /home/lib/zookeeper-3.4.12.tar.gz -C /home /]# cp /home/zookeeper-3.4.12/conf/zoo_sample.cfg /home/zookeeper-3.4.12/conf/zoo.cfg /]# mkdir /home/zoodata /]# vi /home/zookeeper-3.4.12/conf/zoo.cfg //Add and save dataDir=/home/zoodata server.1=172.172.0.5:12888:13888 server.2=172.172.0.6:12888:13888 server.3=172.172.0.7:12888:13888 /]# echo 2 >/home/zoodata/myid /]# vi /home/zookeeper-3.4.12/bin/zkServer.sh //Add and save export JAVA_HOME=/home/lib/jdk1.8.0_171/ export PATH=$JAVA_HOME/bin:$PATH ctrl+d Exit container

Configure zookeeper 3

]# docker run --name=zookeeper3 --net elknet --ip 172.172.0.7 --privileged=true -e\ TZ=Asia/Shanghai -v /home/log-script/lib/:/home/lib -itd lc-centos7-ssh bin/bash ]# docker exec -it zookeeper3 bin/bash /]# tar -zxvf /home/lib/zookeeper-3.4.12.tar.gz -C /home /]# cp /home/zookeeper-3.4.12/conf/zoo_sample.cfg /home/zookeeper-3.4.12/conf/zoo.cfg /]# mkdir /home/zoodata /]# vi /home/zookeeper-3.4.12/conf/zoo.cfg //Add and save dataDir=/home/zoodata server.1=172.172.0.5:12888:13888 server.2=172.172.0.6:12888:13888 server.3=172.172.0.7:12888:13888 /]# echo 3 >/home/zoodata/myid /]# vi /home/zookeeper-3.4.12/bin/zkServer.sh //Add and save export JAVA_HOME=/home/lib/jdk1.8.0_171/ export PATH=$JAVA_HOME/bin:$PATH ctrl+d Exit container

start-up

]# docker exec -i zookeeper1 /home/zookeeper-3.4.12/bin/zkServer.sh start ]# docker exec -i zookeeper2 /home/zookeeper-3.4.12/bin/zkServer.sh start ]# docker exec -i zookeeper3 /home/zookeeper-3.4.12/bin/zkServer.sh start ]# docker exec -i zookeeper2 /home/zookeeper-3.4.12/bin/zkServer.sh status

Start kafka

Configure kafka1

]# docker run --name=kafka1 --net elknet --ip 172.172.0.3 --privileged=true -e TZ=Asia/Shanghai \ -v /home/log-script/lib/:/home/lib -itd lc-centos7-ssh bin/bash ]# docker exec -it kafka1 bin/bash /]# tar -zxvf /home/lib/kafka_2.11-1.1.0.tgz -C /home /]# vi /home/kafka_2.11-1.1.0/config/server.properties //Where port and host.name are add configurations broker.id=1 port = 9092 host.name = 172.172.0.3 zookeeper.connect=172.172.0.5:2181,172.172.0.6:2181,172.172.0.7:2181 /]# vi /home/kafka_2.11-1.1.0/bin/kafka-server-start.sh export JAVA_HOME=/home/lib/jdk1.8.0_171/ export PATH=$JAVA_HOME/bin:$PATH //Exit and start docker exec -i kafka1 /home/kafka_2.11-1.1.0/bin/kafka-server-start.sh -daemon \ /home/kafka_2.11-1.1.0/config/server.properties

Configure kafka2

]# docker run --name=kafka2 --net elknet --ip 172.172.0.4 --privileged=true -e TZ=Asia/Shanghai \ -v /home/log-script/lib/:/home/lib -itd lc-centos7-ssh bin/bash ]# docker exec -it kafka2 bin/bash /]# tar -zxvf /home/lib/kafka_2.11-1.1.0.tgz -C /home /]# vi /home/kafka_2.11-1.1.0/config/server.properties //Where port and host.name are add configurations broker.id=2 port = 9092 host.name = 172.172.0.4 zookeeper.connect=172.172.0.5:2181,172.172.0.6:2181,172.172.0.7:2181 /]# vi /home/kafka_2.11-1.1.0/bin/kafka-server-start.sh export JAVA_HOME=/home/lib/jdk1.8.0_171/ export PATH=$JAVA_HOME/bin:$PATH //Exit and start docker exec -i kafka2 /home/kafka_2.11-1.1.0/bin/kafka-server-start.sh -daemon \ /home/kafka_2.11-1.1.0/config/server.properties

Test, production input asd, consumer display asd, successful

kafka2 Create theme on ]# docker exec -it kafka2 bin/bash /]# export JAVA_HOME=/home/lib/jdk1.8.0_171/;export PATH=$JAVA_HOME/bin:$PATH /]# /home/kafka_2.11-1.1.0/bin/kafka-topics.sh --create --zookeeper \ 172.172.0.5:2181 --replication-factor 1 --partitions 2 --topic ecplogs kafka2 production /]# /home/kafka_2.11-1.1.0/bin/kafka-console-producer.sh --broker-list \ 172.172.0.3:9092 --topic ecplogs >asd kafka1 consumption ]# docker exec -it kafka1 bin/bash /]# export JAVA_HOME=/home/lib/jdk1.8.0_171/;export PATH=$JAVA_HOME/bin:$PATH /]# /home/kafka_2.11-1.1.0/bin/kafka-console-consumer.sh --zookeeper 172.172.0.5:2181 --topic \ ecplogs --from-beginning asd

Start logstashes

]# docker run --name=logstashes --net elknet --ip 172.172.0.8 --privileged=true \ -e TZ=Asia/Shanghai -v /home/log-script/lib/:/home/lib -itd lc-centos7-ssh bin/bash ]# docker exec -it logstashes bin/bash /]# tar -zxvf /home/lib/logstash-6.3.0.tar.gz -C /home /]# vi /home/logstash-6.3.0/config/logstash_to_es.conf input { kafka { bootstrap_servers => "172.172.0.3:9092,172.172.0.4:9092" topics => ["ecplogs"] } } output { elasticsearch { hosts => ["172.172.0.9:9200"] index => "ecp-log-%{+YYYY.MM.dd}" } } /]# vi /home/startlogstash.sh #!/bin/bash export JAVA_HOME=/home/lib/jdk1.8.0_171/;export PATH=$JAVA_HOME/bin:$PATH nohup /home/logstash-6.3.0/bin/logstash -f /home/logstash-6.3.0/config/logstash_to_es.conf \ >/dev/null & /]# chmod 777 /home/startlogstash.sh //Exit start up ]# docker exec -i logstashes /home/startlogstash.sh

Start logstashbeat

]# docker run --name=logstashbeat --net elknet --ip 172.172.0.2 --privileged=true \ -e TZ=Asia/Shanghai -v /home/log-script/lib/:/home/lib -itd lc-centos7-ssh bin/bash ]# docker exec -it logstashbeat bin/bash /]# tar -zxvf /home/lib/logstash-6.3.0.tar.gz -C /home /]# vi /home/logstash-6.3.0/config/beat_to_logstash.conf input { beats { port => 5044 } } output { kafka { bootstrap_servers => "172.172.0.3:9092,172.172.0.4:9092" topic_id => "ecplogs" } } /]# vi /home/startlogstash.sh #!/bin/bash export JAVA_HOME=/home/lib/jdk1.8.0_171/ export PATH=$JAVA_HOME/bin:$PATH nohup /home/logstash-6.3.0/bin/logstash -f /home/logstash-6.3.0/config/beat_to_logstash.conf \ >/dev/null & /]# chmod 777 /home/startlogstash.sh //Exit start up ]# docker exec -i logstashbeat /home/startlogstash.sh

Start filebeat

]# docker run --name=filebeat --net elknet --ip 172.172.0.11 --privileged=true \ -e TZ=Asia/Shanghai -v /home/log-script/lib/:/home/lib -itd lc-centos7-ssh bin/bash ]# docker exec -it filebeat bin/bash /]# tar -zxvf /home/lib/filebeat-6.3.0-linux-x86_64.tar.gz -C /home //Modify profile /]# vi /home/filebeat-6.3.0-linux-x86_64/filebeat.yml //Modify the filebeat.inputs section //Comment on the Elasticsearch output section //Modify the output.logstash section filebeat.inputs: - type: log enabled: true paths: - /home/log/*.log output.logstash: hosts: ["172.172.0.2:5044"] //start-up /]#mkdir /home/log /]#nohup /home/filebeat-6.3.0-linux-x86_64/filebeat -c \ /home/filebeat-6.3.0-linux-x86_64/filebeat.yml >/home/log/beat.log &