[关闭]
@zhangyy 2021-10-22T16:38:00.000000Z 字数 7004 阅读 276

ElasticSearch 集群部署

ElasticSearch系列

一：ElasticSearch集群部署

1.1 系统环境准备

  1. 系统：rhel8.3x64
  2. cat /etc/hosts 
  3. ----
  4. 192.168.100.41  rhel01.flyfish.cn
  5. 192.168.100.42  rhel02.flyfish.cn
  6. 192.168.100.43  rhel03.flyfish.cn
  7. 192.168.100.44  rhel04.flyfish.cn
  8. 192.168.100.45  rhel05.flyfish.cn
  9. 192.168.100.46  rhel06.flyfish.cn
  10. 192.168.100.47  rhel07.flyfish.cn
  11. 192.168.100.48  rhel08.flyfish.cn
  13. ----
  14. 本次安装 只涉及到 前三台

image_1et6a2llu1oginvs1an71ugndak9.png-86.7kB

  1. 系统语言：
  3. echo "export LANG=en_US.UTF8" >> ~/.bash_profile
  4. cat ~/.bash_profile
  6. ---
  7. 分区：
  8. pvcreate /dev/sdb
  9. vgcreate esdatavg /dev/sdb
  10. lvcreate -n esdatalv -L 100000M esdatavg
  11. mkfs.xfs /dev/esdatavg/esdatalv
  12. ---
  13. vi /etc/fstab
  14. ---
  15. /dev/esdatavg/esdatalv /esdb xfs defaults 0 0
  16. ---
  17. mkdir /esdb
  18. mount /esdb
  1. mkdir /esdb/soft
  3. groupadd -g 60001 esadmin
  4. useradd -u 61001 -g esadmin esadmin
  5. chown -R esadmin:esadmin /esdb
  6. chmod -R 775 /esdb
  7. echo "esadmin" | passwd --stdin esadmin
  8. systemctl set-default multi-user.target
  10. cat >> /etc/security/limits.conf << EOF
  12. root soft nofile 1048576
  13. root hard nofile 1048576
  14. esadmin soft nproc 1048576
  15. esadmin hard nproc 1048576
  16. esadmin soft nofile 1048576
  17. esadmin hard nofile 1048576
  18. esadmin soft stack 10240
  19. esadmin hard stack 32768
  20. esadmin hard memlock unlimited
  21. esadmin soft memlock unlimited
  22. EOF
  23. ---
  24. ---
  26. cat >> /etc/sysctl.conf << EOF
  27. fs.aio-max-nr = 1048576
  28. fs.file-max = 6815744
  29. net.ipv4.ip_local_port_range = 1024 65535
  30. net.ipv4.tcp_mem = 786432 2097152 3145728
  31. net.ipv4.tcp_rmem = 4096 4096 16777216
  32. net.ipv4.tcp_wmem = 4096 4096 16777216
  33. net.ipv4.tcp_tw_reuse = 1
  34. net.ipv4.tcp_tw_recycle = 1
  35. vm.swappiness=1
  36. vm.min_free_kbytes=204800
  37. vm.max_map_count=2048000
  38. kernel.pid_max=819200
  39. vm.zone_reclaim_mode=0
  40. #vm.nr_hugepages = 0
  41. EOF
  43. sysctl -p 
  1. ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
  2. hwclock
  4. echo "SELINUX=disabled" > /etc/selinux/config
  5. echo "#SELINUXTYPE=targeted " >> /etc/selinux/config
  6. cat /etc/selinux/config
  7. setenforce 0
  8. systemctl stop firewalld.service
  9. systemctl disable firewalld.service
  11. cat >> /home/esadmin/.vimrc << EOF
  12. map <F10>:set paste<CR>
  13. map <F11>:set nopaste<CR>
  14. EOF
  16. shutdown -r now
  1. 做esadmin 的无密码登录
  3. /esdb/soft/fgssh/fgssh -user esadmin -hosts "rhel01.flyfish.cn rhel02.flyfish.cn rhel03.flyfish.cn" -advanced -exverify -confirm
  5. chmod 600 /home/esadmin/.ssh/config
  1. su - esadmin
  2. cd /esdb/
  3. tar zxvf /esdb/soft/jdk-14.0.2_linux-x64_bin.tar.gz
  4. scp -r /esdb/jdk-14.0.2 rhel02.flyfish.cn:/esdb/
  5. scp -r /esdb/jdk-14.0.2 rhel03.flyfish.cn:/esdb/
  1. su - esadmin
  2. cat >> ~/.bash_profile << EOF
  3. export JAVA_HOME=/esdb/jdk-14.0.2
  4. export PATH=/esdb/jdk-14.0.2/bin:$PATH
  5. export LANG=en_US.UTF8
  6. EOF
  7. source ~/.bash_profile
  8. java -version

image_1et6ao7um4hi1gs51b4k6ugmrm.png-77.8kB

image_1et6aopnp1esgrcg1rf71huj5kb13.png-72.1kB

image_1et6aq92ivkl8ugphi1pe3rik1g.png-85.9kB

  1. 全部主机：
  2. mkdir -p /esdb/soft/
  3. mkdir -p /esdb/esapp/esapp9200
  4. mkdir -p /esdb/esdata/esdata9200
  5. mkdir -p /esdb/eslog/eslog9200
  7. chown esadmin:esadmin -R /esdb/
  1. rhel01.flyfish.cn 节点：
  3. cd /esdb/soft
  4. tar zxvf /esdb/soft/elasticsearch-7.9.0-linux-x86_64.tar.gz
  5. cp -r /esdb/soft/elasticsearch-7.9.0/* /esdb/esapp/esapp9200/
  7. vim /esdb/esapp/esapp9200/config/elasticsearch.yml
  8. ---
  9. cluster.name: flyfishescluster1
  10. node.name: 192.168.100.41
  11. path.data: /esdb/esdata/esdata9200
  12. path.logs: /esdb/eslog/eslog9200
  13. node.master: true
  14. node.data: true
  15. bootstrap.memory_lock: true
  16. bootstrap.system_call_filter: false
  17. network.host: 192.168.100.41
  18. http.port: 9200
  19. transport.tcp.port: 9300
  20. http.cors.enabled: true
  21. http.cors.allow-origin: "*"
  22. discovery.seed_hosts: ["192.168.100.41:9300","192.168.100.42:9300","192.168.100.43:9300"]
  23. discovery.zen.fd.ping_timeout: 60s
  24. discovery.zen.fd.ping_retries: 5
  25. cluster.initial_master_nodes:
  26. ["192.168.100.41:9300","192.168.100.42:9300","192.168.100.43:9300"]
  27. ---
  2. vi /esdb/esapp/esapp9200/bin/elasticsearch-env
  3. ---
  4. export JAVA_HOME=/esdb/jdk-14.0.2
  5. export PATH=$JAVA_HOME/bin:$PATH
  6. ---
  1. vi /esdb/esapp/esapp9200/config/jvm.options
  2. ---
  3. -Xms256m
  4. -Xmx256m
  5. -Xmn100m
  6. -Xss228K
  7. -XX:+DisableExplicitGC
  8. -XX:HeapDumpPath=/esdb/eslog/eslog9200
  9. -XX:ErrorFile=/esdb/eslog/eslog9200/hs_err_pid%p.log
  10. 9-:-Xlog:gc*,gc+age=trace,safepoint:file=/esdb/eslog/eslog9200/gc.log:utctime,pid,ta
  11. gs:filecount=32,filesize=64m
  1. cd /esdb/esapp/esapp9200
  2. scp -r * esadmin@rhel02.flyfish.cn:/esdb/esapp/esapp9200/
  3. scp -r * esadmin@rhel03.flyfish.cn:/esdb/esapp/esapp9200/
  1. rhel02.flyfish.cn 主机：
  2.    vi /esdb/esapp/esapp9200/config/elasticsearch.yml
  3. ---
  4. cluster.name: flyfishescluster1
  5. node.name: 192.168.100.42
  6. path.data: /esdb/esdata/esdata9200
  7. path.logs: /esdb/eslog/eslog9200
  8. node.master: true
  9. node.data: true
  10. bootstrap.memory_lock: true
  11. bootstrap.system_call_filter: false
  12. network.host: 192.168.100.42
  13. http.port: 9200
  14. transport.tcp.port: 9300
  15. http.cors.enabled: true
  16. http.cors.allow-origin: "*"
  17. discovery.seed_hosts: ["192.168.100.41:9300","192.168.100.42:9300","192.168.100.43:9300"]
  18. discovery.zen.fd.ping_timeout: 60s
  19. discovery.zen.fd.ping_retries: 5
  20. cluster.initial_master_nodes: ["192.168.100.41:9300","192.168.100.42:9300","192.168.100.43:9300"]
  21. ---
  1. rhel03.flyfish.cn 主机：
  2.    vi /esdb/esapp/esapp9200/config/elasticsearch.yml
  3. ----
  4. cluster.name: flyfishescluster1
  5. node.name: 192.168.100.43
  6. path.data: /esdb/esdata/esdata9200
  7. path.logs: /esdb/eslog/eslog9200
  8. node.master: true
  9. node.data: true
  10. bootstrap.memory_lock: true
  11. bootstrap.system_call_filter: false
  12. network.host: 192.168.100.43
  13. http.port: 9200
  14. transport.tcp.port: 9300
  15. http.cors.enabled: true
  16. http.cors.allow-origin: "*"
  17. discovery.seed_hosts: ["192.168.100.41:9300","192.168.100.42:9300","192.168.100.43:9300"]
  18. discovery.zen.fd.ping_timeout: 60s
  19. discovery.zen.fd.ping_retries: 5
  20. cluster.initial_master_nodes: ["192.168.100.41:9300","192.168.100.42:9300","192.168.100.43:9300"]
  21. ----
  1. 启动es （全部机器执行）
  2.     /esdb/esapp/esapp9200/bin/elasticsearch

image_1f0t4dfno1nc11k8k1kfl1d9o1bi19.png-43.7kB

  1. 测试： 
  2.      curl http://192.168.100.41:9200/_cat/nodes

image_1et6c9rf71f6c1hj1juqcue1jti9.png-125.4kB

  1. curl http://192.168.100.41:9200/_cat/health?pretty -H 'Content-Type: application/json'

image_1et6cb90rv19199k177d1btn1dedm.png-104.9kB

  1. 启动脚本：
  2.   mkdir -p /esdb/scripts/
  3.  vi /esdb/scripts/es_start.sh
  4.   ----
  5. #!/bin/bash
  6. echo "es cluster all node start..."
  7. for host in rhel01.flyfish.cn rhel02.flyfish.cn rhel03.flyfish.cn
  8. #for host in `cat /esdb/scripts/es_host.sh | awk '{print $1}'`
  9. do
  10. echo "--------------------------------"
  11. echo $host "node es start..."
  12. ssh -n -T $host "/esdb/esapp/esapp9200/bin/elasticsearch -d >/dev/null 2>&1 &"
  13. echo $host "node es start cancel..."
  14. done
  15. echo "es cluster all node start OK!"
  16. ----
  1. vi /esdb/scripts/es_kill.sh
  2. #!/bin/bash
  3. ----
  4. ps -ef|grep java|grep esapp|awk '{print $2}'|xargs kill -9
  5. vi /esdb/scripts/es_stop.sh
  6. #!/bin/bash
  7. echo "es cluster all node stop..."
  8. for host in fgesdb01 fgesdb02 fgesdb03
  9. #for host in `cat /esdb/scripts/host.sh | awk '{print $1}'`
  10. do
  11. echo "--------------------------------"
  12. echo $host "node es stop..."
  13. ssh -n -T $host "/esdb/scripts/es_kill.sh"
  14. echo $host "node es stop cancel..."
  15. done
  16. echo "es cluster all node stop OK!"
  18. ----
  1. chmod 755 /esdb/scripts/es_*.sh
  2. scp -r /esdb/scripts rhel02.flyfish.cn:/esdb/
  3. scp -r /esdb/scripts rhel03.flyfish.cn:/esdb/
  4. ssh rhel02.flyfish.cn "chmod -R 755 /esdb/scripts"
  5. ssh rhel03.flyfish.cn "chmod -R 755 /esdb/scripts"
  6. /esdb/scripts/es_stop.sh
  7. /esdb/scripts/es_start.sh
  1. 设置集群分片与副本：
  2. curl -XPUT 192.168.100.41:9200/_template/all_index_shards -H 'Content-Type:application/json' -d '{
  3. "template": "*",
  4. "settings": {
  5. "number_of_shards": 6,
  6. "number_of_replicas": "1"
  7. }
  8. }'

image_1et6hd3t31v3v1beld7t8k9mri1g.png-69.5kB

  1. 设置路由分片防止堆积在一台上面：
  2. curl -XPUT '192.168.100.41:9200/_cluster/settings' -H 'Content-Type: application/json' -d '{
  3. "transient": {
  4. "cluster.routing.allocation.same_shard.host": true
  5. }
  6. }'

image_1et6hen01154s1ik6674gdfo3p1t.png-69.8kB

  1. 插入数据：
  2. curl -XPOST http://192.168.100.41:9200/itpuxdb/_doc/?pretty -H 'Content-Type:application/json' -d '
  3. {
  4. "id" : "1",
  5. "name" : "itpux001",
  6. "city" : "上海",
  7. "course" : "bigdata",
  8. "teacher" : "flyfish",
  9. "pxdate": "20200829"
  10. }'

image_1et6hhhojrb1pdvebg13ak22j2a.png-109.7kB

  1. 查看数据：
  3. curl -XPOST 'http://192.168.100.41:9200/itpuxdb/_search?pretty' -H 'Content-Type:application/json'

image_1et6hjglb17ldo11f0l1li61c1v2n.png-119.4kB

  1. curl http://192.168.100.41:9200/_cat/shards?v

image_1et6hqmho9eu1ujg137n19rlnrr34.png-126.6kB

添加新批注
在作者公开此批注前，只有你和作者可见。
回复批注