[关闭]
@zhangyy 2021-10-22T13:36:54.000000Z 字数 12942 阅读 138

ElasticSearch亿万级别的集群配置

ElasticSearch系列

一：系统环境初始化

  1. 系统：rhel8.3x64-1
  2. cat /etc/hosts 
  3. ----
  4. 192.168.100.41  rhel01.flyfish.cn  协调节点
  5. 192.168.100.42  rhel02.flyfish.cn  协调节点
  6. 192.168.100.43  rhel03.flyfish.cn  协调节点
  7. 192.168.100.44  rhel04.flyfish.cn  主节点
  8. 192.168.100.45  rhel05.flyfish.cn  主机点
  9. 192.168.100.46  rhel06.flyfish.cn  主节点
  10. 192.168.100.47  rhel07.flyfish.cn  数据节点
  11. 192.168.100.48  rhel08.flyfish.cn  数据节点
  12. 192.168.100.49  rhel09.flyfish.cn  数据节点
  13. 192.168.100.50  rhel10.flyfish.cn  扩容数据节点
  14. 192.168.100.51  rhel11.flyfish.cn  扩容数据节点
  15. ----
  16. 先部署前面九台机器，后面 两台作为 扩容节点
  1. 全部系统配置：
  3. 系统语言：
  5. echo "export LANG=en_US.UTF8" >> ~/.bash_profile
  6. cat ~/.bash_profile
  8. ---
  9. 分区：
  10. pvcreate /dev/sdb
  11. vgcreate esdatavg /dev/sdb
  12. lvcreate -n esdatalv -L 100000M esdatavg
  13. mkfs.xfs /dev/esdatavg/esdatalv
  14. ---
  15. vi /etc/fstab
  16. ---
  17. /dev/esdatavg/esdatalv /esdb xfs defaults 0 0
  18. ---
  19. mkdir /esdb
  20. mount /esdb
  1. mkdir -p /esdb/soft
  3. groupadd -g 60001 esadmin
  4. useradd -u 61001 -g esadmin esadmin
  5. chown -R esadmin:esadmin /esdb
  6. chmod -R 775 /esdb
  7. echo "esadmin" | passwd --stdin esadmin
  8. systemctl set-default multi-user.target
  10. chown esadmin:esadmin -R /esdb
  12. cat >> /etc/security/limits.conf << EOF
  14. root soft nofile 1048576
  15. root hard nofile 1048576
  16. esadmin soft nproc 1048576
  17. esadmin hard nproc 1048576
  18. esadmin soft nofile 1048576
  19. esadmin hard nofile 1048576
  20. esadmin soft stack 10240
  21. esadmin hard stack 32768
  22. esadmin hard memlock unlimited
  23. esadmin soft memlock unlimited
  24. EOF
  25. ---
  26. ---
  28. cat >> /etc/sysctl.conf << EOF
  29. fs.aio-max-nr = 1048576
  30. fs.file-max = 6815744
  31. net.ipv4.ip_local_port_range = 1024 65535
  32. net.ipv4.tcp_mem = 786432 2097152 3145728
  33. net.ipv4.tcp_rmem = 4096 4096 16777216
  34. net.ipv4.tcp_wmem = 4096 4096 16777216
  35. net.ipv4.tcp_tw_reuse = 1
  36. net.ipv4.tcp_tw_recycle = 1
  37. vm.swappiness=1
  38. vm.min_free_kbytes=204800
  39. vm.max_map_count=2048000
  40. kernel.pid_max=819200
  41. vm.zone_reclaim_mode=0
  42. #vm.nr_hugepages = 0
  43. EOF
  45. sysctl -p 
  1. ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
  2. hwclock
  4. echo "SELINUX=disabled" > /etc/selinux/config
  5. echo "#SELINUXTYPE=targeted " >> /etc/selinux/config
  6. cat /etc/selinux/config
  7. setenforce 0
  8. systemctl stop firewalld.service
  9. systemctl disable firewalld.service
  11. cat >> /home/esadmin/.vimrc << EOF
  12. map <F10>:set paste<CR>
  13. map <F11>:set nopaste<CR>
  14. EOF
  16. shutdown -r now
  1. 做9台机器的esadmin 无密码登录
  2. 做esadmin 的无密码登录
  4. /root/fgssh -user esadmin -hosts "rhel01.flyfish.cn rhel02.flyfish.cn rhel03.flyfish.cn rhel04.flyfish.cn rhel05.flyfish.cn rhel06.flyfish.cn rhel07.flyfish.cn rhel08.flyfish.cn rhel09.flyfish.cn" -advanced -exverify -confirm
  6. chmod 600 /home/esadmin/.ssh/config

二：ES的配置

2.1 配置jdk

  1. su - esadmin
  2. cd /esdb/
  3. tar zxvf /esdb/soft/jdk-14.0.2_linux-x64_bin.tar.gz
  4. scp -r /esdb/jdk-14.0.2 esadmin@rhel02.flyfish.cn:/esdb/
  5. scp -r /esdb/jdk-14.0.2 esadmin@rhel03.flyfish.cn:/esdb/
  6. scp -r /esdb/jdk-14.0.2 esadmin@rhel04.flyfish.cn:/esdb/
  7. scp -r /esdb/jdk-14.0.2 esadmin@rhel05.flyfish.cn:/esdb/
  8. scp -r /esdb/jdk-14.0.2 esadmin@rhel06.flyfish.cn:/esdb/
  9. scp -r /esdb/jdk-14.0.2 esadmin@rhel07.flyfish.cn:/esdb/
  10. scp -r /esdb/jdk-14.0.2 esadmin@rhel08.flyfish.cn:/esdb/
  11. scp -r /esdb/jdk-14.0.2 esadmin@rhel09.flyfish.cn:/esdb/
  1. 全部主机：
  2. su - esadmin
  3. cat >> ~/.bash_profile << EOF
  4. export JAVA_HOME=/esdb/jdk-14.0.2
  5. export PATH=/esdb/jdk-14.0.2/bin:$PATH
  6. export LANG=en_US.UTF8
  7. EOF
  8. source ~/.bash_profile
  9. java -version

image_1euuo5hugu5kfjbd9f1hau14859.png-77.3kB

image_1euuo5v1m6lo1r4q4vjhc2rqpm.png-97.7kB

2.2 设置ES 的安装目录

  1. 全部主机：
  2. mkdir -p /esdb/soft/
  3. mkdir -p /esdb/esapp/esapp9200
  4. mkdir -p /esdb/esdata/esdata9200
  5. mkdir -p /esdb/eslog/eslog9200
  7. chown esadmin:esadmin -R /esdb/

image_1euuo6p2510f1si0164t1h0q1jsm13.png-51.4kB

image_1euuo7ad71o38fcvcge1bb5sso1g.png-75.7kB

  1. rhel01.flyfish.cn: 操作
  3. cd /esdb/soft
  4. tar zxvf /esdb/soft/elasticsearch-7.9.0-linux-x86_64.tar.gz
  5. cp -r /esdb/elasticsearch-7.9.0/* /esdb/esapp/esapp9200

image_1euuod5nf8md4fch7qim81p791t.png-67.3kB

  1. vi /esdb/esapp/esapp9200/bin/elasticsearch-env
  2. ----
  3. export JAVA_HOME=/esdb/jdk-14.0.2
  4. export PATH=$JAVA_HOME/bin:$PATH
  5. ----
  1. vi /esdb/esapp/esapp9200/config/elasticsearch.yml
  2. ----
  3. cluster.name: flyfishescluster1
  4. node.name: 192.168.100.41
  5. network.host: 192.168.100.41
  6. http.port: 9200
  7. transport.tcp.port: 9300
  8. path.data: /esdb/esdata/esdata9200
  9. path.logs: /esdb/eslog/eslog9200
  10. node.master: false                   
  11. node.data: false
  12. cluster.remote.connect: false
  13. http.cors.enabled: true
  14. http.cors.allow-origin: "*"
  15. bootstrap.memory_lock: true
  16. bootstrap.system_call_filter: false
  17. cluster.initial_master_nodes:
  18. ["192.168.100.44:9300","192.168.100.45:9300","192.168.100.46:9300"]
  19. discovery.seed_hosts: ["192.168.100.44:9300", "192.168.100.45:9300", "192.168.100.46:9300"]
  20. discovery.zen.fd.ping_timeout: 60s
  21. discovery.zen.fd.ping_retries: 5
  22. ----
  1. vim /esdb/esapp/esapp9200/config/jvm.options
  2. ----
  3. -Xms1g
  4. -Xmx1g
  5. -XX:+DisableExplicitGC
  6. -XX:HeapDumpPath=/esdb/eslog/eslog9200
  7. -XX:ErrorFile=/esdb/eslog/eslog9200/hs_err_pid%p.log
  8. 9-:-Xlog:gc*,gc+age=trace,safepoint:file=/esdb/eslog/eslog9200/gc.log:utctime,pid,ta
  9. gs:filecount=32,filesize=64m
  10. ----
  1. 同步所有主机：(所有节点都执行)
  3.        scp -r rhel01.flyfish.cn:/esdb/esapp/esapp9200 /esdb/esapp/

2.3 配置 协调节点

  1. rhel0(1-3).flyfish.cn  前三台做协调节点：
  3. rhel01.flyfish.cn:
  4.     vi /esdb/esapp/esapp9200/config/elasticsearch.yml
  5. ----
  6. cluster.name: flyfishescluster1
  7. node.name: 192.168.100.41
  8. network.host: 192.168.100.41
  9. http.port: 9200
  10. transport.tcp.port: 9300
  11. path.data: /esdb/esdata/esdata9200
  12. path.logs: /esdb/eslog/eslog9200
  13. node.master: false
  14. node.data: false
  15. cluster.remote.connect: false
  16. http.cors.enabled: true
  17. http.cors.allow-origin: "*"
  18. bootstrap.memory_lock: true
  19. bootstrap.system_call_filter: false
  20. cluster.initial_master_nodes:
  21. ["192.168.100.44:9300","192.168.100.45:9300","192.168.100.46:9300"]
  22. discovery.seed_hosts: ["192.168.100.44:9300", "192.168.100.45:9300", "192.168.100.46:9300"]
  23. discovery.zen.fd.ping_timeout: 60s
  24. discovery.zen.fd.ping_retries: 5
  25. -----
  27. rhel02.flyfish.cn:
  28. vi /esdb/esapp/esapp9200/config/elasticsearch.yml
  29. ----
  30. cluster.name: flyfishescluster1
  31. node.name: 192.168.100.42
  32. network.host: 192.168.100.42
  33. http.port: 9200
  34. transport.tcp.port: 9300
  35. path.data: /esdb/esdata/esdata9200
  36. path.logs: /esdb/eslog/eslog9200
  37. node.master: false
  38. node.data: false
  39. cluster.remote.connect: false
  40. http.cors.enabled: true
  41. http.cors.allow-origin: "*"
  42. bootstrap.memory_lock: true
  43. bootstrap.system_call_filter: false
  44. cluster.initial_master_nodes:
  45. ["192.168.100.44:9300","192.168.100.45:9300","192.168.100.46:9300"]
  46. discovery.seed_hosts: ["192.168.100.44:9300", "192.168.100.45:9300", "192.168.100.46:9300"]
  47. discovery.zen.fd.ping_timeout: 60s
  48. discovery.zen.fd.ping_retries: 5
  49. ----
  51. rhel03.flyfish.cn:
  52. vi /esdb/esapp/esapp9200/config/elasticsearch.yml
  54. ---
  55. cluster.name: flyfishescluster1
  56. node.name: 192.168.100.43
  57. network.host: 192.168.100.43
  58. http.port: 9200
  59. transport.tcp.port: 9300
  60. path.data: /esdb/esdata/esdata9200
  61. path.logs: /esdb/eslog/eslog9200
  62. node.master: false
  63. node.data: false
  64. cluster.remote.connect: false
  65. http.cors.enabled: true
  66. http.cors.allow-origin: "*"
  67. bootstrap.memory_lock: true
  68. bootstrap.system_call_filter: false
  69. cluster.initial_master_nodes:
  70. ["192.168.100.44:9300","192.168.100.45:9300","192.168.100.46:9300"]
  71. discovery.seed_hosts: ["192.168.100.44:9300", "192.168.100.45:9300", "192.168.100.46:9300"]
  72. discovery.zen.fd.ping_timeout: 60s
  73. discovery.zen.fd.ping_retries: 5
  74. ---

2.4 配置主节点

  1. rhel0(4-6).flyfish.cn做 主节点：
  2. rhel04.flyfish.cn:
  4. vi /esdb/esapp/esapp9200/config/elasticsearch.yml
  5. ----
  6. cluster.name: flyfishescluster1
  7. node.name: 192.168.100.44
  8. network.host: 192.168.100.44
  9. http.port: 9200
  10. transport.tcp.port: 9300
  11. path.data: /esdb/esdata/esdata9200
  12. path.logs: /esdb/eslog/eslog9200
  13. node.master: true
  14. node.data: false
  15. cluster.remote.connect: false
  16. http.cors.enabled: true
  17. http.cors.allow-origin: "*"
  18. bootstrap.memory_lock: true
  19. bootstrap.system_call_filter: false
  20. cluster.initial_master_nodes:
  21. ["192.168.100.44:9300","192.168.100.45:9300","192.168.100.46:9300"]
  22. discovery.seed_hosts: ["192.168.100.44:9300", "192.168.100.45:9300", "192.168.100.46:9300"]
  23. discovery.zen.fd.ping_timeout: 60s
  24. discovery.zen.fd.ping_retries: 5
  25. -----
  27. rhel05.flyfish.cn:
  28. vi /esdb/esapp/esapp9200/config/elasticsearch.yml
  29. ----
  30. cluster.name: flyfishescluster1
  31. node.name: 192.168.100.45
  32. network.host: 192.168.100.45
  33. http.port: 9200
  34. transport.tcp.port: 9300
  35. path.data: /esdb/esdata/esdata9200
  36. path.logs: /esdb/eslog/eslog9200
  37. node.master: true
  38. node.data: false
  39. cluster.remote.connect: false
  40. http.cors.enabled: true
  41. http.cors.allow-origin: "*"
  42. bootstrap.memory_lock: true
  43. bootstrap.system_call_filter: false
  44. cluster.initial_master_nodes:
  45. ["192.168.100.44:9300","192.168.100.45:9300","192.168.100.46:9300"]
  46. discovery.seed_hosts: ["192.168.100.44:9300", "192.168.100.45:9300", "192.168.100.46:9300"]
  47. discovery.zen.fd.ping_timeout: 60s
  48. discovery.zen.fd.ping_retries: 5
  49. ----
  51. rhel06.flyfish.cn:
  53. vi /esdb/esapp/esapp9200/config/elasticsearch.yml
  54. ---
  55. cluster.name: flyfishescluster1
  56. node.name: 192.168.100.46
  57. network.host: 192.168.100.46
  58. http.port: 9200
  59. transport.tcp.port: 9300
  60. path.data: /esdb/esdata/esdata9200
  61. path.logs: /esdb/eslog/eslog9200
  62. node.master: true
  63. node.data: false
  64. cluster.remote.connect: false
  65. http.cors.enabled: true
  66. http.cors.allow-origin: "*"
  67. bootstrap.memory_lock: true
  68. bootstrap.system_call_filter: false
  69. cluster.initial_master_nodes:
  70. ["192.168.100.44:9300","192.168.100.45:9300","192.168.100.46:9300"]
  71. discovery.seed_hosts: ["192.168.100.44:9300", "192.168.100.45:9300", "192.168.100.46:9300"]
  72. discovery.zen.fd.ping_timeout: 60s
  73. discovery.zen.fd.ping_retries: 5
  74. ----

2.5 配置数据节点

  1. rhel0(7-9).flyfish.cn 做数据节点：
  2. rhel07.flyfish.cn
  3. vi /esdb/esapp/esapp9200/config/elasticsearch.yml
  4. ---
  5. cluster.name: flyfishescluster1
  6. node.name: 192.168.100.47
  7. network.host: 192.168.100.47
  8. http.port: 9200
  9. transport.tcp.port: 9300
  10. path.data: /esdb/esdata/esdata9200
  11. path.logs: /esdb/eslog/eslog9200
  12. node.master: false
  13. node.data: true
  14. cluster.remote.connect: false
  15. http.cors.enabled: true
  16. http.cors.allow-origin: "*"
  17. bootstrap.memory_lock: true
  18. bootstrap.system_call_filter: false
  19. cluster.initial_master_nodes:
  20. ["192.168.100.44:9300","192.168.100.45:9300","192.168.100.46:9300"]
  21. discovery.seed_hosts: ["192.168.100.44:9300", "192.168.100.45:9300", "192.168.100.46:9300"]
  22. discovery.zen.fd.ping_timeout: 60s
  23. discovery.zen.fd.ping_retries: 5
  24. ---
  26. rhel08.flyfish.cn:
  27. vi /esdb/esapp/esapp9200/config/elasticsearch.yml
  28. ----
  29. cluster.name: flyfishescluster1
  30. node.name: 192.168.100.48
  31. network.host: 192.168.100.48
  32. http.port: 9200
  33. transport.tcp.port: 9300
  34. path.data: /esdb/esdata/esdata9200
  35. path.logs: /esdb/eslog/eslog9200
  36. node.master: false
  37. node.data: true
  38. cluster.remote.connect: false
  39. http.cors.enabled: true
  40. http.cors.allow-origin: "*"
  41. bootstrap.memory_lock: true
  42. bootstrap.system_call_filter: false
  43. cluster.initial_master_nodes: ["192.168.100.44:9300","192.168.100.45:9300","192.168.100.46:9300"]
  44. discovery.seed_hosts: ["192.168.100.44:9300", "192.168.100.45:9300", "192.168.100.46:9300"]
  45. discovery.zen.fd.ping_timeout: 60s
  46. discovery.zen.fd.ping_retries: 5
  47. ----
  49. rhel09.flyfish.cn:
  51. vi /esdb/esapp/esapp9200/config/elasticsearch.yml
  52. ----
  53. cluster.name: flyfishescluster1
  54. node.name: 192.168.100.49
  55. network.host: 192.168.100.49
  56. http.port: 9200
  57. transport.tcp.port: 9300
  58. path.data: /esdb/esdata/esdata9200
  59. path.logs: /esdb/eslog/eslog9200
  60. node.master: false
  61. node.data: true
  62. cluster.remote.connect: false
  63. http.cors.enabled: true
  64. http.cors.allow-origin: "*"
  65. bootstrap.memory_lock: true
  66. bootstrap.system_call_filter: false
  67. cluster.initial_master_nodes: ["192.168.100.44:9300","192.168.100.45:9300","192.168.100.46:9300"]
  68. discovery.seed_hosts: ["192.168.100.44:9300", "192.168.100.45:9300", "192.168.100.46:9300"]
  69. discovery.zen.fd.ping_timeout: 60s
  70. discovery.zen.fd.ping_retries: 5
  71. ----

2.6 设置启动停止脚本：

  1. mkdir -p /esdb/scripts
  2. vim /esdb/script/es_start.sh
  3. ----
  4. #!/bin/bash
  5. echo "es cluster all node start..."
  6. for hostm in rhel04.flyfish.cn rhel05.flyfish.cn rhel06.flyfish.cn
  7. #for host in `cat /esdb/scripts/es_host.sh | awk '{print $1}'`
  8. do
  9. echo "--------------------------------"
  10. echo $hostm "node es start..."
  11. ssh -n -T $hostm "/esdb/esapp/esapp9200/bin/elasticsearch -d >/dev/null 2>&1 &"
  12. echo $hostm "node es start cancel..."
  13. done
  14. sleep 120
  15. for hostd in rhel07.flyfish.cn rhel08.flyfish.cn rhel09.flyfish.cn
  16. #for host in `cat /esdb/scripts/es_host.sh | awk '{print $1}'`
  17. do
  18. echo "--------------------------------"
  19. echo $hostd "node es start..."
  20. ssh -n -T $hostd "/esdb/esapp/esapp9200/bin/elasticsearch -d >/dev/null 2>&1 &"
  21. echo $hostd "node es start cancel..."
  22. done
  23. sleep 120
  24. for host in rhel01.flyfish.cn rhel02.flyfish.cn rhel03.flyfish.cn
  25. #for host in `cat /esdb/scripts/es_host.sh | awk '{print $1}'`
  26. do
  27. echo "--------------------------------"
  28. echo $host "node es start..."
  29. ssh -n -T $h0st "/esdb/esapp/esapp9200/bin/elasticsearch -d >/dev/null 2>&1 &"
  30. echo $host "node es start cancel..."
  31. done
  32. echo "es cluster all node start OK!"
  33. ----
  1. vi /esdb/scripts/es_kill.sh
  2. ---
  3. #!/bin/bash
  4. ps -ef|grep java|grep esapp|awk '{print $2}'|xargs kill -9
  5. vi /esdb/scripts/es_stop.sh
  6. #!/bin/bash
  7. echo "es cluster all node stop..."
  8. for host in rhel01.flyfish.cn rhel02.flyfish.cn rhel03.flyfish.cn rhel04.flyfish.cn rhel05.flyfish.cn rhel06.flyfish.cn rhel07.flyfish.cn rhel08.flyfish.cn rhel09.flyfish.cn  
  9. #for host in `cat /esdb/scripts/host.sh | awk '{print $1}'`
  10. do
  11. echo "--------------------------------"
  12. echo $host "node es stop..."
  13. ssh -n -T $host "/esdb/scripts/es_kill.sh"
  14. echo $host "node es stop cancel..."
  15. done
  16. echo "es cluster all node stop OK!"
  17. ---
  19. chmod 755 /esdb/scripts/es_*.sh
  20. /esdb/scripts/es_stop.sh
  21. /esdb/scripts/es_start.sh
  1. curl '192.168.100.44:9200/_cat/nodes?v'

image_1euv29ua01f1rinf1g3qqrn1r0l9.png-100.9kB

2.7 修改es的分片副本

  1. curl -XPUT 192.168.100.41:9200/_template/all_index_shards -H 'Content-Type:application/json' -d '{
  2. "template": "*",
  3. "settings": {
  4. "number_of_shards": 6,
  5. "number_of_replicas": "1"
  6. }
  7. }'

image_1euv377m4uel19pb1vjb235vr59.png-65.4kB

  1. 设置路由分片防止堆积在一台上面：
  3. curl -XPUT '192.168.100.41:9200/_cluster/settings' -H 'Content-Type: application/json' -d '{
  4. "transient": {
  5. "cluster.routing.allocation.same_shard.host": true
  6. }
  7. }'

image_1euv381471mt93kf15v01j5e1tnpm.png-74.7kB

  1. 插入数据：
  2. curl -XPOST http://192.168.100.41:9200/itpuxdb/_doc/?pretty -H 'Content-Type:application/json' -d '
  3. {
  4. "id" : "1",
  5. "name" : "itpux001",
  6. "city" : "上海",
  7. "course" : "bigdata",
  8. "teacher" : "flyfish",
  9. "pxdate": "20200829"
  10. }'

image_1euv38lbcrng19ckkbhq8kd313.png-102.9kB

  1. 查看数据：
  2. curl -XPOST 'http://192.168.100.41:9200/itpuxdb/_search?pretty' -H 'Content-Type:application/json'

image_1euv3f03k1vhtout1oqmjjd1fr11g.png-128kB

  1. curl http://192.168.100.41:9200/_cat/shards?v

image_1euv3g0r91dfa1as5gpi1elck331t.png-119.5kB

添加新批注
在作者公开此批注前，只有你和作者可见。
回复批注