[关闭]
@saltyang 2018-06-19T11:38:53.000000Z 字数 9325 阅读 2503

NFS HA： NFS + Keepalived + Rsync + Inotify

NFS, Keepalived, Rsync, Inotify

存储服务器采用NFS的方式向接收服务器提供存储服务，存储服务器本身采用rsync + inotify实现文件的实时同步，其中Keepalived保证NFS服务的高可用，实现故障转移和自动切换。

本系统采用Keepalived + Keepalived + Rsync + Inotify来保证NFS服务的高可用

  1. Linux: Centos 6.9
  2. Rsync Version: 3.0.6
  3. master_nfs ip: 192.168.1.176
  4. salve_nfs ip: 192.168.1.177
  5. VIP: 192.168.1.178

ENV Prepare

  1. yum install rpcbind nfs-utils;
  2. yum install keepalived rsync inotify-tools; 
  3. chkconfig keepalived on; chkconfig nfs on;

Rsync Server Config

Rsync Server is the backup machine, the first is salve_nfs.

  1. ##rsyncd.conf start##
  2. #工作中指定用户(需要指定用户)
  3. uid = root
  4. gid = root
  5. use chroot = no
  6. #有多少个客户端同时传文件
  7. max connections = 36000
  8. #超时时间
  9. timeout = 300
  10. #进程号文件
  11. pid file = /var/run/rsyncd.pid
  12. #日志文件
  13. lock file = /var/run/rsync.lock
  14. #日志文件
  15. log file = /var/log/rsyncd.log
  16. #模块开始
  17. #这个模块对应的是推送目录
  18. #模块名称随便起
  19. [backup]
  20. #需要同步的目录
  21. path = /data/
  22. #表示出现错误忽略错误
  23. ignore errors = yes
  24. #表示网络权限可写(本地控制真正可写)
  25. read only = false
  26. #这里设置IP或让不让同步
  27. list = false
  28. #指定允许的网段
  29. hosts allow = 192.168.1.0/24
  30. #拒绝链接的地址，一下表示没有拒绝的链接。
  31. hosts deny = 0.0.0.0/32
  32. #不要动的东西(默认情况)
  33. #虚拟用户
  34. auth users = rsync_backup
  35. #虚拟用户的密码文件
  36. secrets file = /etc/rsync.password
  1. echo "rsync_backup:ems" > /etc/rsync.password
  2. chmod 600 /etc/rsync.password
  1. rsync --daemon
  2. ps -ef | grep "rsync" | grep -v "grep" # check daemon success or not
  3. echo "/usr/bin/rsync --daemon" >>/etc/rc.local # add daemon to machine start

Rsync Client Config

Rsync Client is the machine which satrted nfs service, the first is master_nfs.

  1. #!/bin/bash
  3. # rsync server address
  4. RSYNC_SERVER=192.168.1.177
  5. # rsync client monitor doc
  6. SYC_DIR=/data/
  7. # rsync server service module name
  8. DST_DIR=backup
  9. # rsync server service auth user
  10. USER=rsync_backup
  11. # rsync service password file
  12. RSYNC_PASSFILE=/etc/rsync_client.password
  13. INOTIFYWAIT=/usr/bin/inotifywait
  14. LOG_FILE=/var/log/rsyncd.log
  16. if [ ! -e "$SYC_DIR" ] || [ ! -e "${RSYNC_PASSFILE}" ] || [ ! -e "${INOTIFYWAIT}" ] \
  17. || [ ! -e "/usr/bin/rsync" ];
  18. then
  19.     echo "Check File and Folder" > $LOG_FILE
  20.     exit 9
  21. fi
  22. ${INOTIFYWAIT} -mrq --timefmt '%d/%m/%y %H:%M' --format '%T %w%f' -e close_write,delete,create,attrib $SYC_DIR \
  23. | while read file
  24. do
  25.     cd $SYC_DIR
  26.     rsync -aruz -R --delete ./  --timeout=100 $USER@$RSYNC_SERVER::$DST_DIR --password-file=${RSYNC_PASSFILE} >/dev/null 2>&1
  27. done
  28. exit 0
  1. echo ems > /etc/rsync_client.password

Keepalived Config

  1. ! Configuration File for keepalived
  3. global_defs {
  4.    notification_email {
  5.      salt_yang@puyacn.com
  6.    }
  7.    notification_email_from service@webackup.cn
  8.    smtp_server smtp.mxichina.cn
  9.    smtp_connect_timeout 30
  10.    router_id NFS_MASTER
  11. }
  12. vrrp_script chk_nfs {
  13.     script "/etc/keepalived/checknfs.sh check"
  14.     interval 30
  15. }
  17. vrrp_instance NFS_MASTER {
  18.     state MASTER
  19.     interface eth5
  20.     virtual_router_id 54
  21.     priority 100
  22.     advert_int 1
  23.     authentication {
  24.         auth_type PASS
  25.         auth_pass 1111
  26.     }
  27.     virtual_ipaddress {
  28.         192.168.1.178
  29.     }
  30.     track_script {
  31.         chk_nfs
  32.     }
  33.     debug
  34.     nopreempt
  35.     notify_master "/etc/keepalived/checknfs.sh master"
  36.     notify_backup "/etc/keepalived/checknfs.sh backup"
  37.     notify_fault  "/etc/keepalived/checknfs.sh fault"
  38.     notify_stop  "/etc/keepalived/checknfs.sh fault"
  39. }
  1. ! Configuration File for keepalived
  3. global_defs {
  4.    notification_email {
  5.      salt_yang@puyacn.com
  6.    }
  7.    notification_email_from service@webackup.cn
  8.    smtp_server smtp.mxichina.cn
  9.    smtp_connect_timeout 30
  10.    router_id NFS_BACCKUP
  11. }
  12. vrrp_script chk_nfs {
  13.     script "/etc/keepalived/checknfs2.sh check"
  14.     interval 30
  15. }
  17. vrrp_instance NFS_BACKUP {
  18.     state BACKUP
  19.     interface eth5
  20.     virtual_router_id 54
  21.     priority 80
  22.     advert_int 1
  23.     authentication {
  24.         auth_type PASS
  25.         auth_pass 1111
  26.     }
  27.     virtual_ipaddress {
  28.         192.168.1.178
  29.     }
  30.     track_script {
  31.         chk_nfs
  32.     }
  33.     debug
  34.     nopreempt
  35.     notify_master "/etc/keepalived/checknfs.sh master"                          
  36.     notify_backup "/etc/keepalived/checknfs.sh backup"                          
  37.     notify_fault  "/etc/keepalived/checknfs.sh fault"                           
  38.     notify_stop  "/etc/keepalived/checknfs.sh fault"
  39. }
  1. #!/bin/dash
  2. # Script to handle NFS from keepalived.
  3. #
  4. # Usage: checknfs.sh action
  5. #
  6. # Note: you can use $MAINTENANCE (/etc/keepalived/maintenance) to disable NFS checks
  7. # in case of short NFS maintenance
  8. #
  10. # Usage func :
  11. [ "$1" = "--help" ] && { sed -n -e '/^# Usage:/,/^$/ s/^# \?//p' < $0; exit; }
  14. #
  15. # CONFIG
  16. #
  17. # Keepalived
  18. KEEPALIVEDPID=/var/run/keepalived.pid
  19. KEEPALIVED=/etc/init.d/keepalived
  20. # NFSD
  21. NFS_FLAG="nfsd"
  22. # rync + inotify
  23. RSYNC=/usr/bin/rsync
  24. RSYNCDPID=/var/run/rsyncd.pid
  25. INOTIFY_SCRIPT=/usr/local/inotify.sh
  26. INOTIFY_FLAG="inotify"
  27. RSYNC_FLAG="rsync"
  28. # local mount point
  29. MOUNTPOINT="/data"
  30. # warmup delay
  31. MAXWAIT=240
  32. VIP=192.168.1.178
  33. # how to handle potential split-brain
  34. # 0: manual
  35. # 1: invalidate local data
  36. # 2: invalidate remote data
  37. SPLIT_BRAIN_METHOD=1
  38. # maintenance flag: used to do maintenance on NFS without switch between nodes
  39. MAINTENANCE="/etc/keepalived/maintenance"
  41. #
  42. # CONFIG LOGGER
  43. #
  44. # tail -f /var/log/syslog | grep Keep
  45. LOG="logger -t KeepNFS[$$] -p syslog" # do not use -i
  46. LOGINFO="$LOG.info"
  47. LOGWARN="$LOG.warn"
  48. LOGERR="$LOG.err"
  50. check() {
  51.     if ip addr | grep "${VIP}"
  52.     then
  53.         # if nfs server is down, need restart it.
  54.         # Keepalive need switch after restart nfs failed
  55.         $LOGWARN "This is Master."
  56.         if check_nfs
  57.         then
  58.             $LOGWARN "NFS is OK."
  59. 	    else
  60. 	        start_nfs
  61. 		    $LOGWARN "Restart NFS service"
  62.             if check_nfs
  63.             then
  64.                 $LOGWARN "After restart NFS service, but it's also down. Switch keepalive status"
  65. 		        $KEEPALIVED restart
  66.             fi
  67.         fi
  69.         # if inotify server is down, need restart it.
  70.         # nfs is also OK, keepalived needn't  switch after restart nfs failed
  71.         if check_inotify
  72.         then
  73.             $LOGWARN "Inotify is OK."
  74. 	    else
  75. 	        $LOGWARN "Start Inotify script......"
  76.             /bin/bash $INOTIFY_SCRIPT  &
  77. 	        if check_inotify
  78. 		    then
  79. 		        $LOGWARN "After restart Inotify script, but it's also down and need man repair"
  80.             fi
  81.         fi
  83.     else
  84.         $LOGWARN "This is Salve."
  85. 	    if check_rsycn_daemon
  86. 	    then
  87. 	        $LOGWARN "Rsync daemon is OK."
  88.         else
  89.             $LOGWARN "Start Rsync daemon....."
  90. 	        start_rsync_daemon
  91. 	        if check_rsycn_daemon
  92. 		    then
  93. 		        $LOGWARN "After restart Rsync service, but it's also down and need man repair"
  94.             fi
  95.         fi
  96.     fi
  97.     return $?
  98. }
  101. set_fault() {
  102.     set_backup
  103. }
  106. set_backup() {
  107.     if check_nfs
  108.     then
  109.         $LOGWARN "NFS Service is UP, now kill it"
  110.         kill_nfs
  111.     fi
  112.     set_rsync_server
  113. }
  116. set_rsync_server() {
  117.     # start rsync daemon
  118.     if check_rsycn_daemon
  119.     then
  120.        $LOGWARN "[Rsync daemon Service is Up.]"
  121.     else
  122.         $LOGWARN "[Begin to start daemon Service.]"
  123.         start_rsync_daemon
  124.         for i in $( seq 1 $MAXWAIT )
  125.         do
  126.             sleep 1
  127.             if check_rsycn_daemon
  128.             then
  129.                 break
  130.             fi
  131.         done
  132.     fi
  134.     if ! check_rsycn_daemon
  135.     then
  136.         $LOGWARN "[After 240s, rsync daemon doesn't start, need adminstrator to repair]"
  137.     fi
  140.     # kill inotify service
  141.     if check_inotify
  142.     then
  143.         if kill_inotify
  144.         then
  145.             $LOGWARN "[Kill inotify Service.]"
  146.         else
  147.             $LOGWARN "[Kill inotify Service failed, need adminstrator to repair]"
  148.         fi
  149.     fi
  150. }
  153. # WARNING set_master is called at keepalived start
  154. # So if already in "good" state we must do nothing :)
  155. set_master() {
  156.     # Starting NFS
  157.     if [ $( pidof nfsd | wc -w ) -gt 0 ]
  158. 	then
  159. 		$LOGWARN "NFS already started ? What did I have to do ?"
  160.     else
  161.         $LOGWARN "Starting NFS ..."
  162. 		/sbin/service nfs restart
  163. 		for i in $( seq 1 $MAXWAIT )
  164.         do
  165.             sleep 1
  166.             if check_nfs
  167.             then
  168.                 break
  169.             fi
  170.         done
  171.     fi
  173.     if check_nfs
  174.     then
  175.         $LOGWARN "NFS service has started"
  176. 	else
  177. 		$LOGWARN "NFS Service is broken and need a manual repair."
  178.     fi
  180.     # check inotify whether is not active. if not, start it.
  181.     if check_inotify
  182.     then
  183.         $LOGWARN "Inotify has started"
  184. 	else
  185. 	    $LOGWARN "Started Inotify"
  186.         start_inotify
  187.     fi
  189.     # check rsync daemon service. if it's active, kill it.
  190.     if check_rsync_daemon
  191.     then
  192.         if kill_rsync_daemon
  193.         then
  194.             $LOGWARN "[Kill Rsync Service successed!]"
  195.         else
  196.             $LOGWARN "[Kill Rsync Service failed, need adminstrator to repair]"
  197.         fi
  198.     fi
  200. }
  203. # Check that NFS is responding
  204. # Return:
  205. # 0 if nfs is OK (or in maintenance mode)
  206. # 1 if nfs is down
  207. check_nfs() {
  208.     if [ -e $MAINTENANCE ]
  209.     then
  210.         return 0
  211.     fi
  213.     flag_exists=$(ps -ef | grep "${NFS_FLAG}" | grep -v grep | wc -l)
  214.     if [ "${flag_exists}" -eq 0 ]
  215. 	then
  216. 	    $LOGWARN "[NFS service is unavailable.]"
  217.         return 1
  218.     else
  219.         return 0
  221.     fi
  222. }
  225. kill_nfs() {
  226.     /sbin/service nfs stop
  227.     return $?
  228. }
  230. start_nfs() {
  231.     if [ $( pidof rpcbind | wc -w ) -eq 0 ]
  232.     then
  233.         /sbin/service rpcbind start
  234.     fi
  235.     /sbin/service nfs start
  236.     return $?
  237. }
  239. # Check that inotify service is responding
  240. # Return:
  241. # 0 if inotify is OK (or in maintenance mode)
  242. # 1 if inotify is down
  243. check_inotify() {
  244.     flag_exists=$(ps -ef | grep "${INOTIFY_FLAG}" | grep -v grep | wc -l)
  245.     if [ "${flag_exists}" -eq 0 ]
  246. 	then
  247. 	    $LOGWARN "[Inotify daemon service is unavailable.]"
  248.         return 1
  249.     else
  250.         return 0
  251.     fi
  252. }
  254. start_inotify() {
  255.     /bin/bash $INOTIFY_SCRIPT &
  256.     return $?
  257. }
  259. kill_inotify() {
  260.     ps -ef | grep "${INOTIFY_FLAG}" | grep -v grep | awk '{print $2 }' | xargs kill -9
  261.     return $?
  262. }
  265. # Check that rsync daemon is responding
  266. # Return:
  267. # 0 if rsync daemon is OK (or in maintenance mode)
  268. # 1 if rsync daemon is down
  269. check_rsycn_daemon() {
  270.     flag_exists=$(ps -ef | grep "${RSYNC_FLAG}" | grep -v grep | wc -l)
  271.     if [ "${flag_exists}" -eq 0 ]
  272. 	then
  273. 	    $LOGWARN "[Rsync daemon service is unavailable.]"
  274.         return 1
  275.     else
  276.         return 0
  277.     fi
  278. }
  280. # Start Rsync daemon
  281. start_rsync_daemon() {
  282.     if [ -e $RSYNCDPID ]
  283. 	then
  284. 	    rm -rf $RSYNCDPID
  285.     fi
  286.     $RSYNC --daemon
  287. 	return $?
  288. }
  290. # Kill Rsync daemon
  291. kill_rsync_daemon() {
  292.     ps -ef | grep "${RSYNC_FLAG}" | grep -v grep | awk '{print $2 }' | xargs kill -9
  293.     return $?
  294. }
  297. case "$1" in
  298. 	check)
  299. 		check
  300. 		exit $?
  301.     ;;
  302.     backup)
  303.         $LOGWARN "=> set to backup state <="
  304. 		set_backup
  305. 		exit $?
  306.     ;;
  307.     fault)
  308.         $LOGWARN "=> set to fault state <="
  309. 		set_fault
  310. 		exit $?
  311.     ;;
  312.     master)
  313.         $LOGWARN "=> set to master state <="
  314. 		set_master
  315. 		exit $?
  316.     ;;
  317. esac

Creat share dir

Note: Use LVM to aviod nfs state file handle when switch nfs server

  1. # check disk partition
  2. fdisk -l 
  3. # Creat partition and enter partition management
  4. fdisk /dev/sda 
  5. Enter these choices: n -> p(partition type) -> +500M(size) -> t(change partition type) -> 4(partition num) -> 8e(LVM type) -> p(check partition) -> w (write partition) 
  6. # make partition valid and not need restart
  7. partprobe 
  8. # Create PV and check
  9. pvcreate /dev/sda4
  10. pvdisplay
  11. # Create vg and add pv into 
  12. vgcreate VolGroup /dev/sda4 
  13. # If VolGroup is existed, not need to create vg
  14. extendvg VolGroup /dev/sda4 
  15. # Creat a LV and named lvData and check
  16. lvcreate -L 100M -n lvData VolGroup 
  17. lvdisplay 
  18. # Format and mount
  19. mkfs -t ext4 /dev/VolGroup/lvData 
  20. mount /dev/VolGroup/lvData /data
  21. # add fstab in /etc/fstab
  22. /dev/VolGroup/lvData    /data                   ext4    defaults        1 2

Extend share dir 

  1. # Add a disk 
  2. # Creat disk partition
  3. fdisk /dev/sdb 
  4. Enter these choices: n -> p(partition type) -> +20G(size) -> t(change partition type) -> 1(partition num) -> p(check partition) -> w (write partition) 
  5. # make partition valid and not need restart
  6. partprobe 
  7. # Create PV and check
  8. pvcreate /dev/sdb1
  9. mkfs –t ext4 /dev/sdb1
  10. # Extend vg 
  11. vgextend VolGroup /dev/sdb1
  12. # Extend lv
  13. lvextend -L 20G /dev/VolGroup/lvData
  14. resize2fs /dev/VolGroup/lvData
添加新批注
在作者公开此批注前，只有你和作者可见。
回复批注