三千年读史无外乎功名利禄,九万里悟道终归是诗酒田园。

LVS负载均衡集群- NAT和DR模型总结

一、LVS NAT模型

lvs-nat

搭建上述三台虚拟机,环境如上,其中web server网关都指向director的网卡2 IP

确保两个web server能被Director正常访问到:
[root@director ~]# curl http://192.168.118.7
this is node1
[root@director ~]# curl http://192.168.118.8
this is node2
[root@director ~]# iptables -L -n        
Chain INPUT (policy ACCEPT)
target     prot opt source               destination         

Chain FORWARD (policy ACCEPT)
target     prot opt source               destination         

Chain OUTPUT (policy ACCEPT)
target     prot opt source               destination         
[root@director ~]#
打开路由转发:
[root@director ~]# echo 'net.ipv4.ip_forward = 1' >> /etc/sysctl.conf
[root@director ~]# sysctl -p
net.ipv4.ip_forward = 1
[root@director ~]# cat /proc/sys/net/ipv4/ip_forward
1
[root@director ~]#

配置ipvs规则(先安装ipvsadm):

[root@director ~]# yum install ipvsadm -y
[root@director ~]# ipvsadm -A -t 192.168.0.121 -s rr    ##使用rr轮询算法
[root@director ~]# ipvsadm -L -n            ##查看当前ipvs规则
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.0.121:80 rr
[root@director ~]# ipvsadm -a -t 192.168.0.121:80 -r 192.168.118.7 -m
[root@director ~]# ipvsadm -a -t 192.168.0.121:80 -r 192.168.118.8 -m
[root@director ~]# ipvsadm -L -n
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.0.121:80 rr
  -> 192.168.118.7:80             Masq    1      0          0         
  -> 192.168.118.8:80             Masq    1      0          0         
[root@director ~]#
  • 使用curl访问测试:
[root@director ~]# for((i=1;i<=4;i++));do curl 192.168.0.121;done
this is node2
this is node1
this is node2
this is node1
[root@director ~]# ipvsadm -L -n
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.0.121:80 rr
  -> 192.168.118.7:80             Masq    1      0          5         
  -> 192.168.118.8:80             Masq    1      0          4         
[root@director ~]#

到此lvs nat负载均衡集群配置完成

  • 查看ipvsadm服务配置文件:
[root@director ~]# rpm -ql ipvsadm
/etc/sysconfig/ipvsadm-config
/usr/lib/systemd/system/ipvsadm.service
/usr/sbin/ipvsadm
/usr/sbin/ipvsadm-restore
/usr/sbin/ipvsadm-save
/usr/share/doc/ipvsadm-1.27
/usr/share/doc/ipvsadm-1.27/README
/usr/share/man/man8/ipvsadm-restore.8.gz
/usr/share/man/man8/ipvsadm-save.8.gz
/usr/share/man/man8/ipvsadm.8.gz
[root@director ~]# cat /usr/lib/systemd/system/ipvsadm.service
[Unit]
Description=Initialise the Linux Virtual Server
After=syslog.target network.target

[Service]
Type=oneshot
ExecStart=/bin/bash -c "exec /sbin/ipvsadm-restore < /etc/sysconfig/ipvsadm"
ExecStop=/bin/bash -c "exec /sbin/ipvsadm-save -n > /etc/sysconfig/ipvsadm"
ExecStop=/sbin/ipvsadm -C
RemainAfterExit=yes

[Install]
WantedBy=multi-user.target

[root@director ~]#

由此可以知道每次停止ipvsadm的时候规则全部会被清除或者restore旧配置文件

保存与恢复规则

保存当前ipvs规则(ipvsadm-save-S参数):

[root@director ~]# ipvsadm -S > /etc/sysconfig/ipvsadm    ##注意是ipvsadm不是ipvsadm-config
[root@director ~]# cat /etc/sysconfig/ipvsadm
-A -t director:http -s rr
-a -t director:http -r 192.168.118.7:http -m -w 1
-a -t director:http -r 192.168.118.8:http -m -w 1
[root@director ~]#

恢复规则(ipvsadm-restore-R参数):

[root@director ~]# ipvsadm-restore < /etc/sysconfig/ipvsadm
[root@director ~]# ipvsadm -L -n
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.0.121:80 rr
  -> 192.168.118.7:80             Masq    1      0          0         
  -> 192.168.118.8:80             Masq    1      0          0         
[root@director ~]#

修改及删除ipvs规则(-E选项)

  • 比如修改集群服务调度算法为sh算法:
    [root@director ~]# ipvsadm -E -t 192.168.0.121:80 -s sh
    [root@director ~]# ipvsadm -L -n
    IP Virtual Server version 1.2.1 (size=4096)
    Prot LocalAddress:Port Scheduler Flags
    -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
    TCP  192.168.0.121:80 sh
    -> 192.168.118.7:80             Masq    1      0          0         
    -> 192.168.118.8:80             Masq    1      0          0         
    [root@director ~]#
  • 修改real server(RS)的ipvs规则(-e选项)
首先,两个RS的web监听端口都改为8080
[root@director ~]# curl http://192.168.118.7:8080
this is node1
[root@director ~]# curl http://192.168.118.8:8080
this is node2
[root@director ~]#

修改ipvs规则:

[root@director ~]# ipvsadm -e -t 192.168.0.121:80 -r 192.168.118.7:8080 -m
Memory allocation problem
[root@director ~]#

出现报错,"<font color='red'>Memory allocation problem</font>",意味内存分配问题,程序bug。解决办法,清掉规则,全部重现添加规则,或者直接修改之前ipvsadm-save保存的配置文件:

[root@director ~]# cat /etc/sysconfig/ipvsadm
-A -t director:http -s rr
-a -t director:http -r 192.168.118.7:http -m -w 1
-a -t director:http -r 192.168.118.8:http -m -w 1
[root@director ~]#

修改为:

[root@director ~]# cat /etc/sysconfig/ipvsadm
-A -t director:http -s rr
-a -t director:http -r 192.168.118.7:8080 -m -w 1
-a -t director:http -r 192.168.118.8:8080 -m -w 1
[root@director ~]#

然后清除当前规则,重现加载规则:

[root@director ~]# ipvsadm -C
[root@director ~]# ipvsadm -R < /etc/sysconfig/ipvsadm
[root@director ~]# ipvsadm -L -n
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.0.121:80 rr
  -> 192.168.118.7:8080           Masq    1      0          0         
  -> 192.168.118.8:8080           Masq    1      0          0         
[root@director ~]#

客户端访问:

[root@director ~]# for((i=1;i<=4;i++));do curl 192.168.0.121;done
this is node1
this is node2
this is node1
this is node2
[root@director ~]#

出现上述结果说明规则修改无误,端口映射成功。

  • 删除规则

删除RS,使用-d选项:

[root@director ~]# ipvsadm -d -t 192.168.0.121:80 -r 192.168.118.7:8080
[root@director ~]# ipvsadm -L -n
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.0.121:80 rr
  -> 192.168.118.8:8080           Masq    1      0          0         
[root@director ~]#

删除集群服务,使用-D

[root@director ~]# ipvsadm -D -t 192.168.0.121:80
[root@director ~]# ipvsadm -L -n
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
[root@director ~]#
  • 查看规则
    ipvsadm -L|-l [option]
    -n: numeric,基于数字格式显示地址和端口;
    -c: connetction,显示ipvs连接;
    --status: 统计数据;
    --rate:速率;
    --exact: 精确值

    示例:

    [root@director ~]# ipvsadm -L -n --stats
    IP Virtual Server version 1.2.1 (size=4096)
    Prot LocalAddress:Port               Conns   InPkts  OutPkts  InBytes OutBytes
    -> RemoteAddress:Port
    TCP  192.168.0.121:80                   15       92       63     6182     6712
    -> 192.168.118.7:8080                  7       45       26     2838     2840
    -> 192.168.118.8:8080                  8       47       37     3344     3872
    [root@director ~]#

二、LVS DR模型

lvs-dr
需要修改两个内核参数:

arp_ingnore=1,arp_announce=2

Director配置VIP

  • 使用网卡别名,掩码为32位,并且broadcast只广播给自己:
[root@director ~]# ifconfig ens33:0 192.168.0.122/32 broadcast 192.168.0.122 up
  • 增加一条路由,当请求ip192.168.0.122的时候使用网卡ens33:0响应 :
[root@director ~]# route add -host 192.168.0.122 dev ens33:0
[root@director ~]# route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         192.168.0.1     0.0.0.0         UG    100    0        0 ens33
192.168.0.0     0.0.0.0         255.255.0.0     U     100    0        0 ens33
192.168.0.122   0.0.0.0         255.255.255.255 UH    0      0        0 ens33
[root@director ~]#
  • 配置ipvs,先加内核参数,再配置vip
    配置all接口和第一个网卡接口,这里是eth0:
    [root@RS1 ~]# echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
    [root@RS1 ~]# echo 1 > /proc/sys/net/ipv4/conf/eth0/arp_ignore
    [root@RS1 ~]# echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce
    [root@RS1 ~]# echo 2 > /proc/sys/net/ipv4/conf/eth0/arp_announce
    [root@RS1 ~]#

配置RS1VIP,同时增加路由:

[root@RS1 ~]# ifconfig lo:0 192.168.0.122/32 broadcast 192.168.0.122 up
[root@RS1 ~]# route add -host 192.168.0.122 dev lo:0
[root@RS1 ~]#

同理,配置RS2VIP,同时增加路由:

[root@RS2 ~]# echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
[root@RS2 ~]# echo 1 > /proc/sys/net/ipv4/conf/eth1/arp_ignore
[root@RS2 ~]# echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce
[root@RS2 ~]# echo 2 > /proc/sys/net/ipv4/conf/eth1/arp_announce
[root@RS2 ~]# ifconfig lo:0 192.168.0.122/32 broadcast 192.168.0.122 up
[root@RS2 ~]# route add -host 192.168.0.122 dev lo:0
[root@RS2 ~]#

配置directorIPVS规则

先使用curl查看两个RSweb服务是否访问正常

[root@director ~]# curl http://192.168.0.7
this is node1
[root@director ~]# curl http://192.168.0.8
this is node2
[root@director ~]#

配置ipvs

[root@director ~]# ipvsadm -A -t 192.168.0.122:80 -s rr
[root@director ~]# ipvsadm -a -t 192.168.0.122:80 -r 192.168.0.7 -g
[root@director ~]# ipvsadm -a -t 192.168.0.122:80 -r 192.168.0.8 -g
[root@director ~]#

客户端访问:

[root@system ~]# for((i=1;i<=4;i++));do curl 192.168.0.122;done
this is node2
this is node1
this is node2
this is node1
[root@system ~]#

至此lvs dr模型的负载均衡集群配置完成

赞(11)
转载请注明出处:RokasYang's Blog » LVS负载均衡集群-