| 主机名 | IP |
| master | eth0: 192.168.30.130(NAT) eth1: 192.168.17.130(VMNet4) |
| node-1 | eth0: 192.168.30.131(NAT) eth1: 192.168.17.131(VMNet4) |
安装过程省略
master上的配置
[root@master ~]# vim /etc/tgt/targets.confbacking-store /dev/sdb1 initiator-address 192.168.30.131 initiator-address 192.168.17.131 vendor_id sishen product_id target-01 [root@master ~]# /etc/init.d/tgtd restart Stopping SCSI target daemon: initiators still connected [FAILED]Starting SCSI target daemon: [ OK ][root@master ~]# tgttgtadm tgt-admin tgtd tgtimg tgt-setup-lun[root@master ~]# tgt-admin --show Target 1: iqn.2018-06.cn.sishen.www:target_san1 System information: Driver: iscsi State: ready I_T nexus information: LUN information: LUN: 0 Type: controller SCSI ID: IET 00010000 SCSI SN: beaf10 Size: 0 MB, Block size: 1 Online: Yes Removable media: No Prevent removal: No Readonly: No Backing store type: null Backing store path: None Backing store flags: Account information: ACL information: 192.168.30.131 192.168.17.131
在客户端测试通讯
[root@node-1 ~]# ping -c 2 192.168.30.130PING 192.168.30.130 (192.168.30.130) 56(84) bytes of data.64 bytes from 192.168.30.130: icmp_seq=1 ttl=64 time=0.183 ms64 bytes from 192.168.30.130: icmp_seq=2 ttl=64 time=0.267 ms--- 192.168.30.130 ping statistics ---2 packets transmitted, 2 received, 0% packet loss, time 1000msrtt min/avg/max/mdev = 0.183/0.225/0.267/0.042 ms[root@node-1 ~]# ping -c 2 192.168.17.130PING 192.168.17.130 (192.168.17.130) 56(84) bytes of data.64 bytes from 192.168.17.130: icmp_seq=1 ttl=64 time=0.350 ms64 bytes from 192.168.17.130: icmp_seq=2 ttl=64 time=0.154 ms--- 192.168.17.130 ping statistics ---2 packets transmitted, 2 received, 0% packet loss, time 1000msrtt min/avg/max/mdev = 0.154/0.252/0.350/0.098 ms
配置线路1
[root@node-1 ~]# iscsiadm -m discovery -t sendtargets -p 192.168.30.130:3260Starting iscsid: [ OK ]192.168.30.130:3260,1 iqn.2018-06.cn.sishen.www:target_san1[root@node-1 ~]# /etc/init.d/iscsi restart Stopping iscsi: [ OK ]Starting iscsi: [ OK ][root@node-1 ~]# lsblk NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINTsr0 11:0 1 3.6G 0 rom /media/cdromsda 8:0 0 20G 0 disk ├─sda1 8:1 0 200M 0 part /boot├─sda2 8:2 0 2G 0 part [SWAP]└─sda3 8:3 0 17.8G 0 part └─vg_master-LogVol00 (dm-0) 253:0 0 17.8G 0 lvm /sdb 8:16 0 5G 0 disk └─sdb1 8:17 0 5G 0 part
配置线路2
[root@node-1 ~]# iscsiadm -m discovery -t sendtargets -p 192.168.17.130:3260192.168.17.130:3260,1 iqn.2018-06.cn.sishen.www:target_san1
[root@node-1 ~]# tree /var/lib/iscsi//var/lib/iscsi/├── ifaces├── isns├── nodes│ └── iqn.2018-06.cn.sishen.www:target_san1│ ├── 192.168.17.130,3260,1│ │ └── default│ └── 192.168.30.130,3260,1│ └── default├── send_targets│ ├── 192.168.17.130,3260│ │ ├── iqn.2018-06.cn.sishen.www:target_san1,192.168.17.130,3260,1,default -> /var/lib/iscsi/nodes/iqn.2018-06.cn.sishen.www:target_san1/192.168.17.130,3260,1│ │ └── st_config│ └── 192.168.30.130,3260│ ├── iqn.2018-06.cn.sishen.www:target_san1,192.168.30.130,3260,1,default -> /var/lib/iscsi/nodes/iqn.2018-06.cn.sishen.www:target_san1/192.168.30.130,3260,1│ └── st_config├── slp└── static13 directories, 4 files
客户端重启iscsi服务
[root@node-1 ~]# lsblk NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINTsr0 11:0 1 3.6G 0 rom /media/cdromsda 8:0 0 20G 0 disk ├─sda1 8:1 0 200M 0 part /boot├─sda2 8:2 0 2G 0 part [SWAP]└─sda3 8:3 0 17.8G 0 part └─vg_master-LogVol00 (dm-0) 253:0 0 17.8G 0 lvm /sdc 8:32 0 5G 0 disk └─sdc1 8:33 0 5G 0 part sdb 8:16 0 5G 0 disk └─sdb1 8:17 0 5G 0 part
挂在数据测试
[root@node-1 ~]# mount /dev/sdb1 /opt/[root@node-1 ~]# df -h /opt/Filesystem Size Used Avail Use% Mounted on/dev/sdb1 5.0G 139M 4.6G 3% /opt[root@node-1 ~]# ll /dev/sdb1brw-rw---- 1 root disk 8, 17 Jun 4 11:15 /dev/sdb1[root@node-1 ~]# ll /dev/sdc1brw-rw---- 1 root disk 8, 33 Jun 4 11:15 /dev/sdc1
配置多路径实现冗余
[root@node-1 ~]# yum install -y device-mapper-multipath[root@node-1 ~]# cp /usr/share/doc/device-mapper-multipath-0.4.9/multipath.conf /etc/[root@node-1 ~]# /etc/init.d/multipathd restart ux_socket_connect: No such file or directoryStopping multipathd daemon: [FAILED]Starting multipathd daemon: [ OK ][root@node-1 ~]# multipath -ll #没有反应，卸载/opt/下面的挂载重启multipath服务[root@node-1 ~]# umount /opt/[root@node-1 ~]# /etc/init.d/multipathd restart okStopping multipathd daemon: [ OK ]Starting multipathd daemon: [ OK ][root@node-1 ~]# multipath -llmpatha (1IET 00010001) dm-1 sishen,target-01size=5.0G features='0' hwhandler='0' wp=rw|-+- policy='round-robin 0' prio=1 status=active| `- 9:0:0:1 sdc 8:32 active ready running`-+- policy='round-robin 0' prio=1 status=enabled `- 10:0:0:1 sdb 8:16 active ready running[root@node-1 ~]# df -hFilesystem Size Used Avail Use% Mounted on/dev/mapper/vg_master-LogVol00 18G 4.1G 13G 25% /tmpfs 2.0G 72K 2.0G 1% /dev/shm/dev/sda1 194M 35M 150M 19% /boot/dev/sr0 3.6G 3.6G 0 100% /media/cdrom/dev/mapper/mpathap1 5.0G 139M 4.6G 3% /opt[root@node-1 ~]# ll /dev/mapper/mpatha*lrwxrwxrwx 1 root root 7 Jun 4 11:20 /dev/mapper/mpatha -> ../dm-1lrwxrwxrwx 1 root root 7 Jun 4 11:20 /dev/mapper/mpathap1 -> ../dm-2
配置负载均衡
[root@node-1 ~]# multipath -llmpatha (1IET 00010001) dm-1 sishen,target-01 #远程存储设备的产品信息 dm-0 xuegod #厂商,target1 #产品IDsize=5.0G features='0' hwhandler='0' wp=rw|-+- policy='round-robin 0' prio=1 status=active| `- 9:0:0:1 sdc 8:32 active ready running`-+- policy='round-robin 0' prio=1 status=enabled `- 10:0:0:1 sdb 8:16 active ready running[root@node-1 ~]# cp /etc/multipath.conf{,.bak}[root@node-1 ~]# vim /etc/multipath.conf ## Use user friendly names, instead of using WWIDs as names. #注释掉以下三行#defaults { # user_friendly_names yes#}62 multipaths { 63 multipath { 64 wwid "1IET 00010001" # 65 alias data-back 66 path_grouping_policy multibus 67 path_checker readsector0 68 path_selector "round-robin 0" 69 failback manual 70 rr_weight priorities 71 no_path_retry 5 72 } 73 } [root@node-1 ~]# /etc/init.d/multipathd restart okStopping multipathd daemon: [ OK ]Starting multipathd daemon: [ OK ][root@node-1 ~]# /etc/init.d/iscsi restart Stopping iscsi: [ OK ]Starting iscsi: [ OK ][root@node-1 ~]# multipath -llJun 04 11:30:32 | multipath.conf line 67, invalid keyword: path_checkerdata-back (1IET 00010001) dm-1 sishen,target-01size=5.0G features='1 queue_if_no_path' hwhandler='0' wp=rw`-+- policy='round-robin 0' prio=1 status=active |- 12:0:0:1 sdb 8:16 active ready running `- 11:0:0:1 sdc 8:32 active ready running [root@node-1 ~]# vim /etc/multipath.conf 24 ## Use user friendly names, instead of using WWIDs as names.， 25 #defaults { 26 # user_friendly_names yes 27 #}#在62行修改或添加如下内容 62 multipaths { 63 multipath { 64 wwid "1IET 00010001" 65 alias data-back 66 path_grouping_policy multibus 67 path_selector "round-robin 0" 68 failback manual 69 rr_weight priorities 70 no_path_retry 5 71 } 72 } 重启服务
[root@node-1 ~]# /etc/init.d/multipathd restart okStopping multipathd daemon: [ OK ]Starting multipathd daemon: [ OK ][root@node-1 ~]# /etc/init.d/iscsi restart Stopping iscsi: [ OK ]Starting iscsi: [ OK ][root@node-1 ~]# multipath -ll datashare (1IET 00010001) dm-1 sishen,target-01size=5.0G features='1 queue_if_no_path' hwhandler='0' wp=rw`-+- policy='round-robin 0' prio=1 status=active |- 6:0:0:1 sdb 8:16 active ready running `- 5:0:0:1 sdc 8:32 active ready running[root@node-1 ~]# ll /dev/mapper/datashare*lrwxrwxrwx 1 root root 7 Jun 4 13:02 /dev/mapper/datashare -> ../dm-1lrwxrwxrwx 1 root root 7 Jun 4 13:02 /dev/mapper/datasharep1 -> ../dm-2
挂载测试
[root@node-1 ~]# mount /dev/mapper/datasharedatashare datasharep1 [root@node-1 ~]# mount /dev/mapper/datasharep1 /opt/[root@node-1 ~]# df -h /opt/Filesystem Size Used Avail Use% Mounted on/dev/mapper/datasharep1 5.0G 139M 4.6G 3% /opt[root@node-1 ~]# cp install.log /opt/[root@node-1 ~]# ll /opt/total 80-rw-r--r-- 1 root root 61802 Jun 4 13:05 install.logdrwx------ 2 root root 16384 Jun 4 12:37 lost+found
模拟故障测试冗余
[root@master ~]# ifdown eth1 #手动停掉eth1Device state: 3 (disconnected)[root@node-1 ~]# multipath -ll #这里大约需要60秒来识别，所以会卡在这里，等待即可datashare (1IET 00010001) dm-1 sishen,target-01size=5.0G features='1 queue_if_no_path' hwhandler='0' wp=rw`-+- policy='round-robin 0' prio=1 status=active |- 6:0:0:1 sdb 8:16 active faulty running `- 5:0:0:1 sdc 8:32 active ready running
在服务端[root@master ~]# ifdown eth1Device state: 3 (disconnected)在客户端[root@node-1 ~]# multipath -ll #回车之后等待大约60秒datashare (1IET 00010001) dm-1 sishen,target-01size=5.0G features='1 queue_if_no_path' hwhandler='0' wp=rw`-+- policy='round-robin 0' prio=1 status=active |- 6:0:0:1 sdb 8:16 failed faulty running `- 5:0:0:1 sdc 8:32 active ready running #这里也要等待大约60秒测试数据写入[root@node-1 ~]# cp /etc/passwd /opt/[root@node-1 ~]# ll /opt/total 84-rw-r--r-- 1 root root 61802 Jun 4 13:05 install.logdrwx------ 2 root root 16384 Jun 4 12:37 lost+found-rw-r--r-- 1 root root 1779 Jun 4 13:16 passwd在服务端启动eth1[root@master ~]# ifup eth1Active connection state: activatedActive connection path: /org/freedesktop/NetworkManager/ActiveConnection/2在客户端查看负载均衡[root@node-1 ~]# multipath -ll #立刻就会显示结果datashare (1IET 00010001) dm-1 sishen,target-01size=5.0G features='1 queue_if_no_path' hwhandler='0' wp=rw`-+- policy='round-robin 0' prio=1 status=active |- 6:0:0:1 sdb 8:16 active ready running `- 5:0:0:1 sdc 8:32 active ready running#这里说明负载均衡正常