环境:
OS:Centos 6.9
virtualbox虚拟机
1.安装
[root@rac02 ios]# yum install device-mapper-multipath
2.设置开机自启动
[root@cluster01 ~]# chkconfig multipathd on
[root@cluster01 ~]# chkconfig --list |grep mul
multipathd 0:off 1:off 2:on 3:on 4:on 5:on 6:off
3.生成配置文件
[root@cluster01 ~]#mpathconf --enable
执行该语句后,会在/etc目录下生成配置文件
multipath.conf
4.编辑配置文件(基础配置)
vi /etc/multipath.conf
blacklist {
devnode "^sda"
}
defaults {
user_friendly_names yes
path_grouping_policy multibus
failback immediate
no_path_retry fail
getuid_callout "/lib/udev/scsi_id --whitelisted --replace-whitespace --device=/dev/%n"
}
注意这里需要加上如下这句
getuid_callout "/lib/udev/scsi_id --whitelisted --replace-whitespace --device=/dev/%n"
--replace-whitespace:去掉空格
否则生成的uuid是这样的(centos7没有这个问题)
[root@rac01 multipath]# more bindings
# Multipath bindings, Version : 1.0
# NOTE: this file is automatically maintained by the multipath program.
# You should not need to edit this file in normal circumstances.
#
# Format:
# alias wwid
#
mpatha 1ATA VBOX HARDDISK VB779b9035-39d44b9d
mpathb 1ATA VBOX HARDDISK VBeebe1a1e-4d165a69
mpathc 1ATA VBOX HARDDISK VB700c874b-4eda5d88
mpathd 1ATA VBOX HARDDISK VB23597643-632b8a33
mpathe 1ATA VBOX HARDDISK VBbbb16347-1c5fe609
这样话在启动的时候,无法map
执行命令 multipath -v3 会报如下错误:
Aug 30 15:51:08 | mpathe: set ACT_CREATE (map does not exist)
Aug 30 15:51:08 | libdevmapper: libdm-common.c(441): Mangled form of the UUID too long for "mpath-1ATA VBOX HARDDISK VBbbb16347-1c5fe609 ".
Aug 30 15:51:08 | libdevmapper: libdm-common.c(779): Failed to mangle device uuid "mpath-1ATA VBOX HARDDISK VBbbb16347-1c5fe609 ".
5.启动
[root@db11g ~]# modprobe dm-multipath
[root@db11g ~]# modprobe dm-round-robin
[root@db11g ~]# service multipathd start
6.查看mapping情况
[root@rac01 multipath]# multipath -ll
mpathj (1ATA_VBOX_HARDDISK_VBbbb16347-1c5fe609) dm-5 ATA,VBOX HARDDISK
size=10G features='0' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
`- 7:0:0:0 sdf 8:80 active ready running
mpathi (1ATA_VBOX_HARDDISK_VB23597643-632b8a33) dm-4 ATA,VBOX HARDDISK
size=10G features='0' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
`- 6:0:0:0 sde 8:64 active ready running
mpathh (1ATA_VBOX_HARDDISK_VB700c874b-4eda5d88) dm-3 ATA,VBOX HARDDISK
size=1.0G features='0' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
`- 5:0:0:0 sdd 8:48 active ready running
mpathg (1ATA_VBOX_HARDDISK_VBeebe1a1e-4d165a69) dm-2 ATA,VBOX HARDDISK
size=1.0G features='0' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
`- 4:0:0:0 sdc 8:32 active ready running
mpathf (1ATA_VBOX_HARDDISK_VB779b9035-39d44b9d) dm-1 ATA,VBOX HARDDISK
size=1.0G features='0' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
`- 3:0:0:0 sdb 8:16 active ready running
[root@rac01 multipath]# lsblk
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
sr0 11:0 1 1024M 0 rom
sda 8:0 0 35G 0 disk
├─sda1 8:1 0 500M 0 part /boot
├─sda2 8:2 0 4G 0 part [SWAP]
└─sda3 8:3 0 30.5G 0 part
└─vg_root-lvroot (dm-0) 253:0 0 30.5G 0 lvm /
sdb 8:16 0 1G 0 disk
└─mpathf (dm-1) 253:1 0 1G 0 mpath
sdc 8:32 0 1G 0 disk
└─mpathg (dm-2) 253:2 0 1G 0 mpath
sdd 8:48 0 1G 0 disk
└─mpathh (dm-3) 253:3 0 1G 0 mpath
sde 8:64 0 10G 0 disk
└─mpathi (dm-4) 253:4 0 10G 0 mpath
sdf 8:80 0 10G 0 disk
└─mpathj (dm-5) 253:5 0 10G 0 mpath
7.重新编辑配置文件
编辑 /etc/multipath.conf 文件,用于安装oracle rac,添加如下参数:
multipaths {
multipath {
wwid 1ATA_VBOX_HARDDISK_VB779b9035-39d44b9d
alias ocrdisk01
path_grouping_policy multibus
}
multipath {
wwid 1ATA_VBOX_HARDDISK_VBeebe1a1e-4d165a69
alias ocrdisk02
path_grouping_policy multibus
}
multipath {
wwid 1ATA_VBOX_HARDDISK_VB700c874b-4eda5d88
alias ocrdisk03
path_grouping_policy multibus
}
multipath {
wwid 1ATA_VBOX_HARDDISK_VB23597643-632b8a33
alias datadisk01
path_grouping_policy multibus
}
multipath {
wwid 1ATA_VBOX_HARDDISK_VBbbb16347-1c5fe609
alias redodisk01
path_grouping_policy multibus
}
}
完整的配置如下:
[root@rac01 multipath]# more /etc/multipath.conf
blacklist {
devnode "^sda"
}
defaults {
user_friendly_names yes
path_grouping_policy multibus
failback immediate
no_path_retry fail
getuid_callout "/lib/udev/scsi_id --whitelisted --replace-whitespace --device=/dev/%n"
}
multipaths {
multipath {
wwid 1ATA_VBOX_HARDDISK_VB779b9035-39d44b9d
alias ocrdisk01
path_grouping_policy multibus
}
multipath {
wwid 1ATA_VBOX_HARDDISK_VBeebe1a1e-4d165a69
alias ocrdisk02
path_grouping_policy multibus
}
multipath {
wwid 1ATA_VBOX_HARDDISK_VB700c874b-4eda5d88
alias ocrdisk03
path_grouping_policy multibus
}
multipath {
wwid 1ATA_VBOX_HARDDISK_VB23597643-632b8a33
alias datadisk01
path_grouping_policy multibus
}
multipath {
wwid 1ATA_VBOX_HARDDISK_VBbbb16347-1c5fe609
alias redodisk01
path_grouping_policy multibus
}
}
9.重启加载配置
[root@localhost multipath]# multipath -r
查看加载后的配置
[root@localhost multipath]# multipath -ll
[root@rac01 etc]# multipath -ll
ocrdisk03 (1ATA_VBOX_HARDDISK_VB700c874b-4eda5d88) dm-3 ATA,VBOX HARDDISK
size=1.0G features='0' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
`- 5:0:0:0 sdd 8:48 active ready running
datadisk01 (1ATA_VBOX_HARDDISK_VB23597643-632b8a33) dm-4 ATA,VBOX HARDDISK
size=10G features='0' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
`- 6:0:0:0 sde 8:64 active ready running
ocrdisk02 (1ATA_VBOX_HARDDISK_VBeebe1a1e-4d165a69) dm-2 ATA,VBOX HARDDISK
size=1.0G features='0' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
`- 4:0:0:0 sdc 8:32 active ready running
redodisk01 (1ATA_VBOX_HARDDISK_VBbbb16347-1c5fe609) dm-5 ATA,VBOX HARDDISK
size=10G features='0' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
`- 7:0:0:0 sdf 8:80 active ready running
ocrdisk01 (1ATA_VBOX_HARDDISK_VB779b9035-39d44b9d) dm-1 ATA,VBOX HARDDISK
size=1.0G features='0' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
`- 3:0:0:0 sdb 8:16 active ready running
可以看到mapper下面的磁盘名称也改变了
[root@rac01 etc]# ls -al /dev/mapper/
total 0
drwxr-xr-x. 2 root root 180 Aug 30 17:05 .
drwxr-xr-x. 21 root root 4000 Aug 30 15:59 ..
crw-rw----. 1 root root 10, 58 Aug 30 14:52 control
lrwxrwxrwx. 1 root root 7 Aug 30 17:05 datadisk01 -> ../dm-4
lrwxrwxrwx. 1 root root 7 Aug 30 17:05 ocrdisk01 -> ../dm-1
lrwxrwxrwx. 1 root root 7 Aug 30 17:05 ocrdisk02 -> ../dm-2
lrwxrwxrwx. 1 root root 7 Aug 30 17:05 ocrdisk03 -> ../dm-3
lrwxrwxrwx. 1 root root 7 Aug 30 17:05 redodisk01 -> ../dm-5
lrwxrwxrwx. 1 root root 7 Aug 30 14:52 vg_root-lvroot -> ../dm-0
-------------------------------------配置udev------------------------------
1.编辑规则文件
[root@SL0101A-IVDB02 ~]# cd /etc/udev/rules.d/
[root@djp01 rules.d]# vi 99-oracle-asmdevices.rules
文件文件内容如下:
ENV{DM_NAME}=="ocrdisk01", OWNER:="grid", GROUP:="asmadmin", MODE:="660"
ENV{DM_NAME}=="ocrdisk02", OWNER:="grid", GROUP:="asmadmin", MODE:="660"
ENV{DM_NAME}=="ocrdisk03", OWNER:="grid", GROUP:="asmadmin", MODE:="660"
ENV{DM_NAME}=="datadisk01", OWNER:="grid", GROUP:="asmadmin", MODE:="660"
ENV{DM_NAME}=="redodisk01", OWNER:="grid", GROUP:="asmadmin", MODE:="660"
这里的磁盘名称需要跟 /etc/multipath.conf 配置里的对应起来
2.启用
[root@localhost ~]# /sbin/udevadm trigger --type=devices --action=change
[root@localhost bin]# udevadm trigger ##新增磁盘的情况 执行该句即可
如执行上面两个命令不生效的情况下，可以执行如下命令
[root@localhost ~]#/sbin/udevadm control --reload
[root@localhost ~]# /sbin/udevadm trigger --type=devices --action=change
[root@localhost bin]# udevadm trigger
3.查看设备权限
[root@localhost rules.d]# ls -al /dev/dm*
[root@rac01 rules.d]# ls -al /dev/dm*
brw-rw----. 1 root disk 253, 0 Aug 31 08:37 /dev/dm-0
brw-rw----. 1 grid asmadmin 253, 1 Aug 31 08:37 /dev/dm-1
brw-rw----. 1 grid asmadmin 253, 2 Aug 31 08:37 /dev/dm-2
brw-rw----. 1 grid asmadmin 253, 3 Aug 31 08:37 /dev/dm-3
brw-rw----. 1 grid asmadmin 253, 4 Aug 31 08:37 /dev/dm-4
brw-rw----. 1 grid asmadmin 253, 5 Aug 31 08:37 /dev/dm-5
4.安装rac的时候选择发现磁盘路径为:/dev/mapper/*