环境:
OS:Centos 6.9
说明:整个操作过程不会影响rac集群的运行
1.在虚拟机上添加共享磁盘,rac的2个节点分别挂载
2.查看当前的情况
节点1:
[root@rac01 ~]# multipath -ll
mpathc (1ATA_VBOX_HARDDISK_VB1c359161-37143588) dm-8 ATA,VBOX HARDDISK
size=1.0G features='0' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
`- 10:0:0:0 sdi 8:128 active ready running
mpathb (1ATA_VBOX_HARDDISK_VBb0ea4d92-91689dbe) dm-7 ATA,VBOX HARDDISK
size=1.0G features='0' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
`- 9:0:0:0 sdh 8:112 active ready running
mpatha (1ATA_VBOX_HARDDISK_VBa3c4a5e0-94bb5ab8) dm-6 ATA,VBOX HARDDISK
size=1.0G features='0' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
`- 8:0:0:0 sdg 8:96 active ready running
ocrdisk03 (1ATA_VBOX_HARDDISK_VB700c874b-4eda5d88) dm-3 ATA,VBOX HARDDISK
size=1.0G features='0' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
`- 5:0:0:0 sdd 8:48 active ready running
datadisk01 (1ATA_VBOX_HARDDISK_VB23597643-632b8a33) dm-4 ATA,VBOX HARDDISK
size=10G features='0' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
`- 6:0:0:0 sde 8:64 active ready running
ocrdisk02 (1ATA_VBOX_HARDDISK_VBeebe1a1e-4d165a69) dm-2 ATA,VBOX HARDDISK
size=1.0G features='0' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
`- 4:0:0:0 sdc 8:32 active ready running
redodisk01 (1ATA_VBOX_HARDDISK_VBbbb16347-1c5fe609) dm-5 ATA,VBOX HARDDISK
size=10G features='0' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
`- 7:0:0:0 sdf 8:80 active ready running
ocrdisk01 (1ATA_VBOX_HARDDISK_VB779b9035-39d44b9d) dm-1 ATA,VBOX HARDDISK
size=1.0G features='0' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
`- 3:0:0:0 sdb 8:16 active ready running
节点2:
[root@rac02 ~]# multipath -ll
ocrdisk03 (1ATA_VBOX_HARDDISK_VB700c874b-4eda5d88) dm-3 ATA,VBOX HARDDISK
size=1.0G features='0' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
`- 5:0:0:0 sdd 8:48 active ready running
datadisk01 (1ATA_VBOX_HARDDISK_VB23597643-632b8a33) dm-4 ATA,VBOX HARDDISK
size=10G features='0' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
`- 6:0:0:0 sde 8:64 active ready running
ocrdisk02 (1ATA_VBOX_HARDDISK_VBeebe1a1e-4d165a69) dm-2 ATA,VBOX HARDDISK
size=1.0G features='0' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
`- 4:0:0:0 sdc 8:32 active ready running
redodisk01 (1ATA_VBOX_HARDDISK_VBbbb16347-1c5fe609) dm-5 ATA,VBOX HARDDISK
size=10G features='0' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
`- 7:0:0:0 sdf 8:80 active ready running
mpathh (1ATA_VBOX_HARDDISK_VB1c359161-37143588) dm-8 ATA,VBOX HARDDISK
size=1.0G features='0' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
`- 10:0:0:0 sdi 8:128 active ready running
ocrdisk01 (1ATA_VBOX_HARDDISK_VB779b9035-39d44b9d) dm-1 ATA,VBOX HARDDISK
size=1.0G features='0' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
`- 3:0:0:0 sdb 8:16 active ready running
mpathg (1ATA_VBOX_HARDDISK_VBb0ea4d92-91689dbe) dm-7 ATA,VBOX HARDDISK
size=1.0G features='0' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
`- 9:0:0:0 sdh 8:112 active ready running
mpathf (1ATA_VBOX_HARDDISK_VBa3c4a5e0-94bb5ab8) dm-6 ATA,VBOX HARDDISK
size=1.0G features='0' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
`- 8:0:0:0 sdg 8:96 active ready running
可以发现经过多路径映射出来的磁盘符号是不一样的
3.节点1找出原始磁盘和多路径磁盘的对应关系
[root@rac01 multipath]# more bindings
# Multipath bindings, Version : 1.0
# NOTE: this file is automatically maintained by the multipath program.
# You should not need to edit this file in normal circumstances.
#
# Format:
# alias wwid
#
mpathf 1ATA_VBOX_HARDDISK_VB779b9035-39d44b9d
mpathg 1ATA_VBOX_HARDDISK_VBeebe1a1e-4d165a69
mpathh 1ATA_VBOX_HARDDISK_VB700c874b-4eda5d88
mpathi 1ATA_VBOX_HARDDISK_VB23597643-632b8a33
mpathj 1ATA_VBOX_HARDDISK_VBbbb16347-1c5fe609
mpatha 1ATA_VBOX_HARDDISK_VBa3c4a5e0-94bb5ab8
mpathb 1ATA_VBOX_HARDDISK_VBb0ea4d92-91689dbe
mpathc 1ATA_VBOX_HARDDISK_VB1c359161-37143588
[root@rac01 multipath]# lsblk
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
sr0 11:0 1 1024M 0 rom
sda 8:0 0 35G 0 disk
├─sda1 8:1 0 500M 0 part /boot
├─sda2 8:2 0 4G 0 part [SWAP]
└─sda3 8:3 0 30.5G 0 part
└─vg_root-lvroot (dm-0) 253:0 0 30.5G 0 lvm /
sdb 8:16 0 1G 0 disk
└─ocrdisk01 (dm-1) 253:1 0 1G 0 mpath
sdc 8:32 0 1G 0 disk
└─ocrdisk02 (dm-2) 253:2 0 1G 0 mpath
sdd 8:48 0 1G 0 disk
└─ocrdisk03 (dm-3) 253:3 0 1G 0 mpath
sde 8:64 0 10G 0 disk
└─datadisk01 (dm-4) 253:4 0 10G 0 mpath
sdf 8:80 0 10G 0 disk
└─redodisk01 (dm-5) 253:5 0 10G 0 mpath
sdg 8:96 0 1G 0 disk
└─mpatha (dm-6) 253:6 0 1G 0 mpath
sdh 8:112 0 1G 0 disk
└─mpathb (dm-7) 253:7 0 1G 0 mpath
sdi 8:128 0 1G 0 disk
└─mpathc (dm-8) 253:8 0 1G 0 mpath
这里的磁盘对应关系为:
原始磁盘名称 多路径出来的磁盘名 uuid
sdg mpatha 1ATA_VBOX_HARDDISK_VBa3c4a5e0-94bb5ab8
sdh mpathb 1ATA_VBOX_HARDDISK_VBb0ea4d92-91689dbe
sdi mpathc 1ATA_VBOX_HARDDISK_VB1c359161-37143588
4.节点2找出原始磁盘和多路径磁盘的对应关系
[root@rac02 multipath]# cd /etc/multipath
[root@rac02 multipath]# more bindings
# Multipath bindings, Version : 1.0
# NOTE: this file is automatically maintained by the multipath program.
# You should not need to edit this file in normal circumstances.
#
# Format:
# alias wwid
#
mpatha 1ATA_VBOX_HARDDISK_VB779b9035-39d44b9d
mpathb 1ATA_VBOX_HARDDISK_VBeebe1a1e-4d165a69
mpathc 1ATA_VBOX_HARDDISK_VB700c874b-4eda5d88
mpathd 1ATA_VBOX_HARDDISK_VB23597643-632b8a33
mpathe 1ATA_VBOX_HARDDISK_VBbbb16347-1c5fe609
mpathf 1ATA_VBOX_HARDDISK_VBa3c4a5e0-94bb5ab8
mpathg 1ATA_VBOX_HARDDISK_VBb0ea4d92-91689dbe
mpathh 1ATA_VBOX_HARDDISK_VB1c359161-37143588
[root@rac02 multipath]# lsblk
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
sr0 11:0 1 1024M 0 rom
sda 8:0 0 35G 0 disk
├─sda1 8:1 0 500M 0 part /boot
├─sda2 8:2 0 4G 0 part [SWAP]
└─sda3 8:3 0 30.5G 0 part
└─vg_root-lvroot (dm-0) 253:0 0 30.5G 0 lvm /
sdb 8:16 0 1G 0 disk
└─ocrdisk01 (dm-1) 253:1 0 1G 0 mpath
sdc 8:32 0 1G 0 disk
└─ocrdisk02 (dm-2) 253:2 0 1G 0 mpath
sdd 8:48 0 1G 0 disk
└─ocrdisk03 (dm-3) 253:3 0 1G 0 mpath
sde 8:64 0 10G 0 disk
└─datadisk01 (dm-4) 253:4 0 10G 0 mpath
sdf 8:80 0 10G 0 disk
└─redodisk01 (dm-5) 253:5 0 10G 0 mpath
sdg 8:96 0 1G 0 disk
└─mpathf (dm-6) 253:6 0 1G 0 mpath
sdh 8:112 0 1G 0 disk
└─mpathg (dm-7) 253:7 0 1G 0 mpath
sdi 8:128 0 1G 0 disk
└─mpathh (dm-8) 253:8 0 1G 0 mpath
原始磁盘名称 多路径出来的磁盘名 uuid
sdg mpathf 1ATA_VBOX_HARDDISK_VBa3c4a5e0-94bb5ab8
sdh mpathg 1ATA_VBOX_HARDDISK_VBb0ea4d92-91689dbe
sdi mpathh 1ATA_VBOX_HARDDISK_VB1c359161-37143588
5.节点1修改配置文件,配置别名与wwid的对应关系
红色部分为新增
[root@rac01 multipath]# more /etc/multipath.conf
blacklist {
devnode "^sda"
}
defaults {
user_friendly_names yes
path_grouping_policy multibus
failback immediate
no_path_retry fail
getuid_callout "/lib/udev/scsi_id --whitelisted --replace-whitespace --device=/dev/%n"
}
multipaths {
multipath {
wwid 1ATA_VBOX_HARDDISK_VB779b9035-39d44b9d
alias ocrdisk01
path_grouping_policy multibus
}
multipath {
wwid 1ATA_VBOX_HARDDISK_VBeebe1a1e-4d165a69
alias ocrdisk02
path_grouping_policy multibus
}
multipath {
wwid 1ATA_VBOX_HARDDISK_VB700c874b-4eda5d88
alias ocrdisk03
path_grouping_policy multibus
}
multipath {
wwid 1ATA_VBOX_HARDDISK_VB23597643-632b8a33
alias datadisk01
path_grouping_policy multibus
}
multipath {
wwid 1ATA_VBOX_HARDDISK_VBbbb16347-1c5fe609
alias redodisk01
path_grouping_policy multibus
}
multipath {
wwid 1ATA_VBOX_HARDDISK_VBa3c4a5e0-94bb5ab8
alias ocr01bakdisk
path_grouping_policy multibus
}
multipath {
wwid 1ATA_VBOX_HARDDISK_VBb0ea4d92-91689dbe
alias ocr02bakdisk
path_grouping_policy multibus
}
multipath {
wwid 1ATA_VBOX_HARDDISK_VB1c359161-37143588
alias ocr03bakdisk
path_grouping_policy multibus
}
}
节点2操作一样添加新增的项
4.重新reload
节点1和节点2都做同样的操作
[root@rac01 ~]# multipath -r
或是执行 multipath -v3
这个时候就会输出别名了
[root@rac01 multipath]# multipath -ll
ocr02bakdisk (1ATA_VBOX_HARDDISK_VBb0ea4d92-91689dbe) dm-7 ATA,VBOX HARDDISK
size=1.0G features='0' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
`- 9:0:0:0 sdh 8:112 active ready running
ocrdisk03 (1ATA_VBOX_HARDDISK_VB700c874b-4eda5d88) dm-3 ATA,VBOX HARDDISK
size=1.0G features='0' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
`- 5:0:0:0 sdd 8:48 active ready running
datadisk01 (1ATA_VBOX_HARDDISK_VB23597643-632b8a33) dm-4 ATA,VBOX HARDDISK
size=10G features='0' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
`- 6:0:0:0 sde 8:64 active ready running
ocrdisk02 (1ATA_VBOX_HARDDISK_VBeebe1a1e-4d165a69) dm-2 ATA,VBOX HARDDISK
size=1.0G features='0' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
`- 4:0:0:0 sdc 8:32 active ready running
redodisk01 (1ATA_VBOX_HARDDISK_VBbbb16347-1c5fe609) dm-5 ATA,VBOX HARDDISK
size=10G features='0' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
`- 7:0:0:0 sdf 8:80 active ready running
ocrdisk01 (1ATA_VBOX_HARDDISK_VB779b9035-39d44b9d) dm-1 ATA,VBOX HARDDISK
size=1.0G features='0' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
`- 3:0:0:0 sdb 8:16 active ready running
ocr01bakdisk (1ATA_VBOX_HARDDISK_VBa3c4a5e0-94bb5ab8) dm-6 ATA,VBOX HARDDISK
size=1.0G features='0' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
`- 8:0:0:0 sdg 8:96 active ready running
ocr03bakdisk (1ATA_VBOX_HARDDISK_VB1c359161-37143588) dm-8 ATA,VBOX HARDDISK
size=1.0G features='0' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
`- 10:0:0:0 sdi 8:128 active ready running
5.重新加载udev
节点1和节点2做同样的操作
[root@rac01 ~]# cd /etc/udev/rules.d/
添加新增项
[root@rac01 rules.d]# more 99-oracle-asmdevices.rules
ENV{DM_NAME}=="ocrdisk01", OWNER:="grid", GROUP:="asmadmin", MODE:="660"
ENV{DM_NAME}=="ocrdisk02", OWNER:="grid", GROUP:="asmadmin", MODE:="660"
ENV{DM_NAME}=="ocrdisk03", OWNER:="grid", GROUP:="asmadmin", MODE:="660"
ENV{DM_NAME}=="datadisk01", OWNER:="grid", GROUP:="asmadmin", MODE:="660"
ENV{DM_NAME}=="redodisk01", OWNER:="grid", GROUP:="asmadmin", MODE:="660"
ENV{DM_NAME}=="ocr01bakdisk", OWNER:="grid", GROUP:="asmadmin", MODE:="660"
ENV{DM_NAME}=="ocr02bakdisk", OWNER:="grid", GROUP:="asmadmin", MODE:="660"
ENV{DM_NAME}=="ocr03bakdisk", OWNER:="grid", GROUP:="asmadmin", MODE:="660"
6.执行如下命令修改磁盘权限
每个节点都做同样的操作
当前新设备的权限还是属于root,执行如下命令后,权限赋予grid
[root@rac01 rules.d]# udevadm trigger
[root@rac01 rules.d]# ls -al /dev/dm*
brw-rw---- 1 root disk 253, 0 Sep 1 10:42 /dev/dm-0
brw-rw---- 1 grid asmadmin 253, 1 Sep 1 10:42 /dev/dm-1
brw-rw---- 1 grid asmadmin 253, 2 Sep 1 10:42 /dev/dm-2
brw-rw---- 1 grid asmadmin 253, 3 Sep 1 10:42 /dev/dm-3
brw-rw---- 1 grid asmadmin 253, 4 Sep 1 10:42 /dev/dm-4
brw-rw---- 1 grid asmadmin 253, 5 Sep 1 10:42 /dev/dm-5
brw-rw---- 1 grid asmadmin 253, 6 Sep 1 10:42 /dev/dm-6
brw-rw---- 1 grid asmadmin 253, 7 Sep 1 10:42 /dev/dm-7
brw-rw---- 1 grid asmadmin 253, 8 Sep 1 10:42 /dev/dm-8