环境:
OS:Centos 6.9
DB:11.2.0.4
1.虚拟机添加磁盘
2.每个登录查看磁盘情况
节点1:
[root@rac01 bin]# lsblk
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
sr0 11:0 1 1024M 0 rom
sda 8:0 0 35G 0 disk
├─sda1 8:1 0 500M 0 part /boot
├─sda2 8:2 0 4G 0 part [SWAP]
└─sda3 8:3 0 30.5G 0 part
└─vg_root-lvroot (dm-0) 253:0 0 30.5G 0 lvm /
sdb 8:16 0 1G 0 disk
└─ocrdisk01 (dm-1) 253:1 0 1G 0 mpath
sdc 8:32 0 1G 0 disk
└─ocrdisk02 (dm-2) 253:2 0 1G 0 mpath
sdd 8:48 0 1G 0 disk
└─ocrdisk03 (dm-3) 253:3 0 1G 0 mpath
sde 8:64 0 10G 0 disk
└─datadisk01 (dm-4) 253:4 0 10G 0 mpath
sdf 8:80 0 10G 0 disk
└─redodisk01 (dm-5) 253:5 0 10G 0 mpath
sdg 8:96 0 1G 0 disk
└─ocr01bakdisk (dm-6) 253:6 0 1G 0 mpath
sdh 8:112 0 1G 0 disk
└─ocr02bakdisk (dm-7) 253:7 0 1G 0 mpath
sdi 8:128 0 1G 0 disk
└─ocr03bakdisk (dm-8) 253:8 0 1G 0 mpath
sdj 8:144 0 30G 0 disk
└─mpathd (dm-9) 253:9 0 30G 0 mpath
sdk 8:160 0 10G 0 disk
└─mpathe (dm-10) 253:10 0 10G 0 mpath
节点2:
[root@rac02 ~]# lsblk
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
sr0 11:0 1 1024M 0 rom
sda 8:0 0 35G 0 disk
├─sda1 8:1 0 500M 0 part /boot
├─sda2 8:2 0 4G 0 part [SWAP]
└─sda3 8:3 0 30.5G 0 part
└─vg_root-lvroot (dm-0) 253:0 0 30.5G 0 lvm /
sdb 8:16 0 1G 0 disk
└─ocrdisk01 (dm-1) 253:1 0 1G 0 mpath
sdc 8:32 0 1G 0 disk
└─ocrdisk02 (dm-2) 253:2 0 1G 0 mpath
sdd 8:48 0 1G 0 disk
└─ocrdisk03 (dm-3) 253:3 0 1G 0 mpath
sde 8:64 0 10G 0 disk
└─datadisk01 (dm-4) 253:4 0 10G 0 mpath
sdf 8:80 0 10G 0 disk
└─redodisk01 (dm-5) 253:5 0 10G 0 mpath
sdg 8:96 0 1G 0 disk
└─ocr01bakdisk (dm-6) 253:6 0 1G 0 mpath
sdh 8:112 0 1G 0 disk
└─ocr02bakdisk (dm-7) 253:7 0 1G 0 mpath
sdi 8:128 0 1G 0 disk
└─ocr03bakdisk (dm-8) 253:8 0 1G 0 mpath
sdj 8:144 0 30G 0 disk
└─mpathi (dm-9) 253:9 0 30G 0 mpath
sdk 8:160 0 10G 0 disk
└─mpathj (dm-10) 253:10 0 10G 0 mpath
节点3:
[root@rac03 ~]# lsblk
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
sr0 11:0 1 1024M 0 rom
sda 8:0 0 35G 0 disk
├─sda1 8:1 0 300M 0 part /boot
├─sda2 8:2 0 3.9G 0 part [SWAP]
└─sda3 8:3 0 30.8G 0 part
└─vg_root-lvroot (dm-0) 253:0 0 30.8G 0 lvm /
sdb 8:16 0 1G 0 disk
└─ocrdisk01 (dm-1) 253:1 0 1G 0 mpath
sdc 8:32 0 1G 0 disk
└─ocrdisk02 (dm-2) 253:2 0 1G 0 mpath
sdd 8:48 0 1G 0 disk
└─ocrdisk03 (dm-3) 253:3 0 1G 0 mpath
sde 8:64 0 1G 0 disk
└─ocr01bakdisk (dm-4) 253:4 0 1G 0 mpath
sdf 8:80 0 1G 0 disk
└─ocr02bakdisk (dm-5) 253:5 0 1G 0 mpath
sdg 8:96 0 1G 0 disk
└─ocr03bakdisk (dm-6) 253:6 0 1G 0 mpath
sdi 8:128 0 10G 0 disk
└─redodisk01 (dm-7) 253:7 0 10G 0 mpath
sdj 8:144 0 30G 0 disk
└─mpatha (dm-8) 253:8 0 30G 0 mpath
sdk 8:160 0 10G 0 disk
└─mpathb (dm-9) 253:9 0 10G 0 mpath
sdh 8:112 0 10G 0 disk
└─datadisk01 (dm-10) 253:10 0 10G 0 mpath
可以看到,共享磁盘挂载到每个机器后,盘符是不同的,下面找到新增磁盘的uuid.
3.找到新增磁盘的uuid
节点1:
[root@rac01 bin]# /lib/udev/scsi_id --whitelisted --replace-whitespace --device=/dev/mapper/mpathd
1ATA_VBOX_HARDDISK_VB6d153909-ef011b58
[root@rac01 bin]# /lib/udev/scsi_id --whitelisted --replace-whitespace --device=/dev/mapper/mpathe
1ATA_VBOX_HARDDISK_VBc3cdb83e-e09020b3
节点2:
[root@rac02 ~]# /lib/udev/scsi_id --whitelisted --replace-whitespace --device=/dev/mapper/mpathi
1ATA_VBOX_HARDDISK_VB6d153909-ef011b58
[root@rac02 ~]# /lib/udev/scsi_id --whitelisted --replace-whitespace --device=/dev/mapper/mpathj
1ATA_VBOX_HARDDISK_VBc3cdb83e-e09020b3
节点3:
[root@rac03 ~]# /lib/udev/scsi_id --whitelisted --replace-whitespace --device=/dev/mapper/mpatha
1ATA_VBOX_HARDDISK_VB6d153909-ef011b58
[root@rac03 ~]# /lib/udev/scsi_id --whitelisted --replace-whitespace --device=/dev/mapper/mpathb
1ATA_VBOX_HARDDISK_VBc3cdb83e-e09020b3
可以看到,共享磁盘虽然挂载到每个机器后,盘符是不同的,但是磁盘的uuid是相同的.
这里也可以从每个节点上部署的multipath下的bindings文件和wwids查看盘符与uuid的对应关系,下面以节点1为列
[root@rac01 multipath]# more bindings
# Multipath bindings, Version : 1.0
# NOTE: this file is automatically maintained by the multipath program.
# You should not need to edit this file in normal circumstances.
#
# Format:
# alias wwid
#
mpathf 1ATA_VBOX_HARDDISK_VB779b9035-39d44b9d
mpathg 1ATA_VBOX_HARDDISK_VBeebe1a1e-4d165a69
mpathh 1ATA_VBOX_HARDDISK_VB700c874b-4eda5d88
mpathi 1ATA_VBOX_HARDDISK_VB23597643-632b8a33
mpathj 1ATA_VBOX_HARDDISK_VBbbb16347-1c5fe609
mpatha 1ATA_VBOX_HARDDISK_VBa3c4a5e0-94bb5ab8
mpathb 1ATA_VBOX_HARDDISK_VBb0ea4d92-91689dbe
mpathc 1ATA_VBOX_HARDDISK_VB1c359161-37143588
mpathd 1ATA_VBOX_HARDDISK_VB6d153909-ef011b58
mpathe 1ATA_VBOX_HARDDISK_VBc3cdb83e-e09020b3
[root@rac01 multipath]# ls
bak_bindings bindings wwids
[root@rac01 multipath]# more wwids
# Multipath wwids, Version : 1.0
# NOTE: This file is automatically maintained by multipath and multipathd.
# You should not need to edit this file in normal circumstances.
#
# Valid WWIDs:
/1ATA_VBOX_HARDDISK_VB779b9035-39d44b9d/
/1ATA_VBOX_HARDDISK_VBeebe1a1e-4d165a69/
/1ATA_VBOX_HARDDISK_VB700c874b-4eda5d88/
/1ATA_VBOX_HARDDISK_VB23597643-632b8a33/
/1ATA_VBOX_HARDDISK_VBbbb16347-1c5fe609/
/1ATA_VBOX_HARDDISK_VBa3c4a5e0-94bb5ab8/
/1ATA_VBOX_HARDDISK_VBb0ea4d92-91689dbe/
/1ATA_VBOX_HARDDISK_VB1c359161-37143588/
/1ATA_VBOX_HARDDISK_VB6d153909-ef011b58/
/1ATA_VBOX_HARDDISK_VBc3cdb83e-e09020b3/
4.修改配置文件,配置别名与wwid的对应关系,添加红色部分
添加红色部分,每个节点都要进程操作
vi /etc/multipath.conf
multipaths {
multipath {
wwid 1ATA_VBOX_HARDDISK_VB779b9035-39d44b9d
alias ocrdisk01
path_grouping_policy multibus
}
multipath {
wwid 1ATA_VBOX_HARDDISK_VBeebe1a1e-4d165a69
alias ocrdisk02
path_grouping_policy multibus
}
multipath {
wwid 1ATA_VBOX_HARDDISK_VB700c874b-4eda5d88
alias ocrdisk03
path_grouping_policy multibus
}
multipath {
wwid 1ATA_VBOX_HARDDISK_VB23597643-632b8a33
alias datadisk01
path_grouping_policy multibus
}
multipath {
wwid 1ATA_VBOX_HARDDISK_VBbbb16347-1c5fe609
alias redodisk01
path_grouping_policy multibus
}
multipath {
wwid 1ATA_VBOX_HARDDISK_VBa3c4a5e0-94bb5ab8
alias ocr01bakdisk
path_grouping_policy multibus
}
multipath {
wwid 1ATA_VBOX_HARDDISK_VBb0ea4d92-91689dbe
alias ocr02bakdisk
path_grouping_policy multibus
}
multipath {
wwid 1ATA_VBOX_HARDDISK_VB1c359161-37143588
alias ocr03bakdisk
path_grouping_policy multibus
}
multipath {
wwid 1ATA_VBOX_HARDDISK_VB6d153909-ef011b58
alias databak_disk
path_grouping_policy multibus
}
multipath {
wwid 1ATA_VBOX_HARDDISK_VBc3cdb83e-e09020b3
alias redobak_disk
path_grouping_policy multibus
}
}
5.重新reload
每个节点都进行同样的操作
[root@rac01 ~]# multipath -r
或是执行 multipath -v3
这个时候就出现别名了
[root@rac01 multipath]# multipath -ll
redobak_disk (1ATA_VBOX_HARDDISK_VBc3cdb83e-e09020b3) dm-10 ATA,VBOX HARDDISK
size=10G features='0' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
`- 12:0:0:0 sdk 8:160 active ready running
databak_disk (1ATA_VBOX_HARDDISK_VB6d153909-ef011b58) dm-9 ATA,VBOX HARDDISK
size=30G features='0' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
`- 11:0:0:0 sdj 8:144 active ready running
6.重新加载udev
每个节点都需要进行修改
[root@rac01 ~]# cd /etc/udev/rules.d/
添加新增项,红色部分
[root@rac01 rules.d]# more 99-oracle-asmdevices.rules
ENV{DM_NAME}=="ocrdisk01", OWNER:="grid", GROUP:="asmadmin", MODE:="660"
ENV{DM_NAME}=="ocrdisk02", OWNER:="grid", GROUP:="asmadmin", MODE:="660"
ENV{DM_NAME}=="ocrdisk03", OWNER:="grid", GROUP:="asmadmin", MODE:="660"
ENV{DM_NAME}=="datadisk01", OWNER:="grid", GROUP:="asmadmin", MODE:="660"
ENV{DM_NAME}=="redodisk01", OWNER:="grid", GROUP:="asmadmin", MODE:="660"
ENV{DM_NAME}=="ocr01bakdisk", OWNER:="grid", GROUP:="asmadmin", MODE:="660"
ENV{DM_NAME}=="ocr02bakdisk", OWNER:="grid", GROUP:="asmadmin", MODE:="660"
ENV{DM_NAME}=="ocr03bakdisk", OWNER:="grid", GROUP:="asmadmin", MODE:="660"
ENV{DM_NAME}=="databak_disk", OWNER:="grid", GROUP:="asmadmin", MODE:="660"
ENV{DM_NAME}=="redobak_disk", OWNER:="grid", GROUP:="asmadmin", MODE:="660"
[root@rac01 rules.d]# ls -al /dev/dm*
brw-rw---- 1 root disk 253, 0 Sep 25 08:54 /dev/dm-0
brw-rw---- 1 grid asmadmin 253, 1 Sep 25 09:31 /dev/dm-1
brw-rw---- 1 root disk 253, 10 Sep 25 09:27 /dev/dm-10
brw-rw---- 1 grid asmadmin 253, 2 Sep 25 09:31 /dev/dm-2
brw-rw---- 1 grid asmadmin 253, 3 Sep 25 09:31 /dev/dm-3
brw-rw---- 1 grid asmadmin 253, 4 Sep 25 09:31 /dev/dm-4
brw-rw---- 1 grid asmadmin 253, 5 Sep 25 09:31 /dev/dm-5
brw-rw---- 1 grid asmadmin 253, 6 Sep 25 09:27 /dev/dm-6
brw-rw---- 1 grid asmadmin 253, 7 Sep 25 09:27 /dev/dm-7
brw-rw---- 1 grid asmadmin 253, 8 Sep 25 09:27 /dev/dm-8
brw-rw---- 1 root disk 253, 9 Sep 25 09:27 /dev/dm-9
前新设备的权限还是属于root,执行如下命令后,权限赋予grid
[root@rac01 rules.d]#udevadm trigger
[root@rac01 rules.d]# ls -al /dev/dm*
brw-rw---- 1 root disk 253, 0 Sep 25 09:32 /dev/dm-0
brw-rw---- 1 grid asmadmin 253, 1 Sep 25 09:32 /dev/dm-1
brw-rw---- 1 grid asmadmin 253, 10 Sep 25 09:32 /dev/dm-10
brw-rw---- 1 grid asmadmin 253, 2 Sep 25 09:32 /dev/dm-2
brw-rw---- 1 grid asmadmin 253, 3 Sep 25 09:32 /dev/dm-3
brw-rw---- 1 grid asmadmin 253, 4 Sep 25 09:32 /dev/dm-4
brw-rw---- 1 grid asmadmin 253, 5 Sep 25 09:32 /dev/dm-5
brw-rw---- 1 grid asmadmin 253, 6 Sep 25 09:32 /dev/dm-6
brw-rw---- 1 grid asmadmin 253, 7 Sep 25 09:32 /dev/dm-7
brw-rw---- 1 grid asmadmin 253, 8 Sep 25 09:32 /dev/dm-8
brw-rw---- 1 grid asmadmin 253, 9 Sep 25 09:32 /dev/dm-9
至此,添加新的lun完成.