################################################################
# How to restore OCR and Voting disk  on Oracle 11g R2.
################################################################

--Location and status of OCR before starting the test:
 root@host1:/u01/GRID/11.2/cdata # /u01/GRID/11.2/bin/ocrcheck
 Status of Oracle Cluster Registry is as follows :
 Version                  :          3
 Total space (kbytes)     :     262120
 Used space (kbytes)      :       2744
 Available space (kbytes) :     259376
 ID                       :  401168391
 Device/File Name         : +OCRVOTING
 Device/File integrity check succeeded

Device/File not configured

Device/File not configured

Device/File not configured

Device/File not configured

Cluster registry integrity check succeeded

Logical corruption check succeeded

--Check the existency of BACKUPS:
 root@host1:/root # /u01/GRID/11.2/bin/ocrconfig -showbackup

host1     2010/01/21 14:17:54     /u01/GRID/11.2/cdata/cluster01/backup00.ocr

host1     2010/01/21 05:58:31     /u01/GRID/11.2/cdata/cluster01/backup01.ocr

host1     2010/01/21 01:58:30     /u01/GRID/11.2/cdata/cluster01/backup02.ocr

host1     2010/01/20 05:58:21     /u01/GRID/11.2/cdata/cluster01/day.ocr

host1     2010/01/14 23:12:07     /u01/GRID/11.2/cdata/cluster01/week.ocr
 PROT-25: Manual backups for the Oracle Cluster Registry are not available

--Identify all the disks belong the Disk group +OCRVOTING:

NAME                                       PATH
 ------------------------------ ------------------------------------------------------------
 OCRVOTING_0000                 /dev/oracle/asm.25.lun
 OCRVOTING_0001                 /dev/oracle/asm.26.lun
 OCRVOTING_0002                 /dev/oracle/asm.27.lun
 OCRVOTING_0003                 /dev/oracle/asm.28.lun
 OCRVOTING_0004                 /dev/oracle/asm.29.lun

5 rows selected.

--Corrupt tht disks belong the Disk group +OCRVOTING:
 dd if=/tmp/corrupt_disk of=/dev/oracle/asm.25.lun bs=1024 count=1000
 dd if=/tmp/corrupt_disk of=/dev/oracle/asm.26.lun bs=1024 count=1000
 dd if=/tmp/corrupt_disk of=/dev/oracle/asm.27.lun bs=1024 count=1000
 dd if=/tmp/corrupt_disk of=/dev/oracle/asm.28.lun bs=1024 count=1000
 dd if=/tmp/corrupt_disk of=/dev/oracle/asm.29.lun bs=1024 count=1000

--OCR Check after Corruption:
 root@host1:/tmp # /u01/GRID/11.2/bin/ocrcheck
 Status of Oracle Cluster Registry is as follows :
 Version                  :          3
 Total space (kbytes)     :     262120
 Used space (kbytes)      :       2712
 Available space (kbytes) :     259408
 ID                       :  701409037
 Device/File Name         : +OCRVOTING
 Device/File integrity check succeeded

Device/File not configured

Device/File not configured

Device/File not configured

Device/File not configured

Cluster registry integrity check succeeded

Logical corruption check succeeded

--Stop and Start of database instance after corruption
 oracle@host1:/u01/oracle/data $ srvctl stop instance -d DB -i DB1
 oracle@host1:/u01/oracle/data $ srvctl start instance -d DB -i DB1

--Stop and Start entire Cluster:

-host1:
 root@host1:/tmp # /u01/GRID/11.2/bin/crsctl stop crs
 CRS-2791: Starting shutdown of Oracle High Availability Services-managed resources on 'host1'
 CRS-2673: Attempting to stop 'ora.crsd' on 'host1'
 CRS-2790: Starting shutdown of Cluster Ready Services-managed resources on 'host1'
 CRS-2673: Attempting to stop 'ora.OCRVOTING.dg' on 'host1'
 CRS-2673: Attempting to stop 'ora.db.db' on 'host1'
 CRS-2673: Attempting to stop 'ora.LISTENER.lsnr' on 'host1'
 CRS-2677: Stop of 'ora.LISTENER.lsnr' on 'host1' succeeded
 CRS-2673: Attempting to stop 'ora.host1.vip' on 'host1'
 CRS-2677: Stop of 'ora.host1.vip' on 'host1' succeeded
 CRS-2677: Stop of 'ora.OCRVOTING.dg' on 'host1' succeeded
 CRS-2673: Attempting to stop 'ora.scan2.vip' on 'host1'
 CRS-2673: Attempting to stop 'ora.scan3.vip' on 'host1'
 CRS-2673: Attempting to stop 'ora.host2.vip' on 'host1'
 CRS-2677: Stop of 'ora.scan2.vip' on 'host1' succeeded
 CRS-2677: Stop of 'ora.scan3.vip' on 'host1' succeeded
 CRS-2677: Stop of 'ora.host2.vip' on 'host1' succeeded
 CRS-2677: Stop of 'ora.db.db' on 'host1' succeeded
 CRS-2673: Attempting to stop 'ora.DATA1.dg' on 'host1'
 CRS-2673: Attempting to stop 'ora.FRA1.dg' on 'host1'
 CRS-2677: Stop of 'ora.DATA1.dg' on 'host1' succeeded
 CRS-2677: Stop of 'ora.FRA1.dg' on 'host1' succeeded
 CRS-2673: Attempting to stop 'ora.asm' on 'host1'
 CRS-2677: Stop of 'ora.asm' on 'host1' succeeded
 CRS-2673: Attempting to stop 'ora.ons' on 'host1'
 CRS-2673: Attempting to stop 'ora.eons' on 'host1'
 CRS-2677: Stop of 'ora.ons' on 'host1' succeeded
 CRS-2673: Attempting to stop 'ora.net1.network' on 'host1'
 CRS-2677: Stop of 'ora.net1.network' on 'host1' succeeded
 CRS-2677: Stop of 'ora.eons' on 'host1' succeeded
 CRS-2792: Shutdown of Cluster Ready Services-managed resources on 'host1' has completed
 CRS-2677: Stop of 'ora.crsd' on 'host1' succeeded
 CRS-2673: Attempting to stop 'ora.mdnsd' on 'host1'
 CRS-2673: Attempting to stop 'ora.gpnpd' on 'host1'
 CRS-2673: Attempting to stop 'ora.cssdmonitor' on 'host1'
 CRS-2673: Attempting to stop 'ora.ctssd' on 'host1'
 CRS-2673: Attempting to stop 'ora.evmd' on 'host1'
 CRS-2673: Attempting to stop 'ora.asm' on 'host1'
 CRS-2677: Stop of 'ora.cssdmonitor' on 'host1' succeeded
 CRS-2677: Stop of 'ora.mdnsd' on 'host1' succeeded
 CRS-2677: Stop of 'ora.gpnpd' on 'host1' succeeded
 CRS-2677: Stop of 'ora.evmd' on 'host1' succeeded
 CRS-2677: Stop of 'ora.ctssd' on 'host1' succeeded
 CRS-2677: Stop of 'ora.asm' on 'host1' succeeded
 CRS-2673: Attempting to stop 'ora.cssd' on 'host1'
 CRS-2677: Stop of 'ora.cssd' on 'host1' succeeded
 CRS-2673: Attempting to stop 'ora.diskmon' on 'host1'
 CRS-2673: Attempting to stop 'ora.gipcd' on 'host1'
 CRS-2677: Stop of 'ora.gipcd' on 'host1' succeeded
 CRS-2677: Stop of 'ora.diskmon' on 'host1' succeeded
 CRS-2793: Shutdown of Oracle High Availability Services-managed resources on 'host1' has completed
 CRS-4133: Oracle High Availability Services has been stopped.

--host2:
 root@host2:/root # /u01/GRID/11.2/bin/crsctl stop crs
 CRS-2791: Starting shutdown of Oracle High Availability Services-managed resources on 'host2'
 CRS-2673: Attempting to stop 'ora.crsd' on 'host2'
 CRS-2790: Starting shutdown of Cluster Ready Services-managed resources on 'host2'
 CRS-2673: Attempting to stop 'ora.LISTENER_SCAN2.lsnr' on 'host2'
 CRS-2673: Attempting to stop 'ora.LISTENER_SCAN3.lsnr' on 'host2'
 CRS-2673: Attempting to stop 'ora.LISTENER.lsnr' on 'host2'
 CRS-2673: Attempting to stop 'ora.OCRVOTING.dg' on 'host2'
 CRS-2673: Attempting to stop 'ora.db.db' on 'host2'
 CRS-2673: Attempting to stop 'ora.LISTENER_SCAN1.lsnr' on 'host2'
 CRS-2677: Stop of 'ora.LISTENER_SCAN2.lsnr' on 'host2' succeeded
 CRS-2673: Attempting to stop 'ora.scan2.vip' on 'host2'
 CRS-2677: Stop of 'ora.scan2.vip' on 'host2' succeeded
 CRS-2672: Attempting to start 'ora.scan2.vip' on 'host1'
 CRS-2677: Stop of 'ora.LISTENER_SCAN3.lsnr' on 'host2' succeeded
 CRS-2673: Attempting to stop 'ora.scan3.vip' on 'host2'
 CRS-2677: Stop of 'ora.LISTENER.lsnr' on 'host2' succeeded
 CRS-2673: Attempting to stop 'ora.host2.vip' on 'host2'
 CRS-2677: Stop of 'ora.scan3.vip' on 'host2' succeeded
 CRS-2672: Attempting to start 'ora.scan3.vip' on 'host1'
 CRS-2677: Stop of 'ora.host2.vip' on 'host2' succeeded
 CRS-2672: Attempting to start 'ora.host2.vip' on 'host1'
 CRS-2677: Stop of 'ora.LISTENER_SCAN1.lsnr' on 'host2' succeeded
 CRS-2673: Attempting to stop 'ora.scan1.vip' on 'host2'
 CRS-2677: Stop of 'ora.scan1.vip' on 'host2' succeeded
 CRS-2676: Start of 'ora.scan2.vip' on 'host1' succeeded
 CRS-2676: Start of 'ora.scan3.vip' on 'host1' succeeded
 CRS-2676: Start of 'ora.host2.vip' on 'host1' succeeded
 CRS-2677: Stop of 'ora.OCRVOTING.dg' on 'host2' succeeded
 CRS-2677: Stop of 'ora.db.db' on 'host2' succeeded
 CRS-2673: Attempting to stop 'ora.DATA1.dg' on 'host2'
 CRS-2673: Attempting to stop 'ora.FRA1.dg' on 'host2'
 CRS-2677: Stop of 'ora.DATA1.dg' on 'host2' succeeded
 CRS-2677: Stop of 'ora.FRA1.dg' on 'host2' succeeded
 CRS-2673: Attempting to stop 'ora.asm' on 'host2'
 CRS-2677: Stop of 'ora.asm' on 'host2' succeeded
 CRS-2673: Attempting to stop 'ora.ons' on 'host2'
 CRS-2673: Attempting to stop 'ora.eons' on 'host2'
 CRS-2677: Stop of 'ora.ons' on 'host2' succeeded
 CRS-2673: Attempting to stop 'ora.net1.network' on 'host2'
 CRS-2677: Stop of 'ora.net1.network' on 'host2' succeeded
 CRS-2677: Stop of 'ora.eons' on 'host2' succeeded
 CRS-2792: Shutdown of Cluster Ready Services-managed resources on 'host2' has completed
 CRS-2677: Stop of 'ora.crsd' on 'host2' succeeded
 CRS-2673: Attempting to stop 'ora.gpnpd' on 'host2'
 CRS-2673: Attempting to stop 'ora.cssdmonitor' on 'host2'
 CRS-2673: Attempting to stop 'ora.ctssd' on 'host2'
 CRS-2673: Attempting to stop 'ora.evmd' on 'host2'
 CRS-2673: Attempting to stop 'ora.asm' on 'host2'
 CRS-2673: Attempting to stop 'ora.mdnsd' on 'host2'
 CRS-2677: Stop of 'ora.cssdmonitor' on 'host2' succeeded
 CRS-2677: Stop of 'ora.gpnpd' on 'host2' succeeded
 CRS-2677: Stop of 'ora.evmd' on 'host2' succeeded
 CRS-2677: Stop of 'ora.mdnsd' on 'host2' succeeded
 CRS-2677: Stop of 'ora.asm' on 'host2' succeeded
 CRS-2677: Stop of 'ora.ctssd' on 'host2' succeeded
 CRS-2673: Attempting to stop 'ora.cssd' on 'host2'
 CRS-2677: Stop of 'ora.cssd' on 'host2' succeeded
 CRS-2673: Attempting to stop 'ora.diskmon' on 'host2'
 CRS-2673: Attempting to stop 'ora.gipcd' on 'host2'
 CRS-2677: Stop of 'ora.gipcd' on 'host2' succeeded
 CRS-2677: Stop of 'ora.diskmon' on 'host2' succeeded
 CRS-2793: Shutdown of Oracle High Availability Services-managed resources on 'host2' has completed
 CRS-4133: Oracle High Availability Services has been stopped.

--host1
 root@host1:/root # /u01/GRID/11.2/bin/crsctl start crs
 CRS-4123: Oracle High Availability Services has been started.

--host2
 root@host2:/u01/GRID/11.2/cdata/cluster01 # /u01/GRID/11.2/bin/crsctl start crs
 CRS-4123: Oracle High Availability Services has been started.

--CRS Alert log: (Start failed because the Diskgroup is not available)
 2010-01-21 16:29:07.785
 [cssd(10123)]CRS-1705:Found 0 configured voting files but 1 voting files are required, terminating to ensure data integrity; details at (:CSSNM00065:) in /u01/GRID/11.2/log/host1/cssd/ocssd.log
 2010-01-21 16:29:07.785
 [cssd(10123)]CRS-1603:CSSD on node host1 shutdown by user.
 2010-01-21 16:29:07.918
 [ohasd(9931)]CRS-2765:Resource 'ora.cssdmonitor' has failed on server 'host1'.
 2010-01-21 16:30:05.489
 [/u01/GRID/11.2/bin/orarootagent.bin(10113)]CRS-5818:Aborted command 'start for resource: ora.diskmon 1 1' for resource 'ora.diskmon'. Details at (:CRSAGF00113:) in /u01/GRID/11.2/log/host1/agent/ohasd/orarootagent_root/orarootagent_root.log.
 2010-01-21 16:30:09.504
 [ohasd(9931)]CRS-2757:Command 'Start' timed out waiting for response from the resource 'ora.diskmon'. Details at (:CRSPE00111:) in /u01/GRID/11.2/log/host1/ohasd/ohasd.log.
 2010-01-21 16:30:20.687
 [cssd(10622)]CRS-1713:CSSD daemon is started in clustered mode
 2010-01-21 16:30:21.801
 [cssd(10622)]CRS-1705:Found 0 configured voting files but 1 voting files are required, terminating to ensure data integrity; details at (:CSSNM00065:) in /u01/GRID/11.2/log/host1/cssd/ocssd.log
 2010-01-21 16:30:21.801
 [cssd(10622)]CRS-1603:CSSD on node host1 shutdown by user.

--host1 STOP CRS because due to Voting Disk unavailability is not running properly:
 root@host1:/tmp # /u01/GRID/11.2/bin/crsctl stop crs
 CRS-2791: Starting shutdown of Oracle High Availability Services-managed resources on 'host1'
 CRS-2673: Attempting to stop 'ora.crsd' on 'host1'
 CRS-4548: Unable to connect to CRSD
 CRS-2675: Stop of 'ora.crsd' on 'host1' failed
 CRS-2679: Attempting to clean 'ora.crsd' on 'host1'
 CRS-4548: Unable to connect to CRSD
 CRS-2678: 'ora.crsd' on 'host1' has experienced an unrecoverable failure
 CRS-0267: Human intervention required to resume its availability.
 CRS-2795: Shutdown of Oracle High Availability Services-managed resources on 'host1' has failed
 CRS-4687: Shutdown command has completed with error(s).
 CRS-4000: Command Stop failed, or completed with errors.

--Because all the processes are not STOPPING, disable the cluster AUTO Start and reboot
 --the server for cleaning all the pending processes.

root@host1:/tmp # /u01/GRID/11.2/bin/crsctl disable crs
 CRS-4621: Oracle High Availability Services autostart is disabled.

root@host1:/tmp # reboot

--Start the Cluster in EXLUSIVE Mode in order to recreate ASM Diskgroup:
 root@host1:/root # /u01/GRID/11.2/bin/crsctl start crs -excl
 CRS-4123: Oracle High Availability Services has been started.
 CRS-2672: Attempting to start 'ora.gipcd' on 'host1'
 CRS-2672: Attempting to start 'ora.mdnsd' on 'host1'
 CRS-2676: Start of 'ora.gipcd' on 'host1' succeeded
 CRS-2676: Start of 'ora.mdnsd' on 'host1' succeeded
 CRS-2672: Attempting to start 'ora.gpnpd' on 'host1'
 CRS-2676: Start of 'ora.gpnpd' on 'host1' succeeded
 CRS-2672: Attempting to start 'ora.cssdmonitor' on 'host1'
 CRS-2676: Start of 'ora.cssdmonitor' on 'host1' succeeded
 CRS-2672: Attempting to start 'ora.cssd' on 'host1'
 CRS-2679: Attempting to clean 'ora.diskmon' on 'host1'
 CRS-2681: Clean of 'ora.diskmon' on 'host1' succeeded
 CRS-2672: Attempting to start 'ora.diskmon' on 'host1'
 CRS-2676: Start of 'ora.diskmon' on 'host1' succeeded
 CRS-2676: Start of 'ora.cssd' on 'host1' succeeded
 CRS-2672: Attempting to start 'ora.ctssd' on 'host1'
 CRS-2676: Start of 'ora.ctssd' on 'host1' succeeded
 CRS-2672: Attempting to start 'ora.asm' on 'host1'
 CRS-2676: Start of 'ora.asm' on 'host1' succeeded
 CRS-2672: Attempting to start 'ora.crsd' on 'host1'
 CRS-2676: Start of 'ora.crsd' on 'host1' succeeded

--Stop ASM and restart it using a pfile example:
 *.asm_diskgroups='DATA1','FRA1'
 *.asm_diskstring='/dev/oracle/asm*'
 *.diagnostic_dest='/u01/oracle'
 +ASM1.instance_number=1
 +ASM2.instance_number=2
 *.instance_type='asm'
 *.large_pool_size=12M
 *.processes=500
 *.sga_max_size=1G
 *.sga_target=1G
 *.shared_pool_size=300M

--Recreate ASM Diskgroup
 --This command FAILS because asmca is not able to update the OCR:
 asmca -silent -createDiskGroup -diskGroupName OCRVOTING  -disk '/dev/oracle/asm.25.lun' -disk '/dev/oracle/asm.26.lun'  -disk '/dev/oracle/asm.27.lun'  -disk '/dev/oracle/asm.28.lun'  -disk '/dev/oracle/asm.29.lun'  -redundancy HIGH -compatible.asm '11.2.0.0.0'  -compatible.rdbms '11.2.0.0.0' -compatible.advm '11.2.0.0.0'

--Create the Diskgroup Using SQLPLUS Create Diskgroup and save the ASM spfile inside:
 create Diskgroup OCRVOTING high redundancy disk '/dev/oracle/asm.25.lun',
 '/dev/oracle/asm.26.lun', '/dev/oracle/asm.27.lun',
 '/dev/oracle/asm.28.lun', '/dev/oracle/asm.29.lun'
 ATTRIBUTE  'compatible.asm'='11.2.0.0.0', 'compatible.rdbms'='11.2.0.0.0';

create spfile='+OCRVOTING' from pfile='/tmp/asm_pfile.ora';

File created.

SQL> shut immediate
 ASM diskgroups dismounted
 ASM instance shutdown
 SQL> startup
 ASM instance started

Total System Global Area 1069252608 bytes
 Fixed Size                  2154936 bytes
 Variable Size            1041931848 bytes
 ASM Cache                  25165824 bytes
 ASM diskgroups mounted

-- Restore OCR from backup:
 root@host1:/root # /u01/GRID/11.2/bin/ocrconfig -restore /u01/GRID/11.2/cdata/cluster01/backup00.ocr

--Check the OCR status after restore:
 root@host1:/root # /u01/GRID/11.2/bin/ocrcheck
 Status of Oracle Cluster Registry is as follows :
 Version                  :          3
 Total space (kbytes)     :     262120
 Used space (kbytes)      :       2712
 Available space (kbytes) :     259408
 ID                       :  701409037
 Device/File Name         : +OCRVOTING
 Device/File integrity check succeeded

Device/File not configured

Device/File not configured

Device/File not configured

Device/File not configured

Cluster registry integrity check succeeded

Logical corruption check succeeded

--Restore the Voting Disk:
 root@host1:/root # /u01/GRID/11.2/bin/crsctl replace votedisk +OCRVOTING
 Successful addition of voting disk 7s16f9fbf4b64f74bfy0ee8826f15eb4.
 Successful addition of voting disk 9k6af49d3cd54fc5bf28a2fc3899c8c6.
 Successful addition of voting disk 876eb99563924ff6bfc1defe6865deeb.
 Successful addition of voting disk 12230b5ef41f4fc2bf2cae957f765fb0.
 Successful addition of voting disk 47812b7f6p034f33bf13490e6e136b8b.
 Successfully replaced voting disk group with +OCRVOTING.
 CRS-4266: Voting file(s) successfully replaced

--Re-enable CRS auto starup
 root@host1:/root # /u01/GRID/11.2/bin/crsctl enable crs
 CRS-4622: Oracle High Availability Services autostart is enabled.

--Stop CRS on host1
 root@host1:/root # /u01/GRID/11.2/bin/crsctl stop crs
 CRS-2791: Starting shutdown of Oracle High Availability Services-managed resources on 'host1'
 CRS-2673: Attempting to stop 'ora.crsd' on 'host1'
 CRS-2677: Stop of 'ora.crsd' on 'host1' succeeded
 CRS-2673: Attempting to stop 'ora.gpnpd' on 'host1'
 CRS-2673: Attempting to stop 'ora.cssdmonitor' on 'host1'
 CRS-2673: Attempting to stop 'ora.ctssd' on 'host1'
 CRS-2673: Attempting to stop 'ora.asm' on 'host1'
 CRS-2673: Attempting to stop 'ora.mdnsd' on 'host1'
 CRS-2677: Stop of 'ora.cssdmonitor' on 'host1' succeeded
 CRS-2677: Stop of 'ora.gpnpd' on 'host1' succeeded
 CRS-2677: Stop of 'ora.mdnsd' on 'host1' succeeded
 CRS-2677: Stop of 'ora.ctssd' on 'host1' succeeded
 CRS-2677: Stop of 'ora.asm' on 'host1' succeeded
 CRS-2673: Attempting to stop 'ora.cssd' on 'host1'
 CRS-2677: Stop of 'ora.cssd' on 'host1' succeeded
 CRS-2673: Attempting to stop 'ora.diskmon' on 'host1'
 CRS-2673: Attempting to stop 'ora.gipcd' on 'host1'
 CRS-2677: Stop of 'ora.gipcd' on 'host1' succeeded
 CRS-2677: Stop of 'ora.diskmon' on 'host1' succeeded
 CRS-2793: Shutdown of Oracle High Availability Services-managed resources on 'host1' has completed
 CRS-4133: Oracle High Availability Services has been stopped.

--Start CRS on host1
 root@host1:/root # /u01/GRID/11.2/bin/crsctl start crs
 CRS-4123: Oracle High Availability Services has been started.

--Start CRS on host2
 root@host2:/root # /u01/GRID/11.2/bin/crsctl start crs
 CRS-4123: Oracle High Availability Services has been started.

--Check if all the Resources are running:
 root@host1:/root # /u01/GRID/11.2/bin/crsctl stat res -t
 --------------------------------------------------------------------------------
 NAME           TARGET  STATE        SERVER                   STATE_DETAILS
 --------------------------------------------------------------------------------
 Local Resources
 --------------------------------------------------------------------------------
 ora.DATA1.dg
 ONLINE  ONLINE       host1
 ONLINE  ONLINE       host2
 ora.FRA1.dg
 ONLINE  ONLINE       host1
 ONLINE  ONLINE       host2
 ora.LISTENER.lsnr
 ONLINE  ONLINE       host1
 ONLINE  ONLINE       host2
 ora.OCRVOTING.dg
 ONLINE  ONLINE       host1
 ONLINE  ONLINE       host2
 ora.asm
 ONLINE  ONLINE       host1                 Started
 ONLINE  ONLINE       host2                 Started
 ora.eons
 ONLINE  ONLINE       host1
 ONLINE  ONLINE       host2
 ora.gsd
 OFFLINE OFFLINE      host1
 OFFLINE OFFLINE      host2
 ora.net1.network
 ONLINE  ONLINE       host1
 ONLINE  ONLINE       host2
 ora.ons
 ONLINE  ONLINE       host1
 ONLINE  ONLINE       host2
 --------------------------------------------------------------------------------
 Cluster Resources
 --------------------------------------------------------------------------------
 ora.LISTENER_SCAN1.lsnr
 1        ONLINE  ONLINE       host1
 ora.LISTENER_SCAN2.lsnr
 1        ONLINE  ONLINE       host2
 ora.LISTENER_SCAN3.lsnr
 1        ONLINE  ONLINE       host2
 ora.db.db
 1        ONLINE  ONLINE       host1                 Open
 2        ONLINE  ONLINE       host2                 Open
 ora.oc4j
 1        OFFLINE OFFLINE
 ora.scan1.vip
 1        ONLINE  ONLINE       host1
 ora.scan2.vip
 1        ONLINE  ONLINE       host2
 ora.scan3.vip
 1        ONLINE  ONLINE       host2
 ora.host1.vip
 1        ONLINE  ONLINE       host1
 ora.host2.vip
 1        ONLINE  ONLINE       host2