Added ONOS Failure and Cluster Restart testcase

Change-Id: I3ab68b68e90e82ff37ddff09108059a6d9706f14
diff --git a/TestON/tests/USECASE/SegmentRouting/SROnosFailure/README.md b/TestON/tests/USECASE/SegmentRouting/SROnosFailure/README.md
new file mode 100755
index 0000000..94677cb
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SROnosFailure/README.md
@@ -0,0 +1,22 @@
+This test verifies basic control plane resilience from an ONOS Instance failure using SegmentRouting via pingall
+
+It consists of 
+
+1) Configure and Install ONOS
+2) Start Mininet and check flow state
+3) Pingall
+4) Induce a ONOS failure
+5) check flow state
+6) Pingall
+
+Requirements
+
+ - An updated version of the CPQD switch has to be running to make sure it supports group chaining.
+
+The test is executed using the netcfg subsystem:
+    1) APPS=openflow-base,netcfghostprovider,netcfglinksprovider
+
+The test runs for different topologies:
+ - 2x2 Leaf-Spine and 3-node ONOS cluster
+ - 4x4 Leaf-Spine and 3-node ONOS cluster
+ - Single switch and 3-node ONOS cluster
\ No newline at end of file
diff --git a/TestON/tests/USECASE/SegmentRouting/SROnosFailure/SROnosFailure.params b/TestON/tests/USECASE/SegmentRouting/SROnosFailure/SROnosFailure.params
new file mode 100755
index 0000000..1a58556
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SROnosFailure/SROnosFailure.params
@@ -0,0 +1,46 @@
+<PARAMS>
+
+    <testcases>1,2,3</testcases>
+
+    <SCALE>
+        <size>3</size>
+        <max>3</max>
+    </SCALE>
+
+    <DEPENDENCY>
+        <wrapper1>startUp</wrapper1>
+        <topology>cord_fabric.py</topology>
+    </DEPENDENCY>
+
+    <ENV>
+        <cellName>productionCell</cellName>
+        <cellApps>drivers,segmentrouting</cellApps>
+        <diffApps>openflow-base,netcfghostprovider,netcfglinksprovider</diffApps>
+        <cellUser>sdn</cellUser>
+    </ENV>
+
+    <GIT>
+        <pull>False</pull>
+        <branch>master</branch>
+    </GIT>
+
+    <CTRL>
+        <port>6653</port>
+    </CTRL>
+
+    <timers>
+        <LinkDiscovery>12</LinkDiscovery>
+        <SwitchDiscovery>12</SwitchDiscovery>
+    </timers>
+
+    <kill>
+        <switch> spine101 </switch>
+        <dpid> 000000000101 </dpid>
+        <links> leaf1 leaf2 </links>
+    </kill>
+
+    <SLEEP>
+        <startup>10</startup>
+    </SLEEP>
+
+</PARAMS>
diff --git a/TestON/tests/USECASE/SegmentRouting/SROnosFailure/SROnosFailure.py b/TestON/tests/USECASE/SegmentRouting/SROnosFailure/SROnosFailure.py
new file mode 100755
index 0000000..53daa33
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SROnosFailure/SROnosFailure.py
@@ -0,0 +1,106 @@
+# CASE1: 2x2 topo + 3-node ONOS CLUSTER + IP connectivity test + Control plane resilience
+# CASE2: 4x4 topo + 3-node ONOS CLUSTER + IP connectivity test + Control plane resilience
+# CASE3: Single switch + 3-node ONOS CLUSTER + IP connectivity test + Control plane resilience
+
+class SROnosFailure:
+    def __init__( self ):
+        self.default = ''
+
+    def CASE1( self, main ):
+        """
+        Sets up 3-node Onos-cluster
+        Start 2x2 Leaf-Spine topology
+        Pingall
+        Induce ONOS Failure
+        Pingall
+        """
+        description = "ONOS Failure test with 2x2 Leaf-spine "
+        main.case( description )
+        from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
+            Testcaselib as run
+        if not hasattr( main, 'apps' ):
+            run.initTest( main )
+        main.cfgName = '2x2'
+        main.numCtrls = 3
+        run.installOnos( main )
+        run.startMininet( main, 'cord_fabric.py' )
+        # pre-configured routing and bridging test
+        run.checkFlows( main, minFlowCount=116 )
+        run.pingAll( main, "CASE1" )
+        run.killOnos( main, [ 0 ], '4', '8', '2' )
+        run.pingAll( main, 'CASE1_Failure' )
+        run.recoverOnos( main, [ 0 ], '4', '8', '3' )
+        run.checkFlows( main, minFlowCount=116 )
+        run.pingAll( main, 'CASE1_Failure' )
+        # TODO Dynamic config of hosts in subnet
+        # TODO Dynamic config of host not in subnet
+        # TODO Dynamic config of vlan xconnect
+        # TODO Vrouter integration
+        # TODO Mcast integration
+        run.cleanup( main )
+
+    def CASE2( self, main ):
+        """
+        Sets up 3-node Onos-cluster
+        Start 4x4 Leaf-Spine topology
+        Pingall
+        Induce ONOS Failure
+        Pingall
+        """
+        description = "ONOS Failure test with 4x4 Leaf-spine "
+        main.case( description )
+        from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
+            Testcaselib as run
+        if not hasattr( main, 'apps' ):
+            run.initTest( main )
+        main.cfgName = '4x4'
+        main.numCtrls = 3
+        run.installOnos( main )
+        run.startMininet( main, 'cord_fabric.py', args="--leaf=4 --spine=4" )
+        # pre-configured routing and bridging test
+        run.checkFlows( main, minFlowCount=350 )
+        run.pingAll( main, 'CASE2' )
+        run.killOnos( main, [ 0 ], '8', '32', '2' )
+        run.pingAll( main, 'CASE2_Failure' )
+        run.recoverOnos( main, [ 0 ], '8', '32', '3' )
+        run.checkFlows( main, minFlowCount=350 )
+        run.pingAll( main, 'CASE3_Recovery' )
+        # TODO Dynamic config of hosts in subnet
+        # TODO Dynamic config of host not in subnet
+        # TODO Dynamic config of vlan xconnect
+        # TODO Vrouter integration
+        # TODO Mcast integration
+        run.cleanup( main )
+
+    def CASE3( self, main ):
+        """
+        Sets up 3-node Onos-cluster
+        Start single switch topology
+        Pingall
+        Induce ONOS Failure
+        Pingall
+        """
+        description = "ONOS Failure test with single switch "
+        main.case( description )
+        from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
+            Testcaselib as run
+        if not hasattr( main, 'apps' ):
+            run.initTest( main )
+        main.cfgName = '0x1'
+        main.numCtrls = 3
+        run.installOnos( main )
+        run.startMininet( main, 'cord_fabric.py', args="--leaf=1 --spine=0" )
+        # pre-configured routing and bridging test
+        run.checkFlows( main, minFlowCount=15 )
+        run.pingAll( main, 'CASE3' )
+        run.killOnos( main, [ 0 ], '1', '0', '2' )
+        run.pingAll( main, 'CASE3_Failure' )
+        run.recoverOnos( main, [ 0 ], '1', '0', '3' )
+        run.checkFlows( main, minFlowCount=15 )
+        run.pingAll( main, 'CASE3_Failure' )
+        # TODO Dynamic config of hosts in subnet
+        # TODO Dynamic config of host not in subnet
+        # TODO Dynamic config of vlan xconnect
+        # TODO Vrouter integration
+        # TODO Mcast integration
+        run.cleanup( main )
diff --git a/TestON/tests/USECASE/SegmentRouting/SROnosFailure/SROnosFailure.topo b/TestON/tests/USECASE/SegmentRouting/SROnosFailure/SROnosFailure.topo
new file mode 100755
index 0000000..c531f7d
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SROnosFailure/SROnosFailure.topo
@@ -0,0 +1,90 @@
+<TOPOLOGY>
+    <COMPONENT>
+
+        <ONOSbench>
+            <host>localhost</host>
+            <user>sdn</user>
+            <password>rocks</password>
+            <type>OnosDriver</type>
+            <connect_order>1</connect_order>
+            <COMPONENTS>
+                <nodes>1</nodes>
+            </COMPONENTS>
+        </ONOSbench>
+
+        <ONOScli1>
+            <host>OC1</host>
+            <user>sdn</user>
+            <password>rocks</password>
+            <type>OnosCliDriver</type>
+            <connect_order>2</connect_order>
+            <COMPONENTS>
+            </COMPONENTS>
+        </ONOScli1>
+        
+        <ONOScli2>
+            <host>OC2</host>
+            <user>sdn</user>
+            <password>rocks</password>
+            <type>OnosCliDriver</type>
+            <connect_order>3</connect_order>
+            <COMPONENTS>
+            </COMPONENTS>
+        </ONOScli2>
+
+        <ONOScli3>
+            <host>OC3</host>
+            <user>sdn</user>
+            <password>rocks</password>
+            <type>OnosCliDriver</type>
+            <connect_order>4</connect_order>
+            <COMPONENTS>
+            </COMPONENTS>
+        </ONOScli3>
+
+        <ONOSrest1>
+            <host>OC1</host>
+            <port>8181</port>
+            <user>onos</user>
+            <password>rocks</password>
+            <type>OnosRestDriver</type>
+            <connect_order>5</connect_order>
+            <COMPONENTS>
+            </COMPONENTS>
+        </ONOSrest1>
+
+        <ONOSrest2>
+            <host>OC2</host>
+            <port>8181</port>
+            <user>onos</user>
+            <password>rocks</password>
+            <type>OnosRestDriver</type>
+            <connect_order>6</connect_order>
+            <COMPONENTS>
+            </COMPONENTS>
+        </ONOSrest2>
+
+        <ONOSrest3>
+            <host>OC3</host>
+            <port>8181</port>
+            <user>onos</user>
+            <password>rocks</password>
+            <type>OnosRestDriver</type>
+            <connect_order>6</connect_order>
+            <COMPONENTS>
+            </COMPONENTS>
+        </ONOSrest3>
+
+        <Mininet1>
+            <host>OCN</host>
+            <user>sdn</user>
+            <password>rocks</password>
+            <type>MininetCliDriver</type>
+            <connect_order>7</connect_order>
+            <COMPONENTS>
+                <home>~/mininet/custom/</home>
+            </COMPONENTS>
+        </Mininet1>
+
+    </COMPONENT>
+</TOPOLOGY>
diff --git a/TestON/tests/USECASE/SegmentRouting/SROnosFailure/__init__.py b/TestON/tests/USECASE/SegmentRouting/SROnosFailure/__init__.py
new file mode 100755
index 0000000..e69de29
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SROnosFailure/__init__.py