Update HA Tests

    - Add configuration for number of nodes to backup flow rules to
    - Add some more conditions to exit the test early when failing
    - Turn of flow objective intent compiler for more stability

Change-Id: Ie250c8e214b9f44d55abec95b0ef29839f0a4579
diff --git a/TestON/tests/HA/HAstopNodes/HAstopNodes.params b/TestON/tests/HA/HAstopNodes/HAstopNodes.params
index dd035f5..a83e878 100644
--- a/TestON/tests/HA/HAstopNodes/HAstopNodes.params
+++ b/TestON/tests/HA/HAstopNodes/HAstopNodes.params
@@ -24,8 +24,11 @@
     <apps></apps>
     <ONOS_Configuration>
         <org.onosproject.net.intent.impl.compiler.IntentConfigurableRegistrator>
-            <useFlowObjectives>true</useFlowObjectives>
+            <useFlowObjectives>false</useFlowObjectives>
         </org.onosproject.net.intent.impl.compiler.IntentConfigurableRegistrator>
+        <org.onosproject.store.flow.impl.DistributedFlowRuleStore>
+            <backupCount>3</backupCount>
+        </org.onosproject.store.flow.impl.DistributedFlowRuleStore>
     </ONOS_Configuration>
     <ENV>
         <cellName>HA</cellName>
diff --git a/TestON/tests/HA/HAstopNodes/HAstopNodes.py b/TestON/tests/HA/HAstopNodes/HAstopNodes.py
index 7617a7a..9b0f7a7 100644
--- a/TestON/tests/HA/HAstopNodes/HAstopNodes.py
+++ b/TestON/tests/HA/HAstopNodes/HAstopNodes.py
@@ -1808,6 +1808,27 @@
                                  onpass="ONOS nodes stopped successfully",
                                  onfail="ONOS nodes NOT successfully stopped" )
 
+        main.step( "Checking ONOS nodes" )
+        nodeResults = utilities.retry( main.HA.nodesCheck,
+                                       False,
+                                       args=[main.activeNodes],
+                                       sleep=15,
+                                       attempts=5 )
+
+        utilities.assert_equals( expect=True, actual=nodeResults,
+                                 onpass="Nodes check successful",
+                                 onfail="Nodes check NOT successful" )
+
+        if not nodeResults:
+            for i in main.activeNodes:
+                cli = main.CLIs[i]
+                main.log.debug( "{} components not ACTIVE: \n{}".format(
+                    cli.name,
+                    cli.sendline( "scr:list | grep -v ACTIVE" ) ) )
+            main.log.error( "Failed to start ONOS, stopping test" )
+            main.cleanup()
+            main.exit()
+
     def CASE62( self, main ):
         """
         The bring up stopped nodes
@@ -1866,8 +1887,27 @@
         # protocol has had time to work
         main.restartTime = time.time() - restartTime
         main.log.debug( "Restart time: " + str( main.restartTime ) )
-        # TODO: MAke this configurable. Also, we are breaking the above timer
-        time.sleep( 60 )
+
+        main.step( "Checking ONOS nodes" )
+        nodeResults = utilities.retry( main.HA.nodesCheck,
+                                       False,
+                                       args=[main.activeNodes],
+                                       sleep=15,
+                                       attempts=5 )
+
+        utilities.assert_equals( expect=True, actual=nodeResults,
+                                 onpass="Nodes check successful",
+                                 onfail="Nodes check NOT successful" )
+
+        if not nodeResults:
+            for i in main.activeNodes:
+                cli = main.CLIs[i]
+                main.log.debug( "{} components not ACTIVE: \n{}".format(
+                    cli.name,
+                    cli.sendline( "scr:list | grep -v ACTIVE" ) ) )
+            main.log.error( "Failed to start ONOS, stopping test" )
+            main.cleanup()
+            main.exit()
         node = main.activeNodes[0]
         main.log.debug( main.CLIs[node].nodes( jsonFormat=False ) )
         main.log.debug( main.CLIs[node].leaders( jsonFormat=False ) )
@@ -2623,6 +2663,10 @@
                     main.CLIs[i].name,
                     main.CLIs[i].sendline( "scr:list | grep -v ACTIVE" ) ) )
 
+        if not topoResult:
+            main.cleanup()
+            main.exit()
+
     def CASE9( self, main ):
         """
         Link s3-s28 down