Merge branch 'HA' of https://github.com/opennetworkinglab/ONLabTest into devl/merge

Conflicts:
	TestON/core/teston.py
	TestON/drivers/common/cli/emulator/mininetclidriver.py
	TestON/drivers/common/cli/onosdriver.py
	TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py
	TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.py
diff --git a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py
index 8649f98..3e7de57 100644
--- a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py
+++ b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py
@@ -31,14 +31,18 @@
         CASE1 is to compile ONOS and push it to the test machines
 
         Startup sequence:
-        git pull
-        mvn clean install
-        onos-package
         cell <name>
         onos-verify-cell
         NOTE: temporary - onos-remove-raft-logs
+        onos-uninstall
+        start mininet
+        git pull
+        mvn clean install
+        onos-package
         onos-install -f
         onos-wait-for-start
+        start cli sessions
+        start tcpdump
         """
         main.log.report(
             "ONOS HA test: Restart minority of ONOS nodes - initialization" )
@@ -109,8 +113,7 @@
 
         main.step( "Compiling the latest version of ONOS" )
         if PULLCODE:
-            # TODO Configure branch in params
-            main.step( "Git checkout and pull master" )
+            main.step( "Git checkout and pull " + gitBranch )
             main.ONOSbench.gitCheckout( gitBranch )
             gitPullResult = main.ONOSbench.gitPull()
 
@@ -145,7 +148,6 @@
             and onos7InstallResult
 
         main.step( "Checking if ONOS is up yet" )
-        # TODO check bundle:list?
         for i in range( 2 ):
             onos1Isup = main.ONOSbench.isup( ONOS1Ip )
             if not onos1Isup:
@@ -209,8 +211,8 @@
                         and onosIsupResult and cliResults )
 
         utilities.assert_equals( expect=main.TRUE, actual=case1Result,
-                                onpass="Test startup successful",
-                                onfail="Test startup NOT successful" )
+                                 onpass="Test startup successful",
+                                 onfail="Test startup NOT successful" )
 
         if case1Result == main.FALSE:
             main.cleanup()
@@ -266,118 +268,131 @@
         # Manually assign mastership to the controller we want
         roleCall = main.TRUE
         roleCheck = main.TRUE
-
-        # Assign switch
-        deviceId = main.ONOScli1.getDevice( "1000" ).get( 'id' )
-        roleCall = roleCall and main.ONOScli1.deviceRole(
-            deviceId,
-            ONOS1Ip )
-        # Check assignment
-        if ONOS1Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
-            roleCheck = roleCheck and main.TRUE
-        else:
-            roleCheck = roleCheck and main.FALSE
-
-        # Assign switch
-        deviceId = main.ONOScli1.getDevice( "2800" ).get( 'id' )
-        roleCall = roleCall and main.ONOScli1.deviceRole(
-            deviceId,
-            ONOS1Ip )
-        # Check assignment
-        if ONOS1Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
-            roleCheck = roleCheck and main.TRUE
-        else:
-            roleCheck = roleCheck and main.FALSE
-
-        # Assign switch
-        deviceId = main.ONOScli1.getDevice( "2000" ).get( 'id' )
-        roleCall = roleCall and main.ONOScli1.deviceRole(
-            deviceId,
-            ONOS2Ip )
-        # Check assignment
-        if ONOS2Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
-            roleCheck = roleCheck and main.TRUE
-        else:
-            roleCheck = roleCheck and main.FALSE
-
-        # Assign switch
-        deviceId = main.ONOScli1.getDevice( "3000" ).get( 'id' )
-        roleCall = roleCall and main.ONOScli1.deviceRole(
-            deviceId,
-            ONOS2Ip )
-        # Check assignment
-        if ONOS2Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
-            roleCheck = roleCheck and main.TRUE
-        else:
-            roleCheck = roleCheck and main.FALSE
-
-        # Assign switch
-        deviceId = main.ONOScli1.getDevice( "5000" ).get( 'id' )
-        roleCall = roleCall and main.ONOScli1.deviceRole(
-            deviceId,
-            ONOS3Ip )
-        # Check assignment
-        if ONOS3Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
-            roleCheck = roleCheck and main.TRUE
-        else:
-            roleCheck = roleCheck and main.FALSE
-
-        # Assign switch
-        deviceId = main.ONOScli1.getDevice( "6000" ).get( 'id' )
-        roleCall = roleCall and main.ONOScli1.deviceRole(
-            deviceId,
-            ONOS3Ip )
-        # Check assignment
-        if ONOS3Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
-            roleCheck = roleCheck and main.TRUE
-        else:
-            roleCheck = roleCheck and main.FALSE
-
-        # Assign switch
-        deviceId = main.ONOScli1.getDevice( "3004" ).get( 'id' )
-        roleCall = roleCall and main.ONOScli1.deviceRole(
-            deviceId,
-            ONOS4Ip )
-        # Check assignment
-        if ONOS4Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
-            roleCheck = roleCheck and main.TRUE
-        else:
-            roleCheck = roleCheck and main.FALSE
-
-        for i in range( 8, 18 ):
-            dpid = '3' + str( i ).zfill( 3 )
-            deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
+        try:
+            # Assign switch
+            deviceId = main.ONOScli1.getDevice( "1000" ).get( 'id' )
+            assert deviceId, "No device id for s1 in ONOS"
             roleCall = roleCall and main.ONOScli1.deviceRole(
                 deviceId,
-                ONOS5Ip )
+                ONOS1Ip )
             # Check assignment
-            if ONOS5Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
+            if ONOS1Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
                 roleCheck = roleCheck and main.TRUE
             else:
                 roleCheck = roleCheck and main.FALSE
 
-        deviceId = main.ONOScli1.getDevice( "6007" ).get( 'id' )
-        roleCall = roleCall and main.ONOScli1.deviceRole(
-            deviceId,
-            ONOS6Ip )
-        # Check assignment
-        if ONOS6Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
-            roleCheck = roleCheck and main.TRUE
-        else:
-            roleCheck = roleCheck and main.FALSE
-
-        for i in range( 18, 28 ):
-            dpid = '6' + str( i ).zfill( 3 )
-            deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
+            # Assign switch
+            deviceId = main.ONOScli1.getDevice( "2800" ).get( 'id' )
+            assert deviceId, "No device id for s28 in ONOS"
             roleCall = roleCall and main.ONOScli1.deviceRole(
                 deviceId,
-                ONOS7Ip )
+                ONOS1Ip )
             # Check assignment
-            if ONOS7Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
+            if ONOS1Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
                 roleCheck = roleCheck and main.TRUE
             else:
                 roleCheck = roleCheck and main.FALSE
 
+            # Assign switch
+            deviceId = main.ONOScli1.getDevice( "2000" ).get( 'id' )
+            assert deviceId, "No device id for s2 in ONOS"
+            roleCall = roleCall and main.ONOScli1.deviceRole(
+                deviceId,
+                ONOS2Ip )
+            # Check assignment
+            if ONOS2Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
+                roleCheck = roleCheck and main.TRUE
+            else:
+                roleCheck = roleCheck and main.FALSE
+
+            # Assign switch
+            deviceId = main.ONOScli1.getDevice( "3000" ).get( 'id' )
+            assert deviceId, "No device id for s3 in ONOS"
+            roleCall = roleCall and main.ONOScli1.deviceRole(
+                deviceId,
+                ONOS2Ip )
+            # Check assignment
+            if ONOS2Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
+                roleCheck = roleCheck and main.TRUE
+            else:
+                roleCheck = roleCheck and main.FALSE
+
+            # Assign switch
+            deviceId = main.ONOScli1.getDevice( "5000" ).get( 'id' )
+            assert deviceId, "No device id for s5 in ONOS"
+            roleCall = roleCall and main.ONOScli1.deviceRole(
+                deviceId,
+                ONOS3Ip )
+            # Check assignment
+            if ONOS3Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
+                roleCheck = roleCheck and main.TRUE
+            else:
+                roleCheck = roleCheck and main.FALSE
+
+            # Assign switch
+            deviceId = main.ONOScli1.getDevice( "6000" ).get( 'id' )
+            assert deviceId, "No device id for s6 in ONOS"
+            roleCall = roleCall and main.ONOScli1.deviceRole(
+                deviceId,
+                ONOS3Ip )
+            # Check assignment
+            if ONOS3Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
+                roleCheck = roleCheck and main.TRUE
+            else:
+                roleCheck = roleCheck and main.FALSE
+
+            # Assign switch
+            deviceId = main.ONOScli1.getDevice( "3004" ).get( 'id' )
+            assert deviceId, "No device id for s4 in ONOS"
+            roleCall = roleCall and main.ONOScli1.deviceRole(
+                deviceId,
+                ONOS4Ip )
+            # Check assignment
+            if ONOS4Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
+                roleCheck = roleCheck and main.TRUE
+            else:
+                roleCheck = roleCheck and main.FALSE
+
+            for i in range( 8, 18 ):
+                dpid = '3' + str( i ).zfill( 3 )
+                deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
+                assert deviceId, "No device id for s%i in ONOS" % i
+                roleCall = roleCall and main.ONOScli1.deviceRole(
+                    deviceId,
+                    ONOS5Ip )
+                # Check assignment
+                if ONOS5Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
+                    roleCheck = roleCheck and main.TRUE
+                else:
+                    roleCheck = roleCheck and main.FALSE
+
+            deviceId = main.ONOScli1.getDevice( "6007" ).get( 'id' )
+            assert deviceId, "No device id for s7 in ONOS"
+            roleCall = roleCall and main.ONOScli1.deviceRole(
+                deviceId,
+                ONOS6Ip )
+            # Check assignment
+            if ONOS6Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
+                roleCheck = roleCheck and main.TRUE
+            else:
+                roleCheck = roleCheck and main.FALSE
+
+            for i in range( 18, 28 ):
+                dpid = '6' + str( i ).zfill( 3 )
+                deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
+                assert deviceId, "No device id for s%i in ONOS" % i
+                roleCall = roleCall and main.ONOScli1.deviceRole(
+                    deviceId,
+                    ONOS7Ip )
+                # Check assignment
+                if ONOS7Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
+                    roleCheck = roleCheck and main.TRUE
+                else:
+                    roleCheck = roleCheck and main.FALSE
+        except ( AttributeError, AssertionError ):
+            main.log.exception( "Something is wrong with ONOS device view" )
+            main.log.info( main.ONOScli1.devices() )
+
         utilities.assert_equals(
             expect=main.TRUE,
             actual=roleCall,
@@ -443,8 +458,11 @@
         time.sleep( 10 )
 
         main.step( "Add  host intents" )
+        intentIds = []
         # TODO:  move the host numbers to params
+        #        Maybe look at all the paths we ping?
         intentAddResult = True
+        hostResult = main.TRUE
         for i in range( 8, 18 ):
             main.log.info( "Adding host intent between h" + str( i ) +
                            " and h" + str( i + 10 ) )
@@ -461,34 +479,111 @@
                 host1Id = host1Dict.get( 'id', None )
                 host2Id = host2Dict.get( 'id', None )
             if host1Id and host2Id:
-                #Changed onos node to test something
-                tmpResult = main.ONOScli4.addHostIntent(
+                # Changed onos node to test something
+                tmpId = main.ONOScli4.addHostIntent(
                     host1Id,
                     host2Id )
+                if tmpId:
+                    main.log.info( "Added intent with id: " + tmpId )
+                    intentIds.append( tmpId )
+                else:
+                    main.log.error( "addHostIntent reutrned None" )
             else:
                 main.log.error( "Error, getHost() failed" )
                 main.log.warn( json.dumps( json.loads( main.ONOScli1.hosts() ),
                                            sort_keys=True,
                                            indent=4,
                                            separators=( ',', ': ' ) ) )
-                tmpResult = main.FALSE
-            intentAddResult = bool( pingResult and intentAddResult
-                                     and tmpResult )
-            # TODO Check that intents were added?
+                hostResult = main.FALSE
+        onosIds = main.ONOScli1.getAllIntentsId()
+        main.log.info( "Submitted intents: " + str( intentIds ) )
+        main.log.info( "Intents in ONOS: " + str( onosIds ) )
+        for intent in intentIds:
+            if intent in onosIds:
+                pass  # intent submitted is still in onos
+            else:
+                intentAddResult = False
         # Print the intent states
-        intents = main.ONOScli1.intents( )
+        intents = main.ONOScli1.intents()
         intentStates = []
+        installedCheck = True 
+        main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
+        count = 0
         for intent in json.loads( intents ):  # Iter through intents of a node
-            intentStates.append( intent.get( 'state', None ) )
-        out = [ (i, intentStates.count( i ) ) for i in set( intentStates ) ]
-        main.log.info( dict( out ) )
-
+            state = intent.get( 'state', None )
+            if "INSTALLED" not in state:
+                installedCheck = False
+            intentId = intent.get( 'id', None )
+            intentStates.append( ( intentId, state ) )
+        # add submitted intents not in the store
+        tmplist = [ i for i, s in intentStates ]
+        missingIntents = False
+        for i in intentIds:
+            if i not in tmplist:
+                intentStates.append( ( i, " - " ) )
+                missingIntents = True
+        intentStates.sort()
+        for i, s in intentStates:
+            count += 1
+            main.log.info( "%-6s%-15s%-15s" %
+                           ( str( count ), str( i ), str( s ) ) )
+        main.ONOScli1.leaders()
+        main.ONOScli1.partitions()
+        # for node in nodes:
+        #     node.pendingMap()
+        pendingMap = main.ONOScli1.pendingMap()
+        main.ONOScli2.pendingMap()
+        main.ONOScli3.pendingMap()
+        main.ONOScli4.pendingMap()
+        main.ONOScli5.pendingMap()
+        main.ONOScli6.pendingMap()
+        main.ONOScli7.pendingMap()
+        intentAddResult = bool( pingResult and hostResult and intentAddResult
+                                and not missingIntents and installedCheck )
         utilities.assert_equals(
             expect=True,
             actual=intentAddResult,
             onpass="Pushed host intents to ONOS",
             onfail="Error in pushing host intents to ONOS" )
-        # TODO Check if intents all exist in datastore
+
+        if not intentAddResult or "key" in pendingMap:
+            import time
+            installedCheck = True
+            main.log.info( "Sleeping 60 seconds to see if intents are found" )
+            time.sleep( 60 )
+            onosIds = main.ONOScli1.getAllIntentsId()
+            main.log.info( "Submitted intents: " + str( intentIds ) )
+            main.log.info( "Intents in ONOS: " + str( onosIds ) )
+            # Print the intent states
+            intents = main.ONOScli1.intents()
+            intentStates = []
+            main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
+            count = 0
+            for intent in json.loads( intents ):
+                # Iter through intents of a node
+                state = intent.get( 'state', None )
+                if "INSTALLED" not in state:
+                    installedCheck = False
+                intentId = intent.get( 'id', None )
+                intentStates.append( ( intentId, state ) )
+            # add submitted intents not in the store
+            tmplist = [ i for i, s in intentStates ]
+            for i in intentIds:
+                if i not in tmplist:
+                    intentStates.append( ( i, " - " ) )
+            intentStates.sort()
+            for i, s in intentStates:
+                count += 1
+                main.log.info( "%-6s%-15s%-15s" %
+                               ( str( count ), str( i ), str( s ) ) )
+            main.ONOScli1.leaders()
+            main.ONOScli1.pendingMap()
+            main.ONOScli2.pendingMap()
+            main.ONOScli3.pendingMap()
+            main.ONOScli4.pendingMap()
+            main.ONOScli5.pendingMap()
+            main.ONOScli6.pendingMap()
+            main.ONOScli7.pendingMap()
 
     def CASE4( self, main ):
         """
@@ -512,7 +607,7 @@
         if PingResult == main.FALSE:
             main.log.report(
                 "Intents have not been installed correctly, pings failed." )
-            #TODO: pretty print
+            # TODO: pretty print
             main.log.warn( "ONSO1 intents: " )
             main.log.warn( json.dumps( json.loads( main.ONOScli1.intents() ),
                                        sort_keys=True,
@@ -527,6 +622,64 @@
             onpass="Intents have been installed correctly and pings work",
             onfail="Intents have not been installed correctly, pings failed." )
 
+        installedCheck = True
+        if PingResult is not main.TRUE:
+            # Print the intent states
+            intents = main.ONOScli1.intents()
+            intentStates = []
+            main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
+            count = 0
+            # Iter through intents of a node
+            for intent in json.loads( intents ):
+                state = intent.get( 'state', None )
+                if "INSTALLED" not in state:
+                    installedCheck = False
+                intentId = intent.get( 'id', None )
+                intentStates.append( ( intentId, state ) )
+            intentStates.sort()
+            for i, s in intentStates:
+                count += 1
+                main.log.info( "%-6s%-15s%-15s" %
+                               ( str( count ), str( i ), str( s ) ) )
+            main.ONOScli1.leaders()
+            main.ONOScli1.partitions()
+            main.ONOScli1.pendingMap()
+            main.ONOScli2.pendingMap()
+            main.ONOScli3.pendingMap()
+            main.ONOScli4.pendingMap()
+            main.ONOScli5.pendingMap()
+            main.ONOScli6.pendingMap()
+            main.ONOScli7.pendingMap()
+        if not installedCheck:
+            main.log.info( "Waiting 60 seconds to see if intent states change" )
+            time.sleep( 60 )
+            # Print the intent states
+            intents = main.ONOScli1.intents()
+            intentStates = []
+            main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
+            count = 0
+            # Iter through intents of a node
+            for intent in json.loads( intents ):
+                state = intent.get( 'state', None )
+                if "INSTALLED" not in state:
+                    installedCheck = False
+                intentId = intent.get( 'id', None )
+                intentStates.append( ( intentId, state ) )
+            intentStates.sort()
+            for i, s in intentStates:
+                count += 1
+                main.log.info( "%-6s%-15s%-15s" %
+                               ( str( count ), str( i ), str( s ) ) )
+            main.ONOScli1.leaders()
+            main.ONOScli1.partitions()
+            main.ONOScli1.pendingMap()
+            main.ONOScli2.pendingMap()
+            main.ONOScli3.pendingMap()
+            main.ONOScli4.pendingMap()
+            main.ONOScli5.pendingMap()
+            main.ONOScli6.pendingMap()
+            main.ONOScli7.pendingMap()
+
     def CASE5( self, main ):
         """
         Reading state of ONOS
@@ -789,29 +942,39 @@
         main.step( "Get the flows from each controller" )
         global flowState
         flowState = []
-        ONOS1Flows = main.ONOScli1.flows( jsonFormat=True )
-        ONOS2Flows = main.ONOScli2.flows( jsonFormat=True )
-        ONOS3Flows = main.ONOScli3.flows( jsonFormat=True )
-        ONOS4Flows = main.ONOScli4.flows( jsonFormat=True )
-        ONOS5Flows = main.ONOScli5.flows( jsonFormat=True )
-        ONOS6Flows = main.ONOScli6.flows( jsonFormat=True )
-        ONOS7Flows = main.ONOScli7.flows( jsonFormat=True )
-        ONOS1FlowsJson = json.loads( ONOS1Flows )
-        ONOS2FlowsJson = json.loads( ONOS2Flows )
-        ONOS3FlowsJson = json.loads( ONOS3Flows )
-        ONOS4FlowsJson = json.loads( ONOS4Flows )
-        ONOS5FlowsJson = json.loads( ONOS5Flows )
-        ONOS6FlowsJson = json.loads( ONOS6Flows )
-        ONOS7FlowsJson = json.loads( ONOS7Flows )
         flowCheck = main.FALSE
-        if "Error" in ONOS1Flows or not ONOS1Flows\
-                or "Error" in ONOS2Flows or not ONOS2Flows\
-                or "Error" in ONOS3Flows or not ONOS3Flows\
-                or "Error" in ONOS4Flows or not ONOS4Flows\
-                or "Error" in ONOS5Flows or not ONOS5Flows\
-                or "Error" in ONOS6Flows or not ONOS6Flows\
-                or "Error" in ONOS7Flows or not ONOS7Flows:
-            main.log.report( "Error in getting ONOS intents" )
+        try:
+            ONOS1Flows = main.ONOScli1.flows( jsonFormat=True )
+            ONOS2Flows = main.ONOScli2.flows( jsonFormat=True )
+            ONOS3Flows = main.ONOScli3.flows( jsonFormat=True )
+            ONOS4Flows = main.ONOScli4.flows( jsonFormat=True )
+            ONOS5Flows = main.ONOScli5.flows( jsonFormat=True )
+            ONOS6Flows = main.ONOScli6.flows( jsonFormat=True )
+            ONOS7Flows = main.ONOScli7.flows( jsonFormat=True )
+            assert ONOS1Flows, "ONOS1 Flows should not be empty"
+            assert ONOS2Flows, "ONOS2 Flows should not be empty"
+            assert ONOS3Flows, "ONOS3 Flows should not be empty"
+            assert ONOS4Flows, "ONOS4 Flows should not be empty"
+            assert ONOS5Flows, "ONOS5 Flows should not be empty"
+            assert ONOS6Flows, "ONOS6 Flows should not be empty"
+            assert ONOS7Flows, "ONOS7 Flows should not be empty"
+            assert "Error" not in ONOS1Flows, "ONOS1 Flows contains 'Error'"
+            assert "Error" not in ONOS2Flows, "ONOS2 Flows contains 'Error'"
+            assert "Error" not in ONOS3Flows, "ONOS3 Flows contains 'Error'"
+            assert "Error" not in ONOS4Flows, "ONOS4 Flows contains 'Error'"
+            assert "Error" not in ONOS5Flows, "ONOS5 Flows contains 'Error'"
+            assert "Error" not in ONOS6Flows, "ONOS6 Flows contains 'Error'"
+            assert "Error" not in ONOS7Flows, "ONOS7 Flows contains 'Error'"
+            ONOS1FlowsJson = json.loads( ONOS1Flows )
+            ONOS2FlowsJson = json.loads( ONOS2Flows )
+            ONOS3FlowsJson = json.loads( ONOS3Flows )
+            ONOS4FlowsJson = json.loads( ONOS4Flows )
+            ONOS5FlowsJson = json.loads( ONOS5Flows )
+            ONOS6FlowsJson = json.loads( ONOS6Flows )
+            ONOS7FlowsJson = json.loads( ONOS7Flows )
+        except ( ValueError, AssertionError ):  # From json.loads, or asserts
+            main.log.exception( "One or more 'flows' responses from " +
+                                "ONOS couldn't be decoded." )
             main.log.warn( "ONOS1 flows repsponse: " + ONOS1Flows )
             main.log.warn( "ONOS2 flows repsponse: " + ONOS2Flows )
             main.log.warn( "ONOS3 flows repsponse: " + ONOS3Flows )
@@ -819,38 +982,48 @@
             main.log.warn( "ONOS5 flows repsponse: " + ONOS5Flows )
             main.log.warn( "ONOS6 flows repsponse: " + ONOS6Flows )
             main.log.warn( "ONOS7 flows repsponse: " + ONOS7Flows )
-        elif len( ONOS1FlowsJson ) == len( ONOS2FlowsJson )\
-                and len( ONOS1FlowsJson ) == len( ONOS3FlowsJson )\
-                and len( ONOS1FlowsJson ) == len( ONOS4FlowsJson )\
-                and len( ONOS1FlowsJson ) == len( ONOS5FlowsJson )\
-                and len( ONOS1FlowsJson ) == len( ONOS6FlowsJson )\
-                and len( ONOS1FlowsJson ) == len( ONOS7FlowsJson ):
+        else:  # No exceptions
+            if len( ONOS1FlowsJson ) == len( ONOS2FlowsJson )\
+                    and len( ONOS1FlowsJson ) == len( ONOS3FlowsJson )\
+                    and len( ONOS1FlowsJson ) == len( ONOS4FlowsJson )\
+                    and len( ONOS1FlowsJson ) == len( ONOS5FlowsJson )\
+                    and len( ONOS1FlowsJson ) == len( ONOS6FlowsJson )\
+                    and len( ONOS1FlowsJson ) == len( ONOS7FlowsJson ):
                 # TODO: Do a better check, maybe compare flows on switches?
-            flowState = ONOS1Flows
-            flowCheck = main.TRUE
-            main.log.report( "Flow count is consistent across all ONOS nodes" )
-        else:
-            main.log.warn( "ONOS1 flows: " +
-                           json.dumps( ONOS1FlowsJson, sort_keys=True,
-                                       indent=4, separators=( ',', ': ' ) ) )
-            main.log.warn( "ONOS2 flows: " +
-                           json.dumps( ONOS2FlowsJson, sort_keys=True,
-                                       indent=4, separators=( ',', ': ' ) ) )
-            main.log.warn( "ONOS3 flows: " +
-                           json.dumps( ONOS3FlowsJson, sort_keys=True,
-                                       indent=4, separators=( ',', ': ' ) ) )
-            main.log.warn( "ONOS4 flows: " +
-                           json.dumps( ONOS4FlowsJson, sort_keys=True,
-                                       indent=4, separators=( ',', ': ' ) ) )
-            main.log.warn( "ONOS5 flows: " +
-                           json.dumps( ONOS5FlowsJson, sort_keys=True,
-                                       indent=4, separators=( ',', ': ' ) ) )
-            main.log.warn( "ONOS6 flows: " +
-                           json.dumps( ONOS6FlowsJson, sort_keys=True,
-                                       indent=4, separators=( ',', ': ' ) ) )
-            main.log.warn( "ONOS7 flows: " +
-                           json.dumps( ONOS7FlowsJson, sort_keys=True,
-                                       indent=4, separators=( ',', ': ' ) ) )
+                # NOTE Possible issue with this not always being set?
+                flowState = ONOS1Flows
+                flowCheck = main.TRUE
+                main.log.report( "Flow count is consistent across all" +
+                                 " ONOS nodes" )
+            else:
+                main.log.warn( "ONOS1 flows: " +
+                               json.dumps( ONOS1FlowsJson, sort_keys=True,
+                                           indent=4,
+                                           separators=( ',', ': ' ) ) )
+                main.log.warn( "ONOS2 flows: " +
+                               json.dumps( ONOS2FlowsJson, sort_keys=True,
+                                           indent=4,
+                                           separators=( ',', ': ' ) ) )
+                main.log.warn( "ONOS3 flows: " +
+                               json.dumps( ONOS3FlowsJson, sort_keys=True,
+                                           indent=4,
+                                           separators=( ',', ': ' ) ) )
+                main.log.warn( "ONOS4 flows: " +
+                               json.dumps( ONOS4FlowsJson, sort_keys=True,
+                                           indent=4,
+                                           separators=( ',', ': ' ) ) )
+                main.log.warn( "ONOS5 flows: " +
+                               json.dumps( ONOS5FlowsJson, sort_keys=True,
+                                           indent=4,
+                                           separators=( ',', ': ' ) ) )
+                main.log.warn( "ONOS6 flows: " +
+                               json.dumps( ONOS6FlowsJson, sort_keys=True,
+                                           indent=4,
+                                           separators=( ',', ': ' ) ) )
+                main.log.warn( "ONOS7 flows: " +
+                               json.dumps( ONOS7FlowsJson, sort_keys=True,
+                                           indent=4,
+                                           separators=( ',', ': ' ) ) )
         utilities.assert_equals(
             expect=main.TRUE,
             actual=flowCheck,
@@ -862,7 +1035,9 @@
         flows = []
         for i in range( 1, 29 ):
             flows.append( main.Mininet2.getFlowTable( 1.3, "s" + str( i ) ) )
-
+        if flowCheck == main.FALSE:
+            for table in flows:
+                main.log.warn( table )
         # TODO: Compare switch flow tables with ONOS flow tables
 
         main.step( "Start continuous pings" )
@@ -936,13 +1111,13 @@
         devices.append( main.ONOScli6.devices() )
         devices.append( main.ONOScli7.devices() )
         hosts = []
-        hosts.append( main.ONOScli1.hosts() )
-        hosts.append( main.ONOScli2.hosts() )
-        hosts.append( main.ONOScli3.hosts() )
-        hosts.append( main.ONOScli4.hosts() )
-        hosts.append( main.ONOScli5.hosts() )
-        hosts.append( main.ONOScli6.hosts() )
-        hosts.append( main.ONOScli7.hosts() )
+        hosts.append( json.loads( main.ONOScli1.hosts() ) )
+        hosts.append( json.loads( main.ONOScli2.hosts() ) )
+        hosts.append( json.loads( main.ONOScli3.hosts() ) )
+        hosts.append( json.loads( main.ONOScli4.hosts() ) )
+        hosts.append( json.loads( main.ONOScli5.hosts() ) )
+        hosts.append( json.loads( main.ONOScli6.hosts() ) )
+        hosts.append( json.loads( main.ONOScli7.hosts() ) )
         ports = []
         ports.append( main.ONOScli1.ports() )
         ports.append( main.ONOScli2.ports() )
@@ -996,6 +1171,21 @@
             onpass="Hosts view is consistent across all ONOS nodes",
             onfail="ONOS nodes have different views of hosts" )
 
+        ipResult = main.TRUE
+        for controller in range( 0, len( hosts ) ):
+            controllerStr = str( controller + 1 )
+            for host in hosts[ controller ]:
+                if host.get( 'ips', [] ) == []:
+                    main.log.error(
+                        "DEBUG:Error with host ips on controller" +
+                        controllerStr + ": " + str( host ) )
+                    ipResult = main.FALSE
+        utilities.assert_equals(
+            expect=main.TRUE,
+            actual=ipResult,
+            onpass="The ips of the hosts aren't empty",
+            onfail="The ip of at least one host is missing" )
+
         # Strongly connected clusters of devices
         consistentClustersResult = main.TRUE
         for controller in range( len( clusters ) ):
@@ -1022,13 +1212,14 @@
             onfail="ONOS nodes have different views of clusters" )
         # there should always only be one cluster
         numClusters = len( json.loads( clusters[ 0 ] ) )
+        clusterResults = main.FALSE
+        if numClusters == 1:
+            clusterResults = main.TRUE
         utilities.assert_equals(
             expect=1,
             actual=numClusters,
             onpass="ONOS shows 1 SCC",
-            onfail="ONOS shows " +
-            str( numClusters ) +
-            " SCCs" )
+            onfail="ONOS shows " + str( numClusters ) + " SCCs" )
 
         main.step( "Comparing ONOS topology to MN" )
         devicesResults = main.TRUE
@@ -1044,11 +1235,11 @@
             else:
                 currentDevicesResult = main.FALSE
             utilities.assert_equals( expect=main.TRUE,
-                                    actual=currentDevicesResult,
-                                    onpass="ONOS" + controllerStr +
-                                    " Switches view is correct",
-                                    onfail="ONOS" + controllerStr +
-                                    " Switches view is incorrect" )
+                                     actual=currentDevicesResult,
+                                     onpass="ONOS" + controllerStr +
+                                     " Switches view is correct",
+                                     onfail="ONOS" + controllerStr +
+                                     " Switches view is incorrect" )
 
             if ports[ controller ] or "Error" not in ports[ controller ]:
                 currentPortsResult = main.Mininet1.comparePorts(
@@ -1058,11 +1249,11 @@
             else:
                 currentPortsResult = main.FALSE
             utilities.assert_equals( expect=main.TRUE,
-                                    actual=currentPortsResult,
-                                    onpass="ONOS" + controllerStr +
-                                    " ports view is correct",
-                                    onfail="ONOS" + controllerStr +
-                                    " ports view is incorrect" )
+                                     actual=currentPortsResult,
+                                     onpass="ONOS" + controllerStr +
+                                     " ports view is correct",
+                                     onfail="ONOS" + controllerStr +
+                                     " ports view is incorrect" )
 
             if links[ controller ] or "Error" not in links[ controller ]:
                 currentLinksResult = main.Mininet1.compareLinks(
@@ -1072,28 +1263,29 @@
             else:
                 currentLinksResult = main.FALSE
             utilities.assert_equals( expect=main.TRUE,
-                                    actual=currentLinksResult,
-                                    onpass="ONOS" + controllerStr +
-                                    " links view is correct",
-                                    onfail="ONOS" + controllerStr +
-                                    " links view is incorrect" )
+                                     actual=currentLinksResult,
+                                     onpass="ONOS" + controllerStr +
+                                     " links view is correct",
+                                     onfail="ONOS" + controllerStr +
+                                     " links view is incorrect" )
 
             devicesResults = devicesResults and currentDevicesResult
             portsResults = portsResults and currentPortsResult
             linksResults = linksResults and currentLinksResult
 
         topoResult = devicesResults and portsResults and linksResults\
-            and consistentHostsResult and consistentClustersResult
+                     and consistentHostsResult and consistentClustersResult\
+                     and clusterResults and ipResult
         utilities.assert_equals( expect=main.TRUE, actual=topoResult,
-                                onpass="Topology Check Test successful",
-                                onfail="Topology Check Test NOT successful" )
+                                 onpass="Topology Check Test successful",
+                                 onfail="Topology Check Test NOT successful" )
 
         finalAssert = main.TRUE
         finalAssert = finalAssert and topoResult and flowCheck \
-            and intentCheck and consistentMastership and rolesNotNull
+                      and intentCheck and consistentMastership and rolesNotNull
         utilities.assert_equals( expect=main.TRUE, actual=finalAssert,
-                                onpass="State check successful",
-                                onfail="State check NOT successful" )
+                                 onpass="State check successful",
+                                 onfail="State check NOT successful" )
 
     def CASE6( self, main ):
         """
@@ -1130,8 +1322,8 @@
         main.restartTime = time.time()
         caseResults = main.TRUE and onosIsupResult and cliResults
         utilities.assert_equals( expect=main.TRUE, actual=caseResults,
-                                onpass="ONOS restart successful",
-                                onfail="ONOS restart NOT successful" )
+                                 onpass="ONOS restart successful",
+                                 onfail="ONOS restart NOT successful" )
 
     def CASE7( self, main ):
         """
@@ -1259,147 +1451,133 @@
         # NOTE: we expect mastership to change on controller failure
         mastershipCheck = consistentMastership
 
-        while True:
-            whileTime = time.time() - main.restartTime
-            # Gossip store
-            main.step( "Get the intents and compare across all nodes" )
-            ONOS1Intents = main.ONOScli1.intents( jsonFormat=True )
-            ONOS2Intents = main.ONOScli2.intents( jsonFormat=True )
-            ONOS3Intents = main.ONOScli3.intents( jsonFormat=True )
-            ONOS4Intents = main.ONOScli4.intents( jsonFormat=True )
-            ONOS5Intents = main.ONOScli5.intents( jsonFormat=True )
-            ONOS6Intents = main.ONOScli6.intents( jsonFormat=True )
-            ONOS7Intents = main.ONOScli7.intents( jsonFormat=True )
-            intentCheck = main.FALSE
-            if "Error" in ONOS1Intents or not ONOS1Intents\
-                    or "Error" in ONOS2Intents or not ONOS2Intents\
-                    or "Error" in ONOS3Intents or not ONOS3Intents\
-                    or "Error" in ONOS4Intents or not ONOS4Intents\
-                    or "Error" in ONOS5Intents or not ONOS5Intents\
-                    or "Error" in ONOS6Intents or not ONOS6Intents\
-                    or "Error" in ONOS7Intents or not ONOS7Intents:
-                main.log.report( "Error in getting ONOS intents" )
-                main.log.warn( "ONOS1 intents response: " +
-                               repr( ONOS1Intents ) )
-                main.log.warn( "ONOS2 intents response: " +
-                               repr( ONOS2Intents ) )
-                main.log.warn( "ONOS3 intents response: " +
-                               repr( ONOS3Intents ) )
-                main.log.warn( "ONOS4 intents response: " +
-                               repr( ONOS4Intents ) )
-                main.log.warn( "ONOS5 intents response: " +
-                               repr( ONOS5Intents ) )
-                main.log.warn( "ONOS6 intents response: " +
-                               repr( ONOS6Intents ) )
-                main.log.warn( "ONOS7 intents response: " +
-                               repr( ONOS7Intents ) )
-            elif ONOS1Intents == ONOS2Intents\
-                    and ONOS1Intents == ONOS3Intents\
-                    and ONOS1Intents == ONOS4Intents\
-                    and ONOS1Intents == ONOS5Intents\
-                    and ONOS1Intents == ONOS6Intents\
-                    and ONOS1Intents == ONOS7Intents:
-                intentCheck = main.TRUE
-                main.log.report( "Intents are consistent across all" +
-                                 " ONOS nodes" )
-            else:
-                main.log.warn( "ONOS1 intents: " )
-                print json.dumps( json.loads( ONOS1Intents ), sort_keys=True,
-                                  indent=4, separators=( ',', ': ' ) )
-                main.log.warn( "ONOS2 intents: " )
-                print json.dumps( json.loads( ONOS2Intents ), sort_keys=True,
-                                  indent=4, separators=( ',', ': ' ) )
-                main.log.warn( "ONOS3 intents: " )
-                print json.dumps( json.loads( ONOS3Intents ), sort_keys=True,
-                                  indent=4, separators=( ',', ': ' ) )
-                main.log.warn( "ONOS4 intents: " )
-                print json.dumps( json.loads( ONOS4Intents ), sort_keys=True,
-                                  indent=4, separators=( ',', ': ' ) )
-                main.log.warn( "ONOS5 intents: " )
-                print json.dumps( json.loads( ONOS5Intents ), sort_keys=True,
-                                  indent=4, separators=( ',', ': ' ) )
-                main.log.warn( "ONOS6 intents: " )
-                print json.dumps( json.loads( ONOS6Intents ), sort_keys=True,
-                                  indent=4, separators=( ',', ': ' ) )
-                main.log.warn( "ONOS7 intents: " )
-                print json.dumps( json.loads( ONOS7Intents ), sort_keys=True,
-                                  indent=4, separators=( ',', ': ' ) )
-            utilities.assert_equals(
-                expect=main.TRUE,
-                actual=intentCheck,
-                onpass="Intents are consistent across all ONOS nodes",
-                onfail="ONOS nodes have different views of intents" )
-            # Print the intent states
-            intents = []
-            intents.append( ONOS1Intents )
-            intents.append( ONOS2Intents )
-            intents.append( ONOS3Intents )
-            intents.append( ONOS4Intents )
-            intents.append( ONOS5Intents )
-            intents.append( ONOS6Intents )
-            intents.append( ONOS7Intents )
-            intentStates = []
-            for node in intents:  # Iter through ONOS nodes
-                nodeStates = []
-                # Iter through intents of a node
-                for intent in json.loads( node ):
-                    nodeStates.append( intent[ 'state' ] )
-                intentStates.append( nodeStates )
-                out = [ (i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
-                main.log.info( dict( out ) )
+        main.step( "Get the intents and compare across all nodes" )
+        ONOS1Intents = main.ONOScli1.intents( jsonFormat=True )
+        ONOS2Intents = main.ONOScli2.intents( jsonFormat=True )
+        ONOS3Intents = main.ONOScli3.intents( jsonFormat=True )
+        ONOS4Intents = main.ONOScli4.intents( jsonFormat=True )
+        ONOS5Intents = main.ONOScli5.intents( jsonFormat=True )
+        ONOS6Intents = main.ONOScli6.intents( jsonFormat=True )
+        ONOS7Intents = main.ONOScli7.intents( jsonFormat=True )
+        intentCheck = main.FALSE
+        if "Error" in ONOS1Intents or not ONOS1Intents\
+                or "Error" in ONOS2Intents or not ONOS2Intents\
+                or "Error" in ONOS3Intents or not ONOS3Intents\
+                or "Error" in ONOS4Intents or not ONOS4Intents\
+                or "Error" in ONOS5Intents or not ONOS5Intents\
+                or "Error" in ONOS6Intents or not ONOS6Intents\
+                or "Error" in ONOS7Intents or not ONOS7Intents:
+            main.log.report( "Error in getting ONOS intents" )
+            main.log.warn( "ONOS1 intents response: " + repr( ONOS1Intents ) )
+            main.log.warn( "ONOS2 intents response: " + repr( ONOS2Intents ) )
+            main.log.warn( "ONOS3 intents response: " + repr( ONOS3Intents ) )
+            main.log.warn( "ONOS4 intents response: " + repr( ONOS4Intents ) )
+            main.log.warn( "ONOS5 intents response: " + repr( ONOS5Intents ) )
+            main.log.warn( "ONOS6 intents response: " + repr( ONOS6Intents ) )
+            main.log.warn( "ONOS7 intents response: " + repr( ONOS7Intents ) )
+        elif ONOS1Intents == ONOS2Intents\
+                and ONOS1Intents == ONOS3Intents\
+                and ONOS1Intents == ONOS4Intents\
+                and ONOS1Intents == ONOS5Intents\
+                and ONOS1Intents == ONOS6Intents\
+                and ONOS1Intents == ONOS7Intents:
+            intentCheck = main.TRUE
+            main.log.report( "Intents are consistent across all ONOS nodes" )
+        else:
+            main.log.warn( "ONOS1 intents: " )
+            print json.dumps( json.loads( ONOS1Intents ), sort_keys=True,
+                              indent=4, separators=( ',', ': ' ) )
+            main.log.warn( "ONOS2 intents: " )
+            print json.dumps( json.loads( ONOS2Intents ), sort_keys=True,
+                              indent=4, separators=( ',', ': ' ) )
+            main.log.warn( "ONOS3 intents: " )
+            print json.dumps( json.loads( ONOS3Intents ), sort_keys=True,
+                              indent=4, separators=( ',', ': ' ) )
+            main.log.warn( "ONOS4 intents: " )
+            print json.dumps( json.loads( ONOS4Intents ), sort_keys=True,
+                              indent=4, separators=( ',', ': ' ) )
+            main.log.warn( "ONOS5 intents: " )
+            print json.dumps( json.loads( ONOS5Intents ), sort_keys=True,
+                              indent=4, separators=( ',', ': ' ) )
+            main.log.warn( "ONOS6 intents: " )
+            print json.dumps( json.loads( ONOS6Intents ), sort_keys=True,
+                              indent=4, separators=( ',', ': ' ) )
+            main.log.warn( "ONOS7 intents: " )
+            print json.dumps( json.loads( ONOS7Intents ), sort_keys=True,
+                              indent=4, separators=( ',', ': ' ) )
+        utilities.assert_equals(
+            expect=main.TRUE,
+            actual=intentCheck,
+            onpass="Intents are consistent across all ONOS nodes",
+            onfail="ONOS nodes have different views of intents" )
+        # Print the intent states
+        intents = []
+        intents.append( ONOS1Intents )
+        intents.append( ONOS2Intents )
+        intents.append( ONOS3Intents )
+        intents.append( ONOS4Intents )
+        intents.append( ONOS5Intents )
+        intents.append( ONOS6Intents )
+        intents.append( ONOS7Intents )
+        intentStates = []
+        for node in intents:  # Iter through ONOS nodes
+            nodeStates = []
+            # Iter through intents of a node
+            for intent in json.loads( node ):
+                nodeStates.append( intent[ 'state' ] )
+            intentStates.append( nodeStates )
+            out = [ (i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
+            main.log.info( dict( out ) )
 
-
-            # NOTE: Store has no durability, so intents are lost across system
-            # restarts
-            main.step( "Compare current intents with intents before the failure" )
-            # NOTE: this requires case 5 to pass for intentState to be set.
-            #      maybe we should stop the test if that fails?
+        # NOTE: Store has no durability, so intents are lost across system
+        #       restarts
+        main.step( "Compare current intents with intents before the failure" )
+        # NOTE: this requires case 5 to pass for intentState to be set.
+        #      maybe we should stop the test if that fails?
+        sameIntents = main.TRUE
+        if intentState and intentState == ONOS1Intents:
             sameIntents = main.TRUE
-            if intentState and intentState == ONOS1Intents:
-                sameIntents = main.TRUE
-                main.log.report( "Intents are consistent with before failure" )
-            # TODO: possibly the states have changed? we may need to figure out
-            # what the aceptable states are
-            else:
-                try:
-                    main.log.warn( "ONOS1 intents: " )
-                    print json.dumps( json.loads( ONOS1Intents ),
-                                      sort_keys=True, indent=4,
-                                      separators=( ',', ': ' ) )
-                except Exception:
-                    pass
-                sameIntents = main.FALSE
-            utilities.assert_equals(
-                expect=main.TRUE,
-                actual=sameIntents,
-                onpass="Intents are consistent with before failure",
-                onfail="The Intents changed during failure" )
-            intentCheck = intentCheck and sameIntents
+            main.log.report( "Intents are consistent with before failure" )
+        # TODO: possibly the states have changed? we may need to figure out
+        # what the aceptable states are
+        else:
+            try:
+                main.log.warn( "ONOS1 intents: " )
+                print json.dumps( json.loads( ONOS1Intents ),
+                                  sort_keys=True, indent=4,
+                                  separators=( ',', ': ' ) )
+            except Exception:
+                pass
+            sameIntents = main.FALSE
+        utilities.assert_equals(
+            expect=main.TRUE,
+            actual=sameIntents,
+            onpass="Intents are consistent with before failure",
+            onfail="The Intents changed during failure" )
+        intentCheck = intentCheck and sameIntents
 
-            main.step( "Get the OF Table entries and compare to before " +
-                       "component failure" )
-            FlowTables = main.TRUE
-            flows2 = []
-            for i in range( 28 ):
-                main.log.info( "Checking flow table on s" + str( i + 1 ) )
-                tmpFlows = main.Mininet2.getFlowTable( 1.3, "s" + str( i + 1 ) )
-                flows2.append( tmpFlows )
-                tempResult = main.Mininet2.flowComp(
-                    flow1=flows[ i ],
-                    flow2=tmpFlows )
-                FlowTables = FlowTables and tempResult
-                if FlowTables == main.FALSE:
-                    main.log.info( "Differences in flow table for switch: s" +
-                                   str( i + 1 ) )
-            if FlowTables == main.TRUE:
-                main.log.report( "No changes were found in the flow tables" )
-            utilities.assert_equals(
-                expect=main.TRUE,
-                actual=FlowTables,
-                onpass="No changes were found in the flow tables",
-                onfail="Changes were found in the flow tables" )
-            if topoResult == main.TRUE or ( whileTime  > 10 ) :
-                break
+        main.step( "Get the OF Table entries and compare to before " +
+                   "component failure" )
+        FlowTables = main.TRUE
+        flows2 = []
+        for i in range( 28 ):
+            main.log.info( "Checking flow table on s" + str( i + 1 ) )
+            tmpFlows = main.Mininet2.getFlowTable( 1.3, "s" + str( i + 1 ) )
+            flows2.append( tmpFlows )
+            tempResult = main.Mininet2.flowComp(
+                flow1=flows[ i ],
+                flow2=tmpFlows )
+            FlowTables = FlowTables and tempResult
+            if FlowTables == main.FALSE:
+                main.log.info( "Differences in flow table for switch: s" +
+                               str( i + 1 ) )
+        if FlowTables == main.TRUE:
+            main.log.report( "No changes were found in the flow tables" )
+        utilities.assert_equals(
+            expect=main.TRUE,
+            actual=FlowTables,
+            onpass="No changes were found in the flow tables",
+            onfail="Changes were found in the flow tables" )
 
         main.step( "Check the continuous pings to ensure that no packets " +
                    "were dropped during component failure" )
@@ -1472,8 +1650,8 @@
             onpass="Leadership election passed",
             onfail="Something went wrong with Leadership election" )
 
-        result = mastershipCheck and intentCheck and FlowTables and\
-            ( not LossInPings ) and rolesNotNull and leaderResult
+        result = ( mastershipCheck and intentCheck and FlowTables and
+                   ( not LossInPings ) and rolesNotNull and leaderResult )
         result = int( result )
         if result == main.TRUE:
             main.log.report( "Constant State Tests Passed" )
@@ -1519,6 +1697,7 @@
         devicesResults = main.TRUE
         portsResults = main.TRUE
         linksResults = main.TRUE
+        hostsResults = main.TRUE
         topoResult = main.FALSE
         elapsed = 0
         count = 0
@@ -1529,9 +1708,7 @@
             count = count + 1
             if count > 1:
                 # TODO: Depricate STS usage
-                MNTopo = TestONTopology(
-                    main.Mininet1,
-                    ctrls )
+                MNTopo = TestONTopology( main.Mininet1, ctrls )
             cliStart = time.time()
             devices = []
             devices.append( main.ONOScli1.devices() )
@@ -1549,13 +1726,15 @@
             hosts.append( json.loads( main.ONOScli5.hosts() ) )
             hosts.append( json.loads( main.ONOScli6.hosts() ) )
             hosts.append( json.loads( main.ONOScli7.hosts() ) )
+            ipResult = main.TRUE
             for controller in range( 0, len( hosts ) ):
                 controllerStr = str( controller + 1 )
                 for host in hosts[ controller ]:
-                    if host[ 'ips' ] == []:
+                    if host is None or host.get( 'ips', [] ) == []:
                         main.log.error(
                             "DEBUG:Error with host ips on controller" +
                             controllerStr + ": " + str( host ) )
+                        ipResult = main.FALSE
             ports = []
             ports.append( main.ONOScli1.ports() )
             ports.append( main.ONOScli2.ports() )
@@ -1591,47 +1770,58 @@
                         controller ]:
                     currentDevicesResult = main.Mininet1.compareSwitches(
                         MNTopo,
-                        json.loads(
-                            devices[ controller ] ) )
+                        json.loads( devices[ controller ] ) )
                 else:
                     currentDevicesResult = main.FALSE
                 utilities.assert_equals( expect=main.TRUE,
-                                        actual=currentDevicesResult,
-                                        onpass="ONOS" + controllerStr +
-                                        " Switches view is correct",
-                                        onfail="ONOS" + controllerStr +
-                                        " Switches view is incorrect" )
+                                         actual=currentDevicesResult,
+                                         onpass="ONOS" + controllerStr +
+                                         " Switches view is correct",
+                                         onfail="ONOS" + controllerStr +
+                                         " Switches view is incorrect" )
 
                 if ports[ controller ] or "Error" not in ports[ controller ]:
                     currentPortsResult = main.Mininet1.comparePorts(
                         MNTopo,
-                        json.loads(
-                            ports[ controller ] ) )
+                        json.loads( ports[ controller ] ) )
                 else:
                     currentPortsResult = main.FALSE
                 utilities.assert_equals( expect=main.TRUE,
-                                        actual=currentPortsResult,
-                                        onpass="ONOS" + controllerStr +
-                                        " ports view is correct",
-                                        onfail="ONOS" + controllerStr +
-                                        " ports view is incorrect" )
+                                         actual=currentPortsResult,
+                                         onpass="ONOS" + controllerStr +
+                                         " ports view is correct",
+                                         onfail="ONOS" + controllerStr +
+                                         " ports view is incorrect" )
 
                 if links[ controller ] or "Error" not in links[ controller ]:
                     currentLinksResult = main.Mininet1.compareLinks(
                         MNTopo,
-                        json.loads(
-                            links[ controller ] ) )
+                        json.loads( links[ controller ] ) )
                 else:
                     currentLinksResult = main.FALSE
                 utilities.assert_equals( expect=main.TRUE,
-                                        actual=currentLinksResult,
-                                        onpass="ONOS" + controllerStr +
-                                        " links view is correct",
-                                        onfail="ONOS" + controllerStr +
-                                        " links view is incorrect" )
-            devicesResults = devicesResults and currentDevicesResult
-            portsResults = portsResults and currentPortsResult
-            linksResults = linksResults and currentLinksResult
+                                         actual=currentLinksResult,
+                                         onpass="ONOS" + controllerStr +
+                                         " links view is correct",
+                                         onfail="ONOS" + controllerStr +
+                                         " links view is incorrect" )
+
+                if hosts[ controller ] or "Error" not in hosts[ controller ]:
+                    currentHostsResult = main.Mininet1.compareHosts(
+                        MNTopo, hosts[ controller ] )
+                else:
+                    currentHostsResult = main.FALSE
+                utilities.assert_equals( expect=main.TRUE,
+                                         actual=currentHostsResult,
+                                         onpass="ONOS" + controllerStr +
+                                         " hosts exist in Mininet",
+                                         onfail="ONOS" + controllerStr +
+                                         " hosts don't match Mininet" )
+
+                devicesResults = devicesResults and currentDevicesResult
+                portsResults = portsResults and currentPortsResult
+                linksResults = linksResults and currentLinksResult
+                hostsResults = hostsResults and currentHostsResult
 
             # Compare json objects for hosts and dataplane clusters
 
@@ -1688,17 +1878,19 @@
                 onfail="ONOS nodes have different views of clusters" )
             # there should always only be one cluster
             numClusters = len( json.loads( clusters[ 0 ] ) )
+            clusterResults = main.FALSE
+            if numClusters == 1:
+                clusterResults = main.TRUE
             utilities.assert_equals(
                 expect=1,
                 actual=numClusters,
                 onpass="ONOS shows 1 SCC",
-                onfail="ONOS shows " +
-                str( numClusters ) +
-                " SCCs" )
+                onfail="ONOS shows " + str( numClusters ) + " SCCs" )
 
             topoResult = ( devicesResults and portsResults and linksResults
-                           and consistentHostsResult
-                           and consistentClustersResult )
+                           and hostsResults and consistentHostsResult
+                           and consistentClustersResult and clusterResults
+                           and ipResult )
 
         topoResult = topoResult and int( count <= 2 )
         note = "note it takes about " + str( int( cliTime ) ) + \
@@ -1709,8 +1901,8 @@
             str( note ) + " ): " + str( elapsed ) + " seconds, " +
             str( count ) + " tries" )
         utilities.assert_equals( expect=main.TRUE, actual=topoResult,
-                                onpass="Topology Check Test successful",
-                                onfail="Topology Check Test NOT successful" )
+                                 onpass="Topology Check Test successful",
+                                 onfail="Topology Check Test NOT successful" )
         if topoResult == main.TRUE:
             main.log.report( "ONOS topology view matches Mininet topology" )
 
@@ -1724,20 +1916,18 @@
         linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
 
         description = "Turn off a link to ensure that Link Discovery " +\
-            "is working properly"
+                      "is working properly"
         main.log.report( description )
         main.case( description )
 
         main.step( "Kill Link between s3 and s28" )
         LinkDown = main.Mininet1.link( END1="s3", END2="s28", OPTION="down" )
-        main.log.info(
-            "Waiting " +
-            str( linkSleep ) +
-            " seconds for link down to be discovered" )
+        main.log.info( "Waiting " + str( linkSleep ) +
+                       " seconds for link down to be discovered" )
         time.sleep( linkSleep )
         utilities.assert_equals( expect=main.TRUE, actual=LinkDown,
-                                onpass="Link down succesful",
-                                onfail="Failed to bring link down" )
+                                 onpass="Link down succesful",
+                                 onfail="Failed to bring link down" )
         # TODO do some sort of check here
 
     def CASE10( self, main ):
@@ -1750,20 +1940,18 @@
         linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
 
         description = "Restore a link to ensure that Link Discovery is " + \
-            "working properly"
+                      "working properly"
         main.log.report( description )
         main.case( description )
 
         main.step( "Bring link between s3 and s28 back up" )
         LinkUp = main.Mininet1.link( END1="s3", END2="s28", OPTION="up" )
-        main.log.info(
-            "Waiting " +
-            str( linkSleep ) +
-            " seconds for link up to be discovered" )
+        main.log.info( "Waiting " + str( linkSleep ) +
+                       " seconds for link up to be discovered" )
         time.sleep( linkSleep )
         utilities.assert_equals( expect=main.TRUE, actual=LinkUp,
-                                onpass="Link up succesful",
-                                onfail="Failed to bring link up" )
+                                 onpass="Link up succesful",
+                                 onfail="Failed to bring link up" )
         # TODO do some sort of check here
 
     def CASE11( self, main ):
@@ -1795,8 +1983,8 @@
         if device and device[ 'available' ] is False:
             result = main.TRUE
         utilities.assert_equals( expect=main.TRUE, actual=result,
-                                onpass="Kill switch succesful",
-                                onfail="Failed to kill switch?" )
+                                 onpass="Kill switch succesful",
+                                 onfail="Failed to kill switch?" )
 
     def CASE12( self, main ):
         """
@@ -1819,27 +2007,24 @@
         # TODO: New dpid or same? Ask Thomas?
         for peer in links:
             main.Mininet1.addLink( switch, peer )
-        main.Mininet1.assignSwController(
-            sw=switch.split( 's' )[ 1 ],
-            count=numControllers,
-            ip1=ONOS1Ip,
-            port1=ONOS1Port,
-            ip2=ONOS2Ip,
-            port2=ONOS2Port,
-            ip3=ONOS3Ip,
-            port3=ONOS3Port,
-            ip4=ONOS4Ip,
-            port4=ONOS4Port,
-            ip5=ONOS5Ip,
-            port5=ONOS5Port,
-            ip6=ONOS6Ip,
-            port6=ONOS6Port,
-            ip7=ONOS7Ip,
-            port7=ONOS7Port )
-        main.log.info(
-            "Waiting " +
-            str( switchSleep ) +
-            " seconds for switch up to be discovered" )
+        main.Mininet1.assignSwController( sw=switch.split( 's' )[ 1 ],
+                                          count=numControllers,
+                                          ip1=ONOS1Ip,
+                                          port1=ONOS1Port,
+                                          ip2=ONOS2Ip,
+                                          port2=ONOS2Port,
+                                          ip3=ONOS3Ip,
+                                          port3=ONOS3Port,
+                                          ip4=ONOS4Ip,
+                                          port4=ONOS4Port,
+                                          ip5=ONOS5Ip,
+                                          port5=ONOS5Port,
+                                          ip6=ONOS6Ip,
+                                          port6=ONOS6Port,
+                                          ip7=ONOS7Ip,
+                                          port7=ONOS7Port )
+        main.log.info( "Waiting " + str( switchSleep ) +
+                       " seconds for switch up to be discovered" )
         time.sleep( switchSleep )
         device = main.ONOScli1.getDevice( dpid=switchDPID )
         # Peek at the deleted switch
@@ -1848,8 +2033,8 @@
         if device and device[ 'available' ]:
             result = main.TRUE
         utilities.assert_equals( expect=main.TRUE, actual=result,
-                                onpass="add switch succesful",
-                                onfail="Failed to add switch?" )
+                                 onpass="add switch succesful",
+                                 onfail="Failed to add switch?" )
 
     def CASE13( self, main ):
         """
@@ -1911,6 +2096,8 @@
                                                 dstDir + str( testname ) +
                                                 "-ONOS" + str( i + 1 ) + "-" +
                                                 f )
+                main.ONOSbench.handle.expect( "\$" )
+
         # std*.log's
         # NOTE: must end in /
         logFolder = "/opt/onos/var/"
@@ -1926,15 +2113,17 @@
                                                 dstDir + str( testname ) +
                                                 "-ONOS" + str( i + 1 ) + "-" +
                                                 f )
+                main.ONOSbench.handle.expect( "\$" )
         # sleep so scp can finish
         time.sleep( 10 )
+        main.Mininet1.stopNet()
         main.step( "Packing and rotating pcap archives" )
         os.system( "~/TestON/dependencies/rotate.sh " + str( testname ) )
 
         # TODO: actually check something here
         utilities.assert_equals( expect=main.TRUE, actual=main.TRUE,
-                                onpass="Test cleanup successful",
-                                onfail="Test cleanup NOT successful" )
+                                 onpass="Test cleanup successful",
+                                 onfail="Test cleanup NOT successful" )
 
     def CASE14( self, main ):
         """
@@ -2035,11 +2224,13 @@
         elif leader is None or leader == main.FALSE:
             main.log.report(
                 "Leader for the election app should be an ONOS node," +
-                "instead got '" +
-                str( leader ) +
-                "'" )
+                "instead got '" + str( leader ) + "'" )
             leaderResult = main.FALSE
-        withdrawResult = oldLeader.electionTestWithdraw()
+            oldLeader = None
+        else:
+            main.log.error( "Leader election --- why am I HERE?!?")
+        if oldLeader:
+            withdrawResult = oldLeader.electionTestWithdraw()
         utilities.assert_equals(
             expect=main.TRUE,
             actual=withdrawResult,
@@ -2055,10 +2246,8 @@
         for leaderN in leaderList:
             if leaderN == leader:
                 main.log.report(
-                    "ONOS" +
-                    str( controller ) +
-                    " still sees " +
-                    str( leader ) +
+                    "ONOS" + str( controller ) +
+                    " still sees " + str( leader ) +
                     " as leader after they withdrew" )
                 leaderResult = main.FALSE
             elif leaderN == main.FALSE:
@@ -2092,9 +2281,12 @@
             onpass="Leadership election passed",
             onfail="Something went wrong with Leadership election" )
 
-        main.step(
-            "Run for election on old leader( just so everyone is in the hat )" )
-        runResult = oldLeader.electionTestRun()
+        main.step( "Run for election on old leader( just so everyone " +
+                   "is in the hat )" )
+        if oldLeader:
+            runResult = oldLeader.electionTestRun()
+        else:
+            runResult = main.FALSE
         utilities.assert_equals(
             expect=main.TRUE,
             actual=runResult,