diff --git a/TestON/core/logger.py b/TestON/core/logger.py
index 3a24008..80fe0de 100644
--- a/TestON/core/logger.py
+++ b/TestON/core/logger.py
@@ -110,7 +110,7 @@
            
         main.LogFileName = main.logdir + "/" + main.TEST + "_" +str(currentTime) + ".log"
         main.ReportFileName = main.logdir + "/" + main.TEST + "_" + str(currentTime) + ".rpt"
-        main.JenkinsCSV = main.logdir + "/" + main.TEST + ".csv"               
+        main.JenkinsCSV = main.logdir + "/" + main.TEST + ".csv"
  
         #### Add log-level - Report
         logging.addLevelName(9, "REPORT")
@@ -229,14 +229,14 @@
         #main.log.report(testResult)
         main.testResult = testResult
         main.log.exact(testResult)
-        
+
         ##CSV output needed for Jenkin's plot plugin
         #NOTE: the elements were orded based on the colors assigned to the data
         logfile = open(main.JenkinsCSV ,"w")
         logfile.write(",".join( ['Tests Failed', 'Tests Passed', 'Tests Planned'] ) + "\n")
-        logfile.write(",".join( [str(main.TOTAL_TC_FAIL), str(main.TOTAL_TC_PASS), str(main.TOTAL_TC_PLANNED)] ))
-        logfile.close()        
-        
+        logfile.write(",".join( [str(int(main.TOTAL_TC_FAIL)), str(int(main.TOTAL_TC_PASS)), str(int(main.TOTAL_TC_PLANNED))] ))
+        logfile.close()
+
 
 
 
diff --git a/TestON/dependencies/rotate.sh b/TestON/dependencies/rotate.sh
index 235d138..c685a14 100755
--- a/TestON/dependencies/rotate.sh
+++ b/TestON/dependencies/rotate.sh
@@ -51,4 +51,4 @@
 #Maybe this should be an argument? pack-and-rotate supports that
 nr_max=20
 
-pack-rotate-log ${root_dir}'/'${base_name} "${root_dir}/${base_name}*.pcap" ${nr_max}
+pack-rotate-log ${root_dir}'/'${base_name} "${root_dir}/${base_name}*.pcap ${root_dir}/${base_name}*.log " ${nr_max}
diff --git a/TestON/drivers/common/cli/emulator/mininetclidriver.py b/TestON/drivers/common/cli/emulator/mininetclidriver.py
index 17c25a0..bf3a2c6 100644
--- a/TestON/drivers/common/cli/emulator/mininetclidriver.py
+++ b/TestON/drivers/common/cli/emulator/mininetclidriver.py
@@ -1207,7 +1207,7 @@
                         break
             mn_ports.sort(key=float)
             onos_ports.sort(key=float)
-            #print "\nPorts for Switch %s:" % (switch['name'])
+            #print "\nPorts for Switch %s:" % (mn_switch['name'])
             #print "\tmn_ports[] = ", mn_ports
             #print "\tonos_ports[] = ", onos_ports
             mn_ports_log = mn_ports
diff --git a/TestON/drivers/common/cli/emulator/remotemininetdriver.py b/TestON/drivers/common/cli/emulator/remotemininetdriver.py
index 1795a16..ae0f599 100644
--- a/TestON/drivers/common/cli/emulator/remotemininetdriver.py
+++ b/TestON/drivers/common/cli/emulator/remotemininetdriver.py
@@ -114,7 +114,7 @@
         args = utilities.parse_args(["SRC","TARGET","PINGTIME"],**pingParams)
         precmd = "sudo rm /tmp/ping." + args["SRC"]
         self.execute(cmd=precmd,prompt="(.*)",timeout=10)
-        command = "sudo mininet/util/m " + args["SRC"] + " ping "+args ["TARGET"]+" -i .2 -w " + str(args['PINGTIME']) + " > /tmp/ping." + args["SRC"] + " &"
+        command = "sudo mininet/util/m " + args["SRC"] + " ping "+args ["TARGET"]+" -i .2 -w " + str(args['PINGTIME']) + " -D > /tmp/ping." + args["SRC"] + " &"
         main.log.info( command ) 
         self.execute(cmd=command,prompt="(.*)",timeout=10)
         self.handle.sendline("")
@@ -372,57 +372,40 @@
 
     def get_flowTable(self, protoVersion, sw):
         #TODO document usage
-        #FIXME: clean up print statements and such
+        #TODO add option to look at cookies. ignoreing them for now
         self.handle.sendline("cd")
         self.handle.expect(["\$",pexpect.EOF,pexpect.TIMEOUT])
-        #TODO: Write seperate versions of the function for this, possibly a string that tells it which switch is in use?
-        #For 1.0 version of OVS
-        #command = "sudo ovs-ofctl dump-flows " + sw + " | awk '{OFS=\",\" ; print $1 $6 $7 }' |sort -n -k1"
-        #for 1.3 version of OVS
-        #command = "sudo ovs-ofctl dump-flows " + sw + " | awk '{OFS=\",\" ; print $1 $3 $7 $8}' |sort -n -k1"
+        #print "get_flowTable(" + str(protoVersion) +" " + str(sw) +")"
         #NOTE: Use format to force consistent flow table output across versions
         if protoVersion==1.0:
-            command = "sudo ovs-ofctl dump-flows " + sw + " -F OpenFlow10-table_id | awk '{OFS=\",\" ; print $1  $3  $6  $7  $8}' |sort -n -k1"
+            command = "sudo ovs-ofctl dump-flows " + sw + " -F OpenFlow10-table_id | awk '{OFS=\",\" ; print $1  $3  $6  $7  $8}' | cut -d ',' -f 2- | sort -n -k1 -r"
             self.handle.sendline(command)
-            self.handle.expect(["sort -n -k1",pexpect.EOF,pexpect.TIMEOUT])
+            self.handle.expect(["k1 -r",pexpect.EOF,pexpect.TIMEOUT])
             self.handle.expect(["OFPST_FLOW",pexpect.EOF,pexpect.TIMEOUT])
             response = self.handle.before
             #print "response=", response
             return response
         elif protoVersion==1.3:
-            command = "sudo ovs-ofctl dump-flows " + sw + " -O OpenFlow13  | awk '{OFS=\",\" ; print $1  $3  $6  $7}' |sort -n -k1" 
+            command = "sudo ovs-ofctl dump-flows " + sw + " -O OpenFlow13  | awk '{OFS=\",\" ; print $1  $3  $6  $7}' | cut -d ',' -f 2- | sort -n -k1 -r"
             self.handle.sendline(command)
-            #print "ovs-vsctl Command sent status."
-            #self.handle.expect(["sort -n -k1",pexpect.EOF,pexpect.TIMEOUT])
-            #print "sort return status: " 
-            #print self.handle.expect(["sort -n -k1",pexpect.EOF,pexpect.TIMEOUT])
-            #print self.handle.before
+            self.handle.expect(["k1 -r",pexpect.EOF,pexpect.TIMEOUT])
             self.handle.expect(["OFPST_FLOW",pexpect.EOF,pexpect.TIMEOUT])
-            #print "OFPST_FLOW expected status: " 
-            #print self.handle.expect(["OFPST_FLOW",pexpect.EOF,pexpect.TIMEOUT])
-            #print self.handle.before
             response = self.handle.before
             #print "response=", response
-            return response 
-        
+            return response
+        else:
+            main.log.error("Unknown  protoVersion in get_flowTable(). given: ("+str(type(protoVersion))+") '"+str(protoVersion)+"'")
+
 
     def flow_comp(self,flow1,flow2):
-        #print "Inside flow compare function"
-        #print "Flow1 Table:"
-        #for flow in flow1:
-            #print flow1
-        #print "Flow2 Table"
-        #for flow in flow2:
-            #print flow2
-
         if flow1==flow2:
             return main.TRUE
         else:
             main.log.info("Flow tables do not match, printing tables:")
-            #main.log.info("Flow Table 1:")
-            #main.log.info(flow1)
-            #main.log.info("Flow Table 2:")
-            #main.log.info(flow2)
+            main.log.info("Flow Table 1:")
+            main.log.info(flow1)
+            main.log.info("Flow Table 2:")
+            main.log.info(flow2)
             return main.FALSE
 
     def setIpTablesOUTPUT(self, dst_ip, dst_port, action='add', packet_type='tcp',rule='DROP'):
diff --git a/TestON/drivers/common/cli/onosclidriver.py b/TestON/drivers/common/cli/onosclidriver.py
index a6cf341..358945f 100644
--- a/TestON/drivers/common/cli/onosclidriver.py
+++ b/TestON/drivers/common/cli/onosclidriver.py
@@ -693,7 +693,38 @@
             main.log.info(self.name+" ::::::")
             main.cleanup()
             main.exit()
-    
+
+    def roles_not_null(self):
+        '''
+        Iterates through each device and checks if there is a master assigned
+        Returns: main.TRUE if each device has a master
+                 main.FALSE any device has no master
+        '''
+        try:
+            import json
+            raw_roles = self.roles()
+            roles_json = json.loads(raw_roles)
+            #search json for the device with id then return the device
+            for device in roles_json:
+                #print device
+                if device['master'] == "none":
+                    main.log.warn("Device has no master: " + str(device) )
+                    return main.FALSE
+            return main.TRUE
+
+        except pexpect.EOF:
+            main.log.error(self.name + ": EOF exception found")
+            main.log.error(self.name + ":    " + self.handle.before)
+            main.cleanup()
+            main.exit()
+        except:
+            main.log.info(self.name+" ::::::")
+            main.log.error( traceback.print_exc())
+            main.log.info(self.name+" ::::::")
+            main.cleanup()
+            main.exit()
+
+
     def paths(self, src_id, dst_id):
         '''
         Returns string of paths, and the cost.
@@ -1713,7 +1744,7 @@
         onos_node is the ip of one of the onos nodes in the cluster
         role must be either master, standby, or none
 
-        Returns main.TRUE or main.FALSE based argument varification.
+        Returns main.TRUE or main.FALSE based on argument verification.
             When device-role supports errors this should be extended to
             support that output
         '''
@@ -1805,5 +1836,115 @@
             main.cleanup()
             main.exit()
 
+    def election_test_leader(self):
+        '''
+         * CLI command to get the current leader for the Election test application.
+         #NOTE: Requires installation of the onos-app-election feature
+         Returns: Node IP of the leader if one exists
+                  None if none exists
+                  Main.FALSE on error
+        '''
+        try:
+            self.handle.sendline("election-test-leader")
+            self.handle.expect("election-test-leader")
+            self.handle.expect("onos>")
+            response = self.handle.before
+            #Leader
+            node_search = re.search("The\scurrent\sleader\sfor\sthe\sElection\sapp\sis\s(?P<node>.+)\.", response)
+            if node_search:
+                node = node_search.group('node')
+                main.log.info("Election-test-leader found " + node + " as the leader")
+                return node
+            #no leader
+            null_search = re.search("There\sis\scurrently\sno\sleader\selected\sfor\sthe\sElection\sapp", response)
+            if null_search:
+                main.log.info("Election-test-leader found no leader")
+                return None
+            #error
+            main.log.error("Error in election_test_leader: unexpected response")
+            main.log.error( repr(response) )
+            return main.FALSE
+
+        except pexpect.EOF:
+            main.log.error(self.name + ": EOF exception found")
+            main.log.error(self.name + ":    " + self.handle.before)
+            main.cleanup()
+            main.exit()
+        except:
+            main.log.info(self.name+" ::::::")
+            main.log.error( traceback.print_exc())
+            main.log.info(self.name+" ::::::")
+            main.cleanup()
+            main.exit()
+
+    def election_test_run(self):
+        '''
+         * CLI command to run for leadership of the Election test application.
+         #NOTE: Requires installation of the onos-app-election feature
+         Returns: Main.TRUE on success
+                  Main.FALSE on error
+        '''
+        try:
+            self.handle.sendline("election-test-run")
+            self.handle.expect("election-test-run")
+            self.handle.expect("onos>")
+            response = self.handle.before
+            #success
+            search = re.search("Entering\sleadership\selections\sfor\sthe\sElection\sapp.", response)
+            if search:
+                main.log.info("Entering leadership elections for the Election app.")
+                return main.TRUE
+            #error
+            main.log.error("Error in election_test_run: unexpected response")
+            main.log.error( repr(response) )
+            return main.FALSE
+
+        except pexpect.EOF:
+            main.log.error(self.name + ": EOF exception found")
+            main.log.error(self.name + ":    " + self.handle.before)
+            main.cleanup()
+            main.exit()
+        except:
+            main.log.info(self.name+" ::::::")
+            main.log.error( traceback.print_exc())
+            main.log.info(self.name+" ::::::")
+            main.cleanup()
+            main.exit()
+
+    def election_test_withdraw(self):
+        '''
+         * CLI command to withdraw the local node from leadership election for
+         * the Election test application.
+         #NOTE: Requires installation of the onos-app-election feature
+         Returns: Main.TRUE on success
+                  Main.FALSE on error
+        '''
+        try:
+            self.handle.sendline("election-test-withdraw")
+            self.handle.expect("election-test-withdraw")
+            self.handle.expect("onos>")
+            response = self.handle.before
+            #success
+            search = re.search("Withdrawing\sfrom\sleadership\selections\sfor\sthe\sElection\sapp.", response)
+            if search:
+                main.log.info("Withdrawing from leadership elections for the Election app.")
+                return main.TRUE
+            #error
+            main.log.error("Error in election_test_withdraw: unexpected response")
+            main.log.error( repr(response) )
+            return main.FALSE
+
+
+        except pexpect.EOF:
+            main.log.error(self.name + ": EOF exception found")
+            main.log.error(self.name + ":    " + self.handle.before)
+            main.cleanup()
+            main.exit()
+        except:
+            main.log.info(self.name+" ::::::")
+            main.log.error( traceback.print_exc())
+            main.log.info(self.name+" ::::::")
+            main.cleanup()
+            main.exit()
 
     #***********************************
diff --git a/TestON/drivers/common/cli/onosdriver.py b/TestON/drivers/common/cli/onosdriver.py
index 418554d..678bae4 100644
--- a/TestON/drivers/common/cli/onosdriver.py
+++ b/TestON/drivers/common/cli/onosdriver.py
@@ -1072,21 +1072,21 @@
                 if not line.startswith("time="):
                     continue
                 #else
-                print line
+                #print line
                 for var in line.split(","):
                     #print "'"+var+"'"
                     #print "'"+var.strip()+"'"
                     key, value = var.strip().split("=")
                     topology[key] = value
-            print "topology = ", topology
+            #print "topology = ", topology
             devices = topology.get('devices', False)
-            print "devices = ", devices
+            #print "devices = ", devices
             links = topology.get('links', False)
-            print "links = ", links
-            clusters = topology.get('clusters', False)
-            print "clusters = ", clusters
+            #print "links = ", links
+            SCCs = topology.get('SCC(s)', False)
+            #print "SCCs = ", SCCs
             paths = topology.get('paths', False)
-            print "paths = ", paths
+            #print "paths = ", paths
 
             return topology
         except pexpect.EOF:
@@ -1339,3 +1339,26 @@
             main.log.error( traceback.print_exc())
             main.log.info(self.name+" ::::::")
 
+    def check_logs(self, onos_ip):
+        '''
+        runs onos-check-logs on the given onos node
+        returns the response
+        '''
+        try:
+            cmd = "onos-check-logs " + str(onos_ip)
+            self.handle.sendline(cmd)
+            self.handle.expect(cmd)
+            self.handle.expect("\$")
+            response = self.handle.before
+            return response
+        except pexpect.EOF:
+            main.log.error("Lost ssh connection")
+            main.log.error(self.name + ": EOF exception found")
+            main.log.error(self.name + ":    " + self.handle.before)
+        except:
+            main.log.error("Some error in check_logs:")
+            main.log.info(self.name+" ::::::")
+            main.log.error( traceback.print_exc())
+            main.log.info(self.name+" ::::::")
+
+
diff --git a/TestON/drivers/common/clidriver.py b/TestON/drivers/common/clidriver.py
index 00d00ed..878839e 100644
--- a/TestON/drivers/common/clidriver.py
+++ b/TestON/drivers/common/clidriver.py
@@ -48,7 +48,7 @@
         ssh_newkey = 'Are you sure you want to continue connecting'
         refused = "ssh: connect to host "+self.ip_address+" port 22: Connection refused"
         if self.port:
-            self.handle =pexpect.spawn('ssh -p '+self.port+' '+self.user_name+'@'+self.ip_address,maxread=50000)
+            self.handle =pexpect.spawn('ssh -p '+self.port+' '+self.user_name+'@'+self.ip_address, env = {"TERM": "vt100"} , maxread=50000)
         else :
             self.handle =pexpect.spawn('ssh -X '+self.user_name+'@'+self.ip_address,maxread=1000000,timeout=60)
 
@@ -190,7 +190,9 @@
         '''
         ssh_newkey = 'Are you sure you want to continue connecting'
         refused = "ssh: connect to host "+ip_address+" port 22: Connection refused"
-        self.handle =pexpect.spawn('scp '+user_name+'@'+ip_address+':'+filepath+' '+dst_path)
+        cmd = 'scp '+str(user_name)+'@'+str(ip_address)+':'+str(filepath)+' '+str(dst_path)
+        main.log.info("Sending: " + cmd )
+        self.handle =pexpect.spawn( cmd )
         i=self.handle.expect([ssh_newkey,'password:',pexpect.EOF,pexpect.TIMEOUT,refused],120)
         
         if i==0:    
@@ -212,7 +214,9 @@
             main.log.error("ssh: connect to host "+ip_address+" port 22: Connection refused")
             return main.FALSE
 
-        self.handle.sendline("\r")
+        self.handle.sendline("")
+        self.handle.expect("$")
+        print self.handle.before
         
         return self.handle
     
diff --git a/TestON/tests/HATestClusterRestart/HATestClusterRestart.params b/TestON/tests/HATestClusterRestart/HATestClusterRestart.params
index 0a8bd57..68f12a8 100644
--- a/TestON/tests/HATestClusterRestart/HATestClusterRestart.params
+++ b/TestON/tests/HATestClusterRestart/HATestClusterRestart.params
@@ -1,5 +1,5 @@
 <PARAMS>
-    <testcases>1,2,8,3,4,5,[6],7,8,4,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
+    <testcases>1,2,8,3,4,5,[6],8,3,7,4,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
     <ENV>
     <cellName>HA</cellName>
     </ENV>
@@ -28,7 +28,7 @@
         <port7>6633</port7>
     </CTRL>
     <TESTONUSER>admin</TESTONUSER>
-    <TESTONIP>10.128.30.10</TESTONIP>
+    <TESTONIP>10.128.30.9</TESTONIP>
     <PING>
         <source1>h8</source1>
         <source2>h9</source2>
diff --git a/TestON/tests/HATestClusterRestart/HATestClusterRestart.py b/TestON/tests/HATestClusterRestart/HATestClusterRestart.py
index 47dc744..3e0a055 100644
--- a/TestON/tests/HATestClusterRestart/HATestClusterRestart.py
+++ b/TestON/tests/HATestClusterRestart/HATestClusterRestart.py
@@ -82,7 +82,7 @@
         cell_result = main.ONOSbench.set_cell(cell_name)
         verify_result = main.ONOSbench.verify_cell()
 
-        #FIXME:this is short term fix 
+        #FIXME:this is short term fix
         main.log.report("Removing raft logs")
         main.ONOSbench.onos_remove_raft_logs()
         main.log.report("Uninstalling ONOS")
@@ -145,29 +145,32 @@
         main.step("Checking if ONOS is up yet")
         #TODO: Refactor
         # check bundle:list?
-        onos1_isup = main.ONOSbench.isup(ONOS1_ip)
-        if not onos1_isup:
-            main.log.report("ONOS1 didn't start!")
-        onos2_isup = main.ONOSbench.isup(ONOS2_ip)
-        if not onos2_isup:
-            main.log.report("ONOS2 didn't start!")
-        onos3_isup = main.ONOSbench.isup(ONOS3_ip)
-        if not onos3_isup:
-            main.log.report("ONOS3 didn't start!")
-        onos4_isup = main.ONOSbench.isup(ONOS4_ip)
-        if not onos4_isup:
-            main.log.report("ONOS4 didn't start!")
-        onos5_isup = main.ONOSbench.isup(ONOS5_ip)
-        if not onos5_isup:
-            main.log.report("ONOS5 didn't start!")
-        onos6_isup = main.ONOSbench.isup(ONOS6_ip)
-        if not onos6_isup:
-            main.log.report("ONOS6 didn't start!")
-        onos7_isup = main.ONOSbench.isup(ONOS7_ip)
-        if not onos7_isup:
-            main.log.report("ONOS7 didn't start!")
-        onos_isup_result = onos1_isup and onos2_isup and onos3_isup\
-                and onos4_isup and onos5_isup and onos6_isup and onos7_isup
+        for i in range(2):
+            onos1_isup = main.ONOSbench.isup(ONOS1_ip)
+            if not onos1_isup:
+                main.log.report("ONOS1 didn't start!")
+            onos2_isup = main.ONOSbench.isup(ONOS2_ip)
+            if not onos2_isup:
+                main.log.report("ONOS2 didn't start!")
+            onos3_isup = main.ONOSbench.isup(ONOS3_ip)
+            if not onos3_isup:
+                main.log.report("ONOS3 didn't start!")
+            onos4_isup = main.ONOSbench.isup(ONOS4_ip)
+            if not onos4_isup:
+                main.log.report("ONOS4 didn't start!")
+            onos5_isup = main.ONOSbench.isup(ONOS5_ip)
+            if not onos5_isup:
+                main.log.report("ONOS5 didn't start!")
+            onos6_isup = main.ONOSbench.isup(ONOS6_ip)
+            if not onos6_isup:
+                main.log.report("ONOS6 didn't start!")
+            onos7_isup = main.ONOSbench.isup(ONOS7_ip)
+            if not onos7_isup:
+                main.log.report("ONOS7 didn't start!")
+            onos_isup_result = onos1_isup and onos2_isup and onos3_isup\
+                    and onos4_isup and onos5_isup and onos6_isup and onos7_isup
+            if onos_isup_result == main.TRUE:
+                break
         # TODO: if it becomes an issue, we can retry this step  a few times
 
 
@@ -197,9 +200,9 @@
                 onfail="Test startup NOT successful")
 
 
-        #if case1_result==main.FALSE:
-        #    main.cleanup()
-        #    main.exit()
+        if case1_result==main.FALSE:
+            main.cleanup()
+            main.exit()
 
     def CASE2(self,main) :
         '''
@@ -209,25 +212,6 @@
         import json
         import re
 
-
-        '''
-        ONOS1_ip = main.params['CTRL']['ip1']
-        ONOS1_port = main.params['CTRL']['port1']
-        ONOS2_ip = main.params['CTRL']['ip2']
-        ONOS2_port = main.params['CTRL']['port2']
-        ONOS3_ip = main.params['CTRL']['ip3']
-        ONOS3_port = main.params['CTRL']['port3']
-        ONOS4_ip = main.params['CTRL']['ip4']
-        ONOS4_port = main.params['CTRL']['port4']
-        ONOS5_ip = main.params['CTRL']['ip5']
-        ONOS5_port = main.params['CTRL']['port5']
-        ONOS6_ip = main.params['CTRL']['ip6']
-        ONOS6_port = main.params['CTRL']['port6']
-        ONOS7_ip = main.params['CTRL']['ip7']
-        ONOS7_port = main.params['CTRL']['port7']
-        '''
-
-
         main.log.report("Assigning switches to controllers")
         main.case("Assigning Controllers")
         main.step("Assign switches to controllers")
@@ -265,8 +249,217 @@
                 onpass="Switch mastership assigned correctly",
                 onfail="Switches not assigned correctly to controllers")
 
-        #TODO: If assign roles is working reliably then manually 
-        #   assign mastership to the controller we want
+        #Manually assign mastership to the controller we want
+        role_call = main.TRUE
+        role_check = main.TRUE
+
+        device_id = main.ONOScli1.get_device("1000")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS1_ip)
+        if ONOS1_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("2800")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS1_ip)
+        if ONOS1_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("2000")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS2_ip)
+        if ONOS2_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3000")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS2_ip)
+        if ONOS2_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("5000")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS3_ip)
+        if ONOS3_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6000")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS3_ip)
+        if ONOS3_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id =  main.ONOScli1.get_device("3004")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS4_ip)
+        if ONOS4_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3008")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3009")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3010")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3011")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3012")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3013")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3014")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3015")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3016")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3017")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6007")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS6_ip)
+        if ONOS6_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6018")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6019")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6020")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6021")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6022")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6023")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6024")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6025")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6026")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6027")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        utilities.assert_equals(expect = main.TRUE,actual=role_call,
+                onpass="Re-assigned switch mastership to designated controller",
+                onfail="Something wrong with device_role calls")
+
+        utilities.assert_equals(expect = main.TRUE,actual=role_check,
+                onpass="Switches were successfully reassigned to designated controller",
+                onfail="Switches were not successfully reassigned")
+        mastership_check = mastership_check and role_call and role_check
+        utilities.assert_equals(expect = main.TRUE,actual=mastership_check,
+                onpass="Switch mastership correctly assigned",
+                onfail="Error in (re)assigning switch mastership")
 
 
     def CASE3(self,main) :
@@ -274,6 +467,7 @@
         Assign intents
 
         """
+        #FIXME: This should only be called once when we have persistence
         import time
         import json
         import re
@@ -283,6 +477,16 @@
         main.step("Discovering  Hosts( Via pingall for now)")
         #FIXME: Once we have a host discovery mechanism, use that instead
 
+        #install onos-app-fwd
+        main.log.info("Install reactive forwarding app")
+        main.ONOScli1.feature_install("onos-app-fwd")
+        main.ONOScli2.feature_install("onos-app-fwd")
+        main.ONOScli3.feature_install("onos-app-fwd")
+        main.ONOScli4.feature_install("onos-app-fwd")
+        main.ONOScli5.feature_install("onos-app-fwd")
+        main.ONOScli6.feature_install("onos-app-fwd")
+        main.ONOScli7.feature_install("onos-app-fwd")
+
         #REACTIVE FWD test
         ping_result = main.FALSE
         time1 = time.time()
@@ -304,7 +508,7 @@
         #TODO:  move the host numbers to params
         import json
         intents_json= json.loads(main.ONOScli1.hosts())
-        intent_add_result = main.FALSE
+        intent_add_result = True
         for i in range(8,18):
             main.log.info("Adding host intent between h"+str(i)+" and h"+str(i+10))
             host1 =  "00:00:00:00:00:" + str(hex(i)[2:]).zfill(2).upper()
@@ -314,7 +518,10 @@
             host1_id = main.ONOScli1.get_host(host1)['id']
             host2_id = main.ONOScli1.get_host(host2)['id']
             tmp_result = main.ONOScli1.add_host_intent(host1_id, host2_id )
-            intent_add_result = intent_add_result and tmp_result
+            intent_add_result = bool(intent_add_result and tmp_result)
+        utilities.assert_equals(expect=True, actual=intent_add_result,
+                onpass="Switch mastership correctly assigned",
+                onfail="Error in (re)assigning switch mastership")
         #TODO Check if intents all exist in datastore
         #NOTE: Do we need to print this once the test is working?
         #main.log.info(json.dumps(json.loads(main.ONOScli1.intents(json_format=True)),
@@ -360,6 +567,25 @@
 
         main.step("Get the Mastership of each switch from each controller")
         global mastership_state
+        mastership_state = []
+
+        #Assert that each device has a master
+        ONOS1_master_not_null = main.ONOScli1.roles_not_null()
+        ONOS2_master_not_null = main.ONOScli2.roles_not_null()
+        ONOS3_master_not_null = main.ONOScli3.roles_not_null()
+        ONOS4_master_not_null = main.ONOScli4.roles_not_null()
+        ONOS5_master_not_null = main.ONOScli5.roles_not_null()
+        ONOS6_master_not_null = main.ONOScli6.roles_not_null()
+        ONOS7_master_not_null = main.ONOScli7.roles_not_null()
+        roles_not_null = ONOS1_master_not_null and ONOS2_master_not_null and\
+                ONOS3_master_not_null and ONOS4_master_not_null and\
+                ONOS5_master_not_null and ONOS6_master_not_null and\
+                ONOS7_master_not_null
+        utilities.assert_equals(expect = main.TRUE,actual=roles_not_null,
+                onpass="Each device has a master",
+                onfail="Some devices don't have a master assigned")
+
+
         ONOS1_mastership = main.ONOScli1.roles()
         ONOS2_mastership = main.ONOScli2.roles()
         ONOS3_mastership = main.ONOScli3.roles()
@@ -416,6 +642,7 @@
 
         main.step("Get the intents from each controller")
         global intent_state
+        intent_state = []
         ONOS1_intents = main.ONOScli1.intents( json_format=True )
         ONOS2_intents = main.ONOScli2.intents( json_format=True )
         ONOS3_intents = main.ONOScli3.intents( json_format=True )
@@ -470,6 +697,7 @@
 
         main.step("Get the flows from each controller")
         global flow_state
+        flow_state = []
         ONOS1_flows = main.ONOScli1.flows( json_format=True )
         ONOS2_flows = main.ONOScli2.flows( json_format=True )
         ONOS3_flows = main.ONOScli3.flows( json_format=True )
@@ -527,7 +755,7 @@
         global flows
         flows=[]
         for i in range(1,29):
-            flows.append(main.Mininet2.get_flowTable("s"+str(i),1.0))
+            flows.append(main.Mininet2.get_flowTable(1.3, "s"+str(i)))
 
         #TODO: Compare switch flow tables with ONOS flow tables
 
@@ -578,7 +806,6 @@
         devices.append( main.ONOScli5.devices() )
         devices.append( main.ONOScli6.devices() )
         devices.append( main.ONOScli7.devices() )
-        '''
         hosts = []
         hosts.append( main.ONOScli1.hosts() )
         hosts.append( main.ONOScli2.hosts() )
@@ -587,7 +814,6 @@
         hosts.append( main.ONOScli5.hosts() )
         hosts.append( main.ONOScli6.hosts() )
         hosts.append( main.ONOScli7.hosts() )
-        '''
         ports = []
         ports.append( main.ONOScli1.ports() )
         ports.append( main.ONOScli2.ports() )
@@ -604,6 +830,91 @@
         links.append( main.ONOScli5.links() )
         links.append( main.ONOScli6.links() )
         links.append( main.ONOScli7.links() )
+        clusters = []
+        clusters.append( main.ONOScli1.clusters() )
+        clusters.append( main.ONOScli2.clusters() )
+        clusters.append( main.ONOScli3.clusters() )
+        clusters.append( main.ONOScli4.clusters() )
+        clusters.append( main.ONOScli5.clusters() )
+        clusters.append( main.ONOScli6.clusters() )
+        clusters.append( main.ONOScli7.clusters() )
+        paths = []
+        temp_topo = main.ONOSbench.get_topology( main.ONOScli1.topology() )
+        paths.append( temp_topo.get('paths', False) )
+        temp_topo = main.ONOSbench.get_topology( main.ONOScli2.topology() )
+        paths.append( temp_topo.get('paths', False) )
+        temp_topo = main.ONOSbench.get_topology( main.ONOScli3.topology() )
+        paths.append( temp_topo.get('paths', False) )
+        temp_topo = main.ONOSbench.get_topology( main.ONOScli4.topology() )
+        paths.append( temp_topo.get('paths', False) )
+        temp_topo = main.ONOSbench.get_topology( main.ONOScli5.topology() )
+        paths.append( temp_topo.get('paths', False) )
+        temp_topo = main.ONOSbench.get_topology( main.ONOScli6.topology() )
+        paths.append( temp_topo.get('paths', False) )
+        temp_topo = main.ONOSbench.get_topology( main.ONOScli7.topology() )
+        paths.append( temp_topo.get('paths', False) )
+
+        #Compare json objects for hosts, dataplane clusters and paths
+
+        #hosts
+        consistent_hosts_result = main.TRUE
+        for controller in range( len( hosts ) ):
+            if not "Error" in hosts[controller]:
+                if hosts[controller] == hosts[0]:
+                    continue
+                else:#hosts not consistent
+                    main.log.report("hosts from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+                    main.log.warn( repr( hosts[controller] ) )
+                    consistent_hosts_result = main.FALSE
+
+            else:
+                main.log.report("Error in getting ONOS hosts from ONOS" + str(controller + 1) )
+                consistent_hosts_result = main.FALSE
+                main.log.warn("ONOS" + str(controller + 1) + " hosts response: " + repr(hosts[controller]) )
+        utilities.assert_equals(expect = main.TRUE,actual=consistent_hosts_result,
+                onpass="Hosts view is consistent across all ONOS nodes",
+                onfail="ONOS nodes have different views of hosts")
+
+        #Strongly connected clusters of devices
+        consistent_clusters_result = main.TRUE
+        for controller in range( len( clusters ) ):
+            if not "Error" in clusters[controller]:
+                if clusters[controller] == clusters[0]:
+                    continue
+                else:#clusters not consistent
+                    main.log.report("clusters from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+                    consistent_clusters_result = main.FALSE
+
+            else:
+                main.log.report("Error in getting dataplane clusters from ONOS" + str(controller + 1) )
+                consistent_clusters_result = main.FALSE
+                main.log.warn("ONOS" + str(controller + 1) + " clusters response: " + repr(clusters[controller]) )
+        utilities.assert_equals(expect = main.TRUE,actual=consistent_clusters_result,
+                onpass="Clusters view is consistent across all ONOS nodes",
+                onfail="ONOS nodes have different views of clusters")
+        num_clusters =  len(json.loads(clusters[0])) #there should always only be one cluster
+        utilities.assert_equals(expect = 1, actual = num_clusters,
+                onpass="ONOS shows 1 SCC",
+                onfail="ONOS shows "+str(num_clusters) +" SCCs")
+
+
+        #paths
+        consistent_paths_result = main.TRUE
+        for controller in range( len( paths ) ):
+            if not "Error" in paths[controller]:
+                if paths[controller] == paths[0]:
+                    continue
+                else:#paths not consistent
+                    main.log.report("paths from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+                    consistent_paths_result = main.FALSE
+
+            else:
+                main.log.report("Error in getting paths from ONOS" + str(controller + 1) )
+                consistent_paths_result = main.FALSE
+                main.log.warn("ONOS" + str(controller + 1) + " paths response: " + repr(paths[controller]) )
+        utilities.assert_equals(expect = main.TRUE,actual=consistent_paths_result,
+                onpass="Paths count is consistent across all ONOS nodes",
+                onfail="ONOS nodes have different counts of paths")
 
 
         main.step("Comparing ONOS topology to MN")
@@ -639,14 +950,16 @@
             ports_results = ports_results and current_ports_result
             links_results = links_results and current_links_result
 
-        topo_result = devices_results and ports_results and links_results
+        topo_result = devices_results and ports_results and links_results\
+                and consistent_hosts_result and consistent_clusters_result\
+                and consistent_paths_result
         utilities.assert_equals(expect=main.TRUE, actual=topo_result,
                 onpass="Topology Check Test successful",
                 onfail="Topology Check Test NOT successful")
 
         final_assert = main.TRUE
         final_assert = final_assert and topo_result and flow_check \
-                and intent_check and consistent_mastership
+                and intent_check and consistent_mastership and roles_not_null
         utilities.assert_equals(expect=main.TRUE, actual=final_assert,
                 onpass="State check successful",
                 onfail="State check NOT successful")
@@ -708,6 +1021,24 @@
         import json
         main.case("Running ONOS Constant State Tests")
 
+        #Assert that each device has a master
+        ONOS1_master_not_null = main.ONOScli1.roles_not_null()
+        ONOS2_master_not_null = main.ONOScli2.roles_not_null()
+        ONOS3_master_not_null = main.ONOScli3.roles_not_null()
+        ONOS4_master_not_null = main.ONOScli4.roles_not_null()
+        ONOS5_master_not_null = main.ONOScli5.roles_not_null()
+        ONOS6_master_not_null = main.ONOScli6.roles_not_null()
+        ONOS7_master_not_null = main.ONOScli7.roles_not_null()
+        roles_not_null = ONOS1_master_not_null and ONOS2_master_not_null and\
+                ONOS3_master_not_null and ONOS4_master_not_null and\
+                ONOS5_master_not_null and ONOS6_master_not_null and\
+                ONOS7_master_not_null
+        utilities.assert_equals(expect = main.TRUE,actual=roles_not_null,
+                onpass="Each device has a master",
+                onfail="Some devices don't have a master assigned")
+
+
+
         main.step("Check if switch roles are consistent across all nodes")
         ONOS1_mastership = main.ONOScli1.roles()
         ONOS2_mastership = main.ONOScli2.roles()
@@ -739,7 +1070,6 @@
                 and ONOS1_mastership == ONOS5_mastership\
                 and ONOS1_mastership == ONOS6_mastership\
                 and ONOS1_mastership == ONOS7_mastership:
-                    #mastership_state = ONOS1_mastership
                     consistent_mastership = main.TRUE
                     main.log.report("Switch roles are consistent across all ONOS nodes")
         else:
@@ -822,25 +1152,36 @@
                     intent_check = main.TRUE
                     main.log.report("Intents are consistent across all ONOS nodes")
         else:
-            main.log.warn("ONOS1 intents: ", json.dumps(json.loads(ONOS1_intents),
-                sort_keys=True, indent=4, separators=(',', ': ')))
-            main.log.warn("ONOS2 intents: ", json.dumps(json.loads(ONOS2_intents),
-                sort_keys=True, indent=4, separators=(',', ': ')))
-            main.log.warn("ONOS3 intents: ", json.dumps(json.loads(ONOS3_intents),
-                sort_keys=True, indent=4, separators=(',', ': ')))
-            main.log.warn("ONOS4 intents: ", json.dumps(json.loads(ONOS4_intents),
-                sort_keys=True, indent=4, separators=(',', ': ')))
-            main.log.warn("ONOS5 intents: ", json.dumps(json.loads(ONOS5_intents),
-                sort_keys=True, indent=4, separators=(',', ': ')))
-            main.log.warn("ONOS6 intents: ", json.dumps(json.loads(ONOS6_intents),
-                sort_keys=True, indent=4, separators=(',', ': ')))
-            main.log.warn("ONOS7 intents: ", json.dumps(json.loads(ONOS7_intents),
-                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS1 intents: ") 
+            print json.dumps(json.loads(ONOS1_intents),
+                sort_keys=True, indent=4, separators=(',', ': '))
+            main.log.warn("ONOS2 intents: ")
+            print json.dumps(json.loads(ONOS2_intents),
+                sort_keys=True, indent=4, separators=(',', ': '))
+            main.log.warn("ONOS3 intents: ")
+            print json.dumps(json.loads(ONOS3_intents),
+                sort_keys=True, indent=4, separators=(',', ': '))
+            main.log.warn("ONOS4 intents: ")
+            print json.dumps(json.loads(ONOS4_intents),
+                sort_keys=True, indent=4, separators=(',', ': '))
+            main.log.warn("ONOS5 intents: ")
+            print json.dumps(json.loads(ONOS5_intents),
+                sort_keys=True, indent=4, separators=(',', ': '))
+            main.log.warn("ONOS6 intents: ")
+            print json.dumps(json.loads(ONOS6_intents),
+                sort_keys=True, indent=4, separators=(',', ': '))
+            main.log.warn("ONOS7 intents: ")
+            print json.dumps(json.loads(ONOS7_intents),
+                sort_keys=True, indent=4, separators=(',', ': '))
         utilities.assert_equals(expect = main.TRUE,actual=intent_check,
                 onpass="Intents are consistent across all ONOS nodes",
                 onfail="ONOS nodes have different views of intents")
 
+        #NOTE: Hazelcast has no durability, so intents are lost
+        '''
         main.step("Compare current intents with intents before the failure")
+        #NOTE: this requires case 5 to pass for intent_state to be set.
+        #      maybe we should stop the test if that fails?
         if intent_state == ONOS1_intents:
             same_intents = main.TRUE
             main.log.report("Intents are consistent with before failure")
@@ -851,6 +1192,7 @@
                 onpass="Intents are consistent with before failure",
                 onfail="The Intents changed during failure")
         intent_check = intent_check and same_intents
+        '''
 
 
 
@@ -859,12 +1201,12 @@
         flows2=[]
         for i in range(28):
             main.log.info("Checking flow table on s" + str(i+1))
-            tmp_flows = main.Mininet2.get_flowTable("s"+str(i+1),1.0)
+            tmp_flows = main.Mininet2.get_flowTable(1.3, "s"+str(i+1))
             flows2.append(tmp_flows)
-            Flow_Tables = Flow_Tables and main.Mininet2.flow_comp(flow1=flows[i],flow2=tmp_flows)
+            temp_result = main.Mininet2.flow_comp(flow1=flows[i],flow2=tmp_flows)
+            Flow_Tables = Flow_Tables and temp_result
             if Flow_Tables == main.FALSE:
                 main.log.info("Differences in flow table for switch: "+str(i+1))
-                break
         if Flow_Tables == main.TRUE:
             main.log.report("No changes were found in the flow tables")
         utilities.assert_equals(expect=main.TRUE,actual=Flow_Tables,
@@ -880,7 +1222,7 @@
         #NOTE: checkForLoss returns main.FALSE with 0% packet loss
         for i in range(8,18):
             main.log.info("Checking for a loss in pings along flow from s" + str(i))
-            Loss_In_Pings = Loss_In_Pings or main.Mininet2.checkForLoss("/tmp/ping.h"+str(i))
+            Loss_In_Pings = main.Mininet2.checkForLoss("/tmp/ping.h"+str(i)) or Loss_In_Pings
         if Loss_In_Pings == main.TRUE:
             main.log.info("Loss in ping detected")
         elif Loss_In_Pings == main.ERROR:
@@ -891,10 +1233,12 @@
         utilities.assert_equals(expect=main.FALSE,actual=Loss_In_Pings,
                 onpass="No Loss of connectivity",
                 onfail="Loss of dataplane connectivity detected")
+        #NOTE: Since intents are not persisted with Hazelcast, we expect this
+        Loss_In_Pings = main.FALSE
 
 
         #TODO:add topology to this or leave as a seperate case?
-        result = mastership_check and intent_check and Flow_Tables and (not Loss_In_Pings)
+        result = mastership_check and intent_check and Flow_Tables and (not Loss_In_Pings) and roles_not_null 
         result = int(result)
         if result == main.TRUE:
             main.log.report("Constant State Tests Passed")
@@ -936,87 +1280,173 @@
         ports_results = main.TRUE
         links_results = main.TRUE
         topo_result = main.FALSE
-        start_time = time.time()
         elapsed = 0
         count = 0
-        while topo_result == main.FALSE and elapsed < 120:
+        main.step("Collecting topology information from ONOS")
+        start_time = time.time()
+        while topo_result == main.FALSE and elapsed < 60:
             count = count + 1
-            try:
-                main.step("Collecting topology information from ONOS")
-                devices = []
-                devices.append( main.ONOScli1.devices() )
-                devices.append( main.ONOScli2.devices() )
-                devices.append( main.ONOScli3.devices() )
-                devices.append( main.ONOScli4.devices() )
-                devices.append( main.ONOScli5.devices() )
-                devices.append( main.ONOScli6.devices() )
-                devices.append( main.ONOScli7.devices() )
-                '''
-                hosts = []
-                hosts.append( main.ONOScli1.hosts() )
-                hosts.append( main.ONOScli2.hosts() )
-                hosts.append( main.ONOScli3.hosts() )
-                hosts.append( main.ONOScli4.hosts() )
-                hosts.append( main.ONOScli5.hosts() )
-                hosts.append( main.ONOScli6.hosts() )
-                hosts.append( main.ONOScli7.hosts() )
-                '''
-                ports = []
-                ports.append( main.ONOScli1.ports() )
-                ports.append( main.ONOScli2.ports() )
-                ports.append( main.ONOScli3.ports() )
-                ports.append( main.ONOScli4.ports() )
-                ports.append( main.ONOScli5.ports() )
-                ports.append( main.ONOScli6.ports() )
-                ports.append( main.ONOScli7.ports() )
-                links = []
-                links.append( main.ONOScli1.links() )
-                links.append( main.ONOScli2.links() )
-                links.append( main.ONOScli3.links() )
-                links.append( main.ONOScli4.links() )
-                links.append( main.ONOScli5.links() )
-                links.append( main.ONOScli6.links() )
-                links.append( main.ONOScli7.links() )
+            if count > 1:
+                MNTopo = TestONTopology(main.Mininet1, ctrls) # can also add Intent API info for intent operations
+            cli_start = time.time()
+            devices = []
+            devices.append( main.ONOScli1.devices() )
+            devices.append( main.ONOScli2.devices() )
+            devices.append( main.ONOScli3.devices() )
+            devices.append( main.ONOScli4.devices() )
+            devices.append( main.ONOScli5.devices() )
+            devices.append( main.ONOScli6.devices() )
+            devices.append( main.ONOScli7.devices() )
+            hosts = []
+            hosts.append( main.ONOScli1.hosts() )
+            hosts.append( main.ONOScli2.hosts() )
+            hosts.append( main.ONOScli3.hosts() )
+            hosts.append( main.ONOScli4.hosts() )
+            hosts.append( main.ONOScli5.hosts() )
+            hosts.append( main.ONOScli6.hosts() )
+            hosts.append( main.ONOScli7.hosts() )
+            ports = []
+            ports.append( main.ONOScli1.ports() )
+            ports.append( main.ONOScli2.ports() )
+            ports.append( main.ONOScli3.ports() )
+            ports.append( main.ONOScli4.ports() )
+            ports.append( main.ONOScli5.ports() )
+            ports.append( main.ONOScli6.ports() )
+            ports.append( main.ONOScli7.ports() )
+            links = []
+            links.append( main.ONOScli1.links() )
+            links.append( main.ONOScli2.links() )
+            links.append( main.ONOScli3.links() )
+            links.append( main.ONOScli4.links() )
+            links.append( main.ONOScli5.links() )
+            links.append( main.ONOScli6.links() )
+            links.append( main.ONOScli7.links() )
+            clusters = []
+            clusters.append( main.ONOScli1.clusters() )
+            clusters.append( main.ONOScli2.clusters() )
+            clusters.append( main.ONOScli3.clusters() )
+            clusters.append( main.ONOScli4.clusters() )
+            clusters.append( main.ONOScli5.clusters() )
+            clusters.append( main.ONOScli6.clusters() )
+            clusters.append( main.ONOScli7.clusters() )
+            paths = []
+            temp_topo = main.ONOSbench.get_topology( main.ONOScli1.topology() )
+            paths.append( temp_topo.get('paths', False) )
+            temp_topo = main.ONOSbench.get_topology( main.ONOScli2.topology() )
+            paths.append( temp_topo.get('paths', False) )
+            temp_topo = main.ONOSbench.get_topology( main.ONOScli3.topology() )
+            paths.append( temp_topo.get('paths', False) )
+            temp_topo = main.ONOSbench.get_topology( main.ONOScli4.topology() )
+            paths.append( temp_topo.get('paths', False) )
+            temp_topo = main.ONOSbench.get_topology( main.ONOScli5.topology() )
+            paths.append( temp_topo.get('paths', False) )
+            temp_topo = main.ONOSbench.get_topology( main.ONOScli6.topology() )
+            paths.append( temp_topo.get('paths', False) )
+            temp_topo = main.ONOSbench.get_topology( main.ONOScli7.topology() )
+            paths.append( temp_topo.get('paths', False) )
 
-                for controller in range(7): #TODO parameterize the number of controllers
-                    if devices[controller] or not "Error" in devices[controller]:
-                        current_devices_result =  main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
-                    else:
-                        current_devices_result = main.FALSE
-                    utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
-                            onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
-                            onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
 
-                    if ports[controller] or not "Error" in ports[controller]:
-                        current_ports_result =  main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
-                    else:
-                        current_ports_result = main.FALSE
-                    utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
-                            onpass="ONOS"+str(int(controller+1))+" ports view is correct",
-                            onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
+            elapsed = time.time() - start_time
+            cli_time = time.time() - cli_start
+            print "CLI time: " + str(cli_time)
 
-                    if links[controller] or not "Error" in links[controller]:
-                        current_links_result =  main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
-                    else:
-                        current_links_result = main.FALSE
-                    utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
-                            onpass="ONOS"+str(int(controller+1))+" links view is correct",
-                            onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
-            except:
-                main.log.error("something went wrong in topo comparison")
-                main.log.warn( repr( devices ) )
-                main.log.warn( repr( ports ) )
-                main.log.warn( repr( links ) )
+            for controller in range(7): #TODO parameterize the number of controllers
+                if devices[controller] or not "Error" in devices[controller]:
+                    current_devices_result =  main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
+                else:
+                    current_devices_result = main.FALSE
+                utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
+                        onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
+                        onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
 
+                if ports[controller] or not "Error" in ports[controller]:
+                    current_ports_result =  main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
+                else:
+                    current_ports_result = main.FALSE
+                utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
+                        onpass="ONOS"+str(int(controller+1))+" ports view is correct",
+                        onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
+
+                if links[controller] or not "Error" in links[controller]:
+                    current_links_result =  main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
+                else:
+                    current_links_result = main.FALSE
+                utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
+                        onpass="ONOS"+str(int(controller+1))+" links view is correct",
+                        onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
             devices_results = devices_results and current_devices_result
             ports_results = ports_results and current_ports_result
             links_results = links_results and current_links_result
-            topo_result = devices_results and ports_results and links_results
-            elapsed = time.time() - start_time
-        time_threshold = elapsed < 1
-        topo_result = topo_result and time_threshold
-        #TODO make sure this step is non-blocking. IE add a timeout
-        main.log.report("Very crass estimate for topology discovery/convergence: " +\
+
+            #Compare json objects for hosts, dataplane clusters and paths
+
+            #hosts
+            consistent_hosts_result = main.TRUE
+            for controller in range( len( hosts ) ):
+                if not "Error" in hosts[controller]:
+                    if hosts[controller] == hosts[0]:
+                        continue
+                    else:#hosts not consistent
+                        main.log.report("hosts from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+                        main.log.warn( repr( hosts[controller] ) )
+                        consistent_hosts_result = main.FALSE
+
+                else:
+                    main.log.report("Error in getting ONOS hosts from ONOS" + str(controller + 1) )
+                    consistent_hosts_result = main.FALSE
+                    main.log.warn("ONOS" + str(controller + 1) + " hosts response: " + repr(hosts[controller]) )
+            utilities.assert_equals(expect = main.TRUE,actual=consistent_hosts_result,
+                    onpass="Hosts view is consistent across all ONOS nodes",
+                    onfail="ONOS nodes have different views of hosts")
+
+            #Strongly connected clusters of devices
+            consistent_clusters_result = main.TRUE
+            for controller in range( len( clusters ) ):
+                if not "Error" in clusters[controller]:
+                    if clusters[controller] == clusters[0]:
+                        continue
+                    else:#clusters not consistent
+                        main.log.report("clusters from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+                        consistent_clusters_result = main.FALSE
+
+                else:
+                    main.log.report("Error in getting dataplane clusters from ONOS" + str(controller + 1) )
+                    consistent_clusters_result = main.FALSE
+                    main.log.warn("ONOS" + str(controller + 1) + " clusters response: " + repr(clusters[controller]) )
+            utilities.assert_equals(expect = main.TRUE,actual=consistent_clusters_result,
+                    onpass="Clusters view is consistent across all ONOS nodes",
+                    onfail="ONOS nodes have different views of clusters")
+            num_clusters =  len(json.loads(clusters[0])) #there should always only be one cluster
+            utilities.assert_equals(expect = 1, actual = num_clusters,
+                    onpass="ONOS shows 1 SCC",
+                    onfail="ONOS shows "+str(num_clusters) +" SCCs")
+
+
+            #paths
+            consistent_paths_result = main.TRUE
+            for controller in range( len( paths ) ):
+                if not "Error" in paths[controller]:
+                    if paths[controller] == paths[0]:
+                        continue
+                    else:#paths not consistent
+                        main.log.report("paths from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+                        consistent_paths_result = main.FALSE
+
+                else:
+                    main.log.report("Error in getting paths from ONOS" + str(controller + 1) )
+                    consistent_paths_result = main.FALSE
+                    main.log.warn("ONOS" + str(controller + 1) + " paths response: " + repr(paths[controller]) )
+            utilities.assert_equals(expect = main.TRUE,actual=consistent_paths_result,
+                    onpass="Paths count is consistent across all ONOS nodes",
+                    onfail="ONOS nodes have different counts of paths")
+
+
+            topo_result = devices_results and ports_results and links_results\
+                    and consistent_hosts_result and consistent_clusters_result and consistent_paths_result
+
+        topo_result = topo_result and int(count <= 2)
+        note = "note it takes about "+str( int(cli_time) )+" seconds for the test to make all the cli calls to fetch the topology from each ONOS instance"
+        main.log.report("Very crass estimate for topology discovery/convergence("+ str(note) + "): " +\
                 str(elapsed) + " seconds, " + str(count) +" tries" )
         utilities.assert_equals(expect=main.TRUE, actual=topo_result,
                 onpass="Topology Check Test successful",
@@ -1085,13 +1515,18 @@
         #TODO: Make this switch parameterizable
         main.step("Kill s28 ")
         main.log.report("Deleting s28")
-        #FIXME: use new dynamic topo functions
         main.Mininet1.del_switch("s28")
         main.log.info("Waiting " + str(switch_sleep) + " seconds for switch down to be discovered")
         time.sleep(switch_sleep)
+        device = main.ONOScli1.get_device(dpid="0028")
         #Peek at the deleted switch
-        main.log.warn(main.ONOScli1.get_device(dpid="0028"))
-        #TODO do some sort of check here
+        main.log.warn( str(device) )
+        result = main.FALSE
+        if device and device['available'] == False:
+            result = main.TRUE
+        utilities.assert_equals(expect=main.TRUE,actual=result,
+                onpass="Kill switch succesful",
+                onfail="Failed to kill switch?")
 
     def CASE12 (self, main) :
         '''
@@ -1099,7 +1534,6 @@
         '''
         #NOTE: You should probably run a topology check after this
         import time
-        #FIXME: use new dynamic topo functions
         description = "Adding a switch to ensure it is discovered correctly"
         main.log.report(description)
         main.case(description)
@@ -1121,9 +1555,15 @@
                 ip7=ONOS7_ip,port7=ONOS7_port)
         main.log.info("Waiting " + str(switch_sleep) + " seconds for switch up to be discovered")
         time.sleep(switch_sleep)
-        #Peek at the added switch
-        main.log.warn(main.ONOScli1.get_device(dpid="0028"))
-        #TODO do some sort of check here
+        device = main.ONOScli1.get_device(dpid="0028")
+        #Peek at the deleted switch
+        main.log.warn( str(device) )
+        result = main.FALSE
+        if device and device['available'] == True:
+            result = main.TRUE
+        utilities.assert_equals(expect=main.TRUE,actual=result,
+                onpass="add switch succesful",
+                onfail="Failed to add switch?")
 
     def CASE13 (self, main) :
         '''
@@ -1137,8 +1577,26 @@
         main.step("Killing tcpdumps")
         main.Mininet2.stop_tcpdump()
 
+        main.step("Checking ONOS Logs for errors")
+        print "Checking logs for errors on ONOS1:"
+        print main.ONOSbench.check_logs(ONOS1_ip)
+        print "Checking logs for errors on ONOS2:"
+        print main.ONOSbench.check_logs(ONOS2_ip)
+        print "Checking logs for errors on ONOS3:"
+        print main.ONOSbench.check_logs(ONOS3_ip)
+        print "Checking logs for errors on ONOS4:"
+        print main.ONOSbench.check_logs(ONOS4_ip)
+        print "Checking logs for errors on ONOS5:"
+        print main.ONOSbench.check_logs(ONOS5_ip)
+        print "Checking logs for errors on ONOS6:"
+        print main.ONOSbench.check_logs(ONOS6_ip)
+        print "Checking logs for errors on ONOS7:"
+        print main.ONOSbench.check_logs(ONOS7_ip)
+
         main.step("Copying MN pcap and ONOS log files to test station")
         testname = main.TEST
+        teststation_user = main.params['TESTONUSER']
+        teststation_IP = main.params['TESTONIP']
         #NOTE: MN Pcap file is being saved to ~/packet_captures
         #       scp this file as MN and TestON aren't necessarily the same vm
         #FIXME: scp
@@ -1150,19 +1608,26 @@
         #NOTE: must end in /
         dst_dir = "~/packet_captures/"
         for f in log_files:
-            main.ONOSbench.secureCopy( "sdn", ONOS1_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS1_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS1-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS2_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS2_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS2-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS3_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS3_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS3-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS4_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS4_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS4-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS5_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS5_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS5-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS6_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS6_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS6-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS7_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS7_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS7-"+f )
 
         #std*.log's
@@ -1172,24 +1637,29 @@
         #NOTE: must end in /
         dst_dir = "~/packet_captures/"
         for f in log_files:
-            main.ONOSbench.secureCopy( "sdn", ONOS1_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS1_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS1-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS2_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS2_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS2-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS3_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS3_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS3-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS4_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS4_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS4-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS5_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS5_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS5-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS6_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS6_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS6-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS7_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS7_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS7-"+f )
 
 
-
-
         #sleep so scp can finish
         time.sleep(10)
         main.step("Packing and rotating pcap archives")
diff --git a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.params b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.params
index 0a8bd57..5b97b66 100644
--- a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.params
+++ b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.params
@@ -1,5 +1,5 @@
 <PARAMS>
-    <testcases>1,2,8,3,4,5,[6],7,8,4,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
+    <testcases>1,2,8,3,4,5,[6],8,7,4,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
     <ENV>
     <cellName>HA</cellName>
     </ENV>
@@ -28,7 +28,7 @@
         <port7>6633</port7>
     </CTRL>
     <TESTONUSER>admin</TESTONUSER>
-    <TESTONIP>10.128.30.10</TESTONIP>
+    <TESTONIP>10.128.30.9</TESTONIP>
     <PING>
         <source1>h8</source1>
         <source2>h9</source2>
diff --git a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py
index 2af775d..3fbc29d 100644
--- a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py
+++ b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py
@@ -82,7 +82,7 @@
         cell_result = main.ONOSbench.set_cell(cell_name)
         verify_result = main.ONOSbench.verify_cell()
 
-        #FIXME:this is short term fix 
+        #FIXME:this is short term fix
         main.log.report("Removing raft logs")
         main.ONOSbench.onos_remove_raft_logs()
         main.log.report("Uninstalling ONOS")
@@ -145,29 +145,32 @@
         main.step("Checking if ONOS is up yet")
         #TODO: Refactor
         # check bundle:list?
-        onos1_isup = main.ONOSbench.isup(ONOS1_ip)
-        if not onos1_isup:
-            main.log.report("ONOS1 didn't start!")
-        onos2_isup = main.ONOSbench.isup(ONOS2_ip)
-        if not onos2_isup:
-            main.log.report("ONOS2 didn't start!")
-        onos3_isup = main.ONOSbench.isup(ONOS3_ip)
-        if not onos3_isup:
-            main.log.report("ONOS3 didn't start!")
-        onos4_isup = main.ONOSbench.isup(ONOS4_ip)
-        if not onos4_isup:
-            main.log.report("ONOS4 didn't start!")
-        onos5_isup = main.ONOSbench.isup(ONOS5_ip)
-        if not onos5_isup:
-            main.log.report("ONOS5 didn't start!")
-        onos6_isup = main.ONOSbench.isup(ONOS6_ip)
-        if not onos6_isup:
-            main.log.report("ONOS6 didn't start!")
-        onos7_isup = main.ONOSbench.isup(ONOS7_ip)
-        if not onos7_isup:
-            main.log.report("ONOS7 didn't start!")
-        onos_isup_result = onos1_isup and onos2_isup and onos3_isup\
-                and onos4_isup and onos5_isup and onos6_isup and onos7_isup
+        for i in range(2):
+            onos1_isup = main.ONOSbench.isup(ONOS1_ip)
+            if not onos1_isup:
+                main.log.report("ONOS1 didn't start!")
+            onos2_isup = main.ONOSbench.isup(ONOS2_ip)
+            if not onos2_isup:
+                main.log.report("ONOS2 didn't start!")
+            onos3_isup = main.ONOSbench.isup(ONOS3_ip)
+            if not onos3_isup:
+                main.log.report("ONOS3 didn't start!")
+            onos4_isup = main.ONOSbench.isup(ONOS4_ip)
+            if not onos4_isup:
+                main.log.report("ONOS4 didn't start!")
+            onos5_isup = main.ONOSbench.isup(ONOS5_ip)
+            if not onos5_isup:
+                main.log.report("ONOS5 didn't start!")
+            onos6_isup = main.ONOSbench.isup(ONOS6_ip)
+            if not onos6_isup:
+                main.log.report("ONOS6 didn't start!")
+            onos7_isup = main.ONOSbench.isup(ONOS7_ip)
+            if not onos7_isup:
+                main.log.report("ONOS7 didn't start!")
+            onos_isup_result = onos1_isup and onos2_isup and onos3_isup\
+                    and onos4_isup and onos5_isup and onos6_isup and onos7_isup
+            if onos_isup_result == main.TRUE:
+                break
         # TODO: if it becomes an issue, we can retry this step  a few times
 
 
@@ -197,9 +200,9 @@
                 onfail="Test startup NOT successful")
 
 
-        #if case1_result==main.FALSE:
-        #    main.cleanup()
-        #    main.exit()
+        if case1_result==main.FALSE:
+            main.cleanup()
+            main.exit()
 
     def CASE2(self,main) :
         '''
@@ -209,25 +212,6 @@
         import json
         import re
 
-
-        '''
-        ONOS1_ip = main.params['CTRL']['ip1']
-        ONOS1_port = main.params['CTRL']['port1']
-        ONOS2_ip = main.params['CTRL']['ip2']
-        ONOS2_port = main.params['CTRL']['port2']
-        ONOS3_ip = main.params['CTRL']['ip3']
-        ONOS3_port = main.params['CTRL']['port3']
-        ONOS4_ip = main.params['CTRL']['ip4']
-        ONOS4_port = main.params['CTRL']['port4']
-        ONOS5_ip = main.params['CTRL']['ip5']
-        ONOS5_port = main.params['CTRL']['port5']
-        ONOS6_ip = main.params['CTRL']['ip6']
-        ONOS6_port = main.params['CTRL']['port6']
-        ONOS7_ip = main.params['CTRL']['ip7']
-        ONOS7_port = main.params['CTRL']['port7']
-        '''
-
-
         main.log.report("Assigning switches to controllers")
         main.case("Assigning Controllers")
         main.step("Assign switches to controllers")
@@ -265,8 +249,217 @@
                 onpass="Switch mastership assigned correctly",
                 onfail="Switches not assigned correctly to controllers")
 
-        #TODO: If assign roles is working reliably then manually 
-        #   assign mastership to the controller we want
+        #Manually assign mastership to the controller we want
+        role_call = main.TRUE
+        role_check = main.TRUE
+
+        device_id = main.ONOScli1.get_device("1000")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS1_ip)
+        if ONOS1_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("2800")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS1_ip)
+        if ONOS1_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("2000")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS2_ip)
+        if ONOS2_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3000")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS2_ip)
+        if ONOS2_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("5000")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS3_ip)
+        if ONOS3_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6000")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS3_ip)
+        if ONOS3_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id =  main.ONOScli1.get_device("3004")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS4_ip)
+        if ONOS4_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3008")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3009")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3010")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3011")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3012")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3013")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3014")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3015")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3016")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3017")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6007")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS6_ip)
+        if ONOS6_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6018")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6019")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6020")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6021")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6022")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6023")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6024")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6025")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6026")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6027")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        utilities.assert_equals(expect = main.TRUE,actual=role_call,
+                onpass="Re-assigned switch mastership to designated controller",
+                onfail="Something wrong with device_role calls")
+
+        utilities.assert_equals(expect = main.TRUE,actual=role_check,
+                onpass="Switches were successfully reassigned to designated controller",
+                onfail="Switches were not successfully reassigned")
+        mastership_check = mastership_check and role_call and role_check
+        utilities.assert_equals(expect = main.TRUE,actual=mastership_check,
+                onpass="Switch mastership correctly assigned",
+                onfail="Error in (re)assigning switch mastership")
 
 
     def CASE3(self,main) :
@@ -283,6 +476,16 @@
         main.step("Discovering  Hosts( Via pingall for now)")
         #FIXME: Once we have a host discovery mechanism, use that instead
 
+        #install onos-app-fwd
+        main.log.info("Install reactive forwarding app")
+        main.ONOScli1.feature_install("onos-app-fwd")
+        main.ONOScli2.feature_install("onos-app-fwd")
+        main.ONOScli3.feature_install("onos-app-fwd")
+        main.ONOScli4.feature_install("onos-app-fwd")
+        main.ONOScli5.feature_install("onos-app-fwd")
+        main.ONOScli6.feature_install("onos-app-fwd")
+        main.ONOScli7.feature_install("onos-app-fwd")
+
         #REACTIVE FWD test
         ping_result = main.FALSE
         time1 = time.time()
@@ -304,7 +507,7 @@
         #TODO:  move the host numbers to params
         import json
         intents_json= json.loads(main.ONOScli1.hosts())
-        intent_add_result = main.FALSE
+        intent_add_result = True
         for i in range(8,18):
             main.log.info("Adding host intent between h"+str(i)+" and h"+str(i+10))
             host1 =  "00:00:00:00:00:" + str(hex(i)[2:]).zfill(2).upper()
@@ -314,7 +517,10 @@
             host1_id = main.ONOScli1.get_host(host1)['id']
             host2_id = main.ONOScli1.get_host(host2)['id']
             tmp_result = main.ONOScli1.add_host_intent(host1_id, host2_id )
-            intent_add_result = intent_add_result and tmp_result
+            intent_add_result = bool(intent_add_result and tmp_result)
+        utilities.assert_equals(expect=True, actual=intent_add_result,
+                onpass="Switch mastership correctly assigned",
+                onfail="Error in (re)assigning switch mastership")
         #TODO Check if intents all exist in datastore
         #NOTE: Do we need to print this once the test is working?
         #main.log.info(json.dumps(json.loads(main.ONOScli1.intents(json_format=True)),
@@ -360,6 +566,25 @@
 
         main.step("Get the Mastership of each switch from each controller")
         global mastership_state
+        mastership_state = []
+
+        #Assert that each device has a master
+        ONOS1_master_not_null = main.ONOScli1.roles_not_null()
+        ONOS2_master_not_null = main.ONOScli2.roles_not_null()
+        ONOS3_master_not_null = main.ONOScli3.roles_not_null()
+        ONOS4_master_not_null = main.ONOScli4.roles_not_null()
+        ONOS5_master_not_null = main.ONOScli5.roles_not_null()
+        ONOS6_master_not_null = main.ONOScli6.roles_not_null()
+        ONOS7_master_not_null = main.ONOScli7.roles_not_null()
+        roles_not_null = ONOS1_master_not_null and ONOS2_master_not_null and\
+                ONOS3_master_not_null and ONOS4_master_not_null and\
+                ONOS5_master_not_null and ONOS6_master_not_null and\
+                ONOS7_master_not_null
+        utilities.assert_equals(expect = main.TRUE,actual=roles_not_null,
+                onpass="Each device has a master",
+                onfail="Some devices don't have a master assigned")
+
+
         ONOS1_mastership = main.ONOScli1.roles()
         ONOS2_mastership = main.ONOScli2.roles()
         ONOS3_mastership = main.ONOScli3.roles()
@@ -416,6 +641,7 @@
 
         main.step("Get the intents from each controller")
         global intent_state
+        intent_state = []
         ONOS1_intents = main.ONOScli1.intents( json_format=True )
         ONOS2_intents = main.ONOScli2.intents( json_format=True )
         ONOS3_intents = main.ONOScli3.intents( json_format=True )
@@ -470,6 +696,7 @@
 
         main.step("Get the flows from each controller")
         global flow_state
+        flow_state = []
         ONOS1_flows = main.ONOScli1.flows( json_format=True )
         ONOS2_flows = main.ONOScli2.flows( json_format=True )
         ONOS3_flows = main.ONOScli3.flows( json_format=True )
@@ -527,7 +754,7 @@
         global flows
         flows=[]
         for i in range(1,29):
-            flows.append(main.Mininet2.get_flowTable("s"+str(i),1.0))
+            flows.append(main.Mininet2.get_flowTable(1.3, "s"+str(i)))
 
         #TODO: Compare switch flow tables with ONOS flow tables
 
@@ -578,7 +805,6 @@
         devices.append( main.ONOScli5.devices() )
         devices.append( main.ONOScli6.devices() )
         devices.append( main.ONOScli7.devices() )
-        '''
         hosts = []
         hosts.append( main.ONOScli1.hosts() )
         hosts.append( main.ONOScli2.hosts() )
@@ -587,7 +813,6 @@
         hosts.append( main.ONOScli5.hosts() )
         hosts.append( main.ONOScli6.hosts() )
         hosts.append( main.ONOScli7.hosts() )
-        '''
         ports = []
         ports.append( main.ONOScli1.ports() )
         ports.append( main.ONOScli2.ports() )
@@ -604,6 +829,91 @@
         links.append( main.ONOScli5.links() )
         links.append( main.ONOScli6.links() )
         links.append( main.ONOScli7.links() )
+        clusters = []
+        clusters.append( main.ONOScli1.clusters() )
+        clusters.append( main.ONOScli2.clusters() )
+        clusters.append( main.ONOScli3.clusters() )
+        clusters.append( main.ONOScli4.clusters() )
+        clusters.append( main.ONOScli5.clusters() )
+        clusters.append( main.ONOScli6.clusters() )
+        clusters.append( main.ONOScli7.clusters() )
+        paths = []
+        temp_topo = main.ONOSbench.get_topology( main.ONOScli1.topology() )
+        paths.append( temp_topo.get('paths', False) )
+        temp_topo = main.ONOSbench.get_topology( main.ONOScli2.topology() )
+        paths.append( temp_topo.get('paths', False) )
+        temp_topo = main.ONOSbench.get_topology( main.ONOScli3.topology() )
+        paths.append( temp_topo.get('paths', False) )
+        temp_topo = main.ONOSbench.get_topology( main.ONOScli4.topology() )
+        paths.append( temp_topo.get('paths', False) )
+        temp_topo = main.ONOSbench.get_topology( main.ONOScli5.topology() )
+        paths.append( temp_topo.get('paths', False) )
+        temp_topo = main.ONOSbench.get_topology( main.ONOScli6.topology() )
+        paths.append( temp_topo.get('paths', False) )
+        temp_topo = main.ONOSbench.get_topology( main.ONOScli7.topology() )
+        paths.append( temp_topo.get('paths', False) )
+
+        #Compare json objects for hosts, dataplane clusters and paths
+
+        #hosts
+        consistent_hosts_result = main.TRUE
+        for controller in range( len( hosts ) ):
+            if not "Error" in hosts[controller]:
+                if hosts[controller] == hosts[0]:
+                    continue
+                else:#hosts not consistent
+                    main.log.report("hosts from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+                    main.log.warn( repr( hosts[controller] ) )
+                    consistent_hosts_result = main.FALSE
+
+            else:
+                main.log.report("Error in getting ONOS hosts from ONOS" + str(controller + 1) )
+                consistent_hosts_result = main.FALSE
+                main.log.warn("ONOS" + str(controller + 1) + " hosts response: " + repr(hosts[controller]) )
+        utilities.assert_equals(expect = main.TRUE,actual=consistent_hosts_result,
+                onpass="Hosts view is consistent across all ONOS nodes",
+                onfail="ONOS nodes have different views of hosts")
+
+        #Strongly connected clusters of devices
+        consistent_clusters_result = main.TRUE
+        for controller in range( len( clusters ) ):
+            if not "Error" in clusters[controller]:
+                if clusters[controller] == clusters[0]:
+                    continue
+                else:#clusters not consistent
+                    main.log.report("clusters from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+                    consistent_clusters_result = main.FALSE
+
+            else:
+                main.log.report("Error in getting dataplane clusters from ONOS" + str(controller + 1) )
+                consistent_clusters_result = main.FALSE
+                main.log.warn("ONOS" + str(controller + 1) + " clusters response: " + repr(clusters[controller]) )
+        utilities.assert_equals(expect = main.TRUE,actual=consistent_clusters_result,
+                onpass="Clusters view is consistent across all ONOS nodes",
+                onfail="ONOS nodes have different views of clusters")
+        num_clusters =  len(json.loads(clusters[0])) #there should always only be one cluster
+        utilities.assert_equals(expect = 1, actual = num_clusters,
+                onpass="ONOS shows 1 SCC",
+                onfail="ONOS shows "+str(num_clusters) +" SCCs")
+
+
+        #paths
+        consistent_paths_result = main.TRUE
+        for controller in range( len( paths ) ):
+            if not "Error" in paths[controller]:
+                if paths[controller] == paths[0]:
+                    continue
+                else:#paths not consistent
+                    main.log.report("paths from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+                    consistent_paths_result = main.FALSE
+
+            else:
+                main.log.report("Error in getting paths from ONOS" + str(controller + 1) )
+                consistent_paths_result = main.FALSE
+                main.log.warn("ONOS" + str(controller + 1) + " paths response: " + repr(paths[controller]) )
+        utilities.assert_equals(expect = main.TRUE,actual=consistent_paths_result,
+                onpass="Paths count is consistent across all ONOS nodes",
+                onfail="ONOS nodes have different counts of paths")
 
 
         main.step("Comparing ONOS topology to MN")
@@ -639,14 +949,16 @@
             ports_results = ports_results and current_ports_result
             links_results = links_results and current_links_result
 
-        topo_result = devices_results and ports_results and links_results
+        topo_result = devices_results and ports_results and links_results\
+                and consistent_hosts_result and consistent_clusters_result\
+                and consistent_paths_result
         utilities.assert_equals(expect=main.TRUE, actual=topo_result,
                 onpass="Topology Check Test successful",
                 onfail="Topology Check Test NOT successful")
 
         final_assert = main.TRUE
         final_assert = final_assert and topo_result and flow_check \
-                and intent_check and consistent_mastership
+                and intent_check and consistent_mastership and roles_not_null
         utilities.assert_equals(expect=main.TRUE, actual=final_assert,
                 onpass="State check successful",
                 onfail="State check NOT successful")
@@ -656,11 +968,14 @@
         '''
         The Failure case.
         '''
+        import time
         main.log.report("Killing 3 ONOS nodes")
         main.log.case("Restart minority of ONOS nodes")
         #TODO: Randomize these nodes
         main.ONOSbench.onos_kill(ONOS1_ip)
+        time.sleep(10)
         main.ONOSbench.onos_kill(ONOS2_ip)
+        time.sleep(10)
         main.ONOSbench.onos_kill(ONOS3_ip)
 
         main.step("Checking if ONOS is up yet")
@@ -694,6 +1009,24 @@
         import json
         main.case("Running ONOS Constant State Tests")
 
+        #Assert that each device has a master
+        ONOS1_master_not_null = main.ONOScli1.roles_not_null()
+        ONOS2_master_not_null = main.ONOScli2.roles_not_null()
+        ONOS3_master_not_null = main.ONOScli3.roles_not_null()
+        ONOS4_master_not_null = main.ONOScli4.roles_not_null()
+        ONOS5_master_not_null = main.ONOScli5.roles_not_null()
+        ONOS6_master_not_null = main.ONOScli6.roles_not_null()
+        ONOS7_master_not_null = main.ONOScli7.roles_not_null()
+        roles_not_null = ONOS1_master_not_null and ONOS2_master_not_null and\
+                ONOS3_master_not_null and ONOS4_master_not_null and\
+                ONOS5_master_not_null and ONOS6_master_not_null and\
+                ONOS7_master_not_null
+        utilities.assert_equals(expect = main.TRUE,actual=roles_not_null,
+                onpass="Each device has a master",
+                onfail="Some devices don't have a master assigned")
+
+
+
         main.step("Check if switch roles are consistent across all nodes")
         ONOS1_mastership = main.ONOScli1.roles()
         ONOS2_mastership = main.ONOScli2.roles()
@@ -702,20 +1035,6 @@
         ONOS5_mastership = main.ONOScli5.roles()
         ONOS6_mastership = main.ONOScli6.roles()
         ONOS7_mastership = main.ONOScli7.roles()
-        print type(ONOS1_mastership)
-        print ONOS1_mastership
-        print type(ONOS2_mastership)
-        print ONOS2_mastership
-        print type(ONOS3_mastership)
-        print ONOS3_mastership
-        print type(ONOS4_mastership)
-        print ONOS4_mastership
-        print type(ONOS5_mastership)
-        print ONOS5_mastership
-        print type(ONOS6_mastership)
-        print ONOS6_mastership
-        print type(ONOS7_mastership)
-        print ONOS7_mastership
         #print json.dumps(json.loads(ONOS1_mastership), sort_keys=True, indent=4, separators=(',', ': '))
         if "Error" in ONOS1_mastership or not ONOS1_mastership\
                 or "Error" in ONOS2_mastership or not ONOS2_mastership\
@@ -739,7 +1058,6 @@
                 and ONOS1_mastership == ONOS5_mastership\
                 and ONOS1_mastership == ONOS6_mastership\
                 and ONOS1_mastership == ONOS7_mastership:
-                    #mastership_state = ONOS1_mastership
                     consistent_mastership = main.TRUE
                     main.log.report("Switch roles are consistent across all ONOS nodes")
         else:
@@ -822,25 +1140,35 @@
                     intent_check = main.TRUE
                     main.log.report("Intents are consistent across all ONOS nodes")
         else:
-            main.log.warn("ONOS1 intents: ", json.dumps(json.loads(ONOS1_intents),
-                sort_keys=True, indent=4, separators=(',', ': ')))
-            main.log.warn("ONOS2 intents: ", json.dumps(json.loads(ONOS2_intents),
-                sort_keys=True, indent=4, separators=(',', ': ')))
-            main.log.warn("ONOS3 intents: ", json.dumps(json.loads(ONOS3_intents),
-                sort_keys=True, indent=4, separators=(',', ': ')))
-            main.log.warn("ONOS4 intents: ", json.dumps(json.loads(ONOS4_intents),
-                sort_keys=True, indent=4, separators=(',', ': ')))
-            main.log.warn("ONOS5 intents: ", json.dumps(json.loads(ONOS5_intents),
-                sort_keys=True, indent=4, separators=(',', ': ')))
-            main.log.warn("ONOS6 intents: ", json.dumps(json.loads(ONOS6_intents),
-                sort_keys=True, indent=4, separators=(',', ': ')))
-            main.log.warn("ONOS7 intents: ", json.dumps(json.loads(ONOS7_intents),
-                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS1 intents: ") 
+            print json.dumps(json.loads(ONOS1_intents),
+                sort_keys=True, indent=4, separators=(',', ': '))
+            main.log.warn("ONOS2 intents: ")
+            print json.dumps(json.loads(ONOS2_intents),
+                sort_keys=True, indent=4, separators=(',', ': '))
+            main.log.warn("ONOS3 intents: ")
+            print json.dumps(json.loads(ONOS3_intents),
+                sort_keys=True, indent=4, separators=(',', ': '))
+            main.log.warn("ONOS4 intents: ")
+            print json.dumps(json.loads(ONOS4_intents),
+                sort_keys=True, indent=4, separators=(',', ': '))
+            main.log.warn("ONOS5 intents: ")
+            print json.dumps(json.loads(ONOS5_intents),
+                sort_keys=True, indent=4, separators=(',', ': '))
+            main.log.warn("ONOS6 intents: ")
+            print json.dumps(json.loads(ONOS6_intents),
+                sort_keys=True, indent=4, separators=(',', ': '))
+            main.log.warn("ONOS7 intents: ")
+            print json.dumps(json.loads(ONOS7_intents),
+                sort_keys=True, indent=4, separators=(',', ': '))
         utilities.assert_equals(expect = main.TRUE,actual=intent_check,
                 onpass="Intents are consistent across all ONOS nodes",
                 onfail="ONOS nodes have different views of intents")
 
+        #NOTE: Hazelcast has no durability, so intents are lost
         main.step("Compare current intents with intents before the failure")
+        #NOTE: this requires case 5 to pass for intent_state to be set.
+        #      maybe we should stop the test if that fails?
         if intent_state == ONOS1_intents:
             same_intents = main.TRUE
             main.log.report("Intents are consistent with before failure")
@@ -859,12 +1187,12 @@
         flows2=[]
         for i in range(28):
             main.log.info("Checking flow table on s" + str(i+1))
-            tmp_flows = main.Mininet2.get_flowTable("s"+str(i+1),1.0)
+            tmp_flows = main.Mininet2.get_flowTable(1.3, "s"+str(i+1))
             flows2.append(tmp_flows)
-            Flow_Tables = Flow_Tables and main.Mininet2.flow_comp(flow1=flows[i],flow2=tmp_flows)
+            temp_result = main.Mininet2.flow_comp(flow1=flows[i],flow2=tmp_flows)
+            Flow_Tables = Flow_Tables and temp_result
             if Flow_Tables == main.FALSE:
                 main.log.info("Differences in flow table for switch: "+str(i+1))
-                break
         if Flow_Tables == main.TRUE:
             main.log.report("No changes were found in the flow tables")
         utilities.assert_equals(expect=main.TRUE,actual=Flow_Tables,
@@ -880,7 +1208,7 @@
         #NOTE: checkForLoss returns main.FALSE with 0% packet loss
         for i in range(8,18):
             main.log.info("Checking for a loss in pings along flow from s" + str(i))
-            Loss_In_Pings = Loss_In_Pings or main.Mininet2.checkForLoss("/tmp/ping.h"+str(i))
+            Loss_In_Pings = main.Mininet2.checkForLoss("/tmp/ping.h"+str(i)) or Loss_In_Pings
         if Loss_In_Pings == main.TRUE:
             main.log.info("Loss in ping detected")
         elif Loss_In_Pings == main.ERROR:
@@ -894,7 +1222,7 @@
 
 
         #TODO:add topology to this or leave as a seperate case?
-        result = mastership_check and intent_check and Flow_Tables and (not Loss_In_Pings)
+        result = mastership_check and intent_check and Flow_Tables and (not Loss_In_Pings) and roles_not_null 
         result = int(result)
         if result == main.TRUE:
             main.log.report("Constant State Tests Passed")
@@ -936,87 +1264,175 @@
         ports_results = main.TRUE
         links_results = main.TRUE
         topo_result = main.FALSE
-        start_time = time.time()
         elapsed = 0
         count = 0
-        while topo_result == main.FALSE and elapsed < 120:
+        main.step("Collecting topology information from ONOS")
+        start_time = time.time()
+        while topo_result == main.FALSE and elapsed < 60:
             count = count + 1
-            try:
-                main.step("Collecting topology information from ONOS")
-                devices = []
-                devices.append( main.ONOScli1.devices() )
-                devices.append( main.ONOScli2.devices() )
-                devices.append( main.ONOScli3.devices() )
-                devices.append( main.ONOScli4.devices() )
-                devices.append( main.ONOScli5.devices() )
-                devices.append( main.ONOScli6.devices() )
-                devices.append( main.ONOScli7.devices() )
-                '''
-                hosts = []
-                hosts.append( main.ONOScli1.hosts() )
-                hosts.append( main.ONOScli2.hosts() )
-                hosts.append( main.ONOScli3.hosts() )
-                hosts.append( main.ONOScli4.hosts() )
-                hosts.append( main.ONOScli5.hosts() )
-                hosts.append( main.ONOScli6.hosts() )
-                hosts.append( main.ONOScli7.hosts() )
-                '''
-                ports = []
-                ports.append( main.ONOScli1.ports() )
-                ports.append( main.ONOScli2.ports() )
-                ports.append( main.ONOScli3.ports() )
-                ports.append( main.ONOScli4.ports() )
-                ports.append( main.ONOScli5.ports() )
-                ports.append( main.ONOScli6.ports() )
-                ports.append( main.ONOScli7.ports() )
-                links = []
-                links.append( main.ONOScli1.links() )
-                links.append( main.ONOScli2.links() )
-                links.append( main.ONOScli3.links() )
-                links.append( main.ONOScli4.links() )
-                links.append( main.ONOScli5.links() )
-                links.append( main.ONOScli6.links() )
-                links.append( main.ONOScli7.links() )
+            if count > 1:
+                MNTopo = TestONTopology(main.Mininet1, ctrls) # can also add Intent API info for intent operations
+            cli_start = time.time()
+            devices = []
+            devices.append( main.ONOScli1.devices() )
+            devices.append( main.ONOScli2.devices() )
+            devices.append( main.ONOScli3.devices() )
+            devices.append( main.ONOScli4.devices() )
+            devices.append( main.ONOScli5.devices() )
+            devices.append( main.ONOScli6.devices() )
+            devices.append( main.ONOScli7.devices() )
+            hosts = []
+            hosts.append( main.ONOScli1.hosts() )
+            hosts.append( main.ONOScli2.hosts() )
+            hosts.append( main.ONOScli3.hosts() )
+            hosts.append( main.ONOScli4.hosts() )
+            hosts.append( main.ONOScli5.hosts() )
+            hosts.append( main.ONOScli6.hosts() )
+            hosts.append( main.ONOScli7.hosts() )
+            ports = []
+            ports.append( main.ONOScli1.ports() )
+            ports.append( main.ONOScli2.ports() )
+            ports.append( main.ONOScli3.ports() )
+            ports.append( main.ONOScli4.ports() )
+            ports.append( main.ONOScli5.ports() )
+            ports.append( main.ONOScli6.ports() )
+            ports.append( main.ONOScli7.ports() )
+            links = []
+            links.append( main.ONOScli1.links() )
+            links.append( main.ONOScli2.links() )
+            links.append( main.ONOScli3.links() )
+            links.append( main.ONOScli4.links() )
+            links.append( main.ONOScli5.links() )
+            links.append( main.ONOScli6.links() )
+            links.append( main.ONOScli7.links() )
+            clusters = []
+            clusters.append( main.ONOScli1.clusters() )
+            clusters.append( main.ONOScli2.clusters() )
+            clusters.append( main.ONOScli3.clusters() )
+            clusters.append( main.ONOScli4.clusters() )
+            clusters.append( main.ONOScli5.clusters() )
+            clusters.append( main.ONOScli6.clusters() )
+            clusters.append( main.ONOScli7.clusters() )
+            paths = []
+            temp_topo = main.ONOSbench.get_topology( main.ONOScli1.topology() )
+            paths.append( temp_topo.get('paths', False) )
+            temp_topo = main.ONOSbench.get_topology( main.ONOScli2.topology() )
+            paths.append( temp_topo.get('paths', False) )
+            temp_topo = main.ONOSbench.get_topology( main.ONOScli3.topology() )
+            paths.append( temp_topo.get('paths', False) )
+            temp_topo = main.ONOSbench.get_topology( main.ONOScli4.topology() )
+            paths.append( temp_topo.get('paths', False) )
+            temp_topo = main.ONOSbench.get_topology( main.ONOScli5.topology() )
+            paths.append( temp_topo.get('paths', False) )
+            temp_topo = main.ONOSbench.get_topology( main.ONOScli6.topology() )
+            paths.append( temp_topo.get('paths', False) )
+            temp_topo = main.ONOSbench.get_topology( main.ONOScli7.topology() )
+            paths.append( temp_topo.get('paths', False) )
 
-                for controller in range(7): #TODO parameterize the number of controllers
-                    if devices[controller] or not "Error" in devices[controller]:
-                        current_devices_result =  main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
-                    else:
-                        current_devices_result = main.FALSE
-                    utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
-                            onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
-                            onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
 
-                    if ports[controller] or not "Error" in ports[controller]:
-                        current_ports_result =  main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
-                    else:
-                        current_ports_result = main.FALSE
-                    utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
-                            onpass="ONOS"+str(int(controller+1))+" ports view is correct",
-                            onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
+            elapsed = time.time() - start_time
+            cli_time = time.time() - cli_start
+            print "CLI time: " + str(cli_time)
 
-                    if links[controller] or not "Error" in links[controller]:
-                        current_links_result =  main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
-                    else:
-                        current_links_result = main.FALSE
-                    utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
-                            onpass="ONOS"+str(int(controller+1))+" links view is correct",
-                            onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
-            except:
-                main.log.error("something went wrong in topo comparison")
-                main.log.warn( repr( devices ) )
-                main.log.warn( repr( ports ) )
-                main.log.warn( repr( links ) )
+            for controller in range(7): #TODO parameterize the number of controllers
+                if devices[controller] or not "Error" in devices[controller]:
+                    current_devices_result =  main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
+                else:
+                    current_devices_result = main.FALSE
+                utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
+                        onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
+                        onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
 
+                if ports[controller] or not "Error" in ports[controller]:
+                    current_ports_result =  main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
+                else:
+                    current_ports_result = main.FALSE
+                utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
+                        onpass="ONOS"+str(int(controller+1))+" ports view is correct",
+                        onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
+
+                if links[controller] or not "Error" in links[controller]:
+                    current_links_result =  main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
+                else:
+                    current_links_result = main.FALSE
+                utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
+                        onpass="ONOS"+str(int(controller+1))+" links view is correct",
+                        onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
             devices_results = devices_results and current_devices_result
             ports_results = ports_results and current_ports_result
             links_results = links_results and current_links_result
-            topo_result = devices_results and ports_results and links_results
-            elapsed = time.time() - start_time
-        time_threshold = elapsed < 1
-        topo_result = topo_result and time_threshold
-        #TODO make sure this step is non-blocking. IE add a timeout
-        main.log.report("Very crass estimate for topology discovery/convergence: " +\
+
+            #Compare json objects for hosts, dataplane clusters and paths
+
+            #hosts
+            consistent_hosts_result = main.TRUE
+            for controller in range( len( hosts ) ):
+                if not "Error" in hosts[controller]:
+                    if hosts[controller] == hosts[0]:
+                        continue
+                    else:#hosts not consistent
+                        main.log.report("hosts from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+                        main.log.warn( repr( hosts[controller] ) )
+                        consistent_hosts_result = main.FALSE
+
+                else:
+                    main.log.report("Error in getting ONOS hosts from ONOS" + str(controller + 1) )
+                    consistent_hosts_result = main.FALSE
+            if consistent_hosts_result == main.FALSE:
+                for controller in range( len( hosts ) ):
+                    main.log.warn("ONOS" + str(controller + 1) + " hosts response: " + repr(hosts[controller]) )
+            utilities.assert_equals(expect = main.TRUE,actual=consistent_hosts_result,
+                    onpass="Hosts view is consistent across all ONOS nodes",
+                    onfail="ONOS nodes have different views of hosts")
+
+            #Strongly connected clusters of devices
+            consistent_clusters_result = main.TRUE
+            for controller in range( len( clusters ) ):
+                if not "Error" in clusters[controller]:
+                    if clusters[controller] == clusters[0]:
+                        continue
+                    else:#clusters not consistent
+                        main.log.report("clusters from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+                        consistent_clusters_result = main.FALSE
+
+                else:
+                    main.log.report("Error in getting dataplane clusters from ONOS" + str(controller + 1) )
+                    consistent_clusters_result = main.FALSE
+                    main.log.warn("ONOS" + str(controller + 1) + " clusters response: " + repr(clusters[controller]) )
+            utilities.assert_equals(expect = main.TRUE,actual=consistent_clusters_result,
+                    onpass="Clusters view is consistent across all ONOS nodes",
+                    onfail="ONOS nodes have different views of clusters")
+            num_clusters =  len(json.loads(clusters[0])) #there should always only be one cluster
+            utilities.assert_equals(expect = 1, actual = num_clusters,
+                    onpass="ONOS shows 1 SCC",
+                    onfail="ONOS shows "+str(num_clusters) +" SCCs")
+
+
+            #paths
+            consistent_paths_result = main.TRUE
+            for controller in range( len( paths ) ):
+                if not "Error" in paths[controller]:
+                    if paths[controller] == paths[0]:
+                        continue
+                    else:#paths not consistent
+                        main.log.report("paths from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+                        consistent_paths_result = main.FALSE
+
+                else:
+                    main.log.report("Error in getting paths from ONOS" + str(controller + 1) )
+                    consistent_paths_result = main.FALSE
+                    main.log.warn("ONOS" + str(controller + 1) + " paths response: " + repr(paths[controller]) )
+            utilities.assert_equals(expect = main.TRUE,actual=consistent_paths_result,
+                    onpass="Paths count is consistent across all ONOS nodes",
+                    onfail="ONOS nodes have different counts of paths")
+
+
+            topo_result = devices_results and ports_results and links_results\
+                    and consistent_hosts_result and consistent_clusters_result and consistent_paths_result
+
+        topo_result = topo_result and int(count <= 2)
+        note = "note it takes about "+str( int(cli_time) )+" seconds for the test to make all the cli calls to fetch the topology from each ONOS instance"
+        main.log.report("Very crass estimate for topology discovery/convergence("+ str(note) + "): " +\
                 str(elapsed) + " seconds, " + str(count) +" tries" )
         utilities.assert_equals(expect=main.TRUE, actual=topo_result,
                 onpass="Topology Check Test successful",
@@ -1085,13 +1501,18 @@
         #TODO: Make this switch parameterizable
         main.step("Kill s28 ")
         main.log.report("Deleting s28")
-        #FIXME: use new dynamic topo functions
         main.Mininet1.del_switch("s28")
         main.log.info("Waiting " + str(switch_sleep) + " seconds for switch down to be discovered")
         time.sleep(switch_sleep)
+        device = main.ONOScli1.get_device(dpid="0028")
         #Peek at the deleted switch
-        main.log.warn(main.ONOScli1.get_device(dpid="0028"))
-        #TODO do some sort of check here
+        main.log.warn( str(device) )
+        result = main.FALSE
+        if device and device['available'] == False:
+            result = main.TRUE
+        utilities.assert_equals(expect=main.TRUE,actual=result,
+                onpass="Kill switch succesful",
+                onfail="Failed to kill switch?")
 
     def CASE12 (self, main) :
         '''
@@ -1099,7 +1520,6 @@
         '''
         #NOTE: You should probably run a topology check after this
         import time
-        #FIXME: use new dynamic topo functions
         description = "Adding a switch to ensure it is discovered correctly"
         main.log.report(description)
         main.case(description)
@@ -1121,9 +1541,15 @@
                 ip7=ONOS7_ip,port7=ONOS7_port)
         main.log.info("Waiting " + str(switch_sleep) + " seconds for switch up to be discovered")
         time.sleep(switch_sleep)
-        #Peek at the added switch
-        main.log.warn(main.ONOScli1.get_device(dpid="0028"))
-        #TODO do some sort of check here
+        device = main.ONOScli1.get_device(dpid="0028")
+        #Peek at the deleted switch
+        main.log.warn( str(device) )
+        result = main.FALSE
+        if device and device['available'] == True:
+            result = main.TRUE
+        utilities.assert_equals(expect=main.TRUE,actual=result,
+                onpass="add switch succesful",
+                onfail="Failed to add switch?")
 
     def CASE13 (self, main) :
         '''
@@ -1137,8 +1563,26 @@
         main.step("Killing tcpdumps")
         main.Mininet2.stop_tcpdump()
 
+        main.step("Checking ONOS Logs for errors")
+        print "Checking logs for errors on ONOS1:"
+        print main.ONOSbench.check_logs(ONOS1_ip)
+        print "Checking logs for errors on ONOS2:"
+        print main.ONOSbench.check_logs(ONOS2_ip)
+        print "Checking logs for errors on ONOS3:"
+        print main.ONOSbench.check_logs(ONOS3_ip)
+        print "Checking logs for errors on ONOS4:"
+        print main.ONOSbench.check_logs(ONOS4_ip)
+        print "Checking logs for errors on ONOS5:"
+        print main.ONOSbench.check_logs(ONOS5_ip)
+        print "Checking logs for errors on ONOS6:"
+        print main.ONOSbench.check_logs(ONOS6_ip)
+        print "Checking logs for errors on ONOS7:"
+        print main.ONOSbench.check_logs(ONOS7_ip)
+
         main.step("Copying MN pcap and ONOS log files to test station")
         testname = main.TEST
+        teststation_user = main.params['TESTONUSER']
+        teststation_IP = main.params['TESTONIP']
         #NOTE: MN Pcap file is being saved to ~/packet_captures
         #       scp this file as MN and TestON aren't necessarily the same vm
         #FIXME: scp
@@ -1150,19 +1594,26 @@
         #NOTE: must end in /
         dst_dir = "~/packet_captures/"
         for f in log_files:
-            main.ONOSbench.secureCopy( "sdn", ONOS1_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS1_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS1-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS2_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS2_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS2-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS3_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS3_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS3-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS4_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS4_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS4-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS5_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS5_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS5-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS6_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS6_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS6-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS7_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS7_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS7-"+f )
 
         #std*.log's
@@ -1172,24 +1623,29 @@
         #NOTE: must end in /
         dst_dir = "~/packet_captures/"
         for f in log_files:
-            main.ONOSbench.secureCopy( "sdn", ONOS1_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS1_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS1-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS2_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS2_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS2-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS3_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS3_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS3-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS4_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS4_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS4-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS5_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS5_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS5-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS6_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS6_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS6-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS7_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS7_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS7-"+f )
 
 
-
-
         #sleep so scp can finish
         time.sleep(10)
         main.step("Packing and rotating pcap archives")
diff --git a/TestON/tests/HATestSanity/HATestSanity.params b/TestON/tests/HATestSanity/HATestSanity.params
index e1e75f9..5259a47 100644
--- a/TestON/tests/HATestSanity/HATestSanity.params
+++ b/TestON/tests/HATestSanity/HATestSanity.params
@@ -1,5 +1,5 @@
 <PARAMS>
-    <testcases>1,2,8,3,4,5,[6],7,8,4,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
+    <testcases>1,2,8,3,4,5,[6],8,7,4,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
     <ENV>
     <cellName>HA</cellName>
     </ENV>
@@ -28,7 +28,7 @@
         <port7>6633</port7>
     </CTRL>
     <TESTONUSER>admin</TESTONUSER>
-    <TESTONIP>10.128.30.10</TESTONIP>
+    <TESTONIP>10.128.30.9</TESTONIP>
     <PING>
         <source1>h8</source1>
         <source2>h9</source2>
diff --git a/TestON/tests/HATestSanity/HATestSanity.py b/TestON/tests/HATestSanity/HATestSanity.py
index 614d514..4e5b463 100644
--- a/TestON/tests/HATestSanity/HATestSanity.py
+++ b/TestON/tests/HATestSanity/HATestSanity.py
@@ -214,25 +214,6 @@
         import json
         import re
 
-
-        '''
-        ONOS1_ip = main.params['CTRL']['ip1']
-        ONOS1_port = main.params['CTRL']['port1']
-        ONOS2_ip = main.params['CTRL']['ip2']
-        ONOS2_port = main.params['CTRL']['port2']
-        ONOS3_ip = main.params['CTRL']['ip3']
-        ONOS3_port = main.params['CTRL']['port3']
-        ONOS4_ip = main.params['CTRL']['ip4']
-        ONOS4_port = main.params['CTRL']['port4']
-        ONOS5_ip = main.params['CTRL']['ip5']
-        ONOS5_port = main.params['CTRL']['port5']
-        ONOS6_ip = main.params['CTRL']['ip6']
-        ONOS6_port = main.params['CTRL']['port6']
-        ONOS7_ip = main.params['CTRL']['ip7']
-        ONOS7_port = main.params['CTRL']['port7']
-        '''
-
-
         main.log.report("Assigning switches to controllers")
         main.case("Assigning Controllers")
         main.step("Assign switches to controllers")
@@ -270,8 +251,217 @@
                 onpass="Switch mastership assigned correctly",
                 onfail="Switches not assigned correctly to controllers")
 
-        #TODO: If assign roles is working reliably then manually 
-        #   assign mastership to the controller we want
+        #Manually assign mastership to the controller we want
+        role_call = main.TRUE
+        role_check = main.TRUE
+
+        device_id = main.ONOScli1.get_device("1000")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS1_ip)
+        if ONOS1_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("2800")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS1_ip)
+        if ONOS1_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("2000")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS2_ip)
+        if ONOS2_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3000")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS2_ip)
+        if ONOS2_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("5000")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS3_ip)
+        if ONOS3_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6000")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS3_ip)
+        if ONOS3_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id =  main.ONOScli1.get_device("3004")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS4_ip)
+        if ONOS4_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3008")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3009")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3010")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3011")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3012")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3013")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3014")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3015")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3016")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3017")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6007")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS6_ip)
+        if ONOS6_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6018")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6019")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6020")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6021")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6022")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6023")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6024")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6025")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6026")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6027")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        utilities.assert_equals(expect = main.TRUE,actual=role_call,
+                onpass="Re-assigned switch mastership to designated controller",
+                onfail="Something wrong with device_role calls")
+
+        utilities.assert_equals(expect = main.TRUE,actual=role_check,
+                onpass="Switches were successfully reassigned to designated controller",
+                onfail="Switches were not successfully reassigned")
+        mastership_check = mastership_check and role_call and role_check
+        utilities.assert_equals(expect = main.TRUE,actual=mastership_check,
+                onpass="Switch mastership correctly assigned",
+                onfail="Error in (re)assigning switch mastership")
 
 
     def CASE3(self,main) :
@@ -288,6 +478,16 @@
         main.step("Discovering  Hosts( Via pingall for now)")
         #FIXME: Once we have a host discovery mechanism, use that instead
 
+        #install onos-app-fwd
+        main.log.info("Install reactive forwarding app")
+        main.ONOScli1.feature_install("onos-app-fwd")
+        main.ONOScli2.feature_install("onos-app-fwd")
+        main.ONOScli3.feature_install("onos-app-fwd")
+        main.ONOScli4.feature_install("onos-app-fwd")
+        main.ONOScli5.feature_install("onos-app-fwd")
+        main.ONOScli6.feature_install("onos-app-fwd")
+        main.ONOScli7.feature_install("onos-app-fwd")
+
         #REACTIVE FWD test
         ping_result = main.FALSE
         time1 = time.time()
@@ -309,7 +509,7 @@
         #TODO:  move the host numbers to params
         import json
         intents_json= json.loads(main.ONOScli1.hosts())
-        intent_add_result = main.FALSE
+        intent_add_result = True
         for i in range(8,18):
             main.log.info("Adding host intent between h"+str(i)+" and h"+str(i+10))
             host1 =  "00:00:00:00:00:" + str(hex(i)[2:]).zfill(2).upper()
@@ -319,7 +519,10 @@
             host1_id = main.ONOScli1.get_host(host1)['id']
             host2_id = main.ONOScli1.get_host(host2)['id']
             tmp_result = main.ONOScli1.add_host_intent(host1_id, host2_id )
-            intent_add_result = intent_add_result and tmp_result
+            intent_add_result = bool(intent_add_result and tmp_result)
+        utilities.assert_equals(expect=True, actual=intent_add_result,
+                onpass="Switch mastership correctly assigned",
+                onfail="Error in (re)assigning switch mastership")
         #TODO Check if intents all exist in datastore
         #NOTE: Do we need to print this once the test is working?
         #main.log.info(json.dumps(json.loads(main.ONOScli1.intents(json_format=True)),
@@ -365,6 +568,25 @@
 
         main.step("Get the Mastership of each switch from each controller")
         global mastership_state
+        mastership_state = []
+
+        #Assert that each device has a master
+        ONOS1_master_not_null = main.ONOScli1.roles_not_null()
+        ONOS2_master_not_null = main.ONOScli2.roles_not_null()
+        ONOS3_master_not_null = main.ONOScli3.roles_not_null()
+        ONOS4_master_not_null = main.ONOScli4.roles_not_null()
+        ONOS5_master_not_null = main.ONOScli5.roles_not_null()
+        ONOS6_master_not_null = main.ONOScli6.roles_not_null()
+        ONOS7_master_not_null = main.ONOScli7.roles_not_null()
+        roles_not_null = ONOS1_master_not_null and ONOS2_master_not_null and\
+                ONOS3_master_not_null and ONOS4_master_not_null and\
+                ONOS5_master_not_null and ONOS6_master_not_null and\
+                ONOS7_master_not_null
+        utilities.assert_equals(expect = main.TRUE,actual=roles_not_null,
+                onpass="Each device has a master",
+                onfail="Some devices don't have a master assigned")
+
+
         ONOS1_mastership = main.ONOScli1.roles()
         ONOS2_mastership = main.ONOScli2.roles()
         ONOS3_mastership = main.ONOScli3.roles()
@@ -421,6 +643,7 @@
 
         main.step("Get the intents from each controller")
         global intent_state
+        intent_state = []
         ONOS1_intents = main.ONOScli1.intents( json_format=True )
         ONOS2_intents = main.ONOScli2.intents( json_format=True )
         ONOS3_intents = main.ONOScli3.intents( json_format=True )
@@ -475,6 +698,7 @@
 
         main.step("Get the flows from each controller")
         global flow_state
+        flow_state = []
         ONOS1_flows = main.ONOScli1.flows( json_format=True )
         ONOS2_flows = main.ONOScli2.flows( json_format=True )
         ONOS3_flows = main.ONOScli3.flows( json_format=True )
@@ -532,7 +756,7 @@
         global flows
         flows=[]
         for i in range(1,29):
-            flows.append(main.Mininet2.get_flowTable("s"+str(i),1.0))
+            flows.append(main.Mininet2.get_flowTable(1.3, "s"+str(i)))
 
         #TODO: Compare switch flow tables with ONOS flow tables
 
@@ -583,7 +807,6 @@
         devices.append( main.ONOScli5.devices() )
         devices.append( main.ONOScli6.devices() )
         devices.append( main.ONOScli7.devices() )
-        '''
         hosts = []
         hosts.append( main.ONOScli1.hosts() )
         hosts.append( main.ONOScli2.hosts() )
@@ -592,7 +815,6 @@
         hosts.append( main.ONOScli5.hosts() )
         hosts.append( main.ONOScli6.hosts() )
         hosts.append( main.ONOScli7.hosts() )
-        '''
         ports = []
         ports.append( main.ONOScli1.ports() )
         ports.append( main.ONOScli2.ports() )
@@ -609,6 +831,91 @@
         links.append( main.ONOScli5.links() )
         links.append( main.ONOScli6.links() )
         links.append( main.ONOScli7.links() )
+        clusters = []
+        clusters.append( main.ONOScli1.clusters() )
+        clusters.append( main.ONOScli2.clusters() )
+        clusters.append( main.ONOScli3.clusters() )
+        clusters.append( main.ONOScli4.clusters() )
+        clusters.append( main.ONOScli5.clusters() )
+        clusters.append( main.ONOScli6.clusters() )
+        clusters.append( main.ONOScli7.clusters() )
+        paths = []
+        temp_topo = main.ONOSbench.get_topology( main.ONOScli1.topology() )
+        paths.append( temp_topo.get('paths', False) )
+        temp_topo = main.ONOSbench.get_topology( main.ONOScli2.topology() )
+        paths.append( temp_topo.get('paths', False) )
+        temp_topo = main.ONOSbench.get_topology( main.ONOScli3.topology() )
+        paths.append( temp_topo.get('paths', False) )
+        temp_topo = main.ONOSbench.get_topology( main.ONOScli4.topology() )
+        paths.append( temp_topo.get('paths', False) )
+        temp_topo = main.ONOSbench.get_topology( main.ONOScli5.topology() )
+        paths.append( temp_topo.get('paths', False) )
+        temp_topo = main.ONOSbench.get_topology( main.ONOScli6.topology() )
+        paths.append( temp_topo.get('paths', False) )
+        temp_topo = main.ONOSbench.get_topology( main.ONOScli7.topology() )
+        paths.append( temp_topo.get('paths', False) )
+
+        #Compare json objects for hosts, dataplane clusters and paths
+
+        #hosts
+        consistent_hosts_result = main.TRUE
+        for controller in range( len( hosts ) ):
+            if not "Error" in hosts[controller]:
+                if hosts[controller] == hosts[0]:
+                    continue
+                else:#hosts not consistent
+                    main.log.report("hosts from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+                    main.log.warn( repr( hosts[controller] ) )
+                    consistent_hosts_result = main.FALSE
+
+            else:
+                main.log.report("Error in getting ONOS hosts from ONOS" + str(controller + 1) )
+                consistent_hosts_result = main.FALSE
+                main.log.warn("ONOS" + str(controller + 1) + " hosts response: " + repr(hosts[controller]) )
+        utilities.assert_equals(expect = main.TRUE,actual=consistent_hosts_result,
+                onpass="Hosts view is consistent across all ONOS nodes",
+                onfail="ONOS nodes have different views of hosts")
+
+        #Strongly connected clusters of devices
+        consistent_clusters_result = main.TRUE
+        for controller in range( len( clusters ) ):
+            if not "Error" in clusters[controller]:
+                if clusters[controller] == clusters[0]:
+                    continue
+                else:#clusters not consistent
+                    main.log.report("clusters from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+                    consistent_clusters_result = main.FALSE
+
+            else:
+                main.log.report("Error in getting dataplane clusters from ONOS" + str(controller + 1) )
+                consistent_clusters_result = main.FALSE
+                main.log.warn("ONOS" + str(controller + 1) + " clusters response: " + repr(clusters[controller]) )
+        utilities.assert_equals(expect = main.TRUE,actual=consistent_clusters_result,
+                onpass="Clusters view is consistent across all ONOS nodes",
+                onfail="ONOS nodes have different views of clusters")
+        num_clusters =  len(json.loads(clusters[0])) #there should always only be one cluster
+        utilities.assert_equals(expect = 1, actual = num_clusters,
+                onpass="ONOS shows 1 SCC",
+                onfail="ONOS shows "+str(num_clusters) +" SCCs")
+
+
+        #paths
+        consistent_paths_result = main.TRUE
+        for controller in range( len( paths ) ):
+            if not "Error" in paths[controller]:
+                if paths[controller] == paths[0]:
+                    continue
+                else:#paths not consistent
+                    main.log.report("paths from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+                    consistent_paths_result = main.FALSE
+
+            else:
+                main.log.report("Error in getting paths from ONOS" + str(controller + 1) )
+                consistent_paths_result = main.FALSE
+                main.log.warn("ONOS" + str(controller + 1) + " paths response: " + repr(paths[controller]) )
+        utilities.assert_equals(expect = main.TRUE,actual=consistent_paths_result,
+                onpass="Paths count is consistent across all ONOS nodes",
+                onfail="ONOS nodes have different counts of paths")
 
 
         main.step("Comparing ONOS topology to MN")
@@ -644,14 +951,16 @@
             ports_results = ports_results and current_ports_result
             links_results = links_results and current_links_result
 
-        topo_result = devices_results and ports_results and links_results
+        topo_result = devices_results and ports_results and links_results\
+                and consistent_hosts_result and consistent_clusters_result\
+                and consistent_paths_result
         utilities.assert_equals(expect=main.TRUE, actual=topo_result,
                 onpass="Topology Check Test successful",
                 onfail="Topology Check Test NOT successful")
 
         final_assert = main.TRUE
         final_assert = final_assert and topo_result and flow_check \
-                and intent_check and consistent_mastership
+                and intent_check and consistent_mastership and roles_not_null
         utilities.assert_equals(expect=main.TRUE, actual=final_assert,
                 onpass="State check successful",
                 onfail="State check NOT successful")
@@ -676,6 +985,24 @@
         import json
         main.case("Running ONOS Constant State Tests")
 
+        #Assert that each device has a master
+        ONOS1_master_not_null = main.ONOScli1.roles_not_null()
+        ONOS2_master_not_null = main.ONOScli2.roles_not_null()
+        ONOS3_master_not_null = main.ONOScli3.roles_not_null()
+        ONOS4_master_not_null = main.ONOScli4.roles_not_null()
+        ONOS5_master_not_null = main.ONOScli5.roles_not_null()
+        ONOS6_master_not_null = main.ONOScli6.roles_not_null()
+        ONOS7_master_not_null = main.ONOScli7.roles_not_null()
+        roles_not_null = ONOS1_master_not_null and ONOS2_master_not_null and\
+                ONOS3_master_not_null and ONOS4_master_not_null and\
+                ONOS5_master_not_null and ONOS6_master_not_null and\
+                ONOS7_master_not_null
+        utilities.assert_equals(expect = main.TRUE,actual=roles_not_null,
+                onpass="Each device has a master",
+                onfail="Some devices don't have a master assigned")
+
+
+
         main.step("Check if switch roles are consistent across all nodes")
         ONOS1_mastership = main.ONOScli1.roles()
         ONOS2_mastership = main.ONOScli2.roles()
@@ -707,7 +1034,6 @@
                 and ONOS1_mastership == ONOS5_mastership\
                 and ONOS1_mastership == ONOS6_mastership\
                 and ONOS1_mastership == ONOS7_mastership:
-                    #mastership_state = ONOS1_mastership
                     consistent_mastership = main.TRUE
                     main.log.report("Switch roles are consistent across all ONOS nodes")
         else:
@@ -789,25 +1115,35 @@
                     intent_check = main.TRUE
                     main.log.report("Intents are consistent across all ONOS nodes")
         else:
-            main.log.warn("ONOS1 intents: ", json.dumps(json.loads(ONOS1_intents),
-                sort_keys=True, indent=4, separators=(',', ': ')))
-            main.log.warn("ONOS2 intents: ", json.dumps(json.loads(ONOS2_intents),
-                sort_keys=True, indent=4, separators=(',', ': ')))
-            main.log.warn("ONOS3 intents: ", json.dumps(json.loads(ONOS3_intents),
-                sort_keys=True, indent=4, separators=(',', ': ')))
-            main.log.warn("ONOS4 intents: ", json.dumps(json.loads(ONOS4_intents),
-                sort_keys=True, indent=4, separators=(',', ': ')))
-            main.log.warn("ONOS5 intents: ", json.dumps(json.loads(ONOS5_intents),
-                sort_keys=True, indent=4, separators=(',', ': ')))
-            main.log.warn("ONOS6 intents: ", json.dumps(json.loads(ONOS6_intents),
-                sort_keys=True, indent=4, separators=(',', ': ')))
-            main.log.warn("ONOS7 intents: ", json.dumps(json.loads(ONOS7_intents),
-                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS1 intents: ") 
+            print json.dumps(json.loads(ONOS1_intents),
+                sort_keys=True, indent=4, separators=(',', ': '))
+            main.log.warn("ONOS2 intents: ")
+            print json.dumps(json.loads(ONOS2_intents),
+                sort_keys=True, indent=4, separators=(',', ': '))
+            main.log.warn("ONOS3 intents: ")
+            print json.dumps(json.loads(ONOS3_intents),
+                sort_keys=True, indent=4, separators=(',', ': '))
+            main.log.warn("ONOS4 intents: ")
+            print json.dumps(json.loads(ONOS4_intents),
+                sort_keys=True, indent=4, separators=(',', ': '))
+            main.log.warn("ONOS5 intents: ")
+            print json.dumps(json.loads(ONOS5_intents),
+                sort_keys=True, indent=4, separators=(',', ': '))
+            main.log.warn("ONOS6 intents: ")
+            print json.dumps(json.loads(ONOS6_intents),
+                sort_keys=True, indent=4, separators=(',', ': '))
+            main.log.warn("ONOS7 intents: ")
+            print json.dumps(json.loads(ONOS7_intents),
+                sort_keys=True, indent=4, separators=(',', ': '))
         utilities.assert_equals(expect = main.TRUE,actual=intent_check,
                 onpass="Intents are consistent across all ONOS nodes",
                 onfail="ONOS nodes have different views of intents")
 
+        #NOTE: Hazelcast has no durability, so intents are lost
         main.step("Compare current intents with intents before the failure")
+        #NOTE: this requires case 5 to pass for intent_state to be set.
+        #      maybe we should stop the test if that fails?
         if intent_state == ONOS1_intents:
             same_intents = main.TRUE
             main.log.report("Intents are consistent with before failure")
@@ -826,12 +1162,12 @@
         flows2=[]
         for i in range(28):
             main.log.info("Checking flow table on s" + str(i+1))
-            tmp_flows = main.Mininet2.get_flowTable("s"+str(i+1),1.0)
+            tmp_flows = main.Mininet2.get_flowTable(1.3, "s"+str(i+1))
             flows2.append(tmp_flows)
-            Flow_Tables = Flow_Tables and main.Mininet2.flow_comp(flow1=flows[i],flow2=tmp_flows)
+            temp_result = main.Mininet2.flow_comp(flow1=flows[i],flow2=tmp_flows)
+            Flow_Tables = Flow_Tables and temp_result
             if Flow_Tables == main.FALSE:
                 main.log.info("Differences in flow table for switch: "+str(i+1))
-                break
         if Flow_Tables == main.TRUE:
             main.log.report("No changes were found in the flow tables")
         utilities.assert_equals(expect=main.TRUE,actual=Flow_Tables,
@@ -847,7 +1183,7 @@
         #NOTE: checkForLoss returns main.FALSE with 0% packet loss
         for i in range(8,18):
             main.log.info("Checking for a loss in pings along flow from s" + str(i))
-            Loss_In_Pings = Loss_In_Pings or main.Mininet2.checkForLoss("/tmp/ping.h"+str(i))
+            Loss_In_Pings = main.Mininet2.checkForLoss("/tmp/ping.h"+str(i)) or Loss_In_Pings
         if Loss_In_Pings == main.TRUE:
             main.log.info("Loss in ping detected")
         elif Loss_In_Pings == main.ERROR:
@@ -861,7 +1197,7 @@
 
 
         #TODO:add topology to this or leave as a seperate case?
-        result = mastership_check and intent_check and Flow_Tables and (not Loss_In_Pings)
+        result = mastership_check and intent_check and Flow_Tables and (not Loss_In_Pings) and roles_not_null 
         result = int(result)
         if result == main.TRUE:
             main.log.report("Constant State Tests Passed")
@@ -903,89 +1239,173 @@
         ports_results = main.TRUE
         links_results = main.TRUE
         topo_result = main.FALSE
-        start_time = time.time()
         elapsed = 0
         count = 0
-        while topo_result == main.FALSE and elapsed < 120:
-            print "cond 1:" + str(topo_result == main.FALSE)
-            print "cond 2:" + str(elapsed < 120)
+        main.step("Collecting topology information from ONOS")
+        start_time = time.time()
+        while topo_result == main.FALSE and elapsed < 60:
             count = count + 1
-            try:
-                main.step("Collecting topology information from ONOS")
-                devices = []
-                devices.append( main.ONOScli1.devices() )
-                devices.append( main.ONOScli2.devices() )
-                devices.append( main.ONOScli3.devices() )
-                devices.append( main.ONOScli4.devices() )
-                devices.append( main.ONOScli5.devices() )
-                devices.append( main.ONOScli6.devices() )
-                devices.append( main.ONOScli7.devices() )
-                '''
-                hosts = []
-                hosts.append( main.ONOScli1.hosts() )
-                hosts.append( main.ONOScli2.hosts() )
-                hosts.append( main.ONOScli3.hosts() )
-                hosts.append( main.ONOScli4.hosts() )
-                hosts.append( main.ONOScli5.hosts() )
-                hosts.append( main.ONOScli6.hosts() )
-                hosts.append( main.ONOScli7.hosts() )
-                '''
-                ports = []
-                ports.append( main.ONOScli1.ports() )
-                ports.append( main.ONOScli2.ports() )
-                ports.append( main.ONOScli3.ports() )
-                ports.append( main.ONOScli4.ports() )
-                ports.append( main.ONOScli5.ports() )
-                ports.append( main.ONOScli6.ports() )
-                ports.append( main.ONOScli7.ports() )
-                links = []
-                links.append( main.ONOScli1.links() )
-                links.append( main.ONOScli2.links() )
-                links.append( main.ONOScli3.links() )
-                links.append( main.ONOScli4.links() )
-                links.append( main.ONOScli5.links() )
-                links.append( main.ONOScli6.links() )
-                links.append( main.ONOScli7.links() )
+            if count > 1:
+                MNTopo = TestONTopology(main.Mininet1, ctrls) # can also add Intent API info for intent operations
+            cli_start = time.time()
+            devices = []
+            devices.append( main.ONOScli1.devices() )
+            devices.append( main.ONOScli2.devices() )
+            devices.append( main.ONOScli3.devices() )
+            devices.append( main.ONOScli4.devices() )
+            devices.append( main.ONOScli5.devices() )
+            devices.append( main.ONOScli6.devices() )
+            devices.append( main.ONOScli7.devices() )
+            hosts = []
+            hosts.append( main.ONOScli1.hosts() )
+            hosts.append( main.ONOScli2.hosts() )
+            hosts.append( main.ONOScli3.hosts() )
+            hosts.append( main.ONOScli4.hosts() )
+            hosts.append( main.ONOScli5.hosts() )
+            hosts.append( main.ONOScli6.hosts() )
+            hosts.append( main.ONOScli7.hosts() )
+            ports = []
+            ports.append( main.ONOScli1.ports() )
+            ports.append( main.ONOScli2.ports() )
+            ports.append( main.ONOScli3.ports() )
+            ports.append( main.ONOScli4.ports() )
+            ports.append( main.ONOScli5.ports() )
+            ports.append( main.ONOScli6.ports() )
+            ports.append( main.ONOScli7.ports() )
+            links = []
+            links.append( main.ONOScli1.links() )
+            links.append( main.ONOScli2.links() )
+            links.append( main.ONOScli3.links() )
+            links.append( main.ONOScli4.links() )
+            links.append( main.ONOScli5.links() )
+            links.append( main.ONOScli6.links() )
+            links.append( main.ONOScli7.links() )
+            clusters = []
+            clusters.append( main.ONOScli1.clusters() )
+            clusters.append( main.ONOScli2.clusters() )
+            clusters.append( main.ONOScli3.clusters() )
+            clusters.append( main.ONOScli4.clusters() )
+            clusters.append( main.ONOScli5.clusters() )
+            clusters.append( main.ONOScli6.clusters() )
+            clusters.append( main.ONOScli7.clusters() )
+            paths = []
+            temp_topo = main.ONOSbench.get_topology( main.ONOScli1.topology() )
+            paths.append( temp_topo.get('paths', False) )
+            temp_topo = main.ONOSbench.get_topology( main.ONOScli2.topology() )
+            paths.append( temp_topo.get('paths', False) )
+            temp_topo = main.ONOSbench.get_topology( main.ONOScli3.topology() )
+            paths.append( temp_topo.get('paths', False) )
+            temp_topo = main.ONOSbench.get_topology( main.ONOScli4.topology() )
+            paths.append( temp_topo.get('paths', False) )
+            temp_topo = main.ONOSbench.get_topology( main.ONOScli5.topology() )
+            paths.append( temp_topo.get('paths', False) )
+            temp_topo = main.ONOSbench.get_topology( main.ONOScli6.topology() )
+            paths.append( temp_topo.get('paths', False) )
+            temp_topo = main.ONOSbench.get_topology( main.ONOScli7.topology() )
+            paths.append( temp_topo.get('paths', False) )
 
-                for controller in range(7): #TODO parameterize the number of controllers
-                    if devices[controller] or not "Error" in devices[controller]:
-                        current_devices_result =  main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
-                    else:
-                        current_devices_result = main.FALSE
-                    utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
-                            onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
-                            onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
 
-                    if ports[controller] or not "Error" in ports[controller]:
-                        current_ports_result =  main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
-                    else:
-                        current_ports_result = main.FALSE
-                    utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
-                            onpass="ONOS"+str(int(controller+1))+" ports view is correct",
-                            onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
+            elapsed = time.time() - start_time
+            cli_time = time.time() - cli_start
+            print "CLI time: " + str(cli_time)
 
-                    if links[controller] or not "Error" in links[controller]:
-                        current_links_result =  main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
-                    else:
-                        current_links_result = main.FALSE
-                    utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
-                            onpass="ONOS"+str(int(controller+1))+" links view is correct",
-                            onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
-            except:
-                main.log.error("something went wrong in topo comparison")
-                main.log.warn( repr( devices ) )
-                main.log.warn( repr( ports ) )
-                main.log.warn( repr( links ) )
+            for controller in range(7): #TODO parameterize the number of controllers
+                if devices[controller] or not "Error" in devices[controller]:
+                    current_devices_result =  main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
+                else:
+                    current_devices_result = main.FALSE
+                utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
+                        onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
+                        onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
 
+                if ports[controller] or not "Error" in ports[controller]:
+                    current_ports_result =  main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
+                else:
+                    current_ports_result = main.FALSE
+                utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
+                        onpass="ONOS"+str(int(controller+1))+" ports view is correct",
+                        onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
+
+                if links[controller] or not "Error" in links[controller]:
+                    current_links_result =  main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
+                else:
+                    current_links_result = main.FALSE
+                utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
+                        onpass="ONOS"+str(int(controller+1))+" links view is correct",
+                        onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
             devices_results = devices_results and current_devices_result
             ports_results = ports_results and current_ports_result
             links_results = links_results and current_links_result
-            topo_result = devices_results and ports_results and links_results
-            elapsed = time.time() - start_time
-        time_threshold = elapsed < 1
-        topo_result = topo_result and time_threshold
-        #TODO make sure this step is non-blocking. IE add a timeout
-        main.log.report("Very crass estimate for topology discovery/convergence: " +\
+
+            #Compare json objects for hosts, dataplane clusters and paths
+
+            #hosts
+            consistent_hosts_result = main.TRUE
+            for controller in range( len( hosts ) ):
+                if not "Error" in hosts[controller]:
+                    if hosts[controller] == hosts[0]:
+                        continue
+                    else:#hosts not consistent
+                        main.log.report("hosts from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+                        main.log.warn( repr( hosts[controller] ) )
+                        consistent_hosts_result = main.FALSE
+
+                else:
+                    main.log.report("Error in getting ONOS hosts from ONOS" + str(controller + 1) )
+                    consistent_hosts_result = main.FALSE
+                    main.log.warn("ONOS" + str(controller + 1) + " hosts response: " + repr(hosts[controller]) )
+            utilities.assert_equals(expect = main.TRUE,actual=consistent_hosts_result,
+                    onpass="Hosts view is consistent across all ONOS nodes",
+                    onfail="ONOS nodes have different views of hosts")
+
+            #Strongly connected clusters of devices
+            consistent_clusters_result = main.TRUE
+            for controller in range( len( clusters ) ):
+                if not "Error" in clusters[controller]:
+                    if clusters[controller] == clusters[0]:
+                        continue
+                    else:#clusters not consistent
+                        main.log.report("clusters from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+                        consistent_clusters_result = main.FALSE
+
+                else:
+                    main.log.report("Error in getting dataplane clusters from ONOS" + str(controller + 1) )
+                    consistent_clusters_result = main.FALSE
+                    main.log.warn("ONOS" + str(controller + 1) + " clusters response: " + repr(clusters[controller]) )
+            utilities.assert_equals(expect = main.TRUE,actual=consistent_clusters_result,
+                    onpass="Clusters view is consistent across all ONOS nodes",
+                    onfail="ONOS nodes have different views of clusters")
+            num_clusters =  len(json.loads(clusters[0])) #there should always only be one cluster
+            utilities.assert_equals(expect = 1, actual = num_clusters,
+                    onpass="ONOS shows 1 SCC",
+                    onfail="ONOS shows "+str(num_clusters) +" SCCs")
+
+
+            #paths
+            consistent_paths_result = main.TRUE
+            for controller in range( len( paths ) ):
+                if not "Error" in paths[controller]:
+                    if paths[controller] == paths[0]:
+                        continue
+                    else:#paths not consistent
+                        main.log.report("paths from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+                        consistent_paths_result = main.FALSE
+
+                else:
+                    main.log.report("Error in getting paths from ONOS" + str(controller + 1) )
+                    consistent_paths_result = main.FALSE
+                    main.log.warn("ONOS" + str(controller + 1) + " paths response: " + repr(paths[controller]) )
+            utilities.assert_equals(expect = main.TRUE,actual=consistent_paths_result,
+                    onpass="Paths count is consistent across all ONOS nodes",
+                    onfail="ONOS nodes have different counts of paths")
+
+
+            topo_result = devices_results and ports_results and links_results\
+                    and consistent_hosts_result and consistent_clusters_result and consistent_paths_result
+
+        topo_result = topo_result and int(count <= 2)
+        note = "note it takes about "+str( int(cli_time) )+" seconds for the test to make all the cli calls to fetch the topology from each ONOS instance"
+        main.log.report("Very crass estimate for topology discovery/convergence("+ str(note) + "): " +\
                 str(elapsed) + " seconds, " + str(count) +" tries" )
         utilities.assert_equals(expect=main.TRUE, actual=topo_result,
                 onpass="Topology Check Test successful",
@@ -1054,13 +1474,18 @@
         #TODO: Make this switch parameterizable
         main.step("Kill s28 ")
         main.log.report("Deleting s28")
-        #FIXME: use new dynamic topo functions
         main.Mininet1.del_switch("s28")
         main.log.info("Waiting " + str(switch_sleep) + " seconds for switch down to be discovered")
         time.sleep(switch_sleep)
+        device = main.ONOScli1.get_device(dpid="0028")
         #Peek at the deleted switch
-        main.log.warn(main.ONOScli1.get_device(dpid="0028"))
-        #TODO do some sort of check here
+        main.log.warn( str(device) )
+        result = main.FALSE
+        if device and device['available'] == False:
+            result = main.TRUE
+        utilities.assert_equals(expect=main.TRUE,actual=result,
+                onpass="Kill switch succesful",
+                onfail="Failed to kill switch?")
 
     def CASE12 (self, main) :
         '''
@@ -1068,7 +1493,6 @@
         '''
         #NOTE: You should probably run a topology check after this
         import time
-        #FIXME: use new dynamic topo functions
         description = "Adding a switch to ensure it is discovered correctly"
         main.log.report(description)
         main.case(description)
@@ -1090,9 +1514,15 @@
                 ip7=ONOS7_ip,port7=ONOS7_port)
         main.log.info("Waiting " + str(switch_sleep) + " seconds for switch up to be discovered")
         time.sleep(switch_sleep)
-        #Peek at the added switch
-        main.log.warn(main.ONOScli1.get_device(dpid="0028"))
-        #TODO do some sort of check here
+        device = main.ONOScli1.get_device(dpid="0028")
+        #Peek at the deleted switch
+        main.log.warn( str(device) )
+        result = main.FALSE
+        if device and device['available'] == True:
+            result = main.TRUE
+        utilities.assert_equals(expect=main.TRUE,actual=result,
+                onpass="add switch succesful",
+                onfail="Failed to add switch?")
 
     def CASE13 (self, main) :
         '''
@@ -1106,8 +1536,26 @@
         main.step("Killing tcpdumps")
         main.Mininet2.stop_tcpdump()
 
+        main.step("Checking ONOS Logs for errors")
+        print "Checking logs for errors on ONOS1:"
+        print main.ONOSbench.check_logs(ONOS1_ip)
+        print "Checking logs for errors on ONOS2:"
+        print main.ONOSbench.check_logs(ONOS2_ip)
+        print "Checking logs for errors on ONOS3:"
+        print main.ONOSbench.check_logs(ONOS3_ip)
+        print "Checking logs for errors on ONOS4:"
+        print main.ONOSbench.check_logs(ONOS4_ip)
+        print "Checking logs for errors on ONOS5:"
+        print main.ONOSbench.check_logs(ONOS5_ip)
+        print "Checking logs for errors on ONOS6:"
+        print main.ONOSbench.check_logs(ONOS6_ip)
+        print "Checking logs for errors on ONOS7:"
+        print main.ONOSbench.check_logs(ONOS7_ip)
+
         main.step("Copying MN pcap and ONOS log files to test station")
         testname = main.TEST
+        teststation_user = main.params['TESTONUSER']
+        teststation_IP = main.params['TESTONIP']
         #NOTE: MN Pcap file is being saved to ~/packet_captures
         #       scp this file as MN and TestON aren't necessarily the same vm
         #FIXME: scp
@@ -1119,19 +1567,26 @@
         #NOTE: must end in /
         dst_dir = "~/packet_captures/"
         for f in log_files:
-            main.ONOSbench.secureCopy( "sdn", ONOS1_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS1_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS1-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS2_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS2_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS2-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS3_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS3_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS3-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS4_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS4_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS4-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS5_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS5_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS5-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS6_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS6_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS6-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS7_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS7_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS7-"+f )
 
         #std*.log's
@@ -1141,24 +1596,29 @@
         #NOTE: must end in /
         dst_dir = "~/packet_captures/"
         for f in log_files:
-            main.ONOSbench.secureCopy( "sdn", ONOS1_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS1_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS1-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS2_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS2_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS2-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS3_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS3_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS3-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS4_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS4_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS4-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS5_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS5_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS5-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS6_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS6_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS6-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS7_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS7_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS7-"+f )
 
 
-
-
         #sleep so scp can finish
         time.sleep(10)
         main.step("Packing and rotating pcap archives")
@@ -1169,3 +1629,22 @@
         utilities.assert_equals(expect=main.TRUE, actual=main.TRUE,
                 onpass="Test cleanup successful",
                 onfail="Test cleanup NOT successful")
+    def CASE14 ( self, main ) :
+        '''
+        start election app on all onos nodes
+        '''
+        #install app on onos 1
+        #wait for election
+        #check for leader
+        #install on other nodes
+        #check for leader. Should be onos1 and each app shows same leader
+        #
+
+
+
+
+
+        #Next Case
+        #add to reboot case?
+        #check for conistent leader
+        #new leader should have been elected
diff --git a/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.params b/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.params
index e1e75f9..68f12a8 100644
--- a/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.params
+++ b/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.params
@@ -1,9 +1,9 @@
 <PARAMS>
-    <testcases>1,2,8,3,4,5,[6],7,8,4,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
+    <testcases>1,2,8,3,4,5,[6],8,3,7,4,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
     <ENV>
     <cellName>HA</cellName>
     </ENV>
-    <Git>True</Git>
+    <Git>False</Git>
 
     <CTRL>
         <ip1>10.128.30.11</ip1>
@@ -28,7 +28,7 @@
         <port7>6633</port7>
     </CTRL>
     <TESTONUSER>admin</TESTONUSER>
-    <TESTONIP>10.128.30.10</TESTONIP>
+    <TESTONIP>10.128.30.9</TESTONIP>
     <PING>
         <source1>h8</source1>
         <source2>h9</source2>
diff --git a/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.py b/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.py
index e6f2fde..a82bbd5 100644
--- a/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.py
+++ b/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.py
@@ -112,6 +112,8 @@
                         "clean install")
         main.ONOSbench.get_version(report=True)
 
+        cell_result = main.ONOSbench.set_cell("SingleHA")
+        verify_result = main.ONOSbench.verify_cell()
         main.step("Creating ONOS package")
         package_result = main.ONOSbench.onos_package()
 
@@ -123,15 +125,17 @@
         main.step("Checking if ONOS is up yet")
         #TODO: Refactor
         # check bundle:list?
-        #this should be enough for ONOS to start
-        time.sleep(60)
-        onos1_isup = main.ONOSbench.isup(ONOS1_ip)
+        for i in range(2):
+            onos1_isup = main.ONOSbench.isup(ONOS1_ip)
+            if onos1_isup:
+                break
         if not onos1_isup:
             main.log.report("ONOS1 didn't start!")
+
         # TODO: if it becomes an issue, we can retry this step  a few times
 
 
-        cli_result1 = main.ONOScli1.start_onos_cli(ONOS1_ip)
+        cli_result = main.ONOScli1.start_onos_cli(ONOS1_ip)
 
         main.step("Start Packet Capture MN")
         main.Mininet2.start_tcpdump(
@@ -142,7 +146,7 @@
 
         case1_result = (clean_install_result and package_result and
                 cell_result and verify_result and onos1_install_result and
-                onos1_isup and cli1_results)
+                onos1_isup and cli_result)
 
         utilities.assert_equals(expect=main.TRUE, actual=case1_result,
                 onpass="Test startup successful",
@@ -186,8 +190,6 @@
                 onpass="Switch mastership assigned correctly",
                 onfail="Switches not assigned correctly to controllers")
 
-        #TODO: If assign roles is working reliably then manually 
-        #   assign mastership to the controller we want
 
 
     def CASE3(self,main) :
@@ -195,6 +197,7 @@
         Assign intents
 
         """
+        #FIXME: Only call this once when using a persistent datastore!!!!
         import time
         import json
         import re
@@ -204,6 +207,10 @@
         main.step("Discovering  Hosts( Via pingall for now)")
         #FIXME: Once we have a host discovery mechanism, use that instead
 
+        #install onos-app-fwd
+        main.log.info("Install reactive forwarding app")
+        main.ONOScli1.feature_install("onos-app-fwd")
+
         #REACTIVE FWD test
         ping_result = main.FALSE
         time1 = time.time()
@@ -219,7 +226,7 @@
         #TODO:  move the host numbers to params
         import json
         intents_json= json.loads(main.ONOScli1.hosts())
-        intent_add_result = main.FALSE
+        intent_add_result = True
         for i in range(8,18):
             main.log.info("Adding host intent between h"+str(i)+" and h"+str(i+10))
             host1 =  "00:00:00:00:00:" + str(hex(i)[2:]).zfill(2).upper()
@@ -229,7 +236,10 @@
             host1_id = main.ONOScli1.get_host(host1)['id']
             host2_id = main.ONOScli1.get_host(host2)['id']
             tmp_result = main.ONOScli1.add_host_intent(host1_id, host2_id )
-            intent_add_result = intent_add_result and tmp_result
+            intent_add_result = bool(intent_add_result and tmp_result)
+        utilities.assert_equals(expect=True, actual=intent_add_result,
+                onpass="Switch mastership correctly assigned",
+                onfail="Error in (re)assigning switch mastership")
         #TODO Check if intents all exist in datastore
         #NOTE: Do we need to print this once the test is working?
         #main.log.info(json.dumps(json.loads(main.ONOScli1.intents(json_format=True)),
@@ -275,6 +285,15 @@
 
         main.step("Get the Mastership of each switch from each controller")
         global mastership_state
+        mastership_state = []
+
+        #Assert that each device has a master
+        roles_not_null = main.ONOScli1.roles_not_null()
+        utilities.assert_equals(expect = main.TRUE,actual=roles_not_null,
+                onpass="Each device has a master",
+                onfail="Some devices don't have a master assigned")
+
+
         ONOS1_mastership = main.ONOScli1.roles()
         #print json.dumps(json.loads(ONOS1_mastership), sort_keys=True, indent=4, separators=(',', ': '))
         #TODO: Make this a meaningful check
@@ -285,14 +304,11 @@
         else:
             mastership_state = ONOS1_mastership
             consistent_mastership = main.TRUE
-            main.log.report("Switch roles are consistent across all ONOS nodes")
-        utilities.assert_equals(expect = main.TRUE,actual=consistent_mastership,
-                onpass="Switch roles are consistent across all ONOS nodes",
-                onfail="ONOS nodes have different views of switch roles")
 
 
         main.step("Get the intents from each controller")
         global intent_state
+        intent_state = []
         ONOS1_intents = main.ONOScli1.intents( json_format=True )
         intent_check = main.FALSE
         if "Error" in ONOS1_intents or not ONOS1_intents:
@@ -300,14 +316,11 @@
             main.log.warn("ONOS1 intents response: " + repr(ONOS1_intents))
         else:
             intent_check = main.TRUE
-            main.log.report("Intents are consistent across all ONOS nodes")
-        utilities.assert_equals(expect = main.TRUE,actual=intent_check,
-                onpass="Intents are consistent across all ONOS nodes",
-                onfail="ONOS nodes have different views of intents")
 
 
         main.step("Get the flows from each controller")
         global flow_state
+        flow_state = []
         ONOS1_flows = main.ONOScli1.flows( json_format=True )
         flow_check = main.FALSE
         if "Error" in ONOS1_flows or not ONOS1_flows:
@@ -317,42 +330,16 @@
             #TODO: Do a better check, maybe compare flows on switches?
             flow_state = ONOS1_flows
             flow_check = main.TRUE
-            main.log.report("Flow count is consistent across all ONOS nodes")
-        utilities.assert_equals(expect = main.TRUE,actual=flow_check,
-                onpass="The flow count is consistent across all ONOS nodes",
-                onfail="ONOS nodes have different flow counts")
 
 
         main.step("Get the OF Table entries")
         global flows
         flows=[]
         for i in range(1,29):
-            flows.append(main.Mininet2.get_flowTable("s"+str(i),1.0))
+            flows.append(main.Mininet2.get_flowTable(1.3, "s"+str(i)))
 
         #TODO: Compare switch flow tables with ONOS flow tables
 
-        main.step("Start continuous pings")
-        main.Mininet2.pingLong(src=main.params['PING']['source1'],
-                            target=main.params['PING']['target1'],pingTime=500)
-        main.Mininet2.pingLong(src=main.params['PING']['source2'],
-                            target=main.params['PING']['target2'],pingTime=500)
-        main.Mininet2.pingLong(src=main.params['PING']['source3'],
-                            target=main.params['PING']['target3'],pingTime=500)
-        main.Mininet2.pingLong(src=main.params['PING']['source4'],
-                            target=main.params['PING']['target4'],pingTime=500)
-        main.Mininet2.pingLong(src=main.params['PING']['source5'],
-                            target=main.params['PING']['target5'],pingTime=500)
-        main.Mininet2.pingLong(src=main.params['PING']['source6'],
-                            target=main.params['PING']['target6'],pingTime=500)
-        main.Mininet2.pingLong(src=main.params['PING']['source7'],
-                            target=main.params['PING']['target7'],pingTime=500)
-        main.Mininet2.pingLong(src=main.params['PING']['source8'],
-                            target=main.params['PING']['target8'],pingTime=500)
-        main.Mininet2.pingLong(src=main.params['PING']['source9'],
-                            target=main.params['PING']['target9'],pingTime=500)
-        main.Mininet2.pingLong(src=main.params['PING']['source10'],
-                            target=main.params['PING']['target10'],pingTime=500)
-
         main.step("Create TestONTopology object")
         ctrls = []
         count = 1
@@ -417,7 +404,7 @@
 
         final_assert = main.TRUE
         final_assert = final_assert and topo_result and flow_check \
-                and intent_check and consistent_mastership
+                and intent_check and consistent_mastership and roles_not_null
         utilities.assert_equals(expect=main.TRUE, actual=final_assert,
                 onpass="State check successful",
                 onfail="State check NOT successful")
@@ -436,7 +423,7 @@
 
         main.step("Checking if ONOS is up yet")
         count = 0
-        while count < 10
+        while count < 10:
             onos1_isup = main.ONOSbench.isup(ONOS1_ip)
             if onos1_isup == main.TRUE:
                 elapsed = time.time() - start
@@ -446,11 +433,12 @@
 
         cli_result = main.ONOScli1.start_onos_cli(ONOS1_ip)
 
-        case_results = main.TRUE and onosi1_isup and cli_result
+        case_results = main.TRUE and onos1_isup and cli_result
         utilities.assert_equals(expect=main.TRUE, actual=case_results,
                 onpass="ONOS restart successful",
                 onfail="ONOS restart NOT successful")
-        main.log.info("ONOS took %s seconds to restart" % str(elapsed) )
+        main.log.info("ESTIMATE: ONOS took %s seconds to restart" % str(elapsed) )
+        time.sleep(5)
 
     def CASE7(self,main) :
         '''
@@ -460,6 +448,14 @@
         import json
         main.case("Running ONOS Constant State Tests")
 
+        #Assert that each device has a master
+        roles_not_null = main.ONOScli1.roles_not_null()
+        utilities.assert_equals(expect = main.TRUE,actual=roles_not_null,
+                onpass="Each device has a master",
+                onfail="Some devices don't have a master assigned")
+
+
+
         main.step("Check if switch roles are consistent across all nodes")
         ONOS1_mastership = main.ONOScli1.roles()
         #FIXME: Refactor this whole case for single instance
@@ -508,13 +504,14 @@
             main.log.report("Error in getting ONOS intents")
             main.log.warn("ONOS1 intents response: " + repr(ONOS1_intents))
         else:
-            intent_state = ONOS1_intents
             intent_check = main.TRUE
             main.log.report("Intents are consistent across all ONOS nodes")
         utilities.assert_equals(expect = main.TRUE,actual=intent_check,
                 onpass="Intents are consistent across all ONOS nodes",
                 onfail="ONOS nodes have different views of intents")
 
+        #NOTE: Hazelcast has no durability, so intents are lost
+        '''
         main.step("Compare current intents with intents before the failure")
         if intent_state == ONOS1_intents:
             same_intents = main.TRUE
@@ -526,6 +523,7 @@
                 onpass="Intents are consistent with before failure",
                 onfail="The Intents changed during failure")
         intent_check = intent_check and same_intents
+        '''
 
 
 
@@ -534,42 +532,20 @@
         flows2=[]
         for i in range(28):
             main.log.info("Checking flow table on s" + str(i+1))
-            tmp_flows = main.Mininet2.get_flowTable("s"+str(i+1),1.0)
+            tmp_flows = main.Mininet2.get_flowTable(1.3, "s"+str(i+1))
             flows2.append(tmp_flows)
-            Flow_Tables = Flow_Tables and main.Mininet2.flow_comp(flow1=flows[i],flow2=tmp_flows)
+            temp_result = main.Mininet2.flow_comp(flow1=flows[i],flow2=tmp_flows)
+            Flow_Tables = Flow_Tables and temp_result
             if Flow_Tables == main.FALSE:
                 main.log.info("Differences in flow table for switch: "+str(i+1))
-                break
         if Flow_Tables == main.TRUE:
             main.log.report("No changes were found in the flow tables")
         utilities.assert_equals(expect=main.TRUE,actual=Flow_Tables,
                 onpass="No changes were found in the flow tables",
                 onfail="Changes were found in the flow tables")
 
-        main.step("Check the continuous pings to ensure that no packets were dropped during component failure")
-        #FIXME: This check is always failing. Investigate cause
-        #NOTE:  this may be something to do with file permsissions
-        #       or slight change in format
-        main.Mininet2.pingKill(main.params['TESTONUSER'], main.params['TESTONIP'])
-        Loss_In_Pings = main.FALSE 
-        #NOTE: checkForLoss returns main.FALSE with 0% packet loss
-        for i in range(8,18):
-            main.log.info("Checking for a loss in pings along flow from s" + str(i))
-            Loss_In_Pings = Loss_In_Pings or main.Mininet2.checkForLoss("/tmp/ping.h"+str(i))
-        if Loss_In_Pings == main.TRUE:
-            main.log.info("Loss in ping detected")
-        elif Loss_In_Pings == main.ERROR:
-            main.log.info("There are multiple mininet process running")
-        elif Loss_In_Pings == main.FALSE:
-            main.log.info("No Loss in the pings")
-            main.log.report("No loss of dataplane connectivity")
-        utilities.assert_equals(expect=main.FALSE,actual=Loss_In_Pings,
-                onpass="No Loss of connectivity",
-                onfail="Loss of dataplane connectivity detected")
-
-
         #TODO:add topology to this or leave as a seperate case?
-        result = mastership_check and intent_check and Flow_Tables and (not Loss_In_Pings)
+        result = mastership_check and intent_check and Flow_Tables and roles_not_null 
         result = int(result)
         if result == main.TRUE:
             main.log.report("Constant State Tests Passed")
@@ -606,62 +582,62 @@
         ports_results = main.TRUE
         links_results = main.TRUE
         topo_result = main.FALSE
-        start_time = time.time()
         elapsed = 0
         count = 0
-        while topo_result == main.FALSE and elapsed < 120:
+        main.step("Collecting topology information from ONOS")
+        start_time = time.time()
+        while topo_result == main.FALSE and elapsed < 60:
             count = count + 1
-            try:
-                main.step("Collecting topology information from ONOS")
-                devices = []
-                devices.append( main.ONOScli1.devices() )
-                '''
-                hosts = []
-                hosts.append( main.ONOScli1.hosts() )
-                '''
-                ports = []
-                ports.append( main.ONOScli1.ports() )
-                links = []
-                links.append( main.ONOScli1.links() )
-                for controller in range(1): #TODO parameterize the number of controllers
-                    if devices[controller] or not "Error" in devices[controller]:
-                        current_devices_result =  main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
-                    else:
-                        current_devices_result = main.FALSE
-                    utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
-                            onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
-                            onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
+            if count > 1:
+                MNTopo = TestONTopology(main.Mininet1, ctrls) # can also add Intent API info for intent operations
+            cli_start = time.time()
+            devices = []
+            devices.append( main.ONOScli1.devices() )
+            '''
+            hosts = []
+            hosts.append( main.ONOScli1.hosts() )
+            '''
+            ports = []
+            ports.append( main.ONOScli1.ports() )
+            links = []
+            links.append( main.ONOScli1.links() )
+            elapsed = time.time() - start_time
+            print "CLI time: " + str(time.time() - cli_start)
 
-                    if ports[controller] or not "Error" in ports[controller]:
-                        current_ports_result =  main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
-                    else:
-                        current_ports_result = main.FALSE
-                    utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
-                            onpass="ONOS"+str(int(controller+1))+" ports view is correct",
-                            onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
+            for controller in range(1): #TODO parameterize the number of controllers
+                if devices[controller] or not "Error" in devices[controller]:
+                    current_devices_result =  main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
+                else:
+                    current_devices_result = main.FALSE
+                utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
+                        onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
+                        onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
 
-                    if links[controller] or not "Error" in links[controller]:
-                        current_links_result =  main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
-                    else:
-                        current_links_result = main.FALSE
-                    utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
-                            onpass="ONOS"+str(int(controller+1))+" links view is correct",
-                            onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
-            except:
-                main.log.error("something went wrong in topo comparison")
-                main.log.warn( repr( devices ) )
-                main.log.warn( repr( ports ) )
-                main.log.warn( repr( links ) )
+                if ports[controller] or not "Error" in ports[controller]:
+                    current_ports_result =  main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
+                else:
+                    current_ports_result = main.FALSE
+                utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
+                        onpass="ONOS"+str(int(controller+1))+" ports view is correct",
+                        onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
 
+                if links[controller] or not "Error" in links[controller]:
+                    current_links_result =  main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
+                else:
+                    current_links_result = main.FALSE
+                utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
+                        onpass="ONOS"+str(int(controller+1))+" links view is correct",
+                        onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
             devices_results = devices_results and current_devices_result
             ports_results = ports_results and current_ports_result
             links_results = links_results and current_links_result
-            elapsed = time.time() - start_time
-        time_threshold = elapsed < 1
-        topo_result = devices_results and ports_results and links_results and time_threshold
-        #TODO make sure this step is non-blocking. IE add a timeout
-        main.log.report("Very crass estimate for topology discovery/convergence: " +\
+            topo_result = devices_results and ports_results and links_results
+
+        topo_result = topo_result and int(count <= 2)
+        main.log.report("Very crass estimate for topology discovery/convergence(note it takes about 1 seconds to read the topology from each ONOS instance): " +\
                 str(elapsed) + " seconds, " + str(count) +" tries" )
+        if elapsed > 60:
+            main.log.report("Giving up on topology convergence")
         utilities.assert_equals(expect=main.TRUE, actual=topo_result,
                 onpass="Topology Check Test successful",
                 onfail="Topology Check Test NOT successful")
@@ -729,13 +705,18 @@
         #TODO: Make this switch parameterizable
         main.step("Kill s28 ")
         main.log.report("Deleting s28")
-        #FIXME: use new dynamic topo functions
         main.Mininet1.del_switch("s28")
         main.log.info("Waiting " + str(switch_sleep) + " seconds for switch down to be discovered")
         time.sleep(switch_sleep)
+        device = main.ONOScli1.get_device(dpid="0028")
         #Peek at the deleted switch
-        main.log.warn(main.ONOScli1.get_device(dpid="0028"))
-        #TODO do some sort of check here
+        main.log.warn( str(device) )
+        result = main.FALSE
+        if device and device['available'] == False:
+            result = main.TRUE
+        utilities.assert_equals(expect=main.TRUE,actual=result,
+                onpass="Kill switch succesful",
+                onfail="Failed to kill switch?")
 
     def CASE12 (self, main) :
         '''
@@ -743,7 +724,6 @@
         '''
         #NOTE: You should probably run a topology check after this
         import time
-        #FIXME: use new dynamic topo functions
         description = "Adding a switch to ensure it is discovered correctly"
         main.log.report(description)
         main.case(description)
@@ -759,9 +739,15 @@
                 ip1=ONOS1_ip,port1=ONOS1_port)
         main.log.info("Waiting " + str(switch_sleep) + " seconds for switch up to be discovered")
         time.sleep(switch_sleep)
-        #Peek at the added switch
-        main.log.warn(main.ONOScli1.get_device(dpid="0028"))
-        #TODO do some sort of check here
+        device = main.ONOScli1.get_device(dpid="0028")
+        #Peek at the deleted switch
+        main.log.warn( str(device) )
+        result = main.FALSE
+        if device and device['available'] == True:
+            result = main.TRUE
+        utilities.assert_equals(expect=main.TRUE,actual=result,
+                onpass="add switch succesful",
+                onfail="Failed to add switch?")
 
     def CASE13 (self, main) :
         '''
@@ -775,8 +761,13 @@
         main.step("Killing tcpdumps")
         main.Mininet2.stop_tcpdump()
 
+        main.step("Checking ONOS Logs for errors")
+        print "Checking logs for errors on ONOS1:"
+        print main.ONOSbench.check_logs(ONOS1_ip)
         main.step("Copying MN pcap and ONOS log files to test station")
         testname = main.TEST
+        teststation_user = main.params['TESTONUSER']
+        teststation_IP = main.params['TESTONIP']
         #NOTE: MN Pcap file is being saved to ~/packet_captures
         #       scp this file as MN and TestON aren't necessarily the same vm
         #FIXME: scp
@@ -788,8 +779,11 @@
         #NOTE: must end in /
         dst_dir = "~/packet_captures/"
         for f in log_files:
-            main.ONOSbench.secureCopy( "sdn", ONOS1_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS1_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS1-"+f )
+            main.ONOSbench.handle.expect("\$")
+            print main.ONOSbench.handle.before
 
         #std*.log's
         #NOTE: must end in /
@@ -798,7 +792,8 @@
         #NOTE: must end in /
         dst_dir = "~/packet_captures/"
         for f in log_files:
-            main.ONOSbench.secureCopy( "sdn", ONOS1_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS1_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS1-"+f )